From bd6a5818492b1f7371449ad35bce8da7854785fc Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Tue, 12 Nov 2024 14:01:04 +0200 Subject: [PATCH 1/2] CM-41217 Fix go graph vulnerabilities and adding supports of multi commands execution --- .../sca/base_restore_dependencies.py | 25 ++++++++++++------- .../sca/go/restore_go_dependencies.py | 16 +++++++----- .../sca/maven/restore_gradle_dependencies.py | 4 +-- .../sca/maven/restore_maven_dependencies.py | 8 +++--- .../sca/npm/restore_npm_dependencies.py | 18 +++++++------ .../sca/nuget/restore_nuget_dependencies.py | 4 +-- .../sca/sbt/restore_sbt_dependencies.py | 4 +-- .../files_collector/sca/sca_code_scanner.py | 6 +++++ cycode/cli/utils/shell_executor.py | 4 +-- 9 files changed, 54 insertions(+), 35 deletions(-) diff --git a/cycode/cli/files_collector/sca/base_restore_dependencies.py b/cycode/cli/files_collector/sca/base_restore_dependencies.py index b320e029..d221d755 100644 --- a/cycode/cli/files_collector/sca/base_restore_dependencies.py +++ b/cycode/cli/files_collector/sca/base_restore_dependencies.py @@ -13,19 +13,26 @@ def build_dep_tree_path(path: str, generated_file_name: str) -> str: return join_paths(get_file_dir(path), generated_file_name) -def execute_command( - command: List[str], +def execute_commands( + commands: List[List[str]], file_name: str, command_timeout: int, dependencies_file_name: Optional[str] = None, working_directory: Optional[str] = None, ) -> Optional[str]: try: - dependencies = shell(command=command, timeout=command_timeout, working_directory=working_directory) - # Write stdout output to the file if output_file_path is provided + all_dependencies = [] + + # Run all commands and collect outputs + for command in commands: + dependencies = shell(command=command, timeout=command_timeout, working_directory=working_directory) + all_dependencies.append(dependencies) # Collect each command's output + + # Write all collected outputs to the file if dependencies_file_name is provided if dependencies_file_name: - with open(dependencies_file_name, 'w') as output_file: - output_file.write(dependencies) + with open(dependencies_file_name, 'w') as output_file: # Open once in 'w' mode to start fresh + for dependencies in all_dependencies: + output_file.write(dependencies + '\n') except Exception as e: logger.debug('Failed to restore dependencies via shell command, %s', {'filename': file_name}, exc_info=e) return None @@ -62,8 +69,8 @@ def try_restore_dependencies(self, document: Document) -> Optional[Document]: restore_file_content = get_file_content(restore_file_path) else: output_file_path = restore_file_path if self.create_output_file_manually else None - execute_command( - self.get_command(manifest_file_path), + execute_commands( + self.get_commands(manifest_file_path), manifest_file_path, self.command_timeout, output_file_path, @@ -85,7 +92,7 @@ def is_project(self, document: Document) -> bool: pass @abstractmethod - def get_command(self, manifest_file_path: str) -> List[str]: + def get_commands(self, manifest_file_path: str) -> List[str]: pass @abstractmethod diff --git a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py index 512af8a5..e71047af 100644 --- a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py +++ b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py @@ -1,5 +1,5 @@ import os -from typing import List +from typing import List, Optional import click @@ -7,7 +7,7 @@ from cycode.cli.models import Document GO_PROJECT_FILE_EXTENSIONS = ['.mod'] -GO_RESTORE_FILE_NAME = 'go.sum' +GO_RESTORE_FILE_NAME = 'go.mod.graph' BUILD_GO_FILE_NAME = 'go.mod' @@ -18,8 +18,12 @@ def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: i def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in GO_PROJECT_FILE_EXTENSIONS) - def get_command(self, manifest_file_path: str) -> List[str]: - return ['cd', self.prepare_tree_file_path_for_command(manifest_file_path), '&&', 'go', 'list', '-m', '-json'] + def get_commands(self, manifest_file_path: str) -> List[List[str]]: + return [ + ['go', 'list', '-m', '-json', 'all'], + ['echo', '------------------------------------------------------'], + ['go', 'mod', 'graph'], + ] def get_lock_file_name(self) -> str: return GO_RESTORE_FILE_NAME @@ -27,5 +31,5 @@ def get_lock_file_name(self) -> str: def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: return os.path.isfile(restore_file_path) - def prepare_tree_file_path_for_command(self, manifest_file_path: str) -> str: - return manifest_file_path.replace(os.sep + BUILD_GO_FILE_NAME, '') + def get_working_directory(self, document: Document) -> Optional[str]: + return os.path.dirname(document.absolute_path) diff --git a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py index f925c28e..04fc6b9c 100644 --- a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py @@ -18,8 +18,8 @@ def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: i def is_project(self, document: Document) -> bool: return document.path.endswith(BUILD_GRADLE_FILE_NAME) or document.path.endswith(BUILD_GRADLE_KTS_FILE_NAME) - def get_command(self, manifest_file_path: str) -> List[str]: - return ['gradle', 'dependencies', '-b', manifest_file_path, '-q', '--console', 'plain'] + def get_commands(self, manifest_file_path: str) -> List[List[str]]: + return [['gradle', 'dependencies', '-b', manifest_file_path, '-q', '--console', 'plain']] def get_lock_file_name(self) -> str: return BUILD_GRADLE_DEP_TREE_FILE_NAME diff --git a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py index 84732021..a44a27e0 100644 --- a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py @@ -7,7 +7,7 @@ from cycode.cli.files_collector.sca.base_restore_dependencies import ( BaseRestoreDependencies, build_dep_tree_path, - execute_command, + execute_commands, ) from cycode.cli.models import Document from cycode.cli.utils.path_utils import get_file_content, get_file_dir, join_paths @@ -24,8 +24,8 @@ def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: i def is_project(self, document: Document) -> bool: return path.basename(document.path).split('/')[-1] == BUILD_MAVEN_FILE_NAME - def get_command(self, manifest_file_path: str) -> List[str]: - return ['mvn', 'org.cyclonedx:cyclonedx-maven-plugin:2.7.4:makeAggregateBom', '-f', manifest_file_path] + def get_commands(self, manifest_file_path: str) -> List[List[str]]: + return [['mvn', 'org.cyclonedx:cyclonedx-maven-plugin:2.7.4:makeAggregateBom', '-f', manifest_file_path]] def get_lock_file_name(self) -> str: return join_paths('target', MAVEN_CYCLONE_DEP_TREE_FILE_NAME) @@ -52,7 +52,7 @@ def restore_from_secondary_command( ) -> Optional[Document]: # TODO(MarshalX): does it even work? Ignored restore_dependencies_document arg secondary_restore_command = create_secondary_restore_command(manifest_file_path) - backup_restore_content = execute_command(secondary_restore_command, manifest_file_path, self.command_timeout) + backup_restore_content = execute_commands(secondary_restore_command, manifest_file_path, self.command_timeout) restore_dependencies_document = Document( build_dep_tree_path(document.path, MAVEN_DEP_TREE_FILE_NAME), backup_restore_content, self.is_git_diff ) diff --git a/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py index ea6f4061..c3026938 100644 --- a/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py +++ b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py @@ -18,15 +18,17 @@ def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: i def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in NPM_PROJECT_FILE_EXTENSIONS) - def get_command(self, manifest_file_path: str) -> List[str]: + def get_commands(self, manifest_file_path: str) -> List[List[str]]: return [ - 'npm', - 'install', - '--prefix', - self.prepare_manifest_file_path_for_command(manifest_file_path), - '--package-lock-only', - '--ignore-scripts', - '--no-audit', + [ + 'npm', + 'install', + '--prefix', + self.prepare_manifest_file_path_for_command(manifest_file_path), + '--package-lock-only', + '--ignore-scripts', + '--no-audit', + ] ] def get_lock_file_name(self) -> str: diff --git a/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py b/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py index c54c3e5e..0e2ed83d 100644 --- a/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py +++ b/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py @@ -17,8 +17,8 @@ def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: i def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in NUGET_PROJECT_FILE_EXTENSIONS) - def get_command(self, manifest_file_path: str) -> List[str]: - return ['dotnet', 'restore', manifest_file_path, '--use-lock-file', '--verbosity', 'quiet'] + def get_commands(self, manifest_file_path: str) -> List[List[str]]: + return [['dotnet', 'restore', manifest_file_path, '--use-lock-file', '--verbosity', 'quiet']] def get_lock_file_name(self) -> str: return NUGET_LOCK_FILE_NAME diff --git a/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py b/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py index f5073ef0..b8e1c41b 100644 --- a/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py +++ b/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py @@ -12,8 +12,8 @@ class RestoreSbtDependencies(BaseRestoreDependencies): def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in SBT_PROJECT_FILE_EXTENSIONS) - def get_command(self, manifest_file_path: str) -> List[str]: - return ['sbt', 'dependencyLockWrite', '--verbose'] + def get_commands(self, manifest_file_path: str) -> List[List[str]]: + return [['sbt', 'dependencyLockWrite', '--verbose']] def get_lock_file_name(self) -> str: return SBT_LOCK_FILE_NAME diff --git a/cycode/cli/files_collector/sca/sca_code_scanner.py b/cycode/cli/files_collector/sca/sca_code_scanner.py index 9e5ac5b4..d13d486c 100644 --- a/cycode/cli/files_collector/sca/sca_code_scanner.py +++ b/cycode/cli/files_collector/sca/sca_code_scanner.py @@ -5,8 +5,11 @@ from cycode.cli import consts from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies +from cycode.cli.files_collector.sca.go.restore_go_dependencies import RestoreGoDependencies from cycode.cli.files_collector.sca.maven.restore_gradle_dependencies import RestoreGradleDependencies from cycode.cli.files_collector.sca.maven.restore_maven_dependencies import RestoreMavenDependencies +from cycode.cli.files_collector.sca.npm.restore_npm_dependencies import RestoreNpmDependencies +from cycode.cli.files_collector.sca.nuget.restore_nuget_dependencies import RestoreNugetDependencies from cycode.cli.files_collector.sca.sbt.restore_sbt_dependencies import RestoreSbtDependencies from cycode.cli.models import Document from cycode.cli.utils.git_proxy import git_proxy @@ -132,6 +135,9 @@ def restore_handlers(context: click.Context, is_git_diff: bool) -> List[BaseRest RestoreGradleDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), RestoreMavenDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), RestoreSbtDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreGoDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreNugetDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreNpmDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), ] diff --git a/cycode/cli/utils/shell_executor.py b/cycode/cli/utils/shell_executor.py index 5ac79518..1a245882 100644 --- a/cycode/cli/utils/shell_executor.py +++ b/cycode/cli/utils/shell_executor.py @@ -1,5 +1,5 @@ import subprocess -from typing import List, Optional, Union +from typing import List, Optional import click @@ -9,7 +9,7 @@ def shell( - command: Union[str, List[str]], + command: List[List[str]], timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC, working_directory: Optional[str] = None, ) -> Optional[str]: From 953f597a471133c9e6b9ffac557ad5b3a1023b47 Mon Sep 17 00:00:00 2001 From: naftali-hershler Date: Tue, 12 Nov 2024 16:16:15 +0200 Subject: [PATCH 2/2] CM-40909 Add sum-mod extension check for restoring --- .../sca/base_restore_dependencies.py | 7 ++++--- .../sca/go/restore_go_dependencies.py | 18 +++++++++++++++++- cycode/cli/utils/shell_executor.py | 4 ++-- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/cycode/cli/files_collector/sca/base_restore_dependencies.py b/cycode/cli/files_collector/sca/base_restore_dependencies.py index d221d755..81caea1d 100644 --- a/cycode/cli/files_collector/sca/base_restore_dependencies.py +++ b/cycode/cli/files_collector/sca/base_restore_dependencies.py @@ -28,11 +28,12 @@ def execute_commands( dependencies = shell(command=command, timeout=command_timeout, working_directory=working_directory) all_dependencies.append(dependencies) # Collect each command's output + dependencies = '\n'.join(all_dependencies) + # Write all collected outputs to the file if dependencies_file_name is provided if dependencies_file_name: with open(dependencies_file_name, 'w') as output_file: # Open once in 'w' mode to start fresh - for dependencies in all_dependencies: - output_file.write(dependencies + '\n') + output_file.writelines(dependencies) except Exception as e: logger.debug('Failed to restore dependencies via shell command, %s', {'filename': file_name}, exc_info=e) return None @@ -92,7 +93,7 @@ def is_project(self, document: Document) -> bool: pass @abstractmethod - def get_commands(self, manifest_file_path: str) -> List[str]: + def get_commands(self, manifest_file_path: str) -> List[List[str]]: pass @abstractmethod diff --git a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py index e71047af..1986b3a2 100644 --- a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py +++ b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py @@ -1,3 +1,4 @@ +import logging import os from typing import List, Optional @@ -6,15 +7,30 @@ from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document -GO_PROJECT_FILE_EXTENSIONS = ['.mod'] +GO_PROJECT_FILE_EXTENSIONS = ['.mod', '.sum'] GO_RESTORE_FILE_NAME = 'go.mod.graph' BUILD_GO_FILE_NAME = 'go.mod' +BUILD_GO_LOCK_FILE_NAME = 'go.sum' class RestoreGoDependencies(BaseRestoreDependencies): def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: super().__init__(context, is_git_diff, command_timeout, create_output_file_manually=True) + def try_restore_dependencies(self, document: Document) -> Optional[Document]: + manifest_exists = os.path.isfile(self.get_working_directory(document) + os.sep + BUILD_GO_FILE_NAME) + lock_exists = os.path.isfile(self.get_working_directory(document) + os.sep + BUILD_GO_LOCK_FILE_NAME) + + if not manifest_exists or not lock_exists: + logging.info('No manifest go.mod file found' if not manifest_exists else 'No manifest go.sum file found') + + manifest_files_exists = manifest_exists & lock_exists + + if not manifest_files_exists: + return None + + return super().try_restore_dependencies(document) + def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in GO_PROJECT_FILE_EXTENSIONS) diff --git a/cycode/cli/utils/shell_executor.py b/cycode/cli/utils/shell_executor.py index 1a245882..5ac79518 100644 --- a/cycode/cli/utils/shell_executor.py +++ b/cycode/cli/utils/shell_executor.py @@ -1,5 +1,5 @@ import subprocess -from typing import List, Optional +from typing import List, Optional, Union import click @@ -9,7 +9,7 @@ def shell( - command: List[List[str]], + command: Union[str, List[str]], timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC, working_directory: Optional[str] = None, ) -> Optional[str]: