Merge pull request #162 from bancolombia/feature/improve_dependencies…

…_scan

Feature/improve dependencies scan

tools/devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py

@@ -89,7 +89,7 @@ def get_inputs_from_cli(args):
     parser.add_argument("--token_cmdb", required=False, help="Token to connect to the CMDB")
     parser.add_argument("--token_vulnerability_management", required=False, help="Token to connect to the Vulnerability Management")
     parser.add_argument("--token_engine_container", required=False, help="Token to execute engine_container if is necessary")
-    parser.add_argument("--token_engine_dependencies", required=False, help="Token to execute engine_dependencies if is necessary")
+    parser.add_argument("--token_engine_dependencies", required=False, help="Token to execute engine_dependencies if is necessary. If using xray as engine_dependencies tool, the token is the base64 of artifactory server config.")
     args = parser.parse_args()
     return {
         "platform_devops": args.platform_devops,

tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/model/gateways/tool_gateway.py

@@ -4,6 +4,6 @@
 class ToolGateway(metaclass=ABCMeta):
     @abstractmethod
     def run_tool_dependencies_sca(
-        self, remote_config, dir_to_scan_path, bypass_limits_flag, token
+        self, remote_config, file_to_scan, bypass_limits_flag, token
     ) -> str:
         "run tool dependencies sca"

tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/dependencies_sca_scan.py

@@ -12,14 +12,14 @@ def __init__(
         tool_run: ToolGateway,
         tool_deserializator: DeserializatorGateway,
         remote_config,
-        dir_to_scan_path,
+        file_to_scan,
         bypass_limits_flag,
         token,
     ):
         self.tool_run = tool_run
         self.tool_deserializator = tool_deserializator
         self.remote_config = remote_config
-        self.dir_to_scan_path = dir_to_scan_path
+        self.file_to_scan = file_to_scan
         self.bypass_limits_flag = bypass_limits_flag
         self.token = token
 
@@ -31,7 +31,7 @@ def process(self):
         """
         return self.tool_run.run_tool_dependencies_sca(
             self.remote_config,
-            self.dir_to_scan_path,
+            self.file_to_scan,
             self.bypass_limits_flag,
             self.token,
         )

tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/find_artifacts.py

@@ -47,7 +47,6 @@ def compress_and_mv(self, tar_path, package):
                     arcname=os.path.basename(package),
                     filter=lambda x: None if "/.bin/" in x.name else x,
                 )
-                logger.debug(f"File to scan: {tar_path}")
 
         except subprocess.CalledProcessError as e:
             logger.error(f"Error during {package} compression: {e}")
@@ -88,9 +87,14 @@ def find_artifacts(self):
             for file in files
             if os.path.isfile(os.path.join(dir_to_scan_path, file))
         ]
+        file_to_scan = None
         if files:
+            file_to_scan = os.path.join(dir_to_scan_path, "file_to_scan.tar")
+            self.compress_and_mv(file_to_scan, dir_to_scan_path)
             files_string = ", ".join(files)
             logger.debug(f"Files to scan: {files_string}")
             print(f"Files to scan: {files_string}")
+        else:
+            logger.warning("No artifacts found")
 
-        return dir_to_scan_path
+        return file_to_scan

tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/xray_tool/xray_manager_scan.py

@@ -95,23 +95,23 @@ def config_server(self, prefix, token):
         except subprocess.CalledProcessError as error:
             logger.error(f"Error during Xray Server configuration: {error}")
 
-    def scan_dependencies(self, prefix, target_dir_name, bypass_limits_flag):
+    def scan_dependencies(self, prefix, file_to_scan, bypass_limits_flag):
         try:
             if bypass_limits_flag:
                 command = [
                     prefix,
                     "scan",
                     "--format=json",
                     "--bypass-archive-limits",
-                    f"{target_dir_name}/",
+                    f"{file_to_scan}",
                 ]
             else:
-                command = [prefix, "scan", "--format=json", f"{target_dir_name}/"]
+                command = [prefix, "scan", "--format=json", f"{file_to_scan}"]
             result = subprocess.run(
                 command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
             )
             scan_result = json.loads(result.stdout)
-            file_result = os.path.join(target_dir_name, "scan_result.json")
+            file_result = os.path.join(os.getcwd(), "scan_result.json")
             with open(file_result, "w") as file:
                 json.dump(scan_result, file, indent=4)
             return file_result
@@ -121,7 +121,7 @@ def scan_dependencies(self, prefix, target_dir_name, bypass_limits_flag):
     def run_tool_dependencies_sca(
         self,
         remote_config,
-        dir_to_scan_path,
+        file_to_scan,
         bypass_limits_flag,
         token,
     ):
@@ -143,12 +143,8 @@ def run_tool_dependencies_sca(
 
         self.config_server(command_prefix, token)
 
-        results_file = None
-        if len(os.listdir(dir_to_scan_path)) == 0:
-            logger.warning("No artifacts found")
-        else:
-            results_file = self.scan_dependencies(
-                command_prefix, dir_to_scan_path, bypass_limits_flag
-            )
+        results_file = self.scan_dependencies(
+            command_prefix, file_to_scan, bypass_limits_flag
+        )
 
         return results_file

tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/entry_points/entry_point_tool.py

@@ -49,19 +49,20 @@ def init_engine_dependencies(
         bypass_limits_flag = handle_remote_config_patterns.bypass_archive_limits()
         pattern = handle_remote_config_patterns.excluded_files()
 
-        find_artifacts = FindArtifacts(os.getcwd(), pattern, remote_config["PACKAGES_TO_SCAN"])
-        dir_to_scan_path = find_artifacts.find_artifacts()
-
-        dependencies_sca_scan = DependenciesScan(
-            tool_run,
-            tool_deserializator,
-            remote_config,
-            dir_to_scan_path,
-            bypass_limits_flag,
-            token,
+        find_artifacts = FindArtifacts(
+            os.getcwd(), pattern, remote_config["PACKAGES_TO_SCAN"]
         )
-        dependencies_scanned = dependencies_sca_scan.process()
-        if dependencies_scanned:
+        file_to_scan = find_artifacts.find_artifacts()
+        if file_to_scan:
+            dependencies_sca_scan = DependenciesScan(
+                tool_run,
+                tool_deserializator,
+                remote_config,
+                file_to_scan,
+                bypass_limits_flag,
+                token,
+            )
+            dependencies_scanned = dependencies_sca_scan.process()
             deserialized = dependencies_sca_scan.deserializator(dependencies_scanned)
     else:
         print(f"Tool skipped by DevSecOps policy")

tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/domain/usecases/test_dependencies_sca_scan.py

@@ -12,14 +12,14 @@ def test_init():
         "devsecops_engine_tools.engine_sca.engine_dependencies.src.domain.usecases.dependencies_sca_scan.DeserializatorGateway"
     ) as mock_deserializator_gateway:
         remote_config = {"remote_config_key": "remote_config_value"}
-        dir_to_scan_path = "/working/dir"
+        file_to_scan = "/working/dir/file.tar"
         bypass_limits_flag = True
         token = "token"
         dependencies_scan_instance = DependenciesScan(
             mock_tool_gateway,
             mock_deserializator_gateway,
             remote_config,
-            dir_to_scan_path,
+            file_to_scan,
             bypass_limits_flag,
             token,
         )
@@ -30,7 +30,7 @@ def test_init():
             == mock_deserializator_gateway
         )
         assert dependencies_scan_instance.remote_config == remote_config
-        assert dependencies_scan_instance.dir_to_scan_path == dir_to_scan_path
+        assert dependencies_scan_instance.file_to_scan == file_to_scan
         assert dependencies_scan_instance.bypass_limits_flag == bypass_limits_flag
         assert dependencies_scan_instance.token == token
 
@@ -42,22 +42,22 @@ def test_process():
         "devsecops_engine_tools.engine_sca.engine_dependencies.src.domain.usecases.dependencies_sca_scan.DeserializatorGateway"
     ) as mock_deserializator_gateway:
         remote_config = {"remote_config_key": "remote_config_value"}
-        dir_to_scan_path = "/working/dir"
+        file_to_scan = "/working/dir/file.tar"
         bypass_limits_flag = True
         token = "token"
 
         dependencies_scan_instance = DependenciesScan(
             mock_tool_gateway,
             mock_deserializator_gateway,
             remote_config,
-            dir_to_scan_path,
+            file_to_scan,
             bypass_limits_flag,
             token,
         )
         dependencies_scan_instance.process()
 
         mock_tool_gateway.run_tool_dependencies_sca.assert_called_once_with(
-            remote_config, dir_to_scan_path, bypass_limits_flag, token
+            remote_config, file_to_scan, bypass_limits_flag, token
         )
 
 
@@ -68,7 +68,7 @@ def test_deserializator():
         "devsecops_engine_tools.engine_sca.engine_dependencies.src.domain.usecases.dependencies_sca_scan.DeserializatorGateway"
     ) as mock_deserializator_gateway:
         remote_config = {"remote_config_key": "remote_config_value"}
-        dir_to_scan_path = "/working/dir"
+        file_to_scan = "/working/dir/file.tar"
         bypass_limits_flag = True
         token = "token"
         dependencies_scanned = "scanned.json"
@@ -77,7 +77,7 @@ def test_deserializator():
             mock_tool_gateway,
             mock_deserializator_gateway,
             remote_config,
-            dir_to_scan_path,
+            file_to_scan,
             bypass_limits_flag,
             token,
         )

tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/infrastructure/driven_adapters/xray_tool/test_xray_manager_scan.py

@@ -172,14 +172,17 @@ def test_scan_dependencies_success(xray_scan_instance):
         "builtins.open"
     ) as mock_open, patch(
         "os.path.join"
-    ) as mock_path_join:
+    ) as mock_path_join, patch(
+        "os.getcwd"
+    ) as mock_os_getcwd:
         prefix = "jf"
-        target_dir_name = "target_dir"
+        file_to_scan = "target_file.tar"
         bypass_limits_flag = True
         mock_subprocess_run.side_effect = Mock(returncode=0)
+        mock_os_getcwd.return_value = "/working_dir"
 
         xray_scan_instance.scan_dependencies(
-            prefix, target_dir_name, bypass_limits_flag
+            prefix, file_to_scan, bypass_limits_flag
         )
 
         mock_subprocess_run.assert_called_with(
@@ -188,14 +191,14 @@ def test_scan_dependencies_success(xray_scan_instance):
                 "scan",
                 "--format=json",
                 "--bypass-archive-limits",
-                f"{target_dir_name}/",
+                f"{file_to_scan}",
             ],
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             text=True,
         )
         mock_json_loads.assert_any_call
-        mock_path_join.assert_called_with(target_dir_name, "scan_result.json")
+        mock_path_join.assert_called_with("/working_dir", "scan_result.json")
         mock_open.assert_any_call
         mock_json_dump.assert_any_call
 
@@ -205,14 +208,14 @@ def test_scan_dependencies_failure(xray_scan_instance):
         "devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.driven_adapters.prisma_cloud.prisma_cloud_manager_scan.logger.error"
     ) as mock_logger_error:
         prefix = "jf"
-        target_dir_name = "target_dir"
+        file_to_scan = "target_file.tar"
         bypass_limits_flag = False
         mock_subprocess_run.side_effect = subprocess.CalledProcessError(
             returncode=1, cmd="xray scan"
         )
 
         xray_scan_instance.scan_dependencies(
-            prefix, target_dir_name, bypass_limits_flag
+            prefix, file_to_scan, bypass_limits_flag
         )
 
         mock_logger_error.assert_called_with(

tools/devsecops_engine_tools/version.py

@@ -1 +1 @@
-version = '1.7.19'
+version = '1.7.20'