#81 add security scan command

doc/changes/changes_0.7.0.md

@@ -9,6 +9,7 @@ t.b.d.
 ## Features / Enhancements
 
  - #90: Improve test performance
+ - #81: Add security scan command
 
 ## Bug Fixes
 

exasol_script_languages_container_tool/cli/commands/__init__.py

@@ -6,3 +6,4 @@
 from .run_db_tests import run_db_test
 from .save import save
 from .upload import upload
+from .security_scan import security_scan

exasol_script_languages_container_tool/cli/commands/security_scan.py

@@ -0,0 +1,72 @@
+from pathlib import Path
+from typing import Tuple
+
+from exasol_integration_test_docker_environment.cli.cli import cli
+from exasol_integration_test_docker_environment.cli.common import add_options, import_build_steps, set_build_config, \
+    set_docker_repository_config, generate_root_task, run_task
+from exasol_integration_test_docker_environment.cli.options.build_options import build_options
+from exasol_integration_test_docker_environment.cli.options.docker_repository_options import docker_repository_options
+from exasol_integration_test_docker_environment.cli.options.system_options import system_options
+
+from exasol_script_languages_container_tool.cli.options.flavor_options import flavor_options
+from exasol_script_languages_container_tool.lib.tasks.security_scan.security_scan import SecurityScan
+from exasol_script_languages_container_tool.lib.utils.logging_redirection import log_redirector_task_creator_wrapper
+
+
+@cli.command()
+@add_options(flavor_options)
+@add_options(build_options)
+@add_options(docker_repository_options)
+@add_options(system_options)
+def security_scan(flavor_path: Tuple[str, ...],
+           force_rebuild: bool,
+           force_rebuild_from: Tuple[str, ...],
+           force_pull: bool,
+           output_directory: str,
+           temporary_base_directory: str,
+           log_build_context_content: bool,
+           cache_directory: str,
+           build_name: str,
+           source_docker_repository_name: str,
+           source_docker_tag_prefix: str,
+           source_docker_username: str,
+           source_docker_password: str,
+           target_docker_repository_name: str,
+           target_docker_tag_prefix: str,
+           target_docker_username: str,
+           target_docker_password: str,
+           workers: int,
+           task_dependencies_dot_file: str):
+    """
+    This command executes the security scan, which must be defined as separate step in the build steps declaration.
+    The scan runs the docker container of the respective step, passing a folder of the output-dir as argument.
+    If the stages do not exists locally, the system will build or pull them before running the scan.
+    """
+    import_build_steps(flavor_path)
+    set_build_config(force_rebuild,
+                     force_rebuild_from,
+                     force_pull,
+                     log_build_context_content,
+                     output_directory,
+                     temporary_base_directory,
+                     cache_directory,
+                     build_name)
+    set_docker_repository_config(source_docker_password, source_docker_repository_name, source_docker_username,
+                                 source_docker_tag_prefix, "source")
+    set_docker_repository_config(target_docker_password, target_docker_repository_name, target_docker_username,
+                                 target_docker_tag_prefix, "target")
+
+    report_path = Path(output_directory).joinpath("security_scan")
+    task_creator = log_redirector_task_creator_wrapper(lambda: generate_root_task(task_class=SecurityScan,
+                                                                                  flavor_paths=list(flavor_path),
+                                                                                  report_path=report_path
+                                                                                  ))
+
+    success, task = run_task(task_creator, workers, task_dependencies_dot_file)
+
+    if success:
+        with task.security_report_target.open("r") as f:
+            print(f.read())
+    print(f'Full security scan report can be found at:{report_path}')
+    if not success:
+        exit(1)

exasol_script_languages_container_tool/lib/tasks/security_scan/security_scan.py

@@ -0,0 +1,123 @@
+from pathlib import Path
+from typing import Dict
+
+import luigi
+
+import tarfile
+
+from exasol_integration_test_docker_environment.lib.base.flavor_task import FlavorsBaseTask
+from exasol_integration_test_docker_environment.lib.config.build_config import build_config
+from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient
+
+from exasol_script_languages_container_tool.lib.tasks.build.docker_flavor_build_base import DockerFlavorBuildBase
+
+from exasol_script_languages_container_tool.lib.tasks.security_scan.security_scan_parameter import SecurityScanParameter
+
+from docker.models.containers import Container
+
+
+class ScanResult:
+    def __init__(self, is_ok: bool, summary: str, report_dir: Path):
+        self.is_ok = is_ok
+        self.summary = summary
+        self.report_dir = report_dir
+
+
+class SecurityScan(FlavorsBaseTask, SecurityScanParameter):
+
+    def __init__(self, *args, **kwargs):
+        self.security_scanner_futures = None
+        super().__init__(*args, **kwargs)
+        report_path = self.report_path.joinpath("security_report")
+        self.security_report_target = luigi.LocalTarget(str(report_path))
+
+    def register_required(self):
+        tasks = self.create_tasks_for_flavors_with_common_params(
+            SecurityScanner, report_path=self.report_path)  # type: Dict[str,SecurityScanner]
+        self.security_scanner_futures = self.register_dependencies(tasks)
+
+    def run_task(self):
+        security_scanner_results = self.get_values_from_futures(
+            self.security_scanner_futures)
+
+        self.write_report(security_scanner_results)
+        all_result = AllScanResult(security_scanner_results)
+        if not all_result.scans_are_ok:
+            raise RuntimeError(f"Not all security scans were successful.:\n{all_result.get_error_scans_msg()}")
+
+    def write_report(self, security_scanner: Dict[str, ScanResult]):
+        with self.security_report_target.open("w") as out_file:
+
+            for key, value in security_scanner.items():
+                out_file.write("\n")
+                out_file.write(f"============ START SECURITY SCAN REPORT - <{key}> ====================")
+                out_file.write("\n")
+                out_file.write(f"Successful:{value.is_ok}\n")
+                out_file.write(f"Full report:{value.report_dir}\n")
+                out_file.write(f"Summary:\n")
+                out_file.write(value.summary)
+                out_file.write("\n")
+                out_file.write(f"============ END SECURITY SCAN REPORT - <{key}> ====================")
+                out_file.write("\n")
+
+
+class SecurityScanner(DockerFlavorBuildBase, SecurityScanParameter):
+
+    def get_goals(self):
+        return {"security_scan"}
+
+    def get_release_task(self):
+        return self.create_build_tasks(not build_config().force_rebuild)
+
+    def run_task(self):
+        tasks = self.get_release_task()
+
+        tasks_futures = yield from self.run_dependencies(tasks)
+        task_results = self.get_values_from_futures(tasks_futures)
+        flavor_path = Path(self.flavor_path)
+        report_path = self.report_path.joinpath(flavor_path.name)
+        report_path.mkdir(parents=True, exist_ok=True)
+        report_path_abs = report_path.absolute()
+        result = ScanResult(is_ok=False, summary="", report_dir=report_path_abs)
+        assert len(task_results.values()) == 1
+        for task_result in task_results.values():
+            self.logger.info(f"Running security run on image: {task_result.get_target_complete_name()}, report path: "
+                             f"{report_path_abs}")
+
+            report_local_path = "/report"
+            with ContextDockerClient() as docker_client:
+                result_container = docker_client.containers.run(task_result.get_target_complete_name(),
+                                                                command=report_local_path,
+                                                                detach=True, stderr=True)
+                try:
+                    logs = result_container.logs(follow=True).decode("UTF-8")
+                    result_container_result = result_container.wait()
+                    #We don't use mount binding here to exchange the report files, but download them from the container
+                    #Thus we avoid that the files are created by root
+                    self._write_report(result_container, report_path_abs, report_local_path)
+                    result = ScanResult(is_ok=(result_container_result["StatusCode"] == 0),
+                                        summary=logs, report_dir=report_path_abs)
+                finally:
+                    result_container.remove()
+
+        self.return_object(result)
+
+    def _write_report(self, container: Container, report_path_abs: Path, report_local_path: str):
+        tar_file_path = report_path_abs / 'report.tar'
+        with open(tar_file_path, 'wb') as tar_file:
+            bits, stat = container.get_archive(report_local_path)
+            for chunk in bits:
+                tar_file.write(chunk)
+        with tarfile.open(tar_file_path) as tar_file:
+            tar_file.extractall(path=report_path_abs)
+
+
+class AllScanResult:
+    def __init__(self, scan_results_per_flavor: Dict[str, ScanResult]):
+        self.scan_results_per_flavor = scan_results_per_flavor
+        self.scans_are_ok = all(scan_result.is_ok
+                                for scan_result
+                                in scan_results_per_flavor.values())
+
+    def get_error_scans_msg(self):
+        return [f"{key}: '{value.summary}'" for key, value in self.scan_results_per_flavor.items() if not value.is_ok]

exasol_script_languages_container_tool/lib/tasks/security_scan/security_scan_parameter.py

@@ -0,0 +1,6 @@
+import luigi
+from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import DependencyLoggerBaseTask
+
+
+class SecurityScanParameter(DependencyLoggerBaseTask):
+    report_path = luigi.Parameter()

exasol_script_languages_container_tool/main.py

@@ -5,7 +5,7 @@
     build_test_container
 
 from exasol_script_languages_container_tool.cli.commands import build, clean_all_images, clean_flavor_images, export, \
-    generate_language_activation, push, run_db_test, save, upload, clean
+    generate_language_activation, push, run_db_test, save, upload, clean, security_scan
 
 if __name__ == '__main__':
     # Required to announce the commands to click
@@ -21,5 +21,6 @@
                 run_db_test,
                 save,
                 upload,
-                clean]
+                clean,
+                security_scan]
     cli()

setup.py

@@ -13,6 +13,7 @@
  'exasol_script_languages_container_tool.lib.tasks.export',
  'exasol_script_languages_container_tool.lib.tasks.push',
  'exasol_script_languages_container_tool.lib.tasks.save',
+ 'exasol_script_languages_container_tool.lib.tasks.security_scan',
  'exasol_script_languages_container_tool.lib.tasks.test',
  'exasol_script_languages_container_tool.lib.tasks.upload',
  'exasol_script_languages_container_tool.lib.utils']

test/resources/real-test-flavor/real_flavor_base/build_steps.py

@@ -122,15 +122,27 @@ def requires_tasks(self):
     def get_path_in_flavor(self):
         return "flavor_base"
 
-
 class AnalyzeRelease(DockerFlavorAnalyzeImageTask):
     def get_build_step(self) -> str:
         return "release"
 
     def requires_tasks(self):
         return {"flavor_customization": AnalyzeFlavorCustomization,
                 "build_run": AnalyzeBuildRun,
+                "language_deps": AnalyzeLanguageDeps,
                 "language_deps": AnalyzeLanguageDeps}
 
     def get_path_in_flavor(self):
         return "flavor_base"
+
+
+class SecurityScan(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "security_scan"
+
+    def requires_tasks(self):
+        return {"release": AnalyzeRelease}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+

test/resources/real-test-flavor/real_flavor_base/security_scan/Dockerfile

@@ -0,0 +1,6 @@
+FROM {{release}}
+
+RUN echo "Building security scan..."
+
+COPY security_scan/test.sh /test.sh
+ENTRYPOINT ["/test.sh"]

test/resources/real-test-flavor/real_flavor_base/security_scan/test.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+echo Running scan...
+mkdir -p $1
+echo Report 123 >> "$1/report.txt"

test/test_security_scan.py

@@ -0,0 +1,35 @@
+import unittest
+from pathlib import Path
+
+import utils as exaslct_utils
+from exasol_integration_test_docker_environment.testing import utils
+
+
+class DockerSecurityScanTest(unittest.TestCase):
+
+    def setUp(self):
+        print(f"SetUp {self.__class__.__name__}")
+        self.test_environment = exaslct_utils.ExaslctTestEnvironmentWithCleanUp(self, exaslct_utils.EXASLCT_DEFAULT_BIN)
+        self.test_environment.clean_images()
+
+    def tearDown(self):
+        utils.close_environments(self.test_environment)
+
+    def test_docker_build(self):
+        command = f"{self.test_environment.executable} security-scan"
+        completed_process = self.test_environment.run_command(command,
+                                                              track_task_dependencies=True, capture_output=True)
+        output = completed_process.stdout.decode("UTF-8")
+        self.assertIn("============ START SECURITY SCAN REPORT - ", output)
+        self.assertIn("Running scan...", output)
+        self.assertIn("============ END SECURITY SCAN REPORT - ", output)
+
+        report = Path(self.test_environment.temp_dir, "security_scan", "test-flavor", "report", "report.txt")
+        self.assertTrue(report.exists())
+        with open(report) as report_file:
+            report_result = report_file.read()
+            self.assertIn("Report 123", report_result)
+
+
+if __name__ == '__main__':
+    unittest.main()