From 875164b749d8ad483baec54047cbb4c30396df4d Mon Sep 17 00:00:00 2001 From: Tim Yarkov Date: Tue, 4 Apr 2023 20:48:37 +1000 Subject: [PATCH] feat: multi build tool detection Signed-off-by: Tim Yarkov --- scripts/dev_scripts/integration_tests.sh | 2 +- .../dependency_analyzer/cyclonedx_mvn.py | 2 - src/macaron/slsa_analyzer/analyze_context.py | 3 +- src/macaron/slsa_analyzer/analyzer.py | 123 ++++++----- .../build_tool/base_build_tool.py | 62 +----- .../checks/build_as_code_check.py | 77 +++++-- .../checks/build_script_check.py | 23 +- .../checks/build_service_check.py | 208 +++++++++++------- src/macaron/slsa_analyzer/specs/build_spec.py | 4 +- .../cyclonedx_timyarkov_multibuild_test.json | 8 + .../multibuild_test/multibuild_test.json | 38 ++-- .../checks/test_build_as_code_check.py | 67 ++++-- .../checks/test_build_script_check.py | 12 +- .../checks/test_build_service_check.py | 61 +++-- tests/slsa_analyzer/checks/test_vcs_check.py | 3 +- 15 files changed, 394 insertions(+), 299 deletions(-) diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 7e73d8f3e..a2da6f44e 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -69,7 +69,7 @@ python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail echo -e "\n----------------------------------------------------------------------------------" echo "timyarkov/multibuild_test: Analyzing the repo path, the branch name and the commit digest" -echo "with dependency resolution using cyclonedx Gradle plugin (default)." +echo "with dependency resolution using cyclonedx Gradle and Maven plugins (defaults)." echo -e "----------------------------------------------------------------------------------\n" JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/multibuild_test/multibuild_test.json JSON_RESULT=$WORKSPACE/output/reports/github_com/timyarkov/multibuild_test/multibuild_test.json diff --git a/src/macaron/dependency_analyzer/cyclonedx_mvn.py b/src/macaron/dependency_analyzer/cyclonedx_mvn.py index 73ca509d2..cfd195746 100644 --- a/src/macaron/dependency_analyzer/cyclonedx_mvn.py +++ b/src/macaron/dependency_analyzer/cyclonedx_mvn.py @@ -47,8 +47,6 @@ def get_cmd(self) -> list: f"org.cyclonedx:cyclonedx-maven-plugin:{self.tool_version}:makeAggregateBom", "-D", "includeTestScope=true", - "-f", - self.repo_path, ] def collect_dependencies(self, dir_path: str) -> dict[str, DependencyInfo]: diff --git a/src/macaron/slsa_analyzer/analyze_context.py b/src/macaron/slsa_analyzer/analyze_context.py index 474ec9986..a2658caed 100644 --- a/src/macaron/slsa_analyzer/analyze_context.py +++ b/src/macaron/slsa_analyzer/analyze_context.py @@ -13,7 +13,6 @@ from pydriller.git import Git from macaron.database.table_definitions import RepositoryTable, SLSALevelTable -from macaron.slsa_analyzer.build_tool.base_build_tool import NoneBuildTool from macaron.slsa_analyzer.checks.check_result import CheckResult, CheckResultType from macaron.slsa_analyzer.git_service import BaseGitService from macaron.slsa_analyzer.git_service.base_git_service import NoneGitService @@ -117,7 +116,7 @@ def __init__( # Add the data computed at runtime to the dynamic_data attribute. self.dynamic_data: ChecksOutputs = ChecksOutputs( git_service=NoneGitService(), - build_spec=BuildSpec(tool=NoneBuildTool()), + build_spec=BuildSpec(tools=[]), ci_services=[], is_inferred_prov=True, expectation=None, diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index 75fee662e..2605ae652 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -32,7 +32,6 @@ from macaron.slsa_analyzer import git_url from macaron.slsa_analyzer.analyze_context import AnalyzeContext from macaron.slsa_analyzer.build_tool import BUILD_TOOLS -from macaron.slsa_analyzer.build_tool.base_build_tool import NoneBuildTool # To load all checks into the registry from macaron.slsa_analyzer.checks import * # pylint: disable=wildcard-import,unused-wildcard-import # noqa: F401,F403 @@ -246,71 +245,74 @@ def resolve_dependencies(self, main_ctx: AnalyzeContext, sbom_path: str) -> dict deps_resolved: dict[str, DependencyInfo] = {} - build_tool = main_ctx.dynamic_data["build_spec"]["tool"] - if not build_tool or isinstance(build_tool, NoneBuildTool): - logger.info("Unable to find a valid build tool.") + build_tools = main_ctx.dynamic_data["build_spec"]["tools"] + if not build_tools: + logger.info("Unable to find any valid build tools.") return {} - try: - dep_analyzer = build_tool.get_dep_analyzer(main_ctx.repo_path) - except DependencyAnalyzerError as error: - logger.error("Unable to find a dependency analyzer: %s", error) - return {} + # Grab dependencies for each build tool, collate all into the deps_resolved + for tool in build_tools: + try: + dep_analyzer = tool.get_dep_analyzer(main_ctx.repo_path) + except DependencyAnalyzerError as error: + logger.error("Unable to find a dependency analyzer for %s: %s", tool.name, error) + continue + + if isinstance(dep_analyzer, NoneDependencyAnalyzer): + logger.info( + "Dependency analyzer is not available for %s", + tool.name, + ) + continue - if isinstance(dep_analyzer, NoneDependencyAnalyzer): + # Start resolving dependencies. logger.info( - "Dependency analyzer is not available for %s", - main_ctx.dynamic_data["build_spec"]["tool"].name, + "Running %s version %s dependency analyzer on %s", + dep_analyzer.tool_name, + dep_analyzer.tool_version, + main_ctx.repo_path, ) - return {} - # Start resolving dependencies. - logger.info( - "Running %s version %s dependency analyzer on %s", - dep_analyzer.tool_name, - dep_analyzer.tool_version, - main_ctx.repo_path, - ) + log_path = os.path.join( + global_config.build_log_path, + f"{main_ctx.repo_name}.{dep_analyzer.tool_name}.log", + ) - log_path = os.path.join( - global_config.build_log_path, - f"{main_ctx.repo_name}.{dep_analyzer.tool_name}.log", - ) + # Clean up existing SBOM files. + dep_analyzer.remove_sboms(main_ctx.repo_path) + + commands = dep_analyzer.get_cmd() + working_dirs: Iterable[Path] = tool.get_build_dirs(main_ctx.repo_path) + for working_dir in working_dirs: + # Get the absolute path to use as the working dir in the subprocess. + working_dir = Path(main_ctx.repo_path).joinpath(working_dir) + try: + # Suppressing Bandit's B603 report because the repo paths are validated. + analyzer_output = subprocess.run( # nosec B603 + commands, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True, + cwd=str(working_dir), + timeout=defaults.getint("dependency.resolver", "timeout", fallback=1200), + ) + with open(log_path, mode="a", encoding="utf-8") as log_file: + log_file.write(analyzer_output.stdout.decode("utf-8")) - # Clean up existing SBOM files. - dep_analyzer.remove_sboms(main_ctx.repo_path) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: + logger.error(error) + with open(log_path, mode="a", encoding="utf-8") as log_file: + log_file.write(error.output.decode("utf-8")) + except FileNotFoundError as error: + logger.error(error) + + # We collect the generated SBOM as a best effort, even if the build exits with errors. + # TODO: add improvements to help the SBOM build succeed as much as possible. + # Update deps_resolved with new dependencies. + deps_resolved |= dep_analyzer.collect_dependencies(str(working_dir)) + + logger.info("Stored dependency resolver log for %s to %s.", dep_analyzer.tool_name, log_path) - commands = dep_analyzer.get_cmd() - working_dirs: Iterable[Path] = build_tool.get_build_dirs(main_ctx.repo_path) - for working_dir in working_dirs: - # Get the absolute path to use as the working dir in the subprocess. - working_dir = Path(main_ctx.repo_path).joinpath(working_dir) - try: - # Suppressing Bandit's B603 report because the repo paths are validated. - analyzer_output = subprocess.run( # nosec B603 - commands, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - check=True, - cwd=str(working_dir), - timeout=defaults.getint("dependency.resolver", "timeout", fallback=1200), - ) - with open(log_path, mode="a", encoding="utf-8") as log_file: - log_file.write(analyzer_output.stdout.decode("utf-8")) - - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: - logger.error(error) - with open(log_path, mode="a", encoding="utf-8") as log_file: - log_file.write(error.output.decode("utf-8")) - except FileNotFoundError as error: - logger.error(error) - - # We collect the generated SBOM as a best effort, even if the build exits with errors. - # TODO: add improvements to help the SBOM build succeed as much as possible. - # Update deps_resolved with new dependencies. - deps_resolved |= dep_analyzer.collect_dependencies(str(working_dir)) - - logger.info("Stored dependency resolver log to %s.", log_path) return deps_resolved def run_single(self, config: Configuration, existing_records: Optional[dict[str, Record]] = None) -> Record: @@ -649,12 +651,11 @@ def perform_checks(self, analyze_ctx: AnalyzeContext) -> dict[str, CheckResult]: if build_tool.is_detected(analyze_ctx.git_obj.path): logger.info("The repo uses %s build tool.", build_tool.name) - analyze_ctx.dynamic_data["build_spec"]["tool"] = build_tool - break + analyze_ctx.dynamic_data["build_spec"]["tools"].append(build_tool) - if not analyze_ctx.dynamic_data["build_spec"].get("tool"): + if not analyze_ctx.dynamic_data["build_spec"]["tools"]: logger.info( - "Cannot discover any build tool for %s or the build tool is not supported.", + "Cannot discover any build tools for %s or the build tools found are not supported.", analyze_ctx.repo_full_name, ) diff --git a/src/macaron/slsa_analyzer/build_tool/base_build_tool.py b/src/macaron/slsa_analyzer/build_tool/base_build_tool.py index 2235b8e5d..43ebe7067 100644 --- a/src/macaron/slsa_analyzer/build_tool/base_build_tool.py +++ b/src/macaron/slsa_analyzer/build_tool/base_build_tool.py @@ -10,7 +10,7 @@ from collections.abc import Iterable from pathlib import Path -from macaron.dependency_analyzer import DependencyAnalyzer, NoneDependencyAnalyzer +from macaron.dependency_analyzer import DependencyAnalyzer logger: logging.Logger = logging.getLogger(__name__) @@ -171,63 +171,3 @@ def get_build_dirs(self, repo_path: str) -> Iterable[Path]: except StopIteration: pass - - -class NoneBuildTool(BaseBuildTool): - """This class can be used to initialize an empty build tool.""" - - def __init__(self) -> None: - """Initialize instance.""" - super().__init__(name="") - - def is_detected(self, repo_path: str) -> bool: - """Return True if this build tool is used in the target repo. - - Parameters - ---------- - repo_path : str - The path to the target repo. - - Returns - ------- - bool - True if this build tool is detected, else False. - """ - return False - - def prepare_config_files(self, wrapper_path: str, build_dir: str) -> bool: - """Prepare the necessary wrapper files for running the build. - - This method will return False if there is any errors happened during operation. - - Parameters - ---------- - wrapper_path : str - The path where all necessary wrapper files are located. - build_dir : str - The path of the build dir. This is where all files are copied to. - - Returns - ------- - bool - True if succeed else False. - """ - return False - - def load_defaults(self) -> None: - """Load the default values from defaults.ini.""" - - def get_dep_analyzer(self, repo_path: str) -> DependencyAnalyzer: - """Create an invalid DependencyAnalyzer for the empty build tool. - - Parameters - ---------- - repo_path: str - The path to the target repo. - - Returns - ------- - DependencyAnalyzer - The DependencyAnalyzer object. - """ - return NoneDependencyAnalyzer() diff --git a/src/macaron/slsa_analyzer/checks/build_as_code_check.py b/src/macaron/slsa_analyzer/checks/build_as_code_check.py index e25768049..f0455d9c8 100644 --- a/src/macaron/slsa_analyzer/checks/build_as_code_check.py +++ b/src/macaron/slsa_analyzer/checks/build_as_code_check.py @@ -13,7 +13,7 @@ from macaron.database.database_manager import ORMBase from macaron.database.table_definitions import CheckFactsTable from macaron.slsa_analyzer.analyze_context import AnalyzeContext -from macaron.slsa_analyzer.build_tool.base_build_tool import BaseBuildTool, NoneBuildTool +from macaron.slsa_analyzer.build_tool.base_build_tool import BaseBuildTool from macaron.slsa_analyzer.checks.base_check import BaseCheck from macaron.slsa_analyzer.checks.check_result import CheckResult, CheckResultType from macaron.slsa_analyzer.ci_service.base_ci_service import NoneCIService @@ -24,6 +24,7 @@ from macaron.slsa_analyzer.ci_service.travis import Travis from macaron.slsa_analyzer.registry import registry from macaron.slsa_analyzer.slsa_req import ReqName +from macaron.slsa_analyzer.specs.ci_spec import CIInfo logger: logging.Logger = logging.getLogger(__name__) @@ -108,13 +109,19 @@ def _has_deploy_command(self, commands: list[list[str]], build_tool: BaseBuildTo return str(com) return "" - def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResultType: - """Implement the check in this method. + def _check_build_tool( + self, build_tool: BaseBuildTool, ctx: AnalyzeContext, ci_services: list[CIInfo], check_result: CheckResult + ) -> CheckResultType | None: + """Run the check for a single build tool to determine if "build as code" holds for it. Parameters ---------- + build_tool: BaseBuildTool + The build tool to run the check for. ctx : AnalyzeContext The object containing processed data for the target repo. + ci_services: list[CIInfo] + List of CI services in use. check_result : CheckResult The object containing result data of a check. @@ -123,12 +130,7 @@ def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResu CheckResultType The result type of the check (e.g. PASSED). """ - # Get the build tool identified by the mcn_version_control_system_1, which we depend on. - build_tool = ctx.dynamic_data["build_spec"].get("tool") - ci_services = ctx.dynamic_data["ci_services"] - - # Checking if a build tool is discovered for this repo. - if build_tool and not isinstance(build_tool, NoneBuildTool): + if build_tool: for ci_info in ci_services: ci_service = ci_info["service"] # Checking if a CI service is discovered for this repo. @@ -246,7 +248,7 @@ def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResu predicate["invocation"]["configSource"]["digest"]["sha1"] = ctx.commit_sha predicate["invocation"]["configSource"]["entryPoint"] = trigger_link predicate["metadata"]["buildInvocationId"] = html_url - check_result["result_tables"] = [ + check_result["result_tables"].append( BuildAsCodeTable( build_tool_name=build_tool.name, ci_service_name=ci_service.name, @@ -254,7 +256,8 @@ def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResu deploy_command=deploy_cmd, build_status_url=html_url, ) - ] + ) + return CheckResultType.PASSED # We currently don't parse these CI configuration files. @@ -280,23 +283,59 @@ def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResu ] = f"{ctx.remote_path}@refs/heads/{ctx.branch_name}" predicate["invocation"]["configSource"]["digest"]["sha1"] = ctx.commit_sha predicate["invocation"]["configSource"]["entryPoint"] = config_name - check_result["result_tables"] = [ + check_result["result_tables"].append( BuildAsCodeTable( build_tool_name=build_tool.name, ci_service_name=ci_service.name, deploy_command=deploy_kw, ) - ] + ) return CheckResultType.PASSED - pass_msg = f"The target repository does not use {build_tool.name} to deploy." - check_result["justification"].append(pass_msg) - check_result["result_tables"] = [BuildAsCodeTable(build_tool_name=build_tool.name)] + pass_msg = f"The target repository does not use {build_tool.name} to deploy." + check_result["justification"].append(pass_msg) + check_result["result_tables"].append(BuildAsCodeTable(build_tool_name=build_tool.name)) + return CheckResultType.FAILED + + def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResultType: + """Implement the check in this method. + + Parameters + ---------- + ctx : AnalyzeContext + The object containing processed data for the target repo. + check_result : CheckResult + The object containing result data of a check. + + Returns + ------- + CheckResultType + The result type of the check (e.g. PASSED). + """ + # Get the build tool identified by the mcn_version_control_system_1, which we depend on. + build_tools = ctx.dynamic_data["build_spec"]["tools"] + + if not build_tools: + check_result["result_tables"] = [BuildAsCodeTable()] + failed_msg = "The target repository does not have any build tools." + check_result["justification"].append(failed_msg) return CheckResultType.FAILED - check_result["result_tables"] = [BuildAsCodeTable()] - failed_msg = "The target repository does not have a build tool." - check_result["justification"].append(failed_msg) + ci_services = ctx.dynamic_data["ci_services"] + + # Check if "build as code" holds for each build tool. + for tool in build_tools: + res = self._check_build_tool(tool, ctx, ci_services, check_result) + + if res == CheckResultType.PASSED: + # Since the check passing is contingent on at least one passing, + # short-circuit if we do get a pass + # TODO: When more sophisticated build tool detection is + # implemented, consider whether this should be one fail = whole + # check fails instead + return CheckResultType.PASSED + + # No passes, so overall fail return CheckResultType.FAILED diff --git a/src/macaron/slsa_analyzer/checks/build_script_check.py b/src/macaron/slsa_analyzer/checks/build_script_check.py index 242bdc61c..b5d295e12 100644 --- a/src/macaron/slsa_analyzer/checks/build_script_check.py +++ b/src/macaron/slsa_analyzer/checks/build_script_check.py @@ -11,7 +11,6 @@ from macaron.database.database_manager import ORMBase from macaron.database.table_definitions import CheckFactsTable from macaron.slsa_analyzer.analyze_context import AnalyzeContext -from macaron.slsa_analyzer.build_tool.base_build_tool import NoneBuildTool from macaron.slsa_analyzer.checks.base_check import BaseCheck from macaron.slsa_analyzer.checks.check_result import CheckResult, CheckResultType from macaron.slsa_analyzer.registry import registry @@ -59,20 +58,22 @@ def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResu CheckResultType The result type of the check (e.g. PASSED). """ - build_tool = ctx.dynamic_data["build_spec"].get("tool") + build_tools = ctx.dynamic_data["build_spec"]["tools"] - # Check if a build tool is discovered for this repo. + if not build_tools: + failed_msg = "The target repository does not have any build tools." + check_result["justification"].append(failed_msg) + return CheckResultType.FAILED + + # Check if any build tools are discovered for this repo. # TODO: look for build commands in the bash scripts. Currently - # we parse bash scripts that are reachable through CI only. - if build_tool and not isinstance(build_tool, NoneBuildTool): - pass_msg = f"The target repository uses build tool {build_tool.name}." + # we parse bash scripts that are reachable through CI only. + for tool in build_tools: + pass_msg = f"The target repository uses build tool {tool.name}." check_result["justification"].append(pass_msg) - check_result["result_tables"] = [BuildScriptTable(build_tool_name=build_tool.name)] - return CheckResultType.PASSED + check_result["result_tables"].append(BuildScriptTable(build_tool_name=tool.name)) - failed_msg = "The target repository does not have a build tool." - check_result["justification"].append(failed_msg) - return CheckResultType.FAILED + return CheckResultType.PASSED registry.register(BuildScriptCheck()) diff --git a/src/macaron/slsa_analyzer/checks/build_service_check.py b/src/macaron/slsa_analyzer/checks/build_service_check.py index f4838aba3..7d435b6af 100644 --- a/src/macaron/slsa_analyzer/checks/build_service_check.py +++ b/src/macaron/slsa_analyzer/checks/build_service_check.py @@ -12,7 +12,7 @@ from macaron.database.database_manager import ORMBase from macaron.database.table_definitions import CheckFactsTable from macaron.slsa_analyzer.analyze_context import AnalyzeContext -from macaron.slsa_analyzer.build_tool.base_build_tool import BaseBuildTool, NoneBuildTool +from macaron.slsa_analyzer.build_tool.base_build_tool import BaseBuildTool from macaron.slsa_analyzer.checks.base_check import BaseCheck from macaron.slsa_analyzer.checks.check_result import CheckResult, CheckResultType from macaron.slsa_analyzer.ci_service.base_ci_service import NoneCIService @@ -22,6 +22,7 @@ from macaron.slsa_analyzer.ci_service.travis import Travis from macaron.slsa_analyzer.registry import registry from macaron.slsa_analyzer.slsa_req import ReqName +from macaron.slsa_analyzer.specs.ci_spec import CIInfo logger: logging.Logger = logging.getLogger(__name__) @@ -95,121 +96,164 @@ def _has_build_command(self, commands: list[list[str]], build_tool: BaseBuildToo return str(com) return "" - def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResultType: - """Implement the check in this method. + def _check_build_tool( + self, build_tool: BaseBuildTool, ctx: AnalyzeContext, check_result: CheckResult, ci_services: list[CIInfo] + ) -> CheckResultType | None: + """ + Check that a single build tool has a build service associated to it. Parameters ---------- + build_tool : BaseBuildTool + Build tool to analyse for ctx : AnalyzeContext The object containing processed data for the target repo. check_result : CheckResult The object containing result data of a check. + ci_services: list[CIInfo] + List of objects containing information on present CI services. Returns ------- CheckResultType The result type of the check (e.g. PASSED). """ - build_tool = ctx.dynamic_data["build_spec"].get("tool") - ci_services = ctx.dynamic_data["ci_services"] + for ci_info in ci_services: + ci_service = ci_info["service"] + # Checking if a CI service is discovered for this repo. + if isinstance(ci_service, NoneCIService): + continue + for bash_cmd in ci_info["bash_commands"]: + build_cmd = self._has_build_command(bash_cmd["commands"], build_tool) + if build_cmd: + # Get the permalink and HTML hyperlink tag of the CI file that triggered the bash command. + trigger_link = ci_service.api_client.get_file_link( + ctx.repo_full_name, + ctx.commit_sha, + ci_service.api_client.get_relative_path_of_workflow(os.path.basename(bash_cmd["CI_path"])), + ) + # Get the permalink and HTML hyperlink tag of the source file of the bash command. + bash_source_link = ci_service.api_client.get_file_link( + ctx.repo_full_name, ctx.commit_sha, bash_cmd["caller_path"] + ) - # Checking if a build tool is discovered for this repo. - if build_tool and not isinstance(build_tool, NoneBuildTool): - for ci_info in ci_services: - ci_service = ci_info["service"] - # Checking if a CI service is discovered for this repo. - if isinstance(ci_service, NoneCIService): - continue - for bash_cmd in ci_info["bash_commands"]: - build_cmd = self._has_build_command(bash_cmd["commands"], build_tool) - if build_cmd: - # Get the permalink and HTML hyperlink tag of the CI file that triggered the bash command. - trigger_link = ci_service.api_client.get_file_link( - ctx.repo_full_name, - ctx.commit_sha, - ci_service.api_client.get_relative_path_of_workflow(os.path.basename(bash_cmd["CI_path"])), - ) - # Get the permalink and HTML hyperlink tag of the source file of the bash command. - bash_source_link = ci_service.api_client.get_file_link( - ctx.repo_full_name, ctx.commit_sha, bash_cmd["caller_path"] + html_url = ci_service.has_latest_run_passed( + ctx.repo_full_name, + ctx.branch_name, + ctx.commit_sha, + ctx.commit_date, + os.path.basename(bash_cmd["CI_path"]), + ) + + justification: list[str | dict[str, str]] = [ + { + f"The target repository uses build tool {build_tool.name} to deploy": bash_source_link, + "The build is triggered by": trigger_link, + }, + f"Build command: {build_cmd}", + {"The status of the build can be seen at": html_url} + if html_url + else "However, could not find a passing workflow run.", + ] + check_result["justification"].extend(justification) + check_result["result_tables"].append( + BuildServiceTable( + build_tool_name=build_tool.name, + build_trigger=trigger_link, + ci_service_name=ci_service.name, ) + ) - html_url = ci_service.has_latest_run_passed( - ctx.repo_full_name, - ctx.branch_name, - ctx.commit_sha, - ctx.commit_date, - os.path.basename(bash_cmd["CI_path"]), + if ctx.dynamic_data["is_inferred_prov"] and ci_info["provenances"]: + predicate = ci_info["provenances"][0]["predicate"] + predicate["buildType"] = f"Custom {ci_service.name}" + predicate["builder"]["id"] = bash_source_link + predicate["invocation"]["configSource"][ + "uri" + ] = f"{ctx.remote_path}@refs/heads/{ctx.branch_name}" + predicate["invocation"]["configSource"]["digest"]["sha1"] = ctx.commit_sha + predicate["invocation"]["configSource"]["entryPoint"] = trigger_link + predicate["metadata"]["buildInvocationId"] = html_url + return CheckResultType.PASSED + + # We currently don't parse these CI configuration files. + # We just look for a keyword for now. + for unparsed_ci in (Jenkins, Travis, CircleCI, GitLabCI): + if isinstance(ci_service, unparsed_ci): + if build_tool.ci_build_kws[ci_service.name]: + _, config_name = ci_service.has_kws_in_config( + build_tool.ci_build_kws[ci_service.name], repo_path=ctx.repo_path ) + if not config_name: + break - justification: list[str | dict[str, str]] = [ - { - f"The target repository uses build tool {build_tool.name} to deploy": bash_source_link, - "The build is triggered by": trigger_link, - }, - f"Build command: {build_cmd}", - {"The status of the build can be seen at": html_url} - if html_url - else "However, could not find a passing workflow run.", - ] - check_result["justification"].extend(justification) - check_result["result_tables"] = [ + check_result["justification"].append( + f"The target repository uses " + f"build tool {build_tool.name} in {ci_service.name} to " + f"build." + ) + check_result["result_tables"].append( BuildServiceTable( build_tool_name=build_tool.name, - build_trigger=trigger_link, ci_service_name=ci_service.name, ) - ] + ) if ctx.dynamic_data["is_inferred_prov"] and ci_info["provenances"]: predicate = ci_info["provenances"][0]["predicate"] predicate["buildType"] = f"Custom {ci_service.name}" - predicate["builder"]["id"] = bash_source_link + predicate["builder"]["id"] = config_name predicate["invocation"]["configSource"][ "uri" ] = f"{ctx.remote_path}@refs/heads/{ctx.branch_name}" predicate["invocation"]["configSource"]["digest"]["sha1"] = ctx.commit_sha - predicate["invocation"]["configSource"]["entryPoint"] = trigger_link - predicate["metadata"]["buildInvocationId"] = html_url + predicate["invocation"]["configSource"]["entryPoint"] = config_name return CheckResultType.PASSED - # We currently don't parse these CI configuration files. - # We just look for a keyword for now. - for unparsed_ci in (Jenkins, Travis, CircleCI, GitLabCI): - if isinstance(ci_service, unparsed_ci): - if build_tool.ci_build_kws[ci_service.name]: - _, config_name = ci_service.has_kws_in_config( - build_tool.ci_build_kws[ci_service.name], repo_path=ctx.repo_path - ) - if not config_name: - break - - check_result["justification"].append( - f"The target repository uses " - f"build tool {build_tool.name} in {ci_service.name} to " - f"build." - ) - check_result["result_tables"] = [ - BuildServiceTable( - build_tool_name=build_tool.name, - ci_service_name=ci_service.name, - ) - ] - - if ctx.dynamic_data["is_inferred_prov"] and ci_info["provenances"]: - predicate = ci_info["provenances"][0]["predicate"] - predicate["buildType"] = f"Custom {ci_service.name}" - predicate["builder"]["id"] = config_name - predicate["invocation"]["configSource"][ - "uri" - ] = f"{ctx.remote_path}@refs/heads/{ctx.branch_name}" - predicate["invocation"]["configSource"]["digest"]["sha1"] = ctx.commit_sha - predicate["invocation"]["configSource"]["entryPoint"] = config_name - return CheckResultType.PASSED - - fail_msg = "The target repository does not have a build service." + # Nothing found; fail + fail_msg = f"The target repository does not have a build service for {build_tool}." check_result["justification"].append(fail_msg) return CheckResultType.FAILED + def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResultType: + """Implement the check in this method. + + Parameters + ---------- + ctx : AnalyzeContext + The object containing processed data for the target repo. + check_result : CheckResult + The object containing result data of a check. + + Returns + ------- + CheckResultType + The result type of the check (e.g. PASSED). + """ + build_tools = ctx.dynamic_data["build_spec"]["tools"] + ci_services = ctx.dynamic_data["ci_services"] + + # Checking if at least one build tool is discovered for this repo. + # No build tools is auto fail. + # TODO: When more sophisticated build tool detection is + # implemented, consider whether this should be one fail = whole + # check fails instead + all_passing = False + + for tool in build_tools: + res = self._check_build_tool(tool, ctx, check_result, ci_services) + + if res == CheckResultType.PASSED: + # Pass at some point so treat as entire check pass; short-circuit + all_passing = True + break + + if not all_passing or not build_tools: + fail_msg = "The target repository does not have a build service for at least one build tool." + check_result["justification"].append(fail_msg) + return CheckResultType.FAILED + + return CheckResultType.PASSED + registry.register(BuildServiceCheck()) diff --git a/src/macaron/slsa_analyzer/specs/build_spec.py b/src/macaron/slsa_analyzer/specs/build_spec.py index eda968ef5..c1871c5c8 100644 --- a/src/macaron/slsa_analyzer/specs/build_spec.py +++ b/src/macaron/slsa_analyzer/specs/build_spec.py @@ -47,8 +47,8 @@ class BuildSpec(TypedDict): # sourceRmFiles: str ## Rebuild environment prerequisites - tool: BaseBuildTool - """The build tool used for building this artifact.""" + tools: list[BaseBuildTool] + """The build tools used for building this artifact.""" # jdk: str # newline: str ## crlf for Windows, lf for Unix diff --git a/tests/dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test.json b/tests/dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test.json index bdd94588f..bd0414a34 100644 --- a/tests/dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test.json +++ b/tests/dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test.json @@ -14,5 +14,13 @@ "digest": "", "note": "https://github.com/spring-projects/spring-boot is already analyzed.", "available": "DUPLICATED REPO URL" + }, + { + "id": "com.google.code.gson:gson", + "path": "https://github.com/google/gson", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" } ] diff --git a/tests/e2e/expected_results/multibuild_test/multibuild_test.json b/tests/e2e/expected_results/multibuild_test/multibuild_test.json index f42a03c84..2244d9454 100644 --- a/tests/e2e/expected_results/multibuild_test/multibuild_test.json +++ b/tests/e2e/expected_results/multibuild_test/multibuild_test.json @@ -1,6 +1,6 @@ { "metadata": { - "timestamps": "2023-06-14 15:45:41" + "timestamps": "2023-06-18 21:51:40" }, "target": { "info": { @@ -177,40 +177,40 @@ } }, "dependencies": { - "analyzed_deps": 2, - "unique_dep_repos": 1, + "analyzed_deps": 3, + "unique_dep_repos": 2, "checks_summary": [ { - "check_id": "mcn_provenance_expectation_1", - "num_deps_pass": 0 + "check_id": "mcn_build_script_1", + "num_deps_pass": 2 }, { - "check_id": "mcn_provenance_available_1", - "num_deps_pass": 0 + "check_id": "mcn_version_control_system_1", + "num_deps_pass": 2 }, { - "check_id": "mcn_build_as_code_1", - "num_deps_pass": 0 + "check_id": "mcn_build_service_1", + "num_deps_pass": 2 }, { - "check_id": "mcn_version_control_system_1", - "num_deps_pass": 1 + "check_id": "mcn_trusted_builder_level_three_1", + "num_deps_pass": 0 }, { - "check_id": "mcn_trusted_builder_level_three_1", + "check_id": "mcn_provenance_available_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_script_1", - "num_deps_pass": 1 + "check_id": "mcn_build_as_code_1", + "num_deps_pass": 0 }, { "check_id": "mcn_provenance_level_three_1", "num_deps_pass": 0 }, { - "check_id": "mcn_build_service_1", - "num_deps_pass": 1 + "check_id": "mcn_provenance_expectation_1", + "num_deps_pass": 0 } ], "dep_status": [ @@ -225,6 +225,12 @@ "description": "https://github.com/spring-projects/spring-boot is already analyzed.", "report": "", "status": "DUPLICATED REPO URL" + }, + { + "id": "com.google.code.gson:gson", + "description": "Analysis Completed.", + "report": "gson.html", + "status": "AVAILABLE" } ] } diff --git a/tests/slsa_analyzer/checks/test_build_as_code_check.py b/tests/slsa_analyzer/checks/test_build_as_code_check.py index b7a07b0f9..b842e18b0 100644 --- a/tests/slsa_analyzer/checks/test_build_as_code_check.py +++ b/tests/slsa_analyzer/checks/test_build_as_code_check.py @@ -41,7 +41,7 @@ def test_build_as_code_check( ) -> None: """Test the Build As Code Check.""" check = BuildAsCodeCheck() - check_result = CheckResult(justification=[]) # type: ignore + check_result = CheckResult(justification=[], result_tables=[]) # type: ignore bash_commands = BashCommands(caller_path="source_file", CI_path="ci_file", CI_type="github_actions", commands=[[]]) ci_info = CIInfo( service=github_actions_service, @@ -54,22 +54,22 @@ def test_build_as_code_check( # The target repo uses Maven build tool but does not deploy artifacts. use_build_tool = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - use_build_tool.dynamic_data["build_spec"]["tool"] = maven_tool + use_build_tool.dynamic_data["build_spec"]["tools"] = [maven_tool] assert check.run_check(use_build_tool, check_result) == CheckResultType.FAILED # The target repo uses Gradle build tool but does not deploy artifacts. use_build_tool = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - use_build_tool.dynamic_data["build_spec"]["tool"] = gradle_tool + use_build_tool.dynamic_data["build_spec"]["tools"] = [gradle_tool] assert check.run_check(use_build_tool, check_result) == CheckResultType.FAILED # The target repo uses Poetry build tool but does not deploy artifacts. use_build_tool = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - use_build_tool.dynamic_data["build_spec"]["tool"] = poetry_tool + use_build_tool.dynamic_data["build_spec"]["tools"] = [poetry_tool] assert check.run_check(use_build_tool, check_result) == CheckResultType.FAILED # The target repo uses Pip build tool but does not deploy artifacts. use_build_tool = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - use_build_tool.dynamic_data["build_spec"]["tool"] = pip_tool + use_build_tool.dynamic_data["build_spec"]["tools"] = [pip_tool] assert check.run_check(use_build_tool, check_result) == CheckResultType.FAILED # The target repo does not use a build tool. @@ -78,7 +78,7 @@ def test_build_as_code_check( # Use mvn deploy to deploy the artifact. maven_deploy = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - maven_deploy.dynamic_data["build_spec"]["tool"] = maven_tool + maven_deploy.dynamic_data["build_spec"]["tools"] = [maven_tool] bash_commands["commands"] = [["mvn", "deploy"]] maven_deploy.dynamic_data["ci_services"] = [ci_info] assert check.run_check(maven_deploy, check_result) == CheckResultType.PASSED @@ -95,7 +95,7 @@ def test_build_as_code_check( # Use mvn but do not deploy artifacts. no_maven_deploy = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - no_maven_deploy.dynamic_data["build_spec"]["tool"] = maven_tool + no_maven_deploy.dynamic_data["build_spec"]["tools"] = [maven_tool] bash_commands["commands"] = [["mvn", "verify"]] no_maven_deploy.dynamic_data["ci_services"] = [ci_info] assert check.run_check(no_maven_deploy, check_result) == CheckResultType.FAILED @@ -107,42 +107,42 @@ def test_build_as_code_check( # Use gradle to deploy the artifact. gradle_deploy = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - gradle_deploy.dynamic_data["build_spec"]["tool"] = gradle_tool + gradle_deploy.dynamic_data["build_spec"]["tools"] = [gradle_tool] bash_commands["commands"] = [["./gradlew", "publishToSonatype"]] gradle_deploy.dynamic_data["ci_services"] = [ci_info] assert check.run_check(gradle_deploy, check_result) == CheckResultType.PASSED # Use poetry publish to publish the artifact. poetry_publish = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - poetry_publish.dynamic_data["build_spec"]["tool"] = poetry_tool + poetry_publish.dynamic_data["build_spec"]["tools"] = [poetry_tool] bash_commands["commands"] = [["poetry", "publish"]] poetry_publish.dynamic_data["ci_services"] = [ci_info] assert check.run_check(poetry_publish, check_result) == CheckResultType.PASSED # Use Poetry but do not deploy artifacts. no_poetry_deploy = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - no_poetry_deploy.dynamic_data["build_spec"]["tool"] = poetry_tool + no_poetry_deploy.dynamic_data["build_spec"]["tools"] = [poetry_tool] bash_commands["commands"] = [["poetry", "upload"]] no_poetry_deploy.dynamic_data["ci_services"] = [ci_info] assert check.run_check(no_maven_deploy, check_result) == CheckResultType.FAILED # Use twine upload to deploy the artifact. twine_upload = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - twine_upload.dynamic_data["build_spec"]["tool"] = pip_tool + twine_upload.dynamic_data["build_spec"]["tools"] = [pip_tool] bash_commands["commands"] = [["twine", "upload", "dist/*"]] twine_upload.dynamic_data["ci_services"] = [ci_info] assert check.run_check(twine_upload, check_result) == CheckResultType.PASSED # Use flit publish to deploy the artifact. flit_publish = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - flit_publish.dynamic_data["build_spec"]["tool"] = pip_tool + flit_publish.dynamic_data["build_spec"]["tools"] = [pip_tool] bash_commands["commands"] = [["flit", "publish"]] flit_publish.dynamic_data["ci_services"] = [ci_info] assert check.run_check(flit_publish, check_result) == CheckResultType.PASSED # Test Jenkins. maven_deploy = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - maven_deploy.dynamic_data["build_spec"]["tool"] = maven_tool + maven_deploy.dynamic_data["build_spec"]["tools"] = [maven_tool] ci_info["service"] = jenkins_service bash_commands["commands"] = [] maven_deploy.dynamic_data["ci_services"] = [ci_info] @@ -150,7 +150,7 @@ def test_build_as_code_check( # Test Travis. maven_deploy = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - maven_deploy.dynamic_data["build_spec"]["tool"] = maven_tool + maven_deploy.dynamic_data["build_spec"]["tools"] = [maven_tool] ci_info["service"] = travis_service bash_commands["commands"] = [] maven_deploy.dynamic_data["ci_services"] = [ci_info] @@ -158,7 +158,7 @@ def test_build_as_code_check( # Test Circle CI. maven_deploy = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - maven_deploy.dynamic_data["build_spec"]["tool"] = maven_tool + maven_deploy.dynamic_data["build_spec"]["tools"] = [maven_tool] ci_info["service"] = circle_ci_service bash_commands["commands"] = [] maven_deploy.dynamic_data["ci_services"] = [ci_info] @@ -166,12 +166,39 @@ def test_build_as_code_check( # Test GitLab CI. maven_deploy = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - maven_deploy.dynamic_data["build_spec"]["tool"] = maven_tool + maven_deploy.dynamic_data["build_spec"]["tools"] = [maven_tool] ci_info["service"] = gitlab_ci_service bash_commands["commands"] = [] maven_deploy.dynamic_data["ci_services"] = [ci_info] assert check.run_check(maven_deploy, check_result) == CheckResultType.FAILED + # Using both gradle and maven with valid commands to deploy + gradle_deploy = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) + gradle_deploy.dynamic_data["build_spec"]["tools"] = [gradle_tool, maven_tool] + bash_commands["commands"] = [["./gradlew", "publishToSonatype"], ["mvn", "deploy"]] + gradle_deploy.dynamic_data["ci_services"] = [ci_info] + assert check.run_check(gradle_deploy, check_result) == CheckResultType.PASSED + + # Using both gradle and maven, but maven incorrect (singular failure in a list) + gradle_deploy = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) + gradle_deploy.dynamic_data["build_spec"]["tools"] = [gradle_tool, maven_tool] + bash_commands["commands"] = [["./gradlew", "publishToSonatype"], ["mvn", "verify"]] + gradle_deploy.dynamic_data["ci_services"] = [ci_info] + assert check.run_check(gradle_deploy, check_result) == CheckResultType.PASSED + + # Using both gradle and maven, both incorrect + gradle_deploy = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) + gradle_deploy.dynamic_data["build_spec"]["tools"] = [gradle_tool, maven_tool] + bash_commands["commands"] = [["./gradlew", "build"], ["mvn", "verify"]] + gradle_deploy.dynamic_data["ci_services"] = [ci_info] + assert check.run_check(gradle_deploy, check_result) == CheckResultType.FAILED + + # No build tools present + gradle_deploy = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) + gradle_deploy.dynamic_data["build_spec"]["tools"] = [] + gradle_deploy.dynamic_data["ci_services"] = [ci_info] + assert check.run_check(gradle_deploy, check_result) == CheckResultType.FAILED + def test_gha_workflow_deployment( pip_tool: Pip, @@ -179,7 +206,7 @@ def test_gha_workflow_deployment( ) -> None: """Test the use of verified GitHub Actions to deploy.""" check = BuildAsCodeCheck() - check_result = CheckResult(justification=[]) # type: ignore + check_result = CheckResult(justification=[], result_tables=[]) # type: ignore ci_info = CIInfo( service=github_actions_service, bash_commands=[], @@ -193,7 +220,7 @@ def test_gha_workflow_deployment( # This Github Actions workflow uses gh-action-pypi-publish to publish the artifact. gha_deploy = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - gha_deploy.dynamic_data["build_spec"]["tool"] = pip_tool + gha_deploy.dynamic_data["build_spec"]["tools"] = [pip_tool] gha_deploy.dynamic_data["ci_services"] = [ci_info] root = GitHubNode(name="root", node_type=GHWorkflowType.NONE, source_path="", parsed_obj={}, caller_path="") @@ -251,9 +278,9 @@ def test_travis_ci_deploy( latest_release={}, provenances=[], ) - check_result = CheckResult(justification=[]) # type: ignore + check_result = CheckResult(justification=[], result_tables=[]) # type: ignore gradle_deploy = AnalyzeContext("use_build_tool", str(repo_path.absolute()), MagicMock()) - gradle_deploy.dynamic_data["build_spec"]["tool"] = gradle_tool + gradle_deploy.dynamic_data["build_spec"]["tools"] = [gradle_tool] gradle_deploy.dynamic_data["ci_services"] = [ci_info] assert check.run_check(gradle_deploy, check_result) == expect_result diff --git a/tests/slsa_analyzer/checks/test_build_script_check.py b/tests/slsa_analyzer/checks/test_build_script_check.py index ab2afdda7..2ea28c633 100644 --- a/tests/slsa_analyzer/checks/test_build_script_check.py +++ b/tests/slsa_analyzer/checks/test_build_script_check.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock from macaron.slsa_analyzer.analyze_context import AnalyzeContext +from macaron.slsa_analyzer.build_tool.gradle import Gradle from macaron.slsa_analyzer.build_tool.maven import Maven from macaron.slsa_analyzer.checks.build_script_check import BuildScriptCheck from macaron.slsa_analyzer.checks.check_result import CheckResult, CheckResultType @@ -20,13 +21,20 @@ class TestBuildScriptCheck(MacaronTestCase): def test_build_script_check(self) -> None: """Test the Build Script Check.""" check = BuildScriptCheck() - check_result = CheckResult(justification=[]) # type: ignore + check_result = CheckResult(justification=[], result_tables=[]) # type: ignore maven = Maven() + gradle = Gradle() maven.load_defaults() # The target repo uses a build tool. use_build_tool = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - use_build_tool.dynamic_data["build_spec"]["tool"] = maven + use_build_tool.dynamic_data["build_spec"]["tools"] = [maven] + + assert check.run_check(use_build_tool, check_result) == CheckResultType.PASSED + + # The target repo uses multiple build tools + use_build_tool = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) + use_build_tool.dynamic_data["build_spec"]["tools"] = [maven, gradle] assert check.run_check(use_build_tool, check_result) == CheckResultType.PASSED diff --git a/tests/slsa_analyzer/checks/test_build_service_check.py b/tests/slsa_analyzer/checks/test_build_service_check.py index 3998225c1..ea976157a 100644 --- a/tests/slsa_analyzer/checks/test_build_service_check.py +++ b/tests/slsa_analyzer/checks/test_build_service_check.py @@ -40,7 +40,7 @@ class TestBuildServiceCheck(MacaronTestCase): def test_build_service_check(self) -> None: """Test the Build Service Check.""" check = BuildServiceCheck() - check_result = CheckResult(justification=[]) # type: ignore + check_result = CheckResult(justification=[], result_tables=[]) # type: ignore maven = Maven() maven.load_defaults() gradle = Gradle() @@ -74,31 +74,36 @@ def test_build_service_check(self) -> None: # The target repo uses Maven build tool but does not use a service. use_build_tool = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - use_build_tool.dynamic_data["build_spec"]["tool"] = maven + use_build_tool.dynamic_data["build_spec"]["tools"] = [maven] assert check.run_check(use_build_tool, check_result) == CheckResultType.FAILED # The target repo uses Gradle build tool but does not use a service. use_build_tool = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - use_build_tool.dynamic_data["build_spec"]["tool"] = gradle + use_build_tool.dynamic_data["build_spec"]["tools"] = [gradle] assert check.run_check(use_build_tool, check_result) == CheckResultType.FAILED # The target repo uses Poetry build tool but does not use a service. use_build_tool = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - use_build_tool.dynamic_data["build_spec"]["tool"] = poetry + use_build_tool.dynamic_data["build_spec"]["tools"] = [poetry] assert check.run_check(use_build_tool, check_result) == CheckResultType.FAILED # The target repo uses Pip build tool but does not use a service. use_build_tool = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - use_build_tool.dynamic_data["build_spec"]["tool"] = pip + use_build_tool.dynamic_data["build_spec"]["tools"] = [pip] assert check.run_check(use_build_tool, check_result) == CheckResultType.FAILED # The target repo does not use a build tool. no_build_tool = AnalyzeContext("no_build_tool", os.path.abspath("./"), MagicMock()) assert check.run_check(no_build_tool, check_result) == CheckResultType.FAILED + # The target repo has multiple build tools, but does not use a service. + use_build_tool = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) + use_build_tool.dynamic_data["build_spec"]["tools"] = [maven, gradle] + assert check.run_check(use_build_tool, check_result) == CheckResultType.FAILED + # Use mvn build args in CI to build the artifact. maven_build_ci = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - maven_build_ci.dynamic_data["build_spec"]["tool"] = maven + maven_build_ci.dynamic_data["build_spec"]["tools"] = [maven] bash_commands["commands"] = [["mvn", "package"]] maven_build_ci.dynamic_data["ci_services"] = [ci_info] assert check.run_check(maven_build_ci, check_result) == CheckResultType.PASSED @@ -115,7 +120,7 @@ def test_build_service_check(self) -> None: # Use mvn but do not use CI to build artifacts. no_maven_build_ci = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - no_maven_build_ci.dynamic_data["build_spec"]["tool"] = maven + no_maven_build_ci.dynamic_data["build_spec"]["tools"] = [maven] bash_commands["commands"] = [["mvn", "test"]] no_maven_build_ci.dynamic_data["ci_services"] = [ci_info] assert check.run_check(no_maven_build_ci, check_result) == CheckResultType.FAILED @@ -127,56 +132,76 @@ def test_build_service_check(self) -> None: # Use gradle in CI to build the artifact. gradle_build_ci = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - gradle_build_ci.dynamic_data["build_spec"]["tool"] = gradle + gradle_build_ci.dynamic_data["build_spec"]["tools"] = [gradle] bash_commands["commands"] = [["./gradlew", "build"]] gradle_build_ci.dynamic_data["ci_services"] = [ci_info] assert check.run_check(gradle_build_ci, check_result) == CheckResultType.PASSED # Use poetry in CI to build the artifact. poetry_build_ci = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - poetry_build_ci.dynamic_data["build_spec"]["tool"] = poetry + poetry_build_ci.dynamic_data["build_spec"]["tools"] = [poetry] bash_commands["commands"] = [["poetry", "build"]] poetry_build_ci.dynamic_data["ci_services"] = [ci_info] assert check.run_check(poetry_build_ci, check_result) == CheckResultType.PASSED # Use pip in CI to build the artifact. pip_build_ci = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - pip_build_ci.dynamic_data["build_spec"]["tool"] = pip + pip_build_ci.dynamic_data["build_spec"]["tools"] = [pip] bash_commands["commands"] = [["pip", "install"]] pip_build_ci.dynamic_data["ci_services"] = [ci_info] assert check.run_check(pip_build_ci, check_result) == CheckResultType.PASSED # Use flit in CI to build the artifact. flit_build_ci = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - flit_build_ci.dynamic_data["build_spec"]["tool"] = pip + flit_build_ci.dynamic_data["build_spec"]["tools"] = [pip] bash_commands["commands"] = [["flit", "build"]] flit_build_ci.dynamic_data["ci_services"] = [ci_info] assert check.run_check(flit_build_ci, check_result) == CheckResultType.PASSED # Use pip as a module in CI to build the artifact. pip_interpreter_build_ci = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - pip_interpreter_build_ci.dynamic_data["build_spec"]["tool"] = pip + pip_interpreter_build_ci.dynamic_data["build_spec"]["tools"] = [pip] bash_commands["commands"] = [["python", "-m", "pip", "install"]] pip_interpreter_build_ci.dynamic_data["ci_services"] = [ci_info] assert check.run_check(pip_interpreter_build_ci, check_result) == CheckResultType.PASSED # Use pip as a module incorrectly in CI to build the artifact. no_pip_interpreter_build_ci = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - no_pip_interpreter_build_ci.dynamic_data["build_spec"]["tool"] = pip + no_pip_interpreter_build_ci.dynamic_data["build_spec"]["tools"] = [pip] bash_commands["commands"] = [["python", "pip", "install"]] no_pip_interpreter_build_ci.dynamic_data["ci_services"] = [ci_info] assert check.run_check(no_pip_interpreter_build_ci, check_result) == CheckResultType.FAILED # Use pip as a module in CI with invalid goal to build the artifact. no_pip_interpreter_build_ci = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - no_pip_interpreter_build_ci.dynamic_data["build_spec"]["tool"] = pip + no_pip_interpreter_build_ci.dynamic_data["build_spec"]["tools"] = [pip] bash_commands["commands"] = [["python", "-m", "pip", "installl"]] no_pip_interpreter_build_ci.dynamic_data["ci_services"] = [ci_info] assert check.run_check(no_pip_interpreter_build_ci, check_result) == CheckResultType.FAILED + # Maven and Gradle are both used in CI to build the artifact + gradle_build_ci = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) + gradle_build_ci.dynamic_data["build_spec"]["tools"] = [gradle, maven] + bash_commands["commands"] = [["./gradlew", "build"], ["mvn", "package"]] + gradle_build_ci.dynamic_data["ci_services"] = [ci_info] + assert check.run_check(gradle_build_ci, check_result) == CheckResultType.PASSED + + # Maven is used in CI to build the artifact, Gradle is not + gradle_build_ci = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) + gradle_build_ci.dynamic_data["build_spec"]["tools"] = [gradle, maven] + bash_commands["commands"] = [["mvn", "package"]] + gradle_build_ci.dynamic_data["ci_services"] = [ci_info] + assert check.run_check(gradle_build_ci, check_result) == CheckResultType.PASSED + + # No build tools used + gradle_build_ci = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) + gradle_build_ci.dynamic_data["build_spec"]["tools"] = [] + gradle_build_ci.dynamic_data["ci_services"] = [ci_info] + assert check.run_check(gradle_build_ci, check_result) == CheckResultType.FAILED + # Test Jenkins. maven_build_ci = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - maven_build_ci.dynamic_data["build_spec"]["tool"] = maven + maven_build_ci.dynamic_data["build_spec"]["tools"] = [maven] bash_commands["commands"] = [] ci_info["service"] = jenkins maven_build_ci.dynamic_data["ci_services"] = [ci_info] @@ -184,7 +209,7 @@ def test_build_service_check(self) -> None: # Test Travis. maven_build_ci = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - maven_build_ci.dynamic_data["build_spec"]["tool"] = maven + maven_build_ci.dynamic_data["build_spec"]["tools"] = [maven] bash_commands["commands"] = [] ci_info["service"] = travis maven_build_ci.dynamic_data["ci_services"] = [ci_info] @@ -192,7 +217,7 @@ def test_build_service_check(self) -> None: # Test Circle CI. maven_build_ci = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - maven_build_ci.dynamic_data["build_spec"]["tool"] = maven + maven_build_ci.dynamic_data["build_spec"]["tools"] = [maven] bash_commands["commands"] = [] ci_info["service"] = circle_ci maven_build_ci.dynamic_data["ci_services"] = [ci_info] @@ -200,7 +225,7 @@ def test_build_service_check(self) -> None: # Test GitLab CI. maven_build_ci = AnalyzeContext("use_build_tool", os.path.abspath("./"), MagicMock()) - maven_build_ci.dynamic_data["build_spec"]["tool"] = maven + maven_build_ci.dynamic_data["build_spec"]["tools"] = [maven] bash_commands["commands"] = [] ci_info["service"] = gitlab_ci maven_build_ci.dynamic_data["ci_services"] = [ci_info] diff --git a/tests/slsa_analyzer/checks/test_vcs_check.py b/tests/slsa_analyzer/checks/test_vcs_check.py index 937390550..df0f9b0f4 100644 --- a/tests/slsa_analyzer/checks/test_vcs_check.py +++ b/tests/slsa_analyzer/checks/test_vcs_check.py @@ -6,7 +6,6 @@ import os from macaron.slsa_analyzer.analyze_context import AnalyzeContext, ChecksOutputs -from macaron.slsa_analyzer.build_tool.base_build_tool import NoneBuildTool from macaron.slsa_analyzer.checks.check_result import CheckResult, CheckResultType from macaron.slsa_analyzer.checks.vcs_check import VCSCheck from macaron.slsa_analyzer.git_service.base_git_service import NoneGitService @@ -40,7 +39,7 @@ def __init__(self) -> None: self.remote_path = "https://github.com/org/name" self.dynamic_data: ChecksOutputs = ChecksOutputs( git_service=NoneGitService(), - build_spec=BuildSpec(tool=NoneBuildTool()), + build_spec=BuildSpec(tools=[]), ci_services=[], is_inferred_prov=True, expectation=None,