diff --git a/docs/source/pages/cli_usage/command_gen_build_spec.rst b/docs/source/pages/cli_usage/command_gen_build_spec.rst index 0e886c220..26465fd6f 100644 --- a/docs/source/pages/cli_usage/command_gen_build_spec.rst +++ b/docs/source/pages/cli_usage/command_gen_build_spec.rst @@ -39,5 +39,4 @@ Options .. option:: --output-format OUTPUT_FORMAT - The desired output format for the build specification. The default format is `rc-buildspec`, which is the Reproducible-Central build specification. - Other formats may be available depending on your configuration. + The output format. Can be `default-buildspec` (default) or `rc-buildspec` (Reproducible-central build spec) diff --git a/docs/source/pages/output_files.rst b/docs/source/pages/output_files.rst index dddd0861d..a248cba76 100644 --- a/docs/source/pages/output_files.rst +++ b/docs/source/pages/output_files.rst @@ -21,6 +21,7 @@ Top level structure output/ ├── build_log/ + ├── buildspec/ ├── git_repos/ ├── reports/ ├── debug.log @@ -43,8 +44,8 @@ The report files of Macaron (from using the :ref:`analyze command /buildspec/// + +Depending on the chosen output format, the following files may be generated in each directory: +- ``macaron.buildspec`` (default format) +- ``reproducible_central.buildspec`` (when run with the ``rc-buildspec`` output format for Maven artifacts) + +Each file contains the build specification for the corresponding software component. + + .. _output_files_macaron_verify_policy: ------------------------------------- diff --git a/docs/source/pages/supported_technologies/index.rst b/docs/source/pages/supported_technologies/index.rst index ff71dd18b..9740751d6 100644 --- a/docs/source/pages/supported_technologies/index.rst +++ b/docs/source/pages/supported_technologies/index.rst @@ -29,7 +29,8 @@ such as GitHub Actions workflows. Build Specification Generation ------------------------------ -* Maven and Gradle builds for Java artifacts +* Maven and Gradle builds for Java packages +* The built-in ``build`` module and various build tools, like Poetry for Python packages .. _supported_git_services: diff --git a/docs/source/pages/tutorials/rebuild_third_party_artifacts.rst b/docs/source/pages/tutorials/rebuild_third_party_artifacts.rst index 54b11a872..89602cae9 100644 --- a/docs/source/pages/tutorials/rebuild_third_party_artifacts.rst +++ b/docs/source/pages/tutorials/rebuild_third_party_artifacts.rst @@ -16,6 +16,7 @@ These buildspecs help document and automate the build process for packages, enab * - Currently Supported packages * - Maven packages built with Gradle or Maven + * - Python packages built with the built-in ``build`` module and various build tools, like Poetry .. contents:: :local: @@ -31,9 +32,9 @@ Addressing this lack of transparency is critical for improving supply chain secu Background ********** -A build specification is a file that describes all necessary information to rebuild a package from source. This includes metadata such as the build tool, the specific build command to run, the language version, e.g., JDK for Java, and artifact coordinates. Macaron can now generate this file automatically for supported ecosystems, greatly simplifying build from source. +A build specification is a file that describes all necessary information to rebuild a package from source. This includes metadata such as the build tool, the specific build command to run, the language version, e.g., Python or JDK for Java, and artifact coordinates. Macaron can now generate this file automatically for supported ecosystems, greatly simplifying build from source. -The generated buildspec will be stored in an ecosystem- and PURL-specific path under the ``output/`` directory (see more under :ref:`Output Files Guide `). +The generated buildspec will be stored in an ecosystem- and PURL-specific path under the ``output/`` directory (see more under :ref:`Output Files Guide `). ****************************** Installation and Prerequisites @@ -101,7 +102,48 @@ In the example above, the buildspec is located at: Step 3: Review and Use the Buildspec File ***************************************** -The generated buildspec uses the `Reproducible Central buildspec `_ format, for example: +By default we generate the buildspec in JSON format as follows: + +.. code-block:: ini + + { + "macaron_version": "0.18.0", + "group_id": "org.apache.hugegraph", + "artifact_id": "computer-k8s", + "version": "1.0.0", + "git_repo": "https://github.com/apache/hugegraph-computer", + "git_tag": "d2b95262091d6572cc12dcda57d89f9cd44ac88b", + "newline": "lf", + "language_version": [ + "11" + ], + "ecosystem": "maven", + "purl": "pkg:maven/org.apache.hugegraph/computer-k8s@1.0.0", + "language": "java", + "build_tools": [ + "maven" + ], + "build_commands": [ + [ + "mvn", + "-DskipTests=true", + "-Dmaven.test.skip=true", + "-Dmaven.site.skip=true", + "-Drat.skip=true", + "-Dmaven.javadoc.skip=true", + "clean", + "package" + ] + ] + } + +If you use the ``rc-buildspec`` output format, the generated buildspec follows the `Reproducible Central buildspec `_ format. For example, you can generate it with: + +.. code-block:: shell + + ./run_macaron.sh gen-build-spec -purl pkg:maven/org.apache.hugegraph/computer-k8s@1.0.0 --database output/macaron.db --output-format rc-buildspec + +The resulting file will be saved as ``output/buildspec/maven/org_apache_hugegraph/computer-k8s/reproducible_central.buildspec``, and will look like this: .. code-block:: ini @@ -136,18 +178,18 @@ The ``gen-build-spec`` works as follows: - Extracts metadata and build information from Macaron’s local SQLite database. - Parses and modifies build commands from CI/CD configurations to ensure compatibility with rebuild systems. -- Identifies the JDK version by parsing CI/CD configurations or extracting it from the ``META-INF/MANIFEST.MF`` file in Maven Central artifacts. +- Identifies the language version, e.g., JDK version by parsing CI/CD configurations or extracting it from the ``META-INF/MANIFEST.MF`` file in Maven Central artifacts. - Ensures that only the major JDK version is included, as required by the build specification format. -This feature is described in more detail in our accepted ASE 2025 Industry ShowCase paper: `Unlocking Reproducibility: Automating the Re-Build Process for Open-Source Software `_. +The Java support for this feature is described in more detail in our accepted ASE 2025 Industry ShowCase paper: `Unlocking Reproducibility: Automating the Re-Build Process for Open-Source Software `_. *********************************** Frequently Asked Questions (FAQs) *********************************** *Q: What formats are supported for buildspec output?* -A: Currently, only ``rc-buildspec`` is supported. +A: Currently, a default JSON spec and optional ``rc-buildspec`` are supported. *Q: Do I need to analyze the package every time before generating a buildspec?* A: No, you only need to analyze the package once unless you want to update the database with newer information. diff --git a/src/macaron/build_spec_generator/common_spec/base_spec.py b/src/macaron/build_spec_generator/common_spec/base_spec.py index 952412983..b410729fe 100644 --- a/src/macaron/build_spec_generator/common_spec/base_spec.py +++ b/src/macaron/build_spec_generator/common_spec/base_spec.py @@ -25,8 +25,8 @@ class BaseBuildSpecDict(TypedDict, total=False): #: The programming language, e.g., 'java', 'python', 'javascript'. language: Required[str] - #: The build tool or package manager, e.g., 'maven', 'gradle', 'pip', 'poetry', 'npm', 'yarn'. - build_tool: Required[str] + #: The build tools or package managers, e.g., 'maven', 'gradle', 'pip', 'poetry', 'npm', 'yarn'. + build_tools: Required[list[str]] #: The version of Macaron used for generating the spec. macaron_version: Required[str] @@ -73,10 +73,13 @@ class BaseBuildSpecDict(TypedDict, total=False): #: Entry point script, class, or binary for running the project. entry_point: NotRequired[str | None] + #: The build_requires is the required packages that need to be available in the build environment. + build_requires: NotRequired[dict[str, str]] + #: A "back end" is tool that a "front end" (such as pip/build) would call to #: package the source distribution into the wheel format. build_backends would #: be a list of these that were used in building the wheel alongside their version. - build_backends: NotRequired[dict[str, str]] + build_backends: NotRequired[list[str]] class BaseBuildSpec(ABC): @@ -94,21 +97,21 @@ def resolve_fields(self, purl: PackageURL) -> None: """ @abstractmethod - def get_default_build_command( + def get_default_build_commands( self, - build_tool_name: str, - ) -> list[str]: - """Return a default build command for the build tool. + build_tool_names: list[str], + ) -> list[list[str]]: + """Return the default build commands for the build tools. Parameters ---------- - build_tool_name: str - The build tool to get the default build command. + build_tool_names: list[str] + The build tools to get the default build command. Returns ------- - list[str] - The build command as a list[str]. + list[list[str]] + The build command as a list[list[str]]. Raises ------ diff --git a/src/macaron/build_spec_generator/common_spec/core.py b/src/macaron/build_spec_generator/common_spec/core.py index 94dd8985d..26b2f329f 100644 --- a/src/macaron/build_spec_generator/common_spec/core.py +++ b/src/macaron/build_spec_generator/common_spec/core.py @@ -57,6 +57,9 @@ class MacaronBuildToolName(str, Enum): GRADLE = "gradle" PIP = "pip" POETRY = "poetry" + FLIT = "flit" + HATCH = "hatch" + CONDA = "conda" def format_build_command_info(build_command_info: list[GenericBuildCommandInfo]) -> str: @@ -117,18 +120,14 @@ def compose_shell_commands(cmds_sequence: list[list[str]]) -> str: return result -def get_macaron_build_tool_name( +def get_macaron_build_tool_names( build_tool_facts: Sequence[BuildToolFacts], target_language: str -) -> MacaronBuildToolName | None: +) -> list[MacaronBuildToolName] | None: """ - Retrieve the Macaron build tool name for supported projects from the database facts. + Retrieve the Macaron build tool names for supported projects from the database facts. - Iterates over the provided build tool facts and returns the first valid `MacaronBuildToolName` - for a supported language. If no valid build tool name is found, returns None. - - .. note:: - If multiple build tools are present in the database, only the first valid one encountered - in the sequence is returned. + Iterates over the provided build tool facts and returns the list of valid `MacaronBuildToolName` + for a supported language. Parameters ---------- @@ -139,31 +138,27 @@ def get_macaron_build_tool_name( Returns ------- - MacaronBuildToolName or None - The corresponding Macaron build tool name if found, otherwise None. + list[MacaronBuildToolName] None + The corresponding Macaron build tool names, or None otherwise. """ + build_tool_names = [] for fact in build_tool_facts: if fact.language.lower() == target_language: try: - macaron_build_tool_name = MacaronBuildToolName(fact.build_tool_name) + build_tool_names.append(MacaronBuildToolName(fact.build_tool_name)) except ValueError: continue - # TODO: What happen if we report multiple build tools in the database? - return macaron_build_tool_name - - return None + return build_tool_names or None -def get_build_tool_name( +def get_build_tool_names( component_id: int, session: sqlalchemy.orm.Session, target_language: str -) -> MacaronBuildToolName | None: - """ - Retrieve the Macaron build tool name for a given component. +) -> list[MacaronBuildToolName] | None: + """Retrieve the Macaron build tool names for a given component. - Queries the database for build tool facts associated with the specified component ID - and returns the corresponding `MacaronBuildToolName` if found. If no valid build tool - information is available or an error occurs during the query, returns None. + Queries the database for build tool facts associated with the specified component ID. + It returns the corresponding list of `MacaronBuildToolName` if found. Parameters ---------- @@ -176,7 +171,7 @@ def get_build_tool_name( Returns ------- - MacaronBuildToolName or None + list[MacaronBuildToolName] | None The corresponding build tool name for the component if available, otherwise None. """ try: @@ -203,7 +198,7 @@ def get_build_tool_name( [(fact.build_tool_name, fact.language) for fact in build_tool_facts], ) - return get_macaron_build_tool_name(build_tool_facts, target_language) + return get_macaron_build_tool_names(build_tool_facts, target_language) def get_build_command_info( @@ -345,12 +340,17 @@ def gen_generic_build_spec( latest_component_repository.commit_sha, ) - build_tool_name = get_build_tool_name( + build_tool_names = [] + build_tools = get_build_tool_names( component_id=latest_component.id, session=session, target_language=target_language ) - if not build_tool_name: + if not build_tools: raise GenerateBuildSpecError(f"Failed to determine build tool for {purl}.") + # This check is for Pylint, which is not able to iterate over build_tools, even though it cannot be None. + if build_tools is not None: + build_tool_names = [build_tool.value for build_tool in build_tools] + build_command_info = get_build_command_info( component_id=latest_component.id, session=session, @@ -377,7 +377,7 @@ def gen_generic_build_spec( "ecosystem": purl.type, "purl": str(purl), "language": target_language, - "build_tool": build_tool_name.value, + "build_tools": build_tool_names, "build_commands": [selected_build_command], } ) diff --git a/src/macaron/build_spec_generator/common_spec/maven_spec.py b/src/macaron/build_spec_generator/common_spec/maven_spec.py index f893896bc..1d0abf4f8 100644 --- a/src/macaron/build_spec_generator/common_spec/maven_spec.py +++ b/src/macaron/build_spec_generator/common_spec/maven_spec.py @@ -31,45 +31,47 @@ def __init__(self, data: BaseBuildSpecDict): """ self.data = data - def get_default_build_command( + def get_default_build_commands( self, - build_tool_name: str, - ) -> list[str]: - """Return a default build command for the build tool. + build_tool_names: list[str], + ) -> list[list[str]]: + """Return the default build commands for the build tools. Parameters ---------- - build_tool_name: str - The build tool to get the default build command. + build_tool_names: list[str] + The build tools to get the default build command. Returns ------- - list[str] - The build command as a list[str]. + list[list[str]] + The build command as a list[list[str]]. Raises ------ GenerateBuildSpecError If there is no default build command available for the specified build tool. """ - default_build_command = None + default_build_commands = [] - match build_tool_name: - case "maven": - default_build_command = "mvn clean package".split() - case "gradle": - default_build_command = "./gradlew clean assemble publishToMavenLocal".split() - case _: - pass + for build_tool_name in build_tool_names: - if not default_build_command: + match build_tool_name: + case "maven": + default_build_commands.append("mvn clean package".split()) + case "gradle": + default_build_commands.append("./gradlew clean assemble publishToMavenLocal".split()) + case _: + pass + + if not default_build_commands: logger.critical( - "There is no default build command available for the build tool %s.", - build_tool_name, + "There is no default build command available for the build tools %s.", + build_tool_names, ) raise GenerateBuildSpecError("Unable to find a default build command.") - return default_build_command + return default_build_commands def resolve_fields(self, purl: PackageURL) -> None: """ @@ -113,9 +115,9 @@ def resolve_fields(self, purl: PackageURL) -> None: self.data["language_version"] = [major_jdk_version] # Resolve and patch build commands. - selected_build_commands = self.data["build_commands"] or [ - self.get_default_build_command(self.data["build_tool"]) - ] + selected_build_commands = self.data["build_commands"] or self.get_default_build_commands( + self.data["build_tools"] + ) patched_build_commands = patch_commands( cmds_sequence=selected_build_commands, diff --git a/src/macaron/build_spec_generator/common_spec/pypi_spec.py b/src/macaron/build_spec_generator/common_spec/pypi_spec.py index ee8d3da7c..bd2f8b558 100644 --- a/src/macaron/build_spec_generator/common_spec/pypi_spec.py +++ b/src/macaron/build_spec_generator/common_spec/pypi_spec.py @@ -10,6 +10,7 @@ import tomli from packageurl import PackageURL from packaging.requirements import InvalidRequirement, Requirement +from packaging.specifiers import InvalidSpecifier from packaging.utils import InvalidWheelFilename, parse_wheel_filename from macaron.build_spec_generator.build_command_patcher import CLI_COMMAND_PATCHES, patch_commands @@ -39,45 +40,53 @@ def __init__(self, data: BaseBuildSpecDict): """ self.data = data - def get_default_build_command( + def get_default_build_commands( self, - build_tool_name: str, - ) -> list[str]: - """Return a default build command for the build tool. + build_tool_names: list[str], + ) -> list[list[str]]: + """Return the default build commands for the build tools. Parameters ---------- - build_tool_name: str - The build tool to get the default build command. + build_tool_names: list[str] + The build tools to get the default build command. Returns ------- - list[str] - The build command as a list[str]. + list[list[str]] + The build command as a list[list[str]]. Raises ------ GenerateBuildSpecError If there is no default build command available for the specified build tool. """ - default_build_command = None - - match build_tool_name: - case "pip": - default_build_command = "python -m build".split() - case "poetry": - default_build_command = "poetry build".split() - case _: - pass - - if not default_build_command: + default_build_commands = [] + + for build_tool_name in build_tool_names: + + match build_tool_name: + case "pip": + default_build_commands.append("python -m build".split()) + case "poetry": + default_build_commands.append("poetry build".split()) + case "flit": + default_build_commands.append("flit build".split()) + case "hatch": + default_build_commands.append("hatch build".split()) + case "conda": + default_build_commands.append("conda build".split()) + case _: + pass + + if not default_build_commands: logger.critical( - "There is no default build command available for the build tool %s.", - build_tool_name, + "There is no default build command available for the build tools %s.", + build_tool_names, ) raise GenerateBuildSpecError("Unable to find a default build command.") - return default_build_command + return default_build_commands def resolve_fields(self, purl: PackageURL) -> None: """ @@ -95,22 +104,22 @@ def resolve_fields(self, purl: PackageURL) -> None: registry.load_defaults() registry_info = PackageRegistryInfo( - build_tool_name="pip", - build_tool_purl_type="pypi", + ecosystem="pypi", package_registry=registry, metadata=[], ) pypi_package_json = pypi_registry.find_or_create_pypi_asset(purl.name, purl.version, registry_info) patched_build_commands: list[list[str]] = [] + build_requires_set: set[str] = set() + build_backends_set: set[str] = set() + parsed_build_requires: dict[str, str] = {} + python_version_set: set[str] = set() + wheel_name_python_version_list: list[str] = [] + wheel_name_platforms: set[str] = set() if pypi_package_json is not None: if pypi_package_json.package_json or pypi_package_json.download(dest=""): - requires_array: list[str] = [] - build_backends: dict[str, str] = {} - python_version_set: set[str] = set() - wheel_name_python_version_list: list[str] = [] - wheel_name_platforms: set[str] = set() # Get the Python constraints from the PyPI JSON response. json_releases = pypi_package_json.get_releases() @@ -128,59 +137,62 @@ def resolve_fields(self, purl: PackageURL) -> None: wheel_contents, metadata_contents = self.read_directory(pypi_package_json.wheel_path, purl) generator, version = self.read_generator_line(wheel_contents) if generator != "": - build_backends[generator] = "==" + version - if generator != "setuptools": - # Apply METADATA heuristics to determine setuptools version. - if "License-File" in metadata_contents: - build_backends["setuptools"] = "==" + defaults.get( - "heuristic.pypi", "setuptools_version_emitting_license" - ) - elif "Platform: UNKNOWN" in metadata_contents: - build_backends["setuptools"] = "==" + defaults.get( - "heuristic.pypi", "setuptools_version_emitting_platform_unknown" - ) - else: - build_backends["setuptools"] = "==" + defaults.get( - "heuristic.pypi", "default_setuptools" - ) + parsed_build_requires[generator] = "==" + version.replace(" ", "") + # Apply METADATA heuristics to determine setuptools version. + elif "License-File" in metadata_contents: + parsed_build_requires["setuptools"] = "==" + defaults.get( + "heuristic.pypi", "setuptools_version_emitting_license" + ) + elif "Platform: UNKNOWN" in metadata_contents: + parsed_build_requires["setuptools"] = "==" + defaults.get( + "heuristic.pypi", "setuptools_version_emitting_platform_unknown" + ) except SourceCodeError: logger.debug("Could not find pure wheel matching this PURL") logger.debug("From .dist_info:") - logger.debug(build_backends) + logger.debug(parsed_build_requires) try: with pypi_package_json.sourcecode(): try: pyproject_content = pypi_package_json.get_sourcecode_file_contents("pyproject.toml") content = tomli.loads(pyproject_content.decode("utf-8")) - build_system: dict[str, list[str]] = content.get("build-system", {}) - requires_array = build_system.get("requires", []) + requires = json_extract(content, ["build-system", "requires"], list) + if requires: + build_requires_set.update(elem.replace(" ", "") for elem in requires) + backend = json_extract(content, ["build-system", "build-backend"], str) + if backend: + build_backends_set.add(backend.replace(" ", "")) python_version_constraint = json_extract(content, ["project", "requires-python"], str) if python_version_constraint: python_version_set.add(python_version_constraint.replace(" ", "")) - logger.debug("From pyproject.toml:") - logger.debug(requires_array) - except SourceCodeError: - logger.debug("No pyproject.toml found") - except SourceCodeError: - logger.debug("No source distribution found") - - # Merge in pyproject.toml information only when the wheel dist_info does not contain the same + logger.debug( + "After analyzing pyproject.toml from the sdist: build-requires: %s, build_backend: %s", + build_requires_set, + build_backends_set, + ) + except TypeError as error: + logger.debug( + "Found a type error while reading the pyproject.toml file from the sdist: %s", error + ) + except tomli.TOMLDecodeError as error: + logger.debug("Failed to read the pyproject.toml file from the sdist: %s", error) + except SourceCodeError as error: + logger.debug("No pyproject.toml found: %s", error) + except SourceCodeError as error: + logger.debug("No source distribution found: %s", error) + + # Merge in pyproject.toml information only when the wheel dist_info does not contain the same. # Hatch is an interesting example of this merge being required. - for requirement in requires_array: + for requirement in build_requires_set: try: parsed_requirement = Requirement(requirement) - if parsed_requirement.name not in build_backends: - build_backends[parsed_requirement.name] = str(parsed_requirement.specifier) - except InvalidRequirement: - logger.debug("Malformed requirement encountered:") - logger.debug(requirement) - - logger.debug("Combined:") - logger.debug(build_backends) - self.data["build_backends"] = build_backends + if parsed_requirement.name not in parsed_build_requires: + parsed_build_requires[parsed_requirement.name] = str(parsed_requirement.specifier) + except (InvalidRequirement, InvalidSpecifier) as error: + logger.debug("Malformed requirement encountered %s : %s", requirement, error) try: # Get information from the wheel file name. @@ -197,24 +209,35 @@ def resolve_fields(self, purl: PackageURL) -> None: # Use the default build command for pure Python packages. if "any" in wheel_name_platforms: - patched_build_commands = [self.get_default_build_command(self.data["build_tool"])] + patched_build_commands = self.get_default_build_commands(self.data["build_tools"]) + + # If we were not able to find any build and backends, use the default setuptools. + if not parsed_build_requires: + parsed_build_requires["setuptools"] = "==" + defaults.get("heuristic.pypi", "default_setuptools") + if not build_backends_set: + build_backends_set.add("setuptools.build_meta") + + logger.debug("Combined build-requires: %s", parsed_build_requires) + self.data["build_requires"] = parsed_build_requires + self.data["build_backends"] = list(build_backends_set) + + if not patched_build_commands: + # Resolve and patch build commands. + selected_build_commands = self.data["build_commands"] or self.get_default_build_commands( + self.data["build_tools"] + ) - if not patched_build_commands: - # Resolve and patch build commands. - selected_build_commands = self.data["build_commands"] or [ - self.get_default_build_command(self.data["build_tool"]) - ] - patched_build_commands = ( - patch_commands( - cmds_sequence=selected_build_commands, - patches=CLI_COMMAND_PATCHES, - ) - or [] + patched_build_commands = ( + patch_commands( + cmds_sequence=selected_build_commands, + patches=CLI_COMMAND_PATCHES, ) - if not patched_build_commands: - raise GenerateBuildSpecError(f"Failed to patch command sequences {selected_build_commands}.") + or [] + ) + if not patched_build_commands: + raise GenerateBuildSpecError(f"Failed to patch command sequences {selected_build_commands}.") - self.data["build_commands"] = patched_build_commands + self.data["build_commands"] = patched_build_commands def read_directory(self, wheel_path: str, purl: PackageURL) -> tuple[str, str]: """ @@ -278,5 +301,6 @@ def read_generator_line(self, wheel_contents: str) -> tuple[str, str]: for line in wheel_contents.splitlines(): if line.startswith("Generator:"): split_line = line.split(" ") - return split_line[1], split_line[2] + if len(split_line) > 2: + return split_line[1], split_line[2] return "", "" diff --git a/src/macaron/build_spec_generator/reproducible_central/reproducible_central.py b/src/macaron/build_spec_generator/reproducible_central/reproducible_central.py index ba0b61426..5a6ec8389 100644 --- a/src/macaron/build_spec_generator/reproducible_central/reproducible_central.py +++ b/src/macaron/build_spec_generator/reproducible_central/reproducible_central.py @@ -77,9 +77,9 @@ def gen_reproducible_central_build_spec(build_spec: BaseBuildSpecDict) -> str | GenerateBuildSpecError Raised if generation of the build spec fails. """ - if build_spec["build_tool"].upper() not in (e.name for e in ReproducibleCentralBuildTool): + if build_spec["build_tools"][0].upper() not in (e.name for e in ReproducibleCentralBuildTool): raise GenerateBuildSpecError( - f"Build tool {build_spec['build_tool']} is not supported by Reproducible Central. " + f"Build tool {build_spec['build_tools'][0]} is not supported by Reproducible Central. " f"Supported build tools: {[build.name for build in ReproducibleCentralBuildTool]}" ) if build_spec["group_id"] is None: @@ -92,7 +92,7 @@ def gen_reproducible_central_build_spec(build_spec: BaseBuildSpecDict) -> str | "version": build_spec["version"], "git_repo": build_spec["git_repo"], "git_tag": build_spec["git_tag"], - "tool": ReproducibleCentralBuildTool[build_spec["build_tool"].upper()].value, + "tool": ReproducibleCentralBuildTool[build_spec["build_tools"][0].upper()].value, "newline": build_spec["newline"], "buildinfo": f"target/{build_spec['artifact_id']}-{build_spec['version']}.buildinfo", "jdk": build_spec["language_version"][0], diff --git a/src/macaron/config/defaults.ini b/src/macaron/config/defaults.ini index c036598ca..e9c47af34 100644 --- a/src/macaron/config/defaults.ini +++ b/src/macaron/config/defaults.ini @@ -290,14 +290,14 @@ build_configs = pyproject.toml packager = pip - pip3 - flit - conda + build publisher = twine - flit - conda - tox +# build-system information. +build_requires = + setuptools +build_backend = + setuptools.build_meta # These are the Python interpreters that may be used to load modules. interpreter = python @@ -322,6 +322,11 @@ package_lock = poetry.lock builder = poetry poetry-core +# build-system information. +build_requires = + poetry-core +build_backend = + poetry.core.masonry.api # These are the Python interpreters that may be used to load modules. interpreter = python @@ -336,6 +341,82 @@ deploy_arg = [builder.poetry.ci.deploy] github_actions = pypa/gh-action-pypi-publish +# This is the spec for Flit packaging tool. +[builder.flit] +entry_conf = +build_configs = + pyproject.toml + flit.ini +builder = + flit +# build-system information. +build_requires = + flit_core +build_backend = + flit_core.buildapi +# These are the Python interpreters that may be used to load modules. +interpreter = + python + python3 +interpreter_flag = + -m +build_arg = + build +deploy_arg = + publish + +[builder.flit.ci.deploy] +github_actions = pypa/gh-action-pypi-publish + +# This is the spec for the Hatch packaging tool. +[builder.hatch] +entry_conf = +build_configs = + pyproject.toml + hatch.toml +builder = + hatch +# build-system information. +build_requires = + hatchling +build_backend = + hatchling.build +# These are the Python interpreters that may be used to load modules. +interpreter = + python + python3 +interpreter_flag = + -m +build_arg = + build +deploy_arg = + publish + +[builder.hatch.ci.deploy] +github_actions = pypa/gh-action-pypi-publish + +# This is the spec for the Conda packaging tool. +[builder.conda] +entry_conf = +build_configs = + environment.yml + meta.yaml +builder = + conda +# These are the Python interpreters that may be used to load modules. +interpreter = + python + python3 +interpreter_flag = + -m +build_arg = + build +deploy_arg = + publish + +[builder.conda.ci.deploy] +github_actions = pypa/gh-action-pypi-publish + # This is the spec for trusted Docker build tool usages. [builder.docker] entry_conf = diff --git a/src/macaron/provenance/provenance_finder.py b/src/macaron/provenance/provenance_finder.py index 8bf9937e1..4935ca62d 100644 --- a/src/macaron/provenance/provenance_finder.py +++ b/src/macaron/provenance/provenance_finder.py @@ -554,11 +554,7 @@ def get_artifact_hash( return None registry_info = next( - ( - info - for info in package_registries_info - if info.package_registry == pypi_registry and info.build_tool_name in {"pip", "poetry"} - ), + (info for info in package_registries_info if info.package_registry == pypi_registry), None, ) if not registry_info: diff --git a/src/macaron/repo_finder/repo_finder_pypi.py b/src/macaron/repo_finder/repo_finder_pypi.py index 7adcede4a..d6e660e30 100644 --- a/src/macaron/repo_finder/repo_finder_pypi.py +++ b/src/macaron/repo_finder/repo_finder_pypi.py @@ -37,11 +37,7 @@ def find_repo( if package_registries_info: # Find the package registry info object that contains the PyPI registry and has the pypi build tool. pypi_info = next( - ( - info - for info in package_registries_info - if isinstance(info.package_registry, PyPIRegistry) and info.build_tool_name in {"poetry", "pip"} - ), + (info for info in package_registries_info if isinstance(info.package_registry, PyPIRegistry)), None, ) if not pypi_info: diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index 766b553bb..31b4f0937 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -379,7 +379,7 @@ def run_single( ) # Pre-populate all package registries so assets can be stored for later. - package_registries_info = self._populate_package_registry_info() + package_registries_info = self._populate_package_registry_info(parsed_purl) if parsed_purl else [] provenance_is_verified = False provenance_asset = None @@ -1127,18 +1127,14 @@ def _determine_ci_services(self, analyze_ctx: AnalyzeContext, git_service: BaseG "[red]Not Found[/]", ) - def _populate_package_registry_info(self) -> list[PackageRegistryInfo]: + def _populate_package_registry_info(self, parsed_purl: PackageURL) -> list[PackageRegistryInfo]: """Add all possible package registries to the analysis context.""" package_registries = [] for package_registry in PACKAGE_REGISTRIES: - for build_tool in BUILD_TOOLS: - build_tool_name = build_tool.name - if build_tool_name not in package_registry.build_tool_names: - continue + if package_registry.ecosystem == parsed_purl.type: package_registries.append( PackageRegistryInfo( - build_tool_name=build_tool_name, - build_tool_purl_type=build_tool.purl_type, + ecosystem=parsed_purl.type, package_registry=package_registry, ) ) @@ -1149,14 +1145,10 @@ def _determine_package_registries( analyze_ctx: AnalyzeContext, package_registries_info: list[PackageRegistryInfo], ) -> None: - """Determine the package registries used by the software component based on its build tools.""" - build_tools = ( - analyze_ctx.dynamic_data["build_spec"]["tools"] or analyze_ctx.dynamic_data["build_spec"]["purl_tools"] - ) - build_tool_names = {build_tool.name for build_tool in build_tools} + """Determine the package registries used by the software component.""" relevant_package_registries = [] for package_registry in package_registries_info: - if package_registry.build_tool_name not in build_tool_names: + if not package_registry.ecosystem == analyze_ctx.component.type: continue relevant_package_registries.append(package_registry) diff --git a/src/macaron/slsa_analyzer/build_tool/__init__.py b/src/macaron/slsa_analyzer/build_tool/__init__.py index 4638e1e5d..a474a2003 100644 --- a/src/macaron/slsa_analyzer/build_tool/__init__.py +++ b/src/macaron/slsa_analyzer/build_tool/__init__.py @@ -1,8 +1,12 @@ -# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2025, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """The build_tool package contains the supported build tools for Macaron.""" +from macaron.slsa_analyzer.build_tool.conda import Conda +from macaron.slsa_analyzer.build_tool.flit import Flit +from macaron.slsa_analyzer.build_tool.hatch import Hatch + from .base_build_tool import BaseBuildTool from .docker import Docker from .go import Go @@ -15,4 +19,16 @@ # The list of supported build tools. The order of the list determine the order # in which each build tool is checked against the target repository. -BUILD_TOOLS: list[BaseBuildTool] = [Gradle(), Maven(), Poetry(), Pip(), Docker(), NPM(), Yarn(), Go()] +BUILD_TOOLS: list[BaseBuildTool] = [ + Gradle(), + Maven(), + Poetry(), + Flit(), + Hatch(), + Conda(), + Pip(), + Docker(), + NPM(), + Yarn(), + Go(), +] diff --git a/src/macaron/slsa_analyzer/build_tool/base_build_tool.py b/src/macaron/slsa_analyzer/build_tool/base_build_tool.py index dfc286c4a..48ddb8e52 100644 --- a/src/macaron/slsa_analyzer/build_tool/base_build_tool.py +++ b/src/macaron/slsa_analyzer/build_tool/base_build_tool.py @@ -172,6 +172,8 @@ def __init__(self, name: str, language: BuildLanguage, purl_type: str) -> None: self.build_configs: list[str] = [] self.package_lock: list[str] = [] self.builder: list[str] = [] + self.build_requires: list[str] = [] + self.build_backend: list[str] = [] self.packager: list[str] = [] self.publisher: list[str] = [] self.interpreter: list[str] = [] diff --git a/src/macaron/slsa_analyzer/build_tool/conda.py b/src/macaron/slsa_analyzer/build_tool/conda.py new file mode 100644 index 000000000..af72dff05 --- /dev/null +++ b/src/macaron/slsa_analyzer/build_tool/conda.py @@ -0,0 +1,168 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module contains the Conda class which inherits BaseBuildTool. + +This module is used to work with repositories that use Conda for dependency management. +""" + +import logging +import os + +from cyclonedx_py import __version__ as cyclonedx_version + +from macaron.config.defaults import defaults +from macaron.config.global_config import global_config +from macaron.dependency_analyzer.cyclonedx import DependencyAnalyzer +from macaron.dependency_analyzer.cyclonedx_python import CycloneDxPython +from macaron.slsa_analyzer.build_tool.base_build_tool import BaseBuildTool, BuildToolCommand, file_exists +from macaron.slsa_analyzer.build_tool.language import BuildLanguage +from macaron.slsa_analyzer.checks.check_result import Confidence + +logger: logging.Logger = logging.getLogger(__name__) + + +class Conda(BaseBuildTool): + """This class contains the information of the conda build tool.""" + + def __init__(self) -> None: + """Initialize instance.""" + super().__init__(name="conda", language=BuildLanguage.PYTHON, purl_type="pypi") + + def load_defaults(self) -> None: + """Load the default values from defaults.ini.""" + super().load_defaults() + if "builder.conda" in defaults: + for item in defaults["builder.conda"]: + if hasattr(self, item): + setattr(self, item, defaults.get_list("builder.conda", item)) + + if "builder.conda.ci.deploy" in defaults: + for item in defaults["builder.conda.ci.deploy"]: + if item in self.ci_deploy_kws: + self.ci_deploy_kws[item] = defaults.get_list("builder.conda.ci.deploy", item) + + def is_detected(self, repo_path: str) -> bool: + """Return True if this build tool is used in the target repo. + + Parameters + ---------- + repo_path : str + The path to the target repo. + + Returns + ------- + bool + True if this build tool is detected, else False. + """ + return any(file_exists(repo_path, file, filters=self.path_filters) for file in self.build_configs) + + def get_dep_analyzer(self) -> DependencyAnalyzer: + """Create a DependencyAnalyzer for the build tool. + + Returns + ------- + DependencyAnalyzer + The DependencyAnalyzer object. + """ + return CycloneDxPython( + resources_path=global_config.resources_path, + file_name="python_sbom.json", + tool_name="cyclonedx_py", + tool_version=cyclonedx_version, + ) + + def is_deploy_command( + self, cmd: BuildToolCommand, excluded_configs: list[str] | None = None, provenance_workflow: str | None = None + ) -> tuple[bool, Confidence]: + """ + Determine if the command is a deploy command. + + A deploy command usually performs multiple tasks, such as compilation, packaging, and publishing the artifact. + This function filters the build tool commands that are called from the configuration files provided as input. + + Parameters + ---------- + cmd: BuildToolCommand + The build tool command object. + excluded_configs: list[str] | None + Build tool commands that are called from these configuration files are excluded. + provenance_workflow: str | None + The relative path to the root CI file that is captured in a provenance or None if provenance is not found. + + Returns + ------- + tuple[bool, Confidence] + Return True along with the inferred confidence level if the command is a deploy tool command. + """ + # Check the language. + if cmd["language"] is not self.language: + return False, Confidence.HIGH + + build_cmd = cmd["command"] + cmd_program_name = os.path.basename(build_cmd[0]) + + # Some projects use a publisher tool and some use the build tool with deploy arguments. + deploy_tools = self.publisher if self.publisher else self.builder + deploy_args = self.deploy_arg + + # Sometimes conda is called as a Python module. + if cmd_program_name in self.interpreter and len(build_cmd) > 2 and build_cmd[1] in self.interpreter_flag: + # Use the module cmd-line args. + build_cmd = build_cmd[2:] + + if not self.match_cmd_args(cmd=build_cmd, tools=deploy_tools, args=deploy_args): + return False, Confidence.HIGH + + # Check if the CI workflow is a configuration for a known tool. + if excluded_configs and os.path.basename(cmd["ci_path"]) in excluded_configs: + return False, Confidence.HIGH + + return True, self.infer_confidence_deploy_command(cmd, provenance_workflow) + + def is_package_command( + self, cmd: BuildToolCommand, excluded_configs: list[str] | None = None + ) -> tuple[bool, Confidence]: + """ + Determine if the command is a packaging command. + + A packaging command usually performs multiple tasks, such as compilation and creating the artifact. + This function filters the build tool commands that are called from the configuration files provided as input. + + Parameters + ---------- + cmd: BuildToolCommand + The build tool command object. + excluded_configs: list[str] | None + Build tool commands that are called from these configuration files are excluded. + + Returns + ------- + tuple[bool, Confidence] + Return True along with the inferred confidence level if the command is a build tool command. + """ + # Check the language. + if cmd["language"] is not self.language: + return False, Confidence.HIGH + + build_cmd = cmd["command"] + cmd_program_name = os.path.basename(build_cmd[0]) + if not cmd_program_name: + return False, Confidence.HIGH + + builder = self.packager if self.packager else self.builder + build_args = self.build_arg + + # Sometimes conda is called as a Python module. + if cmd_program_name in self.interpreter and len(build_cmd) > 2 and build_cmd[1] in self.interpreter_flag: + # Use the module cmd-line args. + build_cmd = build_cmd[2:] + + if not self.match_cmd_args(cmd=build_cmd, tools=builder, args=build_args): + return False, Confidence.HIGH + + # Check if the CI workflow is a configuration for a known tool. + if excluded_configs and os.path.basename(cmd["ci_path"]) in excluded_configs: + return False, Confidence.HIGH + + return True, Confidence.HIGH diff --git a/src/macaron/slsa_analyzer/build_tool/flit.py b/src/macaron/slsa_analyzer/build_tool/flit.py new file mode 100644 index 000000000..e03eb477b --- /dev/null +++ b/src/macaron/slsa_analyzer/build_tool/flit.py @@ -0,0 +1,181 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module contains the Flit class which inherits BaseBuildTool. + +This module is used to work with repositories that use Flit for dependency management. +""" + +import logging +import os + +from cyclonedx_py import __version__ as cyclonedx_version + +from macaron.config.defaults import defaults +from macaron.config.global_config import global_config +from macaron.dependency_analyzer.cyclonedx import DependencyAnalyzer +from macaron.dependency_analyzer.cyclonedx_python import CycloneDxPython +from macaron.slsa_analyzer.build_tool import pyproject +from macaron.slsa_analyzer.build_tool.base_build_tool import BaseBuildTool, BuildToolCommand, file_exists +from macaron.slsa_analyzer.build_tool.language import BuildLanguage +from macaron.slsa_analyzer.checks.check_result import Confidence + +logger: logging.Logger = logging.getLogger(__name__) + + +class Flit(BaseBuildTool): + """This class contains the information of the flit build tool.""" + + def __init__(self) -> None: + """Initialize instance.""" + super().__init__(name="flit", language=BuildLanguage.PYTHON, purl_type="pypi") + + def load_defaults(self) -> None: + """Load the default values from defaults.ini.""" + super().load_defaults() + if "builder.flit" in defaults: + for item in defaults["builder.flit"]: + if hasattr(self, item): + setattr(self, item, defaults.get_list("builder.flit", item)) + + if "builder.flit.ci.deploy" in defaults: + for item in defaults["builder.flit.ci.deploy"]: + if item in self.ci_deploy_kws: + self.ci_deploy_kws[item] = defaults.get_list("builder.flit.ci.deploy", item) + + def is_detected(self, repo_path: str) -> bool: + """Return True if this build tool is used in the target repo. + + Parameters + ---------- + repo_path : str + The path to the target repo. + + Returns + ------- + bool + True if this build tool is detected, else False. + """ + for config_name in self.build_configs: + if config_path := file_exists(repo_path, config_name, filters=self.path_filters): + if os.path.basename(config_path) == "pyproject.toml": + if pyproject.contains_build_tool("flit", config_path): + return True + # Check the build-system section. + for tool in self.build_requires + self.build_backend: + if pyproject.build_system_contains_tool(tool, config_path): + return True + else: + # For other build configuration files, the presence of the file alone is sufficient. + return True + return False + + def get_dep_analyzer(self) -> DependencyAnalyzer: + """Create a DependencyAnalyzer for the build tool. + + Returns + ------- + DependencyAnalyzer + The DependencyAnalyzer object. + """ + return CycloneDxPython( + resources_path=global_config.resources_path, + file_name="python_sbom.json", + tool_name="cyclonedx_py", + tool_version=cyclonedx_version, + ) + + def is_deploy_command( + self, cmd: BuildToolCommand, excluded_configs: list[str] | None = None, provenance_workflow: str | None = None + ) -> tuple[bool, Confidence]: + """ + Determine if the command is a deploy command. + + A deploy command usually performs multiple tasks, such as compilation, packaging, and publishing the artifact. + This function filters the build tool commands that are called from the configuration files provided as input. + + Parameters + ---------- + cmd: BuildToolCommand + The build tool command object. + excluded_configs: list[str] | None + Build tool commands that are called from these configuration files are excluded. + provenance_workflow: str | None + The relative path to the root CI file that is captured in a provenance or None if provenance is not found. + + Returns + ------- + tuple[bool, Confidence] + Return True along with the inferred confidence level if the command is a deploy tool command. + """ + # Check the language. + if cmd["language"] is not self.language: + return False, Confidence.HIGH + + build_cmd = cmd["command"] + cmd_program_name = os.path.basename(build_cmd[0]) + + # Some projects use a publisher tool and some use the build tool with deploy arguments. + deploy_tools = self.publisher if self.publisher else self.builder + deploy_args = self.deploy_arg + + # Sometimes flit is called as a Python module. + if cmd_program_name in self.interpreter and len(build_cmd) > 2 and build_cmd[1] in self.interpreter_flag: + # Use the module cmd-line args. + build_cmd = build_cmd[2:] + + if not self.match_cmd_args(cmd=build_cmd, tools=deploy_tools, args=deploy_args): + return False, Confidence.HIGH + + # Check if the CI workflow is a configuration for a known tool. + if excluded_configs and os.path.basename(cmd["ci_path"]) in excluded_configs: + return False, Confidence.HIGH + + return True, self.infer_confidence_deploy_command(cmd, provenance_workflow) + + def is_package_command( + self, cmd: BuildToolCommand, excluded_configs: list[str] | None = None + ) -> tuple[bool, Confidence]: + """ + Determine if the command is a packaging command. + + A packaging command usually performs multiple tasks, such as compilation and creating the artifact. + This function filters the build tool commands that are called from the configuration files provided as input. + + Parameters + ---------- + cmd: BuildToolCommand + The build tool command object. + excluded_configs: list[str] | None + Build tool commands that are called from these configuration files are excluded. + + Returns + ------- + tuple[bool, Confidence] + Return True along with the inferred confidence level if the command is a build tool command. + """ + # Check the language. + if cmd["language"] is not self.language: + return False, Confidence.HIGH + + build_cmd = cmd["command"] + cmd_program_name = os.path.basename(build_cmd[0]) + if not cmd_program_name: + return False, Confidence.HIGH + + builder = self.packager if self.packager else self.builder + build_args = self.build_arg + + # Sometimes flit is called as a Python module. + if cmd_program_name in self.interpreter and len(build_cmd) > 2 and build_cmd[1] in self.interpreter_flag: + # Use the module cmd-line args. + build_cmd = build_cmd[2:] + + if not self.match_cmd_args(cmd=build_cmd, tools=builder, args=build_args): + return False, Confidence.HIGH + + # Check if the CI workflow is a configuration for a known tool. + if excluded_configs and os.path.basename(cmd["ci_path"]) in excluded_configs: + return False, Confidence.HIGH + + return True, Confidence.HIGH diff --git a/src/macaron/slsa_analyzer/build_tool/hatch.py b/src/macaron/slsa_analyzer/build_tool/hatch.py new file mode 100644 index 000000000..22e2c2e0a --- /dev/null +++ b/src/macaron/slsa_analyzer/build_tool/hatch.py @@ -0,0 +1,181 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module contains the Hatch class which inherits BaseBuildTool. + +This module is used to work with repositories that use Hatch for dependency management. +""" + +import logging +import os + +from cyclonedx_py import __version__ as cyclonedx_version + +from macaron.config.defaults import defaults +from macaron.config.global_config import global_config +from macaron.dependency_analyzer.cyclonedx import DependencyAnalyzer +from macaron.dependency_analyzer.cyclonedx_python import CycloneDxPython +from macaron.slsa_analyzer.build_tool import pyproject +from macaron.slsa_analyzer.build_tool.base_build_tool import BaseBuildTool, BuildToolCommand, file_exists +from macaron.slsa_analyzer.build_tool.language import BuildLanguage +from macaron.slsa_analyzer.checks.check_result import Confidence + +logger: logging.Logger = logging.getLogger(__name__) + + +class Hatch(BaseBuildTool): + """This class contains the information of the hatch build tool.""" + + def __init__(self) -> None: + """Initialize instance.""" + super().__init__(name="hatch", language=BuildLanguage.PYTHON, purl_type="pypi") + + def load_defaults(self) -> None: + """Load the default values from defaults.ini.""" + super().load_defaults() + if "builder.hatch" in defaults: + for item in defaults["builder.hatch"]: + if hasattr(self, item): + setattr(self, item, defaults.get_list("builder.hatch", item)) + + if "builder.hatch.ci.deploy" in defaults: + for item in defaults["builder.hatch.ci.deploy"]: + if item in self.ci_deploy_kws: + self.ci_deploy_kws[item] = defaults.get_list("builder.hatch.ci.deploy", item) + + def is_detected(self, repo_path: str) -> bool: + """Return True if this build tool is used in the target repo. + + Parameters + ---------- + repo_path : str + The path to the target repo. + + Returns + ------- + bool + True if this build tool is detected, else False. + """ + for config_name in self.build_configs: + if config_path := file_exists(repo_path, config_name, filters=self.path_filters): + if os.path.basename(config_path) == "pyproject.toml": + if pyproject.contains_build_tool("hatch", config_path): + return True + # Check the build-system section. + for tool in self.build_requires + self.build_backend: + if pyproject.build_system_contains_tool(tool, config_path): + return True + else: + # For other build configuration files, the presence of the file alone is sufficient. + return True + return False + + def get_dep_analyzer(self) -> DependencyAnalyzer: + """Create a DependencyAnalyzer for the build tool. + + Returns + ------- + DependencyAnalyzer + The DependencyAnalyzer object. + """ + return CycloneDxPython( + resources_path=global_config.resources_path, + file_name="python_sbom.json", + tool_name="cyclonedx_py", + tool_version=cyclonedx_version, + ) + + def is_deploy_command( + self, cmd: BuildToolCommand, excluded_configs: list[str] | None = None, provenance_workflow: str | None = None + ) -> tuple[bool, Confidence]: + """ + Determine if the command is a deploy command. + + A deploy command usually performs multiple tasks, such as compilation, packaging, and publishing the artifact. + This function filters the build tool commands that are called from the configuration files provided as input. + + Parameters + ---------- + cmd: BuildToolCommand + The build tool command object. + excluded_configs: list[str] | None + Build tool commands that are called from these configuration files are excluded. + provenance_workflow: str | None + The relative path to the root CI file that is captured in a provenance or None if provenance is not found. + + Returns + ------- + tuple[bool, Confidence] + Return True along with the inferred confidence level if the command is a deploy tool command. + """ + # Check the language. + if cmd["language"] is not self.language: + return False, Confidence.HIGH + + build_cmd = cmd["command"] + cmd_program_name = os.path.basename(build_cmd[0]) + + # Some projects use a publisher tool and some use the build tool with deploy arguments. + deploy_tools = self.publisher if self.publisher else self.builder + deploy_args = self.deploy_arg + + # Sometimes hatch is called as a Python module. + if cmd_program_name in self.interpreter and len(build_cmd) > 2 and build_cmd[1] in self.interpreter_flag: + # Use the module cmd-line args. + build_cmd = build_cmd[2:] + + if not self.match_cmd_args(cmd=build_cmd, tools=deploy_tools, args=deploy_args): + return False, Confidence.HIGH + + # Check if the CI workflow is a configuration for a known tool. + if excluded_configs and os.path.basename(cmd["ci_path"]) in excluded_configs: + return False, Confidence.HIGH + + return True, self.infer_confidence_deploy_command(cmd, provenance_workflow) + + def is_package_command( + self, cmd: BuildToolCommand, excluded_configs: list[str] | None = None + ) -> tuple[bool, Confidence]: + """ + Determine if the command is a packaging command. + + A packaging command usually performs multiple tasks, such as compilation and creating the artifact. + This function filters the build tool commands that are called from the configuration files provided as input. + + Parameters + ---------- + cmd: BuildToolCommand + The build tool command object. + excluded_configs: list[str] | None + Build tool commands that are called from these configuration files are excluded. + + Returns + ------- + tuple[bool, Confidence] + Return True along with the inferred confidence level if the command is a build tool command. + """ + # Check the language. + if cmd["language"] is not self.language: + return False, Confidence.HIGH + + build_cmd = cmd["command"] + cmd_program_name = os.path.basename(build_cmd[0]) + if not cmd_program_name: + return False, Confidence.HIGH + + builder = self.packager if self.packager else self.builder + build_args = self.build_arg + + # Sometimes hatch is called as a Python module. + if cmd_program_name in self.interpreter and len(build_cmd) > 2 and build_cmd[1] in self.interpreter_flag: + # Use the module cmd-line args. + build_cmd = build_cmd[2:] + + if not self.match_cmd_args(cmd=build_cmd, tools=builder, args=build_args): + return False, Confidence.HIGH + + # Check if the CI workflow is a configuration for a known tool. + if excluded_configs and os.path.basename(cmd["ci_path"]) in excluded_configs: + return False, Confidence.HIGH + + return True, Confidence.HIGH diff --git a/src/macaron/slsa_analyzer/build_tool/pip.py b/src/macaron/slsa_analyzer/build_tool/pip.py index 5e1bb68a5..2ee2752c7 100644 --- a/src/macaron/slsa_analyzer/build_tool/pip.py +++ b/src/macaron/slsa_analyzer/build_tool/pip.py @@ -15,6 +15,7 @@ from macaron.config.global_config import global_config from macaron.dependency_analyzer.cyclonedx import DependencyAnalyzer from macaron.dependency_analyzer.cyclonedx_python import CycloneDxPython +from macaron.slsa_analyzer.build_tool import pyproject from macaron.slsa_analyzer.build_tool.base_build_tool import BaseBuildTool, BuildToolCommand, file_exists from macaron.slsa_analyzer.build_tool.language import BuildLanguage from macaron.slsa_analyzer.checks.check_result import Confidence @@ -55,7 +56,19 @@ def is_detected(self, repo_path: str) -> bool: bool True if this build tool is detected, else False. """ - return any(file_exists(repo_path, file, filters=self.path_filters) for file in self.build_configs) + for config_name in self.build_configs: + if config_path := file_exists(repo_path, config_name, filters=self.path_filters): + if os.path.basename(config_path) == "pyproject.toml": + # Check the build-system section. If it doesn't exist, by default setuptools should be used. + if pyproject.get_build_system(config_path) is None: + return True + for tool in self.build_requires + self.build_backend: + if pyproject.build_system_contains_tool(tool, config_path): + return True + else: + # TODO: For other build configuration files, like setup.py, we need to improve the logic. + return True + return False def get_dep_analyzer(self) -> DependencyAnalyzer: """Create a DependencyAnalyzer for the build tool. diff --git a/src/macaron/slsa_analyzer/build_tool/poetry.py b/src/macaron/slsa_analyzer/build_tool/poetry.py index c12a40e33..dde2bfa28 100644 --- a/src/macaron/slsa_analyzer/build_tool/poetry.py +++ b/src/macaron/slsa_analyzer/build_tool/poetry.py @@ -6,11 +6,8 @@ This module is used to work with repositories that use Poetry for dependency management. """ -import glob import logging import os -import tomllib -from pathlib import Path from cyclonedx_py import __version__ as cyclonedx_version @@ -18,6 +15,7 @@ from macaron.config.global_config import global_config from macaron.dependency_analyzer.cyclonedx import DependencyAnalyzer from macaron.dependency_analyzer.cyclonedx_python import CycloneDxPython +from macaron.slsa_analyzer.build_tool import pyproject from macaron.slsa_analyzer.build_tool.base_build_tool import BaseBuildTool, BuildToolCommand, file_exists from macaron.slsa_analyzer.build_tool.language import BuildLanguage from macaron.slsa_analyzer.checks.check_result import Confidence @@ -41,9 +39,9 @@ def load_defaults(self) -> None: setattr(self, item, defaults.get_list("builder.poetry", item)) if "builder.pip.ci.deploy" in defaults: - for item in defaults["builder.pip.ci.deploy"]: + for item in defaults["builder.poetry.ci.deploy"]: if item in self.ci_deploy_kws: - self.ci_deploy_kws[item] = defaults.get_list("builder.pip.ci.deploy", item) + self.ci_deploy_kws[item] = defaults.get_list("builder.poetry.ci.deploy", item) def is_detected(self, repo_path: str) -> bool: """Return True if this build tool is used in the target repo. @@ -64,34 +62,17 @@ def is_detected(self, repo_path: str) -> bool: package_lock_exists = file break - for conf in self.build_configs: - # Find the paths of all pyproject.toml files. - pattern = os.path.join(repo_path, "**", conf) - files_detected = glob.glob(pattern, recursive=True) - - if files_detected: - # If a package_lock file exists, and a config file is present, Poetry build tool is detected. + file_paths = (file_exists(repo_path, file, filters=self.path_filters) for file in self.build_configs) + for config_path in file_paths: + if config_path and os.path.basename(config_path) == "pyproject.toml": if package_lock_exists: return True - # TODO: this implementation assumes one build type, so when multiple build types are supported, this - # needs to be updated. - # Take the highest level file, if there are two at the same level, take the first in the list. - file_path = min(files_detected, key=lambda x: len(Path(x).parts)) - try: - # Parse the .toml file - with open(file_path, "rb") as toml_file: - try: - data = tomllib.load(toml_file) - # Check for the existence of a [tool.poetry] section. - if ("tool" in data) and ("poetry" in data["tool"]): - return True - except tomllib.TOMLDecodeError: - logger.debug("Failed to read the %s file: invalid toml file.", conf) - return False - return False - except FileNotFoundError: - logger.debug("Failed to read the %s file.", conf) - return False + if pyproject.contains_build_tool("poetry", config_path): + return True + # Check the build-system section. + for tool in self.build_requires + self.build_backend: + if pyproject.build_system_contains_tool(tool, config_path): + return True return False diff --git a/src/macaron/slsa_analyzer/build_tool/pyproject.py b/src/macaron/slsa_analyzer/build_tool/pyproject.py new file mode 100644 index 000000000..5b327f94c --- /dev/null +++ b/src/macaron/slsa_analyzer/build_tool/pyproject.py @@ -0,0 +1,117 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module provides analysis functions for a pyproject.toml file.""" + +import logging +import tomllib +from pathlib import Path +from typing import Any + +from tomli import TOMLDecodeError + +from macaron.json_tools import json_extract + +logger: logging.Logger = logging.getLogger(__name__) + + +def get_content(pyproject_path: Path) -> dict[str, Any] | None: + """ + Return the pyproject.toml content. + + Parameters + ---------- + pyproject_path : Path + The file path to the pyproject.toml file. + + Returns + ------- + dict[str, Any] | None + The [build-system] section as a dict, or None otherwise. + """ + try: + with open(pyproject_path, "rb") as toml_file: + return tomllib.load(toml_file) + except (FileNotFoundError, TypeError, TOMLDecodeError) as error: + logger.debug("Failed to read the %s file: %s", pyproject_path, error) + return None + + +def contains_build_tool(tool_name: str, pyproject_path: Path) -> bool: + """ + Check if a given build tool is present in the [tool] section of a pyproject.toml file. + + Parameters + ---------- + tool_name : str + The name of the build tool to search for (e.g., 'poetry', 'flit'). + pyproject_path : Path + The file path to the pyproject.toml file. + + Returns + ------- + bool + True if the build tool is found in the [tool] section, False otherwise. + """ + content = get_content(pyproject_path) + if not content: + return False + + # Check for the existence of a [tool.] section. + tools = json_extract(content, ["tool"], dict) + if tools and tool_name in tools: + return True + return False + + +def build_system_contains_tool(tool_name: str, pyproject_path: Path) -> bool: + """ + Check if the [build-system] section lists the specified tool in 'build-backend' or 'requires' in pyproject.toml. + + Parameters + ---------- + tool_name : str + The tool or backend name to search for (e.g., 'setuptools', 'poetry.masonry.api', 'flit_core.buildapi'). + pyproject_path : Path + The file path to the pyproject.toml file. + + Returns + ------- + bool + True if the tool is found in either the 'build-backend' or 'requires' of the [build-system] section, False otherwise. + """ + content = get_content(pyproject_path) + if not content: + return False + + # Check in 'build-backend'. + backend = json_extract(content, ["build-system", "build-backend"], str) + if backend and tool_name in backend: + return True + # Check in 'requires' list. + requires = json_extract(content, ["build-system", "requires"], list) + if requires and any(tool_name in req for req in requires): + return True + + return False + + +def get_build_system(pyproject_path: Path) -> dict[str, str] | None: + """ + Return the [build-system] section in pyproject.toml if it exists. + + Parameters + ---------- + pyproject_path : Path + The file path to the pyproject.toml file. + + Returns + ------- + dict[str, str] | None + The [build-system] section as a dict, or None otherwise. + """ + content = get_content(pyproject_path) + if not content: + return None + + return json_extract(content, ["build-system"], dict) diff --git a/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py b/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py index 075f9ee2e..87f4877d8 100644 --- a/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py +++ b/src/macaron/slsa_analyzer/checks/detect_malicious_metadata_check.py @@ -295,8 +295,7 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: match package_registry_info_entry: # Currently, only PyPI packages are supported. case PackageRegistryInfo( - build_tool_name="pip" | "poetry", - build_tool_purl_type="pypi", + ecosystem="pypi", package_registry=PyPIRegistry(), ) as pypi_registry_info: # Retrieve the pre-existing asset, or create a new one. diff --git a/src/macaron/slsa_analyzer/checks/infer_artifact_pipeline_check.py b/src/macaron/slsa_analyzer/checks/infer_artifact_pipeline_check.py index ce9f6bd2f..a10d14d57 100644 --- a/src/macaron/slsa_analyzer/checks/infer_artifact_pipeline_check.py +++ b/src/macaron/slsa_analyzer/checks/infer_artifact_pipeline_check.py @@ -120,7 +120,7 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: # Look for the artifact in the corresponding registry and find the publish timestamp. artifact_published_date = None for registry_info in ctx.dynamic_data["package_registries"]: - if registry_info.build_tool_purl_type == ctx.component.type: + if registry_info.ecosystem == ctx.component.type: try: artifact_published_date = registry_info.package_registry.find_publish_timestamp(ctx.component.purl) break diff --git a/src/macaron/slsa_analyzer/package_registry/jfrog_maven_registry.py b/src/macaron/slsa_analyzer/package_registry/jfrog_maven_registry.py index 02188de1d..25b3145ed 100644 --- a/src/macaron/slsa_analyzer/package_registry/jfrog_maven_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/jfrog_maven_registry.py @@ -123,7 +123,7 @@ def __init__( self.request_timeout = request_timeout or 10 self.download_timeout = download_timeout or 120 self.enabled = enabled or False - super().__init__("JFrog Maven Registry", {"maven", "gradle"}) + super().__init__("JFrog Maven Registry", "maven") def load_defaults(self) -> None: """Load the .ini configuration for the current package registry. diff --git a/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py b/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py index 1618bbdb5..010cb20cf 100644 --- a/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/maven_central_registry.py @@ -106,7 +106,7 @@ def __init__( self.registry_url_scheme = registry_url_scheme or "" self.registry_url = "" # Created from the registry_url_scheme and registry_url_netloc. self.request_timeout = request_timeout or 10 - super().__init__("Maven Central Registry", {"maven", "gradle"}) + super().__init__("Maven Central Registry", "maven") def load_defaults(self) -> None: """Load the .ini configuration for the current package registry. diff --git a/src/macaron/slsa_analyzer/package_registry/npm_registry.py b/src/macaron/slsa_analyzer/package_registry/npm_registry.py index fe009cc34..5613a62c5 100644 --- a/src/macaron/slsa_analyzer/package_registry/npm_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/npm_registry.py @@ -50,7 +50,7 @@ def __init__( self.attestation_endpoint = attestation_endpoint or "" self.request_timeout = request_timeout or 10 self.enabled = enabled - super().__init__("npm Registry", {"npm", "yarn"}) + super().__init__("npm Registry", "npm") def load_defaults(self) -> None: """Load the .ini configuration for the current package registry. diff --git a/src/macaron/slsa_analyzer/package_registry/package_registry.py b/src/macaron/slsa_analyzer/package_registry/package_registry.py index ca0adfa62..87a130ac8 100644 --- a/src/macaron/slsa_analyzer/package_registry/package_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/package_registry.py @@ -17,36 +17,36 @@ class PackageRegistry(ABC): """Base package registry class.""" - def __init__(self, name: str, build_tool_names: set[str]) -> None: + def __init__(self, name: str, ecosystem: str) -> None: self.name = name - self.build_tool_names = build_tool_names + self.ecosystem = ecosystem self.enabled: bool = True @abstractmethod def load_defaults(self) -> None: """Load the .ini configuration for the current package registry.""" - def is_detected(self, build_tool_name: str) -> bool: + def is_detected(self, ecosystem: str) -> bool: """Detect if artifacts of the repo under analysis can possibly be published to this package registry. - The detection here is based on the repo's detected build tool. - If the package registry is compatible with the given build tool, it can be a - possible place where the artifacts produced from the repo are published. + The detection here is based on the artifact ecosystem. + If the package registry is compatible with the given PURL ecosystem, it can be a + possible place where it is published. Parameters ---------- - build_tool_name: str - The name of a detected build tool of the repository under analysis. + ecosystem: str + The name of the artifact ecosystem under analysis. Returns ------- bool ``True`` if the repo under analysis can be published to this package registry, - based on the given build tool. + based on the given software component ecosystem. """ if not self.enabled: return False - return build_tool_name in self.build_tool_names + return ecosystem == self.ecosystem def find_publish_timestamp(self, purl: str) -> datetime: """Retrieve the publication timestamp for a package specified by its purl from the deps.dev repository by default. diff --git a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py index 0f61e4037..ee2ebf6a6 100644 --- a/src/macaron/slsa_analyzer/package_registry/pypi_registry.py +++ b/src/macaron/slsa_analyzer/package_registry/pypi_registry.py @@ -91,7 +91,7 @@ def __init__( self.request_timeout = request_timeout or 10 self.enabled = enabled self.registry_url = "" - super().__init__("PyPI Registry", {"pip", "poetry"}) + super().__init__("PyPI Registry", "pypi") def load_defaults(self) -> None: """Load the .ini configuration for the current package registry. @@ -555,13 +555,13 @@ class PyPIPackageJsonAsset: #: The asset content. package_json: dict - #: the source code temporary location name + #: The source code temporary location name. package_sourcecode_path: str - #: the wheel temporary location name + #: The wheel temporary location name. wheel_path: str - #: name of the wheel file + #: Name of the wheel file. wheel_filename: str #: The size of the asset (in bytes). This attribute is added to match the AssetLocator diff --git a/src/macaron/slsa_analyzer/specs/package_registry_spec.py b/src/macaron/slsa_analyzer/specs/package_registry_spec.py index 84b2a69e7..364bd932e 100644 --- a/src/macaron/slsa_analyzer/specs/package_registry_spec.py +++ b/src/macaron/slsa_analyzer/specs/package_registry_spec.py @@ -13,12 +13,10 @@ @dataclass class PackageRegistryInfo: - """This class contains data for one package registry that is matched against a repository.""" + """This class contains data for one package registry that is matched against a software component.""" - #: The name of the build tool matched against the repository. - build_tool_name: str #: The purl type of the build tool matched against the repository. - build_tool_purl_type: str + ecosystem: str #: The package registry matched against the repository. This is dependent on the build tool detected. package_registry: PackageRegistry #: The provenances matched against the current repo. diff --git a/tests/build_spec_generator/common_spec/test_core.py b/tests/build_spec_generator/common_spec/test_core.py index 9f2eec87e..7df8b1615 100644 --- a/tests/build_spec_generator/common_spec/test_core.py +++ b/tests/build_spec_generator/common_spec/test_core.py @@ -9,7 +9,7 @@ MacaronBuildToolName, compose_shell_commands, get_language_version, - get_macaron_build_tool_name, + get_macaron_build_tool_names, ) from macaron.build_spec_generator.macaron_db_extractor import GenericBuildCommandInfo from macaron.slsa_analyzer.checks.build_tool_check import BuildToolFacts @@ -52,7 +52,7 @@ def test_compose_shell_commands( ) ], "python", - MacaronBuildToolName.PIP, + [MacaronBuildToolName.PIP], id="python_pip_supported", ), pytest.param( @@ -63,7 +63,7 @@ def test_compose_shell_commands( ) ], "java", - MacaronBuildToolName.GRADLE, + [MacaronBuildToolName.GRADLE], id="build_tool_gradle", ), pytest.param( @@ -74,7 +74,7 @@ def test_compose_shell_commands( ) ], "java", - MacaronBuildToolName.MAVEN, + [MacaronBuildToolName.MAVEN], id="build_tool_maven", ), pytest.param( @@ -104,10 +104,10 @@ def test_compose_shell_commands( def test_get_build_tool_name( build_tool_facts: list[BuildToolFacts], language: str, - expected: MacaronBuildToolName | None, + expected: list[MacaronBuildToolName] | None, ) -> None: """Test build tool name detection.""" - assert get_macaron_build_tool_name(build_tool_facts, target_language=language) == expected + assert get_macaron_build_tool_names(build_tool_facts, target_language=language) == expected @pytest.mark.parametrize( diff --git a/tests/build_spec_generator/reproducible_central/test_reproducible_central.py b/tests/build_spec_generator/reproducible_central/test_reproducible_central.py index 39b8ef5ed..f28b93f66 100644 --- a/tests/build_spec_generator/reproducible_central/test_reproducible_central.py +++ b/tests/build_spec_generator/reproducible_central/test_reproducible_central.py @@ -24,7 +24,7 @@ def fixture_base_build_spec() -> BaseBuildSpecDict: "version": "1.2.3", "git_repo": "https://github.com/oracle/example-artifact.git", "git_tag": "sampletag", - "build_tool": "maven", + "build_tools": ["maven"], "newline": "lf", "language_version": ["17"], "build_commands": [["mvn", "package"]], @@ -45,7 +45,7 @@ def test_successful_build_spec(base_build_spec: BaseBuildSpecDict) -> None: def test_unsupported_build_tool(base_build_spec: BaseBuildSpecDict) -> None: """Test an unsupported build tool name.""" - base_build_spec["build_tool"] = "unsupported_tool" + base_build_spec["build_tools"] = ["unsupported_tool"] with pytest.raises(GenerateBuildSpecError) as excinfo: gen_reproducible_central_build_spec(base_build_spec) assert "is not supported by Reproducible Central" in str(excinfo.value) @@ -60,17 +60,17 @@ def test_missing_group_id(base_build_spec: BaseBuildSpecDict) -> None: @pytest.mark.parametrize( - ("build_tool", "expected"), + ("build_tools", "expected"), [ - ("maven", "mvn"), - ("gradle", "gradle"), - ("MAVEN", "mvn"), - ("GRADLE", "gradle"), + (["maven", "pip"], "mvn"), + (["gradle"], "gradle"), + (["MAVEN"], "mvn"), + (["GRADLE", "pip"], "gradle"), ], ) -def test_build_tool_name_variants(base_build_spec: BaseBuildSpecDict, build_tool: str, expected: str) -> None: +def test_build_tool_name_variants(base_build_spec: BaseBuildSpecDict, build_tools: list[str], expected: str) -> None: """Test the correct handling of build tool names.""" - base_build_spec["build_tool"] = build_tool + base_build_spec["build_tools"] = build_tools content = gen_reproducible_central_build_spec(base_build_spec) assert content assert f"tool={expected}" in content diff --git a/tests/conftest.py b/tests/conftest.py index d4ed2ab1b..77223948f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,9 +20,12 @@ from macaron.parsers.github_workflow_model import Identified, Job, NormalJob, RunStep, Workflow from macaron.slsa_analyzer.analyze_context import AnalyzeContext from macaron.slsa_analyzer.build_tool.base_build_tool import BaseBuildTool +from macaron.slsa_analyzer.build_tool.conda import Conda from macaron.slsa_analyzer.build_tool.docker import Docker +from macaron.slsa_analyzer.build_tool.flit import Flit from macaron.slsa_analyzer.build_tool.go import Go from macaron.slsa_analyzer.build_tool.gradle import Gradle +from macaron.slsa_analyzer.build_tool.hatch import Hatch from macaron.slsa_analyzer.build_tool.maven import Maven from macaron.slsa_analyzer.build_tool.npm import NPM from macaron.slsa_analyzer.build_tool.pip import Pip @@ -149,6 +152,63 @@ def poetry_tool(setup_test) -> Poetry: # type: ignore # pylint: disable=unused- return poetry +@pytest.fixture(autouse=True) +def flit_tool(setup_test) -> Flit: # type: ignore # pylint: disable=unused-argument + """Create a Flit tool instance. + + Parameters + ---------- + setup_test + Depends on setup_test fixture. + + Returns + ------- + Flit + The Flit instance. + """ + flit = Flit() + flit.load_defaults() + return flit + + +@pytest.fixture(autouse=True) +def hatch_tool(setup_test) -> Hatch: # type: ignore # pylint: disable=unused-argument + """Create a Hatch tool instance. + + Parameters + ---------- + setup_test + Depends on setup_test fixture. + + Returns + ------- + Hatch + The Hatch instance. + """ + hatch = Hatch() + hatch.load_defaults() + return hatch + + +@pytest.fixture(autouse=True) +def conda_tool(setup_test) -> Conda: # type: ignore # pylint: disable=unused-argument + """Create a Conda tool instance. + + Parameters + ---------- + setup_test + Depends on setup_test fixture. + + Returns + ------- + Conda + The Conda instance. + """ + conda = Conda() + conda.load_defaults() + return conda + + @pytest.fixture(autouse=True) def pip_tool(setup_test) -> Pip: # type: ignore # pylint: disable=unused-argument """Create a Pip tool instance. @@ -253,6 +313,9 @@ def get_build_tools( gradle_tool: BaseBuildTool, pip_tool: BaseBuildTool, poetry_tool: BaseBuildTool, + flit_tool: BaseBuildTool, + hatch_tool: BaseBuildTool, + conda_tool: BaseBuildTool, docker_tool: BaseBuildTool, ) -> dict[str, BaseBuildTool]: """Create a dictionary to look up build tool fixtures. @@ -268,6 +331,9 @@ def get_build_tools( "gradle": gradle_tool, "pip": pip_tool, "poetry": poetry_tool, + "flit": flit_tool, + "hatch": hatch_tool, + "conda": conda_tool, "docker": docker_tool, } diff --git a/tests/integration/cases/org_apache_hugegraph/computer-k8s/expected_default.buildspec b/tests/integration/cases/org_apache_hugegraph/computer-k8s/expected_default.buildspec index 1bfeba572..86325ad7f 100644 --- a/tests/integration/cases/org_apache_hugegraph/computer-k8s/expected_default.buildspec +++ b/tests/integration/cases/org_apache_hugegraph/computer-k8s/expected_default.buildspec @@ -1 +1,30 @@ -{"macaron_version": "0.18.0", "group_id": "org.apache.hugegraph", "artifact_id": "computer-k8s", "version": "1.0.0", "git_repo": "https://github.com/apache/hugegraph-computer", "git_tag": "d2b95262091d6572cc12dcda57d89f9cd44ac88b", "newline": "lf", "language_version": ["11"], "ecosystem": "maven", "purl": "pkg:maven/org.apache.hugegraph/computer-k8s@1.0.0", "language": "java", "build_tool": "maven", "build_commands": [["mvn", "-DskipTests=true", "-Dmaven.test.skip=true", "-Dmaven.site.skip=true", "-Drat.skip=true", "-Dmaven.javadoc.skip=true", "clean", "package"]]} +{ + "macaron_version": "0.18.0", + "group_id": "org.apache.hugegraph", + "artifact_id": "computer-k8s", + "version": "1.0.0", + "git_repo": "https://github.com/apache/hugegraph-computer", + "git_tag": "d2b95262091d6572cc12dcda57d89f9cd44ac88b", + "newline": "lf", + "language_version": [ + "11" + ], + "ecosystem": "maven", + "purl": "pkg:maven/org.apache.hugegraph/computer-k8s@1.0.0", + "language": "java", + "build_tools": [ + "maven" + ], + "build_commands": [ + [ + "mvn", + "-DskipTests=true", + "-Dmaven.test.skip=true", + "-Dmaven.site.skip=true", + "-Drat.skip=true", + "-Dmaven.javadoc.skip=true", + "clean", + "package" + ] + ] +} diff --git a/tests/integration/cases/pypi_cachetools/expected_default.buildspec b/tests/integration/cases/pypi_cachetools/expected_default.buildspec index ceb606aa5..18e24d556 100644 --- a/tests/integration/cases/pypi_cachetools/expected_default.buildspec +++ b/tests/integration/cases/pypi_cachetools/expected_default.buildspec @@ -12,7 +12,9 @@ "ecosystem": "pypi", "purl": "pkg:pypi/cachetools@6.2.1", "language": "python", - "build_tool": "pip", + "build_tools": [ + "pip" + ], "build_commands": [ [ "python", @@ -20,8 +22,11 @@ "build" ] ], - "build_backends": { + "build_requires": { "setuptools": "==(80.9.0)", "wheel": "" - } + }, + "build_backends": [ + "setuptools.build_meta" + ] } diff --git a/tests/integration/cases/pypi_markdown-it-py/expected_default.buildspec b/tests/integration/cases/pypi_markdown-it-py/expected_default.buildspec new file mode 100644 index 000000000..e7842d046 --- /dev/null +++ b/tests/integration/cases/pypi_markdown-it-py/expected_default.buildspec @@ -0,0 +1,31 @@ +{ + "macaron_version": "0.18.0", + "group_id": null, + "artifact_id": "markdown-it-py", + "version": "4.0.0", + "git_repo": "https://github.com/executablebooks/markdown-it-py", + "git_tag": "c62983f1554124391b47170180e6c62df4d476ca", + "newline": "lf", + "language_version": [ + ">=3.10" + ], + "ecosystem": "pypi", + "purl": "pkg:pypi/markdown-it-py@4.0.0", + "language": "python", + "build_tools": [ + "flit" + ], + "build_commands": [ + [ + "flit", + "build" + ] + ], + "build_requires": { + "flit": "==3.12.0", + "flit_core": "<4,>=3.4" + }, + "build_backends": [ + "flit_core.buildapi" + ] +} diff --git a/tests/integration/cases/pypi_markdown-it-py/test.yaml b/tests/integration/cases/pypi_markdown-it-py/test.yaml new file mode 100644 index 000000000..a57b7d2cf --- /dev/null +++ b/tests/integration/cases/pypi_markdown-it-py/test.yaml @@ -0,0 +1,29 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Test buildspec generation for the Flit build tool. + +tags: +- macaron-python-package +- tutorial + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -purl + - pkg:pypi/markdown-it-py@4.0.0 +- name: Generate the buildspec + kind: gen-build-spec + options: + command_args: + - -purl + - pkg:pypi/markdown-it-py@4.0.0 +- name: Compare Buildspec. + kind: compare + options: + kind: default_build_spec + result: output/buildspec/pypi/markdown-it-py/macaron.buildspec + expected: expected_default.buildspec diff --git a/tests/integration/cases/pypi_toga/expected_default.buildspec b/tests/integration/cases/pypi_toga/expected_default.buildspec index 2418f3909..cc0ef7be9 100644 --- a/tests/integration/cases/pypi_toga/expected_default.buildspec +++ b/tests/integration/cases/pypi_toga/expected_default.buildspec @@ -12,7 +12,9 @@ "ecosystem": "pypi", "purl": "pkg:pypi/toga@0.5.1", "language": "python", - "build_tool": "pip", + "build_tools": [ + "pip" + ], "build_commands": [ [ "python", @@ -20,9 +22,12 @@ "build" ] ], - "build_backends": { + "build_requires": { "setuptools": "==(80.3.1)", - "setuptools_scm": "==8.3.1", - "setuptools_dynamic_dependencies": "==1.0.0" - } + "setuptools_dynamic_dependencies": "==1.0.0", + "setuptools_scm": "==8.3.1" + }, + "build_backends": [ + "setuptools.build_meta" + ] } diff --git a/tests/slsa_analyzer/build_tool/__snapshots__/test_conda.ambr b/tests/slsa_analyzer/build_tool/__snapshots__/test_conda.ambr new file mode 100644 index 000000000..2a8f1092a --- /dev/null +++ b/tests/slsa_analyzer/build_tool/__snapshots__/test_conda.ambr @@ -0,0 +1,10 @@ +# serializer version: 1 +# name: test_get_build_dirs[mock_repo0] + list([ + PosixPath('.'), + ]) +# --- +# name: test_get_build_dirs[mock_repo1] + list([ + ]) +# --- diff --git a/tests/slsa_analyzer/build_tool/__snapshots__/test_flit.ambr b/tests/slsa_analyzer/build_tool/__snapshots__/test_flit.ambr new file mode 100644 index 000000000..2a8f1092a --- /dev/null +++ b/tests/slsa_analyzer/build_tool/__snapshots__/test_flit.ambr @@ -0,0 +1,10 @@ +# serializer version: 1 +# name: test_get_build_dirs[mock_repo0] + list([ + PosixPath('.'), + ]) +# --- +# name: test_get_build_dirs[mock_repo1] + list([ + ]) +# --- diff --git a/tests/slsa_analyzer/build_tool/__snapshots__/test_hatch.ambr b/tests/slsa_analyzer/build_tool/__snapshots__/test_hatch.ambr new file mode 100644 index 000000000..2a8f1092a --- /dev/null +++ b/tests/slsa_analyzer/build_tool/__snapshots__/test_hatch.ambr @@ -0,0 +1,10 @@ +# serializer version: 1 +# name: test_get_build_dirs[mock_repo0] + list([ + PosixPath('.'), + ]) +# --- +# name: test_get_build_dirs[mock_repo1] + list([ + ]) +# --- diff --git a/tests/slsa_analyzer/build_tool/mock_repos/conda_repos/has_conda/meta.yaml b/tests/slsa_analyzer/build_tool/mock_repos/conda_repos/has_conda/meta.yaml new file mode 100644 index 000000000..8e17a3508 --- /dev/null +++ b/tests/slsa_analyzer/build_tool/mock_repos/conda_repos/has_conda/meta.yaml @@ -0,0 +1,2 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. diff --git a/tests/slsa_analyzer/build_tool/mock_repos/conda_repos/no_conda/pyproject.toml b/tests/slsa_analyzer/build_tool/mock_repos/conda_repos/no_conda/pyproject.toml new file mode 100644 index 000000000..8e17a3508 --- /dev/null +++ b/tests/slsa_analyzer/build_tool/mock_repos/conda_repos/no_conda/pyproject.toml @@ -0,0 +1,2 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. diff --git a/tests/slsa_analyzer/build_tool/mock_repos/flit_repos/has_flit_pyproject/pyproject.toml b/tests/slsa_analyzer/build_tool/mock_repos/flit_repos/has_flit_pyproject/pyproject.toml new file mode 100644 index 000000000..81fb17053 --- /dev/null +++ b/tests/slsa_analyzer/build_tool/mock_repos/flit_repos/has_flit_pyproject/pyproject.toml @@ -0,0 +1,6 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +[build-system] +requires = ["flit_core >=3.4,<4"] +build-backend = "flit_core.buildapi" diff --git a/tests/slsa_analyzer/build_tool/mock_repos/flit_repos/no_flit/pyproject.toml b/tests/slsa_analyzer/build_tool/mock_repos/flit_repos/no_flit/pyproject.toml new file mode 100644 index 000000000..8e17a3508 --- /dev/null +++ b/tests/slsa_analyzer/build_tool/mock_repos/flit_repos/no_flit/pyproject.toml @@ -0,0 +1,2 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. diff --git a/tests/slsa_analyzer/build_tool/mock_repos/hatch_repos/has_hatch_pyproject/pyproject.toml b/tests/slsa_analyzer/build_tool/mock_repos/hatch_repos/has_hatch_pyproject/pyproject.toml new file mode 100644 index 000000000..9fc136edc --- /dev/null +++ b/tests/slsa_analyzer/build_tool/mock_repos/hatch_repos/has_hatch_pyproject/pyproject.toml @@ -0,0 +1,6 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/tests/slsa_analyzer/build_tool/mock_repos/hatch_repos/no_hatch/pyproject.toml b/tests/slsa_analyzer/build_tool/mock_repos/hatch_repos/no_hatch/pyproject.toml new file mode 100644 index 000000000..8e17a3508 --- /dev/null +++ b/tests/slsa_analyzer/build_tool/mock_repos/hatch_repos/no_hatch/pyproject.toml @@ -0,0 +1,2 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. diff --git a/tests/slsa_analyzer/build_tool/test_conda.py b/tests/slsa_analyzer/build_tool/test_conda.py new file mode 100644 index 000000000..896abad13 --- /dev/null +++ b/tests/slsa_analyzer/build_tool/test_conda.py @@ -0,0 +1,229 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module tests the Conda build functions.""" + +from pathlib import Path + +import pytest + +from macaron.code_analyzer.call_graph import BaseNode +from macaron.slsa_analyzer.build_tool.base_build_tool import BuildToolCommand +from macaron.slsa_analyzer.build_tool.conda import Conda +from macaron.slsa_analyzer.build_tool.language import BuildLanguage +from tests.slsa_analyzer.mock_git_utils import prepare_repo_for_testing + + +@pytest.mark.parametrize( + "mock_repo", + [ + Path(__file__).parent.joinpath("mock_repos", "conda_repos", "has_conda"), + Path(__file__).parent.joinpath("mock_repos", "conda_repos", "no_conda"), + ], +) +def test_get_build_dirs(snapshot: list, conda_tool: Conda, mock_repo: Path) -> None: + """Test discovering build directories.""" + assert list(conda_tool.get_build_dirs(str(mock_repo))) == snapshot + + +@pytest.mark.parametrize( + ("mock_repo", "expected_value"), + [ + (Path(__file__).parent.joinpath("mock_repos", "conda_repos", "has_conda"), True), + (Path(__file__).parent.joinpath("mock_repos", "conda_repos", "no_conda"), False), + ], +) +def test_conda_build_tool(conda_tool: Conda, macaron_path: str, mock_repo: str, expected_value: bool) -> None: + """Test the Conda build tool.""" + base_dir = Path(__file__).parent + ctx = prepare_repo_for_testing(mock_repo, macaron_path, base_dir) + assert conda_tool.is_detected(ctx.component.repository.fs_path) == expected_value + + +@pytest.mark.parametrize( + ( + "command", + "language", + "language_versions", + "language_distributions", + "ci_path", + "reachable_secrets", + "events", + "excluded_configs", + "expected_result", + ), + [ + ( + ["conda", "publish"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["release"], + ["codeql-analysis.yaml"], + True, + ), + ( + ["conda", "publish"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/conda.yaml", + [{"key", "pass"}], + ["push"], + ["conda.yaml"], + False, + ), + ( + ["python", "-m", "conda", "publish"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["release"], + ["codeql-analysis.yaml"], + True, + ), + ( + ["conda", "publish"], + BuildLanguage.JAVASCRIPT, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["push"], + None, + False, + ), + ], +) +def test_is_conda_deploy_command( + conda_tool: Conda, + command: list[str], + language: str, + language_versions: list[str], + language_distributions: list[str], + ci_path: str, + reachable_secrets: list[str], + events: list[str], + excluded_configs: list[str] | None, + expected_result: bool, +) -> None: + """Test the deploy commend detection function.""" + result, _ = conda_tool.is_deploy_command( + BuildToolCommand( + command=command, + language=language, + language_versions=language_versions, + language_distributions=language_distributions, + language_url=None, + ci_path=ci_path, + step_node=BaseNode(), + reachable_secrets=reachable_secrets, + events=events, + ), + excluded_configs=excluded_configs, + ) + assert result == expected_result + + +@pytest.mark.parametrize( + ( + "command", + "language", + "language_versions", + "language_distributions", + "ci_path", + "reachable_secrets", + "events", + "excluded_configs", + "expected_result", + ), + [ + ( + ["conda", "build"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["release"], + ["codeql-analysis.yaml"], + True, + ), + ( + ["python", "-m", "conda", "build"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["push"], + ["conda.yaml"], + True, + ), + ( + ["python", "-m", "conda", "build"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/conda.yaml", + [{"key", "pass"}], + ["push"], + ["conda.yaml"], + False, + ), + ( + ["conda", "--version"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["release"], + ["codeql-analysis.yaml"], + False, + ), + ( + ["conda", "build"], + BuildLanguage.JAVA, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["push"], + None, + False, + ), + ], +) +def test_is_conda_package_command( + conda_tool: Conda, + command: list[str], + language: str, + language_versions: list[str], + language_distributions: list[str], + ci_path: str, + reachable_secrets: list[str], + events: list[str], + excluded_configs: list[str] | None, + expected_result: bool, +) -> None: + """Test the packaging command detection function.""" + result, _ = conda_tool.is_package_command( + BuildToolCommand( + command=command, + language=language, + language_versions=language_versions, + language_distributions=language_distributions, + language_url=None, + ci_path=ci_path, + step_node=BaseNode(), + reachable_secrets=reachable_secrets, + events=events, + ), + excluded_configs=excluded_configs, + ) + assert result == expected_result diff --git a/tests/slsa_analyzer/build_tool/test_flit.py b/tests/slsa_analyzer/build_tool/test_flit.py new file mode 100644 index 000000000..9a3757c78 --- /dev/null +++ b/tests/slsa_analyzer/build_tool/test_flit.py @@ -0,0 +1,229 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module tests the Flit build functions.""" + +from pathlib import Path + +import pytest + +from macaron.code_analyzer.call_graph import BaseNode +from macaron.slsa_analyzer.build_tool.base_build_tool import BuildToolCommand +from macaron.slsa_analyzer.build_tool.flit import Flit +from macaron.slsa_analyzer.build_tool.language import BuildLanguage +from tests.slsa_analyzer.mock_git_utils import prepare_repo_for_testing + + +@pytest.mark.parametrize( + "mock_repo", + [ + Path(__file__).parent.joinpath("mock_repos", "flit_repos", "has_flit_pyproject"), + Path(__file__).parent.joinpath("mock_repos", "flit_repos", "no_flit"), + ], +) +def test_get_build_dirs(snapshot: list, flit_tool: Flit, mock_repo: Path) -> None: + """Test discovering build directories.""" + assert list(flit_tool.get_build_dirs(str(mock_repo))) == snapshot + + +@pytest.mark.parametrize( + ("mock_repo", "expected_value"), + [ + (Path(__file__).parent.joinpath("mock_repos", "flit_repos", "has_flit_pyproject"), True), + (Path(__file__).parent.joinpath("mock_repos", "flit_repos", "no_flit"), False), + ], +) +def test_flit_build_tool(flit_tool: Flit, macaron_path: str, mock_repo: str, expected_value: bool) -> None: + """Test the Flit build tool.""" + base_dir = Path(__file__).parent + ctx = prepare_repo_for_testing(mock_repo, macaron_path, base_dir) + assert flit_tool.is_detected(ctx.component.repository.fs_path) == expected_value + + +@pytest.mark.parametrize( + ( + "command", + "language", + "language_versions", + "language_distributions", + "ci_path", + "reachable_secrets", + "events", + "excluded_configs", + "expected_result", + ), + [ + ( + ["flit", "publish"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["release"], + ["codeql-analysis.yaml"], + True, + ), + ( + ["flit", "publish"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/flit.yaml", + [{"key", "pass"}], + ["push"], + ["flit.yaml"], + False, + ), + ( + ["python", "-m", "flit", "publish"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["release"], + ["codeql-analysis.yaml"], + True, + ), + ( + ["flit", "publish"], + BuildLanguage.JAVASCRIPT, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["push"], + None, + False, + ), + ], +) +def test_is_flit_deploy_command( + flit_tool: Flit, + command: list[str], + language: str, + language_versions: list[str], + language_distributions: list[str], + ci_path: str, + reachable_secrets: list[str], + events: list[str], + excluded_configs: list[str] | None, + expected_result: bool, +) -> None: + """Test the deploy commend detection function.""" + result, _ = flit_tool.is_deploy_command( + BuildToolCommand( + command=command, + language=language, + language_versions=language_versions, + language_distributions=language_distributions, + language_url=None, + ci_path=ci_path, + step_node=BaseNode(), + reachable_secrets=reachable_secrets, + events=events, + ), + excluded_configs=excluded_configs, + ) + assert result == expected_result + + +@pytest.mark.parametrize( + ( + "command", + "language", + "language_versions", + "language_distributions", + "ci_path", + "reachable_secrets", + "events", + "excluded_configs", + "expected_result", + ), + [ + ( + ["flit", "build"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["release"], + ["codeql-analysis.yaml"], + True, + ), + ( + ["python", "-m", "flit", "build"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["push"], + ["flit.yaml"], + True, + ), + ( + ["python", "-m", "flit", "build"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/flit.yaml", + [{"key", "pass"}], + ["push"], + ["flit.yaml"], + False, + ), + ( + ["flit", "--version"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["release"], + ["codeql-analysis.yaml"], + False, + ), + ( + ["flit", "build"], + BuildLanguage.JAVA, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["push"], + None, + False, + ), + ], +) +def test_is_flit_package_command( + flit_tool: Flit, + command: list[str], + language: str, + language_versions: list[str], + language_distributions: list[str], + ci_path: str, + reachable_secrets: list[str], + events: list[str], + excluded_configs: list[str] | None, + expected_result: bool, +) -> None: + """Test the packaging command detection function.""" + result, _ = flit_tool.is_package_command( + BuildToolCommand( + command=command, + language=language, + language_versions=language_versions, + language_distributions=language_distributions, + language_url=None, + ci_path=ci_path, + step_node=BaseNode(), + reachable_secrets=reachable_secrets, + events=events, + ), + excluded_configs=excluded_configs, + ) + assert result == expected_result diff --git a/tests/slsa_analyzer/build_tool/test_hatch.py b/tests/slsa_analyzer/build_tool/test_hatch.py new file mode 100644 index 000000000..40e8d0f30 --- /dev/null +++ b/tests/slsa_analyzer/build_tool/test_hatch.py @@ -0,0 +1,229 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module tests the Hatch build functions.""" + +from pathlib import Path + +import pytest + +from macaron.code_analyzer.call_graph import BaseNode +from macaron.slsa_analyzer.build_tool.base_build_tool import BuildToolCommand +from macaron.slsa_analyzer.build_tool.hatch import Hatch +from macaron.slsa_analyzer.build_tool.language import BuildLanguage +from tests.slsa_analyzer.mock_git_utils import prepare_repo_for_testing + + +@pytest.mark.parametrize( + "mock_repo", + [ + Path(__file__).parent.joinpath("mock_repos", "hatch_repos", "has_hatch_pyproject"), + Path(__file__).parent.joinpath("mock_repos", "hatch_repos", "no_hatch"), + ], +) +def test_get_build_dirs(snapshot: list, hatch_tool: Hatch, mock_repo: Path) -> None: + """Test discovering build directories.""" + assert list(hatch_tool.get_build_dirs(str(mock_repo))) == snapshot + + +@pytest.mark.parametrize( + ("mock_repo", "expected_value"), + [ + (Path(__file__).parent.joinpath("mock_repos", "hatch_repos", "has_hatch_pyproject"), True), + (Path(__file__).parent.joinpath("mock_repos", "hatch_repos", "no_hatch"), False), + ], +) +def test_hatch_build_tool(hatch_tool: Hatch, macaron_path: str, mock_repo: str, expected_value: bool) -> None: + """Test the Hatch build tool.""" + base_dir = Path(__file__).parent + ctx = prepare_repo_for_testing(mock_repo, macaron_path, base_dir) + assert hatch_tool.is_detected(ctx.component.repository.fs_path) == expected_value + + +@pytest.mark.parametrize( + ( + "command", + "language", + "language_versions", + "language_distributions", + "ci_path", + "reachable_secrets", + "events", + "excluded_configs", + "expected_result", + ), + [ + ( + ["hatch", "publish"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["release"], + ["codeql-analysis.yaml"], + True, + ), + ( + ["hatch", "publish"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/hatch.yaml", + [{"key", "pass"}], + ["push"], + ["hatch.yaml"], + False, + ), + ( + ["python", "-m", "hatch", "publish"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["release"], + ["codeql-analysis.yaml"], + True, + ), + ( + ["hatch", "publish"], + BuildLanguage.JAVASCRIPT, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["push"], + None, + False, + ), + ], +) +def test_is_hatch_deploy_command( + hatch_tool: Hatch, + command: list[str], + language: str, + language_versions: list[str], + language_distributions: list[str], + ci_path: str, + reachable_secrets: list[str], + events: list[str], + excluded_configs: list[str] | None, + expected_result: bool, +) -> None: + """Test the deploy commend detection function.""" + result, _ = hatch_tool.is_deploy_command( + BuildToolCommand( + command=command, + language=language, + language_versions=language_versions, + language_distributions=language_distributions, + language_url=None, + ci_path=ci_path, + step_node=BaseNode(), + reachable_secrets=reachable_secrets, + events=events, + ), + excluded_configs=excluded_configs, + ) + assert result == expected_result + + +@pytest.mark.parametrize( + ( + "command", + "language", + "language_versions", + "language_distributions", + "ci_path", + "reachable_secrets", + "events", + "excluded_configs", + "expected_result", + ), + [ + ( + ["hatch", "build"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["release"], + ["codeql-analysis.yaml"], + True, + ), + ( + ["python", "-m", "hatch", "build"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["push"], + ["hatch.yaml"], + True, + ), + ( + ["python", "-m", "hatch", "build"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/hatch.yaml", + [{"key", "pass"}], + ["push"], + ["hatch.yaml"], + False, + ), + ( + ["hatch", "--version"], + BuildLanguage.PYTHON, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["release"], + ["codeql-analysis.yaml"], + False, + ), + ( + ["hatch", "build"], + BuildLanguage.JAVA, + None, + None, + ".github/workflows/release.yaml", + [{"key", "pass"}], + ["push"], + None, + False, + ), + ], +) +def test_is_hatch_package_command( + hatch_tool: Hatch, + command: list[str], + language: str, + language_versions: list[str], + language_distributions: list[str], + ci_path: str, + reachable_secrets: list[str], + events: list[str], + excluded_configs: list[str] | None, + expected_result: bool, +) -> None: + """Test the packaging command detection function.""" + result, _ = hatch_tool.is_package_command( + BuildToolCommand( + command=command, + language=language, + language_versions=language_versions, + language_distributions=language_distributions, + language_url=None, + ci_path=ci_path, + step_node=BaseNode(), + reachable_secrets=reachable_secrets, + events=events, + ), + excluded_configs=excluded_configs, + ) + assert result == expected_result diff --git a/tests/slsa_analyzer/build_tool/test_pip.py b/tests/slsa_analyzer/build_tool/test_pip.py index 4a7fb447f..1a069f31a 100644 --- a/tests/slsa_analyzer/build_tool/test_pip.py +++ b/tests/slsa_analyzer/build_tool/test_pip.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module tests the Pip build functions.""" @@ -144,7 +144,7 @@ def test_is_pip_deploy_command( [{"key", "pass"}], ["push"], None, - True, + False, ), ( ["python", "-m", "flit", "build"], diff --git a/tests/slsa_analyzer/checks/test_build_as_code_check.py b/tests/slsa_analyzer/checks/test_build_as_code_check.py index b1bd82b12..d34ae64e2 100644 --- a/tests/slsa_analyzer/checks/test_build_as_code_check.py +++ b/tests/slsa_analyzer/checks/test_build_as_code_check.py @@ -89,7 +89,10 @@ def test_no_build_tool(macaron_path: Path) -> None: ("poetry", ["poetry publish"], CheckResultType.PASSED), ("poetry", ["poetry upload"], CheckResultType.FAILED), ("pip", ["twine upload dist/*"], CheckResultType.PASSED), - ("pip", ["flit publish"], CheckResultType.PASSED), + ("pip", ["flit publish"], CheckResultType.FAILED), + ("flit", ["flit publish"], CheckResultType.PASSED), + ("hatch", ["hatch publish"], CheckResultType.PASSED), + ("conda", ["conda publish"], CheckResultType.PASSED), ], ) def test_deploy_commands( diff --git a/tests/slsa_analyzer/checks/test_detect_malicious_metadata_check.py b/tests/slsa_analyzer/checks/test_detect_malicious_metadata_check.py index 07c4684de..783813129 100644 --- a/tests/slsa_analyzer/checks/test_detect_malicious_metadata_check.py +++ b/tests/slsa_analyzer/checks/test_detect_malicious_metadata_check.py @@ -60,7 +60,7 @@ def test_detect_malicious_metadata( # Set up the context object with PyPIRegistry instance. ctx = MockAnalyzeContext(macaron_path=macaron_path, output_dir="", purl=purl) pypi_registry = PyPIRegistry() - ctx.dynamic_data["package_registries"] = [PackageRegistryInfo("pip", "pypi", pypi_registry)] + ctx.dynamic_data["package_registries"] = [PackageRegistryInfo("pypi", pypi_registry)] ctx.dynamic_data["force_analyze_source"] = force_analyze_source mock_global_config.resources_path = os.path.join(MACARON_PATH, "resources") diff --git a/tests/slsa_analyzer/checks/test_repo_verification_check.py b/tests/slsa_analyzer/checks/test_repo_verification_check.py index dcc15af43..1d2f01bb4 100644 --- a/tests/slsa_analyzer/checks/test_repo_verification_check.py +++ b/tests/slsa_analyzer/checks/test_repo_verification_check.py @@ -23,9 +23,7 @@ def test_repo_verification_pass(maven_tool: BaseBuildTool, macaron_path: Path) - ctx = MockAnalyzeContext(macaron_path=macaron_path, output_dir="", purl="pkg:maven/test/test") maven_registry = MavenCentralRegistry() - ctx.dynamic_data["package_registries"] = [ - PackageRegistryInfo(maven_tool.name, maven_tool.purl_type, maven_registry) - ] + ctx.dynamic_data["package_registries"] = [PackageRegistryInfo(maven_tool.purl_type, maven_registry)] ctx.dynamic_data["repo_verification"] = [ RepositoryVerificationResult( status=RepositoryVerificationStatus.PASSED, @@ -43,9 +41,7 @@ def test_repo_verification_fail(maven_tool: BaseBuildTool, macaron_path: Path) - ctx = MockAnalyzeContext(macaron_path=macaron_path, output_dir="", purl="pkg:maven/test/test") maven_registry = MavenCentralRegistry() - ctx.dynamic_data["package_registries"] = [ - PackageRegistryInfo(maven_tool.name, maven_tool.purl_type, maven_registry) - ] + ctx.dynamic_data["package_registries"] = [PackageRegistryInfo(maven_tool.purl_type, maven_registry)] ctx.dynamic_data["repo_verification"] = [ RepositoryVerificationResult( status=RepositoryVerificationStatus.FAILED, @@ -63,9 +59,7 @@ def test_check_unknown_for_unknown_repo_verification(maven_tool: BaseBuildTool, ctx = MockAnalyzeContext(macaron_path=macaron_path, output_dir="", purl="pkg:maven/test/test") maven_registry = MavenCentralRegistry() - ctx.dynamic_data["package_registries"] = [ - PackageRegistryInfo(maven_tool.name, maven_tool.purl_type, maven_registry) - ] + ctx.dynamic_data["package_registries"] = [PackageRegistryInfo(maven_tool.purl_type, maven_registry)] ctx.dynamic_data["repo_verification"] = [ RepositoryVerificationResult( status=RepositoryVerificationStatus.UNKNOWN, @@ -83,6 +77,6 @@ def test_check_unknown_for_unsupported_build_tools(pip_tool: BaseBuildTool, maca ctx = MockAnalyzeContext(macaron_path=macaron_path, output_dir="", purl="pkg:pypi/test/test") pypi_registry = PyPIRegistry() - ctx.dynamic_data["package_registries"] = [PackageRegistryInfo(pip_tool.name, pip_tool.purl_type, pypi_registry)] + ctx.dynamic_data["package_registries"] = [PackageRegistryInfo(pip_tool.purl_type, pypi_registry)] assert check.run_check(ctx).result_type == CheckResultType.UNKNOWN diff --git a/tests/slsa_analyzer/package_registry/test_jfrog_maven_registry.py b/tests/slsa_analyzer/package_registry/test_jfrog_maven_registry.py index ef7276dcf..de9609dfe 100644 --- a/tests/slsa_analyzer/package_registry/test_jfrog_maven_registry.py +++ b/tests/slsa_analyzer/package_registry/test_jfrog_maven_registry.py @@ -10,11 +10,6 @@ from macaron.config.defaults import load_defaults from macaron.errors import ConfigurationError -from macaron.slsa_analyzer.build_tool.base_build_tool import BaseBuildTool -from macaron.slsa_analyzer.build_tool.gradle import Gradle -from macaron.slsa_analyzer.build_tool.maven import Maven -from macaron.slsa_analyzer.build_tool.pip import Pip -from macaron.slsa_analyzer.build_tool.poetry import Poetry from macaron.slsa_analyzer.package_registry.jfrog_maven_registry import JFrogMavenAssetMetadata, JFrogMavenRegistry @@ -115,26 +110,25 @@ def test_load_defaults_with_invalid_config(tmp_path: Path, user_config_input: st @pytest.mark.parametrize( - ("build_tool", "expected_result"), + ("ecosystem", "expected_result"), [ - (Maven(), True), - (Gradle(), True), - (Pip(), False), - (Poetry(), False), + ("maven", True), + ("pypi", False), + ("npm", False), ], ) def test_is_detected( jfrog_maven: JFrogMavenRegistry, - build_tool: BaseBuildTool, + ecosystem: str, expected_result: bool, ) -> None: """Test the ``is_detected`` method.""" - assert jfrog_maven.is_detected(build_tool.name) == expected_result + assert jfrog_maven.is_detected(ecosystem) == expected_result # The method always returns False when the jfrog_maven instance is not enabled # (in the ini config). jfrog_maven.enabled = False - assert jfrog_maven.is_detected(build_tool.name) is False + assert jfrog_maven.is_detected(ecosystem) is False @pytest.mark.parametrize( diff --git a/tests/slsa_analyzer/package_registry/test_maven_central_registry.py b/tests/slsa_analyzer/package_registry/test_maven_central_registry.py index 128b57df5..c304074f0 100644 --- a/tests/slsa_analyzer/package_registry/test_maven_central_registry.py +++ b/tests/slsa_analyzer/package_registry/test_maven_central_registry.py @@ -138,21 +138,20 @@ def test_load_defaults_with_invalid_config(tmp_path: Path, user_config_input: st @pytest.mark.parametrize( - ("build_tool_name", "expected_result"), + ("ecosystem", "expected_result"), [ ("maven", True), - ("gradle", True), - ("pip", False), - ("poetry", False), + ("pypi", False), + ("npm", False), ], ) def test_is_detected( maven_central: MavenCentralRegistry, - build_tool_name: str, + ecosystem: str, expected_result: bool, ) -> None: """Test the ``is_detected`` method.""" - assert maven_central.is_detected(build_tool_name) == expected_result + assert maven_central.is_detected(ecosystem) == expected_result @pytest.mark.parametrize( diff --git a/tests/slsa_analyzer/package_registry/test_npm_registry.py b/tests/slsa_analyzer/package_registry/test_npm_registry.py index a180ea78b..53ec3a56b 100644 --- a/tests/slsa_analyzer/package_registry/test_npm_registry.py +++ b/tests/slsa_analyzer/package_registry/test_npm_registry.py @@ -13,7 +13,6 @@ from macaron.config.defaults import load_defaults from macaron.errors import ConfigurationError, InvalidHTTPResponseError -from macaron.slsa_analyzer.build_tool.npm import NPM from macaron.slsa_analyzer.package_registry.npm_registry import NPMAttestationAsset, NPMRegistry @@ -31,7 +30,7 @@ def create_npm_registry() -> NPMRegistry: ) -def test_disable_npm_registry(npm_registry: NPMRegistry, tmp_path: Path, npm_tool: NPM) -> None: +def test_disable_npm_registry(npm_registry: NPMRegistry, tmp_path: Path) -> None: """Test disabling npm registry.""" config = """ [package_registry.npm] @@ -44,7 +43,7 @@ def test_disable_npm_registry(npm_registry: NPMRegistry, tmp_path: Path, npm_too npm_registry.load_defaults() assert npm_registry.enabled is False - assert npm_registry.is_detected(npm_tool.name) is False + assert npm_registry.is_detected("npm") is False @pytest.mark.parametrize( @@ -75,21 +74,17 @@ def test_npm_registry_invalid_config(npm_registry: NPMRegistry, tmp_path: Path, @pytest.mark.parametrize( - ( - "build_tool_name", - "expected", - ), + ("ecosystem", "expected_result"), [ - ("npm", True), - ("yarn", True), - ("go", False), ("maven", False), + ("pypi", False), + ("npm", True), ], ) -def test_is_detected(npm_registry: NPMRegistry, build_tool_name: str, expected: bool) -> None: - """Test that the registry is correctly detected for a build tool.""" +def test_is_detected(npm_registry: NPMRegistry, ecosystem: str, expected_result: bool) -> None: + """Test that the registry is correctly detected for the ecosystem.""" npm_registry.load_defaults() - assert npm_registry.is_detected(build_tool_name) == expected + assert npm_registry.is_detected(ecosystem) == expected_result @pytest.mark.parametrize(