diff --git a/src/macaron/build_spec_generator/common_spec/pypi_spec.py b/src/macaron/build_spec_generator/common_spec/pypi_spec.py index 0f7d78824..bb90ba6a1 100644 --- a/src/macaron/build_spec_generator/common_spec/pypi_spec.py +++ b/src/macaron/build_spec_generator/common_spec/pypi_spec.py @@ -6,6 +6,7 @@ import logging import os import re +from typing import Any import tomli from packageurl import PackageURL @@ -67,15 +68,17 @@ def get_default_build_commands( match build_tool_name: case "pip": - default_build_commands.append("python -m build".split()) + default_build_commands.append("python -m build --wheel -n".split()) case "poetry": default_build_commands.append("poetry build".split()) case "flit": + # We might also want to deal with existence flit.ini, we can do so via + # "python -m flit.tomlify" default_build_commands.append("flit build".split()) case "hatch": default_build_commands.append("hatch build".split()) case "conda": - default_build_commands.append("conda build".split()) + default_build_commands.append('echo("Not supported")'.split()) case _: pass @@ -156,6 +159,7 @@ def resolve_fields(self, purl: PackageURL) -> None: try: with pypi_package_json.sourcecode(): try: + # Get the build time requirements from ["build-system", "requires"] pyproject_content = pypi_package_json.get_sourcecode_file_contents("pyproject.toml") content = tomli.loads(pyproject_content.decode("utf-8")) requires = json_extract(content, ["build-system", "requires"], list) @@ -164,10 +168,10 @@ def resolve_fields(self, purl: PackageURL) -> None: backend = json_extract(content, ["build-system", "build-backend"], str) if backend: build_backends_set.add(backend.replace(" ", "")) - python_version_constraint = json_extract(content, ["project", "requires-python"], str) if python_version_constraint: python_version_set.add(python_version_constraint.replace(" ", "")) + self.apply_tool_specific_inferences(build_requires_set, python_version_set, content) logger.debug( "After analyzing pyproject.toml from the sdist: build-requires: %s, build_backend: %s", build_requires_set, @@ -239,6 +243,40 @@ def resolve_fields(self, purl: PackageURL) -> None: self.data["build_commands"] = patched_build_commands + def apply_tool_specific_inferences( + self, build_requires_set: set[str], python_version_set: set[str], pyproject_contents: dict[str, Any] + ) -> None: + """ + Based on build tools inferred, look into the pyproject.toml for related additional dependencies. + + Parameters + ---------- + build_requires_set: set[str] + Set of build requirements to populate. + python_version_set: set[str] + Set of compatible interpreter versions to populate. + pyproject_contents: dict[str, Any] + Parsed contents of the pyproject.toml file. + """ + # If we have hatch as a build_tool, we will examine [tool.hatch.build.hooks.*] to + # look for any additional build dependencies declared there. + if "hatch" in self.data["build_tools"]: + # Look for [tool.hatch.build.hooks.*] + hatch_build_hooks = json_extract(pyproject_contents, ["tool", "hatch", "build", "hooks"], dict) + if hatch_build_hooks: + for _, section in hatch_build_hooks.items(): + dependencies = section.get("dependencies") + if dependencies: + build_requires_set.update(elem.replace(" ", "") for elem in dependencies) + # If we have flit as a build_tool, we will check if the legacy header [tool.flit.metadata] exists, + # and if so, check to see if we can use its "requires-python". + if "flit" in self.data["build_tools"]: + flit_python_version_constraint = json_extract( + pyproject_contents, ["tool", "flit", "metadata", "requires-python"], str + ) + if flit_python_version_constraint: + python_version_set.add(flit_python_version_constraint.replace(" ", "")) + def read_directory(self, wheel_path: str, purl: PackageURL) -> tuple[str, str]: """ Read in the WHEEL and METADATA file from the .dist_info directory. diff --git a/src/macaron/build_spec_generator/dockerfile/pypi_dockerfile_output.py b/src/macaron/build_spec_generator/dockerfile/pypi_dockerfile_output.py index fd41f063c..ef2360a5c 100644 --- a/src/macaron/build_spec_generator/dockerfile/pypi_dockerfile_output.py +++ b/src/macaron/build_spec_generator/dockerfile/pypi_dockerfile_output.py @@ -38,6 +38,17 @@ def gen_dockerfile(buildspec: BaseBuildSpecDict) -> str: logger.debug("Could not derive a specific interpreter version.") raise GenerateBuildSpecError("Could not derive specific interpreter version.") backend_install_commands: str = " && ".join(build_backend_commands(buildspec)) + build_tool_install: str = "" + if ( + buildspec["build_tools"][0] != "pip" + and buildspec["build_tools"][0] != "conda" + and buildspec["build_tools"][0] != "flit" + ): + build_tool_install = f"pip install {buildspec['build_tools'][0]} && " + elif buildspec["build_tools"][0] == "flit": + build_tool_install = ( + f"pip install {buildspec['build_tools'][0]} && if test -f \"flit.ini\"; then python -m flit.tomlify; fi && " + ) dockerfile_content = f""" #syntax=docker/dockerfile:1.10 FROM oraclelinux:9 @@ -87,7 +98,7 @@ def gen_dockerfile(buildspec: BaseBuildSpecDict) -> str: EOF # Run the build - RUN /deps/bin/python -m build --wheel -n + RUN {"source /deps/bin/activate && " + build_tool_install + " ".join(x for x in buildspec["build_commands"][0])} """ return dedent(dockerfile_content) @@ -148,4 +159,6 @@ def build_backend_commands(buildspec: BaseBuildSpecDict) -> list[str]: commands: list[str] = [] for backend, version_constraint in buildspec["build_requires"].items(): commands.append(f'/deps/bin/pip install "{backend}{version_constraint}"') + # For a stable order on the install commands + commands.sort() return commands diff --git a/tests/build_spec_generator/dockerfile/__snapshots__/test_pypi_dockerfile_output.ambr b/tests/build_spec_generator/dockerfile/__snapshots__/test_pypi_dockerfile_output.ambr index a39631a05..696ee6f8d 100644 --- a/tests/build_spec_generator/dockerfile/__snapshots__/test_pypi_dockerfile_output.ambr +++ b/tests/build_spec_generator/dockerfile/__snapshots__/test_pypi_dockerfile_output.ambr @@ -50,7 +50,7 @@ EOF # Run the build - RUN /deps/bin/python -m build --wheel -n + RUN source /deps/bin/activate && python -m build ''' # --- diff --git a/tests/build_spec_generator/dockerfile/compare_dockerfile_buildspec.py b/tests/build_spec_generator/dockerfile/compare_dockerfile_buildspec.py new file mode 100644 index 000000000..8c181d8d5 --- /dev/null +++ b/tests/build_spec_generator/dockerfile/compare_dockerfile_buildspec.py @@ -0,0 +1,106 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""Script to compare a generated dockerfile buildspec.""" + +import argparse +import logging +from collections.abc import Callable + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +logging.basicConfig(format="[%(filename)s:%(lineno)s %(tag)s] %(message)s") + + +def log_with_tag(tag: str) -> Callable[[str], None]: + """Generate a log function that prints the name of the file and a tag at the beginning of each line.""" + + def log_fn(msg: str) -> None: + logger.info(msg, extra={"tag": tag}) + + return log_fn + + +log_info = log_with_tag("INFO") +log_err = log_with_tag("ERROR") +log_passed = log_with_tag("PASSED") +log_failed = log_with_tag("FAILED") + + +def log_diff(result: str, expected: str) -> None: + """Pretty-print the diff of two strings.""" + output = [ + *("---- Result ---", result), + *("---- Expected ---", expected), + "-----------------", + ] + log_info("\n".join(output)) + + +def main() -> int: + """Compare a Macaron generated dockerfile buildspec. + + Returns + ------- + int + 0 if the generated dockerfile matches the expected output, or non-zero otherwise. + """ + parser = argparse.ArgumentParser() + parser.add_argument("result_dockerfile", help="the result dockerfile buildspec") + parser.add_argument("expected_dockerfile_buildspec", help="the expected buildspec dockerfile") + args = parser.parse_args() + + # Load both files + with open(args.result_dockerfile, encoding="utf-8") as file: + buildspec = normalize(file.read()) + + with open(args.expected_dockerfile_buildspec, encoding="utf-8") as file: + expected_buildspec = normalize(file.read()) + + log_info( + f"Comparing the dockerfile buildspec {args.result_dockerfile} with the expected " + + "output dockerfile {args.expected_dockerfile_buildspec}" + ) + + # Compare the files + return compare(buildspec, expected_buildspec) + + +def normalize(contents: str) -> list[str]: + """Convert string of file contents to list of its non-empty lines""" + return [line.strip() for line in contents.splitlines() if line.strip()] + + +def compare(buildspec: list[str], expected_buildspec: list[str]) -> int: + """Compare the lines in the two files directly. + + Early return when an unexpected difference is found. If the lengths + mismatch, but the first safe_index_max lines are the same, print + the missing/extra lines. + + Returns + ------- + int + 0 if the generated dockerfile matches the expected output, or non-zero otherwise. + """ + safe_index_max = min(len(buildspec), len(expected_buildspec)) + for index in range(safe_index_max): + if buildspec[index] != expected_buildspec[index]: + # Log error + log_err("Mismatch found:") + # Log diff + log_diff(buildspec[index], expected_buildspec[index]) + return 1 + if safe_index_max < len(expected_buildspec): + log_err("Mismatch found: result is missing trailing lines") + log_diff("", "\n".join(expected_buildspec[safe_index_max:])) + return 1 + if safe_index_max < len(buildspec): + log_err("Mismatch found: result has extra trailing lines") + log_diff("\n".join(buildspec[safe_index_max:]), "") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/integration/cases/pypi_cachetools/expected_default.buildspec b/tests/integration/cases/pypi_cachetools/expected_default.buildspec index 5af209e96..0b5d8acfa 100644 --- a/tests/integration/cases/pypi_cachetools/expected_default.buildspec +++ b/tests/integration/cases/pypi_cachetools/expected_default.buildspec @@ -19,7 +19,9 @@ [ "python", "-m", - "build" + "build", + "--wheel", + "-n" ] ], "build_requires": { diff --git a/tests/integration/cases/pypi_cachetools/expected_dockerfile.buildspec b/tests/integration/cases/pypi_cachetools/expected_dockerfile.buildspec new file mode 100644 index 000000000..749757f91 --- /dev/null +++ b/tests/integration/cases/pypi_cachetools/expected_dockerfile.buildspec @@ -0,0 +1,50 @@ + +#syntax=docker/dockerfile:1.10 +FROM oraclelinux:9 + +# Install core tools +RUN dnf -y install which wget tar git + +# Install compiler and make +RUN dnf -y install gcc make + +# Download and unzip interpreter +RUN <=3.4" + /deps/bin/pip install build +EOF + +# Run the build +RUN source /deps/bin/activate && pip install flit && if test -f "flit.ini"; then python -m flit.tomlify; fi && flit build diff --git a/tests/integration/cases/pypi_markdown-it-py/test.yaml b/tests/integration/cases/pypi_markdown-it-py/test.yaml index a57b7d2cf..d3b0b365a 100644 --- a/tests/integration/cases/pypi_markdown-it-py/test.yaml +++ b/tests/integration/cases/pypi_markdown-it-py/test.yaml @@ -27,3 +27,17 @@ steps: kind: default_build_spec result: output/buildspec/pypi/markdown-it-py/macaron.buildspec expected: expected_default.buildspec +- name: Generate the buildspec + kind: gen-build-spec + options: + command_args: + - -purl + - pkg:pypi/markdown-it-py@4.0.0 + - --output-format + - dockerfile +- name: Compare Dockerfile + kind: compare + options: + kind: dockerfile_build_spec + result: output/buildspec/pypi/markdown-it-py/dockerfile.buildspec + expected: expected_dockerfile.buildspec diff --git a/tests/integration/cases/pypi_toga/expected_default.buildspec b/tests/integration/cases/pypi_toga/expected_default.buildspec index ffb146e81..819113207 100644 --- a/tests/integration/cases/pypi_toga/expected_default.buildspec +++ b/tests/integration/cases/pypi_toga/expected_default.buildspec @@ -19,7 +19,9 @@ [ "python", "-m", - "build" + "build", + "--wheel", + "-n" ] ], "build_requires": { diff --git a/tests/integration/cases/pypi_toga/expected_dockerfile.buildspec b/tests/integration/cases/pypi_toga/expected_dockerfile.buildspec new file mode 100644 index 000000000..47e1e012a --- /dev/null +++ b/tests/integration/cases/pypi_toga/expected_dockerfile.buildspec @@ -0,0 +1,50 @@ + +#syntax=docker/dockerfile:1.10 +FROM oraclelinux:9 + +# Install core tools +RUN dnf -y install which wget tar git + +# Install compiler and make +RUN dnf -y install gcc make + +# Download and unzip interpreter +RUN < None: "find_source": ["tests", "find_source", "compare_source_reports.py"], "rc_build_spec": ["tests", "build_spec_generator", "reproducible_central", "compare_rc_build_spec.py"], "default_build_spec": ["tests", "build_spec_generator", "common_spec", "compare_default_buildspec.py"], + "dockerfile_build_spec": ["tests", "build_spec_generator", "dockerfile", "compare_dockerfile_buildspec.py"], } VALIDATE_SCHEMA_SCRIPTS: dict[str, Sequence[str]] = {