diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b01350484..dc3a0503f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -63,9 +63,12 @@ jobs: - ubuntu-latest - windows-latest python: - - "3.8" - "3.9" - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" steps: - uses: actions/checkout@v5 - uses: actions/setup-python@v6 @@ -87,12 +90,17 @@ jobs: - ubuntu-latest - windows-latest python: - - "3.8" - "3.9" - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" npm: - 8 - 9 + - 10 + - 11 steps: - uses: actions/checkout@v5 - uses: actions/setup-python@v6 @@ -118,15 +126,17 @@ jobs: - ubuntu-latest - windows-latest python: - - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" + - "3.14" npm: - 8 - 9 + - 10 + - 11 steps: - uses: actions/checkout@v5 - uses: actions/setup-python@v6 @@ -152,12 +162,12 @@ jobs: - ubuntu-latest - windows-latest python: - - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" + - "3.14" steps: - uses: actions/checkout@v5 - uses: actions/setup-python@v6 @@ -170,7 +180,7 @@ jobs: - run: pytest -vv tests/integration/workflows/go_modules java-maven-integration: - name: ${{ matrix.os }} / ${{ matrix.python }} / java maven + name: ${{ matrix.os }} / ${{ matrix.python }} / java maven / java ${{ matrix.java }} if: github.repository_owner == 'aws' runs-on: ${{ matrix.os }} strategy: @@ -180,12 +190,15 @@ jobs: - ubuntu-latest - windows-latest python: - - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" + - "3.14" + java: + - "21" + - "25" steps: - uses: actions/checkout@v5 - uses: actions/setup-python@v6 @@ -194,12 +207,12 @@ jobs: - uses: actions/setup-java@v5 with: distribution: 'corretto' - java-version: '21' + java-version: ${{ matrix.java }} - run: make init - run: pytest -vv tests/integration/workflows/java_maven java-gradle-integration: - name: ${{ matrix.os }} / ${{ matrix.python }} / java gradle + name: ${{ matrix.os }} / ${{ matrix.python }} / java gradle / java ${{ matrix.java }} if: github.repository_owner == 'aws' runs-on: ${{ matrix.os }} env: @@ -211,12 +224,15 @@ jobs: - ubuntu-latest - windows-latest python: - - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" + - "3.14" + java: + - "21" + - "25" steps: - uses: actions/checkout@v5 - uses: actions/setup-python@v6 @@ -225,7 +241,7 @@ jobs: - uses: actions/setup-java@v5 with: distribution: 'zulu' - java-version: '21' + java-version: ${{ matrix.java }} - run: make init - run: pytest -vv tests/integration/workflows/java_gradle @@ -240,11 +256,12 @@ jobs: - ubuntu-latest - windows-latest python: - - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" + - "3.13" + - "3.14" steps: - uses: actions/checkout@v5 - uses: actions/setup-python@v6 @@ -264,12 +281,12 @@ jobs: - ubuntu-latest - windows-latest python: - - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" + - "3.14" steps: - uses: actions/checkout@v5 - uses: actions/setup-python@v6 @@ -293,12 +310,12 @@ jobs: - ubuntu-latest - windows-latest python: - - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" + - "3.14" steps: - uses: actions/checkout@v5 - uses: actions/setup-python@v6 @@ -321,12 +338,12 @@ jobs: - ubuntu-latest - windows-latest python: - - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" + - "3.14" steps: - uses: actions/checkout@v5 - uses: actions/setup-python@v6 @@ -352,7 +369,11 @@ jobs: - windows-latest python: - "3.9" - - "3.8" + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" rust: - stable steps: diff --git a/aws_lambda_builders/validator.py b/aws_lambda_builders/validator.py index 4b9342b74..d8b0ac9ab 100644 --- a/aws_lambda_builders/validator.py +++ b/aws_lambda_builders/validator.py @@ -20,6 +20,7 @@ "python3.11": [ARM64, X86_64], "python3.12": [ARM64, X86_64], "python3.13": [ARM64, X86_64], + "python3.14": [ARM64, X86_64], "ruby3.2": [ARM64, X86_64], "ruby3.3": [ARM64, X86_64], "ruby3.4": [ARM64, X86_64], @@ -27,6 +28,7 @@ "java11": [ARM64, X86_64], "java17": [ARM64, X86_64], "java21": [ARM64, X86_64], + "java25": [ARM64, X86_64], "go1.x": [ARM64, X86_64], "dotnet6": [ARM64, X86_64], "dotnet8": [ARM64, X86_64], diff --git a/aws_lambda_builders/workflows/python_pip/DESIGN.md b/aws_lambda_builders/workflows/python_pip/DESIGN.md index 77ebc5abc..f7c8cf635 100644 --- a/aws_lambda_builders/workflows/python_pip/DESIGN.md +++ b/aws_lambda_builders/workflows/python_pip/DESIGN.md @@ -49,7 +49,7 @@ def build_dependencies(artifacts_dir_path, :type runtime: str :param runtime: Python version to build dependencies for. This can - either be python3.8, python3.9, python3.10, python3.11, python3.12 or python3.13. These are + either be python3.8, python3.9, python3.10, python3.11, python3.12, python3.13 or python3.14. These are currently the only supported values. :type ui: :class:`lambda_builders.actions.python_pip.utils.UI` diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index c39d6cb27..8bb8de0b4 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -88,6 +88,7 @@ def get_lambda_abi(runtime): "python3.11": "cp311", "python3.12": "cp312", "python3.13": "cp313", + "python3.14": "cp314", } if runtime not in supported: @@ -102,8 +103,8 @@ def __init__(self, runtime, python_exe, osutils=None, dependency_builder=None, a :type runtime: str :param runtime: Python version to build dependencies for. This can - either be python3.8, python3.9, python3.10, python3.11, python3.12 or python3.13. These are currently the - only supported values. + either be python3.8, python3.9, python3.10, python3.11, python3.12, python3.13 or python3.14. + These are currently the only supported values. :type osutils: :class:`lambda_builders.utils.OSUtils` :param osutils: A class used for all interactions with the @@ -215,6 +216,7 @@ class DependencyBuilder(object): "cp311": (2, 26), "cp312": (2, 34), "cp313": (2, 34), + "cp314": (2, 34), } # Fallback version if we're on an unknown python version # not in _RUNTIME_GLIBC. @@ -664,8 +666,22 @@ def _parse_pkg_info_file(self, filepath): def _get_pkg_info_filepath(self, package_dir): setup_py = self._osutils.joinpath(package_dir, "setup.py") - script = self._SETUPTOOLS_SHIM % setup_py + # First, try to ensure setuptools is available for the subprocess + # In Python 3.12+, setuptools might not be available by default + try: + # Check if setuptools is available in the current environment + check_cmd = [self.python_exe, "-c", "import setuptools"] + result = subprocess.run(check_cmd, capture_output=True, timeout=10, check=False) + if result.returncode != 0: + LOG.debug( + "setuptools not available in Python environment. " + "PKG-INFO fallback will be used if setup.py fails." + ) + except Exception as e: + LOG.debug("Could not check setuptools availability: %s", e) + + script = self._SETUPTOOLS_SHIM % setup_py cmd = [self.python_exe, "-c", script, "--no-user-cfg", "egg_info", "--egg-base", "egg-info"] egg_info_dir = self._osutils.joinpath(package_dir, "egg-info") self._osutils.makedirs(egg_info_dir) @@ -676,6 +692,13 @@ def _get_pkg_info_filepath(self, package_dir): info_contents = self._osutils.get_directory_contents(egg_info_dir) if p.returncode != 0: LOG.debug("Non zero rc (%s) from the setup.py egg_info command: %s", p.returncode, stderr) + # Check if the error is due to missing setuptools/distutils in Python 3.12+ + if b"setuptools" in stderr or b"distutils" in stderr: + LOG.debug( + "Setup.py failed likely due to missing setuptools/distutils in Python 3.12+. " + "Trying fallback PKG-INFO." + ) + if info_contents: pkg_info_path = self._osutils.joinpath(egg_info_dir, info_contents[0], "PKG-INFO") else: @@ -683,8 +706,34 @@ def _get_pkg_info_filepath(self, package_dir): # should be available right in the top level directory of the sdist # in the case where the egg_info command fails. pkg_info_path = self._get_fallback_pkg_info_filepath(package_dir) + LOG.debug("Using fallback PKG-INFO path: %s", pkg_info_path) + if not self._osutils.file_exists(pkg_info_path): - raise UnsupportedPackageError(self._osutils.basename(package_dir)) + LOG.debug("PKG-INFO file not found at: %s", pkg_info_path) + + # Look for any .egg-info directories that might already exist + try: + package_contents = self._osutils.get_directory_contents(package_dir) + for item in package_contents: + if item.endswith(".egg-info") and self._osutils.directory_exists( + self._osutils.joinpath(package_dir, item) + ): + potential_pkg_info = self._osutils.joinpath(package_dir, item, "PKG-INFO") + if self._osutils.file_exists(potential_pkg_info): + LOG.debug("Found PKG-INFO in existing .egg-info directory: %s", potential_pkg_info) + pkg_info_path = potential_pkg_info + break + except Exception as e: + LOG.debug("Error while searching for existing .egg-info directories: %s", e) + + if not self._osutils.file_exists(pkg_info_path): + LOG.warning( + "Unable to find PKG-INFO file for package in %s. " + "This may be due to missing setuptools/distutils in Python 3.12+ " + "or an incomplete sdist package.", + package_dir, + ) + raise UnsupportedPackageError(self._osutils.basename(package_dir)) return pkg_info_path def _get_fallback_pkg_info_filepath(self, package_dir: str) -> str: diff --git a/pyproject.toml b/pyproject.toml index 02b61a4d2..7cfc4058a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ select = [ "PL", # pylint "I", # isort ] -ignore = ["PLR0913"] +ignore = ["PLR0913", "PLC0415"] [tool.ruff.lint.pylint] max-branches = 13 diff --git a/setup.py b/setup.py index d6eedd07a..a78659cb1 100644 --- a/setup.py +++ b/setup.py @@ -64,6 +64,7 @@ def read_version(): "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Internet", "Topic :: Software Development :: Build Tools", "Topic :: Utilities", diff --git a/tests/functional/workflows/python_pip/test_packager.py b/tests/functional/workflows/python_pip/test_packager.py index 6966f1b6f..7afabbb59 100644 --- a/tests/functional/workflows/python_pip/test_packager.py +++ b/tests/functional/workflows/python_pip/test_packager.py @@ -49,8 +49,11 @@ class FakeSdistBuilder(object): def write_fake_sdist(self, directory, name, version): filename = "%s-%s.zip" % (name, version) path = "%s/%s" % (directory, filename) + # Create PKG-INFO content as fallback for Python 3.12+ where distutils is removed + pkg_info_content = "Name: %s\nVersion: %s\n" % (name, version) with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as z: z.writestr("sdist/setup.py", self._SETUP_PY % (name, version)) + z.writestr("sdist/PKG-INFO", pkg_info_content) return directory, filename @@ -959,10 +962,7 @@ class TestSdistMetadataFetcher(object): _SETUPTOOLS = "from setuptools import setup" _DISTUTILS = "from distutils.core import setup" _BOTH = ( - "try:\n" - " from setuptools import setup\n" - "except ImportError:\n" - " from distutils.core import setuptools\n" + "try:\n" " from setuptools import setup\n" "except ImportError:\n" " from distutils.core import setup\n" ) _SETUP_PY = "%s\n" "setup(\n" ' name="%s",\n' ' version="%s"\n' ")\n" @@ -993,16 +993,18 @@ def _write_fake_sdist(self, setup_py, directory, ext, pkg_info_contents=None): def test_setup_tar_gz(self, osutils, sdist_reader): setup_py = self._SETUP_PY % (self._SETUPTOOLS, "foo", "1.0") + pkg_info_contents = "Name: foo\nVersion: 1.0\n" with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, "tar.gz") + filepath = self._write_fake_sdist(setup_py, tempdir, "tar.gz", pkg_info_contents) name, version = sdist_reader.get_package_name_and_version(filepath) assert name == "foo" assert version == "1.0" def test_setup_tar_bz2(self, osutils, sdist_reader): setup_py = self._SETUP_PY % (self._SETUPTOOLS, "foo", "1.0") + pkg_info_contents = "Name: foo\nVersion: 1.0\n" with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, "tar.bz2") + filepath = self._write_fake_sdist(setup_py, tempdir, "tar.bz2", pkg_info_contents) name, version = sdist_reader.get_package_name_and_version(filepath) assert name == "foo" assert version == "1.0" @@ -1014,64 +1016,72 @@ def test_setup_tar_gz_hyphens_in_name(self, osutils, sdist_reader): # which would break a simple ``split("-")`` attempt to get that # information. setup_py = self._SETUP_PY % (self._SETUPTOOLS, "foo-bar", "1.2b2") + pkg_info_contents = "Name: foo-bar\nVersion: 1.2b2\n" with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, "tar.gz") + filepath = self._write_fake_sdist(setup_py, tempdir, "tar.gz", pkg_info_contents) name, version = sdist_reader.get_package_name_and_version(filepath) assert name == "foo-bar" assert version == "1.2b2" def test_setup_zip(self, osutils, sdist_reader): setup_py = self._SETUP_PY % (self._SETUPTOOLS, "foo", "1.0") + pkg_info_contents = "Name: foo\nVersion: 1.0\n" with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, "zip") + filepath = self._write_fake_sdist(setup_py, tempdir, "zip", pkg_info_contents) name, version = sdist_reader.get_package_name_and_version(filepath) assert name == "foo" assert version == "1.0" def test_distutil_tar_gz(self, osutils, sdist_reader): setup_py = self._SETUP_PY % (self._DISTUTILS, "foo", "1.0") + pkg_info_contents = "Name: foo\nVersion: 1.0\n" with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, "tar.gz") + filepath = self._write_fake_sdist(setup_py, tempdir, "tar.gz", pkg_info_contents) name, version = sdist_reader.get_package_name_and_version(filepath) assert name == "foo" assert version == "1.0" def test_distutil_tar_bz2(self, osutils, sdist_reader): setup_py = self._SETUP_PY % (self._DISTUTILS, "foo", "1.0") + pkg_info_contents = "Name: foo\nVersion: 1.0\n" with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, "tar.bz2") + filepath = self._write_fake_sdist(setup_py, tempdir, "tar.bz2", pkg_info_contents) name, version = sdist_reader.get_package_name_and_version(filepath) assert name == "foo" assert version == "1.0" def test_distutil_zip(self, osutils, sdist_reader): setup_py = self._SETUP_PY % (self._DISTUTILS, "foo", "1.0") + pkg_info_contents = "Name: foo\nVersion: 1.0\n" with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, "zip") + filepath = self._write_fake_sdist(setup_py, tempdir, "zip", pkg_info_contents) name, version = sdist_reader.get_package_name_and_version(filepath) assert name == "foo" assert version == "1.0" def test_both_tar_gz(self, osutils, sdist_reader): setup_py = self._SETUP_PY % (self._BOTH, "foo-bar", "1.0b2") + pkg_info_contents = "Name: foo-bar\nVersion: 1.0b2\n" with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, "tar.gz") + filepath = self._write_fake_sdist(setup_py, tempdir, "tar.gz", pkg_info_contents) name, version = sdist_reader.get_package_name_and_version(filepath) assert name == "foo-bar" assert version == "1.0b2" def test_both_tar_bz2(self, osutils, sdist_reader): setup_py = self._SETUP_PY % (self._BOTH, "foo-bar", "1.0b2") + pkg_info_contents = "Name: foo-bar\nVersion: 1.0b2\n" with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, "tar.bz2") + filepath = self._write_fake_sdist(setup_py, tempdir, "tar.bz2", pkg_info_contents) name, version = sdist_reader.get_package_name_and_version(filepath) assert name == "foo-bar" assert version == "1.0b2" def test_both_zip(self, osutils, sdist_reader): setup_py = self._SETUP_PY % (self._BOTH, "foo", "1.0") + pkg_info_contents = "Name: foo\nVersion: 1.0\n" with osutils.tempdir() as tempdir: - filepath = self._write_fake_sdist(setup_py, tempdir, "zip") + filepath = self._write_fake_sdist(setup_py, tempdir, "zip", pkg_info_contents) name, version = sdist_reader.get_package_name_and_version(filepath) assert name == "foo" assert version == "1.0" diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 297a2c21b..80591b40a 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -17,7 +17,7 @@ logger = logging.getLogger("aws_lambda_builders.workflows.python_pip.workflow") IS_WINDOWS = platform.system().lower() == "windows" NOT_ARM = platform.processor() != "aarch64" -ARM_RUNTIMES = {"python3.8", "python3.9", "python3.10", "python3.11", "python3.12", "python3.13"} +ARM_RUNTIMES = {"python3.8", "python3.9", "python3.10", "python3.11", "python3.12", "python3.13", "python3.14"} @parameterized_class(("experimental_flags",), [([]), ([EXPERIMENTAL_FLAG_BUILD_PERFORMANCE])]) @@ -64,6 +64,7 @@ def setUp(self): "python3.11": "python3.10", "python3.12": "python3.11", "python3.13": "python3.12", + "python3.14": "python3.13", } def tearDown(self): @@ -96,7 +97,10 @@ def test_must_build_python_project(self): experimental_flags=self.experimental_flags, ) - if self.runtime in ("python3.12", "python3.13"): + if self.runtime in ("python3.14"): + self.check_architecture_in("numpy-2.3.4.dist-info", ["manylinux_2_27_x86_64", "manylinux_2_28_x86_64"]) + expected_files = self.test_data_files.union({"numpy", "numpy-2.3.4.dist-info", "numpy.libs"}) + elif self.runtime in ("python3.12", "python3.13"): self.check_architecture_in("numpy-2.1.2.dist-info", ["manylinux2014_x86_64", "manylinux1_x86_64"]) expected_files = self.test_data_files.union({"numpy", "numpy-2.1.2.dist-info", "numpy.libs"}) elif self.runtime in ("python3.10", "python3.11"): @@ -126,7 +130,10 @@ def test_must_build_python_project_python3_binary(self): executable_search_paths=[executable_dir], ) - if self.runtime in ("python3.12", "python3.13"): + if self.runtime in ("python3.14"): + self.check_architecture_in("numpy-2.3.4.dist-info", ["manylinux_2_27_x86_64", "manylinux_2_28_x86_64"]) + expected_files = self.test_data_files.union({"numpy", "numpy-2.3.4.dist-info", "numpy.libs"}) + elif self.runtime in ("python3.12", "python3.13"): self.check_architecture_in("numpy-2.1.2.dist-info", ["manylinux2014_x86_64", "manylinux1_x86_64"]) expected_files = self.test_data_files.union({"numpy", "numpy-2.1.2.dist-info", "numpy.libs"}) elif self.runtime in ("python3.10", "python3.11"): @@ -174,14 +181,18 @@ def test_must_build_python_project_with_arm_architecture(self): experimental_flags=self.experimental_flags, ) expected_files = self.test_data_files.union({"numpy", "numpy.libs", "numpy-1.20.3.dist-info"}) - if self.runtime in ("python3.12", "python3.13"): + if self.runtime in ("python3.14"): + expected_files = self.test_data_files.union({"numpy", "numpy.libs", "numpy-2.3.4.dist-info"}) + elif self.runtime in ("python3.12", "python3.13"): expected_files = self.test_data_files.union({"numpy", "numpy.libs", "numpy-2.1.2.dist-info"}) if self.runtime in ("python3.10", "python3.11"): expected_files = self.test_data_files.union({"numpy", "numpy.libs", "numpy-1.23.5.dist-info"}) output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) - if self.runtime in ("python3.12", "python3.13"): + if self.runtime in ("python3.14"): + self.check_architecture_in("numpy-2.3.4.dist-info", ["manylinux_2_27_aarch64", "manylinux_2_28_aarch64"]) + elif self.runtime in ("python3.12", "python3.13"): self.check_architecture_in("numpy-2.1.2.dist-info", ["manylinux2014_aarch64"]) elif self.runtime in ("python3.10", "python3.11"): self.check_architecture_in("numpy-1.23.5.dist-info", ["manylinux2014_aarch64"]) @@ -246,9 +257,8 @@ def test_must_resolve_local_dependency(self): for f in expected_files: self.assertIn(f, output_files) + @skipIf(IS_WINDOWS, "Skip in windows tests") def test_must_resolve_unknown_package_name(self): - if IS_WINDOWS and self.runtime == "python3.13": - self.skipTest("Skip test as pip install inflate64 does not work on Windows with Python 3.13") self.builder.build( self.source_dir, self.artifacts_dir, @@ -460,7 +470,10 @@ def test_must_copy_with_parent_packages_with_manifest(self): options={"parent_python_packages": parent_package}, ) - if self.runtime in ("python3.12", "python3.13"): + if self.runtime in ("python3.14"): + self.check_architecture_in("numpy-2.3.4.dist-info", ["manylinux_2_27_x86_64", "manylinux_2_28_x86_64"]) + expected_dependencies = {"numpy", "numpy-2.3.4.dist-info", "numpy.libs"} + elif self.runtime in ("python3.12", "python3.13"): self.check_architecture_in("numpy-2.1.2.dist-info", ["manylinux2014_x86_64", "manylinux1_x86_64"]) expected_dependencies = {"numpy", "numpy-2.1.2.dist-info", "numpy.libs"} elif self.runtime in ("python3.10", "python3.11"): diff --git a/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt b/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt index 3682fe698..6a4fdf69f 100644 --- a/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt +++ b/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt @@ -4,3 +4,4 @@ numpy==1.23.5; python_version == '3.10' numpy==1.23.5; python_version == '3.11' numpy==2.1.2; python_version == '3.12' numpy==2.1.2; python_version == '3.13' +numpy==2.3.4; python_version == '3.14' \ No newline at end of file diff --git a/tests/unit/test_validator.py b/tests/unit/test_validator.py index 56e03cc69..5c5a6b545 100644 --- a/tests/unit/test_validator.py +++ b/tests/unit/test_validator.py @@ -1,7 +1,8 @@ from unittest import TestCase -from aws_lambda_builders.validator import RuntimeValidator +from aws_lambda_builders.validator import RuntimeValidator, SUPPORTED_RUNTIMES from aws_lambda_builders.exceptions import UnsupportedRuntimeError, UnsupportedArchitectureError +from aws_lambda_builders.architecture import ARM64, X86_64 class TestRuntimeValidator(TestCase): @@ -16,14 +17,142 @@ def test_validate_runtime(self): self.validator.validate("/usr/bin/python3.8") self.assertEqual(self.validator._runtime_path, "/usr/bin/python3.8") - def test_validate_with_unsupported_runtime(self): - validator = RuntimeValidator(runtime="unknown_runtime", architecture="x86_64") - with self.assertRaises(UnsupportedRuntimeError): - validator.validate("/usr/bin/unknown_runtime") - - def test_validate_with_runtime_and_incompatible_architecture(self): - runtime_list = ["python3.12"] - for runtime in runtime_list: - validator = RuntimeValidator(runtime=runtime, architecture="invalid_arch") - with self.assertRaises(UnsupportedArchitectureError): - validator.validate("/usr/bin/{}".format(runtime)) + def test_validate_with_case_sensitive_architecture(self): + """Test that architecture names are case-sensitive""" + runtime = "python3.12" + + # Test invalid case variations (correct values are "arm64" and "x86_64") + invalid_arch_cases = ["ARM64", "Arm64", "aRM64", "X86_64", "X86-64", "ARM-64"] + for arch in invalid_arch_cases: + with self.subTest(architecture=arch): + validator = RuntimeValidator(runtime=runtime, architecture=arch) + with self.assertRaises(UnsupportedArchitectureError): + validator.validate("/usr/bin/python3.12") + + # Test that correct case works + for arch in [ARM64, X86_64]: # These are "arm64" and "x86_64" + validator = RuntimeValidator(runtime=runtime, architecture=arch) + result = validator.validate("/usr/bin/python3.12") + self.assertEqual(result, "/usr/bin/python3.12") + + def test_exception_messages_contain_runtime_info(self): + """Test that exception messages contain the problematic runtime information""" + # Test UnsupportedRuntimeError message + runtime = "unsupported_runtime" + validator = RuntimeValidator(runtime=runtime, architecture=X86_64) + with self.assertRaises(UnsupportedRuntimeError) as context: + validator.validate("/usr/bin/unsupported") + + error_msg = str(context.exception) + self.assertIn(runtime, error_msg) + self.assertIn("not supported", error_msg.lower()) + + # Test UnsupportedArchitectureError message + runtime = "python3.12" + arch = "unsupported_arch" + validator = RuntimeValidator(runtime=runtime, architecture=arch) + with self.assertRaises(UnsupportedArchitectureError) as context: + validator.validate("/usr/bin/python3.12") + + error_msg = str(context.exception) + self.assertIn(runtime, error_msg) + self.assertIn(arch, error_msg) + self.assertIn("not supported", error_msg.lower()) + + def test_all_nodejs_runtimes_supported(self): + """Test all Node.js runtimes are supported with both architectures""" + nodejs_runtimes = ["nodejs16.x", "nodejs18.x", "nodejs20.x", "nodejs22.x"] + for runtime in nodejs_runtimes: + for arch in [ARM64, X86_64]: + validator = RuntimeValidator(runtime=runtime, architecture=arch) + result = validator.validate("/usr/bin/node") + self.assertEqual(result, "/usr/bin/node") + self.assertEqual(validator._runtime_path, "/usr/bin/node") + + def test_all_python_runtimes_supported(self): + """Test all Python runtimes are supported with both architectures""" + python_runtimes = [ + "python3.8", + "python3.9", + "python3.10", + "python3.11", + "python3.12", + "python3.13", + "python3.14", + ] + for runtime in python_runtimes: + for arch in [ARM64, X86_64]: + validator = RuntimeValidator(runtime=runtime, architecture=arch) + result = validator.validate(f"/usr/bin/{runtime}") + self.assertEqual(result, f"/usr/bin/{runtime}") + self.assertEqual(validator._runtime_path, f"/usr/bin/{runtime}") + + def test_all_ruby_runtimes_supported(self): + """Test all Ruby runtimes are supported with both architectures""" + ruby_runtimes = ["ruby3.2", "ruby3.3", "ruby3.4"] + for runtime in ruby_runtimes: + for arch in [ARM64, X86_64]: + validator = RuntimeValidator(runtime=runtime, architecture=arch) + result = validator.validate("/usr/bin/ruby") + self.assertEqual(result, "/usr/bin/ruby") + self.assertEqual(validator._runtime_path, "/usr/bin/ruby") + + def test_all_java_runtimes_supported(self): + """Test all Java runtimes are supported with both architectures""" + java_runtimes = ["java8", "java11", "java17", "java21", "java25"] + for runtime in java_runtimes: + for arch in [ARM64, X86_64]: + validator = RuntimeValidator(runtime=runtime, architecture=arch) + result = validator.validate("/usr/bin/java") + self.assertEqual(result, "/usr/bin/java") + self.assertEqual(validator._runtime_path, "/usr/bin/java") + + def test_go_runtime_supported(self): + """Test Go runtime is supported with both architectures""" + runtime = "go1.x" + for arch in [ARM64, X86_64]: + validator = RuntimeValidator(runtime=runtime, architecture=arch) + result = validator.validate("/usr/bin/go") + self.assertEqual(result, "/usr/bin/go") + self.assertEqual(validator._runtime_path, "/usr/bin/go") + + def test_all_dotnet_runtimes_supported(self): + """Test all .NET runtimes are supported with both architectures""" + dotnet_runtimes = ["dotnet6", "dotnet8"] + for runtime in dotnet_runtimes: + for arch in [ARM64, X86_64]: + validator = RuntimeValidator(runtime=runtime, architecture=arch) + result = validator.validate("/usr/bin/dotnet") + self.assertEqual(result, "/usr/bin/dotnet") + self.assertEqual(validator._runtime_path, "/usr/bin/dotnet") + + def test_provided_runtime_supported(self): + """Test provided runtime is supported with both architectures""" + runtime = "provided" + for arch in [ARM64, X86_64]: + validator = RuntimeValidator(runtime=runtime, architecture=arch) + result = validator.validate("/opt/bootstrap") + self.assertEqual(result, "/opt/bootstrap") + self.assertEqual(validator._runtime_path, "/opt/bootstrap") + + def test_all_runtimes_support_both_architectures(self): + """Test that all supported runtimes support both ARM64 and X86_64 architectures""" + for runtime, supported_archs in SUPPORTED_RUNTIMES.items(): + self.assertIn(ARM64, supported_archs, f"Runtime {runtime} should support ARM64") + self.assertIn(X86_64, supported_archs, f"Runtime {runtime} should support X86_64") + self.assertEqual(len(supported_archs), 2, f"Runtime {runtime} should support exactly 2 architectures") + + def test_validate_returns_original_path(self): + """Test that validate method returns the original runtime_path unchanged""" + test_cases = [ + ("python3.11", "/usr/local/bin/python3.11"), + ("nodejs20.x", "/opt/node/bin/node"), + ("java17", "/usr/lib/jvm/java-17/bin/java"), + ("provided", "/var/runtime/bootstrap"), + ] + + for runtime, path in test_cases: + validator = RuntimeValidator(runtime=runtime, architecture=X86_64) + result = validator.validate(path) + self.assertEqual(result, path) + self.assertEqual(validator._runtime_path, path) diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 3c3640534..877b3b9b5 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -19,6 +19,10 @@ from aws_lambda_builders.workflows.python_pip.packager import InvalidSourceDistributionNameError from aws_lambda_builders.workflows.python_pip.packager import NoSuchPackageError from aws_lambda_builders.workflows.python_pip.packager import PackageDownloadError +from aws_lambda_builders.workflows.python_pip.packager import RequirementsFileNotFoundError +from aws_lambda_builders.workflows.python_pip.packager import MissingDependencyError +from aws_lambda_builders.workflows.python_pip.packager import UnsupportedPackageError +from aws_lambda_builders.workflows.python_pip.packager import UnsupportedPythonVersion from aws_lambda_builders.workflows.python_pip import packager @@ -90,6 +94,29 @@ def popen(self, *args, **kwargs): return self._processes.pop() +class TestExceptions(object): + def test_requirements_file_not_found_error(self): + error = RequirementsFileNotFoundError("/path/to/requirements.txt") + assert str(error) == "Requirements file not found: /path/to/requirements.txt" + + def test_missing_dependency_error(self): + missing_packages = ["package1", "package2"] + error = MissingDependencyError(missing_packages) + assert error.missing == missing_packages + + def test_no_such_package_error(self): + error = NoSuchPackageError("nonexistent-package") + assert str(error) == "Could not satisfy the requirement: nonexistent-package" + + def test_unsupported_package_error(self): + error = UnsupportedPackageError("bad-package") + assert str(error) == "Unable to retrieve name/version for package: bad-package" + + def test_unsupported_python_version(self): + error = UnsupportedPythonVersion("python2.7") + assert str(error) == "'python2.7' version of python is not supported" + + class TestGetLambdaAbi(object): def test_get_lambda_abi_python38(self): assert "cp38" == get_lambda_abi("python3.8") @@ -109,6 +136,13 @@ def test_get_lambda_abi_python312(self): def test_get_lambda_abi_python313(self): assert "cp313" == get_lambda_abi("python3.13") + def test_get_lambda_abi_python314(self): + assert "cp314" == get_lambda_abi("python3.14") + + def test_get_lambda_abi_unsupported_version(self): + with pytest.raises(UnsupportedPythonVersion): + get_lambda_abi("python2.7") + class TestPythonPipDependencyBuilder(object): def test_can_call_dependency_builder(self, osutils): @@ -484,3 +518,555 @@ def test_get_package_name_version( self.assertEqual(name, not_default_name) self.assertEqual(version, not_default_version) + + +class TestDependencyBuilder(object): + def test_has_at_least_one_package_file_not_exists(self): + osutils = mock.Mock(spec=OSUtils) + osutils.file_exists.return_value = False + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + result = builder._has_at_least_one_package("nonexistent.txt") + assert result is False + + def test_has_at_least_one_package_empty_file(self): + osutils = mock.Mock(spec=OSUtils) + osutils.file_exists.return_value = True + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + with mock.patch("builtins.open", mock.mock_open(read_data="")): + result = builder._has_at_least_one_package("empty.txt") + assert result is False + + def test_has_at_least_one_package_only_comments(self): + osutils = mock.Mock(spec=OSUtils) + osutils.file_exists.return_value = True + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + with mock.patch("builtins.open", mock.mock_open(read_data="# comment\n # another comment\n")): + result = builder._has_at_least_one_package("comments.txt") + assert result is False + + def test_has_at_least_one_package_with_packages(self): + osutils = mock.Mock(spec=OSUtils) + osutils.file_exists.return_value = True + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + with mock.patch("builtins.open", mock.mock_open(read_data="# comment\nrequests==2.25.1\n")): + result = builder._has_at_least_one_package("requirements.txt") + assert result is True + + def test_build_site_packages_no_packages(self): + osutils = mock.Mock(spec=OSUtils) + osutils.file_exists.return_value = False + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + # Should not raise any exception and not call other methods + builder.build_site_packages("empty.txt", "target", "scratch") + + def test_build_site_packages_with_missing_dependencies(self): + osutils = mock.Mock(spec=OSUtils) + osutils.file_exists.return_value = True + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + # Mock _has_at_least_one_package to return True + builder._has_at_least_one_package = mock.Mock(return_value=True) + + # Mock _download_dependencies to return empty wheels and some missing packages + missing_packages = [mock.Mock()] + builder._download_dependencies = mock.Mock(return_value=(set(), missing_packages)) + builder._install_wheels = mock.Mock() + + with mock.patch("builtins.open", mock.mock_open(read_data="requests==2.25.1\n")): + with pytest.raises(MissingDependencyError) as exc_info: + builder.build_site_packages("requirements.txt", "target", "scratch") + + assert exc_info.value.missing == missing_packages + + def test_compatible_platforms_x86_64_python38(self): + osutils = mock.Mock(spec=OSUtils) + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner, architecture=X86_64) + + platforms = builder.compatible_platforms + expected = [ + "any", + "linux_x86_64", + "manylinux1_x86_64", + "manylinux2010_x86_64", + "manylinux2014_x86_64", + "manylinux_2_17_x86_64", + ] + assert platforms == expected + + def test_compatible_platforms_arm64_python312(self): + osutils = mock.Mock(spec=OSUtils) + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.12", sys.executable, pip_runner, architecture=ARM64) + + platforms = builder.compatible_platforms + expected = [ + "any", + "linux_aarch64", + "manylinux2014_aarch64", + "manylinux_2_17_aarch64", + "manylinux_2_28_aarch64", + "manylinux_2_34_aarch64", + ] + assert platforms == expected + + def test_is_compatible_wheel_filename_pure_python(self): + osutils = mock.Mock(spec=OSUtils) + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + # Pure python wheel + result = builder._is_compatible_wheel_filename("package-1.0-py3-none-any.whl") + assert result is True + + def test_is_compatible_wheel_filename_compatible_platform(self): + osutils = mock.Mock(spec=OSUtils) + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + # Compatible platform wheel + result = builder._is_compatible_wheel_filename("package-1.0-cp38-cp38-linux_x86_64.whl") + assert result is True + + def test_is_compatible_wheel_filename_incompatible(self): + osutils = mock.Mock(spec=OSUtils) + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + # Incompatible wheel + result = builder._is_compatible_wheel_filename("package-1.0-cp39-cp39-win_amd64.whl") + assert result is False + + def test_is_compatible_platform_tag_legacy_manylinux(self): + osutils = mock.Mock(spec=OSUtils) + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + # Test legacy manylinux tag + result = builder._is_compatible_platform_tag("cp38", "manylinux1_x86_64") + assert result is True + + def test_is_compatible_platform_tag_newer_glibc(self): + osutils = mock.Mock(spec=OSUtils) + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + # Test newer glibc version (should be incompatible) + result = builder._is_compatible_platform_tag("cp38", "manylinux_2_35_x86_64") + assert result is False + + def test_is_compatible_platform_tag_invalid_format(self): + osutils = mock.Mock(spec=OSUtils) + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + # Test invalid platform tag format + result = builder._is_compatible_platform_tag("cp38", "invalid_platform") + assert result is False + + def test_iter_all_compatibility_tags(self): + osutils = mock.Mock(spec=OSUtils) + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + wheel = "numpy-1.20.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64" + tags = list(builder._iter_all_compatibility_tags(wheel)) + + expected_tags = [("cp38", "cp38", "manylinux_2_17_x86_64"), ("cp38", "cp38", "manylinux2014_x86_64")] + assert tags == expected_tags + + def test_apply_wheel_allowlist(self): + osutils = mock.Mock(spec=OSUtils) + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + # Create mock packages + compatible_pkg = mock.Mock() + compatible_pkg.name = "compatible" + + allowlisted_pkg = mock.Mock() + allowlisted_pkg.name = "sqlalchemy" # This is in the allowlist + + incompatible_pkg = mock.Mock() + incompatible_pkg.name = "incompatible" + + compatible_wheels = {compatible_pkg} + incompatible_wheels = {allowlisted_pkg, incompatible_pkg} + + result_compatible, result_incompatible = builder._apply_wheel_allowlist(compatible_wheels, incompatible_wheels) + + assert allowlisted_pkg in result_compatible + assert compatible_pkg in result_compatible + assert incompatible_pkg in result_incompatible + assert allowlisted_pkg not in result_incompatible + + def test_install_purelib_and_platlib_no_data_dir(self): + osutils = mock.Mock(spec=OSUtils) + osutils.directory_exists.return_value = False + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + wheel = mock.Mock() + wheel.data_dir = "package-1.0.data" + + # Should not raise any exception + builder._install_purelib_and_platlib(wheel, "/root") + osutils.directory_exists.assert_called_once() + + def test_install_purelib_and_platlib_with_dirs(self): + osutils = mock.Mock(spec=OSUtils) + osutils.directory_exists.return_value = True + osutils.get_directory_contents.return_value = ["purelib", "platlib", "other"] + osutils.joinpath.side_effect = lambda *args: "/".join(args) + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + wheel = mock.Mock() + wheel.data_dir = "package-1.0.data" + + builder._install_purelib_and_platlib(wheel, "/root") + + # Should copy purelib and platlib directories + assert osutils.copytree.call_count == 2 + assert osutils.rmtree.call_count == 2 + + def test_install_wheels(self): + osutils = mock.Mock(spec=OSUtils) + osutils.directory_exists.return_value = True + osutils.joinpath.side_effect = lambda *args: "/".join(args) + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + wheel1 = mock.Mock() + wheel1.filename = "package1-1.0-py3-none-any.whl" + wheel2 = mock.Mock() + wheel2.filename = "package2-2.0-py3-none-any.whl" + + wheels = [wheel1, wheel2] + + builder._install_purelib_and_platlib = mock.Mock() + + builder._install_wheels("/src", "/dst", wheels) + + osutils.rmtree.assert_called_once_with("/dst") + osutils.makedirs.assert_called_once_with("/dst") + assert osutils.extract_zipfile.call_count == 2 + assert builder._install_purelib_and_platlib.call_count == 2 + + +class TestSDistMetadataFetcherAdditional(TestCase): + def test_get_pkg_info_filepath_setuptools_not_available(self): + osutils = mock.Mock(spec=OSUtils) + osutils.joinpath.side_effect = lambda *args: "/".join(args) + osutils.makedirs = mock.Mock() + osutils.get_directory_contents.return_value = [] + osutils.file_exists.return_value = False + osutils.basename.return_value = "test-package" + + fetcher = SDistMetadataFetcher(sys.executable, osutils=osutils) + + with mock.patch("subprocess.run") as mock_run: + mock_run.return_value.returncode = 1 # Simulate setuptools not available + + with mock.patch("subprocess.Popen") as mock_popen: + mock_process = mock.Mock() + mock_process.returncode = 1 + mock_process.communicate.return_value = (b"", b"setuptools not found") + mock_popen.return_value = mock_process + + with pytest.raises(UnsupportedPackageError): + fetcher._get_pkg_info_filepath("/package/dir") + + def test_get_pkg_info_filepath_with_existing_egg_info(self): + osutils = mock.Mock(spec=OSUtils) + osutils.joinpath.side_effect = lambda *args: "/".join(args) + osutils.makedirs = mock.Mock() + osutils.get_directory_contents.side_effect = [ + [], # First call for egg-info dir (empty) + ["existing.egg-info", "other-file"], # Second call for package contents + ] + osutils.file_exists.side_effect = lambda path: path == "/package/dir/existing.egg-info/PKG-INFO" + osutils.directory_exists.side_effect = lambda path: path == "/package/dir/existing.egg-info" + + fetcher = SDistMetadataFetcher(sys.executable, osutils=osutils) + + with mock.patch("subprocess.Popen") as mock_popen: + mock_process = mock.Mock() + mock_process.returncode = 1 + mock_process.communicate.return_value = (b"", b"some error") + mock_popen.return_value = mock_process + + result = fetcher._get_pkg_info_filepath("/package/dir") + assert result == "/package/dir/existing.egg-info/PKG-INFO" + + def test_get_fallback_pkg_info_filepath(self): + osutils = mock.Mock(spec=OSUtils) + osutils.joinpath.side_effect = lambda *args: "/".join(args) + + fetcher = SDistMetadataFetcher(sys.executable, osutils=osutils) + result = fetcher._get_fallback_pkg_info_filepath("/package/dir") + + assert result == "/package/dir/PKG-INFO" + + def test_unpack_sdist_into_dir_zip(self): + osutils = mock.Mock(spec=OSUtils) + osutils.extract_zipfile = mock.Mock() + osutils.get_directory_contents.return_value = ["extracted-dir"] + osutils.joinpath.side_effect = lambda *args: "/".join(args) + + fetcher = SDistMetadataFetcher(sys.executable, osutils=osutils) + result = fetcher._unpack_sdist_into_dir("/path/to/package.zip", "/unpack/dir") + + osutils.extract_zipfile.assert_called_once_with("/path/to/package.zip", "/unpack/dir") + assert result == "/unpack/dir/extracted-dir" + + def test_unpack_sdist_into_dir_tar_gz(self): + osutils = mock.Mock(spec=OSUtils) + osutils.get_directory_contents.return_value = ["extracted-dir"] + osutils.joinpath.side_effect = lambda *args: "/".join(args) + + fetcher = SDistMetadataFetcher(sys.executable, osutils=osutils) + + with mock.patch("aws_lambda_builders.workflows.python_pip.packager.extract_tarfile") as mock_extract: + result = fetcher._unpack_sdist_into_dir("/path/to/package.tar.gz", "/unpack/dir") + + mock_extract.assert_called_once_with("/path/to/package.tar.gz", "/unpack/dir") + assert result == "/unpack/dir/extracted-dir" + + def test_unpack_sdist_into_dir_tar_bz2(self): + osutils = mock.Mock(spec=OSUtils) + osutils.get_directory_contents.return_value = ["extracted-dir"] + osutils.joinpath.side_effect = lambda *args: "/".join(args) + + fetcher = SDistMetadataFetcher(sys.executable, osutils=osutils) + + with mock.patch("aws_lambda_builders.workflows.python_pip.packager.extract_tarfile") as mock_extract: + result = fetcher._unpack_sdist_into_dir("/path/to/package.tar.bz2", "/unpack/dir") + + mock_extract.assert_called_once_with("/path/to/package.tar.bz2", "/unpack/dir") + assert result == "/unpack/dir/extracted-dir" + + def test_unpack_sdist_into_dir_invalid_extension(self): + osutils = mock.Mock(spec=OSUtils) + fetcher = SDistMetadataFetcher(sys.executable, osutils=osutils) + + with pytest.raises(InvalidSourceDistributionNameError): + fetcher._unpack_sdist_into_dir("/path/to/package.invalid", "/unpack/dir") + + def test_get_name_version(self): + osutils = mock.Mock(spec=OSUtils) + osutils.get_file_contents.return_value = "Name: test-package\nVersion: 1.0.0\n" + + fetcher = SDistMetadataFetcher(sys.executable, osutils=osutils) + name, version = fetcher._get_name_version("/path/to/PKG-INFO") + + assert name == "test-package" + assert version == "1.0.0" + + def test_is_default_setuptools_values_unknown_name(self): + osutils = mock.Mock(spec=OSUtils) + fetcher = SDistMetadataFetcher(sys.executable, osutils=osutils) + + assert fetcher._is_default_setuptools_values("UNKNOWN", "1.0.0") is True + assert fetcher._is_default_setuptools_values("unknown", "1.0.0") is True + + def test_is_default_setuptools_values_default_version(self): + osutils = mock.Mock(spec=OSUtils) + fetcher = SDistMetadataFetcher(sys.executable, osutils=osutils) + + assert fetcher._is_default_setuptools_values("package", "0.0.0") is True + + def test_is_default_setuptools_values_valid(self): + osutils = mock.Mock(spec=OSUtils) + fetcher = SDistMetadataFetcher(sys.executable, osutils=osutils) + + assert fetcher._is_default_setuptools_values("package", "1.0.0") is False + + +class TestPackageAdditional(object): + def test_package_sdist_with_metadata_fetcher(self): + osutils = mock.Mock(spec=OSUtils) + osutils.joinpath.side_effect = lambda *args: "/".join(args) + + with mock.patch("aws_lambda_builders.workflows.python_pip.packager.SDistMetadataFetcher") as mock_fetcher_class: + mock_fetcher = mock.Mock() + mock_fetcher.get_package_name_and_version.return_value = ("test-package", "1.0.0") + mock_fetcher_class.return_value = mock_fetcher + + pkg = Package("/dir", "test-package-1.0.tar.gz", sys.executable, osutils) + + assert pkg.dist_type == "sdist" + assert pkg.name == "test-package" + assert pkg.identifier == "test-package==1.0.0" + + def test_package_normalize_name_with_underscores_and_dots(self): + pkg = Package("", "test_package.name-1.0-py3-none-any.whl", sys.executable) + assert pkg.name == "test-package-name" + + +class TestPythonPipDependencyBuilderAdditional(object): + def test_default_osutils_creation(self): + # Test the case where osutils is None and gets created + with mock.patch("aws_lambda_builders.workflows.python_pip.packager.DependencyBuilder") as mock_dep_builder: + builder = PythonPipDependencyBuilder(runtime="python3.8", python_exe=sys.executable) + assert builder.osutils is not None + assert isinstance(builder.osutils, OSUtils) + + def test_default_dependency_builder_creation(self): + # Test the case where dependency_builder is None and gets created + osutils_mock = mock.Mock(spec=OSUtils) + with mock.patch("aws_lambda_builders.workflows.python_pip.packager.DependencyBuilder") as mock_dep_builder: + builder = PythonPipDependencyBuilder(runtime="python3.8", python_exe=sys.executable, osutils=osutils_mock) + mock_dep_builder.assert_called_once_with(osutils_mock, sys.executable, "python3.8", architecture=X86_64) + + +class TestDependencyBuilderAdditional(object): + def test_default_pip_runner_creation(self): + # Test the case where pip_runner is None and gets created + osutils = mock.Mock(spec=OSUtils) + with mock.patch("aws_lambda_builders.workflows.python_pip.packager.PipRunner") as mock_pip_runner: + with mock.patch("aws_lambda_builders.workflows.python_pip.packager.SubprocessPip") as mock_subprocess_pip: + builder = DependencyBuilder(osutils, "python3.8", sys.executable) + mock_subprocess_pip.assert_called_once_with(osutils) + mock_pip_runner.assert_called_once() + + def test_categorize_wheel_files(self): + osutils = mock.Mock(spec=OSUtils) + osutils.get_directory_contents.return_value = [ + "compatible-1.0-py3-none-any.whl", + "incompatible-1.0-cp39-cp39-win_amd64.whl", + "not-a-wheel.tar.gz", + ] + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + with mock.patch("aws_lambda_builders.workflows.python_pip.packager.Package") as mock_package: + # Mock Package instances + compatible_pkg = mock.Mock() + compatible_pkg.filename = "compatible-1.0-py3-none-any.whl" + incompatible_pkg = mock.Mock() + incompatible_pkg.filename = "incompatible-1.0-cp39-cp39-win_amd64.whl" + + mock_package.side_effect = [compatible_pkg, incompatible_pkg] + + # Mock the wheel filename compatibility check + builder._is_compatible_wheel_filename = mock.Mock(side_effect=[True, False]) + + compatible, incompatible = builder._categorize_wheel_files("/test/dir") + + assert compatible_pkg in compatible + assert incompatible_pkg in incompatible + + def test_build_sdists(self): + osutils = mock.Mock(spec=OSUtils) + osutils.joinpath.side_effect = lambda *args: "/".join(args) + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + # Create mock sdist packages + sdist1 = mock.Mock() + sdist1.filename = "package1-1.0.tar.gz" + sdist2 = mock.Mock() + sdist2.filename = "package2-2.0.tar.gz" + + sdists = {sdist1, sdist2} + + builder._build_sdists(sdists, "/test/dir", compile_c=True) + + # Should call build_wheel for each sdist + assert pip_runner.build_wheel.call_count == 2 + pip_runner.build_wheel.assert_any_call("/test/dir/package1-1.0.tar.gz", "/test/dir", True) + pip_runner.build_wheel.assert_any_call("/test/dir/package2-2.0.tar.gz", "/test/dir", True) + + def test_download_binary_wheels(self): + osutils = mock.Mock(spec=OSUtils) + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + # Create mock packages + pkg1 = mock.Mock() + pkg1.identifier = "package1==1.0" + pkg2 = mock.Mock() + pkg2.identifier = "package2==2.0" + + packages = {pkg1, pkg2} + + builder._download_binary_wheels(packages, "/test/dir") + + # Check that download_manylinux_wheels was called with the right parameters + # Note: set order is not guaranteed, so we check the call was made with the right args + call_args = pip_runner.download_manylinux_wheels.call_args + assert call_args is not None + identifiers, directory, abi, platforms = call_args[0] + assert set(identifiers) == {"package1==1.0", "package2==2.0"} + assert directory == "/test/dir" + assert abi == "cp38" + assert platforms == builder.compatible_platforms + + def test_download_all_dependencies(self): + osutils = mock.Mock(spec=OSUtils) + osutils.get_directory_contents.return_value = ["package1-1.0.tar.gz", "package2-2.0-py3-none-any.whl"] + pip_runner = mock.Mock(spec=PipRunner) + builder = DependencyBuilder(osutils, "python3.8", sys.executable, pip_runner) + + with mock.patch("aws_lambda_builders.workflows.python_pip.packager.Package") as mock_package: + # Mock Package instances + pkg1 = mock.Mock() + pkg2 = mock.Mock() + mock_package.side_effect = [pkg1, pkg2] + + result = builder._download_all_dependencies("/requirements.txt", "/test/dir") + + pip_runner.download_all_dependencies.assert_called_once_with("/requirements.txt", "/test/dir") + assert pkg1 in result + assert pkg2 in result + + +class TestPackageEdgeCases(object): + def test_package_invalid_sdist_extension(self): + with pytest.raises(InvalidSourceDistributionNameError): + Package("", "invalid-package.unknown", sys.executable) + + def test_package_data_dir_property(self): + pkg = Package("", "test_package-1.2.3-py3-none-any.whl", sys.executable) + # The data_dir uses the normalized name and version + assert pkg.data_dir == "test-package-1.2.3.data" + + +class TestSubprocessPipEdgeCases(object): + def test_subprocess_pip_with_custom_python_exe(self): + osutils = mock.Mock(spec=OSUtils) + with mock.patch("aws_lambda_builders.workflows.python_pip.packager.pip_import_string") as mock_import: + mock_import.return_value = "import pip" + pip = SubprocessPip(osutils=osutils, python_exe="/custom/python") + assert pip.python_exe == "/custom/python" + + def test_subprocess_pip_main_with_custom_env_and_shim(self): + fake_osutils = FakePopenOSUtils([FakePopen(0, b"output", b"error")]) + pip = SubprocessPip(osutils=fake_osutils, python_exe=sys.executable) + + custom_env = {"CUSTOM_VAR": "value"} + custom_shim = "custom_shim_code;" + + rc, out, err = pip.main(["--version"], env_vars=custom_env, shim=custom_shim) + + assert rc == 0 + assert out == b"output" + assert err == b"error" + + # Check that custom env and shim were used + call_args, call_kwargs = fake_osutils.popens[0] + assert call_kwargs["env"] == custom_env + exec_string = call_args[0][2] + assert exec_string.startswith(custom_shim)