From f4359e84ce17a3a63ffdd33ad48711b3a93bb2e1 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Sat, 13 Apr 2024 19:22:41 -0400 Subject: [PATCH] Add `shared-scripts` option for the `wheel` target (#1386) --- backend/src/hatchling/builders/wheel.py | 69 +++++++ docs/history/hatchling.md | 4 + docs/plugins/builder/wheel.md | 3 +- tests/backend/builders/test_wheel.py | 194 ++++++++++++++++++ tests/helpers/helpers.py | 10 +- .../wheel/standard_default_shared_scripts.py | 87 ++++++++ tests/helpers/templates/wheel/utils.py | 21 +- 7 files changed, 379 insertions(+), 9 deletions(-) create mode 100644 tests/helpers/templates/wheel/standard_default_shared_scripts.py diff --git a/backend/src/hatchling/builders/wheel.py b/backend/src/hatchling/builders/wheel.py index 0262fd439..1acac4b6a 100644 --- a/backend/src/hatchling/builders/wheel.py +++ b/backend/src/hatchling/builders/wheel.py @@ -123,6 +123,10 @@ def write_metadata(self, relative_path: str, contents: str | bytes) -> tuple[str relative_path = f'{self.metadata_directory}/{normalize_archive_path(relative_path)}' return self.write_file(relative_path, contents) + def write_shared_script(self, relative_path: str, contents: str | bytes) -> tuple[str, str, str]: + relative_path = f'{self.shared_data_directory}/scripts/{normalize_archive_path(relative_path)}' + return self.write_file(relative_path, contents) + def add_shared_file(self, shared_file: IncludedFile) -> tuple[str, str, str]: shared_file.distribution_path = f'{self.shared_data_directory}/data/{shared_file.distribution_path}' return self.add_file(shared_file) @@ -163,6 +167,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self.__core_metadata_constructor: Callable[..., str] | None = None self.__shared_data: dict[str, str] | None = None + self.__shared_scripts: dict[str, str] | None = None self.__extra_metadata: dict[str, str] | None = None self.__strict_naming: bool | None = None self.__macos_max_compat: bool | None = None @@ -289,6 +294,40 @@ def shared_data(self) -> dict[str, str]: return self.__shared_data + @property + def shared_scripts(self) -> dict[str, str]: + if self.__shared_scripts is None: + shared_scripts = self.target_config.get('shared-scripts', {}) + if not isinstance(shared_scripts, dict): + message = f'Field `tool.hatch.build.targets.{self.plugin_name}.shared-scripts` must be a mapping' + raise TypeError(message) + + for i, (source, relative_path) in enumerate(shared_scripts.items(), 1): + if not source: + message = ( + f'Source #{i} in field `tool.hatch.build.targets.{self.plugin_name}.shared-scripts` ' + f'cannot be an empty string' + ) + raise ValueError(message) + + if not isinstance(relative_path, str): + message = ( + f'Path for source `{source}` in field ' + f'`tool.hatch.build.targets.{self.plugin_name}.shared-scripts` must be a string' + ) + raise TypeError(message) + + if not relative_path: + message = ( + f'Path for source `{source}` in field ' + f'`tool.hatch.build.targets.{self.plugin_name}.shared-scripts` cannot be an empty string' + ) + raise ValueError(message) + + self.__shared_scripts = normalize_inclusion_map(shared_scripts, self.root) + + return self.__shared_scripts + @property def extra_metadata(self) -> dict[str, str]: if self.__extra_metadata is None: @@ -547,6 +586,7 @@ def write_data( self, archive: WheelArchive, records: RecordFile, build_data: dict[str, Any], extra_dependencies: Sequence[str] ) -> None: self.add_shared_data(archive, records) + self.add_shared_scripts(archive, records) # Ensure metadata is written last, see https://peps.python.org/pep-0427/#recommended-archiver-features self.write_metadata(archive, records, build_data, extra_dependencies=extra_dependencies) @@ -556,6 +596,35 @@ def add_shared_data(self, archive: WheelArchive, records: RecordFile) -> None: record = archive.add_shared_file(shared_file) records.write(record) + def add_shared_scripts(self, archive: WheelArchive, records: RecordFile) -> None: + import re + from io import BytesIO + + # https://packaging.python.org/en/latest/specifications/binary-distribution-format/#recommended-installer-features + shebang = re.compile(rb'^#!.*(?:pythonw?|pypyw?)[0-9.]*(.*)', flags=re.DOTALL) + + for shared_script in self.recurse_explicit_files(self.config.shared_scripts): + with open(shared_script.path, 'rb') as f: + content = BytesIO() + for line in f: + # Ignore leading blank lines + if not line.strip(): + continue + + match = shebang.match(line) + if match is None: + content.write(line) + else: + content.write(b'#!python') + if remaining := match.group(1): + content.write(remaining) + + content.write(f.read()) + break + + record = archive.write_shared_script(shared_script.distribution_path, content.getvalue()) + records.write(record) + def write_metadata( self, archive: WheelArchive, diff --git a/docs/history/hatchling.md b/docs/history/hatchling.md index 5ca2c6eb3..ecc5f3b93 100644 --- a/docs/history/hatchling.md +++ b/docs/history/hatchling.md @@ -8,6 +8,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## Unreleased +***Added:*** + +- Add `shared-scripts` option for the `wheel` target + ***Fixed:*** - Set the `packaging` dependency version as `>=23.2` to avoid its URL validation which can conflict with context formatting diff --git a/docs/plugins/builder/wheel.md b/docs/plugins/builder/wheel.md index 62a29d1a1..9ae3bc176 100644 --- a/docs/plugins/builder/wheel.md +++ b/docs/plugins/builder/wheel.md @@ -17,7 +17,8 @@ The builder plugin name is `wheel`. | Option | Default | Description | | --- | --- | --- | | `core-metadata-version` | `"2.3"` | The version of [core metadata](https://packaging.python.org/specifications/core-metadata/) to use | -| `shared-data` | | A mapping similar to the [forced inclusion](../../config/build.md#forced-inclusion) option corresponding to [data](https://peps.python.org/pep-0427/#the-data-directory) that will be installed globally in a given Python environment, usually under `#!python sys.prefix` | +| `shared-data` | | A mapping similar to the [forced inclusion](../../config/build.md#forced-inclusion) option corresponding to the `data` subdirectory within the standard [data directory](https://packaging.python.org/en/latest/specifications/binary-distribution-format/#the-data-directory) that will be installed globally in a given Python environment, usually under `#!python sys.prefix` | +| `shared-scripts` | | A mapping similar to the [forced inclusion](../../config/build.md#forced-inclusion) option corresponding to the `scripts` subdirectory within the standard [data directory](https://packaging.python.org/en/latest/specifications/binary-distribution-format/#the-data-directory) that will be installed in a given Python environment, usually under `Scripts` on Windows or `bin` otherwise, and would normally be available on PATH | | `extra-metadata` | | A mapping similar to the [forced inclusion](../../config/build.md#forced-inclusion) option corresponding to extra [metadata](https://peps.python.org/pep-0427/#the-dist-info-directory) that will be shipped in a directory named `extra_metadata` | | `strict-naming` | `true` | Whether or not file names should contain the normalized version of the project name | | `macos-max-compat` | `true` | Whether or not on macOS, when build hooks have set the `infer_tag` [build data](#build-data), the wheel name should signal broad support rather than specific versions for newer SDK versions.

Note: The default will become `false`, and this option eventually removed, sometime after consumers like pip start supporting these newer SDK versions. | diff --git a/tests/backend/builders/test_wheel.py b/tests/backend/builders/test_wheel.py index 11adab9e7..476eb9104 100644 --- a/tests/backend/builders/test_wheel.py +++ b/tests/backend/builders/test_wheel.py @@ -360,6 +360,95 @@ def test_order(self, isolation): } +class TestSharedScripts: + def test_default(self, isolation): + builder = WheelBuilder(str(isolation)) + + assert builder.config.shared_scripts == builder.config.shared_scripts == {} + + def test_invalid_type(self, isolation): + config = {'tool': {'hatch': {'build': {'targets': {'wheel': {'shared-scripts': 42}}}}}} + builder = WheelBuilder(str(isolation), config=config) + + with pytest.raises(TypeError, match='Field `tool.hatch.build.targets.wheel.shared-scripts` must be a mapping'): + _ = builder.config.shared_scripts + + def test_absolute(self, isolation): + config = { + 'tool': { + 'hatch': {'build': {'targets': {'wheel': {'shared-scripts': {str(isolation / 'source'): '/target/'}}}}} + } + } + builder = WheelBuilder(str(isolation), config=config) + + assert builder.config.shared_scripts == {str(isolation / 'source'): 'target'} + + def test_relative(self, isolation): + config = {'tool': {'hatch': {'build': {'targets': {'wheel': {'shared-scripts': {'../source': '/target/'}}}}}}} + builder = WheelBuilder(str(isolation / 'foo'), config=config) + + assert builder.config.shared_scripts == {str(isolation / 'source'): 'target'} + + def test_source_empty_string(self, isolation): + config = {'tool': {'hatch': {'build': {'targets': {'wheel': {'shared-scripts': {'': '/target/'}}}}}}} + builder = WheelBuilder(str(isolation), config=config) + + with pytest.raises( + ValueError, + match='Source #1 in field `tool.hatch.build.targets.wheel.shared-scripts` cannot be an empty string', + ): + _ = builder.config.shared_scripts + + def test_relative_path_not_string(self, isolation): + config = {'tool': {'hatch': {'build': {'targets': {'wheel': {'shared-scripts': {'source': 0}}}}}}} + builder = WheelBuilder(str(isolation), config=config) + + with pytest.raises( + TypeError, + match='Path for source `source` in field `tool.hatch.build.targets.wheel.shared-scripts` must be a string', + ): + _ = builder.config.shared_scripts + + def test_relative_path_empty_string(self, isolation): + config = {'tool': {'hatch': {'build': {'targets': {'wheel': {'shared-scripts': {'source': ''}}}}}}} + builder = WheelBuilder(str(isolation), config=config) + + with pytest.raises( + ValueError, + match=( + 'Path for source `source` in field `tool.hatch.build.targets.wheel.shared-scripts` ' + 'cannot be an empty string' + ), + ): + _ = builder.config.shared_scripts + + def test_order(self, isolation): + config = { + 'tool': { + 'hatch': { + 'build': { + 'targets': { + 'wheel': { + 'shared-scripts': { + '../very-nested': 'target1/embedded', + '../source1': '/target2/', + '../source2': '/target1/', + } + } + } + } + } + } + } + builder = WheelBuilder(str(isolation / 'foo'), config=config) + + assert builder.config.shared_scripts == { + str(isolation / 'source2'): 'target1', + str(isolation / 'very-nested'): f'target1{os.sep}embedded', + str(isolation / 'source1'): 'target2', + } + + class TestExtraMetadata: def test_default(self, isolation): builder = WheelBuilder(str(isolation)) @@ -1832,6 +1921,111 @@ def test_default_shared_data(self, hatch, helpers, temp_dir, config_file): ) helpers.assert_files(extraction_directory, expected_files) + def test_default_shared_scripts(self, hatch, helpers, temp_dir, config_file): + config_file.model.template.plugins['default']['src-layout'] = False + config_file.save() + + project_name = 'My.App' + + with temp_dir.as_cwd(): + result = hatch('new', project_name) + + assert result.exit_code == 0, result.output + + project_path = temp_dir / 'my-app' + + shared_data_path = temp_dir / 'data' + shared_data_path.ensure_dir_exists() + + binary_contents = os.urandom(1024) + (shared_data_path / 'binary').write_bytes(binary_contents) + (shared_data_path / 'other_script.sh').write_text( + helpers.dedent( + """ + + #!/bin/sh arg1 arg2 + echo "Hello, World!" + """ + ) + ) + (shared_data_path / 'python_script.sh').write_text( + helpers.dedent( + """ + + #!/usr/bin/env python3.11 arg1 arg2 + print("Hello, World!") + """ + ) + ) + (shared_data_path / 'pythonw_script.sh').write_text( + helpers.dedent( + """ + + #!/usr/bin/pythonw3.11 arg1 arg2 + print("Hello, World!") + """ + ) + ) + (shared_data_path / 'pypy_script.sh').write_text( + helpers.dedent( + """ + + #!/usr/bin/env pypy + print("Hello, World!") + """ + ) + ) + (shared_data_path / 'pypyw_script.sh').write_text( + helpers.dedent( + """ + + #!pypyw3.11 arg1 arg2 + print("Hello, World!") + """ + ) + ) + + config = { + 'project': {'name': project_name, 'requires-python': '>3', 'dynamic': ['version']}, + 'tool': { + 'hatch': { + 'version': {'path': 'my_app/__about__.py'}, + 'build': {'targets': {'wheel': {'versions': ['standard'], 'shared-scripts': {'../data': '/'}}}}, + }, + }, + } + builder = WheelBuilder(str(project_path), config=config) + + build_path = project_path / 'dist' + build_path.mkdir() + + with project_path.as_cwd(): + artifacts = list(builder.build(directory=str(build_path))) + + assert len(artifacts) == 1 + expected_artifact = artifacts[0] + + build_artifacts = list(build_path.iterdir()) + assert len(build_artifacts) == 1 + assert expected_artifact == str(build_artifacts[0]) + + extraction_directory = temp_dir / '_archive' + extraction_directory.mkdir() + + with zipfile.ZipFile(str(expected_artifact), 'r') as zip_archive: + zip_archive.extractall(str(extraction_directory)) + + metadata_directory = f'{builder.project_id}.dist-info' + shared_data_directory = f'{builder.project_id}.data' + expected_files = helpers.get_template_files( + 'wheel.standard_default_shared_scripts', + project_name, + metadata_directory=metadata_directory, + shared_data_directory=shared_data_directory, + binary_contents=binary_contents, + ) + helpers.assert_files(extraction_directory, expected_files) + def test_default_extra_metadata(self, hatch, helpers, temp_dir, config_file): config_file.model.template.plugins['default']['src-layout'] = False config_file.save() diff --git a/tests/helpers/helpers.py b/tests/helpers/helpers.py index 3e0223505..25a200815 100644 --- a/tests/helpers/helpers.py +++ b/tests/helpers/helpers.py @@ -78,8 +78,14 @@ def assert_files(directory, expected_files, *, check_contents=True): seen_relative_file_paths.add(relative_file_path) if check_contents and relative_file_path in expected_relative_files: - with open(os.path.join(start, relative_file_path), encoding='utf-8') as f: - assert f.read() == expected_relative_files[relative_file_path], relative_file_path + file_path = os.path.join(start, relative_file_path) + expected_contents = expected_relative_files[relative_file_path] + try: + with open(file_path, encoding='utf-8') as f: + assert f.read() == expected_contents, relative_file_path + except UnicodeDecodeError: + with open(file_path, 'rb') as f: + assert f.read() == expected_contents, (relative_file_path, expected_contents) else: # no cov pass diff --git a/tests/helpers/templates/wheel/standard_default_shared_scripts.py b/tests/helpers/templates/wheel/standard_default_shared_scripts.py new file mode 100644 index 000000000..b89c997bd --- /dev/null +++ b/tests/helpers/templates/wheel/standard_default_shared_scripts.py @@ -0,0 +1,87 @@ +from hatch.template import File +from hatch.utils.fs import Path +from hatchling.__about__ import __version__ +from hatchling.metadata.spec import DEFAULT_METADATA_VERSION + +from ..new.feature_no_src_layout import get_files as get_template_files +from .utils import update_record_file_contents + + +def get_files(**kwargs): + metadata_directory = kwargs.get('metadata_directory', '') + shared_data_directory = kwargs.get('shared_data_directory', '') + binary_contents = kwargs.get('binary_contents', b'') + + files = [] + for f in get_template_files(**kwargs): + if str(f.path) == 'LICENSE.txt': + files.append(File(Path(metadata_directory, 'licenses', f.path), f.contents)) + + if f.path.parts[0] != kwargs['package_name']: + continue + + files.append(f) + + files.extend(( + File(Path(shared_data_directory, 'scripts', 'binary'), binary_contents), + File( + Path(shared_data_directory, 'scripts', 'other_script.sh'), + """\ +#!/bin/sh arg1 arg2 +echo "Hello, World!" +""", + ), + File( + Path(shared_data_directory, 'scripts', 'python_script.sh'), + """\ +#!python arg1 arg2 +print("Hello, World!") +""", + ), + File( + Path(shared_data_directory, 'scripts', 'pythonw_script.sh'), + """\ +#!python arg1 arg2 +print("Hello, World!") +""", + ), + File( + Path(shared_data_directory, 'scripts', 'pypy_script.sh'), + """\ +#!python +print("Hello, World!") +""", + ), + File( + Path(shared_data_directory, 'scripts', 'pypyw_script.sh'), + """\ +#!python arg1 arg2 +print("Hello, World!") +""", + ), + File( + Path(metadata_directory, 'WHEEL'), + f"""\ +Wheel-Version: 1.0 +Generator: hatchling {__version__} +Root-Is-Purelib: true +Tag: py3-none-any +""", + ), + File( + Path(metadata_directory, 'METADATA'), + f"""\ +Metadata-Version: {DEFAULT_METADATA_VERSION} +Name: {kwargs['project_name']} +Version: 0.0.1 +License-File: LICENSE.txt +Requires-Python: >3 +""", + ), + )) + + record_file = File(Path(metadata_directory, 'RECORD'), '') + update_record_file_contents(record_file, files) + files.append(record_file) + + return files diff --git a/tests/helpers/templates/wheel/utils.py b/tests/helpers/templates/wheel/utils.py index a37d1d0fd..42d5c04fc 100644 --- a/tests/helpers/templates/wheel/utils.py +++ b/tests/helpers/templates/wheel/utils.py @@ -15,14 +15,23 @@ def update_record_file_contents(record_file, files, generated_files=()): f.path.parts, ), ): - raw_contents = template_file.contents.encode('utf-8') + if isinstance(template_file.contents, bytes): + is_binary = True + raw_contents = template_file.contents + else: + is_binary = False + raw_contents = template_file.contents.encode('utf-8') template_file_path = str(template_file.path) - if os.linesep != '\n' and ( - 'LICENSE' in template_file_path - or ( - not template_file.path.parts[0].endswith('.dist-info') - and all(f not in template_file_path for f in generated_files) + if ( + not is_binary + and os.linesep != '\n' + and ( + 'LICENSE' in template_file_path + or ( + not template_file.path.parts[0].endswith('.dist-info') + and all(f not in template_file_path for f in generated_files) + ) ) ): raw_contents = raw_contents.replace(b'\n', b'\r\n')