From 57cbfe1a86a2bced96d1948528dcf9cc440a800c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Thu, 16 Feb 2023 14:04:54 +0900 Subject: [PATCH] refactor: starlark reimplementation of pip_repository (#1043) Previously we were using pip_parse python scripts. This has a few drawbacks: * Requires system python to be present. * Usage of a Python script makes it harder to reason as there is an extra layer of abstraction. * Extending/reusing code between multi_pip_parse and pip_parse is hard. Now we use Starlark to parse the requirements.txt into requirements.bzl. --- docs/pip.md | 3 +- docs/pip_repository.md | 29 ++- examples/pip_parse_vendored/BUILD.bazel | 1 - examples/pip_parse_vendored/requirements.bzl | 11 +- python/extensions.bzl | 14 +- python/pip.bzl | 11 +- python/pip_install/BUILD.bazel | 2 - python/pip_install/pip_repository.bzl | 201 +++++++++++++----- .../pip_repository_requirements.bzl.tmpl | 52 +++++ ...ip_repository_requirements_bzlmod.bzl.tmpl | 24 +++ python/pip_install/private/srcs.bzl | 2 - .../tools/lock_file_generator/BUILD.bazel | 50 ----- .../tools/lock_file_generator/__init__.py | 14 -- .../lock_file_generator_test.py | 163 -------------- 14 files changed, 259 insertions(+), 318 deletions(-) create mode 100644 python/pip_install/pip_repository_requirements.bzl.tmpl create mode 100644 python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl delete mode 100644 python/pip_install/tools/lock_file_generator/BUILD.bazel delete mode 100644 python/pip_install/tools/lock_file_generator/__init__.py delete mode 100644 python/pip_install/tools/lock_file_generator/lock_file_generator_test.py diff --git a/docs/pip.md b/docs/pip.md index 2f5b92ebf..528abf737 100644 --- a/docs/pip.md +++ b/docs/pip.md @@ -168,7 +168,7 @@ install_deps() ## pip_parse 
-pip_parse(requirements, requirements_lock, name, bzlmod, kwargs)
+pip_parse(requirements, requirements_lock, name, kwargs)
Accepts a locked/compiled requirements file and installs the dependencies listed within. @@ -264,7 +264,6 @@ See the example in rules_python/examples/pip_parse_vendored. | requirements | Deprecated. See requirements_lock. | None | | requirements_lock | A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that wheels are fetched/built only for the targets specified by 'build/run/test'. Note that if your lockfile is platform-dependent, you can use the requirements_[platform] attributes. | None | | name | The name of the generated repository. The generated repositories containing each requirement will be of the form <name>_<requirement-name>. | "pip_parsed_deps" | -| bzlmod | Whether this rule is being run under a bzlmod module extension. | False | | kwargs | Additional arguments to the [pip_repository](./pip_repository.md) repository rule. | none | diff --git a/docs/pip_repository.md b/docs/pip_repository.md index 7abb503c7..2ccdc6485 100644 --- a/docs/pip_repository.md +++ b/docs/pip_repository.md @@ -7,8 +7,8 @@ ## pip_repository 
-pip_repository(name, annotations, bzlmod, download_only, enable_implicit_namespace_pkgs,
-               environment, extra_pip_args, isolated, pip_data_exclude, python_interpreter,
+pip_repository(name, annotations, download_only, enable_implicit_namespace_pkgs, environment,
+               extra_pip_args, isolated, pip_data_exclude, python_interpreter,
                python_interpreter_target, quiet, repo_mapping, repo_prefix, requirements_darwin,
                requirements_linux, requirements_lock, requirements_windows, timeout)
@@ -60,7 +60,6 @@ py_binary( | :------------- | :------------- | :------------- | :------------- | :------------- | | name | A unique name for this repository. | Name | required | | | annotations | Optional annotations to apply to packages | Dictionary: String -> String | optional | {} | -| bzlmod | Whether this repository rule is invoked under bzlmod, in which case we do not create the install_deps() macro. | Boolean | optional | False | | download_only | Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of --platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different platform from the host platform. | Boolean | optional | False | | enable_implicit_namespace_pkgs | If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary and py_test targets must specify either legacy_create_init=False or the global Bazel option --incompatible_default_to_explicit_init_py to prevent __init__.py being automatically generated in every directory.

This option is required to support some packages which cannot handle the conversion to pkg-util style. | Boolean | optional | False | | environment | Environment variables to set in the pip subprocess. Can be used to set common variables such as http_proxy, https_proxy and no_proxy Note that pip is run with "--isolated" on the CLI so PIP_<VAR>_<NAME> style env vars are ignored, but env vars that control requests and urllib3 can be passed. | Dictionary: String -> String | optional | {} | @@ -79,6 +78,30 @@ py_binary( | timeout | Timeout (in seconds) on the rule's execution duration. | Integer | optional | 600 | + + +## pip_repository_bzlmod + +
+pip_repository_bzlmod(name, repo_mapping, requirements_darwin, requirements_linux,
+                      requirements_lock, requirements_windows)
+
+ +A rule for bzlmod pip_repository creation. Intended for private use only. + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this repository. | Name | required | | +| repo_mapping | A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.<p>For example, an entry "@foo": "@bar" declares that, for any time this repository depends on @foo (such as a dependency on @foo//some:target, it should actually resolve that dependency within globally-declared @bar (@bar//some:target). | Dictionary: String -> String | required | | +| requirements_darwin | Override the requirements_lock attribute when the host platform is Mac OS | Label | optional | None | +| requirements_linux | Override the requirements_lock attribute when the host platform is Linux | Label | optional | None | +| requirements_lock | A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that wheels are fetched/built only for the targets specified by 'build/run/test'. | Label | optional | None | +| requirements_windows | Override the requirements_lock attribute when the host platform is Windows | Label | optional | None | + + ## whl_library diff --git a/examples/pip_parse_vendored/BUILD.bazel b/examples/pip_parse_vendored/BUILD.bazel index 9585195f1..56630e513 100644 --- a/examples/pip_parse_vendored/BUILD.bazel +++ b/examples/pip_parse_vendored/BUILD.bazel @@ -20,7 +20,6 @@ genrule( # Replace the bazel 6.0.0 specific comment with something that bazel 5.4.0 would produce. # This enables this example to be run as a test under bazel 5.4.0. """sed -e 's#@//#//#'""", - """tr "'" '"' """, """sed 's#"@python39_.*//:bin/python3"#interpreter#' >$@""", ]), ) diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index e13503ac6..015df9340 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -14,29 +14,20 @@ all_whl_requirements = ["@pip_certifi//:whl", "@pip_charset_normalizer//:whl", " _packages = [("pip_certifi", "certifi==2022.12.7 --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"), ("pip_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("pip_idna", "idna==3.4 --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("pip_requests", "requests==2.28.1 --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("pip_urllib3", "urllib3==1.26.13 --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")] _config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": interpreter, "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600} _annotations = {} -_bzlmod = False def _clean_name(name): return name.replace("-", "_").replace(".", "_").lower() def requirement(name): - if _bzlmod: - return "@@pip//:" + _clean_name(name) + "_pkg" return "@pip_" + _clean_name(name) + "//:pkg" def whl_requirement(name): - if _bzlmod: - return "@@pip//:" + _clean_name(name) + "_whl" return "@pip_" + _clean_name(name) + "//:whl" def data_requirement(name): - if _bzlmod: - return "@@pip//:" + _clean_name(name) + "_data" return "@pip_" + _clean_name(name) + "//:data" def dist_info_requirement(name): - if _bzlmod: - return "@@pip//:" + _clean_name(name) + "_dist_info" return "@pip_" + _clean_name(name) + "//:dist_info" def entry_point(pkg, script = None): @@ -46,7 +37,7 @@ def entry_point(pkg, script = None): def _get_annotation(requirement): # This expects to parse `setuptools==58.2.0 --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11` - # down wo `setuptools`. + # down to `setuptools`. name = requirement.split(" ")[0].split("=")[0].split("[")[0] return _annotations.get(name) diff --git a/python/extensions.bzl b/python/extensions.bzl index bc0d570c5..01f731f14 100644 --- a/python/extensions.bzl +++ b/python/extensions.bzl @@ -14,9 +14,8 @@ "Module extensions for use with bzlmod" -load("@rules_python//python:pip.bzl", "pip_parse") load("@rules_python//python:repositories.bzl", "python_register_toolchains") -load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "use_isolated", "whl_library") +load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "pip_repository_bzlmod", "use_isolated", "whl_library") load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies") load("@rules_python//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") load("@rules_python//python/private:coverage_deps.bzl", "install_coverage_deps") @@ -68,7 +67,7 @@ def _pip_impl(module_ctx): # Parse the requirements file directly in starlark to get the information # needed for the whl_libary declarations below. This is needed to contain - # the pip_parse logic to a single module extension. + # the pip_repository logic to a single module extension. requirements_lock_content = module_ctx.read(requrements_lock) parse_result = parse_requirements(requirements_lock_content) requirements = parse_result.requirements @@ -76,14 +75,9 @@ def _pip_impl(module_ctx): # Create the repository where users load the `requirement` macro. Under bzlmod # this does not create the install_deps() macro. - pip_parse( + pip_repository_bzlmod( name = attr.name, requirements_lock = attr.requirements_lock, - bzlmod = True, - timeout = attr.timeout, - python_interpreter = attr.python_interpreter, - python_interpreter_target = attr.python_interpreter_target, - quiet = attr.quiet, ) for name, requirement_line in requirements: @@ -114,7 +108,7 @@ def _pip_parse_ext_attrs(): "name": attr.string(mandatory = True), }, **pip_repository_attrs) - # Like the pip_parse macro, we end up setting this manually so + # Like the pip_repository rule, we end up setting this manually so # don't allow users to override it. attrs.pop("repo_prefix") diff --git a/python/pip.bzl b/python/pip.bzl index 3d45aed61..3c0630130 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -47,7 +47,7 @@ def pip_install(requirements = None, name = "pip", **kwargs): print("pip_install is deprecated. Please switch to pip_parse. pip_install will be removed in a future release.") pip_parse(requirements = requirements, name = name, **kwargs) -def pip_parse(requirements = None, requirements_lock = None, name = "pip_parsed_deps", bzlmod = False, **kwargs): +def pip_parse(requirements = None, requirements_lock = None, name = "pip_parsed_deps", **kwargs): """Accepts a locked/compiled requirements file and installs the dependencies listed within. Those dependencies become available in a generated `requirements.bzl` file. @@ -143,14 +143,9 @@ def pip_parse(requirements = None, requirements_lock = None, name = "pip_parsed_ requirements (Label): Deprecated. See requirements_lock. name (str, optional): The name of the generated repository. The generated repositories containing each requirement will be of the form `_`. - bzlmod (bool, optional): Whether this rule is being run under a bzlmod module extension. **kwargs (dict): Additional arguments to the [`pip_repository`](./pip_repository.md) repository rule. """ - - # Don't try to fetch dependencies under bzlmod because they are already fetched via the internal_deps - # module extention, and because the maybe-install pattern doesn't work under bzlmod. - if not bzlmod: - pip_install_dependencies() + pip_install_dependencies() # Temporary compatibility shim. # pip_install was previously document to use requirements while pip_parse was using requirements_lock. @@ -160,8 +155,6 @@ def pip_parse(requirements = None, requirements_lock = None, name = "pip_parsed_ pip_repository( name = name, requirements_lock = reqs_to_use, - repo_prefix = "{}_".format(name), - bzlmod = bzlmod, **kwargs ) diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel index 451e7fab7..281ccba6a 100644 --- a/python/pip_install/BUILD.bazel +++ b/python/pip_install/BUILD.bazel @@ -4,7 +4,6 @@ filegroup( "BUILD.bazel", "//python/pip_install/tools/dependency_resolver:distribution", "//python/pip_install/tools/lib:distribution", - "//python/pip_install/tools/lock_file_generator:distribution", "//python/pip_install/tools/wheel_installer:distribution", "//python/pip_install/private:distribution", ], @@ -24,7 +23,6 @@ filegroup( srcs = [ "//python/pip_install/tools/dependency_resolver:py_srcs", "//python/pip_install/tools/lib:py_srcs", - "//python/pip_install/tools/lock_file_generator:py_srcs", "//python/pip_install/tools/wheel_installer:py_srcs", ], visibility = ["//python/pip_install/private:__pkg__"], diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 68fde0e37..c20fbd4cd 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -257,83 +257,151 @@ A requirements_lock attribute must be specified, or a platform-specific lockfile """) return requirements_txt -# Keep in sync with `_clean_name` in generated requirements.bzl +# Keep in sync with `_clean_pkg_name` in generated bzlmod requirements.bzl def _clean_pkg_name(name): return name.replace("-", "_").replace(".", "_").lower() -def _bzlmod_pkg_aliases(rctx, requirements_txt): +def _bzlmod_pkg_aliases(repo_name, bzl_packages): """Create alias declarations for each python dependency. - The aliases should be appended to the pip_parse repo's BUILD.bazel file. These aliases + The aliases should be appended to the pip_repository BUILD.bazel file. These aliases allow users to use requirement() without needed a corresponding `use_repo()` for each dep when using bzlmod. Args: - rctx: the repository context - requirements_txt: label to the requirements lock file + repo_name: the repository name of the parent that is visible to the users. + bzl_packages: the list of packages to setup. """ - requirements = parse_requirements(rctx.read(requirements_txt)).requirements - build_content = "" - for requirement in requirements: + for name in bzl_packages: build_content += """\ alias( name = "{name}_pkg", - actual = "@{repo_prefix}{dep}//:pkg", + actual = "@{repo_name}_{dep}//:pkg", ) alias( name = "{name}_whl", - actual = "@{repo_prefix}{dep}//:whl", + actual = "@{repo_name}_{dep}//:whl", ) alias( name = "{name}_data", - actual = "@{repo_prefix}{dep}//:data", + actual = "@{repo_name}_{dep}//:data", ) alias( name = "{name}_dist_info", - actual = "@{repo_prefix}{dep}//:dist_info", + actual = "@{repo_name}_{dep}//:dist_info", ) """.format( - name = _clean_pkg_name(requirement[0]), - repo_prefix = rctx.attr.repo_prefix, - dep = _clean_pkg_name(requirement[0]), + name = name, + repo_name = repo_name, + dep = name, ) return build_content -def _pip_repository_impl(rctx): - python_interpreter = _resolve_python_interpreter(rctx) +def _pip_repository_bzlmod_impl(rctx): + requirements_txt = locked_requirements_label(rctx, rctx.attr) + content = rctx.read(requirements_txt) + parsed_requirements_txt = parse_requirements(content) + + packages = [(_clean_pkg_name(name), requirement) for name, requirement in parsed_requirements_txt.requirements] + + bzl_packages = sorted([name for name, _ in packages]) + + repo_name = rctx.attr.name.split("~")[-1] - # Write the annotations file to pass to the wheel maker - annotations = {package: json.decode(data) for (package, data) in rctx.attr.annotations.items()} - annotations_file = rctx.path("annotations.json") - rctx.file(annotations_file, json.encode_indent(annotations, indent = " " * 4)) + build_contents = _BUILD_FILE_CONTENTS + _bzlmod_pkg_aliases(repo_name, bzl_packages) + rctx.file("BUILD.bazel", build_contents) + rctx.template("requirements.bzl", rctx.attr._template, substitutions = { + "%%ALL_REQUIREMENTS%%": _format_repr_list([ + "@{}//:{}_pkg".format(repo_name, p) + for p in bzl_packages + ]), + "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([ + "@{}//:{}_whl".format(repo_name, p) + for p in bzl_packages + ]), + "%%NAME%%": rctx.attr.name, + "%%REQUIREMENTS_LOCK%%": str(requirements_txt), + }) + +pip_repository_bzlmod_attrs = { + "requirements_darwin": attr.label( + allow_single_file = True, + doc = "Override the requirements_lock attribute when the host platform is Mac OS", + ), + "requirements_linux": attr.label( + allow_single_file = True, + doc = "Override the requirements_lock attribute when the host platform is Linux", + ), + "requirements_lock": attr.label( + allow_single_file = True, + doc = """ +A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead +of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that +wheels are fetched/built only for the targets specified by 'build/run/test'. +""", + ), + "requirements_windows": attr.label( + allow_single_file = True, + doc = "Override the requirements_lock attribute when the host platform is Windows", + ), + "_template": attr.label( + default = ":pip_repository_requirements_bzlmod.bzl.tmpl", + ), +} + +pip_repository_bzlmod = repository_rule( + attrs = pip_repository_bzlmod_attrs, + doc = """A rule for bzlmod pip_repository creation. Intended for private use only.""", + implementation = _pip_repository_bzlmod_impl, +) + +def _pip_repository_impl(rctx): requirements_txt = locked_requirements_label(rctx, rctx.attr) - args = [ - python_interpreter, - "-m", - "python.pip_install.tools.lock_file_generator.lock_file_generator", - "--requirements_lock", - rctx.path(requirements_txt), - "--requirements_lock_label", - str(requirements_txt), - # pass quiet and timeout args through to child repos. - "--quiet", - str(rctx.attr.quiet), - "--timeout", - str(rctx.attr.timeout), - "--annotations", - annotations_file, - "--bzlmod", - str(rctx.attr.bzlmod).lower(), + content = rctx.read(requirements_txt) + parsed_requirements_txt = parse_requirements(content) + + packages = [(_clean_pkg_name(name), requirement) for name, requirement in parsed_requirements_txt.requirements] + + bzl_packages = sorted([name for name, _ in packages]) + + imports = [ + 'load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library")', ] - args += ["--python_interpreter", _get_python_interpreter_attr(rctx)] + annotations = {} + for pkg, annotation in rctx.attr.annotations.items(): + filename = "{}.annotation.json".format(_clean_pkg_name(pkg)) + rctx.file(filename, json.encode_indent(json.decode(annotation))) + annotations[pkg] = "@{name}//:{filename}".format(name = rctx.attr.name, filename = filename) + + tokenized_options = [] + for opt in parsed_requirements_txt.options: + for p in opt.split(" "): + tokenized_options.append(p) + + options = tokenized_options + rctx.attr.extra_pip_args + + config = { + "download_only": rctx.attr.download_only, + "enable_implicit_namespace_pkgs": rctx.attr.enable_implicit_namespace_pkgs, + "environment": rctx.attr.environment, + "extra_pip_args": options, + "isolated": use_isolated(rctx, rctx.attr), + "pip_data_exclude": rctx.attr.pip_data_exclude, + "python_interpreter": _get_python_interpreter_attr(rctx), + "quiet": rctx.attr.quiet, + "repo": rctx.attr.name, + "repo_prefix": "{}_".format(rctx.attr.name), + "timeout": rctx.attr.timeout, + } + if rctx.attr.python_interpreter_target: args += ["--python_interpreter_target", str(rctx.attr.python_interpreter_target)] if rctx.attr.pip_platform_definitions: @@ -358,13 +426,29 @@ def _pip_repository_impl(rctx): if result.return_code: fail("rules_python failed: %s (%s)" % (result.stdout, result.stderr)) - # We need a BUILD file to load the generated requirements.bzl - build_contents = _BUILD_FILE_CONTENTS - - if rctx.attr.bzlmod: - build_contents += _bzlmod_pkg_aliases(rctx, requirements_txt) - - rctx.file("BUILD.bazel", build_contents + "\n# The requirements.bzl file was generated by running:\n# " + " ".join([str(a) for a in args])) + rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) + rctx.template("requirements.bzl", rctx.attr._template, substitutions = { + "%%ALL_REQUIREMENTS%%": _format_repr_list([ + "@{}_{}//:pkg".format(rctx.attr.name, p) + for p in bzl_packages + ]), + "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([ + "@{}_{}//:whl".format(rctx.attr.name, p) + for p in bzl_packages + ]), + "%%ANNOTATIONS%%": _format_dict(_repr_dict(annotations)), + "%%CONFIG%%": _format_dict(_repr_dict(config)), + "%%EXTRA_PIP_ARGS%%": json.encode(options), + "%%IMPORTS%%": "\n".join(sorted(imports)), + "%%NAME%%": rctx.attr.name, + "%%PACKAGES%%": _format_repr_list( + [ + ("{}_{}".format(rctx.attr.name, p), r) + for p, r in packages + ], + ), + "%%REQUIREMENTS_LOCK%%": str(requirements_txt), + }) return @@ -457,12 +541,6 @@ pip_repository_attrs = { "annotations": attr.string_dict( doc = "Optional annotations to apply to packages", ), - "bzlmod": attr.bool( - default = False, - doc = """Whether this repository rule is invoked under bzlmod, in which case -we do not create the install_deps() macro. -""", - ), "pip_platform_definitions": attr.label_keyed_string_dict( doc = """ A map of select keys to platform definitions in the form ---" @@ -488,6 +566,9 @@ wheels are fetched/built only for the targets specified by 'build/run/test'. allow_single_file = True, doc = "Override the requirements_lock attribute when the host platform is Windows", ), + "_template": attr.label( + default = ":pip_repository_requirements.bzl.tmpl", + ), } pip_repository_attrs.update(**common_attrs) @@ -644,6 +725,22 @@ def package_annotation( srcs_exclude_glob = srcs_exclude_glob, )) +# pip_repository implementation + +def _format_list(items): + return "[{}]".format(", ".join(items)) + +def _format_repr_list(strings): + return _format_list( + [repr(s) for s in strings], + ) + +def _repr_dict(items): + return {k: repr(v) for k, v in items.items()} + +def _format_dict(items): + return "{{{}}}".format(", ".join(sorted(['"{}": {}'.format(k, v) for k, v in items.items()]))) + _PLATFORM_ALIAS_TMPL = """ alias( name = "pkg", diff --git a/python/pip_install/pip_repository_requirements.bzl.tmpl b/python/pip_install/pip_repository_requirements.bzl.tmpl new file mode 100644 index 000000000..bf6a05362 --- /dev/null +++ b/python/pip_install/pip_repository_requirements.bzl.tmpl @@ -0,0 +1,52 @@ +"""Starlark representation of locked requirements. + +@generated by rules_python pip_parse repository rule +from %%REQUIREMENTS_LOCK%% +""" + +%%IMPORTS%% + +all_requirements = %%ALL_REQUIREMENTS%% + +all_whl_requirements = %%ALL_WHL_REQUIREMENTS%% + +_packages = %%PACKAGES%% +_config = %%CONFIG%% +_annotations = %%ANNOTATIONS%% + +def _clean_name(name): + return name.replace("-", "_").replace(".", "_").lower() + +def requirement(name): + return "@%%NAME%%_" + _clean_name(name) + "//:pkg" + +def whl_requirement(name): + return "@%%NAME%%_" + _clean_name(name) + "//:whl" + +def data_requirement(name): + return "@%%NAME%%_" + _clean_name(name) + "//:data" + +def dist_info_requirement(name): + return "@%%NAME%%_" + _clean_name(name) + "//:dist_info" + +def entry_point(pkg, script = None): + if not script: + script = pkg + return "@%%NAME%%_" + _clean_name(pkg) + "//:rules_python_wheel_entry_point_" + script + +def _get_annotation(requirement): + # This expects to parse `setuptools==58.2.0 --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11` + # down to `setuptools`. + name = requirement.split(" ")[0].split("=")[0].split("[")[0] + return _annotations.get(name) + +def install_deps(**whl_library_kwargs): + whl_config = dict(_config) + whl_config.update(whl_library_kwargs) + for name, requirement in _packages: + whl_library( + name = name, + requirement = requirement, + annotation = _get_annotation(requirement), + **whl_config + ) diff --git a/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl b/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl new file mode 100644 index 000000000..462829d07 --- /dev/null +++ b/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl @@ -0,0 +1,24 @@ +"""Starlark representation of locked requirements. + +@generated by rules_python pip_parse repository rule +from %%REQUIREMENTS_LOCK%%. +""" + +all_requirements = %%ALL_REQUIREMENTS%% + +all_whl_requirements = %%ALL_WHL_REQUIREMENTS%% + +def _clean_name(name): + return name.replace("-", "_").replace(".", "_").lower() + +def requirement(name): + return "@@%%NAME%%//:" + _clean_name(name) + "_pkg" + +def whl_requirement(name): + return "@@%%NAME%%//:" + _clean_name(name) + "_whl" + +def data_requirement(name): + return "@@%%NAME%%//:" + _clean_name(name) + "_data" + +def dist_info_requirement(name): + return "@@%%NAME%%//:" + _clean_name(name) + "_dist_info" diff --git a/python/pip_install/private/srcs.bzl b/python/pip_install/private/srcs.bzl index 57644f612..f3064a3ae 100644 --- a/python/pip_install/private/srcs.bzl +++ b/python/pip_install/private/srcs.bzl @@ -13,8 +13,6 @@ PIP_INSTALL_PY_SRCS = [ "@rules_python//python/pip_install/tools/lib:annotation.py", "@rules_python//python/pip_install/tools/lib:arguments.py", "@rules_python//python/pip_install/tools/lib:bazel.py", - "@rules_python//python/pip_install/tools/lock_file_generator:__init__.py", - "@rules_python//python/pip_install/tools/lock_file_generator:lock_file_generator.py", "@rules_python//python/pip_install/tools/wheel_installer:namespace_pkgs.py", "@rules_python//python/pip_install/tools/wheel_installer:wheel.py", "@rules_python//python/pip_install/tools/wheel_installer:wheel_installer.py", diff --git a/python/pip_install/tools/lock_file_generator/BUILD.bazel b/python/pip_install/tools/lock_file_generator/BUILD.bazel deleted file mode 100644 index 804f36a94..000000000 --- a/python/pip_install/tools/lock_file_generator/BUILD.bazel +++ /dev/null @@ -1,50 +0,0 @@ -load("//python:defs.bzl", "py_binary", "py_library", "py_test") -load("//python/pip_install:repositories.bzl", "requirement") - -py_library( - name = "lib", - srcs = [ - "lock_file_generator.py", - ], - deps = [ - "//python/pip_install/tools/lib", - requirement("pip"), - ], -) - -py_binary( - name = "lock_file_generator", - srcs = [ - "lock_file_generator.py", - ], - deps = [":lib"], -) - -py_test( - name = "lock_file_generator_test", - size = "small", - srcs = [ - "lock_file_generator_test.py", - ], - deps = [ - ":lib", - ], -) - -filegroup( - name = "distribution", - srcs = glob( - ["*"], - exclude = ["*_test.py"], - ), - visibility = ["//python/pip_install:__subpackages__"], -) - -filegroup( - name = "py_srcs", - srcs = glob( - include = ["**/*.py"], - exclude = ["**/*_test.py"], - ), - visibility = ["//python/pip_install:__subpackages__"], -) diff --git a/python/pip_install/tools/lock_file_generator/__init__.py b/python/pip_install/tools/lock_file_generator/__init__.py deleted file mode 100644 index bbdfb4c58..000000000 --- a/python/pip_install/tools/lock_file_generator/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/python/pip_install/tools/lock_file_generator/lock_file_generator_test.py b/python/pip_install/tools/lock_file_generator/lock_file_generator_test.py deleted file mode 100644 index be244b1c0..000000000 --- a/python/pip_install/tools/lock_file_generator/lock_file_generator_test.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import json -import tempfile -import unittest -from pathlib import Path -from textwrap import dedent - -from pip._internal.req.req_install import InstallRequirement - -from python.pip_install.tools.lock_file_generator import lock_file_generator - - -class TestParseRequirementsToBzl(unittest.TestCase): - maxDiff = None - - def test_generated_requirements_bzl(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - requirements_lock = Path(temp_dir) / "requirements.txt" - comments_and_flags = "#comment\n--require-hashes True\n" - requirement_string = "foo==0.0.0 --hash=sha256:hashofFoowhl" - requirements_lock.write_bytes( - bytes(comments_and_flags + requirement_string, encoding="utf-8") - ) - args = argparse.Namespace() - args.requirements_lock = str(requirements_lock.resolve()) - args.repo = ("pip_parsed_deps_pypi__",) - args.repo_prefix = "pip_parsed_deps_pypi__" - extra_pip_args = ["--index-url=pypi.org/simple"] - pip_data_exclude = ["**.foo"] - args.extra_pip_args = json.dumps({"arg": extra_pip_args}) - args.pip_data_exclude = json.dumps({"arg": pip_data_exclude}) - args.python_interpreter = "/custom/python3" - args.python_interpreter_target = "@custom_python//:exec" - args.environment = json.dumps({"arg": {}}) - whl_library_args = lock_file_generator.parse_whl_library_args(args) - contents = lock_file_generator.generate_parsed_requirements_contents( - requirements_lock=args.requirements_lock, - repo=args.repo, - repo_prefix=args.repo_prefix, - whl_library_args=whl_library_args, - ) - library_target = "@pip_parsed_deps_pypi__foo//:pkg" - whl_target = "@pip_parsed_deps_pypi__foo//:whl" - all_requirements = 'all_requirements = ["{library_target}"]'.format( - library_target=library_target - ) - all_whl_requirements = 'all_whl_requirements = ["{whl_target}"]'.format( - whl_target=whl_target - ) - self.assertIn(all_requirements, contents, contents) - self.assertIn(all_whl_requirements, contents, contents) - self.assertIn(requirement_string, contents, contents) - all_flags = extra_pip_args + ["--require-hashes", "True"] - self.assertIn( - "'extra_pip_args': {}".format(repr(all_flags)), contents, contents - ) - self.assertIn( - "'pip_data_exclude': {}".format(repr(pip_data_exclude)), - contents, - contents, - ) - self.assertIn("'python_interpreter': '/custom/python3'", contents, contents) - self.assertIn( - "'python_interpreter_target': '@custom_python//:exec'", - contents, - contents, - ) - # Assert it gets set to an empty dict by default. - self.assertIn("'environment': {}", contents, contents) - - def test_parse_install_requirements_with_args(self): - # Test requirements files with varying arguments - for requirement_args in ("", "--index-url https://index.python.com"): - with tempfile.TemporaryDirectory() as temp_dir: - requirements_lock = Path(temp_dir) / "requirements.txt" - requirements_lock.write_text( - dedent( - """\ - {} - - wheel==0.37.1 \\ - --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a \\ - --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4 - # via -r requirements.in - setuptools==58.2.0 \\ - --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11 \ - --hash=sha256:2c55bdb85d5bb460bd2e3b12052b677879cffcf46c0c688f2e5bf51d36001145 - # via -r requirements.in - """.format( - requirement_args - ) - ) - ) - - install_req_and_lines = lock_file_generator.parse_install_requirements( - str(requirements_lock), ["-v"] - ) - - # There should only be two entries for the two requirements - self.assertEqual(len(install_req_and_lines), 2) - - # The first index in each tuple is expected to be an `InstallRequirement` object - self.assertIsInstance(install_req_and_lines[0][0], InstallRequirement) - self.assertIsInstance(install_req_and_lines[1][0], InstallRequirement) - - # Ensure the requirements text is correctly parsed with the trailing arguments - self.assertTupleEqual( - install_req_and_lines[0][1:], - ( - "wheel==0.37.1 --hash=sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a --hash=sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4", - ), - ) - self.assertTupleEqual( - install_req_and_lines[1][1:], - ( - "setuptools==58.2.0 --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11 --hash=sha256:2c55bdb85d5bb460bd2e3b12052b677879cffcf46c0c688f2e5bf51d36001145", - ), - ) - - def test_parse_install_requirements_pinned_direct_reference(self): - # Test PEP-440 direct references - with tempfile.TemporaryDirectory() as temp_dir: - requirements_lock = Path(temp_dir) / "requirements.txt" - requirements_lock.write_text( - dedent( - """\ - onnx @ https://files.pythonhosted.org/packages/24/93/f5b001dc0f5de84ce049a34ff382032cd9478e1080aa6ac48470fa810577/onnx-1.11.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl \ - --hash=sha256:67c6d2654c1c203e5c839a47900b51f588fd0de71bbd497fb193d30a0b3ec1e9 - """ - ) - ) - - install_req_and_lines = lock_file_generator.parse_install_requirements( - str(requirements_lock), ["-v"] - ) - - self.assertEqual(len(install_req_and_lines), 1) - self.assertEqual(install_req_and_lines[0][0].name, "onnx") - - self.assertTupleEqual( - install_req_and_lines[0][1:], - ( - "onnx @ https://files.pythonhosted.org/packages/24/93/f5b001dc0f5de84ce049a34ff382032cd9478e1080aa6ac48470fa810577/onnx-1.11.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl --hash=sha256:67c6d2654c1c203e5c839a47900b51f588fd0de71bbd497fb193d30a0b3ec1e9", - ), - ) - - -if __name__ == "__main__": - unittest.main()