From c58ff912c1713bcb0aa619f6ee6becd92695c6f8 Mon Sep 17 00:00:00 2001 From: Jesse Schalken Date: Sat, 30 Jul 2022 01:51:21 +1000 Subject: [PATCH 1/5] Add option to use "pip download" instead of "pip wheel" --- python/pip_install/extract_wheels/arguments.py | 6 ++++++ .../pip_install/extract_wheels/extract_single_wheel.py | 2 +- python/pip_install/extract_wheels/extract_wheels.py | 2 +- python/pip_install/pip_repository.bzl | 10 ++++++++++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/python/pip_install/extract_wheels/arguments.py b/python/pip_install/extract_wheels/arguments.py index d7d34523a7..ce77bb028e 100644 --- a/python/pip_install/extract_wheels/arguments.py +++ b/python/pip_install/extract_wheels/arguments.py @@ -39,6 +39,12 @@ def parse_common_args(parser: ArgumentParser) -> ArgumentParser: required=True, help="Prefix to prepend to packages", ) + parser.add_argument( + "--download_only", + action="store_true", + help="Use 'pip download' instead of 'pip wheel'. Disables building wheels from source, but allows use of " + "--platform, --python-version, --implementation, and --abi in --extra_pip_args.", + ) return parser diff --git a/python/pip_install/extract_wheels/extract_single_wheel.py b/python/pip_install/extract_wheels/extract_single_wheel.py index c69fbd5958..442a600068 100644 --- a/python/pip_install/extract_wheels/extract_single_wheel.py +++ b/python/pip_install/extract_wheels/extract_single_wheel.py @@ -36,7 +36,7 @@ def main() -> None: pip_args = ( [sys.executable, "-m", "pip"] + (["--isolated"] if args.isolated else []) - + ["wheel", "--no-deps"] + + ["download" if args.download_only else "wheel", "--no-deps"] + deserialized_args["extra_pip_args"] ) diff --git a/python/pip_install/extract_wheels/extract_wheels.py b/python/pip_install/extract_wheels/extract_wheels.py index 7e583eb442..2addaf89fd 100644 --- a/python/pip_install/extract_wheels/extract_wheels.py +++ b/python/pip_install/extract_wheels/extract_wheels.py @@ -80,7 +80,7 @@ def main() -> None: pip_args = ( [sys.executable, "-m", "pip"] + (["--isolated"] if args.isolated else []) - + ["wheel", "-r", args.requirements] + + ["download" if args.download_only else "wheel", "-r", args.requirements] + ["--wheel-dir", os.getcwd()] + deserialized_args["extra_pip_args"] ) diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 706edef5de..d473202f20 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -130,6 +130,9 @@ def _parse_optional_attrs(rctx, args): struct(arg = rctx.attr.extra_pip_args).to_json(), ] + if rctx.attr.download_only: + args.append("--download_only") + if rctx.attr.pip_data_exclude != None: args += [ "--pip_data_exclude", @@ -281,6 +284,13 @@ to control this flag. """, default = True, ), + "download_only": attr.bool( + doc = """ +Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of +--platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different +platform from the host platform. + """ + ), "pip_data_exclude": attr.string_list( doc = "Additional data exclusion parameters to add to the pip packages BUILD file.", ), From 071994653fa1c73824a7118448ba2c95fea40def Mon Sep 17 00:00:00 2001 From: Jesse Schalken Date: Sat, 30 Jul 2022 23:06:39 +1000 Subject: [PATCH 2/5] Update examples/pip_parse_vendored/requirements.bzl with download_only option --- examples/pip_parse_vendored/requirements.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index 038b612309..58c6e7bad1 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -12,7 +12,7 @@ all_requirements = ["@pip_certifi//:pkg", "@pip_charset_normalizer//:pkg", "@pip all_whl_requirements = ["@pip_certifi//:whl", "@pip_charset_normalizer//:whl", "@pip_idna//:whl", "@pip_requests//:whl", "@pip_urllib3//:whl"] _packages = [("pip_certifi", "certifi==2021.10.8 --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"), ("pip_charset_normalizer", "charset-normalizer==2.0.12 --hash=sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597 --hash=sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"), ("pip_idna", "idna==3.3 --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"), ("pip_requests", "requests==2.27.1 --hash=sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61 --hash=sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"), ("pip_urllib3", "urllib3==1.26.9 --hash=sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14 --hash=sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e")] -_config = {"enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": interpreter, "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600} +_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": interpreter, "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600} _annotations = {} def _clean_name(name): From a390752432f3aad5d221f9aac42983e4f952d1b5 Mon Sep 17 00:00:00 2001 From: Jesse Schalken Date: Sat, 30 Jul 2022 23:15:11 +1000 Subject: [PATCH 3/5] Update docs/pip_repository.md --- docs/pip_repository.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/pip_repository.md b/docs/pip_repository.md index ef7f72bcd2..c66d8bfd91 100644 --- a/docs/pip_repository.md +++ b/docs/pip_repository.md @@ -5,10 +5,10 @@ ## pip_repository 
-pip_repository(name, annotations, enable_implicit_namespace_pkgs, environment, extra_pip_args,
-               incremental, isolated, pip_data_exclude, python_interpreter, python_interpreter_target,
-               quiet, repo_prefix, requirements, requirements_darwin, requirements_linux,
-               requirements_lock, requirements_windows, timeout)
+pip_repository(name, annotations, download_only, enable_implicit_namespace_pkgs, environment,
+               extra_pip_args, incremental, isolated, pip_data_exclude, python_interpreter,
+               python_interpreter_target, quiet, repo_prefix, requirements, requirements_darwin,
+               requirements_linux, requirements_lock, requirements_windows, timeout)
A rule for importing `requirements.txt` dependencies into Bazel. @@ -58,6 +58,7 @@ py_binary( | :-------------: | :-------------: | :-------------: | :-------------: | :-------------: | | name | A unique name for this repository. | Name | required | | | annotations | Optional annotations to apply to packages | Dictionary: String -> String | optional | {} | +| download_only | Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of --platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different platform from the host platform. | Boolean | optional | False | | enable_implicit_namespace_pkgs | If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary and py_test targets must specify either legacy_create_init=False or the global Bazel option --incompatible_default_to_explicit_init_py to prevent __init__.py being automatically generated in every directory.

This option is required to support some packages which cannot handle the conversion to pkg-util style. | Boolean | optional | False | | environment | Environment variables to set in the pip subprocess. Can be used to set common variables such as http_proxy, https_proxy and no_proxy Note that pip is run with "--isolated" on the CLI so PIP_<VAR>_<NAME> style env vars are ignored, but env vars that control requests and urllib3 can be passed. | Dictionary: String -> String | optional | {} | | extra_pip_args | Extra arguments to pass on to pip. Must not contain spaces. | List of strings | optional | [] | @@ -81,9 +82,9 @@ py_binary( ## whl_library 
-whl_library(name, annotation, enable_implicit_namespace_pkgs, environment, extra_pip_args, isolated,
-            pip_data_exclude, python_interpreter, python_interpreter_target, quiet, repo, repo_prefix,
-            requirement, timeout)
+whl_library(name, annotation, download_only, enable_implicit_namespace_pkgs, environment,
+            extra_pip_args, isolated, pip_data_exclude, python_interpreter, python_interpreter_target,
+            quiet, repo, repo_prefix, requirement, timeout)
@@ -97,6 +98,7 @@ Instantiated from pip_repository and inherits config options from there. | :-------------: | :-------------: | :-------------: | :-------------: | :-------------: | | name | A unique name for this repository. | Name | required | | | annotation | Optional json encoded file containing annotation to apply to the extracted wheel. See package_annotation | Label | optional | None | +| download_only | Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of --platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different platform from the host platform. | Boolean | optional | False | | enable_implicit_namespace_pkgs | If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary and py_test targets must specify either legacy_create_init=False or the global Bazel option --incompatible_default_to_explicit_init_py to prevent __init__.py being automatically generated in every directory.

This option is required to support some packages which cannot handle the conversion to pkg-util style. | Boolean | optional | False | | environment | Environment variables to set in the pip subprocess. Can be used to set common variables such as http_proxy, https_proxy and no_proxy Note that pip is run with "--isolated" on the CLI so PIP_<VAR>_<NAME> style env vars are ignored, but env vars that control requests and urllib3 can be passed. | Dictionary: String -> String | optional | {} | | extra_pip_args | Extra arguments to pass on to pip. Must not contain spaces. | List of strings | optional | [] | From 88cd68d4b8f1449d38eac3da8e4ed6ff997c21f4 Mon Sep 17 00:00:00 2001 From: Jesse Schalken Date: Sat, 30 Jul 2022 23:19:02 +1000 Subject: [PATCH 4/5] Fix order of common_attrs keys to be lexicographic --- python/pip_install/pip_repository.bzl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index d473202f20..1d1d005149 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -253,6 +253,13 @@ common_env = [ ] common_attrs = { + "download_only": attr.bool( + doc = """ +Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of +--platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different +platform from the host platform. + """ + ), "enable_implicit_namespace_pkgs": attr.bool( default = False, doc = """ @@ -284,13 +291,6 @@ to control this flag. """, default = True, ), - "download_only": attr.bool( - doc = """ -Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of ---platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different -platform from the host platform. - """ - ), "pip_data_exclude": attr.string_list( doc = "Additional data exclusion parameters to add to the pip packages BUILD file.", ), From 1d09c2958e15b6d68354ae15511e903063574f86 Mon Sep 17 00:00:00 2001 From: Jesse Schalken Date: Sat, 30 Jul 2022 23:39:43 +1000 Subject: [PATCH 5/5] Fix buildifier lint --- python/pip_install/pip_repository.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 1d1d005149..eab59f35cd 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -258,7 +258,7 @@ common_attrs = { Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of --platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different platform from the host platform. - """ + """, ), "enable_implicit_namespace_pkgs": attr.bool( default = False,