From ecdfac282d4014e07c214a7780e0f76e2e345d7a Mon Sep 17 00:00:00 2001 From: ryan Date: Sun, 24 Mar 2024 06:37:18 +0000 Subject: [PATCH 1/5] add initial plumbing to pyInfo --- python/private/common/common.bzl | 2 ++ python/private/common/common_bazel.bzl | 5 +++-- python/private/common/providers.bzl | 19 ++++++++++++++++++- python/private/common/py_executable.bzl | 2 +- python/private/common/py_library.bzl | 2 +- 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/python/private/common/common.bzl b/python/private/common/common.bzl index 75c117f5cd..eed7f02a5f 100644 --- a/python/private/common/common.bzl +++ b/python/private/common/common.bzl @@ -356,6 +356,7 @@ def create_py_info(ctx, *, direct_sources, imports): uses_shared_libraries = False has_py2_only_sources = ctx.attr.srcs_version in ("PY2", "PY2ONLY") has_py3_only_sources = ctx.attr.srcs_version in ("PY3", "PY3ONLY") + pyc_mode = "auto" transitive_sources_depsets = [] # list of depsets transitive_sources_files = [] # list of Files for target in ctx.attr.deps: @@ -366,6 +367,7 @@ def create_py_info(ctx, *, direct_sources, imports): uses_shared_libraries = uses_shared_libraries or info.uses_shared_libraries has_py2_only_sources = has_py2_only_sources or info.has_py2_only_sources has_py3_only_sources = has_py3_only_sources or info.has_py3_only_sources + pyc_mode = pyc_mode or info.pyc_mode else: # TODO(b/228692666): Remove this once non-PyInfo targets are no # longer supported in `deps`. diff --git a/python/private/common/common_bazel.bzl b/python/private/common/common_bazel.bzl index 7277337849..936f232a71 100644 --- a/python/private/common/common_bazel.bzl +++ b/python/private/common/common_bazel.bzl @@ -50,7 +50,7 @@ def collect_cc_info(ctx, extra_deps = []): return _cc_common.merge_cc_infos(cc_infos = cc_infos) -def maybe_precompile(ctx, srcs): +def maybe_precompile(ctx, srcs, pyc_mode): """Computes all the outputs (maybe precompiled) from the input srcs. See create_binary_semantics_struct for details about this function. @@ -58,12 +58,13 @@ def maybe_precompile(ctx, srcs): Args: ctx: Rule ctx. srcs: List of Files; the inputs to maybe precompile. + pyc_mode: mode to determine compilation process for pycs and what outputs will be returned Returns: List of Files; the desired output files derived from the input sources. """ _ = ctx # @unused - + _ = pyc_mode # Precompilation isn't implemented yet, so just return srcs as-is return srcs diff --git a/python/private/common/providers.bzl b/python/private/common/providers.bzl index f36a2d12c1..2ac966565e 100644 --- a/python/private/common/providers.bzl +++ b/python/private/common/providers.bzl @@ -24,6 +24,8 @@ DEFAULT_BOOTSTRAP_TEMPLATE = Label("//python/private:python_bootstrap_template.t _PYTHON_VERSION_VALUES = ["PY2", "PY3"] +_PYC_MODE_VALUES = ["auto", "py_only", "pyc_only", "pyc_checked_hash", "pyc_unchecked_hash"] + # Helper to make the provider definitions not crash under Bazel 5.4: # Bazel 5.4 doesn't support the `init` arg of `provider()`, so we have to # not pass that when using Bazel 5.4. But, not passing the `init` arg @@ -196,7 +198,8 @@ def _PyInfo_init( uses_shared_libraries = False, imports = depset(), has_py2_only_sources = False, - has_py3_only_sources = False): + has_py3_only_sources = False, + pyc_mode = "auto"): _check_arg_type("transitive_sources", "depset", transitive_sources) # Verify it's postorder compatible, but retain is original ordering. @@ -206,12 +209,18 @@ def _PyInfo_init( _check_arg_type("imports", "depset", imports) _check_arg_type("has_py2_only_sources", "bool", has_py2_only_sources) _check_arg_type("has_py3_only_sources", "bool", has_py3_only_sources) + if pyc_mode not in _PYC_MODE_VALUES: + fail("invalid compile mode: '{}'; must be one of {}".format( + pyc_mode, + _PYC_MODE_VALUES, + )) return { "has_py2_only_sources": has_py2_only_sources, "has_py3_only_sources": has_py2_only_sources, "imports": imports, "transitive_sources": transitive_sources, "uses_shared_libraries": uses_shared_libraries, + "pyc_mode": pyc_mode, } PyInfo, _unused_raw_py_info_ctor = _define_provider( @@ -236,6 +245,14 @@ as a `.so` file). This field is currently unused in Bazel and may go away in the future. """, + "pyc_mode": ( + "Mode used to determine how to compile python files to bytecode pyc" + + "Valid values are `\"auto\"`, no compilation is done" + + "`\"py_only\"` no compilation is done and return only .py" + + "`\"pyc_only\"` compile py source files to pyc and only return .pyc files not located in pycache" + + "`\"pyc_checked_hash\"` return compiled __pycache__/*.pyc files using checked_hash + py files" + + "`\"pyc_unchecked_hash\"` return compiled __pycache__/*.pyc files using unchecked_hash + py files" + ) }, ) diff --git a/python/private/common/py_executable.bzl b/python/private/common/py_executable.bzl index df739273e4..2e3c3d70c6 100644 --- a/python/private/common/py_executable.bzl +++ b/python/private/common/py_executable.bzl @@ -127,7 +127,7 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment = main_py = determine_main(ctx) direct_sources = filter_to_py_srcs(ctx.files.srcs) - output_sources = semantics.maybe_precompile(ctx, direct_sources) + output_sources = semantics.maybe_precompile(ctx, direct_sources, ctx.attr.pyc_mode) imports = collect_imports(ctx, semantics) executable, files_to_build = _compute_outputs(ctx, output_sources) diff --git a/python/private/common/py_library.bzl b/python/private/common/py_library.bzl index 28ee7bf4b6..58a2749c44 100644 --- a/python/private/common/py_library.bzl +++ b/python/private/common/py_library.bzl @@ -57,7 +57,7 @@ def py_library_impl(ctx, *, semantics): """ check_native_allowed(ctx) direct_sources = filter_to_py_srcs(ctx.files.srcs) - output_sources = depset(semantics.maybe_precompile(ctx, direct_sources)) + output_sources = depset(semantics.maybe_precompile(ctx, direct_sources, ctx.attr.pyc_mode)) runfiles = collect_runfiles(ctx = ctx, files = output_sources) cc_info = semantics.get_cc_info_for_library(ctx) From d708ee2270eaf8c550d68f3678aaf76a605b5f08 Mon Sep 17 00:00:00 2001 From: ryan Date: Sun, 24 Mar 2024 07:16:28 +0000 Subject: [PATCH 2/5] add tool to compile --- python/defs.bzl | 3 ++ python/private/common/common_bazel.bzl | 32 ++++++++++++++++--- python/py_precompiler_toolchain.bzl | 34 ++++++++++++++++++++ tools/private/precompiler/BUILD.bazel | 13 ++++++++ tools/private/precompiler/precompiler.py | 40 ++++++++++++++++++++++++ 5 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 python/py_precompiler_toolchain.bzl create mode 100644 tools/private/precompiler/BUILD.bazel create mode 100644 tools/private/precompiler/precompiler.py diff --git a/python/defs.bzl b/python/defs.bzl index bd89f5b1f2..e94b2a3991 100644 --- a/python/defs.bzl +++ b/python/defs.bzl @@ -22,6 +22,7 @@ load("//python:py_runtime_info.bzl", internal_PyRuntimeInfo = "PyRuntimeInfo") load("//python:py_runtime_pair.bzl", _py_runtime_pair = "py_runtime_pair") load("//python:py_test.bzl", _py_test = "py_test") load(":current_py_toolchain.bzl", _current_py_toolchain = "current_py_toolchain") +load(":py_precompiler_toolchain.bzl", _py_precompiler_toolchain = "py_precompiler_toolchain") load(":py_import.bzl", _py_import = "py_import") # Patching placeholder: end of loads @@ -32,6 +33,8 @@ PyRuntimeInfo = internal_PyRuntimeInfo current_py_toolchain = _current_py_toolchain +py_precompiler_toolchain = _py_precompiler_toolchain + py_import = _py_import # Re-exports of Starlark-defined symbols in @bazel_tools//tools/python. diff --git a/python/private/common/common_bazel.bzl b/python/private/common/common_bazel.bzl index 936f232a71..b298250ebf 100644 --- a/python/private/common/common_bazel.bzl +++ b/python/private/common/common_bazel.bzl @@ -63,10 +63,34 @@ def maybe_precompile(ctx, srcs, pyc_mode): Returns: List of Files; the desired output files derived from the input sources. """ - _ = ctx # @unused - _ = pyc_mode - # Precompilation isn't implemented yet, so just return srcs as-is - return srcs + if pyc_mode == "auto" or pyc_mode == "py_only": + return srcs + compilation_mode = "CHECKED_HASH" if pyc_mode == "pyc_checked_hash" else "UNCHECKED_HASH" + + pycs = [] + pyc_file_extension = ".cpython-" + ctx.toolchains["//python:precompiler_toolchain_type"].python_version + ".pyc" + for src in srcs: + if src.extension != "py" or "site-packages" not in src.path: # TODO @ryang debug replace this prior to submission + continue + + basename = src.basename + dirname = src.path.rsplit("/", 1)[0].split("/", 2)[2] + + pyc_out = dirname + "/__pycache__/" + basename.replace(".py", pyc_file_extension) + pyc = ctx.actions.declare_file(pyc_out) + + ctx.actions.run( + inputs = [src], + outputs = [pyc], + executable = ctx.toolchains["//python:precompiler_toolchain_type"].interpreter, + args = [ctx.toolchains["//python:precompiler_toolchain_type"].precompiler_src] + [src.path, pyc.path, compilation_mode], + toolchain = "//python:precompiler_toolchain_type", + ) + + if pyc_mode == "pyc_only": + return pycs + + return srcs + pycs def get_imports(ctx): """Gets the imports from a rule's `imports` attribute. diff --git a/python/py_precompiler_toolchain.bzl b/python/py_precompiler_toolchain.bzl new file mode 100644 index 0000000000..d5715fba8c --- /dev/null +++ b/python/py_precompiler_toolchain.bzl @@ -0,0 +1,34 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Toolchain setup for precompiling py files into pyc.""" +def _py_precompiler_toolchain_impl(ctx): + return [platform_common.ToolchainInfo( + interpreter = ctx.attr.interpreter, + precompiler_src = ctx.attr.precompiler_src, + python_version = ctx.attr.python_version, + )] + +py_precompiler_toolchain = rule( + doc = """ + This rule exists so that the precompiler toolchain can be set for precompiling python files to pyc without. + This cannot be a regular py_binary, otherwise there will be a circular dependency. + """, + implementation = _py_precompiler_toolchain_impl, + attrs = { + "interpreter": attr.label(cfg="exec"), + "precompiler_src": attr.label(cfg="exec", default = Label("//tools/precompiler:precompiler.py")), + "python_version": attr.string(doc = "python version. I.e 310 for python 3.10. Used for generating magic tag for pyc file", mandatory = True), + }, +) diff --git a/tools/private/precompiler/BUILD.bazel b/tools/private/precompiler/BUILD.bazel new file mode 100644 index 0000000000..35fcf1a14a --- /dev/null +++ b/tools/private/precompiler/BUILD.bazel @@ -0,0 +1,13 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/tools/private/precompiler/precompiler.py b/tools/private/precompiler/precompiler.py new file mode 100644 index 0000000000..7719f4365f --- /dev/null +++ b/tools/private/precompiler/precompiler.py @@ -0,0 +1,40 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A small cli tool for use to precompile py source files + +Args: + src_path: str, source py file path + pyc_path: str, file path where pyc should be written to + invalidation_mode: str, mode to determine compilation based on PycInvalidationMode. Currently one of [UNCHECKED_HASH, CHECKED_HASH] +""" +from sys import argv +from py_compile import compile, PycInvalidationMode + +def main(): + src_path = arg[1] + pyc_path = arg[2] + invalidation_mode = arg[3] + + pyc_invalidation_mode = { + "UNCHECKED_HASH": PycInvalidationMode.UNCHECKED_HASH, + "CHECKED_HASH": PycInvalidationMode.CHECKED_HASH, + } + if invalidation_mode not in pyc_invalidation_mode: + raise RuntimeError(f"{invalidation_mode} is not a valid choice. Choose 1 of {pyc_invalidation_mode.keys()}") + + compile(argv1, cfile=argv[2], invalidation_mode=pyc_invalidation_mode[invalidation_mode]) + +if __name__ == "__main__": + main() \ No newline at end of file From d4e44d22ca194741f55b8d805be80cc8a58e5ce5 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 25 Mar 2024 01:05:06 +0000 Subject: [PATCH 3/5] failing when running bazelisk test //:pip_parse_entry_points_test --- python/BUILD.bazel | 9 +++++++++ python/private/common/attributes.bzl | 3 +++ python/private/common/common.bzl | 10 +++++++++- python/private/common/py_executable.bzl | 2 +- python/private/common/py_executable_bazel.bzl | 1 + python/private/common/py_library.bzl | 2 +- .../private/common/py_library_rule_bazel.bzl | 5 +++-- python/py_precompiler_toolchain.bzl | 1 + .../pip_repository_entry_points/BUILD.bazel | 20 ++++++++++++++++++- .../pip_repository_entry_points/WORKSPACE | 1 + 10 files changed, 48 insertions(+), 6 deletions(-) diff --git a/python/BUILD.bazel b/python/BUILD.bazel index cc6348acef..6b480cc54c 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -51,6 +51,11 @@ bzl_library( srcs = ["current_py_toolchain.bzl"], ) +bzl_library( + name = "py_precompiler_toolchain_bzl", + srcs = ["py_precompiler_toolchain.bzl"], +) + bzl_library( name = "defs_bzl", srcs = [ @@ -63,6 +68,7 @@ bzl_library( ":py_import_bzl", ":py_info_bzl", ":py_library_bzl", + ":py_precompiler_toolchain_bzl", ":py_runtime_bzl", ":py_runtime_info_bzl", ":py_runtime_pair_bzl", @@ -290,6 +296,9 @@ alias( actual = "@bazel_tools//tools/python:toolchain_type", ) +# Toolchain type for Python rules precompiler +toolchain_type(name = "precompiler_toolchain_type") + # Definitions for a Python toolchain that, at execution time, attempts to detect # a platform runtime having the appropriate major Python version. Consider this # a toolchain of last resort. diff --git a/python/private/common/attributes.bzl b/python/private/common/attributes.bzl index 5ddca72a65..1bae2ec0bf 100644 --- a/python/private/common/attributes.bzl +++ b/python/private/common/attributes.bzl @@ -183,6 +183,9 @@ Targets that only provide data files used at runtime belong in the `data` attribute. """, ), + # optional attribute, default being auto + # determines what mode to compile py file to pyc + "pyc_mode": attr.string(default = "auto"), # Required attribute, but details vary by rule. # Use create_srcs_attr to create one. "srcs": None, diff --git a/python/private/common/common.bzl b/python/private/common/common.bzl index eed7f02a5f..7dc1c22506 100644 --- a/python/private/common/common.bzl +++ b/python/private/common/common.bzl @@ -53,6 +53,7 @@ def create_binary_semantics_struct( get_native_deps_user_link_flags, get_stamp_flag, maybe_precompile, + pyc_mode, should_build_native_deps_dso, should_create_init_files, should_include_build_data): @@ -94,6 +95,9 @@ def create_binary_semantics_struct( maybe_precompile: Callable that may optional precompile the input `.py` sources and returns the full set of desired outputs derived from the source files (e.g., both py and pyc, only one of them, etc). + pyc_mode: str mode of precompilation for the input.py. + possible values passed in can be found in _PYC_MODE_VALUES + See PyInfo for details regarding the different modes should_build_native_deps_dso: Callable that returns bool; True if building a native deps DSO is supported, False if not. should_create_init_files: Callable that returns bool; True if @@ -119,6 +123,7 @@ def create_binary_semantics_struct( get_native_deps_user_link_flags = get_native_deps_user_link_flags, get_stamp_flag = get_stamp_flag, maybe_precompile = maybe_precompile, + pyc_mode = pyc_mode, should_build_native_deps_dso = should_build_native_deps_dso, should_create_init_files = should_create_init_files, should_include_build_data = should_include_build_data, @@ -128,7 +133,8 @@ def create_library_semantics_struct( *, get_cc_info_for_library, get_imports, - maybe_precompile): + maybe_precompile, + pyc_mode): """Create a `LibrarySemantics` struct. Call this instead of a raw call to `struct(...)`; it'll help ensure all @@ -139,6 +145,7 @@ def create_library_semantics_struct( see py_library_impl for arg details. get_imports: Callable; see create_binary_semantics_struct. maybe_precompile: Callable; see create_binary_semantics_struct. + pyc_mode: mode for precompiling pyc files; see PyInfo for details Returns: a `LibrarySemantics` struct. """ @@ -147,6 +154,7 @@ def create_library_semantics_struct( get_cc_info_for_library = get_cc_info_for_library, get_imports = get_imports, maybe_precompile = maybe_precompile, + pyc_mode = pyc_mode, ) def create_cc_details_struct( diff --git a/python/private/common/py_executable.bzl b/python/private/common/py_executable.bzl index 2e3c3d70c6..6c606be44a 100644 --- a/python/private/common/py_executable.bzl +++ b/python/private/common/py_executable.bzl @@ -127,7 +127,7 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment = main_py = determine_main(ctx) direct_sources = filter_to_py_srcs(ctx.files.srcs) - output_sources = semantics.maybe_precompile(ctx, direct_sources, ctx.attr.pyc_mode) + output_sources = semantics.maybe_precompile(ctx, direct_sources, semantics.pyc_mode) imports = collect_imports(ctx, semantics) executable, files_to_build = _compute_outputs(ctx, output_sources) diff --git a/python/private/common/py_executable_bazel.bzl b/python/private/common/py_executable_bazel.bzl index ff4e3c1fb0..4d896a76b1 100644 --- a/python/private/common/py_executable_bazel.bzl +++ b/python/private/common/py_executable_bazel.bzl @@ -123,6 +123,7 @@ def create_binary_semantics_bazel(): get_native_deps_user_link_flags = _get_native_deps_user_link_flags, get_stamp_flag = _get_stamp_flag, maybe_precompile = maybe_precompile, + pyc_mode = lambda ctx: ctx.attr.pyc_mode, should_build_native_deps_dso = lambda ctx: False, should_create_init_files = _should_create_init_files, should_include_build_data = lambda ctx: False, diff --git a/python/private/common/py_library.bzl b/python/private/common/py_library.bzl index 58a2749c44..3599bc3ddf 100644 --- a/python/private/common/py_library.bzl +++ b/python/private/common/py_library.bzl @@ -57,7 +57,7 @@ def py_library_impl(ctx, *, semantics): """ check_native_allowed(ctx) direct_sources = filter_to_py_srcs(ctx.files.srcs) - output_sources = depset(semantics.maybe_precompile(ctx, direct_sources, ctx.attr.pyc_mode)) + output_sources = depset(semantics.maybe_precompile(ctx, direct_sources, semantics.pyc_mode)) runfiles = collect_runfiles(ctx = ctx, files = output_sources) cc_info = semantics.get_cc_info_for_library(ctx) diff --git a/python/private/common/py_library_rule_bazel.bzl b/python/private/common/py_library_rule_bazel.bzl index 453abcb816..2ec68dc7f0 100644 --- a/python/private/common/py_library_rule_bazel.bzl +++ b/python/private/common/py_library_rule_bazel.bzl @@ -28,17 +28,18 @@ _BAZEL_LIBRARY_ATTRS = union_attrs( IMPORTS_ATTRS, ) -def create_library_semantics_bazel(): +def create_library_semantics_bazel(pyc_mode): return create_library_semantics_struct( get_imports = get_imports, maybe_precompile = maybe_precompile, get_cc_info_for_library = collect_cc_info, + pyc_mode = pyc_mode, ) def _py_library_impl(ctx): return bazel_py_library_impl( ctx, - semantics = create_library_semantics_bazel(), + semantics = create_library_semantics_bazel(ctx.attr.pyc_mode), ) py_library = create_py_library_rule( diff --git a/python/py_precompiler_toolchain.bzl b/python/py_precompiler_toolchain.bzl index d5715fba8c..0a52705d4b 100644 --- a/python/py_precompiler_toolchain.bzl +++ b/python/py_precompiler_toolchain.bzl @@ -31,4 +31,5 @@ py_precompiler_toolchain = rule( "precompiler_src": attr.label(cfg="exec", default = Label("//tools/precompiler:precompiler.py")), "python_version": attr.string(doc = "python version. I.e 310 for python 3.10. Used for generating magic tag for pyc file", mandatory = True), }, + toolchains = ["//python:precompiler_toolchain_type"], ) diff --git a/tests/integration/pip_repository_entry_points/BUILD.bazel b/tests/integration/pip_repository_entry_points/BUILD.bazel index c39b1f0a2d..a688762b98 100644 --- a/tests/integration/pip_repository_entry_points/BUILD.bazel +++ b/tests/integration/pip_repository_entry_points/BUILD.bazel @@ -1,5 +1,5 @@ load("@pip//:requirements.bzl", "entry_point") -load("@rules_python//python:defs.bzl", "py_test") +load("@rules_python//python:defs.bzl", "py_test", "py_precompiler_toolchain") load("@rules_python//python:pip.bzl", "compile_pip_requirements") # This rule adds a convenient way to update the requirements file. @@ -30,3 +30,21 @@ py_test( main = "pip_repository_entry_points_test.py", deps = ["@rules_python//python/runfiles"], ) + +toolchain( + name = "precompiler_3_10_python_linux_x86_64_toolchain", + target_compatible_with = [ + # "@rules_python//python/config_settings:is_python_3.10", # doesn't have mandatory ConstraintValueInfo TODO(ryang) remove after debugging + "@platforms//os:linux", + ], + exec_compatible_with = ["@platforms//os:linux"], + toolchain = ":precompiler_toolchain_impl", + toolchain_type = "@rules_python//python:precompiler_toolchain_type", + visibility = ["//visibility:public"], +) + +py_precompiler_toolchain( + name = "precompiler_toolchain_impl", + interpreter = ":bin/python3.10", + python_version="310", +) diff --git a/tests/integration/pip_repository_entry_points/WORKSPACE b/tests/integration/pip_repository_entry_points/WORKSPACE index 0ae087b2e0..af4be49b53 100644 --- a/tests/integration/pip_repository_entry_points/WORKSPACE +++ b/tests/integration/pip_repository_entry_points/WORKSPACE @@ -9,6 +9,7 @@ load("@rules_python//python:repositories.bzl", "py_repositories", "python_regist py_repositories() +register_toolchains("//:precompiler_3_10_python_linux_x86_64_toolchain") # This toolchain is explicitly 3.10 while `rules_python` is 3.9 to act as # a regression test, ensuring 3.10 is functional python_register_toolchains( From 8c3ced0173a22c3997106c4f3cab5cc403a8bdad Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 25 Mar 2024 01:23:18 +0000 Subject: [PATCH 4/5] solved toolchain type not found --- python/private/common/py_executable.bzl | 4 +++- .../pip_repository_entry_points/BUILD.bazel | 4 ++-- .../pip_repository_entry_points/MODULE.bazel | 6 ++++++ tools/{private => }/precompiler/BUILD.bazel | 12 +++++++++++- tools/{private => }/precompiler/precompiler.py | 0 5 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 tests/integration/pip_repository_entry_points/MODULE.bazel rename tools/{private => }/precompiler/BUILD.bazel (74%) rename tools/{private => }/precompiler/precompiler.py (100%) diff --git a/python/private/common/py_executable.bzl b/python/private/common/py_executable.bzl index 6c606be44a..d98107361f 100644 --- a/python/private/common/py_executable.bzl +++ b/python/private/common/py_executable.bzl @@ -64,6 +64,8 @@ _CC_TOOLCHAINS = [config_common.toolchain_type( mandatory = False, )] if hasattr(config_common, "toolchain_type") else [] +_PY_PRECOMPILER_TOOLCHAIN = "//python:precompiler_toolchain_type" + # Non-Google-specific attributes for executables # These attributes are for rules that accept Python sources. EXECUTABLE_ATTRS = union_attrs( @@ -854,7 +856,7 @@ def create_base_executable_rule(*, attrs, fragments = [], **kwargs): return rule( # TODO: add ability to remove attrs, i.e. for imports attr attrs = dicts.add(EXECUTABLE_ATTRS, attrs), - toolchains = [TOOLCHAIN_TYPE] + _CC_TOOLCHAINS, + toolchains = [_PY_PRECOMPILER_TOOLCHAIN, TOOLCHAIN_TYPE] + _CC_TOOLCHAINS, fragments = fragments, **kwargs ) diff --git a/tests/integration/pip_repository_entry_points/BUILD.bazel b/tests/integration/pip_repository_entry_points/BUILD.bazel index a688762b98..8a0333c6f1 100644 --- a/tests/integration/pip_repository_entry_points/BUILD.bazel +++ b/tests/integration/pip_repository_entry_points/BUILD.bazel @@ -38,13 +38,13 @@ toolchain( "@platforms//os:linux", ], exec_compatible_with = ["@platforms//os:linux"], - toolchain = ":precompiler_toolchain_impl", + toolchain = ":precompiler_toolchain_py_310", toolchain_type = "@rules_python//python:precompiler_toolchain_type", visibility = ["//visibility:public"], ) py_precompiler_toolchain( - name = "precompiler_toolchain_impl", + name = "precompiler_toolchain_py_310", interpreter = ":bin/python3.10", python_version="310", ) diff --git a/tests/integration/pip_repository_entry_points/MODULE.bazel b/tests/integration/pip_repository_entry_points/MODULE.bazel new file mode 100644 index 0000000000..00bb18361f --- /dev/null +++ b/tests/integration/pip_repository_entry_points/MODULE.bazel @@ -0,0 +1,6 @@ +############################################################################### +# Bazel now uses Bzlmod by default to manage external dependencies. +# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel. +# +# For more details, please check https://github.com/bazelbuild/bazel/issues/18958 +############################################################################### diff --git a/tools/private/precompiler/BUILD.bazel b/tools/precompiler/BUILD.bazel similarity index 74% rename from tools/private/precompiler/BUILD.bazel rename to tools/precompiler/BUILD.bazel index 35fcf1a14a..891ad04b5c 100644 --- a/tools/private/precompiler/BUILD.bazel +++ b/tools/precompiler/BUILD.bazel @@ -10,4 +10,14 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +filegroup( + name = "precompiler_py", + srcs = [ + "precompiler.py", + ], +) \ No newline at end of file diff --git a/tools/private/precompiler/precompiler.py b/tools/precompiler/precompiler.py similarity index 100% rename from tools/private/precompiler/precompiler.py rename to tools/precompiler/precompiler.py From 7aca6b559141294575154facb81134fc7389dc3f Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 25 Mar 2024 04:36:35 +0000 Subject: [PATCH 5/5] not working if anything other than auto is selected --- python/private/common/attributes.bzl | 5 ++++- python/py_precompiler_toolchain.bzl | 3 +-- .../pip_repository_entry_points/BUILD.bazel | 18 ++++++++++++++---- .../pip_repository_entry_points/wrapper.sh | 5 +++++ 4 files changed, 24 insertions(+), 7 deletions(-) create mode 100644 tests/integration/pip_repository_entry_points/wrapper.sh diff --git a/python/private/common/attributes.bzl b/python/private/common/attributes.bzl index 1bae2ec0bf..09530f4194 100644 --- a/python/private/common/attributes.bzl +++ b/python/private/common/attributes.bzl @@ -185,7 +185,10 @@ attribute. ), # optional attribute, default being auto # determines what mode to compile py file to pyc - "pyc_mode": attr.string(default = "auto"), + "pyc_mode": attr.string( + values = ["auto", "py_only", "pyc_only", "pyc_checked_hash", "pyc_unchecked_hash"], + default = "auto", + ), # Required attribute, but details vary by rule. # Use create_srcs_attr to create one. "srcs": None, diff --git a/python/py_precompiler_toolchain.bzl b/python/py_precompiler_toolchain.bzl index 0a52705d4b..abf1c1a58d 100644 --- a/python/py_precompiler_toolchain.bzl +++ b/python/py_precompiler_toolchain.bzl @@ -28,8 +28,7 @@ py_precompiler_toolchain = rule( implementation = _py_precompiler_toolchain_impl, attrs = { "interpreter": attr.label(cfg="exec"), - "precompiler_src": attr.label(cfg="exec", default = Label("//tools/precompiler:precompiler.py")), + "precompiler_src": attr.label(cfg="exec", default = Label("//tools/precompiler:precompiler_py")), "python_version": attr.string(doc = "python version. I.e 310 for python 3.10. Used for generating magic tag for pyc file", mandatory = True), }, - toolchains = ["//python:precompiler_toolchain_type"], ) diff --git a/tests/integration/pip_repository_entry_points/BUILD.bazel b/tests/integration/pip_repository_entry_points/BUILD.bazel index 8a0333c6f1..bd187e0676 100644 --- a/tests/integration/pip_repository_entry_points/BUILD.bazel +++ b/tests/integration/pip_repository_entry_points/BUILD.bazel @@ -1,5 +1,5 @@ load("@pip//:requirements.bzl", "entry_point") -load("@rules_python//python:defs.bzl", "py_test", "py_precompiler_toolchain") +load("@rules_python//python:defs.bzl", "py_test", "py_precompiler_toolchain", "py_library") load("@rules_python//python:pip.bzl", "compile_pip_requirements") # This rule adds a convenient way to update the requirements file. @@ -16,9 +16,15 @@ pip_sphinx = entry_point( pip_yamllint = entry_point("yamllint") +py_library( + name = "pip_repository_entry_points_test_py", + srcs = ["pip_repository_entry_points_test.py"], + deps = ["@rules_python//python/runfiles"], + pyc_mode = "pyc_only",#"auto", +) py_test( name = "pip_parse_entry_points_test", - srcs = ["pip_repository_entry_points_test.py"], + srcs = ["pip_repository_entry_points_test_py"], data = [ pip_sphinx, pip_yamllint, @@ -28,7 +34,6 @@ py_test( "YAMLLINT_ENTRY_POINT": "$(rootpath {})".format(pip_yamllint), }, main = "pip_repository_entry_points_test.py", - deps = ["@rules_python//python/runfiles"], ) toolchain( @@ -43,8 +48,13 @@ toolchain( visibility = ["//visibility:public"], ) +sh_binary( + name = "python310", # TODO (ryang) figure out how to use the existing python toolchain? + srcs = ["wrapper.sh"], +) + py_precompiler_toolchain( name = "precompiler_toolchain_py_310", - interpreter = ":bin/python3.10", + interpreter = ":python310", python_version="310", ) diff --git a/tests/integration/pip_repository_entry_points/wrapper.sh b/tests/integration/pip_repository_entry_points/wrapper.sh new file mode 100644 index 0000000000..38cfab3387 --- /dev/null +++ b/tests/integration/pip_repository_entry_points/wrapper.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -euo pipefail + +/bin/python3.10 $@ \ No newline at end of file