add initial plumbing to pyInfo

python/BUILD.bazel

@@ -51,6 +51,11 @@ bzl_library(
     srcs = ["current_py_toolchain.bzl"],
 )
 
+bzl_library(
+    name = "py_precompiler_toolchain_bzl",
+    srcs = ["py_precompiler_toolchain.bzl"],
+)
+
 bzl_library(
     name = "defs_bzl",
     srcs = [
@@ -63,6 +68,7 @@ bzl_library(
         ":py_import_bzl",
         ":py_info_bzl",
         ":py_library_bzl",
+        ":py_precompiler_toolchain_bzl",
         ":py_runtime_bzl",
         ":py_runtime_info_bzl",
         ":py_runtime_pair_bzl",
@@ -290,6 +296,9 @@ alias(
     actual = "@bazel_tools//tools/python:toolchain_type",
 )
 
+# Toolchain type for Python rules precompiler
+toolchain_type(name = "precompiler_toolchain_type")
+
 # Definitions for a Python toolchain that, at execution time, attempts to detect
 # a platform runtime having the appropriate major Python version. Consider this
 # a toolchain of last resort.

python/defs.bzl

@@ -22,6 +22,7 @@ load("//python:py_runtime_info.bzl", internal_PyRuntimeInfo = "PyRuntimeInfo")
 load("//python:py_runtime_pair.bzl", _py_runtime_pair = "py_runtime_pair")
 load("//python:py_test.bzl", _py_test = "py_test")
 load(":current_py_toolchain.bzl", _current_py_toolchain = "current_py_toolchain")
+load(":py_precompiler_toolchain.bzl", _py_precompiler_toolchain = "py_precompiler_toolchain")
 load(":py_import.bzl", _py_import = "py_import")
 
 # Patching placeholder: end of loads
@@ -32,6 +33,8 @@ PyRuntimeInfo = internal_PyRuntimeInfo
 
 current_py_toolchain = _current_py_toolchain
 
+py_precompiler_toolchain = _py_precompiler_toolchain
+
 py_import = _py_import
 
 # Re-exports of Starlark-defined symbols in @bazel_tools//tools/python.

python/private/common/attributes.bzl

@@ -183,6 +183,12 @@ Targets that only provide data files used at runtime belong in the `data`
 attribute.
 """,
         ),
+        # optional attribute, default being auto
+        # determines what mode to compile py file to pyc
+        "pyc_mode": attr.string(
+            values = ["auto", "py_only", "pyc_only", "pyc_checked_hash", "pyc_unchecked_hash"],
+            default = "auto",
+        ),
         # Required attribute, but details vary by rule.
         # Use create_srcs_attr to create one.
         "srcs": None,

python/private/common/common.bzl

@@ -53,6 +53,7 @@ def create_binary_semantics_struct(
         get_native_deps_user_link_flags,
         get_stamp_flag,
         maybe_precompile,
+        pyc_mode,
         should_build_native_deps_dso,
         should_create_init_files,
         should_include_build_data):
@@ -94,6 +95,9 @@ def create_binary_semantics_struct(
         maybe_precompile: Callable that may optional precompile the input `.py`
             sources and returns the full set of desired outputs derived from
             the source files (e.g., both py and pyc, only one of them, etc).
+        pyc_mode: str mode of precompilation for the input.py. 
+            possible values passed in can be found in _PYC_MODE_VALUES
+            See PyInfo for details regarding the different modes
         should_build_native_deps_dso: Callable that returns bool; True if
             building a native deps DSO is supported, False if not.
         should_create_init_files: Callable that returns bool; True if
@@ -119,6 +123,7 @@ def create_binary_semantics_struct(
         get_native_deps_user_link_flags = get_native_deps_user_link_flags,
         get_stamp_flag = get_stamp_flag,
         maybe_precompile = maybe_precompile,
+        pyc_mode = pyc_mode,
         should_build_native_deps_dso = should_build_native_deps_dso,
         should_create_init_files = should_create_init_files,
         should_include_build_data = should_include_build_data,
@@ -128,7 +133,8 @@ def create_library_semantics_struct(
         *,
         get_cc_info_for_library,
         get_imports,
-        maybe_precompile):
+        maybe_precompile,
+        pyc_mode):
     """Create a `LibrarySemantics` struct.
 
     Call this instead of a raw call to `struct(...)`; it'll help ensure all
@@ -139,6 +145,7 @@ def create_library_semantics_struct(
             see py_library_impl for arg details.
         get_imports: Callable; see create_binary_semantics_struct.
         maybe_precompile: Callable; see create_binary_semantics_struct.
+        pyc_mode: mode for precompiling pyc files; see PyInfo for details
     Returns:
         a `LibrarySemantics` struct.
     """
@@ -147,6 +154,7 @@ def create_library_semantics_struct(
         get_cc_info_for_library = get_cc_info_for_library,
         get_imports = get_imports,
         maybe_precompile = maybe_precompile,
+        pyc_mode = pyc_mode,
     )
 
 def create_cc_details_struct(
@@ -356,6 +364,7 @@ def create_py_info(ctx, *, direct_sources, imports):
     uses_shared_libraries = False
     has_py2_only_sources = ctx.attr.srcs_version in ("PY2", "PY2ONLY")
     has_py3_only_sources = ctx.attr.srcs_version in ("PY3", "PY3ONLY")
+    pyc_mode = "auto"
     transitive_sources_depsets = []  # list of depsets
     transitive_sources_files = []  # list of Files
     for target in ctx.attr.deps:
@@ -366,6 +375,7 @@ def create_py_info(ctx, *, direct_sources, imports):
             uses_shared_libraries = uses_shared_libraries or info.uses_shared_libraries
             has_py2_only_sources = has_py2_only_sources or info.has_py2_only_sources
             has_py3_only_sources = has_py3_only_sources or info.has_py3_only_sources
+            pyc_mode = pyc_mode or info.pyc_mode
         else:
             # TODO(b/228692666): Remove this once non-PyInfo targets are no
             # longer supported in `deps`.

python/private/common/common_bazel.bzl

@@ -50,22 +50,47 @@ def collect_cc_info(ctx, extra_deps = []):
 
     return _cc_common.merge_cc_infos(cc_infos = cc_infos)
 
-def maybe_precompile(ctx, srcs):
+def maybe_precompile(ctx, srcs, pyc_mode):
     """Computes all the outputs (maybe precompiled) from the input srcs.
 
     See create_binary_semantics_struct for details about this function.
 
     Args:
         ctx: Rule ctx.
         srcs: List of Files; the inputs to maybe precompile.
+        pyc_mode: mode to determine compilation process for pycs and what outputs will be returned
 
     Returns:
         List of Files; the desired output files derived from the input sources.
     """
-    _ = ctx  # @unused
+    if pyc_mode == "auto" or pyc_mode == "py_only":
+        return srcs
+    compilation_mode = "CHECKED_HASH" if pyc_mode == "pyc_checked_hash" else "UNCHECKED_HASH"
+
+    pycs = []  
+    pyc_file_extension = ".cpython-" + ctx.toolchains["//python:precompiler_toolchain_type"].python_version + ".pyc"
+    for src in srcs:
+        if src.extension != "py" or "site-packages" not in src.path: # TODO @ryang debug replace this prior to submission
+            continue
+
+        basename = src.basename
+        dirname = src.path.rsplit("/", 1)[0].split("/", 2)[2]
+
+        pyc_out = dirname + "/__pycache__/" + basename.replace(".py", pyc_file_extension)
+        pyc = ctx.actions.declare_file(pyc_out)
+
+        ctx.actions.run(
+            inputs = [src],
+            outputs = [pyc],
+            executable = ctx.toolchains["//python:precompiler_toolchain_type"].interpreter,
+            args = [ctx.toolchains["//python:precompiler_toolchain_type"].precompiler_src] + [src.path, pyc.path, compilation_mode],
+            toolchain = "//python:precompiler_toolchain_type",
+        )
+
+    if pyc_mode == "pyc_only":
+        return pycs
 
-    # Precompilation isn't implemented yet, so just return srcs as-is
-    return srcs
+    return srcs + pycs
 
 def get_imports(ctx):
     """Gets the imports from a rule's `imports` attribute.

python/private/common/providers.bzl

@@ -24,6 +24,8 @@ DEFAULT_BOOTSTRAP_TEMPLATE = Label("//python/private:python_bootstrap_template.t
 
 _PYTHON_VERSION_VALUES = ["PY2", "PY3"]
 
+_PYC_MODE_VALUES = ["auto", "py_only", "pyc_only", "pyc_checked_hash", "pyc_unchecked_hash"]
+
 # Helper to make the provider definitions not crash under Bazel 5.4:
 # Bazel 5.4 doesn't support the `init` arg of `provider()`, so we have to
 # not pass that when using Bazel 5.4. But, not passing the `init` arg
@@ -196,7 +198,8 @@ def _PyInfo_init(
         uses_shared_libraries = False,
         imports = depset(),
         has_py2_only_sources = False,
-        has_py3_only_sources = False):
+        has_py3_only_sources = False,
+        pyc_mode = "auto"):
     _check_arg_type("transitive_sources", "depset", transitive_sources)
 
     # Verify it's postorder compatible, but retain is original ordering.
@@ -206,12 +209,18 @@ def _PyInfo_init(
     _check_arg_type("imports", "depset", imports)
     _check_arg_type("has_py2_only_sources", "bool", has_py2_only_sources)
     _check_arg_type("has_py3_only_sources", "bool", has_py3_only_sources)
+    if pyc_mode not in _PYC_MODE_VALUES:
+        fail("invalid compile mode: '{}'; must be one of {}".format(
+            pyc_mode,
+            _PYC_MODE_VALUES,
+        ))
     return {
         "has_py2_only_sources": has_py2_only_sources,
         "has_py3_only_sources": has_py2_only_sources,
         "imports": imports,
         "transitive_sources": transitive_sources,
         "uses_shared_libraries": uses_shared_libraries,
+        "pyc_mode": pyc_mode,
     }
 
 PyInfo, _unused_raw_py_info_ctor = _define_provider(
@@ -236,6 +245,14 @@ as a `.so` file).
 
 This field is currently unused in Bazel and may go away in the future.
 """,
+    "pyc_mode": (
+        "Mode used to determine how to compile python files to bytecode pyc" +
+        "Valid values are `\"auto\"`, no compilation is done" +
+        "`\"py_only\"` no compilation is done and return only .py" +
+        "`\"pyc_only\"` compile py source files to pyc and only return .pyc files not located in pycache" +
+        "`\"pyc_checked_hash\"` return compiled __pycache__/*.pyc files using checked_hash + py files" +
+        "`\"pyc_unchecked_hash\"` return compiled __pycache__/*.pyc files using unchecked_hash + py files"
+    )
     },
 )
 

python/private/common/py_executable.bzl

@@ -64,6 +64,8 @@ _CC_TOOLCHAINS = [config_common.toolchain_type(
     mandatory = False,
 )] if hasattr(config_common, "toolchain_type") else []
 
+_PY_PRECOMPILER_TOOLCHAIN = "//python:precompiler_toolchain_type"
+
 # Non-Google-specific attributes for executables
 # These attributes are for rules that accept Python sources.
 EXECUTABLE_ATTRS = union_attrs(
@@ -127,7 +129,7 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment =
 
     main_py = determine_main(ctx)
     direct_sources = filter_to_py_srcs(ctx.files.srcs)
-    output_sources = semantics.maybe_precompile(ctx, direct_sources)
+    output_sources = semantics.maybe_precompile(ctx, direct_sources, semantics.pyc_mode)
     imports = collect_imports(ctx, semantics)
     executable, files_to_build = _compute_outputs(ctx, output_sources)
 
@@ -854,7 +856,7 @@ def create_base_executable_rule(*, attrs, fragments = [], **kwargs):
     return rule(
         # TODO: add ability to remove attrs, i.e. for imports attr
         attrs = dicts.add(EXECUTABLE_ATTRS, attrs),
-        toolchains = [TOOLCHAIN_TYPE] + _CC_TOOLCHAINS,
+        toolchains = [_PY_PRECOMPILER_TOOLCHAIN, TOOLCHAIN_TYPE] + _CC_TOOLCHAINS,
         fragments = fragments,
         **kwargs
     )

python/private/common/py_executable_bazel.bzl

@@ -123,6 +123,7 @@ def create_binary_semantics_bazel():
         get_native_deps_user_link_flags = _get_native_deps_user_link_flags,
         get_stamp_flag = _get_stamp_flag,
         maybe_precompile = maybe_precompile,
+        pyc_mode = lambda ctx: ctx.attr.pyc_mode,
         should_build_native_deps_dso = lambda ctx: False,
         should_create_init_files = _should_create_init_files,
         should_include_build_data = lambda ctx: False,

python/private/common/py_library.bzl

@@ -57,7 +57,7 @@ def py_library_impl(ctx, *, semantics):
     """
     check_native_allowed(ctx)
     direct_sources = filter_to_py_srcs(ctx.files.srcs)
-    output_sources = depset(semantics.maybe_precompile(ctx, direct_sources))
+    output_sources = depset(semantics.maybe_precompile(ctx, direct_sources, semantics.pyc_mode))
     runfiles = collect_runfiles(ctx = ctx, files = output_sources)
 
     cc_info = semantics.get_cc_info_for_library(ctx)

python/private/common/py_library_rule_bazel.bzl

@@ -28,17 +28,18 @@ _BAZEL_LIBRARY_ATTRS = union_attrs(
     IMPORTS_ATTRS,
 )
 
-def create_library_semantics_bazel():
+def create_library_semantics_bazel(pyc_mode):
     return create_library_semantics_struct(
         get_imports = get_imports,
         maybe_precompile = maybe_precompile,
         get_cc_info_for_library = collect_cc_info,
+        pyc_mode = pyc_mode,
     )
 
 def _py_library_impl(ctx):
     return bazel_py_library_impl(
         ctx,
-        semantics = create_library_semantics_bazel(),
+        semantics = create_library_semantics_bazel(ctx.attr.pyc_mode),
     )
 
 py_library = create_py_library_rule(

python/py_precompiler_toolchain.bzl

@@ -0,0 +1,34 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Toolchain setup for precompiling py files into pyc."""
+def _py_precompiler_toolchain_impl(ctx):
+    return [platform_common.ToolchainInfo(
+        interpreter = ctx.attr.interpreter,
+        precompiler_src = ctx.attr.precompiler_src,
+        python_version = ctx.attr.python_version,
+    )]
+
+py_precompiler_toolchain = rule(
+    doc = """
+    This rule exists so that the precompiler toolchain can be set for precompiling python files to pyc without.
+    This cannot be a regular py_binary, otherwise there will be a circular dependency.
+    """,
+    implementation = _py_precompiler_toolchain_impl,
+    attrs = {
+        "interpreter": attr.label(cfg="exec"),
+        "precompiler_src": attr.label(cfg="exec", default = Label("//tools/precompiler:precompiler_py")),
+        "python_version": attr.string(doc = "python version. I.e 310 for python 3.10. Used for generating magic tag for pyc file", mandatory = True),
+    },
+)

tests/integration/pip_repository_entry_points/BUILD.bazel

@@ -1,5 +1,5 @@
 load("@pip//:requirements.bzl", "entry_point")
-load("@rules_python//python:defs.bzl", "py_test")
+load("@rules_python//python:defs.bzl", "py_test", "py_precompiler_toolchain", "py_library")
 load("@rules_python//python:pip.bzl", "compile_pip_requirements")
 
 # This rule adds a convenient way to update the requirements file.
@@ -16,9 +16,15 @@ pip_sphinx = entry_point(
 
 pip_yamllint = entry_point("yamllint")
 
+py_library(
+    name = "pip_repository_entry_points_test_py",
+    srcs = ["pip_repository_entry_points_test.py"],
+    deps = ["@rules_python//python/runfiles"],
+    pyc_mode = "pyc_only",#"auto",
+)
 py_test(
     name = "pip_parse_entry_points_test",
-    srcs = ["pip_repository_entry_points_test.py"],
+    srcs = ["pip_repository_entry_points_test_py"],
     data = [
         pip_sphinx,
         pip_yamllint,
@@ -28,5 +34,27 @@ py_test(
         "YAMLLINT_ENTRY_POINT": "$(rootpath {})".format(pip_yamllint),
     },
     main = "pip_repository_entry_points_test.py",
-    deps = ["@rules_python//python/runfiles"],
+)
+
+toolchain(
+    name = "precompiler_3_10_python_linux_x86_64_toolchain",
+    target_compatible_with = [
+        # "@rules_python//python/config_settings:is_python_3.10", # doesn't have mandatory ConstraintValueInfo TODO(ryang) remove after debugging
+        "@platforms//os:linux",
+    ],
+    exec_compatible_with = ["@platforms//os:linux"],
+    toolchain = ":precompiler_toolchain_py_310",
+    toolchain_type = "@rules_python//python:precompiler_toolchain_type",
+    visibility = ["//visibility:public"],
+)
+
+sh_binary(
+    name = "python310", # TODO (ryang) figure out how to use the existing python toolchain?
+    srcs = ["wrapper.sh"],
+)
+
+py_precompiler_toolchain(
+    name = "precompiler_toolchain_py_310",
+    interpreter = ":python310",
+    python_version="310",
 )

tests/integration/pip_repository_entry_points/MODULE.bazel

@@ -0,0 +1,6 @@
+###############################################################################
+# Bazel now uses Bzlmod by default to manage external dependencies.
+# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel.
+#
+# For more details, please check https://github.com/bazelbuild/bazel/issues/18958
+###############################################################################