diff --git a/python/private/common/attributes.bzl b/python/private/common/attributes.bzl new file mode 100644 index 000000000..7e28ed9d6 --- /dev/null +++ b/python/private/common/attributes.bzl @@ -0,0 +1,227 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Attributes for Python rules.""" + +load(":common/cc/cc_info.bzl", _CcInfo = "CcInfo") +load(":common/python/common.bzl", "union_attrs") +load(":common/python/providers.bzl", "PyInfo") +load( + ":common/python/semantics.bzl", + "DEPS_ATTR_ALLOW_RULES", + "PLATFORMS_LOCATION", + "SRCS_ATTR_ALLOW_FILES", + "TOOLS_REPO", +) + +PackageSpecificationInfo = _builtins.toplevel.PackageSpecificationInfo + +_STAMP_VALUES = [-1, 0, 1] + +def create_stamp_attr(**kwargs): + return {"stamp": attr.int(values = _STAMP_VALUES, **kwargs)} + +def create_srcs_attr(*, mandatory): + return { + "srcs": attr.label_list( + # Google builds change the set of allowed files. + allow_files = SRCS_ATTR_ALLOW_FILES, + mandatory = mandatory, + # Necessary for --compile_one_dependency to work. + flags = ["DIRECT_COMPILE_TIME_INPUT"], + ), + } + +SRCS_VERSION_ALL_VALUES = ["PY2", "PY2ONLY", "PY2AND3", "PY3", "PY3ONLY"] +SRCS_VERSION_NON_CONVERSION_VALUES = ["PY2AND3", "PY2ONLY", "PY3ONLY"] + +def create_srcs_version_attr(values): + return { + "srcs_version": attr.string( + default = "PY2AND3", + values = values, + ), + } + +def copy_common_binary_kwargs(kwargs): + return { + key: kwargs[key] + for key in BINARY_ATTR_NAMES + if key in kwargs + } + +def copy_common_test_kwargs(kwargs): + return { + key: kwargs[key] + for key in TEST_ATTR_NAMES + if key in kwargs + } + +CC_TOOLCHAIN = { + # NOTE: The `cc_helper.find_cpp_toolchain()` function expects the attribute + # name to be this name. + "_cc_toolchain": attr.label(default = "@" + TOOLS_REPO + "//tools/cpp:current_cc_toolchain"), +} + +# The common "data" attribute definition. +DATA_ATTRS = { + # NOTE: The "flags" attribute is deprecated, but there isn't an alternative + # way to specify that constraints should be ignored. + "data": attr.label_list( + allow_files = True, + flags = ["SKIP_CONSTRAINTS_OVERRIDE"], + ), +} + +NATIVE_RULES_ALLOWLIST_ATTRS = { + "_native_rules_allowlist": attr.label( + default = configuration_field( + fragment = "py", + name = "native_rules_allowlist", + ), + providers = [PackageSpecificationInfo], + ), +} + +# Attributes common to all rules. +COMMON_ATTRS = union_attrs( + DATA_ATTRS, + NATIVE_RULES_ALLOWLIST_ATTRS, + { + # NOTE: This attribute is deprecated and slated for removal. + "distribs": attr.string_list(), + # TODO(b/148103851): This attribute is deprecated and slated for + # removal. + # NOTE: The license attribute is missing in some Java integration tests, + # so fallback to a regular string_list for that case. + # buildifier: disable=attr-license + "licenses": attr.license() if hasattr(attr, "license") else attr.string_list(), + }, + allow_none = True, +) + +# Attributes common to rules accepting Python sources and deps. +PY_SRCS_ATTRS = union_attrs( + { + "deps": attr.label_list( + providers = [[PyInfo], [_CcInfo]], + # TODO(b/228692666): Google-specific; remove these allowances once + # the depot is cleaned up. + allow_rules = DEPS_ATTR_ALLOW_RULES, + ), + # Required attribute, but details vary by rule. + # Use create_srcs_attr to create one. + "srcs": None, + # NOTE: In Google, this attribute is deprecated, and can only + # effectively be PY3 or PY3ONLY. Externally, with Bazel, this attribute + # has a separate story. + # Required attribute, but the details vary by rule. + # Use create_srcs_version_attr to create one. + "srcs_version": None, + }, + allow_none = True, +) + +# Attributes specific to Python executable-equivalent rules. Such rules may not +# accept Python sources (e.g. some packaged-version of a py_test/py_binary), but +# still accept Python source-agnostic settings. +AGNOSTIC_EXECUTABLE_ATTRS = union_attrs( + DATA_ATTRS, + { + "env": attr.string_dict( + doc = """\ +Dictionary of strings; optional; values are subject to `$(location)` and "Make +variable" substitution. + +Specifies additional environment variables to set when the target is executed by +`test` or `run`. +""", + ), + # The value is required, but varies by rule and/or rule type. Use + # create_stamp_attr to create one. + "stamp": None, + }, + allow_none = True, +) + +# Attributes specific to Python test-equivalent executable rules. Such rules may +# not accept Python sources (e.g. some packaged-version of a py_test/py_binary), +# but still accept Python source-agnostic settings. +AGNOSTIC_TEST_ATTRS = union_attrs( + AGNOSTIC_EXECUTABLE_ATTRS, + # Tests have stamping disabled by default. + create_stamp_attr(default = 0), + { + "env_inherit": attr.string_list( + doc = """\ +List of strings; optional + +Specifies additional environment variables to inherit from the external +environment when the test is executed by bazel test. +""", + ), + # TODO(b/176993122): Remove when Bazel automatically knows to run on darwin. + "_apple_constraints": attr.label_list( + default = [ + PLATFORMS_LOCATION + "/os:ios", + PLATFORMS_LOCATION + "/os:macos", + PLATFORMS_LOCATION + "/os:tvos", + PLATFORMS_LOCATION + "/os:visionos", + PLATFORMS_LOCATION + "/os:watchos", + ], + ), + }, +) + +# Attributes specific to Python binary-equivalent executable rules. Such rules may +# not accept Python sources (e.g. some packaged-version of a py_test/py_binary), +# but still accept Python source-agnostic settings. +AGNOSTIC_BINARY_ATTRS = union_attrs( + AGNOSTIC_EXECUTABLE_ATTRS, + create_stamp_attr(default = -1), +) + +# Attribute names common to all Python rules +COMMON_ATTR_NAMES = [ + "compatible_with", + "deprecation", + "distribs", # NOTE: Currently common to all rules, but slated for removal + "exec_compatible_with", + "exec_properties", + "features", + "restricted_to", + "tags", + "target_compatible_with", + # NOTE: The testonly attribute requires careful handling: None/unset means + # to use the `package(default_testonly`) value, which isn't observable + # during the loading phase. + "testonly", + "toolchains", + "visibility", +] + COMMON_ATTRS.keys() + +# Attribute names common to all test=True rules +TEST_ATTR_NAMES = COMMON_ATTR_NAMES + [ + "args", + "size", + "timeout", + "flaky", + "shard_count", + "local", +] + AGNOSTIC_TEST_ATTRS.keys() + +# Attribute names common to all executable=True rules +BINARY_ATTR_NAMES = COMMON_ATTR_NAMES + [ + "args", + "output_licenses", # NOTE: Common to all rules, but slated for removal +] + AGNOSTIC_BINARY_ATTRS.keys() diff --git a/python/private/common/attributes_bazel.bzl b/python/private/common/attributes_bazel.bzl new file mode 100644 index 000000000..f87245d6f --- /dev/null +++ b/python/private/common/attributes_bazel.bzl @@ -0,0 +1,30 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Attributes specific to the Bazel implementation of the Python rules.""" + +IMPORTS_ATTRS = { + "imports": attr.string_list( + doc = """ +List of import directories to be added to the PYTHONPATH. + +Subject to "Make variable" substitution. These import directories will be added +for this rule and all rules that depend on it (note: not the rules this rule +depends on. Each directory will be added to `PYTHONPATH` by `py_binary` rules +that depend on this rule. The strings are repo-runfiles-root relative, + +Absolute paths (paths that start with `/`) and paths that references a path +above the execution root are not allowed and will result in an error. +""", + ), +} diff --git a/python/private/common/common.bzl b/python/private/common/common.bzl new file mode 100644 index 000000000..97ed3e3ee --- /dev/null +++ b/python/private/common/common.bzl @@ -0,0 +1,528 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Various things common to Bazel and Google rule implementations.""" + +load(":common/cc/cc_helper.bzl", "cc_helper") +load( + ":common/python/providers.bzl", + "PyInfo", +) +load( + ":common/python/semantics.bzl", + "NATIVE_RULES_MIGRATION_FIX_CMD", + "NATIVE_RULES_MIGRATION_HELP_URL", + "TOOLS_REPO", +) + +_testing = _builtins.toplevel.testing +_platform_common = _builtins.toplevel.platform_common +_coverage_common = _builtins.toplevel.coverage_common +_py_builtins = _builtins.internal.py_builtins +PackageSpecificationInfo = _builtins.toplevel.PackageSpecificationInfo + +TOOLCHAIN_TYPE = "@" + TOOLS_REPO + "//tools/python:toolchain_type" + +# Extensions without the dot +_PYTHON_SOURCE_EXTENSIONS = ["py"] + +# NOTE: Must stay in sync with the value used in rules_python +_MIGRATION_TAG = "__PYTHON_RULES_MIGRATION_DO_NOT_USE_WILL_BREAK__" + +def create_binary_semantics_struct( + *, + create_executable, + get_cc_details_for_binary, + get_central_uncachable_version_file, + get_coverage_deps, + get_debugger_deps, + get_extra_common_runfiles_for_binary, + get_extra_providers, + get_extra_write_build_data_env, + get_interpreter_path, + get_imports, + get_native_deps_dso_name, + get_native_deps_user_link_flags, + get_stamp_flag, + maybe_precompile, + should_build_native_deps_dso, + should_create_init_files, + should_include_build_data): + """Helper to ensure a semantics struct has all necessary fields. + + Call this instead of a raw call to `struct(...)`; it'll help ensure all + the necessary functions are being correctly provided. + + Args: + create_executable: Callable; creates a binary's executable output. See + py_executable.bzl#py_executable_base_impl for details. + get_cc_details_for_binary: Callable that returns a `CcDetails` struct; see + `create_cc_detail_struct`. + get_central_uncachable_version_file: Callable that returns an optional + Artifact; this artifact is special: it is never cached and is a copy + of `ctx.version_file`; see py_builtins.copy_without_caching + get_coverage_deps: Callable that returns a list of Targets for making + coverage work; only called if coverage is enabled. + get_debugger_deps: Callable that returns a list of Targets that provide + custom debugger support; only called for target-configuration. + get_extra_common_runfiles_for_binary: Callable that returns a runfiles + object of extra runfiles a binary should include. + get_extra_providers: Callable that returns extra providers; see + py_executable.bzl#_create_providers for details. + get_extra_write_build_data_env: Callable that returns a dict[str, str] + of additional environment variable to pass to build data generation. + get_interpreter_path: Callable that returns an optional string, which is + the path to the Python interpreter to use for running the binary. + get_imports: Callable that returns a list of the target's import + paths (from the `imports` attribute, so just the target's own import + path strings, not from dependencies). + get_native_deps_dso_name: Callable that returns a string, which is the + basename (with extension) of the native deps DSO library. + get_native_deps_user_link_flags: Callable that returns a list of strings, + which are any extra linker flags to pass onto the native deps DSO + linking action. + get_stamp_flag: Callable that returns bool of if the --stamp flag was + enabled or not. + maybe_precompile: Callable that may optional precompile the input `.py` + sources and returns the full set of desired outputs derived from + the source files (e.g., both py and pyc, only one of them, etc). + should_build_native_deps_dso: Callable that returns bool; True if + building a native deps DSO is supported, False if not. + should_create_init_files: Callable that returns bool; True if + `__init__.py` files should be generated, False if not. + should_include_build_data: Callable that returns bool; True if + build data should be generated, False if not. + Returns: + A "BinarySemantics" struct. + """ + return struct( + # keep-sorted + create_executable = create_executable, + get_cc_details_for_binary = get_cc_details_for_binary, + get_central_uncachable_version_file = get_central_uncachable_version_file, + get_coverage_deps = get_coverage_deps, + get_debugger_deps = get_debugger_deps, + get_extra_common_runfiles_for_binary = get_extra_common_runfiles_for_binary, + get_extra_providers = get_extra_providers, + get_extra_write_build_data_env = get_extra_write_build_data_env, + get_imports = get_imports, + get_interpreter_path = get_interpreter_path, + get_native_deps_dso_name = get_native_deps_dso_name, + get_native_deps_user_link_flags = get_native_deps_user_link_flags, + get_stamp_flag = get_stamp_flag, + maybe_precompile = maybe_precompile, + should_build_native_deps_dso = should_build_native_deps_dso, + should_create_init_files = should_create_init_files, + should_include_build_data = should_include_build_data, + ) + +def create_library_semantics_struct( + *, + get_cc_info_for_library, + get_imports, + maybe_precompile): + """Create a `LibrarySemantics` struct. + + Call this instead of a raw call to `struct(...)`; it'll help ensure all + the necessary functions are being correctly provided. + + Args: + get_cc_info_for_library: Callable that returns a CcInfo for the library; + see py_library_impl for arg details. + get_imports: Callable; see create_binary_semantics_struct. + maybe_precompile: Callable; see create_binary_semantics_struct. + Returns: + a `LibrarySemantics` struct. + """ + return struct( + # keep sorted + get_cc_info_for_library = get_cc_info_for_library, + get_imports = get_imports, + maybe_precompile = maybe_precompile, + ) + +def create_cc_details_struct( + *, + cc_info_for_propagating, + cc_info_for_self_link, + cc_info_with_extra_link_time_libraries, + extra_runfiles, + cc_toolchain): + """Creates a CcDetails struct. + + Args: + cc_info_for_propagating: CcInfo that is propagated out of the target + by returning it within a PyCcLinkParamsProvider object. + cc_info_for_self_link: CcInfo that is used when linking for the + binary (or its native deps DSO) itself. This may include extra + information that isn't propagating (e.g. a custom malloc) + cc_info_with_extra_link_time_libraries: CcInfo of extra link time + libraries that MUST come after `cc_info_for_self_link` (or possibly + always last; not entirely clear) when passed to + `link.linking_contexts`. + extra_runfiles: runfiles of extra files needed at runtime, usually as + part of `cc_info_with_extra_link_time_libraries`; should be added to + runfiles. + cc_toolchain: CcToolchain that should be used when building. + + Returns: + A `CcDetails` struct. + """ + return struct( + cc_info_for_propagating = cc_info_for_propagating, + cc_info_for_self_link = cc_info_for_self_link, + cc_info_with_extra_link_time_libraries = cc_info_with_extra_link_time_libraries, + extra_runfiles = extra_runfiles, + cc_toolchain = cc_toolchain, + ) + +def create_executable_result_struct(*, extra_files_to_build, output_groups): + """Creates a `CreateExecutableResult` struct. + + This is the return value type of the semantics create_executable function. + + Args: + extra_files_to_build: depset of File; additional files that should be + included as default outputs. + output_groups: dict[str, depset[File]]; additional output groups that + should be returned. + + Returns: + A `CreateExecutableResult` struct. + """ + return struct( + extra_files_to_build = extra_files_to_build, + output_groups = output_groups, + ) + +def union_attrs(*attr_dicts, allow_none = False): + """Helper for combining and building attriute dicts for rules. + + Similar to dict.update, except: + * Duplicate keys raise an error if they aren't equal. This is to prevent + unintentionally replacing an attribute with a potentially incompatible + definition. + * None values are special: They mean the attribute is required, but the + value should be provided by another attribute dict (depending on the + `allow_none` arg). + Args: + *attr_dicts: The dicts to combine. + allow_none: bool, if True, then None values are allowed. If False, + then one of `attrs_dicts` must set a non-None value for keys + with a None value. + + Returns: + dict of attributes. + """ + result = {} + missing = {} + for attr_dict in attr_dicts: + for attr_name, value in attr_dict.items(): + if value == None and not allow_none: + if attr_name not in result: + missing[attr_name] = None + else: + if attr_name in missing: + missing.pop(attr_name) + + if attr_name not in result or result[attr_name] == None: + result[attr_name] = value + elif value != None and result[attr_name] != value: + fail("Duplicate attribute name: '{}': existing={}, new={}".format( + attr_name, + result[attr_name], + value, + )) + + # Else, they're equal, so do nothing. This allows merging dicts + # that both define the same key from a common place. + + if missing and not allow_none: + fail("Required attributes missing: " + csv(missing.keys())) + return result + +def csv(values): + """Convert a list of strings to comma separated value string.""" + return ", ".join(sorted(values)) + +def filter_to_py_srcs(srcs): + """Filters .py files from the given list of files""" + + # TODO(b/203567235): Get the set of recognized extensions from + # elsewhere, as there may be others. e.g. Bazel recognizes .py3 + # as a valid extension. + return [f for f in srcs if f.extension == "py"] + +def collect_imports(ctx, semantics): + return depset(direct = semantics.get_imports(ctx), transitive = [ + dep[PyInfo].imports + for dep in ctx.attr.deps + if PyInfo in dep + ]) + +def collect_runfiles(ctx, files): + """Collects the necessary files from the rule's context. + + This presumes the ctx is for a py_binary, py_test, or py_library rule. + + Args: + ctx: rule ctx + files: depset of extra files to include in the runfiles. + Returns: + runfiles necessary for the ctx's target. + """ + return ctx.runfiles( + transitive_files = files, + # This little arg carries a lot of weight, but because Starlark doesn't + # have a way to identify if a target is just a File, the equivalent + # logic can't be re-implemented in pure-Starlark. + # + # Under the hood, it calls the Java `Runfiles#addRunfiles(ctx, + # DEFAULT_RUNFILES)` method, which is the what the Java implementation + # of the Python rules originally did, and the details of how that method + # works have become relied on in various ways. Specifically, what it + # does is visit the srcs, deps, and data attributes in the following + # ways: + # + # For each target in the "data" attribute... + # If the target is a File, then add that file to the runfiles. + # Otherwise, add the target's **data runfiles** to the runfiles. + # + # Note that, contray to best practice, the default outputs of the + # targets in `data` are *not* added, nor are the default runfiles. + # + # This ends up being important for several reasons, some of which are + # specific to Google-internal features of the rules. + # * For Python executables, we have to use `data_runfiles` to avoid + # conflicts for the build data files. Such files have + # target-specific content, but uses a fixed location, so if a + # binary has another binary in `data`, and both try to specify a + # file for that file path, then a warning is printed and an + # arbitrary one will be used. + # * For rules with _entirely_ different sets of files in data runfiles + # vs default runfiles vs default outputs. For example, + # proto_library: documented behavior of this rule is that putting it + # in the `data` attribute will cause the transitive closure of + # `.proto` source files to be included. This set of sources is only + # in the `data_runfiles` (`default_runfiles` is empty). + # * For rules with a _subset_ of files in data runfiles. For example, + # a certain Google rule used for packaging arbitrary binaries will + # generate multiple versions of a binary (e.g. different archs, + # stripped vs un-stripped, etc) in its default outputs, but only + # one of them in the runfiles; this helps avoid large, unused + # binaries contributing to remote executor input limits. + # + # Unfortunately, the above behavior also results in surprising behavior + # in some cases. For example, simple custom rules that only return their + # files in their default outputs won't have their files included. Such + # cases must either return their files in runfiles, or use `filegroup()` + # which will do so for them. + # + # For each target in "srcs" and "deps"... + # Add the default runfiles of the target to the runfiles. While this + # is desirable behavior, it also ends up letting a `py_library` + # be put in `srcs` and still mostly work. + # TODO(b/224640180): Reject py_library et al rules in srcs. + collect_default = True, + ) + +def create_py_info(ctx, *, direct_sources, imports): + """Create PyInfo provider. + + Args: + ctx: rule ctx. + direct_sources: depset of Files; the direct, raw `.py` sources for the + target. This should only be Python source files. It should not + include pyc files. + imports: depset of strings; the import path values to propagate. + + Returns: + A tuple of the PyInfo instance and a depset of the + transitive sources collected from dependencies (the latter is only + necessary for deprecated extra actions support). + """ + uses_shared_libraries = False + has_py2_only_sources = ctx.attr.srcs_version in ("PY2", "PY2ONLY") + has_py3_only_sources = ctx.attr.srcs_version in ("PY3", "PY3ONLY") + transitive_sources_depsets = [] # list of depsets + transitive_sources_files = [] # list of Files + for target in ctx.attr.deps: + # PyInfo may not be present for e.g. cc_library rules. + if PyInfo in target: + info = target[PyInfo] + transitive_sources_depsets.append(info.transitive_sources) + uses_shared_libraries = uses_shared_libraries or info.uses_shared_libraries + has_py2_only_sources = has_py2_only_sources or info.has_py2_only_sources + has_py3_only_sources = has_py3_only_sources or info.has_py3_only_sources + else: + # TODO(b/228692666): Remove this once non-PyInfo targets are no + # longer supported in `deps`. + files = target.files.to_list() + for f in files: + if f.extension == "py": + transitive_sources_files.append(f) + uses_shared_libraries = ( + uses_shared_libraries or + cc_helper.is_valid_shared_library_artifact(f) + ) + deps_transitive_sources = depset( + direct = transitive_sources_files, + transitive = transitive_sources_depsets, + ) + + # We only look at data to calculate uses_shared_libraries, if it's already + # true, then we don't need to waste time looping over it. + if not uses_shared_libraries: + # Similar to the above, except we only calculate uses_shared_libraries + for target in ctx.attr.data: + # TODO(b/234730058): Remove checking for PyInfo in data once depot + # cleaned up. + if PyInfo in target: + info = target[PyInfo] + uses_shared_libraries = info.uses_shared_libraries + else: + files = target.files.to_list() + for f in files: + uses_shared_libraries = cc_helper.is_valid_shared_library_artifact(f) + if uses_shared_libraries: + break + if uses_shared_libraries: + break + + # TODO(b/203567235): Set `uses_shared_libraries` field, though the Bazel + # docs indicate it's unused in Bazel and may be removed. + py_info = PyInfo( + transitive_sources = depset( + transitive = [deps_transitive_sources, direct_sources], + ), + imports = imports, + # NOTE: This isn't strictly correct, but with Python 2 gone, + # the srcs_version logic is largely defunct, so shouldn't matter in + # practice. + has_py2_only_sources = has_py2_only_sources, + has_py3_only_sources = has_py3_only_sources, + uses_shared_libraries = uses_shared_libraries, + ) + return py_info, deps_transitive_sources + +def create_instrumented_files_info(ctx): + return _coverage_common.instrumented_files_info( + ctx, + source_attributes = ["srcs"], + dependency_attributes = ["deps", "data"], + extensions = _PYTHON_SOURCE_EXTENSIONS, + ) + +def create_output_group_info(transitive_sources, extra_groups): + return OutputGroupInfo( + compilation_prerequisites_INTERNAL_ = transitive_sources, + compilation_outputs = transitive_sources, + **extra_groups + ) + +def maybe_add_test_execution_info(providers, ctx): + """Adds ExecutionInfo, if necessary for proper test execution. + + Args: + providers: Mutable list of providers; may have ExecutionInfo + provider appended. + ctx: Rule ctx. + """ + + # When built for Apple platforms, require the execution to be on a Mac. + # TODO(b/176993122): Remove when bazel automatically knows to run on darwin. + if target_platform_has_any_constraint(ctx, ctx.attr._apple_constraints): + providers.append(_testing.ExecutionInfo({"requires-darwin": ""})) + +_BOOL_TYPE = type(True) + +def is_bool(v): + return type(v) == _BOOL_TYPE + +def target_platform_has_any_constraint(ctx, constraints): + """Check if target platform has any of a list of constraints. + + Args: + ctx: rule context. + constraints: label_list of constraints. + + Returns: + True if target platform has at least one of the constraints. + """ + for constraint in constraints: + constraint_value = constraint[_platform_common.ConstraintValueInfo] + if ctx.target_platform_has_constraint(constraint_value): + return True + return False + +def check_native_allowed(ctx): + """Check if the usage of the native rule is allowed. + + Args: + ctx: rule context to check + """ + if not ctx.fragments.py.disallow_native_rules: + return + + if _MIGRATION_TAG in ctx.attr.tags: + return + + # NOTE: The main repo name is empty in *labels*, but not in + # ctx.workspace_name + is_main_repo = not bool(ctx.label.workspace_name) + if is_main_repo: + check_label = ctx.label + else: + # package_group doesn't allow @repo syntax, so we work around that + # by prefixing external repos with a fake package path. This also + # makes it easy to enable or disable all external repos. + check_label = Label("@//__EXTERNAL_REPOS__/{workspace}/{package}".format( + workspace = ctx.label.workspace_name, + package = ctx.label.package, + )) + allowlist = ctx.attr._native_rules_allowlist + if allowlist: + allowed = ctx.attr._native_rules_allowlist[PackageSpecificationInfo].contains(check_label) + allowlist_help = str(allowlist.label).replace("@//", "//") + else: + allowed = False + allowlist_help = ("no allowlist specified; all disallowed; specify one " + + "with --python_native_rules_allowlist") + if not allowed: + if ctx.attr.generator_function: + generator = "{generator_function}(name={generator_name}) in {generator_location}".format( + generator_function = ctx.attr.generator_function, + generator_name = ctx.attr.generator_name, + generator_location = ctx.attr.generator_location, + ) + else: + generator = "No generator (called directly in BUILD file)" + + msg = ( + "{target} not allowed to use native.{rule}\n" + + "Generated by: {generator}\n" + + "Allowlist: {allowlist}\n" + + "Migrate to using @rules_python, see {help_url}\n" + + "FIXCMD: {fix_cmd} --target={target} --rule={rule} " + + "--generator_name={generator_name} --location={generator_location}" + ) + fail(msg.format( + target = str(ctx.label).replace("@//", "//"), + rule = _py_builtins.get_rule_name(ctx), + generator = generator, + allowlist = allowlist_help, + generator_name = ctx.attr.generator_name, + generator_location = ctx.attr.generator_location, + help_url = NATIVE_RULES_MIGRATION_HELP_URL, + fix_cmd = NATIVE_RULES_MIGRATION_FIX_CMD, + )) diff --git a/python/private/common/common_bazel.bzl b/python/private/common/common_bazel.bzl new file mode 100644 index 000000000..51b06fb83 --- /dev/null +++ b/python/private/common/common_bazel.bzl @@ -0,0 +1,104 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Common functions that are specific to Bazel rule implementation""" + +load(":common/cc/cc_common.bzl", _cc_common = "cc_common") +load(":common/cc/cc_info.bzl", _CcInfo = "CcInfo") +load(":common/paths.bzl", "paths") +load(":common/python/common.bzl", "is_bool") +load(":common/python/providers.bzl", "PyCcLinkParamsProvider") + +_py_builtins = _builtins.internal.py_builtins + +def collect_cc_info(ctx, extra_deps = []): + """Collect C++ information from dependencies for Bazel. + + Args: + ctx: Rule ctx; must have `deps` attribute. + extra_deps: list of Target to also collect C+ information from. + + Returns: + CcInfo provider of merged information. + """ + deps = ctx.attr.deps + if extra_deps: + deps = list(deps) + deps.extend(extra_deps) + cc_infos = [] + for dep in deps: + if _CcInfo in dep: + cc_infos.append(dep[_CcInfo]) + + if PyCcLinkParamsProvider in dep: + cc_infos.append(dep[PyCcLinkParamsProvider].cc_info) + + return _cc_common.merge_cc_infos(cc_infos = cc_infos) + +def maybe_precompile(ctx, srcs): + """Computes all the outputs (maybe precompiled) from the input srcs. + + See create_binary_semantics_struct for details about this function. + + Args: + ctx: Rule ctx. + srcs: List of Files; the inputs to maybe precompile. + + Returns: + List of Files; the desired output files derived from the input sources. + """ + _ = ctx # @unused + + # Precompilation isn't implemented yet, so just return srcs as-is + return srcs + +def get_imports(ctx): + """Gets the imports from a rule's `imports` attribute. + + See create_binary_semantics_struct for details about this function. + + Args: + ctx: Rule ctx. + + Returns: + List of strings. + """ + prefix = "{}/{}".format( + ctx.workspace_name, + _py_builtins.get_label_repo_runfiles_path(ctx.label), + ) + result = [] + for import_str in ctx.attr.imports: + import_str = ctx.expand_make_variables("imports", import_str, {}) + if import_str.startswith("/"): + continue + + # To prevent "escaping" out of the runfiles tree, we normalize + # the path and ensure it doesn't have up-level references. + import_path = paths.normalize("{}/{}".format(prefix, import_str)) + if import_path.startswith("../") or import_path == "..": + fail("Path '{}' references a path above the execution root".format( + import_str, + )) + result.append(import_path) + return result + +def convert_legacy_create_init_to_int(kwargs): + """Convert "legacy_create_init" key to int, in-place. + + Args: + kwargs: The kwargs to modify. The key "legacy_create_init", if present + and bool, will be converted to its integer value, in place. + """ + if is_bool(kwargs.get("legacy_create_init")): + kwargs["legacy_create_init"] = 1 if kwargs["legacy_create_init"] else 0 diff --git a/python/private/common/providers.bzl b/python/private/common/providers.bzl new file mode 100644 index 000000000..a9df61bda --- /dev/null +++ b/python/private/common/providers.bzl @@ -0,0 +1,212 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Providers for Python rules.""" + +load(":common/python/semantics.bzl", "TOOLS_REPO") + +_CcInfo = _builtins.toplevel.CcInfo + +# NOTE: This is copied to PyRuntimeInfo.java +DEFAULT_STUB_SHEBANG = "#!/usr/bin/env python3" + +# NOTE: This is copied to PyRuntimeInfo.java +DEFAULT_BOOTSTRAP_TEMPLATE = "@" + TOOLS_REPO + "//tools/python:python_bootstrap_template.txt" +_PYTHON_VERSION_VALUES = ["PY2", "PY3"] + +def _PyRuntimeInfo_init( + *, + interpreter_path = None, + interpreter = None, + files = None, + coverage_tool = None, + coverage_files = None, + python_version, + stub_shebang = None, + bootstrap_template = None): + if (interpreter_path and interpreter) or (not interpreter_path and not interpreter): + fail("exactly one of interpreter or interpreter_path must be specified") + + if interpreter_path and files != None: + fail("cannot specify 'files' if 'interpreter_path' is given") + + if (coverage_tool and not coverage_files) or (not coverage_tool and coverage_files): + fail( + "coverage_tool and coverage_files must both be set or neither must be set, " + + "got coverage_tool={}, coverage_files={}".format( + coverage_tool, + coverage_files, + ), + ) + + if python_version not in _PYTHON_VERSION_VALUES: + fail("invalid python_version: '{}'; must be one of {}".format( + python_version, + _PYTHON_VERSION_VALUES, + )) + + if files != None and type(files) != type(depset()): + fail("invalid files: got value of type {}, want depset".format(type(files))) + + if interpreter: + if files == None: + files = depset() + else: + files = None + + if coverage_files == None: + coverage_files = depset() + + if not stub_shebang: + stub_shebang = DEFAULT_STUB_SHEBANG + + return { + "bootstrap_template": bootstrap_template, + "coverage_files": coverage_files, + "coverage_tool": coverage_tool, + "files": files, + "interpreter": interpreter, + "interpreter_path": interpreter_path, + "python_version": python_version, + "stub_shebang": stub_shebang, + } + +# TODO(#15897): Rename this to PyRuntimeInfo when we're ready to replace the Java +# implemented provider with the Starlark one. +PyRuntimeInfo, _unused_raw_py_runtime_info_ctor = provider( + doc = """Contains information about a Python runtime, as returned by the `py_runtime` +rule. + +A Python runtime describes either a *platform runtime* or an *in-build runtime*. +A platform runtime accesses a system-installed interpreter at a known path, +whereas an in-build runtime points to a `File` that acts as the interpreter. In +both cases, an "interpreter" is really any executable binary or wrapper script +that is capable of running a Python script passed on the command line, following +the same conventions as the standard CPython interpreter. +""", + init = _PyRuntimeInfo_init, + fields = { + "bootstrap_template": ( + "See py_runtime_rule.bzl%py_runtime.bootstrap_template for docs." + ), + "coverage_files": ( + "The files required at runtime for using `coverage_tool`. " + + "Will be `None` if no `coverage_tool` was provided." + ), + "coverage_tool": ( + "If set, this field is a `File` representing tool used for collecting code coverage information from python tests. Otherwise, this is `None`." + ), + "files": ( + "If this is an in-build runtime, this field is a `depset` of `File`s" + + "that need to be added to the runfiles of an executable target that " + + "uses this runtime (in particular, files needed by `interpreter`). " + + "The value of `interpreter` need not be included in this field. If " + + "this is a platform runtime then this field is `None`." + ), + "interpreter": ( + "If this is an in-build runtime, this field is a `File` representing " + + "the interpreter. Otherwise, this is `None`. Note that an in-build " + + "runtime can use either a prebuilt, checked-in interpreter or an " + + "interpreter built from source." + ), + "interpreter_path": ( + "If this is a platform runtime, this field is the absolute " + + "filesystem path to the interpreter on the target platform. " + + "Otherwise, this is `None`." + ), + "python_version": ( + "Indicates whether this runtime uses Python major version 2 or 3. " + + "Valid values are (only) `\"PY2\"` and " + + "`\"PY3\"`." + ), + "stub_shebang": ( + "\"Shebang\" expression prepended to the bootstrapping Python stub " + + "script used when executing `py_binary` targets. Does not " + + "apply to Windows." + ), + }, +) + +def _check_arg_type(name, required_type, value): + value_type = type(value) + if value_type != required_type: + fail("parameter '{}' got value of type '{}', want '{}'".format( + name, + value_type, + required_type, + )) + +def _PyInfo_init( + *, + transitive_sources, + uses_shared_libraries = False, + imports = depset(), + has_py2_only_sources = False, + has_py3_only_sources = False): + _check_arg_type("transitive_sources", "depset", transitive_sources) + + # Verify it's postorder compatible, but retain is original ordering. + depset(transitive = [transitive_sources], order = "postorder") + + _check_arg_type("uses_shared_libraries", "bool", uses_shared_libraries) + _check_arg_type("imports", "depset", imports) + _check_arg_type("has_py2_only_sources", "bool", has_py2_only_sources) + _check_arg_type("has_py3_only_sources", "bool", has_py3_only_sources) + return { + "has_py2_only_sources": has_py2_only_sources, + "has_py3_only_sources": has_py2_only_sources, + "imports": imports, + "transitive_sources": transitive_sources, + "uses_shared_libraries": uses_shared_libraries, + } + +PyInfo, _unused_raw_py_info_ctor = provider( + "Encapsulates information provided by the Python rules.", + init = _PyInfo_init, + fields = { + "has_py2_only_sources": "Whether any of this target's transitive sources requires a Python 2 runtime.", + "has_py3_only_sources": "Whether any of this target's transitive sources requires a Python 3 runtime.", + "imports": """\ +A depset of import path strings to be added to the `PYTHONPATH` of executable +Python targets. These are accumulated from the transitive `deps`. +The order of the depset is not guaranteed and may be changed in the future. It +is recommended to use `default` order (the default). +""", + "transitive_sources": """\ +A (`postorder`-compatible) depset of `.py` files appearing in the target's +`srcs` and the `srcs` of the target's transitive `deps`. +""", + "uses_shared_libraries": """ +Whether any of this target's transitive `deps` has a shared library file (such +as a `.so` file). + +This field is currently unused in Bazel and may go away in the future. +""", + }, +) + +def _PyCcLinkParamsProvider_init(cc_info): + return { + "cc_info": _CcInfo(linking_context = cc_info.linking_context), + } + +# buildifier: disable=name-conventions +PyCcLinkParamsProvider, _unused_raw_py_cc_link_params_provider_ctor = provider( + doc = ("Python-wrapper to forward CcInfo.linking_context. This is to " + + "allow Python targets to propagate C++ linking information, but " + + "without the Python target appearing to be a valid C++ rule dependency"), + init = _PyCcLinkParamsProvider_init, + fields = { + "cc_info": "A CcInfo instance; it has only linking_context set", + }, +) diff --git a/python/private/common/py_binary_bazel.bzl b/python/private/common/py_binary_bazel.bzl new file mode 100644 index 000000000..3a5df737b --- /dev/null +++ b/python/private/common/py_binary_bazel.bzl @@ -0,0 +1,48 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule implementation of py_binary for Bazel.""" + +load(":common/python/attributes.bzl", "AGNOSTIC_BINARY_ATTRS") +load( + ":common/python/py_executable_bazel.bzl", + "create_executable_rule", + "py_executable_bazel_impl", +) +load(":common/python/semantics.bzl", "TOOLS_REPO") + +_PY_TEST_ATTRS = { + "_collect_cc_coverage": attr.label( + default = "@" + TOOLS_REPO + "//tools/test:collect_cc_coverage", + executable = True, + cfg = "exec", + ), + "_lcov_merger": attr.label( + default = configuration_field(fragment = "coverage", name = "output_generator"), + executable = True, + cfg = "exec", + ), +} + +def _py_binary_impl(ctx): + return py_executable_bazel_impl( + ctx = ctx, + is_test = False, + inherited_environment = [], + ) + +py_binary = create_executable_rule( + implementation = _py_binary_impl, + attrs = AGNOSTIC_BINARY_ATTRS | _PY_TEST_ATTRS, + executable = True, +) diff --git a/python/private/common/py_binary_macro.bzl b/python/private/common/py_binary_macro.bzl new file mode 100644 index 000000000..24e5c6dbe --- /dev/null +++ b/python/private/common/py_binary_macro.bzl @@ -0,0 +1,21 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation of macro-half of py_binary rule.""" + +load(":common/python/common_bazel.bzl", "convert_legacy_create_init_to_int") +load(":common/python/py_binary_bazel.bzl", py_binary_rule = "py_binary") + +def py_binary(**kwargs): + convert_legacy_create_init_to_int(kwargs) + py_binary_rule(**kwargs) diff --git a/python/private/common/py_executable.bzl b/python/private/common/py_executable.bzl new file mode 100644 index 000000000..9db92b18e --- /dev/null +++ b/python/private/common/py_executable.bzl @@ -0,0 +1,845 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Common functionality between test/binary executables.""" + +load(":common/cc/cc_common.bzl", _cc_common = "cc_common") +load(":common/cc/cc_helper.bzl", "cc_helper") +load( + ":common/python/attributes.bzl", + "AGNOSTIC_EXECUTABLE_ATTRS", + "COMMON_ATTRS", + "PY_SRCS_ATTRS", + "SRCS_VERSION_ALL_VALUES", + "create_srcs_attr", + "create_srcs_version_attr", +) +load( + ":common/python/common.bzl", + "TOOLCHAIN_TYPE", + "check_native_allowed", + "collect_imports", + "collect_runfiles", + "create_instrumented_files_info", + "create_output_group_info", + "create_py_info", + "csv", + "filter_to_py_srcs", + "union_attrs", +) +load( + ":common/python/providers.bzl", + "PyCcLinkParamsProvider", + "PyRuntimeInfo", +) +load( + ":common/python/semantics.bzl", + "ALLOWED_MAIN_EXTENSIONS", + "BUILD_DATA_SYMLINK_PATH", + "IS_BAZEL", + "PY_RUNTIME_ATTR_NAME", +) + +_py_builtins = _builtins.internal.py_builtins + +# Non-Google-specific attributes for executables +# These attributes are for rules that accept Python sources. +EXECUTABLE_ATTRS = union_attrs( + COMMON_ATTRS, + AGNOSTIC_EXECUTABLE_ATTRS, + PY_SRCS_ATTRS, + { + # TODO(b/203567235): In the Java impl, any file is allowed. While marked + # label, it is more treated as a string, and doesn't have to refer to + # anything that exists because it gets treated as suffix-search string + # over `srcs`. + "main": attr.label( + allow_single_file = True, + doc = """\ +Optional; the name of the source file that is the main entry point of the +application. This file must also be listed in `srcs`. If left unspecified, +`name`, with `.py` appended, is used instead. If `name` does not match any +filename in `srcs`, `main` must be specified. +""", + ), + # TODO(b/203567235): In Google, this attribute is deprecated, and can + # only effectively be PY3. Externally, with Bazel, this attribute has + # a separate story. + "python_version": attr.string( + # TODO(b/203567235): In the Java impl, the default comes from + # --python_version. Not clear what the Starlark equivalent is. + default = "PY3", + # NOTE: Some tests care about the order of these values. + values = ["PY2", "PY3"], + ), + }, + create_srcs_version_attr(values = SRCS_VERSION_ALL_VALUES), + create_srcs_attr(mandatory = True), + allow_none = True, +) + +def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment = []): + """Base rule implementation for a Python executable. + + Google and Bazel call this common base and apply customizations using the + semantics object. + + Args: + ctx: The rule ctx + semantics: BinarySemantics struct; see create_binary_semantics_struct() + is_test: bool, True if the rule is a test rule (has `test=True`), + False if not (has `executable=True`) + inherited_environment: List of str; additional environment variable + names that should be inherited from the runtime environment when the + executable is run. + Returns: + DefaultInfo provider for the executable + """ + _validate_executable(ctx) + + main_py = determine_main(ctx) + direct_sources = filter_to_py_srcs(ctx.files.srcs) + output_sources = semantics.maybe_precompile(ctx, direct_sources) + imports = collect_imports(ctx, semantics) + executable, files_to_build = _compute_outputs(ctx, output_sources) + + runtime_details = _get_runtime_details(ctx, semantics) + if ctx.configuration.coverage_enabled: + extra_deps = semantics.get_coverage_deps(ctx, runtime_details) + else: + extra_deps = [] + + # The debugger dependency should be prevented by select() config elsewhere, + # but just to be safe, also guard against adding it to the output here. + if not _is_tool_config(ctx): + extra_deps.extend(semantics.get_debugger_deps(ctx, runtime_details)) + + cc_details = semantics.get_cc_details_for_binary(ctx, extra_deps = extra_deps) + native_deps_details = _get_native_deps_details( + ctx, + semantics = semantics, + cc_details = cc_details, + is_test = is_test, + ) + runfiles_details = _get_base_runfiles_for_binary( + ctx, + executable = executable, + extra_deps = extra_deps, + files_to_build = files_to_build, + extra_common_runfiles = [ + runtime_details.runfiles, + cc_details.extra_runfiles, + native_deps_details.runfiles, + semantics.get_extra_common_runfiles_for_binary(ctx), + ], + semantics = semantics, + ) + exec_result = semantics.create_executable( + ctx, + executable = executable, + main_py = main_py, + imports = imports, + is_test = is_test, + runtime_details = runtime_details, + cc_details = cc_details, + native_deps_details = native_deps_details, + runfiles_details = runfiles_details, + ) + files_to_build = depset(transitive = [ + exec_result.extra_files_to_build, + files_to_build, + ]) + extra_exec_runfiles = ctx.runfiles(transitive_files = files_to_build) + runfiles_details = struct( + default_runfiles = runfiles_details.default_runfiles.merge(extra_exec_runfiles), + data_runfiles = runfiles_details.data_runfiles.merge(extra_exec_runfiles), + ) + + legacy_providers, modern_providers = _create_providers( + ctx = ctx, + executable = executable, + runfiles_details = runfiles_details, + main_py = main_py, + imports = imports, + direct_sources = direct_sources, + files_to_build = files_to_build, + runtime_details = runtime_details, + cc_info = cc_details.cc_info_for_propagating, + inherited_environment = inherited_environment, + semantics = semantics, + output_groups = exec_result.output_groups, + ) + return struct( + legacy_providers = legacy_providers, + providers = modern_providers, + ) + +def _validate_executable(ctx): + if ctx.attr.python_version != "PY3": + fail("It is not allowed to use Python 2") + check_native_allowed(ctx) + +def _compute_outputs(ctx, output_sources): + # TODO: This should use the configuration instead of the Bazel OS. + if _py_builtins.get_current_os_name() == "windows": + executable = ctx.actions.declare_file(ctx.label.name + ".exe") + else: + executable = ctx.actions.declare_file(ctx.label.name) + + # TODO(b/208657718): Remove output_sources from the default outputs + # once the depot is cleaned up. + return executable, depset([executable] + output_sources) + +def _get_runtime_details(ctx, semantics): + """Gets various information about the Python runtime to use. + + While most information comes from the toolchain, various legacy and + compatibility behaviors require computing some other information. + + Args: + ctx: Rule ctx + semantics: A `BinarySemantics` struct; see `create_binary_semantics_struct` + + Returns: + A struct; see inline-field comments of the return value for details. + """ + + # Bazel has --python_path. This flag has a computed default of "python" when + # its actual default is null (see + # BazelPythonConfiguration.java#getPythonPath). This flag is only used if + # toolchains are not enabled and `--python_top` isn't set. Note that Google + # used to have a variant of this named --python_binary, but it has since + # been removed. + # + # TOOD(bazelbuild/bazel#7901): Remove this once --python_path flag is removed. + + if IS_BAZEL: + flag_interpreter_path = ctx.fragments.bazel_py.python_path + toolchain_runtime, effective_runtime = _maybe_get_runtime_from_ctx(ctx) + if not effective_runtime: + # Clear these just in case + toolchain_runtime = None + effective_runtime = None + + else: # Google code path + flag_interpreter_path = None + toolchain_runtime, effective_runtime = _maybe_get_runtime_from_ctx(ctx) + if not effective_runtime: + fail("Unable to find Python runtime") + + if effective_runtime: + direct = [] # List of files + transitive = [] # List of depsets + if effective_runtime.interpreter: + direct.append(effective_runtime.interpreter) + transitive.append(effective_runtime.files) + + if ctx.configuration.coverage_enabled: + if effective_runtime.coverage_tool: + direct.append(effective_runtime.coverage_tool) + if effective_runtime.coverage_files: + transitive.append(effective_runtime.coverage_files) + runtime_files = depset(direct = direct, transitive = transitive) + else: + runtime_files = depset() + + executable_interpreter_path = semantics.get_interpreter_path( + ctx, + runtime = effective_runtime, + flag_interpreter_path = flag_interpreter_path, + ) + + return struct( + # Optional PyRuntimeInfo: The runtime found from toolchain resolution. + # This may be None because, within Google, toolchain resolution isn't + # yet enabled. + toolchain_runtime = toolchain_runtime, + # Optional PyRuntimeInfo: The runtime that should be used. When + # toolchain resolution is enabled, this is the same as + # `toolchain_resolution`. Otherwise, this probably came from the + # `_python_top` attribute that the Google implementation still uses. + # This is separate from `toolchain_runtime` because toolchain_runtime + # is propagated as a provider, while non-toolchain runtimes are not. + effective_runtime = effective_runtime, + # str; Path to the Python interpreter to use for running the executable + # itself (not the bootstrap script). Either an absolute path (which + # means it is platform-specific), or a runfiles-relative path (which + # means the interpreter should be within `runtime_files`) + executable_interpreter_path = executable_interpreter_path, + # runfiles: Additional runfiles specific to the runtime that should + # be included. For in-build runtimes, this shold include the interpreter + # and any supporting files. + runfiles = ctx.runfiles(transitive_files = runtime_files), + ) + +def _maybe_get_runtime_from_ctx(ctx): + """Finds the PyRuntimeInfo from the toolchain or attribute, if available. + + Returns: + 2-tuple of toolchain_runtime, effective_runtime + """ + if ctx.fragments.py.use_toolchains: + toolchain = ctx.toolchains[TOOLCHAIN_TYPE] + + if not hasattr(toolchain, "py3_runtime"): + fail("Python toolchain field 'py3_runtime' is missing") + if not toolchain.py3_runtime: + fail("Python toolchain missing py3_runtime") + py3_runtime = toolchain.py3_runtime + + # Hack around the fact that the autodetecting Python toolchain, which is + # automatically registered, does not yet support Windows. In this case, + # we want to return null so that _get_interpreter_path falls back on + # --python_path. See tools/python/toolchain.bzl. + # TODO(#7844): Remove this hack when the autodetecting toolchain has a + # Windows implementation. + if py3_runtime.interpreter_path == "/_magic_pyruntime_sentinel_do_not_use": + return None, None + + if py3_runtime.python_version != "PY3": + fail("Python toolchain py3_runtime must be python_version=PY3, got {}".format( + py3_runtime.python_version, + )) + toolchain_runtime = toolchain.py3_runtime + effective_runtime = toolchain_runtime + else: + toolchain_runtime = None + attr_target = getattr(ctx.attr, PY_RUNTIME_ATTR_NAME) + + # In Bazel, --python_top is null by default. + if attr_target and PyRuntimeInfo in attr_target: + effective_runtime = attr_target[PyRuntimeInfo] + else: + return None, None + + return toolchain_runtime, effective_runtime + +def _get_base_runfiles_for_binary( + ctx, + *, + executable, + extra_deps, + files_to_build, + extra_common_runfiles, + semantics): + """Returns the set of runfiles necessary prior to executable creation. + + NOTE: The term "common runfiles" refers to the runfiles that both the + default and data runfiles have in common. + + Args: + ctx: The rule ctx. + executable: The main executable output. + extra_deps: List of Targets; additional targets whose runfiles + will be added to the common runfiles. + files_to_build: depset of File of the default outputs to add into runfiles. + extra_common_runfiles: List of runfiles; additional runfiles that + will be added to the common runfiles. + semantics: A `BinarySemantics` struct; see `create_binary_semantics_struct`. + + Returns: + struct with attributes: + * default_runfiles: The default runfiles + * data_runfiles: The data runfiles + """ + common_runfiles = collect_runfiles(ctx, depset( + direct = [executable], + transitive = [files_to_build], + )) + if extra_deps: + common_runfiles = common_runfiles.merge_all([ + t[DefaultInfo].default_runfiles + for t in extra_deps + ]) + common_runfiles = common_runfiles.merge_all(extra_common_runfiles) + + if semantics.should_create_init_files(ctx): + common_runfiles = _py_builtins.merge_runfiles_with_generated_inits_empty_files_supplier( + ctx = ctx, + runfiles = common_runfiles, + ) + + # Don't include build_data.txt in data runfiles. This allows binaries to + # contain other binaries while still using the same fixed location symlink + # for the build_data.txt file. Really, the fixed location symlink should be + # removed and another way found to locate the underlying build data file. + data_runfiles = common_runfiles + + if is_stamping_enabled(ctx, semantics) and semantics.should_include_build_data(ctx): + default_runfiles = common_runfiles.merge(_create_runfiles_with_build_data( + ctx, + semantics.get_central_uncachable_version_file(ctx), + semantics.get_extra_write_build_data_env(ctx), + )) + else: + default_runfiles = common_runfiles + + return struct( + default_runfiles = default_runfiles, + data_runfiles = data_runfiles, + ) + +def _create_runfiles_with_build_data( + ctx, + central_uncachable_version_file, + extra_write_build_data_env): + return ctx.runfiles( + symlinks = { + BUILD_DATA_SYMLINK_PATH: _write_build_data( + ctx, + central_uncachable_version_file, + extra_write_build_data_env, + ), + }, + ) + +def _write_build_data(ctx, central_uncachable_version_file, extra_write_build_data_env): + # TODO: Remove this logic when a central file is always available + if not central_uncachable_version_file: + version_file = ctx.actions.declare_file(ctx.label.name + "-uncachable_version_file.txt") + _py_builtins.copy_without_caching( + ctx = ctx, + read_from = ctx.version_file, + write_to = version_file, + ) + else: + version_file = central_uncachable_version_file + + direct_inputs = [ctx.info_file, version_file] + + # A "constant metadata" file is basically a special file that doesn't + # support change detection logic and reports that it is unchanged. i.e., it + # behaves like ctx.version_file and is ignored when computing "what inputs + # changed" (see https://bazel.build/docs/user-manual#workspace-status). + # + # We do this so that consumers of the final build data file don't have + # to transitively rebuild everything -- the `uncachable_version_file` file + # isn't cachable, which causes the build data action to always re-run. + # + # While this technically means a binary could have stale build info, + # it ends up not mattering in practice because the volatile information + # doesn't meaningfully effect other outputs. + # + # This is also done for performance and Make It work reasons: + # * Passing the transitive dependencies into the action requires passing + # the runfiles, but actions don't directly accept runfiles. While + # flattening the depsets can be deferred, accessing the + # `runfiles.empty_filenames` attribute will will invoke the empty + # file supplier a second time, which is too much of a memory and CPU + # performance hit. + # * Some targets specify a directory in `data`, which is unsound, but + # mostly works. Google's RBE, unfortunately, rejects it. + # * A binary's transitive closure may be so large that it exceeds + # Google RBE limits for action inputs. + build_data = _py_builtins.declare_constant_metadata_file( + ctx = ctx, + name = ctx.label.name + ".build_data.txt", + root = ctx.bin_dir, + ) + + ctx.actions.run( + executable = ctx.executable._build_data_gen, + env = { + # NOTE: ctx.info_file is undocumented; see + # https://github.com/bazelbuild/bazel/issues/9363 + "INFO_FILE": ctx.info_file.path, + "OUTPUT": build_data.path, + "PLATFORM": cc_helper.find_cpp_toolchain(ctx).toolchain_id, + "TARGET": str(ctx.label), + "VERSION_FILE": version_file.path, + } | extra_write_build_data_env, + inputs = depset( + direct = direct_inputs, + ), + outputs = [build_data], + mnemonic = "PyWriteBuildData", + progress_message = "Generating %{label} build_data.txt", + ) + return build_data + +def _get_native_deps_details(ctx, *, semantics, cc_details, is_test): + if not semantics.should_build_native_deps_dso(ctx): + return struct(dso = None, runfiles = ctx.runfiles()) + + cc_info = cc_details.cc_info_for_self_link + + if not cc_info.linking_context.linker_inputs: + return struct(dso = None, runfiles = ctx.runfiles()) + + dso = ctx.actions.declare_file(semantics.get_native_deps_dso_name(ctx)) + share_native_deps = ctx.fragments.cpp.share_native_deps() + cc_feature_config = cc_configure_features( + ctx, + cc_toolchain = cc_details.cc_toolchain, + # See b/171276569#comment18: this feature string is just to allow + # Google's RBE to know the link action is for the Python case so it can + # take special actions (though as of Jun 2022, no special action is + # taken). + extra_features = ["native_deps_link"], + ) + if share_native_deps: + linked_lib = _create_shared_native_deps_dso( + ctx, + cc_info = cc_info, + is_test = is_test, + requested_features = cc_feature_config.requested_features, + feature_configuration = cc_feature_config.feature_configuration, + ) + ctx.actions.symlink( + output = dso, + target_file = linked_lib, + progress_message = "Symlinking shared native deps for %{label}", + ) + else: + linked_lib = dso + _cc_common.link( + name = ctx.label.name, + actions = ctx.actions, + linking_contexts = [cc_info.linking_context], + output_type = "dynamic_library", + never_link = True, + native_deps = True, + feature_configuration = cc_feature_config.feature_configuration, + cc_toolchain = cc_details.cc_toolchain, + test_only_target = is_test, + stamp = 1 if is_stamping_enabled(ctx, semantics) else 0, + main_output = linked_lib, + use_shareable_artifact_factory = True, + # NOTE: Only flags not captured by cc_info.linking_context need to + # be manually passed + user_link_flags = semantics.get_native_deps_user_link_flags(ctx), + ) + return struct( + dso = dso, + runfiles = ctx.runfiles(files = [dso]), + ) + +def _create_shared_native_deps_dso( + ctx, + *, + cc_info, + is_test, + feature_configuration, + requested_features): + linkstamps = cc_info.linking_context.linkstamps() + + partially_disabled_thin_lto = ( + _cc_common.is_enabled( + feature_name = "thin_lto_linkstatic_tests_use_shared_nonlto_backends", + feature_configuration = feature_configuration, + ) and not _cc_common.is_enabled( + feature_name = "thin_lto_all_linkstatic_use_shared_nonlto_backends", + feature_configuration = feature_configuration, + ) + ) + dso_hash = _get_shared_native_deps_hash( + linker_inputs = cc_helper.get_static_mode_params_for_dynamic_library_libraries( + depset([ + lib + for linker_input in cc_info.linking_context.linker_inputs.to_list() + for lib in linker_input.libraries + ]), + ), + link_opts = [ + flag + for input in cc_info.linking_context.linker_inputs.to_list() + for flag in input.user_link_flags + ], + linkstamps = [linkstamp.file() for linkstamp in linkstamps.to_list()], + build_info_artifacts = _cc_common.get_build_info(ctx) if linkstamps else [], + features = requested_features, + is_test_target_partially_disabled_thin_lto = is_test and partially_disabled_thin_lto, + ) + return ctx.actions.declare_shareable_artifact("_nativedeps/%x.so" % dso_hash) + +# This is a minimal version of NativeDepsHelper.getSharedNativeDepsPath, see +# com.google.devtools.build.lib.rules.nativedeps.NativeDepsHelper#getSharedNativeDepsPath +# The basic idea is to take all the inputs that affect linking and encode (via +# hashing) them into the filename. +# TODO(b/234232820): The settings that affect linking must be kept in sync with the actual +# C++ link action. For more information, see the large descriptive comment on +# NativeDepsHelper#getSharedNativeDepsPath. +def _get_shared_native_deps_hash( + *, + linker_inputs, + link_opts, + linkstamps, + build_info_artifacts, + features, + is_test_target_partially_disabled_thin_lto): + # NOTE: We use short_path because the build configuration root in which + # files are always created already captures the configuration-specific + # parts, so no need to include them manually. + parts = [] + for artifact in linker_inputs: + parts.append(artifact.short_path) + parts.append(str(len(link_opts))) + parts.extend(link_opts) + for artifact in linkstamps: + parts.append(artifact.short_path) + for artifact in build_info_artifacts: + parts.append(artifact.short_path) + parts.extend(sorted(features)) + + # Sharing of native dependencies may cause an {@link + # ActionConflictException} when ThinLTO is disabled for test and test-only + # targets that are statically linked, but enabled for other statically + # linked targets. This happens in case the artifacts for the shared native + # dependency are output by {@link Action}s owned by the non-test and test + # targets both. To fix this, we allow creation of multiple artifacts for the + # shared native library - one shared among the test and test-only targets + # where ThinLTO is disabled, and the other shared among other targets where + # ThinLTO is enabled. See b/138118275 + parts.append("1" if is_test_target_partially_disabled_thin_lto else "0") + + return hash("".join(parts)) + +def determine_main(ctx): + """Determine the main entry point .py source file. + + Args: + ctx: The rule ctx. + + Returns: + Artifact; the main file. If one can't be found, an error is raised. + """ + if ctx.attr.main: + proposed_main = ctx.attr.main.label.name + if not proposed_main.endswith(tuple(ALLOWED_MAIN_EXTENSIONS)): + fail("main must end in '.py'") + else: + if ctx.label.name.endswith(".py"): + fail("name must not end in '.py'") + proposed_main = ctx.label.name + ".py" + + main_files = [src for src in ctx.files.srcs if _path_endswith(src.short_path, proposed_main)] + if not main_files: + if ctx.attr.main: + fail("could not find '{}' as specified by 'main' attribute".format(proposed_main)) + else: + fail(("corresponding default '{}' does not appear in srcs. Add " + + "it or override default file name with a 'main' attribute").format( + proposed_main, + )) + + elif len(main_files) > 1: + if ctx.attr.main: + fail(("file name '{}' specified by 'main' attributes matches multiple files. " + + "Matches: {}").format( + proposed_main, + csv([f.short_path for f in main_files]), + )) + else: + fail(("default main file '{}' matches multiple files in srcs. Perhaps specify " + + "an explicit file with 'main' attribute? Matches were: {}").format( + proposed_main, + csv([f.short_path for f in main_files]), + )) + return main_files[0] + +def _path_endswith(path, endswith): + # Use slash to anchor each path to prevent e.g. + # "ab/c.py".endswith("b/c.py") from incorrectly matching. + return ("/" + path).endswith("/" + endswith) + +def is_stamping_enabled(ctx, semantics): + """Tells if stamping is enabled or not. + + Args: + ctx: The rule ctx + semantics: a semantics struct (see create_semantics_struct). + Returns: + bool; True if stamping is enabled, False if not. + """ + if _is_tool_config(ctx): + return False + + stamp = ctx.attr.stamp + if stamp == 1: + return True + elif stamp == 0: + return False + elif stamp == -1: + return semantics.get_stamp_flag(ctx) + else: + fail("Unsupported `stamp` value: {}".format(stamp)) + +def _is_tool_config(ctx): + # NOTE: The is_tool_configuration() function is only usable by builtins. + # See https://github.com/bazelbuild/bazel/issues/14444 for the FR for + # a more public API. Outside of builtins, ctx.bin_dir.path can be + # checked for `/host/` or `-exec-`. + return ctx.configuration.is_tool_configuration() + +def _create_providers( + *, + ctx, + executable, + main_py, + direct_sources, + files_to_build, + runfiles_details, + imports, + cc_info, + inherited_environment, + runtime_details, + output_groups, + semantics): + """Creates the providers an executable should return. + + Args: + ctx: The rule ctx. + executable: File; the target's executable file. + main_py: File; the main .py entry point. + direct_sources: list of Files; the direct, raw `.py` sources for the target. + This should only be Python source files. It should not include pyc + files. + files_to_build: depset of Files; the files for DefaultInfo.files + runfiles_details: runfiles that will become the default and data runfiles. + imports: depset of strings; the import paths to propagate + cc_info: optional CcInfo; Linking information to propagate as + PyCcLinkParamsProvider. Note that only the linking information + is propagated, not the whole CcInfo. + inherited_environment: list of strings; Environment variable names + that should be inherited from the environment the executuble + is run within. + runtime_details: struct of runtime information; see _get_runtime_details() + output_groups: dict[str, depset[File]]; used to create OutputGroupInfo + semantics: BinarySemantics struct; see create_binary_semantics() + + Returns: + A two-tuple of: + 1. A dict of legacy providers. + 2. A list of modern providers. + """ + providers = [ + DefaultInfo( + executable = executable, + files = files_to_build, + default_runfiles = _py_builtins.make_runfiles_respect_legacy_external_runfiles( + ctx, + runfiles_details.default_runfiles, + ), + data_runfiles = _py_builtins.make_runfiles_respect_legacy_external_runfiles( + ctx, + runfiles_details.data_runfiles, + ), + ), + create_instrumented_files_info(ctx), + _create_run_environment_info(ctx, inherited_environment), + ] + + # TODO(b/265840007): Make this non-conditional once Google enables + # --incompatible_use_python_toolchains. + if runtime_details.toolchain_runtime: + providers.append(runtime_details.toolchain_runtime) + + # TODO(b/163083591): Remove the PyCcLinkParamsProvider once binaries-in-deps + # are cleaned up. + if cc_info: + providers.append( + PyCcLinkParamsProvider(cc_info = cc_info), + ) + + py_info, deps_transitive_sources = create_py_info( + ctx, + direct_sources = depset(direct_sources), + imports = imports, + ) + + # TODO(b/253059598): Remove support for extra actions; https://github.com/bazelbuild/bazel/issues/16455 + listeners_enabled = _py_builtins.are_action_listeners_enabled(ctx) + if listeners_enabled: + _py_builtins.add_py_extra_pseudo_action( + ctx = ctx, + dependency_transitive_python_sources = deps_transitive_sources, + ) + + providers.append(py_info) + providers.append(create_output_group_info(py_info.transitive_sources, output_groups)) + + extra_legacy_providers, extra_providers = semantics.get_extra_providers( + ctx, + main_py = main_py, + runtime_details = runtime_details, + ) + providers.extend(extra_providers) + return extra_legacy_providers, providers + +def _create_run_environment_info(ctx, inherited_environment): + expanded_env = {} + for key, value in ctx.attr.env.items(): + expanded_env[key] = _py_builtins.expand_location_and_make_variables( + ctx = ctx, + attribute_name = "env[{}]".format(key), + expression = value, + targets = ctx.attr.data, + ) + return RunEnvironmentInfo( + environment = expanded_env, + inherited_environment = inherited_environment, + ) + +def create_base_executable_rule(*, attrs, fragments = [], **kwargs): + """Create a function for defining for Python binary/test targets. + + Args: + attrs: Rule attributes + fragments: List of str; extra config fragments that are required. + **kwargs: Additional args to pass onto `rule()` + + Returns: + A rule function + """ + if "py" not in fragments: + # The list might be frozen, so use concatentation + fragments = fragments + ["py"] + return rule( + # TODO: add ability to remove attrs, i.e. for imports attr + attrs = EXECUTABLE_ATTRS | attrs, + toolchains = [TOOLCHAIN_TYPE] + cc_helper.use_cpp_toolchain(), + fragments = fragments, + **kwargs + ) + +def cc_configure_features(ctx, *, cc_toolchain, extra_features): + """Configure C++ features for Python purposes. + + Args: + ctx: Rule ctx + cc_toolchain: The CcToolchain the target is using. + extra_features: list of strings; additional features to request be + enabled. + + Returns: + struct of the feature configuration and all requested features. + """ + requested_features = ["static_linking_mode"] + requested_features.extend(extra_features) + requested_features.extend(ctx.features) + if "legacy_whole_archive" not in ctx.disabled_features: + requested_features.append("legacy_whole_archive") + feature_configuration = _cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc_toolchain, + requested_features = requested_features, + unsupported_features = ctx.disabled_features, + ) + return struct( + feature_configuration = feature_configuration, + requested_features = requested_features, + ) + +only_exposed_for_google_internal_reason = struct( + create_runfiles_with_build_data = _create_runfiles_with_build_data, +) diff --git a/python/private/common/py_executable_bazel.bzl b/python/private/common/py_executable_bazel.bzl new file mode 100644 index 000000000..7c7ecb01d --- /dev/null +++ b/python/private/common/py_executable_bazel.bzl @@ -0,0 +1,480 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation for Bazel Python executable.""" + +load(":common/paths.bzl", "paths") +load(":common/python/attributes_bazel.bzl", "IMPORTS_ATTRS") +load( + ":common/python/common.bzl", + "create_binary_semantics_struct", + "create_cc_details_struct", + "create_executable_result_struct", + "union_attrs", +) +load(":common/python/common_bazel.bzl", "collect_cc_info", "get_imports", "maybe_precompile") +load(":common/python/providers.bzl", "DEFAULT_STUB_SHEBANG") +load( + ":common/python/py_executable.bzl", + "create_base_executable_rule", + "py_executable_base_impl", +) +load(":common/python/semantics.bzl", "TOOLS_REPO") + +_py_builtins = _builtins.internal.py_builtins +_EXTERNAL_PATH_PREFIX = "external" +_ZIP_RUNFILES_DIRECTORY_NAME = "runfiles" + +BAZEL_EXECUTABLE_ATTRS = union_attrs( + IMPORTS_ATTRS, + { + "legacy_create_init": attr.int( + default = -1, + values = [-1, 0, 1], + doc = """\ +Whether to implicitly create empty `__init__.py` files in the runfiles tree. +These are created in every directory containing Python source code or shared +libraries, and every parent directory of those directories, excluding the repo +root directory. The default, `-1` (auto), means true unless +`--incompatible_default_to_explicit_init_py` is used. If false, the user is +responsible for creating (possibly empty) `__init__.py` files and adding them to +the `srcs` of Python targets as required. + """, + ), + "_bootstrap_template": attr.label( + allow_single_file = True, + default = "@" + TOOLS_REPO + "//tools/python:python_bootstrap_template.txt", + ), + "_launcher": attr.label( + cfg = "target", + default = "@" + TOOLS_REPO + "//tools/launcher:launcher", + executable = True, + ), + "_py_interpreter": attr.label( + default = configuration_field( + fragment = "bazel_py", + name = "python_top", + ), + ), + # TODO: This appears to be vestigial. It's only added because + # GraphlessQueryTest.testLabelsOperator relies on it to test for + # query behavior of implicit dependencies. + "_py_toolchain_type": attr.label( + default = "@" + TOOLS_REPO + "//tools/python:toolchain_type", + ), + "_windows_launcher_maker": attr.label( + default = "@" + TOOLS_REPO + "//tools/launcher:launcher_maker", + cfg = "exec", + executable = True, + ), + "_zipper": attr.label( + cfg = "exec", + executable = True, + default = "@" + TOOLS_REPO + "//tools/zip:zipper", + ), + }, +) + +def create_executable_rule(*, attrs, **kwargs): + return create_base_executable_rule( + attrs = BAZEL_EXECUTABLE_ATTRS | attrs, + fragments = ["py", "bazel_py"], + **kwargs + ) + +def py_executable_bazel_impl(ctx, *, is_test, inherited_environment): + """Common code for executables for Baze.""" + result = py_executable_base_impl( + ctx = ctx, + semantics = create_binary_semantics_bazel(), + is_test = is_test, + inherited_environment = inherited_environment, + ) + return struct( + providers = result.providers, + **result.legacy_providers + ) + +def create_binary_semantics_bazel(): + return create_binary_semantics_struct( + # keep-sorted start + create_executable = _create_executable, + get_cc_details_for_binary = _get_cc_details_for_binary, + get_central_uncachable_version_file = lambda ctx: None, + get_coverage_deps = _get_coverage_deps, + get_debugger_deps = _get_debugger_deps, + get_extra_common_runfiles_for_binary = lambda ctx: ctx.runfiles(), + get_extra_providers = _get_extra_providers, + get_extra_write_build_data_env = lambda ctx: {}, + get_imports = get_imports, + get_interpreter_path = _get_interpreter_path, + get_native_deps_dso_name = _get_native_deps_dso_name, + get_native_deps_user_link_flags = _get_native_deps_user_link_flags, + get_stamp_flag = _get_stamp_flag, + maybe_precompile = maybe_precompile, + should_build_native_deps_dso = lambda ctx: False, + should_create_init_files = _should_create_init_files, + should_include_build_data = lambda ctx: False, + # keep-sorted end + ) + +def _get_coverage_deps(ctx, runtime_details): + _ = ctx, runtime_details # @unused + return [] + +def _get_debugger_deps(ctx, runtime_details): + _ = ctx, runtime_details # @unused + return [] + +def _get_extra_providers(ctx, main_py, runtime_details): + _ = ctx, main_py, runtime_details # @unused + return {}, [] + +def _get_stamp_flag(ctx): + # NOTE: Undocumented API; private to builtins + return ctx.configuration.stamp_binaries + +def _should_create_init_files(ctx): + if ctx.attr.legacy_create_init == -1: + return not ctx.fragments.py.default_to_explicit_init_py + else: + return bool(ctx.attr.legacy_create_init) + +def _create_executable( + ctx, + *, + executable, + main_py, + imports, + is_test, + runtime_details, + cc_details, + native_deps_details, + runfiles_details): + _ = is_test, cc_details, native_deps_details # @unused + + common_bootstrap_template_kwargs = dict( + main_py = main_py, + imports = imports, + runtime_details = runtime_details, + ) + + # TODO: This should use the configuration instead of the Bazel OS. + # This is just legacy behavior. + is_windows = _py_builtins.get_current_os_name() == "windows" + + if is_windows: + if not executable.extension == "exe": + fail("Should not happen: somehow we are generating a non-.exe file on windows") + base_executable_name = executable.basename[0:-4] + else: + base_executable_name = executable.basename + + zip_bootstrap = ctx.actions.declare_file(base_executable_name + ".temp", sibling = executable) + zip_file = ctx.actions.declare_file(base_executable_name + ".zip", sibling = executable) + + _expand_bootstrap_template( + ctx, + output = zip_bootstrap, + is_for_zip = True, + **common_bootstrap_template_kwargs + ) + _create_zip_file( + ctx, + output = zip_file, + original_nonzip_executable = executable, + executable_for_zip_file = zip_bootstrap, + runfiles = runfiles_details.default_runfiles, + ) + + extra_files_to_build = [] + + # NOTE: --build_python_zip defauls to true on Windows + build_zip_enabled = ctx.fragments.py.build_python_zip + + # When --build_python_zip is enabled, then the zip file becomes + # one of the default outputs. + if build_zip_enabled: + extra_files_to_build.append(zip_file) + + # The logic here is a bit convoluted. Essentially, there are 3 types of + # executables produced: + # 1. (non-Windows) A bootstrap template based program. + # 2. (non-Windows) A self-executable zip file of a bootstrap template based program. + # 3. (Windows) A native Windows executable that finds and launches + # the actual underlying Bazel program (one of the above). Note that + # it implicitly assumes one of the above is located next to it, and + # that --build_python_zip defaults to true for Windows. + + should_create_executable_zip = False + bootstrap_output = None + if not is_windows: + if build_zip_enabled: + should_create_executable_zip = True + else: + bootstrap_output = executable + else: + _create_windows_exe_launcher( + ctx, + output = executable, + use_zip_file = build_zip_enabled, + python_binary_path = runtime_details.executable_interpreter_path, + ) + if not build_zip_enabled: + # On Windows, the main executable has an "exe" extension, so + # here we re-use the un-extensioned name for the bootstrap output. + bootstrap_output = ctx.actions.declare_file(base_executable_name) + + # The launcher looks for the non-zip executable next to + # itself, so add it to the default outputs. + extra_files_to_build.append(bootstrap_output) + + if should_create_executable_zip: + if bootstrap_output != None: + fail("Should not occur: bootstrap_output should not be used " + + "when creating an executable zip") + _create_executable_zip_file(ctx, output = executable, zip_file = zip_file) + elif bootstrap_output: + _expand_bootstrap_template( + ctx, + output = bootstrap_output, + is_for_zip = build_zip_enabled, + **common_bootstrap_template_kwargs + ) + else: + # Otherwise, this should be the Windows case of launcher + zip. + # Double check this just to make sure. + if not is_windows or not build_zip_enabled: + fail(("Should not occur: The non-executable-zip and " + + "non-boostrap-template case should have windows and zip " + + "both true, but got " + + "is_windows={is_windows} " + + "build_zip_enabled={build_zip_enabled}").format( + is_windows = is_windows, + build_zip_enabled = build_zip_enabled, + )) + + return create_executable_result_struct( + extra_files_to_build = depset(extra_files_to_build), + output_groups = {"python_zip_file": depset([zip_file])}, + ) + +def _expand_bootstrap_template( + ctx, + *, + output, + main_py, + imports, + is_for_zip, + runtime_details): + runtime = runtime_details.effective_runtime + if (ctx.configuration.coverage_enabled and + runtime and + runtime.coverage_tool): + coverage_tool_runfiles_path = "{}/{}".format( + ctx.workspace_name, + runtime.coverage_tool.short_path, + ) + else: + coverage_tool_runfiles_path = "" + + if runtime: + shebang = runtime.stub_shebang + template = runtime.bootstrap_template + else: + shebang = DEFAULT_STUB_SHEBANG + template = ctx.file._bootstrap_template + + ctx.actions.expand_template( + template = template, + output = output, + substitutions = { + "%coverage_tool%": coverage_tool_runfiles_path, + "%import_all%": "True" if ctx.fragments.bazel_py.python_import_all_repositories else "False", + "%imports%": ":".join(imports.to_list()), + "%is_zipfile%": "True" if is_for_zip else "False", + "%main%": "{}/{}".format( + ctx.workspace_name, + main_py.short_path, + ), + "%python_binary%": runtime_details.executable_interpreter_path, + "%shebang%": shebang, + "%target%": str(ctx.label), + "%workspace_name%": ctx.workspace_name, + }, + is_executable = True, + ) + +def _create_windows_exe_launcher( + ctx, + *, + output, + python_binary_path, + use_zip_file): + launch_info = ctx.actions.args() + launch_info.use_param_file("%s", use_always = True) + launch_info.set_param_file_format("multiline") + launch_info.add("binary_type=Python") + launch_info.add(ctx.workspace_name, format = "workspace_name=%s") + launch_info.add( + "1" if ctx.configuration.runfiles_enabled() else "0", + format = "symlink_runfiles_enabled=%s", + ) + launch_info.add(python_binary_path, format = "python_bin_path=%s") + launch_info.add("1" if use_zip_file else "0", format = "use_zip_file=%s") + + ctx.actions.run( + executable = ctx.executable._windows_launcher_maker, + arguments = [ctx.executable._launcher.path, launch_info, output.path], + inputs = [ctx.executable._launcher], + outputs = [output], + mnemonic = "PyBuildLauncher", + progress_message = "Creating launcher for %{label}", + # Needed to inherit PATH when using non-MSVC compilers like MinGW + use_default_shell_env = True, + ) + +def _create_zip_file(ctx, *, output, original_nonzip_executable, executable_for_zip_file, runfiles): + workspace_name = ctx.workspace_name + legacy_external_runfiles = _py_builtins.get_legacy_external_runfiles(ctx) + + manifest = ctx.actions.args() + manifest.use_param_file("@%s", use_always = True) + manifest.set_param_file_format("multiline") + + manifest.add("__main__.py={}".format(executable_for_zip_file.path)) + manifest.add("__init__.py=") + manifest.add( + "{}=".format( + _get_zip_runfiles_path("__init__.py", workspace_name, legacy_external_runfiles), + ), + ) + for path in runfiles.empty_filenames.to_list(): + manifest.add("{}=".format(_get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles))) + + def map_zip_runfiles(file): + if file != original_nonzip_executable and file != output: + return "{}={}".format( + _get_zip_runfiles_path(file.short_path, workspace_name, legacy_external_runfiles), + file.path, + ) + else: + return None + + manifest.add_all(runfiles.files, map_each = map_zip_runfiles, allow_closure = True) + + inputs = [executable_for_zip_file] + if _py_builtins.is_bzlmod_enabled(ctx): + zip_repo_mapping_manifest = ctx.actions.declare_file( + output.basename + ".repo_mapping", + sibling = output, + ) + _py_builtins.create_repo_mapping_manifest( + ctx = ctx, + runfiles = runfiles, + output = zip_repo_mapping_manifest, + ) + manifest.add("{}/_repo_mapping={}".format( + _ZIP_RUNFILES_DIRECTORY_NAME, + zip_repo_mapping_manifest.path, + )) + inputs.append(zip_repo_mapping_manifest) + + for artifact in runfiles.files.to_list(): + # Don't include the original executable because it isn't used by the + # zip file, so no need to build it for the action. + # Don't include the zipfile itself because it's an output. + if artifact != original_nonzip_executable and artifact != output: + inputs.append(artifact) + + zip_cli_args = ctx.actions.args() + zip_cli_args.add("cC") + zip_cli_args.add(output) + + ctx.actions.run( + executable = ctx.executable._zipper, + arguments = [zip_cli_args, manifest], + inputs = depset(inputs), + outputs = [output], + use_default_shell_env = True, + mnemonic = "PythonZipper", + progress_message = "Building Python zip: %{label}", + ) + +def _get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles): + if legacy_external_runfiles and path.startswith(_EXTERNAL_PATH_PREFIX): + zip_runfiles_path = paths.relativize(path, _EXTERNAL_PATH_PREFIX) + else: + # NOTE: External runfiles (artifacts in other repos) will have a leading + # path component of "../" so that they refer outside the main workspace + # directory and into the runfiles root. By normalizing, we simplify e.g. + # "workspace/../foo/bar" to simply "foo/bar". + zip_runfiles_path = paths.normalize("{}/{}".format(workspace_name, path)) + return "{}/{}".format(_ZIP_RUNFILES_DIRECTORY_NAME, zip_runfiles_path) + +def _create_executable_zip_file(ctx, *, output, zip_file): + ctx.actions.run_shell( + command = "echo '{shebang}' | cat - {zip} > {output}".format( + shebang = "#!/usr/bin/env python3", + zip = zip_file.path, + output = output.path, + ), + inputs = [zip_file], + outputs = [output], + use_default_shell_env = True, + mnemonic = "BuildBinary", + progress_message = "Build Python zip executable: %{label}", + ) + +def _get_cc_details_for_binary(ctx, extra_deps): + cc_info = collect_cc_info(ctx, extra_deps = extra_deps) + return create_cc_details_struct( + cc_info_for_propagating = cc_info, + cc_info_for_self_link = cc_info, + cc_info_with_extra_link_time_libraries = None, + extra_runfiles = ctx.runfiles(), + # Though the rules require the CcToolchain, it isn't actually used. + cc_toolchain = None, + ) + +def _get_interpreter_path(ctx, *, runtime, flag_interpreter_path): + if runtime: + if runtime.interpreter_path: + interpreter_path = runtime.interpreter_path + else: + interpreter_path = "{}/{}".format( + ctx.workspace_name, + runtime.interpreter.short_path, + ) + + # NOTE: External runfiles (artifacts in other repos) will have a + # leading path component of "../" so that they refer outside the + # main workspace directory and into the runfiles root. By + # normalizing, we simplify e.g. "workspace/../foo/bar" to simply + # "foo/bar" + interpreter_path = paths.normalize(interpreter_path) + + elif flag_interpreter_path: + interpreter_path = flag_interpreter_path + else: + fail("Unable to determine interpreter path") + + return interpreter_path + +def _get_native_deps_dso_name(ctx): + _ = ctx # @unused + fail("Building native deps DSO not supported.") + +def _get_native_deps_user_link_flags(ctx): + _ = ctx # @unused + fail("Building native deps DSO not supported.") diff --git a/python/private/common/py_library.bzl b/python/private/common/py_library.bzl new file mode 100644 index 000000000..62f974f4b --- /dev/null +++ b/python/private/common/py_library.bzl @@ -0,0 +1,99 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation of py_library rule.""" + +load( + ":common/python/attributes.bzl", + "COMMON_ATTRS", + "PY_SRCS_ATTRS", + "SRCS_VERSION_ALL_VALUES", + "create_srcs_attr", + "create_srcs_version_attr", +) +load( + ":common/python/common.bzl", + "check_native_allowed", + "collect_imports", + "collect_runfiles", + "create_instrumented_files_info", + "create_output_group_info", + "create_py_info", + "filter_to_py_srcs", + "union_attrs", +) +load(":common/python/providers.bzl", "PyCcLinkParamsProvider") + +_py_builtins = _builtins.internal.py_builtins + +LIBRARY_ATTRS = union_attrs( + COMMON_ATTRS, + PY_SRCS_ATTRS, + create_srcs_version_attr(values = SRCS_VERSION_ALL_VALUES), + create_srcs_attr(mandatory = False), +) + +def py_library_impl(ctx, *, semantics): + """Abstract implementation of py_library rule. + + Args: + ctx: The rule ctx + semantics: A `LibrarySemantics` struct; see `create_library_semantics_struct` + + Returns: + A list of modern providers to propagate. + """ + check_native_allowed(ctx) + direct_sources = filter_to_py_srcs(ctx.files.srcs) + output_sources = depset(semantics.maybe_precompile(ctx, direct_sources)) + runfiles = collect_runfiles(ctx = ctx, files = output_sources) + + cc_info = semantics.get_cc_info_for_library(ctx) + py_info, deps_transitive_sources = create_py_info( + ctx, + direct_sources = depset(direct_sources), + imports = collect_imports(ctx, semantics), + ) + + # TODO(b/253059598): Remove support for extra actions; https://github.com/bazelbuild/bazel/issues/16455 + listeners_enabled = _py_builtins.are_action_listeners_enabled(ctx) + if listeners_enabled: + _py_builtins.add_py_extra_pseudo_action( + ctx = ctx, + dependency_transitive_python_sources = deps_transitive_sources, + ) + + return [ + DefaultInfo(files = output_sources, runfiles = runfiles), + py_info, + create_instrumented_files_info(ctx), + PyCcLinkParamsProvider(cc_info = cc_info), + create_output_group_info(py_info.transitive_sources, extra_groups = {}), + ] + +def create_py_library_rule(*, attrs = {}, **kwargs): + """Creates a py_library rule. + + Args: + attrs: dict of rule attributes. + **kwargs: Additional kwargs to pass onto the rule() call. + Returns: + A rule object + """ + return rule( + attrs = LIBRARY_ATTRS | attrs, + # TODO(b/253818097): fragments=py is only necessary so that + # RequiredConfigFragmentsTest passes + fragments = ["py"], + **kwargs + ) diff --git a/python/private/common/py_library_bazel.bzl b/python/private/common/py_library_bazel.bzl new file mode 100644 index 000000000..b844b97e9 --- /dev/null +++ b/python/private/common/py_library_bazel.bzl @@ -0,0 +1,59 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation of py_library for Bazel.""" + +load( + ":common/python/attributes_bazel.bzl", + "IMPORTS_ATTRS", +) +load( + ":common/python/common.bzl", + "create_library_semantics_struct", + "union_attrs", +) +load( + ":common/python/common_bazel.bzl", + "collect_cc_info", + "get_imports", + "maybe_precompile", +) +load( + ":common/python/py_library.bzl", + "LIBRARY_ATTRS", + "create_py_library_rule", + bazel_py_library_impl = "py_library_impl", +) + +_BAZEL_LIBRARY_ATTRS = union_attrs( + LIBRARY_ATTRS, + IMPORTS_ATTRS, +) + +def create_library_semantics_bazel(): + return create_library_semantics_struct( + get_imports = get_imports, + maybe_precompile = maybe_precompile, + get_cc_info_for_library = collect_cc_info, + ) + +def _py_library_impl(ctx): + return bazel_py_library_impl( + ctx, + semantics = create_library_semantics_bazel(), + ) + +py_library = create_py_library_rule( + implementation = _py_library_impl, + attrs = _BAZEL_LIBRARY_ATTRS, +) diff --git a/python/private/common/py_library_macro.bzl b/python/private/common/py_library_macro.bzl new file mode 100644 index 000000000..729c426f1 --- /dev/null +++ b/python/private/common/py_library_macro.bzl @@ -0,0 +1,19 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation of macro-half of py_library rule.""" + +load(":common/python/py_library_bazel.bzl", py_library_rule = "py_library") + +def py_library(**kwargs): + py_library_rule(**kwargs) diff --git a/python/private/common/py_runtime_macro.bzl b/python/private/common/py_runtime_macro.bzl new file mode 100644 index 000000000..6b27bccfc --- /dev/null +++ b/python/private/common/py_runtime_macro.bzl @@ -0,0 +1,22 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Macro to wrap the py_runtime rule.""" + +load(":common/python/py_runtime_rule.bzl", py_runtime_rule = "py_runtime") + +# NOTE: The function name is purposefully selected to match the underlying +# rule name so that e.g. 'generator_function' shows as the same name so +# that it is less confusing to users. +def py_runtime(**kwargs): + py_runtime_rule(**kwargs) diff --git a/python/private/common/py_runtime_rule.bzl b/python/private/common/py_runtime_rule.bzl new file mode 100644 index 000000000..22efaa6b7 --- /dev/null +++ b/python/private/common/py_runtime_rule.bzl @@ -0,0 +1,214 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation of py_runtime rule.""" + +load(":common/paths.bzl", "paths") +load(":common/python/attributes.bzl", "NATIVE_RULES_ALLOWLIST_ATTRS") +load(":common/python/common.bzl", "check_native_allowed") +load(":common/python/providers.bzl", "DEFAULT_BOOTSTRAP_TEMPLATE", "DEFAULT_STUB_SHEBANG", _PyRuntimeInfo = "PyRuntimeInfo") + +_py_builtins = _builtins.internal.py_builtins + +def _py_runtime_impl(ctx): + check_native_allowed(ctx) + interpreter_path = ctx.attr.interpreter_path or None # Convert empty string to None + interpreter = ctx.file.interpreter + if (interpreter_path and interpreter) or (not interpreter_path and not interpreter): + fail("exactly one of the 'interpreter' or 'interpreter_path' attributes must be specified") + + runtime_files = depset(transitive = [ + t[DefaultInfo].files + for t in ctx.attr.files + ]) + + hermetic = bool(interpreter) + if not hermetic: + if runtime_files: + fail("if 'interpreter_path' is given then 'files' must be empty") + if not paths.is_absolute(interpreter_path): + fail("interpreter_path must be an absolute path") + + if ctx.attr.coverage_tool: + coverage_di = ctx.attr.coverage_tool[DefaultInfo] + + if _py_builtins.is_singleton_depset(coverage_di.files): + coverage_tool = coverage_di.files.to_list()[0] + elif coverage_di.files_to_run and coverage_di.files_to_run.executable: + coverage_tool = coverage_di.files_to_run.executable + else: + fail("coverage_tool must be an executable target or must produce exactly one file.") + + coverage_files = depset(transitive = [ + coverage_di.files, + coverage_di.default_runfiles.files, + ]) + else: + coverage_tool = None + coverage_files = None + + python_version = ctx.attr.python_version + if python_version == "_INTERNAL_SENTINEL": + if ctx.fragments.py.use_toolchains: + fail( + "When using Python toolchains, this attribute must be set explicitly to either 'PY2' " + + "or 'PY3'. See https://github.com/bazelbuild/bazel/issues/7899 for more " + + "information. You can temporarily avoid this error by reverting to the legacy " + + "Python runtime mechanism (`--incompatible_use_python_toolchains=false`).", + ) + else: + python_version = ctx.fragments.py.default_python_version + + # TODO: Uncomment this after --incompatible_python_disable_py2 defaults to true + # if ctx.fragments.py.disable_py2 and python_version == "PY2": + # fail("Using Python 2 is not supported and disabled; see " + + # "https://github.com/bazelbuild/bazel/issues/15684") + + return [ + _PyRuntimeInfo( + interpreter_path = interpreter_path or None, + interpreter = interpreter, + files = runtime_files if hermetic else None, + coverage_tool = coverage_tool, + coverage_files = coverage_files, + python_version = python_version, + stub_shebang = ctx.attr.stub_shebang, + bootstrap_template = ctx.file.bootstrap_template, + ), + DefaultInfo( + files = runtime_files, + runfiles = ctx.runfiles(), + ), + ] + +# Bind to the name "py_runtime" to preserve the kind/rule_class it shows up +# as elsewhere. +py_runtime = rule( + implementation = _py_runtime_impl, + doc = """ +Represents a Python runtime used to execute Python code. + +A `py_runtime` target can represent either a *platform runtime* or an *in-build +runtime*. A platform runtime accesses a system-installed interpreter at a known +path, whereas an in-build runtime points to an executable target that acts as +the interpreter. In both cases, an "interpreter" means any executable binary or +wrapper script that is capable of running a Python script passed on the command +line, following the same conventions as the standard CPython interpreter. + +A platform runtime is by its nature non-hermetic. It imposes a requirement on +the target platform to have an interpreter located at a specific path. An +in-build runtime may or may not be hermetic, depending on whether it points to +a checked-in interpreter or a wrapper script that accesses the system +interpreter. + +# Example + +``` +py_runtime( + name = "python-2.7.12", + files = glob(["python-2.7.12/**"]), + interpreter = "python-2.7.12/bin/python", +) + +py_runtime( + name = "python-3.6.0", + interpreter_path = "/opt/pyenv/versions/3.6.0/bin/python", +) +``` +""", + fragments = ["py"], + attrs = NATIVE_RULES_ALLOWLIST_ATTRS | { + "bootstrap_template": attr.label( + allow_single_file = True, + default = DEFAULT_BOOTSTRAP_TEMPLATE, + doc = """ +The bootstrap script template file to use. Should have %python_binary%, +%workspace_name%, %main%, and %imports%. + +This template, after expansion, becomes the executable file used to start the +process, so it is responsible for initial bootstrapping actions such as finding +the Python interpreter, runfiles, and constructing an environment to run the +intended Python application. + +While this attribute is currently optional, it will become required when the +Python rules are moved out of Bazel itself. + +The exact variable names expanded is an unstable API and is subject to change. +The API will become more stable when the Python rules are moved out of Bazel +itself. + +See @bazel_tools//tools/python:python_bootstrap_template.txt for more variables. +""", + ), + "coverage_tool": attr.label( + allow_files = False, + doc = """ +This is a target to use for collecting code coverage information from `py_binary` +and `py_test` targets. + +If set, the target must either produce a single file or be an executable target. +The path to the single file, or the executable if the target is executable, +determines the entry point for the python coverage tool. The target and its +runfiles will be added to the runfiles when coverage is enabled. + +The entry point for the tool must be loadable by a Python interpreter (e.g. a +`.py` or `.pyc` file). It must accept the command line arguments +of coverage.py (https://coverage.readthedocs.io), at least including +the `run` and `lcov` subcommands. +""", + ), + "files": attr.label_list( + allow_files = True, + doc = """ +For an in-build runtime, this is the set of files comprising this runtime. +These files will be added to the runfiles of Python binaries that use this +runtime. For a platform runtime this attribute must not be set. +""", + ), + "interpreter": attr.label( + allow_single_file = True, + doc = """ +For an in-build runtime, this is the target to invoke as the interpreter. For a +platform runtime this attribute must not be set. +""", + ), + "interpreter_path": attr.string(doc = """ +For a platform runtime, this is the absolute path of a Python interpreter on +the target platform. For an in-build runtime this attribute must not be set. +"""), + "python_version": attr.string( + default = "_INTERNAL_SENTINEL", + values = ["PY2", "PY3", "_INTERNAL_SENTINEL"], + doc = """ +Whether this runtime is for Python major version 2 or 3. Valid values are `"PY2"` +and `"PY3"`. + +The default value is controlled by the `--incompatible_py3_is_default` flag. +However, in the future this attribute will be mandatory and have no default +value. + """, + ), + "stub_shebang": attr.string( + default = DEFAULT_STUB_SHEBANG, + doc = """ +"Shebang" expression prepended to the bootstrapping Python stub script +used when executing `py_binary` targets. + +See https://github.com/bazelbuild/bazel/issues/8685 for +motivation. + +Does not apply to Windows. +""", + ), + }, +) diff --git a/python/private/common/py_test_bazel.bzl b/python/private/common/py_test_bazel.bzl new file mode 100644 index 000000000..fde3a5a47 --- /dev/null +++ b/python/private/common/py_test_bazel.bzl @@ -0,0 +1,55 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule implementation of py_test for Bazel.""" + +load(":common/python/attributes.bzl", "AGNOSTIC_TEST_ATTRS") +load(":common/python/common.bzl", "maybe_add_test_execution_info") +load( + ":common/python/py_executable_bazel.bzl", + "create_executable_rule", + "py_executable_bazel_impl", +) +load(":common/python/semantics.bzl", "TOOLS_REPO") + +_BAZEL_PY_TEST_ATTRS = { + # This *might* be a magic attribute to help C++ coverage work. There's no + # docs about this; see TestActionBuilder.java + "_collect_cc_coverage": attr.label( + default = "@" + TOOLS_REPO + "//tools/test:collect_cc_coverage", + executable = True, + cfg = "exec", + ), + # This *might* be a magic attribute to help C++ coverage work. There's no + # docs about this; see TestActionBuilder.java + "_lcov_merger": attr.label( + default = configuration_field(fragment = "coverage", name = "output_generator"), + cfg = "exec", + executable = True, + ), +} + +def _py_test_impl(ctx): + providers = py_executable_bazel_impl( + ctx = ctx, + is_test = True, + inherited_environment = ctx.attr.env_inherit, + ) + maybe_add_test_execution_info(providers.providers, ctx) + return providers + +py_test = create_executable_rule( + implementation = _py_test_impl, + attrs = AGNOSTIC_TEST_ATTRS | _BAZEL_PY_TEST_ATTRS, + test = True, +) diff --git a/python/private/common/py_test_macro.bzl b/python/private/common/py_test_macro.bzl new file mode 100644 index 000000000..4faede68a --- /dev/null +++ b/python/private/common/py_test_macro.bzl @@ -0,0 +1,21 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation of macro-half of py_test rule.""" + +load(":common/python/common_bazel.bzl", "convert_legacy_create_init_to_int") +load(":common/python/py_test_bazel.bzl", py_test_rule = "py_test") + +def py_test(**kwargs): + convert_legacy_create_init_to_int(kwargs) + py_test_rule(**kwargs) diff --git a/python/private/common/semantics.bzl b/python/private/common/semantics.bzl new file mode 100644 index 000000000..487ff303e --- /dev/null +++ b/python/private/common/semantics.bzl @@ -0,0 +1,34 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains constants that vary between Bazel and Google-internal""" + +IMPORTS_ATTR_SUPPORTED = True + +TOOLS_REPO = "bazel_tools" +PLATFORMS_LOCATION = "@platforms/" + +SRCS_ATTR_ALLOW_FILES = [".py", ".py3"] + +DEPS_ATTR_ALLOW_RULES = None + +PY_RUNTIME_ATTR_NAME = "_py_interpreter" + +BUILD_DATA_SYMLINK_PATH = None + +IS_BAZEL = True + +NATIVE_RULES_MIGRATION_HELP_URL = "https://github.com/bazelbuild/bazel/issues/17773" +NATIVE_RULES_MIGRATION_FIX_CMD = "add_python_loads" + +ALLOWED_MAIN_EXTENSIONS = [".py"]