+py_binary(name, data, deps, env, imports, main, srcs) ++ + + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this target. | Name | required | | +| data | - | List of labels | optional | [] | +| deps | - | List of labels | optional | [] | +| env | - | Dictionary: String -> String | optional | {} | +| imports | - | List of strings | optional | [] | +| main | - | Label | required | | +| srcs | - | List of labels | optional | [] | + + + + +## py_test + +
+py_test(name, data, deps, env, imports, main, srcs) ++ + + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this target. | Name | required | | +| data | - | List of labels | optional | [] | +| deps | - | List of labels | optional | [] | +| env | - | Dictionary: String -> String | optional | {} | +| imports | - | List of strings | optional | [] | +| main | - | Label | required | | +| srcs | - | List of labels | optional | [] | + + + + +## py_base.implementation + +
+py_base.implementation(ctx) ++ + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| ctx |
-
| none | + + diff --git a/docs/py_library.md b/docs/py_library.md new file mode 100755 index 00000000..6985aadf --- /dev/null +++ b/docs/py_library.md @@ -0,0 +1,101 @@ + + +Implementation for the py_library rule + + + +## py_library + ++py_library(name, data, deps, imports, srcs) ++ + + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this target. | Name | required | | +| data | - | List of labels | optional | [] | +| deps | - | List of labels | optional | [] | +| imports | - | List of strings | optional | [] | +| srcs | - | List of labels | optional | [] | + + + + +## py_library_utils.make_srcs_depset + +
+py_library_utils.make_srcs_depset(ctx) ++ + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| ctx |
-
| none | + + + + +## py_library_utils.make_imports_depset + ++py_library_utils.make_imports_depset(ctx) ++ + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| ctx |
-
| none | + + + + +## py_library_utils.make_merged_runfiles + ++py_library_utils.make_merged_runfiles(ctx, extra_depsets, extra_runfiles, extra_runfiles_depsets) ++ + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| ctx |
-
| none | +| extra_depsets |-
|[] |
+| extra_runfiles | -
|[] |
+| extra_runfiles_depsets | -
|[] |
+
+
+
+
+## py_library_utils.implementation
+
++py_library_utils.implementation(ctx) ++ + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| ctx |
-
| none | + + diff --git a/docs/rules.md b/docs/rules.md index 5193d1c2..4fd2b5d8 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -2,3 +2,87 @@ Public API re-exports + + +## py_wheel + ++py_wheel(name, src) ++ + + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this target. | Name | required | | +| src | - | Label | optional | None | + + + + +## py_binary + +
+py_binary(name, srcs, main, kwargs) ++ +Wrapper macro for the py_binary rule, setting a default for imports. + +It also creates a virtualenv to constrain the interpreter and packages used at runtime, +you can `bazel run [name].venv` to produce this, then use it in the editor. + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| name | name of the rule | none | +| srcs | python source files |
[] |
+| main | the entry point. If absent, then the first entry in srcs is used. | None |
+| kwargs | see [py_binary attributes](./py_binary) | none |
+
+
+
+
+## py_library
+
++py_library(name, kwargs) ++ +Wrapper macro for the py_library rule, setting a default for imports + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| name | name of the rule | none | +| kwargs | see [py_library attributes](./py_library) | none | + + + + +## py_test + +
+py_test(name, main, srcs, kwargs) ++ +Identical to py_binary, but produces a target that can be used with `bazel test`. + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| name |
-
| none | +| main |-
|None |
+| srcs | -
|[] |
+| kwargs | -
| none | + + diff --git a/examples/foo/BUILD.bazel b/examples/foo/BUILD.bazel new file mode 100644 index 00000000..130a2d3f --- /dev/null +++ b/examples/foo/BUILD.bazel @@ -0,0 +1,8 @@ +load("//py:defs.bzl", "py_library") + +py_library( + name = "foo", + srcs = ["__init__.py"], + imports = [".."], + visibility = ["//:__subpackages__"], +) diff --git a/examples/foo/__init__.py b/examples/foo/__init__.py new file mode 100644 index 00000000..39fe20c8 --- /dev/null +++ b/examples/foo/__init__.py @@ -0,0 +1,2 @@ +def get_branding(): + return "rules_py" diff --git a/internal_deps.bzl b/internal_deps.bzl index 5b61b43a..029a9106 100644 --- a/internal_deps.bzl +++ b/internal_deps.bzl @@ -62,10 +62,3 @@ def rules_py_internal_deps(): "https://github.com/bazelbuild/stardoc/releases/download/0.5.0/stardoc-0.5.0.tar.gz", ], ) - - maybe( - http_archive, - name = "aspect_bazel_lib", - sha256 = "8c8cf0554376746e2451de85c4a7670cc8d7400c1f091574c1c1ed2a65021a4c", - url = "https://github.com/aspect-build/bazel-lib/releases/download/v0.2.6/bazel_lib-0.2.6.tar.gz", - ) diff --git a/internal_python_deps.bzl b/internal_python_deps.bzl new file mode 100644 index 00000000..8c5c20eb --- /dev/null +++ b/internal_python_deps.bzl @@ -0,0 +1,16 @@ +"""Our "development" Python dependencies + +Users should *not* need to install these. If users see a load() +statement from these, that's a bug in our distribution. + +These happen after the regular internal dependencies loads as we need to reference the resolved interpreter +""" + +load("@rules_python//python:pip.bzl", "pip_parse") + +def rules_py_internal_pypi_deps(interpreter): + pip_parse( + name = "pypi", + python_interpreter_target = interpreter, + requirements_lock = "//py/tests/external-deps:requirements.txt", + ) diff --git a/py/BUILD.bazel b/py/BUILD.bazel index 60d18870..07e4e641 100644 --- a/py/BUILD.bazel +++ b/py/BUILD.bazel @@ -3,12 +3,6 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") # For stardoc to reference the files exports_files(["defs.bzl"]) -bzl_library( - name = "defs", - srcs = ["defs.bzl"], - visibility = ["//visibility:public"], -) - bzl_library( name = "repositories", srcs = ["repositories.bzl"], @@ -18,3 +12,14 @@ bzl_library( "@bazel_tools//tools/build_defs/repo:utils.bzl", ], ) + +bzl_library( + name = "defs", + srcs = ["defs.bzl"], + visibility = ["//visibility:public"], + deps = [ + "//py/private:py_binary", + "//py/private:py_library", + "//py/private:py_wheel", + ], +) diff --git a/py/defs.bzl b/py/defs.bzl index fff639fd..5816b784 100644 --- a/py/defs.bzl +++ b/py/defs.bzl @@ -1 +1,80 @@ "Public API re-exports" + +load("//py/private:py_binary.bzl", _py_binary = "py_binary", _py_test = "py_test") +load("//py/private:py_library.bzl", _py_library = "py_library") +load("//py/private:py_wheel.bzl", "py_wheel_lib") + +def py_library(name, **kwargs): + """Wrapper macro for the py_library rule, setting a default for imports + + Args: + name: name of the rule + **kwargs: see [py_library attributes](./py_library) + """ + _py_library( + name = name, + imports = kwargs.pop("imports", []) + ["."], + **kwargs + ) + +def py_binary(name, srcs = [], main = None, **kwargs): + """Wrapper macro for the py_binary rule, setting a default for imports. + + It also creates a virtualenv to constrain the interpreter and packages used at runtime, + you can `bazel run [name].venv` to produce this, then use it in the editor. + + Args: + name: name of the rule + srcs: python source files + main: the entry point. If absent, then the first entry in srcs is used. + **kwargs: see [py_binary attributes](./py_binary) + """ + _py_binary( + name = name, + srcs = srcs, + main = main if main != None else srcs[0], + imports = kwargs.pop("imports", []) + ["."], + **kwargs + ) + + native.filegroup( + name = "%s_create_venv_files" % name, + srcs = [name], + tags = ["manual"], + output_group = "create_venv", + ) + + native.sh_binary( + name = "%s.venv" % name, + tags = ["manual"], + srcs = [":%s_create_venv_files" % name], + ) + +def py_test(name, main = None, srcs = [], **kwargs): + "Identical to py_binary, but produces a target that can be used with `bazel test`." + _py_test( + name = name, + srcs = srcs, + main = main if main != None else srcs[0], + imports = kwargs.pop("imports", []) + ["."], + **kwargs + ) + + native.filegroup( + name = "%s_create_venv_files" % name, + srcs = [name], + tags = ["manual"], + output_group = "create_venv", + ) + + native.sh_binary( + name = "%s.venv" % name, + tags = ["manual"], + srcs = [":%s_create_venv_files" % name], + ) + +py_wheel = rule( + implementation = py_wheel_lib.implementation, + attrs = py_wheel_lib.attrs, + provides = py_wheel_lib.provides, +) diff --git a/py/private/BUILD.bazel b/py/private/BUILD.bazel index e69de29b..5b5ef639 100644 --- a/py/private/BUILD.bazel +++ b/py/private/BUILD.bazel @@ -0,0 +1,56 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +exports_files( + [ + "entry.tmpl.sh", + ], + visibility = ["//visibility:public"], +) + +exports_files( + [ + "py_binary.bzl", + "py_library.bzl", + ], + visibility = ["//docs:__pkg__"], +) + +bzl_library( + name = "py_binary", + srcs = ["py_binary.bzl"], + visibility = ["//:__subpackages__"], + deps = [ + ":py_library", + ":utils", + "@aspect_bazel_lib//lib:paths", + ], +) + +bzl_library( + name = "py_library", + srcs = ["py_library.bzl"], + visibility = ["//:__subpackages__"], + deps = [ + ":providers", + "@bazel_skylib//lib:paths", + ], +) + +bzl_library( + name = "providers", + srcs = ["providers.bzl"], + visibility = ["//py:__subpackages__"], +) + +bzl_library( + name = "py_wheel", + srcs = ["py_wheel.bzl"], + visibility = ["//py:__subpackages__"], + deps = [":providers"], +) + +bzl_library( + name = "utils", + srcs = ["utils.bzl"], + visibility = ["//py:__subpackages__"], +) diff --git a/py/private/entry.tmpl.sh b/py/private/entry.tmpl.sh new file mode 100644 index 00000000..3bfd1d31 --- /dev/null +++ b/py/private/entry.tmpl.sh @@ -0,0 +1,139 @@ +#!{{BASH_BIN}} + +{{BASH_RLOCATION_FN}} + +runfiles_export_envvars + +set -o errexit -o nounset -o pipefail + +PWD=$(pwd) + +function alocation { + local P=$1 + if [[ "${P:0:1}" == "/" ]]; then + echo "${P}" + else + echo "${PWD}/${P}" + fi +} + +export BAZEL_WORKSPACE_NAME="{{BAZEL_WORKSPACE_NAME}}" + +function wheel_location { + local P=$1 + if [[ "${P:0:3}" == "../" ]]; then + echo $(rlocation "${P:3}") + else + echo $(rlocation "${BAZEL_WORKSPACE_NAME}/${P}") + fi +} + +export -f wheel_location + +# Resolved from the py_interpreter via PyInterpreterInfo. +PYTHON_LOCATION="$(rlocation {{PYTHON_INTERPRETER_PATH}})" +PYTHON="${PYTHON_LOCATION} {{INTERPRETER_FLAGS}}" +PYTHON_BIN_DIR=$(dirname "${PYTHON}") +PIP_LOCATION="${PYTHON_BIN_DIR}/pip" +PYTHON_SITE_PACKAGES=$(${PYTHON} -c 'import site; print(site.getsitepackages()[0])') +PTH_FILE="$(alocation "$(rlocation {{PTH_FILE}})")" +PIP_FIND_LINKS_SH=$(rlocation {{PIP_FIND_LINKS_SH}}) +PIP_FIND_LINKS=$("${PIP_FIND_LINKS_SH}" | tr '\n' ' ') +ENTRYPOINT="$(rlocation {{BINARY_ENTRY_POINT}})" + +# Convenience vars for the Python virtual env that's created. +RUNFILES_VENV_LOCATION=$(alocation "${RUNFILES_DIR}/{{VENV_NAME}}") +VENV_LOCATION="{{VENV_LOCATION}}" +VBIN_LOCATION="${VENV_LOCATION}/bin" +VPIP_LOCATION="${VBIN_LOCATION}/pip" +VPYTHON="${VBIN_LOCATION}/python3 {{INTERPRETER_FLAGS}}" +VPIP="${VPYTHON} -m pip" + +# Create a virtual env to run inside. This allows us to not have to manipulate the PYTHON_PATH to find external +# dependencies. +# We can also now specify the `-I` (isolated) flag to Python, stopping Python from adding the script path to sys.path[0] +# which we have no control over otherwise. +# This does however have some side effects as now all other PYTHON* env vars are ignored. + +# The venv is intentionally created without pip, as when the venv is created with pip, `ensurepip` is used which will +# use the bundled version of pip, which does not match the version of pip bundled with the interpreter distro. +# So we symlink in this ourselves. +VENV_FLAGS=( + "--without-pip" + "--clear" +) +${PYTHON} -m venv "${VENV_LOCATION}" "${VENV_FLAGS[@]}" + +# Activate the venv, disable changing the prompt +export VIRTUAL_ENV_DISABLE_PROMPT=1 +. "${VBIN_LOCATION}/activate" +unset VIRTUAL_ENV_DISABLE_PROMPT + +# Now symlink in pip from the toolchain +# Also link to `pip` as well as `pip3`. Python venv will also link `pip3.x`, but this seems unnecessary for this use +ln -snf "${PIP_LOCATION}" "${VPIP_LOCATION}" +ln -snf "${VPIP_LOCATION}" "${VBIN_LOCATION}/pip3" + +# Need to symlink in the pip site-packages folder not just the binary. +# Ask Python where the site-packages folder is and symlink the pip package in from the toolchain +VENV_SITE_PACKAGES=$(${VPYTHON} -c 'import site; print(site.getsitepackages()[0])') +ln -snf "${PYTHON_SITE_PACKAGES}/pip" "${VENV_SITE_PACKAGES}/pip" +ln -snf "${PYTHON_SITE_PACKAGES}/_distutils_hack" "${VENV_SITE_PACKAGES}/_distutils_hack" +ln -snf "${PYTHON_SITE_PACKAGES}/setuptools" "${VENV_SITE_PACKAGES}/setuptools" + +INSTALL_WHEELS={{INSTALL_WHEELS}} +if [ "$INSTALL_WHEELS" = true ]; then + # Call to pip to "install" our dependencies. The `find-links` section in the config points to the external downloaded wheels, + # while `--no-index` ensures we don't reach out to PyPi + # We may hit command line length limits if passing a large number of find-links flags, so set them on the PIP_FIND_LINKS env var + export PIP_FIND_LINKS + + # TODO: This can likely be generated by an action up front, but this is fine for now + read -r -a WHEELS <<< "${PIP_FIND_LINKS}" + REQUIREMENTS_FILE="$(mktemp)" + printf "%s\n" "${WHEELS[@]}" > "${REQUIREMENTS_FILE}" + + PIP_FLAGS=( + "--quiet" + "--no-compile" + "--require-virtualenv" + "--no-input" + "--no-cache-dir" + "--disable-pip-version-check" + "--no-python-version-warning" + "--only-binary=:all:" + "--no-dependencies" + "--no-index" + ) + + ${VPIP} install "${PIP_FLAGS[@]}" -r "${REQUIREMENTS_FILE}" + rm "${REQUIREMENTS_FILE}" + + unset PIP_FIND_LINKS +fi + +# Create the site-packages pth file containing all our first party dependency paths. These are from all direct and transitive +# py_library rules. +# The .pth file adds to the interpreters sys.path, without having to set `PYTHONPATH`. This allows us to still +# run with the interpreter with the `-I` flag. This stops some import mechanisms breaking out the sandbox by using +# relative imports. +# This is cat'd in so we don't have to have more fun with runfiles symlink paths. +cat "${PTH_FILE}" > "${VENV_SITE_PACKAGES}/first_party.pth" + +# Set all the env vars here, just before we launch +{{PYTHON_ENV}} + +# We can stop here an not run the py_binary / py_test entrypoint and just create the venv. +# This can be useful for editor support. +RUN_BINARY_ENTRY_POINT={{RUN_BINARY_ENTRY_POINT}} +if [ "$RUN_BINARY_ENTRY_POINT" = true ]; then + # Finally, launch the entrypoint + ${VPYTHON} "${ENTRYPOINT}" -- "$@" +fi + +# Deactivate the venv +deactivate + +# Unset any set env vars +{{PYTHON_ENV_UNSET}} +unset BAZEL_WORKSPACE_NAME diff --git a/py/private/providers.bzl b/py/private/providers.bzl new file mode 100644 index 00000000..407629ec --- /dev/null +++ b/py/private/providers.bzl @@ -0,0 +1,7 @@ +PyWheelInfo = provider( + doc = "Provides information about a Python Wheel", + fields = { + "files": "Depset of all files including deps for this wheel", + "default_runfiles": "Runfiles of all files including deps for this wheel", + }, +) diff --git a/py/private/py_binary.bzl b/py/private/py_binary.bzl new file mode 100644 index 00000000..4fa37d20 --- /dev/null +++ b/py/private/py_binary.bzl @@ -0,0 +1,215 @@ +"Implementation for the py_binary and py_test rules." + +load("@aspect_bazel_lib//lib:paths.bzl", "BASH_RLOCATION_FUNCTION", "to_manifest_path") +load("//py/private:py_library.bzl", _py_library = "py_library_utils") +load("//py/private:providers.bzl", "PyWheelInfo") +load("//py/private:utils.bzl", "dict_to_exports") + +PY_TOOLCHAIN = "@bazel_tools//tools/python:toolchain_type" +SH_TOOLCHAIN = "@bazel_tools//tools/sh:toolchain_type" + +def _strip_external(path): + return path[len("external/"):] if path.startswith("external/") else path + +def _wheel_path_map(file): + return file.short_path + +def _resolve_toolchain(ctx): + toolchain_info = ctx.toolchains[PY_TOOLCHAIN] + + if not toolchain_info.py3_runtime: + fail("A py3_runtime must be set on the Python toolchain") + + py3_toolchain = toolchain_info.py3_runtime + + interpreter_path = None + if py3_toolchain.interpreter_path: + interpreter_path = py3_toolchain.interpreter_path + else: + interpreter_path = to_manifest_path(ctx, py3_toolchain.interpreter) + + if interpreter_path == None: + fail("Unable to resolve a path to the Python interperter") + + return struct( + toolchain = py3_toolchain, + path = interpreter_path, + flags = ["-B", "-s", "-I"], + ) + +def _py_binary_rule_imp(ctx): + bash_bin = ctx.toolchains[SH_TOOLCHAIN].path + interpreter = _resolve_toolchain(ctx) + main = ctx.file.main + + runfiles_files = [] + ctx.files._runfiles_lib + + entry = ctx.actions.declare_file(ctx.attr.name) + env = dict({ + "BAZEL_TARGET": ctx.label, + "BAZEL_WORKSPACE": ctx.workspace_name, + "BAZEL_TARGET_NAME": ctx.attr.name, + }, **ctx.attr.env) + + # Get each path to every wheel we need, this includes the transitive wheels + # As these are just filegroups, then we need to dig into the default_runfiles to get the transitive files + # Create a depset for all these + wheels_depsets = [ + target[PyWheelInfo].files + for target in ctx.attr.deps + if PyWheelInfo in target + ] + wheels_depset = depset( + transitive = wheels_depsets, + ) + + # To avoid calling to_list, and then either creating a lot of extra symlinks or adding a large number + # of find-links flags to pip, we can create a conf file and add a file-links section. + # Create this via the an args action so we can work directly with the depset + pip_find_links_sh = ctx.actions.declare_file("%s.pip.conf.sh" % ctx.attr.name) + runfiles_files.append(pip_find_links_sh) + + find_links_lines = ctx.actions.args() + + # Note the format here is set to multiline so that each line isn't shell quoted + find_links_lines.set_param_file_format(format = "multiline") + + find_links_lines.add("#!%s" % bash_bin) + find_links_lines.add_all(wheels_depset, map_each = _wheel_path_map, format_each = "echo $(wheel_location %s)") + + ctx.actions.write( + output = pip_find_links_sh, + content = find_links_lines, + ) + + # Create a depset from the `imports` depsets, then pass this to Args to create the `.pth` file. + # This avoids having to call `.to_list` on the depset and taking the perf hit. + # We also need to collect our own "imports" attr. + # Can reuse the helper from py_library, as it's the same process + imports_depset = _py_library.make_imports_depset(ctx) + + pth = ctx.actions.declare_file("%s.pth" % ctx.attr.name) + runfiles_files.append(pth) + + pth_lines = ctx.actions.args() + + # The venv is created at the root of the runfiles tree, in 'VENV_NAME', the full path is "${RUNFILES_DIR}/${VENV_NAME}", + # but depending on if we are running as the top level binary or a tool, then $RUNFILES_DIR may be absolute or relative. + # Paths in the .pth are relative to the site-packages folder where they reside. + # All "import" paths from `py_library` start with the workspace name, so we need to go back up the tree for + # each segment from site-packages in the venv to the root of the runfiles tree. + # Four .. will get us back to the root of the venv: + # {name}.runfiles/.{name}.venv/lib/python{version}/site-packages/first_party.pth + escape = ([".."] * 4) + pth_lines.add_all(imports_depset, format_each = "/".join(escape) + "/%s") + + ctx.actions.write( + output = pth, + content = pth_lines, + ) + + common_substitutions = { + "{{BASH_BIN}}": bash_bin, + "{{BASH_RLOCATION_FN}}": BASH_RLOCATION_FUNCTION, + "{{BAZEL_WORKSPACE_NAME}}": ctx.workspace_name, + "{{BINARY_ENTRY_POINT}}": to_manifest_path(ctx, main), + "{{INTERPRETER_FLAGS}}": " ".join(interpreter.flags), + "{{INTERPRETER_FLAGS_PARTS}}": " ".join(['"%s", ' % f for f in interpreter.flags]), + "{{INSTALL_WHEELS}}": str(len(wheels_depsets) > 0).lower(), + "{{PIP_FIND_LINKS_SH}}": to_manifest_path(ctx, pip_find_links_sh), + "{{PTH_FILE}}": to_manifest_path(ctx, pth), + "{{PYTHON_INTERPRETER_PATH}}": interpreter.path, + "{{RUN_BINARY_ENTRY_POINT}}": "true", + "{{VENV_NAME}}": ".%s.venv" % ctx.attr.name, + "{{VENV_LOCATION}}": "${RUNFILES_VENV_LOCATION}", + "{{PYTHON_ENV}}": "\n".join(dict_to_exports(env)).strip(), + "{{PYTHON_ENV_UNSET}}": "\n".join(["unset %s" % k for k in env.keys()]).strip(), + } + + ctx.actions.expand_template( + template = ctx.file._entry, + output = entry, + substitutions = common_substitutions, + is_executable = True, + ) + + create_venv_bin = ctx.actions.declare_file("%s_create_venv.sh" % ctx.attr.name) + ctx.actions.expand_template( + template = ctx.file._entry, + output = create_venv_bin, + substitutions = dict( + common_substitutions, + **{ + "{{RUN_BINARY_ENTRY_POINT}}": "false", + "{{VENV_LOCATION}}": "${BUILD_WORKSPACE_DIRECTORY}/$@", + } + ), + is_executable = True, + ) + + srcs_depset = _py_library.make_srcs_depset(ctx) + + runfiles = _py_library.make_merged_runfiles( + ctx, + extra_depsets = [ + interpreter.toolchain.files, + wheels_depset, + srcs_depset, + ], + extra_runfiles = runfiles_files, + extra_runfiles_depsets = [ + target[PyWheelInfo].default_runfiles + for target in ctx.attr.deps + if PyWheelInfo in target + ], + ) + + return [ + DefaultInfo( + files = depset([entry, main]), + runfiles = runfiles, + executable = entry, + ), + OutputGroupInfo( + create_venv = [create_venv_bin], + ), + # Return PyInfo? + ] + +py_base = struct( + implementation = _py_binary_rule_imp, + attrs = dict({ + "env": attr.string_dict( + default = {}, + ), + "main": attr.label( + allow_single_file = True, + mandatory = True, + ), + "_entry": attr.label( + allow_single_file = True, + default = "//py/private:entry.tmpl.sh", + ), + "_runfiles_lib": attr.label( + default = "@bazel_tools//tools/bash/runfiles", + ), + }, **_py_library.attrs), + toolchains = [ + SH_TOOLCHAIN, + PY_TOOLCHAIN, + ], +) + +py_binary = rule( + implementation = py_base.implementation, + attrs = py_base.attrs, + toolchains = py_base.toolchains, + executable = True, +) + +py_test = rule( + implementation = py_base.implementation, + attrs = py_base.attrs, + toolchains = py_base.toolchains, + test = True, +) diff --git a/py/private/py_library.bzl b/py/private/py_library.bzl new file mode 100644 index 00000000..ba0f69ae --- /dev/null +++ b/py/private/py_library.bzl @@ -0,0 +1,107 @@ +"Implementation for the py_library rule" + +load("@bazel_skylib//lib:paths.bzl", "paths") +load("//py/private:providers.bzl", "PyWheelInfo") + +def _make_srcs_depset(ctx): + return depset( + order = "postorder", + direct = ctx.files.srcs, + transitive = [ + target[PyInfo].transitive_sources + for target in ctx.attr.deps + if PyInfo in target + ], + ) + +def _make_import_path(workspace, base, imp): + if imp.startswith(".."): + return paths.normalize(paths.join(workspace, *base.split("/")[0:-len(imp.split("/"))])) + else: + return paths.normalize(paths.join(workspace, base, imp)) + +def _make_imports_depset(ctx): + base = paths.dirname(ctx.build_file_path) + import_paths = [ + _make_import_path(ctx.workspace_name, base, im) + for im in ctx.attr.imports + ] + + return depset( + direct = import_paths, + transitive = [ + target[PyInfo].imports + for target in ctx.attr.deps + if PyInfo in target + ], + ) + +def _make_merged_runfiles(ctx, extra_depsets = [], extra_runfiles = [], extra_runfiles_depsets = []): + runfiles_targets = ctx.attr.deps + ctx.attr.data + runfiles = ctx.runfiles( + files = ctx.files.data + extra_runfiles, + transitive_files = depset( + transitive = extra_depsets, + ), + ) + + runfiles = runfiles.merge_all([ + target[DefaultInfo].default_runfiles + for target in runfiles_targets + ] + extra_runfiles_depsets) + + return runfiles + +def _py_library_impl(ctx): + transitive_srcs = _make_srcs_depset(ctx) + imports = _make_imports_depset(ctx) + runfiles = _make_merged_runfiles(ctx) + + return [ + DefaultInfo( + files = depset(direct = ctx.files.srcs, transitive = [transitive_srcs]), + default_runfiles = runfiles, + ), + PyInfo( + imports = imports, + transitive_sources = transitive_srcs, + has_py2_only_sources = False, + has_py3_only_sources = True, + uses_shared_libraries = False, + ), + ] + +_attrs = dict({ + "srcs": attr.label_list( + allow_files = True, + ), + "deps": attr.label_list( + allow_files = True, + # Ideally we'd have a PyWheelInfo provider here so we can restrict the dependency set + providers = [[PyInfo], [PyWheelInfo]], + ), + "data": attr.label_list( + allow_files = True, + ), + "imports": attr.string_list(), +}) + +_providers = [ + DefaultInfo, + PyInfo, +] + +py_library_utils = struct( + make_srcs_depset = _make_srcs_depset, + make_imports_depset = _make_imports_depset, + make_merged_runfiles = _make_merged_runfiles, + implementation = _py_library_impl, + attrs = _attrs, + py_library_providers = _providers, +) + +py_library = rule( + implementation = py_library_utils.implementation, + attrs = py_library_utils.attrs, + provides = py_library_utils.py_library_providers, +) diff --git a/py/private/py_wheel.bzl b/py/private/py_wheel.bzl new file mode 100644 index 00000000..6601623d --- /dev/null +++ b/py/private/py_wheel.bzl @@ -0,0 +1,31 @@ +"Represent a python wheel file" + +load("//py/private:providers.bzl", "PyWheelInfo") + +_ATTRS = { + "src": attr.label( + allow_files = [".whl"], + ), +} + +def _make_py_wheel_info_from_filegroup(wheel_filegroup): + files_depsets = [] + files_depsets.append(wheel_filegroup[DefaultInfo].files) + files_depsets.append(wheel_filegroup[DefaultInfo].default_runfiles.files) + + return PyWheelInfo( + files = depset(transitive = files_depsets), + default_runfiles = wheel_filegroup[DefaultInfo].default_runfiles, + ) + +def _py_wheel_impl(ctx): + py_wheel_info = _make_py_wheel_info_from_filegroup(ctx.attr.src) + return [ + py_wheel_info, + ] + +py_wheel_lib = struct( + implementation = _py_wheel_impl, + attrs = _ATTRS, + provides = [PyWheelInfo], +) diff --git a/py/private/utils.bzl b/py/private/utils.bzl new file mode 100644 index 00000000..0be9fa60 --- /dev/null +++ b/py/private/utils.bzl @@ -0,0 +1,5 @@ +def dict_to_exports(env): + return [ + "export %s=\"%s\"" % (k, v) + for (k, v) in env.items() + ] diff --git a/py/repositories.bzl b/py/repositories.bzl index d9a05d9b..e205fd3d 100644 --- a/py/repositories.bzl +++ b/py/repositories.bzl @@ -28,17 +28,15 @@ def rules_py_dependencies(): maybe( http_archive, name = "aspect_bazel_lib", - sha256 = "5f5f1237601d41d61608ad0b9541614935839232940010f9e62163c3e53dc1b7", - strip_prefix = "bazel-lib-0.5.0", - url = "https://github.com/aspect-build/bazel-lib/archive/refs/tags/v0.5.0.tar.gz", + sha256 = "b3de6702d48904e8dbe9b45d29e5f07d3258d826981fda87424462b36f16b35f", + strip_prefix = "bazel-lib-0.8.3", + url = "https://github.com/aspect-build/bazel-lib/archive/refs/tags/v0.8.3.tar.gz", ) maybe( http_archive, name = "rules_python", - sha256 = "6c719f2981f47c5d9d0fe9ffe5d4a1cf4dbdc4c3f248d5631a3fc05af563e88e", - strip_prefix = "651ce7f4e620a8419cd051f6a7ebc11a67adc556", - # This is the HEAD of https://github.com/bazelbuild/rules_python/pull/618 as of 2022 March 4 - # TODO: replace with rules_python 0.7.0 when released - url = "https://github.com/bazelbuild/rules_python/archive/651ce7f4e620a8419cd051f6a7ebc11a67adc556.zip", + sha256 = "9fcf91dbcc31fde6d1edb15f117246d912c33c36f44cf681976bd886538deba6", + strip_prefix = "rules_python-0.8.0", + url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.8.0.tar.gz", ) diff --git a/py/tests/BUILD.bazel b/py/tests/BUILD.bazel deleted file mode 100644 index e69de29b..00000000 diff --git a/py/tests/external-deps/BUILD.bazel b/py/tests/external-deps/BUILD.bazel new file mode 100644 index 00000000..0fc6c9ac --- /dev/null +++ b/py/tests/external-deps/BUILD.bazel @@ -0,0 +1,47 @@ +load("@rules_python//python/pip_install:requirements.bzl", "compile_pip_requirements") +load("//py:defs.bzl", "py_binary", "py_library", "py_wheel") +load("@python_toolchain//:defs.bzl", "host_platform") +load("@aspect_bazel_lib//lib:write_source_files.bzl", "write_source_files") + +compile_pip_requirements( + name = "requirements", + requirements_in = "requirements.in", + requirements_txt = "requirements.txt", +) + +py_wheel( + name = "django", + src = "@pypi_django//:whl", +) + +py_library( + name = "lib", + srcs = ["lib.py"], +) + +py_binary( + name = "main", + srcs = ["__main__.py"], + deps = [ + ":django", + ":lib", + ], +) + +genrule( + name = "run_main", + outs = ["out"], + cmd = "|".join([ + "$(execpath main)", + """sed "s#$$(pwd)#(pwd)#" """, + "sed 's#^.*execroot/aspect_rules_py/external/python_toolchain_%s#(py_toolchain)#'" % host_platform, + ]) + "> $(execpath out)", + tools = ["main"], +) + +write_source_files( + name = "test", + files = { + "expected": "out", + }, +) diff --git a/py/tests/external-deps/__main__.py b/py/tests/external-deps/__main__.py new file mode 100644 index 00000000..07630027 --- /dev/null +++ b/py/tests/external-deps/__main__.py @@ -0,0 +1,22 @@ +import os +import site +import sys +import django + +print(f'Python: {sys.executable}') +print(f'version: {sys.version}') +print(f'version info: {sys.version_info}') +print(f'cwd: {os.getcwd()}') +print(f'site-packages folder: {site.getsitepackages()}') + +print('\nsys path:') +for entry in sys.path: + print(entry) + +print(f'\nEntrypoint Path: {__file__}') + +print(f'\nDjango location: {django.__file__}') +print(f'Django version: {django.__version__}') + +from lib import greet +print(greet("Matt")) diff --git a/py/tests/external-deps/expected b/py/tests/external-deps/expected new file mode 100755 index 00000000..aa9aaee2 --- /dev/null +++ b/py/tests/external-deps/expected @@ -0,0 +1,19 @@ +Python: (pwd)/bazel-out/host/bin/py/tests/external-deps/main.runfiles/.main.venv/bin/python3 +version: 3.9.10 (main, Feb 27 2022, 23:39:20) +[Clang 13.0.1 ] +version info: sys.version_info(major=3, minor=9, micro=10, releaselevel='final', serial=0) +cwd: (pwd) +site-packages folder: ['(pwd)/bazel-out/host/bin/py/tests/external-deps/main.runfiles/.main.venv/lib/python3.9/site-packages'] + +sys path: +(py_toolchain)/lib/python39.zip +(py_toolchain)/lib/python3.9 +(py_toolchain)/lib/python3.9/lib-dynload +(pwd)/bazel-out/host/bin/py/tests/external-deps/main.runfiles/.main.venv/lib/python3.9/site-packages +(pwd)/bazel-out/host/bin/py/tests/external-deps/main.runfiles/aspect_rules_py/py/tests/external-deps + +Entrypoint Path: (pwd)/bazel-out/host/bin/py/tests/external-deps/main.runfiles/aspect_rules_py/py/tests/external-deps/__main__.py + +Django location: (pwd)/bazel-out/host/bin/py/tests/external-deps/main.runfiles/.main.venv/lib/python3.9/site-packages/django/__init__.py +Django version: 4.0.2 +Hello Matt diff --git a/py/tests/external-deps/lib.py b/py/tests/external-deps/lib.py new file mode 100644 index 00000000..2ba5041e --- /dev/null +++ b/py/tests/external-deps/lib.py @@ -0,0 +1,2 @@ +def greet(name): + return f"Hello {name}" diff --git a/py/tests/external-deps/requirements.in b/py/tests/external-deps/requirements.in new file mode 100644 index 00000000..57ac677c --- /dev/null +++ b/py/tests/external-deps/requirements.in @@ -0,0 +1 @@ +django==4.0.2 \ No newline at end of file diff --git a/py/tests/external-deps/requirements.txt b/py/tests/external-deps/requirements.txt new file mode 100644 index 00000000..49ffa9f3 --- /dev/null +++ b/py/tests/external-deps/requirements.txt @@ -0,0 +1,18 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# bazel run //py/tests/external-deps:requirements.update +# +asgiref==3.5.0 \ + --hash=sha256:2f8abc20f7248433085eda803936d98992f1343ddb022065779f37c5da0181d0 \ + --hash=sha256:88d59c13d634dcffe0510be048210188edd79aeccb6a6c9028cdad6f31d730a9 + # via django +django==4.0.2 \ + --hash=sha256:110fb58fb12eca59e072ad59fc42d771cd642dd7a2f2416582aa9da7a8ef954a \ + --hash=sha256:996495c58bff749232426c88726d8cd38d24c94d7c1d80835aafffa9bc52985a + # via -r py/tests/external-deps/requirements.in +sqlparse==0.4.2 \ + --hash=sha256:0c00730c74263a94e5a9919ade150dfc3b19c574389985446148402998287dae \ + --hash=sha256:48719e356bb8b42991bdbb1e8b83223757b93789c00910a616a071910ca4a64d + # via django