Gazelle extension for Python

.bazelrc

@@ -3,7 +3,15 @@
 # This lets us glob() up all the files inside the examples to make them inputs to tests
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh
-build --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import,examples/relative_requirements
-query --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import,examples/relative_requirements
+build --deleted_packages=examples/build_file_generation,examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import,examples/relative_requirements
+query --deleted_packages=examples/build_file_generation,examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import,examples/relative_requirements
 
 test --test_output=errors
+
+# Do NOT implicitly create empty __init__.py files in the runfiles tree.
+# By default, these are created in every directory containing Python source code
+# or shared libraries, and every parent directory of those directories,
+# excluding the repo root directory. With this flag set, we are responsible for
+# creating (possibly empty) __init__.py files and adding them to the srcs of
+# Python targets as required.
+build --incompatible_default_to_explicit_init_py

.github/CODEOWNERS

@@ -10,6 +10,9 @@
 /python/whl.bzl @thundergolfer @andyscott
 /python/requirements.txt @thundergolfer @andyscott
 
+# Directory containing the Gazelle extension and Go code.
+/gazelle/ @f0rmiga
+
 # The proposals dir corresponds to the Bazel proposals process, documented
 # here: https://bazel.build/designs/index.html
 /proposals/ @brandjon @lberki

.gitignore

@@ -41,3 +41,8 @@
 # vim swap files
 *.swp
 *.swo
+
+# Go/Gazelle files
+# These otherwise match patterns above
+!go.mod
+!BUILD.out

BUILD

@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+load("@bazel_gazelle//:def.bzl", "gazelle")
+
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # Apache 2.0
@@ -51,3 +53,19 @@ filegroup(
     ],
     visibility = ["//visibility:public"],
 )
+
+# Gazelle configuration options.
+# See https://github.com/bazelbuild/bazel-gazelle#running-gazelle-with-bazel
+# gazelle:prefix github.com/bazelbuild/rules_python
+# gazelle:exclude bazel-out
+gazelle(name = "gazelle")
+
+gazelle(
+    name = "update_go_deps",
+    args = [
+        "-from_file=go.mod",
+        "-to_macro=gazelle/deps.bzl%gazelle_deps",
+        "-prune",
+    ],
+    command = "update-repos",
+)

WORKSPACE

@@ -24,3 +24,8 @@ rules_python_internal_deps()
 load("//:internal_setup.bzl", "rules_python_internal_setup")
 
 rules_python_internal_setup()
+
+load("//gazelle:deps.bzl", "gazelle_deps")
+
+# gazelle:repository_macro gazelle/deps.bzl%gazelle_deps
+gazelle_deps()

examples/build_file_generation/.bazelversion

@@ -0,0 +1 @@
+4.0.0

examples/build_file_generation/.gitignore

@@ -0,0 +1 @@
+bazel-*

examples/build_file_generation/BUILD

@@ -0,0 +1,37 @@
+load("@bazel_gazelle//:def.bzl", "gazelle")
+load("@rules_python//gazelle:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS")
+load("@rules_python//gazelle/manifest:defs.bzl", "gazelle_python_manifest")
+load("@rules_python//python:defs.bzl", "py_library")
+
+# Gazelle python extension needs a manifest file mapping from
+# an import to the installed package that provides it.
+# This macro produces two targets:
+# - //:gazelle_python_manifest.update can be used with `bazel run`
+#   to recalculate the manifest
+# - //:gazelle_python_manifest.test is a test target ensuring that
+#   the manifest doesn't need to be updated
+gazelle_python_manifest(
+    name = "gazelle_python_manifest",
+    modules_mapping = "@modules_map//:modules_mapping.json",
+    pip_deps_repository_name = "pip",
+    requirements = "//:requirements_lock.txt",
+)
+
+# Our gazelle target points to the python gazelle binary.
+# This is the simple case where we only need one language supported.
+# If you also had proto, go, or other gazelle-supported languages,
+# you would also need a gazelle_binary rule.
+# See https://github.com/bazelbuild/bazel-gazelle/blob/master/extend.rst#example
+gazelle(
+    name = "gazelle",
+    data = GAZELLE_PYTHON_RUNTIME_DEPS,
+    gazelle = "@rules_python//gazelle:gazelle_python_binary",
+)
+
+# This rule is auto-generated and managed by Gazelle,
+# because it found the __init__.py file in this folder.
+py_library(
+    name = "build_file_generation",
+    srcs = ["__init__.py"],
+    visibility = ["//:__subpackages__"],
+)

examples/build_file_generation/README.md

@@ -0,0 +1,20 @@
+# Build file generation with Gazelle
+
+This example shows a project that has Gazelle setup with the rules_python
+extension, so that targets like `py_library` and `py_binary` can be
+automatically created just by running
+
+```sh
+$ bazel run //:gazelle
+```
+
+As a demo, try creating a `__main__.py` file in this directory, then
+re-run that gazelle command. You'll see that a `py_binary` target
+is created in the `BUILD` file.
+
+Or, try importing the `requests` library in `__init__.py`.
+You'll see that `deps = ["@pip//pypi__requests"]` is automatically
+added to the `py_library` target in the `BUILD` file.
+
+For more information on the behavior of the rules_python gazelle extension,
+see the README.md file in the /gazelle folder.

examples/build_file_generation/WORKSPACE

@@ -0,0 +1,73 @@
+workspace(name = "build_file_generation_example")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+######################################################################
+# We need rules_go and bazel_gazelle, to build the gazelle plugin from source.
+# Setup instructions for this section are at
+# https://github.com/bazelbuild/bazel-gazelle#running-gazelle-with-bazel
+
+# Note, you could omit the rules_go dependency, if you have some way to statically
+# compile the gazelle binary for your workspace and distribute it to users on all
+# needed platforms.
+http_archive(
+    name = "io_bazel_rules_go",
+    sha256 = "69de5c704a05ff37862f7e0f5534d4f479418afc21806c887db544a316f3cb6b",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.27.0/rules_go-v0.27.0.tar.gz",
+        "https://github.com/bazelbuild/rules_go/releases/download/v0.27.0/rules_go-v0.27.0.tar.gz",
+    ],
+)
+
+# NB: bazel-gazelle version must be after 18 August 2021
+# to include https://github.com/bazelbuild/bazel-gazelle/commit/2834ea4
+http_archive(
+    name = "bazel_gazelle",
+    sha256 = "0bb8056ab9ed4cbcab5b74348d8530c0e0b939987b0cfe36c1ab53d35a99e4de",
+    strip_prefix = "bazel-gazelle-2834ea44b3ec6371c924baaf28704730ec9d4559",
+    urls = [
+        # No release since March, and we need subsequent fixes
+        "https://github.com/bazelbuild/bazel-gazelle/archive/2834ea44b3ec6371c924baaf28704730ec9d4559.zip",
+    ],
+)
+
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
+
+go_rules_dependencies()
+
+go_register_toolchains(version = "1.16.5")
+
+gazelle_dependencies()
+
+######################################################################
+# Remaining setup is for rules_python
+
+local_repository(
+    name = "rules_python",
+    path = "../..",
+)
+
+load("@rules_python//python:pip.bzl", "pip_install")
+
+pip_install(
+    # Uses the default repository name "pip"
+    requirements = "//:requirements_lock.txt",
+)
+
+# The rules_python gazelle extension has some third-party go dependencies
+# which we need to fetch in order to compile it.
+load("@rules_python//gazelle:deps.bzl", _py_gazelle_deps = "gazelle_deps")
+
+_py_gazelle_deps()
+
+load("@rules_python//gazelle/modules_mapping:def.bzl", "modules_mapping")
+
+# This repository rule fetches the metadata for python packages we
+# depend on. That data is required for the gazelle_python_manifest
+# rule to update our manifest file.
+# To see what this rule does, try `bazel run @modules_map//:print`
+modules_mapping(
+    name = "modules_map",
+    requirements = "//:requirements_lock.txt",
+)

examples/build_file_generation/__init__.py

@@ -0,0 +1 @@
+print("hello")

examples/build_file_generation/gazelle_python.yaml

@@ -0,0 +1,130 @@
+# GENERATED FILE - DO NOT EDIT!
+#
+# To update this file, run:
+#   bazel run //:gazelle_python_manifest.update
+
+manifest:
+  modules_mapping:
+    certifi: certifi
+    certifi.__init__: certifi
+    certifi.__main__: certifi
+    certifi.core: certifi
+    chardet: chardet
+    chardet.__init__: chardet
+    chardet.big5freq: chardet
+    chardet.big5prober: chardet
+    chardet.chardistribution: chardet
+    chardet.charsetgroupprober: chardet
+    chardet.charsetprober: chardet
+    chardet.cli: chardet
+    chardet.cli.__init__: chardet
+    chardet.cli.chardetect: chardet
+    chardet.codingstatemachine: chardet
+    chardet.compat: chardet
+    chardet.cp949prober: chardet
+    chardet.enums: chardet
+    chardet.escprober: chardet
+    chardet.escsm: chardet
+    chardet.eucjpprober: chardet
+    chardet.euckrfreq: chardet
+    chardet.euckrprober: chardet
+    chardet.euctwfreq: chardet
+    chardet.euctwprober: chardet
+    chardet.gb2312freq: chardet
+    chardet.gb2312prober: chardet
+    chardet.hebrewprober: chardet
+    chardet.jisfreq: chardet
+    chardet.jpcntx: chardet
+    chardet.langbulgarianmodel: chardet
+    chardet.langcyrillicmodel: chardet
+    chardet.langgreekmodel: chardet
+    chardet.langhebrewmodel: chardet
+    chardet.langhungarianmodel: chardet
+    chardet.langthaimodel: chardet
+    chardet.langturkishmodel: chardet
+    chardet.latin1prober: chardet
+    chardet.mbcharsetprober: chardet
+    chardet.mbcsgroupprober: chardet
+    chardet.mbcssm: chardet
+    chardet.sbcharsetprober: chardet
+    chardet.sbcsgroupprober: chardet
+    chardet.sjisprober: chardet
+    chardet.universaldetector: chardet
+    chardet.utf8prober: chardet
+    chardet.version: chardet
+    idna: idna
+    idna.__init__: idna
+    idna.codec: idna
+    idna.compat: idna
+    idna.core: idna
+    idna.idnadata: idna
+    idna.intranges: idna
+    idna.package_data: idna
+    idna.uts46data: idna
+    requests: requests
+    requests.__init__: requests
+    requests.__version__: requests
+    requests._internal_utils: requests
+    requests.adapters: requests
+    requests.api: requests
+    requests.auth: requests
+    requests.certs: requests
+    requests.compat: requests
+    requests.cookies: requests
+    requests.exceptions: requests
+    requests.help: requests
+    requests.hooks: requests
+    requests.models: requests
+    requests.packages: requests
+    requests.sessions: requests
+    requests.status_codes: requests
+    requests.structures: requests
+    requests.utils: requests
+    urllib3: urllib3
+    urllib3.__init__: urllib3
+    urllib3._collections: urllib3
+    urllib3._version: urllib3
+    urllib3.connection: urllib3
+    urllib3.connectionpool: urllib3
+    urllib3.contrib: urllib3
+    urllib3.contrib.__init__: urllib3
+    urllib3.contrib._appengine_environ: urllib3
+    urllib3.contrib._securetransport: urllib3
+    urllib3.contrib._securetransport.__init__: urllib3
+    urllib3.contrib._securetransport.bindings: urllib3
+    urllib3.contrib._securetransport.low_level: urllib3
+    urllib3.contrib.appengine: urllib3
+    urllib3.contrib.ntlmpool: urllib3
+    urllib3.contrib.pyopenssl: urllib3
+    urllib3.contrib.securetransport: urllib3
+    urllib3.contrib.socks: urllib3
+    urllib3.exceptions: urllib3
+    urllib3.fields: urllib3
+    urllib3.filepost: urllib3
+    urllib3.packages: urllib3
+    urllib3.packages.__init__: urllib3
+    urllib3.packages.backports: urllib3
+    urllib3.packages.backports.__init__: urllib3
+    urllib3.packages.backports.makefile: urllib3
+    urllib3.packages.six: urllib3
+    urllib3.packages.ssl_match_hostname: urllib3
+    urllib3.packages.ssl_match_hostname.__init__: urllib3
+    urllib3.packages.ssl_match_hostname._implementation: urllib3
+    urllib3.poolmanager: urllib3
+    urllib3.request: urllib3
+    urllib3.response: urllib3
+    urllib3.util: urllib3
+    urllib3.util.__init__: urllib3
+    urllib3.util.connection: urllib3
+    urllib3.util.proxy: urllib3
+    urllib3.util.queue: urllib3
+    urllib3.util.request: urllib3
+    urllib3.util.response: urllib3
+    urllib3.util.retry: urllib3
+    urllib3.util.ssl_: urllib3
+    urllib3.util.ssltransport: urllib3
+    urllib3.util.timeout: urllib3
+    urllib3.util.url: urllib3
+    urllib3.util.wait: urllib3
+  pip_deps_repository_name: pip
+integrity: 575d259c512b4b80f9923d1623d2aae3038654b731a4e088bf268e01138b6411

examples/build_file_generation/requirements.txt

@@ -0,0 +1 @@
+requests==2.25.1

examples/build_file_generation/requirements_lock.txt

@@ -0,0 +1,26 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes --output-file=requirements_lock.txt requirements.txt
+#
+certifi==2020.12.5 \
+    --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \
+    --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830
+    # via requests
+chardet==3.0.4 \
+    --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
+    --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691
+    # via requests
+idna==2.10 \
+    --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \
+    --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0
+    # via requests
+requests==2.25.1 \
+    --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \
+    --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e
+    # via -r requirements.txt
+urllib3==1.26.5 \
+    --hash=sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c \
+    --hash=sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098
+    # via requests