feat(gazelle)!: Move the plugin to a separate workspace (bazelbuild#972)

feat!(gazelle): Move the plugin to a separate workspace

Summary:
* Move go.mod to gazelle.
* Move gazelle definition.
* Fix file distribution for the gazelle module.
* Update the example test.
* Include rules_python_gazelle_plugin during integration tests
* Update ignored packages
* Update CI configuration

.bazelci/presubmit.yml

@@ -10,17 +10,11 @@ buildifier:
     # As a regression test for #225, check that wheel targets still build when
     # their package path is qualified with the repo name.
     - "@rules_python//examples/wheel/..."
-    - "-//gazelle/..."
   build_flags:
     - "--keep_going"
   test_targets:
     - "--"
     - "..."
-    # The gazelle tests are not compatible with Windows, so we only test them
-    # on Linux. The build file generation, which uses this Gazelle extension,
-    # runs on all platforms, and is asserted by the build_file_generation
-    # integration tests below.
-    - "-//gazelle/..."
   test_flags:
     - "--test_tag_filters=-integration-test"
 .reusable_build_test_all: &reusable_build_test_all
@@ -30,8 +24,9 @@ tasks:
   gazelle_extension:
     name: Test the Gazelle extension
     platform: ubuntu2004
-    build_targets: ["//gazelle/..."]
-    test_targets: ["//gazelle/..."]
+    build_targets: ["//..."]
+    test_targets: ["//..."]
+    working_directory: gazelle
   ubuntu:
     <<: *reusable_config
     name: Default test on Ubuntu

.github/workflows/workspace_snippet.sh

@@ -40,7 +40,7 @@ register_toolchains(
 )
 \`\`\`
 
-## Using WORKSPACE:
+## Using WORKSPACE
 
 Paste this snippet into your \`WORKSPACE\` file:
 
@@ -58,4 +58,18 @@ load("@rules_python//python:repositories.bzl", "py_repositories")
 
 py_repositories()
 \`\`\`
+
+### Gazelle plugin
+
+Paste this snippet into your \`WORKSPACE\` file:
+
+\`\`\`starlark
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+http_archive(
+    name = "rules_python_gazelle_plugin",
+    sha256 = "${SHA}",
+    strip_prefix = "${PREFIX}/gazelle",
+    url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/${TAG}.tar.gz",
+)
+\`\`\`
 EOF

.gitignore

@@ -43,10 +43,5 @@ user.bazelrc
 *.swp
 *.swo
 
-# Go/Gazelle files
-# These otherwise match patterns above
-!go.mod
-!BUILD.out
-
 # Python cache
 **/__pycache__/

BUILD.bazel

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("@bazel_gazelle//:def.bzl", "gazelle")
 load(":version.bzl", "BAZEL_VERSION")
 
 package(default_visibility = ["//visibility:public"])
@@ -32,10 +31,10 @@ filegroup(
         "WORKSPACE",
         "internal_deps.bzl",
         "internal_setup.bzl",
-        "//gazelle:distribution",
         "//python:distribution",
         "//python/pip_install:distribution",
         "//tools:distribution",
+        "@rules_python_gazelle_plugin//:distribution",
     ],
     visibility = [
         "//examples:__pkg__",
@@ -61,23 +60,6 @@ filegroup(
     visibility = ["//visibility:public"],
 )
 
-# Gazelle configuration options.
-# See https://github.com/bazelbuild/bazel-gazelle#running-gazelle-with-bazel
-# gazelle:prefix github.com/bazelbuild/rules_python
-# gazelle:exclude bazel-out
-# gazelle:exclude examples/**
-gazelle(name = "gazelle")
-
-gazelle(
-    name = "gazelle_update_repos",
-    args = [
-        "-from_file=go.mod",
-        "-to_macro=gazelle/deps.bzl%gazelle_deps",
-        "-prune",
-    ],
-    command = "update-repos",
-)
-
 genrule(
     name = "assert_bazelversion",
     srcs = [".bazelversion"],

WORKSPACE

@@ -34,11 +34,6 @@ python_register_multi_toolchains(
     python_versions = MINOR_MAPPING.values(),
 )
 
-load("//gazelle:deps.bzl", "gazelle_deps")
-
-# gazelle:repository_macro gazelle/deps.bzl%gazelle_deps
-gazelle_deps()
-
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 # Used for Bazel CI
@@ -58,3 +53,17 @@ rbe_preconfig(
     name = "buildkite_config",
     toolchain = "ubuntu1804-bazel-java11",
 )
+
+local_repository(
+    name = "rules_python_gazelle_plugin",
+    path = "gazelle",
+)
+
+# The rules_python gazelle extension has some third-party go dependencies
+# which we need to fetch in order to compile it.
+load("@rules_python_gazelle_plugin//:deps.bzl", _py_gazelle_deps = "gazelle_deps")
+
+# See: https://github.com/bazelbuild/rules_python/blob/main/gazelle/README.md
+# This rule loads and compiles various go dependencies that running gazelle
+# for python requirements.
+_py_gazelle_deps()

examples/build_file_generation/BUILD.bazel

@@ -4,11 +4,11 @@
 # ruleset. When the symbol is loaded you can use the rule.
 load("@bazel_gazelle//:def.bzl", "gazelle")
 load("@pip//:requirements.bzl", "all_whl_requirements")
-load("@rules_python//gazelle:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS")
-load("@rules_python//gazelle/manifest:defs.bzl", "gazelle_python_manifest")
-load("@rules_python//gazelle/modules_mapping:def.bzl", "modules_mapping")
 load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
 load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+load("@rules_python_gazelle_plugin//:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS")
+load("@rules_python_gazelle_plugin//manifest:defs.bzl", "gazelle_python_manifest")
+load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping")
 
 compile_pip_requirements(
     name = "requirements",
@@ -53,7 +53,7 @@ gazelle_python_manifest(
 gazelle(
     name = "gazelle",
     data = GAZELLE_PYTHON_RUNTIME_DEPS,
-    gazelle = "@rules_python//gazelle:gazelle_python_binary",
+    gazelle = "@rules_python_gazelle_plugin//python:gazelle_binary",
 )
 
 # This rule is auto-generated and managed by Gazelle,

examples/build_file_generation/WORKSPACE

@@ -82,6 +82,22 @@ local_repository(
 #     url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.16.1.tar.gz",
 # )
 
+# We import the repository-local rules_python_gazelle_plugin version in order to
+# be able to test development changes to the plugin.
+local_repository(
+    name = "rules_python_gazelle_plugin",
+    path = "../../gazelle",
+)
+
+# When loading the gazelle plugin outside this repo, use the http_archive rule as follows:
+#
+#http_archive(
+#    name = "rules_python_gazelle_plugin",
+#    sha256 = "497ca47374f48c8b067d786b512ac10a276211810f4a580178ee9b9ad139323a",
+#    strip_prefix = "rules_python-0.16.1/gazelle",
+#    url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.16.1.tar.gz",
+#)
+
 # Next we load the toolchain from rules_python.
 load("@rules_python//python:repositories.bzl", "python_register_toolchains")
 
@@ -124,7 +140,7 @@ install_deps()
 
 # The rules_python gazelle extension has some third-party go dependencies
 # which we need to fetch in order to compile it.
-load("@rules_python//gazelle:deps.bzl", _py_gazelle_deps = "gazelle_deps")
+load("@rules_python_gazelle_plugin//:deps.bzl", _py_gazelle_deps = "gazelle_deps")
 
 # See: https://github.com/bazelbuild/rules_python/blob/main/gazelle/README.md
 # This rule loads and compiles various go dependencies that running gazelle

examples/build_file_generation/gazelle_python.yaml

@@ -114,4 +114,4 @@ manifest:
     zipp.py310compat: zipp
   pip_repository:
     name: pip
-integrity: 4153df7683d64d7d6ad56c14ea1c7f7bec84a2ddf9ef8f075d1bb9313b8d11aa
+integrity: 2c84a3cabeaff134a1d045e5a173a3178086f236ab20f895ffbd7f3b7a6e5bb0

gazelle/.bazelrc

@@ -0,0 +1,13 @@
+test --test_output=errors
+
+# Do NOT implicitly create empty __init__.py files in the runfiles tree.
+# By default, these are created in every directory containing Python source code
+# or shared libraries, and every parent directory of those directories,
+# excluding the repo root directory. With this flag set, we are responsible for
+# creating (possibly empty) __init__.py files and adding them to the srcs of
+# Python targets as required.
+build --incompatible_default_to_explicit_init_py
+
+# Windows makes use of runfiles for some rules
+build --enable_runfiles
+startup --windows_enable_symlinks

gazelle/.gitignore

@@ -0,0 +1,12 @@
+# Bazel directories
+/bazel-*
+/bazel-bin
+/bazel-genfiles
+/bazel-out
+/bazel-testlogs
+user.bazelrc
+
+# Go/Gazelle files
+# These otherwise match patterns above
+!go.mod
+!BUILD.out

gazelle/BUILD.bazel

@@ -1,22 +1,35 @@
-alias(
-    name = "gazelle",
-    actual = "//gazelle/python",
-    visibility = ["//visibility:public"],
-)
+load("@bazel_gazelle//:def.bzl", "gazelle")
+
+# Gazelle configuration options.
+# See https://github.com/bazelbuild/bazel-gazelle#running-gazelle-with-bazel
+# gazelle:prefix github.com/bazelbuild/rules_python/gazelle
+# gazelle:exclude bazel-out
+gazelle(name = "gazelle")
 
-alias(
-    name = "gazelle_python_binary",
-    actual = "//gazelle/python:gazelle_binary",
-    visibility = ["//visibility:public"],
+gazelle(
+    name = "gazelle_update_repos",
+    args = [
+        "-from_file=go.mod",
+        "-to_macro=deps.bzl%gazelle_deps",
+        "-prune",
+    ],
+    command = "update-repos",
 )
 
 filegroup(
     name = "distribution",
-    srcs = glob(["**"]) + [
-        "//gazelle/manifest:distribution",
-        "//gazelle/modules_mapping:distribution",
-        "//gazelle/python:distribution",
-        "//gazelle/pythonconfig:distribution",
+    srcs = [
+        ":BUILD.bazel",
+        ":README.md",
+        ":WORKSPACE",
+        ":def.bzl",
+        ":deps.bzl",
+        ":go.mod",
+        ":go.sum",
+        "//manifest:distribution",
+        "//modules_mapping:distribution",
+        "//python:distribution",
+        "//pythonconfig:distribution",
     ],
-    visibility = ["//:__pkg__"],
+    visibility = ["@rules_python//:__pkg__"],
 )

gazelle/README.md

@@ -17,9 +17,17 @@ depends on.
 Add this to your `WORKSPACE`:
 
 ```starlark
+http_archive(
+    name = "rules_python_gazelle_plugin",
+    sha256 = "",
+    strip_prefix = "rules_python-0.17.0/gazelle",
+    url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.17.0.tar.gz",
+)
+
 # To compile the rules_python gazelle extension from source,
 # we must fetch some third-party go dependencies that it uses.
-load("@rules_python//gazelle:deps.bzl", _py_gazelle_deps = "gazelle_deps")
+
+load("@rules_python_gazelle_plugin//:deps.bzl", _py_gazelle_deps = "gazelle_deps")
 
 _py_gazelle_deps()
 ```
@@ -40,8 +48,8 @@ To keep the metadata updated, put this in your `BUILD.bazel` file next to `gazel
 
 ```starlark
 load("@pip//:requirements.bzl", "all_whl_requirements")
-load("@rules_python//gazelle/manifest:defs.bzl", "gazelle_python_manifest")
-load("@rules_python//gazelle/modules_mapping:def.bzl", "modules_mapping")
+load("@rules_python_gazelle_plugin//manifest:defs.bzl", "gazelle_python_manifest")
+load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping")
 
 # This rule fetches the metadata for python packages we depend on. That data is
 # required for the gazelle_python_manifest rule to update our manifest file.
@@ -75,7 +83,7 @@ with the rules_python extension included. This typically goes in your root
 
 ```
 load("@bazel_gazelle//:def.bzl", "gazelle")
-load("@rules_python//gazelle:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS")
+load("@rules_python_gazelle_plugin//:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS")
 
 # Our gazelle target points to the python gazelle binary.
 # This is the simple case where we only need one language supported.
@@ -85,7 +93,7 @@ load("@rules_python//gazelle:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS")
 gazelle(
     name = "gazelle",
     data = GAZELLE_PYTHON_RUNTIME_DEPS,
-    gazelle = "@rules_python//gazelle:gazelle_python_binary",
+    gazelle = "@rules_python_gazelle_plugin//python:gazelle_binary",
 )
 ```
 

gazelle/WORKSPACE

@@ -0,0 +1,47 @@
+workspace(name = "rules_python_gazelle_plugin")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "io_bazel_rules_go",
+    sha256 = "099a9fb96a376ccbbb7d291ed4ecbdfd42f6bc822ab77ae6f1b5cb9e914e94fa",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip",
+        "https://github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip",
+    ],
+)
+
+http_archive(
+    name = "bazel_gazelle",
+    sha256 = "448e37e0dbf61d6fa8f00aaa12d191745e14f07c31cabfa731f0c8e8a4f41b97",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.28.0/bazel-gazelle-v0.28.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.28.0/bazel-gazelle-v0.28.0.tar.gz",
+    ],
+)
+
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
+
+go_rules_dependencies()
+
+go_register_toolchains(version = "1.19.4")
+
+gazelle_dependencies()
+
+local_repository(
+    name = "rules_python",
+    path = "..",
+)
+
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+
+python_register_toolchains(
+    name = "python39",
+    python_version = "3.9",
+)
+
+load("//:deps.bzl", _py_gazelle_deps = "gazelle_deps")
+
+# gazelle:repository_macro deps.bzl%gazelle_deps
+_py_gazelle_deps()

gazelle/def.bzl

@@ -2,6 +2,6 @@
 """
 
 GAZELLE_PYTHON_RUNTIME_DEPS = [
-    "@rules_python//gazelle/python:parse",
-    "@rules_python//gazelle/python:std_modules",
+    "@rules_python_gazelle_plugin//python:parse",
+    "@rules_python_gazelle_plugin//python:std_modules",
 ]

go.mod → gazelle/go.mod

@@ -1,4 +1,4 @@
-module github.com/bazelbuild/rules_python
+module github.com/bazelbuild/rules_python/gazelle
 
 go 1.19