refactor: consume tools from source if unstamped

.bazelrc

@@ -2,14 +2,14 @@
 # Settings that apply only to CI are in .aspect/bazelrc/ci.bazelrc
 
 # Import Aspect bazelrc presets
-import %workspace%/.aspect/bazelrc/convenience.bazelrc
-import %workspace%/.aspect/bazelrc/correctness.bazelrc
-import %workspace%/.aspect/bazelrc/performance.bazelrc
-import %workspace%/.aspect/bazelrc/debug.bazelrc
-import %workspace%/.aspect/bazelrc/javascript.bazelrc
+import .aspect/bazelrc/convenience.bazelrc
+import .aspect/bazelrc/correctness.bazelrc
+import .aspect/bazelrc/performance.bazelrc
+import .aspect/bazelrc/debug.bazelrc
+import .aspect/bazelrc/javascript.bazelrc
 
 # Use a try-import for Bazel 6 settings since this repository is also tested against Bazel 5 on CI
-try-import %workspace%/.aspect/bazelrc/bazel6.bazelrc
+try-import .aspect/bazelrc/bazel6.bazelrc
 
 ### PROJECT SPECIFIC OPTIONS ###
 
@@ -19,10 +19,6 @@ build --embed_label=v1.2.3
 # Mock versioning command to test the --stamp behavior
 build --workspace_status_command="echo BUILD_SCM_VERSION 1.2.3"
 
-# For testing always use toolchains from source
-build --@aspect_bazel_lib//tools:use_source_toolchains
-build --extra_toolchains=//tools:all
-
 # For releasing, use --workspace_status_command and stamp
 common:release --workspace_status_command "${PWD}/workspace_status.sh"
 common:release -c opt

.github/workflows/release.yml

@@ -27,11 +27,6 @@ jobs:
           # Bazelisk will download bazel to here
           XDG_CACHE_HOME: ~/.cache/bazel-repo
         run: bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc test //...
-      - name: Check Versions
-        env:
-          # Bazelisk will download bazel to here
-          XDG_CACHE_HOME: ~/.cache/bazel-repo
-        run: ./scripts/check_versions.sh ${{ env.GITHUB_REF_NAME }}
       - name: Build release artifacts
         run: |
           if [ -n "$(git status --porcelain)" ]; then

.github/workflows/release_prep.sh

@@ -7,6 +7,10 @@ set -o errexit -o nounset -o pipefail
 cat >.git/info/attributes <<EOF
 # Omit folders that users don't need, making the distribution artifact smaller
 lib/tests export-ignore
+e2e export-ignore
+
+# Substitution for the _VERSION_PRIVATE placeholder at the top of this file
+tools/version.bzl export-subst
 EOF
 
 # Set by GH actions, see

.pre-commit-config.yaml

@@ -38,11 +38,3 @@ repos:
         exclude: &exclude_pattern "^(docs|e2e|lib/tests)/"
       - id: mixed-line-ending
       - id: trailing-whitespace
-
-  - repo: local
-    hooks:
-      - id: check-versions
-        name: Check Versions
-        entry: ./scripts/check_versions.sh
-        language: script
-        stages: [pre-push]

MODULE.bazel

@@ -25,6 +25,21 @@ register_toolchains(
     "@expand_template_toolchains//:all",
 )
 
+# To allow /tools to be built from source
+bazel_dep(name = "rules_go", version = "0.41.0", repo_name = "io_bazel_rules_go")
+bazel_dep(name = "gazelle", version = "0.33.0")
+
+
+go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
+go_deps.from_file(go_mod = "//:go.mod")
+
+use_repo(
+    go_deps,
+    "org_golang_x_exp",
+    "com_github_bmatcuk_doublestar_v4"
+)
+
+
 # Development-only dependencies
 
 bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.4.1", dev_dependency = True)

WORKSPACE

@@ -29,22 +29,27 @@ load("//lib:host_repo.bzl", "host_repo")
 
 host_repo(name = "aspect_bazel_lib_host")
 
+############################################
+# rules_go
+
 load("//:deps.bzl", "go_dependencies")
 
 # gazelle:repository_macro deps.bzl%go_dependencies
 # gazelle:repository go_repository name=org_golang_x_tools importpath=golang.org/x/tools
 # https://github.com/bazelbuild/bazel-gazelle/issues/1217#issuecomment-1152236735
 go_dependencies()
 
-############################################
-# Gazelle, for generating bzl_library targets
 load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
-load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
 
 go_rules_dependencies()
 
 go_register_toolchains(version = "1.18.3")
 
+############################################
+# Gazelle, for generating bzl_library targets
+
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+
 gazelle_dependencies()
 
 # Buildifier

deps.bzl

@@ -24,8 +24,8 @@ def go_dependencies():
         name = "org_golang_x_exp",
         build_file_proto_mode = "disable_global",
         importpath = "golang.org/x/exp",
-        sum = "h1:m9O6OTJ627iFnN2JIWfdqlZCzneRO6EEBsHXI25P8ws=",
-        version = "v0.0.0-20221230185412-738e83a70c30",
+        sum = "h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw=",
+        version = "v0.0.0-20230713183714-613f0c0eb8a1",
     )
     go_repository(
         name = "org_golang_x_mod",

e2e/copy_to_directory/WORKSPACE

@@ -6,3 +6,24 @@ local_repository(
 load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies")
 
 aspect_bazel_lib_dependencies()
+
+############################################
+# rules_go is needed to consume tools from sources
+
+load("@aspect_bazel_lib//:internal_deps.bzl", "bazel_lib_internal_deps")
+
+bazel_lib_internal_deps()
+
+load("@aspect_bazel_lib//:deps.bzl", "go_dependencies")
+
+go_dependencies()
+
+load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
+
+go_rules_dependencies()
+
+go_register_toolchains(version = "1.18.3")
+
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+
+gazelle_dependencies()

e2e/smoke/WORKSPACE

@@ -10,3 +10,24 @@ aspect_bazel_lib_dependencies()
 register_jq_toolchains()
 
 register_yq_toolchains()
+
+############################################
+# rules_go is needed to consume tools from sources
+
+load("@aspect_bazel_lib//:internal_deps.bzl", "bazel_lib_internal_deps")
+
+bazel_lib_internal_deps()
+
+load("@aspect_bazel_lib//:deps.bzl", "go_dependencies")
+
+go_dependencies()
+
+load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
+
+go_rules_dependencies()
+
+go_register_toolchains(version = "1.18.3")
+
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+
+gazelle_dependencies()

lib/BUILD.bazel

@@ -235,6 +235,7 @@ bzl_library(
         "//lib/private/docs:expand_template_toolchain",
         "//lib/private/docs:jq_toolchain",
         "//lib/private/docs:local_config_platform",
+        "//lib/private/docs:source_toolchains_repo",
         "//lib/private/docs:yq_toolchain",
     ],
 )

lib/private/copy_directory_toolchain.bzl

@@ -6,7 +6,8 @@
 # `tools/copy_directory/mirror_release.sh`. To calculate for a specific release run
 # `tools/copy_directory/mirror_release.sh <release_version>`
 
-load("//tools:versions.bzl", "COPY_DIRECTORY_INTEGRITY", "COPY_DIRECTORY_VERSION")
+load("//tools:sri.bzl", "COPY_DIRECTORY_INTEGRITY")
+load("//tools:version.bzl", "VERSION")
 
 # Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl
 COPY_DIRECTORY_PLATFORMS = {
@@ -158,7 +159,7 @@ def _copy_directory_platform_repo_impl(rctx):
 
     # https://github.com/aspect-build/bazel-lib/releases/download/v1.19.0/copy_directory-linux_amd64
     url = "https://github.com/aspect-build/bazel-lib/releases/download/v{0}/copy_directory-{1}{2}".format(
-        COPY_DIRECTORY_VERSION,
+        VERSION,
         release_platform,
         ".exe" if is_windows else "",
     )

lib/private/copy_to_directory_toolchain.bzl

@@ -6,7 +6,8 @@
 # `tools/copy_to_directory/mirror_release.sh`. To calculate for a specific release run
 # `tools/copy_to_directory/mirror_release.sh <release_version>`
 
-load("//tools:versions.bzl", "COPY_TO_DIRECTORY_INTEGRITY", "COPY_TO_DIRECTORY_VERSION")
+load("//tools:sri.bzl", "COPY_TO_DIRECTORY_INTEGRITY")
+load("//tools:version.bzl", "VERSION")
 
 # Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl
 COPY_TO_DIRECTORY_PLATFORMS = {
@@ -158,7 +159,7 @@ def _copy_to_directory_platform_repo_impl(rctx):
 
     # https://github.com/aspect-build/bazel-lib/releases/download/v1.19.0/copy_to_directory-linux_amd64
     url = "https://github.com/aspect-build/bazel-lib/releases/download/v{0}/copy_to_directory-{1}{2}".format(
-        COPY_TO_DIRECTORY_VERSION,
+        VERSION,
         release_platform,
         ".exe" if is_windows else "",
     )

lib/private/docs/BUILD.bazel

@@ -244,13 +244,24 @@ bzl_library(
 bzl_library(
     name = "copy_directory_toolchain",
     srcs = ["//lib/private:copy_directory_toolchain.bzl"],
-    deps = ["//tools:versions"],
+    deps = [
+        "//tools:sri",
+        "//tools:version",
+    ],
 )
 
 bzl_library(
     name = "copy_to_directory_toolchain",
     srcs = ["//lib/private:copy_to_directory_toolchain.bzl"],
-    deps = ["//tools:versions"],
+    deps = [
+        "//tools:sri",
+        "//tools:version",
+    ],
+)
+
+bzl_library(
+    name = "source_toolchains_repo",
+    srcs = ["//lib/private:source_toolchains_repo.bzl"],
 )
 
 bzl_library(
@@ -263,7 +274,7 @@ bzl_library(
     srcs = ["//lib/private:expand_template_toolchain.bzl"],
     deps = [
         "//lib:stamping",
-        "//tools:versions",
+        "//tools:sri",
     ],
 )
 

lib/private/expand_template_toolchain.bzl

@@ -6,7 +6,8 @@
 # `tools/expand_template/mirror_release.sh`. To calculate for a specific release run
 # `tools/expand_template/mirror_release.sh <release_version>`
 
-load("//tools:versions.bzl", "EXPAND_TEMPLATE_INTEGRITY", "EXPAND_TEMPLATE_VERSION")
+load("//tools:sri.bzl", "EXPAND_TEMPLATE_INTEGRITY")
+load("//tools:version.bzl", "VERSION")
 
 # Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl
 EXPAND_TEMPLATE_PLATFORMS = {
@@ -158,7 +159,7 @@ def _expand_template_platform_repo_impl(rctx):
 
     # https://github.com/aspect-build/bazel-lib/releases/download/v1.19.0/expand_template-linux_amd64
     url = "https://github.com/aspect-build/bazel-lib/releases/download/v{0}/expand_template-{1}{2}".format(
-        EXPAND_TEMPLATE_VERSION,
+        VERSION,
         release_platform,
         ".exe" if is_windows else "",
     )

lib/private/source_toolchains_repo.bzl

@@ -0,0 +1,90 @@
+"""Create a repository to hold the toolchains
+
+This follows guidance here:
+https://docs.bazel.build/versions/main/skylark/deploying.html#registering-toolchains
+"
+Note that in order to resolve toolchains in the analysis phase
+Bazel needs to analyze all toolchain targets that are registered.
+Bazel will not need to analyze all targets referenced by toolchain.toolchain attribute.
+If in order to register toolchains you need to perform complex computation in the repository,
+consider splitting the repository with toolchain targets
+from the repository with <LANG>_toolchain targets.
+Former will be always fetched,
+and the latter will only be fetched when user actually needs to build <LANG> code.
+"
+The "complex computation" in our case is simply downloading large artifacts.
+This guidance tells us how to avoid that: we put the toolchain targets in the alias repository
+with only the toolchain attribute pointing into the platform-specific repositories.
+"""
+
+# Add more platforms as needed to mirror all the binaries
+# published by the upstream project.
+
+DEFS_TMPL = """\
+# Generated by source_toolchains_repo.bzl for {toolchain_type}
+load("@bazel_skylib//lib:structs.bzl", "structs")
+
+# Forward all the providers
+def _resolved_toolchain_impl(ctx):
+    toolchain_info = ctx.toolchains["{toolchain_type}"]
+    return [toolchain_info] + structs.to_dict(toolchain_info).values()
+
+# Copied from java_toolchain_alias
+# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl
+resolved_toolchain = rule(
+    implementation = _resolved_toolchain_impl,
+    toolchains = ["{toolchain_type}"],
+    incompatible_use_toolchain_transition = True,
+)
+"""
+
+BUILD_TMPL = """\
+# Generated by source_toolchains_repo.bzl
+#
+# These can be registered in the workspace file or passed to --extra_toolchains flag.
+# By default all of these toolchains are registered by the oci_register_toolchains macro
+# so you don't normally need to interact with these targets.
+
+load(":defs.bzl", "resolved_toolchain")
+load("{toolchain_rule_load_from}", toolchain_rule = "{toolchain_rule}")
+
+resolved_toolchain(name = "current_toolchain", visibility = ["//visibility:public"])
+
+toolchain_rule(
+    name = "source",
+    bin = "{binary}",
+    visibility = ["//visibility:public"],
+)
+
+toolchain(
+    name = "toolchain",
+    toolchain = ":source",
+    toolchain_type = "{toolchain_type}",
+)
+"""
+
+def _source_toolchains_repo_impl(rctx):
+    # Expose a concrete toolchain which is the result of Bazel resolving the toolchain
+    # for the execution or target platform.
+    # Workaround for https://github.com/bazelbuild/bazel/issues/14009
+    rctx.file("defs.bzl", DEFS_TMPL.format(
+        toolchain_type = rctx.attr.toolchain_type,
+    ))
+
+    rctx.file("BUILD.bazel", BUILD_TMPL.format(
+        toolchain_type = rctx.attr.toolchain_type,
+        toolchain_rule_load_from = rctx.attr.toolchain_rule_load_from,
+        toolchain_rule = rctx.attr.toolchain_rule,
+        binary = rctx.attr.binary,
+    ))
+
+source_toolchains_repo = repository_rule(
+    _source_toolchains_repo_impl,
+    doc = "Creates a repository with toolchain definitions for source binaries.",
+    attrs = {
+        "toolchain_type": attr.string(doc = "Label to the toolchain_type", mandatory = True),
+        "toolchain_rule_load_from": attr.string(doc = "Label to the concrete toolchain rule to load from", mandatory = True),
+        "toolchain_rule": attr.string(doc = "Name of the concerete toolchain rule", mandatory = True),
+        "binary": attr.string(doc = "Label to the binary", mandatory = True),
+    },
+)