feat(dev): add aspect-based implementation of extra actions for tests (…

…#5923)

* feat(dev): add aspect-based implementation of extra actions for tests

* chore(dev): fix bzl_library visibility

* chore: remove debug print statements

kythe/cxx/indexer/cxx/testdata/BUILD

@@ -1,8 +1,8 @@
 load(
     "//tools/build_rules/verifier_test:cc_indexer_test.bzl",
-    "cc_extract_kzip",
     "cc_extractor_test",
     "cc_indexer_test",
+    "cc_write_kzip",
     "objc_indexer_test",
 )
 load("//kythe/go/test/tools/empty_corpus_checker:empty_corpus_test.bzl", "empty_corpus_test")
@@ -707,7 +707,7 @@ cc_indexer_test(
     tags = ["basic"],
 )
 
-cc_extract_kzip(
+cc_write_kzip(
     name = "usr_macro_default_kzip",
     srcs = ["basic/usr_macro_default.cc"],
     corpus = "unit",
@@ -746,7 +746,7 @@ cc_indexer_test(
     tags = ["basic"],
 )
 
-cc_extract_kzip(
+cc_write_kzip(
     name = "usr_var_default_kzip",
     srcs = ["basic/usr_var_default.cc"],
     corpus = "unit",
@@ -922,7 +922,7 @@ cc_indexer_test(
     ],
 )
 
-cc_extract_kzip(
+cc_write_kzip(
     name = "vname_corpus_kzip",
     srcs = ["basic/vname_corpus.cc"],
     corpus = "unit",

kythe/cxx/indexer/cxx/testdata/proto/cc_proto_verifier_test.bzl

@@ -13,7 +13,7 @@
 # limitations under the License.
 """C++ protocol buffer metadata test rules."""
 
-load("//tools/build_rules/verifier_test:cc_indexer_test.bzl", "cc_extract_kzip", "cc_index", "cc_kythe_proto_library")
+load("//tools/build_rules/verifier_test:cc_indexer_test.bzl", "cc_index", "cc_kythe_proto_library", "cc_write_kzip")
 load("//tools/build_rules/verifier_test:verifier_test.bzl", "index_compilation", "verifier_test")
 load(
     "//kythe/cxx/indexer/proto/testdata:proto_verifier_test.bzl",
@@ -74,7 +74,7 @@ def cc_proto_verifier_test(
     ]
 
     cc_kzip = _invoke(
-        cc_extract_kzip,
+        cc_write_kzip,
         name = name + "_cc_kzip",
         srcs = srcs,
         deps = cc_proto_libs + cc_deps,
@@ -106,7 +106,7 @@ def cc_proto_verifier_test(
             "--claim_unknown=false",
         ]
         claim_deps = [claim_file]
-    
+
     aw_opt = []
     if experimental_set_aliases_as_writes:
         aw_opt = ["--experimental_set_aliases_as_writes"]

tools/build_rules/extra_aspects/BUILD

@@ -0,0 +1,38 @@
+"""Rules for Aspect-based extra actions."""
+
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load(":extra_actions.bzl", "spawn_info_proto_writer")
+
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+bzl_library(
+    name = "actions",
+    srcs = ["actions.bzl"],
+    deps = [
+        "@bazel_skylib//lib:sets",
+    ],
+)
+
+bzl_library(
+    name = "extra_actions",
+    srcs = ["extra_actions.bzl"],
+    deps = [
+        ":actions",
+        "@bazel_skylib//lib:sets",
+    ],
+)
+
+bzl_library(
+    name = "config",
+    srcs = ["config.bzl"],
+    deps = [
+        ":actions",
+        ":extra_actions",
+        "@bazel_skylib//lib:sets",
+        "@bazel_skylib//lib:structs",
+    ],
+)
+
+spawn_info_proto_writer(name = "spawn-info-writer")

tools/build_rules/extra_aspects/actions.bzl

@@ -0,0 +1,86 @@
+# Copyright 2023 The Kythe Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Starlark utilities for selecting actions."""
+
+load("@bazel_skylib//lib:sets.bzl", "sets")
+
+def select_target_actions(target, rule, config):
+    """Given a target and a rule, finds actions matching the specified config.
+
+    Args:
+      target: The Target object provided to an aspect implementation.
+      rule: The rule_attributes from an aspect's ctx.rule parameter.
+      config: A KytheActionSelectionInfo instance, configuring action selection.
+
+    Returns:
+      A list of actions selected by the provided configuration.
+    """
+    actions = []
+    if sets.length(config.rules) == 0 or sets.contains(config.rules, rule.kind):
+        actions.extend(_filter_actions(target.actions, config.mnemonics, config.exclude_input_extensions))
+    for attr in config.aspect_rule_attrs.get(rule.kind, []):
+        for dep in getattr(rule.attr, attr, []):
+            # {lang}_proto_library targets are implemented as aspects
+            # with a single "dep" pointing at original proto_library target.
+            # All of the actions come from the aspect as applied to the dep,
+            # which is not a top-level dep, so we need to inspect those as well.
+            # TODO(shahms): It would be nice if there was some way to tell whether or
+            # not an action came from an aspect or the underlying rule. We could then
+            # more generally inspect deps to find actions rather than limiting it to
+            # allowlisted kinds.
+            # This could be accomplished by having `aspect_ids` populated on deps.
+            # We could look specifically for "<merged target" in the string-ified dep,
+            # but that's pretty ugly.
+            # It looks like adding an "aspect_ids" list attribute should be straightforward.
+            # https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/analysis/configuredtargets/MergedConfiguredTarget.java#L54
+            # It could also just use ActionOwner information.
+            if str(dep).startswith("<merged target") and hasattr(dep, "actions"):
+                actions.extend(_filter_actions(dep.actions, config.mnemonics, config.exclude_input_extensions))
+    return actions
+
+def config(rules, aspect_rule_attrs, mnemonics, exclude_input_extensions):
+    """Create an action selection config.
+
+    Args:
+     rules: A Skylib set of rule kinds from which to select actions.
+     aspect_rule_attrs: A dict of {kind: [attrs]} of rule kind attributes from which to select actions.
+     mnemonics: A Skylib set of action mnemonics to select.
+     exclude_input_extensions: A Skylib set of file extensions to whose actions to exclude.
+
+    Returns:
+      A struct with corresponding attributes.
+    """
+    return struct(
+        rules = rules,
+        aspect_rule_attrs = aspect_rule_attrs,
+        mnemonics = mnemonics,
+        exclude_input_extensions = exclude_input_extensions,
+    )
+
+def _has_extension(inputs, extensions):
+    if not extensions or sets.length(extensions) == 0:
+        return False
+    for input in inputs.to_list():
+        if sets.contains(extensions, input.extension):
+            return True
+    return False
+
+def _filter_actions(actions, mnemonics, exclude_extensions = None):
+    return [
+        a
+        for a in actions
+        if sets.contains(mnemonics, a.mnemonic) and
+           not _has_extension(a.inputs, exclude_extensions)
+    ]

tools/build_rules/extra_aspects/config.bzl

@@ -0,0 +1,148 @@
+# Copyright 2023 The Kythe Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Kythe extractor configuration rules and providers."""
+
+load("@bazel_skylib//lib:sets.bzl", "sets")
+load("@bazel_skylib//lib:structs.bzl", "structs")
+load(":actions.bzl", _action_selection_config = "config")
+load(
+    ":extra_actions.bzl",
+    "ActionListenerConfigInfo",
+    "ActionListenerInfo",
+    "ProtoWriterInfo",
+    "action_listener_aspect",
+    "extract_target",
+)
+
+KytheExtractorConfigInfo = provider(
+    doc = "Configuration options for Kythe extraction.",
+    fields = {
+        "allowed_rule_kinds": "A Skylib set of allowed rule kinds (or None to allow everything).",
+        "action_listeners": "A depset of ActionListenerInfo from the provided target.",
+        "action_selection_config": "A KytheActionSelectionInfo instance.",
+    },
+)
+
+def _use_default_proto_writer(action_listener, proto_writer):
+    if not getattr(action_listener, "write_extra_action", None):
+        keys = structs.to_dict(action_listener)
+        keys["write_extra_action"] = proto_writer.write_extra_action
+        return ActionListenerInfo(**keys)
+    return action_listener
+
+def _get_transitive_action_listeners(action_listeners, proto_writer):
+    return depset(direct = [
+        _use_default_proto_writer(t[ActionListenerInfo], proto_writer[ProtoWriterInfo])
+        for t in action_listeners
+        if ActionListenerInfo in t
+    ], transitive = [
+        t[ActionListenerConfigInfo].action_listeners
+        for t in action_listeners
+        if ActionListenerConfigInfo in t
+    ])
+
+def _config_impl(ctx):
+    action_listeners = _get_transitive_action_listeners(
+        ctx.attr.action_listeners,
+        ctx.attr.proto_writer,
+    ).to_list()
+
+    # TODO(b/182403136): Workaround for error applying aspects to jsunit_test rules.
+    # They complain about a "non-executable dependency" so we provide a do-nothing executable
+    # file to silence this.
+    dummy_executable = ctx.actions.declare_file(ctx.label.name + ".dummy_exec")
+    ctx.actions.write(
+        output = dummy_executable,
+        content = "",
+        is_executable = True,
+    )
+
+    return [
+        KytheExtractorConfigInfo(
+            action_listeners = action_listeners,
+            allowed_rule_kinds = _allowed_rule_kinds(ctx.attr.rules, ctx.attr.aspect_rule_attrs),
+            action_selection_config = _action_selection_config(
+                rules = sets.make(ctx.attr.rules),
+                aspect_rule_attrs = ctx.attr.aspect_rule_attrs,
+                mnemonics = sets.union(*[a.mnemonics for a in action_listeners]),
+                exclude_input_extensions = sets.make(ctx.attr.exclude_input_extensions),
+            ),
+        ),
+        DefaultInfo(
+            executable = dummy_executable,
+        ),
+    ]
+
+kythe_extractor_config = rule(
+    implementation = _config_impl,
+    attrs = {
+        "rules": attr.string_list(mandatory = True),
+        "aspect_rule_attrs": attr.string_list_dict(),
+        "action_listeners": attr.label_list(
+            aspects = [action_listener_aspect],
+            providers = [
+                [ActionListenerInfo],
+                [ActionListenerConfigInfo],
+            ],
+            allow_rules = ["action_listener"],
+        ),
+        "proto_writer": attr.label(
+            default = Label("//tools/build_rules/extra_aspects:spawn-info-writer"),
+            providers = [ProtoWriterInfo],
+        ),
+        "exclude_input_extensions": attr.string_list(),
+    },
+    provides = [KytheExtractorConfigInfo],
+)
+
+def _action_listener_config_impl(ctx):
+    return [
+        ActionListenerConfigInfo(
+            action_listeners = _get_transitive_action_listeners(ctx.attr.action_listeners, ctx.attr.proto_writer),
+        ),
+    ]
+
+action_listener_config = rule(
+    implementation = _action_listener_config_impl,
+    attrs = {
+        "action_listeners": attr.label_list(
+            aspects = [action_listener_aspect],
+            providers = [
+                [ActionListenerInfo],
+                [ActionListenerConfigInfo],
+            ],
+            allow_rules = ["action_listener"],
+        ),
+        "proto_writer": attr.label(
+            mandatory = True,
+            providers = [ProtoWriterInfo],
+        ),
+    },
+    provides = [ActionListenerConfigInfo],
+)
+
+def _allowed_rule_kinds(rules, aspect_rule_attrs):
+    if not rules:
+        return None  # An empty `rules` attribute means allow all kinds.
+    return sets.union(sets.make(rules), sets.make(aspect_rule_attrs.keys()))
+
+def run_configured_extractor(target, ctx, config):
+    # If there is an explicit allowlist of rule kinds, only extract those.
+    # Attempting to extract an unknown rule kind is not an error.
+    # As command-line aspects are called for all top-level targets,
+    # they must be able to quietly handle this case.
+    if config.allowed_rule_kinds and not sets.contains(config.allowed_rule_kinds, ctx.rule.kind):
+        return depset()
+    return extract_target(target, ctx, config)

tools/build_rules/extra_aspects/cxx/BUILD

@@ -0,0 +1,46 @@
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load("//tools/build_rules/extra_aspects:config.bzl", "kythe_extractor_config")
+load(":config.bzl", "cpp_compile_info_proto_writer")
+
+kythe_extractor_config(
+    name = "extractor-config",
+    action_listeners = [
+        "//kythe/extractors:extract_kzip_cxx",
+    ],
+    aspect_rule_attrs = {"cc_proto_library": []},
+    proto_writer = ":cpp-compile-info-writer",
+    rules = [
+        "cc_binary",
+        "cc_library",
+        "cc_test",
+        "cc_stubby_library",
+        "py_extension",
+        "proto_library",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+cpp_compile_info_proto_writer(name = "cpp-compile-info-writer")
+
+bzl_library(
+    name = "config",
+    srcs = ["config.bzl"],
+    deps = ["//tools/build_rules/extra_aspects:extra_actions"],
+)
+
+bzl_library(
+    name = "aspect",
+    srcs = ["aspect.bzl"],
+    deps = [
+        "//tools/build_rules/extra_aspects:config",
+    ],
+)
+
+bzl_library(
+    name = "rules",
+    srcs = ["rules.bzl"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":aspect",
+    ],
+)