From 6da37475c52997a9d0f7959865bc9b7746b2eb6d Mon Sep 17 00:00:00 2001 From: Matt Stark Date: Fri, 19 Apr 2024 12:57:32 +1000 Subject: [PATCH 1/3] Add bzlmod lockfile to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a6ef824c..7f69f3af 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /bazel-* +MODULE.bazel.lock From 14efc8cae901a14b76aeba2be33c2b315b26ff35 Mon Sep 17 00:00:00 2001 From: Matt Stark Date: Fri, 19 Apr 2024 12:57:32 +1000 Subject: [PATCH 2/3] Add a rule to create providers for directories. --- MODULE.bazel | 1 + rules/directories/BUILD | 1 + rules/directories/directory.bzl | 106 +++++++++++++++++++++++ rules/directories/providers.bzl | 33 +++++++ tests/directories/BUILD | 30 +++++++ tests/directories/directory_test.bzl | 88 +++++++++++++++++++ tests/directories/testdata/BUILD | 38 ++++++++ tests/directories/testdata/dir/subdir/f2 | 0 tests/directories/testdata/f1 | 0 tests/subpackages_tests.bzl | 2 + 10 files changed, 299 insertions(+) create mode 100644 rules/directories/BUILD create mode 100644 rules/directories/directory.bzl create mode 100644 rules/directories/providers.bzl create mode 100644 tests/directories/BUILD create mode 100644 tests/directories/directory_test.bzl create mode 100644 tests/directories/testdata/BUILD create mode 100644 tests/directories/testdata/dir/subdir/f2 create mode 100644 tests/directories/testdata/f1 diff --git a/MODULE.bazel b/MODULE.bazel index 298974a3..31e2613e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -18,6 +18,7 @@ bazel_dep(name = "platforms", version = "0.0.4") bazel_dep(name = "stardoc", version = "0.5.6", dev_dependency = True, repo_name = "io_bazel_stardoc") bazel_dep(name = "rules_pkg", version = "0.9.1", dev_dependency = True) bazel_dep(name = "rules_cc", version = "0.0.2", dev_dependency = True) +bazel_dep(name = "rules_testing", version = "0.6.0", dev_dependency = True) # Needed for bazelci and for building distribution tarballs. # If using an unreleased version of bazel_skylib via git_override, apply diff --git a/rules/directories/BUILD b/rules/directories/BUILD new file mode 100644 index 00000000..5b01f6e3 --- /dev/null +++ b/rules/directories/BUILD @@ -0,0 +1 @@ +licenses(["notice"]) diff --git a/rules/directories/directory.bzl b/rules/directories/directory.bzl new file mode 100644 index 00000000..4c94a911 --- /dev/null +++ b/rules/directories/directory.bzl @@ -0,0 +1,106 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Skylib module containing rules to create metadata about directories.""" + +load(":providers.bzl", "DirectoryInfo") + +visibility("public") + +def _directory_impl(ctx): + if ctx.label.workspace_root: + pkg_path = ctx.label.workspace_root + "/" + ctx.label.package + else: + pkg_path = ctx.label.package + source_dir = pkg_path.rstrip("/") + source_prefix = source_dir + "/" + + # Declare a generated file so that we can get the path to generated files. + f = ctx.actions.declare_file(ctx.label.name) + ctx.actions.write(f, "") + generated_dir = f.path.rsplit("/", 1)[0] + generated_prefix = generated_dir + "/" + + root_metadata = struct( + directories = {}, + files = [], + source_path = source_dir, + generated_path = generated_dir, + human_readable = "@@%s//%s" % (ctx.label.repo_name, ctx.label.package), + ) + + # Topologically order directories so we can use them for DFS. + topo = [root_metadata] + for src in ctx.files.srcs: + prefix = source_prefix if src.is_source else generated_prefix + if not src.path.startswith(prefix): + fail("{path} is not contained within {prefix}".format( + path = src.path, + prefix = root_metadata.human_readable, + )) + relative = src.path[len(prefix):].split("/") + current_path = root_metadata + for dirname in relative[:-1]: + if dirname not in current_path.directories: + dir_metadata = struct( + directories = {}, + files = [], + source_path = "%s/%s" % (current_path.source_path, dirname), + generated_path = "%s/%s" % (current_path.generated_path, dirname), + human_readable = "%s/%s" % (current_path.human_readable, dirname), + ) + current_path.directories[dirname] = dir_metadata + topo.append(dir_metadata) + + current_path = current_path.directories[dirname] + current_path.files.append(src) + + # The output DirectoryInfos. Key them by something arbitrary but unique. + # In this case, we choose source_path. + out = {} + for dir_metadata in reversed(topo): + child_dirs = { + dirname: out[subdir_metadata.source_path] + for dirname, subdir_metadata in dir_metadata.directories.items() + } + files = sorted(dir_metadata.files, key = lambda file: file.basename) + direct_files = depset(files) + transitive_files = depset(transitive = [direct_files] + [ + d.transitive_files + for d in child_dirs.values() + ], order = "preorder") + out[dir_metadata.source_path] = DirectoryInfo( + directories = child_dirs, + files = {f.basename: f for f in files}, + direct_files = direct_files, + transitive_files = transitive_files, + source_path = dir_metadata.source_path, + generated_path = dir_metadata.generated_path, + human_readable = dir_metadata.human_readable, + ) + + root_directory = out[root_metadata.source_path] + + return [ + root_directory, + DefaultInfo(files = root_directory.transitive_files), + ] + +directory = rule( + implementation = _directory_impl, + attrs = { + "srcs": attr.label_list(allow_files = True), + }, + provides = [DirectoryInfo], +) diff --git a/rules/directories/providers.bzl b/rules/directories/providers.bzl new file mode 100644 index 00000000..42b862d8 --- /dev/null +++ b/rules/directories/providers.bzl @@ -0,0 +1,33 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Skylib module containing providers for directories.""" + +visibility("public") + +DirectoryInfo = provider( + doc = "Information about a directory", + # @unsorted-dict-items + fields = { + "directories": "(Dict[str, DirectoryInfo]) The subdirectories contained directly within", + "files": "(Dict[str, File]) The files contained directly within the directory, keyed by their path relative to this directory.", + "direct_files": "(Depset[File])", + # A transitive_directories would be useful here, but is blocked on + # https://github.com/bazelbuild/starlark/issues/272 + "transitive_files": "(Depset[File])", + "source_path": "(string) Path to all source files contained within this directory", + "generated_path": "(string) Path to all generated files contained within this directory", + "human_readable": "(string) A human readable identifier for a directory. Useful for providing error messages to a user.", + }, +) diff --git a/tests/directories/BUILD b/tests/directories/BUILD new file mode 100644 index 00000000..271d5af4 --- /dev/null +++ b/tests/directories/BUILD @@ -0,0 +1,30 @@ +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load( + ":directory_test.bzl", + "directory_test", + "outside_testdata_test", +) + +analysis_test( + name = "directory_test", + impl = directory_test, + targets = { + "root": "//tests/directories/testdata:root", + "f1": "//tests/directories/testdata:f1_filegroup", + "f2": "//tests/directories/testdata:f2_filegroup", + "f3": "//tests/directories/testdata:f3", + }, +) + +filegroup( + name = "outside_testdata", + srcs = ["BUILD"], + visibility = ["//tests/directories/testdata:__pkg__"], +) + +analysis_test( + name = "outside_testdata_test", + expect_failure = True, + impl = outside_testdata_test, + target = "//tests/directories/testdata:outside_testdata", +) diff --git a/tests/directories/directory_test.bzl b/tests/directories/directory_test.bzl new file mode 100644 index 00000000..19e2da1b --- /dev/null +++ b/tests/directories/directory_test.bzl @@ -0,0 +1,88 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for directory rules.""" + +load("@rules_testing//lib:truth.bzl", "matching", "subjects") +load("//rules/directories:providers.bzl", "DirectoryInfo") + +visibility("private") + +_depset_as_list_subject = lambda value, *, meta: subjects.collection( + value.to_list(), + meta = meta, +) + +_directory_info_subject = lambda value, *, meta: subjects.struct( + value, + meta = meta, + attrs = dict( + directories = subjects.dict, + files = subjects.dict, + direct_files = _depset_as_list_subject, + transitive_files = _depset_as_list_subject, + source_path = subjects.str, + generated_path = subjects.str, + human_readable = subjects.str, + ), +) + +def _directory_subject(env, directory_info): + return env.expect.that_value( + value = directory_info, + expr = "DirectoryInfo(%r)" % directory_info.source_path, + factory = _directory_info_subject, + ) + +# buildifier: disable=function-docstring +def outside_testdata_test(env, target): + env.expect.that_target(target).failures().contains_exactly_predicates([ + matching.contains("tests/directories/BUILD is not contained within @@//tests/directories/testdata"), + ]) + +# buildifier: disable=function-docstring +def directory_test(env, targets): + f1 = targets.f1.files.to_list()[0] + f2 = targets.f2.files.to_list()[0] + f3 = targets.f3.files.to_list()[0] + + env.expect.that_collection(targets.root.files.to_list()).contains_exactly( + [f1, f2, f3], + ) + + root = _directory_subject(env, targets.root[DirectoryInfo]) + root.directories().keys().contains_exactly(["dir", "newdir"]) + root.files().contains_exactly({"f1": f1}) + root.direct_files().contains_exactly([f1]) + root.transitive_files().contains_exactly([f1, f2, f3]).in_order() + root.human_readable().equals("@@//tests/directories/testdata") + env.expect.that_str(root.actual.source_path + "/f1").equals(f1.path) + + dir = _directory_subject(env, root.actual.directories["dir"]) + dir.directories().keys().contains_exactly(["subdir"]) + dir.human_readable().equals("@@//tests/directories/testdata/dir") + + subdir = _directory_subject(env, dir.actual.directories["subdir"]) + subdir.directories().keys().contains_exactly([]) + subdir.files().contains_exactly({"f2": f2}) + subdir.direct_files().contains_exactly([f2]) + subdir.transitive_files().contains_exactly([f2]).in_order() + env.expect.that_str(subdir.actual.source_path + "/f2").equals(f2.path) + + newdir = _directory_subject(env, root.actual.directories["newdir"]) + newdir.directories().keys().contains_exactly([]) + newdir.files().contains_exactly({"f3": f3}) + newdir.direct_files().contains_exactly([f3]) + newdir.transitive_files().contains_exactly([f3]).in_order() + env.expect.that_str(newdir.actual.generated_path + "/f3").equals(f3.path) diff --git a/tests/directories/testdata/BUILD b/tests/directories/testdata/BUILD new file mode 100644 index 00000000..ca19ebec --- /dev/null +++ b/tests/directories/testdata/BUILD @@ -0,0 +1,38 @@ +load("@rules_testing//lib:util.bzl", "util") +load("//rules:copy_file.bzl", "copy_file") +load("//rules/directories:directory.bzl", "directory") + +package(default_visibility = ["//tests/directories:__pkg__"]) + +copy_file( + name = "f3", + src = "dir/subdir/f2", + out = "newdir/f3", +) + +directory( + name = "root", + srcs = glob( + ["**"], + exclude = ["BUILD"], + ) + [":f3"], +) + +util.helper_target( + directory, + name = "outside_testdata", + srcs = glob( + ["**"], + exclude = ["BUILD"], + ) + ["//tests/directories:outside_testdata"], +) + +filegroup( + name = "f1_filegroup", + srcs = ["f1"], +) + +filegroup( + name = "f2_filegroup", + srcs = ["dir/subdir/f2"], +) diff --git a/tests/directories/testdata/dir/subdir/f2 b/tests/directories/testdata/dir/subdir/f2 new file mode 100644 index 00000000..e69de29b diff --git a/tests/directories/testdata/f1 b/tests/directories/testdata/f1 new file mode 100644 index 00000000..e69de29b diff --git a/tests/subpackages_tests.bzl b/tests/subpackages_tests.bzl index 3c494d68..a805e0b3 100644 --- a/tests/subpackages_tests.bzl +++ b/tests/subpackages_tests.bzl @@ -25,6 +25,7 @@ def _all_test(env): "copy_directory", "copy_file", "diff_test", + "directories", "expand_template", "select_file", "write_file", @@ -42,6 +43,7 @@ def _all_test(env): "common_settings", "copy_directory", "copy_file", + "directories", "expand_template", "select_file", "write_file", From f68bc00fcc77bf07c1749e4b57d1e4f3392b848b Mon Sep 17 00:00:00 2001 From: Matt Stark Date: Fri, 19 Apr 2024 12:57:32 +1000 Subject: [PATCH 3/3] Add a rule to create providers for subdirectories --- rules/directories/subdirectory.bzl | 36 ++++++++++++++++++ rules/directories/utils.bzl | 55 ++++++++++++++++++++++++++++ tests/directories/BUILD | 20 ++++++++++ tests/directories/directory_test.bzl | 28 ++++++++++++++ tests/directories/testdata/BUILD | 20 ++++++++++ 5 files changed, 159 insertions(+) create mode 100644 rules/directories/subdirectory.bzl create mode 100644 rules/directories/utils.bzl diff --git a/rules/directories/subdirectory.bzl b/rules/directories/subdirectory.bzl new file mode 100644 index 00000000..5f464cb6 --- /dev/null +++ b/rules/directories/subdirectory.bzl @@ -0,0 +1,36 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Skylib module containing rules to create metadata about subdirectories.""" + +load(":providers.bzl", "DirectoryInfo") +load(":utils.bzl", "get_subdirectory") + +visibility("public") + +def _subdirectory_impl(ctx): + dir = get_subdirectory(ctx.attr.parent[DirectoryInfo], ctx.attr.path) + return [ + dir, + DefaultInfo(files = dir.transitive_files), + ] + +subdirectory = rule( + implementation = _subdirectory_impl, + attrs = { + "parent": attr.label(providers = [DirectoryInfo], mandatory = True), + "path": attr.string(mandatory = True), + }, + provides = [DirectoryInfo], +) diff --git a/rules/directories/utils.bzl b/rules/directories/utils.bzl new file mode 100644 index 00000000..c8bb28b3 --- /dev/null +++ b/rules/directories/utils.bzl @@ -0,0 +1,55 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Skylib module containing utility functions related to directories.""" + +visibility("public") + +_DIR_NOT_FOUND = """{directory} does not contain a directory named {dirname}. +Instead, it contains the directories {children}.""" + +def _check_path_relative(path): + if path.startswith("/"): + fail("Path must be relative. Got {path}".format(path = path)) + +def get_direct_subdirectory(directory, dirname): + """Gets the direct subdirectory of a directory. + + Args: + directory: (DirectoryInfo) The directory to look within. + dirname: (string) The name of the directory to look for. + Returns: + (DirectoryInfo) The directory contained within.""" + if dirname not in directory.directories: + fail(_DIR_NOT_FOUND.format( + directory = directory.human_readable, + dirname = repr(dirname), + children = repr(sorted(directory.directories)), + )) + return directory.directories[dirname] + +def get_subdirectory(directory, path): + """Gets a subdirectory contained within a tree of another directory. + + Args: + directory: (DirectoryInfo) The directory to look within. + path: (string) The path of the directory to look for within it. + Returns: + (DirectoryInfo) The directory contained within. + """ + _check_path_relative(path) + + for dirname in path.split("/"): + directory = get_direct_subdirectory(directory, dirname) + return directory diff --git a/tests/directories/BUILD b/tests/directories/BUILD index 271d5af4..9552948e 100644 --- a/tests/directories/BUILD +++ b/tests/directories/BUILD @@ -2,7 +2,9 @@ load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load( ":directory_test.bzl", "directory_test", + "nonexistent_subdirectory_test", "outside_testdata_test", + "subdirectory_test", ) analysis_test( @@ -28,3 +30,21 @@ analysis_test( impl = outside_testdata_test, target = "//tests/directories/testdata:outside_testdata", ) + +analysis_test( + name = "subdirectory_test", + impl = subdirectory_test, + targets = { + "root": "//tests/directories/testdata:root", + "dir": "//tests/directories/testdata:dir", + "subdir": "//tests/directories/testdata:subdir", + "f2": "//tests/directories/testdata:f2_filegroup", + }, +) + +analysis_test( + name = "nonexistent_subdirectory_test", + expect_failure = True, + impl = nonexistent_subdirectory_test, + target = "//tests/directories/testdata:nonexistent_subdirectory", +) diff --git a/tests/directories/directory_test.bzl b/tests/directories/directory_test.bzl index 19e2da1b..903fb498 100644 --- a/tests/directories/directory_test.bzl +++ b/tests/directories/directory_test.bzl @@ -86,3 +86,31 @@ def directory_test(env, targets): newdir.direct_files().contains_exactly([f3]) newdir.transitive_files().contains_exactly([f3]).in_order() env.expect.that_str(newdir.actual.generated_path + "/f3").equals(f3.path) + +_NONEXISTENT_DIR_ERR = """@@//tests/directories/testdata/dir does not contain a directory named "nonexistent". +Instead, it contains the directories ["subdir"].""" + +# buildifier: disable=function-docstring +def nonexistent_subdirectory_test(env, target): + env.expect.that_target(target).failures().contains_exactly_predicates([ + matching.contains(_NONEXISTENT_DIR_ERR), + ]) + +# buildifier: disable=function-docstring +def subdirectory_test(env, targets): + f2 = targets.f2.files.to_list()[0] + + root = targets.root[DirectoryInfo] + want_dir = root.directories["dir"] + want_subdir = want_dir.directories["subdir"] + + # Use that_str because it supports equality checks. They're not strings. + env.expect.that_str(targets.dir[DirectoryInfo]).equals(want_dir) + env.expect.that_str(targets.subdir[DirectoryInfo]).equals(want_subdir) + + env.expect.that_collection( + targets.dir.files.to_list(), + ).contains_exactly([f2]) + env.expect.that_collection( + targets.subdir.files.to_list(), + ).contains_exactly([f2]) diff --git a/tests/directories/testdata/BUILD b/tests/directories/testdata/BUILD index ca19ebec..e9f0c46d 100644 --- a/tests/directories/testdata/BUILD +++ b/tests/directories/testdata/BUILD @@ -1,6 +1,7 @@ load("@rules_testing//lib:util.bzl", "util") load("//rules:copy_file.bzl", "copy_file") load("//rules/directories:directory.bzl", "directory") +load("//rules/directories:subdirectory.bzl", "subdirectory") package(default_visibility = ["//tests/directories:__pkg__"]) @@ -36,3 +37,22 @@ filegroup( name = "f2_filegroup", srcs = ["dir/subdir/f2"], ) + +subdirectory( + name = "dir", + parent = ":root", + path = "dir", +) + +subdirectory( + name = "subdir", + parent = ":root", + path = "dir/subdir", +) + +util.helper_target( + subdirectory, + name = "nonexistent_subdirectory", + parent = ":root", + path = "dir/nonexistent", +)