From 00a4bfb555fd004abacf267874866c7c6d0869ff Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Tue, 26 Mar 2024 09:44:55 -0400 Subject: [PATCH] Allow a no-cargo setup for bzlmod (#2565) Usage looks like so: ``` crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate") crate.spec(package = "anyhow", version = "1.0.77") .... crate.from_specs(name = "crates") ``` It might make sense to merge the annotation attributes into the spec so we don't have to duplicate things, but we can probably iterate on this in the future, this API is still experimental, yeah? --- .bazelci/presubmit.yml | 6 + crate_universe/extension.bzl | 136 +++++++++++++++--- crate_universe/private/crates_vendor.bzl | 2 +- examples/bzlmod/hello_world_no_cargo/.bazelrc | 3 + .../bzlmod/hello_world_no_cargo/.gitignore | 1 + .../bzlmod/hello_world_no_cargo/BUILD.bazel | 11 ++ .../bzlmod/hello_world_no_cargo/MODULE.bazel | 26 ++++ .../bzlmod/hello_world_no_cargo/src/main.rs | 18 +++ 8 files changed, 186 insertions(+), 17 deletions(-) create mode 100644 examples/bzlmod/hello_world_no_cargo/.bazelrc create mode 100644 examples/bzlmod/hello_world_no_cargo/.gitignore create mode 100644 examples/bzlmod/hello_world_no_cargo/BUILD.bazel create mode 100644 examples/bzlmod/hello_world_no_cargo/MODULE.bazel create mode 100644 examples/bzlmod/hello_world_no_cargo/src/main.rs diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 7115591e99..a94ae32bd9 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -686,6 +686,12 @@ tasks: - "@rules_rust//tools/rust_analyzer:gen_rust_project" test_targets: - "//..." + bzlmod_no_cargo: + name: Cargo-less bzlmod + platform: ubuntu2004 + working_directory: examples/bzlmod/hello_world_no_cargo + build_targets: + - "//..." buildifier: version: latest diff --git a/crate_universe/extension.bzl b/crate_universe/extension.bzl index 8e6b816290..0fddfa30c2 100644 --- a/crate_universe/extension.bzl +++ b/crate_universe/extension.bzl @@ -40,8 +40,35 @@ _generate_repo = repository_rule( ), ) -def _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations): - cargo_lockfile = module_ctx.path(cfg.cargo_lockfile) +def _annotations_for_repo(module_annotations, repo_specific_annotations): + """Merges the set of global annotations with the repo-specific ones + + Args: + module_annotations (dict): The annotation tags that apply to all repos, keyed by crate. + repo_specific_annotations (dict): The annotation tags that apply to only this repo, keyed by crate. + """ + + if not repo_specific_annotations: + return module_annotations + + annotations = dict(module_annotations) + for crate, values in repo_specific_annotations.items(): + _get_or_insert(annotations, crate, []).extend(values) + return annotations + +def _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations, cargo_lockfile = None, manifests = {}, packages = {}): + """Generates repositories for the transitive closure of crates defined by manifests and packages. + + Args: + module_ctx (module_ctx): The module context object. + cargo_bazel (function): A function that can be called to execute cargo_bazel. + cfg (object): The module tag from `from_cargo` or `from_specs` + annotations (dict): The set of annotation tag classes that apply to this closure, keyed by crate name. + cargo_lockfile (path): Path to Cargo.lock, if we have one. This is optional for `from_specs` closures. + manifests (dict): The set of Cargo.toml manifests that apply to this closure, if any, keyed by path. + packages (dict): The set of extra cargo crate tags that apply to this closure, if any, keyed by package name. + """ + tag_path = module_ctx.path(cfg.name) rendering_config = json.decode(render_config( @@ -67,22 +94,21 @@ def _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations): ), ) - manifests = {module_ctx.path(m): m for m in cfg.manifests} splicing_manifest = tag_path.get_child("splicing_manifest.json") module_ctx.file( splicing_manifest, executable = False, content = generate_splicing_manifest( - packages = {}, + packages = packages, splicing_config = "", cargo_config = cfg.cargo_config, - manifests = {str(k): str(v) for k, v in manifests.items()}, + manifests = manifests, manifest_to_path = module_ctx.path, ), ) splicing_output_dir = tag_path.get_child("splicing-output") - cargo_bazel([ + splice_args = [ "splice", "--output-dir", splicing_output_dir, @@ -90,9 +116,13 @@ def _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations): config_file, "--splicing-manifest", splicing_manifest, - "--cargo-lockfile", - cargo_lockfile, - ]) + ] + if cargo_lockfile: + splice_args.extend([ + "--cargo-lockfile", + cargo_lockfile, + ]) + cargo_bazel(splice_args) # Create a lockfile, since we need to parse it to generate spoke # repos. @@ -102,7 +132,7 @@ def _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations): cargo_bazel([ "generate", "--cargo-lockfile", - cargo_lockfile, + cargo_lockfile or splicing_output_dir.get_child("Cargo.lock"), "--config", config_file, "--splicing-manifest", @@ -181,6 +211,15 @@ def _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations): else: fail("Invalid repo: expected Http or Git to exist for crate %s-%s, got %s" % (name, version, repo)) +def _package_to_json(p): + # Avoid adding unspecified properties. + # If we add them as empty strings, cargo-bazel will be unhappy. + return json.encode({ + k: v + for k, v in structs.to_dict(p).items() + if v + }) + def _crate_impl(module_ctx): cargo_bazel = get_cargo_bazel_runner(module_ctx) all_repos = [] @@ -216,19 +255,34 @@ def _crate_impl(module_ctx): ).append(annotation) local_repos = [] - for cfg in mod.tags.from_cargo: + + for cfg in mod.tags.from_cargo + mod.tags.from_specs: if cfg.name in local_repos: fail("Defined two crate universes with the same name in the same MODULE.bazel file. Use the name tag to give them different names.") elif cfg.name in all_repos: fail("Defined two crate universes with the same name in different MODULE.bazel files. Either give one a different name, or use use_extension(isolate=True)") - - annotations = {k: v for k, v in module_annotations.items()} - for crate, values in repo_specific_annotations.get(cfg.name, {}).items(): - _get_or_insert(annotations, crate, []).extend(values) - _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations) all_repos.append(cfg.name) local_repos.append(cfg.name) + for cfg in mod.tags.from_cargo: + annotations = _annotations_for_repo( + module_annotations, + repo_specific_annotations.get(cfg.name), + ) + + cargo_lockfile = module_ctx.path(cfg.cargo_lockfile) + manifests = {str(module_ctx.path(m)): str(m) for m in cfg.manifests} + _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations, cargo_lockfile = cargo_lockfile, manifests = manifests) + + for cfg in mod.tags.from_specs: + annotations = _annotations_for_repo( + module_annotations, + repo_specific_annotations.get(cfg.name), + ) + + packages = {p.package: _package_to_json(p) for p in mod.tags.spec} + _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations, packages = packages) + for repo in repo_specific_annotations: if repo not in local_repos: fail("Annotation specified for repo %s, but the module defined repositories %s" % (repo, local_repos)) @@ -290,10 +344,60 @@ _annotation = tag_class( ), ) +_from_specs = tag_class( + doc = "Generates a repo @crates from the defined `spec` tags", + attrs = dict( + name = attr.string(doc = "The name of the repo to generate", default = "crates"), + cargo_config = CRATES_VENDOR_ATTRS["cargo_config"], + generate_binaries = CRATES_VENDOR_ATTRS["generate_binaries"], + generate_build_scripts = CRATES_VENDOR_ATTRS["generate_build_scripts"], + supported_platform_triples = CRATES_VENDOR_ATTRS["supported_platform_triples"], + ), +) + +# This should be kept in sync with crate_universe/private/crate.bzl. +_spec = tag_class( + attrs = dict( + package = attr.string( + doc = "The explicit name of the package.", + mandatory = True, + ), + version = attr.string( + doc = "The exact version of the crate. Cannot be used with `git`.", + ), + artifact = attr.string( + doc = "Set to 'bin' to pull in a binary crate as an artifact dependency. Requires a nightly Cargo.", + ), + lib = attr.bool( + doc = "If using `artifact = 'bin'`, additionally setting `lib = True` declares a dependency on both the package's library and binary, as opposed to just the binary.", + ), + default_features = attr.bool( + doc = "Maps to the `default-features` flag.", + ), + features = attr.string_list( + doc = "A list of features to use for the crate.", + ), + git = attr.string( + doc = "The Git url to use for the crate. Cannot be used with `version`.", + ), + branch = attr.string( + doc = "The git branch of the remote crate. Tied with the `git` param. Only one of branch, tag or rev may be specified. Specifying `rev` is recommended for fully-reproducible builds.", + ), + tag = attr.string( + doc = "The git tag of the remote crate. Tied with the `git` param. Only one of branch, tag or rev may be specified. Specifying `rev` is recommended for fully-reproducible builds.", + ), + rev = attr.string( + doc = "The git revision of the remote crate. Tied with the `git` param. Only one of branch, tag or rev may be specified.", + ), + ), +) + crate = module_extension( implementation = _crate_impl, tag_classes = dict( from_cargo = _from_cargo, annotation = _annotation, + from_specs = _from_specs, + spec = _spec, ), ) diff --git a/crate_universe/private/crates_vendor.bzl b/crate_universe/private/crates_vendor.bzl index b29c3c0fa7..6bd24b1e12 100644 --- a/crate_universe/private/crates_vendor.bzl +++ b/crate_universe/private/crates_vendor.bzl @@ -118,7 +118,7 @@ def _write_splicing_manifest(ctx): return args, runfiles def generate_splicing_manifest(packages, splicing_config, cargo_config, manifests, manifest_to_path): - # Deserialize information about direct packges + # Deserialize information about direct packages direct_packages_info = { # Ensure the data is using kebab-case as that's what `cargo_toml::DependencyDetail` expects. pkg: kebab_case_keys(dict(json.decode(data))) diff --git a/examples/bzlmod/hello_world_no_cargo/.bazelrc b/examples/bzlmod/hello_world_no_cargo/.bazelrc new file mode 100644 index 0000000000..c75f46eee5 --- /dev/null +++ b/examples/bzlmod/hello_world_no_cargo/.bazelrc @@ -0,0 +1,3 @@ +common --experimental_enable_bzlmod +common --noenable_workspace +common --enable_runfiles diff --git a/examples/bzlmod/hello_world_no_cargo/.gitignore b/examples/bzlmod/hello_world_no_cargo/.gitignore new file mode 100644 index 0000000000..a6ef824c1f --- /dev/null +++ b/examples/bzlmod/hello_world_no_cargo/.gitignore @@ -0,0 +1 @@ +/bazel-* diff --git a/examples/bzlmod/hello_world_no_cargo/BUILD.bazel b/examples/bzlmod/hello_world_no_cargo/BUILD.bazel new file mode 100644 index 0000000000..ad4cb5914a --- /dev/null +++ b/examples/bzlmod/hello_world_no_cargo/BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_rust//rust:defs.bzl", "rust_binary") + +package(default_visibility = ["//visibility:public"]) + +rust_binary( + name = "hello_world", + srcs = ["src/main.rs"], + deps = [ + "@crates//:anyhow", + ], +) diff --git a/examples/bzlmod/hello_world_no_cargo/MODULE.bazel b/examples/bzlmod/hello_world_no_cargo/MODULE.bazel new file mode 100644 index 0000000000..87459edb6d --- /dev/null +++ b/examples/bzlmod/hello_world_no_cargo/MODULE.bazel @@ -0,0 +1,26 @@ +"""bazelbuild/rules_rust - bzlmod no-cargo example""" + +module(name = "hello_world_no_cargo") + +bazel_dep( + name = "rules_rust", + version = "0.0.0", +) +local_path_override( + module_name = "rules_rust", + path = "../../..", +) + +rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") +rust.toolchain(edition = "2021") +use_repo(rust, "rust_toolchains") + +register_toolchains("@rust_toolchains//:all") + +crate = use_extension("@rules_rust//crate_universe:extension.bzl", "crate") +crate.spec( + package = "anyhow", + version = "1.0.77", +) +crate.from_specs() +use_repo(crate, "crates") diff --git a/examples/bzlmod/hello_world_no_cargo/src/main.rs b/examples/bzlmod/hello_world_no_cargo/src/main.rs new file mode 100644 index 0000000000..1aa867e38a --- /dev/null +++ b/examples/bzlmod/hello_world_no_cargo/src/main.rs @@ -0,0 +1,18 @@ +// Copyright 2015 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +fn main() -> anyhow::Result<()> { + println!("Hello, world!"); + Ok(()) +}