From a8cdb51855fd85191e7808eea8eeda18042930ff Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Mon, 25 Apr 2022 09:45:15 -0700 Subject: [PATCH] crate_universe: Added tests for serialized config files. --- crate_universe/BUILD.bazel | 1 + crate_universe/private/crates_repository.bzl | 6 +- crate_universe/private/crates_vendor.bzl | 18 ++-- crate_universe/private/generate_utils.bzl | 85 +++++++++++++++---- crate_universe/private/splicing_utils.bzl | 53 +++++++++--- crate_universe/src/config.rs | 32 +++++++ crate_universe/src/splicing.rs | 49 +++++++++++ .../test_data/serialized_configs/BUILD.bazel | 64 ++++++++++++++ 8 files changed, 261 insertions(+), 47 deletions(-) create mode 100644 crate_universe/test_data/serialized_configs/BUILD.bazel diff --git a/crate_universe/BUILD.bazel b/crate_universe/BUILD.bazel index 47f3bc8b78..523e3254a0 100644 --- a/crate_universe/BUILD.bazel +++ b/crate_universe/BUILD.bazel @@ -95,6 +95,7 @@ rust_test( data = glob(["test_data/**"]) + [ "@rules_rust//rust/toolchain:current_exec_cargo_files", "@rules_rust//rust/toolchain:current_exec_rustc_files", + "//crate_universe/test_data/serialized_configs", ], proc_macro_deps = all_crate_deps( proc_macro_dev = True, diff --git a/crate_universe/private/crates_repository.bzl b/crate_universe/private/crates_repository.bzl index 41ace04894..a25e3a06e5 100644 --- a/crate_universe/private/crates_repository.bzl +++ b/crate_universe/private/crates_repository.bzl @@ -27,7 +27,7 @@ def _crates_repository_impl(repository_ctx): generator, generator_sha256 = get_generator(repository_ctx, host_triple.triple) # Generate a config file for all settings - config = generate_config(repository_ctx) + config_path = generate_config(repository_ctx) # Locate the lockfile lockfile = get_lockfile(repository_ctx) @@ -46,7 +46,7 @@ def _crates_repository_impl(repository_ctx): generator = generator, lockfile_path = lockfile.path, lockfile_kind = lockfile.kind, - config = config.path, + config = config_path, splicing_manifest = splicing_manifest, cargo = cargo_path, rustc = rustc_path, @@ -74,7 +74,7 @@ def _crates_repository_impl(repository_ctx): execute_generator( repository_ctx = repository_ctx, generator = generator, - config = config.path, + config = config_path, splicing_manifest = splicing_manifest, lockfile_path = lockfile.path, lockfile_kind = lockfile.kind, diff --git a/crate_universe/private/crates_vendor.bzl b/crate_universe/private/crates_vendor.bzl index ceaba69a2e..6846af4005 100644 --- a/crate_universe/private/crates_vendor.bzl +++ b/crate_universe/private/crates_vendor.bzl @@ -1,6 +1,6 @@ """Rules for vendoring Bazel targets into existing workspaces""" -load("//crate_universe/private:generate_utils.bzl", "collect_crate_annotations", "render_config") +load("//crate_universe/private:generate_utils.bzl", "compile_config", "render_config") load("//crate_universe/private:splicing_utils.bzl", "kebab_case_keys", "splicing_config") load("//crate_universe/private:urls.bzl", "CARGO_BAZEL_LABEL") load("//rust/platform:triple_mappings.bzl", "SUPPORTED_PLATFORM_TRIPLES") @@ -104,14 +104,6 @@ def _write_extra_manifests_manifest(ctx): return args, runfiles def _write_config_file(ctx): - annotations = collect_crate_annotations(ctx.attr.annotations, str(ctx.label)) - unexpected = [] - for id, annotation in annotations.items(): - if annotation.get("additive_build_file", None): - unexpected.append(id) - if unexpected: - fail("The following annotations use `additive_build_file` which is not supported for `crates_vendor`: {}".format(unexpected)) - rendering_config = dict(json.decode(render_config())) output_pkg = _get_output_package(ctx) @@ -135,16 +127,16 @@ def _write_config_file(ctx): ctx.workspace_name, output_pkg, ), - "repository_name": ctx.attr.repository_name or ctx.label.name, "vendor_mode": ctx.attr.mode, }) - config_data = struct( - annotations = annotations, - rendering = rendering_config, + config_data = compile_config( + crate_annotations = ctx.attr.annotations, generate_build_scripts = ctx.attr.generate_build_scripts, cargo_config = None, + render_config = rendering_config, supported_platform_triples = ctx.attr.supported_platform_triples, + repository_name = ctx.attr.repository_name or ctx.label.name, ) config = _write_data_file( diff --git a/crate_universe/private/generate_utils.bzl b/crate_universe/private/generate_utils.bzl index 2329b557b6..95ce36f7a4 100644 --- a/crate_universe/private/generate_utils.bzl +++ b/crate_universe/private/generate_utils.bzl @@ -182,31 +182,58 @@ def _read_cargo_config(repository_ctx): return repository_ctx.read(config) return None +def _update_render_config(config, repository_name): + """Add the repository name to the render config + + Args: + config (dict): A `render_config` struct + repository_name (str): The name of the repository that owns the config + + Returns: + struct: An updated `render_config`. + """ + + # Add the repository name as it's very relevant to rendering. + config.update({"repository_name": repository_name}) + + return struct(**config) + def _get_render_config(repository_ctx): if repository_ctx.attr.render_config: config = dict(json.decode(repository_ctx.attr.render_config)) else: config = dict(json.decode(render_config())) - # Add the repository name as it's very relevant to rendering. - config.update({"repository_name": repository_ctx.name}) + return config - return struct(**config) +def compile_config(crate_annotations, generate_build_scripts, cargo_config, render_config, supported_platform_triples, repository_name, repository_ctx = None): + """Create a config file for generating crate targets -def generate_config(repository_ctx): - """Generate a config file from various attributes passed to the rule. + [cargo_config]: https://doc.rust-lang.org/cargo/reference/config.html Args: - repository_ctx (repository_ctx): The rule's context object. + crate_annotations (dict): Extra settings to apply to crates. See + `crates_repository.annotations` or `crates_vendor.annotations`. + generate_build_scripts (bool): Whether or not to globally disable build scripts. + cargo_config (str): The optional contents of a [Cargo config][cargo_config]. + render_config (dict): The deserialized dict of the `render_config` function. + supported_platform_triples (list): A list of platform triples + repository_name (str): The name of the repository being generated + repository_ctx (repository_ctx, optional): A repository context object used for enabling + certain functionality. Returns: - struct: A struct containing the path to a config and it's contents + struct: A struct matching a `cargo_bazel::config::Config`. """ - annotations = collect_crate_annotations(repository_ctx.attr.annotations, repository_ctx.name) + annotations = collect_crate_annotations(crate_annotations, repository_name) # Load additive build files if any have been provided. - for data in annotations.values(): + unexpected = [] + for name, data in annotations.items(): f = data.pop("additive_build_file", None) + if f and not repository_ctx: + unexpected.append(name) + f = None content = [x for x in [ data.pop("additive_build_file_content", None), repository_ctx.read(Label(f)) if f else None, @@ -214,12 +241,40 @@ def generate_config(repository_ctx): if content: data.update({"additive_build_file_content": "\n".join(content)}) + if unexpected: + fail("The following annotations use `additive_build_file` which is not supported for {}: {}".format(repository_name, unexpected)) + config = struct( - generate_build_scripts = repository_ctx.attr.generate_build_scripts, + generate_build_scripts = generate_build_scripts, annotations = annotations, + cargo_config = cargo_config, + rendering = _update_render_config( + config = render_config, + repository_name = repository_name, + ), + supported_platform_triples = supported_platform_triples, + ) + + return config + +def generate_config(repository_ctx): + """Generate a config file from various attributes passed to the rule. + + Args: + repository_ctx (repository_ctx): The rule's context object. + + Returns: + struct: A struct containing the path to a config and it's contents + """ + + config = compile_config( + crate_annotations = repository_ctx.attr.annotations, + generate_build_scripts = repository_ctx.attr.generate_build_scripts, cargo_config = _read_cargo_config(repository_ctx), - rendering = _get_render_config(repository_ctx), + render_config = _get_render_config(repository_ctx), supported_platform_triples = repository_ctx.attr.supported_platform_triples, + repository_name = repository_ctx.name, + repository_ctx = repository_ctx, ) config_path = repository_ctx.path("cargo-bazel.json") @@ -228,13 +283,7 @@ def generate_config(repository_ctx): json.encode_indent(config, indent = " " * 4), ) - # This was originally written to return a struct and not just the config path - # so splicing can have access to some rendering information embedded in the config - # If splicing should no longer need that info, it'd be simpler to just return a `path`. - return struct( - path = config_path, - info = config, - ) + return config_path def get_lockfile(repository_ctx): """Locate the lockfile and identify the it's type (Cargo or Bazel). diff --git a/crate_universe/private/splicing_utils.bzl b/crate_universe/private/splicing_utils.bzl index 249e7a44b9..44ca076812 100644 --- a/crate_universe/private/splicing_utils.bzl +++ b/crate_universe/private/splicing_utils.bzl @@ -92,24 +92,48 @@ def kebab_case_keys(data): for (key, val) in data.items() } -def create_splicing_manifest(repository_ctx): +def compile_splicing_manifest(splicing_config, manifests, cargo_config_path, packages): """Produce a manifest containing required components for splciing a new Cargo workspace + [cargo_config]: https://doc.rust-lang.org/cargo/reference/config.html + [cargo_toml]: https://doc.rust-lang.org/cargo/reference/manifest.html + Args: - repository_ctx (repository_ctx): The rule's context object. + splicing_config (dict): A deserialized `splicing_config` + manifests (dict): A mapping of paths to Bazel labels which represent [Cargo manifests][cargo_toml]. + cargo_config_path (str): The absolute path to a [Cargo config][cargo_config]. + packages (dict): A set of crates (packages) specifications to depend on Returns: - path: The path to a json encoded manifest + dict: A dictionary representation of a `cargo_bazel::splicing::SplicingManifest` """ - repo_dir = repository_ctx.path(".") # Deserialize information about direct packges direct_packages_info = { # Ensure the data is using kebab-case as that's what `cargo_toml::DependencyDetail` expects. pkg: kebab_case_keys(dict(json.decode(data))) - for (pkg, data) in repository_ctx.attr.packages.items() + for (pkg, data) in packages.items() } + # Auto-generated splicier manifest values + splicing_manifest_content = { + "cargo_config": cargo_config_path, + "direct_packages": direct_packages_info, + "manifests": manifests, + } + + return dict(splicing_config.items() + splicing_manifest_content.items()) + +def create_splicing_manifest(repository_ctx): + """Produce a manifest containing required components for splciing a new Cargo workspace + + Args: + repository_ctx (repository_ctx): The rule's context object. + + Returns: + path: The path to a json encoded manifest + """ + manifests = {str(repository_ctx.path(m)): str(m) for m in repository_ctx.attr.manifests} if repository_ctx.attr.cargo_config: @@ -120,19 +144,22 @@ def create_splicing_manifest(repository_ctx): # Load user configurable splicing settings config = json.decode(repository_ctx.attr.splicing_config or splicing_config()) - # Auto-generated splicier manifest values - splicing_manifest_content = { - "cargo_config": cargo_config, - "direct_packages": direct_packages_info, - "manifests": manifests, - } + repo_dir = repository_ctx.path(".") - # Serialize information required for splicing splicing_manifest = repository_ctx.path("{}/splicing_manifest.json".format(repo_dir)) + + data = compile_splicing_manifest( + splicing_config = config, + manifests = manifests, + cargo_config_path = cargo_config, + packages = repository_ctx.attr.packages, + ) + + # Serialize information required for splicing repository_ctx.file( splicing_manifest, json.encode_indent( - dict(dict(config).items() + splicing_manifest_content.items()), + data, indent = " " * 4, ), ) diff --git a/crate_universe/src/config.rs b/crate_universe/src/config.rs index 66e3a7ebae..ec3a0f7b29 100644 --- a/crate_universe/src/config.rs +++ b/crate_universe/src/config.rs @@ -492,4 +492,36 @@ mod test { id.version = "<1".to_owned(); assert!(!id.matches(&package)); } + + #[test] + fn deserialize_config() { + let runfiles = runfiles::Runfiles::create().unwrap(); + let path = runfiles + .rlocation("rules_rust/crate_universe/test_data/serialized_configs/config.json"); + + let content = std::fs::read_to_string(path).unwrap(); + + let config: Config = serde_json::from_str(&content).unwrap(); + + println!("{:#?}", config); + // Annotations + let annotation = config + .annotations + .get(&CrateId::new("rand".to_owned(), "0.8.5".to_owned())) + .unwrap(); + assert_eq!( + annotation.crate_features, + Some(BTreeSet::from(["small_rng".to_owned()])) + ); + + // Global settings + assert!(config.cargo_config.is_none()); + assert!(!config.generate_build_scripts); + + // Render Config + assert_eq!( + config.rendering.platforms_template, + "//custom/platform:{triple}" + ); + } } diff --git a/crate_universe/src/splicing.rs b/crate_universe/src/splicing.rs index 0de1daadb0..74e012e9a6 100644 --- a/crate_universe/src/splicing.rs +++ b/crate_universe/src/splicing.rs @@ -493,3 +493,52 @@ pub fn generate_lockfile( Ok(lockfile) } + +#[cfg(test)] +mod test { + use super::*; + + use std::path::PathBuf; + + #[test] + fn deserialize_splicing_manifest() { + let runfiles = runfiles::Runfiles::create().unwrap(); + let path = runfiles.rlocation( + "rules_rust/crate_universe/test_data/serialized_configs/splicing_manifest.json", + ); + + let content = std::fs::read_to_string(path).unwrap(); + + let manifest: SplicingManifest = serde_json::from_str(&content).unwrap(); + + // Check splicing configs + assert_eq!(manifest.resolver_version, cargo_toml::Resolver::V2); + + // Check manifests + assert_eq!(manifest.manifests.len(), 1); + let maniefst_label = manifest + .manifests + .get(&PathBuf::from("/tmp/abs/path/workspace/Cargo.toml")) + .unwrap(); + assert_eq!(maniefst_label, &Label::from_str("//:Cargo.toml").unwrap()); + + // Check packages + assert_eq!(manifest.direct_packages.len(), 1); + let package = manifest.direct_packages.get("rand").unwrap(); + assert_eq!( + package, + &cargo_toml::DependencyDetail { + default_features: Some(false), + features: vec!["small_rng".to_owned()], + version: Some("0.8.5".to_owned()), + ..Default::default() + } + ); + + // Check cargo config + assert_eq!( + manifest.cargo_config, + Some(PathBuf::from("/tmp/abs/path/workspace/.cargo/config.toml")) + ); + } +} diff --git a/crate_universe/test_data/serialized_configs/BUILD.bazel b/crate_universe/test_data/serialized_configs/BUILD.bazel new file mode 100644 index 0000000000..81d8bc41ad --- /dev/null +++ b/crate_universe/test_data/serialized_configs/BUILD.bazel @@ -0,0 +1,64 @@ +load("@bazel_skylib//rules:write_file.bzl", "write_file") +load("//crate_universe:defs.bzl", "crate", "render_config", "splicing_config") + +# buildifier: disable=bzl-visibility +load("//crate_universe/private:generate_utils.bzl", "compile_config") + +# buildifier: disable=bzl-visibility +load("//crate_universe/private:splicing_utils.bzl", "compile_splicing_manifest") + +write_file( + name = "config", + out = "config.json", + content = [json.encode( + compile_config( + cargo_config = None, + crate_annotations = { + "rand": [crate.annotation( + crate_features = ["small_rng"], + version = "0.8.5", + )], + }, + generate_build_scripts = False, + render_config = json.decode(render_config( + platforms_template = "//custom/platform:{triple}", + )), + repository_name = "mock_config", + supported_platform_triples = [ + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + "x86_64-apple-darwin", + ], + ), + ).strip()], + newline = "unix", +) + +write_file( + name = "splicing_manifest", + out = "splicing_manifest.json", + content = [json.encode(compile_splicing_manifest( + cargo_config_path = "/tmp/abs/path/workspace/.cargo/config.toml", + manifests = {"/tmp/abs/path/workspace/Cargo.toml": "//:Cargo.toml"}, + packages = { + "rand": crate.spec( + default_features = False, + features = ["small_rng"], + version = "0.8.5", + ), + }, + splicing_config = dict(json.decode(splicing_config( + resolver_version = "2", + ))), + )).strip()], + newline = "unix", +) + +filegroup( + name = "serialized_configs", + srcs = [ + "config.json", + "splicing_manifest.json", + ], + visibility = ["//crate_universe:__pkg__"], +)