From f3d15526d284f920b1d8864321a6011fefb8695e Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 25 Oct 2020 21:05:38 +1000 Subject: [PATCH] $(location ...) expansion and runtime data support for build scripts - rustc_env now expands $(rootpath //package:target), for passing the location of files or tools that are needed at compile time - build_script_env now expands $(execpath //package:target), for passing the location of files or tools that are needed at build script runtime - cargo_script_build() now passes the data argument to the build script runner, so the data is available at runtime Ideally both build and run would work with execpath/location, but the approach of walking back to the root folder that cargo_build_script() is using doesn't seem to be possible at compile time, as ${pwd} is being substituted after the starlark code is executed. So at compile time, we use the relative rootpath instead. --- cargo/cargo_build_script.bzl | 41 ++++++++++++++++++++++++----- docs/cargo_build_script.md | 17 ++++++++---- docs/flatten.md | 27 ++++++++++++------- docs/rust.md | 8 +++--- docs/rust_wasm_bindgen.md | 2 +- examples/env_locations/BUILD | 46 +++++++++++++++++++++++++++++++++ examples/env_locations/build.rs | 9 +++++++ examples/env_locations/main.rs | 8 ++++++ rust/private/clippy.bzl | 1 + rust/private/rust.bzl | 3 +++ rust/private/rustc.bzl | 15 +++++++++-- 11 files changed, 148 insertions(+), 29 deletions(-) create mode 100644 examples/env_locations/BUILD create mode 100644 examples/env_locations/build.rs create mode 100644 examples/env_locations/main.rs diff --git a/cargo/cargo_build_script.bzl b/cargo/cargo_build_script.bzl index 9877422d60..6bb91927bd 100644 --- a/cargo/cargo_build_script.bzl +++ b/cargo/cargo_build_script.bzl @@ -4,6 +4,20 @@ load("@io_bazel_rules_rust//rust:private/rustc.bzl", "BuildInfo", "DepInfo", "ge load("@io_bazel_rules_rust//rust:private/utils.bzl", "find_toolchain") load("@io_bazel_rules_rust//rust:rust.bzl", "rust_binary") +def _expand_location(ctx, data, subfolder_level, env_string): + expanded = ctx.expand_location(env_string, data) + + # if a variable was expanded, make path relative to working directory + if env_string != expanded: + expanded = "/".join([".."] * subfolder_level) + "/" + expanded + return expanded + +def _expand_locations(ctx, subfolder_level): + "Expand $(execroot ...) references in user-provided env vars." + env = ctx.attr.build_script_env + data = getattr(ctx.attr, "data", []) + return dict([(k, _expand_location(ctx, data, subfolder_level, v)) for (k, v) in env.items()]) + def _build_script_impl(ctx): """The implementation for the `_build_script_run` rule. @@ -88,14 +102,14 @@ def _build_script_impl(ctx): for f in ctx.attr.crate_features: env["CARGO_FEATURE_" + f.upper().replace("-", "_")] = "1" - env.update(ctx.attr.build_script_env) + env.update(_expand_locations(ctx, manifest_dir.count("/") + 1)) tools = depset( direct = [ script, ctx.executable._cargo_build_script_runner, toolchain.rustc, - ], + ] + ctx.files.data, transitive = toolchain_tools, ) @@ -164,6 +178,10 @@ _build_script_run = rule( "build_script_env": attr.string_dict( doc = "Environment variables for build scripts.", ), + "data": attr.label_list( + doc = "Data or tools required by the build script.", + allow_files = True, + ), "_cc_toolchain": attr.label( default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"), ), @@ -188,6 +206,7 @@ def cargo_build_script( version = None, deps = [], build_script_env = {}, + data = [], **kwargs): """Compile and execute a rust build script to generate build attributes @@ -220,10 +239,16 @@ def cargo_build_script( cargo_build_script( name = "build_script", srcs = ["build.rs"], - # Data are shipped during execution. - data = ["src/lib.rs"], - # Environment variables passed during build.rs execution - build_script_env = {"CARGO_PKG_VERSION": "0.1.2"}, + # Optional environment variables passed during build.rs compilation + rustc_env = { + "CARGO_PKG_VERSION": "0.1.2", + }, + # Optional environment variables passed during build.rs execution + build_script_env = { + "SOME_TOOL_OR_FILE": "$(execroot @tool//:binary)" + } + # Optional data/tool dependencies + data = ["@tool//:binary"], ) rust_library( @@ -245,6 +270,7 @@ def cargo_build_script( version (str, optional): The semantic version (semver) of the crate. deps (list, optional): The dependencies of the crate defined by `crate_name`. build_script_env (dict, optional): Environment variables for build scripts. + data (list, optional): Files or tools needed by the build script. **kwargs: Forwards to the underlying `rust_binary` rule. """ rust_binary( @@ -252,6 +278,7 @@ def cargo_build_script( crate_features = crate_features, version = version, deps = deps, + data = data, **kwargs ) _build_script_run( @@ -261,5 +288,5 @@ def cargo_build_script( crate_features = crate_features, version = version, build_script_env = build_script_env, - deps = deps, + data = data, ) diff --git a/docs/cargo_build_script.md b/docs/cargo_build_script.md index 51477960fc..bee5801398 100644 --- a/docs/cargo_build_script.md +++ b/docs/cargo_build_script.md @@ -6,7 +6,7 @@ ## cargo_build_script 
-cargo_build_script(name, crate_name, crate_features, version, deps, build_script_env, kwargs)
+cargo_build_script(name, crate_name, crate_features, version, deps, build_script_env, data, kwargs)
Compile and execute a rust build script to generate build attributes @@ -40,10 +40,16 @@ load("@io_bazel_rules_rust//cargo:cargo_build_script.bzl", "cargo_build_script") cargo_build_script( name = "build_script", srcs = ["build.rs"], - # Data are shipped during execution. - data = ["src/lib.rs"], - # Environment variables passed during build.rs execution - build_script_env = {"CARGO_PKG_VERSION": "0.1.2"}, + # Optional environment variables passed during build.rs compilation + rustc_env = { + "CARGO_PKG_VERSION": "0.1.2", + }, + # Optional environment variables passed during build.rs execution + build_script_env = { + "SOME_TOOL_OR_FILE": "$(execroot @tool//:binary)" + } + # Optional data/tool dependencies + data = ["@tool//:binary"], ) rust_library( @@ -69,6 +75,7 @@ The `hello_lib` target will be build with the flags and the environment variable | version | The semantic version (semver) of the crate. | None | | deps | The dependencies of the crate defined by crate_name. | [] | | build_script_env | Environment variables for build scripts. | {} | +| data | Files or tools needed by the build script. | [] | | kwargs | Forwards to the underlying rust_binary rule. | none | diff --git a/docs/flatten.md b/docs/flatten.md index 17c41d6bc2..93494930f8 100644 --- a/docs/flatten.md +++ b/docs/flatten.md @@ -122,7 +122,7 @@ Run the benchmark test using: `bazel run //fibonacci:fibonacci_bench`. | edition | The rust edition to use for this crate. Defaults to the edition specified in the rust_toolchain. | String | optional | "" | | out_dir_tar | __Deprecated__, do not use, see [#cargo_build_script] instead. | Label | optional | None | | proc_macro_deps | List of rust_library targets with kind proc-macro used to help build this library target. | List of labels | optional | [] | -| rustc_env | Dictionary of additional "key": "value" environment variables to set for rustc. | Dictionary: String -> String | optional | {} | +| rustc_env | Dictionary of additional "key": "value" environment variables to set for rustc.

Supports $(rootpath ...) expansion, so you can use this to pass in the path to a generated file or an external tool. | Dictionary: String -> String | optional | {} | | rustc_flags | List of compiler flags passed to rustc. | List of strings | optional | [] | | srcs | List of Rust .rs source files used to build the library.

If srcs contains more than one file, then there must be a file either named lib.rs. Otherwise, crate_root must be set to the source file that is the root of the crate to be passed to rustc to build this crate. | List of labels | optional | [] | | version | A version to inject in the cargo environment variable. | String | optional | "0.0.0" | @@ -239,7 +239,7 @@ Hello world | out_binary | - | Boolean | optional | False | | out_dir_tar | __Deprecated__, do not use, see [#cargo_build_script] instead. | Label | optional | None | | proc_macro_deps | List of rust_library targets with kind proc-macro used to help build this library target. | List of labels | optional | [] | -| rustc_env | Dictionary of additional "key": "value" environment variables to set for rustc. | Dictionary: String -> String | optional | {} | +| rustc_env | Dictionary of additional "key": "value" environment variables to set for rustc.

Supports $(rootpath ...) expansion, so you can use this to pass in the path to a generated file or an external tool. | Dictionary: String -> String | optional | {} | | rustc_flags | List of compiler flags passed to rustc. | List of strings | optional | [] | | srcs | List of Rust .rs source files used to build the library.

If srcs contains more than one file, then there must be a file either named lib.rs. Otherwise, crate_root must be set to the source file that is the root of the crate to be passed to rustc to build this crate. | List of labels | optional | [] | | version | A version to inject in the cargo environment variable. | String | optional | "0.0.0" | @@ -535,7 +535,7 @@ INFO: Elapsed time: 1.245s, Critical Path: 1.01s | edition | The rust edition to use for this crate. Defaults to the edition specified in the rust_toolchain. | String | optional | "" | | out_dir_tar | __Deprecated__, do not use, see [#cargo_build_script] instead. | Label | optional | None | | proc_macro_deps | List of rust_library targets with kind proc-macro used to help build this library target. | List of labels | optional | [] | -| rustc_env | Dictionary of additional "key": "value" environment variables to set for rustc. | Dictionary: String -> String | optional | {} | +| rustc_env | Dictionary of additional "key": "value" environment variables to set for rustc.

Supports $(rootpath ...) expansion, so you can use this to pass in the path to a generated file or an external tool. | Dictionary: String -> String | optional | {} | | rustc_flags | List of compiler flags passed to rustc. | List of strings | optional | [] | | srcs | List of Rust .rs source files used to build the library.

If srcs contains more than one file, then there must be a file either named lib.rs. Otherwise, crate_root must be set to the source file that is the root of the crate to be passed to rustc to build this crate. | List of labels | optional | [] | | version | A version to inject in the cargo environment variable. | String | optional | "0.0.0" | @@ -792,7 +792,7 @@ Run the test with `bazel build //hello_lib:hello_lib_test`. | edition | The rust edition to use for this crate. Defaults to the edition specified in the rust_toolchain. | String | optional | "" | | out_dir_tar | __Deprecated__, do not use, see [#cargo_build_script] instead. | Label | optional | None | | proc_macro_deps | List of rust_library targets with kind proc-macro used to help build this library target. | List of labels | optional | [] | -| rustc_env | Dictionary of additional "key": "value" environment variables to set for rustc. | Dictionary: String -> String | optional | {} | +| rustc_env | Dictionary of additional "key": "value" environment variables to set for rustc.

Supports $(rootpath ...) expansion, so you can use this to pass in the path to a generated file or an external tool. | Dictionary: String -> String | optional | {} | | rustc_flags | List of compiler flags passed to rustc. | List of strings | optional | [] | | srcs | List of Rust .rs source files used to build the library.

If srcs contains more than one file, then there must be a file either named lib.rs. Otherwise, crate_root must be set to the source file that is the root of the crate to be passed to rustc to build this crate. | List of labels | optional | [] | | version | A version to inject in the cargo environment variable. | String | optional | "0.0.0" | @@ -963,7 +963,7 @@ The tools required for the `rust_wasm_bindgen` rule. | Name | Description | Type | Mandatory | Default | | :------------- | :------------- | :------------- | :------------- | :------------- | | name | A unique name for this target. | Name | required | | -| bindgen | The label of a bindgen executable. | Label | optional | None | +| bindgen | The label of a wasm-bindgen executable. | Label | optional | None | @@ -971,7 +971,7 @@ The tools required for the `rust_wasm_bindgen` rule. ## cargo_build_script 
-cargo_build_script(name, crate_name, crate_features, version, deps, build_script_env, kwargs)
+cargo_build_script(name, crate_name, crate_features, version, deps, build_script_env, data, kwargs)
Compile and execute a rust build script to generate build attributes @@ -1005,10 +1005,16 @@ load("@io_bazel_rules_rust//cargo:cargo_build_script.bzl", "cargo_build_script") cargo_build_script( name = "build_script", srcs = ["build.rs"], - # Data are shipped during execution. - data = ["src/lib.rs"], - # Environment variables passed during build.rs execution - build_script_env = {"CARGO_PKG_VERSION": "0.1.2"}, + # Optional environment variables passed during build.rs compilation + rustc_env = { + "CARGO_PKG_VERSION": "0.1.2", + }, + # Optional environment variables passed during build.rs execution + build_script_env = { + "SOME_TOOL_OR_FILE": "$(execroot @tool//:binary)" + } + # Optional data/tool dependencies + data = ["@tool//:binary"], ) rust_library( @@ -1034,6 +1040,7 @@ The `hello_lib` target will be build with the flags and the environment variable | version | The semantic version (semver) of the crate. | None | | deps | The dependencies of the crate defined by crate_name. | [] | | build_script_env | Environment variables for build scripts. | {} | +| data | Files or tools needed by the build script. | [] | | kwargs | Forwards to the underlying rust_binary rule. | none | diff --git a/docs/rust.md b/docs/rust.md index d85c2fe970..a5fcd8567b 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -103,7 +103,7 @@ Run the benchmark test using: `bazel run //fibonacci:fibonacci_bench`. | edition | The rust edition to use for this crate. Defaults to the edition specified in the rust_toolchain. | String | optional | "" | | out_dir_tar | __Deprecated__, do not use, see [#cargo_build_script] instead. | Label | optional | None | | proc_macro_deps | List of rust_library targets with kind proc-macro used to help build this library target. | List of labels | optional | [] | -| rustc_env | Dictionary of additional "key": "value" environment variables to set for rustc. | Dictionary: String -> String | optional | {} | +| rustc_env | Dictionary of additional "key": "value" environment variables to set for rustc.

Supports $(rootpath ...) expansion, so you can use this to pass in the path to a generated file or an external tool. | Dictionary: String -> String | optional | {} | | rustc_flags | List of compiler flags passed to rustc. | List of strings | optional | [] | | srcs | List of Rust .rs source files used to build the library.

If srcs contains more than one file, then there must be a file either named lib.rs. Otherwise, crate_root must be set to the source file that is the root of the crate to be passed to rustc to build this crate. | List of labels | optional | [] | | version | A version to inject in the cargo environment variable. | String | optional | "0.0.0" | @@ -220,7 +220,7 @@ Hello world | out_binary | - | Boolean | optional | False | | out_dir_tar | __Deprecated__, do not use, see [#cargo_build_script] instead. | Label | optional | None | | proc_macro_deps | List of rust_library targets with kind proc-macro used to help build this library target. | List of labels | optional | [] | -| rustc_env | Dictionary of additional "key": "value" environment variables to set for rustc. | Dictionary: String -> String | optional | {} | +| rustc_env | Dictionary of additional "key": "value" environment variables to set for rustc.

Supports $(rootpath ...) expansion, so you can use this to pass in the path to a generated file or an external tool. | Dictionary: String -> String | optional | {} | | rustc_flags | List of compiler flags passed to rustc. | List of strings | optional | [] | | srcs | List of Rust .rs source files used to build the library.

If srcs contains more than one file, then there must be a file either named lib.rs. Otherwise, crate_root must be set to the source file that is the root of the crate to be passed to rustc to build this crate. | List of labels | optional | [] | | version | A version to inject in the cargo environment variable. | String | optional | "0.0.0" | @@ -314,7 +314,7 @@ INFO: Elapsed time: 1.245s, Critical Path: 1.01s | edition | The rust edition to use for this crate. Defaults to the edition specified in the rust_toolchain. | String | optional | "" | | out_dir_tar | __Deprecated__, do not use, see [#cargo_build_script] instead. | Label | optional | None | | proc_macro_deps | List of rust_library targets with kind proc-macro used to help build this library target. | List of labels | optional | [] | -| rustc_env | Dictionary of additional "key": "value" environment variables to set for rustc. | Dictionary: String -> String | optional | {} | +| rustc_env | Dictionary of additional "key": "value" environment variables to set for rustc.

Supports $(rootpath ...) expansion, so you can use this to pass in the path to a generated file or an external tool. | Dictionary: String -> String | optional | {} | | rustc_flags | List of compiler flags passed to rustc. | List of strings | optional | [] | | srcs | List of Rust .rs source files used to build the library.

If srcs contains more than one file, then there must be a file either named lib.rs. Otherwise, crate_root must be set to the source file that is the root of the crate to be passed to rustc to build this crate. | List of labels | optional | [] | | version | A version to inject in the cargo environment variable. | String | optional | "0.0.0" | @@ -473,7 +473,7 @@ Run the test with `bazel build //hello_lib:hello_lib_test`. | edition | The rust edition to use for this crate. Defaults to the edition specified in the rust_toolchain. | String | optional | "" | | out_dir_tar | __Deprecated__, do not use, see [#cargo_build_script] instead. | Label | optional | None | | proc_macro_deps | List of rust_library targets with kind proc-macro used to help build this library target. | List of labels | optional | [] | -| rustc_env | Dictionary of additional "key": "value" environment variables to set for rustc. | Dictionary: String -> String | optional | {} | +| rustc_env | Dictionary of additional "key": "value" environment variables to set for rustc.

Supports $(rootpath ...) expansion, so you can use this to pass in the path to a generated file or an external tool. | Dictionary: String -> String | optional | {} | | rustc_flags | List of compiler flags passed to rustc. | List of strings | optional | [] | | srcs | List of Rust .rs source files used to build the library.

If srcs contains more than one file, then there must be a file either named lib.rs. Otherwise, crate_root must be set to the source file that is the root of the crate to be passed to rustc to build this crate. | List of labels | optional | [] | | version | A version to inject in the cargo environment variable. | String | optional | "0.0.0" | diff --git a/docs/rust_wasm_bindgen.md b/docs/rust_wasm_bindgen.md index 7713f6f299..58e1daff24 100644 --- a/docs/rust_wasm_bindgen.md +++ b/docs/rust_wasm_bindgen.md @@ -38,6 +38,6 @@ The tools required for the `rust_wasm_bindgen` rule. | Name | Description | Type | Mandatory | Default | | :------------- | :------------- | :------------- | :------------- | :------------- | | name | A unique name for this target. | Name | required | | -| bindgen | The label of a bindgen executable. | Label | optional | None | +| bindgen | The label of a wasm-bindgen executable. | Label | optional | None | diff --git a/examples/env_locations/BUILD b/examples/env_locations/BUILD new file mode 100644 index 0000000000..6d2b554067 --- /dev/null +++ b/examples/env_locations/BUILD @@ -0,0 +1,46 @@ +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_test", +) +load("@io_bazel_rules_rust//cargo:cargo_build_script.bzl", "cargo_build_script") + +# generate a file +genrule( + name = "data_generator", + outs = ["generated.data"], + cmd = "echo hello > $@", +) + +_data = [ + # provide the generated file as a dep + "generated.data", + # we should also be able to access external binaries + # such as protoc. + "@com_google_protobuf//:protoc", +] + +cargo_build_script( + name = "build", + srcs = ["build.rs"], + build_script_env = { + "GENERATED_DATA": "$(execpath generated.data)", + "SOME_TOOL": "$(execpath @com_google_protobuf//:protoc)", + }, + data = _data, +) + +rust_test( + name = "test", + srcs = [ + "main.rs", + ], + data = _data, + edition = "2018", + rustc_env = { + "GENERATED_DATA": "$(rootpath generated.data)", + "SOME_TOOL": "$(rootpath @com_google_protobuf//:protoc)", + }, + deps = [ + ":build", + ], +) diff --git a/examples/env_locations/build.rs b/examples/env_locations/build.rs new file mode 100644 index 0000000000..be9e9a6e0c --- /dev/null +++ b/examples/env_locations/build.rs @@ -0,0 +1,9 @@ +use std::{fs,env}; + +fn main() { + let generated_data = fs::read_to_string(env::var("GENERATED_DATA").unwrap()).unwrap(); + // our generated data file should be readable as the build script runs + assert_eq!(generated_data, "hello\n"); + // and we should be able to read (and thus execute) our tool + assert_eq!(fs::read(env::var("SOME_TOOL").unwrap()).unwrap().is_empty(), false); +} diff --git a/examples/env_locations/main.rs b/examples/env_locations/main.rs new file mode 100644 index 0000000000..4976031965 --- /dev/null +++ b/examples/env_locations/main.rs @@ -0,0 +1,8 @@ +#[test] +fn test() { + let generated_data = std::fs::read_to_string(env!("GENERATED_DATA")).unwrap(); + // our generated data file should be readable + assert_eq!(generated_data, "hello\n"); + // and we should be able to read (and thus execute) our tool + assert_eq!(std::fs::read(env!("SOME_TOOL")).unwrap().is_empty(), false); +} diff --git a/rust/private/clippy.bzl b/rust/private/clippy.bzl index f01cc6b4ae..610aeea4ae 100644 --- a/rust/private/clippy.bzl +++ b/rust/private/clippy.bzl @@ -92,6 +92,7 @@ def _clippy_aspect_impl(target, ctx): build_env_file = build_env_file, build_flags_files = build_flags_files, maker_path = clippy_marker.path, + aspect = True, ) # Deny the default-on clippy warning levels. diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index dfa2fc3a3e..e6daaca1ea 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -417,6 +417,9 @@ _rust_common_attrs = { "rustc_env": attr.string_dict( doc = _tidy(""" Dictionary of additional `"key": "value"` environment variables to set for rustc. + + Supports $(rootpath ...) expansion, so you can use this to pass in the path to + a generated file or an external tool. """), ), "crate_features": attr.string_list( diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 5ad97c9eeb..ce46354afc 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -311,6 +311,15 @@ def get_linker_and_args(ctx, cc_toolchain, feature_configuration, rpaths): return ld, link_args, link_env +def _expand_locations(ctx, env, aspect): + "Expand $(rootpath ...) references in user-provided env vars." + if aspect: + data = getattr(ctx.rule.attr, "data", []) + else: + data = getattr(ctx.attr, "data", []) + + return dict([(k, ctx.expand_location(v, data)) for (k, v) in env.items()]) + def _process_build_scripts( ctx, file, @@ -400,7 +409,8 @@ def construct_arguments( out_dir, build_env_file, build_flags_files, - maker_path = None): + maker_path = None, + aspect = False): """Builds an Args object containing common rustc flags Args: @@ -418,6 +428,7 @@ def construct_arguments( build_env_file (str): The output file of a `cargo_build_script` action containing rustc environment variables build_flags_files (list): The output files of a `cargo_build_script` actions containing rustc build flags maker_path (File): An optional clippy marker file + aspect (bool): True if called in an aspect context. Returns: tuple: A tuple of the following items @@ -540,7 +551,7 @@ def construct_arguments( env["CARGO_BIN_EXE_" + dep_crate_info.output.basename] = dep_crate_info.output.short_path # Update environment with user provided variables. - env.update(crate_info.rustc_env) + env.update(_expand_locations(ctx, crate_info.rustc_env, aspect)) # This empty value satisfies Clippy, which otherwise complains about the # sysroot being undefined.