rust_toolchain now generates a Rust sysroot (#1119)

* `rust_toolchain` now recreates a Rust sysroot

* Regenerate documentation

* Updated docs

* Address merge conflicts

* Don't use leading `/`

* Regenerate documentation

* no explicit `--sysroot` flag.

* Better fix for `rust_doc_test`

* Set `--sysroot` in for `rust_doc_test`

* Update rust/private/toolchain_utils.bzl

Co-authored-by: Krasimir Georgiev <krasimir@google.com>

* Moved `generate_sysroot`

Co-authored-by: Krasimir Georgiev <krasimir@google.com>

docs/flatten.md

@@ -1110,9 +1110,9 @@ Run the test with `bazel build //hello_lib:hello_lib_test`.
 
 <pre>
 rust_toolchain(<a href="#rust_toolchain-name">name</a>, <a href="#rust_toolchain-allocator_library">allocator_library</a>, <a href="#rust_toolchain-binary_ext">binary_ext</a>, <a href="#rust_toolchain-cargo">cargo</a>, <a href="#rust_toolchain-clippy_driver">clippy_driver</a>, <a href="#rust_toolchain-debug_info">debug_info</a>,
-               <a href="#rust_toolchain-default_edition">default_edition</a>, <a href="#rust_toolchain-dylib_ext">dylib_ext</a>, <a href="#rust_toolchain-exec_triple">exec_triple</a>, <a href="#rust_toolchain-opt_level">opt_level</a>, <a href="#rust_toolchain-os">os</a>, <a href="#rust_toolchain-rust_doc">rust_doc</a>, <a href="#rust_toolchain-rust_lib">rust_lib</a>, <a href="#rust_toolchain-rust_std">rust_std</a>,
-               <a href="#rust_toolchain-rustc">rustc</a>, <a href="#rust_toolchain-rustc_lib">rustc_lib</a>, <a href="#rust_toolchain-rustc_srcs">rustc_srcs</a>, <a href="#rust_toolchain-rustfmt">rustfmt</a>, <a href="#rust_toolchain-staticlib_ext">staticlib_ext</a>, <a href="#rust_toolchain-stdlib_linkflags">stdlib_linkflags</a>, <a href="#rust_toolchain-target_json">target_json</a>,
-               <a href="#rust_toolchain-target_triple">target_triple</a>)
+               <a href="#rust_toolchain-default_edition">default_edition</a>, <a href="#rust_toolchain-dylib_ext">dylib_ext</a>, <a href="#rust_toolchain-exec_triple">exec_triple</a>, <a href="#rust_toolchain-llvm_tools">llvm_tools</a>, <a href="#rust_toolchain-opt_level">opt_level</a>, <a href="#rust_toolchain-os">os</a>, <a href="#rust_toolchain-rust_doc">rust_doc</a>, <a href="#rust_toolchain-rust_lib">rust_lib</a>,
+               <a href="#rust_toolchain-rust_std">rust_std</a>, <a href="#rust_toolchain-rustc">rustc</a>, <a href="#rust_toolchain-rustc_lib">rustc_lib</a>, <a href="#rust_toolchain-rustc_srcs">rustc_srcs</a>, <a href="#rust_toolchain-rustfmt">rustfmt</a>, <a href="#rust_toolchain-staticlib_ext">staticlib_ext</a>, <a href="#rust_toolchain-stdlib_linkflags">stdlib_linkflags</a>,
+               <a href="#rust_toolchain-target_json">target_json</a>, <a href="#rust_toolchain-target_triple">target_triple</a>)
 </pre>
 
 Declares a Rust toolchain for use.
@@ -1170,6 +1170,7 @@ See @rules_rust//rust:repositories.bzl for examples of defining the @rust_cpuX r
 | <a id="rust_toolchain-default_edition"></a>default_edition |  The edition to use for rust_* rules that don't specify an edition.   | String | optional | "2018" |
 | <a id="rust_toolchain-dylib_ext"></a>dylib_ext |  The extension for dynamic libraries created from rustc.   | String | required |  |
 | <a id="rust_toolchain-exec_triple"></a>exec_triple |  The platform triple for the toolchains execution environment. For more details see: https://docs.bazel.build/versions/master/skylark/rules.html#configurations   | String | required |  |
+| <a id="rust_toolchain-llvm_tools"></a>llvm_tools |  LLVM tools that are shipped with the Rust toolchain.   | <a href="https://bazel.build/docs/build-ref.html#labels">List of labels</a> | optional | [] |
 | <a id="rust_toolchain-opt_level"></a>opt_level |  Rustc optimization levels.   | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {"dbg": "0", "fastbuild": "0", "opt": "3"} |
 | <a id="rust_toolchain-os"></a>os |  The operating system for the current toolchain   | String | required |  |
 | <a id="rust_toolchain-rust_doc"></a>rust_doc |  The location of the <code>rustdoc</code> binary. Can be a direct source or a filegroup containing one item.   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | required |  |

docs/rust_repositories.md

@@ -35,9 +35,9 @@ A dedicated filegroup-like rule for Rust stdlib artifacts.
 
 <pre>
 rust_toolchain(<a href="#rust_toolchain-name">name</a>, <a href="#rust_toolchain-allocator_library">allocator_library</a>, <a href="#rust_toolchain-binary_ext">binary_ext</a>, <a href="#rust_toolchain-cargo">cargo</a>, <a href="#rust_toolchain-clippy_driver">clippy_driver</a>, <a href="#rust_toolchain-debug_info">debug_info</a>,
-               <a href="#rust_toolchain-default_edition">default_edition</a>, <a href="#rust_toolchain-dylib_ext">dylib_ext</a>, <a href="#rust_toolchain-exec_triple">exec_triple</a>, <a href="#rust_toolchain-opt_level">opt_level</a>, <a href="#rust_toolchain-os">os</a>, <a href="#rust_toolchain-rust_doc">rust_doc</a>, <a href="#rust_toolchain-rust_lib">rust_lib</a>, <a href="#rust_toolchain-rust_std">rust_std</a>,
-               <a href="#rust_toolchain-rustc">rustc</a>, <a href="#rust_toolchain-rustc_lib">rustc_lib</a>, <a href="#rust_toolchain-rustc_srcs">rustc_srcs</a>, <a href="#rust_toolchain-rustfmt">rustfmt</a>, <a href="#rust_toolchain-staticlib_ext">staticlib_ext</a>, <a href="#rust_toolchain-stdlib_linkflags">stdlib_linkflags</a>, <a href="#rust_toolchain-target_json">target_json</a>,
-               <a href="#rust_toolchain-target_triple">target_triple</a>)
+               <a href="#rust_toolchain-default_edition">default_edition</a>, <a href="#rust_toolchain-dylib_ext">dylib_ext</a>, <a href="#rust_toolchain-exec_triple">exec_triple</a>, <a href="#rust_toolchain-llvm_tools">llvm_tools</a>, <a href="#rust_toolchain-opt_level">opt_level</a>, <a href="#rust_toolchain-os">os</a>, <a href="#rust_toolchain-rust_doc">rust_doc</a>, <a href="#rust_toolchain-rust_lib">rust_lib</a>,
+               <a href="#rust_toolchain-rust_std">rust_std</a>, <a href="#rust_toolchain-rustc">rustc</a>, <a href="#rust_toolchain-rustc_lib">rustc_lib</a>, <a href="#rust_toolchain-rustc_srcs">rustc_srcs</a>, <a href="#rust_toolchain-rustfmt">rustfmt</a>, <a href="#rust_toolchain-staticlib_ext">staticlib_ext</a>, <a href="#rust_toolchain-stdlib_linkflags">stdlib_linkflags</a>,
+               <a href="#rust_toolchain-target_json">target_json</a>, <a href="#rust_toolchain-target_triple">target_triple</a>)
 </pre>
 
 Declares a Rust toolchain for use.
@@ -95,6 +95,7 @@ See @rules_rust//rust:repositories.bzl for examples of defining the @rust_cpuX r
 | <a id="rust_toolchain-default_edition"></a>default_edition |  The edition to use for rust_* rules that don't specify an edition.   | String | optional | "2018" |
 | <a id="rust_toolchain-dylib_ext"></a>dylib_ext |  The extension for dynamic libraries created from rustc.   | String | required |  |
 | <a id="rust_toolchain-exec_triple"></a>exec_triple |  The platform triple for the toolchains execution environment. For more details see: https://docs.bazel.build/versions/master/skylark/rules.html#configurations   | String | required |  |
+| <a id="rust_toolchain-llvm_tools"></a>llvm_tools |  LLVM tools that are shipped with the Rust toolchain.   | <a href="https://bazel.build/docs/build-ref.html#labels">List of labels</a> | optional | [] |
 | <a id="rust_toolchain-opt_level"></a>opt_level |  Rustc optimization levels.   | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {"dbg": "0", "fastbuild": "0", "opt": "3"} |
 | <a id="rust_toolchain-os"></a>os |  The operating system for the current toolchain   | String | required |  |
 | <a id="rust_toolchain-rust_doc"></a>rust_doc |  The location of the <code>rustdoc</code> binary. Can be a direct source or a filegroup containing one item.   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | required |  |

rust/private/rustc.bzl

@@ -380,19 +380,16 @@ def collect_inputs(
 
     nolinkstamp_compile_inputs = depset(
         getattr(files, "data", []) +
-        [toolchain.rustc] +
-        toolchain.crosstool_files +
         ([build_info.rustc_env, build_info.flags] if build_info else []) +
         ([toolchain.target_json] if toolchain.target_json else []) +
         ([] if linker_script == None else [linker_script]),
         transitive = [
-            toolchain.rustc_lib,
-            toolchain.rust_std,
             linker_depset,
             crate_info.srcs,
             dep_info.transitive_crate_outputs,
             depset(additional_transitive_inputs),
             crate_info.compile_data,
+            toolchain.all_files,
         ],
     )
 
@@ -654,7 +651,7 @@ def construct_arguments(
         data_paths,
     ))
 
-    # Set the SYSROOT to the directory of the rust_std files passed to the toolchain
+    # Ensure the sysroot is set for the target platform
     env["SYSROOT"] = toolchain.sysroot
 
     # extra_rustc_flags apply to the target configuration, not the exec configuration.

rust/private/rustdoc.bzl

@@ -119,6 +119,17 @@ def rustdoc_compile_action(
         force_link = True,
     )
 
+    # Because rustdoc tests compile tests outside of the sandbox, the sysroot
+    # must be updated to the `short_path` equivilant as it will now be
+    # a part of runfiles.
+    if is_test:
+        if "SYSROOT" in env:
+            env.update({"SYSROOT": "${{pwd}}/{}".format(toolchain.sysroot_short_path)})
+
+        # `rustdoc` does not support the SYSROOT environment variable. To account
+        # for this, the flag must be explicitly passed to the `rustdoc` binary.
+        args.rustc_flags.add("--sysroot=${{pwd}}/{}".format(toolchain.sysroot_short_path))
+
     return struct(
         executable = ctx.executable._process_wrapper,
         inputs = depset([crate_info.output], transitive = [compile_inputs]),

rust/toolchain.bzl

@@ -210,6 +210,173 @@ def _make_libstd_and_allocator_ccinfo(ctx, rust_std, allocator_library):
         )
     return None
 
+def _symlink_sysroot_tree(ctx, name, target):
+    """Generate a set of symlinks to files from another target
+
+    Args:
+        ctx (ctx): The toolchain's context object
+        name (str): The name of the sysroot directory (typically `ctx.label.name`)
+        target (Target): A target owning files to symlink
+
+    Returns:
+        depset[File]: A depset of the generated symlink files
+    """
+    tree_files = []
+    for file in target.files.to_list():
+        # Parse the path to the file relative to the workspace root so a
+        # symlink matching this path can be created within the sysroot.
+
+        # The code blow attempts to parse any workspace names out of the
+        # path. For local targets, this code is a noop.
+        if target.label.workspace_root:
+            file_path = file.path.split(target.label.workspace_root, 1)[-1]
+        else:
+            file_path = file.path
+
+        symlink = ctx.actions.declare_file("{}/{}".format(name, file_path.lstrip("/")))
+
+        ctx.actions.symlink(
+            output = symlink,
+            target_file = file,
+        )
+
+        tree_files.append(symlink)
+
+    return depset(tree_files)
+
+def _symlink_sysroot_bin(ctx, name, directory, target):
+    """Crete a symlink to a target file.
+
+    Args:
+        ctx (ctx): The rule's context object
+        name (str): A common name for the output directory
+        directory (str): The directory under `name` to put the file in
+        target (File): A File object to symlink to
+
+    Returns:
+        File: A newly generated symlink file
+    """
+    symlink = ctx.actions.declare_file("{}/{}/{}".format(
+        name,
+        directory,
+        target.basename,
+    ))
+
+    ctx.actions.symlink(
+        output = symlink,
+        target_file = target,
+        is_executable = True,
+    )
+
+    return symlink
+
+def _generate_sysroot(
+        ctx,
+        rustc,
+        rustdoc,
+        rustc_lib,
+        cargo = None,
+        clippy = None,
+        llvm_tools = None,
+        rust_std = None,
+        rustfmt = None):
+    """Generate a rust sysroot from collection of toolchain components
+
+    Args:
+        ctx (ctx): A context object from a `rust_toolchain` rule.
+        rustc (File): The path to a `rustc` executable.
+        rustdoc (File): The path to a `rustdoc` executable.
+        rustc_lib (Target): A collection of Files containing dependencies of `rustc`.
+        cargo (File, optional): The path to a `cargo` executable.
+        clippy (File, optional): The path to a `clippy-driver` executable.
+        llvm_tools (Target, optional): A collection of llvm tools used by `rustc`.
+        rust_std (Target, optional): A collection of Files containing Rust standard library components.
+        rustfmt (File, optional): The path to a `rustfmt` executable.
+
+    Returns:
+        struct: A struct of generated files representing the new sysroot
+    """
+    name = ctx.label.name
+
+    # Define runfiles
+    direct_files = []
+    transitive_file_sets = []
+
+    # Rustc
+    sysroot_rustc = _symlink_sysroot_bin(ctx, name, "bin", rustc)
+    direct_files.extend([sysroot_rustc, rustc])
+
+    # Rustc dependencies
+    sysroot_rustc_lib = None
+    if rustc_lib:
+        sysroot_rustc_lib = _symlink_sysroot_tree(ctx, name, rustc_lib)
+        transitive_file_sets.extend([sysroot_rustc_lib, rustc_lib.files])
+
+    # Rustdoc
+    sysroot_rustdoc = _symlink_sysroot_bin(ctx, name, "bin", rustdoc)
+    direct_files.extend([sysroot_rustdoc, rustdoc])
+
+    # Clippy
+    sysroot_clippy = None
+    if clippy:
+        sysroot_clippy = _symlink_sysroot_bin(ctx, name, "bin", clippy)
+        direct_files.extend([sysroot_clippy, clippy])
+
+    # Cargo
+    sysroot_cargo = None
+    if cargo:
+        sysroot_cargo = _symlink_sysroot_bin(ctx, name, "bin", cargo)
+        direct_files.extend([sysroot_cargo, cargo])
+
+    # Rustfmt
+    sysroot_rustfmt = None
+    if rustfmt:
+        sysroot_rustfmt = _symlink_sysroot_bin(ctx, name, "bin", rustfmt)
+        direct_files.extend([sysroot_rustfmt, rustfmt])
+
+    # Llvm tools
+    sysroot_llvm_tools = None
+    if llvm_tools:
+        sysroot_llvm_tools = _symlink_sysroot_tree(ctx, name, llvm_tools)
+        transitive_file_sets.extend([sysroot_llvm_tools, llvm_tools.files])
+
+    # Rust standard library
+    sysroot_rust_std = None
+    if rust_std:
+        sysroot_rust_std = _symlink_sysroot_tree(ctx, name, rust_std)
+        transitive_file_sets.extend([sysroot_rust_std, rust_std.files])
+
+    # Declare a file in the root of the sysroot to make locating the sysroot easy
+    sysroot_anchor = ctx.actions.declare_file("{}/rust.sysroot".format(name))
+    ctx.actions.write(
+        output = sysroot_anchor,
+        content = "\n".join([
+            "cargo: {}".format(cargo),
+            "clippy: {}".format(clippy),
+            "llvm_tools: {}".format(llvm_tools),
+            "rust_std: {}".format(rust_std),
+            "rustc_lib: {}".format(rustc_lib),
+            "rustc: {}".format(rustc),
+            "rustdoc: {}".format(rustdoc),
+            "rustfmt: {}".format(rustfmt),
+        ]),
+    )
+
+    # Create a depset of all sysroot files (symlinks and their real paths)
+    all_files = depset(direct_files, transitive = transitive_file_sets)
+
+    return struct(
+        all_files = all_files,
+        cargo = sysroot_cargo,
+        clippy = sysroot_clippy,
+        rust_std = sysroot_rust_std,
+        rustc = sysroot_rustc,
+        rustc_lib = sysroot_rustc_lib,
+        rustdoc = sysroot_rustdoc,
+        rustfmt = sysroot_rustfmt,
+        sysroot_anchor = sysroot_anchor,
+    )
+
 def _rust_toolchain_impl(ctx):
     """The rust_toolchain implementation
 
@@ -243,6 +410,18 @@ def _rust_toolchain_impl(ctx):
     else:
         rust_std = ctx.attr.rust_std
 
+    sysroot = _generate_sysroot(
+        ctx = ctx,
+        rustc = ctx.file.rustc,
+        rustdoc = ctx.file.rust_doc,
+        rustc_lib = ctx.attr.rustc_lib,
+        rust_std = rust_std,
+        rustfmt = ctx.file.rustfmt,
+        clippy = ctx.file.clippy_driver,
+        cargo = ctx.file.cargo,
+        llvm_tools = ctx.attr.llvm_tools,
+    )
+
     expanded_stdlib_linkflags = []
     for flag in ctx.attr.stdlib_linkflags:
         expanded_stdlib_linkflags.append(
@@ -265,30 +444,26 @@ def _rust_toolchain_impl(ctx):
         linking_context = linking_context,
     )
 
-    # In cases where the toolchain uses the Rust standard library, calculate sysroot path
-    sysroot_path = None
-    rust_std_files_list = []
-    if rust_std:
-        # Calculate the rustc sysroot path by using a file from the rust-std bundle
-        rust_std_files_list = rust_std.files.to_list()
-        if not rust_std_files_list:
-            fail("The `rust_std` cannot be represented by an empty list")
-        sysroot_path = rust_std_files_list[0].dirname
+    # Determine the path and short_path of the sysroot
+    sysroot_path = sysroot.sysroot_anchor.dirname
+    sysroot_short_path, _, _ = sysroot.sysroot_anchor.short_path.rpartition("/")
 
     toolchain = platform_common.ToolchainInfo(
-        rustc = ctx.file.rustc,
-        rust_doc = ctx.file.rust_doc,
-        rustfmt = ctx.file.rustfmt,
-        cargo = ctx.file.cargo,
-        clippy_driver = ctx.file.clippy_driver,
+        all_files = sysroot.all_files,
+        rustc = sysroot.rustc,
+        rust_doc = sysroot.rustdoc,
+        rustfmt = sysroot.rustfmt,
+        cargo = sysroot.cargo,
+        clippy_driver = sysroot.clippy,
         target_json = ctx.file.target_json,
         target_flag_value = ctx.file.target_json.path if ctx.file.target_json else ctx.attr.target_triple,
-        rustc_lib = depset(ctx.files.rustc_lib),
+        rustc_lib = sysroot.rustc_lib,
         rustc_srcs = ctx.attr.rustc_srcs,
-        rust_std = rust_std.files,
-        rust_std_paths = depset([file.dirname for file in rust_std_files_list]),
-        rust_lib = rust_std.files,  # `rust_lib` is deprecated and only exists for legacy support.
+        rust_std = sysroot.rust_std,
+        rust_std_paths = depset([file.dirname for file in sysroot.rust_std.to_list()]),
+        rust_lib = sysroot.rust_std,  # `rust_lib` is deprecated and only exists for legacy support.
         sysroot = sysroot_path,
+        sysroot_short_path = sysroot_short_path,
         binary_ext = ctx.attr.binary_ext,
         staticlib_ext = ctx.attr.staticlib_ext,
         dylib_ext = ctx.attr.dylib_ext,
@@ -355,6 +530,10 @@ rust_toolchain = rule(
             ),
             mandatory = True,
         ),
+        "llvm_tools": attr.label_list(
+            doc = "LLVM tools that are shipped with the Rust toolchain.",
+            allow_files = True,
+        ),
         "opt_level": attr.string_dict(
             doc = "Rustc optimization levels.",
             default = {