Pull/expose rustfmt binaries (bazelbuild#291)

* Fetch and expose the rustfmt binary

Pulls in the version of rustfmt that is shipped with Rust 1.39.0.
Distribution path/version found in official release channel TOML. See
http://static.rust-lang.org/dist/channel-rust-1.39.0.toml, and
rust-lang/rust-forge#215 for more information on this.

Relates to bazelbuild#87.

* Set LC_ALL in fetch_shas.sh

The order of `sort`'s output depends on this variable, and different
users may have it set differently.

* Add rustfmt hashes to known_shas

rustfmt is versioned independently of the other Rust tools, which
necessitates keeping track of its version separately. Binaries are also
not provided for FreeBSD, and so a separate targets list is also used.

* Expose rustfmt binary via the Rust toolchain

* Add test that rustfmt binary works

bindgen/bindgen.bzl

@@ -14,6 +14,7 @@
 
 load("@io_bazel_rules_rust//rust:rust.bzl", "rust_library")
 load("@io_bazel_rules_rust//rust:private/legacy_cc_starlark_api_shim.bzl", "get_libs_for_static_executable")
+load("@io_bazel_rules_rust//rust:private/utils.bzl", "find_toolchain")
 
 def rust_bindgen_library(
         name,
@@ -42,6 +43,8 @@ def rust_bindgen_library(
     )
 
 def _rust_bindgen_impl(ctx):
+    rust_toolchain = find_toolchain(ctx)
+
     # nb. We can't grab the cc_library`s direct headers, so a header must be provided.
     cc_lib = ctx.attr.cc_lib
     header = ctx.file.header
@@ -51,7 +54,7 @@ def _rust_bindgen_impl(ctx):
 
     toolchain = ctx.toolchains["@io_bazel_rules_rust//bindgen:bindgen_toolchain"]
     bindgen_bin = toolchain.bindgen
-    rustfmt_bin = toolchain.rustfmt
+    rustfmt_bin = toolchain.rustfmt or rust_toolchain.rustfmt
     clang_bin = toolchain.clang
     libclang = toolchain.libclang
 
@@ -144,6 +147,7 @@ rust_bindgen = rule(
     outputs = {"out": "%{name}.rs"},
     toolchains = [
         "@io_bazel_rules_rust//bindgen:bindgen_toolchain",
+        "@io_bazel_rules_rust//rust:toolchain",
     ],
 )
 

rust/known_shas.bzl

@@ -54,6 +54,8 @@ FILE_KEY_TO_SHA = {
     "2018-11-09/rustc-nightly-x86_64-apple-darwin": "55ca5ad85b0afd61a419e374f8e6320b4f4fe30f8092005cdec9e63103812ea7",
     "2018-11-09/rustc-nightly-x86_64-unknown-freebsd": "bd6bb0228aeab01f425cb2ad55b2e0409b43e79450c2830183a6878cc2d2bdc4",
     "2018-11-09/rustc-nightly-x86_64-unknown-linux-gnu": "2c475f886123353c9388322da6e13a67b6ae902d8c249f8e95fde67429f7bf37",
+    "2019-11-07/rustfmt-1.4.8-x86_64-apple-darwin": "9ff48a5a0ec693e28a3cf408019ba67544dea4b0ea119ad572c2f83d387d9ae5",
+    "2019-11-07/rustfmt-1.4.8-x86_64-unknown-linux-gnu": "4d6f813ef721821352a5e447ba1b6a69c04e2b43cec24d379e0c7a0528932d26",
     "2020-02-16/rustc-nightly-x86_64-apple-darwin": "db0338b3e1934147dce0bf6420d9c147caa6aef2db1aca44ca8fef47b7247615",
     "2020-02-16/rustc-nightly-x86_64-unknown-freebsd": "c76fa125e6d17b16a96b01a875d826f20849b09970b49ed1183601a0e7803f6f",
     "2020-02-16/rustc-nightly-x86_64-unknown-linux-gnu": "456af585ad4408ab5f0c7500264ebb4a5f6338c0aed642edb81224ec6146b546",

rust/repositories.bzl

@@ -13,6 +13,7 @@ def rust_repositories():
     """
 
     RUST_VERSION = "1.39.0"
+    RUSTFMT_VERSION = "1.4.8"
 
     maybe(
         http_archive,
@@ -28,20 +29,23 @@ def rust_repositories():
         exec_triple = "x86_64-unknown-linux-gnu",
         extra_target_triples = ["wasm32-unknown-unknown"],
         version = RUST_VERSION,
+        rustfmt_version = RUSTFMT_VERSION,
     )
 
     rust_repository_set(
         name = "rust_darwin_x86_64",
         exec_triple = "x86_64-apple-darwin",
         extra_target_triples = ["wasm32-unknown-unknown"],
         version = RUST_VERSION,
+        rustfmt_version = RUSTFMT_VERSION,
     )
 
     rust_repository_set(
         name = "rust_freebsd_x86_64",
         exec_triple = "x86_64-unknown-freebsd",
         extra_target_triples = ["wasm32-unknown-unknown"],
         version = RUST_VERSION,
+        rustfmt_version = RUSTFMT_VERSION,
     )
 
 def _check_version_valid(version, iso_date, param_prefix = ""):
@@ -97,6 +101,31 @@ filegroup(
         target_triple = target_triple,
     )
 
+def BUILD_for_rustfmt(target_triple):
+    """Emits a BUILD file the compiler .tar.gz."""
+
+    system = triple_to_system(target_triple)
+    return """
+load("@io_bazel_rules_rust//rust:toolchain.bzl", "rust_toolchain")
+
+filegroup(
+    name = "rustfmt_bin",
+    srcs = ["bin/rustfmt{binary_ext}"],
+    visibility = ["//visibility:public"],
+)
+
+sh_binary(
+    name = "rustfmt",
+    srcs = [":rustfmt_bin"],
+    visibility = ["//visibility:public"],
+)
+""".format(
+        binary_ext = system_to_binary_ext(system),
+        staticlib_ext = system_to_staticlib_ext(system),
+        dylib_ext = system_to_dylib_ext(system),
+        target_triple = target_triple,
+    )
+
 def BUILD_for_stdlib(target_triple):
     """Emits a BUILD file the stdlib .tar.gz."""
 
@@ -140,6 +169,7 @@ rust_toolchain(
     rust_doc = "@{workspace_name}//:rustdoc",
     rust_lib = "@{workspace_name}//:rust_lib-{target_triple}",
     rustc = "@{workspace_name}//:rustc",
+    rustfmt = "@{workspace_name}//:rustfmt_bin",
     rustc_lib = "@{workspace_name}//:rustc_lib",
     staticlib_ext = "{staticlib_ext}",
     dylib_ext = "{dylib_ext}",
@@ -237,6 +267,20 @@ def load_arbitrary_tool(ctx, tool_name, param_prefix, tool_subdirectory, version
         stripPrefix = "{}/{}".format(tool_path, tool_subdirectory),
     )
 
+def _load_rustfmt(ctx):
+    target_triple = ctx.attr.exec_triple
+    load_arbitrary_tool(
+        ctx,
+        iso_date = ctx.attr.iso_date,
+        param_prefix = "rustfmt_",
+        target_triple = target_triple,
+        tool_name = "rustfmt",
+        tool_subdirectory = "rustfmt-preview",
+        version = ctx.attr.rustfmt_version,
+    )
+
+    return BUILD_for_rustfmt(target_triple)
+
 def _load_rust_compiler(ctx):
     """Loads a rust compiler and yields corresponding BUILD for it
 
@@ -301,6 +345,10 @@ def _rust_toolchain_repository_impl(ctx):
     _check_version_valid(ctx.attr.version, ctx.attr.iso_date)
 
     BUILD_components = [_load_rust_compiler(ctx)]
+
+    if ctx.attr.rustfmt_version:
+        BUILD_components.append(_load_rustfmt(ctx))
+
     for target_triple in [ctx.attr.exec_triple] + ctx.attr.extra_target_triples:
         BUILD_components.append(_load_rust_stdlib(ctx, target_triple))
 
@@ -342,6 +390,7 @@ Args:
 rust_toolchain_repository = repository_rule(
     attrs = {
         "version": attr.string(mandatory = True),
+        "rustfmt_version": attr.string(),
         "iso_date": attr.string(),
         "exec_triple": attr.string(mandatory = True),
         "extra_target_triples": attr.string_list(),
@@ -372,7 +421,7 @@ rust_toolchain_repository_proxy = repository_rule(
     implementation = _rust_toolchain_repository_proxy_impl,
 )
 
-def rust_repository_set(name, version, exec_triple, extra_target_triples, iso_date = None):
+def rust_repository_set(name, version, exec_triple, extra_target_triples, iso_date = None, rustfmt_version = None):
     """Assembles a remote repository for the given toolchain params, produces a proxy repository
     to contain the toolchain declaration, and registers the toolchains.
 
@@ -395,6 +444,7 @@ def rust_repository_set(name, version, exec_triple, extra_target_triples, iso_da
         iso_date = iso_date,
         toolchain_name_prefix = DEFAULT_TOOLCHAIN_NAME_PREFIX,
         version = version,
+        rustfmt_version = rustfmt_version,
     )
 
     rust_toolchain_repository_proxy(

rust/toolchain.bzl

@@ -15,6 +15,7 @@ def _rust_toolchain_impl(ctx):
     toolchain = platform_common.ToolchainInfo(
         rustc = ctx.file.rustc,
         rust_doc = ctx.file.rust_doc,
+        rustfmt = ctx.file.rustfmt,
         rustc_lib = ctx.attr.rustc_lib,
         rust_lib = ctx.attr.rust_lib,
         staticlib_ext = ctx.attr.staticlib_ext,
@@ -40,6 +41,10 @@ rust_toolchain = rule(
             doc = "The location of the `rustdoc` binary. Can be a direct source or a filegroup containing one item.",
             allow_single_file = True,
         ),
+        "rustfmt": attr.label(
+            doc = "The location of the `rustfmt` binary. Can be a direct source or a filegroup containing one item.",
+            allow_single_file = True,
+        ),
         "rustc_lib": attr.label(
             doc = "The libraries used by rustc during compilation.",
         ),

test/rustfmt/BUILD

@@ -0,0 +1,17 @@
+load("@io_bazel_rules_rust//test/rustfmt:rustfmt_generator.bzl", "rustfmt_generator")
+
+rustfmt_generator(
+    name = "formatted",
+    src = ":unformatted.rs",
+)
+
+sh_test(
+    name = "rustfmt_test",
+    size = "small",
+    data = [
+        ":formatted.rs",
+        ":unformatted.rs",
+    ],
+    srcs = [":rustfmt_test.sh"],
+    deps = ["@bazel_tools//tools/bash/runfiles"],
+)

test/rustfmt/rustfmt_generator.bzl

@@ -0,0 +1,29 @@
+load("@io_bazel_rules_rust//rust:private/utils.bzl", "find_toolchain")
+
+def _rustfmt_generator_impl(ctx):
+    toolchain = find_toolchain(ctx)
+    rustfmt_bin = toolchain.rustfmt
+    output = ctx.outputs.out
+
+    ctx.actions.run_shell(
+        inputs = depset([rustfmt_bin, ctx.file.src]),
+        outputs = [output],
+        command = "{} --emit stdout --quiet {} > {}".format(rustfmt_bin.path, ctx.file.src.path, output.path),
+        tools = [rustfmt_bin],
+    )
+
+
+rustfmt_generator = rule(
+    _rustfmt_generator_impl,
+    doc = "Given an unformatted Rust source file, output the file after being run through rustfmt.",
+    attrs = {
+        "src": attr.label(
+            doc = "The file to be formatted.",
+            allow_single_file = True,
+        )
+    },
+    outputs = {"out": "%{name}.rs"},
+    toolchains = [
+        "@io_bazel_rules_rust//rust:toolchain",
+    ],
+)

test/rustfmt/rustfmt_test.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -euxo pipefail
+
+formatted="$(rlocation io_bazel_rules_rust/test/rustfmt/formatted.rs)"
+unformatted="$(rlocation io_bazel_rules_rust/test/rustfmt/unformatted.rs)"
+
+# Ensure that the file was formatted
+! diff "$unformatted" "$formatted"

test/rustfmt/unformatted.rs

@@ -0,0 +1 @@
+fn example(){println!("test");}

util/fetch_shas.sh

@@ -5,11 +5,15 @@
 #
 # Should be run from workspace root.
 
+export LC_ALL=C
+
 TOOLS="$(cat ./util/fetch_shas_TOOLS.txt)"
 TARGETS="$(cat ./util/fetch_shas_TARGETS.txt)"
 VERSIONS="$(cat ./util/fetch_shas_VERSIONS.txt)"
 BETA_ISO_DATES="$(cat ./util/fetch_shas_BETA_ISO_DATES.txt)"
 NIGHTLY_ISO_DATES="$(cat ./util/fetch_shas_NIGHTLY_ISO_DATES.txt)"
+RUSTFMT_TARGETS="$(cat ./util/fetch_shas_RUSTFMT_TARGETS.txt)"
+RUSTFMT_VERSIONS="$(cat ./util/fetch_shas_RUSTFMT_VERSIONS.txt)"
 
 enumerate_keys() {
   for TOOL in $TOOLS
@@ -34,6 +38,16 @@ enumerate_keys() {
   done
 }
 
+enumerate_rustfmt_keys() {
+    for RUSTFMT_TARGET in $RUSTFMT_TARGETS
+    do
+        for RUSTFMT_VERSION in $RUSTFMT_VERSIONS
+        do
+            echo "$RUSTFMT_VERSION-$RUSTFMT_TARGET"
+        done
+    done
+}
+
 emit_bzl_file_contents() {
   echo "$@" \
     | parallel --trim lr -d ' ' --will-cite 'printf "%s %s\n", {}, $(curl https://static.rust-lang.org/dist/{}.tar.gz.sha256 | cut -f1 --delimiter=" ")' \
@@ -46,4 +60,4 @@ emit_bzl_file_contents() {
   echo "}"
 }
 
-echo "$(emit_bzl_file_contents $(enumerate_keys))" > ./rust/known_shas.bzl
+echo "$(emit_bzl_file_contents $(enumerate_keys) $(enumerate_rustfmt_keys))" > ./rust/known_shas.bzl

util/fetch_shas_RUSTFMT_TARGETS.txt

@@ -0,0 +1,2 @@
+x86_64-unknown-linux-gnu
+x86_64-apple-darwin

util/fetch_shas_RUSTFMT_VERSIONS.txt

@@ -0,0 +1 @@
+2019-11-07/rustfmt-1.4.8