Give gnrt two subcommands (gen, download) and implement download

The download command pulls a crate from crates.io via `curl`, verifies
the license is allowed, untars it into third_party/rust/ and writes a
README.chromium file for it.

R=collinbaker@chromium.org

Bug: 1291994
Change-Id: I64ef8b23420826c8aea1089bb8b85daf1cdbe1c3
Cq-Include-Trybots: luci.chromium.try:android-rust-arm-dbg,android-rust-arm-rel,linux-rust-x64-dbg,linux-rust-x64-rel
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3913769
Reviewed-by: Collin Baker <collinbaker@chromium.org>
Reviewed-by: Collin Baker <collinbaker@google.com>
Commit-Queue: danakj <danakj@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1053245}

tools/crates/gnrt/BUILD.gn

@@ -19,6 +19,7 @@ rust_static_library("gnrt_lib") {
   sources = [
     "crates.rs",
     "deps.rs",
+    "download.rs",
     "gn.rs",
     "lib.rs",
     "manifest.rs",

tools/crates/gnrt/download.rs

@@ -0,0 +1,135 @@
+// Copyright 2022 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+use std::io::{Read, Write};
+use std::process::{self, ExitCode};
+
+use crate::crates;
+use crate::manifest::CargoManifest;
+use crate::paths;
+
+/// Runs the download subcommand, which downloads a crate from crates.io and unpacks it into the
+/// Chromium tree.
+pub fn download(
+    name: &str,
+    version: semver::Version,
+    security: bool,
+    paths: &paths::ChromiumPaths,
+) -> ExitCode {
+    let vendored_crate = crates::ChromiumVendoredCrate {
+        name: name.to_string(),
+        epoch: crates::Epoch::from_version(&version),
+    };
+    let build_path = paths.third_party.join(vendored_crate.build_path());
+    let crate_path = paths.third_party.join(vendored_crate.crate_path());
+
+    let url = format!(
+        "{dir}/{name}/{name}-{version}.{suffix}",
+        dir = CRATES_IO_DOWNLOAD_URL,
+        suffix = CRATES_IO_DOWNLOAD_SUFFIX
+    );
+    let curl_out = process::Command::new("curl")
+        .arg("--fail")
+        .arg(url.to_string())
+        .output()
+        .expect("Failed to run curl");
+    if !curl_out.status.success() {
+        eprintln!("gnrt: {}", String::from_utf8(curl_out.stderr).unwrap());
+        return ExitCode::FAILURE;
+    }
+
+    // Makes the directory where the build file will go. The crate's source code will go below it.
+    // This directory and its parents are allowed to exist already.
+    std::fs::create_dir_all(&build_path)
+        .expect("Could not make the third-party directory '{build_path}' for the crate");
+    // Makes the directory where the source code will be unzipped. It should not exist or we'd be
+    // clobbering existing files.
+    std::fs::create_dir(&crate_path).expect("Crate directory '{crate_path}' already exists");
+
+    let mut untar = process::Command::new("tar")
+        // Extract and unzip from stdin.
+        .arg("xzf")
+        .arg("-")
+        // Drop the first path component, which is the crate's name-version.
+        .arg("--strip-components=1")
+        // Unzip into the crate's directory in third_party/rust.
+        .arg(format!("--directory={}", crate_path.display()))
+        // The input is the downloaded file.
+        .stdin(process::Stdio::piped())
+        .stdout(process::Stdio::piped())
+        .stderr(process::Stdio::piped())
+        .spawn()
+        .expect("Failed to run tar");
+
+    if untar.stdin.take().unwrap().write_all(&curl_out.stdout).is_err() {
+        eprintln!("gnrt: Failed to pipe input to tar, it exited early");
+    }
+
+    if !untar.wait().expect("Failed to wait for tar").success() {
+        let mut stderr_buf = Vec::new();
+        untar.stderr.unwrap().read_to_end(&mut stderr_buf).expect("Failed to read stderr from tar");
+        eprintln!("gnrt: {}", String::from_utf8(stderr_buf).unwrap());
+        return ExitCode::FAILURE;
+    }
+
+    let cargo: CargoManifest = {
+        let str = std::fs::read_to_string(crate_path.join("Cargo.toml"))
+            .expect("Unable to open downloaded Cargo.toml");
+        toml::de::from_str(&str).expect("Unable to parse downloaded Cargo.toml")
+    };
+
+    let (_, readme_license) = ALLOWED_LICENSES
+        .iter()
+        .find(|(allowed_license, _)| &cargo.package.license == *allowed_license)
+        .expect("License in downloaded Cargo.toml is not in ALLOWED_LICENSES");
+
+    let readme = gen_readme_chromium_text(&cargo, readme_license, security);
+    std::fs::write(build_path.join("README.chromium"), readme)
+        .expect("Failed to write README.chromium");
+
+    println!("gnrt: Downloaded {name} {version} to {path}", path = crate_path.display());
+
+    ExitCode::SUCCESS
+}
+
+/// Generate the contents of the README.chromium file.
+fn gen_readme_chromium_text(manifest: &CargoManifest, license: &str, security: bool) -> String {
+    format!(
+        "Name: {crate_name}\n\
+         URL: {url}\n\
+         Description: {description}\n\
+         Version: {version}\n\
+         Security Critical: {security}\n\
+         License: {license}\n",
+        crate_name = manifest.package.name,
+        url = format!("{}/{}", CRATES_IO_VIEW_URL, manifest.package.name),
+        description = manifest.package.description.as_ref().unwrap_or(&"".to_string()),
+        version = manifest.package.version,
+    )
+}
+
+static CRATES_IO_DOWNLOAD_URL: &str = "https://static.crates.io/crates";
+static CRATES_IO_DOWNLOAD_SUFFIX: &str = "crate";
+static CRATES_IO_VIEW_URL: &str = "https://crates.io/crates";
+
+// Allowed licenses, in the format they are specified in Cargo.toml files from
+// crates.io, and the format to write to README.chromium.
+static ALLOWED_LICENSES: [(&str, &str); 15] = [
+    // ("Cargo.toml string", "License for README.chromium")
+    ("Apache-2.0", "Apache 2.0"),
+    ("MIT OR Apache-2.0", "Apache 2.0"),
+    ("MIT/Apache-2.0", "Apache 2.0"),
+    ("Apache-2.0 / MIT", "Apache 2.0"),
+    ("Apache-2.0 OR MIT", "Apache 2.0"),
+    ("Apache-2.0/MIT", "Apache 2.0"),
+    ("MIT", "MIT"),
+    ("Unlicense OR MIT", "MIT"),
+    ("Unlicense/MIT", "MIT"),
+    ("Apache-2.0 OR BSL-1.0", "Apache 2.0"),
+    ("BSD-3-Clause", "BSD 3-Clause"),
+    ("ISC", "ISC"),
+    ("MIT OR Zlib OR Apache-2.0", "Apache 2.0"),
+    ("0BSD OR MIT OR Apache-2.0", "Apache 2.0"),
+    ("Unicode-DFS-2016", "Unicode License Agreement - Data Files and Software (2016)"),
+];

tools/crates/gnrt/lib.rs

@@ -4,6 +4,7 @@
 
 pub mod crates;
 pub mod deps;
+pub mod download;
 pub mod gn;
 pub mod manifest;
 pub mod paths;

tools/crates/gnrt/main.rs

@@ -17,22 +17,48 @@ use clap::arg;
 
 fn main() -> ExitCode {
     let args = clap::Command::new("gnrt")
-        .about("Generate GN build rules from third_party/rust crates")
-        .arg(
-            arg!(--"output-cargo-toml" "Output third_party/rust/Cargo.toml then exit immediately")
+        .subcommand(clap::Command::new("gen")
+            .about("Generate GN build rules from third_party/rust crates")
+            .arg(arg!(--"output-cargo-toml" "Output third_party/rust/Cargo.toml then exit \
+                immediately"))
+            .arg(arg!(--"skip-patch" "Don't apply gnrt_build_patch after generating build files. \
+                Useful when updating the patch."))
+            .arg(arg!(--"for-std" "(WIP) Generate build files for Rust std library instead of \
+                third_party/rust"))
+        )
+        .subcommand(clap::Command::new("download")
+            .about("Download the crate with the given name and version to third_party/rust.")
+            .arg(arg!([NAME] "Name of the crate to download").required(true))
+            .arg(arg!([VERSION] "Version of the crate to download").required(true))
+            .arg(
+                arg!(--"security-critical" <YESNO> "Whether the crate is considered to be \
+                    security critical."
+                ).possible_values(["yes", "no"]).required(true)
+            )
         )
-        .arg(arg!(--"skip-patch" "Don't apply gnrt_build_patch after generating build files. Useful when updating the patch."))
-        .arg(arg!(--"for-std" "(WIP) Generate build files for Rust std library instead of third_party/rust"))
         .get_matches();
 
     let paths = paths::ChromiumPaths::new().unwrap();
 
-    if args.is_present("for-std") {
-        // This is not fully implemented. Currently, it will print data helpful
-        // for development then quit.
-        generate_for_std(&args, &paths)
-    } else {
-        generate_for_third_party(&args, &paths)
+    match args.subcommand() {
+        Some(("gen", args)) => {
+            if args.is_present("for-std") {
+                // This is not fully implemented. Currently, it will print data helpful
+                // for development then quit.
+                generate_for_std(&args, &paths)
+            } else {
+                generate_for_third_party(&args, &paths)
+            }
+        }
+        Some(("download", args)) => {
+            let security = args.value_of("security-critical").unwrap() == "yes";
+            let name = args.value_of("NAME").unwrap();
+            use std::str::FromStr;
+            let version = semver::Version::from_str(args.value_of("VERSION").unwrap())
+                .expect("Invalid version specified");
+            download::download(name, version, security, &paths)
+        }
+        _ => unreachable!("Invalid subcommand"),
     }
 }
 

tools/crates/gnrt/manifest.rs

@@ -126,6 +126,8 @@ pub struct CargoPackage {
     pub edition: Edition,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub description: Option<String>,
+    #[serde(default, skip_serializing_if = "String::is_empty")]
+    pub license: String,
 }
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
@@ -165,7 +167,7 @@ pub fn generate_fake_cargo_toml<Iter: IntoIterator<Item = PatchSpecification>>(
     third_party_manifest: ThirdPartyManifest,
     patches: Iter,
 ) -> CargoManifest {
-    let ThirdPartyManifest { workspace, mut dependency_spec } = third_party_manifest;
+    let ThirdPartyManifest { workspace, mut dependency_spec, .. } = third_party_manifest;
 
     // Hack: set all `allow_first_party_usage` fields to true so they are
     // suppressed in the Cargo.toml.
@@ -197,6 +199,7 @@ pub fn generate_fake_cargo_toml<Iter: IntoIterator<Item = PatchSpecification>>(
         authors: Vec::new(),
         edition: Edition("2021".to_string()),
         description: None,
+        license: "".to_string(),
     };
 
     CargoManifest {

tools/crates/gnrt/manifest_unittest.rs

@@ -104,6 +104,7 @@ fn test() {
             authors: Vec::new(),
             edition: Edition("2021".to_string()),
             description: None,
+            license: "funtimes".to_string(),
         },
         workspace: None,
         dependency_spec: DependencySpec {
@@ -133,6 +134,7 @@ fn test() {
 name = \"chromium\"
 version = \"0.1.0\"
 edition = \"2021\"
+license = \"funtimes\"
 [patch.crates-io.foo_v1]
 path = \"third_party/rust/foo/v1/crate\"
 package = \"foo\"
@@ -149,6 +151,7 @@ version = \"1.2.3\"
 authors = [\"alice@foo.com\", \"bob@foo.com\"]
 edition = \"2021\"
 description = \"A library to foo the bars\"
+license = \"funtimes\"
 "
     ))
     .unwrap();
@@ -161,6 +164,7 @@ description = \"A library to foo the bars\"
             authors: vec!["alice@foo.com".to_string(), "bob@foo.com".to_string()],
             edition: Edition("2021".to_string()),
             description: Some("A library to foo the bars".to_string()),
+            license: "funtimes".to_string(),
         }
     )
 }