Add rust-analyzer submodule

The current plan is that submodule tracks the `release` branch of
rust-analyzer, which is updated once a week.

rust-analyzer is a workspace (with a virtual manifest), the actual
binary is provide by `crates/rust-analyzer` package.

Note that we intentionally don't add rust-analyzer to `Kind::Test`,
for two reasons.

*First*, at the moment rust-analyzer's test suite does a couple of
things which might not work in the context of rust repository. For
example, it shells out directly to `rustup` and `rustfmt`. So, making
this work requires non-trivial efforts.

*Second*, it seems unlikely that running tests in rust-lang/rust repo
would provide any additional guarantees. rust-analyzer builds with
stable and does not depend on the specifics of the compiler, so
changes to compiler can't break ra, unless they break stability
guarantee. Additionally, rust-analyzer itself is gated on bors, so we
are pretty confident that test suite passes.

.gitmodules

@@ -41,3 +41,6 @@
 [submodule "src/doc/embedded-book"]
 	path = src/doc/embedded-book
 	url = https://github.com/rust-embedded/book.git
+[submodule "src/tools/rust-analyzer"]
+	path = src/tools/rust-analyzer
+	url = https://github.com/rust-analyzer/rust-analyzer.git

rustfmt.toml

@@ -27,6 +27,7 @@ ignore = [
     "src/tools/clippy",
     "src/tools/miri",
     "src/tools/rls",
+    "src/tools/rust-analyzer",
     "src/tools/rust-installer",
     "src/tools/rustfmt",
 

src/bootstrap/bootstrap.py

@@ -869,7 +869,7 @@ def ensure_vendored(self):
         # the rust git repository is updated. Normal development usually does
         # not use vendoring, so hopefully this isn't too much of a problem.
         if self.use_vendored_sources and not os.path.exists(vendor_dir):
-            run([self.cargo(), "vendor"],
+            run([self.cargo(), "vendor", "--sync=./src/tools/rust-analyzer/Cargo.toml"],
                 verbose=self.verbose, cwd=self.rust_root)
 
 

src/bootstrap/builder.rs

@@ -368,6 +368,7 @@ impl<'a> Builder<'a> {
                 tool::RustInstaller,
                 tool::Cargo,
                 tool::Rls,
+                tool::RustAnalyzer,
                 tool::Rustdoc,
                 tool::Clippy,
                 tool::CargoClippy,
@@ -462,6 +463,7 @@ impl<'a> Builder<'a> {
                 dist::PlainSourceTarball,
                 dist::Cargo,
                 dist::Rls,
+                dist::RustAnalyzer,
                 dist::Rustfmt,
                 dist::Clippy,
                 dist::Miri,
@@ -474,6 +476,7 @@ impl<'a> Builder<'a> {
                 install::Std,
                 install::Cargo,
                 install::Rls,
+                install::RustAnalyzer,
                 install::Rustfmt,
                 install::Clippy,
                 install::Miri,

src/bootstrap/dist.rs

@@ -30,6 +30,8 @@ pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
         format!("{}-{}", component, builder.cargo_package_vers())
     } else if component == "rls" {
         format!("{}-{}", component, builder.rls_package_vers())
+    } else if component == "rust-analyzer" {
+        format!("{}-{}", component, builder.rust_analyzer_package_vers())
     } else if component == "clippy" {
         format!("{}-{}", component, builder.clippy_package_vers())
     } else if component == "miri" {
@@ -1107,7 +1109,10 @@ impl Step for PlainSourceTarball {
         if builder.rust_info.is_git() {
             // Vendor all Cargo dependencies
             let mut cmd = Command::new(&builder.initial_cargo);
-            cmd.arg("vendor").current_dir(&plain_dst_src);
+            cmd.arg("vendor")
+                .arg("--sync")
+                .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml"))
+                .current_dir(&plain_dst_src);
             builder.run(&mut cmd);
         }
 
@@ -1337,6 +1342,93 @@ impl Step for Rls {
     }
 }
 
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustAnalyzer {
+    pub compiler: Compiler,
+    pub target: Interned<String>,
+}
+
+impl Step for RustAnalyzer {
+    type Output = PathBuf;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("rust-analyzer")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(RustAnalyzer {
+            compiler: run.builder.compiler_for(
+                run.builder.top_stage,
+                run.builder.config.build,
+                run.target,
+            ),
+            target: run.target,
+        });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> PathBuf {
+        let compiler = self.compiler;
+        let target = self.target;
+        assert!(builder.config.extended);
+
+        let src = builder.src.join("src/tools/rust-analyzer");
+        let release_num = builder.release_num("rust-analyzer/crates/rust-analyzer");
+        let name = pkgname(builder, "rust-analyzer");
+        let version = builder.rust_analyzer_info.version(builder, &release_num);
+
+        let tmp = tmpdir(builder);
+        let image = tmp.join("rust-analyzer-image");
+        drop(fs::remove_dir_all(&image));
+        builder.create_dir(&image);
+
+        // Prepare the image directory
+        // We expect rust-analyer to always build, as it doesn't depend on rustc internals
+        // and doesn't have associated toolstate.
+        let rust_analyzer = builder
+            .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
+            .expect("rust-analyzer always builds");
+
+        builder.install(&rust_analyzer, &image.join("bin"), 0o755);
+        let doc = image.join("share/doc/rust-analyzer");
+        builder.install(&src.join("README.md"), &doc, 0o644);
+        builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
+        builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
+
+        // Prepare the overlay
+        let overlay = tmp.join("rust-analyzer-overlay");
+        drop(fs::remove_dir_all(&overlay));
+        t!(fs::create_dir_all(&overlay));
+        builder.install(&src.join("README.md"), &overlay, 0o644);
+        builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644);
+        builder.install(&src.join("LICENSE-MIT"), &doc, 0o644);
+        builder.create(&overlay.join("version"), &version);
+
+        // Generate the installer tarball
+        let mut cmd = rust_installer(builder);
+        cmd.arg("generate")
+            .arg("--product-name=Rust")
+            .arg("--rel-manifest-dir=rustlib")
+            .arg("--success-message=rust-analyzer-ready-to-serve.")
+            .arg("--image-dir")
+            .arg(&image)
+            .arg("--work-dir")
+            .arg(&tmpdir(builder))
+            .arg("--output-dir")
+            .arg(&distdir(builder))
+            .arg("--non-installed-overlay")
+            .arg(&overlay)
+            .arg(format!("--package-name={}-{}", name, target))
+            .arg("--legacy-manifest-dirs=rustlib,cargo")
+            .arg("--component-name=rust-analyzer-preview");
+
+        builder.info(&format!("Dist rust-analyzer stage{} ({})", compiler.stage, target));
+        let _time = timeit(builder);
+        builder.run(&mut cmd);
+        distdir(builder).join(format!("{}-{}.tar.gz", name, target))
+    }
+}
+
 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Clippy {
     pub compiler: Compiler,
@@ -1656,6 +1748,7 @@ impl Step for Extended {
         let cargo_installer = builder.ensure(Cargo { compiler, target });
         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
         let rls_installer = builder.ensure(Rls { compiler, target });
+        let rust_analyzer_installer = builder.ensure(RustAnalyzer { compiler, target });
         let llvm_tools_installer = builder.ensure(LlvmTools { target });
         let clippy_installer = builder.ensure(Clippy { compiler, target });
         let miri_installer = builder.ensure(Miri { compiler, target });
@@ -1690,6 +1783,7 @@ impl Step for Extended {
         tarballs.push(rustc_installer);
         tarballs.push(cargo_installer);
         tarballs.extend(rls_installer.clone());
+        tarballs.push(rust_analyzer_installer.clone());
         tarballs.push(clippy_installer);
         tarballs.extend(miri_installer.clone());
         tarballs.extend(rustfmt_installer.clone());
@@ -1767,6 +1861,7 @@ impl Step for Extended {
             if rls_installer.is_none() {
                 contents = filter(&contents, "rls");
             }
+            contents = filter(&contents, "rust-analyzer");
             if miri_installer.is_none() {
                 contents = filter(&contents, "miri");
             }
@@ -1813,6 +1908,7 @@ impl Step for Extended {
             if rls_installer.is_some() {
                 prepare("rls");
             }
+            prepare("rust-analyzer");
             if miri_installer.is_some() {
                 prepare("miri");
             }
@@ -1846,6 +1942,8 @@ impl Step for Extended {
                     format!("{}-{}", name, target)
                 } else if name == "rls" {
                     "rls-preview".to_string()
+                } else if name == "rust-analyzer" {
+                    "rust-analyzer-preview".to_string()
                 } else if name == "clippy" {
                     "clippy-preview".to_string()
                 } else if name == "miri" {
@@ -1868,6 +1966,7 @@ impl Step for Extended {
             if rls_installer.is_some() {
                 prepare("rls");
             }
+            prepare("rust-analyzer");
             if miri_installer.is_some() {
                 prepare("miri");
             }
@@ -1967,6 +2066,23 @@ impl Step for Extended {
                         .arg(etc.join("msi/remove-duplicates.xsl")),
                 );
             }
+            builder.run(
+                Command::new(&heat)
+                    .current_dir(&exe)
+                    .arg("dir")
+                    .arg("rust-analyzer")
+                    .args(&heat_flags)
+                    .arg("-cg")
+                    .arg("RustAnalyzerGroup")
+                    .arg("-dr")
+                    .arg("RustAnalyzer")
+                    .arg("-var")
+                    .arg("var.RustAnalyzerDir")
+                    .arg("-out")
+                    .arg(exe.join("RustAnalyzerGroup.wxs"))
+                    .arg("-t")
+                    .arg(etc.join("msi/remove-duplicates.xsl")),
+            );
             builder.run(
                 Command::new(&heat)
                     .current_dir(&exe)
@@ -2060,6 +2176,7 @@ impl Step for Extended {
                 if rls_installer.is_some() {
                     cmd.arg("-dRlsDir=rls");
                 }
+                cmd.arg("-dRustAnalyzerDir=rust-analyzer");
                 if miri_installer.is_some() {
                     cmd.arg("-dMiriDir=miri");
                 }
@@ -2079,6 +2196,7 @@ impl Step for Extended {
             if rls_installer.is_some() {
                 candle("RlsGroup.wxs".as_ref());
             }
+            candle("RustAnalyzerGroup.wxs".as_ref());
             if miri_installer.is_some() {
                 candle("MiriGroup.wxs".as_ref());
             }
@@ -2116,6 +2234,7 @@ impl Step for Extended {
             if rls_installer.is_some() {
                 cmd.arg("RlsGroup.wixobj");
             }
+            cmd.arg("RustAnalyzerGroup.wixobj");
             if miri_installer.is_some() {
                 cmd.arg("MiriGroup.wixobj");
             }
@@ -2209,6 +2328,7 @@ impl Step for HashSign {
         cmd.arg(addr);
         cmd.arg(builder.package_vers(&builder.release_num("cargo")));
         cmd.arg(builder.package_vers(&builder.release_num("rls")));
+        cmd.arg(builder.package_vers(&builder.release_num("rust-analyzer/crates/rust-analyzer")));
         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
         cmd.arg(builder.package_vers(&builder.release_num("miri")));
         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));

src/bootstrap/install.rs

@@ -32,6 +32,9 @@ pub fn install_cargo(builder: &Builder<'_>, stage: u32, host: Interned<String>)
 pub fn install_rls(builder: &Builder<'_>, stage: u32, host: Interned<String>) {
     install_sh(builder, "rls", "rls", stage, Some(host));
 }
+pub fn install_rust_analyzer(builder: &Builder<'_>, stage: u32, host: Interned<String>) {
+    install_sh(builder, "rust-analyzer", "rust-analyzer", stage, Some(host));
+}
 pub fn install_clippy(builder: &Builder<'_>, stage: u32, host: Interned<String>) {
     install_sh(builder, "clippy", "clippy", stage, Some(host));
 }
@@ -216,6 +219,16 @@ install!((self, builder, _config),
             );
         }
     };
+    RustAnalyzer, "rust-analyzer", Self::should_build(_config), only_hosts: true, {
+        builder.ensure(dist::RustAnalyzer { compiler: self.compiler, target: self.target });
+        if Self::should_install(builder) {
+            install_rust_analyzer(builder, self.compiler.stage, self.target);
+        } else {
+            builder.info(
+                &format!("skipping Install rust-analyzer stage{} ({})", self.compiler.stage, self.target),
+            );
+        }
+    };
     Clippy, "clippy", Self::should_build(_config), only_hosts: true, {
         builder.ensure(dist::Clippy { compiler: self.compiler, target: self.target });
         if Self::should_install(builder) {

src/bootstrap/lib.rs

@@ -225,6 +225,7 @@ pub struct Build {
     rust_info: channel::GitInfo,
     cargo_info: channel::GitInfo,
     rls_info: channel::GitInfo,
+    rust_analyzer_info: channel::GitInfo,
     clippy_info: channel::GitInfo,
     miri_info: channel::GitInfo,
     rustfmt_info: channel::GitInfo,
@@ -349,6 +350,8 @@ impl Build {
         let rust_info = channel::GitInfo::new(ignore_git, &src);
         let cargo_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/cargo"));
         let rls_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/rls"));
+        let rust_analyzer_info =
+            channel::GitInfo::new(ignore_git, &src.join("src/tools/rust-analyzer"));
         let clippy_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/clippy"));
         let miri_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/miri"));
         let rustfmt_info = channel::GitInfo::new(ignore_git, &src.join("src/tools/rustfmt"));
@@ -405,6 +408,7 @@ impl Build {
             rust_info,
             cargo_info,
             rls_info,
+            rust_analyzer_info,
             clippy_info,
             miri_info,
             rustfmt_info,
@@ -1034,6 +1038,11 @@ impl Build {
         self.package_vers(&self.release_num("rls"))
     }
 
+    /// Returns the value of `package_vers` above for rust-analyzer
+    fn rust_analyzer_package_vers(&self) -> String {
+        self.package_vers(&self.release_num("rust-analyzer/crates/rust-analyzer"))
+    }
+
     /// Returns the value of `package_vers` above for clippy
     fn clippy_package_vers(&self) -> String {
         self.package_vers(&self.release_num("clippy"))

src/bootstrap/tool.rs

@@ -641,7 +641,7 @@ macro_rules! tool_extended {
     }
 }
 
-// Note: tools need to be also added to `Builder::get_step_descriptions` in `build.rs`
+// Note: tools need to be also added to `Builder::get_step_descriptions` in `builder.rs`
 // to make `./x.py build <tool>` work.
 tool_extended!((self, builder),
     Cargofmt, rustfmt, "src/tools/rustfmt", "cargo-fmt", stable=true, {};
@@ -658,6 +658,7 @@ tool_extended!((self, builder),
         self.extra_features.push("clippy".to_owned());
     };
     Rustfmt, rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, {};
+    RustAnalyzer, rust_analyzer, "src/tools/rust-analyzer/crates/rust-analyzer", "rust-analyzer", stable=false, {};
 );
 
 impl<'a> Builder<'a> {

src/tools/tidy/src/lib.rs

@@ -56,6 +56,7 @@ fn filter_dirs(path: &Path) -> bool {
         "src/tools/clippy",
         "src/tools/miri",
         "src/tools/rls",
+        "src/tools/rust-analyzer",
         "src/tools/rust-installer",
         "src/tools/rustfmt",
         "src/doc/book",