Auto merge of rust-lang#12976 - tjdevries:scip, r=Veykril

feat: emit SCIP from rust-analyzer

hi rust-analyzer team

I'm one of the engineers at Sourcegraph (and have done a few small changes related to the LSIF work done in rust-analyzer). Recently, we've moved to a new protocol as the primary way to interact with Sourcegraph (LSIF is still possible to upload, so existing jobs will not stop working any time soon). This new protocol is SCIP (I linked a blog post below with more information).

I've implemented SCIP support (based largely on the existing LSIF support). In addition to supporting the existing features that `rust-analyzer`'s LSIF support does, this PR adds the ability to move between crates on sourcegraph.com. So if both your project and a dependency are indexed, you would be able to hop to the particular version and view the source code. I'd be happy to record a demo of that on my local instance if you're interested.

There are a few TODO's left in the code (some that you might have insights on) which I'm happy to fix in this PR, but I just wanted to open this up for discussion first.

Thanks for your time :)

TJ

- [announcing scip](https://about.sourcegraph.com/blog/announcing-scip)

crates/ide/src/lib.rs

@@ -87,7 +87,7 @@ pub use crate::{
     },
     join_lines::JoinLinesConfig,
     markup::Markup,
-    moniker::{MonikerKind, MonikerResult, PackageInformation},
+    moniker::{MonikerDescriptorKind, MonikerKind, MonikerResult, PackageInformation},
     move_item::Direction,
     navigation_target::NavigationTarget,
     prime_caches::ParallelPrimeCachesProgress,

crates/ide/src/moniker.rs

@@ -13,17 +13,39 @@ use syntax::{AstNode, SyntaxKind::*, T};
 
 use crate::{doc_links::token_as_doc_comment, RangeInfo};
 
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum MonikerDescriptorKind {
+    Namespace,
+    Type,
+    Term,
+    Method,
+    TypeParameter,
+    Parameter,
+    Macro,
+    Meta,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct MonikerDescriptor {
+    pub name: Name,
+    pub desc: MonikerDescriptorKind,
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct MonikerIdentifier {
-    crate_name: String,
-    path: Vec<Name>,
+    pub crate_name: String,
+    pub description: Vec<MonikerDescriptor>,
 }
 
 impl ToString for MonikerIdentifier {
     fn to_string(&self) -> String {
         match self {
-            MonikerIdentifier { path, crate_name } => {
-                format!("{}::{}", crate_name, path.iter().map(|x| x.to_string()).join("::"))
+            MonikerIdentifier { description, crate_name } => {
+                format!(
+                    "{}::{}",
+                    crate_name,
+                    description.iter().map(|x| x.name.to_string()).join("::")
+                )
             }
         }
     }
@@ -42,6 +64,12 @@ pub struct MonikerResult {
     pub package_information: PackageInformation,
 }
 
+impl MonikerResult {
+    pub fn from_def(db: &RootDatabase, def: Definition, from_crate: Crate) -> Option<Self> {
+        def_to_moniker(db, def, from_crate)
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct PackageInformation {
     pub name: String,
@@ -105,13 +133,23 @@ pub(crate) fn def_to_moniker(
     def: Definition,
     from_crate: Crate,
 ) -> Option<MonikerResult> {
-    if matches!(def, Definition::GenericParam(_) | Definition::SelfType(_) | Definition::Local(_)) {
+    if matches!(
+        def,
+        Definition::GenericParam(_)
+            | Definition::Label(_)
+            | Definition::DeriveHelper(_)
+            | Definition::BuiltinAttr(_)
+            | Definition::ToolModule(_)
+    ) {
         return None;
     }
+
     let module = def.module(db)?;
     let krate = module.krate();
-    let mut path = vec![];
-    path.extend(module.path_to_root(db).into_iter().filter_map(|x| x.name(db)));
+    let mut description = vec![];
+    description.extend(module.path_to_root(db).into_iter().filter_map(|x| {
+        Some(MonikerDescriptor { name: x.name(db)?, desc: MonikerDescriptorKind::Namespace })
+    }));
 
     // Handle associated items within a trait
     if let Some(assoc) = def.as_assoc_item(db) {
@@ -120,31 +158,98 @@ pub(crate) fn def_to_moniker(
             AssocItemContainer::Trait(trait_) => {
                 // Because different traits can have functions with the same name,
                 // we have to include the trait name as part of the moniker for uniqueness.
-                path.push(trait_.name(db));
+                description.push(MonikerDescriptor {
+                    name: trait_.name(db),
+                    desc: MonikerDescriptorKind::Type,
+                });
             }
             AssocItemContainer::Impl(impl_) => {
                 // Because a struct can implement multiple traits, for implementations
                 // we add both the struct name and the trait name to the path
                 if let Some(adt) = impl_.self_ty(db).as_adt() {
-                    path.push(adt.name(db));
+                    description.push(MonikerDescriptor {
+                        name: adt.name(db),
+                        desc: MonikerDescriptorKind::Type,
+                    });
                 }
 
                 if let Some(trait_) = impl_.trait_(db) {
-                    path.push(trait_.name(db));
+                    description.push(MonikerDescriptor {
+                        name: trait_.name(db),
+                        desc: MonikerDescriptorKind::Type,
+                    });
                 }
             }
         }
     }
 
     if let Definition::Field(it) = def {
-        path.push(it.parent_def(db).name(db));
+        description.push(MonikerDescriptor {
+            name: it.parent_def(db).name(db),
+            desc: MonikerDescriptorKind::Type,
+        });
     }
 
-    path.push(def.name(db)?);
+    let name_desc = match def {
+        // These are handled by top-level guard (for performance).
+        Definition::GenericParam(_)
+        | Definition::Label(_)
+        | Definition::DeriveHelper(_)
+        | Definition::BuiltinAttr(_)
+        | Definition::ToolModule(_) => return None,
+
+        Definition::Local(local) => {
+            if !local.is_param(db) {
+                return None;
+            }
+
+            MonikerDescriptor { name: local.name(db), desc: MonikerDescriptorKind::Parameter }
+        }
+        Definition::Macro(m) => {
+            MonikerDescriptor { name: m.name(db), desc: MonikerDescriptorKind::Macro }
+        }
+        Definition::Function(f) => {
+            MonikerDescriptor { name: f.name(db), desc: MonikerDescriptorKind::Method }
+        }
+        Definition::Variant(v) => {
+            MonikerDescriptor { name: v.name(db), desc: MonikerDescriptorKind::Type }
+        }
+        Definition::Const(c) => {
+            MonikerDescriptor { name: c.name(db)?, desc: MonikerDescriptorKind::Term }
+        }
+        Definition::Trait(trait_) => {
+            MonikerDescriptor { name: trait_.name(db), desc: MonikerDescriptorKind::Type }
+        }
+        Definition::TypeAlias(ta) => {
+            MonikerDescriptor { name: ta.name(db), desc: MonikerDescriptorKind::TypeParameter }
+        }
+        Definition::Module(m) => {
+            MonikerDescriptor { name: m.name(db)?, desc: MonikerDescriptorKind::Namespace }
+        }
+        Definition::BuiltinType(b) => {
+            MonikerDescriptor { name: b.name(), desc: MonikerDescriptorKind::Type }
+        }
+        Definition::SelfType(imp) => MonikerDescriptor {
+            name: imp.self_ty(db).as_adt()?.name(db),
+            desc: MonikerDescriptorKind::Type,
+        },
+        Definition::Field(it) => {
+            MonikerDescriptor { name: it.name(db), desc: MonikerDescriptorKind::Term }
+        }
+        Definition::Adt(adt) => {
+            MonikerDescriptor { name: adt.name(db), desc: MonikerDescriptorKind::Type }
+        }
+        Definition::Static(s) => {
+            MonikerDescriptor { name: s.name(db), desc: MonikerDescriptorKind::Meta }
+        }
+    };
+
+    description.push(name_desc);
+
     Some(MonikerResult {
         identifier: MonikerIdentifier {
             crate_name: krate.display_name(db)?.crate_name().to_string(),
-            path,
+            description,
         },
         kind: if krate == from_crate { MonikerKind::Export } else { MonikerKind::Import },
         package_information: {

crates/rust-analyzer/Cargo.toml

@@ -23,6 +23,7 @@ crossbeam-channel = "0.5.5"
 dissimilar = "1.0.4"
 itertools = "0.10.3"
 lsp-types = { version = "0.93.0", features = ["proposed"] }
+scip = "0.1.1"
 parking_lot = "0.12.1"
 xflags = "0.2.4"
 oorandom = "11.1.3"

crates/rust-analyzer/src/bin/main.rs

@@ -93,6 +93,7 @@ fn try_main() -> Result<()> {
         flags::RustAnalyzerCmd::Ssr(cmd) => cmd.run()?,
         flags::RustAnalyzerCmd::Search(cmd) => cmd.run()?,
         flags::RustAnalyzerCmd::Lsif(cmd) => cmd.run()?,
+        flags::RustAnalyzerCmd::Scip(cmd) => cmd.run()?,
     }
     Ok(())
 }

crates/rust-analyzer/src/cli.rs

@@ -9,6 +9,7 @@ mod analysis_stats;
 mod diagnostics;
 mod ssr;
 mod lsif;
+mod scip;
 
 mod progress_report;
 

crates/rust-analyzer/src/cli/flags.rs

@@ -112,6 +112,10 @@ xflags::xflags! {
         cmd lsif
             required path: PathBuf
         {}
+
+        cmd scip
+            required path: PathBuf
+        {}
     }
 }
 
@@ -140,6 +144,7 @@ pub enum RustAnalyzerCmd {
     Search(Search),
     ProcMacro(ProcMacro),
     Lsif(Lsif),
+    Scip(Scip),
 }
 
 #[derive(Debug)]
@@ -207,6 +212,11 @@ pub struct Lsif {
     pub path: PathBuf,
 }
 
+#[derive(Debug)]
+pub struct Scip {
+    pub path: PathBuf,
+}
+
 impl RustAnalyzer {
     pub const HELP: &'static str = Self::HELP_;