Skip to content

Commit

Permalink
feat(rust_indexer): remove need for save-analysis files to be saved t…
Browse files Browse the repository at this point in the history 
…o fs (#5193)
  • Loading branch information
@wcalandro
wcalandro committed Jan 25, 2022
1 parent 8694856 commit 3806d62
Show file tree
Hide file tree
Showing 9 changed files with 53 additions and 279 deletions.
7 changes: 0 additions & 7 deletions kythe/rust/indexer/BUILD
View file
Expand Up @@ -35,10 +35,7 @@ rust_binary(
edition = "2021",
deps = [
":kythe_rust_indexer",
"//kythe/proto:analysis_rust_proto",
"//kythe/proto:storage_rust_proto",
"//kythe/rust/cargo:anyhow",
"//kythe/rust/cargo:tempdir",
"@rules_rust//proto/raze:protobuf",
],
)
Expand All @@ -53,12 +50,9 @@ rust_binary(
deps = [
":kythe_rust_indexer",
"//kythe/proto:analysis_rust_proto",
"//kythe/proto:storage_rust_proto",
"//kythe/rust/cargo:anyhow",
"//kythe/rust/cargo:base64",
"//kythe/rust/cargo:clap",
"//kythe/rust/cargo:serde_json",
"//kythe/rust/cargo:tempdir",
"@rules_rust//proto/raze:protobuf",
],
)
Expand All @@ -72,7 +66,6 @@ rust_test(
name = "integration_tests",
srcs = glob(["tests/*.rs"]),
data = [
"tests/testanalysis.json",
"tests/testkzip.kzip",
],
deps = [
Expand Down
59 changes: 2 additions & 57 deletions kythe/rust/indexer/src/bin/bazel/main.rs
View file
Expand Up @@ -14,15 +14,8 @@
extern crate kythe_rust_indexer;
use kythe_rust_indexer::{indexer::KytheIndexer, providers::*, writer::CodedOutputStreamWriter};

use analysis_rust_proto::*;
use anyhow::{Context, Result};
use std::{
env,
fs::File,
io::Write,
path::{Path, PathBuf},
};
use tempdir::TempDir;
use std::{env, fs::File, path::Path};

fn main() -> Result<()> {
// Accepts kzip path as an argument
Expand Down Expand Up @@ -51,55 +44,7 @@ fn main() -> Result<()> {
let mut indexer = KytheIndexer::new(&mut writer);

for unit in compilation_units {
// Create a temporary directory to store required files
let temp_dir =
TempDir::new("rust_indexer").context("Couldn't create temporary directory")?;
let temp_path = PathBuf::from(temp_dir.path());

// Extract the analysis files from the kzip into the temporary directory
extract_analysis_from_kzip(&unit, &temp_path, &mut kzip_provider)?;

// Index the CompilationUnit
indexer.index_cu(&unit, &temp_path, &mut kzip_provider)?;
}
Ok(())
}

/// Takes analysis files from a kzip loaded into `provider` and extracts them
/// to `temp_path` using the file names and digests in the CompilationUnit
pub fn extract_analysis_from_kzip(
c_unit: &CompilationUnit,
temp_path: &Path,
provider: &mut dyn FileProvider,
) -> Result<()> {
for required_input in c_unit.get_required_input() {
let input_path = required_input.get_info().get_path();
let input_path_buf = PathBuf::from(input_path);

// save_analysis files are JSON files
if let Some(os_str) = input_path_buf.extension() {
if let Some("json") = os_str.to_str() {
let digest = required_input.get_info().get_digest();
let file_contents = provider.contents(input_path, digest).with_context(|| {
format!(
"Failed to get contents of file \"{}\" with digest \"{}\"",
input_path, digest
)
})?;

let output_path = temp_path.join(input_path_buf.file_name().unwrap());
let mut output_file =
File::create(&output_path).context("Failed to create file")?;
output_file.write_all(&file_contents).with_context(|| {
format!(
"Failed to copy contents of \"{}\" with digest \"{}\" to \"{}\"",
input_path,
digest,
output_path.display()
)
})?;
}
}
indexer.index_cu(&unit, &mut kzip_provider)?;
}
Ok(())
}
92 changes: 3 additions & 89 deletions kythe/rust/indexer/src/bin/proxy/main.rs
View file
Expand Up @@ -12,68 +12,23 @@
// See the License for the specific language governing permissions and
// limitations under the License.
extern crate kythe_rust_indexer;
use kythe_rust_indexer::{
error::KytheError, indexer::KytheIndexer, providers::*, proxyrequests, writer::ProxyWriter,
};
use kythe_rust_indexer::{indexer::KytheIndexer, providers::*, proxyrequests, writer::ProxyWriter};

use analysis_rust_proto::*;
use anyhow::{anyhow, Context, Result};
use clap::{App, Arg};
use serde_json::Value;
use std::{
fs::File,
io::{self, Write},
path::{Path, PathBuf},
};
use tempdir::TempDir;
use std::io::{self, Write};

fn main() -> Result<()> {
let matches = App::new("Kythe Rust Extractor")
.arg(
Arg::with_name("tmp_directory")
.long("tmp_directory")
.required(false)
.takes_value(true)
.help("A directory where to write save_analysis files"),
)
.get_matches();

// Get the absolute path of the tmp_directory argument or default to /tmp
let tmp_path_arg = {
if let Some(arg) = matches.value_of("tmp_directory") {
let path = PathBuf::from(arg);
assert!(path.is_dir(), "tmp_directory argument \"{}\" does not exist", arg);
path
} else {
std::env::temp_dir()
}
};

// Check if the tmp_directory exists
if !tmp_path_arg.exists() {
eprintln!("tmp_directory path is not a directory or does not exist");
std::process::exit(1);
}

let mut file_provider = ProxyFileProvider::new();
let mut kythe_writer = ProxyWriter::new();
let mut indexer = KytheIndexer::new(&mut kythe_writer);

// Request and process
loop {
let unit = request_compilation_unit()?;
// Retrieve the save_analysis file
let analysis_temp_dir = TempDir::new_in(tmp_path_arg.clone(), "rust_indexer_tmp")
.context("Couldn't create temporary directory")?;
let analysis_temp_path = PathBuf::from(analysis_temp_dir.path());
let write_res = write_analysis_to_directory(&unit, &analysis_temp_path, &mut file_provider);
if write_res.is_err() {
send_done(false, write_res.err().unwrap().to_string())?;
continue;
}

// Index the CompilationUnit and let the proxy know we are done
let index_res = indexer.index_cu(&unit, &analysis_temp_path, &mut file_provider);
let index_res = indexer.index_cu(&unit, &mut file_provider);
if index_res.is_ok() {
send_done(true, String::new())?;
} else {
Expand All @@ -82,47 +37,6 @@ fn main() -> Result<()> {
}
}

/// Takes analysis files present in a CompilationUnit, requests the files
/// from the proxy, and writes them to `temp_path`
pub fn write_analysis_to_directory(
c_unit: &CompilationUnit,
temp_path: &Path,
provider: &mut dyn FileProvider,
) -> std::result::Result<(), KytheError> {
for required_input in c_unit.get_required_input() {
let input_path = required_input.get_info().get_path();
let input_path_buf = PathBuf::from(input_path);

// save_analysis files are JSON files
if let Some(os_str) = input_path_buf.extension() {
if let Some("json") = os_str.to_str() {
let digest = required_input.get_info().get_digest();
let file_contents = provider.contents(input_path, digest).map_err(|err| {
KytheError::IndexerError(format!(
"Failed to get contents of file \"{}\" with digest \"{}\": {:?}",
input_path, digest, err
))
})?;

let output_path = temp_path.join(input_path_buf.file_name().unwrap());
let mut output_file = File::create(&output_path).map_err(|err| {
KytheError::IndexerError(format!("Failed to create file: {:?}", err))
})?;
output_file.write_all(&file_contents).map_err(|err| {
KytheError::IndexerError(format!(
"Failed to copy contents of \"{}\" with digest \"{}\" to \"{}\": {:?}",
input_path,
digest,
output_path.display(),
err
))
})?;
}
}
}
Ok(())
}

fn request_compilation_unit() -> Result<CompilationUnit> {
println!(r#"{{"req":"analysis_wire"}}"#);
io::stdout().flush().context("Failed to flush stdout")?;
Expand Down
33 changes: 14 additions & 19 deletions kythe/rust/indexer/src/indexer/analyzers.rs
View file
Expand Up @@ -20,8 +20,7 @@ use super::entries::EntryEmitter;
use super::offset::OffsetIndex;

use analysis_rust_proto::CompilationUnit;
use rls_analysis::Crate;
use rls_data::{Def, DefKind};
use rls_data::{Analysis, Def, DefKind};
use std::collections::HashMap;
use std::ffi::OsStr;
use storage_rust_proto::*;
Expand Down Expand Up @@ -52,8 +51,8 @@ pub struct CrateAnalyzer<'a, 'b> {
file_vnames: &'b HashMap<String, VName>,
// The current CompilationUnit's VName
unit_vname: &'b VName,
// The current crate being analyzed
krate: Crate,
// The save-analysis for the crate
analysis: Analysis,
// A map between a crate's identifier and a string consisting of
// "<disambiguator1>_<disambiguator2>"
krate_ids: HashMap<u32, rls_data::GlobalCrateId>,
Expand Down Expand Up @@ -175,12 +174,12 @@ impl<'a> UnitAnalyzer<'a> {
}

/// Indexes the provided crate
pub fn index_crate(&mut self, krate: Crate) -> Result<(), KytheError> {
pub fn index_crate(&mut self, analysis: Analysis) -> Result<(), KytheError> {
let mut crate_analyzer = CrateAnalyzer::new(
&mut self.emitter,
&self.file_vnames,
&self.unit_storage_vname,
krate,
analysis,
&self.offset_index,
);
crate_analyzer.emit_crate_nodes()?;
Expand All @@ -200,7 +199,7 @@ impl<'a, 'b> CrateAnalyzer<'a, 'b> {
emitter: &'b mut EntryEmitter<'a>,
file_vnames: &'b HashMap<String, VName>,
unit_vname: &'b VName,
krate: Crate,
analysis: Analysis,
offset_index: &'b OffsetIndex,
) -> Self {
// Initialize the type_vnames HashMap with builtin types
Expand Down Expand Up @@ -246,7 +245,7 @@ impl<'a, 'b> CrateAnalyzer<'a, 'b> {
Self {
emitter,
file_vnames,
krate,
analysis,
unit_vname,
krate_ids: HashMap::new(),
krate_vname: VName::new(),
Expand All @@ -273,12 +272,8 @@ impl<'a, 'b> CrateAnalyzer<'a, 'b> {
/// Generates and emits package nodes for the main crate and external crates
/// NOTE: Must be called first to populate the self.krate_ids HashMap
pub fn emit_crate_nodes(&mut self) -> Result<(), KytheError> {
let krate_analysis = &self.krate.analysis;
let krate_prelude = &krate_analysis.prelude.as_ref().ok_or_else(|| {
KytheError::IndexerError(format!(
"Crate \"{}\" did not have prelude data",
&self.krate.id.name
))
let krate_prelude = &self.analysis.prelude.as_ref().ok_or_else(|| {
KytheError::IndexerError("Crate did not have prelude data".to_string())
})?;

// First emit the node for our own crate and add it to the hashmap
Expand Down Expand Up @@ -316,7 +311,7 @@ impl<'a, 'b> CrateAnalyzer<'a, 'b> {
// Create a HashMap mapping the implementation Id to the implementation
// It might be a safe assumption that the index is the Id, but we can't be too
// careful
let impls = self.krate.analysis.impls.clone();
let impls = self.analysis.impls.clone();
let mut impl_map: HashMap<u32, rls_data::Impl> = HashMap::new();
for implementation in impls.iter() {
impl_map.insert(implementation.id, implementation.clone());
Expand All @@ -327,7 +322,7 @@ impl<'a, 'b> CrateAnalyzer<'a, 'b> {
// Create a HashMap betwen a method definition Id and the struct and trait being
// implemented on
let mut method_index: HashMap<rls_data::Id, MethodImpl> = HashMap::new();
let relations = &self.krate.analysis.relations;
let relations = &self.analysis.relations;
for relation in relations.iter() {
// If this is an implementation relation
if let rls_data::RelationKind::Impl { id: impl_id, .. } = relation.kind {
Expand Down Expand Up @@ -400,7 +395,7 @@ impl<'a, 'b> CrateAnalyzer<'a, 'b> {
/// Emit Kythe graph information for the definitions in the crate
pub fn emit_definitions(&mut self) -> Result<(), KytheError> {
// We must clone to avoid double borrowing "self"
let defs = self.krate.analysis.defs.clone();
let defs = self.analysis.defs.clone();

for def in &defs {
let file_vname = self.file_vnames.get(def.span.file_name.to_str().unwrap());
Expand Down Expand Up @@ -697,7 +692,7 @@ impl<'a, 'b> CrateAnalyzer<'a, 'b> {
/// Emit the Kythe edges for cross references for imports in this crate
pub fn emit_import_xrefs(&mut self) -> Result<(), KytheError> {
// We must clone to avoid double borrowing "self"
let imports = self.krate.analysis.imports.clone();
let imports = self.analysis.imports.clone();

let template_vname = self.krate_vname.clone();
let krate_signature = template_vname.get_signature();
Expand Down Expand Up @@ -777,7 +772,7 @@ impl<'a, 'b> CrateAnalyzer<'a, 'b> {
/// Emit the Kythe edges for cross references in this crate
pub fn emit_xrefs(&mut self) -> Result<(), KytheError> {
// We must clone to avoid double borrowing "self"
let refs = self.krate.analysis.refs.clone();
let refs = self.analysis.refs.clone();

let template_vname = self.krate_vname.clone();
let krate_signature = template_vname.get_signature();
Expand Down

0 comments on commit 3806d62

Please sign in to comment.