diff --git a/ql/extractor/src/autobuilder.rs b/ql/extractor/src/autobuilder.rs index a17a49162393..ce58a217d8c1 100644 --- a/ql/extractor/src/autobuilder.rs +++ b/ql/extractor/src/autobuilder.rs @@ -1,48 +1,21 @@ -use clap::Args; use std::env; use std::path::PathBuf; -use std::process::Command; + +use clap::Args; + +use codeql_extractor::autobuilder; #[derive(Args)] // The autobuilder takes no command-line options, but this may change in the future. pub struct Options {} pub fn run(_: Options) -> std::io::Result<()> { - let dist = env::var("CODEQL_DIST").expect("CODEQL_DIST not set"); - let db = env::var("CODEQL_EXTRACTOR_QL_WIP_DATABASE") + let database = env::var("CODEQL_EXTRACTOR_QL_WIP_DATABASE") .expect("CODEQL_EXTRACTOR_QL_WIP_DATABASE not set"); - let codeql = if env::consts::OS == "windows" { - "codeql.exe" - } else { - "codeql" - }; - let codeql: PathBuf = [&dist, codeql].iter().collect(); - let mut cmd = Command::new(codeql); - cmd.arg("database") - .arg("index-files") - .arg("--include-extension=.ql") - .arg("--include-extension=.qll") - .arg("--include-extension=.dbscheme") - .arg("--include-extension=.json") - .arg("--include-extension=.jsonc") - .arg("--include-extension=.jsonl") - .arg("--include=**/qlpack.yml") - .arg("--include=deprecated.blame") - .arg("--size-limit=10m") - .arg("--language=ql") - .arg("--working-dir=.") - .arg(db); - for line in env::var("LGTM_INDEX_FILTERS") - .unwrap_or_default() - .split('\n') - { - if let Some(stripped) = line.strip_prefix("include:") { - cmd.arg("--also-match=".to_owned() + stripped); - } else if let Some(stripped) = line.strip_prefix("exclude:") { - cmd.arg("--exclude=".to_owned() + stripped); - } - } - let exit = &cmd.spawn()?.wait()?; - std::process::exit(exit.code().unwrap_or(1)) + autobuilder::Autobuilder::new("ql", PathBuf::from(database)) + .include_extensions(&[".ql", ".qll", ".dbscheme", ".json", ".jsonc", ".jsonl"]) + .include_globs(&["**/qlpack.yml", "deprecated.blame"]) + .size_limit("10m") + .run() } diff --git a/ruby/extractor/src/autobuilder.rs b/ruby/extractor/src/autobuilder.rs index 48db694df994..7b6e148eb794 100644 --- a/ruby/extractor/src/autobuilder.rs +++ b/ruby/extractor/src/autobuilder.rs @@ -1,45 +1,22 @@ -use clap::Args; use std::env; use std::path::PathBuf; -use std::process::Command; + +use clap::Args; + +use codeql_extractor::autobuilder; #[derive(Args)] // The autobuilder takes no command-line options, but this may change in the future. pub struct Options {} pub fn run(_: Options) -> std::io::Result<()> { - let dist = env::var("CODEQL_DIST").expect("CODEQL_DIST not set"); - let db = env::var("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE") + let database = env::var("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE") .expect("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE not set"); - let codeql = if env::consts::OS == "windows" { - "codeql.exe" - } else { - "codeql" - }; - let codeql: PathBuf = [&dist, codeql].iter().collect(); - let mut cmd = Command::new(codeql); - cmd.arg("database") - .arg("index-files") - .arg("--include-extension=.rb") - .arg("--include-extension=.erb") - .arg("--include-extension=.gemspec") - .arg("--include=**/Gemfile") - .arg("--exclude=**/.git") - .arg("--size-limit=5m") - .arg("--language=ruby") - .arg("--working-dir=.") - .arg(db); - for line in env::var("LGTM_INDEX_FILTERS") - .unwrap_or_default() - .split('\n') - { - if let Some(stripped) = line.strip_prefix("include:") { - cmd.arg("--also-match=".to_owned() + stripped); - } else if let Some(stripped) = line.strip_prefix("exclude:") { - cmd.arg("--exclude=".to_owned() + stripped); - } - } - let exit = &cmd.spawn()?.wait()?; - std::process::exit(exit.code().unwrap_or(1)) + autobuilder::Autobuilder::new("ruby", PathBuf::from(database)) + .include_extensions(&[".rb", ".erb", ".gemspec"]) + .include_globs(&["**/Gemfile"]) + .exclude_globs(&["**/.git"]) + .size_limit("5m") + .run() } diff --git a/shared/tree-sitter-extractor/src/autobuilder.rs b/shared/tree-sitter-extractor/src/autobuilder.rs new file mode 100644 index 000000000000..a6d4a6852e25 --- /dev/null +++ b/shared/tree-sitter-extractor/src/autobuilder.rs @@ -0,0 +1,90 @@ +use std::env; +use std::path::PathBuf; +use std::process::Command; + +pub struct Autobuilder { + include_extensions: Vec, + include_globs: Vec, + exclude_globs: Vec, + language: String, + database: PathBuf, + size_limit: Option, +} + +impl Autobuilder { + pub fn new(language: &str, database: PathBuf) -> Self { + Self { + language: language.to_string(), + database: database, + include_extensions: vec![], + include_globs: vec![], + exclude_globs: vec![], + size_limit: None, + } + } + + pub fn include_extensions(&mut self, exts: &[&str]) -> &mut Self { + self.include_extensions = exts.into_iter().map(|s| String::from(*s)).collect(); + self + } + + pub fn include_globs(&mut self, globs: &[&str]) -> &mut Self { + self.include_globs = globs.into_iter().map(|s| String::from(*s)).collect(); + self + } + + pub fn exclude_globs(&mut self, globs: &[&str]) -> &mut Self { + self.exclude_globs = globs.into_iter().map(|s| String::from(*s)).collect(); + self + } + + pub fn size_limit(&mut self, limit: &str) -> &mut Self { + self.size_limit = Some(limit.to_string()); + self + } + + pub fn run(&self) -> std::io::Result<()> { + let dist = env::var("CODEQL_DIST").expect("CODEQL_DIST not set"); + let codeql = if env::consts::OS == "windows" { + "codeql.exe" + } else { + "codeql" + }; + let codeql: PathBuf = [&dist, codeql].iter().collect(); + let mut cmd = Command::new(codeql); + cmd.arg("database").arg("index-files"); + + for ext in &self.include_extensions { + cmd.arg(format!("--include-extension={}", ext)); + } + + for glob in &self.include_globs { + cmd.arg(format!("--include={}", glob)); + } + + for glob in &self.exclude_globs { + cmd.arg(format!("--exclude={}", glob)); + } + + if let Some(limit) = &self.size_limit { + cmd.arg(format!("--size-limit={}", limit)); + } + + cmd.arg(format!("--language={}", &self.language)); + cmd.arg("--working-dir=."); + cmd.arg(&self.database); + + for line in env::var("LGTM_INDEX_FILTERS") + .unwrap_or_default() + .split('\n') + { + if let Some(stripped) = line.strip_prefix("include:") { + cmd.arg("--also-match=".to_owned() + stripped); + } else if let Some(stripped) = line.strip_prefix("exclude:") { + cmd.arg("--exclude=".to_owned() + stripped); + } + } + let exit = &cmd.spawn()?.wait()?; + std::process::exit(exit.code().unwrap_or(1)) + } +} diff --git a/shared/tree-sitter-extractor/src/lib.rs b/shared/tree-sitter-extractor/src/lib.rs index eb1754179930..b83c4951b73c 100644 --- a/shared/tree-sitter-extractor/src/lib.rs +++ b/shared/tree-sitter-extractor/src/lib.rs @@ -1,3 +1,4 @@ +pub mod autobuilder; pub mod diagnostics; pub mod extractor; pub mod file_paths;