Shared: Share autobuilder code between Ruby and QL

ql/extractor/src/autobuilder.rs

@@ -1,48 +1,21 @@
-use clap::Args;
 use std::env;
 use std::path::PathBuf;
-use std::process::Command;
+
+use clap::Args;
+
+use codeql_extractor::autobuilder;
 
 #[derive(Args)]
 // The autobuilder takes no command-line options, but this may change in the future.
 pub struct Options {}
 
 pub fn run(_: Options) -> std::io::Result<()> {
-    let dist = env::var("CODEQL_DIST").expect("CODEQL_DIST not set");
-    let db = env::var("CODEQL_EXTRACTOR_QL_WIP_DATABASE")
+    let database = env::var("CODEQL_EXTRACTOR_QL_WIP_DATABASE")
         .expect("CODEQL_EXTRACTOR_QL_WIP_DATABASE not set");
-    let codeql = if env::consts::OS == "windows" {
-        "codeql.exe"
-    } else {
-        "codeql"
-    };
-    let codeql: PathBuf = [&dist, codeql].iter().collect();
-    let mut cmd = Command::new(codeql);
-    cmd.arg("database")
-        .arg("index-files")
-        .arg("--include-extension=.ql")
-        .arg("--include-extension=.qll")
-        .arg("--include-extension=.dbscheme")
-        .arg("--include-extension=.json")
-        .arg("--include-extension=.jsonc")
-        .arg("--include-extension=.jsonl")
-        .arg("--include=**/qlpack.yml")
-        .arg("--include=deprecated.blame")
-        .arg("--size-limit=10m")
-        .arg("--language=ql")
-        .arg("--working-dir=.")
-        .arg(db);
 
-    for line in env::var("LGTM_INDEX_FILTERS")
-        .unwrap_or_default()
-        .split('\n')
-    {
-        if let Some(stripped) = line.strip_prefix("include:") {
-            cmd.arg("--also-match=".to_owned() + stripped);
-        } else if let Some(stripped) = line.strip_prefix("exclude:") {
-            cmd.arg("--exclude=".to_owned() + stripped);
-        }
-    }
-    let exit = &cmd.spawn()?.wait()?;
-    std::process::exit(exit.code().unwrap_or(1))
+    autobuilder::Autobuilder::new("ql", PathBuf::from(database))
+        .include_extensions(&[".ql", ".qll", ".dbscheme", ".json", ".jsonc", ".jsonl"])
+        .include_globs(&["**/qlpack.yml", "deprecated.blame"])
+        .size_limit("10m")
+        .run()
 }

ruby/extractor/src/autobuilder.rs

@@ -1,45 +1,22 @@
-use clap::Args;
 use std::env;
 use std::path::PathBuf;
-use std::process::Command;
+
+use clap::Args;
+
+use codeql_extractor::autobuilder;
 
 #[derive(Args)]
 // The autobuilder takes no command-line options, but this may change in the future.
 pub struct Options {}
 
 pub fn run(_: Options) -> std::io::Result<()> {
-    let dist = env::var("CODEQL_DIST").expect("CODEQL_DIST not set");
-    let db = env::var("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE")
+    let database = env::var("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE")
         .expect("CODEQL_EXTRACTOR_RUBY_WIP_DATABASE not set");
-    let codeql = if env::consts::OS == "windows" {
-        "codeql.exe"
-    } else {
-        "codeql"
-    };
-    let codeql: PathBuf = [&dist, codeql].iter().collect();
-    let mut cmd = Command::new(codeql);
-    cmd.arg("database")
-        .arg("index-files")
-        .arg("--include-extension=.rb")
-        .arg("--include-extension=.erb")
-        .arg("--include-extension=.gemspec")
-        .arg("--include=**/Gemfile")
-        .arg("--exclude=**/.git")
-        .arg("--size-limit=5m")
-        .arg("--language=ruby")
-        .arg("--working-dir=.")
-        .arg(db);
 
-    for line in env::var("LGTM_INDEX_FILTERS")
-        .unwrap_or_default()
-        .split('\n')
-    {
-        if let Some(stripped) = line.strip_prefix("include:") {
-            cmd.arg("--also-match=".to_owned() + stripped);
-        } else if let Some(stripped) = line.strip_prefix("exclude:") {
-            cmd.arg("--exclude=".to_owned() + stripped);
-        }
-    }
-    let exit = &cmd.spawn()?.wait()?;
-    std::process::exit(exit.code().unwrap_or(1))
+    autobuilder::Autobuilder::new("ruby", PathBuf::from(database))
+        .include_extensions(&[".rb", ".erb", ".gemspec"])
+        .include_globs(&["**/Gemfile"])
+        .exclude_globs(&["**/.git"])
+        .size_limit("5m")
+        .run()
 }

shared/tree-sitter-extractor/src/autobuilder.rs

@@ -0,0 +1,90 @@
+use std::env;
+use std::path::PathBuf;
+use std::process::Command;
+
+pub struct Autobuilder {
+    include_extensions: Vec<String>,
+    include_globs: Vec<String>,
+    exclude_globs: Vec<String>,
+    language: String,
+    database: PathBuf,
+    size_limit: Option<String>,
+}
+
+impl Autobuilder {
+    pub fn new(language: &str, database: PathBuf) -> Self {
+        Self {
+            language: language.to_string(),
+            database: database,
+            include_extensions: vec![],
+            include_globs: vec![],
+            exclude_globs: vec![],
+            size_limit: None,
+        }
+    }
+
+    pub fn include_extensions(&mut self, exts: &[&str]) -> &mut Self {
+        self.include_extensions = exts.into_iter().map(|s| String::from(*s)).collect();
+        self
+    }
+
+    pub fn include_globs(&mut self, globs: &[&str]) -> &mut Self {
+        self.include_globs = globs.into_iter().map(|s| String::from(*s)).collect();
+        self
+    }
+
+    pub fn exclude_globs(&mut self, globs: &[&str]) -> &mut Self {
+        self.exclude_globs = globs.into_iter().map(|s| String::from(*s)).collect();
+        self
+    }
+
+    pub fn size_limit(&mut self, limit: &str) -> &mut Self {
+        self.size_limit = Some(limit.to_string());
+        self
+    }
+
+    pub fn run(&self) -> std::io::Result<()> {
+        let dist = env::var("CODEQL_DIST").expect("CODEQL_DIST not set");
+        let codeql = if env::consts::OS == "windows" {
+            "codeql.exe"
+        } else {
+            "codeql"
+        };
+        let codeql: PathBuf = [&dist, codeql].iter().collect();
+        let mut cmd = Command::new(codeql);
+        cmd.arg("database").arg("index-files");
+
+        for ext in &self.include_extensions {
+            cmd.arg(format!("--include-extension={}", ext));
+        }
+
+        for glob in &self.include_globs {
+            cmd.arg(format!("--include={}", glob));
+        }
+
+        for glob in &self.exclude_globs {
+            cmd.arg(format!("--exclude={}", glob));
+        }
+
+        if let Some(limit) = &self.size_limit {
+            cmd.arg(format!("--size-limit={}", limit));
+        }
+
+        cmd.arg(format!("--language={}", &self.language));
+        cmd.arg("--working-dir=.");
+        cmd.arg(&self.database);
+
+        for line in env::var("LGTM_INDEX_FILTERS")
+            .unwrap_or_default()
+            .split('\n')
+        {
+            if let Some(stripped) = line.strip_prefix("include:") {
+                cmd.arg("--also-match=".to_owned() + stripped);
+            } else if let Some(stripped) = line.strip_prefix("exclude:") {
+                cmd.arg("--exclude=".to_owned() + stripped);
+            }
+        }
+        let exit = &cmd.spawn()?.wait()?;
+        std::process::exit(exit.code().unwrap_or(1))
+    }
+}

shared/tree-sitter-extractor/src/lib.rs

@@ -1,3 +1,4 @@
+pub mod autobuilder;
 pub mod diagnostics;
 pub mod extractor;
 pub mod file_paths;