Ruby: Merge extractor crates

.github/workflows/ruby-build.yml

@@ -55,35 +55,35 @@ jobs:
         id: cache-extractor
         with:
           path: |
-            ruby/target/release/ruby-autobuilder
-            ruby/target/release/ruby-autobuilder.exe
-            ruby/target/release/ruby-extractor
-            ruby/target/release/ruby-extractor.exe
-            ruby/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}--${{ hashFiles('ruby/**/*.rs') }}
+            ruby/extractor/target/release/autobuilder
+            ruby/extractor/target/release/autobuilder.exe
+            ruby/extractor/target/release/extractor
+            ruby/extractor/target/release/extractor.exe
+            ruby/extractor/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}--${{ hashFiles('ruby/extractor/**/*.rs') }}
       - uses: actions/cache@v3
         if: steps.cache-extractor.outputs.cache-hit != 'true'
         with:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
             ruby/target
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-rust-cargo-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/**/Cargo.lock') }}
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-rust-cargo-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/**/Cargo.lock') }}
       - name: Check formatting
         if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cargo fmt --all -- --check
+        run: cd extractor && cargo fmt --all -- --check
       - name: Build
         if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cargo build --verbose
+        run: cd extractor && cargo build --verbose
       - name: Run tests
         if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cargo test --verbose
+        run: cd extractor && cargo test --verbose
       - name: Release build
         if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cargo build --release
+        run: cd extractor && cargo build --release
       - name: Generate dbscheme
         if: ${{ matrix.os == 'ubuntu-latest' && steps.cache-extractor.outputs.cache-hit != 'true'}}
-        run: target/release/ruby-generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+        run: extractor/target/release/generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v3
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
@@ -98,10 +98,10 @@ jobs:
         with:
           name: extractor-${{ matrix.os }}
           path: |
-            ruby/target/release/ruby-autobuilder
-            ruby/target/release/ruby-autobuilder.exe
-            ruby/target/release/ruby-extractor
-            ruby/target/release/ruby-extractor.exe
+            ruby/extractor/target/release/autobuilder
+            ruby/extractor/target/release/autobuilder.exe
+            ruby/extractor/target/release/extractor
+            ruby/extractor/target/release/extractor.exe
           retention-days: 1
   compile-queries:
     runs-on: ubuntu-latest-xl
@@ -116,21 +116,22 @@ jobs:
           key: ruby-build
       - name: Build Query Pack
         run: |
-          rm -rf target/packs
-          codeql pack create ../misc/suite-helpers --output target/packs
-          codeql pack create ../shared/regex --output target/packs
-          codeql pack create ../shared/ssa --output target/packs
-          codeql pack create ../shared/tutorial --output target/packs
-          codeql pack create ql/lib --output target/packs
-          codeql pack create -j0 ql/src --output target/packs --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-          PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
+          PACKS=${{ runner.temp }}/query-packs
+          rm -rf $PACKS
+          codeql pack create ../misc/suite-helpers --output "$PACKS"
+          codeql pack create ../shared/regex --output "$PACKS"
+          codeql pack create ../shared/ssa --output "$PACKS"
+          codeql pack create ../shared/tutorial --output "$PACKS"
+          codeql pack create ql/lib --output "$PACKS"
+          codeql pack create -j0 ql/src --output "$PACKS" --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          PACK_FOLDER=$(readlink -f "$PACKS"/codeql/ruby-queries/*)
           codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
           (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
       - uses: actions/upload-artifact@v3
         with:
           name: codeql-ruby-queries
           path: |
-            ruby/target/packs/*
+            ${{ runner.temp }}/query-packs/*
           retention-days: 1
 
   package:
@@ -158,12 +159,12 @@ jobs:
           mkdir -p ruby
           cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
           mkdir -p ruby/tools/{linux64,osx64,win64}
-          cp linux64/ruby-autobuilder ruby/tools/linux64/autobuilder
-          cp osx64/ruby-autobuilder ruby/tools/osx64/autobuilder
-          cp win64/ruby-autobuilder.exe ruby/tools/win64/autobuilder.exe
-          cp linux64/ruby-extractor ruby/tools/linux64/extractor
-          cp osx64/ruby-extractor ruby/tools/osx64/extractor
-          cp win64/ruby-extractor.exe ruby/tools/win64/extractor.exe
+          cp linux64/autobuilder ruby/tools/linux64/autobuilder
+          cp osx64/autobuilder ruby/tools/osx64/autobuilder
+          cp win64/autobuilder.exe ruby/tools/win64/autobuilder.exe
+          cp linux64/extractor ruby/tools/linux64/extractor
+          cp osx64/extractor ruby/tools/osx64/extractor
+          cp win64/extractor.exe ruby/tools/win64/extractor.exe
           chmod +x ruby/tools/{linux64,osx64}/{autobuilder,extractor}
           zip -rq codeql-ruby.zip ruby
       - uses: actions/upload-artifact@v3

ruby/.gitignore

@@ -1,4 +1,4 @@
-/target
+extractor/target
 extractor-pack
 .vscode/launch.json
 .cache

ruby/actions/create-extractor-pack/action.yml

@@ -10,7 +10,7 @@ runs:
       uses: actions/cache@v3
       with:
         path: ruby/extractor-pack
-        key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/scripts/create-extractor-pack.sh', 'ruby/**/Cargo.lock', 'ruby/actions/create-extractor-pack/action.yml') }}-${{ hashFiles('ruby/**/*.rs') }}-${{ hashFiles('ruby/codeql-extractor.yml', 'ruby/downgrades', 'ruby/tools', 'ruby/ql/lib/ruby.dbscheme', 'ruby/ql/lib/ruby.dbscheme.stats') }}
+        key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/scripts/create-extractor-pack.sh', 'ruby/extractor/**/Cargo.lock', 'ruby/actions/create-extractor-pack/action.yml') }}-${{ hashFiles('ruby/extractor/**/*.rs') }}-${{ hashFiles('ruby/codeql-extractor.yml', 'ruby/downgrades', 'ruby/tools', 'ruby/ql/lib/ruby.dbscheme', 'ruby/ql/lib/ruby.dbscheme.stats') }}
     - name: Cache cargo
       uses: actions/cache@v3
       if: steps.cache-extractor.outputs.cache-hit != 'true'
@@ -19,7 +19,7 @@ runs:
           ~/.cargo/registry
           ~/.cargo/git
           ruby/target
-        key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-qltest-cargo-${{ hashFiles('ruby/rust-toolchain.toml', 'ruby/scripts/create-extractor-pack.sh', 'ruby/**/Cargo.lock') }}
+        key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-qltest-cargo-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/scripts/create-extractor-pack.sh', 'ruby/extractor/**/Cargo.lock') }}
     - name: Build Extractor
       if: steps.cache-extractor.outputs.cache-hit != 'true'
       shell: bash

ruby/extractor/Cargo.toml

@@ -8,7 +8,6 @@ edition = "2018"
 
 [dependencies]
 flate2 = "1.0"
-node-types = { path = "../node-types" }
 tree-sitter = "0.20"
 tree-sitter-embedded-template = { git = "https://github.com/tree-sitter/tree-sitter-embedded-template.git", rev = "203f7bd3c1bbfbd98fc19add4b8fcb213c059205" }
 tree-sitter-ruby = { git = "https://github.com/tree-sitter/tree-sitter-ruby.git", rev = "206c7077164372c596ffa8eaadb9435c28941364" }

ruby/extractor/src/main.rs → ruby/extractor/src/bin/extractor.rs

@@ -1,8 +1,3 @@
-mod diagnostics;
-mod extractor;
-mod file_paths;
-mod trap;
-
 #[macro_use]
 extern crate lazy_static;
 extern crate num_cpus;
@@ -16,6 +11,8 @@ use std::io::BufRead;
 use std::path::{Path, PathBuf};
 use tree_sitter::{Language, Parser, Range};
 
+use ruby_extractor::{diagnostics, extractor, file_paths, node_types, trap};
+
 /**
  * Gets the number of threads the extractor should use, by reading the
  * CODEQL_THREADS environment variable and using it as described in the
@@ -46,6 +43,7 @@ lazy_static! {
     static ref CP_NUMBER: regex::Regex = regex::Regex::new("cp([0-9]+)").unwrap();
 }
 
+/// Returns the `encoding::Encoding` corresponding to the given encoding name, if one exists.
 fn encoding_from_name(encoding_name: &str) -> Option<&(dyn encoding::Encoding + Send + Sync)> {
     match encoding::label::encoding_from_whatwg_label(encoding_name) {
         s @ Some(_) => s,

ruby/generator/src/main.rs → ruby/extractor/src/bin/generator.rs

@@ -1,17 +1,14 @@
-mod dbscheme;
-mod language;
-mod ql;
-mod ql_gen;
-
 use clap::arg;
-use language::Language;
 use std::collections::BTreeMap as Map;
 use std::collections::BTreeSet as Set;
 use std::fs::File;
 use std::io::LineWriter;
 use std::io::Write;
 use std::path::PathBuf;
 
+use ruby_extractor::generator::{dbscheme, language::Language, ql, ql_gen};
+use ruby_extractor::node_types;
+
 /// Given the name of the parent node, and its field information, returns a pair,
 /// the first of which is the field's type. The second is an optional dbscheme
 /// entry that should be added.
@@ -573,7 +570,12 @@ fn main() -> std::io::Result<()> {
             node_types: tree_sitter_embedded_template::NODE_TYPES,
         },
     ];
-    let mut dbscheme_writer = LineWriter::new(File::create(dbscheme_path)?);
+
+    let dbscheme_file = File::create(dbscheme_path).map_err(|e| {
+        tracing::error!("Failed to create dbscheme file: {}", e);
+        e
+    })?;
+    let mut dbscheme_writer = LineWriter::new(dbscheme_file);
     write!(
         dbscheme_writer,
         "// CodeQL database schema for {}\n\
@@ -596,7 +598,11 @@ fn main() -> std::io::Result<()> {
         ],
     )?;
 
-    let mut ql_writer = LineWriter::new(File::create(ql_library_path)?);
+    let ql_library_file = File::create(ql_library_path).map_err(|e| {
+        tracing::error!("Failed to create ql library file: {}", e);
+        e
+    })?;
+    let mut ql_writer = LineWriter::new(ql_library_file);
     write!(
         ql_writer,
         "/**\n\

ruby/extractor/src/extractor.rs

@@ -1,7 +1,7 @@
 use crate::diagnostics;
 use crate::file_paths;
+use crate::node_types::{self, EntryKind, Field, NodeTypeMap, Storage, TypeName};
 use crate::trap;
-use node_types::{EntryKind, Field, NodeTypeMap, Storage, TypeName};
 use std::collections::BTreeMap as Map;
 use std::collections::BTreeSet as Set;
 use std::fmt;

ruby/generator/src/dbscheme.rs → ruby/extractor/src/generator/dbscheme.rs

@@ -1,6 +1,8 @@
-use crate::ql;
 use std::collections::BTreeSet as Set;
 use std::fmt;
+
+use crate::generator::ql;
+
 /// Represents a distinct entry in the database schema.
 pub enum Entry<'a> {
     /// An entry defining a database table.

ruby/extractor/src/generator/mod.rs

@@ -0,0 +1,4 @@
+pub mod dbscheme;
+pub mod language;
+pub mod ql;
+pub mod ql_gen;

ruby/generator/src/ql_gen.rs → ruby/extractor/src/generator/ql_gen.rs

@@ -1,6 +1,7 @@
-use crate::ql;
 use std::collections::BTreeSet;
 
+use crate::{generator::ql, node_types};
+
 /// Creates the hard-coded `AstNode` class that acts as a supertype of all
 /// classes we generate.
 pub fn create_ast_node_class<'a>(ast_node: &'a str, node_info_table: &'a str) -> ql::Class<'a> {

ruby/extractor/src/lib.rs

@@ -0,0 +1,6 @@
+pub mod diagnostics;
+pub mod extractor;
+pub mod file_paths;
+pub mod generator;
+pub mod node_types;
+pub mod trap;

ruby/scripts/create-extractor-pack.ps1

@@ -1,6 +1,9 @@
+cd extractor
 cargo build --release
+cd ..
+
+extractor\target\release\generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 
-cargo run --release -p ruby-generator -- --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 codeql query format -i ql\lib\codeql/ruby\ast\internal\TreeSitter.qll
 
 rm -Recurse -Force extractor-pack
@@ -9,5 +12,5 @@ cp codeql-extractor.yml, ql\lib\ruby.dbscheme, ql\lib\ruby.dbscheme.stats extrac
 cp -Recurse tools extractor-pack
 cp -Recurse downgrades extractor-pack
 mkdir extractor-pack\tools\win64 | Out-Null
-cp target\release\ruby-extractor.exe extractor-pack\tools\win64\extractor.exe
-cp target\release\ruby-autobuilder.exe extractor-pack\tools\win64\autobuilder.exe
+cp extractor\target\release\extractor.exe extractor-pack\tools\win64\extractor.exe
+cp extractor\target\release\autobuilder.exe extractor-pack\tools\win64\autobuilder.exe

ruby/scripts/create-extractor-pack.sh

@@ -13,14 +13,14 @@ else
   exit 1
 fi
 
-"$CARGO" build --release
+(cd extractor && "$CARGO" build --release)
+extractor/target/release/generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 
-"$CARGO" run --release -p ruby-generator -- --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 codeql query format -i ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 
 rm -rf extractor-pack
 mkdir -p extractor-pack
 cp -r codeql-extractor.yml downgrades tools ql/lib/ruby.dbscheme ql/lib/ruby.dbscheme.stats extractor-pack/
 mkdir -p extractor-pack/tools/${platform}
-cp target/release/ruby-extractor extractor-pack/tools/${platform}/extractor
-cp target/release/ruby-autobuilder extractor-pack/tools/${platform}/autobuilder
+cp extractor/target/release/extractor extractor-pack/tools/${platform}/extractor
+cp extractor/target/release/autobuilder extractor-pack/tools/${platform}/autobuilder