Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion src/commands/scan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,20 @@ pub async fn execute_scan(
"batched files"
);

// 4. Process batches and collect issues
// 4. Build brain context once for the entire scan (if use_brain enabled).
// Extracts defined symbols from the file list and queries the symbol
// index for impact analysis, affected tests, and related patterns.
let brain_ctx = if config.context_chain.use_brain {
crate::engine::review::build_scan_brain_context(
&files,
config.context_chain.impact_depth,
&root_abs,
)
} else {
None
};

// 5. Process batches and collect issues
let mut all_issues = Vec::new();
let mut total_tokens: Option<TokenUsage> = None;
let mut skipped_batches: Vec<(usize, Vec<String>, String)> = Vec::new();
Expand All @@ -165,6 +178,7 @@ pub async fn execute_scan(
&config.rules,
&config.response_format,
None,
brain_ctx.as_deref(),
)
.await
{
Expand Down
74 changes: 70 additions & 4 deletions src/embed/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,80 @@
//! compiled into the binary via `include_bytes!` / `include_str!`.
//! Higher quality at the cost of ~30 MB binary size.
//!
//! Re-exports will be added to this module in Phase 3 when `cora brain`
//! commands are wired up.

// Embedding engine — consumed by brain module (Phase 3).
//! The [`embed_code_dispatch`] function selects the best available backend at
//! compile time: pretrained-embed (768d) → hashing trick (256d) → FTS5-only.

pub mod tokens;

// Pre-trained nomic-embed-code module — only compiled when vendored data
// exists. Excluded from crates.io package (too large for 10 MB upload limit).
#[cfg(feature = "pretrained-embed")]
pub mod token_vocab;

// Re-export tokenizer — used by both backends.
pub use tokens::EMBEDDING_DIM;

// Re-export pretrained constants when feature is enabled.
#[cfg(feature = "pretrained-embed")]
pub use token_vocab::{PRETRAINED_DIM, embed_code_pretrained};

/// Returns the embedding dimensionality used by the active backend.
///
/// - `pretrained-embed` feature → 768
/// - default (hashing trick) → 256
pub const fn active_dims() -> usize {
#[cfg(feature = "pretrained-embed")]
{
PRETRAINED_DIM
}
#[cfg(not(feature = "pretrained-embed"))]
{
EMBEDDING_DIM
}
}

/// Returns a human-readable label for the active embedding provider.
pub const fn active_provider_name() -> &'static str {
#[cfg(feature = "pretrained-embed")]
{
"nomic-embed-code (768d, pretrained)"
}
#[cfg(not(feature = "pretrained-embed"))]
{
"hashing-trick (256d, static)"
}
}

/// Embed a code snippet using the best available backend.
///
/// Returns an f32 vector that can be passed directly to usearch.
///
/// - **Pretrained path** (`pretrained-embed` feature): tokenises → looks up
/// each token in the nomic vocabulary → accumulates int8 vectors → L2-normalises.
/// Returns 768-dim vector.
///
/// - **Hashing-trick fallback**: tokenises → hashes each token into a
/// pseudo-random 256-dim vector → accumulates → L2-normalises.
/// Returns 256-dim vector.
///
/// Both paths share the same [`tokenize_code`] tokenizer.
pub fn embed_code_dispatch(code: &str) -> Vec<f32> {
#[cfg(feature = "pretrained-embed")]
{
embed_code_pretrained(code)
}
#[cfg(not(feature = "pretrained-embed"))]
{
let embedding = tokens::embed_code(code);
embedding.as_slice().iter().map(|&v| v as f32).collect()
}
}

/// Whether the pretrained embedding backend is available (compile-time).
#[expect(
dead_code,
reason = "used by Phase 3+ features; embed module not yet wired at call sites"
)]
pub const fn has_pretrained() -> bool {
cfg!(feature = "pretrained-embed")
}
13 changes: 10 additions & 3 deletions src/embed/token_vocab.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
//! Pre-trained token vocabulary — reserved for Phase 5 (Voyage-4-Nano ONNX).
#![allow(dead_code)]

//! Pre-trained token vocabulary from nomic-embed-code.
//!
//! Provides real 768-dimensional code embeddings distilled from
Expand Down Expand Up @@ -148,6 +145,16 @@ pub fn embed_pretrained(tokens: &HashMap<String, u32>) -> PretrainedEmbedding {
PretrainedEmbedding { vec }
}

/// Convenience function: tokenize → embed using pretrained vocabulary in one step.
///
/// Returns an f32 vector (768 dimensions, L2-normalised) ready for usearch.
/// This is the pretrained counterpart of [`crate::embed::tokens::embed_code`].
pub fn embed_code_pretrained(code: &str) -> Vec<f32> {
let tokens = crate::embed::tokens::tokenize_code(code);
let emb = embed_pretrained(&tokens);
emb.into_vec()
}

/// Compute cosine similarity between two [`PretrainedEmbedding`]s.
///
/// Because vectors are pre-normalised, this is just the dot product.
Expand Down
Loading
Loading