Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 10 additions & 26 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,18 +39,16 @@ jobs:
with:
components: clippy
- uses: Swatinem/rust-cache@v2
- run: cargo clippy --all-targets --all-features -- -D warnings
- run: cargo clippy --all-features -- -D clippy::correctness

# test:
# name: Test
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v4
# - uses: dtolnay/rust-toolchain@stable
# - uses: Swatinem/rust-cache@v2
# - run: cargo test --lib --all-features
# env:
# OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
test:
name: Test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- run: cargo test --lib --all-features

docs:
name: Documentation
Expand All @@ -60,24 +58,10 @@ jobs:
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Check documentation build
run: cargo doc --no-deps --all-features
run: cargo doc -p vectorless --no-deps --all-features
env:
RUSTDOCFLAGS: -D warnings

# doctest:
# name: Doc Tests
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v4
# - uses: dtolnay/rust-toolchain@stable
# - uses: Swatinem/rust-cache@v2
# - name: Run documentation tests
# run: cargo test --doc --all-features
# continue-on-error: true # Allow doctest failures initially
# env:
# OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

# Release build check
build:
name: Build
runs-on: ubuntu-latest
Expand Down
8 changes: 4 additions & 4 deletions python/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@ use ::vectorless::client::{
};
use ::vectorless::error::Error as RustError;
use ::vectorless::metrics::IndexMetrics;
use ::vectorless::metrics::{LlmMetricsReport, MetricsReport, PilotMetricsReport, RetrievalMetricsReport};
use ::vectorless::metrics::{
LlmMetricsReport, MetricsReport, PilotMetricsReport, RetrievalMetricsReport,
};

// ============================================================
// Error Types
Expand Down Expand Up @@ -679,9 +681,7 @@ impl PyLlmMetricsReport {
fn __repr__(&self) -> String {
format!(
"LlmMetricsReport(calls={}, tokens={}, cost=${:.4})",
self.inner.total_calls,
self.inner.total_tokens,
self.inner.estimated_cost_usd,
self.inner.total_calls, self.inner.total_tokens, self.inner.estimated_cost_usd,
)
}
}
Expand Down
5 changes: 4 additions & 1 deletion rust/examples/advanced.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,10 @@ async fn main() -> vectorless::Result<()> {

// Query
let result = client
.query(QueryContext::new("What features does Vectorless provide?").with_doc_ids(vec![doc_id.clone()]))
.query(
QueryContext::new("What features does Vectorless provide?")
.with_doc_ids(vec![doc_id.clone()]),
)
.await?;
println!("Query: What features does Vectorless provide?");
if let Some(item) = result.single() {
Expand Down
19 changes: 8 additions & 11 deletions rust/examples/index_directory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,7 @@ async fn main() -> vectorless::Result<()> {

// Parse CLI arguments
let args: Vec<String> = std::env::args().collect();
let dir = args
.get(1)
.map(|s| s.as_str())
.unwrap_or("./samples");
let dir = args.get(1).map(|s| s.as_str()).unwrap_or("./samples");
let recursive = !args.iter().any(|a| a == "--no-recursive");

// Build engine
Expand All @@ -46,7 +43,11 @@ async fn main() -> vectorless::Result<()> {
.map_err(|e| vectorless::Error::Config(e.to_string()))?;

// Index directory
println!("{}indexing: {}", if recursive { "Recursively " } else { "" }, dir);
println!(
"{}indexing: {}",
if recursive { "Recursively " } else { "" },
dir
);
let ctx = IndexContext::from_dir(dir, recursive);

if ctx.is_empty() {
Expand Down Expand Up @@ -81,9 +82,7 @@ async fn main() -> vectorless::Result<()> {
let query = "What is this about?";
println!("\nQuerying: \"{query}\"");

let answer = engine
.query(vectorless::QueryContext::new(query))
.await?;
let answer = engine.query(vectorless::QueryContext::new(query)).await?;

for item in &answer.items {
println!(" [{} score={:.2}]", item.doc_id, item.score);
Expand All @@ -99,9 +98,7 @@ async fn main() -> vectorless::Result<()> {
println!("\nMetrics:");
println!(
" LLM: {} calls, {} tokens, ${:.4}",
report.llm.total_calls,
report.llm.total_tokens,
report.llm.estimated_cost_usd,
report.llm.total_calls, report.llm.total_tokens, report.llm.estimated_cost_usd,
);
println!(
" Retrieval: {} queries, avg score {:.2}",
Expand Down
7 changes: 2 additions & 5 deletions rust/src/client/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -442,8 +442,7 @@ impl EngineBuilder {
};

// Indexer uses pool.index()
let indexer =
crate::client::indexer::IndexerClient::with_llm(pool.index().clone());
let indexer = crate::client::indexer::IndexerClient::with_llm(pool.index().clone());

// Retriever uses pool.retrieval()
let retrieval_config = config.retrieval.clone();
Expand All @@ -462,9 +461,7 @@ impl EngineBuilder {
retriever = retriever.with_memo_store(memo_store);
} else {
// Create default memo store with model from config
let memo_store = MemoStore::new()
.with_model(retrieval_model)
.with_version(1);
let memo_store = MemoStore::new().with_model(retrieval_model).with_version(1);
retriever = retriever.with_memo_store(memo_store);
}

Expand Down
4 changes: 2 additions & 2 deletions rust/src/client/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,13 @@ use crate::retrieval::{PipelineRetriever, RetrieveEventReceiver};
use crate::storage::{PersistedDocument, Workspace};
use crate::{DocumentTree, Error};

use crate::events::EventEmitter;
use super::index_context::{IndexContext, IndexSource};
use super::indexer::IndexerClient;
use super::query_context::{QueryContext, QueryScope};
use super::retriever::RetrieverClient;
use super::types::{DocumentInfo, FailedItem, IndexItem, IndexMode, IndexResult, QueryResult};
use super::workspace::WorkspaceClient;
use crate::events::EventEmitter;

/// The main Engine client.
///
Expand Down Expand Up @@ -485,7 +485,7 @@ impl Engine {

/// Query a document with streaming results.
///
/// Returns a [`RetrieveEventReceiver`] that yields [`RetrieveEvent`](crate::retrieval::RetrieveEvent)s
/// Returns a receiver that yields retrieval events
/// as the retrieval pipeline progresses through each stage.
///
/// Only supports single-document scope (via `with_doc_ids` with one ID).
Expand Down
20 changes: 16 additions & 4 deletions rust/src/client/indexer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@ use crate::index::{
use crate::llm::LlmClient;
use crate::storage::{DocumentMeta, PersistedDocument};

use crate::events::{EventEmitter, IndexEvent};
use super::index_context::IndexSource;
use super::types::{IndexOptions, IndexedDocument};
use crate::events::{EventEmitter, IndexEvent};

/// Document indexing client.
///
Expand Down Expand Up @@ -115,7 +115,11 @@ impl IndexerClient {
let validation = crate::utils::validate_file(&path)?;
if !validation.valid {
return Err(Error::Parse(
validation.errors.first().cloned().unwrap_or_else(|| "Invalid file".to_string()),
validation
.errors
.first()
.cloned()
.unwrap_or_else(|| "Invalid file".to_string()),
));
}
for warning in &validation.warnings {
Expand Down Expand Up @@ -162,7 +166,11 @@ impl IndexerClient {
let validation = crate::utils::validate_content(content, format);
if !validation.valid {
return Err(Error::Parse(
validation.errors.first().cloned().unwrap_or_else(|| "Invalid content".to_string()),
validation
.errors
.first()
.cloned()
.unwrap_or_else(|| "Invalid content".to_string()),
));
}

Expand Down Expand Up @@ -199,7 +207,11 @@ impl IndexerClient {
let validation = crate::utils::validate_bytes(bytes, format);
if !validation.valid {
return Err(Error::Parse(
validation.errors.first().cloned().unwrap_or_else(|| "Invalid bytes".to_string()),
validation
.errors
.first()
.cloned()
.unwrap_or_else(|| "Invalid bytes".to_string()),
));
}

Expand Down
4 changes: 3 additions & 1 deletion rust/src/client/query_context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,9 @@ mod tests {
#[test]
fn test_single_doc_scope() {
let ctx = QueryContext::new("test").with_doc_ids(vec!["doc-1".to_string()]);
assert!(matches!(ctx.scope, QueryScope::Documents(ref ids) if ids == &["doc-1".to_string()]));
assert!(
matches!(ctx.scope, QueryScope::Documents(ref ids) if ids == &["doc-1".to_string()])
);
}

#[test]
Expand Down
2 changes: 1 addition & 1 deletion rust/src/client/retriever.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@ use std::sync::Arc;

use tracing::info;

use crate::events::{EventEmitter, QueryEvent};
use super::types::QueryResultItem;
use crate::config::Config;
use crate::document::{DocumentTree, ReasoningIndex};
use crate::error::{Error, Result};
use crate::events::{EventEmitter, QueryEvent};
use crate::retrieval::stream::RetrieveEventReceiver;
use crate::retrieval::{RetrieveOptions, RetrieveResponse};

Expand Down
4 changes: 2 additions & 2 deletions rust/src/client/workspace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ use tracing::{debug, info};
use crate::error::Result;
use crate::storage::{PersistedDocument, Workspace};

use crate::events::{EventEmitter, WorkspaceEvent};
use super::types::DocumentInfo;
use crate::events::{EventEmitter, WorkspaceEvent};

/// Workspace management client.
///
Expand Down Expand Up @@ -230,4 +230,4 @@ impl WorkspaceClient {
pub async fn set_graph(&self, graph: &crate::graph::DocumentGraph) -> Result<()> {
self.workspace.set_graph(graph).await
}
}
}
5 changes: 3 additions & 2 deletions rust/src/config/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ mod validator;
pub(crate) use loader::ConfigLoader;
pub(crate) use types::{
CacheConfig, CompressionAlgorithm, ConcurrencyConfig, Config, FallbackBehavior, FallbackConfig,
IndexerConfig, LlmClientConfig, LlmConfig, LlmMetricsConfig, LlmPoolConfig, MetricsConfig, OnAllFailedBehavior,
PilotMetricsConfig, RetrievalConfig, RetrievalMetricsConfig, SufficiencyConfig, SummaryConfig,
IndexerConfig, LlmClientConfig, LlmConfig, LlmMetricsConfig, LlmPoolConfig, MetricsConfig,
OnAllFailedBehavior, PilotMetricsConfig, RetrievalConfig, RetrievalMetricsConfig,
SufficiencyConfig, SummaryConfig,
};
2 changes: 1 addition & 1 deletion rust/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
//! Error types for the vectorless library.
//!
//! This module provides a comprehensive error type hierarchy for all operations.
//! All errors are consolidated into [`Error`] with specific variants for each category.
//! All errors are consolidated into [`enum@Error`] with specific variants for each category.

use thiserror::Error;

Expand Down
2 changes: 1 addition & 1 deletion rust/src/index/pipeline/context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,7 @@ impl IndexContext {

/// Compute SHA-256 hash of the source content.
fn compute_source_hash(input: &IndexInput) -> String {
use sha2::{Sha256, Digest};
use sha2::{Digest, Sha256};
let hash = match input {
IndexInput::File(path) => {
// Hash the file path as proxy — actual content may not be readable yet
Expand Down
4 changes: 3 additions & 1 deletion rust/src/index/pipeline/executor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,9 @@ impl PipelineExecutor {
/// 7. `reasoning_index` - Build pre-computed reasoning index
/// 8. `optimize` - Optimize tree
pub fn with_llm(client: LlmClient) -> Self {
tracing::info!("PipelineExecutor::with_llm — cloning client to ParseStage + EnhanceStage + context");
tracing::info!(
"PipelineExecutor::with_llm — cloning client to ParseStage + EnhanceStage + context"
);
let orchestrator = PipelineOrchestrator::new()
.with_llm_client(client.clone())
.stage_with_priority(ParseStage::with_llm_client(client.clone()), 10)
Expand Down
4 changes: 3 additions & 1 deletion rust/src/index/pipeline/orchestrator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -528,7 +528,9 @@ impl PipelineOrchestrator {
ctx.stage_results.contains_key(name)
});
if all_completed {
let names: Vec<&str> = group.stage_indices.iter()
let names: Vec<&str> = group
.stage_indices
.iter()
.map(|&i| self.stages[i].stage.name())
.collect();
info!("Skipping already completed parallel group: {:?}", names);
Expand Down
4 changes: 3 additions & 1 deletion rust/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,4 +73,6 @@ pub use graph::DocumentGraph;
pub use events::{EventEmitter, IndexEvent, QueryEvent, WorkspaceEvent};

// Runtime metrics reports
pub use metrics::{IndexMetrics, LlmMetricsReport, MetricsReport, PilotMetricsReport, RetrievalMetricsReport};
pub use metrics::{
IndexMetrics, LlmMetricsReport, MetricsReport, PilotMetricsReport, RetrievalMetricsReport,
};
9 changes: 4 additions & 5 deletions rust/src/utils/validation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,11 +73,10 @@ pub fn validate_file(path: &Path) -> Result<SourceValidation> {

let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("");
if DocumentFormat::from_extension(ext).is_none() {
return Ok(SourceValidation::invalid(vec![format!(
"Unsupported format: .{}",
ext
)])
.with_warnings(warnings));
return Ok(
SourceValidation::invalid(vec![format!("Unsupported format: .{}", ext)])
.with_warnings(warnings),
);
}

Ok(SourceValidation::valid().with_warnings(warnings))
Expand Down
Loading