From 58e4a3b75ce09dea650377d9eec00091ec2854ca Mon Sep 17 00:00:00 2001 From: renardeinside Date: Tue, 17 Feb 2026 16:45:16 +0100 Subject: [PATCH 1/9] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Migrate=20MCP=20server?= =?UTF-8?q?=20to=20rmcp=20framework=20with=20modular=20architecture?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 103 +- Cargo.toml | 1 + crates/cli/src/dev/mcp.rs | 31 +- crates/mcp/Cargo.toml | 3 +- crates/mcp/src/context.rs | 50 + crates/mcp/src/core.rs | 633 ----------- crates/mcp/src/indexing.rs | 213 ++++ crates/mcp/src/info_content.rs | 15 + crates/mcp/src/lib.rs | 7 +- crates/mcp/src/resources.rs | 18 + crates/mcp/src/server.rs | 1622 +++++----------------------- crates/mcp/src/tools/databricks.rs | 406 +++++++ crates/mcp/src/tools/devserver.rs | 106 ++ crates/mcp/src/tools/docs.rs | 92 ++ crates/mcp/src/tools/mod.rs | 40 + crates/mcp/src/tools/project.rs | 401 +++++++ crates/mcp/src/tools/registry.rs | 158 +++ crates/mcp/src/validation.rs | 58 + 18 files changed, 1918 insertions(+), 2039 deletions(-) create mode 100644 crates/mcp/src/context.rs delete mode 100644 crates/mcp/src/core.rs create mode 100644 crates/mcp/src/indexing.rs create mode 100644 crates/mcp/src/info_content.rs create mode 100644 crates/mcp/src/resources.rs create mode 100644 crates/mcp/src/tools/databricks.rs create mode 100644 crates/mcp/src/tools/devserver.rs create mode 100644 crates/mcp/src/tools/docs.rs create mode 100644 crates/mcp/src/tools/mod.rs create mode 100644 crates/mcp/src/tools/project.rs create mode 100644 crates/mcp/src/tools/registry.rs create mode 100644 crates/mcp/src/validation.rs diff --git a/Cargo.lock b/Cargo.lock index fc9843e7..c9b6e08e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -253,8 +253,7 @@ name = "apx-mcp" version = "0.3.0-rc1" dependencies = [ "apx-core", - "futures-util", - "reqwest 0.13.1", + "rmcp", "schemars 1.2.0", "serde", "serde_json", @@ -1188,8 +1187,18 @@ version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.21.3", + "darling_macro 0.21.3", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core 0.23.0", + "darling_macro 0.23.0", ] [[package]] @@ -1206,13 +1215,37 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.114", +] + [[package]] name = "darling_macro" version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ - "darling_core", + "darling_core 0.21.3", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", "quote", "syn 2.0.114", ] @@ -1678,6 +1711,21 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -1740,6 +1788,7 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -3559,6 +3608,12 @@ dependencies = [ "regex", ] +[[package]] +name = "pastey" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b867cad97c0791bbd3aaa6472142568c6c9e8f71937e98379f584cfb0cf35bec" + [[package]] name = "pbkdf2" version = "0.12.2" @@ -4440,6 +4495,41 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rmcp" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bef41ebc9ebed2c1b1d90203e9d1756091e8a00bbc3107676151f39868ca0ee" +dependencies = [ + "async-trait", + "base64 0.22.1", + "chrono", + "futures", + "pastey", + "pin-project-lite", + "rmcp-macros", + "schemars 1.2.0", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "rmcp-macros" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e88ad84b8b6237a934534a62b379a5be6388915663c0cc598ceb9b3292bbbfe" +dependencies = [ + "darling 0.23.0", + "proc-macro2", + "quote", + "serde_json", + "syn 2.0.114", +] + [[package]] name = "rsqlite-vfs" version = "0.1.0" @@ -4639,6 +4729,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54e910108742c57a770f492731f99be216a52fadd361b06c8fb59d74ccc267d2" dependencies = [ + "chrono", "dyn-clone", "ref-cast", "schemars_derive 1.2.0", @@ -4882,7 +4973,7 @@ version = "3.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" dependencies = [ - "darling", + "darling 0.21.3", "proc-macro2", "quote", "syn 2.0.114", diff --git a/Cargo.toml b/Cargo.toml index 092fd77b..11d4cd86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ tokio = { version = "1.49", features = ["rt-multi-thread", "macros", "sync", "pr tokio-stream = "0.1.17" tokio-util = { version = "0.7", features = ["io"] } futures-util = "0.3.31" +rmcp = { version = "0.15", features = ["server", "transport-io", "schemars"] } # Web server axum = { version = "0.8.8", features = ["ws"] } diff --git a/crates/cli/src/dev/mcp.rs b/crates/cli/src/dev/mcp.rs index b14904dc..0b096c33 100644 --- a/crates/cli/src/dev/mcp.rs +++ b/crates/cli/src/dev/mcp.rs @@ -1,8 +1,8 @@ use crate::run_cli_async_helper; -use apx_core::common::run_preflight_checks; use apx_core::components::new_cache_state; use apx_core::interop::get_databricks_sdk_version; -use apx_mcp::server::{AppContext, IndexState, SdkIndexParams, build_server}; +use apx_mcp::context::{AppContext, IndexState, SdkIndexParams}; +use apx_mcp::server::run_server; use clap::Args; use std::sync::Arc; use tokio::sync::{Mutex, broadcast}; @@ -12,11 +12,6 @@ pub struct McpArgs {} pub async fn run(_args: McpArgs) -> i32 { run_cli_async_helper(|| async { - let app_dir = std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from(".")); - - // Run preflight checks (installs deps if needed) - run_preflight_checks(&app_dir).await?; - // Create shutdown channel let (shutdown_tx, _) = broadcast::channel::<()>(1); @@ -49,22 +44,14 @@ pub async fn run(_args: McpArgs) -> i32 { sdk_doc_index: Arc::clone(&sdk_doc_index), }; - // Build server with SDK params - all indexing happens sequentially in one task - let server = build_server( - AppContext { - app_dir, - sdk_doc_index, - cache_state, - index_state, - shutdown_tx: shutdown_tx.clone(), - }, - Some(sdk_params), - ); + let ctx = AppContext { + sdk_doc_index, + cache_state, + index_state, + shutdown_tx: shutdown_tx.clone(), + }; - server - .run_stdio(shutdown_tx) - .await - .map_err(|e| format!("MCP server error: {e}")) + run_server(ctx, Some(sdk_params)).await }) .await } diff --git a/crates/mcp/Cargo.toml b/crates/mcp/Cargo.toml index e26330ce..7053f2d4 100644 --- a/crates/mcp/Cargo.toml +++ b/crates/mcp/Cargo.toml @@ -12,5 +12,4 @@ serde_yaml.workspace = true tokio.workspace = true tracing.workspace = true schemars.workspace = true -futures-util.workspace = true -reqwest.workspace = true +rmcp.workspace = true diff --git a/crates/mcp/src/context.rs b/crates/mcp/src/context.rs new file mode 100644 index 00000000..4bb7cc6d --- /dev/null +++ b/crates/mcp/src/context.rs @@ -0,0 +1,50 @@ +use apx_core::components::SharedCacheState; +use apx_core::search::docs_index::SDKDocsIndex; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use tokio::sync::{Mutex, Notify, broadcast}; + +/// Parameters for SDK indexing, pre-computed synchronously to avoid Python GIL issues +#[derive(Debug)] +pub struct SdkIndexParams { + pub sdk_version: Option, + pub sdk_doc_index: Arc>>, +} + +/// State for tracking index readiness +#[derive(Clone, Debug)] +pub struct IndexState { + /// Notifies waiters when component index is ready + pub component_ready: Arc, + /// Notifies waiters when SDK docs index is ready + pub sdk_ready: Arc, + /// Whether component indexing has completed (for late subscribers) + pub component_indexed: Arc, + /// Whether SDK indexing has completed (for late subscribers) + pub sdk_indexed: Arc, +} + +impl Default for IndexState { + fn default() -> Self { + Self { + component_ready: Arc::new(Notify::new()), + sdk_ready: Arc::new(Notify::new()), + component_indexed: Arc::new(AtomicBool::new(false)), + sdk_indexed: Arc::new(AtomicBool::new(false)), + } + } +} + +impl IndexState { + pub fn new() -> Self { + Self::default() + } +} + +#[derive(Debug)] +pub struct AppContext { + pub sdk_doc_index: Arc>>, + pub cache_state: SharedCacheState, + pub index_state: IndexState, + pub shutdown_tx: broadcast::Sender<()>, +} diff --git a/crates/mcp/src/core.rs b/crates/mcp/src/core.rs deleted file mode 100644 index 4ed387f3..00000000 --- a/crates/mcp/src/core.rs +++ /dev/null @@ -1,633 +0,0 @@ -use futures_util::future::BoxFuture; -use schemars::schema_for; -use serde::de::DeserializeOwned; -use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; -use std::collections::HashMap; -use std::future::Future; -use std::sync::Arc; -use tokio::sync::broadcast; - -// JSON-RPC constants -pub const JSONRPC_VERSION: &str = "2.0"; -pub const PROTOCOL_VERSION: &str = "2025-11-25"; -pub const PARSE_ERROR: i32 = -32700; -pub const METHOD_NOT_FOUND: i32 = -32601; -pub const INVALID_PARAMS: i32 = -32602; - -// JSON-RPC 2.0 Types -#[derive(Debug, Deserialize)] -pub struct JsonRpcRequest { - pub jsonrpc: String, - pub id: Option, - pub method: String, - pub params: Option, -} - -impl JsonRpcRequest { - pub fn validate(&self) -> Result<(), &'static str> { - if self.jsonrpc != JSONRPC_VERSION { - return Err("Invalid JSON-RPC version, expected 2.0"); - } - Ok(()) - } -} - -#[derive(Debug, Serialize)] -pub struct JsonRpcResponse { - pub jsonrpc: &'static str, - pub id: Value, - pub result: Value, -} - -impl JsonRpcResponse { - pub fn new(id: Value, result: Value) -> Self { - Self { - jsonrpc: JSONRPC_VERSION, - id, - result, - } - } - - #[allow(clippy::expect_used)] - pub fn to_json(&self) -> String { - serde_json::to_string(self).expect("JsonRpcResponse serialization should never fail") - } -} - -#[derive(Debug, Serialize)] -pub struct JsonRpcError { - pub jsonrpc: &'static str, - pub id: Value, - pub error: ErrorObject, -} - -impl JsonRpcError { - pub fn new(id: Value, code: i32, message: impl Into) -> Self { - Self { - jsonrpc: JSONRPC_VERSION, - id, - error: ErrorObject { - code, - message: message.into(), - }, - } - } - - pub fn parse_error() -> Self { - Self::new(Value::Null, PARSE_ERROR, "Parse error") - } - - pub fn method_not_found(id: Value, method: &str) -> Self { - Self::new(id, METHOD_NOT_FOUND, format!("Method not found: {method}")) - } - - pub fn invalid_params(id: Value, message: impl Into) -> Self { - Self::new(id, INVALID_PARAMS, message) - } - - #[allow(clippy::expect_used)] - pub fn to_json(&self) -> String { - serde_json::to_string(self).expect("JsonRpcError serialization should never fail") - } -} - -#[derive(Debug, Serialize)] -pub struct ErrorObject { - pub code: i32, - pub message: String, -} - -// MCP Protocol Types -#[derive(Debug, Serialize)] -pub struct ServerInfo { - pub name: String, - pub version: String, -} - -#[derive(Debug, Serialize)] -pub struct ServerCapabilities { - pub resources: ResourceCapabilities, - pub tools: ToolCapabilities, -} - -#[derive(Debug, Serialize)] -pub struct ResourceCapabilities {} - -#[derive(Debug, Serialize)] -pub struct ToolCapabilities {} - -#[derive(Debug, Serialize, Clone)] -pub struct Resource { - pub uri: String, - pub name: String, - pub description: String, - #[serde(rename = "mimeType")] - pub mime_type: String, -} - -#[derive(Debug, Serialize)] -pub struct ResourceContent { - pub uri: String, - #[serde(rename = "mimeType")] - pub mime_type: String, - pub text: String, -} - -#[derive(Debug, Serialize, Clone)] -pub struct Tool { - pub name: String, - pub description: String, - #[serde(rename = "inputSchema")] - pub input_schema: Value, -} - -#[derive(Debug, Serialize)] -pub struct ToolResult { - pub content: Vec, - #[serde(rename = "isError")] - pub is_error: bool, - #[serde(rename = "structuredContent", skip_serializing_if = "Option::is_none")] - pub structured_content: Option, -} - -impl ToolResult { - pub fn success(text: String) -> Self { - Self { - content: vec![ContentBlock::text(text)], - is_error: false, - structured_content: None, - } - } - - pub fn error(text: String) -> Self { - Self { - content: vec![ContentBlock::text(text)], - is_error: true, - structured_content: None, - } - } - - pub fn structured(value: Value) -> Self { - let text_fallback = serde_json::to_string_pretty(&value).unwrap_or_default(); - Self { - content: vec![ContentBlock::text(text_fallback)], - is_error: false, - structured_content: Some(value), - } - } - - pub fn structured_error(value: Value) -> Self { - let text_fallback = serde_json::to_string_pretty(&value).unwrap_or_default(); - Self { - content: vec![ContentBlock::text(text_fallback)], - is_error: true, - structured_content: Some(value), - } - } - - pub fn from_serializable(value: &impl serde::Serialize) -> Self { - match serde_json::to_value(value) { - Ok(v) => Self::structured(v), - Err(e) => Self::error(format!("Failed to serialize response: {e}")), - } - } -} - -#[derive(Debug, Serialize, Clone)] -#[serde(tag = "type")] -pub enum ContentBlock { - #[serde(rename = "text")] - Text { text: String }, -} - -impl ContentBlock { - pub fn text(text: String) -> Self { - Self::Text { text } - } -} - -use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; - -pub type ToolHandler = - Arc, Value) -> BoxFuture<'static, ToolResult> + Send + Sync>; -pub type ResourceHandler = - Arc) -> BoxFuture<'static, Result> + Send + Sync>; - -pub struct ToolDef { - pub description: String, - pub input_schema: Value, - pub handler: ToolHandler, -} - -impl std::fmt::Debug for ToolDef { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ToolDef") - .field("description", &self.description) - .finish_non_exhaustive() - } -} - -pub struct ResourceDef { - pub name: String, - pub description: String, - pub mime_type: String, - pub handler: ResourceHandler, -} - -impl std::fmt::Debug for ResourceDef { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ResourceDef") - .field("name", &self.name) - .field("description", &self.description) - .finish_non_exhaustive() - } -} - -#[derive(Debug)] -pub struct McpServer { - pub ctx: Arc, - pub tools: HashMap>, - pub resources: HashMap>, -} - -impl McpServer { - pub fn new(ctx: C) -> Self { - Self { - ctx: Arc::new(ctx), - tools: HashMap::new(), - resources: HashMap::new(), - } - } - - pub fn tool(mut self, name: &str, description: &str, handler: F) -> Self - where - A: DeserializeOwned + schemars::JsonSchema + Send + 'static, - F: Fn(Arc, A) -> Fut + Send + Sync + 'static, - Fut: Future + Send + 'static, - { - let schema = schema_for!(A); - #[allow(clippy::expect_used)] - let input_schema = - serde_json::to_value(&schema).expect("Tool input schema should serialize"); - let handler = Arc::new(handler); - let handler = Arc::new( - move |ctx: Arc, arguments: Value| -> BoxFuture<'static, ToolResult> { - let handler = Arc::clone(&handler); - Box::pin(async move { - let parsed: Result = serde_json::from_value(arguments); - match parsed { - Ok(args) => handler(ctx, args).await, - Err(err) => ToolResult::error(format!("Invalid tool arguments: {err}")), - } - }) - }, - ); - - self.tools.insert( - name.to_string(), - ToolDef { - description: description.to_string(), - input_schema, - handler, - }, - ); - self - } - - pub fn resource( - mut self, - uri: &str, - name: &str, - description: &str, - mime_type: &str, - handler: F, - ) -> Self - where - F: Fn(Arc) -> Fut + Send + Sync + 'static, - Fut: Future> + Send + 'static, - { - let handler = Arc::new(handler); - let handler = Arc::new( - move |ctx: Arc| -> BoxFuture<'static, Result> { - let handler = Arc::clone(&handler); - Box::pin(async move { handler(ctx).await }) - }, - ); - self.resources.insert( - uri.to_string(), - ResourceDef { - name: name.to_string(), - description: description.to_string(), - mime_type: mime_type.to_string(), - handler, - }, - ); - self - } - - pub fn list_tools(&self) -> Vec { - self.tools - .iter() - .map(|(name, def)| Tool { - name: name.clone(), - description: def.description.clone(), - input_schema: def.input_schema.clone(), - }) - .collect() - } - - pub fn list_resources(&self) -> Vec { - self.resources - .iter() - .map(|(uri, def)| Resource { - uri: uri.clone(), - name: def.name.clone(), - description: def.description.clone(), - mime_type: def.mime_type.clone(), - }) - .collect() - } - - pub async fn call_tool(&self, name: &str, arguments: Value) -> Result { - let tool = self - .tools - .get(name) - .ok_or_else(|| format!("Unknown tool: {name}"))?; - Ok((tool.handler)(Arc::clone(&self.ctx), arguments).await) - } - - pub async fn read_resource(&self, uri: &str) -> Result { - let resource = self - .resources - .get(uri) - .ok_or_else(|| format!("Resource not found: {uri}"))?; - let text = (resource.handler)(Arc::clone(&self.ctx)).await?; - Ok(ResourceContent { - uri: uri.to_string(), - mime_type: resource.mime_type.clone(), - text, - }) - } - - pub fn initialize_result( - &self, - protocol_version: &str, - server_name: &str, - server_version: &str, - ) -> Value { - json!({ - "protocolVersion": protocol_version, - "capabilities": ServerCapabilities { - resources: ResourceCapabilities {}, - tools: ToolCapabilities {}, - }, - "serverInfo": ServerInfo { - name: server_name.to_string(), - version: server_version.to_string(), - } - }) - } - - pub async fn run_stdio(self, shutdown_tx: broadcast::Sender<()>) -> Result<(), String> { - let stdin = tokio::io::stdin(); - let mut stdout = tokio::io::stdout(); - let mut reader = BufReader::new(stdin); - let mut line = String::new(); - - loop { - line.clear(); - match reader.read_line(&mut line).await { - Ok(0) => { - // EOF - signal shutdown to background tasks - let _ = shutdown_tx.send(()); - break; - } - Ok(_) => { - let trimmed = line.trim(); - if trimmed.is_empty() { - continue; - } - - if let Some(response) = self.handle_message(trimmed).await { - self.write_response(&mut stdout, &response).await?; - } - } - Err(e) => { - // Error - signal shutdown to background tasks - let _ = shutdown_tx.send(()); - return Err(format!("Error reading from stdin: {e}")); - } - } - } - - Ok(()) - } - - async fn write_response( - &self, - stdout: &mut tokio::io::Stdout, - response: &str, - ) -> Result<(), String> { - stdout - .write_all(response.as_bytes()) - .await - .map_err(|e| format!("Failed to write response: {e}"))?; - stdout - .write_all(b"\n") - .await - .map_err(|e| format!("Failed to write newline: {e}"))?; - stdout - .flush() - .await - .map_err(|e| format!("Failed to flush stdout: {e}"))?; - Ok(()) - } - - async fn handle_message(&self, msg: &str) -> Option { - let request: JsonRpcRequest = match serde_json::from_str(msg) { - Ok(req) => req, - Err(_) => return Some(JsonRpcError::parse_error().to_json()), - }; - - if request.validate().is_err() { - return Some(JsonRpcError::parse_error().to_json()); - } - - // Notifications (no id) MUST NOT receive a response per JSON-RPC spec - let id = request.id?; - let params = request.params.unwrap_or(Value::Null); - - let response = match request.method.as_str() { - "initialize" => JsonRpcResponse::new( - id, - self.initialize_result(PROTOCOL_VERSION, "apx", env!("CARGO_PKG_VERSION")), - ) - .to_json(), - "ping" => JsonRpcResponse::new(id, json!({})).to_json(), - "resources/list" => { - JsonRpcResponse::new(id, json!({ "resources": self.list_resources() })).to_json() - } - "resources/read" => self.handle_resources_read(id, params).await, - "tools/list" => { - JsonRpcResponse::new(id, json!({ "tools": self.list_tools() })).to_json() - } - "tools/call" => self.handle_tools_call(id, params).await, - method => JsonRpcError::method_not_found(id, method).to_json(), - }; - - Some(response) - } - - async fn handle_resources_read(&self, id: Value, params: Value) -> String { - let uri = match params.get("uri").and_then(|v| v.as_str()) { - Some(u) => u, - None => return JsonRpcError::invalid_params(id, "Missing 'uri' parameter").to_json(), - }; - - match self.read_resource(uri).await { - Ok(content) => JsonRpcResponse::new(id, json!({ "contents": vec![content] })).to_json(), - Err(err) => JsonRpcError::invalid_params(id, err).to_json(), - } - } - - async fn handle_tools_call(&self, id: Value, params: Value) -> String { - let tool_name = match params.get("name").and_then(|v| v.as_str()) { - Some(n) => n, - None => return JsonRpcError::invalid_params(id, "Missing 'name' parameter").to_json(), - }; - - let arguments = params.get("arguments").cloned().unwrap_or(Value::Null); - - match self.call_tool(tool_name, arguments).await { - Ok(result) => { - #[allow(clippy::expect_used)] - let value = serde_json::to_value(result) - .expect("ToolResult serialization should never fail"); - JsonRpcResponse::new(id, value).to_json() - } - Err(err) => JsonRpcError::invalid_params(id, err).to_json(), - } - } -} - -#[cfg(test)] -#[allow(clippy::unwrap_used)] -mod tests { - use super::*; - - #[test] - fn content_block_text_serialization() { - let block = ContentBlock::text("hello".to_string()); - let json = serde_json::to_value(&block).unwrap(); - assert_eq!(json, json!({"type": "text", "text": "hello"})); - } - - #[test] - fn tool_result_success_serialization() { - let result = ToolResult::success("ok".to_string()); - let json = serde_json::to_value(&result).unwrap(); - assert_eq!(json["isError"], false); - assert_eq!(json["content"][0]["type"], "text"); - assert_eq!(json["content"][0]["text"], "ok"); - assert!(json.get("structuredContent").is_none()); - } - - #[test] - fn tool_result_error_serialization() { - let result = ToolResult::error("bad".to_string()); - let json = serde_json::to_value(&result).unwrap(); - assert_eq!(json["isError"], true); - assert_eq!(json["content"][0]["text"], "bad"); - assert!(json.get("structuredContent").is_none()); - } - - #[test] - fn tool_result_structured_serialization() { - let data = json!({"count": 3, "items": [1, 2, 3]}); - let result = ToolResult::structured(data.clone()); - let json = serde_json::to_value(&result).unwrap(); - assert_eq!(json["isError"], false); - assert_eq!(json["structuredContent"], data); - // text fallback should contain the JSON - let text = json["content"][0]["text"].as_str().unwrap(); - assert!(text.contains("\"count\": 3")); - } - - #[test] - fn tool_result_structured_error_serialization() { - let data = json!({"status": "failed", "errors": "something broke"}); - let result = ToolResult::structured_error(data.clone()); - let json = serde_json::to_value(&result).unwrap(); - assert_eq!(json["isError"], true); - assert_eq!(json["structuredContent"], data); - } - - #[test] - fn tool_result_from_serializable_success() { - #[derive(Serialize)] - struct Resp { - count: i32, - } - let result = ToolResult::from_serializable(&Resp { count: 42 }); - let json = serde_json::to_value(&result).unwrap(); - assert_eq!(json["isError"], false); - assert_eq!(json["structuredContent"]["count"], 42); - } - - fn make_server() -> McpServer<()> { - McpServer::new(()) - } - - #[tokio::test] - async fn handle_ping() { - let server = make_server(); - let msg = r#"{"jsonrpc":"2.0","id":1,"method":"ping"}"#; - let response = server.handle_message(msg).await.unwrap(); - let parsed: Value = serde_json::from_str(&response).unwrap(); - assert_eq!(parsed["result"], json!({})); - assert_eq!(parsed["id"], 1); - } - - #[tokio::test] - async fn handle_notification_returns_none() { - let server = make_server(); - // No "id" field = notification - let msg = r#"{"jsonrpc":"2.0","method":"notifications/initialized"}"#; - assert!(server.handle_message(msg).await.is_none()); - - let msg = r#"{"jsonrpc":"2.0","method":"notifications/cancelled"}"#; - assert!(server.handle_message(msg).await.is_none()); - - let msg = r#"{"jsonrpc":"2.0","method":"some/custom/notification"}"#; - assert!(server.handle_message(msg).await.is_none()); - } - - #[tokio::test] - async fn handle_unknown_method_returns_error() { - let server = make_server(); - let msg = r#"{"jsonrpc":"2.0","id":42,"method":"nonexistent/method"}"#; - let response = server.handle_message(msg).await.unwrap(); - let parsed: Value = serde_json::from_str(&response).unwrap(); - assert_eq!(parsed["error"]["code"], METHOD_NOT_FOUND); - assert!( - parsed["error"]["message"] - .as_str() - .unwrap() - .contains("nonexistent/method") - ); - } - - #[tokio::test] - async fn handle_invalid_json_returns_parse_error() { - let server = make_server(); - let response = server.handle_message("not json").await.unwrap(); - let parsed: Value = serde_json::from_str(&response).unwrap(); - assert_eq!(parsed["error"]["code"], PARSE_ERROR); - } - - #[test] - fn protocol_version_is_updated() { - assert_eq!(PROTOCOL_VERSION, "2025-11-25"); - } -} diff --git a/crates/mcp/src/indexing.rs b/crates/mcp/src/indexing.rs new file mode 100644 index 00000000..344d565b --- /dev/null +++ b/crates/mcp/src/indexing.rs @@ -0,0 +1,213 @@ +use crate::context::{AppContext, SdkIndexParams}; +use apx_core::databricks_sdk_doc::SDKSource; +use apx_core::search::ComponentIndex; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; +use tokio::sync::Notify; +use tokio::sync::broadcast; + +/// Initialize all indexes in background (component index, then SDK docs index) +/// +/// All database operations use synchronous SQLite calls wrapped in spawn_blocking. +/// +/// This is called when the MCP server starts. +pub fn init_all_indexes( + ctx: &AppContext, + mut shutdown_rx: broadcast::Receiver<()>, + sdk_params: Option, +) { + let cache_state = ctx.cache_state.clone(); + let index_state = ctx.index_state.clone(); + + // Check for legacy LanceDB directory + apx_core::search::common::check_legacy_lancedb(); + + tokio::spawn(async move { + // Mark as running + { + let mut guard = cache_state.lock().await; + guard.is_running = true; + } + + // ============================================ + // Phase 1: Component Index (ensure exists, skip project-specific sync) + // ============================================ + tracing::info!("Ensuring component search index exists on MCP start"); + + let ensure_result = tokio::select! { + result = tokio::task::spawn_blocking(ensure_search_index) => { + Some(result.unwrap_or_else(|e| Err(format!("spawn_blocking panicked: {e}")))) + }, + _ = shutdown_rx.recv() => { + tracing::info!("Shutdown signal received during search index check, stopping"); + None + } + }; + + if let Some(Err(e)) = ensure_result { + tracing::warn!("Failed to ensure search index: {}", e); + } + + // Mark component indexing as complete + index_state.component_indexed.store(true, Ordering::SeqCst); + index_state.component_ready.notify_waiters(); + tracing::debug!("Component index ready"); + + // ============================================ + // Phase 2: SDK Docs Index (after component index) + // ============================================ + if let Some(params) = sdk_params { + tracing::info!("Initializing Databricks SDK documentation index"); + + let version = match params.sdk_version { + Some(v) => { + tracing::debug!("Using pre-computed SDK version: {}", v); + v + } + None => { + tracing::warn!( + "Databricks SDK not installed. The docs tool will not be available." + ); + index_state.sdk_indexed.store(true, Ordering::SeqCst); + index_state.sdk_ready.notify_waiters(); + + // Mark as done + let mut guard = cache_state.lock().await; + guard.is_running = false; + return; + } + }; + + // Create SDK docs index (sync, but cheap) + let mut index = match apx_core::search::docs_index::SDKDocsIndex::new() { + Ok(idx) => { + tracing::debug!("SDKDocsIndex created successfully"); + idx + } + Err(e) => { + tracing::warn!( + "Failed to initialize SDK doc index: {}. The docs tool will not be available.", + e + ); + index_state.sdk_indexed.store(true, Ordering::SeqCst); + index_state.sdk_ready.notify_waiters(); + + let mut guard = cache_state.lock().await; + guard.is_running = false; + return; + } + }; + + // Bootstrap the index (async: download + sync: build) + tracing::info!("Bootstrapping SDK docs (this may download SDK if not cached)"); + let bootstrap_start = std::time::Instant::now(); + let bootstrap_result = tokio::select! { + result = index.bootstrap_with_version(&SDKSource::DatabricksSdkPython, &version) => Some(result), + _ = shutdown_rx.recv() => { + tracing::info!("Shutdown signal received during SDK doc bootstrapping"); + None + } + }; + tracing::debug!("SDK bootstrap completed in {:?}", bootstrap_start.elapsed()); + + match bootstrap_result { + Some(Ok(true)) => { + tracing::info!("SDK docs indexed successfully"); + *params.sdk_doc_index.lock().await = Some(index); + } + Some(Ok(false)) => { + tracing::info!("SDK docs already indexed"); + *params.sdk_doc_index.lock().await = Some(index); + } + Some(Err(e)) => { + tracing::warn!( + "Failed to bootstrap SDK docs: {}. The docs tool will not be available.", + e + ); + } + None => { + tracing::debug!("Shutdown during SDK bootstrap"); + } + } + + // Mark SDK indexing as complete + index_state.sdk_indexed.store(true, Ordering::SeqCst); + index_state.sdk_ready.notify_waiters(); + tracing::debug!("SDK doc index ready"); + } else { + // No SDK params, mark as ready immediately + index_state.sdk_indexed.store(true, Ordering::SeqCst); + index_state.sdk_ready.notify_waiters(); + } + + // Mark as done + { + let mut guard = cache_state.lock().await; + guard.is_running = false; + } + }); +} + +/// Rebuild the search index from registry.json files (sync) +pub fn rebuild_search_index() -> Result<(), String> { + let index = ComponentIndex::new()?; + index.build_index_from_registries() +} + +/// Ensure search index exists and is valid, build/rebuild if needed (sync) +fn ensure_search_index() -> Result<(), String> { + let index = ComponentIndex::new()?; + + match index.validate_index() { + Ok(true) => { + tracing::debug!("Search index validated successfully"); + Ok(()) + } + Ok(false) => { + tracing::info!("Search index not found, building from registry indexes"); + index.build_index_from_registries() + } + Err(e) => { + tracing::warn!("Search index corrupted ({}), rebuilding...", e); + index.build_index_from_registries() + } + } +} + +/// Wait for an index to be ready with timeout (15 seconds) +pub async fn wait_for_index_ready( + ready_notify: &Notify, + is_ready: &AtomicBool, + index_name: &str, +) -> Result<(), String> { + const TIMEOUT_SECS: u64 = 15; + + // Check if already ready + if is_ready.load(Ordering::SeqCst) { + return Ok(()); + } + + tracing::debug!( + "Waiting up to {}s for {} index to be ready", + TIMEOUT_SECS, + index_name + ); + + // Wait with timeout + match tokio::time::timeout(Duration::from_secs(TIMEOUT_SECS), ready_notify.notified()).await { + Ok(_) => { + tracing::debug!("{} index is now ready", index_name); + Ok(()) + } + Err(_) => { + tracing::warn!( + "{} index not ready after {}s timeout", + index_name, + TIMEOUT_SECS + ); + Err(format!( + "{index_name} index is not yet ready, please rerun the query in 5 seconds" + )) + } + } +} diff --git a/crates/mcp/src/info_content.rs b/crates/mcp/src/info_content.rs new file mode 100644 index 00000000..3699598e --- /dev/null +++ b/crates/mcp/src/info_content.rs @@ -0,0 +1,15 @@ +pub const APX_INFO_CONTENT: &str = r#" +This project uses apx toolkit to build a Databricks app. +apx bundles together a set of tools and libraries to help you with the complete app development lifecycle: develop, build and deploy. + +## Technology Stack + +- **Backend**: Python + FastAPI + Pydantic +- **Frontend**: React + TypeScript + shadcn/ui +- **Build Tools**: uv (Python), bun (JavaScript/TypeScript) + +## Tool Usage + +All project-scoped tools require an `app_path` parameter โ€” the absolute path to the project directory. +Global tools (like `docs`) do not require `app_path`. +"#; diff --git a/crates/mcp/src/lib.rs b/crates/mcp/src/lib.rs index 76970da1..917bffcd 100644 --- a/crates/mcp/src/lib.rs +++ b/crates/mcp/src/lib.rs @@ -9,5 +9,10 @@ clippy::dbg_macro )] -pub mod core; +pub mod context; +pub mod indexing; +pub mod info_content; +pub mod resources; pub mod server; +pub mod tools; +pub mod validation; diff --git a/crates/mcp/src/resources.rs b/crates/mcp/src/resources.rs new file mode 100644 index 00000000..9f3ea2fc --- /dev/null +++ b/crates/mcp/src/resources.rs @@ -0,0 +1,18 @@ +use crate::info_content::APX_INFO_CONTENT; +use rmcp::model::*; + +pub fn list_resources() -> Vec { + let mut raw = RawResource::new("apx://info", "apx-info".to_string()); + raw.description = Some("Information about apx toolkit".to_string()); + raw.mime_type = Some("text/plain".to_string()); + vec![raw.no_annotation()] +} + +pub fn read_resource(uri: &str) -> Result { + match uri { + "apx://info" => Ok(ReadResourceResult { + contents: vec![ResourceContents::text(APX_INFO_CONTENT, uri)], + }), + _ => Err(format!("Resource not found: {uri}")), + } +} diff --git a/crates/mcp/src/server.rs b/crates/mcp/src/server.rs index 0cec61f0..b3085dc4 100644 --- a/crates/mcp/src/server.rs +++ b/crates/mcp/src/server.rs @@ -1,1400 +1,272 @@ -use crate::core::{McpServer, ToolResult}; -use apx_core::common::read_project_metadata; -use apx_core::components::{SharedCacheState, needs_registry_refresh, sync_registry_indexes}; -use apx_core::databricks_sdk_doc::SDKSource; -use apx_core::dotenv::DotenvFile; -use apx_core::interop::generate_openapi_spec; -use apx_core::search::ComponentIndex; -use apx_core::search::docs_index::SDKDocsIndex; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use std::collections::{HashMap, HashSet}; -use std::path::{Path, PathBuf}; -use std::process::Stdio; +use crate::context::{AppContext, SdkIndexParams}; +use crate::indexing::init_all_indexes; +use crate::info_content::APX_INFO_CONTENT; +use crate::tools::AppPathArgs; +use crate::tools::databricks::DatabricksAppsLogsArgs; +use crate::tools::devserver::LogsToolArgs; +use crate::tools::docs::DocsArgs; +use crate::tools::project::GetRouteInfoArgs; +use crate::tools::registry::{AddComponentArgs, SearchRegistryComponentsArgs}; +use rmcp::handler::server::router::tool::ToolRouter; +use rmcp::handler::server::wrapper::Parameters; +use rmcp::model::*; +use rmcp::{RoleServer, ServerHandler, service::RequestContext, tool, tool_handler, tool_router}; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::{Duration, Instant}; -use tokio::process::Command; -use tokio::sync::{Mutex, Notify, broadcast}; -/// Parameters for SDK indexing, pre-computed synchronously to avoid Python GIL issues -#[derive(Debug)] -pub struct SdkIndexParams { - pub sdk_version: Option, - pub sdk_doc_index: Arc>>, +#[derive(Clone)] +pub struct ApxServer { + pub ctx: Arc, + tool_router: ToolRouter, } -/// State for tracking index readiness -#[derive(Clone, Debug)] -pub struct IndexState { - /// Notifies waiters when component index is ready - pub component_ready: Arc, - /// Notifies waiters when SDK docs index is ready - pub sdk_ready: Arc, - /// Whether component indexing has completed (for late subscribers) - pub component_indexed: Arc, - /// Whether SDK indexing has completed (for late subscribers) - pub sdk_indexed: Arc, -} - -impl Default for IndexState { - fn default() -> Self { - Self { - component_ready: Arc::new(Notify::new()), - sdk_ready: Arc::new(Notify::new()), - component_indexed: Arc::new(AtomicBool::new(false)), - sdk_indexed: Arc::new(AtomicBool::new(false)), - } - } -} - -impl IndexState { - pub fn new() -> Self { - Self::default() +impl std::fmt::Debug for ApxServer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ApxServer").finish_non_exhaustive() } } -#[derive(Debug)] -pub struct AppContext { - pub app_dir: PathBuf, - pub sdk_doc_index: Arc>>, - pub cache_state: SharedCacheState, - pub index_state: IndexState, - pub shutdown_tx: broadcast::Sender<()>, -} - -/// Initialize all indexes in background (component index, then SDK docs index) -/// -/// All database operations use synchronous SQLite calls wrapped in spawn_blocking. -/// -/// This is called when the MCP server starts. -pub fn init_all_indexes( - ctx: &AppContext, - mut shutdown_rx: broadcast::Receiver<()>, - sdk_params: Option, -) { - let app_dir = ctx.app_dir.clone(); - let cache_state = ctx.cache_state.clone(); - let index_state = ctx.index_state.clone(); +// All tool definitions in a single #[tool_router] block. +// Heavy logic is delegated to handler methods in tools/*.rs modules. +#[tool_router] +impl ApxServer { + pub fn new(ctx: AppContext, sdk_params: Option) -> Self { + // Initialize all indexes in background + let shutdown_rx = ctx.shutdown_tx.subscribe(); + init_all_indexes(&ctx, shutdown_rx, sdk_params); - // Check for legacy LanceDB directory - apx_core::search::common::check_legacy_lancedb(); - - tokio::spawn(async move { - // Mark as running - { - let mut guard = cache_state.lock().await; - guard.is_running = true; - } - - // ============================================ - // Phase 1: Component Index - // ============================================ - tracing::info!("Syncing registry indexes on MCP start"); - - // Check for shutdown during sync - let sync_result = tokio::select! { - result = sync_registry_indexes(&app_dir, false) => Some(result), - _ = shutdown_rx.recv() => { - tracing::info!("Shutdown signal received during registry sync, stopping"); - None - } - }; - - match sync_result { - Some(Ok(refreshed)) => { - if refreshed { - tracing::info!("Registry indexes refreshed, rebuilding search index"); - - let rebuild_result = tokio::select! { - result = tokio::task::spawn_blocking(rebuild_search_index) => { - Some(result.unwrap_or_else(|e| Err(format!("spawn_blocking panicked: {e}")))) - }, - _ = shutdown_rx.recv() => { - tracing::info!("Shutdown signal received during search index rebuild, stopping"); - None - } - }; - - if let Some(Err(e)) = rebuild_result { - tracing::warn!("Failed to rebuild search index: {}", e); - } - } else { - // Check if search index exists, build if not - let ensure_result = tokio::select! { - result = tokio::task::spawn_blocking(ensure_search_index) => { - Some(result.unwrap_or_else(|e| Err(format!("spawn_blocking panicked: {e}")))) - }, - _ = shutdown_rx.recv() => { - tracing::info!("Shutdown signal received during search index check, stopping"); - None - } - }; - - if let Some(Err(e)) = ensure_result { - tracing::warn!("Failed to ensure search index: {}", e); - } - } - } - Some(Err(e)) => tracing::warn!("Failed to sync registry indexes: {}", e), - None => { - // Shutdown was signaled during sync - } - } - - // Mark component indexing as complete - index_state.component_indexed.store(true, Ordering::SeqCst); - index_state.component_ready.notify_waiters(); - tracing::debug!("Component index ready"); - - // ============================================ - // Phase 2: SDK Docs Index (after component index) - // ============================================ - if let Some(params) = sdk_params { - tracing::info!("Initializing Databricks SDK documentation index"); - - let version = match params.sdk_version { - Some(v) => { - tracing::debug!("Using pre-computed SDK version: {}", v); - v - } - None => { - tracing::warn!( - "Databricks SDK not installed. The docs tool will not be available." - ); - index_state.sdk_indexed.store(true, Ordering::SeqCst); - index_state.sdk_ready.notify_waiters(); - - // Mark as done - let mut guard = cache_state.lock().await; - guard.is_running = false; - return; - } - }; - - // Create SDK docs index (sync, but cheap) - let mut index = match SDKDocsIndex::new() { - Ok(idx) => { - tracing::debug!("SDKDocsIndex created successfully"); - idx - } - Err(e) => { - tracing::warn!( - "Failed to initialize SDK doc index: {}. The docs tool will not be available.", - e - ); - index_state.sdk_indexed.store(true, Ordering::SeqCst); - index_state.sdk_ready.notify_waiters(); - - let mut guard = cache_state.lock().await; - guard.is_running = false; - return; - } - }; - - // Bootstrap the index (async: download + sync: build) - tracing::info!("Bootstrapping SDK docs (this may download SDK if not cached)"); - let bootstrap_start = std::time::Instant::now(); - let bootstrap_result = tokio::select! { - result = index.bootstrap_with_version(&SDKSource::DatabricksSdkPython, &version) => Some(result), - _ = shutdown_rx.recv() => { - tracing::info!("Shutdown signal received during SDK doc bootstrapping"); - None - } - }; - tracing::debug!("SDK bootstrap completed in {:?}", bootstrap_start.elapsed()); - - match bootstrap_result { - Some(Ok(true)) => { - tracing::info!("SDK docs indexed successfully"); - *params.sdk_doc_index.lock().await = Some(index); - } - Some(Ok(false)) => { - tracing::info!("SDK docs already indexed"); - *params.sdk_doc_index.lock().await = Some(index); - } - Some(Err(e)) => { - tracing::warn!( - "Failed to bootstrap SDK docs: {}. The docs tool will not be available.", - e - ); - } - None => { - tracing::debug!("Shutdown during SDK bootstrap"); - } - } - - // Mark SDK indexing as complete - index_state.sdk_indexed.store(true, Ordering::SeqCst); - index_state.sdk_ready.notify_waiters(); - tracing::debug!("SDK doc index ready"); - } else { - // No SDK params, mark as ready immediately - index_state.sdk_indexed.store(true, Ordering::SeqCst); - index_state.sdk_ready.notify_waiters(); - } - - // Mark as done - { - let mut guard = cache_state.lock().await; - guard.is_running = false; - } - }); -} - -/// Rebuild the search index from registry.json files (sync) -fn rebuild_search_index() -> Result<(), String> { - let index = ComponentIndex::new()?; - index.build_index_from_registries() -} - -/// Ensure search index exists and is valid, build/rebuild if needed (sync) -fn ensure_search_index() -> Result<(), String> { - let index = ComponentIndex::new()?; - - match index.validate_index() { - Ok(true) => { - tracing::debug!("Search index validated successfully"); - Ok(()) - } - Ok(false) => { - tracing::info!("Search index not found, building from registry indexes"); - index.build_index_from_registries() - } - Err(e) => { - tracing::warn!("Search index corrupted ({}), rebuilding...", e); - index.build_index_from_registries() + Self { + ctx: Arc::new(ctx), + tool_router: Self::tool_router(), } } -} -/// Wait for an index to be ready with timeout (15 seconds) -async fn wait_for_index_ready( - ready_notify: &Notify, - is_ready: &AtomicBool, - index_name: &str, -) -> Result<(), String> { - const TIMEOUT_SECS: u64 = 15; + // --- Dev server tools --- - // Check if already ready - if is_ready.load(Ordering::SeqCst) { - return Ok(()); - } - - tracing::debug!( - "Waiting up to {}s for {} index to be ready", - TIMEOUT_SECS, - index_name - ); - - // Wait with timeout - match tokio::time::timeout(Duration::from_secs(TIMEOUT_SECS), ready_notify.notified()).await { - Ok(_) => { - tracing::debug!("{} index is now ready", index_name); - Ok(()) - } - Err(_) => { - tracing::warn!( - "{} index not ready after {}s timeout", - index_name, - TIMEOUT_SECS - ); - Err(format!( - "{index_name} index is not yet ready, please rerun the query in 5 seconds" - )) - } + #[tool( + name = "start", + description = "Start development server and return the URL", + annotations(destructive_hint = true, read_only_hint = false) + )] + async fn start( + &self, + Parameters(args): Parameters, + ) -> Result { + self.handle_start(args).await } -} - -pub fn build_server(ctx: AppContext, sdk_params: Option) -> McpServer { - // Initialize all indexes in background (component + SDK docs, sequentially) - let shutdown_rx = ctx.shutdown_tx.subscribe(); - init_all_indexes(&ctx, shutdown_rx, sdk_params); - McpServer::new(ctx) - .resource( - "apx://info", - "apx-info", - "Information about apx toolkit", - "text/plain", - apx_info_resource, - ) - .resource( - "apx://routes", - "api-routes", - "List of API routes from OpenAPI schema", - "application/json", - routes_resource, - ) - .tool( - "start", - "Start development server and return the URL", - start_tool, - ) - .tool( - "stop", - "Stop the development server", - stop_tool, - ) - .tool( - "restart", - "Restart the development server (preserves port if possible)", - restart_tool, - ) - .tool( - "logs", - "Fetch recent dev server logs", - logs_tool, - ) - .tool( - "refresh_openapi", - "Regenerate OpenAPI schema and API client", - refresh_openapi_tool, - ) - .tool( - "check", - "Check the project code for errors (runs tsc and ty checks in parallel)", - check_tool, + #[tool( + name = "stop", + description = "Stop the development server", + annotations( + destructive_hint = true, + read_only_hint = false, + idempotent_hint = true ) - .tool( - "databricks_apps_logs", - "Fetch Databricks Apps logs from an already deployed app using the Databricks CLI", - databricks_apps_logs_tool, + )] + async fn stop( + &self, + Parameters(args): Parameters, + ) -> Result { + self.handle_stop(args).await + } + + #[tool( + name = "restart", + description = "Restart the development server (preserves port if possible)", + annotations(destructive_hint = true, read_only_hint = false) + )] + async fn restart( + &self, + Parameters(args): Parameters, + ) -> Result { + self.handle_restart(args).await + } + + #[tool( + name = "logs", + description = "Fetch recent dev server logs", + annotations(read_only_hint = true) + )] + async fn logs( + &self, + Parameters(args): Parameters, + ) -> Result { + self.handle_logs(args).await + } + + // --- Project tools --- + + #[tool( + name = "check", + description = "Check the project code for errors (runs tsc and ty checks in parallel)", + annotations(read_only_hint = true) + )] + async fn check( + &self, + Parameters(args): Parameters, + ) -> Result { + self.handle_check(args).await + } + + #[tool( + name = "refresh_openapi", + description = "Regenerate OpenAPI schema and API client", + annotations( + destructive_hint = true, + read_only_hint = false, + idempotent_hint = true ) - .tool( - "search_registry_components", - "Search shadcn registry components using semantic search. Supports filtering by category, type, and registry.", - search_registry_components_tool, - ) - .tool( - "add_component", - "Add a component to the project. Component ID can be 'component-name' (from default registry) or '@registry-name/component-name'.", - add_component_tool, - ) - .tool( - "docs", - "Search Databricks SDK documentation for relevant code examples and API references", - docs_tool, - ) - .tool( - "get_route_info", - "Get code example for using a specific API route", - get_route_info_tool, - ) -} - -// --- Resources --- - -async fn apx_info_resource(_ctx: Arc) -> Result { - Ok(APX_INFO_CONTENT.to_string()) -} - -#[derive(Serialize)] -struct RouteInfo { - id: String, - method: String, - path: String, - description: String, -} - -async fn routes_resource(ctx: Arc) -> Result { - let metadata = read_project_metadata(&ctx.app_dir)?; - let (openapi_content, _) = - generate_openapi_spec(&ctx.app_dir, &metadata.app_entrypoint, &metadata.app_slug)?; - - let openapi: Value = serde_json::from_str(&openapi_content) - .map_err(|e| format!("Failed to parse OpenAPI schema: {e}"))?; - - let routes = parse_openapi_operations(&openapi)?; - - serde_json::to_string_pretty(&routes).map_err(|e| format!("Failed to serialize routes: {e}")) -} - -fn parse_openapi_operations(openapi: &Value) -> Result, String> { - let mut routes = Vec::new(); - - let paths = openapi - .get("paths") - .and_then(|p| p.as_object()) - .ok_or_else(|| "OpenAPI schema missing 'paths' object".to_string())?; - - for (path, path_item) in paths { - let methods_obj = path_item - .as_object() - .ok_or_else(|| format!("Path '{path}' is not an object"))?; - - for (method, operation) in methods_obj { - // Skip non-HTTP method keys like "parameters", "summary", etc. - let method_upper = method.to_uppercase(); - if !matches!( - method_upper.as_str(), - "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS" - ) { - continue; - } - - let operation_obj = operation - .as_object() - .ok_or_else(|| format!("Operation '{method}' at path '{path}' is not an object"))?; - - let operation_id = operation_obj - .get("operationId") - .and_then(|v| v.as_str()) - .unwrap_or("unknown") - .to_string(); - - let description = operation_obj - .get("summary") - .or_else(|| operation_obj.get("description")) - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - - routes.push(RouteInfo { - id: operation_id, - method: method_upper, - path: path.clone(), - description, - }); + )] + async fn refresh_openapi( + &self, + Parameters(args): Parameters, + ) -> Result { + self.handle_refresh_openapi(args).await + } + + #[tool( + name = "get_route_info", + description = "Get code example for using a specific API route", + annotations(read_only_hint = true) + )] + async fn get_route_info( + &self, + Parameters(args): Parameters, + ) -> Result { + self.handle_get_route_info(args).await + } + + #[tool( + name = "routes", + description = "List all API routes from the OpenAPI schema", + annotations(read_only_hint = true) + )] + async fn routes( + &self, + Parameters(args): Parameters, + ) -> Result { + self.handle_routes(args).await + } + + // --- Databricks tools --- + + #[tool( + name = "databricks_apps_logs", + description = "Fetch Databricks Apps logs from an already deployed app using the Databricks CLI", + annotations(read_only_hint = true) + )] + async fn databricks_apps_logs( + &self, + Parameters(args): Parameters, + ) -> Result { + self.handle_databricks_apps_logs(args).await + } + + // --- Registry tools --- + + #[tool( + name = "search_registry_components", + description = "Search shadcn registry components using semantic search. Supports filtering by category, type, and registry.", + annotations(read_only_hint = true) + )] + async fn search_registry_components( + &self, + Parameters(args): Parameters, + ) -> Result { + self.handle_search_registry_components(args).await + } + + #[tool( + name = "add_component", + description = "Add a component to the project. Component ID can be 'component-name' (from default registry) or '@registry-name/component-name'.", + annotations(destructive_hint = true, read_only_hint = false) + )] + async fn add_component( + &self, + Parameters(args): Parameters, + ) -> Result { + self.handle_add_component(args).await + } + + // --- Docs tools --- + + #[tool( + name = "docs", + description = "Search Databricks SDK documentation for relevant code examples and API references", + annotations(read_only_hint = true) + )] + async fn docs( + &self, + Parameters(args): Parameters, + ) -> Result { + self.handle_docs(args).await + } +} + +#[tool_handler] +impl ServerHandler for ApxServer { + fn get_info(&self) -> ServerInfo { + ServerInfo { + protocol_version: ProtocolVersion::LATEST, + capabilities: ServerCapabilities::builder() + .enable_resources() + .enable_tools() + .build(), + server_info: Implementation { + name: "apx".into(), + version: env!("CARGO_PKG_VERSION").into(), + title: Some("apx - the toolkit for building Databricks Apps".into()), + description: None, + icons: None, + website_url: Some("https://databricks-solutions.github.io/apx/docs/reference/mcp".into()), + }, + instructions: Some(APX_INFO_CONTENT.to_string()), } } - Ok(routes) -} - -// --- Tools --- - -#[derive(Debug, Deserialize, schemars::JsonSchema)] -pub struct EmptyArgs {} - -#[derive(Debug, Deserialize, schemars::JsonSchema)] -pub struct LogsArgs { - #[serde(default = "default_logs_duration")] - pub duration: String, -} - -fn default_logs_duration() -> String { - apx_core::ops::logs::DEFAULT_LOG_DURATION.to_string() -} - -#[derive(Debug, Deserialize, schemars::JsonSchema)] -pub struct DatabricksAppsLogsArgs { - #[serde(default)] - pub app_name: Option, - #[serde(default = "default_tail_lines")] - pub tail_lines: i32, - #[serde(default)] - pub search: Option, - #[serde(default)] - pub source: Option>, - #[serde(default)] - pub profile: Option, - #[serde(default)] - pub target: Option, - #[serde(default = "default_output")] - pub output: String, - #[serde(default = "default_timeout_seconds")] - pub timeout_seconds: f64, - #[serde(default = "default_max_output_chars")] - pub max_output_chars: i32, -} - -fn default_tail_lines() -> i32 { - 200 -} - -fn default_output() -> String { - "text".to_string() -} - -fn default_timeout_seconds() -> f64 { - 60.0 -} - -fn default_max_output_chars() -> i32 { - 20000 -} - -#[derive(Debug, Deserialize, schemars::JsonSchema)] -pub struct SearchRegistryComponentsArgs { - pub query: String, - #[serde(default = "default_search_limit")] - pub limit: usize, - // Note: The following fields are defined for JSON schema but not yet implemented - #[serde(default)] - #[allow(dead_code)] - pub categories: Option>, - #[serde(default)] - #[allow(dead_code)] - pub item_types: Option>, - #[serde(default)] - #[allow(dead_code)] - pub registries: Option>, -} - -fn default_search_limit() -> usize { - 10 -} - -#[derive(Debug, Deserialize, schemars::JsonSchema)] -pub struct AddComponentArgs { - /// Component ID: either "component-name" (from default registry) or "@registry-name/component-name" - pub component_id: String, - /// Force overwrite existing files - #[serde(default)] - pub force: bool, -} - -#[derive(Debug, Deserialize, schemars::JsonSchema)] -pub struct DocsArgs { - /// Documentation source (currently only "databricks-sdk-python" is supported) - pub source: SDKSource, - /// Search query (e.g., "create cluster", "list jobs", "databricks connect") - pub query: String, - /// Maximum number of results to return - #[serde(default = "default_docs_limit")] - pub num_results: usize, -} - -fn default_docs_limit() -> usize { - 5 -} - -#[derive(Debug, Deserialize, schemars::JsonSchema)] -pub struct GetRouteInfoArgs { - /// Operation ID from the OpenAPI schema (e.g., "listItems", "createItem") - pub operation_id: String, -} - -async fn start_tool(ctx: Arc, _args: EmptyArgs) -> ToolResult { - use apx_core::dev::common::CLIENT_HOST; - use apx_core::ops::dev::start_dev_server; - - match start_dev_server(&ctx.app_dir).await { - Ok(port) => { - ToolResult::success(format!("Dev server started at http://{CLIENT_HOST}:{port}")) - } - Err(e) => ToolResult::error(e), + async fn list_resources( + &self, + _request: Option, + _ctx: RequestContext, + ) -> Result { + Ok(ListResourcesResult { + resources: crate::resources::list_resources(), + next_cursor: None, + meta: None, + }) } -} - -async fn stop_tool(ctx: Arc, _args: EmptyArgs) -> ToolResult { - use apx_core::ops::dev::stop_dev_server; - - match stop_dev_server(&ctx.app_dir).await { - Ok(true) => ToolResult::success("Dev server stopped".to_string()), - Ok(false) => ToolResult::success("No dev server running".to_string()), - Err(e) => ToolResult::error(e), - } -} - -async fn restart_tool(ctx: Arc, _args: EmptyArgs) -> ToolResult { - use apx_core::ops::dev::restart_dev_server; - - match restart_dev_server(&ctx.app_dir).await { - Ok(port) => ToolResult::success(format!("Dev server restarted at http://localhost:{port}")), - Err(e) => ToolResult::error(e), - } -} - -async fn logs_tool(ctx: Arc, args: LogsArgs) -> ToolResult { - use apx_core::ops::logs::fetch_logs_structured; - - match fetch_logs_structured(&ctx.app_dir, &args.duration).await { - Ok(entries) => { - #[derive(Serialize)] - struct LogsResponse { - duration: String, - count: usize, - entries: Vec, - } - - let response = LogsResponse { - duration: args.duration, - count: entries.len(), - entries, - }; - - ToolResult::from_serializable(&response) - } - Err(e) => ToolResult::error(e), - } -} - -async fn refresh_openapi_tool(ctx: Arc, _args: EmptyArgs) -> ToolResult { - use apx_core::api_generator::generate_openapi; - - match generate_openapi(&ctx.app_dir) { - Ok(()) => ToolResult::success("OpenAPI regenerated".to_string()), - Err(e) => ToolResult::error(e), - } -} - -async fn check_tool(ctx: Arc, _args: EmptyArgs) -> ToolResult { - use apx_core::common::OutputMode; - use apx_core::ops::check::run_check; - #[derive(Serialize)] - struct CheckResponse { - status: String, - #[serde(skip_serializing_if = "Option::is_none")] - errors: Option, - } - - let response = match run_check(&ctx.app_dir, OutputMode::Quiet).await { - Ok(()) => CheckResponse { - status: "passed".to_string(), - errors: None, - }, - Err(e) => CheckResponse { - status: "failed".to_string(), - errors: Some(e), - }, - }; - - if response.errors.is_some() { - match serde_json::to_value(&response) { - Ok(value) => ToolResult::structured_error(value), - Err(e) => ToolResult::error(format!("Failed to serialize response: {e}")), - } - } else { - ToolResult::from_serializable(&response) + async fn read_resource( + &self, + request: ReadResourceRequestParams, + _ctx: RequestContext, + ) -> Result { + crate::resources::read_resource(request.uri.as_str()).map_err(|e| { + rmcp::ErrorData::resource_not_found( + e, + Some(serde_json::json!({ "uri": request.uri.as_str() })), + ) + }) } } -async fn databricks_apps_logs_tool( - ctx: Arc, - args: DatabricksAppsLogsArgs, -) -> ToolResult { - let cwd = &ctx.app_dir; - let mut resolved_from_yml = false; - - // Load env vars from .env if present - let dotenv_path = cwd.join(".env"); - let dotenv_vars: HashMap = if dotenv_path.exists() { - DotenvFile::read(&dotenv_path) - .map(|dotenv| dotenv.get_vars()) - .unwrap_or_default() - } else { - HashMap::new() - }; - - // Resolve app_name if not provided - let app_name = match args.app_name.as_ref() { - Some(name) if !name.trim().is_empty() => name.trim().to_string(), - _ => match resolve_app_name_from_databricks_yml(cwd) { - Ok(name) => { - resolved_from_yml = true; - name - } - Err(e) => { - return ToolResult::error(format!("Failed to auto-detect app name: {e}")); - } - }, - }; - - // Build command and track arguments for response - let mut cmd_args = vec!["apps".to_string(), "logs".to_string(), app_name.clone()]; - let mut cmd = Command::new("databricks"); - cmd.args(&cmd_args) - .arg("--tail-lines") - .arg(args.tail_lines.to_string()) - .current_dir(cwd) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - cmd_args.push("--tail-lines".to_string()); - cmd_args.push(args.tail_lines.to_string()); - - let mut push_flag_value = |flag: &str, value: Option<&str>| { - if let Some(value) = value.map(str::trim).filter(|v| !v.is_empty()) { - cmd.arg(flag).arg(value); - cmd_args.push(flag.to_string()); - cmd_args.push(value.to_string()); - } - }; - - push_flag_value("--search", args.search.as_deref()); - push_flag_value("-p", args.profile.as_deref()); - push_flag_value("-t", args.target.as_deref()); - - if let Some(sources) = &args.source { - for source in sources { - cmd.arg("--source").arg(source); - cmd_args.push("--source".to_string()); - cmd_args.push(source.clone()); - } - } - - cmd.arg("-o").arg(&args.output); - cmd_args.push("-o".to_string()); - cmd_args.push(args.output.clone()); - - if !dotenv_vars.is_empty() { - cmd.envs(&dotenv_vars); - } - - let mut full_command = vec!["databricks".to_string()]; - full_command.extend(cmd_args.clone()); - let cmd_str = full_command.join(" "); - - // Run command with timeout - let start = Instant::now(); - let result = - tokio::time::timeout(Duration::from_secs_f64(args.timeout_seconds), cmd.output()).await; - - let (returncode, stdout, stderr, duration_ms) = match result { - Ok(Ok(output)) => { - let duration_ms = start.elapsed().as_millis() as i64; - let returncode = output.status.code().unwrap_or(0); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - (returncode, stdout, stderr, duration_ms) - } - Ok(Err(e)) => { - if e.kind() == std::io::ErrorKind::NotFound { - return ToolResult::error( - "Databricks CLI executable not found (`databricks`). \ - Please install Databricks CLI v0.280.0 or higher and ensure it's on PATH." - .to_string(), - ); - } - return ToolResult::error(format!("Failed to execute command: {e}")); - } - Err(_) => { - return ToolResult::error(format!( - "Timed out after {}s running: {}", - args.timeout_seconds, cmd_str - )); - } - }; - - let stdout_t = truncate(&stdout, args.max_output_chars); - let stderr_t = truncate(&stderr, args.max_output_chars); - - if returncode != 0 { - let combined = format!("{stderr}\n{stdout}").to_lowercase(); - // Check for unsupported subcommand error - if combined.contains("unknown command \"logs\"") - || combined.contains("unknown command logs") - || combined.contains("unknown subcommand") - || combined.contains("no such command") - { - return ToolResult::error(format!( - "Databricks CLI does not support `databricks apps logs` in this version. \ - Please upgrade Databricks CLI to v0.280.0 or higher.\n\n\ - Command: {cmd_str}\n\ - Exit code: {returncode}\n\ - stderr:\n{stderr_t}\n\ - stdout:\n{stdout_t}" - )); - } - - // Forward any other CLI error - return ToolResult::error(format!( - "`databricks apps logs` failed.\n\n\ - Command: {cmd_str}\n\ - Exit code: {returncode}\n\ - stderr:\n{stderr_t}\n\ - stdout:\n{stdout_t}" - )); - } - - // Build success response - #[derive(Serialize)] - struct DatabricksAppsLogsResponse { - app_name: String, - resolved_from_databricks_yml: bool, - command: Vec, - cwd: String, - returncode: i32, - stdout: String, - stderr: String, - duration_ms: i64, - } - - let response = DatabricksAppsLogsResponse { - app_name, - resolved_from_databricks_yml: resolved_from_yml, - command: full_command, - cwd: cwd.to_string_lossy().to_string(), - returncode, - stdout: stdout_t, - stderr: stderr_t, - duration_ms, - }; - - ToolResult::from_serializable(&response) -} - -// Helper functions - -fn truncate(s: &str, max_chars: i32) -> String { - if max_chars <= 0 { - return String::new(); - } - let max_chars = max_chars as usize; - if s.len() <= max_chars { - return s.to_string(); - } - let head_len = max_chars.saturating_sub(50); - let tail_len = if max_chars >= 100 { 40 } else { 0 }; - let head = &s[..head_len]; - let tail = if tail_len > 0 { - &s[s.len().saturating_sub(tail_len)..] - } else { - "" - }; - let truncated = s.len() - head_len - tail_len; - format!("{head}\n\n...[truncated {truncated} chars]...\n\n{tail}") -} - -fn resolve_app_name_from_databricks_yml(project_dir: &Path) -> Result { - let yml_path = project_dir.join("databricks.yml"); - if !yml_path.exists() { - return Err(format!( - "Could not auto-detect app name because databricks.yml was not found at {}. \ - Please pass app_name explicitly.", - yml_path.display() - )); - } - - let contents = std::fs::read_to_string(&yml_path) - .map_err(|e| format!("Failed to read databricks.yml: {e}"))?; - - let data: Value = serde_yaml::from_str(&contents) - .map_err(|e| format!("Failed to parse databricks.yml: {e}"))?; - - let resources = data - .get("resources") - .ok_or_else(|| "databricks.yml 'resources' must be a mapping/object".to_string())?; - - let apps = resources - .get("apps") - .ok_or_else(|| "databricks.yml 'resources.apps' must be a mapping/object".to_string())?; - - let apps_obj = apps - .as_object() - .ok_or_else(|| "databricks.yml 'resources.apps' must be a mapping/object".to_string())?; - - let mut app_names = HashSet::new(); - for app_def in apps_obj.values() { - if let Some(app_obj) = app_def.as_object() - && let Some(name_val) = app_obj.get("name") - && let Some(name_str) = name_val.as_str() - { - let name = name_str.trim(); - if !name.is_empty() { - app_names.insert(name.to_string()); - } - } - } - - let mut app_names_vec: Vec = app_names.into_iter().collect(); - app_names_vec.sort(); - - match app_names_vec.len() { - 1 => Ok(app_names_vec[0].clone()), - 0 => Err( - "Could not auto-detect app name because no apps were found in databricks.yml under \ - resources.apps.*.name. Please pass app_name explicitly." - .to_string(), - ), - _ => Err(format!( - "Could not auto-detect app name because multiple apps were found in databricks.yml \ - ({}). Please pass app_name explicitly.", - app_names_vec.join(", ") - )), - } -} - -async fn search_registry_components_tool( - ctx: Arc, - args: SearchRegistryComponentsArgs, -) -> ToolResult { - use apx_core::common::read_project_metadata; - use apx_core::components::UiConfig; - - // Wait for component index to be ready (15 second timeout) - if let Err(e) = wait_for_index_ready( - &ctx.index_state.component_ready, - &ctx.index_state.component_indexed, - "Component", - ) - .await - { - return ToolResult::error(e); - } - - // Check if registry indexes need refresh - if let Ok(metadata) = read_project_metadata(&ctx.app_dir) { - let cfg = UiConfig::from_metadata(&metadata, &ctx.app_dir); - if needs_registry_refresh(&cfg.registries) { - tracing::info!("Registry indexes stale, refreshing..."); - if let Ok(true) = sync_registry_indexes(&ctx.app_dir, false).await { - // Rebuild search index (sync, in spawn_blocking) - let rebuild_result = tokio::task::spawn_blocking(rebuild_search_index).await; - if let Ok(Err(e)) = rebuild_result { - tracing::warn!("Failed to rebuild search index after refresh: {}", e); - } - } - } - } - - // Search in spawn_blocking (sync SQLite operations) - let query = args.query.clone(); - let limit = args.limit; - let search_results = match tokio::task::spawn_blocking(move || { - let index = ComponentIndex::new()?; - index.search(&query, limit) - }) - .await - { - Ok(Ok(results)) => results, - Ok(Err(e)) => return ToolResult::error(format!("Search failed: {e}")), - Err(e) => return ToolResult::error(format!("Search task panicked: {e}")), - }; - - #[derive(serde::Serialize)] - struct SearchResponse { - query: String, - results: Vec, - } - - #[derive(serde::Serialize)] - struct SearchResultItem { - id: String, - name: String, - registry: String, - score: f32, - } - - let response = SearchResponse { - query: args.query, - results: search_results - .into_iter() - .map(|r| SearchResultItem { - id: r.id, - name: r.name, - registry: r.registry, - score: r.score, - }) - .collect(), - }; - - ToolResult::from_serializable(&response) -} - -async fn add_component_tool(ctx: Arc, args: AddComponentArgs) -> ToolResult { - use apx_core::components::add::{ComponentInput, add_components}; - - // Parse component ID to extract registry and component name - let input = if args.component_id.starts_with('@') { - if let Some((prefix, name)) = args.component_id.split_once('/') { - ComponentInput::with_registry(name, prefix) - } else { - return ToolResult::error(format!( - "Invalid component ID format: {}. Expected '@registry-name/component-name'", - args.component_id - )); - } - } else { - ComponentInput::new(args.component_id.clone()) - }; - - match add_components(&ctx.app_dir, &[input], args.force).await { - Ok(_result) => { - // Component added and cached - index will be updated on next search - // Note: We don't rebuild the entire index here as it's expensive. - // The component is already cached, and the next cache population - // cycle will include it in the index. - tracing::info!("Component {} added successfully", args.component_id); - ToolResult::success(format!( - "Successfully added component: {}", - args.component_id - )) - } - Err(e) => ToolResult::error(format!("Failed to add component: {e}")), - } -} - -async fn docs_tool(ctx: Arc, args: DocsArgs) -> ToolResult { - // Wait for SDK index to be ready (15 second timeout) - if let Err(e) = wait_for_index_ready( - &ctx.index_state.sdk_ready, - &ctx.index_state.sdk_indexed, - "SDK documentation", - ) - .await - { - return ToolResult::error(e); - } - - // Get the SDK doc index - let index_guard = ctx.sdk_doc_index.lock().await; - - let index = match index_guard.as_ref() { - Some(idx) => idx, - None => { - return ToolResult::error( - "SDK documentation is not available. The Databricks SDK may not be installed or the index failed to bootstrap.".to_string() - ); - } - }; - - // Search (sync, in spawn_blocking) - let source = args.source.clone(); - let query = args.query.clone(); - let num_results = args.num_results; - - // We need to clone the connection from the index to use in spawn_blocking - // Since SDKDocsIndex is not Send (Mutex), we search directly here. - // The search is fast enough that blocking briefly is acceptable, but we still - // use search_sync to avoid async across the Mutex. - match index.search_sync(&source, &query, num_results) { - Ok(results) => { - // Drop index_guard before building response - drop(index_guard); - - #[derive(Serialize)] - struct DocsResponse { - source: String, - query: String, - results: Vec, - } - - #[derive(Serialize)] - struct DocsResult { - text: String, - source_file: String, - score: f32, - } - - let response = DocsResponse { - source: match args.source { - SDKSource::DatabricksSdkPython => "databricks-sdk-python".to_string(), - }, - query: args.query, - results: results - .into_iter() - .map(|r| DocsResult { - text: r.text, - source_file: r.source_file, - score: r.score, - }) - .collect(), - }; - - ToolResult::from_serializable(&response) - } - Err(e) => ToolResult::error(e), - } -} - -async fn get_route_info_tool(ctx: Arc, args: GetRouteInfoArgs) -> ToolResult { - let metadata = match read_project_metadata(&ctx.app_dir) { - Ok(m) => m, - Err(e) => return ToolResult::error(e), - }; - - let openapi_content = - match generate_openapi_spec(&ctx.app_dir, &metadata.app_entrypoint, &metadata.app_slug) { - Ok((content, _)) => content, - Err(e) => return ToolResult::error(format!("Failed to generate OpenAPI spec: {e}")), - }; - - let openapi: Value = match serde_json::from_str(&openapi_content) { - Ok(spec) => spec, - Err(e) => return ToolResult::error(format!("Failed to parse OpenAPI schema: {e}")), - }; - - // Find the operation by operationId - let paths = match openapi.get("paths").and_then(|p| p.as_object()) { - Some(p) => p, - None => return ToolResult::error("OpenAPI schema missing 'paths' object".to_string()), - }; - - let mut found_method = None; - for (_, path_item) in paths { - if let Some(methods_obj) = path_item.as_object() { - for (method, operation) in methods_obj { - if let Some(operation_obj) = operation.as_object() - && let Some(op_id) = operation_obj.get("operationId").and_then(|v| v.as_str()) - && op_id == args.operation_id - { - found_method = Some(method.to_uppercase()); - break; - } - } - if found_method.is_some() { - break; - } - } - } - - let method = match found_method { - Some(m) => m, - None => { - return ToolResult::error(format!( - "Operation ID '{}' not found in OpenAPI schema", - args.operation_id - )); - } - }; - - // Generate the appropriate code example based on the HTTP method - let example = if method == "GET" { - generate_query_example(&args.operation_id) - } else { - generate_mutation_example(&args.operation_id) - }; - - #[derive(Serialize)] - struct RouteInfoResponse { - operation_id: String, - method: String, - example: String, - } - - let response = RouteInfoResponse { - operation_id: args.operation_id, - method, - example, - }; - - ToolResult::from_serializable(&response) -} - -fn generate_query_example(operation_id: &str) -> String { - // Convert operationId to PascalCase for the hook name - let capitalized = capitalize_first(operation_id); - let hook_name = format!("use{capitalized}"); - let suspense_hook_name = format!("{hook_name}Suspense"); - let result_type = format!("{capitalized}QueryResult"); - let error_type = format!("{capitalized}QueryError"); - - format!( - r#"// Standard query hook -import {{ {hook_name} }} from "@/lib/api"; -import selector from "@/lib/selector"; - -const Component = () => {{ - const {{ data, isLoading, error }} = {hook_name}(selector()); - - if (isLoading) return
Loading...
; - if (error) return
Error: {{error.message}}
; - - return
{{/* render data */}}
; -}}; - -// Suspense query hook (use with React Suspense boundary) -import {{ {suspense_hook_name} }} from "@/lib/api"; -import selector from "@/lib/selector"; - -const SuspenseComponent = () => {{ - // No loading/error states needed - handled by Suspense boundary - const {{ data }} = {suspense_hook_name}(selector()); - return
{{/* render data */}}
; -}}; - -// Usage with Suspense boundary: -// }}> -// -// - -// Available types for this query: -// import type {{ {result_type}, {error_type} }} from "@/lib/api";"# - ) -} - -fn generate_mutation_example(operation_id: &str) -> String { - // Convert operationId to PascalCase for the hook name - let capitalized = capitalize_first(operation_id); - let hook_name = format!("use{capitalized}"); - let body_type = format!("{capitalized}MutationBody"); - let result_type = format!("{capitalized}MutationResult"); - let error_type = format!("{capitalized}MutationError"); - - format!( - r#"import {{ {hook_name} }} from "@/lib/api"; - -const Component = () => {{ - const {{ mutate, isPending }} = {hook_name}(); - - const handleSubmit = () => {{ - mutate({{ data: {{ /* request body */ }} }}); - }}; - - return ; -}}; - -// Available types for this mutation: -// import type {{ {body_type}, {result_type}, {error_type} }} from "@/lib/api";"# - ) -} - -fn capitalize_first(s: &str) -> String { - let mut chars = s.chars(); - match chars.next() { - None => String::new(), - Some(first) => first.to_uppercase().collect::() + chars.as_str(), - } -} - -const APX_INFO_CONTENT: &str = r#" - -this project uses apx toolkit to build a Databricks app. -apx bundles together a set of tools and libraries to help you with the complete app development lifecycle: develop, build and deploy. - -## Technology Stack - -- **Backend**: Python + FastAPI + Pydantic -- **Frontend**: React + TypeScript + shadcn/ui -- **Build Tools**: uv (Python), bun (JavaScript/TypeScript) - -"#; - -#[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used)] -mod tests { - use super::*; - - #[test] - fn truncate_empty_string() { - assert_eq!(truncate("", 100), ""); - } - - #[test] - fn truncate_short_string() { - assert_eq!(truncate("hello", 100), "hello"); - } - - #[test] - fn truncate_zero_max() { - assert_eq!(truncate("hello", 0), ""); - } - - #[test] - fn truncate_negative_max() { - assert_eq!(truncate("hello", -1), ""); - } - - #[test] - fn truncate_long_string() { - let long = "a".repeat(1000); - let result = truncate(&long, 200); - assert!(result.contains("truncated")); - assert!(result.len() < 1000); - } - - #[test] - fn parse_openapi_operations_basic() { - let openapi = serde_json::json!({ - "paths": { - "/items": { - "get": { - "operationId": "listItems", - "summary": "List all items" - }, - "post": { - "operationId": "createItem", - "description": "Create a new item" - } - } - } - }); - - let routes = parse_openapi_operations(&openapi).unwrap(); - assert_eq!(routes.len(), 2); - - let get_route = routes.iter().find(|r| r.method == "GET").unwrap(); - assert_eq!(get_route.id, "listItems"); - assert_eq!(get_route.description, "List all items"); - - let post_route = routes.iter().find(|r| r.method == "POST").unwrap(); - assert_eq!(post_route.id, "createItem"); - assert_eq!(post_route.description, "Create a new item"); - } - - #[test] - fn parse_openapi_operations_skips_non_methods() { - let openapi = serde_json::json!({ - "paths": { - "/items": { - "parameters": [{"name": "id"}], - "get": { - "operationId": "listItems", - "summary": "List items" - } - } - } - }); - - let routes = parse_openapi_operations(&openapi).unwrap(); - assert_eq!(routes.len(), 1); - assert_eq!(routes[0].method, "GET"); - } - - #[test] - fn parse_openapi_operations_missing_paths() { - let openapi = serde_json::json!({}); - assert!(parse_openapi_operations(&openapi).is_err()); - } - - #[test] - fn resolve_app_name_from_databricks_yml_basic() { - let dir = std::env::temp_dir().join("apx_test_resolve_app"); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).unwrap(); - - let yml_content = r#" -resources: - apps: - my_app: - name: my-cool-app - source_code_path: ./src -"#; - std::fs::write(dir.join("databricks.yml"), yml_content).unwrap(); - - let result = resolve_app_name_from_databricks_yml(&dir); - assert_eq!(result.unwrap(), "my-cool-app"); - - let _ = std::fs::remove_dir_all(&dir); - } - - #[test] - fn resolve_app_name_multiple_apps_returns_error() { - let dir = std::env::temp_dir().join("apx_test_resolve_multi"); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).unwrap(); - - let yml_content = r#" -resources: - apps: - app1: - name: first-app - app2: - name: second-app -"#; - std::fs::write(dir.join("databricks.yml"), yml_content).unwrap(); - - let result = resolve_app_name_from_databricks_yml(&dir); - assert!(result.is_err()); - assert!(result.unwrap_err().contains("multiple apps")); - - let _ = std::fs::remove_dir_all(&dir); - } - - #[test] - fn resolve_app_name_no_file_returns_error() { - let dir = std::env::temp_dir().join("apx_test_resolve_nofile"); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).unwrap(); - - let result = resolve_app_name_from_databricks_yml(&dir); - assert!(result.is_err()); - - let _ = std::fs::remove_dir_all(&dir); - } +pub async fn run_server( + ctx: AppContext, + sdk_params: Option, +) -> Result<(), String> { + use rmcp::ServiceExt; + + let shutdown_tx = ctx.shutdown_tx.clone(); + let server = ApxServer::new(ctx, sdk_params); + let transport = rmcp::transport::io::stdio(); + let service = server + .serve(transport) + .await + .map_err(|e| format!("MCP server initialization error: {e}"))?; + service + .waiting() + .await + .map_err(|e| format!("MCP server error: {e}"))?; + let _ = shutdown_tx.send(()); + Ok(()) } diff --git a/crates/mcp/src/tools/databricks.rs b/crates/mcp/src/tools/databricks.rs new file mode 100644 index 00000000..f291dbda --- /dev/null +++ b/crates/mcp/src/tools/databricks.rs @@ -0,0 +1,406 @@ +use crate::server::ApxServer; +use crate::tools::ToolResultExt; +use crate::validation::validate_app_path; +use apx_core::dotenv::DotenvFile; +use rmcp::model::*; +use rmcp::schemars; +use serde::Serialize; +use serde_json::Value; +use std::collections::{HashMap, HashSet}; +use std::path::Path; +use std::process::Stdio; +use std::time::{Duration, Instant}; +use tokio::process::Command; + +pub(crate) fn truncate(s: &str, max_chars: i32) -> String { + if max_chars <= 0 { + return String::new(); + } + let max_chars = max_chars as usize; + if s.len() <= max_chars { + return s.to_string(); + } + let head_len = max_chars.saturating_sub(50); + let tail_len = if max_chars >= 100 { 40 } else { 0 }; + let head = &s[..head_len]; + let tail = if tail_len > 0 { + &s[s.len().saturating_sub(tail_len)..] + } else { + "" + }; + let truncated = s.len() - head_len - tail_len; + format!("{head}\n\n...[truncated {truncated} chars]...\n\n{tail}") +} + +pub(crate) fn resolve_app_name_from_databricks_yml( + project_dir: &Path, +) -> Result { + let yml_path = project_dir.join("databricks.yml"); + if !yml_path.exists() { + return Err(format!( + "Could not auto-detect app name because databricks.yml was not found at {}. \ + Please pass app_name explicitly.", + yml_path.display() + )); + } + + let contents = std::fs::read_to_string(&yml_path) + .map_err(|e| format!("Failed to read databricks.yml: {e}"))?; + + let data: Value = serde_yaml::from_str(&contents) + .map_err(|e| format!("Failed to parse databricks.yml: {e}"))?; + + let resources = data + .get("resources") + .ok_or_else(|| "databricks.yml 'resources' must be a mapping/object".to_string())?; + + let apps = resources + .get("apps") + .ok_or_else(|| "databricks.yml 'resources.apps' must be a mapping/object".to_string())?; + + let apps_obj = apps + .as_object() + .ok_or_else(|| "databricks.yml 'resources.apps' must be a mapping/object".to_string())?; + + let mut app_names = HashSet::new(); + for app_def in apps_obj.values() { + if let Some(app_obj) = app_def.as_object() + && let Some(name_val) = app_obj.get("name") + && let Some(name_str) = name_val.as_str() + { + let name = name_str.trim(); + if !name.is_empty() { + app_names.insert(name.to_string()); + } + } + } + + let mut app_names_vec: Vec = app_names.into_iter().collect(); + app_names_vec.sort(); + + match app_names_vec.len() { + 1 => Ok(app_names_vec[0].clone()), + 0 => Err( + "Could not auto-detect app name because no apps were found in databricks.yml under \ + resources.apps.*.name. Please pass app_name explicitly." + .to_string(), + ), + _ => Err(format!( + "Could not auto-detect app name because multiple apps were found in databricks.yml \ + ({}). Please pass app_name explicitly.", + app_names_vec.join(", ") + )), + } +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct DatabricksAppsLogsArgs { + /// Absolute path to the project directory + pub app_path: String, + /// Name of the Databricks app (auto-detected from databricks.yml if not provided) + #[serde(default)] + pub app_name: Option, + /// Number of tail lines to fetch (default: 200) + #[serde(default = "default_tail_lines")] + pub tail_lines: i32, + /// Search string to filter logs + #[serde(default)] + pub search: Option, + /// Log sources to include + #[serde(default)] + pub source: Option>, + /// Databricks CLI profile + #[serde(default)] + pub profile: Option, + /// Databricks CLI target + #[serde(default)] + pub target: Option, + /// Output format (default: "text") + #[serde(default = "default_output")] + pub output: String, + /// Timeout in seconds (default: 60) + #[serde(default = "default_timeout_seconds")] + pub timeout_seconds: f64, + /// Maximum output characters (default: 20000) + #[serde(default = "default_max_output_chars")] + pub max_output_chars: i32, +} + +fn default_tail_lines() -> i32 { + 200 +} + +fn default_output() -> String { + "text".to_string() +} + +fn default_timeout_seconds() -> f64 { + 60.0 +} + +fn default_max_output_chars() -> i32 { + 20000 +} + +impl ApxServer { + pub async fn handle_databricks_apps_logs( + &self, + args: DatabricksAppsLogsArgs, + ) -> Result { + let cwd = validate_app_path(&args.app_path) + .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; + + let mut resolved_from_yml = false; + + // Load env vars from .env if present + let dotenv_path = cwd.join(".env"); + let dotenv_vars: HashMap = if dotenv_path.exists() { + DotenvFile::read(&dotenv_path) + .map(|dotenv| dotenv.get_vars()) + .unwrap_or_default() + } else { + HashMap::new() + }; + + // Resolve app_name if not provided + let app_name = match args.app_name.as_ref() { + Some(name) if !name.trim().is_empty() => name.trim().to_string(), + _ => match resolve_app_name_from_databricks_yml(&cwd) { + Ok(name) => { + resolved_from_yml = true; + name + } + Err(e) => { + return Ok(CallToolResult::error(vec![Content::text(format!( + "Failed to auto-detect app name: {e}" + ))])); + } + }, + }; + + // Build command and track arguments for response + let mut cmd_args = vec!["apps".to_string(), "logs".to_string(), app_name.clone()]; + let mut cmd = Command::new("databricks"); + cmd.args(&cmd_args) + .arg("--tail-lines") + .arg(args.tail_lines.to_string()) + .current_dir(&cwd) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + cmd_args.push("--tail-lines".to_string()); + cmd_args.push(args.tail_lines.to_string()); + + let mut push_flag_value = |flag: &str, value: Option<&str>| { + if let Some(value) = value.map(str::trim).filter(|v| !v.is_empty()) { + cmd.arg(flag).arg(value); + cmd_args.push(flag.to_string()); + cmd_args.push(value.to_string()); + } + }; + + push_flag_value("--search", args.search.as_deref()); + push_flag_value("-p", args.profile.as_deref()); + push_flag_value("-t", args.target.as_deref()); + + if let Some(sources) = &args.source { + for src in sources { + cmd.arg("--source").arg(src); + cmd_args.push("--source".to_string()); + cmd_args.push(src.clone()); + } + } + + cmd.arg("-o").arg(&args.output); + cmd_args.push("-o".to_string()); + cmd_args.push(args.output.clone()); + + if !dotenv_vars.is_empty() { + cmd.envs(&dotenv_vars); + } + + let mut full_command = vec!["databricks".to_string()]; + full_command.extend(cmd_args.clone()); + let cmd_str = full_command.join(" "); + + // Run command with timeout + let start = Instant::now(); + let result = tokio::time::timeout( + Duration::from_secs_f64(args.timeout_seconds), + cmd.output(), + ) + .await; + + let (returncode, stdout, stderr, duration_ms) = match result { + Ok(Ok(cmd_output)) => { + let duration_ms = start.elapsed().as_millis() as i64; + let returncode = cmd_output.status.code().unwrap_or(0); + let stdout = String::from_utf8_lossy(&cmd_output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&cmd_output.stderr).to_string(); + (returncode, stdout, stderr, duration_ms) + } + Ok(Err(e)) => { + if e.kind() == std::io::ErrorKind::NotFound { + return Ok(CallToolResult::error(vec![Content::text( + "Databricks CLI executable not found (`databricks`). \ + Please install Databricks CLI v0.280.0 or higher and ensure it's on PATH.", + )])); + } + return Ok(CallToolResult::error(vec![Content::text(format!( + "Failed to execute command: {e}" + ))])); + } + Err(_) => { + return Ok(CallToolResult::error(vec![Content::text(format!( + "Timed out after {}s running: {}", + args.timeout_seconds, cmd_str + ))])); + } + }; + + let stdout_t = truncate(&stdout, args.max_output_chars); + let stderr_t = truncate(&stderr, args.max_output_chars); + + if returncode != 0 { + let combined = format!("{stderr}\n{stdout}").to_lowercase(); + if combined.contains("unknown command \"logs\"") + || combined.contains("unknown command logs") + || combined.contains("unknown subcommand") + || combined.contains("no such command") + { + return Ok(CallToolResult::error(vec![Content::text(format!( + "Databricks CLI does not support `databricks apps logs` in this version. \ + Please upgrade Databricks CLI to v0.280.0 or higher.\n\n\ + Command: {cmd_str}\n\ + Exit code: {returncode}\n\ + stderr:\n{stderr_t}\n\ + stdout:\n{stdout_t}" + ))])); + } + + return Ok(CallToolResult::error(vec![Content::text(format!( + "`databricks apps logs` failed.\n\n\ + Command: {cmd_str}\n\ + Exit code: {returncode}\n\ + stderr:\n{stderr_t}\n\ + stdout:\n{stdout_t}" + ))])); + } + + #[derive(Serialize)] + struct DatabricksAppsLogsResponse { + app_name: String, + resolved_from_databricks_yml: bool, + command: Vec, + cwd: String, + returncode: i32, + stdout: String, + stderr: String, + duration_ms: i64, + } + + let response = DatabricksAppsLogsResponse { + app_name, + resolved_from_databricks_yml: resolved_from_yml, + command: full_command, + cwd: cwd.to_string_lossy().to_string(), + returncode, + stdout: stdout_t, + stderr: stderr_t, + duration_ms, + }; + + Ok(CallToolResult::from_serializable(&response)) + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used)] +mod tests { + use super::*; + + #[test] + fn truncate_empty_string() { + assert_eq!(truncate("", 100), ""); + } + + #[test] + fn truncate_short_string() { + assert_eq!(truncate("hello", 100), "hello"); + } + + #[test] + fn truncate_zero_max() { + assert_eq!(truncate("hello", 0), ""); + } + + #[test] + fn truncate_negative_max() { + assert_eq!(truncate("hello", -1), ""); + } + + #[test] + fn truncate_long_string() { + let long = "a".repeat(1000); + let result = truncate(&long, 200); + assert!(result.contains("truncated")); + assert!(result.len() < 1000); + } + + #[test] + fn resolve_app_name_from_databricks_yml_basic() { + let dir = std::env::temp_dir().join("apx_test_resolve_app"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + + let yml_content = r#" +resources: + apps: + my_app: + name: my-cool-app + source_code_path: ./src +"#; + std::fs::write(dir.join("databricks.yml"), yml_content).unwrap(); + + let result = resolve_app_name_from_databricks_yml(&dir); + assert_eq!(result.unwrap(), "my-cool-app"); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn resolve_app_name_multiple_apps_returns_error() { + let dir = std::env::temp_dir().join("apx_test_resolve_multi"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + + let yml_content = r#" +resources: + apps: + app1: + name: first-app + app2: + name: second-app +"#; + std::fs::write(dir.join("databricks.yml"), yml_content).unwrap(); + + let result = resolve_app_name_from_databricks_yml(&dir); + assert!(result.is_err()); + assert!(result.unwrap_err().contains("multiple apps")); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn resolve_app_name_no_file_returns_error() { + let dir = std::env::temp_dir().join("apx_test_resolve_nofile"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + + let result = resolve_app_name_from_databricks_yml(&dir); + assert!(result.is_err()); + + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/mcp/src/tools/devserver.rs b/crates/mcp/src/tools/devserver.rs new file mode 100644 index 00000000..9975f9d3 --- /dev/null +++ b/crates/mcp/src/tools/devserver.rs @@ -0,0 +1,106 @@ +use crate::server::ApxServer; +use crate::tools::{AppPathArgs, ToolResultExt}; +use crate::validation::validate_app_path; +use rmcp::model::*; +use rmcp::schemars; +use serde::Serialize; + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct LogsToolArgs { + /// Absolute path to the project directory + pub app_path: String, + /// Log duration (e.g. '5m', '1h') + #[serde(default = "default_logs_duration")] + pub duration: String, +} + +fn default_logs_duration() -> String { + apx_core::ops::logs::DEFAULT_LOG_DURATION.to_string() +} + +impl ApxServer { + pub async fn handle_start( + &self, + args: AppPathArgs, + ) -> Result { + let path = validate_app_path(&args.app_path) + .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; + + use apx_core::dev::common::CLIENT_HOST; + use apx_core::ops::dev::start_dev_server; + + match start_dev_server(&path).await { + Ok(port) => Ok(CallToolResult::success(vec![Content::text(format!( + "Dev server started at http://{CLIENT_HOST}:{port}" + ))])), + Err(e) => Ok(CallToolResult::error(vec![Content::text(e)])), + } + } + + pub async fn handle_stop( + &self, + args: AppPathArgs, + ) -> Result { + let path = validate_app_path(&args.app_path) + .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; + + use apx_core::ops::dev::stop_dev_server; + + match stop_dev_server(&path).await { + Ok(true) => Ok(CallToolResult::success(vec![Content::text( + "Dev server stopped", + )])), + Ok(false) => Ok(CallToolResult::success(vec![Content::text( + "No dev server running", + )])), + Err(e) => Ok(CallToolResult::error(vec![Content::text(e)])), + } + } + + pub async fn handle_restart( + &self, + args: AppPathArgs, + ) -> Result { + let path = validate_app_path(&args.app_path) + .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; + + use apx_core::ops::dev::restart_dev_server; + + match restart_dev_server(&path).await { + Ok(port) => Ok(CallToolResult::success(vec![Content::text(format!( + "Dev server restarted at http://localhost:{port}" + ))])), + Err(e) => Ok(CallToolResult::error(vec![Content::text(e)])), + } + } + + pub async fn handle_logs( + &self, + args: LogsToolArgs, + ) -> Result { + let path = validate_app_path(&args.app_path) + .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; + + use apx_core::ops::logs::fetch_logs_structured; + + match fetch_logs_structured(&path, &args.duration).await { + Ok(entries) => { + #[derive(Serialize)] + struct LogsResponse { + duration: String, + count: usize, + entries: Vec, + } + + let response = LogsResponse { + duration: args.duration, + count: entries.len(), + entries, + }; + + Ok(CallToolResult::from_serializable(&response)) + } + Err(e) => Ok(CallToolResult::error(vec![Content::text(e)])), + } + } +} diff --git a/crates/mcp/src/tools/docs.rs b/crates/mcp/src/tools/docs.rs new file mode 100644 index 00000000..9d3dd28e --- /dev/null +++ b/crates/mcp/src/tools/docs.rs @@ -0,0 +1,92 @@ +use crate::indexing::wait_for_index_ready; +use crate::server::ApxServer; +use crate::tools::ToolResultExt; +use apx_core::databricks_sdk_doc::SDKSource; +use rmcp::model::*; +use rmcp::schemars; +use serde::Serialize; + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct DocsArgs { + /// Documentation source (currently only "databricks-sdk-python" is supported) + pub source: SDKSource, + /// Search query (e.g., "create cluster", "list jobs", "databricks connect") + pub query: String, + /// Maximum number of results to return (default: 5) + #[serde(default = "default_docs_limit")] + pub num_results: usize, +} + +fn default_docs_limit() -> usize { + 5 +} + +impl ApxServer { + pub async fn handle_docs( + &self, + args: DocsArgs, + ) -> Result { + let ctx = &self.ctx; + + // Wait for SDK index to be ready (15 second timeout) + if let Err(e) = wait_for_index_ready( + &ctx.index_state.sdk_ready, + &ctx.index_state.sdk_indexed, + "SDK documentation", + ) + .await + { + return Ok(CallToolResult::error(vec![Content::text(e)])); + } + + // Get the SDK doc index + let index_guard = ctx.sdk_doc_index.lock().await; + + let index = match index_guard.as_ref() { + Some(idx) => idx, + None => { + return Ok(CallToolResult::error(vec![Content::text( + "SDK documentation is not available. The Databricks SDK may not be installed or the index failed to bootstrap." + )])); + } + }; + + match index.search_sync(&args.source, &args.query, args.num_results) { + Ok(results) => { + drop(index_guard); + + #[derive(Serialize)] + struct DocsResponse { + source: String, + query: String, + results: Vec, + } + + #[derive(Serialize)] + struct DocsResult { + text: String, + source_file: String, + score: f32, + } + + let response = DocsResponse { + source: match args.source { + SDKSource::DatabricksSdkPython => "databricks-sdk-python".to_string(), + }, + query: args.query, + results: results + .into_iter() + .map(|r| DocsResult { + text: r.text, + source_file: r.source_file, + score: r.score, + }) + .collect(), + }; + + Ok(CallToolResult::from_serializable(&response)) + } + Err(e) => Ok(CallToolResult::error(vec![Content::text(e)])), + } + } +} diff --git a/crates/mcp/src/tools/mod.rs b/crates/mcp/src/tools/mod.rs new file mode 100644 index 00000000..c75ad402 --- /dev/null +++ b/crates/mcp/src/tools/mod.rs @@ -0,0 +1,40 @@ +pub mod databricks; +pub mod devserver; +pub mod docs; +pub mod project; +pub mod registry; + +use rmcp::model::CallToolResult; +use rmcp::model::Content; +use rmcp::schemars; + +/// Shared args for tools that only need an app path. +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct AppPathArgs { + /// Absolute path to the project directory + pub app_path: String, +} + +/// Extension trait for building `CallToolResult` from serializable values. +pub trait ToolResultExt { + fn from_serializable(value: &impl serde::Serialize) -> Self; +} + +impl ToolResultExt for CallToolResult { + fn from_serializable(value: &impl serde::Serialize) -> Self { + match serde_json::to_value(value) { + Ok(v) => { + let text_fallback = serde_json::to_string_pretty(&v).unwrap_or_default(); + CallToolResult { + content: vec![Content::text(text_fallback)], + structured_content: Some(v), + is_error: Some(false), + meta: None, + } + } + Err(e) => CallToolResult::error(vec![Content::text(format!( + "Failed to serialize response: {e}" + ))]), + } + } +} diff --git a/crates/mcp/src/tools/project.rs b/crates/mcp/src/tools/project.rs new file mode 100644 index 00000000..3a5fa156 --- /dev/null +++ b/crates/mcp/src/tools/project.rs @@ -0,0 +1,401 @@ +use crate::server::ApxServer; +use crate::tools::{AppPathArgs, ToolResultExt}; +use crate::validation::validate_app_path; +use rmcp::model::*; +use rmcp::schemars; +use serde::Serialize; +use serde_json::Value; + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct GetRouteInfoArgs { + /// Absolute path to the project directory + pub app_path: String, + /// Operation ID from the OpenAPI schema (e.g., "listItems", "createItem") + pub operation_id: String, +} + +#[derive(Serialize)] +struct RouteInfo { + id: String, + method: String, + path: String, + description: String, +} + +fn parse_openapi_operations(openapi: &Value) -> Result, String> { + let mut routes = Vec::new(); + + let paths = openapi + .get("paths") + .and_then(|p| p.as_object()) + .ok_or_else(|| "OpenAPI schema missing 'paths' object".to_string())?; + + for (path, path_item) in paths { + let methods_obj = path_item + .as_object() + .ok_or_else(|| format!("Path '{path}' is not an object"))?; + + for (method, operation) in methods_obj { + let method_upper = method.to_uppercase(); + if !matches!( + method_upper.as_str(), + "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS" + ) { + continue; + } + + let operation_obj = operation.as_object().ok_or_else(|| { + format!("Operation '{method}' at path '{path}' is not an object") + })?; + + let operation_id = operation_obj + .get("operationId") + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_string(); + + let description = operation_obj + .get("summary") + .or_else(|| operation_obj.get("description")) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + + routes.push(RouteInfo { + id: operation_id, + method: method_upper, + path: path.clone(), + description, + }); + } + } + + Ok(routes) +} + +fn capitalize_first(s: &str) -> String { + let mut chars = s.chars(); + match chars.next() { + None => String::new(), + Some(first) => first.to_uppercase().collect::() + chars.as_str(), + } +} + +fn generate_query_example(operation_id: &str) -> String { + let capitalized = capitalize_first(operation_id); + let hook_name = format!("use{capitalized}"); + let suspense_hook_name = format!("{hook_name}Suspense"); + let result_type = format!("{capitalized}QueryResult"); + let error_type = format!("{capitalized}QueryError"); + + format!( + r#"// Standard query hook +import {{ {hook_name} }} from "@/lib/api"; +import selector from "@/lib/selector"; + +const Component = () => {{ + const {{ data, isLoading, error }} = {hook_name}(selector()); + + if (isLoading) return
Loading...
; + if (error) return
Error: {{error.message}}
; + + return
{{/* render data */}}
; +}}; + +// Suspense query hook (use with React Suspense boundary) +import {{ {suspense_hook_name} }} from "@/lib/api"; +import selector from "@/lib/selector"; + +const SuspenseComponent = () => {{ + // No loading/error states needed - handled by Suspense boundary + const {{ data }} = {suspense_hook_name}(selector()); + return
{{/* render data */}}
; +}}; + +// Usage with Suspense boundary: +// }}> +// +// + +// Available types for this query: +// import type {{ {result_type}, {error_type} }} from "@/lib/api";"# + ) +} + +fn generate_mutation_example(operation_id: &str) -> String { + let capitalized = capitalize_first(operation_id); + let hook_name = format!("use{capitalized}"); + let body_type = format!("{capitalized}MutationBody"); + let result_type = format!("{capitalized}MutationResult"); + let error_type = format!("{capitalized}MutationError"); + + format!( + r#"import {{ {hook_name} }} from "@/lib/api"; + +const Component = () => {{ + const {{ mutate, isPending }} = {hook_name}(); + + const handleSubmit = () => {{ + mutate({{ data: {{ /* request body */ }} }}); + }}; + + return ; +}}; + +// Available types for this mutation: +// import type {{ {body_type}, {result_type}, {error_type} }} from "@/lib/api";"# + ) +} + +impl ApxServer { + pub async fn handle_check( + &self, + args: AppPathArgs, + ) -> Result { + let path = validate_app_path(&args.app_path) + .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; + + use apx_core::common::OutputMode; + use apx_core::ops::check::run_check; + + #[derive(Serialize)] + struct CheckResponse { + status: String, + #[serde(skip_serializing_if = "Option::is_none")] + errors: Option, + } + + let response = match run_check(&path, OutputMode::Quiet).await { + Ok(()) => CheckResponse { + status: "passed".to_string(), + errors: None, + }, + Err(e) => CheckResponse { + status: "failed".to_string(), + errors: Some(e), + }, + }; + + if response.errors.is_some() { + match serde_json::to_value(&response) { + Ok(value) => Ok(CallToolResult::structured_error(value)), + Err(e) => Ok(CallToolResult::error(vec![Content::text(format!( + "Failed to serialize response: {e}" + ))])), + } + } else { + Ok(CallToolResult::from_serializable(&response)) + } + } + + pub async fn handle_refresh_openapi( + &self, + args: AppPathArgs, + ) -> Result { + let path = validate_app_path(&args.app_path) + .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; + + use apx_core::api_generator::generate_openapi; + + match generate_openapi(&path) { + Ok(()) => Ok(CallToolResult::success(vec![Content::text( + "OpenAPI regenerated", + )])), + Err(e) => Ok(CallToolResult::error(vec![Content::text(e)])), + } + } + + pub async fn handle_get_route_info( + &self, + args: GetRouteInfoArgs, + ) -> Result { + let path = validate_app_path(&args.app_path) + .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; + + use apx_core::common::read_project_metadata; + use apx_core::interop::generate_openapi_spec; + + let metadata = match read_project_metadata(&path) { + Ok(m) => m, + Err(e) => return Ok(CallToolResult::error(vec![Content::text(e)])), + }; + + let openapi_content = + match generate_openapi_spec(&path, &metadata.app_entrypoint, &metadata.app_slug) { + Ok((content, _)) => content, + Err(e) => { + return Ok(CallToolResult::error(vec![Content::text(format!( + "Failed to generate OpenAPI spec: {e}" + ))])) + } + }; + + let openapi: Value = match serde_json::from_str(&openapi_content) { + Ok(spec) => spec, + Err(e) => { + return Ok(CallToolResult::error(vec![Content::text(format!( + "Failed to parse OpenAPI schema: {e}" + ))])) + } + }; + + let paths = match openapi.get("paths").and_then(|p| p.as_object()) { + Some(p) => p, + None => { + return Ok(CallToolResult::error(vec![Content::text( + "OpenAPI schema missing 'paths' object", + )])) + } + }; + + let mut found_method = None; + for (_, path_item) in paths { + if let Some(methods_obj) = path_item.as_object() { + for (method, operation) in methods_obj { + if let Some(operation_obj) = operation.as_object() + && let Some(op_id) = + operation_obj.get("operationId").and_then(|v| v.as_str()) + && op_id == args.operation_id + { + found_method = Some(method.to_uppercase()); + break; + } + } + if found_method.is_some() { + break; + } + } + } + + let method = match found_method { + Some(m) => m, + None => { + return Ok(CallToolResult::error(vec![Content::text(format!( + "Operation ID '{}' not found in OpenAPI schema", + args.operation_id + ))])) + } + }; + + let example = if method == "GET" { + generate_query_example(&args.operation_id) + } else { + generate_mutation_example(&args.operation_id) + }; + + #[derive(Serialize)] + struct RouteInfoResponse { + operation_id: String, + method: String, + example: String, + } + + let response = RouteInfoResponse { + operation_id: args.operation_id, + method, + example, + }; + + Ok(CallToolResult::from_serializable(&response)) + } + + pub async fn handle_routes( + &self, + args: AppPathArgs, + ) -> Result { + let path = validate_app_path(&args.app_path) + .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; + + use apx_core::common::read_project_metadata; + use apx_core::interop::generate_openapi_spec; + + let metadata = match read_project_metadata(&path) { + Ok(m) => m, + Err(e) => return Ok(CallToolResult::error(vec![Content::text(e)])), + }; + + let (openapi_content, _) = + match generate_openapi_spec(&path, &metadata.app_entrypoint, &metadata.app_slug) { + Ok(result) => result, + Err(e) => { + return Ok(CallToolResult::error(vec![Content::text(format!( + "Failed to generate OpenAPI spec: {e}" + ))])) + } + }; + + let openapi: Value = match serde_json::from_str(&openapi_content) { + Ok(spec) => spec, + Err(e) => { + return Ok(CallToolResult::error(vec![Content::text(format!( + "Failed to parse OpenAPI schema: {e}" + ))])) + } + }; + + match parse_openapi_operations(&openapi) { + Ok(routes) => Ok(CallToolResult::from_serializable(&routes)), + Err(e) => Ok(CallToolResult::error(vec![Content::text(e)])), + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used)] +mod tests { + use super::*; + + #[test] + fn parse_openapi_operations_basic() { + let openapi = serde_json::json!({ + "paths": { + "/items": { + "get": { + "operationId": "listItems", + "summary": "List all items" + }, + "post": { + "operationId": "createItem", + "description": "Create a new item" + } + } + } + }); + + let routes = parse_openapi_operations(&openapi).unwrap(); + assert_eq!(routes.len(), 2); + + let get_route = routes.iter().find(|r| r.method == "GET").unwrap(); + assert_eq!(get_route.id, "listItems"); + assert_eq!(get_route.description, "List all items"); + + let post_route = routes.iter().find(|r| r.method == "POST").unwrap(); + assert_eq!(post_route.id, "createItem"); + assert_eq!(post_route.description, "Create a new item"); + } + + #[test] + fn parse_openapi_operations_skips_non_methods() { + let openapi = serde_json::json!({ + "paths": { + "/items": { + "parameters": [{"name": "id"}], + "get": { + "operationId": "listItems", + "summary": "List items" + } + } + } + }); + + let routes = parse_openapi_operations(&openapi).unwrap(); + assert_eq!(routes.len(), 1); + assert_eq!(routes[0].method, "GET"); + } + + #[test] + fn parse_openapi_operations_missing_paths() { + let openapi = serde_json::json!({}); + assert!(parse_openapi_operations(&openapi).is_err()); + } +} diff --git a/crates/mcp/src/tools/registry.rs b/crates/mcp/src/tools/registry.rs new file mode 100644 index 00000000..6ee5ecce --- /dev/null +++ b/crates/mcp/src/tools/registry.rs @@ -0,0 +1,158 @@ +use crate::indexing::{rebuild_search_index, wait_for_index_ready}; +use crate::server::ApxServer; +use crate::tools::ToolResultExt; +use crate::validation::validate_app_path; +use apx_core::components::{needs_registry_refresh, sync_registry_indexes}; +use apx_core::search::ComponentIndex; +use rmcp::model::*; +use rmcp::schemars; + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct SearchRegistryComponentsArgs { + /// Absolute path to the project directory + pub app_path: String, + /// Search query + pub query: String, + /// Maximum number of results (default: 10) + #[serde(default = "default_search_limit")] + pub limit: usize, +} + +fn default_search_limit() -> usize { + 10 +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct AddComponentArgs { + /// Absolute path to the project directory + pub app_path: String, + /// Component ID: "component-name" or "@registry-name/component-name" + pub component_id: String, + /// Force overwrite existing files + #[serde(default)] + pub force: bool, +} + +impl ApxServer { + pub async fn handle_search_registry_components( + &self, + args: SearchRegistryComponentsArgs, + ) -> Result { + let path = validate_app_path(&args.app_path) + .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; + + let ctx = &self.ctx; + + // Wait for component index to be ready (15 second timeout) + if let Err(e) = wait_for_index_ready( + &ctx.index_state.component_ready, + &ctx.index_state.component_indexed, + "Component", + ) + .await + { + return Ok(CallToolResult::error(vec![Content::text(e)])); + } + + // Check if registry indexes need refresh + if let Ok(metadata) = apx_core::common::read_project_metadata(&path) { + let cfg = apx_core::components::UiConfig::from_metadata(&metadata, &path); + if needs_registry_refresh(&cfg.registries) { + tracing::info!("Registry indexes stale, refreshing..."); + if let Ok(true) = sync_registry_indexes(&path, false).await { + let rebuild_result = tokio::task::spawn_blocking(rebuild_search_index).await; + if let Ok(Err(e)) = rebuild_result { + tracing::warn!("Failed to rebuild search index after refresh: {}", e); + } + } + } + } + + // Search in spawn_blocking (sync SQLite operations) + let search_query = args.query.clone(); + let limit = args.limit; + let search_results = match tokio::task::spawn_blocking(move || { + let index = ComponentIndex::new()?; + index.search(&search_query, limit) + }) + .await + { + Ok(Ok(results)) => results, + Ok(Err(e)) => { + return Ok(CallToolResult::error(vec![Content::text(format!( + "Search failed: {e}" + ))])) + } + Err(e) => { + return Ok(CallToolResult::error(vec![Content::text(format!( + "Search task panicked: {e}" + ))])) + } + }; + + #[derive(serde::Serialize)] + struct SearchResponse { + query: String, + results: Vec, + } + + #[derive(serde::Serialize)] + struct SearchResultItem { + id: String, + name: String, + registry: String, + score: f32, + } + + let response = SearchResponse { + query: args.query, + results: search_results + .into_iter() + .map(|r| SearchResultItem { + id: r.id, + name: r.name, + registry: r.registry, + score: r.score, + }) + .collect(), + }; + + Ok(CallToolResult::from_serializable(&response)) + } + + pub async fn handle_add_component( + &self, + args: AddComponentArgs, + ) -> Result { + let path = validate_app_path(&args.app_path) + .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; + + use apx_core::components::add::{ComponentInput, add_components}; + + let input = if args.component_id.starts_with('@') { + if let Some((prefix, name)) = args.component_id.split_once('/') { + ComponentInput::with_registry(name, prefix) + } else { + return Ok(CallToolResult::error(vec![Content::text(format!( + "Invalid component ID format: {}. Expected '@registry-name/component-name'", + args.component_id + ))])); + } + } else { + ComponentInput::new(args.component_id.clone()) + }; + + match add_components(&path, &[input], args.force).await { + Ok(_result) => { + tracing::info!("Component {} added successfully", args.component_id); + Ok(CallToolResult::success(vec![Content::text(format!( + "Successfully added component: {}", + args.component_id + ))])) + } + Err(e) => Ok(CallToolResult::error(vec![Content::text(format!( + "Failed to add component: {e}" + ))])), + } + } +} diff --git a/crates/mcp/src/validation.rs b/crates/mcp/src/validation.rs new file mode 100644 index 00000000..8cb88eb8 --- /dev/null +++ b/crates/mcp/src/validation.rs @@ -0,0 +1,58 @@ +use std::path::PathBuf; + +/// Validate that `app_path` is an absolute path to an existing directory. +/// Returns the canonicalized path (symlinks and `..` segments resolved). +pub fn validate_app_path(app_path: &str) -> Result { + let path = PathBuf::from(app_path); + if !path.is_absolute() { + return Err(format!( + "app_path must be an absolute path, got: {app_path}" + )); + } + if !path.exists() { + return Err(format!("app_path does not exist: {app_path}")); + } + if !path.is_dir() { + return Err(format!("app_path is not a directory: {app_path}")); + } + path.canonicalize() + .map_err(|e| format!("Failed to canonicalize path '{app_path}': {e}")) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + use std::fs; + + #[test] + fn rejects_relative_path() { + let err = validate_app_path("relative/path").unwrap_err(); + assert!(err.contains("absolute path"), "got: {err}"); + } + + #[test] + fn rejects_nonexistent_path() { + let err = validate_app_path("/tmp/__apx_test_nonexistent_dir__").unwrap_err(); + assert!(err.contains("does not exist"), "got: {err}"); + } + + #[test] + fn rejects_file_not_dir() { + let tmp = std::env::temp_dir().join("apx_test_validate_file"); + fs::write(&tmp, "").unwrap(); + let err = validate_app_path(tmp.to_str().unwrap()).unwrap_err(); + assert!(err.contains("not a directory"), "got: {err}"); + fs::remove_file(&tmp).unwrap(); + } + + #[test] + fn valid_dir_returns_canonical_path() { + let tmp = std::env::temp_dir().join("apx_test_validate_dir"); + fs::create_dir_all(&tmp).unwrap(); + let result = validate_app_path(tmp.to_str().unwrap()).unwrap(); + // canonicalize resolves symlinks, so compare canonical forms + assert_eq!(result, tmp.canonicalize().unwrap()); + fs::remove_dir(&tmp).unwrap(); + } +} From fe519389a5081c0d46132cd96285df8f83348f08 Mon Sep 17 00:00:00 2001 From: renardeinside Date: Tue, 17 Feb 2026 16:53:00 +0100 Subject: [PATCH 2/9] =?UTF-8?q?=F0=9F=8E=A8=20style:=20apply=20cargo=20fmt?= =?UTF-8?q?=20formatting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/mcp/src/server.rs | 9 ++++----- crates/mcp/src/tools/databricks.rs | 11 +++-------- crates/mcp/src/tools/devserver.rs | 15 +++------------ crates/mcp/src/tools/docs.rs | 7 ++----- crates/mcp/src/tools/project.rs | 23 ++++++++++------------- crates/mcp/src/tools/registry.rs | 4 ++-- 6 files changed, 24 insertions(+), 45 deletions(-) diff --git a/crates/mcp/src/server.rs b/crates/mcp/src/server.rs index b3085dc4..b7f0793e 100644 --- a/crates/mcp/src/server.rs +++ b/crates/mcp/src/server.rs @@ -218,7 +218,9 @@ impl ServerHandler for ApxServer { title: Some("apx - the toolkit for building Databricks Apps".into()), description: None, icons: None, - website_url: Some("https://databricks-solutions.github.io/apx/docs/reference/mcp".into()), + website_url: Some( + "https://databricks-solutions.github.io/apx/docs/reference/mcp".into(), + ), }, instructions: Some(APX_INFO_CONTENT.to_string()), } @@ -250,10 +252,7 @@ impl ServerHandler for ApxServer { } } -pub async fn run_server( - ctx: AppContext, - sdk_params: Option, -) -> Result<(), String> { +pub async fn run_server(ctx: AppContext, sdk_params: Option) -> Result<(), String> { use rmcp::ServiceExt; let shutdown_tx = ctx.shutdown_tx.clone(); diff --git a/crates/mcp/src/tools/databricks.rs b/crates/mcp/src/tools/databricks.rs index f291dbda..3d68e8cf 100644 --- a/crates/mcp/src/tools/databricks.rs +++ b/crates/mcp/src/tools/databricks.rs @@ -32,9 +32,7 @@ pub(crate) fn truncate(s: &str, max_chars: i32) -> String { format!("{head}\n\n...[truncated {truncated} chars]...\n\n{tail}") } -pub(crate) fn resolve_app_name_from_databricks_yml( - project_dir: &Path, -) -> Result { +pub(crate) fn resolve_app_name_from_databricks_yml(project_dir: &Path) -> Result { let yml_path = project_dir.join("databricks.yml"); if !yml_path.exists() { return Err(format!( @@ -226,11 +224,8 @@ impl ApxServer { // Run command with timeout let start = Instant::now(); - let result = tokio::time::timeout( - Duration::from_secs_f64(args.timeout_seconds), - cmd.output(), - ) - .await; + let result = + tokio::time::timeout(Duration::from_secs_f64(args.timeout_seconds), cmd.output()).await; let (returncode, stdout, stderr, duration_ms) = match result { Ok(Ok(cmd_output)) => { diff --git a/crates/mcp/src/tools/devserver.rs b/crates/mcp/src/tools/devserver.rs index 9975f9d3..20b20bc1 100644 --- a/crates/mcp/src/tools/devserver.rs +++ b/crates/mcp/src/tools/devserver.rs @@ -19,10 +19,7 @@ fn default_logs_duration() -> String { } impl ApxServer { - pub async fn handle_start( - &self, - args: AppPathArgs, - ) -> Result { + pub async fn handle_start(&self, args: AppPathArgs) -> Result { let path = validate_app_path(&args.app_path) .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; @@ -37,10 +34,7 @@ impl ApxServer { } } - pub async fn handle_stop( - &self, - args: AppPathArgs, - ) -> Result { + pub async fn handle_stop(&self, args: AppPathArgs) -> Result { let path = validate_app_path(&args.app_path) .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; @@ -74,10 +68,7 @@ impl ApxServer { } } - pub async fn handle_logs( - &self, - args: LogsToolArgs, - ) -> Result { + pub async fn handle_logs(&self, args: LogsToolArgs) -> Result { let path = validate_app_path(&args.app_path) .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; diff --git a/crates/mcp/src/tools/docs.rs b/crates/mcp/src/tools/docs.rs index 9d3dd28e..944136ac 100644 --- a/crates/mcp/src/tools/docs.rs +++ b/crates/mcp/src/tools/docs.rs @@ -22,10 +22,7 @@ fn default_docs_limit() -> usize { } impl ApxServer { - pub async fn handle_docs( - &self, - args: DocsArgs, - ) -> Result { + pub async fn handle_docs(&self, args: DocsArgs) -> Result { let ctx = &self.ctx; // Wait for SDK index to be ready (15 second timeout) @@ -46,7 +43,7 @@ impl ApxServer { Some(idx) => idx, None => { return Ok(CallToolResult::error(vec![Content::text( - "SDK documentation is not available. The Databricks SDK may not be installed or the index failed to bootstrap." + "SDK documentation is not available. The Databricks SDK may not be installed or the index failed to bootstrap.", )])); } }; diff --git a/crates/mcp/src/tools/project.rs b/crates/mcp/src/tools/project.rs index 3a5fa156..502da653 100644 --- a/crates/mcp/src/tools/project.rs +++ b/crates/mcp/src/tools/project.rs @@ -44,9 +44,9 @@ fn parse_openapi_operations(openapi: &Value) -> Result, String> { continue; } - let operation_obj = operation.as_object().ok_or_else(|| { - format!("Operation '{method}' at path '{path}' is not an object") - })?; + let operation_obj = operation + .as_object() + .ok_or_else(|| format!("Operation '{method}' at path '{path}' is not an object"))?; let operation_id = operation_obj .get("operationId") @@ -148,10 +148,7 @@ const Component = () => {{ } impl ApxServer { - pub async fn handle_check( - &self, - args: AppPathArgs, - ) -> Result { + pub async fn handle_check(&self, args: AppPathArgs) -> Result { let path = validate_app_path(&args.app_path) .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; @@ -226,7 +223,7 @@ impl ApxServer { Err(e) => { return Ok(CallToolResult::error(vec![Content::text(format!( "Failed to generate OpenAPI spec: {e}" - ))])) + ))])); } }; @@ -235,7 +232,7 @@ impl ApxServer { Err(e) => { return Ok(CallToolResult::error(vec![Content::text(format!( "Failed to parse OpenAPI schema: {e}" - ))])) + ))])); } }; @@ -244,7 +241,7 @@ impl ApxServer { None => { return Ok(CallToolResult::error(vec![Content::text( "OpenAPI schema missing 'paths' object", - )])) + )])); } }; @@ -273,7 +270,7 @@ impl ApxServer { return Ok(CallToolResult::error(vec![Content::text(format!( "Operation ID '{}' not found in OpenAPI schema", args.operation_id - ))])) + ))])); } }; @@ -320,7 +317,7 @@ impl ApxServer { Err(e) => { return Ok(CallToolResult::error(vec![Content::text(format!( "Failed to generate OpenAPI spec: {e}" - ))])) + ))])); } }; @@ -329,7 +326,7 @@ impl ApxServer { Err(e) => { return Ok(CallToolResult::error(vec![Content::text(format!( "Failed to parse OpenAPI schema: {e}" - ))])) + ))])); } }; diff --git a/crates/mcp/src/tools/registry.rs b/crates/mcp/src/tools/registry.rs index 6ee5ecce..d7713795 100644 --- a/crates/mcp/src/tools/registry.rs +++ b/crates/mcp/src/tools/registry.rs @@ -81,12 +81,12 @@ impl ApxServer { Ok(Err(e)) => { return Ok(CallToolResult::error(vec![Content::text(format!( "Search failed: {e}" - ))])) + ))])); } Err(e) => { return Ok(CallToolResult::error(vec![Content::text(format!( "Search task panicked: {e}" - ))])) + ))])); } }; From c95f37ed9c737fe50b4759e8a2da311bdae36047 Mon Sep 17 00:00:00 2001 From: renardeinside Date: Tue, 17 Feb 2026 17:58:53 +0100 Subject: [PATCH 3/9] =?UTF-8?q?=E2=9C=A8=20feat:=20global=20SDK=20docs=20w?= =?UTF-8?q?ith=20per-project=20version=20detection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/cli/src/dev/mcp.rs | 26 +++++++---- crates/core/src/interop.rs | 64 ++++++++++++++++++++++++++++ crates/core/src/search/docs_index.rs | 38 +++++++++-------- crates/mcp/src/context.rs | 2 +- crates/mcp/src/indexing.rs | 20 +-------- crates/mcp/src/tools/docs.rs | 39 +++++++++++++++-- 6 files changed, 140 insertions(+), 49 deletions(-) diff --git a/crates/cli/src/dev/mcp.rs b/crates/cli/src/dev/mcp.rs index 0b096c33..76578013 100644 --- a/crates/cli/src/dev/mcp.rs +++ b/crates/cli/src/dev/mcp.rs @@ -22,18 +22,26 @@ pub async fn run(_args: McpArgs) -> i32 { let cache_state = new_cache_state(); // Get SDK version via subprocess before spawning async task + const DEFAULT_SDK_VERSION: &str = "0.89.0"; let sdk_version = match get_databricks_sdk_version() { - Ok(version) => { - if let Some(ref v) = version { - tracing::info!("Found Databricks SDK version: {}", v); - } else { - tracing::debug!("Databricks SDK not installed"); - } - version + Ok(Some(v)) => { + tracing::info!("Found Databricks SDK version: {}", v); + v + } + Ok(None) => { + tracing::info!( + "Databricks SDK not installed, using default version {}", + DEFAULT_SDK_VERSION + ); + DEFAULT_SDK_VERSION.to_string() } Err(e) => { - tracing::warn!("Failed to get Databricks SDK version: {}", e); - None + tracing::warn!( + "Failed to detect SDK version: {}, using default {}", + e, + DEFAULT_SDK_VERSION + ); + DEFAULT_SDK_VERSION.to_string() } }; diff --git a/crates/core/src/interop.rs b/crates/core/src/interop.rs index f512203d..ac5312e3 100644 --- a/crates/core/src/interop.rs +++ b/crates/core/src/interop.rs @@ -164,6 +164,70 @@ print(json.dumps(app.openapi(), indent=2)) Ok((spec_json, app_slug.to_string())) } +/// Get the Databricks SDK version for a specific project directory via subprocess. +/// +/// Uses `uv run --directory ` to run in the project's venv context. +pub fn get_databricks_sdk_version_for_project( + project_dir: &Path, +) -> Result, String> { + debug!( + "get_databricks_sdk_version_for_project: checking project at {}", + project_dir.display() + ); + + let uv_path = match try_resolve_uv() { + Ok(resolved) => resolved.path, + Err(e) => { + debug!("get_databricks_sdk_version_for_project: failed to resolve uv: {e}"); + return Ok(None); + } + }; + + let dir_str = project_dir.to_str().unwrap_or("."); + let output = Command::new(&uv_path) + .args([ + "run", + "--directory", + dir_str, + "--no-sync", + "python", + "-c", + "import importlib.metadata; print(importlib.metadata.version('databricks-sdk'))", + ]) + .output(); + + match output { + Ok(o) if o.status.success() => { + let version = String::from_utf8_lossy(&o.stdout).trim().to_string(); + if version.is_empty() { + debug!("get_databricks_sdk_version_for_project: empty output"); + Ok(None) + } else { + debug!( + "get_databricks_sdk_version_for_project: found version {}", + version + ); + Ok(Some(version)) + } + } + Ok(o) => { + let stderr = String::from_utf8_lossy(&o.stderr); + debug!( + "get_databricks_sdk_version_for_project: subprocess failed: {}", + stderr + ); + Ok(None) + } + Err(e) => { + debug!( + "get_databricks_sdk_version_for_project: failed to run uv: {}", + e + ); + Ok(None) + } + } +} + /// Get the installed Databricks SDK version via subprocess pub fn get_databricks_sdk_version() -> Result, String> { debug!("get_databricks_sdk_version: Starting subprocess call"); diff --git a/crates/core/src/search/docs_index.rs b/crates/core/src/search/docs_index.rs index 16c015ff..c2c5bb16 100644 --- a/crates/core/src/search/docs_index.rs +++ b/crates/core/src/search/docs_index.rs @@ -7,7 +7,6 @@ use std::sync::{Arc, Mutex}; use crate::common::Timer; use crate::databricks_sdk_doc::{SDKSource, download_and_extract_sdk, load_doc_files}; -use crate::interop::get_databricks_sdk_version; use crate::search::common; const CHUNK_SIZE: usize = 2000; // characters (no tokenizer needed for FTS) @@ -170,18 +169,6 @@ impl SDKDocsIndex { common::table_exists(&conn, table_name) } - /// Bootstrap: download docs and build index - #[allow(dead_code)] - pub async fn bootstrap(&mut self, source: &SDKSource) -> Result { - match source { - SDKSource::DatabricksSdkPython => { - let version = get_databricks_sdk_version()? - .ok_or_else(|| "databricks-sdk is not installed".to_string())?; - self.bootstrap_with_version(source, &version).await - } - } - } - /// Bootstrap with a pre-computed SDK version pub async fn bootstrap_with_version( &mut self, @@ -334,6 +321,22 @@ impl SDKDocsIndex { Ok(()) } + /// Switch to a different SDK version, bootstrapping it if needed. + /// + /// This is cheap when the version is already indexed (just a `table_exists` check), + /// and lazy-downloads otherwise. + pub async fn ensure_version( + &mut self, + source: &SDKSource, + version: &str, + ) -> Result<(), String> { + if self.version.as_deref() == Some(version) { + return Ok(()); // Already on this version + } + self.bootstrap_with_version(source, version).await?; + Ok(()) + } + /// Search for relevant documentation chunks using FTS5 (sync) pub fn search_sync( &self, @@ -343,12 +346,11 @@ impl SDKDocsIndex { ) -> Result, String> { match source { SDKSource::DatabricksSdkPython => { - let version = get_databricks_sdk_version()? - .ok_or_else(|| { - "databricks-sdk is not installed. Please install databricks-sdk to use this feature.".to_string() - })?; + let version = self.version.as_ref().ok_or_else(|| { + "SDK docs index not initialized. No version has been bootstrapped.".to_string() + })?; - let table_name = Self::table_name(&version); + let table_name = Self::table_name(version); let conn = self.conn.lock().map_err(|e| format!("Lock error: {e}"))?; diff --git a/crates/mcp/src/context.rs b/crates/mcp/src/context.rs index 4bb7cc6d..1b17af96 100644 --- a/crates/mcp/src/context.rs +++ b/crates/mcp/src/context.rs @@ -7,7 +7,7 @@ use tokio::sync::{Mutex, Notify, broadcast}; /// Parameters for SDK indexing, pre-computed synchronously to avoid Python GIL issues #[derive(Debug)] pub struct SdkIndexParams { - pub sdk_version: Option, + pub sdk_version: String, pub sdk_doc_index: Arc>>, } diff --git a/crates/mcp/src/indexing.rs b/crates/mcp/src/indexing.rs index 344d565b..d02e5dfe 100644 --- a/crates/mcp/src/indexing.rs +++ b/crates/mcp/src/indexing.rs @@ -59,24 +59,8 @@ pub fn init_all_indexes( if let Some(params) = sdk_params { tracing::info!("Initializing Databricks SDK documentation index"); - let version = match params.sdk_version { - Some(v) => { - tracing::debug!("Using pre-computed SDK version: {}", v); - v - } - None => { - tracing::warn!( - "Databricks SDK not installed. The docs tool will not be available." - ); - index_state.sdk_indexed.store(true, Ordering::SeqCst); - index_state.sdk_ready.notify_waiters(); - - // Mark as done - let mut guard = cache_state.lock().await; - guard.is_running = false; - return; - } - }; + let version = params.sdk_version; + tracing::debug!("Using SDK version: {}", version); // Create SDK docs index (sync, but cheap) let mut index = match apx_core::search::docs_index::SDKDocsIndex::new() { diff --git a/crates/mcp/src/tools/docs.rs b/crates/mcp/src/tools/docs.rs index 944136ac..9bb5caf6 100644 --- a/crates/mcp/src/tools/docs.rs +++ b/crates/mcp/src/tools/docs.rs @@ -2,6 +2,7 @@ use crate::indexing::wait_for_index_ready; use crate::server::ApxServer; use crate::tools::ToolResultExt; use apx_core::databricks_sdk_doc::SDKSource; +use apx_core::interop::get_databricks_sdk_version_for_project; use rmcp::model::*; use rmcp::schemars; use serde::Serialize; @@ -15,6 +16,9 @@ pub struct DocsArgs { /// Maximum number of results to return (default: 5) #[serde(default = "default_docs_limit")] pub num_results: usize, + /// Optional project path. When provided, detects and uses the project's SDK version. + #[serde(default)] + pub app_path: Option, } fn default_docs_limit() -> usize { @@ -37,17 +41,46 @@ impl ApxServer { } // Get the SDK doc index - let index_guard = ctx.sdk_doc_index.lock().await; + let mut index_guard = ctx.sdk_doc_index.lock().await; - let index = match index_guard.as_ref() { + let index = match index_guard.as_mut() { Some(idx) => idx, None => { return Ok(CallToolResult::error(vec![Content::text( - "SDK documentation is not available. The Databricks SDK may not be installed or the index failed to bootstrap.", + "SDK documentation is not available. The index failed to bootstrap.", )])); } }; + // If app_path is provided, detect that project's SDK version and switch if different + if let Some(ref app_path) = args.app_path { + let project_dir = std::path::Path::new(app_path); + match get_databricks_sdk_version_for_project(project_dir) { + Ok(Some(project_version)) => { + if let Err(e) = index.ensure_version(&args.source, &project_version).await { + tracing::warn!( + "Failed to switch to project SDK version {}: {}. Using current version.", + project_version, + e + ); + } + } + Ok(None) => { + tracing::debug!( + "No SDK version detected for project at {}, using default", + app_path + ); + } + Err(e) => { + tracing::debug!( + "Failed to detect SDK version for project at {}: {}", + app_path, + e + ); + } + } + } + match index.search_sync(&args.source, &args.query, args.num_results) { Ok(results) => { drop(index_guard); From 17f5382d15c9b69b6a62533eabab2081b7d5f959 Mon Sep 17 00:00:00 2001 From: renardeinside Date: Tue, 17 Feb 2026 23:53:28 +0100 Subject: [PATCH 4/9] improve db layout --- Cargo.lock | 537 +++++++++++++++++++--- Cargo.toml | 5 +- crates/agent/Cargo.toml | 1 + crates/agent/src/server.rs | 19 +- crates/cli/Cargo.toml | 1 + crates/cli/src/dev/logs.rs | 25 +- crates/cli/src/dev/mcp.rs | 6 + crates/common/Cargo.toml | 1 - crates/common/src/lib.rs | 4 +- crates/common/src/storage.rs | 424 +---------------- crates/core/Cargo.toml | 3 +- crates/core/src/flux/mod.rs | 4 +- crates/core/src/ops/dev.rs | 4 +- crates/core/src/ops/logs.rs | 23 +- crates/core/src/ops/startup_logs.rs | 26 +- crates/core/src/search/common.rs | 156 +------ crates/core/src/search/component_index.rs | 239 +++++----- crates/core/src/search/docs_index.rs | 229 +++++---- crates/db/Cargo.toml | 14 + crates/db/src/dev.rs | 141 ++++++ crates/db/src/lib.rs | 37 ++ crates/db/src/logs.rs | 419 +++++++++++++++++ crates/mcp/Cargo.toml | 1 + crates/mcp/src/context.rs | 2 + crates/mcp/src/indexing.rs | 50 +- crates/mcp/src/tools/docs.rs | 5 +- crates/mcp/src/tools/registry.rs | 26 +- 27 files changed, 1443 insertions(+), 959 deletions(-) create mode 100644 crates/db/Cargo.toml create mode 100644 crates/db/src/dev.rs create mode 100644 crates/db/src/lib.rs create mode 100644 crates/db/src/logs.rs diff --git a/Cargo.lock b/Cargo.lock index c9b6e08e..efd9150f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,6 +52,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -122,6 +128,7 @@ name = "apx-agent" version = "0.3.0-rc1" dependencies = [ "apx-common", + "apx-db", "axum", "chrono", "clap", @@ -150,6 +157,7 @@ dependencies = [ "apx-common", "apx-core", "apx-databricks-sdk", + "apx-db", "apx-mcp", "chrono", "clap", @@ -175,7 +183,6 @@ version = "0.3.0-rc1" dependencies = [ "chrono", "dirs 5.0.1", - "rusqlite", "serde", "serde_json", "tracing", @@ -188,6 +195,7 @@ dependencies = [ "apx-agent", "apx-common", "apx-databricks-sdk", + "apx-db", "axum", "biome_css_parser", "biome_css_syntax", @@ -208,7 +216,6 @@ dependencies = [ "rand 0.8.5", "rayon", "reqwest 0.13.1", - "rusqlite", "schemars 1.2.0", "serde", "serde_json", @@ -216,6 +223,7 @@ dependencies = [ "serde_yaml", "sha2", "similar", + "sqlx", "sysinfo", "tar", "tempfile", @@ -248,11 +256,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "apx-db" +version = "0.3.0-rc1" +dependencies = [ + "apx-common", + "dirs 5.0.1", + "sqlx", + "tokio", + "tracing", +] + [[package]] name = "apx-mcp" version = "0.3.0-rc1" dependencies = [ "apx-core", + "apx-db", "rmcp", "schemars 1.2.0", "serde", @@ -317,6 +337,15 @@ dependencies = [ "system-deps", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -433,6 +462,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "biome_console" version = "0.5.8" @@ -964,6 +999,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "console" version = "0.15.11" @@ -989,6 +1033,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "constant_time_eq" version = "0.3.1" @@ -1128,6 +1178,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1276,6 +1335,17 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.5.5" @@ -1336,6 +1406,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", "subtle", ] @@ -1432,6 +1503,12 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dpi" version = "0.1.2" @@ -1479,6 +1556,9 @@ name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +dependencies = [ + "serde", +] [[package]] name = "embed-resource" @@ -1569,22 +1649,32 @@ dependencies = [ ] [[package]] -name = "fallible-iterator" -version = "0.2.0" +name = "etcetera" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] [[package]] -name = "fallible-iterator" -version = "0.3.0" +name = "event-listener" +version = "5.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] [[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" +name = "fallible-iterator" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastrand" @@ -1638,6 +1728,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1646,9 +1747,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.2.0" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" [[package]] name = "foreign-types" @@ -1753,6 +1854,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.31" @@ -2167,20 +2279,28 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + [[package]] name = "hashlink" -version = "0.11.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea0b22561a9c04a7cb1a302c013e0259cd3b4bb619f145b32f72b8b4bcbed230" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" dependencies = [ - "hashbrown 0.16.1", + "hashbrown 0.15.5", ] [[package]] @@ -2201,6 +2321,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + [[package]] name = "hmac" version = "0.12.1" @@ -2210,6 +2339,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "html5ever" version = "0.29.1" @@ -2767,6 +2905,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "libappindicator" @@ -2827,9 +2968,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.36.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b4103cffefa72eb8428cb6b47d6627161e51c2739fc5e3b734584157bc642a" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ "cc", "pkg-config", @@ -3103,12 +3244,48 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + [[package]] name = "num-conv" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -3116,6 +3293,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -3576,6 +3754,12 @@ dependencies = [ "system-deps", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.5" @@ -3624,6 +3808,15 @@ dependencies = [ "hmac", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -3858,6 +4051,27 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -3905,7 +4119,7 @@ dependencies = [ "base64 0.22.1", "byteorder", "bytes", - "fallible-iterator 0.2.0", + "fallible-iterator", "hmac", "md-5", "memchr", @@ -3921,7 +4135,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b858f82211e84682fecd373f68e1ceae642d8d751a1ebd13f33de6257b3e20" dependencies = [ "bytes", - "fallible-iterator 0.2.0", + "fallible-iterator", "postgres-protocol", ] @@ -4531,28 +4745,23 @@ dependencies = [ ] [[package]] -name = "rsqlite-vfs" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a1f2315036ef6b1fbacd1972e8ee7688030b0a2121edfc2a6550febd41574d" -dependencies = [ - "hashbrown 0.16.1", - "thiserror 2.0.18", -] - -[[package]] -name = "rusqlite" -version = "0.38.0" +name = "rsa" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1c93dd1c9683b438c392c492109cb702b8090b2bfc8fed6f6e4eb4523f17af3" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "bitflags 2.10.0", - "fallible-iterator 0.3.0", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", - "sqlite-wasm-rs", + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", ] [[package]] @@ -5077,6 +5286,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "simd-adler32" version = "0.3.8" @@ -5132,6 +5351,9 @@ name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] [[package]] name = "socket2" @@ -5192,15 +5414,210 @@ dependencies = [ ] [[package]] -name = "sqlite-wasm-rs" -version = "0.5.2" +name = "spin" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4206ed3a67690b9c29b77d728f6acc3ce78f16bf846d83c94f76400320181b" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" dependencies = [ - "cc", - "js-sys", - "rsqlite-vfs", - "wasm-bindgen", + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64 0.22.1", + "bytes", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap 2.13.0", + "log", + "memchr", + "once_cell", + "percent-encoding", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.114", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "dotenvy", + "either", + "heck 0.5.0", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.114", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.10.0", + "byteorder", + "bytes", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.5", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.18", + "tracing", + "whoami 1.6.1", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.10.0", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.5", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.18", + "tracing", + "whoami 1.6.1", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +dependencies = [ + "atoi", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror 2.0.18", + "tracing", + "url", ] [[package]] @@ -5837,7 +6254,7 @@ dependencies = [ "async-trait", "byteorder", "bytes", - "fallible-iterator 0.2.0", + "fallible-iterator", "futures-channel", "futures-util", "log", @@ -5851,7 +6268,7 @@ dependencies = [ "socket2", "tokio", "tokio-util", - "whoami", + "whoami 2.1.0", ] [[package]] @@ -6529,6 +6946,12 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasite" version = "1.0.2" @@ -6749,6 +7172,16 @@ dependencies = [ "winsafe", ] +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite 0.1.0", +] + [[package]] name = "whoami" version = "2.1.0" @@ -6756,7 +7189,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fae98cf96deed1b7572272dfc777713c249ae40aa1cf8862e091e8b745f5361" dependencies = [ "libredox", - "wasite", + "wasite 1.0.2", "web-sys", ] diff --git a/Cargo.toml b/Cargo.toml index 11d4cd86..e71d5a7a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["crates/common", "crates/agent", "crates/studio", "crates/core", "crates/mcp", "crates/cli", "crates/databricks_sdk", "crates/apx"] +members = ["crates/common", "crates/agent", "crates/studio", "crates/core", "crates/mcp", "crates/cli", "crates/databricks_sdk", "crates/apx", "crates/db"] resolver = "2" [workspace.package] @@ -15,6 +15,7 @@ apx-core = { path = "crates/core" } apx-mcp = { path = "crates/mcp" } apx-cli = { path = "crates/cli" } apx-databricks-sdk = { path = "crates/databricks_sdk" } +apx-db = { path = "crates/db" } # Serialization serde = { version = "1.0.228", features = ["derive"] } @@ -24,7 +25,7 @@ toml = "0.8.20" toml_edit = "0.24.0" # Database -rusqlite = { version = "0.38.0", features = ["bundled"] } +sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite"] } # Async runtime tokio = { version = "1.49", features = ["rt-multi-thread", "macros", "sync", "process", "io-util", "signal", "io-std", "net"] } diff --git a/crates/agent/Cargo.toml b/crates/agent/Cargo.toml index 53b5c44d..5f11fcfe 100644 --- a/crates/agent/Cargo.toml +++ b/crates/agent/Cargo.toml @@ -10,6 +10,7 @@ path = "src/main.rs" [dependencies] apx-common.workspace = true +apx-db.workspace = true axum.workspace = true tokio.workspace = true serde.workspace = true diff --git a/crates/agent/src/server.rs b/crates/agent/src/server.rs index 83040671..d5ab2a55 100644 --- a/crates/agent/src/server.rs +++ b/crates/agent/src/server.rs @@ -3,7 +3,8 @@ //! This module implements an Axum HTTP server that receives OpenTelemetry logs //! via OTLP HTTP protocol, supporting both JSON and Protobuf content types. -use apx_common::{FLUX_PORT, LogRecord, Storage}; +use apx_common::{FLUX_PORT, LogRecord}; +use apx_db::LogsDb; use axum::{ Router, body::Bytes, @@ -21,7 +22,7 @@ use tracing::{debug, error, info}; /// Application state shared across handlers. #[derive(Clone, Debug)] struct AppState { - storage: Storage, + storage: LogsDb, } /// Run the flux server (entry point for `apx-agent`). @@ -34,7 +35,9 @@ pub async fn run_server() -> Result<(), String> { eprintln!("[{now}] Flux daemon starting..."); // Open storage - let storage = Storage::open()?; + let storage = LogsDb::open() + .await + .map_err(|e| format!("Storage error: {e}"))?; eprintln!("[{now}] Storage initialized"); // Start cleanup scheduler as a background task @@ -49,14 +52,14 @@ pub async fn run_server() -> Result<(), String> { /// Periodic cleanup loop that runs within the daemon process. /// Deletes logs older than 7 days every hour. -async fn run_cleanup_loop(storage: Storage) { +async fn run_cleanup_loop(storage: LogsDb) { // Cleanup interval: 1 hour let interval = Duration::from_secs(60 * 60); info!("Cleanup scheduler started (interval: 1 hour, retention: 7 days)"); // Run initial cleanup - match storage.cleanup_old_logs() { + match storage.cleanup_old_logs().await { Ok(deleted) if deleted > 0 => info!("Initial cleanup: removed {} old log records", deleted), Ok(_) => debug!("Initial cleanup: no old records to remove"), Err(e) => error!("Initial cleanup failed: {}", e), @@ -65,7 +68,7 @@ async fn run_cleanup_loop(storage: Storage) { loop { tokio::time::sleep(interval).await; - match storage.cleanup_old_logs() { + match storage.cleanup_old_logs().await { Ok(deleted) if deleted > 0 => { info!("Cleanup: removed {} old log records", deleted); } @@ -80,7 +83,7 @@ async fn run_cleanup_loop(storage: Storage) { } /// Start the flux HTTP server with the given storage. -async fn run_http_server(storage: Storage) -> Result<(), String> { +async fn run_http_server(storage: LogsDb) -> Result<(), String> { let state = AppState { storage }; let app = Router::new() @@ -143,7 +146,7 @@ async fn handle_logs( debug!("Received {} log records", records.len()); - match state.storage.insert_logs(&records) { + match state.storage.insert_logs(&records).await { Ok(count) => { debug!("Stored {} log records", count); StatusCode::OK diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index ca8333dc..75986742 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -7,6 +7,7 @@ rust-version.workspace = true [dependencies] apx-common = { path = "../common" , version = "0.3.0-rc1" } apx-core = { path = "../core" , version = "0.3.0-rc1" } +apx-db.workspace = true apx-databricks-sdk = { path = "../databricks_sdk" , version = "0.3.0-rc1" } apx-mcp = { path = "../mcp" , version = "0.3.0-rc1" } clap.workspace = true diff --git a/crates/cli/src/dev/logs.rs b/crates/cli/src/dev/logs.rs index 80321c18..919fb07c 100644 --- a/crates/cli/src/dev/logs.rs +++ b/crates/cli/src/dev/logs.rs @@ -9,12 +9,13 @@ use tracing::debug; use crate::common::resolve_app_dir; use crate::run_cli_async_helper; -use apx_common::{LogAggregator, Storage, db_path, should_skip_log}; +use apx_common::{LogAggregator, should_skip_log}; use apx_core::dev::common::{lock_path, read_lock}; use apx_core::ops::logs::{ DEFAULT_LOG_DURATION, format_aggregated_record, format_log_record, parse_duration, since_timestamp_nanos, }; +use apx_db::LogsDb; #[derive(Args, Debug, Clone)] pub struct LogsArgs { @@ -59,7 +60,7 @@ async fn run_async(args: LogsArgs) -> Result<(), String> { } // Check if database exists - let db_path = db_path()?; + let db_path = apx_db::logs_db_path()?; if !db_path.exists() { println!("โš ๏ธ No logs database found at {}\n", db_path.display()); println!("Logs will appear here once the dev server is started and produces output."); @@ -67,7 +68,9 @@ async fn run_async(args: LogsArgs) -> Result<(), String> { } // Open storage - let storage = Storage::open().map_err(|e| format!("Failed to open logs database: {e}"))?; + let storage = LogsDb::open() + .await + .map_err(|e| format!("Failed to open logs database: {e}"))?; let duration = parse_duration(&args.duration)?; let since_ns = since_timestamp_nanos(duration); @@ -76,13 +79,13 @@ async fn run_async(args: LogsArgs) -> Result<(), String> { println!("๐Ÿ“œ Streaming logs... (Ctrl+C to stop)\n"); follow_logs(&storage, &app_path_canonical, since_ns, &lock_path).await } else { - read_logs(&storage, &app_path_canonical, since_ns) + read_logs(&storage, &app_path_canonical, since_ns).await } } /// Read logs from database, filtered by app path and timestamp -fn read_logs(storage: &Storage, app_path: &str, since_ns: i64) -> Result<(), String> { - let records = storage.query_logs(Some(app_path), since_ns, None)?; +async fn read_logs(storage: &LogsDb, app_path: &str, since_ns: i64) -> Result<(), String> { + let records = storage.query_logs(Some(app_path), since_ns, None).await?; let filtered: Vec<_> = records.iter().filter(|r| !should_skip_log(r)).collect(); @@ -118,7 +121,7 @@ fn read_logs(storage: &Storage, app_path: &str, since_ns: i64) -> Result<(), Str /// Follow logs for new entries async fn follow_logs( - storage: &Storage, + storage: &LogsDb, app_path: &str, since_ns: i64, lock_path: &std::path::Path, @@ -126,10 +129,10 @@ async fn follow_logs( use chrono::Utc; // First, read existing logs - read_logs(storage, app_path, since_ns)?; + read_logs(storage, app_path, since_ns).await?; // Track last seen ID for incremental queries - let mut last_id = storage.get_latest_id()?; + let mut last_id = storage.get_latest_id().await?; // Track if server was initially running let server_was_running = lock_path.exists(); @@ -156,7 +159,7 @@ async fn follow_logs( } // Poll for new logs - let new_records = storage.query_logs_after_id(Some(app_path), last_id)?; + let new_records = storage.query_logs_after_id(Some(app_path), last_id).await?; for record in &new_records { if !should_skip_log(record) { @@ -168,7 +171,7 @@ async fn follow_logs( } // Update last_id - if let Ok(new_id) = storage.get_latest_id() + if let Ok(new_id) = storage.get_latest_id().await && new_id > last_id { last_id = new_id; diff --git a/crates/cli/src/dev/mcp.rs b/crates/cli/src/dev/mcp.rs index 76578013..29003cfa 100644 --- a/crates/cli/src/dev/mcp.rs +++ b/crates/cli/src/dev/mcp.rs @@ -1,6 +1,7 @@ use crate::run_cli_async_helper; use apx_core::components::new_cache_state; use apx_core::interop::get_databricks_sdk_version; +use apx_db::DevDb; use apx_mcp::context::{AppContext, IndexState, SdkIndexParams}; use apx_mcp::server::run_server; use clap::Args; @@ -52,7 +53,12 @@ pub async fn run(_args: McpArgs) -> i32 { sdk_doc_index: Arc::clone(&sdk_doc_index), }; + let dev_db = DevDb::open() + .await + .map_err(|e| format!("Failed to open dev database: {e}"))?; + let ctx = AppContext { + dev_db, sdk_doc_index, cache_state, index_state, diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 9873245c..aeeeb3e4 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -7,7 +7,6 @@ rust-version.workspace = true [dependencies] serde.workspace = true serde_json.workspace = true -rusqlite.workspace = true chrono.workspace = true dirs.workspace = true tracing.workspace = true diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 8e272df3..60b41986 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -24,8 +24,8 @@ use std::time::Duration; // Re-export commonly used types pub use storage::{ - AggregatedRecord, LogAggregator, LogRecord, Storage, db_path, flux_dir, get_aggregation_key, - should_skip_log, source_label, + AggregatedRecord, LogAggregator, LogRecord, flux_dir, get_aggregation_key, should_skip_log, + source_label, }; /// Flux port for OTLP HTTP receiver diff --git a/crates/common/src/storage.rs b/crates/common/src/storage.rs index 2cfeacca..48d34185 100644 --- a/crates/common/src/storage.rs +++ b/crates/common/src/storage.rs @@ -1,23 +1,14 @@ -//! SQLite storage for flux OTEL logs. +//! Pure types and logic for flux OTEL logs. //! -//! This module handles all database operations for storing and retrieving -//! OpenTelemetry logs in a local SQLite database. +//! This module contains log record types, filtering, and aggregation logic. +//! Database operations have been moved to the `apx-db` crate. -use rusqlite::{Connection, params}; use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; -use tracing::debug; +use std::path::PathBuf; /// Directory for flux data (~/.apx/logs) const FLUX_DIR: &str = ".apx/logs"; -/// Database filename -const DB_FILENAME: &str = "db"; - -/// Retention period in seconds (7 days) -const RETENTION_SECONDS: i64 = 7 * 24 * 60 * 60; - /// A log record to be inserted into the database. #[derive(Debug, Clone)] pub struct LogRecord { @@ -248,415 +239,8 @@ impl LogAggregator { } } -/// Thread-safe storage handle. -#[derive(Clone)] -pub struct Storage { - conn: Arc>, -} - -impl std::fmt::Debug for Storage { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Storage").finish_non_exhaustive() - } -} - -impl Storage { - /// Open or create the database at the default location (~/.apx/logs/db). - pub fn open() -> Result { - let path = db_path()?; - Self::open_at(&path) - } - - /// Open or create the database at a specific path. - pub fn open_at(path: &Path) -> Result { - // Ensure parent directory exists - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent) - .map_err(|e| format!("Failed to create database directory: {e}"))?; - } - - let conn = Connection::open(path).map_err(|e| format!("Failed to open database: {e}"))?; - - // Enable WAL mode for better concurrency - conn.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;") - .map_err(|e| format!("Failed to set pragmas: {e}"))?; - - let storage = Self { - conn: Arc::new(Mutex::new(conn)), - }; - - storage.init_schema()?; - Ok(storage) - } - - /// Initialize the database schema. - fn init_schema(&self) -> Result<(), String> { - let conn = self.conn.lock().map_err(|e| format!("Lock error: {e}"))?; - - conn.execute_batch( - r#" - CREATE TABLE IF NOT EXISTS logs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - timestamp_ns INTEGER NOT NULL, - observed_timestamp_ns INTEGER NOT NULL, - severity_number INTEGER, - severity_text TEXT, - body TEXT, - service_name TEXT, - app_path TEXT, - resource_attributes TEXT, - log_attributes TEXT, - trace_id TEXT, - span_id TEXT, - created_at INTEGER DEFAULT (strftime('%s', 'now')) - ); - - CREATE INDEX IF NOT EXISTS idx_logs_timestamp ON logs(timestamp_ns); - CREATE INDEX IF NOT EXISTS idx_logs_app_path ON logs(app_path); - CREATE INDEX IF NOT EXISTS idx_logs_service ON logs(service_name); - CREATE INDEX IF NOT EXISTS idx_logs_created ON logs(created_at); - "#, - ) - .map_err(|e| format!("Failed to initialize schema: {e}"))?; - - debug!("Flux storage schema initialized"); - Ok(()) - } - - /// Insert a batch of log records. - pub fn insert_logs(&self, records: &[LogRecord]) -> Result { - if records.is_empty() { - return Ok(0); - } - - let mut conn = self.conn.lock().map_err(|e| format!("Lock error: {e}"))?; - let tx = conn - .transaction() - .map_err(|e| format!("Transaction error: {e}"))?; - - let mut count = 0; - { - let mut stmt = tx - .prepare_cached( - r#" - INSERT INTO logs ( - timestamp_ns, observed_timestamp_ns, severity_number, severity_text, - body, service_name, app_path, resource_attributes, log_attributes, - trace_id, span_id - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) - "#, - ) - .map_err(|e| format!("Prepare error: {e}"))?; - - for record in records { - stmt.execute(params![ - record.timestamp_ns, - record.observed_timestamp_ns, - record.severity_number, - record.severity_text, - record.body, - record.service_name, - record.app_path, - record.resource_attributes, - record.log_attributes, - record.trace_id, - record.span_id, - ]) - .map_err(|e| format!("Insert error: {e}"))?; - count += 1; - } - } - - tx.commit().map_err(|e| format!("Commit error: {e}"))?; - Ok(count) - } - - /// Query logs for a specific app path since a given timestamp. - /// Uses COALESCE to fall back to observed_timestamp_ns when timestamp_ns is 0, - /// which happens with OpenTelemetry tracing bridge logs. - pub fn query_logs( - &self, - app_path: Option<&str>, - since_ns: i64, - limit: Option, - ) -> Result, String> { - let conn = self.conn.lock().map_err(|e| format!("Lock error: {e}"))?; - - // Use effective_ts to handle logs where timestamp_ns is 0 (e.g., APX internal logs) - const EFFECTIVE_TS: &str = "COALESCE(NULLIF(timestamp_ns, 0), observed_timestamp_ns)"; - - let sql = match (app_path, limit) { - (Some(_), Some(lim)) => format!( - r#" - SELECT timestamp_ns, observed_timestamp_ns, severity_number, severity_text, - body, service_name, app_path, resource_attributes, log_attributes, - trace_id, span_id - FROM logs - WHERE (app_path LIKE ?1 OR ?1 LIKE '%' || app_path || '%') - AND {EFFECTIVE_TS} >= ?2 - ORDER BY {EFFECTIVE_TS} ASC - LIMIT {lim} - "# - ), - (Some(_), None) => format!( - r#" - SELECT timestamp_ns, observed_timestamp_ns, severity_number, severity_text, - body, service_name, app_path, resource_attributes, log_attributes, - trace_id, span_id - FROM logs - WHERE (app_path LIKE ?1 OR ?1 LIKE '%' || app_path || '%') - AND {EFFECTIVE_TS} >= ?2 - ORDER BY {EFFECTIVE_TS} ASC - "# - ), - (None, Some(lim)) => format!( - r#" - SELECT timestamp_ns, observed_timestamp_ns, severity_number, severity_text, - body, service_name, app_path, resource_attributes, log_attributes, - trace_id, span_id - FROM logs - WHERE {EFFECTIVE_TS} >= ?1 - ORDER BY {EFFECTIVE_TS} ASC - LIMIT {lim} - "# - ), - (None, None) => format!( - r#" - SELECT timestamp_ns, observed_timestamp_ns, severity_number, severity_text, - body, service_name, app_path, resource_attributes, log_attributes, - trace_id, span_id - FROM logs - WHERE {EFFECTIVE_TS} >= ?1 - ORDER BY {EFFECTIVE_TS} ASC - "# - ), - }; - - let mut stmt = conn - .prepare(&sql) - .map_err(|e| format!("Prepare error: {e}"))?; - - let rows = if let Some(path) = app_path { - let pattern = format!("%{path}%"); - stmt.query_map(params![pattern, since_ns], map_row) - .map_err(|e| format!("Query error: {e}"))? - } else { - stmt.query_map(params![since_ns], map_row) - .map_err(|e| format!("Query error: {e}"))? - }; - - let mut records = Vec::new(); - for row in rows { - records.push(row.map_err(|e| format!("Row error: {e}"))?); - } - - Ok(records) - } - - /// Get the latest log ID for change detection in follow mode. - pub fn get_latest_id(&self) -> Result { - let conn = self.conn.lock().map_err(|e| format!("Lock error: {e}"))?; - let id: i64 = conn - .query_row("SELECT COALESCE(MAX(id), 0) FROM logs", [], |row| { - row.get(0) - }) - .map_err(|e| format!("Query error: {e}"))?; - Ok(id) - } - - /// Query logs newer than a given ID (for follow mode). - /// Uses COALESCE for ordering to handle logs where timestamp_ns is 0. - pub fn query_logs_after_id( - &self, - app_path: Option<&str>, - after_id: i64, - ) -> Result, String> { - let conn = self.conn.lock().map_err(|e| format!("Lock error: {e}"))?; - - // Use effective_ts to handle logs where timestamp_ns is 0 (e.g., APX internal logs) - const EFFECTIVE_TS: &str = "COALESCE(NULLIF(timestamp_ns, 0), observed_timestamp_ns)"; - - let (sql, needs_app_path) = if app_path.is_some() { - ( - format!( - r#" - SELECT timestamp_ns, observed_timestamp_ns, severity_number, severity_text, - body, service_name, app_path, resource_attributes, log_attributes, - trace_id, span_id - FROM logs - WHERE id > ?1 AND (app_path LIKE ?2 OR ?2 LIKE '%' || app_path || '%') - ORDER BY {EFFECTIVE_TS} ASC - "# - ), - true, - ) - } else { - ( - format!( - r#" - SELECT timestamp_ns, observed_timestamp_ns, severity_number, severity_text, - body, service_name, app_path, resource_attributes, log_attributes, - trace_id, span_id - FROM logs - WHERE id > ?1 - ORDER BY {EFFECTIVE_TS} ASC - "# - ), - false, - ) - }; - - let mut stmt = conn - .prepare(&sql) - .map_err(|e| format!("Prepare error: {e}"))?; - - let rows = if let Some(app) = app_path.filter(|_| needs_app_path) { - let pattern = format!("%{app}%"); - stmt.query_map(params![after_id, pattern], map_row) - .map_err(|e| format!("Query error: {e}"))? - } else { - stmt.query_map(params![after_id], map_row) - .map_err(|e| format!("Query error: {e}"))? - }; - - let mut records = Vec::new(); - for row in rows { - records.push(row.map_err(|e| format!("Row error: {e}"))?); - } - - Ok(records) - } - - /// Delete logs older than the retention period (7 days). - pub fn cleanup_old_logs(&self) -> Result { - let conn = self.conn.lock().map_err(|e| format!("Lock error: {e}"))?; - - let cutoff = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs() as i64 - RETENTION_SECONDS) - .unwrap_or(0); - - let deleted = conn - .execute("DELETE FROM logs WHERE created_at < ?1", params![cutoff]) - .map_err(|e| format!("Delete error: {e}"))?; - - if deleted > 0 { - debug!("Cleaned up {} old log records", deleted); - } - - Ok(deleted) - } - - /// Get the total count of logs. - #[allow(dead_code)] - pub fn count_logs(&self) -> Result { - let conn = self.conn.lock().map_err(|e| format!("Lock error: {e}"))?; - let count: i64 = conn - .query_row("SELECT COUNT(*) FROM logs", [], |row| row.get(0)) - .map_err(|e| format!("Query error: {e}"))?; - Ok(count) - } -} - -/// Map a database row to a LogRecord. -fn map_row(row: &rusqlite::Row) -> rusqlite::Result { - Ok(LogRecord { - timestamp_ns: row.get(0)?, - observed_timestamp_ns: row.get(1)?, - severity_number: row.get(2)?, - severity_text: row.get(3)?, - body: row.get(4)?, - service_name: row.get(5)?, - app_path: row.get(6)?, - resource_attributes: row.get(7)?, - log_attributes: row.get(8)?, - trace_id: row.get(9)?, - span_id: row.get(10)?, - }) -} - /// Get the flux directory path (~/.apx/logs). pub fn flux_dir() -> Result { let home = dirs::home_dir().ok_or("Could not determine home directory")?; Ok(home.join(FLUX_DIR)) } - -/// Get the database path (~/.apx/logs/db). -pub fn db_path() -> Result { - Ok(flux_dir()?.join(DB_FILENAME)) -} - -#[cfg(test)] -#[allow(clippy::unwrap_used)] -mod tests { - use super::*; - use std::fs; - use std::sync::atomic::{AtomicU64, Ordering}; - - static TEST_COUNTER: AtomicU64 = AtomicU64::new(0); - - fn temp_db_path() -> PathBuf { - let counter = TEST_COUNTER.fetch_add(1, Ordering::SeqCst); - let dir = std::env::temp_dir().join(format!("apx-test-{}-{}", std::process::id(), counter)); - fs::create_dir_all(&dir).unwrap(); - dir.join("test.db") - } - - #[test] - fn test_storage_create_and_insert() { - let db_path = temp_db_path(); - let storage = Storage::open_at(&db_path).unwrap(); - - let record = LogRecord { - timestamp_ns: 1234567890000000000, - observed_timestamp_ns: 1234567890000000000, - severity_number: Some(9), - severity_text: Some("INFO".to_string()), - body: Some("Test log message".to_string()), - service_name: Some("test_app".to_string()), - app_path: Some("/tmp/test".to_string()), - resource_attributes: None, - log_attributes: None, - trace_id: None, - span_id: None, - }; - - let count = storage.insert_logs(&[record]).unwrap(); - assert_eq!(count, 1); - - let total = storage.count_logs().unwrap(); - assert_eq!(total, 1); - - // Cleanup - let _ = fs::remove_dir_all(db_path.parent().unwrap()); - } - - #[test] - fn test_storage_query() { - let db_path = temp_db_path(); - let storage = Storage::open_at(&db_path).unwrap(); - - let record = LogRecord { - timestamp_ns: 1234567890000000000, - observed_timestamp_ns: 1234567890000000000, - severity_number: Some(9), - severity_text: Some("INFO".to_string()), - body: Some("Test log message".to_string()), - service_name: Some("test_app".to_string()), - app_path: Some("/tmp/test".to_string()), - resource_attributes: None, - log_attributes: None, - trace_id: None, - span_id: None, - }; - - storage.insert_logs(&[record]).unwrap(); - - let records = storage.query_logs(Some("/tmp/test"), 0, None).unwrap(); - assert_eq!(records.len(), 1); - assert_eq!(records[0].body, Some("Test log message".to_string())); - - // Cleanup - let _ = fs::remove_dir_all(db_path.parent().unwrap()); - } -} diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index cd7d49dd..fa1fd6cf 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -43,7 +43,8 @@ url.workspace = true zip.workspace = true rayon.workspace = true hex.workspace = true -rusqlite.workspace = true +apx-db.workspace = true +sqlx.workspace = true similar.workspace = true tempfile.workspace = true rand.workspace = true diff --git a/crates/core/src/flux/mod.rs b/crates/core/src/flux/mod.rs index 37e528a9..d53d2687 100644 --- a/crates/core/src/flux/mod.rs +++ b/crates/core/src/flux/mod.rs @@ -28,8 +28,8 @@ use tracing::{debug, info, warn}; // Re-export from apx-common crate pub use apx_common::{ - FLUX_PORT, FluxLock, Storage, db_path, flux_dir, is_flux_listening, is_running, log_path, - read_lock, remove_lock, write_lock, + FLUX_PORT, FluxLock, flux_dir, is_flux_listening, is_running, log_path, read_lock, remove_lock, + write_lock, }; // ============================================================================ diff --git a/crates/core/src/ops/dev.rs b/crates/core/src/ops/dev.rs index 46683e48..5ec52ee7 100644 --- a/crates/core/src/ops/dev.rs +++ b/crates/core/src/ops/dev.rs @@ -126,7 +126,7 @@ async fn wait_for_healthy_with_logs( let start_time = Instant::now(); let deadline = start_time + Duration::from_secs(config.timeout_secs); - let mut log_streamer = StartupLogStreamer::new(app_dir); + let mut log_streamer = StartupLogStreamer::new(app_dir).await; let mut attempt_count = 0u32; let mut last_overall_status: Option = None; let mut first_response_logged = false; @@ -139,7 +139,7 @@ async fn wait_for_healthy_with_logs( return Err("Startup interrupted by user".to_string()); } _ = tokio::time::sleep(Duration::from_millis(config.retry_delay_ms)) => { - log_streamer.print_new_logs(); + log_streamer.print_new_logs().await; attempt_count += 1; let elapsed_ms = start_time.elapsed().as_millis(); diff --git a/crates/core/src/ops/logs.rs b/crates/core/src/ops/logs.rs index d1d833af..033ab700 100644 --- a/crates/core/src/ops/logs.rs +++ b/crates/core/src/ops/logs.rs @@ -3,9 +3,8 @@ use serde::Serialize; use std::path::Path; use std::time::Duration; -use apx_common::{ - AggregatedRecord, LogAggregator, LogRecord, Storage, db_path, should_skip_log, source_label, -}; +use apx_common::{AggregatedRecord, LogAggregator, LogRecord, should_skip_log, source_label}; +use apx_db::LogsDb; pub const DEFAULT_LOG_DURATION: &str = "10m"; @@ -14,24 +13,28 @@ pub const DEFAULT_LOG_DURATION: &str = "10m"; // --------------------------------------------------------------------------- /// Query and filter logs for the given app directory and duration string. -fn query_filtered_logs(app_dir: &Path, duration: &str) -> Result, String> { +async fn query_filtered_logs(app_dir: &Path, duration: &str) -> Result, String> { let app_path_canonical = app_dir .canonicalize() .unwrap_or_else(|_| app_dir.to_path_buf()) .display() .to_string(); - let db_path = db_path()?; + let db_path = apx_db::logs_db_path()?; if !db_path.exists() { return Ok(Vec::new()); } - let storage = Storage::open().map_err(|e| format!("Failed to open logs database: {e}"))?; + let storage = LogsDb::open() + .await + .map_err(|e| format!("Failed to open logs database: {e}"))?; let duration = parse_duration(duration)?; let since_ns = since_timestamp_nanos(duration); - let records = storage.query_logs(Some(&app_path_canonical), since_ns, None)?; + let records = storage + .query_logs(Some(&app_path_canonical), since_ns, None) + .await?; Ok(records .into_iter() .filter(|r| !should_skip_log(r)) @@ -44,10 +47,10 @@ fn query_filtered_logs(app_dir: &Path, duration: &str) -> Result, /// Fetch dev server logs for the given duration without following. pub async fn fetch_logs(app_dir: &Path, duration: &str) -> Result { - let filtered = query_filtered_logs(app_dir, duration)?; + let filtered = query_filtered_logs(app_dir, duration).await?; if filtered.is_empty() { - let db_path = db_path()?; + let db_path = apx_db::logs_db_path()?; if !db_path.exists() { return Ok("No logs database found.".to_string()); } @@ -88,7 +91,7 @@ pub async fn fetch_logs_structured( app_dir: &Path, duration: &str, ) -> Result, String> { - let filtered = query_filtered_logs(app_dir, duration)?; + let filtered = query_filtered_logs(app_dir, duration).await?; let mut aggregator = LogAggregator::new(); let mut entries = Vec::new(); diff --git a/crates/core/src/ops/startup_logs.rs b/crates/core/src/ops/startup_logs.rs index 2327ce60..5476f622 100644 --- a/crates/core/src/ops/startup_logs.rs +++ b/crates/core/src/ops/startup_logs.rs @@ -5,12 +5,13 @@ use chrono::{Local, TimeZone, Utc}; use std::path::Path; -use apx_common::{LogRecord, Storage, should_skip_log}; +use apx_common::{LogRecord, should_skip_log}; +use apx_db::LogsDb; /// Simple log streamer that prints logs line-by-line to stdout. pub struct StartupLogStreamer { last_log_id: i64, - storage: Option, + storage: Option, app_path: String, } @@ -26,18 +27,18 @@ impl std::fmt::Debug for StartupLogStreamer { impl StartupLogStreamer { /// Create a new log streamer for the given app directory. - pub fn new(app_dir: &Path) -> Self { + pub async fn new(app_dir: &Path) -> Self { let app_path = app_dir .canonicalize() .unwrap_or_else(|_| app_dir.to_path_buf()) .display() .to_string(); - let storage = Storage::open().ok(); - let last_log_id = storage - .as_ref() - .and_then(|s| s.get_latest_id().ok()) - .unwrap_or(0); + let storage = LogsDb::open().await.ok(); + let last_log_id = match &storage { + Some(s) => s.get_latest_id().await.unwrap_or(0), + None => 0, + }; Self { last_log_id, @@ -48,14 +49,17 @@ impl StartupLogStreamer { /// Print any new logs since the last call. /// Returns the number of new log lines printed. - pub fn print_new_logs(&mut self) -> usize { + pub async fn print_new_logs(&mut self) -> usize { let storage = match &self.storage { Some(s) => s, None => return 0, }; // Query logs since last ID - let records = match storage.query_logs_after_id(Some(&self.app_path), self.last_log_id) { + let records = match storage + .query_logs_after_id(Some(&self.app_path), self.last_log_id) + .await + { Ok(r) => r, Err(_) => return 0, }; @@ -69,7 +73,7 @@ impl StartupLogStreamer { } // Update last_log_id - if let Ok(new_id) = storage.get_latest_id() + if let Ok(new_id) = storage.get_latest_id().await && new_id > self.last_log_id { self.last_log_id = new_id; diff --git a/crates/core/src/search/common.rs b/crates/core/src/search/common.rs index c5d0c68e..f49753ef 100644 --- a/crates/core/src/search/common.rs +++ b/crates/core/src/search/common.rs @@ -1,150 +1,26 @@ -//! Common utilities for the SQLite FTS5 search index. -//! -//! Provides a global SQLite connection singleton at `~/.apx/search.db` with WAL mode. -//! All search operations (component index, SDK docs index) share this single connection. +//! Common utilities for the search index. -use rusqlite::Connection; -use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex, OnceLock}; - -/// Global SQLite connection singleton -static SEARCH_DB: OnceLock>> = OnceLock::new(); - -/// Get the default search database path (~/.apx/search.db) -fn default_db_path() -> Result { - let home = dirs::home_dir().ok_or_else(|| "Could not determine home directory".to_string())?; - Ok(home.join(".apx").join("search.db")) -} - -/// Open a SQLite connection with WAL mode and recommended pragmas. -fn open_connection(path: &Path) -> Result { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent) - .map_err(|e| format!("Failed to create database directory: {e}"))?; - } - - let conn = - Connection::open(path).map_err(|e| format!("Failed to open search database: {e}"))?; - - conn.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;") - .map_err(|e| format!("Failed to set pragmas: {e}"))?; - - Ok(conn) -} - -/// Get or initialize the global search database connection at `~/.apx/search.db`. -pub fn get_connection() -> Result>, String> { - if let Some(conn) = SEARCH_DB.get() { - return Ok(Arc::clone(conn)); - } - - let path = default_db_path()?; - let conn = open_connection(&path)?; - let arc = Arc::new(Mutex::new(conn)); - - // Another thread may have initialized it; that's fine, we just use theirs - Ok(Arc::clone(SEARCH_DB.get_or_init(|| arc))) -} - -/// Get a search database connection at a specific path (for testing). -pub fn get_connection_at(path: &Path) -> Result>, String> { - let conn = open_connection(path)?; - Ok(Arc::new(Mutex::new(conn))) -} - -/// Check if a table exists in the database. -pub fn table_exists(conn: &Connection, table_name: &str) -> Result { - let exists: bool = conn - .query_row( - "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)", - [table_name], - |row| row.get(0), - ) - .map_err(|e| format!("Failed to check table existence: {e}"))?; - Ok(exists) -} - -/// Check if the legacy LanceDB directory exists and log a warning. -pub fn check_legacy_lancedb() { +/// Check if the legacy LanceDB directory or old search.db exists and log a warning. +pub fn check_legacy_paths() { if let Some(home) = dirs::home_dir() { - let legacy_path = home.join(".apx").join("db"); - if legacy_path.is_dir() { + let legacy_lancedb = home.join(".apx").join("db"); + if legacy_lancedb.is_dir() { tracing::warn!( "Legacy LanceDB directory found at {}. It is no longer used. \ Remove it with: rm -rf {}", - legacy_path.display(), - legacy_path.display() + legacy_lancedb.display(), + legacy_lancedb.display() ); } - } -} - -/// Sanitize a query string for FTS5 MATCH syntax. -/// Wraps each whitespace-separated term in double quotes for safe literal matching. -pub fn sanitize_fts5_query(query: &str) -> String { - query - .split_whitespace() - .map(|term| { - // Strip any existing quotes and special FTS5 operators - let clean: String = term - .chars() - .filter(|c| c.is_alphanumeric() || *c == '_' || *c == '-' || *c == '.') - .collect(); - if clean.is_empty() { - String::new() - } else { - format!("\"{clean}\"") - } - }) - .filter(|s| !s.is_empty()) - .collect::>() - .join(" ") -} - -#[cfg(test)] -#[allow(clippy::unwrap_used)] -mod tests { - use super::*; - - #[test] - fn test_sanitize_fts5_query_basic() { - assert_eq!(sanitize_fts5_query("hello world"), "\"hello\" \"world\""); - } - - #[test] - fn test_sanitize_fts5_query_special_chars() { - assert_eq!( - sanitize_fts5_query("hello* OR world"), - "\"hello\" \"OR\" \"world\"" - ); - } - #[test] - fn test_sanitize_fts5_query_empty() { - assert_eq!(sanitize_fts5_query(""), ""); - assert_eq!(sanitize_fts5_query(" "), ""); - } - - #[test] - fn test_table_exists() { - let conn = Connection::open_in_memory().unwrap(); - conn.execute_batch("CREATE TABLE test_table (id INTEGER)") - .unwrap(); - - assert!(table_exists(&conn, "test_table").unwrap()); - assert!(!table_exists(&conn, "nonexistent").unwrap()); - } - - #[test] - fn test_get_connection_at() { - let dir = tempfile::TempDir::new().unwrap(); - let path = dir.path().join("test_search.db"); - let conn = get_connection_at(&path).unwrap(); - - // Verify connection works - let guard = conn.lock().unwrap(); - guard - .execute_batch("CREATE TABLE test (id INTEGER)") - .unwrap(); + let legacy_search_db = home.join(".apx").join("search.db"); + if legacy_search_db.exists() { + tracing::warn!( + "Legacy search database found at {}. Search now uses ~/.apx/dev/db. \ + Remove it with: rm {}", + legacy_search_db.display(), + legacy_search_db.display() + ); + } } } diff --git a/crates/core/src/search/component_index.rs b/crates/core/src/search/component_index.rs index 1e1ee4a4..7ad63b92 100644 --- a/crates/core/src/search/component_index.rs +++ b/crates/core/src/search/component_index.rs @@ -1,10 +1,10 @@ //! Component registry indexing and search using SQLite FTS5. -use rusqlite::Connection; -use std::sync::{Arc, Mutex}; +use sqlx::Row; +use sqlx::sqlite::SqlitePool; -use super::common; use crate::components::cache::get_all_registry_indexes; +use apx_db::dev::{sanitize_fts5_query, table_exists}; const TABLE_NAME: &str = "components_fts_v1"; @@ -31,22 +31,21 @@ pub struct SearchResult { } /// Component search index using SQLite FTS5 -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ComponentIndex { - conn: Arc>, + pool: SqlitePool, } impl ComponentIndex { - /// Create a new component index using the global search database - pub fn new() -> Result { - let conn = common::get_connection()?; - Ok(Self { conn }) + /// Create a new component index using the provided pool + pub fn new(pool: SqlitePool) -> Self { + Self { pool } } - /// Create with a custom connection (for testing) + /// Create with a specific pool (for testing or custom setups) #[allow(dead_code)] - pub fn with_connection(conn: Arc>) -> Self { - Self { conn } + pub fn with_pool(pool: SqlitePool) -> Self { + Self { pool } } /// Get the FTS table name @@ -55,13 +54,12 @@ impl ComponentIndex { } /// Validate that the index exists - pub fn validate_index(&self) -> Result { - let conn = self.conn.lock().map_err(|e| format!("Lock error: {e}"))?; - common::table_exists(&conn, TABLE_NAME) + pub async fn validate_index(&self) -> Result { + table_exists(&self.pool, TABLE_NAME).await } /// Build index from registry.json files - pub fn build_index_from_registries(&self) -> Result<(), String> { + pub async fn build_index_from_registries(&self) -> Result<(), String> { tracing::info!("Building component FTS5 index from registry indexes"); let all_indexes = get_all_registry_indexes() @@ -105,61 +103,60 @@ impl ComponentIndex { tracing::info!("Indexing {} components", records.len()); - let conn = self.conn.lock().map_err(|e| format!("Lock error: {e}"))?; - // Drop existing table if it exists - conn.execute_batch(&format!("DROP TABLE IF EXISTS {TABLE_NAME}")) + sqlx::query(&format!("DROP TABLE IF EXISTS {TABLE_NAME}")) + .execute(&self.pool) + .await .map_err(|e| format!("Failed to drop existing table: {e}"))?; // Create FTS5 virtual table - conn.execute_batch(&format!( + sqlx::query(&format!( "CREATE VIRTUAL TABLE {TABLE_NAME} USING fts5(\ id UNINDEXED, name, registry UNINDEXED, text, \ tokenize='porter unicode61'\ )" )) + .execute(&self.pool) + .await .map_err(|e| format!("Failed to create FTS5 table: {e}"))?; // Insert all records in a transaction - let tx = conn - .unchecked_transaction() + let mut tx = self + .pool + .begin() + .await .map_err(|e| format!("Transaction error: {e}"))?; - { - let mut stmt = tx - .prepare(&format!( - "INSERT INTO {TABLE_NAME} (id, name, registry, text) VALUES (?1, ?2, ?3, ?4)" - )) - .map_err(|e| format!("Prepare error: {e}"))?; - - for record in &records { - stmt.execute(rusqlite::params![ - record.id, - record.name, - record.registry, - record.text, - ]) - .map_err(|e| format!("Insert error: {e}"))?; - } + for record in &records { + sqlx::query(&format!( + "INSERT INTO {TABLE_NAME} (id, name, registry, text) VALUES (?1, ?2, ?3, ?4)" + )) + .bind(&record.id) + .bind(&record.name) + .bind(&record.registry) + .bind(&record.text) + .execute(&mut *tx) + .await + .map_err(|e| format!("Insert error: {e}"))?; } - tx.commit().map_err(|e| format!("Commit error: {e}"))?; + tx.commit() + .await + .map_err(|e| format!("Commit error: {e}"))?; tracing::info!("Component FTS5 index built successfully"); Ok(()) } /// Search for components using FTS5 - pub fn search(&self, query: &str, limit: usize) -> Result, String> { + pub async fn search(&self, query: &str, limit: usize) -> Result, String> { tracing::debug!("search: Starting search for query '{}'", query); - let conn = self.conn.lock().map_err(|e| format!("Lock error: {e}"))?; - - if !common::table_exists(&conn, TABLE_NAME)? { + if !table_exists(&self.pool, TABLE_NAME).await? { return Err("Index not built. Please ensure components are indexed.".to_string()); } - let sanitized = common::sanitize_fts5_query(query); + let sanitized = sanitize_fts5_query(query); if sanitized.is_empty() { return Ok(Vec::new()); } @@ -170,31 +167,24 @@ impl ComponentIndex { // Fetch more results for reranking let fts_limit = (limit * 3).max(30); - let mut stmt = conn - .prepare(&format!( - "SELECT id, name, registry, rank FROM {TABLE_NAME} \ - WHERE {TABLE_NAME} MATCH ?1 \ - ORDER BY rank \ - LIMIT ?2" - )) - .map_err(|e| format!("Prepare error: {e}"))?; - - let rows = stmt - .query_map(rusqlite::params![sanitized, fts_limit as i64], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, f64>(3)?, - )) - }) - .map_err(|e| format!("Query error: {e}"))?; + let rows = sqlx::query(&format!( + "SELECT id, name, registry, rank FROM {TABLE_NAME} \ + WHERE {TABLE_NAME} MATCH ?1 \ + ORDER BY rank \ + LIMIT ?2" + )) + .bind(&sanitized) + .bind(fts_limit as i64) + .fetch_all(&self.pool) + .await + .map_err(|e| format!("Query error: {e}"))?; let mut results = Vec::new(); - for (rank, row_result) in rows.enumerate() { - let (id, name, registry, _fts_rank) = - row_result.map_err(|e| format!("Row error: {e}"))?; + for (rank, row) in rows.iter().enumerate() { + let id: String = row.get("id"); + let name: String = row.get("name"); + let registry: String = row.get("registry"); let name_lower = name.to_lowercase(); @@ -246,76 +236,73 @@ impl ComponentIndex { mod tests { use super::*; - fn test_index() -> ComponentIndex { - let conn = Connection::open_in_memory().unwrap(); - let conn = Arc::new(Mutex::new(conn)); - ComponentIndex::with_connection(conn) + async fn test_index() -> ComponentIndex { + let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); + ComponentIndex::with_pool(pool) } - #[test] - fn test_validate_index_empty() { - let index = test_index(); - assert!(!index.validate_index().unwrap()); + #[tokio::test] + async fn test_validate_index_empty() { + let index = test_index().await; + assert!(!index.validate_index().await.unwrap()); } - #[test] - fn test_search_no_index() { - let index = test_index(); - let result = index.search("button", 10); + #[tokio::test] + async fn test_search_no_index() { + let index = test_index().await; + let result = index.search("button", 10).await; assert!(result.is_err()); } - #[test] - fn test_build_and_search() { - let index = test_index(); + #[tokio::test] + async fn test_build_and_search() { + let index = test_index().await; // Create the FTS table manually and insert test data - { - let conn = index.conn.lock().unwrap(); - conn.execute_batch(&format!( - "CREATE VIRTUAL TABLE {TABLE_NAME} USING fts5(\ - id UNINDEXED, name, registry UNINDEXED, text, \ - tokenize='porter unicode61'\ - )" - )) - .unwrap(); - - conn.execute( - &format!( - "INSERT INTO {TABLE_NAME} (id, name, registry, text) VALUES (?1, ?2, ?3, ?4)" - ), - rusqlite::params![ - "button", - "button", - "", - "button A styled button component shadcn" - ], - ) - .unwrap(); - - conn.execute( - &format!( - "INSERT INTO {TABLE_NAME} (id, name, registry, text) VALUES (?1, ?2, ?3, ?4)" - ), - rusqlite::params!["card", "card", "", "card A card container component shadcn"], - ) - .unwrap(); - - conn.execute( - &format!( - "INSERT INTO {TABLE_NAME} (id, name, registry, text) VALUES (?1, ?2, ?3, ?4)" - ), - rusqlite::params![ - "@custom/button", - "button", - "custom", - "button A custom button component" - ], - ) - .unwrap(); - } + sqlx::query(&format!( + "CREATE VIRTUAL TABLE {TABLE_NAME} USING fts5(\ + id UNINDEXED, name, registry UNINDEXED, text, \ + tokenize='porter unicode61'\ + )" + )) + .execute(&index.pool) + .await + .unwrap(); - let results = index.search("button", 10).unwrap(); + sqlx::query(&format!( + "INSERT INTO {TABLE_NAME} (id, name, registry, text) VALUES (?1, ?2, ?3, ?4)" + )) + .bind("button") + .bind("button") + .bind("") + .bind("button A styled button component shadcn") + .execute(&index.pool) + .await + .unwrap(); + + sqlx::query(&format!( + "INSERT INTO {TABLE_NAME} (id, name, registry, text) VALUES (?1, ?2, ?3, ?4)" + )) + .bind("card") + .bind("card") + .bind("") + .bind("card A card container component shadcn") + .execute(&index.pool) + .await + .unwrap(); + + sqlx::query(&format!( + "INSERT INTO {TABLE_NAME} (id, name, registry, text) VALUES (?1, ?2, ?3, ?4)" + )) + .bind("@custom/button") + .bind("button") + .bind("custom") + .bind("button A custom button component") + .execute(&index.pool) + .await + .unwrap(); + + let results = index.search("button", 10).await.unwrap(); assert!(!results.is_empty()); // Both buttons should be in results, card should not diff --git a/crates/core/src/search/docs_index.rs b/crates/core/src/search/docs_index.rs index c2c5bb16..afabef75 100644 --- a/crates/core/src/search/docs_index.rs +++ b/crates/core/src/search/docs_index.rs @@ -1,13 +1,13 @@ //! SDK documentation indexing and search using SQLite FTS5. use rayon::prelude::*; -use rusqlite::Connection; use serde::{Deserialize, Serialize}; -use std::sync::{Arc, Mutex}; +use sqlx::Row; +use sqlx::sqlite::SqlitePool; use crate::common::Timer; use crate::databricks_sdk_doc::{SDKSource, download_and_extract_sdk, load_doc_files}; -use crate::search::common; +use apx_db::dev::{sanitize_fts5_query, table_exists}; const CHUNK_SIZE: usize = 2000; // characters (no tokenizer needed for FTS) const CHUNK_OVERLAP: usize = 200; // characters overlap @@ -129,27 +129,26 @@ fn chunk_text( } /// SDK documentation index using SQLite FTS5 -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct SDKDocsIndex { - conn: Arc>, + pool: SqlitePool, version: Option, } impl SDKDocsIndex { - /// Create a new SDK docs index using the global search database - pub fn new() -> Result { - let conn = common::get_connection()?; - Ok(Self { - conn, + /// Create a new SDK docs index using the provided pool + pub fn new(pool: SqlitePool) -> Self { + Self { + pool, version: None, - }) + } } - /// Create with custom connection (for testing) + /// Create with a specific pool (for testing or custom setups) #[allow(dead_code)] - pub fn with_connection(conn: Arc>) -> Self { + pub fn with_pool(pool: SqlitePool) -> Self { Self { - conn, + pool, version: None, } } @@ -164,9 +163,8 @@ impl SDKDocsIndex { } /// Check if the index table exists - fn table_exists_sync(&self, table_name: &str) -> Result { - let conn = self.conn.lock().map_err(|e| format!("Lock error: {e}"))?; - common::table_exists(&conn, table_name) + async fn table_exists_check(&self, table_name: &str) -> Result { + table_exists(&self.pool, table_name).await } /// Bootstrap with a pre-computed SDK version @@ -182,8 +180,8 @@ impl SDKDocsIndex { let table_name = Self::table_name(version); - // Check if already indexed (sync, but cheap) - if self.table_exists_sync(&table_name)? { + // Check if already indexed + if self.table_exists_check(&table_name).await? { tracing::info!("SDK docs already indexed for version {}", version); return Ok(false); } @@ -191,16 +189,20 @@ impl SDKDocsIndex { // Download and extract (async) let docs_path = download_and_extract_sdk(version).await?; - // Build index (sync, wrapped in spawn_blocking by caller) - self.build_index(&table_name, &docs_path)?; + // Build index (async) + self.build_index(&table_name, &docs_path).await?; Ok(true) } } } - /// Build index from a docs path (sync) - fn build_index(&self, table_name: &str, docs_path: &std::path::Path) -> Result<(), String> { + /// Build index from a docs path + async fn build_index( + &self, + table_name: &str, + docs_path: &std::path::Path, + ) -> Result<(), String> { let overall_timer = Timer::start("build_index"); // Load documentation files @@ -211,7 +213,7 @@ impl SDKDocsIndex { let files = load_doc_files(docs_path)?; load_timer.lap(&format!("Loaded {} documentation files", files.len())); - // Chunk all files in parallel + // Chunk all files in parallel (CPU-bound, uses rayon) let chunk_timer = Timer::start("chunk_text_parallel"); if let Some(doc) = files.first() { @@ -255,23 +257,25 @@ impl SDKDocsIndex { chunk_timer.lap(&format!("Created {} text chunks", doc_chunks.len())); - // Database operations + // Database operations (async) let db_timer = Timer::start("database_operations"); - let conn = self.conn.lock().map_err(|e| format!("Lock error: {e}"))?; - // Drop existing table if it exists - conn.execute_batch(&format!("DROP TABLE IF EXISTS {table_name}")) + sqlx::query(&format!("DROP TABLE IF EXISTS \"{table_name}\"")) + .execute(&self.pool) + .await .map_err(|e| format!("Failed to drop existing table: {e}"))?; // Create FTS5 virtual table - conn.execute_batch(&format!( + sqlx::query(&format!( "CREATE VIRTUAL TABLE \"{table_name}\" USING fts5(\ id UNINDEXED, text, source_file UNINDEXED, \ chunk_index UNINDEXED, service, entity, operation, symbols, \ tokenize='porter unicode61'\ )" )) + .execute(&self.pool) + .await .map_err(|e| format!("Failed to create FTS5 table: {e}"))?; db_timer.lap("Created FTS5 table"); @@ -279,35 +283,34 @@ impl SDKDocsIndex { // Insert all chunks in a transaction let insert_timer = Timer::start("insert_chunks"); - let tx = conn - .unchecked_transaction() + let mut tx = self + .pool + .begin() + .await .map_err(|e| format!("Transaction error: {e}"))?; - { - let mut stmt = tx - .prepare(&format!( - "INSERT INTO \"{table_name}\" \ - (id, text, source_file, chunk_index, service, entity, operation, symbols) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)" - )) - .map_err(|e| format!("Prepare error: {e}"))?; - - for chunk in &doc_chunks { - stmt.execute(rusqlite::params![ - chunk.id, - chunk.text, - chunk.source_file, - chunk.chunk_index as i64, - chunk.service, - chunk.entity, - chunk.operation, - chunk.symbols, - ]) - .map_err(|e| format!("Insert error: {e}"))?; - } + for chunk in &doc_chunks { + sqlx::query(&format!( + "INSERT INTO \"{table_name}\" \ + (id, text, source_file, chunk_index, service, entity, operation, symbols) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)" + )) + .bind(&chunk.id) + .bind(&chunk.text) + .bind(&chunk.source_file) + .bind(chunk.chunk_index as i64) + .bind(&chunk.service) + .bind(&chunk.entity) + .bind(&chunk.operation) + .bind(&chunk.symbols) + .execute(&mut *tx) + .await + .map_err(|e| format!("Insert error: {e}"))?; } - tx.commit().map_err(|e| format!("Commit error: {e}"))?; + tx.commit() + .await + .map_err(|e| format!("Commit error: {e}"))?; insert_timer.finish(); db_timer.finish(); @@ -337,8 +340,8 @@ impl SDKDocsIndex { Ok(()) } - /// Search for relevant documentation chunks using FTS5 (sync) - pub fn search_sync( + /// Search for relevant documentation chunks using FTS5 + pub async fn search( &self, source: &SDKSource, query: &str, @@ -352,45 +355,36 @@ impl SDKDocsIndex { let table_name = Self::table_name(version); - let conn = self.conn.lock().map_err(|e| format!("Lock error: {e}"))?; - - if !common::table_exists(&conn, &table_name)? { + if !table_exists(&self.pool, &table_name).await? { return Err(format!( "SDK docs not indexed for version {version}. Index will be built on next server start." )); } - let sanitized = common::sanitize_fts5_query(query); + let sanitized = sanitize_fts5_query(query); if sanitized.is_empty() { return Ok(Vec::new()); } tracing::debug!("search: Executing FTS5 query for '{}'", query); - let mut stmt = conn - .prepare(&format!( - "SELECT text, source_file, rank FROM \"{table_name}\" \ - WHERE \"{table_name}\" MATCH ?1 \ - ORDER BY rank \ - LIMIT ?2" - )) - .map_err(|e| format!("Prepare error: {e}"))?; - - let rows = stmt - .query_map(rusqlite::params![sanitized, limit as i64], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, f64>(2)?, - )) - }) - .map_err(|e| format!("Query error: {e}"))?; + let rows = sqlx::query(&format!( + "SELECT text, source_file, rank FROM \"{table_name}\" \ + WHERE \"{table_name}\" MATCH ?1 \ + ORDER BY rank \ + LIMIT ?2" + )) + .bind(&sanitized) + .bind(limit as i64) + .fetch_all(&self.pool) + .await + .map_err(|e| format!("Query error: {e}"))?; let mut results = Vec::new(); - for (rank, row_result) in rows.enumerate() { - let (text, source_file, _fts_rank) = - row_result.map_err(|e| format!("Row error: {e}"))?; + for (rank, row) in rows.iter().enumerate() { + let text: String = row.get("text"); + let source_file: String = row.get("source_file"); let score = 1.0 / (1.0 + rank as f32); results.push(DocSearchResult { @@ -460,62 +454,57 @@ mod tests { assert!(chunks.is_empty(), "Empty text should produce no chunks"); } - #[test] - fn test_sdk_docs_index_creation() { - let conn = Connection::open_in_memory().unwrap(); - let conn = Arc::new(Mutex::new(conn)); - let index = SDKDocsIndex::with_connection(conn); + #[tokio::test] + async fn test_sdk_docs_index_creation() { + let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); + let index = SDKDocsIndex::with_pool(pool); assert!(index.version.is_none()); } - #[test] - fn test_fts5_search_with_data() { - let conn = Connection::open_in_memory().unwrap(); + #[tokio::test] + async fn test_fts5_search_with_data() { + let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); let table_name = "sdk_docs_fts_test_v1"; - conn.execute_batch(&format!( + sqlx::query(&format!( "CREATE VIRTUAL TABLE \"{table_name}\" USING fts5(\ id UNINDEXED, text, source_file UNINDEXED, \ chunk_index UNINDEXED, service, entity, operation, symbols, \ tokenize='porter unicode61'\ )" )) + .execute(&pool) + .await .unwrap(); - conn.execute( - &format!( - "INSERT INTO \"{table_name}\" \ - (id, text, source_file, chunk_index, service, entity, operation, symbols) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)" - ), - rusqlite::params![ - "test.rst:0", - "ClustersAPI clusters create This is about creating clusters", - "test.rst", - 0, - "clusters", - "ClustersAPI", - "create", - "clusters create ClustersAPI" - ], - ) + sqlx::query(&format!( + "INSERT INTO \"{table_name}\" \ + (id, text, source_file, chunk_index, service, entity, operation, symbols) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)" + )) + .bind("test.rst:0") + .bind("ClustersAPI clusters create This is about creating clusters") + .bind("test.rst") + .bind(0i64) + .bind("clusters") + .bind("ClustersAPI") + .bind("create") + .bind("clusters create ClustersAPI") + .execute(&pool) + .await .unwrap(); // Verify the data is searchable - let mut stmt = conn - .prepare(&format!( - "SELECT text, source_file FROM \"{table_name}\" \ - WHERE \"{table_name}\" MATCH '\"clusters\"' LIMIT 5" - )) - .unwrap(); - - let results: Vec<(String, String)> = stmt - .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) - .unwrap() - .filter_map(|r| r.ok()) - .collect(); + let results = sqlx::query(&format!( + "SELECT text, source_file FROM \"{table_name}\" \ + WHERE \"{table_name}\" MATCH '\"clusters\"' LIMIT 5" + )) + .fetch_all(&pool) + .await + .unwrap(); assert_eq!(results.len(), 1); - assert!(results[0].0.contains("clusters")); + let text: String = results[0].get("text"); + assert!(text.contains("clusters")); } } diff --git a/crates/db/Cargo.toml b/crates/db/Cargo.toml new file mode 100644 index 00000000..0ad4db5e --- /dev/null +++ b/crates/db/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "apx-db" +version = "0.3.0-rc1" +edition.workspace = true +rust-version.workspace = true + +[dependencies] +apx-common.workspace = true +sqlx.workspace = true +dirs.workspace = true +tracing.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } diff --git a/crates/db/src/dev.rs b/crates/db/src/dev.rs new file mode 100644 index 00000000..852a2d7f --- /dev/null +++ b/crates/db/src/dev.rs @@ -0,0 +1,141 @@ +//! Async dev database operations using SQLx. +//! +//! Provides [`DevDb`] as the connection pool for the dev database at `~/.apx/dev/db`. +//! This database holds search indexes (FTS5) and will hold future dev-related tables. + +use sqlx::sqlite::{ + SqliteConnectOptions, SqliteJournalMode, SqlitePool, SqlitePoolOptions, SqliteSynchronous, +}; +use std::path::Path; + +/// Async dev database handle. +#[derive(Clone, Debug)] +pub struct DevDb { + pool: SqlitePool, +} + +impl DevDb { + /// Open or create the dev database at the default location (`~/.apx/dev/db`). + pub async fn open() -> Result { + let path = super::dev_db_path()?; + Self::open_at(&path).await + } + + /// Open or create the dev database at a specific path. + pub async fn open_at(path: &Path) -> Result { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| format!("Failed to create database directory: {e}"))?; + } + + let opts = SqliteConnectOptions::new() + .filename(path) + .create_if_missing(true) + .journal_mode(SqliteJournalMode::Wal) + .synchronous(SqliteSynchronous::Normal); + + let pool = SqlitePoolOptions::new() + .max_connections(5) + .connect_with(opts) + .await + .map_err(|e| format!("Failed to open dev database: {e}"))?; + + Ok(Self { pool }) + } + + /// Get a reference to the underlying connection pool. + pub fn pool(&self) -> &SqlitePool { + &self.pool + } +} + +/// Check if a table exists in the database. +pub async fn table_exists(pool: &SqlitePool, table_name: &str) -> Result { + let row: (bool,) = + sqlx::query_as("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)") + .bind(table_name) + .fetch_one(pool) + .await + .map_err(|e| format!("Failed to check table existence: {e}"))?; + Ok(row.0) +} + +/// Sanitize a query string for FTS5 MATCH syntax. +/// Wraps each whitespace-separated term in double quotes for safe literal matching. +pub fn sanitize_fts5_query(query: &str) -> String { + query + .split_whitespace() + .map(|term| { + let clean: String = term + .chars() + .filter(|c| c.is_alphanumeric() || *c == '_' || *c == '-' || *c == '.') + .collect(); + if clean.is_empty() { + String::new() + } else { + format!("\"{clean}\"") + } + }) + .filter(|s| !s.is_empty()) + .collect::>() + .join(" ") +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn test_sanitize_fts5_query_basic() { + assert_eq!(sanitize_fts5_query("hello world"), "\"hello\" \"world\""); + } + + #[test] + fn test_sanitize_fts5_query_special_chars() { + assert_eq!( + sanitize_fts5_query("hello* OR world"), + "\"hello\" \"OR\" \"world\"" + ); + } + + #[test] + fn test_sanitize_fts5_query_empty() { + assert_eq!(sanitize_fts5_query(""), ""); + assert_eq!(sanitize_fts5_query(" "), ""); + } + + #[tokio::test] + async fn test_table_exists() { + let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); + sqlx::query("CREATE TABLE test_table (id INTEGER)") + .execute(&pool) + .await + .unwrap(); + + assert!(table_exists(&pool, "test_table").await.unwrap()); + assert!(!table_exists(&pool, "nonexistent").await.unwrap()); + } + + #[tokio::test] + async fn test_dev_db_open() { + let dir = std::env::temp_dir().join(format!( + "apx-dev-test-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + let db = DevDb::open_at(&dir.join("test.db")).await.unwrap(); + + // Verify the pool works + sqlx::query("CREATE TABLE test (id INTEGER)") + .execute(db.pool()) + .await + .unwrap(); + + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/db/src/lib.rs b/crates/db/src/lib.rs new file mode 100644 index 00000000..2887114a --- /dev/null +++ b/crates/db/src/lib.rs @@ -0,0 +1,37 @@ +#![forbid(unsafe_code)] +#![deny(warnings, unused_must_use, dead_code, missing_debug_implementations)] +#![deny( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::todo, + clippy::unimplemented, + clippy::dbg_macro +)] + +//! Database layer for APX using SQLx with SQLite. +//! +//! Provides async connection pools for two databases: +//! - **Logs DB** (`~/.apx/logs/db`) โ€” OTLP log storage +//! - **Dev DB** (`~/.apx/dev/db`) โ€” search indexes and future dev-related tables + +pub mod dev; +pub mod logs; + +pub use dev::DevDb; +pub use logs::LogsDb; +pub use sqlx::sqlite::SqlitePool; + +use std::path::PathBuf; + +/// Get the logs database path (`~/.apx/logs/db`). +pub fn logs_db_path() -> Result { + let home = dirs::home_dir().ok_or("Could not determine home directory")?; + Ok(home.join(".apx").join("logs").join("db")) +} + +/// Get the dev database path (`~/.apx/dev/db`). +pub fn dev_db_path() -> Result { + let home = dirs::home_dir().ok_or("Could not determine home directory")?; + Ok(home.join(".apx").join("dev").join("db")) +} diff --git a/crates/db/src/logs.rs b/crates/db/src/logs.rs new file mode 100644 index 00000000..32a975e3 --- /dev/null +++ b/crates/db/src/logs.rs @@ -0,0 +1,419 @@ +//! Async logs database operations using SQLx. +//! +//! Provides [`LogsDb`] for all CRUD operations on the OTLP logs table +//! at `~/.apx/logs/db`. + +use apx_common::LogRecord; +use sqlx::Row; +use sqlx::sqlite::{ + SqliteConnectOptions, SqliteJournalMode, SqlitePool, SqlitePoolOptions, SqliteSynchronous, +}; +use std::path::Path; +use tracing::debug; + +/// Retention period in seconds (7 days). +const RETENTION_SECONDS: i64 = 7 * 24 * 60 * 60; + +/// Async logs database handle. +#[derive(Clone, Debug)] +pub struct LogsDb { + pool: SqlitePool, +} + +impl LogsDb { + /// Open or create the database at the default location (`~/.apx/logs/db`). + pub async fn open() -> Result { + let path = super::logs_db_path()?; + Self::open_at(&path).await + } + + /// Open or create the database at a specific path. + pub async fn open_at(path: &Path) -> Result { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| format!("Failed to create database directory: {e}"))?; + } + + let opts = SqliteConnectOptions::new() + .filename(path) + .create_if_missing(true) + .journal_mode(SqliteJournalMode::Wal) + .synchronous(SqliteSynchronous::Normal); + + let pool = SqlitePoolOptions::new() + .max_connections(5) + .connect_with(opts) + .await + .map_err(|e| format!("Failed to open database: {e}"))?; + + let db = Self { pool }; + db.init_schema().await?; + Ok(db) + } + + /// Initialize the database schema. + async fn init_schema(&self) -> Result<(), String> { + sqlx::query( + r#"CREATE TABLE IF NOT EXISTS logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp_ns INTEGER NOT NULL, + observed_timestamp_ns INTEGER NOT NULL, + severity_number INTEGER, + severity_text TEXT, + body TEXT, + service_name TEXT, + app_path TEXT, + resource_attributes TEXT, + log_attributes TEXT, + trace_id TEXT, + span_id TEXT, + created_at INTEGER DEFAULT (strftime('%s', 'now')) + )"#, + ) + .execute(&self.pool) + .await + .map_err(|e| format!("Failed to initialize schema: {e}"))?; + + for idx_sql in [ + "CREATE INDEX IF NOT EXISTS idx_logs_timestamp ON logs(timestamp_ns)", + "CREATE INDEX IF NOT EXISTS idx_logs_app_path ON logs(app_path)", + "CREATE INDEX IF NOT EXISTS idx_logs_service ON logs(service_name)", + "CREATE INDEX IF NOT EXISTS idx_logs_created ON logs(created_at)", + ] { + sqlx::query(idx_sql) + .execute(&self.pool) + .await + .map_err(|e| format!("Index error: {e}"))?; + } + + debug!("Flux storage schema initialized"); + Ok(()) + } + + /// Insert a batch of log records. + pub async fn insert_logs(&self, records: &[LogRecord]) -> Result { + if records.is_empty() { + return Ok(0); + } + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| format!("Transaction error: {e}"))?; + + let mut count = 0; + for record in records { + sqlx::query( + r#"INSERT INTO logs ( + timestamp_ns, observed_timestamp_ns, severity_number, severity_text, + body, service_name, app_path, resource_attributes, log_attributes, + trace_id, span_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"#, + ) + .bind(record.timestamp_ns) + .bind(record.observed_timestamp_ns) + .bind(record.severity_number) + .bind(record.severity_text.as_deref()) + .bind(record.body.as_deref()) + .bind(record.service_name.as_deref()) + .bind(record.app_path.as_deref()) + .bind(record.resource_attributes.as_deref()) + .bind(record.log_attributes.as_deref()) + .bind(record.trace_id.as_deref()) + .bind(record.span_id.as_deref()) + .execute(&mut *tx) + .await + .map_err(|e| format!("Insert error: {e}"))?; + count += 1; + } + + tx.commit() + .await + .map_err(|e| format!("Commit error: {e}"))?; + Ok(count) + } + + /// Query logs for a specific app path since a given timestamp. + pub async fn query_logs( + &self, + app_path: Option<&str>, + since_ns: i64, + limit: Option, + ) -> Result, String> { + let effective_ts = "COALESCE(NULLIF(timestamp_ns, 0), observed_timestamp_ns)"; + + let sql = match (app_path, limit) { + (Some(_), Some(lim)) => format!( + r#"SELECT timestamp_ns, observed_timestamp_ns, severity_number, severity_text, + body, service_name, app_path, resource_attributes, log_attributes, + trace_id, span_id + FROM logs + WHERE (app_path LIKE ?1 OR ?1 LIKE '%' || app_path || '%') + AND {effective_ts} >= ?2 + ORDER BY {effective_ts} ASC + LIMIT {lim}"# + ), + (Some(_), None) => format!( + r#"SELECT timestamp_ns, observed_timestamp_ns, severity_number, severity_text, + body, service_name, app_path, resource_attributes, log_attributes, + trace_id, span_id + FROM logs + WHERE (app_path LIKE ?1 OR ?1 LIKE '%' || app_path || '%') + AND {effective_ts} >= ?2 + ORDER BY {effective_ts} ASC"# + ), + (None, Some(lim)) => format!( + r#"SELECT timestamp_ns, observed_timestamp_ns, severity_number, severity_text, + body, service_name, app_path, resource_attributes, log_attributes, + trace_id, span_id + FROM logs + WHERE {effective_ts} >= ?1 + ORDER BY {effective_ts} ASC + LIMIT {lim}"# + ), + (None, None) => format!( + r#"SELECT timestamp_ns, observed_timestamp_ns, severity_number, severity_text, + body, service_name, app_path, resource_attributes, log_attributes, + trace_id, span_id + FROM logs + WHERE {effective_ts} >= ?1 + ORDER BY {effective_ts} ASC"# + ), + }; + + let rows = if let Some(path) = app_path { + let pattern = format!("%{path}%"); + sqlx::query(&sql) + .bind(&pattern) + .bind(since_ns) + .fetch_all(&self.pool) + .await + } else { + sqlx::query(&sql).bind(since_ns).fetch_all(&self.pool).await + } + .map_err(|e| format!("Query error: {e}"))?; + + let records = rows.iter().map(row_to_log_record).collect(); + Ok(records) + } + + /// Get the latest log ID for change detection in follow mode. + pub async fn get_latest_id(&self) -> Result { + let row = sqlx::query("SELECT COALESCE(MAX(id), 0) as max_id FROM logs") + .fetch_one(&self.pool) + .await + .map_err(|e| format!("Query error: {e}"))?; + let id: i64 = row.get("max_id"); + Ok(id) + } + + /// Query logs newer than a given ID (for follow mode). + pub async fn query_logs_after_id( + &self, + app_path: Option<&str>, + after_id: i64, + ) -> Result, String> { + let effective_ts = "COALESCE(NULLIF(timestamp_ns, 0), observed_timestamp_ns)"; + + let (sql, has_app_path) = if app_path.is_some() { + ( + format!( + r#"SELECT timestamp_ns, observed_timestamp_ns, severity_number, severity_text, + body, service_name, app_path, resource_attributes, log_attributes, + trace_id, span_id + FROM logs + WHERE id > ?1 AND (app_path LIKE ?2 OR ?2 LIKE '%' || app_path || '%') + ORDER BY {effective_ts} ASC"# + ), + true, + ) + } else { + ( + format!( + r#"SELECT timestamp_ns, observed_timestamp_ns, severity_number, severity_text, + body, service_name, app_path, resource_attributes, log_attributes, + trace_id, span_id + FROM logs + WHERE id > ?1 + ORDER BY {effective_ts} ASC"# + ), + false, + ) + }; + + let rows = if has_app_path { + let pattern = format!("%{}%", app_path.unwrap_or("")); + sqlx::query(&sql) + .bind(after_id) + .bind(&pattern) + .fetch_all(&self.pool) + .await + } else { + sqlx::query(&sql).bind(after_id).fetch_all(&self.pool).await + } + .map_err(|e| format!("Query error: {e}"))?; + + let records = rows.iter().map(row_to_log_record).collect(); + Ok(records) + } + + /// Delete logs older than the retention period (7 days). + pub async fn cleanup_old_logs(&self) -> Result { + let cutoff = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64 - RETENTION_SECONDS) + .unwrap_or(0); + + let result = sqlx::query("DELETE FROM logs WHERE created_at < ?") + .bind(cutoff) + .execute(&self.pool) + .await + .map_err(|e| format!("Delete error: {e}"))?; + + let deleted = result.rows_affected() as usize; + if deleted > 0 { + debug!("Cleaned up {} old log records", deleted); + } + Ok(deleted) + } + + /// Get the total count of logs. + #[allow(dead_code)] + pub async fn count_logs(&self) -> Result { + let row = sqlx::query("SELECT COUNT(*) as cnt FROM logs") + .fetch_one(&self.pool) + .await + .map_err(|e| format!("Query error: {e}"))?; + let count: i64 = row.get("cnt"); + Ok(count) + } +} + +/// Map a SQLx row to a LogRecord. +fn row_to_log_record(row: &sqlx::sqlite::SqliteRow) -> LogRecord { + LogRecord { + timestamp_ns: row.get("timestamp_ns"), + observed_timestamp_ns: row.get("observed_timestamp_ns"), + severity_number: row.get("severity_number"), + severity_text: row.get("severity_text"), + body: row.get("body"), + service_name: row.get("service_name"), + app_path: row.get("app_path"), + resource_attributes: row.get("resource_attributes"), + log_attributes: row.get("log_attributes"), + trace_id: row.get("trace_id"), + span_id: row.get("span_id"), + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + async fn temp_db() -> LogsDb { + let dir = std::env::temp_dir().join(format!( + "apx-db-test-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + LogsDb::open_at(&dir.join("test.db")).await.unwrap() + } + + #[tokio::test] + async fn test_create_and_insert() { + let db = temp_db().await; + + let record = LogRecord { + timestamp_ns: 1234567890000000000, + observed_timestamp_ns: 1234567890000000000, + severity_number: Some(9), + severity_text: Some("INFO".to_string()), + body: Some("Test log message".to_string()), + service_name: Some("test_app".to_string()), + app_path: Some("/tmp/test".to_string()), + resource_attributes: None, + log_attributes: None, + trace_id: None, + span_id: None, + }; + + let count = db.insert_logs(&[record]).await.unwrap(); + assert_eq!(count, 1); + + let total = db.count_logs().await.unwrap(); + assert_eq!(total, 1); + } + + #[tokio::test] + async fn test_query() { + let db = temp_db().await; + + let record = LogRecord { + timestamp_ns: 1234567890000000000, + observed_timestamp_ns: 1234567890000000000, + severity_number: Some(9), + severity_text: Some("INFO".to_string()), + body: Some("Test log message".to_string()), + service_name: Some("test_app".to_string()), + app_path: Some("/tmp/test".to_string()), + resource_attributes: None, + log_attributes: None, + trace_id: None, + span_id: None, + }; + + db.insert_logs(&[record]).await.unwrap(); + + let records = db.query_logs(Some("/tmp/test"), 0, None).await.unwrap(); + assert_eq!(records.len(), 1); + assert_eq!(records[0].body, Some("Test log message".to_string())); + } + + #[tokio::test] + async fn test_query_after_id() { + let db = temp_db().await; + + let record = LogRecord { + timestamp_ns: 1234567890000000000, + observed_timestamp_ns: 1234567890000000000, + severity_number: Some(9), + severity_text: Some("INFO".to_string()), + body: Some("First".to_string()), + service_name: Some("test_app".to_string()), + app_path: Some("/tmp/test".to_string()), + resource_attributes: None, + log_attributes: None, + trace_id: None, + span_id: None, + }; + + db.insert_logs(&[record]).await.unwrap(); + let id = db.get_latest_id().await.unwrap(); + + let record2 = LogRecord { + timestamp_ns: 1234567891000000000, + observed_timestamp_ns: 1234567891000000000, + severity_number: Some(9), + severity_text: Some("INFO".to_string()), + body: Some("Second".to_string()), + service_name: Some("test_app".to_string()), + app_path: Some("/tmp/test".to_string()), + resource_attributes: None, + log_attributes: None, + trace_id: None, + span_id: None, + }; + + db.insert_logs(&[record2]).await.unwrap(); + + let records = db.query_logs_after_id(Some("/tmp/test"), id).await.unwrap(); + assert_eq!(records.len(), 1); + assert_eq!(records[0].body, Some("Second".to_string())); + } +} diff --git a/crates/mcp/Cargo.toml b/crates/mcp/Cargo.toml index 7053f2d4..f6650caa 100644 --- a/crates/mcp/Cargo.toml +++ b/crates/mcp/Cargo.toml @@ -6,6 +6,7 @@ rust-version.workspace = true [dependencies] apx-core = { path = "../core" , version = "0.3.0-rc1" } +apx-db.workspace = true serde.workspace = true serde_json.workspace = true serde_yaml.workspace = true diff --git a/crates/mcp/src/context.rs b/crates/mcp/src/context.rs index 1b17af96..473b929e 100644 --- a/crates/mcp/src/context.rs +++ b/crates/mcp/src/context.rs @@ -1,5 +1,6 @@ use apx_core::components::SharedCacheState; use apx_core::search::docs_index::SDKDocsIndex; +use apx_db::DevDb; use std::sync::Arc; use std::sync::atomic::AtomicBool; use tokio::sync::{Mutex, Notify, broadcast}; @@ -43,6 +44,7 @@ impl IndexState { #[derive(Debug)] pub struct AppContext { + pub dev_db: DevDb, pub sdk_doc_index: Arc>>, pub cache_state: SharedCacheState, pub index_state: IndexState, diff --git a/crates/mcp/src/indexing.rs b/crates/mcp/src/indexing.rs index d02e5dfe..0b94e8ea 100644 --- a/crates/mcp/src/indexing.rs +++ b/crates/mcp/src/indexing.rs @@ -1,6 +1,7 @@ use crate::context::{AppContext, SdkIndexParams}; use apx_core::databricks_sdk_doc::SDKSource; use apx_core::search::ComponentIndex; +use apx_db::SqlitePool; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use tokio::sync::Notify; @@ -18,9 +19,10 @@ pub fn init_all_indexes( ) { let cache_state = ctx.cache_state.clone(); let index_state = ctx.index_state.clone(); + let pool = ctx.dev_db.pool().clone(); // Check for legacy LanceDB directory - apx_core::search::common::check_legacy_lancedb(); + apx_core::search::common::check_legacy_paths(); tokio::spawn(async move { // Mark as running @@ -35,8 +37,8 @@ pub fn init_all_indexes( tracing::info!("Ensuring component search index exists on MCP start"); let ensure_result = tokio::select! { - result = tokio::task::spawn_blocking(ensure_search_index) => { - Some(result.unwrap_or_else(|e| Err(format!("spawn_blocking panicked: {e}")))) + result = ensure_search_index(pool.clone()) => { + Some(result) }, _ = shutdown_rx.recv() => { tracing::info!("Shutdown signal received during search index check, stopping"); @@ -62,25 +64,9 @@ pub fn init_all_indexes( let version = params.sdk_version; tracing::debug!("Using SDK version: {}", version); - // Create SDK docs index (sync, but cheap) - let mut index = match apx_core::search::docs_index::SDKDocsIndex::new() { - Ok(idx) => { - tracing::debug!("SDKDocsIndex created successfully"); - idx - } - Err(e) => { - tracing::warn!( - "Failed to initialize SDK doc index: {}. The docs tool will not be available.", - e - ); - index_state.sdk_indexed.store(true, Ordering::SeqCst); - index_state.sdk_ready.notify_waiters(); - - let mut guard = cache_state.lock().await; - guard.is_running = false; - return; - } - }; + // Create SDK docs index (async) + let mut index = apx_core::search::docs_index::SDKDocsIndex::new(pool.clone()); + tracing::debug!("SDKDocsIndex created successfully"); // Bootstrap the index (async: download + sync: build) tracing::info!("Bootstrapping SDK docs (this may download SDK if not cached)"); @@ -132,28 +118,28 @@ pub fn init_all_indexes( }); } -/// Rebuild the search index from registry.json files (sync) -pub fn rebuild_search_index() -> Result<(), String> { - let index = ComponentIndex::new()?; - index.build_index_from_registries() +/// Rebuild the search index from registry.json files (async) +pub async fn rebuild_search_index(pool: SqlitePool) -> Result<(), String> { + let index = ComponentIndex::new(pool); + index.build_index_from_registries().await } -/// Ensure search index exists and is valid, build/rebuild if needed (sync) -fn ensure_search_index() -> Result<(), String> { - let index = ComponentIndex::new()?; +/// Ensure search index exists and is valid, build/rebuild if needed (async) +async fn ensure_search_index(pool: SqlitePool) -> Result<(), String> { + let index = ComponentIndex::new(pool); - match index.validate_index() { + match index.validate_index().await { Ok(true) => { tracing::debug!("Search index validated successfully"); Ok(()) } Ok(false) => { tracing::info!("Search index not found, building from registry indexes"); - index.build_index_from_registries() + index.build_index_from_registries().await } Err(e) => { tracing::warn!("Search index corrupted ({}), rebuilding...", e); - index.build_index_from_registries() + index.build_index_from_registries().await } } } diff --git a/crates/mcp/src/tools/docs.rs b/crates/mcp/src/tools/docs.rs index 9bb5caf6..c1951431 100644 --- a/crates/mcp/src/tools/docs.rs +++ b/crates/mcp/src/tools/docs.rs @@ -81,7 +81,10 @@ impl ApxServer { } } - match index.search_sync(&args.source, &args.query, args.num_results) { + match index + .search(&args.source, &args.query, args.num_results) + .await + { Ok(results) => { drop(index_guard); diff --git a/crates/mcp/src/tools/registry.rs b/crates/mcp/src/tools/registry.rs index d7713795..0fd0123d 100644 --- a/crates/mcp/src/tools/registry.rs +++ b/crates/mcp/src/tools/registry.rs @@ -60,32 +60,22 @@ impl ApxServer { if needs_registry_refresh(&cfg.registries) { tracing::info!("Registry indexes stale, refreshing..."); if let Ok(true) = sync_registry_indexes(&path, false).await { - let rebuild_result = tokio::task::spawn_blocking(rebuild_search_index).await; - if let Ok(Err(e)) = rebuild_result { + let pool = self.ctx.dev_db.pool().clone(); + if let Err(e) = rebuild_search_index(pool.clone()).await { tracing::warn!("Failed to rebuild search index after refresh: {}", e); } } } } - // Search in spawn_blocking (sync SQLite operations) - let search_query = args.query.clone(); - let limit = args.limit; - let search_results = match tokio::task::spawn_blocking(move || { - let index = ComponentIndex::new()?; - index.search(&search_query, limit) - }) - .await - { - Ok(Ok(results)) => results, - Ok(Err(e)) => { - return Ok(CallToolResult::error(vec![Content::text(format!( - "Search failed: {e}" - ))])); - } + // Search using async DB layer + let pool = self.ctx.dev_db.pool().clone(); + let index = ComponentIndex::new(pool); + let search_results = match index.search(&args.query, args.limit).await { + Ok(results) => results, Err(e) => { return Ok(CallToolResult::error(vec![Content::text(format!( - "Search task panicked: {e}" + "Search failed: {e}" ))])); } }; From 3b72ea44b624b70b9e1789fe1193e925988161c8 Mon Sep 17 00:00:00 2001 From: renardeinside Date: Wed, 18 Feb 2026 01:07:56 +0100 Subject: [PATCH 5/9] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20decouple?= =?UTF-8?q?=20core=20stdout=20output=20from=20MCP=20stdio=20transport?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/cli/src/__generate_openapi.rs | 4 +- crates/cli/src/build.rs | 2 +- crates/cli/src/dev/mcp.rs | 31 +++--- crates/cli/src/dev/restart.rs | 3 +- crates/cli/src/dev/start.rs | 6 +- crates/cli/src/dev/stop.rs | 3 +- crates/cli/src/lib.rs | 2 +- crates/core/src/api_generator.rs | 5 +- crates/core/src/common.rs | 27 ++++- crates/core/src/databricks_sdk_doc.rs | 4 +- crates/core/src/dev/process.rs | 3 + crates/core/src/interop.rs | 19 ++-- crates/core/src/lib.rs | 3 +- crates/core/src/openapi/mod.rs | 2 +- crates/core/src/ops/check.rs | 10 +- crates/core/src/ops/dev.rs | 129 ++++++++++++++-------- crates/core/src/ops/startup_logs.rs | 8 +- crates/core/src/search/component_index.rs | 66 +++++++++++ crates/core/src/sources/databricks_sdk.rs | 46 ++++++++ crates/db/src/dev.rs | 8 +- crates/mcp/src/indexing.rs | 22 +++- crates/mcp/src/tools/devserver.rs | 9 +- crates/mcp/src/tools/docs.rs | 20 +++- crates/mcp/src/tools/project.rs | 50 +++++---- 24 files changed, 350 insertions(+), 132 deletions(-) diff --git a/crates/cli/src/__generate_openapi.rs b/crates/cli/src/__generate_openapi.rs index 07365ae9..3fcd44fa 100644 --- a/crates/cli/src/__generate_openapi.rs +++ b/crates/cli/src/__generate_openapi.rs @@ -9,8 +9,8 @@ pub struct GenerateOpenapiArgs { pub app_dir: PathBuf, } -pub fn run(args: GenerateOpenapiArgs) -> i32 { - match generate_openapi(&args.app_dir) { +pub async fn run(args: GenerateOpenapiArgs) -> i32 { + match generate_openapi(&args.app_dir).await { Ok(()) => { println!("regenerated"); 0 diff --git a/crates/cli/src/build.rs b/crates/cli/src/build.rs index 320f0bfb..6c7690af 100644 --- a/crates/cli/src/build.rs +++ b/crates/cli/src/build.rs @@ -57,7 +57,7 @@ async fn run_inner(args: BuildArgs) -> Result<(), String> { fs::write(build_dir.join(".gitignore"), "*\n") .map_err(|err| format!("Failed to write build .gitignore: {err}"))?; - generate_openapi(&app_path)?; + generate_openapi(&app_path).await?; if args.skip_ui_build { println!("Skipping UI build"); diff --git a/crates/cli/src/dev/mcp.rs b/crates/cli/src/dev/mcp.rs index 29003cfa..ed3891c1 100644 --- a/crates/cli/src/dev/mcp.rs +++ b/crates/cli/src/dev/mcp.rs @@ -1,5 +1,6 @@ use crate::run_cli_async_helper; use apx_core::components::new_cache_state; +use apx_core::databricks_sdk_doc::fetch_latest_sdk_version; use apx_core::interop::get_databricks_sdk_version; use apx_db::DevDb; use apx_mcp::context::{AppContext, IndexState, SdkIndexParams}; @@ -29,20 +30,22 @@ pub async fn run(_args: McpArgs) -> i32 { tracing::info!("Found Databricks SDK version: {}", v); v } - Ok(None) => { - tracing::info!( - "Databricks SDK not installed, using default version {}", - DEFAULT_SDK_VERSION - ); - DEFAULT_SDK_VERSION.to_string() - } - Err(e) => { - tracing::warn!( - "Failed to detect SDK version: {}, using default {}", - e, - DEFAULT_SDK_VERSION - ); - DEFAULT_SDK_VERSION.to_string() + Ok(None) | Err(_) => { + tracing::info!("SDK not detected locally, fetching latest version from GitHub"); + match fetch_latest_sdk_version().await { + Ok(v) => { + tracing::info!("Latest SDK version from GitHub: {}", v); + v + } + Err(e) => { + tracing::warn!( + "Failed to fetch latest SDK version: {}. Using default {}", + e, + DEFAULT_SDK_VERSION + ); + DEFAULT_SDK_VERSION.to_string() + } + } } }; diff --git a/crates/cli/src/dev/restart.rs b/crates/cli/src/dev/restart.rs index 40764138..58f5cd30 100644 --- a/crates/cli/src/dev/restart.rs +++ b/crates/cli/src/dev/restart.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use crate::common::resolve_app_dir; use crate::run_cli_async_helper; +use apx_core::common::OutputMode; use apx_core::ops::dev::restart_dev_server; #[derive(Args, Debug, Clone)] @@ -21,6 +22,6 @@ pub async fn run(args: RestartArgs) -> i32 { async fn run_inner(args: RestartArgs) -> Result<(), String> { let app_dir = resolve_app_dir(args.app_path); - restart_dev_server(&app_dir).await?; + restart_dev_server(&app_dir, OutputMode::Interactive).await?; Ok(()) } diff --git a/crates/cli/src/dev/start.rs b/crates/cli/src/dev/start.rs index 8656e88c..9a7f3cbf 100644 --- a/crates/cli/src/dev/start.rs +++ b/crates/cli/src/dev/start.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use crate::common::resolve_app_dir; use crate::run_cli_async_helper; +use apx_core::common::OutputMode; use apx_core::ops::dev::stop_dev_server; use apx_core::ops::dev::{spawn_server, start_dev_server}; @@ -46,7 +47,7 @@ pub async fn run(args: StartArgs) -> i32 { async fn run_detached(args: StartArgs) -> Result<(), String> { let app_dir = resolve_app_dir(args.app_path); - let _ = start_dev_server(&app_dir).await?; + let _ = start_dev_server(&app_dir, OutputMode::Interactive).await?; Ok(()) } @@ -58,6 +59,7 @@ async fn run_attached(args: StartArgs) -> Result<(), String> { None, args.skip_credentials_validation, args.timeout, + OutputMode::Interactive, ) .await?; @@ -71,6 +73,6 @@ async fn run_attached(args: StartArgs) -> Result<(), String> { // Run logs command (will return on Ctrl+C) let _ = super::logs::run(logs_args).await; - stop_dev_server(&app_dir).await?; + stop_dev_server(&app_dir, OutputMode::Interactive).await?; Ok(()) } diff --git a/crates/cli/src/dev/stop.rs b/crates/cli/src/dev/stop.rs index 96f91a7a..a923b669 100644 --- a/crates/cli/src/dev/stop.rs +++ b/crates/cli/src/dev/stop.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use crate::common::resolve_app_dir; use crate::run_cli_async_helper; +use apx_core::common::OutputMode; use apx_core::ops::dev::stop_dev_server; #[derive(Args, Debug, Clone)] @@ -21,6 +22,6 @@ pub async fn run(args: StopArgs) -> i32 { async fn run_inner(args: StopArgs) -> Result<(), String> { let app_dir = resolve_app_dir(args.app_path); - stop_dev_server(&app_dir).await?; + stop_dev_server(&app_dir, OutputMode::Interactive).await?; Ok(()) } diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 83f63ce7..4294c60e 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -149,7 +149,7 @@ async fn run_cli_async(args: Vec) -> i32 { FluxCommands::Stop(args) => flux::stop::run(args).await, }, Some(Commands::Upgrade) => upgrade::run().await, - Some(Commands::GenerateOpenapi(args)) => __generate_openapi::run(args), + Some(Commands::GenerateOpenapi(args)) => __generate_openapi::run(args).await, None => { let mut cmd = Cli::command(); let _ = cmd.print_help(); diff --git a/crates/core/src/api_generator.rs b/crates/core/src/api_generator.rs index 858e6288..d05eb65b 100644 --- a/crates/core/src/api_generator.rs +++ b/crates/core/src/api_generator.rs @@ -15,7 +15,7 @@ use crate::download::resolve_uv; use crate::interop::generate_openapi_spec; use crate::openapi; -pub fn generate_openapi(project_root: &Path) -> Result<(), String> { +pub async fn generate_openapi(project_root: &Path) -> Result<(), String> { let metadata = read_project_metadata(project_root)?; let app_slug = metadata.app_slug.clone(); let app_entrypoint = metadata.app_entrypoint.clone(); @@ -23,7 +23,8 @@ pub fn generate_openapi(project_root: &Path) -> Result<(), String> { // Ensure _metadata.py exists before importing the Python module write_metadata_file(project_root, &metadata)?; - let (spec_json, app_slug) = generate_openapi_spec(project_root, &app_entrypoint, &app_slug)?; + let (spec_json, app_slug) = + generate_openapi_spec(project_root, &app_entrypoint, &app_slug).await?; let api_ts_path = project_root .join("src") diff --git a/crates/core/src/common.rs b/crates/core/src/common.rs index 3fb6f326..643daa8c 100644 --- a/crates/core/src/common.rs +++ b/crates/core/src/common.rs @@ -535,7 +535,7 @@ pub async fn run_preflight_checks(app_dir: &Path) -> Result Result println!("{msg}"), + OutputMode::Quiet => eprintln!("{msg}"), + } +} + +/// Create a spinner appropriate for the given output mode. +/// Interactive: visible spinner on stdout. Quiet: hidden (no output). +pub fn spinner_for_mode(message: &str, mode: OutputMode) -> ProgressBar { + match mode { + OutputMode::Interactive => spinner(message), + OutputMode::Quiet => { + let pb = ProgressBar::hidden(); + pb.set_message(message.to_string()); + pb + } + } +} + // Spinner utilities for CLI operations +#[allow(clippy::print_stdout)] pub fn spinner(message: &str) -> ProgressBar { let spinner = ProgressBar::new_spinner(); spinner.set_style( @@ -684,6 +707,7 @@ pub fn format_elapsed_ms(start: Instant) -> String { format!("{seconds}s {remaining_ms}ms") } +#[allow(clippy::print_stdout)] pub fn run_with_spinner(description: &str, success_message: &str, f: F) -> Result<(), String> where F: FnOnce() -> Result<(), String>, @@ -698,6 +722,7 @@ where result } +#[allow(clippy::print_stdout)] pub async fn run_with_spinner_async( description: &str, success_message: &str, diff --git a/crates/core/src/databricks_sdk_doc.rs b/crates/core/src/databricks_sdk_doc.rs index 75c0c203..50a85bcd 100644 --- a/crates/core/src/databricks_sdk_doc.rs +++ b/crates/core/src/databricks_sdk_doc.rs @@ -2,4 +2,6 @@ //! //! This module re-exports from `sources::databricks_sdk` for backward compatibility. -pub use crate::sources::databricks_sdk::{SDKSource, download_and_extract_sdk, load_doc_files}; +pub use crate::sources::databricks_sdk::{ + SDKSource, download_and_extract_sdk, fetch_latest_sdk_version, load_doc_files, +}; diff --git a/crates/core/src/dev/process.rs b/crates/core/src/dev/process.rs index 43991bcb..72fb2c2c 100644 --- a/crates/core/src/dev/process.rs +++ b/crates/core/src/dev/process.rs @@ -2,6 +2,9 @@ //! //! Manages frontend (Vite/Bun), backend (uvicorn), and database (PGlite) processes. //! Subprocess stdout/stderr are captured and forwarded to flux for centralized logging. +// Runs inside the dev server child process (spawned with Stdio::null()), +// never in the MCP server process โ€” stdout output here is safe. +#![allow(clippy::print_stdout)] use std::collections::HashMap; use std::fmt; diff --git a/crates/core/src/interop.rs b/crates/core/src/interop.rs index ac5312e3..8f6a20ef 100644 --- a/crates/core/src/interop.rs +++ b/crates/core/src/interop.rs @@ -75,46 +75,46 @@ pub fn extract_templates() -> Result { resources::templates_dir() } -pub fn generate_openapi_spec( +pub async fn generate_openapi_spec( project_root: &Path, app_entrypoint: &str, app_slug: &str, ) -> Result<(String, String), String> { // Try to fetch from running server first (200ms timeout) - if let Some(spec_json) = try_fetch_openapi_from_server(project_root) { + if let Some(spec_json) = try_fetch_openapi_from_server(project_root).await { debug!("Got OpenAPI spec from running server"); return Ok((spec_json, app_slug.to_string())); } // Fall back to subprocess method - generate_openapi_spec_from_module(project_root, app_entrypoint, app_slug) + generate_openapi_spec_from_module(project_root, app_entrypoint, app_slug).await } /// Try to fetch OpenAPI spec from a running dev server. /// Returns None if server is not running or doesn't respond within 200ms. -fn try_fetch_openapi_from_server(project_root: &Path) -> Option { +async fn try_fetch_openapi_from_server(project_root: &Path) -> Option { let lock_file = lock_path(project_root); let lock = read_lock(&lock_file).ok()?; let url = format!("http://{}:{}/openapi.json", CLIENT_HOST, lock.port); debug!("Trying to fetch OpenAPI from server at {}", url); - let client = reqwest::blocking::Client::builder() + let client = reqwest::Client::builder() .timeout(Duration::from_millis(200)) .build() .ok()?; - let response = client.get(&url).send().ok()?; + let response = client.get(&url).send().await.ok()?; if !response.status().is_success() { return None; } - response.text().ok() + response.text().await.ok() } /// Generate OpenAPI spec by running a Python subprocess via `uv run`. -fn generate_openapi_spec_from_module( +async fn generate_openapi_spec_from_module( project_root: &Path, app_entrypoint: &str, app_slug: &str, @@ -144,11 +144,12 @@ print(json.dumps(app.openapi(), indent=2)) ); let uv_path = try_resolve_uv()?.path; - let output = Command::new(&uv_path) + let output = tokio::process::Command::new(&uv_path) .args(["run", "--no-sync", "python", "-c", &script]) .arg(project_root.to_string_lossy().as_ref()) .current_dir(project_root) .output() + .await .map_err(|e| format!("Failed to run uv for OpenAPI generation: {e}"))?; if !output.status.success() { diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 751e9a5d..4a81c326 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -6,7 +6,8 @@ clippy::panic, clippy::todo, clippy::unimplemented, - clippy::dbg_macro + clippy::dbg_macro, + clippy::print_stdout )] pub mod agent; diff --git a/crates/core/src/openapi/mod.rs b/crates/core/src/openapi/mod.rs index a8e92ee1..4d172142 100644 --- a/crates/core/src/openapi/mod.rs +++ b/crates/core/src/openapi/mod.rs @@ -12,7 +12,7 @@ mod spec; pub use emitter::generate; #[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::print_stdout)] mod tests { use super::*; diff --git a/crates/core/src/ops/check.rs b/crates/core/src/ops/check.rs index 22ebd512..79b91694 100644 --- a/crates/core/src/ops/check.rs +++ b/crates/core/src/ops/check.rs @@ -1,7 +1,7 @@ use std::path::Path; use crate::common::{ - BunCommand, OutputMode, ensure_entrypoint_deps, run_preflight_checks, spinner, + BunCommand, OutputMode, emit, ensure_entrypoint_deps, run_preflight_checks, spinner, }; use crate::download::resolve_uv; use crate::frontend::prepare_frontend_args; @@ -207,11 +207,3 @@ async fn generate_route_tree(app_dir: &Path, mode: OutputMode) -> Result<(), Str } Ok(()) } - -/// Print a message to stdout (Interactive) or stderr (Quiet). -fn emit(mode: OutputMode, msg: &str) { - match mode { - OutputMode::Interactive => println!("{msg}"), - OutputMode::Quiet => eprintln!("{msg}"), - } -} diff --git a/crates/core/src/ops/dev.rs b/crates/core/src/ops/dev.rs index 5ec52ee7..7766e012 100644 --- a/crates/core/src/ops/dev.rs +++ b/crates/core/src/ops/dev.rs @@ -4,7 +4,8 @@ use std::process::Stdio; use std::time::{Duration, Instant}; use crate::common::{ - ApxCommand, ensure_dir, format_elapsed_ms, handle_spawn_error, run_preflight_checks, spinner, + ApxCommand, OutputMode, emit, ensure_dir, format_elapsed_ms, handle_spawn_error, + run_preflight_checks, spinner_for_mode, }; use crate::dev::client::{HealthCheckConfig, health, status, stop as stop_server}; use crate::dev::common::{ @@ -21,7 +22,10 @@ fn prepare_app_dir(app_dir: &Path) -> Result<(), String> { Ok(()) } -pub async fn resolve_existing_server(app_dir: &Path) -> Result, String> { +pub async fn resolve_existing_server( + app_dir: &Path, + mode: OutputMode, +) -> Result, String> { let lock_path = lock_path(app_dir); if !lock_path.exists() { return Ok(None); @@ -30,7 +34,7 @@ pub async fn resolve_existing_server(app_dir: &Path) -> Result, Stri let lock = read_lock(&lock_path)?; if !is_process_running(lock.pid) { - println!("๐Ÿงน Cleaning up stale lock file..."); + emit(mode, "๐Ÿงน Cleaning up stale lock file..."); remove_lock(&lock_path)?; return Ok(None); } @@ -38,7 +42,7 @@ pub async fn resolve_existing_server(app_dir: &Path) -> Result, Stri match health(lock.port).await { Ok(true) => Ok(Some(lock.port)), Ok(false) | Err(_) => { - println!("๐Ÿงน Cleaning up stale lock file..."); + emit(mode, "๐Ÿงน Cleaning up stale lock file..."); remove_lock(&lock_path)?; Ok(None) } @@ -48,41 +52,50 @@ pub async fn resolve_existing_server(app_dir: &Path) -> Result, Stri /// Start a dev server for the given app directory. /// If a server is already running and healthy, returns its port. /// Otherwise spawns a new server subprocess. -pub async fn start_dev_server(app_dir: &Path) -> Result { - if let Some(port) = resolve_existing_server(app_dir).await? { +pub async fn start_dev_server(app_dir: &Path, mode: OutputMode) -> Result { + if let Some(port) = resolve_existing_server(app_dir, mode).await? { return Ok(port); } - spawn_server(app_dir, None, false, 60).await + spawn_server(app_dir, None, false, 60, mode).await } /// Run preflight checks and display progress. -async fn run_preflight(app_dir: &Path) -> Result<(), String> { - println!("๐Ÿ›ซ Preflight check started..."); +async fn run_preflight(app_dir: &Path, mode: OutputMode) -> Result<(), String> { + emit(mode, "๐Ÿ›ซ Preflight check started..."); let preflight_start = Instant::now(); - let preflight_spinner = spinner(" Running preflight checks..."); + let preflight_spinner = spinner_for_mode(" Running preflight checks...", mode); let result = run_preflight_checks(app_dir).await; preflight_spinner.finish_and_clear(); match result { Ok(preflight) => { - println!(" โœ“ verified project layout ({}ms)", preflight.layout_ms); - println!(" โœ“ uv sync ({}ms)", preflight.uv_sync_ms); - println!(" โœ“ version file ({}ms)", preflight.version_ms); + emit( + mode, + &format!(" โœ“ verified project layout ({}ms)", preflight.layout_ms), + ); + emit(mode, &format!(" โœ“ uv sync ({}ms)", preflight.uv_sync_ms)); + emit( + mode, + &format!(" โœ“ version file ({}ms)", preflight.version_ms), + ); if let Some(bun_ms) = preflight.bun_install_ms { - println!(" โœ“ bun install ({bun_ms}ms)"); + emit(mode, &format!(" โœ“ bun install ({bun_ms}ms)")); } else { - println!(" โœ“ node_modules (cached)"); + emit(mode, " โœ“ node_modules (cached)"); } - println!( - "โœ… Ready for takeoff! ({})\n", - format_elapsed_ms(preflight_start) + emit( + mode, + &format!( + "โœ… Ready for takeoff! ({})\n", + format_elapsed_ms(preflight_start) + ), ); Ok(()) } Err(e) => { - println!("โŒ Preflight check failed\n"); + emit(mode, "โŒ Preflight check failed\n"); Err(e) } } @@ -94,14 +107,17 @@ const PORT_WAIT_TIMEOUT_MS: u64 = 2000; const PORT_WAIT_INTERVAL_MS: u64 = 100; /// Wait for a port to become available, with timeout. -async fn wait_for_port_available(port: u16) -> Result<(), String> { +async fn wait_for_port_available(port: u16, mode: OutputMode) -> Result<(), String> { let max_attempts = PORT_WAIT_TIMEOUT_MS / PORT_WAIT_INTERVAL_MS; for attempt in 0..max_attempts { if TcpListener::bind((BIND_HOST, port)).is_ok() { return Ok(()); } if attempt == 0 { - println!("โณ Waiting for port {port} to become available..."); + emit( + mode, + &format!("โณ Waiting for port {port} to become available..."), + ); } tokio::time::sleep(Duration::from_millis(PORT_WAIT_INTERVAL_MS)).await; } @@ -115,6 +131,7 @@ async fn wait_for_healthy_with_logs( port: u16, config: &HealthCheckConfig, app_dir: &Path, + mode: OutputMode, ) -> Result<(), String> { use crate::ops::startup_logs::StartupLogStreamer; @@ -126,7 +143,7 @@ async fn wait_for_healthy_with_logs( let start_time = Instant::now(); let deadline = start_time + Duration::from_secs(config.timeout_secs); - let mut log_streamer = StartupLogStreamer::new(app_dir).await; + let mut log_streamer = StartupLogStreamer::new(app_dir, mode).await; let mut attempt_count = 0u32; let mut last_overall_status: Option = None; let mut first_response_logged = false; @@ -178,7 +195,7 @@ async fn wait_for_healthy_with_logs( ); if status_response.db_status != "healthy" { - println!("โš ๏ธ Database not available: local development will work but DB features disabled"); + emit(mode, "โš ๏ธ Database not available: local development will work but DB features disabled"); } return Ok(()); @@ -238,15 +255,16 @@ pub async fn spawn_server( preferred_port: Option, skip_credentials_validation: bool, timeout_secs: u64, + mode: OutputMode, ) -> Result { let start_time = Instant::now(); prepare_app_dir(app_dir)?; - run_preflight(app_dir).await?; + run_preflight(app_dir, mode).await?; let lock_path = lock_path(app_dir); - println!("๐Ÿš€ Starting dev server..."); + emit(mode, "๐Ÿš€ Starting dev server..."); if let Err(e) = flux::ensure_running() { debug!("Failed to start flux: {e}. Logs may not be collected."); @@ -261,7 +279,7 @@ pub async fn spawn_server( let port = registry.get_or_allocate_port(app_dir, preferred_port)?; registry.save()?; - wait_for_port_available(port).await?; + wait_for_port_available(port, mode).await?; let apx_cmd = ApxCommand::new().await?; @@ -307,13 +325,13 @@ pub async fn spawn_server( .spawn() .map_err(|err| handle_spawn_error("apx", err))?; - println!("โณ Waiting for dev server to become healthy...\n"); + emit(mode, "โณ Waiting for dev server to become healthy...\n"); let config = HealthCheckConfig { timeout_secs, ..HealthCheckConfig::default() }; - let health_result = wait_for_healthy_with_logs(port, &config, app_dir).await; + let health_result = wait_for_healthy_with_logs(port, &config, app_dir, mode).await; if let Err(e) = health_result { debug!("Health checks failed, attempting graceful shutdown."); @@ -346,21 +364,24 @@ pub async fn spawn_server( let lock = DevLock::new(pid, port, command, app_dir); write_lock(&lock_path, &lock)?; - println!( - "โœ… Dev server started at http://localhost:{port} in {}\n", - format_elapsed_ms(start_time) + emit( + mode, + &format!( + "โœ… Dev server started at http://localhost:{port} in {}\n", + format_elapsed_ms(start_time) + ), ); Ok(port) } /// Stop the dev server for the given app directory. /// Returns true if a server was found and stopped, false if no server was running. -pub async fn stop_dev_server(app_dir: &Path) -> Result { +pub async fn stop_dev_server(app_dir: &Path, mode: OutputMode) -> Result { let lock_path = lock_path(app_dir); debug!(path = %lock_path.display(), "Checking for dev server lockfile."); if !lock_path.exists() { debug!("No dev server lockfile found."); - println!("โš ๏ธ No dev server running\n"); + emit(mode, "โš ๏ธ No dev server running\n"); return Ok(false); } @@ -372,15 +393,18 @@ pub async fn stop_dev_server(app_dir: &Path) -> Result { ); let start_time = Instant::now(); - let stop_spinner = spinner("Stopping dev server..."); + let stop_spinner = spinner_for_mode("Stopping dev server...", mode); match stop_server(lock.port).await { Ok(()) => { debug!("Dev server stopped gracefully via HTTP."); stop_spinner.finish_and_clear(); - println!( - "โœ… Dev server stopped in {}\n", - format_elapsed_ms(start_time) + emit( + mode, + &format!( + "โœ… Dev server stopped in {}\n", + format_elapsed_ms(start_time) + ), ); return Ok(true); } @@ -395,16 +419,19 @@ pub async fn stop_dev_server(app_dir: &Path) -> Result { Ok(()) => { debug!("Dev server process tree killed; removing lockfile."); remove_lock(&lock_path)?; - println!( - "โœ… Dev server stopped in {}\n", - format_elapsed_ms(start_time) + emit( + mode, + &format!( + "โœ… Dev server stopped in {}\n", + format_elapsed_ms(start_time) + ), ); Ok(true) } Err(err) => { warn!(error = %err, pid = lock.pid, "Failed to kill dev server process tree."); remove_lock(&lock_path)?; - println!("โœ… Dev server already stopped\n"); + emit(mode, "โœ… Dev server already stopped\n"); Ok(true) } } @@ -412,21 +439,27 @@ pub async fn stop_dev_server(app_dir: &Path) -> Result { /// Restart the dev server for the given app directory. /// Preserves the port if an existing server is found. -pub async fn restart_dev_server(app_dir: &Path) -> Result { +pub async fn restart_dev_server(app_dir: &Path, mode: OutputMode) -> Result { let lock_path = lock_path(app_dir); let preferred_port = if lock_path.exists() { let lock = read_lock(&lock_path)?; - println!( - "Found existing dev server at http://localhost:{port}", - port = lock.port + emit( + mode, + &format!( + "Found existing dev server at http://localhost:{port}", + port = lock.port + ), ); - stop_dev_server(app_dir).await?; + stop_dev_server(app_dir, mode).await?; Some(lock.port) } else { None }; - let port = spawn_server(app_dir, preferred_port, false, 60).await?; - println!("โœ… Dev server restarted at http://localhost:{port}\n"); + let port = spawn_server(app_dir, preferred_port, false, 60, mode).await?; + emit( + mode, + &format!("โœ… Dev server restarted at http://localhost:{port}\n"), + ); Ok(port) } diff --git a/crates/core/src/ops/startup_logs.rs b/crates/core/src/ops/startup_logs.rs index 5476f622..cd378741 100644 --- a/crates/core/src/ops/startup_logs.rs +++ b/crates/core/src/ops/startup_logs.rs @@ -8,11 +8,14 @@ use std::path::Path; use apx_common::{LogRecord, should_skip_log}; use apx_db::LogsDb; +use crate::common::{OutputMode, emit}; + /// Simple log streamer that prints logs line-by-line to stdout. pub struct StartupLogStreamer { last_log_id: i64, storage: Option, app_path: String, + mode: OutputMode, } impl std::fmt::Debug for StartupLogStreamer { @@ -27,7 +30,7 @@ impl std::fmt::Debug for StartupLogStreamer { impl StartupLogStreamer { /// Create a new log streamer for the given app directory. - pub async fn new(app_dir: &Path) -> Self { + pub async fn new(app_dir: &Path, mode: OutputMode) -> Self { let app_path = app_dir .canonicalize() .unwrap_or_else(|_| app_dir.to_path_buf()) @@ -44,6 +47,7 @@ impl StartupLogStreamer { last_log_id, storage, app_path, + mode, } } @@ -67,7 +71,7 @@ impl StartupLogStreamer { let mut count = 0; for record in &records { if !should_skip_log(record) { - println!("{}", format_startup_log(record)); + emit(self.mode, &format_startup_log(record)); count += 1; } } diff --git a/crates/core/src/search/component_index.rs b/crates/core/src/search/component_index.rs index 7ad63b92..47477c46 100644 --- a/crates/core/src/search/component_index.rs +++ b/crates/core/src/search/component_index.rs @@ -319,4 +319,70 @@ mod tests { custom_btn.score ); } + + /// Regression test: multi-term queries should return partial matches instead + /// of empty results when not all terms appear in a single document. + /// See: "@animate-ui number counter ticker" returning [] while "animate-ui" alone works. + #[tokio::test] + async fn test_multiterm_query_returns_partial_matches() { + let index = test_index().await; + + sqlx::query(&format!( + "CREATE VIRTUAL TABLE {TABLE_NAME} USING fts5(\ + id UNINDEXED, name, registry UNINDEXED, text, \ + tokenize='porter unicode61'\ + )" + )) + .execute(&index.pool) + .await + .unwrap(); + + // "animate" in name + "ui component" in text โ†’ phrase "animate-ui" matches + // via FTS5 cross-column phrase matching (animate in name, ui in text). + // Also has "number", "counter" but NOT "ticker". + sqlx::query(&format!( + "INSERT INTO {TABLE_NAME} (id, name, registry, text) VALUES (?1, ?2, ?3, ?4)" + )) + .bind("@animate-ui/number-ticker") + .bind("animate") + .bind("animate-ui") + .bind("animate number counter ui component") + .execute(&index.pool) + .await + .unwrap(); + + // Unrelated component + sqlx::query(&format!( + "INSERT INTO {TABLE_NAME} (id, name, registry, text) VALUES (?1, ?2, ?3, ?4)" + )) + .bind("button") + .bind("button") + .bind("") + .bind("button A styled button component shadcn") + .execute(&index.pool) + .await + .unwrap(); + + // Single-term query should find the component + let results = index.search("animate", 10).await.unwrap(); + assert!( + results.iter().any(|r| r.id == "@animate-ui/number-ticker"), + "Single-term 'animate' should match. Got: {results:?}" + ); + + // Multi-term query with a term NOT in the document ("ticker") should + // still return partial matches, not empty. + let results = index + .search("@animate-ui number counter ticker", 10) + .await + .unwrap(); + assert!( + !results.is_empty(), + "Multi-term query '@animate-ui number counter ticker' should return partial matches, not empty" + ); + assert!( + results.iter().any(|r| r.id == "@animate-ui/number-ticker"), + "Should find number-ticker component. Got: {results:?}" + ); + } } diff --git a/crates/core/src/sources/databricks_sdk.rs b/crates/core/src/sources/databricks_sdk.rs index 85786e99..2f15ff39 100644 --- a/crates/core/src/sources/databricks_sdk.rs +++ b/crates/core/src/sources/databricks_sdk.rs @@ -165,6 +165,35 @@ pub async fn download_and_extract_sdk(version: &str) -> Result Ok(docs_path) } +/// Fetch the latest SDK version from GitHub releases. +/// Returns the version string (e.g. "0.47.0") without the "v" prefix. +pub async fn fetch_latest_sdk_version() -> Result { + let url = format!("https://api.github.com/repos/{GITHUB_REPO}/releases/latest"); + let client = reqwest::Client::new(); + let response = client + .get(&url) + .header("User-Agent", "apx-cli") + .timeout(std::time::Duration::from_secs(5)) + .send() + .await + .map_err(|e| format!("Failed to fetch latest SDK version: {e}"))?; + + if !response.status().is_success() { + return Err(format!("GitHub API returned HTTP {}", response.status())); + } + + let body: serde_json::Value = response + .json() + .await + .map_err(|e| format!("Failed to parse GitHub response: {e}"))?; + + let tag = body["tag_name"] + .as_str() + .ok_or("Missing tag_name in GitHub response")?; + + Ok(tag.strip_prefix('v').unwrap_or(tag).to_string()) +} + /// Extract method name from signature like "create(spark_version: str, ...)" -> "create" fn extract_method_name(signature: &str) -> Option<&str> { let name_part = signature.split('(').next()?; @@ -563,3 +592,20 @@ pub fn load_doc_files(docs_path: &Path) -> Result, String> { load_timer.finish(); Ok(files) } + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_fetch_latest_sdk_version() { + let version = fetch_latest_sdk_version().await.unwrap(); + assert!(!version.is_empty()); + assert!(!version.starts_with('v')); + assert!( + version.contains('.'), + "Expected semver format, got: {version}" + ); + } +} diff --git a/crates/db/src/dev.rs b/crates/db/src/dev.rs index 852a2d7f..3abbee9b 100644 --- a/crates/db/src/dev.rs +++ b/crates/db/src/dev.rs @@ -62,6 +62,8 @@ pub async fn table_exists(pool: &SqlitePool, table_name: &str) -> Result String { query .split_whitespace() @@ -78,7 +80,7 @@ pub fn sanitize_fts5_query(query: &str) -> String { }) .filter(|s| !s.is_empty()) .collect::>() - .join(" ") + .join(" OR ") } #[cfg(test)] @@ -88,14 +90,14 @@ mod tests { #[test] fn test_sanitize_fts5_query_basic() { - assert_eq!(sanitize_fts5_query("hello world"), "\"hello\" \"world\""); + assert_eq!(sanitize_fts5_query("hello world"), "\"hello\" OR \"world\""); } #[test] fn test_sanitize_fts5_query_special_chars() { assert_eq!( sanitize_fts5_query("hello* OR world"), - "\"hello\" \"OR\" \"world\"" + "\"hello\" OR \"OR\" OR \"world\"" ); } diff --git a/crates/mcp/src/indexing.rs b/crates/mcp/src/indexing.rs index 0b94e8ea..cad4c415 100644 --- a/crates/mcp/src/indexing.rs +++ b/crates/mcp/src/indexing.rs @@ -144,17 +144,27 @@ async fn ensure_search_index(pool: SqlitePool) -> Result<(), String> { } } -/// Wait for an index to be ready with timeout (15 seconds) +/// Wait for an index to be ready with timeout (15 seconds). +/// +/// Returns `true` if the index was already ready, `false` if we had to wait. +/// Returns `Err` if the timeout expired before the index became ready. pub async fn wait_for_index_ready( ready_notify: &Notify, is_ready: &AtomicBool, index_name: &str, -) -> Result<(), String> { +) -> Result { const TIMEOUT_SECS: u64 = 15; - // Check if already ready + // Register as a waiter BEFORE checking the flag to avoid a race where + // notify_waiters() fires between our is_ready check and the notified() call. + // (notify_waiters does not store a permit, so late registrations miss it.) + let notified = ready_notify.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + + // Fast path: already ready if is_ready.load(Ordering::SeqCst) { - return Ok(()); + return Ok(true); } tracing::debug!( @@ -164,10 +174,10 @@ pub async fn wait_for_index_ready( ); // Wait with timeout - match tokio::time::timeout(Duration::from_secs(TIMEOUT_SECS), ready_notify.notified()).await { + match tokio::time::timeout(Duration::from_secs(TIMEOUT_SECS), notified).await { Ok(_) => { tracing::debug!("{} index is now ready", index_name); - Ok(()) + Ok(false) } Err(_) => { tracing::warn!( diff --git a/crates/mcp/src/tools/devserver.rs b/crates/mcp/src/tools/devserver.rs index 20b20bc1..663fbcfd 100644 --- a/crates/mcp/src/tools/devserver.rs +++ b/crates/mcp/src/tools/devserver.rs @@ -23,10 +23,11 @@ impl ApxServer { let path = validate_app_path(&args.app_path) .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; + use apx_core::common::OutputMode; use apx_core::dev::common::CLIENT_HOST; use apx_core::ops::dev::start_dev_server; - match start_dev_server(&path).await { + match start_dev_server(&path, OutputMode::Quiet).await { Ok(port) => Ok(CallToolResult::success(vec![Content::text(format!( "Dev server started at http://{CLIENT_HOST}:{port}" ))])), @@ -38,9 +39,10 @@ impl ApxServer { let path = validate_app_path(&args.app_path) .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; + use apx_core::common::OutputMode; use apx_core::ops::dev::stop_dev_server; - match stop_dev_server(&path).await { + match stop_dev_server(&path, OutputMode::Quiet).await { Ok(true) => Ok(CallToolResult::success(vec![Content::text( "Dev server stopped", )])), @@ -58,9 +60,10 @@ impl ApxServer { let path = validate_app_path(&args.app_path) .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; + use apx_core::common::OutputMode; use apx_core::ops::dev::restart_dev_server; - match restart_dev_server(&path).await { + match restart_dev_server(&path, OutputMode::Quiet).await { Ok(port) => Ok(CallToolResult::success(vec![Content::text(format!( "Dev server restarted at http://localhost:{port}" ))])), diff --git a/crates/mcp/src/tools/docs.rs b/crates/mcp/src/tools/docs.rs index c1951431..37221fc1 100644 --- a/crates/mcp/src/tools/docs.rs +++ b/crates/mcp/src/tools/docs.rs @@ -30,15 +30,16 @@ impl ApxServer { let ctx = &self.ctx; // Wait for SDK index to be ready (15 second timeout) - if let Err(e) = wait_for_index_ready( + let was_already_ready = match wait_for_index_ready( &ctx.index_state.sdk_ready, &ctx.index_state.sdk_indexed, "SDK documentation", ) .await { - return Ok(CallToolResult::error(vec![Content::text(e)])); - } + Ok(ready) => ready, + Err(e) => return Ok(CallToolResult::error(vec![Content::text(e)])), + }; // Get the SDK doc index let mut index_guard = ctx.sdk_doc_index.lock().await; @@ -93,6 +94,8 @@ impl ApxServer { source: String, query: String, results: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + note: Option, } #[derive(Serialize)] @@ -102,6 +105,16 @@ impl ApxServer { score: f32, } + let note = if results.is_empty() && !was_already_ready { + Some( + "Index was still initializing when this query arrived. \ + Results may be incomplete โ€” retry in a few seconds." + .to_string(), + ) + } else { + None + }; + let response = DocsResponse { source: match args.source { SDKSource::DatabricksSdkPython => "databricks-sdk-python".to_string(), @@ -115,6 +128,7 @@ impl ApxServer { score: r.score, }) .collect(), + note, }; Ok(CallToolResult::from_serializable(&response)) diff --git a/crates/mcp/src/tools/project.rs b/crates/mcp/src/tools/project.rs index 502da653..dad25392 100644 --- a/crates/mcp/src/tools/project.rs +++ b/crates/mcp/src/tools/project.rs @@ -192,9 +192,7 @@ impl ApxServer { let path = validate_app_path(&args.app_path) .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; - use apx_core::api_generator::generate_openapi; - - match generate_openapi(&path) { + match apx_core::api_generator::generate_openapi(&path).await { Ok(()) => Ok(CallToolResult::success(vec![Content::text( "OpenAPI regenerated", )])), @@ -217,15 +215,20 @@ impl ApxServer { Err(e) => return Ok(CallToolResult::error(vec![Content::text(e)])), }; - let openapi_content = - match generate_openapi_spec(&path, &metadata.app_entrypoint, &metadata.app_slug) { - Ok((content, _)) => content, - Err(e) => { - return Ok(CallToolResult::error(vec![Content::text(format!( - "Failed to generate OpenAPI spec: {e}" - ))])); - } - }; + let openapi_content = match generate_openapi_spec( + &path, + &metadata.app_entrypoint, + &metadata.app_slug, + ) + .await + { + Ok((content, _)) => content, + Err(e) => { + return Ok(CallToolResult::error(vec![Content::text(format!( + "Failed to generate OpenAPI spec: {e}" + ))])); + } + }; let openapi: Value = match serde_json::from_str(&openapi_content) { Ok(spec) => spec, @@ -311,15 +314,20 @@ impl ApxServer { Err(e) => return Ok(CallToolResult::error(vec![Content::text(e)])), }; - let (openapi_content, _) = - match generate_openapi_spec(&path, &metadata.app_entrypoint, &metadata.app_slug) { - Ok(result) => result, - Err(e) => { - return Ok(CallToolResult::error(vec![Content::text(format!( - "Failed to generate OpenAPI spec: {e}" - ))])); - } - }; + let (openapi_content, _) = match generate_openapi_spec( + &path, + &metadata.app_entrypoint, + &metadata.app_slug, + ) + .await + { + Ok(result) => result, + Err(e) => { + return Ok(CallToolResult::error(vec![Content::text(format!( + "Failed to generate OpenAPI spec: {e}" + ))])); + } + }; let openapi: Value = match serde_json::from_str(&openapi_content) { Ok(spec) => spec, From 864c65d863d10deb30f98128d0c971ec5ebad1b9 Mon Sep 17 00:00:00 2001 From: renardeinside Date: Wed, 18 Feb 2026 01:17:33 +0100 Subject: [PATCH 6/9] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20deps:=20switch=20TLS?= =?UTF-8?q?=20backend=20from=20aws-lc-rs/rustls=20to=20native-tls?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 418 +++++++++++++---------------------------------------- Cargo.toml | 4 +- 2 files changed, 106 insertions(+), 316 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index efd9150f..caffb4b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -358,28 +358,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "aws-lc-rs" -version = "1.15.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b7b6141e96a8c160799cc2d5adecd5cbbe5054cb8c7c4af53da0f83bb7ad256" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.37.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c34dda4df7017c8db52132f0f8a2e0f8161649d15723ed63fc00c82d0f2081a" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", -] - [[package]] name = "axum" version = "0.8.8" @@ -882,12 +860,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "cfg_aliases" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" - [[package]] name = "chrono" version = "0.4.43" @@ -974,15 +946,6 @@ version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" -[[package]] -name = "cmake" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" -dependencies = [ - "cc", -] - [[package]] name = "colorchoice" version = "1.0.4" @@ -1096,7 +1059,7 @@ dependencies = [ "bitflags 2.10.0", "core-foundation 0.10.1", "core-graphics-types", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1586,15 +1549,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - [[package]] name = "enumflags2" version = "0.7.12" @@ -1751,6 +1705,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared 0.1.1", +] + [[package]] name = "foreign-types" version = "0.5.0" @@ -1758,7 +1721,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" dependencies = [ "foreign-types-macros", - "foreign-types-shared", + "foreign-types-shared 0.3.1", ] [[package]] @@ -1772,6 +1735,12 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "foreign-types-shared" version = "0.3.1" @@ -1787,12 +1756,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "fsevent-sys" version = "4.1.0" @@ -2048,10 +2011,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", - "js-sys", "libc", "wasi 0.11.1+wasi-snapshot-preview1", - "wasm-bindgen", ] [[package]] @@ -2438,31 +2399,31 @@ dependencies = [ ] [[package]] -name = "hyper-rustls" -version = "0.27.7" +name = "hyper-timeout" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "http", "hyper", "hyper-util", - "rustls", - "rustls-pki-types", + "pin-project-lite", "tokio", - "tokio-rustls", "tower-service", ] [[package]] -name = "hyper-timeout" -version = "0.5.2" +name = "hyper-tls" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ + "bytes", + "http-body-util", "hyper", "hyper-util", - "pin-project-lite", + "native-tls", "tokio", + "tokio-native-tls", "tower-service", ] @@ -2485,11 +2446,9 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", - "system-configuration", "tokio", "tower-service", "tracing", - "windows-registry", ] [[package]] @@ -3004,12 +2963,6 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" -[[package]] -name = "lru-slab" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" - [[package]] name = "lzma-rs" version = "0.3.0" @@ -3157,6 +3110,23 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "ndk" version = "0.9.0" @@ -3557,11 +3527,49 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags 2.10.0", + "cfg-if", + "foreign-types 0.3.2", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "openssl-probe" -version = "0.2.1" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] [[package]] name = "opentelemetry" @@ -4306,62 +4314,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "quinn" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror 2.0.18", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" -dependencies = [ - "aws-lc-rs", - "bytes", - "getrandom 0.3.4", - "lru-slab", - "rand 0.9.2", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror 2.0.18", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.60.2", -] - [[package]] name = "quote" version = "1.0.44" @@ -4653,31 +4605,26 @@ checksum = "04e9018c9d814e5f30cc16a0f03271aeab3571e609612d9fe78c1aa8d11c2f62" dependencies = [ "base64 0.22.1", "bytes", - "encoding_rs", "futures-channel", "futures-core", "futures-util", - "h2", "http", "http-body", "http-body-util", "hyper", - "hyper-rustls", + "hyper-tls", "hyper-util", "js-sys", "log", - "mime", + "native-tls", "percent-encoding", "pin-project-lite", - "quinn", - "rustls", "rustls-pki-types", - "rustls-platform-verifier", "serde", "serde_json", "sync_wrapper", "tokio", - "tokio-rustls", + "tokio-native-tls", "tokio-util", "tower", "tower-http", @@ -4695,20 +4642,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "194d8e591e405d1eecf28819740abed6d719d1a2db87fc0bcdedee9a26d55560" -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.17", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - [[package]] name = "rmcp" version = "0.15.0" @@ -4798,81 +4731,15 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "rustls" -version = "0.23.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" -dependencies = [ - "aws-lc-rs", - "once_cell", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-native-certs" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" -dependencies = [ - "openssl-probe", - "rustls-pki-types", - "schannel", - "security-framework", -] - [[package]] name = "rustls-pki-types" version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ - "web-time", "zeroize", ] -[[package]] -name = "rustls-platform-verifier" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" -dependencies = [ - "core-foundation 0.10.1", - "core-foundation-sys", - "jni", - "log", - "once_cell", - "rustls", - "rustls-native-certs", - "rustls-platform-verifier-android", - "rustls-webpki", - "security-framework", - "security-framework-sys", - "webpki-root-certs", - "windows-sys 0.61.2", -] - -[[package]] -name = "rustls-platform-verifier-android" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" - -[[package]] -name = "rustls-webpki" -version = "0.103.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" -dependencies = [ - "aws-lc-rs", - "ring", - "rustls-pki-types", - "untrusted", -] - [[package]] name = "rustversion" version = "1.0.22" @@ -4978,12 +4845,12 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "security-framework" -version = "3.5.1" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.10.0", - "core-foundation 0.10.1", + "core-foundation 0.9.4", "core-foundation-sys", "libc", "security-framework-sys", @@ -5741,27 +5608,6 @@ dependencies = [ "windows 0.57.0", ] -[[package]] -name = "system-configuration" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" -dependencies = [ - "bitflags 2.10.0", - "core-foundation 0.9.4", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "system-deps" version = "6.2.2" @@ -6245,6 +6091,16 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-postgres" version = "0.7.16" @@ -6271,16 +6127,6 @@ dependencies = [ "whoami 2.1.0", ] -[[package]] -name = "tokio-rustls" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" -dependencies = [ - "rustls", - "tokio", -] - [[package]] name = "tokio-stream" version = "0.1.18" @@ -6300,12 +6146,10 @@ checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" dependencies = [ "futures-util", "log", - "rustls", - "rustls-pki-types", + "native-tls", "tokio", - "tokio-rustls", + "tokio-native-tls", "tungstenite", - "webpki-roots 0.26.11", ] [[package]] @@ -6656,9 +6500,8 @@ dependencies = [ "http", "httparse", "log", + "native-tls", "rand 0.9.2", - "rustls", - "rustls-pki-types", "sha1", "thiserror 2.0.18", "utf-8", @@ -6786,12 +6629,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - [[package]] name = "url" version = "2.5.8" @@ -7097,33 +6934,6 @@ dependencies = [ "system-deps", ] -[[package]] -name = "webpki-root-certs" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36a29fc0408b113f68cf32637857ab740edfafdf460c326cd2afaa2d84cc05dc" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "webpki-roots" -version = "0.26.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" -dependencies = [ - "webpki-roots 1.0.5", -] - -[[package]] -name = "webpki-roots" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "webview2-com" version = "0.38.2" @@ -7386,17 +7196,6 @@ dependencies = [ "windows-link 0.1.3", ] -[[package]] -name = "windows-registry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" -dependencies = [ - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - [[package]] name = "windows-result" version = "0.1.2" @@ -7460,15 +7259,6 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.59.0" diff --git a/Cargo.toml b/Cargo.toml index e71d5a7a..f4010327 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,8 +36,8 @@ rmcp = { version = "0.15", features = ["server", "transport-io", "schemars"] } # Web server axum = { version = "0.8.8", features = ["ws"] } -reqwest = { version = "0.13.1", features = ["blocking", "json", "stream"] } -tokio-tungstenite = { version = "0.28", features = ["connect", "handshake", "stream", "rustls-tls-webpki-roots"] } +reqwest = { version = "0.13.1", default-features = false, features = ["blocking", "json", "stream", "native-tls"] } +tokio-tungstenite = { version = "0.28", features = ["connect", "handshake", "stream", "native-tls"] } # OpenTelemetry opentelemetry = "0.29" From 3a8cf4bcbf62f1b83d8847ca770fea5b9d21e356 Mon Sep 17 00:00:00 2001 From: renardeinside Date: Wed, 18 Feb 2026 01:28:46 +0100 Subject: [PATCH 7/9] =?UTF-8?q?=F0=9F=90=9B=20fix:=20re-enable=20http2,=20?= =?UTF-8?q?charset,=20and=20system-proxy=20features=20for=20reqwest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 126 +++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 +- 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index caffb4b1..e7564949 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1549,6 +1549,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "enumflags2" version = "0.7.12" @@ -2398,6 +2407,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + [[package]] name = "hyper-timeout" version = "0.5.2" @@ -2446,9 +2471,11 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -4605,17 +4632,21 @@ checksum = "04e9018c9d814e5f30cc16a0f03271aeab3571e609612d9fe78c1aa8d11c2f62" dependencies = [ "base64 0.22.1", "bytes", + "encoding_rs", "futures-channel", "futures-core", "futures-util", + "h2", "http", "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-tls", "hyper-util", "js-sys", "log", + "mime", "native-tls", "percent-encoding", "pin-project-lite", @@ -4642,6 +4673,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "194d8e591e405d1eecf28819740abed6d719d1a2db87fc0bcdedee9a26d55560" +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rmcp" version = "0.15.0" @@ -4731,6 +4776,19 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -4740,6 +4798,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-webpki" +version = "0.103.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -5608,6 +5677,27 @@ dependencies = [ "windows 0.57.0", ] +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "system-deps" version = "6.2.2" @@ -6127,6 +6217,16 @@ dependencies = [ "whoami 2.1.0", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.18" @@ -6629,6 +6729,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -7196,6 +7302,17 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + [[package]] name = "windows-result" version = "0.1.2" @@ -7259,6 +7376,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.59.0" diff --git a/Cargo.toml b/Cargo.toml index f4010327..dd1013c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ rmcp = { version = "0.15", features = ["server", "transport-io", "schemars"] } # Web server axum = { version = "0.8.8", features = ["ws"] } -reqwest = { version = "0.13.1", default-features = false, features = ["blocking", "json", "stream", "native-tls"] } +reqwest = { version = "0.13.1", default-features = false, features = ["blocking", "json", "stream", "native-tls", "http2", "charset", "system-proxy"] } tokio-tungstenite = { version = "0.28", features = ["connect", "handshake", "stream", "native-tls"] } # OpenTelemetry From 57fea16e5ab61414f5229af62f7726a7d405a232 Mon Sep 17 00:00:00 2001 From: renardeinside Date: Wed, 18 Feb 2026 01:42:53 +0100 Subject: [PATCH 8/9] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20list=5Fregistry?= =?UTF-8?q?=5Fcomponents=20MCP=20tool=20to=20browse=20full=20registry=20ca?= =?UTF-8?q?talogs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/mcp/src/server.rs | 14 ++++- crates/mcp/src/tools/registry.rs | 89 +++++++++++++++++++++++++++++++- 2 files changed, 101 insertions(+), 2 deletions(-) diff --git a/crates/mcp/src/server.rs b/crates/mcp/src/server.rs index b7f0793e..e613df8a 100644 --- a/crates/mcp/src/server.rs +++ b/crates/mcp/src/server.rs @@ -6,7 +6,7 @@ use crate::tools::databricks::DatabricksAppsLogsArgs; use crate::tools::devserver::LogsToolArgs; use crate::tools::docs::DocsArgs; use crate::tools::project::GetRouteInfoArgs; -use crate::tools::registry::{AddComponentArgs, SearchRegistryComponentsArgs}; +use crate::tools::registry::{AddComponentArgs, ListRegistryComponentsArgs, SearchRegistryComponentsArgs}; use rmcp::handler::server::router::tool::ToolRouter; use rmcp::handler::server::wrapper::Parameters; use rmcp::model::*; @@ -188,6 +188,18 @@ impl ApxServer { self.handle_add_component(args).await } + #[tool( + name = "list_registry_components", + description = "List all available components in a registry. Use default shadcn registry if no registry specified.", + annotations(read_only_hint = true) + )] + async fn list_registry_components( + &self, + Parameters(args): Parameters, + ) -> Result { + self.handle_list_registry_components(args).await + } + // --- Docs tools --- #[tool( diff --git a/crates/mcp/src/tools/registry.rs b/crates/mcp/src/tools/registry.rs index 0fd0123d..7ba1082a 100644 --- a/crates/mcp/src/tools/registry.rs +++ b/crates/mcp/src/tools/registry.rs @@ -2,7 +2,7 @@ use crate::indexing::{rebuild_search_index, wait_for_index_ready}; use crate::server::ApxServer; use crate::tools::ToolResultExt; use crate::validation::validate_app_path; -use apx_core::components::{needs_registry_refresh, sync_registry_indexes}; +use apx_core::components::{get_all_registry_indexes, needs_registry_refresh, sync_registry_indexes}; use apx_core::search::ComponentIndex; use rmcp::model::*; use rmcp::schemars; @@ -33,6 +33,14 @@ pub struct AddComponentArgs { pub force: bool, } +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct ListRegistryComponentsArgs { + /// Absolute path to the project directory + pub app_path: String, + /// Registry name (e.g. "@animate-ui"). Omit or leave empty for default shadcn registry. + pub registry: Option, +} + impl ApxServer { pub async fn handle_search_registry_components( &self, @@ -145,4 +153,83 @@ impl ApxServer { ))])), } } + + pub async fn handle_list_registry_components( + &self, + args: ListRegistryComponentsArgs, + ) -> Result { + let path = validate_app_path(&args.app_path) + .map_err(|e| rmcp::ErrorData::invalid_params(e, None))?; + + // Check if registry indexes need refresh + if let Ok(metadata) = apx_core::common::read_project_metadata(&path) { + let cfg = apx_core::components::UiConfig::from_metadata(&metadata, &path); + if needs_registry_refresh(&cfg.registries) { + tracing::info!("Registry indexes stale, refreshing..."); + if let Ok(true) = sync_registry_indexes(&path, false).await { + let pool = self.ctx.dev_db.pool().clone(); + if let Err(e) = rebuild_search_index(pool.clone()).await { + tracing::warn!("Failed to rebuild search index after refresh: {}", e); + } + } + } + } + + let all_indexes = match get_all_registry_indexes() { + Ok(indexes) => indexes, + Err(e) => { + return Ok(CallToolResult::error(vec![Content::text(format!( + "Failed to load registry indexes: {e}" + ))])); + } + }; + + // Determine which registry key to look up + let registry_key = match &args.registry { + Some(name) if !name.is_empty() => name.trim_start_matches('@').to_string(), + _ => "ui".to_string(), + }; + + let items = match all_indexes.get(®istry_key) { + Some(items) => items, + None => { + let available: Vec<&String> = all_indexes.keys().collect(); + return Ok(CallToolResult::error(vec![Content::text(format!( + "Registry '{}' not found. Available registries: {:?}", + registry_key, available + ))])); + } + }; + + #[derive(serde::Serialize)] + struct ListResponse { + registry: String, + total: usize, + items: Vec, + } + + #[derive(serde::Serialize)] + struct ListItem { + name: String, + description: Option, + dependencies: usize, + registry_dependencies: usize, + } + + let response = ListResponse { + registry: registry_key, + total: items.len(), + items: items + .iter() + .map(|item| ListItem { + name: item.name.clone(), + description: item.description.clone(), + dependencies: item.dependencies.len(), + registry_dependencies: item.registry_dependencies.len(), + }) + .collect(), + }; + + Ok(CallToolResult::from_serializable(&response)) + } } From d09b7783c360ca3201fdba8bcb745590054edbb7 Mon Sep 17 00:00:00 2001 From: renardeinside Date: Wed, 18 Feb 2026 01:46:43 +0100 Subject: [PATCH 9/9] =?UTF-8?q?=F0=9F=8E=A8=20style:=20format=20import=20l?= =?UTF-8?q?ines=20with=20cargo=20fmt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/mcp/src/server.rs | 4 +++- crates/mcp/src/tools/registry.rs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/mcp/src/server.rs b/crates/mcp/src/server.rs index e613df8a..e7bfbd30 100644 --- a/crates/mcp/src/server.rs +++ b/crates/mcp/src/server.rs @@ -6,7 +6,9 @@ use crate::tools::databricks::DatabricksAppsLogsArgs; use crate::tools::devserver::LogsToolArgs; use crate::tools::docs::DocsArgs; use crate::tools::project::GetRouteInfoArgs; -use crate::tools::registry::{AddComponentArgs, ListRegistryComponentsArgs, SearchRegistryComponentsArgs}; +use crate::tools::registry::{ + AddComponentArgs, ListRegistryComponentsArgs, SearchRegistryComponentsArgs, +}; use rmcp::handler::server::router::tool::ToolRouter; use rmcp::handler::server::wrapper::Parameters; use rmcp::model::*; diff --git a/crates/mcp/src/tools/registry.rs b/crates/mcp/src/tools/registry.rs index 7ba1082a..f5d76dae 100644 --- a/crates/mcp/src/tools/registry.rs +++ b/crates/mcp/src/tools/registry.rs @@ -2,7 +2,9 @@ use crate::indexing::{rebuild_search_index, wait_for_index_ready}; use crate::server::ApxServer; use crate::tools::ToolResultExt; use crate::validation::validate_app_path; -use apx_core::components::{get_all_registry_indexes, needs_registry_refresh, sync_registry_indexes}; +use apx_core::components::{ + get_all_registry_indexes, needs_registry_refresh, sync_registry_indexes, +}; use apx_core::search::ComponentIndex; use rmcp::model::*; use rmcp::schemars;