diff --git a/hyperdrive/packages/spider/spider/Cargo.toml b/hyperdrive/packages/spider/spider/Cargo.toml index c76145793..d97888cf2 100644 --- a/hyperdrive/packages/spider/spider/Cargo.toml +++ b/hyperdrive/packages/spider/spider/Cargo.toml @@ -15,16 +15,16 @@ version = "0.4" [dependencies.hyperprocess_macro] git = "https://github.com/hyperware-ai/hyperprocess-macro" -rev = "ed99c19" +rev = "66884c0" [dependencies.hyperware-anthropic-sdk] git = "https://github.com/hyperware-ai/hyperware-anthropic-sdk" -rev = "363630c" +rev = "c0cbd5e" [dependencies.hyperware_process_lib] features = ["hyperapp"] git = "https://github.com/hyperware-ai/process_lib" -rev = "753dac3" +rev = "4beff93" [dependencies.serde] features = ["derive"] diff --git a/hyperdrive/packages/spider/spider/src/lib.rs b/hyperdrive/packages/spider/spider/src/lib.rs index 8ed1d71f0..27216616f 100644 --- a/hyperdrive/packages/spider/spider/src/lib.rs +++ b/hyperdrive/packages/spider/spider/src/lib.rs @@ -17,7 +17,7 @@ use hyperware_process_lib::{ our, println, Address, LazyLoadBlob, ProcessId, }; #[cfg(not(feature = "simulation-mode"))] -use spider_caller_utils::anthropic_api_key_manager::request_api_key_remote_rpc; +use spider_dev_caller_utils::anthropic_api_key_manager::request_api_key_remote_rpc; mod provider; use provider::create_llm_provider; @@ -26,15 +26,16 @@ mod types; use types::{ AddMcpServerRequest, ApiKey, ApiKeyInfo, ChatClient, ChatRequest, ChatResponse, ConfigResponse, ConnectMcpServerRequest, Conversation, ConversationMetadata, CreateSpiderKeyRequest, - DisconnectMcpServerRequest, GetConfigRequest, GetConversationRequest, HypergridConnection, - HypergridMessage, HypergridMessageType, JsonRpcNotification, JsonRpcRequest, - ListApiKeysRequest, ListConversationsRequest, ListMcpServersRequest, ListSpiderKeysRequest, - McpCapabilities, McpClientInfo, McpInitializeParams, McpRequestType, McpServer, - McpServerDetails, McpToolCallParams, McpToolInfo, Message, OAuthExchangeRequest, - OAuthRefreshRequest, OAuthTokenResponse, PendingMcpRequest, ProcessRequest, ProcessResponse, - RemoveApiKeyRequest, RemoveMcpServerRequest, RevokeSpiderKeyRequest, SetApiKeyRequest, - SpiderApiKey, SpiderState, Tool, ToolCall, ToolExecutionResult, ToolResult, TrialNotification, - UpdateConfigRequest, WsClientMessage, WsConnection, WsServerMessage, + DisconnectMcpServerRequest, ErrorResponse, GetConfigRequest, GetConversationRequest, + HypergridConnection, HypergridMessage, HypergridMessageType, JsonRpcNotification, + JsonRpcRequest, ListApiKeysRequest, ListConversationsRequest, ListMcpServersRequest, + ListSpiderKeysRequest, McpCapabilities, McpClientInfo, McpInitializeParams, McpRequestType, + McpServer, McpServerDetails, McpToolCallParams, McpToolInfo, Message, OAuthCodeExchangeRequest, + OAuthExchangeRequest, OAuthRefreshRequest, OAuthRefreshTokenRequest, OAuthTokenResponse, + PendingMcpRequest, ProcessRequest, ProcessResponse, RemoveApiKeyRequest, + RemoveMcpServerRequest, RevokeSpiderKeyRequest, SetApiKeyRequest, SpiderApiKey, SpiderState, + Tool, ToolCall, ToolExecutionResult, ToolResponseContent, ToolResponseContentItem, ToolResult, + TrialNotification, UpdateConfigRequest, WsClientMessage, WsConnection, WsServerMessage, }; mod utils; @@ -44,7 +45,11 @@ use utils::{ }; mod tool_providers; -use tool_providers::{hypergrid::HypergridToolProvider, ToolProvider}; +use tool_providers::{ + build_container::{BuildContainerExt, BuildContainerToolProvider}, + hypergrid::{HypergridExt, HypergridToolProvider}, + ToolProvider, +}; const ICON: &str = include_str!("./icon"); @@ -78,7 +83,7 @@ const HYPERGRID: &str = "operator:hypergrid:ware.hypr"; } ], save_config = hyperware_process_lib::hyperapp::SaveOptions::OnDiff, - wit_world = "spider-dot-os-v0" + wit_world = "spider-sys-v0" )] impl SpiderState { #[init] @@ -86,31 +91,90 @@ impl SpiderState { add_to_homepage("Spider", Some(ICON), Some("/"), None); self.default_llm_provider = "anthropic".to_string(); - self.max_tokens = 4096; + self.max_tokens = 32_000; self.temperature = 1.0; + // Only set if empty (preserves existing value from deserialized state) self.next_channel_id = 1000; // Start channel IDs at 1000 let our_node = our().node.clone(); println!("Spider MCP client initialized on node: {}", our_node); - // Register Hypergrid tool provider if not already registered + // Register Build Container tool provider + let build_container_provider = BuildContainerToolProvider::new(); + + // Always register the provider (even if server exists) + self.tool_provider_registry + .register(Box::new(build_container_provider)); + + // Check if build container server exists + let has_build_container = self + .mcp_servers + .iter() + .any(|s| s.transport.transport_type == "build_container"); + + if !has_build_container { + // Create new build container server + let build_container_provider = BuildContainerToolProvider::new(); + let build_container_tools = build_container_provider.get_tools(self); + + let build_container_server = McpServer { + id: "build_container".to_string(), + name: "Build Container".to_string(), + transport: types::TransportConfig { + transport_type: "build_container".to_string(), + command: None, + args: None, + url: None, + hypergrid_token: None, + hypergrid_client_id: None, + hypergrid_node: None, + }, + tools: build_container_tools, + connected: true, // Always mark as connected + }; + + self.mcp_servers.push(build_container_server); + println!("Spider: Build Container MCP server initialized"); + } else { + // Server exists, refresh its tools from the provider + println!("Spider: Refreshing Build Container tools on startup"); + + // Get fresh tools from provider + let build_container_provider = BuildContainerToolProvider::new(); + let fresh_tools = build_container_provider.get_tools(self); + + // Update the existing server's tools + if let Some(server) = self + .mcp_servers + .iter_mut() + .find(|s| s.id == "build_container") + { + server.tools = fresh_tools; + println!( + "Spider: Build Container tools refreshed with {} tools", + server.tools.len() + ); + } + } + + // Register Hypergrid tool provider + let hypergrid_provider = HypergridToolProvider::new("hypergrid_default".to_string()); + + // Always register the provider (even if server exists) + self.tool_provider_registry + .register(Box::new(hypergrid_provider)); + + // Check if hypergrid server exists let has_hypergrid = self .mcp_servers .iter() .any(|s| s.transport.transport_type == "hypergrid"); - // Only create the hypergrid MCP server if none exists if !has_hypergrid { - // Register the Hypergrid tool provider + // Create new hypergrid server let hypergrid_provider = HypergridToolProvider::new("hypergrid_default".to_string()); - - // Get ALL tools from the provider (not filtered) let hypergrid_tools = hypergrid_provider.get_tools(self); - // Register the provider for later use - self.tool_provider_registry - .register(Box::new(hypergrid_provider)); - let hypergrid_server = McpServer { id: "hypergrid_default".to_string(), name: "Hypergrid".to_string(), @@ -118,9 +182,7 @@ impl SpiderState { transport_type: "hypergrid".to_string(), command: None, args: None, - url: Some( - "http://localhost:8080/operator:hypergrid:ware.hypr/shim/mcp".to_string(), - ), + url: Some(format!("http://localhost:8080/{HYPERGRID}/shim/mcp")), hypergrid_token: None, hypergrid_client_id: None, hypergrid_node: None, @@ -132,7 +194,24 @@ impl SpiderState { self.mcp_servers.push(hypergrid_server); println!("Spider: Hypergrid MCP server initialized (unconfigured)"); } else { - println!("Spider: Hypergrid MCP server already exists, skipping initialization"); + println!("Spider: Refreshing Hypergrid tools on startup"); + + // Get fresh tools from provider + let hypergrid_provider = HypergridToolProvider::new("hypergrid_default".to_string()); + let fresh_tools = hypergrid_provider.get_tools(self); + + // Update the existing server's tools + if let Some(server) = self + .mcp_servers + .iter_mut() + .find(|s| s.id == "hypergrid_default") + { + server.tools = fresh_tools; + println!( + "Spider: Hypergrid tools refreshed with {} tools", + server.tools.len() + ); + } // Restore hypergrid connections for configured servers for server in self.mcp_servers.iter() { @@ -233,7 +312,7 @@ impl SpiderState { println!("Auto-reconnecting to MCP server: {}", server_id); // Retry logic with exponential backoff - let max_retries = 3; + let max_retries = 10; let mut retry_delay_ms = 1000u64; // Start with 1 second let mut success = false; @@ -361,6 +440,9 @@ impl SpiderState { }, ); + // Clean up disconnected Build Container MCP connections + self.cleanup_disconnected_build_containers(); + // Send auth success response let response = WsServerMessage::AuthSuccess { message: "Authenticated successfully".to_string(), @@ -546,6 +628,10 @@ impl SpiderState { // Parse the message as JSON let message_str = String::from_utf8(message_bytes).unwrap_or_default(); + println!( + "Spider: Received WebSocket message on channel {}: {}", + channel_id, message_str + ); if let Ok(json_msg) = serde_json::from_str::(&message_str) { self.handle_mcp_message(channel_id, json_msg); } else { @@ -571,6 +657,10 @@ impl SpiderState { server.connected = false; println!("Spider: MCP server {} disconnected", server.name); } + + // Also remove any ws_mcp server that was created for this connection + let ws_mcp_server_id = format!("ws_mcp_{}", channel_id); + self.mcp_servers.retain(|s| s.id != ws_mcp_server_id); } // Clean up any pending requests for this connection @@ -1059,6 +1149,8 @@ impl SpiderState { default_llm_provider: self.default_llm_provider.clone(), max_tokens: self.max_tokens, temperature: self.temperature, + build_container_ws_uri: self.build_container_ws_uri.clone(), + build_container_api_key: self.build_container_api_key.clone(), }) } @@ -1081,6 +1173,46 @@ impl SpiderState { self.temperature = temp; } + // Track if build container settings changed + let mut build_container_changed = false; + + if let Some(uri) = request.build_container_ws_uri { + if self.build_container_ws_uri != uri { + self.build_container_ws_uri = uri; + build_container_changed = true; + } + } + + if let Some(key) = request.build_container_api_key { + if self.build_container_api_key != key { + self.build_container_api_key = key; + build_container_changed = true; + } + } + + // If build container settings changed, update the tools list + if build_container_changed { + // Try multiple tool names since the provider has tools with hyphens + let provider = self + .tool_provider_registry + .find_provider_for_tool("init-build-container", self) + .or_else(|| { + self.tool_provider_registry + .find_provider_for_tool("load-project", self) + }); + + if let Some(provider) = provider { + let updated_tools = provider.get_tools(self); + if let Some(server) = self + .mcp_servers + .iter_mut() + .find(|s| s.id == "build_container") + { + server.tools = updated_tools; + } + } + } + Ok("Configuration updated".to_string()) } @@ -1172,14 +1304,14 @@ impl SpiderState { let state = parts.get(1).unwrap_or(&"").to_string(); // Prepare the request body - let body = serde_json::json!({ - "code": code, - "state": state, - "grant_type": "authorization_code", - "client_id": "9d1c250a-e61b-44d9-88ed-5944d1962f5e", - "redirect_uri": "https://console.anthropic.com/oauth/code/callback", - "code_verifier": req.verifier - }); + let body = OAuthCodeExchangeRequest { + code, + state, + grant_type: "authorization_code".to_string(), + client_id: "9d1c250a-e61b-44d9-88ed-5944d1962f5e".to_string(), + redirect_uri: "https://console.anthropic.com/oauth/code/callback".to_string(), + code_verifier: req.verifier, + }; // Prepare headers let mut headers = std::collections::HashMap::new(); @@ -1189,7 +1321,9 @@ impl SpiderState { let url = url::Url::parse("https://console.anthropic.com/v1/oauth/token") .map_err(|e| format!("Invalid URL: {}", e))?; - let body_bytes = body.to_string().into_bytes(); + let body_bytes = serde_json::to_string(&body) + .map_err(|e| format!("Failed to serialize request: {}", e))? + .into_bytes(); let response = send_request_await_response(Method::POST, url, Some(headers), 30000, body_bytes) .await @@ -1225,11 +1359,11 @@ impl SpiderState { use hyperware_process_lib::http::Method; // Prepare the request body - let body = serde_json::json!({ - "grant_type": "refresh_token", - "refresh_token": req.refresh_token, - "client_id": "9d1c250a-e61b-44d9-88ed-5944d1962f5e" - }); + let body = OAuthRefreshTokenRequest { + grant_type: "refresh_token".to_string(), + refresh_token: req.refresh_token, + client_id: "9d1c250a-e61b-44d9-88ed-5944d1962f5e".to_string(), + }; // Prepare headers let mut headers = std::collections::HashMap::new(); @@ -1239,7 +1373,9 @@ impl SpiderState { let url = url::Url::parse("https://console.anthropic.com/v1/oauth/token") .map_err(|e| format!("Invalid URL: {}", e))?; - let body_bytes = body.to_string().into_bytes(); + let body_bytes = serde_json::to_string(&body) + .map_err(|e| format!("Failed to serialize request: {}", e))? + .into_bytes(); let response = send_request_await_response(Method::POST, url, Some(headers), 30000, body_bytes) .await @@ -1297,6 +1433,62 @@ impl SpiderState { .any(|k| k.key == key && k.permissions.contains(&permission.to_string())) } + fn cleanup_disconnected_build_containers(&mut self) { + // Find all ws_mcp_* servers that are disconnected + let disconnected_server_ids: Vec = self + .mcp_servers + .iter() + .filter(|s| { + // Only cleanup ws_mcp_* servers (Build Container connections) + s.id.starts_with("ws_mcp_") && !s.connected + }) + .map(|s| s.id.clone()) + .collect(); + + if !disconnected_server_ids.is_empty() { + println!( + "Spider: Cleaning up {} disconnected Build Container MCP connections", + disconnected_server_ids.len() + ); + + for server_id in disconnected_server_ids { + // Extract channel_id from server_id (format: "ws_mcp_{channel_id}") + if let Some(channel_str) = server_id.strip_prefix("ws_mcp_") { + if let Ok(old_channel_id) = channel_str.parse::() { + // Remove from ws_connections if it exists + if self.ws_connections.remove(&old_channel_id).is_some() { + println!( + "Spider: Removed ws_connection for channel {}", + old_channel_id + ); + } + + // Clean up any pending MCP requests for this server + let requests_to_remove: Vec = self + .pending_mcp_requests + .iter() + .filter(|(_, req)| req.server_id == server_id) + .map(|(id, _)| id.clone()) + .collect(); + + for req_id in requests_to_remove { + self.pending_mcp_requests.remove(&req_id); + self.tool_responses.remove(&req_id); + } + } + } + + // Remove the server from mcp_servers list + self.mcp_servers.retain(|s| s.id != server_id); + println!("Spider: Removed Build Container MCP server {}", server_id); + } + + println!("Spider: Build Container cleanup complete"); + } else { + println!("Spider: No disconnected Build Container MCP connections to clean up"); + } + } + // Streaming version of chat for WebSocket clients async fn process_chat_request_with_streaming( &mut self, @@ -1437,22 +1629,6 @@ impl SpiderState { } }; - // Collect available tools from connected MCP servers - let available_tools: Vec = if let Some(ref mcp_server_ids) = request.mcp_servers { - self.mcp_servers - .iter() - .filter(|s| s.connected && mcp_server_ids.contains(&s.id)) - .flat_map(|s| s.tools.clone()) - .collect() - } else { - // Use all connected servers if none specified - self.mcp_servers - .iter() - .filter(|s| s.connected) - .flat_map(|s| s.tools.clone()) - .collect() - }; - // Start the agentic loop - runs indefinitely until the agent stops making tool calls let mut working_messages = request.messages.clone(); let mut iteration_count = 0; @@ -1460,6 +1636,29 @@ impl SpiderState { let response = loop { iteration_count += 1; + // Collect available tools from connected MCP servers - refreshed each iteration + // This ensures newly available tools (e.g., after load-project) are immediately available + let available_tools: Vec = if let Some(ref mcp_server_ids) = request.mcp_servers { + self.mcp_servers + .iter() + .filter(|s| { + s.connected && ( + mcp_server_ids.contains(&s.id) || + // If build_container is selected, also include ws_mcp_* servers + (mcp_server_ids.contains(&"build_container".to_string()) && s.id.starts_with("ws_mcp_")) + ) + }) + .flat_map(|s| s.tools.clone()) + .collect() + } else { + // Use all connected servers if none specified + self.mcp_servers + .iter() + .filter(|s| s.connected) + .flat_map(|s| s.tools.clone()) + .collect() + }; + // Check for cancellation if let Some(ch_id) = channel_id { if let Some(cancel_flag) = self.active_chat_cancellation.get(&ch_id) { @@ -1529,12 +1728,20 @@ impl SpiderState { }; // Check if the response contains tool calls + println!("[DEBUG] LLM response received:"); + println!("[DEBUG] - content: {}", llm_response.content); + println!( + "[DEBUG] - has tool_calls_json: {}", + llm_response.tool_calls_json.is_some() + ); + if let Some(ref tool_calls_json) = llm_response.tool_calls_json { // The agent wants to use tools - execute them println!( "Spider: Iteration {} - Agent requested tool calls", iteration_count ); + println!("[DEBUG] Tool calls JSON: {}", tool_calls_json); // Send streaming update for tool calls if let Some(ch_id) = channel_id { @@ -1597,27 +1804,91 @@ impl SpiderState { // Continue the loop - the agent will decide what to do next continue; } else { - // No tool calls - the agent has decided to provide a final response - // Break the loop and return this response + // No tool calls - check if the agent is actually done println!( - "Spider: Iteration {} - Agent provided final response (no tool calls)", + "Spider: Iteration {} - No tool calls, checking if agent is done", iteration_count ); - // Send the final assistant message to the client - if let Some(ch_id) = channel_id { - let msg_update = WsServerMessage::Message { - message: llm_response.clone(), + // Check if response is just a "." - if so, continue immediately + let completion_status = if llm_response.content.trim() == "." { + println!("[DEBUG] Response is just '.', treating as continue"); + "continue".to_string() + } else if llm_provider == "anthropic" { + // Use the same API key that was used for the main request + use crate::provider::AnthropicProvider; + + // The api_key variable already contains the correct key for this conversation + let is_oauth = is_oauth_token(&api_key); + let anthropic_provider = AnthropicProvider::new(api_key.clone(), is_oauth); + + anthropic_provider + .check_tool_loop_completion(&llm_response.content) + .await + } else { + // For non-Anthropic providers, assume done + "done".to_string() + }; + + if completion_status == "continue" { + println!( + "[DEBUG] Agent indicated it wants to continue, sending continue message" + ); + + // Add the assistant's response to messages + working_messages.push(llm_response.clone()); + + // Send the assistant message to the client (but skip if it's just ".") + if let Some(ch_id) = channel_id { + if llm_response.content.trim() != "." { + let msg_update = WsServerMessage::Message { + message: llm_response.clone(), + }; + let json = serde_json::to_string(&msg_update).unwrap(); + send_ws_push( + ch_id, + WsMessageType::Text, + LazyLoadBlob::new(Some("application/json"), json), + ); + } + } + + // Add a continue message and loop + let continue_message = Message { + role: "user".to_string(), + content: "continue".to_string(), + tool_calls_json: None, + tool_results_json: None, + timestamp: Utc::now().timestamp() as u64, }; - let json = serde_json::to_string(&msg_update).unwrap(); - send_ws_push( - ch_id, - WsMessageType::Text, - LazyLoadBlob::new(Some("application/json"), json), + working_messages.push(continue_message); + + // Continue the loop + continue; + } else { + // Agent is done (or error/failed to parse) + println!( + "Spider: Iteration {} - Agent provided final response (completion check: {})", + iteration_count, completion_status ); - } - break llm_response; + // Send the final assistant message to the client (but skip if it's just ".") + if let Some(ch_id) = channel_id { + if llm_response.content.trim() != "." { + let msg_update = WsServerMessage::Message { + message: llm_response.clone(), + }; + let json = serde_json::to_string(&msg_update).unwrap(); + send_ws_push( + ch_id, + WsMessageType::Text, + LazyLoadBlob::new(Some("application/json"), json), + ); + } + } + + break llm_response; + } } }; @@ -1688,6 +1959,11 @@ impl SpiderState { } fn handle_mcp_message(&mut self, channel_id: u32, message: Value) { + println!( + "Spider: handle_mcp_message received on channel {}: {:?}", + channel_id, message + ); + // Find the connection for this channel let conn = match self.ws_connections.get(&channel_id) { Some(c) => c.clone(), @@ -1702,7 +1978,37 @@ impl SpiderState { // Check if this is a response to a pending request if let Some(id) = message.get("id").and_then(|v| v.as_str()) { + println!("Spider: Message has id: {}", id); + + // Check if this is a spider/* method response (not in pending_mcp_requests) + // These are direct responses to spider/* methods like load-project, auth, etc. + if id.starts_with("load-project-") + || id.starts_with("start-package-") + || id.starts_with("persist") + || id.starts_with("auth_") + { + println!("Spider: Handling spider/* method response with id: {}", id); + // Store the response for the waiting execute_*_impl method + let result = if let Some(result_value) = message.get("result") { + result_value.clone() + } else if let Some(error) = message.get("error") { + serde_json::to_value(ErrorResponse { + error: error.clone(), + }) + .unwrap_or_else(|_| Value::Null) + } else { + serde_json::to_value(ErrorResponse { + error: Value::String("Invalid response format".to_string()), + }) + .unwrap_or_else(|_| Value::Null) + }; + self.tool_responses.insert(id.to_string(), result); + println!("Spider: Stored response for id {} in tool_responses", id); + return; + } + if let Some(pending) = self.pending_mcp_requests.remove(id) { + println!("Spider: Found pending request for id: {}", id); match pending.request_type { McpRequestType::Initialize => { self.handle_initialize_response(channel_id, &conn, &message); @@ -1714,6 +2020,8 @@ impl SpiderState { self.handle_tool_call_response(&pending, &message); } } + } else { + println!("Spider: No pending request found for id: {}", id); } } @@ -1845,10 +2153,88 @@ impl SpiderState { ws_conn.tools = tools.clone(); } - // Update server with tools and mark as connected - if let Some(server) = self.mcp_servers.iter_mut().find(|s| s.id == conn.server_id) { - server.tools = tools; - server.connected = true; + // For build container connections, we need special handling + if conn.server_id == "build_container_self_hosted" + || conn.server_id.starts_with("build_container_") + { + // Create or update a separate ws-mcp server entry for the remote tools + let ws_mcp_server_id = format!("ws_mcp_{}", channel_id); + + // Check if this ws-mcp server already exists + if let Some(server) = self + .mcp_servers + .iter_mut() + .find(|s| s.id == ws_mcp_server_id) + { + server.tools = tools; + server.connected = true; + println!( + "Spider: Updated ws-mcp server {} with {} tools", + ws_mcp_server_id, + server.tools.len() + ); + } else { + // Create a new MCP server entry for ws-mcp tools + let ws_mcp_server = McpServer { + id: ws_mcp_server_id.clone(), + name: "Build Container MCP".to_string(), + transport: crate::types::TransportConfig { + transport_type: "websocket".to_string(), + command: None, + args: None, + url: Some(self.build_container_ws_uri.clone()), + hypergrid_token: None, + hypergrid_client_id: None, + hypergrid_node: None, + }, + tools, + connected: true, + }; + self.mcp_servers.push(ws_mcp_server); + println!( + "Spider: Created new ws-mcp server {} with {} tools", + ws_mcp_server_id, tool_count + ); + } + + // Make sure the build_container server retains its native tools + // by refreshing them from the tool provider + if let Some(provider) = self + .tool_provider_registry + .find_provider_for_tool("load-project", self) + { + let native_tools = provider.get_tools(self); + if let Some(server) = self + .mcp_servers + .iter_mut() + .find(|s| s.id == "build_container") + { + server.tools = native_tools; + server.connected = true; + println!( + "Spider: Refreshed build_container server with {} native tools", + server.tools.len() + ); + } + } + } else { + // For non-build-container connections, update normally + if let Some(server) = + self.mcp_servers.iter_mut().find(|s| s.id == conn.server_id) + { + server.tools = tools; + server.connected = true; + println!( + "Spider: Updated MCP server {} with {} tools", + conn.server_id, + server.tools.len() + ); + } else { + println!( + "Spider: Warning - could not find MCP server with id {}", + conn.server_id + ); + } } } } else if let Some(error) = message.get("error") { @@ -1869,13 +2255,15 @@ impl SpiderState { let result = if let Some(result_value) = message.get("result") { result_value.clone() } else if let Some(error) = message.get("error") { - serde_json::json!({ - "error": error + serde_json::to_value(ErrorResponse { + error: error.clone(), }) + .unwrap_or_else(|_| Value::Null) } else { - serde_json::json!({ - "error": "Invalid MCP response format" + serde_json::to_value(ErrorResponse { + error: Value::String("Invalid MCP response format".to_string()), }) + .unwrap_or_else(|_| Value::Null) }; self.tool_responses @@ -1889,6 +2277,113 @@ impl SpiderState { parameters: &Value, conversation_id: Option, ) -> Result { + println!( + "[DEBUG] execute_mcp_tool called with server_id: {}, tool_name: {}", + server_id, tool_name + ); + println!("[DEBUG] parameters: {}", parameters); + println!( + "Spider: Available MCP servers: {:?}", + self.mcp_servers + .iter() + .map(|s| (&s.id, s.connected)) + .collect::>() + ); + + // Special handling for ws_mcp servers (build container WebSocket connections) + if server_id.starts_with("ws_mcp_") { + // Extract channel_id from server_id (format: "ws_mcp_{channel_id}") + let channel_id = server_id + .strip_prefix("ws_mcp_") + .and_then(|s| s.parse::().ok()) + .ok_or_else(|| format!("Invalid ws_mcp server id: {}", server_id))?; + + println!( + "Spider: Looking for WebSocket connection with channel_id {} for server {}", + channel_id, server_id + ); + println!( + "Spider: Available ws_connections: {:?}", + self.ws_connections.keys().collect::>() + ); + + // Verify the connection exists + if !self.ws_connections.contains_key(&channel_id) { + return Err(format!( + "No WebSocket connection found for server {}", + server_id + )); + } + + // Execute via WebSocket using MCP protocol + let request_id = format!("tool_{}_{}", channel_id, Uuid::new_v4()); + let tool_request = JsonRpcRequest { + jsonrpc: "2.0".to_string(), + method: "tools/call".to_string(), + params: Some( + serde_json::to_value(McpToolCallParams { + name: tool_name.to_string(), + arguments: parameters.clone(), + }) + .unwrap(), + ), + id: request_id.clone(), + }; + + // Store pending request + self.pending_mcp_requests.insert( + request_id.clone(), + PendingMcpRequest { + request_id: request_id.clone(), + conversation_id, + server_id: server_id.to_string(), + request_type: McpRequestType::ToolCall { + tool_name: tool_name.to_string(), + }, + }, + ); + + // Send the request + let request_json = serde_json::to_string(&tool_request).unwrap(); + let blob = LazyLoadBlob::new(Some("application/json"), request_json.into_bytes()); + send_ws_client_push(channel_id, WsMessageType::Text, blob); + + // Wait for response + let start = std::time::Instant::now(); + let timeout = std::time::Duration::from_secs(30); + + loop { + if start.elapsed() > timeout { + self.pending_mcp_requests.remove(&request_id); + return Err(format!("Tool call timed out: {}", tool_name)); + } + + if let Some(result) = self.tool_responses.remove(&request_id) { + // Parse the MCP result format + if let Some(content) = result.get("content") { + return Ok(serde_json::to_value(ToolExecutionResult { + result: content.clone(), + success: true, + }) + .unwrap()); + } else if let Some(error) = result.get("error") { + return Err(format!("Tool execution failed: {}", error)); + } else { + // Fallback: return the raw result wrapped in ToolExecutionResult + return Ok(serde_json::to_value(ToolExecutionResult { + result: result, + success: true, + }) + .unwrap()); + } + } + + // Sleep briefly before checking again + let _ = hyperware_process_lib::hyperapp::sleep(100).await; + } + } + + // Regular MCP server handling let server = self .mcp_servers .iter() @@ -1905,179 +2400,221 @@ impl SpiderState { // Execute the tool based on transport type match server.transport.transport_type.as_str() { "hypergrid" => { - // Map old tool names to new ones for backward compatibility - let normalized_tool_name = match tool_name { - "authorize" => "hypergrid_authorize", - "search-registry" => "hypergrid_search", - "call-provider" => "hypergrid_call", - name => name, - }; - - // Handle the different hypergrid tools - match normalized_tool_name { - "hypergrid_authorize" => { - println!( - "Spider: hypergrid_authorize called for server_id: {}", - server_id - ); - println!(" Parameters received: {:?}", parameters); - - // Update hypergrid credentials - let new_url = parameters - .get("url") - .and_then(|v| v.as_str()) - .ok_or_else(|| "Missing url parameter".to_string())?; - let new_token = parameters - .get("token") - .and_then(|v| v.as_str()) - .ok_or_else(|| "Missing token parameter".to_string())?; - let new_client_id = parameters - .get("client_id") - .and_then(|v| v.as_str()) - .ok_or_else(|| "Missing client_id parameter".to_string())?; - let new_node = parameters - .get("node") - .and_then(|v| v.as_str()) - .ok_or_else(|| "Missing node parameter".to_string())?; - - println!("Spider: Authorizing hypergrid with:"); - println!(" - URL: {}", new_url); - println!(" - Token: {}...", &new_token[..new_token.len().min(20)]); - println!(" - Client ID: {}", new_client_id); - println!(" - Node: {}", new_node); - - // Test new connection - println!("Spider: Testing hypergrid connection..."); - self.test_hypergrid_connection(new_url, new_token, new_client_id) - .await?; - println!("Spider: Connection test successful!"); - - // Create or update the hypergrid connection - let hypergrid_conn = HypergridConnection { - server_id: server_id.to_string(), - url: new_url.to_string(), - token: new_token.to_string(), - client_id: new_client_id.to_string(), - node: new_node.to_string(), - last_retry: Instant::now(), - retry_count: 0, - connected: true, - }; - - self.hypergrid_connections - .insert(server_id.to_string(), hypergrid_conn); - println!("Spider: Stored hypergrid connection in memory"); + // Use the hypergrid tool provider + if let Some(provider) = self + .tool_provider_registry + .find_provider_for_tool(tool_name, self) + { + let command = provider.prepare_execution(tool_name, parameters, self)?; + self.execute_tool_command(command, conversation_id).await + } else { + // Map old tool names to new ones for backward compatibility + let normalized_tool_name = match tool_name { + "authorize" => "hypergrid_authorize", + "search-registry" => "hypergrid_search", + "call-provider" => "hypergrid_call", + name => name, + }; - // Update transport config - if let Some(server) = - self.mcp_servers.iter_mut().find(|s| s.id == server_id) - { - println!("Spider: Updating server '{}' transport config", server.name); - server.transport.url = Some(new_url.to_string()); - server.transport.hypergrid_token = Some(new_token.to_string()); - server.transport.hypergrid_client_id = Some(new_client_id.to_string()); - server.transport.hypergrid_node = Some(new_node.to_string()); - println!("Spider: Server transport config updated successfully"); - println!("Spider: State should auto-save due to SaveOptions::OnDiff"); - } else { - println!( - "Spider: WARNING - Could not find server with id: {}", - server_id - ); - } + // Try with normalized name + if let Some(provider) = self + .tool_provider_registry + .find_provider_for_tool(normalized_tool_name, self) + { + let command = + provider.prepare_execution(normalized_tool_name, parameters, self)?; + self.execute_tool_command(command, conversation_id).await + } else { + // Fall back to old implementation for backward compatibility + match normalized_tool_name { + "hypergrid_authorize" => { + println!( + "Spider: hypergrid_authorize called for server_id: {}", + server_id + ); + println!(" Parameters received: {:?}", parameters); + + // Update hypergrid credentials + let new_url = parameters + .get("url") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing url parameter".to_string())?; + let new_token = parameters + .get("token") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing token parameter".to_string())?; + let new_client_id = parameters + .get("client_id") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing client_id parameter".to_string())?; + let new_node = parameters + .get("node") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing node parameter".to_string())?; + + println!("Spider: Authorizing hypergrid with:"); + println!(" - URL: {}", new_url); + println!(" - Token: {}...", &new_token[..new_token.len().min(20)]); + println!(" - Client ID: {}", new_client_id); + println!(" - Node: {}", new_node); + + // Test new connection + println!("Spider: Testing hypergrid connection..."); + self.test_hypergrid_connection(new_url, new_token, new_client_id) + .await?; + println!("Spider: Connection test successful!"); + + // Create or update the hypergrid connection + let hypergrid_conn = HypergridConnection { + server_id: server_id.to_string(), + url: new_url.to_string(), + token: new_token.to_string(), + client_id: new_client_id.to_string(), + node: new_node.to_string(), + last_retry: Instant::now(), + retry_count: 0, + connected: true, + }; + + self.hypergrid_connections + .insert(server_id.to_string(), hypergrid_conn); + println!("Spider: Stored hypergrid connection in memory"); + + // Update transport config + if let Some(server) = + self.mcp_servers.iter_mut().find(|s| s.id == server_id) + { + println!( + "Spider: Updating server '{}' transport config", + server.name + ); + server.transport.url = Some(new_url.to_string()); + server.transport.hypergrid_token = Some(new_token.to_string()); + server.transport.hypergrid_client_id = + Some(new_client_id.to_string()); + server.transport.hypergrid_node = Some(new_node.to_string()); + println!( + "Spider: Server transport config updated successfully" + ); + println!( + "Spider: State should auto-save due to SaveOptions::OnDiff" + ); + } else { + println!( + "Spider: WARNING - Could not find server with id: {}", + server_id + ); + } - Ok(serde_json::json!({ - "content": [{ - "type": "text", - "text": format!("✅ Successfully authorized! Hypergrid is now configured with:\n- Node: {}\n- Client ID: {}\n- URL: {}", new_node, new_client_id, new_url) - }] - })) - } - "hypergrid_search" => { - // Check if configured - let hypergrid_conn = self.hypergrid_connections.get(server_id) + Ok(serde_json::to_value(ToolResponseContent { + content: vec![ToolResponseContentItem { + content_type: "text".to_string(), + text: format!("✅ Successfully authorized! Hypergrid is now configured with:\n- Node: {}\n- Client ID: {}\n- URL: {}", new_node, new_client_id, new_url), + }], + }) + .map_err(|e| format!("Failed to serialize response: {}", e))?) + } + "hypergrid_search" => { + // Check if configured + let hypergrid_conn = self.hypergrid_connections.get(server_id) .ok_or_else(|| "Hypergrid not configured. Please use hypergrid_authorize first with your credentials.".to_string())?; - let query = parameters - .get("query") - .and_then(|v| v.as_str()) - .ok_or_else(|| "Missing query parameter".to_string())?; - - let response = self - .call_hypergrid_api( - &hypergrid_conn.url, - &hypergrid_conn.token, - &hypergrid_conn.client_id, - &HypergridMessage { - request: HypergridMessageType::SearchRegistry( - query.to_string(), - ), - }, - ) - .await?; - - Ok(serde_json::json!({ - "content": [{ - "type": "text", - "text": response - }] - })) - } - "hypergrid_call" => { - // Check if configured - let hypergrid_conn = self.hypergrid_connections.get(server_id) + let query = parameters + .get("query") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing query parameter".to_string())?; + + let response = self + .call_hypergrid_api( + &hypergrid_conn.url, + &hypergrid_conn.token, + &hypergrid_conn.client_id, + &HypergridMessage { + request: HypergridMessageType::SearchRegistry( + query.to_string(), + ), + }, + ) + .await?; + + Ok(serde_json::to_value(ToolResponseContent { + content: vec![ToolResponseContentItem { + content_type: "text".to_string(), + text: response, + }], + }) + .map_err(|e| format!("Failed to serialize response: {}", e))?) + } + "hypergrid_call" => { + // Check if configured + let hypergrid_conn = self.hypergrid_connections.get(server_id) .ok_or_else(|| "Hypergrid not configured. Please use hypergrid_authorize first with your credentials.".to_string())?; - let provider_id = parameters - .get("providerId") - .and_then(|v| v.as_str()) - .ok_or_else(|| "Missing providerId parameter".to_string())?; - let provider_name = parameters - .get("providerName") - .and_then(|v| v.as_str()) - .ok_or_else(|| "Missing providerName parameter".to_string())?; - // Support both "callArgs" (old) and "arguments" (new) parameter names - let call_args = parameters - .get("arguments") - .or_else(|| parameters.get("callArgs")) - .and_then(|v| v.as_array()) - .ok_or_else(|| "Missing arguments parameter".to_string())?; - - // Convert callArgs to Vec<(String, String)> - let mut arguments = Vec::new(); - for arg in call_args { - if let Some(pair) = arg.as_array() { - if pair.len() == 2 { - if let (Some(key), Some(val)) = - (pair[0].as_str(), pair[1].as_str()) - { - arguments.push((key.to_string(), val.to_string())); + let provider_id = parameters + .get("providerId") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing providerId parameter".to_string())?; + let provider_name = parameters + .get("providerName") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing providerName parameter".to_string())?; + // Support both "callArgs" (old) and "arguments" (new) parameter names + let call_args = parameters + .get("arguments") + .or_else(|| parameters.get("callArgs")) + .and_then(|v| v.as_array()) + .ok_or_else(|| "Missing arguments parameter".to_string())?; + + // Convert callArgs to Vec<(String, String)> + let mut arguments = Vec::new(); + for arg in call_args { + if let Some(pair) = arg.as_array() { + if pair.len() == 2 { + if let (Some(key), Some(val)) = + (pair[0].as_str(), pair[1].as_str()) + { + arguments.push((key.to_string(), val.to_string())); + } + } } } + + let response = self + .call_hypergrid_api( + &hypergrid_conn.url, + &hypergrid_conn.token, + &hypergrid_conn.client_id, + &HypergridMessage { + request: HypergridMessageType::CallProvider { + provider_id: provider_id.to_string(), + provider_name: provider_name.to_string(), + arguments, + }, + }, + ) + .await?; + + Ok(serde_json::to_value(ToolResponseContent { + content: vec![ToolResponseContentItem { + content_type: "text".to_string(), + text: response, + }], + }) + .map_err(|e| format!("Failed to serialize response: {}", e))?) } + _ => Err(format!("Unknown hypergrid tool: {}", tool_name)), } - - let response = self - .call_hypergrid_api( - &hypergrid_conn.url, - &hypergrid_conn.token, - &hypergrid_conn.client_id, - &HypergridMessage { - request: HypergridMessageType::CallProvider { - provider_id: provider_id.to_string(), - provider_name: provider_name.to_string(), - arguments, - }, - }, - ) - .await?; - - Ok(serde_json::json!({ - "content": [{ - "type": "text", - "text": response - }] - })) } - _ => Err(format!("Unknown hypergrid tool: {}", tool_name)), + } + } + "build_container" => { + // Native build container tools are handled by the tool provider + if let Some(provider) = self + .tool_provider_registry + .find_provider_for_tool(tool_name, self) + { + let command = provider.prepare_execution(tool_name, parameters, self)?; + self.execute_tool_command(command, conversation_id).await + } else { + Err(format!("Unknown build container tool: {}", tool_name)) } } "stdio" | "websocket" => { @@ -2140,14 +2677,20 @@ impl SpiderState { if let Some(response) = self.tool_responses.remove(&request_id) { self.pending_mcp_requests.remove(&request_id); + println!("[DEBUG] Tool response received:"); + println!("[DEBUG] - response: {}", response); + // Parse the MCP result if let Some(content) = response.get("content") { - return Ok(serde_json::to_value(ToolExecutionResult { + let result = serde_json::to_value(ToolExecutionResult { result: content.clone(), success: true, }) - .unwrap()); + .unwrap(); + println!("[DEBUG] - returning content result: {}", result); + return Ok(result); } else { + println!("[DEBUG] - returning full response: {}", response); return Ok(response); } } @@ -2170,7 +2713,7 @@ impl SpiderState { // Execute via HTTP // This is a placeholder - actual implementation would make HTTP requests Ok(serde_json::to_value(ToolExecutionResult { - result: serde_json::json!(format!( + result: Value::String(format!( "HTTP execution of {} with params: {}", tool_name, parameters )), @@ -2190,12 +2733,20 @@ impl SpiderState { tool_calls_json: &str, conversation_id: Option, ) -> Result, String> { + println!("[DEBUG] process_tool_calls called"); + println!("[DEBUG] tool_calls_json: {}", tool_calls_json); + let tool_calls: Vec = serde_json::from_str(tool_calls_json) .map_err(|e| format!("Failed to parse tool calls: {}", e))?; + println!("[DEBUG] Parsed {} tool calls", tool_calls.len()); let mut results = Vec::new(); for tool_call in tool_calls { + println!("[DEBUG] Processing tool call:"); + println!("[DEBUG] - id: {}", tool_call.id); + println!("[DEBUG] - tool_name: {}", tool_call.tool_name); + println!("[DEBUG] - parameters: {}", tool_call.parameters); // Find which MCP server has this tool and get its ID let server_id = self .mcp_servers @@ -2204,6 +2755,7 @@ impl SpiderState { .map(|s| s.id.clone()); let result = if let Some(server_id) = server_id { + println!("[DEBUG] Found tool in server: {}", server_id); let params: Value = serde_json::from_str(&tool_call.parameters) .unwrap_or(Value::Object(serde_json::Map::new())); match self @@ -2215,14 +2767,24 @@ impl SpiderState { ) .await { - Ok(res) => res.to_string(), - Err(e) => format!(r#"{{"error":"{}"}}"#, e), + Ok(res) => { + let result_str = res.to_string(); + println!("[DEBUG] Tool execution successful: {}", result_str); + result_str + } + Err(e) => { + let error_str = format!(r#"{{"error":"{}"}}"#, e); + println!("[DEBUG] Tool execution error: {}", error_str); + error_str + } } } else { - format!( + let error_str = format!( r#"{{"error":"Tool {} not found in any connected MCP server"}}"#, tool_call.tool_name - ) + ); + println!("[DEBUG] {}", error_str); + error_str }; results.push(ToolResult { @@ -2231,6 +2793,7 @@ impl SpiderState { }); } + println!("[DEBUG] Returning {} tool results", results.len()); Ok(results) } @@ -2333,4 +2896,75 @@ impl SpiderState { Ok(response_text) } + + // Execute tool commands returned by tool providers + async fn execute_tool_command( + &mut self, + command: tool_providers::ToolExecutionCommand, + _conversation_id: Option, + ) -> Result { + use tool_providers::ToolExecutionCommand; + + match command { + ToolExecutionCommand::InitBuildContainer { metadata } => { + self.execute_init_build_container_impl(metadata).await + } + ToolExecutionCommand::LoadProject { + project_uuid, + name, + initial_zip, + channel_id, + } => { + self.execute_load_project_impl(project_uuid, name, initial_zip, channel_id) + .await + } + ToolExecutionCommand::StartPackage { + channel_id, + package_dir, + } => { + self.execute_start_package_impl(channel_id, package_dir) + .await + } + ToolExecutionCommand::Persist { + channel_id, + directories, + } => self.execute_persist_impl(channel_id, directories).await, + ToolExecutionCommand::GetProjects => { + // Return the project name to UUID mapping as JSON + Ok(serde_json::to_value(&self.project_name_to_uuids) + .map_err(|e| format!("Failed to serialize project mapping: {}", e))?) + } + ToolExecutionCommand::DoneBuildContainer { + metadata, + channel_id, + } => { + self.execute_done_build_container_impl(metadata, channel_id) + .await + } + ToolExecutionCommand::HypergridAuthorize { + server_id, + url, + token, + client_id, + node, + name, + } => { + self.execute_hypergrid_authorize_impl(server_id, url, token, client_id, node, name) + .await + } + ToolExecutionCommand::HypergridSearch { server_id, query } => { + self.execute_hypergrid_search_impl(server_id, query).await + } + ToolExecutionCommand::HypergridCall { + server_id, + provider_id, + provider_name, + call_args, + } => { + self.execute_hypergrid_call_impl(server_id, provider_id, provider_name, call_args) + .await + } + ToolExecutionCommand::DirectResult(result) => result, + } + } } diff --git a/hyperdrive/packages/spider/spider/src/provider/anthropic.rs b/hyperdrive/packages/spider/spider/src/provider/anthropic.rs index adab2dbde..3ecdfc3bf 100644 --- a/hyperdrive/packages/spider/spider/src/provider/anthropic.rs +++ b/hyperdrive/packages/spider/spider/src/provider/anthropic.rs @@ -5,22 +5,86 @@ use chrono::Utc; use serde_json::Value; use hyperware_anthropic_sdk::{ - AnthropicClient, Content, CreateMessageRequest, Message as SdkMessage, ResponseContentBlock, - Role, Tool as SdkTool, ToolChoice, + AnthropicClient, CacheControl, Content, ContentBlock, CreateMessageRequest, + Message as SdkMessage, ResponseContentBlock, Role, SystemPromptBlock, Tool as SdkTool, + ToolChoice, }; +use hyperware_process_lib::println; + use crate::provider::LlmProvider; use crate::types::{Message, Tool, ToolCall, ToolResult}; -pub(crate) struct AnthropicProvider { +pub struct AnthropicProvider { api_key: String, is_oauth: bool, } impl AnthropicProvider { - pub(crate) fn new(api_key: String, is_oauth: bool) -> Self { + pub fn new(api_key: String, is_oauth: bool) -> Self { Self { api_key, is_oauth } } + + /// Check if the tool loop is actually done by asking Sonnet 4 + pub async fn check_tool_loop_completion(&self, agent_message: &str) -> String { + // Create a specific prompt to check if the agent is done + let prompt = format!( + r#"The following is a response from an LLM agent to serve a user request, possibly after a tool loop. Respond with `done` (and nothing else) if this message seems to imply the agent is finished replying; `continue` (and nothing else) if it seems to imply the agent is not yet done with serving the request; error and one-sentence explanation else. If the agent is asking for input from the user, you must reply `done`. +""" +{} +""""#, + agent_message + ); + + // Create a message to send to Sonnet 4 + let check_message = Message { + role: "user".to_string(), + content: prompt, + tool_calls_json: None, + tool_results_json: None, + timestamp: Utc::now().timestamp() as u64, + }; + + // Use Sonnet 4 specifically for this check + match self + .complete_with_retry( + &[check_message], + &[], + Some("claude-sonnet-4-20250514"), + 100, + 0.0, + ) + .await + { + Ok(response) => { + let response_text = response.content.trim().to_lowercase(); + + // Parse the response + if response_text == "done" { + "done".to_string() + } else if response_text == "continue" { + "continue".to_string() + } else if response_text.starts_with("error") { + println!( + "[DEBUG] Tool loop completion check error: {}", + response_text + ); + "done".to_string() // Behave like done on error + } else { + // Failed to parse - behave like done but log error + println!( + "[DEBUG] Failed to parse tool loop completion check response: {}", + response_text + ); + "done".to_string() + } + } + Err(e) => { + println!("[DEBUG] Error checking tool loop completion: {}", e); + "done".to_string() // Default to done on error + } + } + } } impl LlmProvider for AnthropicProvider { @@ -48,9 +112,11 @@ impl AnthropicProvider { // Transform MCP JSON Schema to Anthropic-compatible format fn transform_mcp_to_anthropic_schema(&self, mcp_schema: &Value) -> Value { // Start with basic structure - let mut anthropic_schema = serde_json::json!({ - "type": "object" - }); + let mut anthropic_schema = Value::Object(serde_json::Map::new()); + anthropic_schema + .as_object_mut() + .unwrap() + .insert("type".to_string(), Value::String("object".to_string())); if let Some(t) = mcp_schema.get("type") { anthropic_schema["type"] = t.clone(); @@ -242,10 +308,11 @@ impl AnthropicProvider { AnthropicClient::new(&self.api_key) }; - // Convert our Message format to SDK Message format + // Convert our Message format to SDK Message format with caching on the final message let mut sdk_messages = Vec::new(); + let messages_count = messages.len(); - for msg in messages { + for (index, msg) in messages.iter().enumerate() { let role = match msg.role.as_str() { "user" => Role::User, "assistant" => Role::Assistant, @@ -253,6 +320,9 @@ impl AnthropicProvider { _ => Role::User, }; + // Check if this is the final message + let is_final_message = index == messages_count - 1; + // Handle different message types let content = if let Some(tool_results_json) = &msg.tool_results_json { // Parse tool results and format them for the SDK @@ -267,77 +337,99 @@ impl AnthropicProvider { result.tool_call_id, result.result )); } - Content::Text(result_text) + + // Add cache control to final message + if is_final_message { + Content::Blocks(vec![ContentBlock::Text { + text: result_text, + cache_control: Some(CacheControl::ephemeral()), + }]) + } else { + Content::Text(result_text) + } } else if let Some(_tool_calls_json) = &msg.tool_calls_json { - // For now, include tool calls as text in the message - // The SDK will handle tool use blocks separately - Content::Text(format!("{}\n[Tool calls pending]", msg.content)) + // Add cache control to final message + if is_final_message { + Content::Blocks(vec![ContentBlock::Text { + text: msg.content.clone(), + cache_control: Some(CacheControl::ephemeral()), + }]) + } else { + Content::Text(msg.content.clone()) + } } else { - Content::Text(msg.content.clone()) + // Add cache control to final message + if is_final_message { + Content::Blocks(vec![ContentBlock::Text { + text: msg.content.clone(), + cache_control: Some(CacheControl::ephemeral()), + }]) + } else { + Content::Text(msg.content.clone()) + } }; sdk_messages.push(SdkMessage { role, content }); } - // Convert our Tool format to SDK Tool format - let sdk_tools: Vec = tools - .iter() - .map(|tool| { - // Parse the MCP schema from either inputSchema or parameters - let mcp_schema = if let Some(ref input_schema_json) = tool.input_schema_json { - serde_json::from_str::(input_schema_json) - .unwrap_or_else(|_| serde_json::json!({})) - } else { - serde_json::from_str::(&tool.parameters) - .unwrap_or_else(|_| serde_json::json!({})) - }; - - // Transform MCP schema to Anthropic-compatible format - let anthropic_schema = self.transform_mcp_to_anthropic_schema(&mcp_schema); - - // Debug: Log the transformed schema - println!( - "Spider: Tool {} transformed schema: {}", - tool.name, - serde_json::to_string_pretty(&anthropic_schema) - .unwrap_or_else(|_| "error".to_string()) - ); - - // Extract required fields from the transformed schema - let required = anthropic_schema - .get("required") - .and_then(|r| r.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|v| v.as_str().map(String::from)) - .collect() - }) - .unwrap_or_else(Vec::new); - - SdkTool::new( - tool.name.clone(), - tool.description.clone(), - anthropic_schema["properties"].clone(), - required, - None, - //anthropic_schema.get("type").and_then(|v| v.as_str()).map(|s| s.to_string()), - ) - }) - .collect(); + // Convert our Tool format to SDK Tool format with caching on the last tool + let mut sdk_tools: Vec = Vec::new(); + let tools_count = tools.len(); + + for (index, tool) in tools.iter().enumerate() { + // Parse the MCP schema from either inputSchema or parameters + let mcp_schema = if let Some(ref input_schema_json) = tool.input_schema_json { + serde_json::from_str::(input_schema_json) + .unwrap_or_else(|_| Value::Object(serde_json::Map::new())) + } else { + serde_json::from_str::(&tool.parameters) + .unwrap_or_else(|_| Value::Object(serde_json::Map::new())) + }; + + // Transform MCP schema to Anthropic-compatible format + let anthropic_schema = self.transform_mcp_to_anthropic_schema(&mcp_schema); + + // Extract required fields from the transformed schema + let required = anthropic_schema + .get("required") + .and_then(|r| r.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect() + }) + .unwrap_or_else(Vec::new); + + let mut sdk_tool = SdkTool::new( + tool.name.clone(), + tool.description.clone(), + anthropic_schema["properties"].clone(), + required, + None, + //anthropic_schema.get("type").and_then(|v| v.as_str()).map(|s| s.to_string()), + ); + + // Add cache control to the last tool to cache all tool definitions + if index == tools_count - 1 && tools_count > 0 { + sdk_tool = sdk_tool.with_cache_control(CacheControl::ephemeral()); + } + + sdk_tools.push(sdk_tool); + } // Create the request with the specified model or default let model_id = model.unwrap_or("claude-sonnet-4-20250514"); let mut request = CreateMessageRequest::new(model_id, sdk_messages, max_tokens) .with_temperature(temperature); - // Add system prompt for OAuth tokens + // Add system prompt for OAuth tokens with caching if self.is_oauth { - request = - request.with_system("You are Claude Code, Anthropic's official CLI for Claude."); + request = request.with_system_blocks(vec![SystemPromptBlock::text( + "You are Claude Code, Anthropic's official CLI for Claude.", + ) + .with_cache_control(CacheControl::ephemeral())]); } - println!("Tools: {sdk_tools:?}"); - // Add tools if any if !sdk_tools.is_empty() { request = request @@ -348,16 +440,21 @@ impl AnthropicProvider { } // Send the message using the SDK - let response = client - .send_message(request) - .await - .map_err(|e| format!("Failed to send message via SDK: {:?}", e))?; + println!("[DEBUG] Sending request to Anthropic API"); + //println!("[DEBUG] Request: {:?}", request); + let response = client.send_message(request).await.map_err(|e| { + println!("[DEBUG] ERROR: Failed to send message via SDK: {:?}", e); + format!("Failed to send message via SDK: {:?}", e) + })?; + + println!("[DEBUG] Received response from Anthropic API"); + println!("[DEBUG] Raw SDK response: {:?}", response); // Convert SDK response back to our Message format let mut content_text = String::new(); let mut tool_calls: Vec = Vec::new(); - for block in &response.content { + for block in response.content.iter() { match block { ResponseContentBlock::Text { text, .. } => { if !content_text.is_empty() { @@ -376,9 +473,16 @@ impl AnthropicProvider { } } - Ok(Message { + // Replace empty content with "." to avoid Anthropic API issues + let final_content = if content_text.is_empty() { + ".".to_string() + } else { + content_text.clone() + }; + + let final_message = Message { role: "assistant".to_string(), - content: content_text, + content: final_content, tool_calls_json: if tool_calls.is_empty() { None } else { @@ -386,6 +490,8 @@ impl AnthropicProvider { }, tool_results_json: None, timestamp: Utc::now().timestamp() as u64, - }) + }; + + Ok(final_message) } } diff --git a/hyperdrive/packages/spider/spider/src/provider/mod.rs b/hyperdrive/packages/spider/spider/src/provider/mod.rs index 4712fb64e..fba53996b 100644 --- a/hyperdrive/packages/spider/spider/src/provider/mod.rs +++ b/hyperdrive/packages/spider/spider/src/provider/mod.rs @@ -3,8 +3,8 @@ use std::pin::Pin; use crate::types::{Message, Tool}; -mod anthropic; -use anthropic::AnthropicProvider; +pub mod anthropic; +pub use anthropic::AnthropicProvider; pub(crate) trait LlmProvider { fn complete<'a>( diff --git a/hyperdrive/packages/spider/spider/src/tool_providers/build_container.rs b/hyperdrive/packages/spider/spider/src/tool_providers/build_container.rs new file mode 100644 index 000000000..ac408dbbd --- /dev/null +++ b/hyperdrive/packages/spider/spider/src/tool_providers/build_container.rs @@ -0,0 +1,1366 @@ +use crate::tool_providers::{ToolExecutionCommand, ToolProvider}; +use crate::types::{ + BuildContainerRequest, InitializeParams, JsonRpcRequest, LoadProjectParams, McpCapabilities, + McpClientInfo, McpRequestType, PendingMcpRequest, PersistParams, SpiderAuthParams, + SpiderAuthRequest, SpiderState, StartPackageParams, Tool, ToolResponseContent, + ToolResponseContentItem, WsConnection, +}; +use hyperware_process_lib::{ + http::{ + client::{open_ws_connection, send_ws_client_push}, + server::WsMessageType, + }, + hyperapp::sleep, + vfs::open_file, + LazyLoadBlob, Request, +}; +use serde_json::Value; +use std::collections::HashMap; +use std::time::Duration; +use uuid::Uuid; + +pub struct BuildContainerToolProvider { + provider_id: String, +} + +const CONSTRUCTOR_SERVER_URL: &str = "http://localhost:8090"; + +impl BuildContainerToolProvider { + pub fn new() -> Self { + Self { + provider_id: "build_container".to_string(), + } + } + + fn create_init_build_container_tool(&self) -> Tool { + Tool { + name: "init-build-container".to_string(), + description: "Initialize a new build container for remote compilation and development (hosted mode only)".to_string(), + parameters: r#"{"type":"object","properties":{"metadata":{"type":"object","description":"Optional metadata about the project (type, estimated duration, etc.)"}}}"#.to_string(), + input_schema_json: Some(r#"{"type":"object","properties":{"metadata":{"type":"object","description":"Optional metadata about the project (type, estimated duration, etc.)"}}}"#.to_string()), + } + } + + fn create_load_project_tool(&self) -> Tool { + Tool { + name: "load-project".to_string(), + description: "Load a project into the build container. Creates a directory at `~/` which should be used as the working directory for all subsequent file operations and development work. A project name is required - if the user doesn't specify one explicitly, create a descriptive name based on their input or the project context.".to_string(), + parameters: r#"{"type":"object","required":["name"],"properties":{"project_uuid":{"type":"string","description":"Optional unique identifier for the project"},"name":{"type":"string","description":"Required project name. If user doesn't specify, create a descriptive name based on their input or project context"},"initial_zip":{"type":"string","description":"Optional VFS path to a zip file to extract in container's $HOME/ directory (e.g., /spider:dev.hypr/projects//backup.zip)"}}}"#.to_string(), + input_schema_json: Some(r#"{"type":"object","required":["name"],"properties":{"project_uuid":{"type":"string","description":"Optional unique identifier for the project"},"name":{"type":"string","description":"Required project name. If user doesn't specify, create a descriptive name based on their input or project context"},"initial_zip":{"type":"string","description":"Optional VFS path to a zip file to extract in container's $HOME/ directory (e.g., /spider:dev.hypr/projects//backup.zip)"}}}"#.to_string()), + } + } + + fn create_start_package_tool(&self) -> Tool { + Tool { + name: "start-package".to_string(), + description: "Deploy a built package from the build container to the Hyperware node. A package is distinguishable by a pkg/ directory inside of it. Do not use this tool on the pkg/ directory, but the directory that contains the pkg/".to_string(), + parameters: r#"{"type":"object","required":["package_dir"],"properties":{"package_dir":{"type":"string","description":"Path to the package directory that was built with 'kit build'"}}}"#.to_string(), + input_schema_json: Some(r#"{"type":"object","required":["package_dir"],"properties":{"package_dir":{"type":"string","description":"Path to the package directory that was built with 'kit build'"}}}"#.to_string()), + } + } + + fn create_persist_tool(&self) -> Tool { + Tool { + name: "persist".to_string(), + description: "Persist directories from the build container by creating a zip file".to_string(), + parameters: r#"{"type":"object","required":["directories"],"properties":{"directories":{"type":"array","items":{"type":"string"},"description":"List of directory paths to persist"}}}"#.to_string(), + input_schema_json: Some(r#"{"type":"object","required":["directories"],"properties":{"directories":{"type":"array","items":{"type":"string"},"description":"List of directory paths to persist"}}}"#.to_string()), + } + } + + fn create_done_build_container_tool(&self) -> Tool { + Tool { + name: "done-build-container".to_string(), + description: "Notify that work with the build container is complete and it can be torn down (hosted mode only)".to_string(), + parameters: r#"{"type":"object","properties":{"metadata":{"type":"object","description":"Optional metadata about completion status"}}}"#.to_string(), + input_schema_json: Some(r#"{"type":"object","properties":{"metadata":{"type":"object","description":"Optional metadata about completion status"}}}"#.to_string()), + } + } + + fn create_get_projects_tool(&self) -> Tool { + Tool { + name: "get-projects".to_string(), + description: "Get a mapping of project names to their associated UUIDs".to_string(), + parameters: r#"{"type":"object","properties":{}}"#.to_string(), + input_schema_json: Some(r#"{"type":"object","properties":{}}"#.to_string()), + } + } +} + +impl ToolProvider for BuildContainerToolProvider { + fn get_tools(&self, state: &SpiderState) -> Vec { + let mut tools = Vec::new(); + + // Always provide get-projects tool + tools.push(self.create_get_projects_tool()); + + // Check if we're in self-hosted mode + let is_self_hosted = + !state.build_container_ws_uri.is_empty() && !state.build_container_api_key.is_empty(); + + if !is_self_hosted { + // Hosted mode: show init_build_container + tools.push(self.create_init_build_container_tool()); + } + + // Check if we have an active build container connection + let has_connection = state.ws_connections.values().any(|conn| { + conn.server_id.starts_with("build_container_") + || conn.server_id == "build_container_self_hosted" + }); + + // Always show load_project in self-hosted mode, or if we have a connection in hosted mode + if is_self_hosted { + tools.push(self.create_load_project_tool()); + } else if has_connection { + tools.push(self.create_load_project_tool()); + } + + // Show other tools if we have an active build container connection + if has_connection { + tools.push(self.create_start_package_tool()); + tools.push(self.create_persist_tool()); + + if !is_self_hosted { + // Only show done_build_container in hosted mode + tools.push(self.create_done_build_container_tool()); + } + } + + tools + } + + fn should_include_tool(&self, tool_name: &str, state: &SpiderState) -> bool { + let is_self_hosted = + !state.build_container_ws_uri.is_empty() && !state.build_container_api_key.is_empty(); + let has_connection = state + .ws_connections + .values() + .any(|conn| conn.server_id.starts_with("build_container_")); + + match tool_name { + "get-projects" => true, // Always available + "init-build-container" => !is_self_hosted, + "load-project" => is_self_hosted || has_connection, + "start-package" | "persist" => has_connection, + "done-build-container" => !is_self_hosted && has_connection, + _ => false, + } + } + + fn prepare_execution( + &self, + tool_name: &str, + parameters: &Value, + state: &SpiderState, + ) -> Result { + match tool_name { + "get-projects" => Ok(ToolExecutionCommand::GetProjects), + "init-build-container" => { + let metadata = parameters.get("metadata").cloned(); + + Ok(ToolExecutionCommand::InitBuildContainer { metadata }) + } + "load-project" => { + let project_uuid = parameters + .get("project_uuid") + .and_then(|v| v.as_str()) + .map(String::from); + + // Name is now required + let name = parameters + .get("name") + .and_then(|v| v.as_str()) + .map(String::from) + .ok_or_else(|| "Project name is required. Please provide a descriptive name for the project.".to_string())?; + + let initial_zip = parameters + .get("initial_zip") + .and_then(|v| v.as_str()) + .map(String::from); + + // Check if we need to establish connection for self-hosted mode + let is_self_hosted = !state.build_container_ws_uri.is_empty() + && !state.build_container_api_key.is_empty(); + let channel_id = if is_self_hosted + && !state.ws_connections.values().any(|conn| { + conn.server_id.starts_with("build_container_") + || conn.server_id == "build_container_self_hosted" + }) { + // Need to establish connection first (this will be handled in execute) + None + } else { + // Find existing build container connection + state + .ws_connections + .iter() + .find(|(_, conn)| { + conn.server_id.starts_with("build_container_") + || conn.server_id == "build_container_self_hosted" + }) + .map(|(id, _)| *id) + }; + + Ok(ToolExecutionCommand::LoadProject { + project_uuid, + name, + initial_zip, + channel_id, + }) + } + "start-package" => { + let package_dir = parameters + .get("package_dir") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing package_dir parameter".to_string())? + .to_string(); + + // Find the build container connection + let channel_id = state + .ws_connections + .iter() + .find(|(_, conn)| conn.server_id.starts_with("build_container_")) + .map(|(id, _)| *id) + .ok_or_else(|| { + "No build container connection found. Call init-build-container first." + .to_string() + })?; + + Ok(ToolExecutionCommand::StartPackage { + channel_id, + package_dir, + }) + } + "persist" => { + let directories = parameters + .get("directories") + .and_then(|v| v.as_array()) + .ok_or_else(|| "Missing directories parameter".to_string())?; + + let dir_strings: Vec = directories + .iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect(); + + if dir_strings.is_empty() { + return Err("No valid directories provided".to_string()); + } + + // Find the build container connection + let channel_id = state + .ws_connections + .iter() + .find(|(_, conn)| conn.server_id.starts_with("build_container_")) + .map(|(id, _)| *id) + .ok_or_else(|| { + "No build container connection found. Call init-build-container first." + .to_string() + })?; + + Ok(ToolExecutionCommand::Persist { + channel_id, + directories: dir_strings, + }) + } + "done-build-container" => { + let metadata = parameters.get("metadata").cloned(); + + // Find any active build container connection + let channel_id = state + .ws_connections + .iter() + .find(|(_, conn)| conn.server_id.starts_with("build_container_")) + .map(|(id, _)| *id); + + Ok(ToolExecutionCommand::DoneBuildContainer { + metadata, + channel_id, + }) + } + _ => Err(format!("Unknown build container tool: {}", tool_name)), + } + } + + fn get_provider_id(&self) -> &str { + &self.provider_id + } +} + +// Extension trait for build container operations +pub trait BuildContainerExt { + async fn execute_init_build_container_impl( + &mut self, + metadata: Option, + ) -> Result; + async fn execute_load_project_impl( + &mut self, + project_uuid: Option, + name: String, // Now required + initial_zip: Option, + channel_id: Option, + ) -> Result; + async fn execute_start_package_impl( + &mut self, + channel_id: u32, + package_dir: String, + ) -> Result; + async fn execute_persist_impl( + &mut self, + channel_id: u32, + directories: Vec, + ) -> Result; + async fn execute_done_build_container_impl( + &mut self, + metadata: Option, + channel_id: Option, + ) -> Result; + async fn connect_to_self_hosted_container(&mut self) -> Result; + fn request_build_container_tools_list(&mut self, channel_id: u32); + fn send_tools_list_request(&mut self, channel_id: u32); + async fn deploy_package_to_app_store( + &self, + package_name: &str, + publisher: &str, + version_hash: &str, + package_zip: &str, + metadata: Value, + ) -> Result<(), String>; +} + +impl BuildContainerExt for SpiderState { + async fn execute_init_build_container_impl( + &mut self, + metadata: Option, + ) -> Result { + use hyperware_process_lib::http::client::send_request_await_response; + use hyperware_process_lib::http::Method; + + // Use hardcoded constructor URL + let constructor_url = format!("{CONSTRUCTOR_SERVER_URL}/init-build-container"); + + // Prepare request body + let body = BuildContainerRequest { + metadata: metadata.clone(), + }; + + // Make HTTP request to constructor + let mut headers = HashMap::new(); + headers.insert("Content-Type".to_string(), "application/json".to_string()); + + let url = url::Url::parse(&constructor_url) + .map_err(|e| format!("Invalid constructor URL: {}", e))?; + + let response = send_request_await_response( + Method::POST, + url, + Some(headers), + 30000, + serde_json::to_string(&body) + .map_err(|e| format!("Failed to serialize request: {}", e))? + .into_bytes(), + ) + .await + .map_err(|e| format!("Failed to initialize build container: {:?}", e))?; + + if !response.status().is_success() { + let error_text = String::from_utf8_lossy(response.body()); + return Err(format!( + "Constructor error (status {}): {}", + response.status(), + error_text + )); + } + + // Parse response + let response_data: Value = serde_json::from_slice(response.body()) + .map_err(|e| format!("Failed to parse constructor response: {}", e))?; + + let ws_uri = response_data + .get("ws_uri") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing ws_uri in response".to_string())?; + + let api_key = response_data + .get("api_key") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing api_key in response".to_string())?; + + // Generate a unique project UUID since we don't require it anymore + let project_uuid = Uuid::new_v4().to_string(); + + // Connect to the build container's ws-mcp server + let channel_id = self.next_channel_id; + self.next_channel_id += 1; + + // Open WebSocket connection + open_ws_connection(ws_uri.to_string(), None, channel_id) + .await + .map_err(|e| format!("Failed to open WS connection to {ws_uri}: {e}"))?; + + // Store connection info for the build container + let server_id = format!("build_container_{}", project_uuid); + self.ws_connections.insert( + channel_id, + WsConnection { + server_id: server_id.clone(), + server_name: format!("Build Container {}", project_uuid), + channel_id, + tools: Vec::new(), + initialized: false, + }, + ); + + // Send authentication message + let auth_request = SpiderAuthRequest { + jsonrpc: "2.0".to_string(), + method: "spider/authorization".to_string(), + params: SpiderAuthParams { + api_key: api_key.to_string(), + }, + id: format!("auth_{}", channel_id), + }; + + let blob = LazyLoadBlob::new( + None::, + serde_json::to_string(&auth_request) + .map_err(|e| format!("Failed to serialize auth request: {}", e))? + .into_bytes(), + ); + send_ws_client_push(channel_id, WsMessageType::Text, blob); + + // Update build container tools to show additional tools now that we're connected + if let Some(provider) = self + .tool_provider_registry + .find_provider_for_tool("init-build-container", self) + { + let updated_tools = provider.get_tools(self); + if let Some(server) = self + .mcp_servers + .iter_mut() + .find(|s| s.id == "build_container") + { + server.tools = updated_tools; + } + } + + Ok(serde_json::to_value(ToolResponseContent { + content: vec![ToolResponseContentItem { + content_type: "text".to_string(), + text: format!( + "✅ Build container initialized successfully!\n- WebSocket: {}\n- Ready for remote compilation", + ws_uri + ), + }], + }) + .map_err(|e| format!("Failed to serialize response: {}", e))?) + } + + async fn execute_load_project_impl( + &mut self, + project_uuid: Option, + name: String, // Now required + initial_zip: Option, + mut channel_id: Option, + ) -> Result { + // Check if we need to connect to self-hosted container first + let is_self_hosted = + !self.build_container_ws_uri.is_empty() && !self.build_container_api_key.is_empty(); + + if is_self_hosted && channel_id.is_none() { + // Connect to self-hosted container + channel_id = Some(self.connect_to_self_hosted_container().await?); + } + + let channel_id = + channel_id.ok_or_else(|| "No build container connection available".to_string())?; + + // Generate project UUID if not provided + let project_uuid = project_uuid.unwrap_or_else(|| Uuid::new_v4().to_string()); + + // Handle initial_zip - must be a VFS path if provided + let initial_zip_content = if let Some(zip_path) = &initial_zip { + // Validate it's a proper VFS path + if !zip_path.starts_with('/') { + return Err(format!( + "Invalid VFS path '{}'. VFS paths must start with '/' (e.g., /spider:dev.hypr/projects//backup.zip). \ + To load a persisted project, first use 'get-projects' to find available projects, \ + then provide the full VFS path to the backup zip file.", + zip_path + )); + } + + // Load the zip file from VFS + match open_file(zip_path, false, None) { + Ok(file) => { + match file.read() { + Ok(data) => { + if data.is_empty() { + return Err(format!( + "The zip file at '{}' exists but is empty. \ + Please ensure the project was properly persisted with the 'persist' tool.", + zip_path + )); + } + // Encode to base64 for transmission + use base64::{engine::general_purpose, Engine as _}; + Some(general_purpose::STANDARD.encode(&data)) + } + Err(e) => { + return Err(format!( + "Failed to read zip file at '{}': {:?}. \ + Please verify the file exists and you have read permissions. \ + Use 'get-projects' to see available projects.", + zip_path, e + )); + } + } + } + Err(e) => { + // Provide helpful suggestions based on the error + let suggestion = if zip_path.contains("/projects/") { + "Use 'get-projects' to list available projects and their UUIDs, \ + then check the VFS for the correct backup.zip path." + } else { + "Make sure the path follows the format: /spider:dev.hypr/projects//backup.zip" + }; + + return Err(format!( + "Cannot open zip file at '{}': {:?}. \ + {}. The file may not exist or the path may be incorrect.", + zip_path, e, suggestion + )); + } + } + } else { + None + }; + + // Update project name to UUID mapping (name is now always present) + self.project_name_to_uuids + .entry(name.clone()) + .or_insert_with(Vec::new) + .push(project_uuid.clone()); + println!( + "Spider: Added project '{}' with UUID {}", + name, project_uuid + ); + + // Send spider/load-project request over WebSocket + let request_id = format!("load-project-{}", Uuid::new_v4()); + println!( + "Spider: Sending load-project request with id: {}", + request_id + ); + let request = JsonRpcRequest { + jsonrpc: "2.0".to_string(), + method: "spider/load-project".to_string(), + params: Some( + serde_json::to_value(LoadProjectParams { + project_uuid: project_uuid.clone(), + name: Some(name.clone()), + initial_zip: initial_zip_content, + }) + .map_err(|e| format!("Failed to serialize params: {}", e))?, + ), + id: request_id.clone(), + }; + + let request_json = serde_json::to_string(&request) + .map_err(|e| format!("Failed to serialize request: {}", e))?; + + println!( + "Spider: Sending request to channel {}: {}", + channel_id, request_json + ); + let blob = LazyLoadBlob::new(None::, request_json.into_bytes()); + send_ws_client_push(channel_id, WsMessageType::Text, blob); + + // Wait for response (with timeout) + let start = std::time::Instant::now(); + let timeout = Duration::from_secs(30); + + println!("Spider: Waiting for response with id: {}", request_id); + loop { + if start.elapsed() > timeout { + println!( + "Spider: Timeout waiting for response, tool_responses keys: {:?}", + self.tool_responses.keys().collect::>() + ); + return Err("Timeout waiting for load-project response".to_string()); + } + + if let Some(response) = self.tool_responses.remove(&request_id) { + println!( + "Spider: Found response for id {}: {:?}", + request_id, response + ); + // Check if response contains an error + if let Some(error) = response.get("error") { + return Err(format!("Failed to load project: {}", error)); + } + + // Extract project_uuid from response + let returned_uuid = response + .get("project_uuid") + .and_then(|v| v.as_str()) + .unwrap_or(&project_uuid); + + // After successful load-project, ws-mcp may have new tools available + // Send tools/list and wait for the response to ensure tools are updated + println!("Spider: Requesting updated tools list after successful load-project"); + let tools_request_id = format!("tools_refresh_{}", channel_id); + let tools_request = JsonRpcRequest { + jsonrpc: "2.0".to_string(), + method: "tools/list".to_string(), + params: None, + id: tools_request_id.clone(), + }; + + // Store pending request + if let Some(conn) = self.ws_connections.get(&channel_id) { + self.pending_mcp_requests.insert( + tools_request_id.clone(), + PendingMcpRequest { + request_id: tools_request_id.clone(), + conversation_id: None, + server_id: conn.server_id.clone(), + request_type: McpRequestType::ToolsList, + }, + ); + } + + println!( + "Spider: Sending tools/list request with id: {}", + tools_request_id + ); + let blob = LazyLoadBlob::new( + None::, + serde_json::to_string(&tools_request).unwrap().into_bytes(), + ); + send_ws_client_push(channel_id, WsMessageType::Text, blob); + + // Wait for the tools/list response with a short timeout + let tools_start = std::time::Instant::now(); + let tools_timeout = std::time::Duration::from_secs(5); + + println!("Spider: Waiting for tools/list response after load-project"); + loop { + if tools_start.elapsed() > tools_timeout { + println!( + "Spider: Timeout waiting for tools/list response, continuing anyway" + ); + break; // Don't fail, just continue without updated tools + } + + // Check if the tools have been updated (handle_tools_list_response will update them) + // We just need to wait a bit for the response to be processed + if !self.pending_mcp_requests.contains_key(&tools_request_id) { + println!("Spider: Tools list updated successfully"); + break; + } + + // Sleep briefly before checking again + let _ = sleep(100).await; + } + + return Ok(serde_json::to_value(ToolResponseContent { + content: vec![ToolResponseContentItem { + content_type: "text".to_string(), + text: format!( + "✅ Project loaded successfully!\n- UUID: {}\n- Directory created in container", + returned_uuid + ), + }], + }) + .map_err(|e| format!("Failed to serialize response: {}", e))?); + } + + // Sleep briefly before checking again + sleep(100).await; + } + } + + async fn execute_start_package_impl( + &mut self, + channel_id: u32, + package_dir: String, + ) -> Result { + // Send spider/start-package request over WebSocket + let request_id = format!("start-package-{}", Uuid::new_v4()); + let request = JsonRpcRequest { + jsonrpc: "2.0".to_string(), + method: "spider/start-package".to_string(), + params: Some( + serde_json::to_value(StartPackageParams { + package_dir: package_dir.clone(), + }) + .map_err(|e| format!("Failed to serialize params: {}", e))?, + ), + id: request_id.clone(), + }; + + let request_json = serde_json::to_string(&request) + .map_err(|e| format!("Failed to serialize request: {}", e))?; + + let blob = LazyLoadBlob::new(None::, request_json.into_bytes()); + send_ws_client_push(channel_id, WsMessageType::Text, blob); + + // Wait for response (with timeout) + let start = std::time::Instant::now(); + let timeout = Duration::from_secs(30); + + loop { + if start.elapsed() > timeout { + return Err("Timeout waiting for start-package response".to_string()); + } + + if let Some(response) = self.tool_responses.remove(&request_id) { + // Check if response contains an error + if let Some(error) = response.get("error") { + return Err(format!("Failed to start package: {}", error)); + } + + // Extract package_zip from response + let Some(package_zip) = response.get("package_zip").and_then(|v| v.as_str()) else { + return Err("No package_zip in response".to_string()); + }; + + // Extract metadata fields from response + let package_name = response + .get("package_name") + .and_then(|v| v.as_str()) + .ok_or_else(|| "No package_name in response".to_string())?; + + let our_node = hyperware_process_lib::our().node.clone(); + let publisher = response + .get("publisher") + .and_then(|v| v.as_str()) + .unwrap_or(&our_node); + + let version_hash = response + .get("version_hash") + .and_then(|v| v.as_str()) + .ok_or_else(|| "No version_hash in response".to_string())?; + + // Get the full metadata object from response + let metadata = response + .get("metadata") + .ok_or_else(|| "No metadata in response".to_string())?; + + // Deploy the package to the Hyperware node using app-store + match self + .deploy_package_to_app_store( + package_name, + publisher, + version_hash, + package_zip, + metadata.clone(), + ) + .await + { + Ok(_) => { + return Ok(serde_json::to_value(ToolResponseContent { + content: vec![ToolResponseContentItem { + content_type: "text".to_string(), + text: format!( + "✅ Package '{}' from {} deployed and installed successfully!\n- Publisher: {}\n- Version hash: {}", + package_name, + package_dir, + publisher, + &version_hash[..8] // Show first 8 chars of hash + ), + }], + }) + .map_err(|e| format!("Failed to serialize response: {}", e))?); + } + Err(e) => { + return Err(format!("Failed to deploy package: {}", e)); + } + } + } + + // Sleep briefly before checking again + sleep(100).await; + } + } + + async fn execute_persist_impl( + &mut self, + channel_id: u32, + directories: Vec, + ) -> Result { + use hyperware_process_lib::{ + our, + vfs::{create_drive, open_dir, open_file}, + }; + + // Send spider/persist request over WebSocket + let request_id = format!("persist_{}", Uuid::new_v4()); + let request = JsonRpcRequest { + jsonrpc: "2.0".to_string(), + method: "spider/persist".to_string(), + params: Some( + serde_json::to_value(PersistParams { + directories: directories.clone(), + }) + .map_err(|e| format!("Failed to serialize params: {}", e))?, + ), + id: request_id.clone(), + }; + + let request_json = serde_json::to_string(&request) + .map_err(|e| format!("Failed to serialize request: {}", e))?; + + let blob = LazyLoadBlob::new(None::, request_json.into_bytes()); + send_ws_client_push(channel_id, WsMessageType::Text, blob); + + // Wait for response (with timeout) + let start = std::time::Instant::now(); + let timeout = Duration::from_secs(30); + + loop { + if start.elapsed() > timeout { + return Err("Timeout waiting for persist response".to_string()); + } + + if let Some(response) = self.tool_responses.remove(&request_id) { + // Check if response contains persisted_zip + if let Some(persisted_zip) = response.get("persisted_zip").and_then(|v| v.as_str()) + { + // Get project_uuid from response or generate one + let project_uuid = response + .get("project_uuid") + .and_then(|v| v.as_str()) + .unwrap_or_else(|| { + // If no project_uuid in response, try to extract from the first directory path + // Assuming directories are like /home/user//... + directories + .first() + .and_then(|dir| { + let parts: Vec<&str> = dir.split('/').collect(); + // Look for a UUID-like string in the path + parts + .iter() + .find(|part| part.len() == 36 && part.contains('-')) + .copied() + }) + .unwrap_or("unknown") + }) + .to_string(); + + // Create projects drive if it doesn't exist + let projects_drive = match create_drive(our().package_id(), "projects", None) { + Ok(drive_path) => drive_path, + Err(e) => { + println!("Warning: Failed to create projects drive: {:?}", e); + // Still return success but without saving to VFS + return Ok(serde_json::to_value(ToolResponseContent { + content: vec![ToolResponseContentItem { + content_type: "text".to_string(), + text: format!( + "✅ Persisted {} directories successfully! (Note: Could not save backup to VFS)", + directories.len() + ), + }], + }) + .map_err(|e| format!("Failed to serialize response: {}", e))?); + } + }; + + // Create project-specific directory path + let project_dir = format!("{}/{}", projects_drive, project_uuid); + + // Create the project directory + match open_dir(&project_dir, true, None) { + Ok(_) => { + println!("Spider: Created/opened project directory: {}", project_dir); + } + Err(e) => { + println!( + "Warning: Failed to create project directory {}: {:?}", + project_dir, e + ); + // Still try to continue - maybe we can write files directly + } + } + + // Generate timestamp for the zip file + let timestamp = chrono::Utc::now().format("%Y%m%d-%H%M%S").to_string(); + let zip_filename = format!("{}/backup-{}.zip", project_dir, timestamp); + let manifest_filename = format!("{}/manifest-{}.json", project_dir, timestamp); + + // Decode the base64 zip data + use base64::{engine::general_purpose, Engine as _}; + let zip_bytes = general_purpose::STANDARD + .decode(persisted_zip) + .map_err(|e| format!("Failed to decode base64 zip: {}", e))?; + + // Save the zip file + match open_file(&zip_filename, true, None) { + Ok(file) => { + file.write(&zip_bytes) + .map_err(|e| format!("Failed to write zip file: {:?}", e))?; + println!("Saved project backup to: {}", zip_filename); + } + Err(e) => { + println!("Warning: Failed to save zip file: {:?}", e); + } + } + + // Create and save manifest + let manifest = serde_json::json!({ + "project_uuid": project_uuid, + "timestamp": timestamp, + "directories": directories, + "zip_file": zip_filename, + "size_bytes": zip_bytes.len(), + }); + + let manifest_json = serde_json::to_string_pretty(&manifest) + .map_err(|e| format!("Failed to serialize manifest: {}", e))?; + + match open_file(&manifest_filename, true, None) { + Ok(file) => { + file.write(manifest_json.as_bytes()) + .map_err(|e| format!("Failed to write manifest: {:?}", e))?; + println!("Saved manifest to: {}", manifest_filename); + } + Err(e) => { + println!("Warning: Failed to save manifest: {:?}", e); + } + } + + return Ok(serde_json::to_value(ToolResponseContent { + content: vec![ToolResponseContentItem { + content_type: "text".to_string(), + text: format!( + "✅ Persisted {} directories successfully!\n📁 Project: {}\n💾 Backup: {}\n📝 Manifest: {}", + directories.len(), + project_uuid, + zip_filename, + manifest_filename + ), + }], + }) + .map_err(|e| format!("Failed to serialize response: {}", e))?); + } else if let Some(error) = response.get("error") { + return Err(format!("Failed to persist directories: {}", error)); + } else { + return Err("Invalid response from persist operation".to_string()); + } + } + + // Sleep briefly before checking again + let _ = sleep(100).await; + } + } + + async fn execute_done_build_container_impl( + &mut self, + metadata: Option, + channel_id: Option, + ) -> Result { + use hyperware_process_lib::http::client::send_request_await_response; + use hyperware_process_lib::http::Method; + + if let Some(channel_id) = channel_id { + // Get server_id before removing the connection + let server_id = self + .ws_connections + .get(&channel_id) + .map(|conn| conn.server_id.clone()); + + // Send close message + send_ws_client_push(channel_id, WsMessageType::Close, LazyLoadBlob::default()); + + // Remove the connection + self.ws_connections.remove(&channel_id); + + // Clean up any pending requests for this connection + if let Some(sid) = server_id { + self.pending_mcp_requests + .retain(|_, req| req.server_id != sid); + } + } + + // Use hardcoded constructor URL + let constructor_url = format!("{CONSTRUCTOR_SERVER_URL}/done-build-container"); + + // Prepare request body + let body = BuildContainerRequest { + metadata: metadata.clone(), + }; + + // Make HTTP request to constructor + let mut headers = HashMap::new(); + headers.insert("Content-Type".to_string(), "application/json".to_string()); + + let url = url::Url::parse(&constructor_url) + .map_err(|e| format!("Invalid constructor URL: {}", e))?; + + let response = send_request_await_response( + Method::POST, + url, + Some(headers), + 30000, + serde_json::to_string(&body) + .map_err(|e| format!("Failed to serialize request: {}", e))? + .into_bytes(), + ) + .await + .map_err(|e| format!("Failed to tear down build container: {:?}", e))?; + + if !response.status().is_success() { + let error_text = String::from_utf8_lossy(response.body()); + return Err(format!( + "Constructor error (status {}): {}", + response.status(), + error_text + )); + } + + // Update build container tools to hide additional tools now that we're disconnected + if let Some(provider) = self + .tool_provider_registry + .find_provider_for_tool("init-build-container", self) + { + let updated_tools = provider.get_tools(self); + if let Some(server) = self + .mcp_servers + .iter_mut() + .find(|s| s.id == "build_container") + { + server.tools = updated_tools; + } + } + + Ok(serde_json::to_value(ToolResponseContent { + content: vec![ToolResponseContentItem { + content_type: "text".to_string(), + text: "✅ Build container has been torn down successfully!".to_string(), + }], + }) + .map_err(|e| format!("Failed to serialize response: {}", e))?) + } + + async fn connect_to_self_hosted_container(&mut self) -> Result { + // Check if we already have a connection to self-hosted container + if let Some((channel_id, _)) = self + .ws_connections + .iter() + .find(|(_, conn)| conn.server_id == "build_container_self_hosted") + { + println!( + "Spider: Reusing existing self-hosted build container connection on channel {}", + channel_id + ); + return Ok(*channel_id); + } + + // Connect to self-hosted container using configured WS URI and API key + let channel_id = self.next_channel_id; + self.next_channel_id += 1; + + println!( + "Spider: Opening new WebSocket connection to self-hosted build container on channel {}", + channel_id + ); + + // Open WebSocket connection + open_ws_connection(self.build_container_ws_uri.clone(), None, channel_id) + .await + .map_err(|e| { + format!( + "Failed to open WS connection to {}: {e}", + self.build_container_ws_uri + ) + })?; + + // Store connection info for the build container + let server_id = "build_container_self_hosted".to_string(); + self.ws_connections.insert( + channel_id, + WsConnection { + server_id: server_id.clone(), + server_name: "Self-Hosted Build Container".to_string(), + channel_id, + tools: Vec::new(), + initialized: false, + }, + ); + + // Send authentication message + let auth_id = format!("auth_{}", channel_id); + let auth_request = SpiderAuthRequest { + jsonrpc: "2.0".to_string(), + method: "spider/authorization".to_string(), + params: SpiderAuthParams { + api_key: self.build_container_api_key.clone(), + }, + id: auth_id.clone(), + }; + + println!("Spider: Sending authorization request with id: {}", auth_id); + let blob = LazyLoadBlob::new( + None::, + serde_json::to_string(&auth_request) + .map_err(|e| format!("Failed to serialize auth request: {}", e))? + .into_bytes(), + ); + send_ws_client_push(channel_id, WsMessageType::Text, blob); + + // Wait for authentication response + let start = std::time::Instant::now(); + let timeout = Duration::from_secs(10); + + println!("Spider: Waiting for authorization response..."); + loop { + if start.elapsed() > timeout { + println!("Spider: Authorization timeout after 10 seconds"); + return Err("Timeout waiting for authorization response".to_string()); + } + + if let Some(response) = self.tool_responses.remove(&auth_id) { + println!("Spider: Got authorization response: {:?}", response); + if response.get("status").and_then(|s| s.as_str()) == Some("authenticated") { + println!("Spider: Successfully authenticated with self-hosted build container"); + break; + } else if let Some(error) = response.get("error") { + return Err(format!("Authorization failed: {}", error)); + } else { + return Err("Invalid authorization response".to_string()); + } + } + + // Sleep briefly before checking again + let _ = sleep(100).await; + } + + // Now send initialize request after successful authentication + println!("Spider: Sending initialize request after successful authentication"); + self.request_build_container_tools_list(channel_id); + + // Update build container tools to show additional tools now that we're connected + if let Some(provider) = self + .tool_provider_registry + .find_provider_for_tool("load-project", self) + { + let updated_tools = provider.get_tools(self); + if let Some(server) = self + .mcp_servers + .iter_mut() + .find(|s| s.id == "build_container") + { + server.tools = updated_tools; + } + } + + Ok(channel_id) + } + + fn request_build_container_tools_list(&mut self, channel_id: u32) { + use crate::types::{McpRequestType, PendingMcpRequest}; + + // First send initialize request + let init_request_id = format!("init_build_container_{}", channel_id); + let init_request = JsonRpcRequest { + jsonrpc: "2.0".to_string(), + method: "initialize".to_string(), + params: Some( + serde_json::to_value(InitializeParams { + protocol_version: "2024-11-05".to_string(), + client_info: McpClientInfo { + name: "spider".to_string(), + version: "1.0.0".to_string(), + }, + capabilities: McpCapabilities {}, + }) + .unwrap_or_else(|_| Value::Null), + ), + id: init_request_id.clone(), + }; + + // Store pending request for initialize + if let Some(conn) = self.ws_connections.get(&channel_id) { + self.pending_mcp_requests.insert( + init_request_id.clone(), + PendingMcpRequest { + request_id: init_request_id.clone(), + conversation_id: None, + server_id: conn.server_id.clone(), + request_type: McpRequestType::Initialize, + }, + ); + } + + println!( + "Spider: Sending initialize request with id: {}", + init_request_id + ); + let init_blob = LazyLoadBlob::new( + None::, + serde_json::to_string(&init_request).unwrap().into_bytes(), + ); + send_ws_client_push(channel_id, WsMessageType::Text, init_blob); + + // Note: The actual tools/list request will be sent when we receive the initialize response + // This is handled in handle_initialize_response in lib.rs which calls request_tools_list + } + + fn send_tools_list_request(&mut self, channel_id: u32) { + use crate::types::{McpRequestType, PendingMcpRequest}; + + let request_id = format!("tools_refresh_{}", channel_id); + let tools_request = JsonRpcRequest { + jsonrpc: "2.0".to_string(), + method: "tools/list".to_string(), + params: None, + id: request_id.clone(), + }; + + // Store pending request + if let Some(conn) = self.ws_connections.get(&channel_id) { + self.pending_mcp_requests.insert( + request_id.clone(), + PendingMcpRequest { + request_id: request_id.clone(), + conversation_id: None, + server_id: conn.server_id.clone(), + request_type: McpRequestType::ToolsList, + }, + ); + } + + println!("Spider: Sending tools/list request with id: {}", request_id); + let blob = LazyLoadBlob::new( + None::, + serde_json::to_string(&tools_request).unwrap().into_bytes(), + ); + send_ws_client_push(channel_id, WsMessageType::Text, blob); + } + + async fn deploy_package_to_app_store( + &self, + package_name: &str, + publisher: &str, + version_hash: &str, + package_zip: &str, + metadata: Value, + ) -> Result<(), String> { + use base64::Engine; + + println!("Spider: Deploying package {} to app-store", package_name); + + // Decode the base64 package zip + let package_bytes = base64::engine::general_purpose::STANDARD + .decode(package_zip) + .map_err(|e| format!("Failed to decode package zip: {}", e))?; + + // Create NewPackage request + let new_package_request = serde_json::json!({ + "NewPackage": { + "package_id": { + "package_name": package_name, + "publisher_node": publisher, + }, + "mirror": true + } + }); + + // Send NewPackage request to app-store with the zip as blob + let blob = LazyLoadBlob::new(None::, package_bytes); + let request = Request::to(("our", "main", "app-store", "sys")) + .body(serde_json::to_vec(&new_package_request).map_err(|e| e.to_string())?) + .blob(blob) + .expects_response(15); + + let response = request + .send_and_await_response(15) + .map_err(|e| format!("Failed to send new-package request: {:?}", e))? + .map_err(|e| format!("New-package request failed: {:?}", e))?; + + // Parse response + let response_body = String::from_utf8(response.body().to_vec()) + .map_err(|e| format!("Failed to parse response body: {}", e))?; + let response_json: Value = serde_json::from_str(&response_body) + .map_err(|e| format!("Failed to parse response JSON: {}", e))?; + + // Check if NewPackage was successful + if let Some(new_package_response) = response_json.get("NewPackageResponse") { + if new_package_response != &serde_json::Value::String("Success".to_string()) { + return Err(format!("Failed to add package: {:?}", new_package_response)); + } + } else { + return Err(format!( + "Unexpected response from app-store: {:?}", + response_json + )); + } + + println!("Spider: Package added successfully, now installing..."); + + // Parse metadata to create OnchainMetadata + let onchain_metadata = serde_json::json!({ + "name": metadata.get("name").and_then(|v| v.as_str()).unwrap_or(package_name), + "description": metadata.get("description").and_then(|v| v.as_str()).unwrap_or(""), + "image": metadata.get("image").and_then(|v| v.as_str()).unwrap_or(""), + "external_url": metadata.get("external_url").and_then(|v| v.as_str()).unwrap_or(""), + "animation_url": metadata.get("animation_url").and_then(|v| v.as_str()), + "properties": { + "package_name": package_name, + "publisher": publisher, + "current_version": metadata.get("current_version").and_then(|v| v.as_str()).unwrap_or("1.0.0"), + "mirrors": metadata.get("mirrors").and_then(|v| v.as_array()).unwrap_or(&vec![]).clone(), + "code_hashes": metadata.get("code_hashes").and_then(|v| v.as_array()).unwrap_or(&vec![]).clone(), + "license": metadata.get("license").and_then(|v| v.as_str()), + "screenshots": metadata.get("screenshots").and_then(|v| v.as_array()).map(|v| v.clone()), + "wit_version": metadata.get("wit_version").and_then(|v| v.as_u64()).map(|v| v as u32), + "dependencies": metadata.get("dependencies").and_then(|v| v.as_array()).map(|v| v.clone()), + "api_includes": metadata.get("api_includes").and_then(|v| v.as_array()).map(|v| v.clone()), + } + }); + + // Create Install request + let install_request = serde_json::json!({ + "Install": { + "package_id": { + "package_name": package_name, + "publisher_node": publisher, + }, + "version_hash": version_hash, + "metadata": onchain_metadata + } + }); + + // Send Install request to app-store + let request = Request::to(("our", "main", "app-store", "sys")) + .body(serde_json::to_vec(&install_request).map_err(|e| e.to_string())?) + .expects_response(15); + + let response = request + .send_and_await_response(15) + .map_err(|e| format!("Failed to send install request: {:?}", e))? + .map_err(|e| format!("Install request failed: {:?}", e))?; + + // Parse response + let response_body = String::from_utf8(response.body().to_vec()) + .map_err(|e| format!("Failed to parse response body: {}", e))?; + let response_json: Value = serde_json::from_str(&response_body) + .map_err(|e| format!("Failed to parse response JSON: {}", e))?; + + // Check if Install was successful + if let Some(install_response) = response_json.get("InstallResponse") { + if install_response == &serde_json::Value::String("Success".to_string()) { + println!("Spider: Package {} installed successfully!", package_name); + Ok(()) + } else { + Err(format!("Failed to install package: {:?}", install_response)) + } + } else { + Err(format!( + "Unexpected response from app-store: {:?}", + response_json + )) + } + } +} diff --git a/hyperdrive/packages/spider/spider/src/tool_providers/hypergrid.rs b/hyperdrive/packages/spider/spider/src/tool_providers/hypergrid.rs index 2c4a65a06..82839468c 100644 --- a/hyperdrive/packages/spider/spider/src/tool_providers/hypergrid.rs +++ b/hyperdrive/packages/spider/spider/src/tool_providers/hypergrid.rs @@ -1,6 +1,10 @@ -use crate::tool_providers::ToolProvider; -use crate::types::{SpiderState, Tool}; +use crate::tool_providers::{ToolExecutionCommand, ToolProvider}; +use crate::types::{ + HypergridConnection, HypergridMessage, HypergridMessageType, SpiderState, Tool, + ToolResponseContent, ToolResponseContentItem, +}; use serde_json::Value; +use std::time::Instant; pub struct HypergridToolProvider { server_id: String, @@ -68,18 +72,399 @@ impl ToolProvider for HypergridToolProvider { } } - fn execute_tool( + fn prepare_execution( &self, - _tool_name: &str, - _parameters: &Value, - _state: &mut SpiderState, - ) -> Result { - // This is a placeholder - the actual execution still happens in lib.rs - // The provider is responsible for tool registration and visibility logic only - Err("Tool execution should be handled by the main Spider implementation".to_string()) + tool_name: &str, + parameters: &Value, + state: &SpiderState, + ) -> Result { + match tool_name { + "hypergrid_authorize" => { + let url = parameters + .get("url") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing url parameter".to_string())? + .to_string(); + + let token = parameters + .get("token") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing token parameter".to_string())? + .to_string(); + + let client_id = parameters + .get("client_id") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing client_id parameter".to_string())? + .to_string(); + + let node = parameters + .get("node") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing node parameter".to_string())? + .to_string(); + + let name = parameters + .get("name") + .and_then(|v| v.as_str()) + .map(String::from); + + Ok(ToolExecutionCommand::HypergridAuthorize { + server_id: self.server_id.clone(), + url, + token, + client_id, + node, + name, + }) + } + "hypergrid_search" => { + // Check if configured + if !state.hypergrid_connections.contains_key(&self.server_id) { + return Err("Hypergrid not configured. Please use hypergrid_authorize first with your credentials.".to_string()); + } + + let query = parameters + .get("query") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing query parameter".to_string())? + .to_string(); + + Ok(ToolExecutionCommand::HypergridSearch { + server_id: self.server_id.clone(), + query, + }) + } + "hypergrid_call" => { + // Check if configured + if !state.hypergrid_connections.contains_key(&self.server_id) { + return Err("Hypergrid not configured. Please use hypergrid_authorize first with your credentials.".to_string()); + } + + let provider_id = parameters + .get("providerId") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing providerId parameter".to_string())? + .to_string(); + + let provider_name = parameters + .get("providerName") + .and_then(|v| v.as_str()) + .ok_or_else(|| "Missing providerName parameter".to_string())? + .to_string(); + + // Support both "callArgs" (old) and "arguments" (new) parameter names + let call_args = parameters + .get("callArgs") + .or_else(|| parameters.get("arguments")) + .and_then(|v| v.as_array()) + .ok_or_else(|| "Missing callArgs or arguments parameter".to_string())?; + + let args: Vec<(String, String)> = call_args + .iter() + .filter_map(|arg| { + arg.as_array().and_then(|pair| { + if pair.len() == 2 { + let key = pair[0].as_str()?.to_string(); + let value = pair[1].as_str()?.to_string(); + Some((key, value)) + } else { + None + } + }) + }) + .collect(); + + Ok(ToolExecutionCommand::HypergridCall { + server_id: self.server_id.clone(), + provider_id, + provider_name, + call_args: args, + }) + } + _ => Err(format!("Unknown hypergrid tool: {}", tool_name)), + } } fn get_provider_id(&self) -> &str { &self.server_id } } + +// Extension trait for hypergrid operations +pub trait HypergridExt { + async fn execute_hypergrid_authorize_impl( + &mut self, + server_id: String, + url: String, + token: String, + client_id: String, + node: String, + name: Option, + ) -> Result; + async fn execute_hypergrid_search_impl( + &mut self, + server_id: String, + query: String, + ) -> Result; + async fn execute_hypergrid_call_impl( + &mut self, + server_id: String, + provider_id: String, + provider_name: String, + call_args: Vec<(String, String)>, + ) -> Result; + async fn test_hypergrid_connection( + &self, + url: &str, + token: &str, + client_id: &str, + ) -> Result<(), String>; + async fn call_hypergrid_api( + &self, + url: &str, + token: &str, + client_id: &str, + message: &HypergridMessage, + ) -> Result; +} + +impl HypergridExt for SpiderState { + async fn execute_hypergrid_authorize_impl( + &mut self, + server_id: String, + url: String, + token: String, + client_id: String, + node: String, + name: Option, + ) -> Result { + println!( + "Spider: hypergrid_authorize called for server_id: {}", + server_id + ); + println!("Spider: Authorizing hypergrid with:"); + println!(" - URL: {}", url); + println!(" - Token: {}...", &token[..token.len().min(20)]); + println!(" - Client ID: {}", client_id); + println!(" - Node: {}", node); + if let Some(ref n) = name { + println!(" - Name: {}", n); + } + + // Test new connection + println!("Spider: Testing hypergrid connection..."); + self.test_hypergrid_connection(&url, &token, &client_id) + .await?; + println!("Spider: Connection test successful!"); + + // Create or update the hypergrid connection + let hypergrid_conn = HypergridConnection { + server_id: server_id.clone(), + url: url.clone(), + token: token.clone(), + client_id: client_id.clone(), + node: node.clone(), + last_retry: Instant::now(), + retry_count: 0, + connected: true, + }; + + self.hypergrid_connections + .insert(server_id.clone(), hypergrid_conn); + println!("Spider: Stored hypergrid connection in memory"); + + // Update transport config + if let Some(server) = self.mcp_servers.iter_mut().find(|s| s.id == server_id) { + println!("Spider: Updating server '{}' transport config", server.name); + server.transport.url = Some(url.clone()); + server.transport.hypergrid_token = Some(token.clone()); + server.transport.hypergrid_client_id = Some(client_id.clone()); + server.transport.hypergrid_node = Some(node.clone()); + println!("Spider: Server transport config updated successfully"); + println!("Spider: State should auto-save due to SaveOptions::OnDiff"); + } else { + println!( + "Spider: WARNING - Could not find server with id: {}", + server_id + ); + } + + Ok(serde_json::to_value(ToolResponseContent { + content: vec![ToolResponseContentItem { + content_type: "text".to_string(), + text: format!("✅ Successfully authorized! Hypergrid is now configured with:\n- Node: {}\n- Client ID: {}\n- URL: {}", node, client_id, url), + }], + }) + .map_err(|e| format!("Failed to serialize response: {}", e))?) + } + + async fn execute_hypergrid_search_impl( + &mut self, + server_id: String, + query: String, + ) -> Result { + let hypergrid_conn = self.hypergrid_connections.get(&server_id) + .ok_or_else(|| "Hypergrid not configured. Please use hypergrid_authorize first with your credentials.".to_string())?; + + let response = self + .call_hypergrid_api( + &hypergrid_conn.url, + &hypergrid_conn.token, + &hypergrid_conn.client_id, + &HypergridMessage { + request: HypergridMessageType::SearchRegistry(query), + }, + ) + .await?; + + Ok(serde_json::to_value(ToolResponseContent { + content: vec![ToolResponseContentItem { + content_type: "text".to_string(), + text: response, + }], + }) + .map_err(|e| format!("Failed to serialize response: {}", e))?) + } + + async fn execute_hypergrid_call_impl( + &mut self, + server_id: String, + provider_id: String, + provider_name: String, + call_args: Vec<(String, String)>, + ) -> Result { + let hypergrid_conn = self.hypergrid_connections.get(&server_id) + .ok_or_else(|| "Hypergrid not configured. Please use hypergrid_authorize first with your credentials.".to_string())?; + + let response = self + .call_hypergrid_api( + &hypergrid_conn.url, + &hypergrid_conn.token, + &hypergrid_conn.client_id, + &HypergridMessage { + request: HypergridMessageType::CallProvider { + provider_id, + provider_name, + arguments: call_args, + }, + }, + ) + .await?; + + Ok(serde_json::to_value(ToolResponseContent { + content: vec![ToolResponseContentItem { + content_type: "text".to_string(), + text: response, + }], + }) + .map_err(|e| format!("Failed to serialize response: {}", e))?) + } + + async fn test_hypergrid_connection( + &self, + url: &str, + token: &str, + client_id: &str, + ) -> Result<(), String> { + use hyperware_process_lib::http::client::send_request_await_response; + use hyperware_process_lib::http::Method; + use std::collections::HashMap; + + println!( + "Spider: test_hypergrid_connection - Testing connection to {}", + url + ); + + let test_message = HypergridMessage { + request: HypergridMessageType::SearchRegistry("test".to_string()), + }; + + let body = serde_json::to_string(&test_message) + .map_err(|e| format!("Failed to serialize test message: {}", e))?; + + let mut headers = HashMap::new(); + headers.insert("Content-Type".to_string(), "application/json".to_string()); + headers.insert("X-Auth-Token".to_string(), token.to_string()); + headers.insert("X-Client-Id".to_string(), client_id.to_string()); + + let parsed_url = url::Url::parse(url).map_err(|e| format!("Invalid URL: {}", e))?; + + println!("Spider: test_hypergrid_connection - Sending test request..."); + let response = send_request_await_response( + Method::POST, + parsed_url, + Some(headers), + 30000, + body.into_bytes(), + ) + .await + .map_err(|e| { + println!( + "Spider: test_hypergrid_connection - Request failed: {:?}", + e + ); + format!("Connection test failed: {:?}", e) + })?; + + if !response.status().is_success() { + let error_text = String::from_utf8_lossy(response.body()); + println!( + "Spider: test_hypergrid_connection - Server returned error: {}", + error_text + ); + return Err(format!( + "Hypergrid server error (status {}): {}", + response.status(), + error_text + )); + } + + println!("Spider: test_hypergrid_connection - Connection test successful!"); + Ok(()) + } + + async fn call_hypergrid_api( + &self, + url: &str, + token: &str, + client_id: &str, + message: &HypergridMessage, + ) -> Result { + use hyperware_process_lib::http::client::send_request_await_response; + use hyperware_process_lib::http::Method; + use std::collections::HashMap; + + let body = serde_json::to_string(message) + .map_err(|e| format!("Failed to serialize message: {}", e))?; + + let mut headers = HashMap::new(); + headers.insert("Content-Type".to_string(), "application/json".to_string()); + headers.insert("X-Auth-Token".to_string(), token.to_string()); + headers.insert("X-Client-Id".to_string(), client_id.to_string()); + + let parsed_url = url::Url::parse(url).map_err(|e| format!("Invalid URL: {}", e))?; + + let response = send_request_await_response( + Method::POST, + parsed_url, + Some(headers), + 30000, + body.into_bytes(), + ) + .await + .map_err(|e| format!("API call failed: {:?}", e))?; + + if !response.status().is_success() { + let error_text = String::from_utf8_lossy(response.body()); + return Err(format!( + "Hypergrid API error (status {}): {}", + response.status(), + error_text + )); + } + + let response_text = String::from_utf8_lossy(response.body()).to_string(); + Ok(response_text) + } +} diff --git a/hyperdrive/packages/spider/spider/src/tool_providers/mod.rs b/hyperdrive/packages/spider/spider/src/tool_providers/mod.rs index 51a1ed305..9710b302d 100644 --- a/hyperdrive/packages/spider/spider/src/tool_providers/mod.rs +++ b/hyperdrive/packages/spider/spider/src/tool_providers/mod.rs @@ -1,19 +1,67 @@ +pub mod build_container; pub mod hypergrid; use crate::types::{SpiderState, Tool}; use serde_json::Value; +pub enum ToolExecutionCommand { + // Build container commands + InitBuildContainer { + metadata: Option, + }, + LoadProject { + project_uuid: Option, + name: String, // Now required + initial_zip: Option, + channel_id: Option, + }, + StartPackage { + channel_id: u32, + package_dir: String, + }, + Persist { + channel_id: u32, + directories: Vec, + }, + DoneBuildContainer { + metadata: Option, + channel_id: Option, + }, + GetProjects, + // Hypergrid commands + HypergridAuthorize { + server_id: String, + url: String, + token: String, + client_id: String, + node: String, + name: Option, + }, + HypergridSearch { + server_id: String, + query: String, + }, + HypergridCall { + server_id: String, + provider_id: String, + provider_name: String, + call_args: Vec<(String, String)>, + }, + // Direct result (for synchronous operations) + DirectResult(Result), +} + pub trait ToolProvider: Send + Sync { fn get_tools(&self, state: &SpiderState) -> Vec; fn should_include_tool(&self, tool_name: &str, state: &SpiderState) -> bool; - fn execute_tool( + fn prepare_execution( &self, tool_name: &str, parameters: &Value, - state: &mut SpiderState, - ) -> Result; + state: &SpiderState, + ) -> Result; fn get_provider_id(&self) -> &str; } @@ -52,9 +100,13 @@ impl ToolProviderRegistry { tools } - pub fn find_provider_for_tool(&self, tool_name: &str) -> Option<&dyn ToolProvider> { + pub fn find_provider_for_tool( + &self, + tool_name: &str, + state: &SpiderState, + ) -> Option<&dyn ToolProvider> { for provider in &self.providers { - let tools = provider.get_tools(&SpiderState::default()); + let tools = provider.get_tools(state); if tools.iter().any(|t| t.name == tool_name) { return Some(provider.as_ref()); } diff --git a/hyperdrive/packages/spider/spider/src/types.rs b/hyperdrive/packages/spider/spider/src/types.rs index b8e507c1d..4dd98f117 100644 --- a/hyperdrive/packages/spider/spider/src/types.rs +++ b/hyperdrive/packages/spider/spider/src/types.rs @@ -7,6 +7,14 @@ use serde_json::Value; use crate::tool_providers::ToolProviderRegistry; +fn default_empty_string() -> String { + String::new() +} + +fn default_project_mapping() -> HashMap> { + HashMap::new() +} + #[derive(Default, Serialize, Deserialize)] pub struct SpiderState { pub api_keys: Vec<(String, ApiKey)>, @@ -16,6 +24,12 @@ pub struct SpiderState { pub default_llm_provider: String, pub max_tokens: u32, pub temperature: f32, + #[serde(default = "default_empty_string")] + pub build_container_ws_uri: String, + #[serde(default = "default_empty_string")] + pub build_container_api_key: String, + #[serde(default = "default_project_mapping")] + pub project_name_to_uuids: HashMap>, // project name -> list of UUIDs #[serde(skip)] pub ws_connections: HashMap, // channel_id -> connection info #[serde(skip)] @@ -300,6 +314,10 @@ pub(crate) struct UpdateConfigRequest { #[serde(rename = "maxTokens")] pub(crate) max_tokens: Option, pub(crate) temperature: Option, + #[serde(rename = "buildContainerWsUri")] + pub(crate) build_container_ws_uri: Option, + #[serde(rename = "buildContainerApiKey")] + pub(crate) build_container_api_key: Option, #[serde(rename = "authKey")] pub(crate) auth_key: String, } @@ -334,6 +352,10 @@ pub(crate) struct ConfigResponse { #[serde(rename = "maxTokens")] pub(crate) max_tokens: u32, pub(crate) temperature: f32, + #[serde(rename = "buildContainerWsUri")] + pub(crate) build_container_ws_uri: String, + #[serde(rename = "buildContainerApiKey")] + pub(crate) build_container_api_key: String, } #[derive(Serialize, Deserialize, Debug, PartialEq)] @@ -563,3 +585,88 @@ pub(crate) struct OAuthRefreshRequest { #[serde(rename = "refreshToken")] pub(crate) refresh_token: String, } + +// Tool response types +#[derive(Serialize, Deserialize, Debug, Clone)] +pub(crate) struct ToolResponseContent { + pub(crate) content: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub(crate) struct ToolResponseContentItem { + #[serde(rename = "type")] + pub(crate) content_type: String, + pub(crate) text: String, +} + +// Error response types +#[derive(Serialize, Deserialize, Debug, Clone)] +pub(crate) struct ErrorResponse { + pub(crate) error: Value, +} + +// OAuth request types +#[derive(Serialize, Deserialize, Debug)] +pub(crate) struct OAuthCodeExchangeRequest { + pub(crate) code: String, + pub(crate) state: String, + pub(crate) grant_type: String, + pub(crate) client_id: String, + pub(crate) redirect_uri: String, + pub(crate) code_verifier: String, +} + +#[derive(Serialize, Deserialize, Debug)] +pub(crate) struct OAuthRefreshTokenRequest { + pub(crate) grant_type: String, + pub(crate) refresh_token: String, + pub(crate) client_id: String, +} + +// Build container request/response types +#[derive(Serialize, Deserialize, Debug)] +pub(crate) struct BuildContainerRequest { + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) metadata: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +pub(crate) struct SpiderAuthRequest { + pub(crate) jsonrpc: String, + pub(crate) method: String, + pub(crate) params: SpiderAuthParams, + pub(crate) id: String, +} + +#[derive(Serialize, Deserialize, Debug)] +pub(crate) struct SpiderAuthParams { + pub(crate) api_key: String, +} + +#[derive(Serialize, Deserialize, Debug)] +pub(crate) struct LoadProjectParams { + pub(crate) project_uuid: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) initial_zip: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +pub(crate) struct StartPackageParams { + pub(crate) package_dir: String, +} + +#[derive(Serialize, Deserialize, Debug)] +pub(crate) struct PersistParams { + pub(crate) directories: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +pub(crate) struct InitializeParams { + #[serde(rename = "protocolVersion")] + pub(crate) protocol_version: String, + #[serde(rename = "clientInfo")] + pub(crate) client_info: McpClientInfo, + pub(crate) capabilities: McpCapabilities, +} diff --git a/hyperdrive/packages/spider/ui/src/App.css b/hyperdrive/packages/spider/ui/src/App.css index 511a85c12..5da5333ed 100644 --- a/hyperdrive/packages/spider/ui/src/App.css +++ b/hyperdrive/packages/spider/ui/src/App.css @@ -309,6 +309,14 @@ html, body, #root { background: rgba(255, 255, 255, 0.08); } +.form-help-text { + display: block; + margin-top: 0.25rem; + font-size: 0.85rem; + color: #888; + font-style: italic; +} + .api-key-form, .spider-key-form, .mcp-server-form, diff --git a/hyperdrive/packages/spider/ui/src/components/Chat.tsx b/hyperdrive/packages/spider/ui/src/components/Chat.tsx index 6c1ad09d4..4fc5f1fed 100644 --- a/hyperdrive/packages/spider/ui/src/components/Chat.tsx +++ b/hyperdrive/packages/spider/ui/src/components/Chat.tsx @@ -20,11 +20,50 @@ function ToolCallModal({ toolCall, toolResult, onClose }: { onClose: () => void; }) { const copyToClipboard = (text: string) => { - navigator.clipboard.writeText(text).then(() => { - // Could add a toast notification here - }).catch(err => { - console.error('Failed to copy:', err); - }); + // Check if clipboard API is available + if (navigator.clipboard && navigator.clipboard.writeText) { + navigator.clipboard.writeText(text).then(() => { + // Could add a toast notification here + }).catch(err => { + console.error('Failed to copy:', err); + // Fallback to legacy method + fallbackCopyToClipboard(text); + }); + } else { + // Use fallback method + fallbackCopyToClipboard(text); + } + }; + + const fallbackCopyToClipboard = (text: string) => { + // Create a temporary textarea element + const textarea = document.createElement('textarea'); + textarea.value = text; + textarea.style.position = 'fixed'; + textarea.style.top = '0'; + textarea.style.left = '0'; + textarea.style.width = '2em'; + textarea.style.height = '2em'; + textarea.style.padding = '0'; + textarea.style.border = 'none'; + textarea.style.outline = 'none'; + textarea.style.boxShadow = 'none'; + textarea.style.background = 'transparent'; + + document.body.appendChild(textarea); + textarea.focus(); + textarea.select(); + + try { + const successful = document.execCommand('copy'); + if (!successful) { + console.error('Fallback copy failed'); + } + } catch (err) { + console.error('Fallback copy error:', err); + } + + document.body.removeChild(textarea); }; return ( @@ -38,7 +77,7 @@ function ToolCallModal({ toolCall, toolResult, onClose }: {

Tool Call

- + + {showSelfHosting && ( +
+
+ + setBuildContainerWsUri(e.target.value)} + placeholder="ws://localhost:8091" + /> + + WebSocket URI for your self-hosted build container + +
+ +
+ + setBuildContainerApiKey(e.target.value)} + placeholder="Enter API key" + /> + + API key for authenticating with your self-hosted build container + +
+
+ )} +
+
); -} \ No newline at end of file +} diff --git a/hyperdrive/packages/spider/ui/src/store/spider.ts b/hyperdrive/packages/spider/ui/src/store/spider.ts index 7c15b96a5..b28892063 100644 --- a/hyperdrive/packages/spider/ui/src/store/spider.ts +++ b/hyperdrive/packages/spider/ui/src/store/spider.ts @@ -66,6 +66,8 @@ interface SpiderConfig { defaultLlmProvider: string; maxTokens: number; temperature: number; + buildContainerWsUri: string; + buildContainerApiKey: string; } interface SpiderStore { @@ -163,6 +165,8 @@ export const useSpiderStore = create((set, get) => ({ defaultLlmProvider: 'anthropic', maxTokens: 4096, temperature: 0.7, + buildContainerWsUri: '', + buildContainerApiKey: '', }, isLoading: false, error: null, diff --git a/hyperdrive/packages/spider/ui/src/utils/api.ts b/hyperdrive/packages/spider/ui/src/utils/api.ts index 834aa2fdc..9a848bb5d 100644 --- a/hyperdrive/packages/spider/ui/src/utils/api.ts +++ b/hyperdrive/packages/spider/ui/src/utils/api.ts @@ -158,6 +158,8 @@ export async function updateConfig(config: Partial): Promise