feat: tool to fetch markdown content of an url

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["crates/core", "crates/arey", "crates/tools-search", "crates/mcp"]
+members = ["crates/core", "crates/arey", "crates/tools-search", "crates/tools-fetch", "crates/mcp"]
 resolver = "2"
 
 [workspace.package]
@@ -45,6 +45,7 @@ tokio = { version = "1.51", features = [
     "sync",
     "signal",
     "time",
+    "process",
 ] }
 tokio-stream = "0.1.18"
 tracing = "0.1"

crates/arey/Cargo.toml

@@ -11,6 +11,7 @@ path = "src/main.rs"
 arey-core = { path = "../core", version = "*" }
 arey-mcp = { path = "../mcp" }
 arey-tools-search = { path = "../tools-search" }
+arey-tools-fetch = { path = "../tools-fetch" }
 anyhow.workspace = true
 async-stream.workspace = true
 chrono.workspace = true

crates/arey/src/cli/chat/commands.rs

@@ -560,7 +560,9 @@ pub fn parse_command_line(line: &str) -> Vec<String> {
     };
 
     // Special handling for commands that accept multi-word arguments
-    if !args.is_empty() && (args[0] == "/system" || args[0] == "/sys" || args[0] == "/tool") {
+    // Only /system and /sys need to join all args into one (for prompts)
+    // /tool should keep args separate for multiple tool names
+    if !args.is_empty() && (args[0] == "/system" || args[0] == "/sys") {
         if args.len() > 1 {
             // Join all arguments after the command
             let mut processed = vec![args[0].clone()];
@@ -637,52 +639,70 @@ fn format_message_block(messages: &[arey_core::completion::ChatMessage]) -> Resu
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::cli::chat::test_utils::{MockTool, create_test_config_with_custom_agent};
+    use crate::cli::chat::test_utils::{
+        AnotherMockTool, MockTool, create_test_config_with_custom_agent,
+    };
     use arey_core::completion::{ChatMessage, SenderType};
     use arey_core::registry::ToolRegistry;
     use arey_core::tools::{Tool, ToolCall};
     use std::sync::Arc;
     use tokio::sync::Mutex;
+    use yare::parameterized;
+
+    #[parameterized(
+        normal_spaces = { "/system you are an expert", Some("you are an expert") },
+        apostrophes = { "/system you're an expert", Some("you're an expert") },
+        quotes = { "/system \"quoted text\"", Some("quoted text") },
+        punctuation = { "/system hello, world!", Some("hello, world!") },
+        no_args = { "/system", None },
+        alias = { "/sys you're an expert", Some("you're an expert") },
+    )]
+    fn test_system_command_parsing(input: &str, expected_prompt: Option<&str>) {
+        let processed_args = parse_command_line(input);
+
+        let result = CliCommand::try_parse_from(processed_args);
+        assert!(result.is_ok(), "Failed to parse '{}': {:?}", input, result);
+
+        let cli_command = result.unwrap();
+        match cli_command.command {
+            Command::System { prompt } => {
+                assert_eq!(
+                    prompt,
+                    expected_prompt.map(|s| s.to_string()),
+                    "Failed for input: '{}'",
+                    input
+                );
+            }
+            _ => panic!(
+                "Expected System command for input: '{}', got {:?}",
+                input, cli_command.command
+            ),
+        }
+    }
 
-    #[test]
-    fn test_system_command_parsing_edge_cases() {
-        let test_cases = vec![
-            // Test normal case with spaces
-            ("/system you are an expert", Some("you are an expert")),
-            // Test apostrophes (the main issue)
-            ("/system you're an expert", Some("you're an expert")),
-            // Test quotes (should still work with shlex)
-            ("/system \"quoted text\"", Some("quoted text")),
-            // Test mixed punctuation
-            ("/system hello, world!", Some("hello, world!")),
-            // Test no arguments (should work)
-            ("/system", None),
-            // Test alias
-            ("/sys you're an expert", Some("you're an expert")),
-        ];
-
-        for (input, expected_prompt) in test_cases {
-            // Use the shared parsing function
-            let processed_args = parse_command_line(input);
-
-            let result = CliCommand::try_parse_from(processed_args);
-            assert!(result.is_ok(), "Failed to parse '{}': {:?}", input, result);
-
-            let cli_command = result.unwrap();
-            match cli_command.command {
-                Command::System { prompt } => {
-                    assert_eq!(
-                        prompt,
-                        expected_prompt.map(|s| s.to_string()),
-                        "Failed for input: '{}'",
-                        input
-                    );
-                }
-                _ => panic!(
-                    "Expected System command for input: '{}', got {:?}",
-                    input, cli_command.command
-                ),
+    #[parameterized(
+        single_tool = { "/tool search", vec!["search"] },
+        multiple_tools = { "/tool search fetch", vec!["search", "fetch"] },
+        multiple_tools_extra_spaces = { "/tool search  fetch", vec!["search", "fetch"] },
+        clear_command = { "/tool clear", vec!["clear"] },
+        no_arguments = { "/tool", vec![] },
+    )]
+    fn test_tool_command_parsing(input: &str, expected_names: Vec<&str>) {
+        let processed_args = parse_command_line(input);
+
+        let result = CliCommand::try_parse_from(processed_args);
+        assert!(result.is_ok(), "Failed to parse '{}': {:?}", input, result);
+
+        let cli_command = result.unwrap();
+        match cli_command.command {
+            Command::Tool { names } => {
+                let expected: Vec<String> = expected_names.iter().map(|s| s.to_string()).collect();
+                assert_eq!(names, expected, "Failed for input: '{}'", input);
             }
+            _ => panic!(
+                "Expected Tool command for input: '{}', got {:?}",
+                input, cli_command.command
+            ),
         }
     }
 
@@ -937,6 +957,40 @@ USER: Run tool
         Ok(())
     }
 
+    #[tokio::test]
+    async fn test_tool_command_set_multiple() -> Result<()> {
+        // Create Chat instance with multiple mock tools
+        let config = create_test_config_with_custom_agent()?;
+        let mock_tool1: Arc<dyn Tool> = Arc::new(MockTool);
+        let mock_tool2: Arc<dyn Tool> = Arc::new(AnotherMockTool);
+        let mut tool_registry = ToolRegistry::new();
+        tool_registry.register(mock_tool1)?;
+        tool_registry.register(mock_tool2)?;
+
+        let mut chat = Chat::new(&config, Some("test-model-1".to_string()), tool_registry)?;
+        chat.load_session().await?;
+        let chat_session = Arc::new(Mutex::new(chat));
+
+        // Test setting multiple tools
+        let set_tool_cmd = Command::Tool {
+            names: vec!["mock_tool".to_string(), "another_mock_tool".to_string()],
+        };
+        let result = set_tool_cmd.execute(chat_session.clone()).await?;
+        assert!(result, "execute should return true to continue REPL");
+
+        // Check that both tools are set
+        {
+            let chat_guard = chat_session.lock().await;
+            let tools = chat_guard.tools();
+            assert_eq!(tools.len(), 2);
+            let tool_names: Vec<String> = tools.iter().map(|t| t.name()).collect();
+            assert!(tool_names.contains(&"mock_tool".to_string()));
+            assert!(tool_names.contains(&"another_mock_tool".to_string()));
+        }
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn test_tool_command_clear() -> Result<()> {
         // Create Chat instance with a mock tool

crates/arey/src/cli/chat/test_utils.rs

@@ -241,3 +241,26 @@ impl Tool for MockTool {
         Ok(Value::String("mock tool output".to_string()))
     }
 }
+
+/// Another mock tool for testing multiple tools
+#[derive(Debug)]
+pub struct AnotherMockTool;
+
+#[async_trait]
+impl Tool for AnotherMockTool {
+    fn name(&self) -> String {
+        "another_mock_tool".to_string()
+    }
+
+    fn description(&self) -> String {
+        "Another mock tool for testing".to_string()
+    }
+
+    fn parameters(&self) -> Value {
+        Value::Object(serde_json::Map::new())
+    }
+
+    async fn execute(&self, _args: &Value) -> Result<Value, ToolError> {
+        Ok(Value::String("another mock tool output".to_string()))
+    }
+}

crates/arey/src/ext/mod.rs

@@ -8,8 +8,8 @@ use arey_core::{config::Config, registry::ToolRegistry, tools::Tool};
 /// Retrieves all available tools from the configuration.
 ///
 /// This function iterates through the tools defined in the provided configuration
-/// and initializes each one based on its name. Currently, only the "search" tool
-/// is supported.
+/// and initializes each one based on its name. Currently, "search" and "fetch" tools
+/// are supported. The fetch tool is always available.
 ///
 /// # Arguments
 ///
@@ -27,12 +27,19 @@ use arey_core::{config::Config, registry::ToolRegistry, tools::Tool};
 /// - An unknown tool name is found in the configuration.
 pub fn get_tools(config: &Config) -> Result<ToolRegistry> {
     let mut registry = ToolRegistry::new();
+
+    registry.register(Arc::new(arey_tools_fetch::FetchTool::new()))?;
+
     for (name, tool_config) in &config.tools {
         let tool: Arc<dyn Tool> = match name.as_str() {
             "search" => Arc::new(
                 arey_tools_search::SearchTool::from_config(tool_config)
                     .with_context(|| format!("Failed to initialize tool: {name}"))?,
             ),
+            "fetch" => {
+                tracing::debug!("Fetch tool already registered, skipping config");
+                continue;
+            }
             _ => return Err(anyhow!("Unknown tool in config: {}", name)),
         };
         registry.register(tool)?;

crates/core/Cargo.toml

@@ -41,7 +41,7 @@ llama-cpp-2 = "=0.1.143"
 [dev-dependencies]
 tempfile = { workspace = true, features = [] }
 wiremock = { workspace = true, features = [] }
-tokio = { version = "1.51", features = ["macros", "rt-multi-thread"] }
+tokio.workspace = true
 reqwest.workspace = true
 yare.workspace = true
 

crates/mcp/Cargo.toml

@@ -13,22 +13,15 @@ arey-core = { path = "../core" }
 rmcp = { version = "1.4.0", features = ["client", "transport-child-process", "transport-io"] }
 anyhow.workspace = true
 async-trait.workspace = true
-tokio = { version = "1.50", features = [
-    "macros",
-    "process",
-    "rt-multi-thread",
-    "sync",
-    "signal",
-    "time",
-] }
+tokio.workspace = true
 tracing.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 serde_yaml.workspace = true
 
 [dev-dependencies]
 arey-mcp = { path = ".", features = ["test_utils"] }
-tokio = { version = "1.50", features = ["macros", "test-util"] }
+tokio.workspace = true
 serde_yaml.workspace = true
 yare.workspace = true
 

crates/tools-fetch/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "arey-tools-fetch"
+version = "0.0.7"
+edition.workspace = true
+authors.workspace = true
+description = "Fetch tool for arey - extracts readable content from URLs"
+readme.workspace = true
+license.workspace = true
+
+[dependencies]
+arey-core = { path = "../core" }
+
+anyhow.workspace = true
+async-trait.workspace = true
+reqwest.workspace = true
+readabilityrs = "0.1"
+serde.workspace = true
+serde_json.workspace = true
+tracing.workspace = true
+
+[dev-dependencies]
+tokio.workspace = true
+wiremock.workspace = true