fix(walker): handle large directories

crates/forge_api/src/forge_api.rs

@@ -4,7 +4,7 @@ use std::sync::Arc;
 use anyhow::Result;
 use forge_app::{
     ConversationService, EnvironmentService, FileDiscoveryService, ForgeApp, McpConfigManager,
-    ProviderService, Services, WorkflowService,
+    ProviderService, Services, Walker, WorkflowService,
 };
 use forge_domain::*;
 use forge_infra::ForgeInfra;
@@ -35,7 +35,9 @@ impl ForgeAPI<ForgeServices<ForgeInfra>, ForgeInfra> {
 #[async_trait::async_trait]
 impl<A: Services, F: CommandInfra> API for ForgeAPI<A, F> {
     async fn discover(&self) -> Result<Vec<File>> {
-        self.app.collect(None).await
+        let environment = self.app.get_environment();
+        let config = Walker::unlimited().cwd(environment.cwd);
+        self.app.collect_files(config).await
     }
 
     async fn tools(&self) -> anyhow::Result<Vec<ToolDefinition>> {

crates/forge_app/src/app.rs

@@ -11,7 +11,7 @@ use crate::services::TemplateService;
 use crate::tool_registry::ToolRegistry;
 use crate::{
     AttachmentService, ConversationService, EnvironmentService, FileDiscoveryService,
-    ProviderService, Services, WorkflowService,
+    ProviderService, Services, Walker, WorkflowService,
 };
 
 /// ForgeApp handles the core chat functionality by orchestrating various
@@ -48,20 +48,23 @@ impl<S: Services> ForgeApp<S> {
         let models = services.models().await?;
 
         // Discover files using the discovery service
-        let workflow = WorkflowService::read_workflow(services.as_ref(), None)
-            .await
-            .unwrap_or_default();
+        let workflow = services.read_merged(None).await.unwrap_or_default();
         let max_depth = workflow.max_walker_depth;
+        let environment = services.get_environment();
+
+        let mut walker = Walker::conservative().cwd(environment.cwd.clone());
+
+        if let Some(depth) = max_depth {
+            walker = walker.max_depth(depth);
+        };
+
         let files = services
-            .collect(max_depth)
+            .collect_files(walker)
             .await?
             .into_iter()
             .map(|f| f.path)
             .collect::<Vec<_>>();
 
-        // Get environment for orchestrator creation
-        let environment = services.get_environment();
-
         // Register templates using workflow path or environment fallback
         let template_path = workflow
             .templates

crates/forge_app/src/lib.rs

@@ -14,7 +14,9 @@ mod tool_executor;
 mod tool_registry;
 mod truncation;
 mod utils;
+mod walker;
 
 pub use app::*;
 pub use fmt_output::*;
 pub use services::*;
+pub use walker::*;

crates/forge_app/src/services.rs

@@ -7,6 +7,8 @@ use forge_domain::{
 };
 use merge::Merge;
 
+use crate::Walker;
+
 #[derive(Debug)]
 pub struct ShellOutput {
     pub output: CommandOutput,
@@ -180,7 +182,7 @@ pub trait WorkflowService {
 
 #[async_trait::async_trait]
 pub trait FileDiscoveryService: Send + Sync {
-    async fn collect(&self, max_depth: Option<usize>) -> anyhow::Result<Vec<File>>;
+    async fn collect_files(&self, config: Walker) -> anyhow::Result<Vec<File>>;
 }
 
 #[async_trait::async_trait]
@@ -425,8 +427,8 @@ impl<I: Services> WorkflowService for I {
 
 #[async_trait::async_trait]
 impl<I: Services> FileDiscoveryService for I {
-    async fn collect(&self, max_depth: Option<usize>) -> anyhow::Result<Vec<File>> {
-        self.file_discovery_service().collect(max_depth).await
+    async fn collect_files(&self, config: Walker) -> anyhow::Result<Vec<File>> {
+        self.file_discovery_service().collect_files(config).await
     }
 }
 

crates/forge_app/src/walker.rs

@@ -0,0 +1,75 @@
+use std::path::PathBuf;
+
+use derive_setters::Setters;
+
+/// Configuration for filesystem walking operations
+#[derive(Debug, Clone, Setters)]
+#[setters(strip_option, into)]
+pub struct Walker {
+    /// Base directory to start walking from
+    pub cwd: PathBuf,
+    /// Maximum depth of directory traversal (None for unlimited)
+    pub max_depth: Option<usize>,
+    /// Maximum number of entries per directory (None for unlimited)
+    pub max_breadth: Option<usize>,
+    /// Maximum size of individual files to process (None for unlimited)
+    pub max_file_size: Option<u64>,
+    /// Maximum number of files to process in total (None for unlimited)
+    pub max_files: Option<usize>,
+    /// Maximum total size of all files combined (None for unlimited)
+    pub max_total_size: Option<u64>,
+    /// Whether to skip binary files
+    pub skip_binary: bool,
+}
+
+impl Walker {
+    /// Creates a new WalkerConfig with conservative default limits
+    pub fn conservative() -> Self {
+        Self {
+            cwd: PathBuf::new(),
+            max_depth: Some(5),
+            max_breadth: Some(10),
+            max_file_size: Some(1024 * 1024), // 1MB
+            max_files: Some(100),
+            max_total_size: Some(10 * 1024 * 1024), // 10MB
+            skip_binary: true,
+        }
+    }
+
+    /// Creates a new WalkerConfig with no limits (use with caution)
+    pub fn unlimited() -> Self {
+        Self {
+            cwd: PathBuf::new(),
+            max_depth: None,
+            max_breadth: None,
+            max_file_size: None,
+            max_files: None,
+            max_total_size: None,
+            skip_binary: false,
+        }
+    }
+}
+
+impl Default for Walker {
+    fn default() -> Self {
+        Self::conservative()
+    }
+}
+
+/// Represents a file or directory found during filesystem traversal
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct WalkedFile {
+    /// Relative path from the base directory
+    pub path: String,
+    /// File name (None for root directory)
+    pub file_name: Option<String>,
+    /// Size in bytes
+    pub size: u64,
+}
+
+impl WalkedFile {
+    /// Returns true if this represents a directory
+    pub fn is_dir(&self) -> bool {
+        self.path.ends_with('/')
+    }
+}

crates/forge_infra/src/forge_infra.rs

@@ -205,10 +205,7 @@ impl McpServerInfra for ForgeInfra {
 
 #[async_trait::async_trait]
 impl WalkerInfra for ForgeInfra {
-    async fn walk(
-        &self,
-        config: forge_services::WalkerConfig,
-    ) -> anyhow::Result<Vec<forge_services::WalkedFile>> {
+    async fn walk(&self, config: forge_app::Walker) -> anyhow::Result<Vec<forge_app::WalkedFile>> {
         self.walker_service.walk(config).await
     }
 }

crates/forge_infra/src/walker.rs

@@ -1,5 +1,5 @@
 use anyhow::Result;
-use forge_services::{WalkedFile, WalkerConfig};
+use forge_app::{WalkedFile, Walker};
 
 pub struct ForgeWalkerService;
 
@@ -8,7 +8,7 @@ impl ForgeWalkerService {
         Self
     }
 
-    pub async fn walk(&self, config: WalkerConfig) -> Result<Vec<WalkedFile>> {
+    pub async fn walk(&self, config: Walker) -> Result<Vec<WalkedFile>> {
         // Convert domain config to forge_walker config
         let mut walker = if config.max_depth.is_none()
             && config.max_breadth.is_none()
@@ -65,7 +65,7 @@ mod tests {
         std::fs::write(fixture.path().join("test.txt"), "test content").unwrap();
 
         let service = ForgeWalkerService::new();
-        let config = WalkerConfig::conservative().cwd(fixture.path().to_path_buf());
+        let config = Walker::conservative().cwd(fixture.path().to_path_buf());
 
         let actual = service.walk(config).await.unwrap();
 
@@ -80,7 +80,7 @@ mod tests {
         std::fs::write(fixture.path().join("test.txt"), "test content").unwrap();
 
         let service = ForgeWalkerService::new();
-        let config = WalkerConfig::unlimited().cwd(fixture.path().to_path_buf());
+        let config = Walker::unlimited().cwd(fixture.path().to_path_buf());
 
         let actual = service.walk(config).await.unwrap();
 

crates/forge_services/src/discovery.rs

@@ -1,10 +1,10 @@
 use std::sync::Arc;
 
 use anyhow::Result;
-use forge_app::FileDiscoveryService;
+use forge_app::{FileDiscoveryService, Walker};
 use forge_domain::File;
 
-use crate::{EnvironmentInfra, WalkerConfig, WalkerInfra};
+use crate::{EnvironmentInfra, WalkerInfra};
 
 pub struct ForgeDiscoveryService<F> {
     service: Arc<F>,
@@ -17,15 +17,7 @@ impl<F> ForgeDiscoveryService<F> {
 }
 
 impl<F: EnvironmentInfra + WalkerInfra> ForgeDiscoveryService<F> {
-    async fn discover_with_depth(&self, max_depth: Option<usize>) -> Result<Vec<File>> {
-        let cwd = self.service.get_environment().cwd.clone();
-
-        let config = if let Some(depth) = max_depth {
-            WalkerConfig::unlimited().cwd(cwd).max_depth(depth)
-        } else {
-            WalkerConfig::unlimited().cwd(cwd)
-        };
-
+    async fn discover_with_config(&self, config: Walker) -> Result<Vec<File>> {
         let files = self.service.walk(config).await?;
         Ok(files
             .into_iter()
@@ -38,7 +30,7 @@ impl<F: EnvironmentInfra + WalkerInfra> ForgeDiscoveryService<F> {
 impl<F: EnvironmentInfra + WalkerInfra + Send + Sync> FileDiscoveryService
     for ForgeDiscoveryService<F>
 {
-    async fn collect(&self, max_depth: Option<usize>) -> Result<Vec<File>> {
-        self.discover_with_depth(max_depth).await
+    async fn collect_files(&self, config: Walker) -> Result<Vec<File>> {
+        self.discover_with_config(config).await
     }
 }

crates/forge_services/src/infra.rs

@@ -2,83 +2,12 @@ use std::path::{Path, PathBuf};
 
 use anyhow::Result;
 use bytes::Bytes;
-use derive_setters::Setters;
+use forge_app::{WalkedFile, Walker};
 use forge_domain::{
     CommandOutput, Environment, McpServerConfig, ToolDefinition, ToolName, ToolOutput,
 };
 use forge_snaps::Snapshot;
 
-/// Configuration for filesystem walking operations
-#[derive(Debug, Clone, Setters)]
-#[setters(strip_option, into)]
-pub struct WalkerConfig {
-    /// Base directory to start walking from
-    pub cwd: PathBuf,
-    /// Maximum depth of directory traversal (None for unlimited)
-    pub max_depth: Option<usize>,
-    /// Maximum number of entries per directory (None for unlimited)
-    pub max_breadth: Option<usize>,
-    /// Maximum size of individual files to process (None for unlimited)
-    pub max_file_size: Option<u64>,
-    /// Maximum number of files to process in total (None for unlimited)
-    pub max_files: Option<usize>,
-    /// Maximum total size of all files combined (None for unlimited)
-    pub max_total_size: Option<u64>,
-    /// Whether to skip binary files
-    pub skip_binary: bool,
-}
-
-impl WalkerConfig {
-    /// Creates a new WalkerConfig with conservative default limits
-    pub fn conservative() -> Self {
-        Self {
-            cwd: PathBuf::new(),
-            max_depth: Some(5),
-            max_breadth: Some(10),
-            max_file_size: Some(1024 * 1024), // 1MB
-            max_files: Some(100),
-            max_total_size: Some(10 * 1024 * 1024), // 10MB
-            skip_binary: true,
-        }
-    }
-
-    /// Creates a new WalkerConfig with no limits (use with caution)
-    pub fn unlimited() -> Self {
-        Self {
-            cwd: PathBuf::new(),
-            max_depth: None,
-            max_breadth: None,
-            max_file_size: None,
-            max_files: None,
-            max_total_size: None,
-            skip_binary: false,
-        }
-    }
-}
-
-impl Default for WalkerConfig {
-    fn default() -> Self {
-        Self::conservative()
-    }
-}
-/// Represents a file or directory found during filesystem traversal
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct WalkedFile {
-    /// Relative path from the base directory
-    pub path: String,
-    /// File name (None for root directory)
-    pub file_name: Option<String>,
-    /// Size in bytes
-    pub size: u64,
-}
-
-impl WalkedFile {
-    /// Returns true if this represents a directory
-    pub fn is_dir(&self) -> bool {
-        self.path.ends_with('/')
-    }
-}
-
 pub trait EnvironmentInfra: Send + Sync {
     fn get_environment(&self) -> Environment;
 }
@@ -227,5 +156,5 @@ pub trait McpServerInfra: Send + Sync + 'static {
 pub trait WalkerInfra: Send + Sync {
     /// Walks the filesystem starting from the given directory with the
     /// specified configuration
-    async fn walk(&self, config: WalkerConfig) -> anyhow::Result<Vec<WalkedFile>>;
+    async fn walk(&self, config: Walker) -> anyhow::Result<Vec<WalkedFile>>;
 }

crates/forge_services/src/tool_services/fs_search.rs

@@ -3,12 +3,11 @@ use std::path::Path;
 use std::sync::Arc;
 
 use anyhow::Context;
-use forge_app::{FsSearchService, Match, MatchResult, SearchResult};
+use forge_app::{FsSearchService, Match, MatchResult, SearchResult, Walker};
 use grep_searcher::sinks::UTF8;
 
 use crate::infra::WalkerInfra;
 use crate::utils::assert_absolute_path;
-use crate::WalkerConfig;
 
 // Using FSSearchInput from forge_domain
 
@@ -168,7 +167,7 @@ impl<W: WalkerInfra> ForgeFsSearch<W> {
             #[allow(unused_mut)]
             let mut paths = self
                 .walker
-                .walk(WalkerConfig::unlimited().cwd(dir.to_path_buf()))
+                .walk(Walker::unlimited().cwd(dir.to_path_buf()))
                 .await
                 .with_context(|| format!("Failed to walk directory '{}'", dir.display()))?
                 .into_iter()
@@ -191,18 +190,18 @@ impl<W: WalkerInfra> ForgeFsSearch<W> {
 mod test {
     use std::sync::Arc;
 
+    use forge_app::{WalkedFile, Walker};
     use tokio::fs;
 
     use super::*;
-    use crate::infra::WalkedFile;
     use crate::utils::TempDir;
 
     // Mock WalkerInfra for testing
     struct MockInfra;
 
     #[async_trait::async_trait]
     impl WalkerInfra for MockInfra {
-        async fn walk(&self, config: crate::WalkerConfig) -> anyhow::Result<Vec<WalkedFile>> {
+        async fn walk(&self, config: Walker) -> anyhow::Result<Vec<WalkedFile>> {
             // Simple mock that just returns files in the directory
             let mut files = Vec::new();
             let metadata = tokio::fs::metadata(&config.cwd).await?;