From f37981ec071d40faeefec2c262b64fcb6545a22e Mon Sep 17 00:00:00 2001 From: elasticdotventures Date: Sat, 5 Jul 2025 08:15:09 +0000 Subject: [PATCH 1/8] checkpoint 1, broken --- README.md | 57 +++++++++++++++-------------------- src/bin/cratedocs.rs | 35 ---------------------- src/tools/docs/docs.rs | 68 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 14f9a2d..c89b998 100644 --- a/README.md +++ b/README.md @@ -189,47 +189,38 @@ in `mcp_settings.json` ``` -## License - -MIT License -## MCP Tool: `list_crate_items` +### 4. `list_crate_items` -The `list_crate_items` tool enumerates all items in a specified Rust crate and version, optionally filtering by item type, visibility, or module path. This is useful for quickly exploring the structure of a crate, generating concise listings for LLMs, or programmatically analyzing crate APIs. +Enumerates all items in a specified Rust crate and version, optionally filtering by item type, visibility, or module path. Useful for exploring crate structure, generating concise listings for LLMs, or programmatically analyzing crate APIs. -### Usage +**Parameters:** +- `crate_name` (required): The name of the crate +- `version` (required): The version of the crate +- `item_type` (optional): Filter by item type (struct, enum, trait, fn, macro, mod) +- `visibility` (optional): Filter by visibility (pub, private) +- `module` (optional): Filter by module path (e.g., serde::de) -```sh -cargo run --bin cratedocs -- list-crate-items --crate-name serde --version 1.0.0 +**Example:** +```json +{ + "name": "list_crate_items", + "arguments": { + "crate_name": "serde", + "version": "1.0.0", + "item_type": "struct" + } +} ``` -#### With filters: - -- Filter by item type (e.g., struct, enum, trait, fn, macro, mod): - - ```sh - cargo run --bin cratedocs -- list-crate-items --crate-name serde --version 1.0.0 --item-type struct - ``` - -- Filter by visibility (e.g., pub, private): - - ```sh - cargo run --bin cratedocs -- list-crate-items --crate-name serde --version 1.0.0 --visibility pub - ``` - -- Filter by module path: - - ```sh - cargo run --bin cratedocs -- list-crate-items --crate-name serde --version 1.0.0 --module serde::de - ``` - -### Output - -The output is a concise, categorized list (JSON or markdown) showing each item's name, type, visibility, and module path. - -**Example (stub output):** +**Example Output (stub):** ``` Stub: list_crate_items for crate: serde, version: 1.0.0, filters: Some(ItemListFilters { item_type: Some("struct"), visibility: None, module: None }) ``` When implemented, the output will be a structured list of items matching the filters. + + +## License + +MIT License diff --git a/src/bin/cratedocs.rs b/src/bin/cratedocs.rs index 8b50a6e..02f1d7d 100644 --- a/src/bin/cratedocs.rs +++ b/src/bin/cratedocs.rs @@ -84,24 +84,6 @@ enum Commands { #[arg(short, long)] debug: bool, }, - /// List all items in a crate (using rust-analyzer) - ListCrateItems { - /// Crate name (e.g., serde) - #[arg(long)] - crate_name: String, - /// Crate version (e.g., 1.0.0) - #[arg(long)] - version: String, - /// Filter by item type (struct, enum, trait, fn, macro, mod) - #[arg(long)] - item_type: Option, - /// Filter by visibility (pub, private) - #[arg(long)] - visibility: Option, - /// Filter by module path (e.g., serde::de) - #[arg(long)] - module: Option, - }, } #[tokio::main] @@ -136,23 +118,6 @@ async fn main() -> Result<()> { max_tokens, debug }).await, - Commands::ListCrateItems { - crate_name, - version, - item_type, - visibility, - module, - } => { - use cratedocs_mcp::tools::item_list::{list_crate_items, ItemListFilters}; - let filters = ItemListFilters { - item_type, - visibility, - module, - }; - let result = list_crate_items(&crate_name, &version, Some(filters)).await?; - println!("{}", result); - Ok(()) - } } } diff --git a/src/tools/docs/docs.rs b/src/tools/docs/docs.rs index 486a9f5..710bfb6 100644 --- a/src/tools/docs/docs.rs +++ b/src/tools/docs/docs.rs @@ -1,3 +1,4 @@ +use crate::tools::item_list; use std::{future::Future, pin::Pin, sync::Arc}; use mcp_core::{ @@ -321,6 +322,36 @@ impl mcp_server::Router for DocRouter { "required": ["crate_name", "item_path"] }), ), + Tool::new( + "list_crate_items".to_string(), + "Enumerate all items in a Rust crate (optionally filtered by type, visibility, or module). Returns a concise, categorized list.".to_string(), + json!({ + "type": "object", + "properties": { + "crate_name": { + "type": "string", + "description": "The name of the crate" + }, + "version": { + "type": "string", + "description": "The version of the crate" + }, + "item_type": { + "type": "string", + "description": "Filter by item type (struct, enum, trait, fn, macro, mod)" + }, + "visibility": { + "type": "string", + "description": "Filter by visibility (pub, private)" + }, + "module": { + "type": "string", + "description": "Filter by module path (e.g., serde::de)" + } + }, + "required": ["crate_name", "version"] + }), + ), ] } @@ -386,6 +417,43 @@ impl mcp_server::Router for DocRouter { let doc = this.lookup_item(crate_name, item_path, version).await?; Ok(vec![Content::text(doc)]) } + "list_crate_items" => { + let crate_name = arguments + .get("crate_name") + .and_then(|v| v.as_str()) + .ok_or_else(|| ToolError::InvalidParameters("crate_name is required".to_string()))? + .to_string(); + let version = arguments + .get("version") + .and_then(|v| v.as_str()) + .ok_or_else(|| ToolError::InvalidParameters("version is required".to_string()))? + .to_string(); + let item_type = arguments + .get("item_type") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + let visibility = arguments + .get("visibility") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + let module = arguments + .get("module") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + let filters = cratedocs_mcp::tools::item_list::ItemListFilters { + item_type, + visibility, + module, + }; + let result = cratedocs_mcp::tools::item_list::list_crate_items( + &crate_name, + &version, + Some(filters), + ) + .await + .map_err(|e| ToolError::ExecutionError(format!("list_crate_items failed: {}", e)))?; + Ok(vec![Content::text(result)]) + } _ => Err(ToolError::NotFound(format!("Tool {} not found", tool_name))), } }) From ae1fc4297a160d1b56e8fbed8f118c7be5b15264 Mon Sep 17 00:00:00 2001 From: elasticdotventures Date: Sat, 5 Jul 2025 08:16:47 +0000 Subject: [PATCH 2/8] checkpoint 2, working --- src/tools/docs/docs.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/docs/docs.rs b/src/tools/docs/docs.rs index 710bfb6..e2f8842 100644 --- a/src/tools/docs/docs.rs +++ b/src/tools/docs/docs.rs @@ -440,12 +440,12 @@ impl mcp_server::Router for DocRouter { .get("module") .and_then(|v| v.as_str()) .map(|s| s.to_string()); - let filters = cratedocs_mcp::tools::item_list::ItemListFilters { + let filters = item_list::ItemListFilters { item_type, visibility, module, }; - let result = cratedocs_mcp::tools::item_list::list_crate_items( + let result = item_list::list_crate_items( &crate_name, &version, Some(filters), From fd18fe7cf57bb6b2d56bc9fba7a90221f04fa4b5 Mon Sep 17 00:00:00 2001 From: elasticdotventures Date: Sat, 5 Jul 2025 09:39:04 +0000 Subject: [PATCH 3/8] checkpoint, moved list_crate_items moved to tools --- src/bin/cratedocs.rs | 47 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/src/bin/cratedocs.rs b/src/bin/cratedocs.rs index 02f1d7d..aef8cf0 100644 --- a/src/bin/cratedocs.rs +++ b/src/bin/cratedocs.rs @@ -40,11 +40,11 @@ enum Commands { }, /// Test tools directly from the CLI Test { - /// The tool to test (lookup_crate, search_crates, lookup_item) + /// The tool to test (lookup_crate, search_crates, lookup_item, list_crate_items) #[arg(long, default_value = "lookup_crate")] tool: String, - /// Crate name for lookup_crate and lookup_item + /// Crate name for lookup_crate, lookup_item, and list_crate_items #[arg(long)] crate_name: Option, @@ -64,6 +64,18 @@ enum Commands { #[arg(long)] limit: Option, + /// Filter by item type for list_crate_items (e.g., struct, enum, trait) + #[arg(long)] + item_type: Option, + + /// Filter by visibility for list_crate_items (e.g., pub, private) + #[arg(long)] + visibility: Option, + + /// Filter by module path for list_crate_items (e.g., serde::de) + #[arg(long)] + module: Option, + /// Output format (markdown, text, json) #[arg(long, default_value = "markdown")] format: Option, @@ -71,11 +83,11 @@ enum Commands { /// Output file path (if not specified, results will be printed to stdout) #[arg(long)] output: Option, - + /// Summarize output by stripping LICENSE and VERSION sections (TL;DR mode) #[arg(long)] tldr: bool, - + /// Maximum number of tokens for output (token-aware truncation) #[arg(long)] max_tokens: Option, @@ -100,6 +112,9 @@ async fn main() -> Result<()> { query, version, limit, + item_type, + visibility, + module, format, output, tldr, @@ -112,6 +127,9 @@ async fn main() -> Result<()> { query, version, limit, + item_type, + visibility, + module, format, output, tldr, @@ -212,6 +230,9 @@ struct TestToolConfig { query: Option, version: Option, limit: Option, + item_type: Option, + visibility: Option, + module: Option, format: Option, output: Option, tldr: bool, @@ -233,6 +254,9 @@ async fn run_test_tool(config: TestToolConfig) -> Result<()> { tldr, max_tokens, debug, + item_type, + visibility, + module, } = config; // Print help information if the tool is "help" if tool == "help" { @@ -308,6 +332,21 @@ async fn run_test_tool(config: TestToolConfig) -> Result<()> { "limit": limit, }) }, + "list_crate_items" => { + let crate_name = crate_name.ok_or_else(|| + anyhow::anyhow!("--crate-name is required for list_crate_items tool"))?; + let version = version.ok_or_else(|| + anyhow::anyhow!("--version is required for list_crate_items tool"))?; + + let arguments = json!({ + "crate_name": crate_name, + "version": version, + "item_type": item_type, + "visibility": visibility, + "module": module, + }); + arguments + }, _ => return Err(anyhow::anyhow!("Unknown tool: {}", tool)), }; From fc3f2f955a261e9dd814bf53d87f93d1eb189559 Mon Sep 17 00:00:00 2001 From: elasticdotventures Date: Sat, 5 Jul 2025 09:41:39 +0000 Subject: [PATCH 4/8] checkpoint, 1 test fails --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index c89b998..d8a9a80 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,9 @@ cargo run --bin cratedocs http --debug ### Directly Testing Documentation Tools +# Enumerate crate items +cargo run --bin cratedocs test --tool list_crate_items --crate-name serde --version 1.0.0 --item-type struct +cargo run --bin cratedocs test --tool list_crate_items --crate-name tokio --version 1.28.0 --visibility pub --module tokio::sync You can directly test the documentation tools from the command line without starting a server: ```bash From b2df17188adee35ef60b5a6758fbeb9596d625b7 Mon Sep 17 00:00:00 2001 From: elasticdotventures Date: Sat, 5 Jul 2025 09:51:06 +0000 Subject: [PATCH 5/8] added version tool --- .gitignore | 1 + README.md | 3 +- src/bin/cratedocs.rs | 6 +++ src/tools/docs/tests.rs | 4 +- src/tools/item_list.rs | 77 ++++++++++++++++++++++++++++++++++++++ tests/integration_tests.rs | 6 +-- 6 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 src/tools/item_list.rs diff --git a/.gitignore b/.gitignore index b197cae..7fc7d40 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ Thumbs.db *.swo output_tests target/* +context_portal/* diff --git a/README.md b/README.md index d8a9a80..40f667d 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,10 @@ This is an MCP (Model Context Protocol) server that provides tools for Rust crat ## Installation ```bash -git clone https://github.com/d6e/cratedocs-mcp.git +git clone https://github.com/promptexecution/cratedocs-mcp.git cd cratedocs-mcp cargo build --release +cargo install --path . ``` ## Running the Server diff --git a/src/bin/cratedocs.rs b/src/bin/cratedocs.rs index aef8cf0..c782baa 100644 --- a/src/bin/cratedocs.rs +++ b/src/bin/cratedocs.rs @@ -22,6 +22,8 @@ struct Cli { #[derive(Subcommand)] enum Commands { + /// Output the version and exit + Version, /// Run the server in stdin/stdout mode Stdio { /// Enable debug logging @@ -103,6 +105,10 @@ async fn main() -> Result<()> { let cli = Cli::parse(); match cli.command { + Commands::Version => { + println!("{}", env!("CARGO_PKG_VERSION")); + Ok(()) + }, Commands::Stdio { debug } => run_stdio_server(debug).await, Commands::Http { address, debug } => run_http_server(address, debug).await, Commands::Test { diff --git a/src/tools/docs/tests.rs b/src/tools/docs/tests.rs index 71acf32..0b6fa3b 100644 --- a/src/tools/docs/tests.rs +++ b/src/tools/docs/tests.rs @@ -75,8 +75,8 @@ async fn test_list_tools() { let router = DocRouter::new(); let tools = router.list_tools(); - // Should have exactly 3 tools - assert_eq!(tools.len(), 3); + // Should have exactly 4 tools (lookup_crate, search_crates, lookup_item, list_crate_items) + assert_eq!(tools.len(), 4); // Check tool names let tool_names: Vec = tools.iter().map(|t| t.name.clone()).collect(); diff --git a/src/tools/item_list.rs b/src/tools/item_list.rs new file mode 100644 index 0000000..68316ad --- /dev/null +++ b/src/tools/item_list.rs @@ -0,0 +1,77 @@ +use anyhow::Result; + +/// Represents filters for item listing. +#[derive(Debug)] +pub struct ItemListFilters { + pub item_type: Option, + pub visibility: Option, + pub module: Option, +} + +/// Stub for the crate item enumeration tool. +/// This will use rust-analyzer to enumerate items in a crate. +pub async fn list_crate_items( + crate_name: &str, + version: &str, + filters: Option, +) -> Result { + // 🦨 skunky: Implementation pending. Will use rust-analyzer APIs. + Ok(format!( + "Stub: list_crate_items for crate: {}, version: {}, filters: {:?}", + crate_name, version, filters + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio; + + #[tokio::test] + async fn test_basic_call_returns_stub() { + let result = list_crate_items("serde", "1.0.0", None).await.unwrap(); + assert!(result.contains("Stub: list_crate_items for crate: serde, version: 1.0.0"), "Stub output missing expected text"); + } + + #[tokio::test] + async fn test_with_item_type_filter() { + let filters = ItemListFilters { + item_type: Some("struct".to_string()), + visibility: None, + module: None, + }; + let result = list_crate_items("serde", "1.0.0", Some(filters)).await.unwrap(); + assert!(result.contains("filters: Some"), "Stub output missing filters"); + assert!(result.contains("struct"), "Stub output missing item_type"); + } + + #[tokio::test] + async fn test_with_visibility_filter() { + let filters = ItemListFilters { + item_type: None, + visibility: Some("pub".to_string()), + module: None, + }; + let result = list_crate_items("serde", "1.0.0", Some(filters)).await.unwrap(); + assert!(result.contains("filters: Some"), "Stub output missing filters"); + assert!(result.contains("pub"), "Stub output missing visibility"); + } + + #[tokio::test] + async fn test_with_module_filter() { + let filters = ItemListFilters { + item_type: None, + visibility: None, + module: Some("serde::de".to_string()), + }; + let result = list_crate_items("serde", "1.0.0", Some(filters)).await.unwrap(); + assert!(result.contains("filters: Some"), "Stub output missing filters"); + assert!(result.contains("serde::de"), "Stub output missing module filter"); + } + + #[tokio::test] + async fn test_invalid_crate_name() { + let result = list_crate_items("not_a_real_crate", "0.0.1", None).await.unwrap(); + assert!(result.contains("not_a_real_crate"), "Stub output missing invalid crate name"); + } +} diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index c1631df..fdea00b 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -13,7 +13,7 @@ async fn test_doc_router_initialization() { // Tools should be available and correctly configured let tools = router.list_tools(); - assert_eq!(tools.len(), 3); + assert_eq!(tools.len(), 4); // Check specific tool schemas let lookup_crate_tool = tools.iter().find(|t| t.name == "lookup_crate").unwrap(); @@ -68,8 +68,8 @@ async fn test_end_to_end_crate_lookup() { // The response should be HTML from docs.rs match &content[0] { mcp_core::Content::Text(text) => { - assert!(text.text.contains("")); - assert!(text.text.contains("serde")); + // Output is now markdown, not HTML + assert!(text.text.to_lowercase().contains("serde")); }, _ => panic!("Expected text content"), } From 579f0c1ed169a4907bd52f14e00a1fa4e919ed17 Mon Sep 17 00:00:00 2001 From: elasticdotventures Date: Sat, 5 Jul 2025 10:43:36 +0000 Subject: [PATCH 6/8] --tldr added tag stripping --- src/bin/cratedocs.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/bin/cratedocs.rs b/src/bin/cratedocs.rs index c782baa..0d05b1e 100644 --- a/src/bin/cratedocs.rs +++ b/src/bin/cratedocs.rs @@ -210,6 +210,8 @@ fn apply_tldr(input: &str) -> String { let tldr_section_re = Regex::new(r"(?i)^\s*#+\s*license\b|^\s*#+\s*version(s)?\b|^\s*#+license\b|^\s*#+version(s)?\b").unwrap(); // Match any heading (for ending the skip) let heading_re = Regex::new(r"^\s*#+").unwrap(); + // Match tags including start, end, and inline attributes + let detail_tag_re = Regex::new(r"<[/]?detail.*?>").unwrap(); for line in input.lines() { // Start skipping if we hit a LICENSE or VERSION(S) heading @@ -222,10 +224,12 @@ fn apply_tldr(input: &str) -> String { skip = false; } if !skip { - output.push(line); + // Remove tags from the line + let cleaned_line = detail_tag_re.replace_all(line, "").to_string(); + output.push(cleaned_line.to_string()); } } - output.join("\n") + output.iter().map(|s| s.as_str()).collect::>().join("\n") } /// Configuration for the test tool @@ -594,3 +598,4 @@ Another version section. assert!(output.contains("Some real documentation here.")); } } + From 58b7680c8cd45a92940b02c435dec1cfa66e2f54 Mon Sep 17 00:00:00 2001 From: elasticdotventures Date: Sat, 5 Jul 2025 11:05:38 +0000 Subject: [PATCH 7/8] checkpoint, syn in - but missing --- Cargo.lock | 8 ++-- Cargo.toml | 2 + src/tools/item_list.rs | 102 +++++++++++++++++++---------------------- 3 files changed, 54 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b61328c..b1a5047 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -443,6 +443,7 @@ dependencies = [ "anyhow", "axum", "clap", + "flate2", "futures", "html2md", "hyper 0.14.32", @@ -455,6 +456,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "syn", "tokenizers", "tokio", "tokio-util", @@ -610,7 +612,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -2455,9 +2457,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.99" +version = "2.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e02e925281e18ffd9d640e234264753c43edc62d64b2d4cf898f1bc5e75f3fc2" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index c153ee9..efc576f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,8 @@ rand = "0.8" clap = { version = "4.4", features = ["derive"] } html2md = "0.2.14" regex = "1" +syn = { version = "2.0.104", features = ["full"] } +flate2 = "1.1.2" [dev-dependencies] # Testing utilities diff --git a/src/tools/item_list.rs b/src/tools/item_list.rs index 68316ad..7d2e10f 100644 --- a/src/tools/item_list.rs +++ b/src/tools/item_list.rs @@ -1,4 +1,11 @@ use anyhow::Result; +use reqwest; +use std::fs; +use std::path::Path; +use tar::Archive; +use flate2::read::GzDecoder; +use syn::{File, Item}; +use tokio::fs as tokio_fs; /// Represents filters for item listing. #[derive(Debug)] @@ -8,6 +15,27 @@ pub struct ItemListFilters { pub module: Option, } +/// Utility function to download and cache crate source. +async fn download_and_cache_crate(crate_name: &str, version: &str) -> Result { + let cache_dir = Path::new("./cache"); + let crate_dir = cache_dir.join(format!("{}-{}", crate_name, version)); + + if crate_dir.exists() { + return Ok(crate_dir.to_string_lossy().to_string()); + } + + let url = format!("https://crates.io/api/v1/crates/{}/{}/download", crate_name, version); + let response = reqwest::get(&url).await?; + let tarball = response.bytes().await?; + + fs::create_dir_all(&cache_dir)?; + let tar_gz = GzDecoder::new(&*tarball); + let mut archive = Archive::new(tar_gz); + archive.unpack(&cache_dir)?; + + Ok(crate_dir.to_string_lossy().to_string()) +} + /// Stub for the crate item enumeration tool. /// This will use rust-analyzer to enumerate items in a crate. pub async fn list_crate_items( @@ -15,63 +43,27 @@ pub async fn list_crate_items( version: &str, filters: Option, ) -> Result { - // 🦨 skunky: Implementation pending. Will use rust-analyzer APIs. - Ok(format!( - "Stub: list_crate_items for crate: {}, version: {}, filters: {:?}", - crate_name, version, filters - )) -} + let crate_path = download_and_cache_crate(crate_name, version).await?; + let mut items = Vec::new(); -#[cfg(test)] -mod tests { - use super::*; - use tokio; + for entry in fs::read_dir(crate_path)? { + let entry = entry?; + let path = entry.path(); + if path.extension().and_then(|ext| ext.to_str()) == Some("rs") { + let content = fs::read_to_string(&path)?; + let parsed_file: File = syn::parse_file(&content)?; - #[tokio::test] - async fn test_basic_call_returns_stub() { - let result = list_crate_items("serde", "1.0.0", None).await.unwrap(); - assert!(result.contains("Stub: list_crate_items for crate: serde, version: 1.0.0"), "Stub output missing expected text"); + for item in parsed_file.items { + match item { + Item::Struct(_) if filters.as_ref().map_or(true, |f| f.item_type.as_deref() == Some("struct")) => items.push(format!("{:?}", item)), + Item::Enum(_) if filters.as_ref().map_or(true, |f| f.item_type.as_deref() == Some("enum")) => items.push(format!("{:?}", item)), + Item::Trait(_) if filters.as_ref().map_or(true, |f| f.item_type.as_deref() == Some("trait")) => items.push(format!("{:?}", item)), + Item::Fn(_) if filters.as_ref().map_or(true, |f| f.item_type.as_deref() == Some("fn")) => items.push(format!("{:?}", item)), + _ => {} + } + } + } } - #[tokio::test] - async fn test_with_item_type_filter() { - let filters = ItemListFilters { - item_type: Some("struct".to_string()), - visibility: None, - module: None, - }; - let result = list_crate_items("serde", "1.0.0", Some(filters)).await.unwrap(); - assert!(result.contains("filters: Some"), "Stub output missing filters"); - assert!(result.contains("struct"), "Stub output missing item_type"); - } - - #[tokio::test] - async fn test_with_visibility_filter() { - let filters = ItemListFilters { - item_type: None, - visibility: Some("pub".to_string()), - module: None, - }; - let result = list_crate_items("serde", "1.0.0", Some(filters)).await.unwrap(); - assert!(result.contains("filters: Some"), "Stub output missing filters"); - assert!(result.contains("pub"), "Stub output missing visibility"); - } - - #[tokio::test] - async fn test_with_module_filter() { - let filters = ItemListFilters { - item_type: None, - visibility: None, - module: Some("serde::de".to_string()), - }; - let result = list_crate_items("serde", "1.0.0", Some(filters)).await.unwrap(); - assert!(result.contains("filters: Some"), "Stub output missing filters"); - assert!(result.contains("serde::de"), "Stub output missing module filter"); - } - - #[tokio::test] - async fn test_invalid_crate_name() { - let result = list_crate_items("not_a_real_crate", "0.0.1", None).await.unwrap(); - assert!(result.contains("not_a_real_crate"), "Stub output missing invalid crate name"); - } + Ok(items.join("\n")) } From a13d2bea1b78e321a20f957ed4e22c3b1e3ebccd Mon Sep 17 00:00:00 2001 From: elasticdotventures Date: Sat, 5 Jul 2025 11:12:34 +0000 Subject: [PATCH 8/8] list_crate_items appears to work! --- Cargo.lock | 35 +++++++++++++++++++++ Cargo.toml | 1 + src/tools/item_list.rs | 70 +++++++++++++++++++++++++++++++++--------- 3 files changed, 91 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1a5047..6d79a86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -457,6 +457,7 @@ dependencies = [ "serde", "serde_json", "syn", + "tar", "tokenizers", "tokio", "tokio-util", @@ -684,6 +685,18 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + [[package]] name = "flate2" version = "1.1.2" @@ -1358,6 +1371,7 @@ checksum = "1580801010e535496706ba011c15f8532df6b42297d2e471fec38ceadd8c0638" dependencies = [ "bitflags 2.9.0", "libc", + "redox_syscall", ] [[package]] @@ -2510,6 +2524,17 @@ dependencies = [ "libc", ] +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.18.0" @@ -3389,6 +3414,16 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "xattr" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af3a19837351dc82ba89f8a125e22a3c475f05aba604acc023d62b2739ae2909" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "xml5ever" version = "0.18.1" diff --git a/Cargo.toml b/Cargo.toml index efc576f..4c4f286 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,7 @@ html2md = "0.2.14" regex = "1" syn = { version = "2.0.104", features = ["full"] } flate2 = "1.1.2" +tar = "0.4.44" [dev-dependencies] # Testing utilities diff --git a/src/tools/item_list.rs b/src/tools/item_list.rs index 7d2e10f..6e0c4dc 100644 --- a/src/tools/item_list.rs +++ b/src/tools/item_list.rs @@ -5,7 +5,6 @@ use std::path::Path; use tar::Archive; use flate2::read::GzDecoder; use syn::{File, Item}; -use tokio::fs as tokio_fs; /// Represents filters for item listing. #[derive(Debug)] @@ -46,24 +45,65 @@ pub async fn list_crate_items( let crate_path = download_and_cache_crate(crate_name, version).await?; let mut items = Vec::new(); - for entry in fs::read_dir(crate_path)? { - let entry = entry?; - let path = entry.path(); - if path.extension().and_then(|ext| ext.to_str()) == Some("rs") { - let content = fs::read_to_string(&path)?; - let parsed_file: File = syn::parse_file(&content)?; + // Most crates have their source in a "src" subdirectory + let src_path = Path::new(&crate_path).join("src"); - for item in parsed_file.items { - match item { - Item::Struct(_) if filters.as_ref().map_or(true, |f| f.item_type.as_deref() == Some("struct")) => items.push(format!("{:?}", item)), - Item::Enum(_) if filters.as_ref().map_or(true, |f| f.item_type.as_deref() == Some("enum")) => items.push(format!("{:?}", item)), - Item::Trait(_) if filters.as_ref().map_or(true, |f| f.item_type.as_deref() == Some("trait")) => items.push(format!("{:?}", item)), - Item::Fn(_) if filters.as_ref().map_or(true, |f| f.item_type.as_deref() == Some("fn")) => items.push(format!("{:?}", item)), - _ => {} + fn visit_rs_files(dir: &Path, cb: &mut F) { + if let Ok(entries) = fs::read_dir(dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + visit_rs_files(&path, cb); + } else if path.extension().and_then(|ext| ext.to_str()) == Some("rs") { + cb(&path); } } } } - Ok(items.join("\n")) + visit_rs_files(&src_path, &mut |path: &Path| { + if let Ok(content) = fs::read_to_string(path) { + if let Ok(parsed_file) = syn::parse_file(&content) { + for item in parsed_file.items { + if let Item::Struct(s) = &item { + if filters.as_ref().map_or(true, |f| f.item_type.as_deref().map_or(true, |ty| ty == "struct")) { + items.push(("Structs", format!("{}", s.ident))); + } + } + if let Item::Enum(e) = &item { + if filters.as_ref().map_or(true, |f| f.item_type.as_deref().map_or(true, |ty| ty == "enum")) { + items.push(("Enums", format!("{}", e.ident))); + } + } + if let Item::Trait(t) = &item { + if filters.as_ref().map_or(true, |f| f.item_type.as_deref().map_or(true, |ty| ty == "trait")) { + items.push(("Traits", format!("{}", t.ident))); + } + } + if let Item::Fn(f) = &item { + if filters.as_ref().map_or(true, |f| f.item_type.as_deref().map_or(true, |ty| ty == "fn")) { + items.push(("Functions", format!("{}", f.sig.ident))); + } + } + } + } + } + }); + + use std::collections::BTreeMap; + let mut grouped: BTreeMap<&str, Vec> = BTreeMap::new(); + for (kind, name) in items { + grouped.entry(kind).or_default().push(name); + } + + let mut output = String::new(); + for (kind, names) in grouped { + output.push_str(&format!("## {}\n", kind)); + for name in names { + output.push_str(&format!("- {}\n", name)); + } + output.push('\n'); + } + + Ok(output) }