Skip to content

fix: Align mcp-tester with WASM MCP server implementations #51

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 25, 2025
Merged

fix: Align mcp-tester with WASM MCP server implementations #51

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion examples/26-server-tester/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,7 @@ async fn main() -> Result<()> {
cli.insecure,
cli.api_key.as_deref(),
cli.transport.as_deref(),
cli.verbose > 0,
)
.await
},
Expand Down Expand Up @@ -409,6 +410,7 @@ async fn run_tools_test(
insecure: bool,
api_key: Option<&str>,
transport: Option<&str>,
verbose: bool,
) -> Result<TestReport> {
let mut tester = ServerTester::new(
url,
Expand All @@ -421,7 +423,10 @@ async fn run_tools_test(
println!("{}", "Discovering and testing tools...".green());
println!();

tester.run_tools_discovery(test_all).await
// Pass verbose flag to the tester for detailed output
tester
.run_tools_discovery_with_verbose(test_all, verbose)
.await
}

async fn run_diagnostics(
Expand Down
72 changes: 64 additions & 8 deletions examples/26-server-tester/src/scenario_executor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,12 +237,62 @@ impl<'a> ScenarioExecutor<'a> {

match operation {
Operation::ToolCall { tool, arguments } => {
let result = self.tester.test_tool(&tool, arguments).await?;
Ok(json!({
"success": result.status == crate::report::TestStatus::Passed,
"result": result.details,
"error": result.error
}))
// Call the tool directly to get raw response for assertions
match self.tester.transport_type {
crate::tester::TransportType::Http => {
if let Some(ref client) = self.tester.pmcp_client {
match client.call_tool(tool.clone(), arguments).await {
Ok(result) => {
// Extract the text content from the response
let content_text = result
.content
.into_iter()
.filter_map(|c| match c {
pmcp::types::Content::Text { text } => Some(text),
_ => None,
})
.collect::<Vec<_>>()
.join("\n");

// Check if the content indicates an error
if content_text.starts_with("Error:") {
Ok(json!({
"success": false,
"result": null,
"error": content_text
}))
} else {
Ok(json!({
"success": true,
"result": content_text,
"error": null
}))
}
},
Err(e) => Ok(json!({
"success": false,
"result": null,
"error": e.to_string()
})),
}
} else {
Ok(json!({
"success": false,
"result": null,
"error": "Client not initialized"
}))
}
},
_ => {
// Fall back to test_tool for other transport types
let result = self.tester.test_tool(&tool, arguments).await?;
Ok(json!({
"success": result.status == crate::report::TestStatus::Passed,
"result": result.details,
"error": result.error
}))
},
}
},

Operation::ListTools => {
Expand Down Expand Up @@ -523,7 +573,10 @@ impl<'a> ScenarioExecutor<'a> {
},

Assertion::Success => {
let has_error = response.get("error").is_some();
let has_error = response
.get("error")
.and_then(|e| if e.is_null() { None } else { Some(e) })
.is_some();
AssertionResult {
assertion: "Success".to_string(),
passed: !has_error,
Expand All @@ -538,7 +591,10 @@ impl<'a> ScenarioExecutor<'a> {
},

Assertion::Failure => {
let has_error = response.get("error").is_some();
let has_error = response
.get("error")
.and_then(|e| if e.is_null() { None } else { Some(e) })
.is_some();
AssertionResult {
assertion: "Failure".to_string(),
passed: has_error,
Expand Down
59 changes: 56 additions & 3 deletions examples/26-server-tester/src/tester.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ pub enum TransportType {

pub struct ServerTester {
url: String,
transport_type: TransportType,
pub transport_type: TransportType,
http_config: Option<StreamableHttpTransportConfig>,
json_rpc_client: Option<Client>,
#[allow(dead_code)]
Expand All @@ -53,7 +53,7 @@ pub struct ServerTester {
server_info: Option<InitializeResult>,
tools: Option<Vec<ToolInfo>>,
// Store the initialized pmcp client for reuse across tests
pmcp_client: Option<pmcp::Client<StreamableHttpTransport>>,
pub pmcp_client: Option<pmcp::Client<StreamableHttpTransport>>,
stdio_client: Option<pmcp::Client<StdioTransport>>,
}

Expand Down Expand Up @@ -288,13 +288,33 @@ impl ServerTester {
}

pub async fn run_tools_discovery(&mut self, test_all: bool) -> Result<TestReport> {
self.run_tools_discovery_with_verbose(test_all, false).await
}

pub async fn run_tools_discovery_with_verbose(
&mut self,
test_all: bool,
verbose: bool,
) -> Result<TestReport> {
let mut report = TestReport::new();
let start = Instant::now();

// Initialize
let init_result = self.test_initialize().await;
report.add_test(init_result.clone());

if verbose && init_result.status == TestStatus::Passed {
println!(" ✓ Server initialized successfully");
if let Some(ref server) = self.server_info {
println!(
" Server: {} v{}",
server.server_info.name, server.server_info.version
);
}
} else if verbose && init_result.status != TestStatus::Passed {
println!(" ✗ Initialization failed: {:?}", init_result.error);
}

if init_result.status != TestStatus::Passed {
return Ok(report);
}
Expand All @@ -303,6 +323,32 @@ impl ServerTester {
let tools_result = self.test_tools_list().await;
report.add_test(tools_result.clone());

if verbose {
if tools_result.status == TestStatus::Passed {
if let Some(ref tools) = self.tools {
println!(" ✓ Found {} tools:", tools.len());
for tool in tools {
println!(
" • {} - {}",
tool.name,
tool.description.as_deref().unwrap_or("No description")
);
}
} else {
println!(" ✓ No tools found");
}
} else {
println!(" ✗ Failed to list tools: {:?}", tools_result.error);
if verbose {
// Print the actual error details
println!(
" Error details: {}",
tools_result.error.as_deref().unwrap_or("Unknown error")
);
}
}
}

if tools_result.status == TestStatus::Passed && test_all {
let tools_to_test: Vec<(String, Value)> = self
.tools
Expand All @@ -319,7 +365,14 @@ impl ServerTester {
.unwrap_or_default();

for (tool_name, test_args) in tools_to_test {
report.add_test(self.test_tool(&tool_name, test_args).await?);
let test_result = self.test_tool(&tool_name, test_args.clone()).await?;
if verbose {
println!(" Testing tool '{}': {:?}", tool_name, test_result.status);
if test_result.status != TestStatus::Passed {
println!(" Error: {:?}", test_result.error);
}
}
report.add_test(test_result);
}
}

Expand Down
77 changes: 71 additions & 6 deletions examples/wasm-mcp-server/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ cargo build --target wasm32-unknown-unknown --release
cd deployments/cloudflare
make deploy

# Live at: https://mcp-sdk-worker.guy-ernest.workers.dev
# Your deployment will be available at: https://<your-worker-name>.workers.dev
```

### Deploy to Fermyon Spin
Expand All @@ -54,7 +54,7 @@ make deploy
cd deployments/fermyon-spin
make deploy

# Live at: https://mcp-fermyon-spin-3juc7zc4.fermyon.app/
# Your deployment will be available at: https://<your-app-name>.fermyon.app/
```

## 🏗️ Architecture
Expand Down Expand Up @@ -116,7 +116,58 @@ Reports the runtime environment (Cloudflare vs Fermyon)

## 🧪 Testing

### Test Any Deployment
### Using MCP Tester with Scenario Files

The repository includes comprehensive test scenarios that can be run with the `mcp-tester` tool:

```bash
# Test with simple calculator scenario
mcp-tester scenario <deployment-url> test-scenarios/calculator-simple.json

# Test with comprehensive calculator tests (including error cases)
mcp-tester scenario <deployment-url> test-scenarios/calculator-test.yaml

# Test with minimal tool listing
mcp-tester scenario <deployment-url> test-scenarios/minimal-test.json
```

#### Example: Testing Cloudflare Deployment
```bash
# From the rust-mcp-sdk root directory
# Replace <your-worker-name> with your Cloudflare Worker subdomain
./target/release/mcp-tester scenario \
https://<your-worker-name>.workers.dev \
examples/wasm-mcp-server/test-scenarios/calculator-test.yaml
```

#### Example: Testing Fermyon Spin Deployment
```bash
# From the rust-mcp-sdk root directory
# Replace <your-app-name> with your Fermyon app URL
./target/release/mcp-tester scenario \
https://<your-app-name>.fermyon.app/ \
examples/wasm-mcp-server/test-scenarios/calculator-test.yaml
```

### Available Test Scenarios

1. **`calculator-simple.json`** - Basic calculator operations
- Tests addition, multiplication, division, and subtraction
- Validates correct results for each operation

2. **`calculator-test.yaml`** - Comprehensive calculator test suite
- Tests all arithmetic operations with various inputs
- Tests negative numbers and decimals
- Tests error handling (division by zero, invalid operations, missing parameters)
- Tests large numbers and edge cases

3. **`minimal-test.json`** - Minimal connectivity test
- Simply lists available tools
- Quick smoke test for deployment health

### Manual Testing with curl

You can also test deployments manually:

```bash
# Initialize connection
Expand All @@ -135,6 +186,17 @@ curl -X POST <deployment-url> \
-d '{"jsonrpc":"2.0","id":"3","method":"tools/call","params":{"name":"calculator","arguments":{"operation":"add","a":5,"b":3}}}'
```

### Building the MCP Tester

If you need to build the mcp-tester tool:

```bash
# From the rust-mcp-sdk root directory
cargo build --release --package mcp-server-tester

# The binary will be at: ./target/release/mcp-tester
```

## 📊 Deployment Comparison

| Platform | Build Target | Runtime | Global Edge | Cold Start | State Management |
Expand Down Expand Up @@ -222,6 +284,9 @@ MIT

---

**Current Production Deployments:**
- 🌐 Cloudflare: https://mcp-sdk-worker.guy-ernest.workers.dev
- 🔄 Fermyon: https://mcp-fermyon-spin-3juc7zc4.fermyon.app/
**Example Deployments for Testing:**
You can test the MCP protocol with these example deployments:
- 🌐 Cloudflare Example: https://mcp-sdk-worker.guy-ernest.workers.dev
- 🔄 Fermyon Example: https://mcp-fermyon-spin-3juc7zc4.fermyon.app/

Note: These are example deployments for testing. Deploy your own instances using the instructions above.
52 changes: 51 additions & 1 deletion examples/wasm-mcp-server/deployments/cloudflare/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,51 @@ make deploy
make test-prod
```

## Testing with MCP Tester

### Automated Scenario Testing

The deployment can be tested using the mcp-tester tool with predefined scenarios:

```bash
# From the rust-mcp-sdk root directory
# Replace <your-worker-name> with your deployed Worker subdomain

# Test with comprehensive calculator scenario
./target/release/mcp-tester scenario \
https://<your-worker-name>.workers.dev \
examples/wasm-mcp-server/test-scenarios/calculator-test.yaml

# Quick connectivity test
./target/release/mcp-tester scenario \
https://<your-worker-name>.workers.dev \
examples/wasm-mcp-server/test-scenarios/minimal-test.json

# Basic calculator operations test
./target/release/mcp-tester scenario \
https://<your-worker-name>.workers.dev \
examples/wasm-mcp-server/test-scenarios/calculator-simple.json
```

### Expected Test Results

All scenarios should pass with output like:
```
╔════════════════════════════════════════════════════════════╗
║ MCP SERVER TESTING TOOL v0.1.0 ║
╚════════════════════════════════════════════════════════════╝

TEST RESULTS
════════════════════════════════════════════════════════════
✓ Test Addition - 10 + 5
✓ Test Multiplication - 4 * 7
✓ Test Division - 20 / 4
✓ Test Division by Zero (error case)
✓ Test Invalid Operation (error case)

SUMMARY: PASSED
```

## Configuration

The `wrangler.toml` file specifies:
Expand All @@ -50,6 +95,11 @@ The `wrangler.toml` file specifies:
- If runtime fails: Check the JavaScript wrapper initialization
- For CORS issues: Headers are set in the Rust code

## Live Deployment
## Deployment URL

After deploying, your MCP server will be available at:
🌐 `https://<your-worker-name>.workers.dev`

### Example Deployment for Testing
You can test the MCP protocol with this example deployment:
🌐 https://mcp-sdk-worker.guy-ernest.workers.dev
Loading
Loading