From af426a0e9e7e6a80348f39905a20ba177f12e9c6 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 4 Sep 2025 12:56:24 +0000 Subject: [PATCH 01/22] chore: update contributors data 2025-09-04 --- src/data/contributors.json | 88 +++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/src/data/contributors.json b/src/data/contributors.json index b65ebae..584973a 100644 --- a/src/data/contributors.json +++ b/src/data/contributors.json @@ -1,14 +1,14 @@ { - "generated_at": "2025-09-03T19:07:43.751Z", + "generated_at": "2025-09-04T12:56:24.256Z", "total_contributors": 10, - "total_contributions": 505, - "total_impact_score": 359, - "total_recent_activity": 361, + "total_contributions": 519, + "total_impact_score": 373, + "total_recent_activity": 375, "scoring_method": "simplified_recent_activity", - "total_additions": 313932, - "total_deletions": 77961, - "total_changes": 391893, - "total_commits_analyzed": 505, + "total_additions": 321436, + "total_deletions": 78154, + "total_changes": 399590, + "total_commits_analyzed": 519, "contributors": [ { "id": 43811028, @@ -25,7 +25,7 @@ "followers": 22, "following": 2, "created_at": "2018-10-03T10:10:15Z", - "contributions": 169, + "contributions": 179, "repositories": [ "python-utcp", "utcp-specification", @@ -38,18 +38,18 @@ "utcp-agent" ], "repo_count": 9, - "impact_score": 169, - "total_prs": 166, - "total_merged_prs": 133, - "total_recent_commits": 169, - "total_commits": 169, + "impact_score": 179, + "total_prs": 169, + "total_merged_prs": 136, + "total_recent_commits": 179, + "total_commits": 179, "total_reviews": 9, - "last_activity": "2025-09-02T14:45:10Z", + "last_activity": "2025-09-04T12:36:01Z", "pr_success_rate": 80, - "total_additions": 210552, - "total_deletions": 54504, - "total_changes": 265056, - "commits_analyzed": 169 + "total_additions": 213252, + "total_deletions": 54522, + "total_changes": 267774, + "commits_analyzed": 179 }, { "id": 170965471, @@ -63,7 +63,7 @@ "blog": null, "hireable": true, "public_repos": 28, - "followers": 30, + "followers": 31, "following": 59, "created_at": "2024-05-27T16:22:55Z", "contributions": 244, @@ -79,9 +79,9 @@ "total_reviews": 3, "last_activity": "2025-08-29T12:15:29Z", "pr_success_rate": 74, - "total_additions": 39386, - "total_deletions": 11377, - "total_changes": 50763, + "total_additions": 39387, + "total_deletions": 11378, + "total_changes": 50765, "commits_analyzed": 244 }, { @@ -99,7 +99,7 @@ "followers": 3, "following": 3, "created_at": "2020-08-11T00:28:15Z", - "contributions": 38, + "contributions": 42, "repositories": [ "python-utcp", "utcp-specification", @@ -107,18 +107,18 @@ "utcp-mcp" ], "repo_count": 4, - "impact_score": 38, - "total_prs": 68, - "total_merged_prs": 57, - "total_recent_commits": 38, - "total_commits": 38, + "impact_score": 42, + "total_prs": 70, + "total_merged_prs": 60, + "total_recent_commits": 42, + "total_commits": 42, "total_reviews": 0, - "last_activity": "2025-08-21T12:32:23Z", - "pr_success_rate": 84, - "total_additions": 12605, - "total_deletions": 5194, - "total_changes": 17799, - "commits_analyzed": 38 + "last_activity": "2025-09-04T11:01:20Z", + "pr_success_rate": 86, + "total_additions": 17408, + "total_deletions": 5368, + "total_changes": 22776, + "commits_analyzed": 42 }, { "id": 5594869, @@ -144,13 +144,13 @@ ], "repo_count": 4, "impact_score": 24, - "total_prs": 54, + "total_prs": 55, "total_merged_prs": 44, "total_recent_commits": 24, "total_commits": 24, "total_reviews": 0, "last_activity": "2025-09-02T14:17:44Z", - "pr_success_rate": 81, + "pr_success_rate": 80, "total_additions": 12023, "total_deletions": 3946, "total_changes": 15969, @@ -212,13 +212,13 @@ ], "repo_count": 3, "impact_score": 7, - "total_prs": 51, + "total_prs": 52, "total_merged_prs": 43, "total_recent_commits": 7, "total_commits": 7, "total_reviews": 5, "last_activity": "2025-09-02T14:17:22Z", - "pr_success_rate": 84, + "pr_success_rate": 83, "total_additions": 28358, "total_deletions": 916, "total_changes": 29274, @@ -279,13 +279,13 @@ ], "repo_count": 1, "impact_score": 3, - "total_prs": 20, - "total_merged_prs": 19, + "total_prs": 22, + "total_merged_prs": 22, "total_recent_commits": 4, "total_commits": 4, "total_reviews": 0, "last_activity": "2025-08-01T15:02:36Z", - "pr_success_rate": 95, + "pr_success_rate": 100, "total_additions": 7, "total_deletions": 7, "total_changes": 14, @@ -345,13 +345,13 @@ ], "repo_count": 1, "impact_score": 1, - "total_prs": 6, + "total_prs": 7, "total_merged_prs": 6, "total_recent_commits": 1, "total_commits": 1, "total_reviews": 0, "last_activity": "2025-07-24T16:29:37Z", - "pr_success_rate": 100, + "pr_success_rate": 86, "total_additions": 318, "total_deletions": 0, "total_changes": 318, From 12cd0d28bf7d32b741c19082c712dba49e2ef26c Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 4 Sep 2025 13:03:07 +0000 Subject: [PATCH 02/22] chore: update contributors data 2025-09-04 --- src/data/contributors.json | 51 +++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/src/data/contributors.json b/src/data/contributors.json index 584973a..243c34d 100644 --- a/src/data/contributors.json +++ b/src/data/contributors.json @@ -1,14 +1,14 @@ { - "generated_at": "2025-09-04T12:56:24.256Z", - "total_contributors": 10, - "total_contributions": 519, - "total_impact_score": 373, - "total_recent_activity": 375, + "generated_at": "2025-09-04T13:03:07.067Z", + "total_contributors": 11, + "total_contributions": 520, + "total_impact_score": 374, + "total_recent_activity": 376, "scoring_method": "simplified_recent_activity", - "total_additions": 321436, - "total_deletions": 78154, - "total_changes": 399590, - "total_commits_analyzed": 519, + "total_additions": 321480, + "total_deletions": 78198, + "total_changes": 399678, + "total_commits_analyzed": 520, "contributors": [ { "id": 43811028, @@ -291,6 +291,39 @@ "total_changes": 14, "commits_analyzed": 4 }, + { + "id": 65916846, + "login": "actions-user", + "name": "actions-user", + "avatar_url": "https://avatars.githubusercontent.com/u/65916846?v=4", + "html_url": "https://github.com/actions-user", + "bio": null, + "company": "@actions", + "location": null, + "blog": "https://github.com/actions", + "hireable": false, + "public_repos": 0, + "followers": 3231, + "following": 0, + "created_at": "2020-05-25T17:35:50Z", + "contributions": 1, + "repositories": [ + "utcp-specification" + ], + "repo_count": 1, + "impact_score": 1, + "total_prs": 22, + "total_merged_prs": 22, + "total_recent_commits": 1, + "total_commits": 1, + "total_reviews": 0, + "last_activity": "2025-09-04T12:56:24Z", + "pr_success_rate": 100, + "total_additions": 44, + "total_deletions": 44, + "total_changes": 88, + "commits_analyzed": 1 + }, { "id": 210855846, "login": "aliraza1006", From f32dbfcb89ecf50c020c37ef0d68d1349031e824 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 5 Sep 2025 06:38:23 +0000 Subject: [PATCH 03/22] chore: update contributors data 2025-09-05 --- src/data/contributors.json | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/data/contributors.json b/src/data/contributors.json index 243c34d..08a952f 100644 --- a/src/data/contributors.json +++ b/src/data/contributors.json @@ -1,14 +1,14 @@ { - "generated_at": "2025-09-04T13:03:07.067Z", + "generated_at": "2025-09-05T06:38:23.414Z", "total_contributors": 11, - "total_contributions": 520, - "total_impact_score": 374, - "total_recent_activity": 376, + "total_contributions": 521, + "total_impact_score": 375, + "total_recent_activity": 377, "scoring_method": "simplified_recent_activity", - "total_additions": 321480, - "total_deletions": 78198, - "total_changes": 399678, - "total_commits_analyzed": 520, + "total_additions": 321522, + "total_deletions": 78207, + "total_changes": 399729, + "total_commits_analyzed": 521, "contributors": [ { "id": 43811028, @@ -269,7 +269,7 @@ "location": null, "blog": null, "hireable": false, - "public_repos": 17, + "public_repos": 18, "followers": 2, "following": 4, "created_at": "2023-11-08T02:28:44Z", @@ -303,26 +303,26 @@ "blog": "https://github.com/actions", "hireable": false, "public_repos": 0, - "followers": 3231, + "followers": 3235, "following": 0, "created_at": "2020-05-25T17:35:50Z", - "contributions": 1, + "contributions": 2, "repositories": [ "utcp-specification" ], "repo_count": 1, - "impact_score": 1, + "impact_score": 2, "total_prs": 22, "total_merged_prs": 22, - "total_recent_commits": 1, - "total_commits": 1, + "total_recent_commits": 2, + "total_commits": 2, "total_reviews": 0, - "last_activity": "2025-09-04T12:56:24Z", + "last_activity": "2025-09-04T13:03:07Z", "pr_success_rate": 100, - "total_additions": 44, - "total_deletions": 44, - "total_changes": 88, - "commits_analyzed": 1 + "total_additions": 86, + "total_deletions": 53, + "total_changes": 139, + "commits_analyzed": 2 }, { "id": 210855846, From 313b30ac8f2e86159513460282ee3451c1ad9f73 Mon Sep 17 00:00:00 2001 From: Luca Perrozzi Date: Fri, 5 Sep 2025 08:35:06 +0000 Subject: [PATCH 04/22] Add comprehensive v1.0 documentation - Add complete communication protocols documentation (HTTP, WebSocket, SSE, CLI, Text, MCP) - Add implementation guide with Python examples and multi-language references - Add for-tool-providers guide with practical examples - Add migration guide from v0.1 to v1.0 - Update sidebar navigation structure - Address incomplete current version documentation gap --- docs/for-tool-providers.md | 689 +++++++++++++++++++++++++++++++++ docs/implementation.md | 580 +++++++++++++++++++++++++++ docs/migration-v0.1-to-v1.0.md | 601 ++++++++++++++++++++++++++++ docs/providers/cli.md | 323 ++++++++++++++++ docs/providers/http.md | 258 ++++++++++++ docs/providers/index.md | 92 +++++ docs/providers/mcp.md | 336 ++++++++++++++++ docs/providers/sse.md | 397 +++++++++++++++++++ docs/providers/text.md | 448 +++++++++++++++++++++ docs/providers/websocket.md | 366 +++++++++++++++++ sidebars.ts | 54 ++- 11 files changed, 4134 insertions(+), 10 deletions(-) create mode 100644 docs/for-tool-providers.md create mode 100644 docs/implementation.md create mode 100644 docs/migration-v0.1-to-v1.0.md create mode 100644 docs/providers/cli.md create mode 100644 docs/providers/http.md create mode 100644 docs/providers/index.md create mode 100644 docs/providers/mcp.md create mode 100644 docs/providers/sse.md create mode 100644 docs/providers/text.md create mode 100644 docs/providers/websocket.md diff --git a/docs/for-tool-providers.md b/docs/for-tool-providers.md new file mode 100644 index 0000000..cf5b80d --- /dev/null +++ b/docs/for-tool-providers.md @@ -0,0 +1,689 @@ +--- +id: for-tool-providers +title: For Tool Providers +sidebar_position: 2 +--- + +# For Tool Providers + +:::info Language Note +This guide uses **Python** examples with the reference implementation. UTCP implementations are available in multiple languages - check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) for TypeScript, Go, and other language implementations. +::: + +This guide helps you expose your tools through UTCP so they can be discovered and used by AI agents and other applications. + +## Overview + +As a tool provider, you'll create a **UTCP Manual** - a standardized description of your tools that tells clients how to call them directly using their native protocols. This eliminates the need for wrapper servers and allows direct communication. + +## Quick Start + +### 1. Create a Simple Manual + +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "info": { + "title": "My API Tools", + "version": "1.0.0", + "description": "Collection of useful API tools" + }, + "tools": [ + { + "name": "get_user", + "description": "Retrieve user information by ID", + "inputs": { + "type": "object", + "properties": { + "user_id": {"type": "string", "description": "User identifier"} + }, + "required": ["user_id"] + }, + "outputs": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + "email": {"type": "string"} + } + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/users/${user_id}", + "http_method": "GET", + "headers": { + "Authorization": "Bearer ${API_TOKEN}" + } + } + } + ] +} +``` + +### 2. Expose via Discovery Endpoint + +```python +from fastapi import FastAPI + +app = FastAPI() + +@app.get("/utcp") +def get_utcp_manual(): + return { + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "tools": [ + # ... your tools here + ] + } + +# Your existing API endpoints remain unchanged +@app.get("/users/{user_id}") +def get_user(user_id: str): + return {"id": user_id, "name": "John Doe", "email": "john@example.com"} +``` + +## Manual Structure + +### Required Fields + +| Field | Type | Description | +|-------|------|-------------| +| `manual_version` | string | Version of your manual (semantic versioning) | +| `utcp_version` | string | UTCP specification version | +| `tools` | array | List of available tools | + +### Optional Fields + +| Field | Type | Description | +|-------|------|-------------| +| `info` | object | Metadata about your API | +| `auth` | object | Default authentication for all tools | +| `variables` | object | Default variable values | + +### Tool Definition + +Each tool must include: + +| Field | Type | Description | +|-------|------|-------------| +| `name` | string | Unique tool identifier | +| `description` | string | Human-readable description | +| `inputs` | object | JSON Schema for input parameters | +| `tool_call_template` | object | How to call the tool | + +Optional tool fields: + +| Field | Type | Description | +|-------|------|-------------| +| `outputs` | object | JSON Schema for expected outputs | +| `tags` | array | Tags for categorization and search | +| `examples` | array | Usage examples | + +## Communication Protocols + +### HTTP Tools + +Most common for REST APIs: + +```json +{ + "name": "create_user", + "description": "Create a new user account", + "inputs": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string", "format": "email"} + }, + "required": ["name", "email"] + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/users", + "http_method": "POST", + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer ${API_TOKEN}" + }, + "body": { + "name": "${name}", + "email": "${email}" + } + } +} +``` + +### CLI Tools + +For command-line applications: + +```json +{ + "name": "git_status", + "description": "Get git repository status", + "inputs": { + "type": "object", + "properties": { + "repo_path": {"type": "string", "description": "Path to git repository"} + }, + "required": ["repo_path"] + }, + "tool_call_template": { + "call_template_type": "cli", + "command": "git", + "args": ["status", "--porcelain"], + "working_directory": "${repo_path}" + } +} +``` + +### WebSocket Tools + +For real-time communication: + +```json +{ + "name": "subscribe_updates", + "description": "Subscribe to real-time updates", + "inputs": { + "type": "object", + "properties": { + "channel": {"type": "string"} + }, + "required": ["channel"] + }, + "tool_call_template": { + "call_template_type": "websocket", + "url": "wss://api.example.com/ws", + "message": { + "action": "subscribe", + "channel": "${channel}" + } + } +} +``` + +## Authentication + +### API Key Authentication + +```json +{ + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "X-API-Key", + "location": "header" + } +} +``` + +### Bearer Token + +```json +{ + "auth": { + "auth_type": "api_key", + "api_key": "${ACCESS_TOKEN}", + "var_name": "Authorization", + "location": "header" + } +} +``` + +### OAuth2 + +```json +{ + "auth": { + "auth_type": "oauth2", + "client_id": "${CLIENT_ID}", + "client_secret": "${CLIENT_SECRET}", + "token_url": "https://auth.example.com/token", + "scope": "read:users write:users" + } +} +``` + +### Per-Tool Authentication + +```json +{ + "name": "admin_action", + "description": "Perform admin action", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/admin/action", + "http_method": "POST", + "auth": { + "auth_type": "api_key", + "api_key": "${ADMIN_TOKEN}", + "var_name": "Authorization", + "location": "header" + } + } +} +``` + +## Variable Substitution + +Use `${VARIABLE_NAME}` syntax for dynamic values: + +### From Tool Arguments + +```json +{ + "url": "https://api.example.com/users/${user_id}", + "body": { + "name": "${name}", + "email": "${email}" + } +} +``` + +### From Environment Variables + +```json +{ + "headers": { + "Authorization": "Bearer ${API_TOKEN}", + "X-Client-ID": "${CLIENT_ID}" + } +} +``` + +### Default Values + +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "variables": { + "base_url": "https://api.example.com", + "timeout": 30 + }, + "tools": [ + { + "name": "get_data", + "tool_call_template": { + "call_template_type": "http", + "url": "${base_url}/data", + "timeout": "${timeout}" + } + } + ] +} +``` + +## Implementation Examples + +### FastAPI Implementation (Python) + +```python +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel +from typing import List, Optional + +app = FastAPI() + +class User(BaseModel): + id: str + name: str + email: str + +# Your existing API endpoints +@app.get("/users/{user_id}") +def get_user(user_id: str) -> User: + # Your existing logic + return User(id=user_id, name="John Doe", email="john@example.com") + +@app.post("/users") +def create_user(user: User) -> User: + # Your existing logic + return user + +# UTCP Discovery endpoint +@app.get("/utcp") +def get_utcp_manual(): + return { + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "info": { + "title": "User Management API", + "version": "1.0.0", + "description": "API for managing user accounts" + }, + "tools": [ + { + "name": "get_user", + "description": "Retrieve user information by ID", + "inputs": { + "type": "object", + "properties": { + "user_id": {"type": "string"} + }, + "required": ["user_id"] + }, + "outputs": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + "email": {"type": "string"} + } + }, + "tool_call_template": { + "call_template_type": "http", + "url": f"{BASE_URL}/users/{{user_id}}", + "http_method": "GET", + "headers": { + "Authorization": "Bearer ${API_TOKEN}" + } + } + }, + { + "name": "create_user", + "description": "Create a new user account", + "inputs": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string", "format": "email"} + }, + "required": ["name", "email"] + }, + "tool_call_template": { + "call_template_type": "http", + "url": f"{BASE_URL}/users", + "http_method": "POST", + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer ${API_TOKEN}" + }, + "body": { + "name": "${name}", + "email": "${email}" + } + } + } + ] + } +``` + +### Express.js Implementation + +```javascript +const express = require('express'); +const app = express(); + +// Your existing API endpoints +app.get('/users/:userId', (req, res) => { + // Your existing logic + res.json({ + id: req.params.userId, + name: 'John Doe', + email: 'john@example.com' + }); +}); + +// UTCP Discovery endpoint +app.get('/utcp', (req, res) => { + res.json({ + manual_version: "1.0.0", + utcp_version: "1.0.1", + info: { + title: "User Management API", + version: "1.0.0", + description: "API for managing user accounts" + }, + tools: [ + { + name: "get_user", + description: "Retrieve user information by ID", + inputs: { + type: "object", + properties: { + user_id: { type: "string" } + }, + required: ["user_id"] + }, + tool_call_template: { + call_template_type: "http", + url: `${process.env.BASE_URL}/users/\${user_id}`, + http_method: "GET", + headers: { + "Authorization": "Bearer ${API_TOKEN}" + } + } + } + ] + }); +}); + +app.listen(3000); +``` + +## OpenAPI Integration + +Convert existing OpenAPI specifications to UTCP manuals: + +### Python with utcp-http + +```python +from utcp_http.openapi_converter import OpenApiConverter + +# Convert OpenAPI spec to UTCP manual +converter = OpenApiConverter() +manual = await converter.convert_openapi_to_manual( + "https://api.example.com/openapi.json", + base_url="https://api.example.com" +) + +# Serve the converted manual +@app.get("/utcp") +def get_utcp_manual(): + return manual.model_dump() +``` + +### Customizing OpenAPI Conversion + +```python +# Convert with custom settings +manual = await converter.convert_openapi_to_manual( + "https://api.example.com/openapi.json", + base_url="https://api.example.com", + include_operations=["get", "post"], # Only include specific operations + exclude_paths=["/internal/*"], # Exclude internal paths + auth_template={ # Default auth for all tools + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "X-API-Key", + "location": "header" + } +) +``` + +## Best Practices + +### Manual Design + +1. **Clear Descriptions**: Write clear, concise tool descriptions +2. **Comprehensive Schemas**: Use detailed JSON schemas for inputs/outputs +3. **Consistent Naming**: Use consistent naming conventions +4. **Version Management**: Use semantic versioning for your manual +5. **Documentation**: Include examples and usage notes + +### Security + +1. **Authentication**: Always implement proper authentication +2. **Input Validation**: Validate all inputs on your API side +3. **Rate Limiting**: Implement rate limiting to prevent abuse +4. **HTTPS Only**: Use HTTPS for all production endpoints +5. **Credential Management**: Never hardcode credentials in manuals + +### Performance + +1. **Efficient Endpoints**: Design efficient API endpoints +2. **Caching**: Implement appropriate caching strategies +3. **Pagination**: Use pagination for large result sets +4. **Timeouts**: Set reasonable timeout values +5. **Monitoring**: Monitor API performance and usage + +### Maintenance + +1. **Versioning**: Version your manual and API together +2. **Backward Compatibility**: Maintain backward compatibility when possible +3. **Deprecation**: Provide clear deprecation notices +4. **Testing**: Test your manual with UTCP clients +5. **Documentation**: Keep documentation up to date + +## Testing Your Manual + +### Manual Validation + +```python +from utcp.data.utcp_manual import UtcpManual +import json + +# Load and validate your manual +with open('manual.json', 'r') as f: + manual_data = json.load(f) + +try: + manual = UtcpManual(**manual_data) + print("Manual is valid!") +except Exception as e: + print(f"Manual validation failed: {e}") +``` + +### Integration Testing + +```python +import pytest +from utcp.utcp_client import UtcpClient + +@pytest.mark.asyncio +async def test_manual_integration(): + client = await UtcpClient.create(config={ + "manual_call_templates": [ + { + "name": "test_service", + "call_template_type": "http", + "url": "http://localhost:8000/utcp", + "http_method": "GET" + } + ] + }) + + # Test tool discovery + tools = await client.list_tools() + assert len(tools) > 0 + + # Test tool call + result = await client.call_tool( + "test_service.get_user", + tool_args={"user_id": "123"} + ) + assert "id" in result +``` + +## Migration Strategies + +### From OpenAPI + +1. **Automatic Conversion**: Use OpenAPI converter for initial conversion +2. **Manual Refinement**: Refine converted manual for better descriptions +3. **Authentication Setup**: Configure authentication properly +4. **Testing**: Test converted manual thoroughly + +### From MCP + +1. **Wrapper Approach**: Use UTCP-MCP plugin initially +2. **Gradual Migration**: Migrate tools one by one to native protocols +3. **Direct Implementation**: Implement tools using native UTCP protocols +4. **Deprecation**: Remove MCP dependency once migration is complete + +## Common Patterns + +### CRUD Operations + +```json +{ + "tools": [ + { + "name": "create_resource", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/resources", + "http_method": "POST" + } + }, + { + "name": "get_resource", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/resources/${id}", + "http_method": "GET" + } + }, + { + "name": "update_resource", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/resources/${id}", + "http_method": "PUT" + } + }, + { + "name": "delete_resource", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/resources/${id}", + "http_method": "DELETE" + } + } + ] +} +``` + +### Batch Operations + +```json +{ + "name": "batch_process", + "description": "Process multiple items in batch", + "inputs": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": {"type": "object"} + } + } + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/batch", + "http_method": "POST", + "body": { + "items": "${items}" + } + } +} +``` + +## Next Steps + +1. **Design Your Manual**: Plan your tool structure and descriptions +2. **Choose Protocols**: Select appropriate communication protocols +3. **Implement Discovery**: Add the `/utcp` endpoint to your API +4. **Test Integration**: Test with UTCP clients +5. **Monitor Usage**: Monitor how your tools are being used +6. **Iterate**: Improve based on usage patterns and feedback + +For more information, see: +- [Communication Protocols](./providers/index.md) +- [Implementation Guide](./implementation.md) +- [Security Considerations](./security.md) diff --git a/docs/implementation.md b/docs/implementation.md new file mode 100644 index 0000000..06abbba --- /dev/null +++ b/docs/implementation.md @@ -0,0 +1,580 @@ +--- +id: implementation +title: Implementation Guide +sidebar_position: 4 +--- + +# Implementation Guide + +:::info Language Note +This guide uses **Python** examples with the reference implementation. UTCP implementations are available in multiple languages - check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) for TypeScript, Go, and other language implementations. +::: + +This comprehensive guide walks you through implementing UTCP in your applications, whether you're creating a tool provider or building a client that consumes tools. + +## Quick Start (Python) + +### 1. Install UTCP + +```bash +# Core UTCP library +pip install utcp + +# Protocol plugins (install as needed) +pip install utcp-http utcp-cli utcp-websocket utcp-text +``` + +### 2. Create Your First Tool Provider + +```python +from fastapi import FastAPI + +app = FastAPI() + +@app.get("/utcp") +def get_utcp_manual(): + return { + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "tools": [ + { + "name": "get_weather", + "description": "Get current weather for a location", + "inputs": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "City name"} + }, + "required": ["location"] + }, + "outputs": { + "type": "object", + "properties": { + "temperature": {"type": "number"}, + "conditions": {"type": "string"} + } + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.weather.com/v1/current", + "http_method": "GET", + "query_params": { + "q": "${location}", + "appid": "${WEATHER_API_KEY}" + } + } + } + ] + } +``` + +### 3. Create Your First Client + +```python +import asyncio +from utcp.utcp_client import UtcpClient + +async def main(): + # Create client with manual discovery + client = await UtcpClient.create(config={ + "manual_call_templates": [ + { + "name": "weather_service", + "call_template_type": "http", + "url": "http://localhost:8000/utcp", + "http_method": "GET" + } + ] + }) + + # Call the tool + result = await client.call_tool( + "weather_service.get_weather", + tool_args={"location": "San Francisco"} + ) + print(f"Weather: {result}") + +if __name__ == "__main__": + asyncio.run(main()) +``` + +## Other Language Implementations + +UTCP is available in multiple programming languages: + +| Language | Repository | Status | +|----------|------------|--------| +| **Python** | [python-utcp](https://github.com/universal-tool-calling-protocol/python-utcp) | ✅ Reference Implementation | +| **TypeScript** | [typescript-utcp](https://github.com/universal-tool-calling-protocol/typescript-utcp) | ✅ Stable | +| **Go** | [go-utcp](https://github.com/universal-tool-calling-protocol/go-utcp) | 🚧 In Development | +| **Rust** | [rust-utcp](https://github.com/universal-tool-calling-protocol/rust-utcp) | 🚧 In Development | + +Visit the respective repositories for language-specific documentation and examples. + +## Tool Provider Implementation (Python) + +### Manual Structure + +A UTCP Manual defines your tools and how to call them: + +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "info": { + "title": "Weather API", + "version": "1.0.0", + "description": "Weather data and forecasting tools" + }, + "tools": [ + { + "name": "get_weather", + "description": "Get current weather conditions", + "inputs": { + "type": "object", + "properties": { + "location": {"type": "string"}, + "units": {"type": "string", "enum": ["metric", "imperial"]} + }, + "required": ["location"] + }, + "outputs": { + "type": "object", + "properties": { + "temperature": {"type": "number"}, + "conditions": {"type": "string"}, + "humidity": {"type": "number"} + } + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.weather.com/v1/current", + "http_method": "GET", + "query_params": { + "q": "${location}", + "units": "${units}", + "appid": "${WEATHER_API_KEY}" + }, + "auth": { + "auth_type": "api_key", + "api_key": "${WEATHER_API_KEY}", + "var_name": "appid", + "location": "query" + } + } + } + ] +} +``` + +### Discovery Endpoint (Python Examples) + +Expose your manual via a discovery endpoint: + +#### FastAPI Example + +```python +from fastapi import FastAPI +from utcp.data.utcp_manual import UtcpManual +from utcp.data.tool import Tool, JsonSchema +from utcp.data.call_template import CallTemplate + +app = FastAPI() + +# Define your manual +manual = UtcpManual( + manual_version="1.0.0", + utcp_version="1.0.1", + tools=[ + Tool( + name="get_weather", + description="Get current weather", + inputs=JsonSchema( + type="object", + properties={ + "location": JsonSchema(type="string") + }, + required=["location"] + ), + tool_call_template=CallTemplate( + call_template_type="http", + url="https://api.weather.com/v1/current", + http_method="GET", + query_params={ + "q": "${location}", + "appid": "${WEATHER_API_KEY}" + } + ) + ) + ] +) + +@app.get("/utcp") +def get_manual(): + return manual.model_dump() +``` + +#### Flask Example + +```python +from flask import Flask, jsonify + +app = Flask(__name__) + +@app.route('/utcp') +def get_manual(): + return jsonify({ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "tools": [ + # ... tool definitions + ] + }) +``` + +### Authentication Patterns (Python) + +#### API Key in Header + +```json +{ + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "X-API-Key", + "location": "header" + } +} +``` + +#### Bearer Token + +```json +{ + "auth": { + "auth_type": "api_key", + "api_key": "${ACCESS_TOKEN}", + "var_name": "Authorization", + "location": "header" + } +} +``` + +#### OAuth2 + +```json +{ + "auth": { + "auth_type": "oauth2", + "client_id": "${CLIENT_ID}", + "client_secret": "${CLIENT_SECRET}", + "token_url": "https://auth.example.com/token", + "scope": "read:data" + } +} +``` + +## Client Implementation (Python) + +### Basic Client Setup + +```python +from utcp.utcp_client import UtcpClient + +# Create client with configuration +client = await UtcpClient.create(config={ + "manual_call_templates": [ + { + "name": "my_service", + "call_template_type": "http", + "url": "https://api.example.com/utcp", + "http_method": "GET" + } + ], + "variable_loaders": [ + { + "loader_type": "env", + "prefix": "UTCP_" + } + ] +}) +``` + +### Configuration Options (Python) + +#### File-based Configuration + +```yaml +# utcp-config.yaml +manual_call_templates: + - name: weather_service + call_template_type: http + url: https://weather.example.com/utcp + http_method: GET + - name: database_service + call_template_type: http + url: https://db.example.com/utcp + http_method: GET + +variable_loaders: + - loader_type: env + prefix: UTCP_ + - loader_type: dotenv + file_path: .env +``` + +```python +client = await UtcpClient.create(config="utcp-config.yaml") +``` + +#### Programmatic Configuration + +```python +from utcp.data.utcp_client_config import UtcpClientConfig +from utcp.data.call_template import CallTemplate + +config = UtcpClientConfig( + manual_call_templates=[ + CallTemplate( + name="weather_service", + call_template_type="http", + url="https://weather.example.com/utcp", + http_method="GET" + ) + ] +) + +client = await UtcpClient.create(config=config) +``` + +### Tool Discovery and Search (Python) + +#### List Available Tools + +```python +# Get all tools +tools = await client.list_tools() +for tool in tools: + print(f"{tool.name}: {tool.description}") + +# Search tools by tag +weather_tools = await client.search_tools(tags=["weather"]) + +# Search tools by description +data_tools = await client.search_tools(description="data processing") +``` + +#### Tool Information + +```python +# Get detailed tool information +tool_info = await client.get_tool("weather_service.get_weather") +print(f"Inputs: {tool_info.inputs}") +print(f"Outputs: {tool_info.outputs}") +``` + +### Calling Tools (Python) + +#### Basic Tool Call + +```python +result = await client.call_tool( + "weather_service.get_weather", + tool_args={"location": "New York"} +) +``` + +#### Tool Call with Context + +```python +result = await client.call_tool( + "weather_service.get_weather", + tool_args={"location": "New York"}, + context={"user_id": "123", "session_id": "abc"} +) +``` + +#### Batch Tool Calls + +```python +results = await client.call_tools([ + ("weather_service.get_weather", {"location": "New York"}), + ("weather_service.get_weather", {"location": "London"}), + ("weather_service.get_weather", {"location": "Tokyo"}) +]) +``` + +### Error Handling (Python) + +```python +from utcp.exceptions import UtcpError, ToolNotFoundError, ToolCallError + +try: + result = await client.call_tool("nonexistent.tool", {}) +except ToolNotFoundError as e: + print(f"Tool not found: {e}") +except ToolCallError as e: + print(f"Tool call failed: {e}") +except UtcpError as e: + print(f"UTCP error: {e}") +``` + +## Advanced Implementation Patterns (Python) + +### Custom Communication Protocols + +```python +from utcp.interfaces.communication_protocol import CommunicationProtocol +from utcp.data.call_template import CallTemplate +from typing import Dict, Any, List + +class CustomProtocol(CommunicationProtocol): + def get_supported_call_template_types(self) -> List[str]: + return ["custom"] + + async def call_tool( + self, + call_template: CallTemplate, + tool_args: Dict[str, Any] + ) -> Any: + # Implement your custom protocol logic + return {"result": "custom protocol response"} + +# Register the protocol +from utcp.plugins.discovery import register_communication_protocol +register_communication_protocol(CustomProtocol()) +``` + +### Custom Tool Repository + +```python +from utcp.interfaces.concurrent_tool_repository import ConcurrentToolRepository +from utcp.data.tool import Tool +from typing import List, Optional + +class DatabaseToolRepository(ConcurrentToolRepository): + async def add_tool(self, tool: Tool, manual_name: str) -> None: + # Store tool in database + pass + + async def get_tool(self, namespaced_name: str) -> Optional[Tool]: + # Retrieve tool from database + pass + + async def list_tools(self) -> List[Tool]: + # List all tools from database + pass + + # ... implement other methods + +# Use custom repository +client = await UtcpClient.create( + config=config, + tool_repository=DatabaseToolRepository() +) +``` + +## Testing (Python) + +### Unit Testing Tool Providers + +```python +import pytest +from fastapi.testclient import TestClient +from your_app import app + +client = TestClient(app) + +def test_utcp_manual(): + response = client.get("/utcp") + assert response.status_code == 200 + + manual = response.json() + assert manual["manual_version"] == "1.0.0" + assert len(manual["tools"]) > 0 + + tool = manual["tools"][0] + assert "name" in tool + assert "description" in tool + assert "tool_call_template" in tool +``` + +### Integration Testing + +```python +import pytest +from utcp.utcp_client import UtcpClient + +@pytest.mark.asyncio +async def test_tool_call(): + client = await UtcpClient.create(config={ + "manual_call_templates": [ + { + "name": "test_service", + "call_template_type": "http", + "url": "http://localhost:8000/utcp", + "http_method": "GET" + } + ] + }) + + result = await client.call_tool( + "test_service.get_weather", + tool_args={"location": "Test City"} + ) + + assert "temperature" in result + assert "conditions" in result +``` + +## Language-Specific Resources + +### Python +- **Repository**: [python-utcp](https://github.com/universal-tool-calling-protocol/python-utcp) +- **Documentation**: [Python UTCP Docs](https://python-utcp.readthedocs.io/) +- **Examples**: [Python Examples](https://github.com/universal-tool-calling-protocol/python-utcp/tree/main/examples) + +### TypeScript +- **Repository**: [typescript-utcp](https://github.com/universal-tool-calling-protocol/typescript-utcp) +- **Documentation**: [TypeScript UTCP Docs](https://typescript-utcp.readthedocs.io/) +- **Examples**: [TypeScript Examples](https://github.com/universal-tool-calling-protocol/typescript-utcp/tree/main/examples) + +### Other Languages +Check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) for the latest language implementations and their respective documentation. + +## Best Practices + +### Tool Provider Best Practices + +1. **Versioning**: Use semantic versioning for your manual +2. **Documentation**: Provide clear descriptions for all tools +3. **Validation**: Validate inputs using JSON Schema +4. **Error Handling**: Return meaningful error messages +5. **Rate Limiting**: Implement appropriate rate limits +6. **Monitoring**: Monitor tool usage and performance +7. **Security**: Implement proper authentication and authorization + +### Client Best Practices + +1. **Configuration Management**: Use external configuration files +2. **Error Handling**: Implement comprehensive error handling +3. **Retry Logic**: Implement retry logic for transient failures +4. **Caching**: Cache tool definitions and results when appropriate +5. **Monitoring**: Monitor client performance and errors +6. **Testing**: Write comprehensive tests for tool integrations +7. **Security**: Store credentials securely + +## Next Steps + +1. **Choose Your Language**: Select from available UTCP implementations +2. **Choose Your Protocols**: Select the communication protocols you need +3. **Design Your Tools**: Plan your tool structure and interfaces +4. **Implement Gradually**: Start with simple tools and expand +5. **Test Thoroughly**: Write comprehensive tests +6. **Monitor and Optimize**: Monitor performance and optimize as needed +7. **Scale**: Plan for scaling as your tool ecosystem grows + +For more detailed information, see: +- [Communication Protocols](./providers/index.md) +- [API Reference](./api/index.md) +- [Security Considerations](./security.md) diff --git a/docs/migration-v0.1-to-v1.0.md b/docs/migration-v0.1-to-v1.0.md new file mode 100644 index 0000000..b220557 --- /dev/null +++ b/docs/migration-v0.1-to-v1.0.md @@ -0,0 +1,601 @@ +--- +id: migration-v0.1-to-v1.0 +title: Migration Guide - v0.1 to v1.0 +sidebar_position: 7 +--- + +# Migration Guide: v0.1 to v1.0 + +This guide helps you migrate from UTCP v0.1 to v1.0, which introduces significant architectural improvements and new features. + +## Overview of Changes + +### Major Changes in v1.0 + +1. **Plugin Architecture**: Core functionality split into pluggable components +2. **Improved Data Models**: Enhanced Pydantic models with better validation +3. **New Protocol Support**: Additional communication protocols +4. **Better Error Handling**: More specific exception types +5. **Enhanced Authentication**: Expanded authentication options +6. **Performance Improvements**: Optimized client and protocol implementations + +### Breaking Changes + +| Component | v0.1 | v1.0 | Impact | +|-----------|------|------|--------| +| **Package Structure** | Single package | Core + plugins | High | +| **Client API** | `UtcpClient()` | `UtcpClient.create()` | Medium | +| **Configuration** | Simple dict | Structured config | Medium | +| **Protocol Registration** | Automatic | Plugin-based | High | +| **Error Types** | Generic exceptions | Specific exception types | Low | + +## Installation Changes + +### v0.1 Installation + +```bash +pip install utcp +``` + +### v1.0 Installation + +```bash +# Core package +pip install utcp + +# Protocol plugins (install as needed) +pip install utcp-http utcp-cli utcp-websocket utcp-text utcp-mcp +``` + +## Client Migration + +### v0.1 Client Code + +```python +from utcp import UtcpClient + +# Old way +client = UtcpClient(config={ + "providers": [ + { + "name": "weather_service", + "provider_type": "http", + "url": "https://weather.example.com/utcp", + "http_method": "GET" + } + ] +}) + +# Call tool +result = client.call_tool("weather_service.get_weather", {"location": "NYC"}) +``` + +### v1.0 Client Code + +```python +from utcp.utcp_client import UtcpClient + +# New way - async factory method +client = await UtcpClient.create(config={ + "manual_call_templates": [ + { + "name": "weather_service", + "call_template_type": "http", + "url": "https://weather.example.com/utcp", + "http_method": "GET" + } + ] +}) + +# Call tool - now async +result = await client.call_tool("weather_service.get_weather", {"location": "NYC"}) +``` + +## Configuration Migration + +### v0.1 Configuration + +```yaml +providers: + - name: weather_service + provider_type: http + url: https://weather.example.com/utcp + http_method: GET + - name: file_service + provider_type: cli + command: cat + args: ["${filename}"] + +variables: + API_KEY: your_api_key +``` + +### v1.0 Configuration + +```yaml +manual_call_templates: + - name: weather_service + call_template_type: http + url: https://weather.example.com/utcp + http_method: GET + - name: file_service + call_template_type: cli + command: cat + args: ["${filename}"] + +variable_loaders: + - loader_type: env + prefix: UTCP_ + - loader_type: dotenv + file_path: .env +``` + +## Manual Format Migration + +### v0.1 Manual Format + +```json +{ + "utcp_version": "0.1.0", + "provider_info": { + "name": "weather_api", + "version": "1.0.0" + }, + "tools": [ + { + "name": "get_weather", + "description": "Get weather data", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + } + }, + "provider": { + "provider_type": "http", + "url": "https://api.weather.com/current", + "method": "GET" + } + } + ] +} +``` + +### v1.0 Manual Format + +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "info": { + "title": "Weather API", + "version": "1.0.0", + "description": "Weather data and forecasting tools" + }, + "tools": [ + { + "name": "get_weather", + "description": "Get weather data", + "inputs": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + }, + "outputs": { + "type": "object", + "properties": { + "temperature": {"type": "number"}, + "conditions": {"type": "string"} + } + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.weather.com/current", + "http_method": "GET", + "query_params": { + "location": "${location}" + } + } + } + ] +} +``` + +## Protocol Migration + +### HTTP Protocol Changes + +#### v0.1 HTTP Provider + +```json +{ + "provider_type": "http", + "url": "https://api.example.com/endpoint", + "method": "POST", + "headers": {"Authorization": "Bearer ${TOKEN}"}, + "body": {"data": "${input}"} +} +``` + +#### v1.0 HTTP Call Template + +```json +{ + "call_template_type": "http", + "url": "https://api.example.com/endpoint", + "http_method": "POST", + "headers": {"Authorization": "Bearer ${TOKEN}"}, + "body": {"data": "${input}"}, + "auth": { + "auth_type": "api_key", + "api_key": "${TOKEN}", + "var_name": "Authorization", + "location": "header" + } +} +``` + +### CLI Protocol Changes + +#### v0.1 CLI Provider + +```json +{ + "provider_type": "cli", + "command": "python", + "args": ["script.py", "${input}"], + "cwd": "/app" +} +``` + +#### v1.0 CLI Call Template + +```json +{ + "call_template_type": "cli", + "command": "python", + "args": ["script.py", "${input}"], + "working_directory": "/app", + "timeout": 30, + "environment": { + "PYTHONPATH": "/app/lib" + } +} +``` + +## Authentication Migration + +### v0.1 Authentication + +```json +{ + "provider": { + "provider_type": "http", + "url": "https://api.example.com/data", + "headers": { + "Authorization": "Bearer ${API_TOKEN}" + } + } +} +``` + +### v1.0 Authentication + +```json +{ + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/data", + "auth": { + "auth_type": "api_key", + "api_key": "${API_TOKEN}", + "var_name": "Authorization", + "location": "header" + } + } +} +``` + +## Error Handling Migration + +### v0.1 Error Handling + +```python +try: + result = client.call_tool("service.tool", args) +except Exception as e: + print(f"Error: {e}") +``` + +### v1.0 Error Handling + +```python +from utcp.exceptions import ( + UtcpError, + ToolNotFoundError, + ToolCallError, + AuthenticationError +) + +try: + result = await client.call_tool("service.tool", args) +except ToolNotFoundError as e: + print(f"Tool not found: {e}") +except AuthenticationError as e: + print(f"Authentication failed: {e}") +except ToolCallError as e: + print(f"Tool call failed: {e}") +except UtcpError as e: + print(f"UTCP error: {e}") +``` + +## Step-by-Step Migration + +### Step 1: Update Dependencies + +```bash +# Uninstall old version +pip uninstall utcp + +# Install new version with plugins +pip install utcp utcp-http utcp-cli utcp-websocket utcp-text +``` + +### Step 2: Update Client Code + +```python +# Before +from utcp import UtcpClient + +def main(): + client = UtcpClient(config=config) + result = client.call_tool("service.tool", args) + return result + +# After +import asyncio +from utcp.utcp_client import UtcpClient + +async def main(): + client = await UtcpClient.create(config=config) + result = await client.call_tool("service.tool", args) + return result + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### Step 3: Update Configuration + +```python +# Migration helper function +def migrate_config_v0_to_v1(old_config): + new_config = { + "manual_call_templates": [], + "variable_loaders": [ + {"loader_type": "env", "prefix": "UTCP_"} + ] + } + + for provider in old_config.get("providers", []): + call_template = { + "name": provider["name"], + "call_template_type": provider["provider_type"], + } + + # Migrate HTTP providers + if provider["provider_type"] == "http": + call_template.update({ + "url": provider["url"], + "http_method": provider.get("method", "GET"), + "headers": provider.get("headers", {}), + "body": provider.get("body") + }) + + # Migrate CLI providers + elif provider["provider_type"] == "cli": + call_template.update({ + "command": provider["command"], + "args": provider.get("args", []), + "working_directory": provider.get("cwd") + }) + + new_config["manual_call_templates"].append(call_template) + + return new_config + +# Use migration helper +old_config = load_old_config() +new_config = migrate_config_v0_to_v1(old_config) +client = await UtcpClient.create(config=new_config) +``` + +### Step 4: Update Manual Format + +```python +# Migration helper for manuals +def migrate_manual_v0_to_v1(old_manual): + new_manual = { + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "info": { + "title": old_manual.get("provider_info", {}).get("name", "API"), + "version": old_manual.get("provider_info", {}).get("version", "1.0.0"), + "description": old_manual.get("provider_info", {}).get("description", "") + }, + "tools": [] + } + + for tool in old_manual.get("tools", []): + new_tool = { + "name": tool["name"], + "description": tool["description"], + "inputs": tool.get("parameters", {}), + "tool_call_template": {} + } + + # Migrate provider to call_template + provider = tool.get("provider", {}) + if provider.get("provider_type") == "http": + new_tool["tool_call_template"] = { + "call_template_type": "http", + "url": provider["url"], + "http_method": provider.get("method", "GET"), + "headers": provider.get("headers", {}), + "body": provider.get("body") + } + + new_manual["tools"].append(new_tool) + + return new_manual +``` + +### Step 5: Test Migration + +```python +import pytest +from utcp.utcp_client import UtcpClient + +@pytest.mark.asyncio +async def test_migrated_client(): + # Test with migrated configuration + client = await UtcpClient.create(config=migrated_config) + + # Test tool discovery + tools = await client.list_tools() + assert len(tools) > 0 + + # Test tool calls + result = await client.call_tool("service.tool", {"param": "value"}) + assert result is not None +``` + +## Common Migration Issues + +### Issue 1: Async/Await + +**Problem**: v1.0 client methods are async +**Solution**: Add `async`/`await` keywords + +```python +# Before +result = client.call_tool("tool", args) + +# After +result = await client.call_tool("tool", args) +``` + +### Issue 2: Configuration Format + +**Problem**: Configuration structure changed +**Solution**: Use migration helper or update manually + +```python +# Before +config = {"providers": [...]} + +# After +config = {"manual_call_templates": [...]} +``` + +### Issue 3: Plugin Dependencies + +**Problem**: Protocol implementations not found +**Solution**: Install required plugins + +```bash +pip install utcp-http utcp-cli utcp-websocket +``` + +### Issue 4: Manual Format + +**Problem**: Old manual format not recognized +**Solution**: Update manual structure + +```json +// Before +{"provider": {"provider_type": "http"}} + +// After +{"tool_call_template": {"call_template_type": "http"}} +``` + +## Validation Tools + +### Configuration Validator + +```python +from utcp.data.utcp_client_config import UtcpClientConfig + +def validate_config(config_dict): + try: + config = UtcpClientConfig(**config_dict) + print("Configuration is valid!") + return config + except Exception as e: + print(f"Configuration validation failed: {e}") + return None +``` + +### Manual Validator + +```python +from utcp.data.utcp_manual import UtcpManual + +def validate_manual(manual_dict): + try: + manual = UtcpManual(**manual_dict) + print("Manual is valid!") + return manual + except Exception as e: + print(f"Manual validation failed: {e}") + return None +``` + +## Best Practices for Migration + +1. **Gradual Migration**: Migrate one component at a time +2. **Test Thoroughly**: Test each migrated component +3. **Backup Configurations**: Keep backups of v0.1 configurations +4. **Use Validation**: Validate configurations and manuals +5. **Monitor Performance**: Check for performance regressions +6. **Update Documentation**: Update internal documentation +7. **Train Team**: Ensure team understands new patterns + +## Post-Migration Checklist + +- [ ] All dependencies updated +- [ ] Client code uses async/await +- [ ] Configuration format updated +- [ ] Manual format updated +- [ ] Error handling updated +- [ ] Tests passing +- [ ] Performance acceptable +- [ ] Documentation updated +- [ ] Team trained on changes + +## Getting Help + +If you encounter issues during migration: + +1. **Check Documentation**: Review the [Implementation Guide](./implementation.md) +2. **GitHub Issues**: Search existing issues or create new ones +3. **Discord Community**: Join the [UTCP Discord](https://discord.gg/ZpMbQ8jRbD) +4. **Examples**: Check the [examples repository](https://github.com/universal-tool-calling-protocol/python-utcp/tree/main/examples) + +## Rollback Plan + +If migration issues occur, you can rollback: + +```bash +# Rollback to v0.1 +pip uninstall utcp utcp-http utcp-cli utcp-websocket utcp-text +pip install utcp==0.1.0 + +# Restore old configuration files +cp config-v0.1-backup.yaml config.yaml +``` + +Remember to test the rollback process in a non-production environment first. diff --git a/docs/providers/cli.md b/docs/providers/cli.md new file mode 100644 index 0000000..05caf92 --- /dev/null +++ b/docs/providers/cli.md @@ -0,0 +1,323 @@ +--- +id: cli +title: CLI Protocol +sidebar_position: 4 +--- + +# CLI Protocol + +The CLI protocol plugin (`utcp-cli`) enables UTCP to execute command-line tools and scripts. This is particularly useful for wrapping existing CLI applications and making them available to AI agents. + +## Installation + +```bash +pip install utcp-cli +``` + +## Call Template Structure + +```json +{ + "call_template_type": "cli", + "command": "curl", + "args": [ + "-X", "GET", + "-H", "Authorization: Bearer ${API_TOKEN}", + "https://api.example.com/data" + ], + "working_directory": "/tmp", + "environment": { + "API_TOKEN": "${API_TOKEN}" + }, + "timeout": 30 +} +``` + +## Configuration Options + +### Required Fields + +| Field | Type | Description | +|-------|------|-------------| +| `call_template_type` | string | Must be `"cli"` | +| `command` | string | The command to execute | + +### Optional Fields + +| Field | Type | Description | +|-------|------|-------------| +| `args` | array | Command arguments | +| `working_directory` | string | Working directory for command execution | +| `environment` | object | Environment variables | +| `timeout` | number | Execution timeout in seconds (default: 30) | +| `shell` | boolean | Whether to execute through shell (default: false) | +| `capture_output` | boolean | Whether to capture stdout/stderr (default: true) | + +## Security Considerations + +:::danger Security Warning +CLI protocol executes commands on the local system. Always validate inputs and use with caution. +::: + +### Input Validation + +Always validate and sanitize inputs to prevent command injection: + +```json +{ + "name": "safe_file_read", + "description": "Safely read a file", + "inputs": { + "type": "object", + "properties": { + "filename": { + "type": "string", + "pattern": "^[a-zA-Z0-9._-]+$" + } + }, + "required": ["filename"] + }, + "tool_call_template": { + "call_template_type": "cli", + "command": "cat", + "args": ["${filename}"], + "working_directory": "/safe/directory" + } +} +``` + +### Sandboxing + +Consider running CLI tools in sandboxed environments: + +```json +{ + "call_template_type": "cli", + "command": "docker", + "args": [ + "run", "--rm", "--read-only", + "-v", "/safe/data:/data:ro", + "alpine:latest", + "cat", "/data/${filename}" + ] +} +``` + +## Examples + +### Simple Command Execution + +```json +{ + "name": "get_system_info", + "description": "Get system information", + "inputs": { + "type": "object", + "properties": {} + }, + "tool_call_template": { + "call_template_type": "cli", + "command": "uname", + "args": ["-a"] + } +} +``` + +### File Operations + +```json +{ + "name": "list_directory", + "description": "List files in a directory", + "inputs": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Directory path to list" + } + }, + "required": ["path"] + }, + "tool_call_template": { + "call_template_type": "cli", + "command": "ls", + "args": ["-la", "${path}"], + "timeout": 10 + } +} +``` + +### Script Execution + +```json +{ + "name": "run_analysis", + "description": "Run data analysis script", + "inputs": { + "type": "object", + "properties": { + "input_file": {"type": "string"}, + "output_format": {"type": "string", "enum": ["json", "csv"]} + }, + "required": ["input_file"] + }, + "tool_call_template": { + "call_template_type": "cli", + "command": "python", + "args": [ + "/scripts/analyze.py", + "--input", "${input_file}", + "--format", "${output_format}" + ], + "working_directory": "/workspace", + "environment": { + "PYTHONPATH": "/workspace/lib" + }, + "timeout": 300 + } +} +``` + +### Git Operations + +```json +{ + "name": "git_status", + "description": "Get git repository status", + "inputs": { + "type": "object", + "properties": { + "repo_path": {"type": "string"} + }, + "required": ["repo_path"] + }, + "tool_call_template": { + "call_template_type": "cli", + "command": "git", + "args": ["status", "--porcelain"], + "working_directory": "${repo_path}" + } +} +``` + +## Output Handling + +The CLI protocol captures and returns: + +- **stdout**: Standard output as the primary result +- **stderr**: Standard error (included in error cases) +- **return_code**: Process exit code +- **execution_time**: Time taken to execute + +### Success Response + +```json +{ + "stdout": "file1.txt\nfile2.txt\n", + "stderr": "", + "return_code": 0, + "execution_time": 0.123 +} +``` + +### Error Response + +```json +{ + "stdout": "", + "stderr": "ls: cannot access '/invalid/path': No such file or directory\n", + "return_code": 2, + "execution_time": 0.045 +} +``` + +## Environment Variables + +Set environment variables for command execution: + +```json +{ + "call_template_type": "cli", + "command": "node", + "args": ["app.js"], + "environment": { + "NODE_ENV": "production", + "API_KEY": "${API_KEY}", + "PORT": "3000" + } +} +``` + +## Working Directory + +Specify the working directory for command execution: + +```json +{ + "call_template_type": "cli", + "command": "make", + "args": ["build"], + "working_directory": "/project/src" +} +``` + +## Best Practices + +1. **Validate Inputs**: Always validate and sanitize user inputs +2. **Use Absolute Paths**: Prefer absolute paths for commands and files +3. **Set Timeouts**: Configure appropriate timeouts to prevent hanging +4. **Limit Permissions**: Run with minimal necessary permissions +5. **Sandbox Execution**: Use containers or chroot when possible +6. **Log Execution**: Log all command executions for audit trails +7. **Handle Errors**: Properly handle and report command failures + +## Common Use Cases + +- **Development Tools**: Git, npm, pip, docker commands +- **System Administration**: File operations, process management +- **Data Processing**: Scripts for ETL, analysis, reporting +- **Build Systems**: Make, gradle, webpack execution +- **Testing**: Running test suites and validation scripts + +## Error Handling + +| Error Type | Description | Handling | +|------------|-------------|----------| +| Command Not Found | Command doesn't exist | Raise `CommandNotFoundError` | +| Permission Denied | Insufficient permissions | Raise `PermissionError` | +| Timeout | Command exceeded timeout | Raise `TimeoutError` | +| Non-zero Exit | Command failed | Include stderr in error | + +## Platform Considerations + +### Windows + +```json +{ + "call_template_type": "cli", + "command": "cmd", + "args": ["/c", "dir", "${path}"], + "shell": true +} +``` + +### Unix/Linux + +```json +{ + "call_template_type": "cli", + "command": "ls", + "args": ["-la", "${path}"] +} +``` + +### Cross-platform + +```json +{ + "call_template_type": "cli", + "command": "python", + "args": ["-c", "import os; print(os.listdir('${path}'))"] +} +``` diff --git a/docs/providers/http.md b/docs/providers/http.md new file mode 100644 index 0000000..6f67741 --- /dev/null +++ b/docs/providers/http.md @@ -0,0 +1,258 @@ +--- +id: http +title: HTTP Protocol +sidebar_position: 1 +--- + +# HTTP Protocol + +The HTTP protocol plugin (`utcp-http`) enables UTCP to call REST APIs, webhooks, and any HTTP-based services. It's the most commonly used protocol and provides comprehensive support for modern web APIs. + +## Installation + +```bash +pip install utcp-http +``` + +## Call Template Structure + +```json +{ + "call_template_type": "http", + "url": "https://api.example.com/endpoint", + "http_method": "POST", + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer ${API_TOKEN}" + }, + "body": { + "query": "${query}", + "limit": 10 + }, + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "X-API-Key", + "location": "header" + } +} +``` + +## Configuration Options + +### Required Fields + +| Field | Type | Description | +|-------|------|-------------| +| `call_template_type` | string | Must be `"http"` | +| `url` | string | The HTTP endpoint URL | +| `http_method` | string | HTTP method (GET, POST, PUT, DELETE, etc.) | + +### Optional Fields + +| Field | Type | Description | +|-------|------|-------------| +| `headers` | object | HTTP headers to include | +| `body` | object/string | Request body for POST/PUT requests | +| `query_params` | object | URL query parameters | +| `timeout` | number | Request timeout in seconds (default: 30) | +| `follow_redirects` | boolean | Whether to follow HTTP redirects (default: true) | +| `verify_ssl` | boolean | Whether to verify SSL certificates (default: true) | + +## Authentication + +The HTTP protocol supports multiple authentication methods: + +### API Key Authentication + +```json +{ + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "X-API-Key", + "location": "header" + } +} +``` + +**Locations:** +- `"header"`: Add as HTTP header +- `"query"`: Add as query parameter +- `"cookie"`: Add as cookie + +### Basic Authentication + +```json +{ + "auth": { + "auth_type": "basic", + "username": "${USERNAME}", + "password": "${PASSWORD}" + } +} +``` + +### OAuth2 Bearer Token + +```json +{ + "auth": { + "auth_type": "oauth2", + "client_id": "${CLIENT_ID}", + "client_secret": "${CLIENT_SECRET}", + "token_url": "https://auth.example.com/token", + "scope": "read:data" + } +} +``` + +## Variable Substitution + +Use `${VARIABLE_NAME}` syntax to substitute values at runtime: + +```json +{ + "url": "https://api.example.com/users/${user_id}", + "headers": { + "Authorization": "Bearer ${access_token}" + }, + "body": { + "name": "${user_name}", + "email": "${user_email}" + } +} +``` + +Variables can come from: +- Tool arguments +- Environment variables +- Configuration files +- Runtime context + +## Examples + +### Simple GET Request + +```json +{ + "name": "get_weather", + "description": "Get current weather for a location", + "inputs": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.weather.com/v1/current", + "http_method": "GET", + "query_params": { + "q": "${location}", + "appid": "${WEATHER_API_KEY}" + } + } +} +``` + +### POST with JSON Body + +```json +{ + "name": "create_user", + "description": "Create a new user account", + "inputs": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string"} + }, + "required": ["name", "email"] + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/users", + "http_method": "POST", + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer ${ACCESS_TOKEN}" + }, + "body": { + "name": "${name}", + "email": "${email}", + "created_at": "{{now}}" + } + } +} +``` + +### File Upload + +```json +{ + "name": "upload_file", + "description": "Upload a file to the server", + "inputs": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "description": {"type": "string"} + }, + "required": ["file_path"] + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/upload", + "http_method": "POST", + "headers": { + "Authorization": "Bearer ${ACCESS_TOKEN}" + }, + "body": { + "file": "@${file_path}", + "description": "${description}" + } + } +} +``` + +## Error Handling + +The HTTP protocol handles various error conditions: + +| HTTP Status | Behavior | +|-------------|----------| +| 200-299 | Success - return response body | +| 400-499 | Client error - raise `HttpClientError` | +| 500-599 | Server error - raise `HttpServerError` | +| Timeout | Raise `HttpTimeoutError` | +| Connection | Raise `HttpConnectionError` | + +## Best Practices + +1. **Use HTTPS**: Always use secure connections for production APIs +2. **Set Timeouts**: Configure appropriate timeouts for your use case +3. **Handle Rate Limits**: Implement retry logic for rate-limited APIs +4. **Validate Inputs**: Use JSON Schema to validate tool inputs +5. **Secure Credentials**: Store API keys and tokens securely +6. **Monitor Usage**: Track API usage and performance metrics + +## OpenAPI Integration + +The HTTP protocol can automatically generate UTCP manuals from OpenAPI specifications: + +```python +from utcp_http.openapi_converter import OpenApiConverter + +converter = OpenApiConverter() +manual = await converter.convert_openapi_to_manual( + "https://api.example.com/openapi.json" +) +``` + +## Related Protocols + +- [Server-Sent Events](./sse.md) - For streaming HTTP responses +- [Streamable HTTP](./streamable-http.md) - For chunked HTTP responses +- [WebSocket](./websocket.md) - For bidirectional real-time communication diff --git a/docs/providers/index.md b/docs/providers/index.md new file mode 100644 index 0000000..f1ad2ba --- /dev/null +++ b/docs/providers/index.md @@ -0,0 +1,92 @@ +--- +id: providers +title: Communication Protocols +sidebar_position: 3 +--- + +# Communication Protocols + +UTCP supports multiple communication protocols through its plugin architecture. Each protocol plugin provides the necessary implementation to call tools using that specific transport method. + +## Available Protocols + +### Core Protocols + +| Protocol | Plugin | Status | Description | +|----------|--------|--------|-------------| +| [HTTP](./http.md) | `utcp-http` | ✅ Stable | REST APIs, webhooks, and standard HTTP services | +| [Server-Sent Events](./sse.md) | `utcp-http` | ✅ Stable | Real-time streaming over HTTP | +| [WebSocket](./websocket.md) | `utcp-websocket` | ✅ Stable | Bidirectional real-time communication | +| [CLI](./cli.md) | `utcp-cli` | ✅ Stable | Command-line tools and scripts | +| [Text Files](./text.md) | `utcp-text` | ✅ Stable | Reading local and remote text files | + +### Integration Protocols + +| Protocol | Plugin | Status | Description | +|----------|--------|--------|-------------| +| [MCP](./mcp.md) | `utcp-mcp` | ✅ Stable | Model Context Protocol interoperability | + +### Experimental Protocols + +| Protocol | Plugin | Status | Description | +|----------|--------|--------|-------------| +| [GraphQL](./graphql.md) | `utcp-gql` | 🚧 Beta | GraphQL APIs and subscriptions | +| [gRPC](./grpc.md) | `utcp-grpc` | 🚧 Beta | High-performance RPC calls | +| [TCP](./tcp.md) | `utcp-socket` | 🚧 Beta | Raw TCP socket connections | +| [UDP](./udp.md) | `utcp-socket` | 🚧 Beta | UDP packet-based communication | + +## Protocol Selection Guide + +Choose the right protocol based on your tool's characteristics: + +### Use HTTP when: +- Your tool is a REST API +- You need simple request/response patterns +- You want maximum compatibility + +### Use WebSocket when: +- You need bidirectional communication +- Your tool provides real-time updates +- You want persistent connections + +### Use CLI when: +- Your tool is a command-line application +- You need to execute local scripts +- You're wrapping existing CLI tools + +### Use SSE when: +- You need server-to-client streaming +- You want real-time updates over HTTP +- You need simple event streaming + +## Plugin Architecture + +UTCP's plugin system allows you to: + +1. **Extend existing protocols** with custom authentication or data transformation +2. **Create new protocols** for specialized communication needs +3. **Combine protocols** for complex tool interactions + +### Creating Custom Protocols + +To create a custom protocol plugin, implement the [`CommunicationProtocol`](../api/core/utcp/interfaces/communication_protocol.md) interface: + +```python +from utcp.interfaces.communication_protocol import CommunicationProtocol +from utcp.data.call_template import CallTemplate + +class MyCustomProtocol(CommunicationProtocol): + def get_supported_call_template_types(self) -> List[str]: + return ["my_custom_protocol"] + + async def call_tool(self, call_template: CallTemplate, tool_args: Dict[str, Any]) -> Any: + # Implement your protocol logic here + pass +``` + +## Next Steps + +- Choose a protocol from the list above +- Read the specific protocol documentation +- Check out the [API Reference](../api/index.md) for implementation details +- See [Examples](../examples/index.md) for practical implementations diff --git a/docs/providers/mcp.md b/docs/providers/mcp.md new file mode 100644 index 0000000..f7a1088 --- /dev/null +++ b/docs/providers/mcp.md @@ -0,0 +1,336 @@ +--- +id: mcp +title: Model Context Protocol (MCP) +sidebar_position: 6 +--- + +# Model Context Protocol (MCP) + +The MCP protocol plugin (`utcp-mcp`) provides interoperability with the Model Context Protocol, allowing UTCP clients to call tools exposed through MCP servers. This enables gradual migration from MCP to UTCP or hybrid deployments. + +## Installation + +```bash +pip install utcp-mcp +``` + +## Call Template Structure + +```json +{ + "call_template_type": "mcp", + "server_config": { + "command": "node", + "args": ["/path/to/mcp-server.js"], + "env": { + "API_KEY": "${API_KEY}" + } + }, + "tool_name": "${tool_name}", + "connection_timeout": 30, + "call_timeout": 60 +} +``` + +## Configuration Options + +### Required Fields + +| Field | Type | Description | +|-------|------|-------------| +| `call_template_type` | string | Must be `"mcp"` | +| `server_config` | object | MCP server configuration | +| `tool_name` | string | Name of the MCP tool to call | + +### Server Configuration + +| Field | Type | Description | +|-------|------|-------------| +| `command` | string | Command to start MCP server | +| `args` | array | Command arguments | +| `env` | object | Environment variables | +| `cwd` | string | Working directory | + +### Optional Fields + +| Field | Type | Description | +|-------|------|-------------| +| `connection_timeout` | number | Server connection timeout (default: 30) | +| `call_timeout` | number | Tool call timeout (default: 60) | +| `server_name` | string | Friendly name for the server | +| `auto_restart` | boolean | Auto-restart server on failure (default: true) | + +## Examples + +### File System MCP Server + +```json +{ + "name": "read_file_mcp", + "description": "Read file content via MCP filesystem server", + "inputs": { + "type": "object", + "properties": { + "path": {"type": "string"} + }, + "required": ["path"] + }, + "tool_call_template": { + "call_template_type": "mcp", + "server_config": { + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "/allowed/path"], + "env": {} + }, + "tool_name": "read_file" + } +} +``` + +### Database MCP Server + +```json +{ + "name": "query_database_mcp", + "description": "Query database via MCP server", + "inputs": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "params": {"type": "array"} + }, + "required": ["query"] + }, + "tool_call_template": { + "call_template_type": "mcp", + "server_config": { + "command": "python", + "args": ["-m", "mcp_server_sqlite", "--db-path", "/data/app.db"], + "env": { + "DATABASE_URL": "${DATABASE_URL}" + } + }, + "tool_name": "execute_query", + "call_timeout": 120 + } +} +``` + +### Custom MCP Server + +```json +{ + "name": "custom_mcp_tool", + "description": "Call custom MCP server tool", + "inputs": { + "type": "object", + "properties": { + "input_data": {"type": "object"} + }, + "required": ["input_data"] + }, + "tool_call_template": { + "call_template_type": "mcp", + "server_config": { + "command": "node", + "args": ["./custom-mcp-server.js"], + "cwd": "/app/servers", + "env": { + "NODE_ENV": "production", + "API_KEY": "${MCP_API_KEY}" + } + }, + "tool_name": "process_data", + "server_name": "custom_processor" + } +} +``` + +## Server Management + +### Automatic Server Lifecycle + +The MCP protocol plugin automatically manages server lifecycle: + +1. **Startup**: Launches MCP server when first tool is called +2. **Connection**: Establishes JSON-RPC connection +3. **Tool Discovery**: Retrieves available tools from server +4. **Call Routing**: Routes tool calls to appropriate server +5. **Shutdown**: Gracefully shuts down server when no longer needed + +### Server Pooling + +Multiple tools can share the same MCP server instance: + +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "tools": [ + { + "name": "list_files", + "tool_call_template": { + "call_template_type": "mcp", + "server_config": { + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "/data"] + }, + "tool_name": "list_directory" + } + }, + { + "name": "read_file", + "tool_call_template": { + "call_template_type": "mcp", + "server_config": { + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-filesystem", "/data"] + }, + "tool_name": "read_file" + } + } + ] +} +``` + +## Error Handling + +| Error Type | Description | Handling | +|------------|-------------|----------| +| Server Start Failed | Cannot start MCP server | Raise `MCPServerStartError` | +| Connection Failed | Cannot connect to server | Raise `MCPConnectionError` | +| Tool Not Found | Tool doesn't exist on server | Raise `MCPToolNotFoundError` | +| Call Timeout | Tool call exceeded timeout | Raise `MCPTimeoutError` | +| Server Crashed | MCP server process died | Auto-restart if enabled | + +## Migration from MCP + +### Gradual Migration Strategy + +1. **Wrap Existing MCP Servers**: Use UTCP-MCP plugin to call existing servers +2. **Identify High-Value Tools**: Prioritize frequently used tools for direct migration +3. **Migrate Tool by Tool**: Convert individual tools to native UTCP protocols +4. **Deprecate MCP Servers**: Remove MCP dependency once migration is complete + +### Migration Example + +**Before (Pure MCP):** +```javascript +// MCP Client +const client = new MCPClient(); +await client.connect("filesystem-server"); +const result = await client.callTool("read_file", {path: "/data/file.txt"}); +``` + +**During Migration (UTCP with MCP):** +```python +# UTCP Client with MCP plugin +client = await UtcpClient.create() +result = await client.call_tool("filesystem.read_file", { + "path": "/data/file.txt" +}) +``` + +**After Migration (Pure UTCP):** +```python +# UTCP Client with native protocol +client = await UtcpClient.create() +result = await client.call_tool("filesystem.read_file", { + "path": "/data/file.txt" +}) +``` + +## Best Practices + +1. **Server Reuse**: Share MCP servers across multiple tools when possible +2. **Timeout Configuration**: Set appropriate timeouts for server startup and calls +3. **Error Handling**: Implement retry logic for transient server failures +4. **Resource Management**: Monitor server resource usage and lifecycle +5. **Migration Planning**: Plan gradual migration to native UTCP protocols +6. **Testing**: Test MCP server compatibility thoroughly +7. **Documentation**: Document MCP server dependencies and requirements + +## Performance Considerations + +### Overhead Comparison + +| Aspect | Native UTCP | UTCP-MCP | Pure MCP | +|--------|-------------|----------|----------| +| Latency | Low | Medium | Medium | +| Memory Usage | Low | Medium | High | +| Process Overhead | None | Medium | High | +| Network Hops | 1 | 2 | 2 | + +### Optimization Tips + +1. **Server Pooling**: Reuse servers across multiple tools +2. **Connection Caching**: Cache server connections +3. **Batch Operations**: Group related tool calls when possible +4. **Resource Limits**: Set memory and CPU limits for MCP servers +5. **Health Monitoring**: Monitor server health and performance + +## Compatibility + +### Supported MCP Versions + +- MCP 1.0.x: ✅ Full support +- MCP 0.x: ⚠️ Limited support + +### Known Limitations + +1. **Streaming**: MCP streaming not fully supported +2. **Resources**: MCP resources not mapped to UTCP +3. **Prompts**: MCP prompts not supported +4. **Sampling**: MCP sampling not supported + +## Common Use Cases + +- **Legacy Integration**: Calling existing MCP servers +- **Gradual Migration**: Transitioning from MCP to UTCP +- **Hybrid Deployments**: Using both MCP and UTCP tools +- **Third-party Tools**: Accessing MCP-only tools +- **Development**: Testing MCP compatibility + +## Troubleshooting + +### Server Won't Start + +```bash +# Check server command manually +npx -y @modelcontextprotocol/server-filesystem /path + +# Verify environment variables +echo $API_KEY + +# Check file permissions +ls -la /path/to/mcp-server.js +``` + +### Connection Issues + +```python +# Enable debug logging +import logging +logging.getLogger('utcp.mcp').setLevel(logging.DEBUG) + +# Test connection timeout +{ + "connection_timeout": 60, # Increase timeout + "auto_restart": true # Enable auto-restart +} +``` + +### Tool Not Found + +```python +# List available tools from MCP server +server_tools = await mcp_client.list_tools() +print(f"Available tools: {[tool.name for tool in server_tools]}") +``` + +## Related Documentation + +- [UTCP vs MCP Comparison](../utcp-vs-mcp.md) +- [Migration Guide](../migration/from-mcp.md) +- [HTTP Protocol](./http.md) - Alternative to MCP for REST APIs +- [CLI Protocol](./cli.md) - Alternative to MCP for command-line tools diff --git a/docs/providers/sse.md b/docs/providers/sse.md new file mode 100644 index 0000000..f39d964 --- /dev/null +++ b/docs/providers/sse.md @@ -0,0 +1,397 @@ +--- +id: sse +title: Server-Sent Events (SSE) +sidebar_position: 3 +--- + +# Server-Sent Events (SSE) + +The Server-Sent Events protocol plugin (`utcp-http`) enables UTCP to receive real-time streaming data from HTTP servers. SSE is perfect for tools that need to stream live updates, notifications, or continuous data feeds. + +## Installation + +SSE support is included with the HTTP plugin: + +```bash +pip install utcp-http +``` + +## Call Template Structure + +```json +{ + "call_template_type": "sse", + "url": "https://api.example.com/events", + "headers": { + "Authorization": "Bearer ${API_TOKEN}", + "Accept": "text/event-stream" + }, + "timeout": 60, + "max_events": 10, + "event_filter": { + "type": "data_update" + } +} +``` + +## Configuration Options + +### Required Fields + +| Field | Type | Description | +|-------|------|-------------| +| `call_template_type` | string | Must be `"sse"` | +| `url` | string | SSE endpoint URL | + +### Optional Fields + +| Field | Type | Description | +|-------|------|-------------| +| `headers` | object | HTTP headers for the request | +| `query_params` | object | URL query parameters | +| `timeout` | number | Stream timeout in seconds (default: 60) | +| `max_events` | number | Maximum events to receive (default: unlimited) | +| `event_filter` | object | Filter events by type or data | +| `reconnect` | boolean | Auto-reconnect on connection loss (default: true) | +| `reconnect_delay` | number | Delay between reconnection attempts (default: 3) | + +## Authentication + +SSE uses standard HTTP authentication methods: + +### Bearer Token + +```json +{ + "headers": { + "Authorization": "Bearer ${ACCESS_TOKEN}" + } +} +``` + +### API Key + +```json +{ + "headers": { + "X-API-Key": "${API_KEY}" + } +} +``` + +### Query Parameter Auth + +```json +{ + "query_params": { + "token": "${API_TOKEN}", + "user_id": "${USER_ID}" + } +} +``` + +## Event Handling + +### Basic Event Stream + +```json +{ + "name": "stream_notifications", + "description": "Stream real-time notifications", + "inputs": { + "type": "object", + "properties": { + "user_id": {"type": "string"} + }, + "required": ["user_id"] + }, + "tool_call_template": { + "call_template_type": "sse", + "url": "https://api.example.com/notifications/stream", + "query_params": { + "user_id": "${user_id}" + }, + "headers": { + "Authorization": "Bearer ${ACCESS_TOKEN}" + }, + "timeout": 300 + } +} +``` + +### Filtered Events + +```json +{ + "call_template_type": "sse", + "url": "https://api.example.com/events", + "event_filter": { + "type": "order_update", + "status": ["completed", "cancelled"] + }, + "max_events": 5 +} +``` + +## Examples + +### Stock Price Stream + +```json +{ + "name": "stream_stock_prices", + "description": "Stream real-time stock price updates", + "inputs": { + "type": "object", + "properties": { + "symbols": { + "type": "array", + "items": {"type": "string"} + }, + "duration": {"type": "number", "default": 60} + }, + "required": ["symbols"] + }, + "tool_call_template": { + "call_template_type": "sse", + "url": "https://api.stocks.com/stream", + "query_params": { + "symbols": "${symbols}", + "format": "json" + }, + "headers": { + "Authorization": "Bearer ${STOCK_API_KEY}" + }, + "timeout": "${duration}", + "event_filter": { + "type": "price_update" + } + } +} +``` + +### Log Monitoring + +```json +{ + "name": "monitor_logs", + "description": "Monitor application logs in real-time", + "inputs": { + "type": "object", + "properties": { + "service": {"type": "string"}, + "level": {"type": "string", "enum": ["error", "warn", "info", "debug"]} + }, + "required": ["service"] + }, + "tool_call_template": { + "call_template_type": "sse", + "url": "https://logs.example.com/stream", + "query_params": { + "service": "${service}", + "level": "${level}" + }, + "headers": { + "X-API-Key": "${LOG_API_KEY}" + }, + "timeout": 600, + "max_events": 100 + } +} +``` + +### System Metrics Stream + +```json +{ + "name": "stream_metrics", + "description": "Stream system performance metrics", + "inputs": { + "type": "object", + "properties": { + "metrics": { + "type": "array", + "items": {"type": "string"} + }, + "interval": {"type": "number", "default": 5} + }, + "required": ["metrics"] + }, + "tool_call_template": { + "call_template_type": "sse", + "url": "https://monitoring.example.com/metrics/stream", + "query_params": { + "metrics": "${metrics}", + "interval": "${interval}" + }, + "headers": { + "Authorization": "Bearer ${MONITORING_TOKEN}" + }, + "timeout": 300, + "reconnect": true, + "reconnect_delay": 5 + } +} +``` + +## Event Format + +SSE events follow the standard format: + +``` +event: message +data: {"type": "update", "value": 123} +id: event-123 +retry: 3000 + +event: heartbeat +data: {"timestamp": "2024-01-15T10:30:00Z"} + +data: {"message": "Simple data without event type"} +``` + +### Parsed Event Structure + +```json +{ + "event": "message", + "data": {"type": "update", "value": 123}, + "id": "event-123", + "retry": 3000, + "timestamp": "2024-01-15T10:30:00Z" +} +``` + +## Response Handling + +### Single Event Response + +```json +{ + "events": [ + { + "event": "data", + "data": {"result": "success"}, + "id": "1", + "timestamp": "2024-01-15T10:30:00Z" + } + ], + "total_events": 1, + "duration": 2.5 +} +``` + +### Multiple Events Response + +```json +{ + "events": [ + { + "event": "start", + "data": {"status": "processing"}, + "id": "1" + }, + { + "event": "progress", + "data": {"percent": 50}, + "id": "2" + }, + { + "event": "complete", + "data": {"result": "success"}, + "id": "3" + } + ], + "total_events": 3, + "duration": 15.2 +} +``` + +## Error Handling + +| Error Type | Description | Handling | +|------------|-------------|----------| +| Connection Failed | Cannot connect to SSE endpoint | Raise `SSEConnectionError` | +| Stream Timeout | No events received within timeout | Return partial results | +| Parse Error | Invalid SSE event format | Skip malformed events | +| Authentication Failed | Invalid credentials | Raise `SSEAuthError` | +| Server Error | HTTP 5xx response | Raise `SSEServerError` | + +## Best Practices + +1. **Set Appropriate Timeouts**: Configure timeouts based on expected data frequency +2. **Handle Reconnections**: Enable auto-reconnect for long-running streams +3. **Filter Events**: Use event filters to reduce unnecessary data processing +4. **Monitor Performance**: Track event rates and processing times +5. **Validate Data**: Validate incoming event data against expected schemas +6. **Handle Backpressure**: Implement buffering for high-frequency events +7. **Graceful Shutdown**: Properly close streams when done + +## Advanced Features + +### Custom Event Parsing + +```json +{ + "call_template_type": "sse", + "url": "https://api.example.com/events", + "event_parser": { + "format": "json", + "extract_fields": ["timestamp", "level", "message"] + } +} +``` + +### Event Aggregation + +```json +{ + "call_template_type": "sse", + "url": "https://api.example.com/metrics", + "aggregation": { + "window": 10, + "function": "average", + "field": "value" + } +} +``` + +### Conditional Termination + +```json +{ + "call_template_type": "sse", + "url": "https://api.example.com/events", + "termination_condition": { + "event_type": "complete", + "data_field": "status", + "value": "finished" + } +} +``` + +## Common Use Cases + +- **Real-time Dashboards**: Live metrics, status updates +- **Notifications**: User alerts, system notifications +- **Log Streaming**: Application logs, audit trails +- **Progress Tracking**: Long-running task progress +- **Live Data Feeds**: News, social media, sensor data +- **Chat Applications**: Message streams, typing indicators + +## Protocol Comparison + +| Feature | SSE | WebSocket | HTTP Polling | +|---------|-----|-----------|--------------| +| Server-to-Client | ✅ | ✅ | ✅ | +| Client-to-Server | ❌ | ✅ | ✅ | +| Auto-Reconnect | ✅ | Manual | Manual | +| Overhead | Low | Low | High | +| Browser Support | ✅ | ✅ | ✅ | +| Simplicity | High | Medium | High | + +## Related Protocols + +- [HTTP](./http.md) - For request/response patterns +- [WebSocket](./websocket.md) - For bidirectional communication +- [Streamable HTTP](./streamable-http.md) - For chunked HTTP responses diff --git a/docs/providers/text.md b/docs/providers/text.md new file mode 100644 index 0000000..e869124 --- /dev/null +++ b/docs/providers/text.md @@ -0,0 +1,448 @@ +--- +id: text +title: Text Protocol +sidebar_position: 5 +--- + +# Text Protocol + +The Text protocol plugin (`utcp-text`) enables UTCP to read and process text files from local filesystem or remote URLs. This is useful for tools that need to access documentation, configuration files, logs, or any text-based data. + +## Installation + +```bash +pip install utcp-text +``` + +## Call Template Structure + +```json +{ + "call_template_type": "text", + "file_path": "/path/to/file.txt", + "encoding": "utf-8", + "max_size": 1048576, + "line_range": { + "start": 1, + "end": 100 + } +} +``` + +## Configuration Options + +### Required Fields + +| Field | Type | Description | +|-------|------|-------------| +| `call_template_type` | string | Must be `"text"` | +| `file_path` | string | Path to text file (local or URL) | + +### Optional Fields + +| Field | Type | Description | +|-------|------|-------------| +| `encoding` | string | File encoding (default: "utf-8") | +| `max_size` | number | Maximum file size in bytes (default: 1MB) | +| `line_range` | object | Specific line range to read | +| `pattern` | string | Regex pattern to filter content | +| `transform` | string | Content transformation ("upper", "lower", "strip") | + +## File Sources + +### Local Files + +```json +{ + "call_template_type": "text", + "file_path": "/var/log/application.log", + "encoding": "utf-8" +} +``` + +### Remote URLs + +```json +{ + "call_template_type": "text", + "file_path": "https://example.com/config.txt", + "max_size": 512000 +} +``` + +### Variable Substitution + +```json +{ + "call_template_type": "text", + "file_path": "/data/${filename}", + "encoding": "${file_encoding}" +} +``` + +## Examples + +### Read Configuration File + +```json +{ + "name": "read_config", + "description": "Read application configuration file", + "inputs": { + "type": "object", + "properties": { + "config_name": {"type": "string"} + }, + "required": ["config_name"] + }, + "tool_call_template": { + "call_template_type": "text", + "file_path": "/etc/app/${config_name}.conf", + "encoding": "utf-8", + "max_size": 65536 + } +} +``` + +### Read Log File with Line Range + +```json +{ + "name": "read_recent_logs", + "description": "Read recent log entries", + "inputs": { + "type": "object", + "properties": { + "log_file": {"type": "string"}, + "lines": {"type": "number", "default": 100} + }, + "required": ["log_file"] + }, + "tool_call_template": { + "call_template_type": "text", + "file_path": "/var/log/${log_file}", + "line_range": { + "start": -${lines}, + "end": -1 + } + } +} +``` + +### Read Remote Documentation + +```json +{ + "name": "fetch_documentation", + "description": "Fetch documentation from remote URL", + "inputs": { + "type": "object", + "properties": { + "doc_url": {"type": "string"}, + "section": {"type": "string"} + }, + "required": ["doc_url"] + }, + "tool_call_template": { + "call_template_type": "text", + "file_path": "${doc_url}", + "pattern": "(?s)## ${section}.*?(?=## |$)", + "max_size": 2097152 + } +} +``` + +### Search in File + +```json +{ + "name": "search_in_file", + "description": "Search for pattern in text file", + "inputs": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "search_pattern": {"type": "string"} + }, + "required": ["file_path", "search_pattern"] + }, + "tool_call_template": { + "call_template_type": "text", + "file_path": "${file_path}", + "pattern": "${search_pattern}", + "transform": "strip" + } +} +``` + +## Line Range Options + +### Absolute Line Numbers + +```json +{ + "line_range": { + "start": 10, + "end": 50 + } +} +``` + +### Relative to End (Tail) + +```json +{ + "line_range": { + "start": -100, + "end": -1 + } +} +``` + +### From Start (Head) + +```json +{ + "line_range": { + "start": 1, + "end": 100 + } +} +``` + +## Pattern Matching + +### Simple Text Search + +```json +{ + "pattern": "ERROR" +} +``` + +### Regex Pattern + +```json +{ + "pattern": "\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2} ERROR.*" +} +``` + +### Multi-line Pattern + +```json +{ + "pattern": "(?s)START.*?END" +} +``` + +## Content Transformations + +### Case Transformations + +```json +{ + "transform": "upper" // Convert to uppercase +} +``` + +```json +{ + "transform": "lower" // Convert to lowercase +} +``` + +### Whitespace Handling + +```json +{ + "transform": "strip" // Remove leading/trailing whitespace +} +``` + +### Custom Transformations + +```json +{ + "transform": "normalize_whitespace" // Normalize all whitespace +} +``` + +## Response Format + +### Successful Read + +```json +{ + "content": "File content here...", + "metadata": { + "file_path": "/path/to/file.txt", + "size": 1024, + "lines": 25, + "encoding": "utf-8", + "last_modified": "2024-01-15T10:30:00Z" + } +} +``` + +### Filtered Content + +```json +{ + "content": "Matching lines...", + "metadata": { + "file_path": "/path/to/file.txt", + "total_lines": 1000, + "matched_lines": 5, + "pattern": "ERROR", + "line_range": {"start": 1, "end": 100} + } +} +``` + +## Error Handling + +| Error Type | Description | Handling | +|------------|-------------|----------| +| File Not Found | File doesn't exist | Raise `FileNotFoundError` | +| Permission Denied | No read permission | Raise `PermissionError` | +| File Too Large | Exceeds max_size limit | Raise `FileSizeError` | +| Encoding Error | Invalid file encoding | Raise `EncodingError` | +| Network Error | URL fetch failed | Raise `NetworkError` | + +## Security Considerations + +### Path Traversal Prevention + +```json +{ + "call_template_type": "text", + "file_path": "/safe/directory/${filename}", + "allowed_paths": ["/safe/directory/"] +} +``` + +### File Size Limits + +```json +{ + "max_size": 1048576 // 1MB limit +} +``` + +### URL Restrictions + +```json +{ + "allowed_domains": ["example.com", "docs.company.com"] +} +``` + +## Best Practices + +1. **Set Size Limits**: Always set appropriate max_size limits +2. **Validate Paths**: Validate file paths to prevent directory traversal +3. **Handle Encoding**: Specify encoding explicitly for non-UTF-8 files +4. **Use Line Ranges**: Use line ranges for large files to improve performance +5. **Pattern Efficiency**: Use efficient regex patterns for content filtering +6. **Cache Results**: Cache frequently accessed files +7. **Monitor Access**: Log file access for security auditing + +## Advanced Features + +### Conditional Reading + +```json +{ + "call_template_type": "text", + "file_path": "/var/log/app.log", + "condition": { + "modified_since": "2024-01-15T00:00:00Z" + } +} +``` + +### Multi-file Reading + +```json +{ + "call_template_type": "text", + "file_paths": [ + "/etc/app/config1.txt", + "/etc/app/config2.txt" + ], + "merge_strategy": "concatenate" +} +``` + +### Streaming Large Files + +```json +{ + "call_template_type": "text", + "file_path": "/var/log/huge.log", + "streaming": true, + "chunk_size": 8192 +} +``` + +## Common Use Cases + +- **Configuration Management**: Reading config files, environment files +- **Log Analysis**: Processing application logs, system logs +- **Documentation**: Accessing README files, API docs, manuals +- **Data Processing**: Reading CSV, JSON, XML text files +- **Template Processing**: Reading template files for generation +- **Code Analysis**: Reading source code files for analysis +- **Monitoring**: Reading status files, health check files + +## Performance Considerations + +| File Size | Recommended Approach | +|-----------|---------------------| +| < 1MB | Read entire file | +| 1MB - 10MB | Use line ranges | +| 10MB - 100MB | Use streaming | +| > 100MB | Use external tools | + +## Integration Examples + +### With HTTP Protocol + +```json +{ + "name": "process_uploaded_file", + "description": "Process uploaded text file", + "inputs": { + "type": "object", + "properties": { + "file_url": {"type": "string"} + } + }, + "tool_call_template": { + "call_template_type": "text", + "file_path": "${file_url}", + "max_size": 5242880 + } +} +``` + +### With CLI Protocol + +```json +{ + "name": "analyze_log_file", + "description": "Analyze log file with external tool", + "inputs": { + "type": "object", + "properties": { + "log_path": {"type": "string"} + } + }, + "tool_call_template": { + "call_template_type": "cli", + "command": "log-analyzer", + "args": ["--file", "${log_path}", "--format", "json"] + } +} +``` diff --git a/docs/providers/websocket.md b/docs/providers/websocket.md new file mode 100644 index 0000000..f71e3ad --- /dev/null +++ b/docs/providers/websocket.md @@ -0,0 +1,366 @@ +--- +id: websocket +title: WebSocket Protocol +sidebar_position: 2 +--- + +# WebSocket Protocol + +The WebSocket protocol plugin (`utcp-websocket`) enables UTCP to communicate with WebSocket servers for real-time, bidirectional communication. This is ideal for tools that require persistent connections or real-time updates. + +## Installation + +```bash +pip install utcp-websocket +``` + +## Call Template Structure + +```json +{ + "call_template_type": "websocket", + "url": "wss://api.example.com/ws", + "message": { + "type": "request", + "action": "${action}", + "data": "${data}" + }, + "connection_timeout": 10, + "response_timeout": 30, + "auth": { + "auth_type": "api_key", + "api_key": "${WS_API_KEY}", + "location": "query" + } +} +``` + +## Configuration Options + +### Required Fields + +| Field | Type | Description | +|-------|------|-------------| +| `call_template_type` | string | Must be `"websocket"` | +| `url` | string | WebSocket URL (ws:// or wss://) | +| `message` | object/string | Message to send to the WebSocket | + +### Optional Fields + +| Field | Type | Description | +|-------|------|-------------| +| `headers` | object | HTTP headers for WebSocket handshake | +| `connection_timeout` | number | Connection timeout in seconds (default: 10) | +| `response_timeout` | number | Response timeout in seconds (default: 30) | +| `close_after_response` | boolean | Close connection after receiving response (default: true) | +| `expected_responses` | number | Number of expected response messages (default: 1) | +| `ping_interval` | number | Ping interval in seconds (default: 30) | + +## Authentication + +WebSocket authentication can be handled in several ways: + +### Query Parameter Authentication + +```json +{ + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "token", + "location": "query" + } +} +``` + +### Header Authentication + +```json +{ + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "Authorization", + "location": "header" + } +} +``` + +### Message-based Authentication + +```json +{ + "message": { + "type": "auth", + "token": "${API_KEY}" + } +} +``` + +## Message Formats + +### JSON Messages + +```json +{ + "call_template_type": "websocket", + "url": "wss://api.example.com/ws", + "message": { + "id": "{{uuid}}", + "method": "getData", + "params": { + "query": "${query}", + "limit": 10 + } + } +} +``` + +### Text Messages + +```json +{ + "call_template_type": "websocket", + "url": "wss://api.example.com/ws", + "message": "GET_DATA:${query}" +} +``` + +### Binary Messages + +```json +{ + "call_template_type": "websocket", + "url": "wss://api.example.com/ws", + "message": { + "type": "binary", + "data": "${base64_data}" + } +} +``` + +## Examples + +### Real-time Data Subscription + +```json +{ + "name": "subscribe_stock_price", + "description": "Subscribe to real-time stock price updates", + "inputs": { + "type": "object", + "properties": { + "symbol": {"type": "string"}, + "duration": {"type": "number", "default": 60} + }, + "required": ["symbol"] + }, + "tool_call_template": { + "call_template_type": "websocket", + "url": "wss://api.stocks.com/ws", + "message": { + "action": "subscribe", + "symbol": "${symbol}", + "type": "price" + }, + "response_timeout": "${duration}", + "expected_responses": -1, + "close_after_response": false + } +} +``` + +### Chat Bot Integration + +```json +{ + "name": "send_chat_message", + "description": "Send a message to the chat bot", + "inputs": { + "type": "object", + "properties": { + "message": {"type": "string"}, + "user_id": {"type": "string"} + }, + "required": ["message", "user_id"] + }, + "tool_call_template": { + "call_template_type": "websocket", + "url": "wss://chat.example.com/ws", + "message": { + "type": "message", + "user_id": "${user_id}", + "content": "${message}", + "timestamp": "{{now}}" + }, + "headers": { + "Authorization": "Bearer ${CHAT_TOKEN}" + } + } +} +``` + +### IoT Device Control + +```json +{ + "name": "control_device", + "description": "Send control commands to IoT device", + "inputs": { + "type": "object", + "properties": { + "device_id": {"type": "string"}, + "command": {"type": "string"}, + "value": {"type": "number"} + }, + "required": ["device_id", "command"] + }, + "tool_call_template": { + "call_template_type": "websocket", + "url": "wss://iot.example.com/device/${device_id}", + "message": { + "command": "${command}", + "value": "${value}", + "timestamp": "{{now}}" + }, + "connection_timeout": 5, + "response_timeout": 10 + } +} +``` + +## Connection Management + +### Persistent Connections + +For tools that need to maintain persistent connections: + +```json +{ + "call_template_type": "websocket", + "url": "wss://api.example.com/ws", + "message": {"action": "ping"}, + "close_after_response": false, + "ping_interval": 30 +} +``` + +### Connection Pooling + +The WebSocket protocol automatically manages connection pooling for efficiency: + +- Reuses connections to the same endpoint +- Handles connection lifecycle automatically +- Implements reconnection logic for dropped connections + +## Response Handling + +### Single Response + +```json +{ + "expected_responses": 1, + "close_after_response": true +} +``` + +### Multiple Responses + +```json +{ + "expected_responses": 5, + "response_timeout": 60 +} +``` + +### Streaming Responses + +```json +{ + "expected_responses": -1, + "response_timeout": 300, + "close_after_response": false +} +``` + +## Error Handling + +| Error Type | Description | Handling | +|------------|-------------|----------| +| Connection Failed | Cannot establish WebSocket connection | Raise `WebSocketConnectionError` | +| Authentication Failed | WebSocket handshake authentication failed | Raise `WebSocketAuthError` | +| Timeout | No response within timeout period | Raise `WebSocketTimeoutError` | +| Protocol Error | Invalid WebSocket protocol usage | Raise `WebSocketProtocolError` | +| Connection Closed | Server closed connection unexpectedly | Raise `WebSocketClosedError` | + +## Best Practices + +1. **Use Secure WebSockets**: Always use `wss://` for production +2. **Handle Reconnections**: Implement retry logic for connection failures +3. **Set Appropriate Timeouts**: Configure timeouts based on expected response times +4. **Validate Messages**: Validate both outgoing and incoming messages +5. **Monitor Connections**: Track connection health and performance +6. **Implement Heartbeats**: Use ping/pong for connection health checks +7. **Handle Backpressure**: Manage message queuing for high-throughput scenarios + +## Advanced Features + +### Message Filtering + +Filter incoming messages based on criteria: + +```json +{ + "call_template_type": "websocket", + "url": "wss://api.example.com/ws", + "message": {"subscribe": "all"}, + "message_filter": { + "type": "stock_price", + "symbol": "${symbol}" + } +} +``` + +### Custom Headers + +Include custom headers in the WebSocket handshake: + +```json +{ + "headers": { + "User-Agent": "UTCP-Client/1.0", + "X-Client-ID": "${CLIENT_ID}", + "Authorization": "Bearer ${TOKEN}" + } +} +``` + +### Compression + +Enable WebSocket compression: + +```json +{ + "compression": "deflate", + "compression_threshold": 1024 +} +``` + +## Common Use Cases + +- **Real-time Data**: Stock prices, sensor data, live metrics +- **Chat Applications**: Messaging, notifications, presence +- **Gaming**: Real-time game state, multiplayer coordination +- **IoT Control**: Device commands, status updates +- **Live Updates**: News feeds, social media streams +- **Collaborative Tools**: Document editing, shared whiteboards + +## Protocol Comparison + +| Feature | WebSocket | HTTP | SSE | +|---------|-----------|------|-----| +| Bidirectional | ✅ | ❌ | ❌ | +| Real-time | ✅ | ❌ | ✅ | +| Persistent | ✅ | ❌ | ✅ | +| Overhead | Low | High | Medium | +| Complexity | Medium | Low | Low | diff --git a/sidebars.ts b/sidebars.ts index 2897139..28be80c 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -13,21 +13,55 @@ import type {SidebarsConfig} from '@docusaurus/plugin-content-docs'; Create as many sidebars as you want. */ const sidebars: SidebarsConfig = { - // By default, Docusaurus generates a sidebar from the docs folder structure - tutorialSidebar: [{type: 'autogenerated', dirName: '.'}], - - // But you can create a sidebar manually - /* + // Main documentation sidebar tutorialSidebar: [ - 'intro', - 'hello', + 'index', + 'for-tool-providers', + { + type: 'category', + label: 'Communication Protocols', + items: [ + 'providers/index', + 'providers/http', + 'providers/websocket', + 'providers/sse', + 'providers/cli', + 'providers/text', + 'providers/mcp', + ], + }, + 'implementation', + 'security', + 'utcp-vs-mcp', + 'migration-v0.1-to-v1.0', { type: 'category', - label: 'Tutorial', - items: ['tutorial-basics/create-a-document'], + label: 'API Reference', + items: [ + 'api/index', + { + type: 'category', + label: 'Core', + items: [ + { + type: 'autogenerated', + dirName: 'api/core', + }, + ], + }, + { + type: 'category', + label: 'Plugins', + items: [ + { + type: 'autogenerated', + dirName: 'api/plugins', + }, + ], + }, + ], }, ], - */ }; export default sidebars; From 6c158969e662daf2444bdccb63c99ad637b2aeea Mon Sep 17 00:00:00 2001 From: Luca Perrozzi Date: Fri, 5 Sep 2025 08:51:56 +0000 Subject: [PATCH 05/22] Improve landing page, security, and UTCP vs MCP documentation - Rewrite docs/index.md with better structure, clear value proposition, and navigation - Enhance docs/security.md with comprehensive protocol-specific security guidance - Improve docs/utcp-vs-mcp.md with technical depth, migration guidance, and decision framework - Add language specification notes consistently across documentation - Include practical examples and cross-references throughout --- docs/index.md | 277 +++++++++++------- docs/security.md | 662 +++++++++++++++++++++++++++++++++++++++----- docs/utcp-vs-mcp.md | 599 ++++++++++++++++++++++++++++++++++----- 3 files changed, 1297 insertions(+), 241 deletions(-) diff --git a/docs/index.md b/docs/index.md index 4f0a4e4..816c444 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,152 +4,221 @@ title: Introduction sidebar_position: 1 --- -# Introduction to UTCP 1.0 +# Universal Tool Calling Protocol (UTCP) -The Universal Tool Calling Protocol (UTCP) is a lightweight, secure, and scalable standard for defining and interacting with tools across a wide variety of communication protocols. Version 1.0 introduces a modular core with a plugin-based architecture, making it more extensible, testable, and easier to package. - -## Core Components +:::info Language Examples +This documentation uses **Python** examples. UTCP is available in multiple languages - see [TypeScript](https://github.com/universal-tool-calling-protocol/typescript-utcp), [Go](https://github.com/universal-tool-calling-protocol/go-utcp), and other implementations in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol). +::: -UTCP consists of four main components: +UTCP is a lightweight, secure, and scalable standard that enables AI agents and applications to discover and call tools directly using their native protocols - **no wrapper servers required**. -1. [**Manuals**](./api/core/utcp/data/utcp_manual.md): The standard tool provider description format that contains tool definitions -2. [**Tools**](./api/core/utcp/data/tool.md): The individual capabilities that can be called -3. [**Call Templates**](./api/core/utcp/data/call_template.md): The communication configurations that specify how tools are accessed. Concretely this maps a tool name and provided arguments to an actual API request in a communication protocol. -4. [**UtcpClient**](./api/core/utcp/utcp_client.md): The client that calls tools using the call templates. +## Why UTCP? -## The "Manual" Approach +### The Problem with Current Approaches +Most tool integration solutions force you to: +- Build and maintain wrapper servers for every tool +- Route all traffic through a middleman protocol +- Reimplement existing authentication and security +- Accept additional latency and complexity -UTCP's fundamental philosophy is to act as a descriptive manual rather than a prescriptive middleman: +### The UTCP Solution +UTCP acts as a **"manual"** that tells agents how to call your tools directly: -:::note -A UTCP Manual tells an agent: "Here is a tool. Here is its native endpoint (HTTP, WebSocket, CLI, etc.), and here is how to call it directly." +:::tip Core Philosophy +*"If a human can call your API, an AI agent should be able to call it too - with the same security and no additional infrastructure."* ::: -This approach eliminates the need for wrapper servers and allows direct communication between agents and tools. - -## New Architecture in 1.0 - -UTCP has been refactored into a core library and a set of optional plugins: +## Quick Start (5 Minutes) -### Core Package (`utcp`) -- **Data Models**: Pydantic models for [`Tool`](./api/core/utcp/data/tool.md), [`CallTemplate`](./api/core/utcp/data/call_template.md), [`UtcpManual`](./api/core/utcp/data/utcp_manual.md), and [`Auth`](./api/core/utcp/data/auth.md) -- **Pluggable Interfaces**: [`CommunicationProtocol`](./api/core/utcp/interfaces/communication_protocol.md), [`ConcurrentToolRepository`](./api/core/utcp/interfaces/concurrent_tool_repository.md), [`ToolSearchStrategy`](./api/core/utcp/interfaces/tool_search_strategy.md), [`VariableSubstitutor`](./api/core/utcp/interfaces/variable_substitutor.md), [`ToolPostProcessor`](./api/core/utcp/interfaces/tool_post_processor.md) -- **Default Implementations**: [`UtcpClient`](./api/core/utcp/utcp_client.md), [`InMemToolRepository`](./api/core/utcp/implementations/in_mem_tool_repository.md), [`TagAndDescriptionWordMatchStrategy`](./api/core/utcp/implementations/tag_search.md) +### 1. Install UTCP -### Protocol Plugins -- `utcp-http`: Supports HTTP, SSE, and streamable HTTP, plus an OpenAPI converter -- `utcp-cli`: For wrapping local command-line tools -- `utcp-mcp`: For interoperability with the Model Context Protocol (MCP) -- `utcp-text`: For reading text files -- `utcp-socket`: TCP and UDP protocols (work in progress) -- `utcp-gql`: GraphQL (work in progress) - -## Minimal Example - -Let's see how easy it is to use UTCP with a minimal example. +```bash +# Core library + HTTP support +pip install utcp utcp-http +``` -### 1. Defining a Tool (Tool Provider) +### 2. Expose Your First Tool -Create a simple HTTP endpoint that serves a UTCP Manual (JSON): +Add a discovery endpoint to your existing API: ```python -# app.py from fastapi import FastAPI app = FastAPI() +# Your existing API endpoint (unchanged) +@app.get("/weather") +def get_weather(location: str): + return {"temperature": 22, "conditions": "Sunny"} + +# Add UTCP discovery endpoint @app.get("/utcp") -def utcp_discovery(): +def utcp_manual(): return { "manual_version": "1.0.0", "utcp_version": "1.0.1", - "tools": [ - { - "name": "get_weather", - "description": "Get current weather for a location", - "inputs": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City name" - } - }, - "required": ["location"] - }, - "outputs": { - "type": "object", - "properties": { - "temperature": {"type": "number"}, - "conditions": {"type": "string"} - } - }, - "tool_call_template": { - "call_template_type": "http", - "url": "https://example.com/api/weather", - "http_method": "GET" - } + "tools": [{ + "name": "get_weather", + "description": "Get current weather for a location", + "inputs": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"] + }, + "tool_call_template": { + "call_template_type": "http", + "url": "http://localhost:8000/weather", + "http_method": "GET", + "query_params": {"location": "${location}"} } - ] + }] } - -# Implement the actual weather API endpoint -@app.get("/api/weather") -def get_weather(location: str): - # In a real app, you'd fetch actual weather data - return {"temperature": 22.5, "conditions": "Sunny"} -``` - -Run the server: - -```bash -uvicorn app:app --reload ``` -### 2. Using the Tool (Client) +### 3. Call Your Tool ```python -# client.py import asyncio from utcp.utcp_client import UtcpClient -from utcp_http.http_call_template import HttpCallTemplate async def main(): - # Create a UTCP client with configuration client = await UtcpClient.create(config={ - "manual_call_templates": [ - { - "name": "weather_service", - "call_template_type": "http", - "http_method": "GET", - "url": "http://localhost:8000/utcp" - } - ] + "manual_call_templates": [{ + "name": "weather_api", + "call_template_type": "http", + "url": "http://localhost:8000/utcp", + "http_method": "GET" + }] }) - - # Tools are automatically registered from the manual call templates - # Call a tool by its namespaced name: {manual_name}.{tool_name} + result = await client.call_tool( - "weather_service.get_weather", + "weather_api.get_weather", tool_args={"location": "San Francisco"} ) - print(f"Weather: {result['temperature']}°C, {result['conditions']}") + print(f"Weather: {result}") -if __name__ == "__main__": - asyncio.run(main()) +asyncio.run(main()) ``` -Run the client: +**That's it!** Your tool is now discoverable and callable by any UTCP client. -```bash -python client.py +## Key Benefits + +| Benefit | Description | +|---------|-------------| +| **🚀 Zero Latency Overhead** | Direct tool calls, no proxy servers | +| **🔒 Native Security** | Use your existing authentication and authorization | +| **🌐 Protocol Flexibility** | HTTP, WebSocket, CLI, GraphQL, and more | +| **⚡ Easy Integration** | Add one endpoint, no infrastructure changes | +| **📈 Scalable** | Leverage your existing scaling and monitoring | + +## How It Works + +```mermaid +graph LR + A[AI Agent] -->|1. Discover| B[UTCP Manual] + B -->|2. Learn| C[Tool Definitions] + A -->|3. Call Directly| D[Your API] + D -->|4. Response| A ``` -## Benefits of the UTCP Approach +1. **Discovery**: Agent fetches your UTCP manual +2. **Learning**: Agent understands how to call your tools +3. **Direct Calling**: Agent calls your API directly using native protocols +4. **Response**: Your API responds normally + +## Supported Protocols + +UTCP supports multiple communication protocols through plugins: + +| Protocol | Use Case | Plugin | Status | +|----------|----------|--------|--------| +| **[HTTP](./providers/http.md)** | REST APIs, webhooks | `utcp-http` | ✅ Stable | +| **[WebSocket](./providers/websocket.md)** | Real-time communication | `utcp-websocket` | ✅ Stable | +| **[CLI](./providers/cli.md)** | Command-line tools | `utcp-cli` | ✅ Stable | +| **[Server-Sent Events](./providers/sse.md)** | Streaming data | `utcp-http` | ✅ Stable | +| **[Text Files](./providers/text.md)** | File reading | `utcp-text` | ✅ Stable | +| **[MCP](./providers/mcp.md)** | MCP interoperability | `utcp-mcp` | ✅ Stable | + +[View all protocols →](./providers/index.md) + +## Architecture Overview + +UTCP v1.0 features a modular, plugin-based architecture: + +### Core Components +- **[Manuals](./api/core/utcp/data/utcp_manual.md)**: Tool definitions and metadata +- **[Tools](./api/core/utcp/data/tool.md)**: Individual callable capabilities +- **[Call Templates](./api/core/utcp/data/call_template.md)**: Protocol-specific call instructions +- **[UTCP Client](./api/core/utcp/utcp_client.md)**: Tool discovery and execution engine + +### Plugin System +- **Protocol Plugins**: HTTP, WebSocket, CLI, etc. +- **Custom Protocols**: Extend with your own communication methods +- **Tool Repositories**: Pluggable storage for tool definitions +- **Search Strategies**: Customizable tool discovery algorithms + +[Learn more about the architecture →](./api/index.md) + +## Who Should Use UTCP? + +### 🛠️ Tool Providers +You have APIs, services, or tools that you want AI agents to use: +- **Existing API owners** - Expose your REST APIs to AI agents +- **SaaS providers** - Make your services AI-accessible +- **Enterprise teams** - Enable internal tool usage by AI systems + +[**Get started as a tool provider →**](./for-tool-providers.md) -1. **Direct Communication**: The client calls the tool's native endpoint directly -2. **No Wrapper Infrastructure**: No need to build and maintain wrapper servers -3. **Leverage Existing Systems**: Uses the tool's existing authentication, rate limiting, etc. -4. **Flexible Protocol Support**: Works with any communication protocol (HTTP, WebSockets, CLI, etc.) +### 🤖 Tool Consumers +You're building AI agents or applications that need to call external tools: +- **AI agent developers** - Give your agents access to external capabilities +- **Application builders** - Integrate third-party tools seamlessly +- **Enterprise developers** - Connect to internal and external services + +[**Get started as a tool consumer →**](./implementation.md) + +## UTCP vs Alternatives + +| Feature | UTCP | MCP | Custom Wrappers | +|---------|------|-----|-----------------| +| **Infrastructure** | None required | Wrapper servers | Custom servers | +| **Latency** | Direct calls | Double hop | Variable | +| **Security** | Native | Reimplemented | Custom | +| **Protocols** | Multiple | HTTP streaming | Single | +| **Maintenance** | Minimal | High | Very high | + +[**Detailed comparison with MCP →**](./utcp-vs-mcp.md) + +## Next Steps + +### For Tool Providers +1. **[Read the provider guide](./for-tool-providers.md)** - Learn how to expose your tools +2. **[Choose your protocol](./providers/index.md)** - Select the right communication method +3. **[Implement your manual](./implementation.md)** - Add UTCP to your existing API +4. **[Secure your tools](./security.md)** - Implement proper authentication + +### For Tool Consumers +1. **[Read the implementation guide](./implementation.md)** - Learn how to build UTCP clients +2. **[Explore protocols](./providers/index.md)** - Understand available communication options +3. **[Check examples](https://github.com/universal-tool-calling-protocol/python-utcp/tree/main/examples)** - See real-world implementations +4. **[Join the community](https://discord.gg/ZpMbQ8jRbD)** - Get help and share experiences + +### Migration from Other Systems +- **[From UTCP v0.1](./migration-v0.1-to-v1.0.md)** - Upgrade to the latest version +- **[From MCP](./providers/mcp.md)** - Migrate from Model Context Protocol +- **[From custom solutions](./implementation.md)** - Replace existing tool integrations + +## Community & Support + +- **[GitHub Organization](https://github.com/universal-tool-calling-protocol)** - Source code and issues +- **[Discord Community](https://discord.gg/ZpMbQ8jRbD)** - Real-time help and discussions +- **[Tool Registry](https://utcp.io/registry)** - Discover available tools +- **[RFC Process](./about/RFC.md)** - Contribute to the specification + +--- -In the following sections, we'll explore the details of UTCP's components and how to implement them in your applications. +**Ready to get started?** Choose your path: +- 🛠️ [**I want to expose my tools**](./for-tool-providers.md) +- 🤖 [**I want to call tools**](./implementation.md) +- 📚 [**I want to learn more**](./providers/index.md) diff --git a/docs/security.md b/docs/security.md index b182938..de5d007 100644 --- a/docs/security.md +++ b/docs/security.md @@ -6,123 +6,647 @@ sidebar_position: 6 # Security Considerations -Security is a critical aspect of any protocol that enables tool access and execution. This section outlines key security considerations when implementing and using UTCP. +:::info Language Examples +This guide uses **Python** examples for implementation code. UTCP security principles apply to all language implementations - check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) for language-specific security examples. +::: + +Security is critical when enabling direct tool access through UTCP. This guide covers security considerations specific to UTCP's "manual" approach and provides practical guidance for secure implementations. + +## UTCP Security Model + +### Direct Communication Implications + +UTCP's direct communication model has unique security characteristics: + +**Advantages:** +- No middleman to compromise +- Native security controls remain intact +- Reduced attack surface (no proxy servers) +- Existing monitoring and logging continue to work + +**Considerations:** +- Clients must handle multiple authentication methods +- Manual endpoints become discovery targets +- Variable substitution introduces injection risks + +### Manual Discovery Security + +Secure your UTCP manual endpoints: + +```python +from fastapi import FastAPI, HTTPException, Depends +from fastapi.security import HTTPBearer -## Authentication +app = FastAPI() +security = HTTPBearer() -UTCP supports several authentication methods across different communication protocol types: +@app.get("/utcp") +def get_manual(token: str = Depends(security)): + # Validate discovery access + if not validate_discovery_token(token.credentials): + raise HTTPException(401, "Invalid discovery token") + + return { + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "tools": get_authorized_tools(token.credentials) + } -### API Key Authentication +def get_authorized_tools(token: str): + # Return only tools the client is authorized to see + user_permissions = get_user_permissions(token) + return filter_tools_by_permissions(all_tools, user_permissions) +``` + +## Authentication & Authorization + +### Enhanced Authentication Examples + +#### API Key with Rotation ```json { "auth": { "auth_type": "api_key", - "api_key": "YOUR_API_KEY", + "api_key": "${API_KEY}", "var_name": "X-API-Key", "location": "header" } } ``` -The `location` field specifies where the API key is placed, and can be `header`, `query`, or `cookie`. +**Secure implementation:** +```python +import os +from datetime import datetime, timedelta + +class RotatingAPIKey: + def __init__(self): + self.current_key = os.getenv("API_KEY_CURRENT") + self.next_key = os.getenv("API_KEY_NEXT") + self.rotation_time = datetime.fromisoformat(os.getenv("KEY_ROTATION_TIME")) + + def get_valid_key(self): + if datetime.now() > self.rotation_time: + return self.next_key + return self.current_key +``` -### Basic Authentication +#### OAuth2 with Scope Validation ```json { "auth": { - "auth_type": "basic", - "username": "user", - "password": "pass" + "auth_type": "oauth2", + "client_id": "${CLIENT_ID}", + "client_secret": "${CLIENT_SECRET}", + "token_url": "https://auth.example.com/token", + "scope": "tools:read tools:execute" } } ``` -### OAuth2 Authentication +#### Per-Tool Authorization ```json { - "auth": { - "auth_type": "oauth2", - "client_id": "YOUR_CLIENT_ID", - "client_secret": "YOUR_CLIENT_SECRET", - "token_url": "https://auth.example.com/token", - "scope": "read:tools" + "name": "admin_tool", + "description": "Administrative operations", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/admin/action", + "http_method": "POST", + "auth": { + "auth_type": "api_key", + "api_key": "${ADMIN_TOKEN}", + "var_name": "Authorization", + "location": "header" + } } } ``` -The `scope` field is optional and specifies the level of access that the client is requesting. +## Protocol-Specific Security -## Tool Access Control +### HTTP/HTTPS Security -When exposing tools through UTCP, consider implementing these access controls: - -1. **Tool-Level Permissions**: Define which users/agents can access specific tools -2. **Parameter Constraints**: Restrict parameter values to prevent abuse -3. **Rate Limiting**: Implement per-user/per-tool rate limits -4. **Usage Quotas**: Set maximum usage quotas for tools -5. **Audit Logging**: Log all tool calls for security monitoring +**Required configurations:** +```json +{ + "call_template_type": "http", + "url": "https://api.example.com/endpoint", + "verify_ssl": true, + "timeout": 30, + "headers": { + "User-Agent": "UTCP-Client/1.0", + "X-Request-ID": "${request_id}" + } +} +``` -## Communication Protocol-Specific Considerations +**Security checklist:** +- ✅ Always use HTTPS in production +- ✅ Validate SSL certificates (`verify_ssl: true`) +- ✅ Set appropriate timeouts +- ✅ Include request tracking headers +- ✅ Implement retry limits -### HTTP +### WebSocket Security -- Always use HTTPS, never HTTP +**Secure WebSocket configuration:** +```json +{ + "call_template_type": "websocket", + "url": "wss://api.example.com/ws", + "headers": { + "Authorization": "Bearer ${WS_TOKEN}", + "Origin": "https://trusted-domain.com" + }, + "ping_interval": 30, + "connection_timeout": 10 +} +``` -### CLI +**Security measures:** +- ✅ Use WSS (secure WebSocket) only +- ✅ Validate Origin headers +- ✅ Implement connection timeouts +- ✅ Use heartbeat/ping for connection health +- ✅ Limit concurrent connections per client -:::important +### CLI Security -CLI poses a significant security risk as it executes commands on the local system. +:::danger High Risk Protocol +CLI execution poses significant security risks. Use with extreme caution. ::: -- Validate and sanitize all input parameters -- Run commands with the minimum necessary permissions -- Implement allow-lists for permitted commands -- Sandbox execution environments when possible +**Secure CLI implementation:** +```json +{ + "call_template_type": "cli", + "command": "/usr/local/bin/safe-script", + "args": ["--input", "${sanitized_input}"], + "working_directory": "/safe/sandbox", + "environment": { + "PATH": "/usr/local/bin:/usr/bin", + "HOME": "/tmp/sandbox" + }, + "timeout": 30, + "allowed_exit_codes": [0] +} +``` -### WebSocket +**Security requirements:** +- ✅ Use absolute paths for commands +- ✅ Sanitize all input parameters +- ✅ Run in sandboxed environments +- ✅ Limit environment variables +- ✅ Set strict timeouts +- ✅ Validate exit codes +- ✅ Use minimal user permissions + +**Input sanitization example:** +```python +import re +import shlex + +def sanitize_cli_input(input_value: str) -> str: + # Remove dangerous characters + sanitized = re.sub(r'[;&|`$(){}[\]<>]', '', input_value) + # Escape for shell safety + return shlex.quote(sanitized) +``` -- Use secure WebSocket (WSS) connections +### Server-Sent Events (SSE) Security -## Data Protection +**Secure SSE configuration:** +```json +{ + "call_template_type": "sse", + "url": "https://api.example.com/events", + "headers": { + "Authorization": "Bearer ${SSE_TOKEN}", + "Accept": "text/event-stream", + "Cache-Control": "no-cache" + }, + "timeout": 300, + "max_events": 1000 +} +``` -1. **Data in Transit**: Ensure all communications use TLS 1.2+ encryption -2. **Data at Rest**: Encrypt sensitive configuration data -3. **Sensitive Data in Logs**: Prevent logging of sensitive parameters -4. **PII Handling**: Implement proper controls for personal information +**Security considerations:** +- ✅ Authenticate SSE connections +- ✅ Set maximum event limits +- ✅ Implement connection timeouts +- ✅ Validate event data format +- ✅ Monitor for event flooding -## Secure Implementation Checklist +### Text Protocol Security -- [ ] Use HTTPS/WSS for all network communications -- [ ] Implement proper authentication for all communication protocols -- [ ] Validate all input against schemas before processing -- [ ] Sanitize inputs to prevent injection attacks -- [ ] Implement rate limiting to prevent abuse -- [ ] Set appropriate timeouts for all operations -- [ ] Log security-relevant events -- [ ] Regularly update dependencies -- [ ] Implement proper error handling that doesn't leak sensitive information +**Secure file access:** +```json +{ + "call_template_type": "text", + "file_path": "/safe/data/${filename}", + "max_size": 1048576, + "allowed_paths": ["/safe/data/"], + "encoding": "utf-8" +} +``` + +**Security measures:** +- ✅ Restrict file paths to safe directories +- ✅ Set maximum file size limits +- ✅ Validate file extensions +- ✅ Prevent directory traversal attacks +- ✅ Use safe encoding handling + +**Path validation example:** +```python +import os +from pathlib import Path + +def validate_file_path(file_path: str, allowed_dirs: list) -> bool: + try: + # Resolve to absolute path + abs_path = Path(file_path).resolve() + + # Check if path is within allowed directories + for allowed_dir in allowed_dirs: + if abs_path.is_relative_to(Path(allowed_dir).resolve()): + return True + return False + except (OSError, ValueError): + return False +``` + +### MCP Security + +**Secure MCP server configuration:** +```json +{ + "call_template_type": "mcp", + "server_config": { + "command": "/usr/local/bin/mcp-server", + "args": ["--config", "/safe/config.json"], + "env": { + "MCP_LOG_LEVEL": "INFO" + }, + "timeout": 60 + } +} +``` + +**Security considerations:** +- ✅ Use trusted MCP server implementations +- ✅ Sandbox MCP server processes +- ✅ Limit server resource usage +- ✅ Monitor server health and logs +- ✅ Implement server restart policies -## Common Vulnerabilities to Avoid +## Input Validation & Sanitization -| Vulnerability | Prevention | -|--------------|------------| -| Injection Attacks | Validate and sanitize all inputs | -| Credential Leakage | Use secure credential storage | -| Excessive Permissions | Follow the principle of least privilege | -| Man-in-the-Middle | Use certificate validation and pinning | -| Denial of Service | Implement rate limiting and timeouts | -| Information Disclosure | Ensure errors don't leak sensitive data | +### JSON Schema Validation + +**Comprehensive input validation:** +```json +{ + "inputs": { + "type": "object", + "properties": { + "email": { + "type": "string", + "format": "email", + "maxLength": 254 + }, + "age": { + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + "tags": { + "type": "array", + "items": {"type": "string", "pattern": "^[a-zA-Z0-9_-]+$"}, + "maxItems": 10 + } + }, + "required": ["email"], + "additionalProperties": false + } +} +``` -## Secure Development Lifecycle +### Parameter Sanitization + +**Server-side validation:** +```python +from pydantic import BaseModel, validator, Field +import re + +class ToolInput(BaseModel): + user_id: str = Field(..., regex=r'^[a-zA-Z0-9_-]+$', max_length=50) + query: str = Field(..., max_length=1000) + + @validator('query') + def sanitize_query(cls, v): + # Remove potentially dangerous characters + return re.sub(r'[<>"\']', '', v).strip() +``` -1. **Design**: Conduct threat modeling during protocol design -2. **Implementation**: Follow secure coding practices -3. **Testing**: Perform security testing and code reviews -4. **Deployment**: Use secure deployment practices -5. **Maintenance**: Monitor for security issues and update regularly +## Secure Variable Handling + +### Environment Variable Security + +**Secure variable loading:** +```python +import os +from typing import Dict, Optional + +class SecureVariableLoader: + def __init__(self, allowed_prefixes: list = None): + self.allowed_prefixes = allowed_prefixes or ['UTCP_', 'API_'] + + def load_variable(self, var_name: str) -> Optional[str]: + # Only load variables with allowed prefixes + if not any(var_name.startswith(prefix) for prefix in self.allowed_prefixes): + raise ValueError(f"Variable {var_name} not allowed") + + return os.getenv(var_name) + + def substitute_variables(self, template: str, context: Dict[str, str]) -> str: + # Safely substitute variables + for var_name, value in context.items(): + if self.is_safe_variable(var_name, value): + template = template.replace(f"${{{var_name}}}", value) + return template + + def is_safe_variable(self, name: str, value: str) -> bool: + # Validate variable safety + if len(value) > 10000: # Prevent extremely long values + return False + if any(char in value for char in ['<', '>', '"', "'"]): # Basic XSS prevention + return False + return True +``` + +### Runtime Variable Substitution + +**Secure substitution:** +```python +import re +from typing import Dict + +def secure_substitute(template: str, variables: Dict[str, str]) -> str: + def replace_var(match): + var_name = match.group(1) + if var_name in variables: + value = variables[var_name] + # Validate and sanitize the value + if is_safe_value(value): + return value + else: + raise ValueError(f"Unsafe variable value for {var_name}") + else: + raise ValueError(f"Variable {var_name} not found") + + # Only replace variables with the expected pattern + return re.sub(r'\$\{([a-zA-Z_][a-zA-Z0-9_]*)\}', replace_var, template) + +def is_safe_value(value: str) -> bool: + # Implement your safety checks + return len(value) < 1000 and not any(c in value for c in ['<', '>', '"', "'", ';', '&']) +``` + +## Network & Transport Security + +### TLS Configuration + +**Minimum TLS requirements:** +```python +import ssl +import httpx + +# Configure secure HTTP client +client = httpx.AsyncClient( + verify=True, # Verify SSL certificates + timeout=30.0, + limits=httpx.Limits(max_connections=100, max_keepalive_connections=20) +) + +# For custom SSL context +ssl_context = ssl.create_default_context() +ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2 +ssl_context.check_hostname = True +ssl_context.verify_mode = ssl.CERT_REQUIRED +``` + +### Certificate Validation + +**Enhanced certificate validation:** +```python +import ssl +import certifi + +def create_secure_context(): + context = ssl.create_default_context(cafile=certifi.where()) + context.check_hostname = True + context.verify_mode = ssl.CERT_REQUIRED + context.minimum_version = ssl.TLSVersion.TLSv1_2 + + # Disable weak ciphers + context.set_ciphers('ECDHE+AESGCM:ECDHE+CHACHA20:DHE+AESGCM:DHE+CHACHA20:!aNULL:!MD5:!DSS') + + return context +``` + +## Monitoring & Incident Response + +### Security Logging + +**Comprehensive security logging:** +```python +import logging +import json +from datetime import datetime + +class SecurityLogger: + def __init__(self): + self.logger = logging.getLogger('utcp.security') + + def log_tool_call(self, tool_name: str, user_id: str, success: bool, **kwargs): + log_entry = { + 'timestamp': datetime.utcnow().isoformat(), + 'event_type': 'tool_call', + 'tool_name': tool_name, + 'user_id': user_id, + 'success': success, + 'metadata': kwargs + } + self.logger.info(json.dumps(log_entry)) + + def log_auth_failure(self, tool_name: str, reason: str, **kwargs): + log_entry = { + 'timestamp': datetime.utcnow().isoformat(), + 'event_type': 'auth_failure', + 'tool_name': tool_name, + 'reason': reason, + 'metadata': kwargs + } + self.logger.warning(json.dumps(log_entry)) +``` + +### Anomaly Detection + +**Basic anomaly detection:** +```python +from collections import defaultdict +from datetime import datetime, timedelta + +class AnomalyDetector: + def __init__(self): + self.call_counts = defaultdict(list) + self.rate_limits = { + 'calls_per_minute': 60, + 'calls_per_hour': 1000 + } + + def check_rate_limit(self, user_id: str) -> bool: + now = datetime.utcnow() + user_calls = self.call_counts[user_id] + + # Clean old entries + user_calls[:] = [call_time for call_time in user_calls + if now - call_time < timedelta(hours=1)] + + # Check limits + recent_calls = [call_time for call_time in user_calls + if now - call_time < timedelta(minutes=1)] + + if len(recent_calls) >= self.rate_limits['calls_per_minute']: + return False + if len(user_calls) >= self.rate_limits['calls_per_hour']: + return False + + # Record this call + user_calls.append(now) + return True +``` + +## Security Testing & Validation + +### Testing Methodologies + +**Security test examples:** +```python +import pytest +from utcp.utcp_client import UtcpClient + +@pytest.mark.asyncio +async def test_injection_prevention(): + client = await UtcpClient.create(config=test_config) + + # Test SQL injection attempt + malicious_input = "'; DROP TABLE users; --" + + with pytest.raises(ValueError, match="Invalid input"): + await client.call_tool("db.query", {"query": malicious_input}) + +@pytest.mark.asyncio +async def test_path_traversal_prevention(): + client = await UtcpClient.create(config=test_config) + + # Test directory traversal attempt + malicious_path = "../../../etc/passwd" + + with pytest.raises(ValueError, match="Path not allowed"): + await client.call_tool("file.read", {"path": malicious_path}) + +@pytest.mark.asyncio +async def test_rate_limiting(): + client = await UtcpClient.create(config=test_config) + + # Test rate limiting + for i in range(100): # Exceed rate limit + try: + await client.call_tool("api.test", {}) + except Exception as e: + assert "rate limit" in str(e).lower() + break + else: + pytest.fail("Rate limiting not enforced") +``` + +### Security Automation + +**Automated security checks:** +```python +def validate_manual_security(manual: dict) -> list: + issues = [] + + for tool in manual.get('tools', []): + call_template = tool.get('tool_call_template', {}) + + # Check for HTTP over HTTPS + if call_template.get('call_template_type') == 'http': + url = call_template.get('url', '') + if url.startswith('http://'): + issues.append(f"Tool {tool['name']}: Uses insecure HTTP") + + # Check for hardcoded credentials + template_str = str(call_template) + if any(keyword in template_str.lower() for keyword in ['password', 'secret', 'key']): + if not any(var in template_str for var in ['${', '${']): + issues.append(f"Tool {tool['name']}: May contain hardcoded credentials") + + # Check for CLI security + if call_template.get('call_template_type') == 'cli': + command = call_template.get('command', '') + if not command.startswith('/'): + issues.append(f"Tool {tool['name']}: CLI command should use absolute path") + + return issues +``` -By following these security considerations, UTCP implementations can minimize risks while enabling powerful tool integrations across various communication protocols. +## Security Checklist + +### Tool Provider Security + +- [ ] UTCP manual endpoint requires authentication +- [ ] All tool endpoints use HTTPS/WSS +- [ ] Input validation implemented for all tools +- [ ] Rate limiting configured per user/tool +- [ ] Security logging enabled +- [ ] Credentials stored securely (not hardcoded) +- [ ] SSL certificate validation enabled +- [ ] Appropriate timeouts configured +- [ ] Error messages don't leak sensitive information + +### Tool Consumer Security + +- [ ] Variable substitution is sanitized +- [ ] SSL certificate verification enabled +- [ ] Connection timeouts configured +- [ ] Rate limiting respected +- [ ] Security events logged +- [ ] Credentials rotated regularly +- [ ] Network connections monitored +- [ ] Input validation before tool calls + +### Protocol-Specific Security + +- [ ] **HTTP**: HTTPS only, certificate validation +- [ ] **WebSocket**: WSS only, origin validation +- [ ] **CLI**: Sandboxed execution, input sanitization +- [ ] **SSE**: Authenticated connections, event limits +- [ ] **Text**: Path validation, size limits +- [ ] **MCP**: Trusted servers, resource limits + +By following these security guidelines, you can safely implement UTCP while maintaining strong security posture across all communication protocols. + +For protocol-specific security details, see: +- [HTTP Security](./providers/http.md#security-considerations) +- [WebSocket Security](./providers/websocket.md#security-considerations) +- [CLI Security](./providers/cli.md#security-considerations) +- [SSE Security](./providers/sse.md#security-considerations) +- [Text Security](./providers/text.md#security-considerations) +- [MCP Security](./providers/mcp.md#security-considerations) diff --git a/docs/utcp-vs-mcp.md b/docs/utcp-vs-mcp.md index 3dec676..099f0d9 100644 --- a/docs/utcp-vs-mcp.md +++ b/docs/utcp-vs-mcp.md @@ -4,99 +4,562 @@ title: UTCP vs MCP sidebar_position: 5 --- -# UTCP vs MCP tool calling: A Comparison +# UTCP vs MCP: A Comprehensive Comparison -This page compares the Universal Tool Calling Protocol (UTCP) with the Model Context Protocol's (MCP) tool calling functionality, highlighting their different approaches to agent-tool integration. +:::info Language Examples +This comparison uses **Python** examples. Both UTCP and MCP have implementations in multiple languages - check respective GitHub organizations for language-specific examples. +::: + +This guide compares the Universal Tool Calling Protocol (UTCP) with the Model Context Protocol (MCP), helping you choose the right approach for your AI tool integration needs. ## Video Overview -## Architectural Differences - -| Aspect | MCP | UTCP | -|--------|-----|------| -| **Core Model** | Middleman | Manual | -| **Architecture** | Agents ↔ MCP Server ↔ Tool | Agent ↔ Tool (Direct) | -| **Integration Approach** | Wraps existing tools | Describes how to call existing tools | -| **Network Hops** | Double (Agent → MCP → Tool) | Single (Agent → Tool) | -| **Protocol Dependency** | Hard dependency on protocol for every call | Protocol only needed during discovery | - -## The Middleman vs Manual Philosophies +## Quick Comparison -### The MCP "Middleman" Approach +| Aspect | UTCP | MCP | +|--------|------|-----| +| **Philosophy** | Manual (describes how to call tools) | Middleman (wraps tools in protocol) | +| **Architecture** | Agent → Tool (Direct) | Agent → MCP Server → Tool | +| **Infrastructure** | None required | Wrapper servers needed | +| **Protocols** | HTTP, WebSocket, CLI, SSE, etc. | JSON-RPC over stdio/HTTP | +| **Performance** | Native tool performance | Additional proxy overhead | +| **Maintenance** | Minimal | High (server maintenance) | -MCP positions itself as the "USB-C for AI Agents" — a universal adapter that all tools must plug into. This approach: - -- Forces all traffic through a new protocol layer -- Requires writing "wrappers" for existing tools -- Needs to reinvent solutions for auth, security, and other infrastructure concerns - -This creates what we call the "wrapper tax": the additional infrastructure, development, and maintenance overhead required to adapt existing tools to work with MCP. - -### The UTCP "Manual" Approach +## Architectural Differences -UTCP takes a different approach — it's a "manual" that describes how to call tools directly: +### UTCP: The "Manual" Approach -- Provides all necessary information to call native APIs directly -- Gets out of the way after tool discovery -- Leverages existing infrastructure for auth, security, etc. +UTCP provides a standardized way to describe how to call existing tools directly: -This eliminates the wrapper tax and allows organizations to expose their existing APIs to AI agents without building and maintaining additional infrastructure. +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "tools": [{ + "name": "get_weather", + "description": "Get current weather", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.weather.com/current", + "http_method": "GET", + "query_params": {"location": "${location}"} + } + }] +} +``` + +**Flow:** Agent discovers manual → Agent calls tool directly + +### MCP: The "Middleman" Approach + +MCP requires building servers that wrap your tools: + +```python +# MCP Server (required infrastructure) +from mcp.server import Server +from mcp.types import Tool + +server = Server("weather-server") + +@server.list_tools() +async def list_tools(): + return [Tool(name="get_weather", description="Get weather")] + +@server.call_tool() +async def call_tool(name: str, arguments: dict): + if name == "get_weather": + # Call actual weather API + return await weather_api.get(arguments["location"]) +``` + +**Flow:** Agent → MCP Server → Tool → MCP Server → Agent + +## Technical Comparison + +### Performance Impact + +#### UTCP Performance +```python +# Direct API call - no overhead +import httpx + +async def call_weather_tool(): + async with httpx.AsyncClient() as client: + response = await client.get( + "https://api.weather.com/current", + params={"location": "San Francisco"} + ) + return response.json() + +# Latency: ~100ms (API response time only) +``` + +#### MCP Performance +```python +# Requires MCP server proxy +import mcp + +async def call_weather_tool(): + client = mcp.Client() + await client.connect("weather-server") + result = await client.call_tool("get_weather", {"location": "San Francisco"}) + return result + +# Latency: ~150ms (API + MCP server overhead) +``` + +### Infrastructure Requirements + +#### UTCP Infrastructure +```python +# Add one endpoint to existing API +@app.get("/utcp") +def get_manual(): + return utcp_manual # Static JSON + +# Total infrastructure: 0 additional servers +``` + +#### MCP Infrastructure +```python +# Requires dedicated MCP server +# Plus process management, monitoring, scaling +# Plus client connection management + +# Total infrastructure: N MCP servers (one per tool provider) +``` + +### Protocol Support Comparison + +#### UTCP Protocol Flexibility +```json +{ + "tools": [ + { + "name": "http_tool", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/data" + } + }, + { + "name": "websocket_tool", + "tool_call_template": { + "call_template_type": "websocket", + "url": "wss://api.example.com/ws" + } + }, + { + "name": "cli_tool", + "tool_call_template": { + "call_template_type": "cli", + "command": "git", + "args": ["status"] + } + } + ] +} +``` + +#### MCP Protocol Limitation +```python +# MCP only supports JSON-RPC over stdio/HTTP +# All tools must be wrapped in MCP servers +# Cannot directly call WebSocket, CLI, or other protocols +``` ## Feature Comparison -| Feature | MCP | UTCP | -|---------|-----|------| -| **Tool Discovery** | Via MCP Server | Via manual discovery endpoint | -| **Protocol Support** | HTTP Streaming | HTTP, WebSockets, gRPC, CLI, etc. | -| **Authentication** | Handled by MCP Server | Uses tool's native authentication | -| **Streaming** | Native support | Supported via appropriate communication protocol (SSE, WebSockets) | -| **Implementation Complexity** | High (requires wrapper servers) | Low (simple JSON definitions) | -| **Performance** | Additional overhead due to proxy | Direct, native performance | -| **Evolution Speed** | Slow (all participants must update) | Fast (individual communication protocols can evolve independently) | - -## When to Choose Each Protocol - -### Choose MCP When: - -- You need strict standardization across all tools -- You're building a closed ecosystem where you control all components -- You're willing to invest in building and maintaining wrapper servers +### Authentication & Security + +#### UTCP: Native Authentication +```json +{ + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/data", + "auth": { + "auth_type": "oauth2", + "client_id": "${CLIENT_ID}", + "client_secret": "${CLIENT_SECRET}", + "token_url": "https://auth.example.com/token" + } + } +} +``` + +**Benefits:** +- Uses existing authentication systems +- No credential translation needed +- Native rate limiting and monitoring +- Existing security policies apply + +#### MCP: Server-Mediated Authentication +```python +# MCP server must handle auth translation +class WeatherMCPServer: + def __init__(self, api_key): + self.api_key = api_key # Server stores credentials + + async def call_tool(self, name, args): + # Server makes authenticated call + return await weather_api.get(args["location"], auth=self.api_key) +``` + +**Challenges:** +- Credential management in MCP servers +- Additional security layer to maintain +- Auth translation complexity + +### Streaming & Real-time Data + +#### UTCP: Native Streaming Support +```json +{ + "name": "stream_logs", + "tool_call_template": { + "call_template_type": "sse", + "url": "https://api.example.com/logs/stream", + "timeout": 300 + } +} +``` + +#### MCP: Limited Streaming +```python +# MCP has basic streaming but requires server implementation +# More complex to set up and maintain +``` + +### Error Handling + +#### UTCP: Native Error Responses +```python +# Errors come directly from the tool +try: + result = await client.call_tool("api.get_data", {"id": "123"}) +except httpx.HTTPStatusError as e: + # Native HTTP error with full context + print(f"API returned {e.response.status_code}: {e.response.text}") +``` + +#### MCP: Wrapped Error Responses +```python +# Errors are wrapped by MCP server +try: + result = await mcp_client.call_tool("get_data", {"id": "123"}) +except MCPError as e: + # MCP error - original context may be lost + print(f"MCP error: {e.message}") +``` + +## Migration & Interoperability + +### Migrating from MCP to UTCP + +UTCP provides an MCP plugin for gradual migration: + +```python +# Phase 1: Use existing MCP servers via UTCP +client = await UtcpClient.create(config={ + "manual_call_templates": [{ + "name": "legacy_mcp_service", + "call_template_type": "mcp", + "server_config": { + "command": "node", + "args": ["existing-mcp-server.js"] + } + }] +}) + +# Phase 2: Migrate high-value tools to native UTCP +# Phase 3: Deprecate MCP servers +``` + +[**Complete migration guide →**](./providers/mcp.md) + +### Hybrid Approach + +You can use both protocols simultaneously: + +```python +client = await UtcpClient.create(config={ + "manual_call_templates": [ + { + "name": "native_api", + "call_template_type": "http", + "url": "https://api.example.com/utcp" + }, + { + "name": "legacy_mcp", + "call_template_type": "mcp", + "server_config": {"command": "mcp-server"} + } + ] +}) + +# Call native UTCP tool +result1 = await client.call_tool("native_api.get_data", {}) + +# Call MCP tool through UTCP +result2 = await client.call_tool("legacy_mcp.mcp_tool", {}) +``` + +## Enterprise Decision Factors + +### Total Cost of Ownership + +#### UTCP TCO +``` +Infrastructure: $0 (uses existing APIs) +Development: Low (add one endpoint) +Maintenance: Minimal (static JSON) +Scaling: Automatic (scales with existing API) +Monitoring: Existing tools work +``` + +#### MCP TCO +``` +Infrastructure: High (dedicated servers) +Development: High (build wrapper servers) +Maintenance: High (server management) +Scaling: Complex (scale MCP servers separately) +Monitoring: Additional monitoring stack needed +``` + +### Development Velocity + +#### UTCP Development Speed +```python +# Day 1: Add UTCP endpoint +@app.get("/utcp") +def get_manual(): + return {"tools": [...]} + +# Day 2: Tools are available to AI agents +# No additional infrastructure needed +``` + +#### MCP Development Speed +```python +# Week 1-2: Build MCP server +# Week 3: Deploy and configure server +# Week 4: Set up monitoring and scaling +# Week 5: Handle production issues +# Ongoing: Server maintenance +``` + +### Risk Assessment + +| Risk Factor | UTCP | MCP | +|-------------|------|-----| +| **Single Point of Failure** | None (direct calls) | MCP servers | +| **Vendor Lock-in** | Low (standard protocols) | Medium (MCP-specific) | +| **Maintenance Burden** | Low | High | +| **Security Surface** | Minimal | Expanded | +| **Performance Risk** | Low | Medium | + +## Decision Framework ### Choose UTCP When: -- You want to leverage existing APIs without building wrappers -- You need to support diverse communication protocols -- You value direct, efficient communication -- You prioritize low implementation overhead -- You want to minimize infrastructure costs +✅ **You have existing APIs** that work well +✅ **You want minimal infrastructure** overhead +✅ **You need multiple protocols** (HTTP, WebSocket, CLI, etc.) +✅ **You prioritize performance** and direct communication +✅ **You want to leverage existing** auth, monitoring, scaling +✅ **You have limited resources** for server maintenance +✅ **You need rapid deployment** of AI tool access -## Real-World Example +### Choose MCP When: + +✅ **You need strict protocol standardization** across all tools +✅ **You're building a closed ecosystem** with full control +✅ **You have resources** for building and maintaining servers +✅ **You need MCP-specific features** like resources and prompts +✅ **You're already invested** in MCP infrastructure +✅ **You prefer centralized control** over tool access + +### Hybrid Approach When: + +✅ **You're migrating from MCP** to UTCP gradually +✅ **You have mixed requirements** (some tools need MCP features) +✅ **You want to evaluate both** approaches in production +✅ **You have legacy MCP investments** to preserve + +## Real-World Examples + +### E-commerce API Integration + +#### UTCP Approach +```python +# Existing e-commerce API +@app.get("/products/{product_id}") +def get_product(product_id: str): + return {"id": product_id, "name": "Widget", "price": 29.99} + +# Add UTCP discovery +@app.get("/utcp") +def get_manual(): + return { + "tools": [{ + "name": "get_product", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.shop.com/products/${product_id}", + "http_method": "GET" + } + }] + } + +# Total additional code: ~10 lines +# Additional infrastructure: 0 servers +``` + +#### MCP Approach +```python +# Requires building MCP server +from mcp.server import Server + +server = Server("ecommerce-server") + +@server.call_tool() +async def call_tool(name: str, args: dict): + if name == "get_product": + # Call existing API + response = await httpx.get(f"https://api.shop.com/products/{args['product_id']}") + return response.json() + +# Plus: server deployment, monitoring, scaling +# Total additional code: ~50+ lines +# Additional infrastructure: 1+ servers +``` + +### Database Query Tool + +#### UTCP Approach +```json +{ + "name": "query_database", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.company.com/query", + "http_method": "POST", + "body": {"sql": "${query}"}, + "auth": { + "auth_type": "api_key", + "api_key": "${DB_API_KEY}", + "var_name": "Authorization", + "location": "header" + } + } +} +``` + +#### MCP Approach +```python +# Requires MCP server with database connection +# Plus connection pooling, query validation, etc. +# Much more complex implementation +``` + +## Performance Benchmarks + +### Latency Comparison + +| Scenario | UTCP | MCP | Difference | +|----------|------|-----|------------| +| Simple API call | 50ms | 75ms | +50% overhead | +| Complex query | 200ms | 250ms | +25% overhead | +| File operation | 10ms | 20ms | +100% overhead | +| Streaming data | Real-time | Buffered | Significant delay | + +### Resource Usage + +| Resource | UTCP | MCP | +|----------|------|-----| +| Memory | 0MB (no servers) | 50-200MB per server | +| CPU | 0% (no processing) | 5-15% per server | +| Network | Direct | Double hops | +| Storage | 0GB | Logs, state, config | + +## Migration Timeline + +### From MCP to UTCP + +**Phase 1 (Week 1): Assessment** +- Inventory existing MCP servers +- Identify high-value tools for migration +- Plan migration strategy + +**Phase 2 (Week 2-3): Hybrid Setup** +- Install UTCP with MCP plugin +- Test existing MCP tools through UTCP +- Validate functionality + +**Phase 3 (Week 4-8): Gradual Migration** +- Migrate tools one by one to native UTCP +- Add `/utcp` endpoints to existing APIs +- Update client configurations + +**Phase 4 (Week 9+): Cleanup** +- Deprecate MCP servers +- Remove MCP infrastructure +- Monitor and optimize + +[**Detailed migration guide →**](./migration-v0.1-to-v1.0.md) + +## Community & Ecosystem + +### UTCP Ecosystem +- **Multiple language implementations**: Python, TypeScript, Go, Rust +- **Growing protocol support**: HTTP, WebSocket, CLI, SSE, Text, MCP +- **Active development**: Regular releases and improvements +- **Open governance**: RFC process for changes + +### MCP Ecosystem +- **Anthropic-led development**: Centralized development +- **Growing tool library**: Community-contributed servers +- **IDE integrations**: Claude Desktop, Cline, etc. +- **Established patterns**: Well-documented server patterns -Consider an organization with an existing REST API that they want to expose to AI agents: +## Conclusion -**With MCP:** -1. Build an MCP server that wraps the REST API -2. Translate all calls between MCP format and REST format -3. Maintain and scale this additional server infrastructure -4. Handle authentication translation between MCP and the API +Both UTCP and MCP solve the tool integration problem, but with fundamentally different approaches: -**With UTCP:** -1. Create a simple JSON definition describing the REST API -2. Expose this definition via a discovery endpoint (typically `/utcp`) -3. The AI agent can now call the REST API directly +**UTCP excels when you:** +- Want to leverage existing APIs without additional infrastructure +- Need support for multiple communication protocols +- Prioritize performance and direct communication +- Have limited resources for server maintenance +- Want rapid deployment and minimal complexity -## Code comparison +**MCP excels when you:** +- Need strict protocol standardization +- Are building a controlled ecosystem +- Have resources for server infrastructure +- Need MCP-specific features beyond tool calling +- Prefer centralized tool management -You can find a full typescript example detailing the MCP and UTCP approach [here](https://github.com/universal-tool-calling-protocol/typescript-utcp/tree/main/examples/src/concrete_example). +**For most organizations**, UTCP's "manual" approach offers significant advantages in terms of simplicity, performance, and cost-effectiveness. The ability to expose existing APIs to AI agents with minimal changes and no additional infrastructure makes it an attractive choice for rapid AI tool integration. -## Conclusion +**For gradual adoption**, consider starting with UTCP's MCP plugin to use existing MCP servers while migrating high-value tools to native UTCP protocols over time. -Both MCP and UTCP aim to solve the problem of standardizing tool calling for AI agents, but they take fundamentally different approaches. +## Next Steps -MCP acts as a middleman, requiring all tools to be wrapped in its protocol. This provides standardization but at the cost of additional infrastructure and development overhead. +### To Get Started with UTCP: +1. **[Read the implementation guide](./implementation.md)** - Learn how to implement UTCP +2. **[Choose your protocols](./providers/index.md)** - Select communication methods +3. **[Check examples](https://github.com/universal-tool-calling-protocol/python-utcp/tree/main/examples)** - See real implementations -UTCP acts as a manual, describing how to call tools directly using their native interfaces. This eliminates the wrapper tax and leverages existing infrastructure, at the cost of requiring clients to handle different communication protocols. +### To Migrate from MCP: +1. **[Read the MCP integration guide](./providers/mcp.md)** - Use MCP tools via UTCP +2. **[Plan your migration](./migration-v0.1-to-v1.0.md)** - Step-by-step migration process +3. **[Join the community](https://discord.gg/ZpMbQ8jRbD)** - Get migration support -The choice between them depends on your specific requirements, existing infrastructure, and development resources. However, UTCP's "manual" approach offers significant advantages in terms of simplicity, efficiency, and leveraging existing investments in API infrastructure. +### To Learn More: +- **[UTCP Architecture](./api/index.md)** - Technical deep dive +- **[Security Considerations](./security.md)** - Security best practices +- **[Tool Provider Guide](./for-tool-providers.md)** - Expose your tools From 196b254f93588b5fb9fd23fbc474ccfca88776cf Mon Sep 17 00:00:00 2001 From: perrozzi Date: Fri, 5 Sep 2025 11:15:01 +0200 Subject: [PATCH 06/22] Update docs/index.md Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 816c444..ec2b57d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -214,7 +214,7 @@ You're building AI agents or applications that need to call external tools: - **[GitHub Organization](https://github.com/universal-tool-calling-protocol)** - Source code and issues - **[Discord Community](https://discord.gg/ZpMbQ8jRbD)** - Real-time help and discussions - **[Tool Registry](https://utcp.io/registry)** - Discover available tools -- **[RFC Process](./about/RFC.md)** - Contribute to the specification +- **[RFC Process](/about/RFC)** - Contribute to the specification --- From 2eb14393dd55067d0dec7cb515b627f3b7276378 Mon Sep 17 00:00:00 2001 From: Luca Perrozzi Date: Fri, 5 Sep 2025 09:21:03 +0000 Subject: [PATCH 07/22] Add missing API documentation for core classes - Add class descriptions for UtcpClient, serializers, and auth implementations - Add documentation for tool repository and variable loader classes - Add call template documentation for HTTP, CLI, Text, and SSE protocols - Fix 'No class documentation available' entries with minimal descriptions --- docs/api/core/utcp/data/auth_implementations/api_key_auth.md | 2 +- docs/api/core/utcp/data/auth_implementations/basic_auth.md | 2 +- docs/api/core/utcp/data/auth_implementations/oauth2_auth.md | 2 +- docs/api/core/utcp/data/utcp_manual.md | 2 +- docs/api/core/utcp/data/variable_loader.md | 2 +- .../dot_env_variable_loader.md | 2 +- docs/api/core/utcp/implementations/in_mem_tool_repository.md | 4 ++-- docs/api/core/utcp/utcp_client.md | 2 +- .../cli/src/utcp_cli/cli_call_template.md | 2 +- .../http/src/utcp_http/http_call_template.md | 2 +- .../http/src/utcp_http/sse_call_template.md | 2 +- .../text/src/utcp_text/text_call_template.md | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/api/core/utcp/data/auth_implementations/api_key_auth.md b/docs/api/core/utcp/data/auth_implementations/api_key_auth.md index be07d9c..c59c075 100644 --- a/docs/api/core/utcp/data/auth_implementations/api_key_auth.md +++ b/docs/api/core/utcp/data/auth_implementations/api_key_auth.md @@ -39,7 +39,7 @@ Supports placement in headers, query parameters, or cookies. ### class ApiKeyAuthSerializer ([Serializer](./../../interfaces/serializer.md#serializer)[ApiKeyAuth]) {#apikeyauthserializer} -*No class documentation available* +Serializer for ApiKeyAuth configuration and credentials. #### Methods: diff --git a/docs/api/core/utcp/data/auth_implementations/basic_auth.md b/docs/api/core/utcp/data/auth_implementations/basic_auth.md index ac451d6..c33cf25 100644 --- a/docs/api/core/utcp/data/auth_implementations/basic_auth.md +++ b/docs/api/core/utcp/data/auth_implementations/basic_auth.md @@ -35,7 +35,7 @@ encoded in the Authorization header. ### class BasicAuthSerializer ([Serializer](./../../interfaces/serializer.md#serializer)[BasicAuth]) {#basicauthserializer} -*No class documentation available* +Serializer for BasicAuth configuration and credentials. #### Methods: diff --git a/docs/api/core/utcp/data/auth_implementations/oauth2_auth.md b/docs/api/core/utcp/data/auth_implementations/oauth2_auth.md index b6198da..d19a9ac 100644 --- a/docs/api/core/utcp/data/auth_implementations/oauth2_auth.md +++ b/docs/api/core/utcp/data/auth_implementations/oauth2_auth.md @@ -39,7 +39,7 @@ authentication. The client automatically handles token acquisition and refresh. ### class OAuth2AuthSerializer ([Serializer](./../../interfaces/serializer.md#serializer)[OAuth2Auth]) {#oauth2authserializer} -*No class documentation available* +Serializer for OAuth2Auth configuration and token management. #### Methods: diff --git a/docs/api/core/utcp/data/utcp_manual.md b/docs/api/core/utcp/data/utcp_manual.md index 2bcc886..011ff0e 100644 --- a/docs/api/core/utcp/data/utcp_manual.md +++ b/docs/api/core/utcp/data/utcp_manual.md @@ -70,7 +70,7 @@ exclude=["tool1"] ### class UtcpManualSerializer ([Serializer](./../interfaces/serializer.md#serializer)[UtcpManual]) {#utcpmanualserializer} -*No class documentation available* +Serializer for converting UtcpManual objects to/from dictionary format. #### Methods: diff --git a/docs/api/core/utcp/data/variable_loader.md b/docs/api/core/utcp/data/variable_loader.md index 597cb42..848a45c 100644 --- a/docs/api/core/utcp/data/variable_loader.md +++ b/docs/api/core/utcp/data/variable_loader.md @@ -52,7 +52,7 @@ Variable value if found, None otherwise. ### class VariableLoaderSerializer ([Serializer](./../interfaces/serializer.md#serializer)[VariableLoader]) {#variableloaderserializer} -*No class documentation available* +Abstract base class for loading variables from different sources (environment, files, etc.). #### Fields: diff --git a/docs/api/core/utcp/data/variable_loader_implementations/dot_env_variable_loader.md b/docs/api/core/utcp/data/variable_loader_implementations/dot_env_variable_loader.md index 1a29499..d6824c8 100644 --- a/docs/api/core/utcp/data/variable_loader_implementations/dot_env_variable_loader.md +++ b/docs/api/core/utcp/data/variable_loader_implementations/dot_env_variable_loader.md @@ -61,7 +61,7 @@ Variable value if found in the file, None otherwise. ### class DotEnvVariableLoaderSerializer ([Serializer](./../../interfaces/serializer.md#serializer)[DotEnvVariableLoader]) {#dotenvvariableloaderserializer} -*No class documentation available* +Variable loader implementation for reading variables from .env files. #### Methods: diff --git a/docs/api/core/utcp/implementations/in_mem_tool_repository.md b/docs/api/core/utcp/implementations/in_mem_tool_repository.md index 3b774bb..fe5085c 100644 --- a/docs/api/core/utcp/implementations/in_mem_tool_repository.md +++ b/docs/api/core/utcp/implementations/in_mem_tool_repository.md @@ -9,7 +9,7 @@ sidebar_label: in_mem_tool_repository ### class InMemToolRepository ([ConcurrentToolRepository](./../interfaces/concurrent_tool_repository.md#concurrenttoolrepository)) {#inmemtoolrepository} -*No class documentation available* +In-memory implementation of tool repository for storing and managing UTCP tools. #### Methods: @@ -164,7 +164,7 @@ A list of all manual call templates in the repository. ### class InMemToolRepositoryConfigSerializer ([Serializer](./../interfaces/serializer.md#serializer)[InMemToolRepository]) {#inmemtoolrepositoryconfigserializer} -*No class documentation available* +Serializer for InMemToolRepository configuration to enable persistence and setup. #### Methods: diff --git a/docs/api/core/utcp/utcp_client.md b/docs/api/core/utcp/utcp_client.md index 699279b..dccd176 100644 --- a/docs/api/core/utcp/utcp_client.md +++ b/docs/api/core/utcp/utcp_client.md @@ -9,7 +9,7 @@ sidebar_label: utcp_client ### class UtcpClient {#utcpclient} -*No class documentation available* +Main client for discovering and calling UTCP tools across multiple communication protocols. #### Methods: diff --git a/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md b/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md index abfe4aa..37db0fa 100644 --- a/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md +++ b/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md @@ -39,7 +39,7 @@ Supports environment variable injection and custom working directories. ### class CliCallTemplateSerializer ([Serializer](./../../../../../core/utcp/interfaces/serializer.md#serializer)[CliCallTemplate]) {#clicalltemplateserializer} -*No class documentation available* +Call template for executing command-line tools and scripts. #### Methods: diff --git a/docs/api/plugins/communication_protocols/http/src/utcp_http/http_call_template.md b/docs/api/plugins/communication_protocols/http/src/utcp_http/http_call_template.md index 2420d59..9628946 100644 --- a/docs/api/plugins/communication_protocols/http/src/utcp_http/http_call_template.md +++ b/docs/api/plugins/communication_protocols/http/src/utcp_http/http_call_template.md @@ -48,7 +48,7 @@ URL body, headers or query pattern parameters are passed as query parameters usi ### class HttpCallTemplateSerializer ([Serializer](./../../../../../core/utcp/interfaces/serializer.md#serializer)[HttpCallTemplate]) {#httpcalltemplateserializer} -*No class documentation available* +Call template for HTTP-based tool calls with support for various HTTP methods and authentication. #### Methods: diff --git a/docs/api/plugins/communication_protocols/http/src/utcp_http/sse_call_template.md b/docs/api/plugins/communication_protocols/http/src/utcp_http/sse_call_template.md index 3dd6951..29ce336 100644 --- a/docs/api/plugins/communication_protocols/http/src/utcp_http/sse_call_template.md +++ b/docs/api/plugins/communication_protocols/http/src/utcp_http/sse_call_template.md @@ -49,7 +49,7 @@ or query pattern parameters are passed as query parameters using '?arg_name=\{ar ### class SSECallTemplateSerializer ([Serializer](./../../../../../core/utcp/interfaces/serializer.md#serializer)[SseCallTemplate]) {#ssecalltemplateserializer} -*No class documentation available* +Call template for Server-Sent Events (SSE) streaming connections. #### Methods: diff --git a/docs/api/plugins/communication_protocols/text/src/utcp_text/text_call_template.md b/docs/api/plugins/communication_protocols/text/src/utcp_text/text_call_template.md index 3c5b062..0cb9ef3 100644 --- a/docs/api/plugins/communication_protocols/text/src/utcp_text/text_call_template.md +++ b/docs/api/plugins/communication_protocols/text/src/utcp_text/text_call_template.md @@ -35,7 +35,7 @@ static tool configurations or environments where manuals are distributed as file ### class TextCallTemplateSerializer ([Serializer](./../../../../../core/utcp/interfaces/serializer.md#serializer)[TextCallTemplate]) {#textcalltemplateserializer} -*No class documentation available* +Call template for reading and processing text files from local or remote sources. #### Methods: From e5294286930f7b1b93a45c1978875edcc8c6199c Mon Sep 17 00:00:00 2001 From: Luca Perrozzi Date: Fri, 5 Sep 2025 09:27:47 +0000 Subject: [PATCH 08/22] Fix providers/index.md ID for sidebar compatibility --- docs/providers/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/providers/index.md b/docs/providers/index.md index f1ad2ba..8f3f0f9 100644 --- a/docs/providers/index.md +++ b/docs/providers/index.md @@ -1,5 +1,5 @@ --- -id: providers +id: index title: Communication Protocols sidebar_position: 3 --- From a4c0df02c47fcc901ec8eb80b3a531f06278f304 Mon Sep 17 00:00:00 2001 From: Luca Perrozzi Date: Fri, 5 Sep 2025 09:30:29 +0000 Subject: [PATCH 09/22] Revert "Add missing API documentation for core classes" This reverts commit 2eb14393dd55067d0dec7cb515b627f3b7276378. --- docs/api/core/utcp/data/auth_implementations/api_key_auth.md | 2 +- docs/api/core/utcp/data/auth_implementations/basic_auth.md | 2 +- docs/api/core/utcp/data/auth_implementations/oauth2_auth.md | 2 +- docs/api/core/utcp/data/utcp_manual.md | 2 +- docs/api/core/utcp/data/variable_loader.md | 2 +- .../dot_env_variable_loader.md | 2 +- docs/api/core/utcp/implementations/in_mem_tool_repository.md | 4 ++-- docs/api/core/utcp/utcp_client.md | 2 +- .../cli/src/utcp_cli/cli_call_template.md | 2 +- .../http/src/utcp_http/http_call_template.md | 2 +- .../http/src/utcp_http/sse_call_template.md | 2 +- .../text/src/utcp_text/text_call_template.md | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/api/core/utcp/data/auth_implementations/api_key_auth.md b/docs/api/core/utcp/data/auth_implementations/api_key_auth.md index c59c075..be07d9c 100644 --- a/docs/api/core/utcp/data/auth_implementations/api_key_auth.md +++ b/docs/api/core/utcp/data/auth_implementations/api_key_auth.md @@ -39,7 +39,7 @@ Supports placement in headers, query parameters, or cookies. ### class ApiKeyAuthSerializer ([Serializer](./../../interfaces/serializer.md#serializer)[ApiKeyAuth]) {#apikeyauthserializer} -Serializer for ApiKeyAuth configuration and credentials. +*No class documentation available* #### Methods: diff --git a/docs/api/core/utcp/data/auth_implementations/basic_auth.md b/docs/api/core/utcp/data/auth_implementations/basic_auth.md index c33cf25..ac451d6 100644 --- a/docs/api/core/utcp/data/auth_implementations/basic_auth.md +++ b/docs/api/core/utcp/data/auth_implementations/basic_auth.md @@ -35,7 +35,7 @@ encoded in the Authorization header. ### class BasicAuthSerializer ([Serializer](./../../interfaces/serializer.md#serializer)[BasicAuth]) {#basicauthserializer} -Serializer for BasicAuth configuration and credentials. +*No class documentation available* #### Methods: diff --git a/docs/api/core/utcp/data/auth_implementations/oauth2_auth.md b/docs/api/core/utcp/data/auth_implementations/oauth2_auth.md index d19a9ac..b6198da 100644 --- a/docs/api/core/utcp/data/auth_implementations/oauth2_auth.md +++ b/docs/api/core/utcp/data/auth_implementations/oauth2_auth.md @@ -39,7 +39,7 @@ authentication. The client automatically handles token acquisition and refresh. ### class OAuth2AuthSerializer ([Serializer](./../../interfaces/serializer.md#serializer)[OAuth2Auth]) {#oauth2authserializer} -Serializer for OAuth2Auth configuration and token management. +*No class documentation available* #### Methods: diff --git a/docs/api/core/utcp/data/utcp_manual.md b/docs/api/core/utcp/data/utcp_manual.md index 011ff0e..2bcc886 100644 --- a/docs/api/core/utcp/data/utcp_manual.md +++ b/docs/api/core/utcp/data/utcp_manual.md @@ -70,7 +70,7 @@ exclude=["tool1"] ### class UtcpManualSerializer ([Serializer](./../interfaces/serializer.md#serializer)[UtcpManual]) {#utcpmanualserializer} -Serializer for converting UtcpManual objects to/from dictionary format. +*No class documentation available* #### Methods: diff --git a/docs/api/core/utcp/data/variable_loader.md b/docs/api/core/utcp/data/variable_loader.md index 848a45c..597cb42 100644 --- a/docs/api/core/utcp/data/variable_loader.md +++ b/docs/api/core/utcp/data/variable_loader.md @@ -52,7 +52,7 @@ Variable value if found, None otherwise. ### class VariableLoaderSerializer ([Serializer](./../interfaces/serializer.md#serializer)[VariableLoader]) {#variableloaderserializer} -Abstract base class for loading variables from different sources (environment, files, etc.). +*No class documentation available* #### Fields: diff --git a/docs/api/core/utcp/data/variable_loader_implementations/dot_env_variable_loader.md b/docs/api/core/utcp/data/variable_loader_implementations/dot_env_variable_loader.md index d6824c8..1a29499 100644 --- a/docs/api/core/utcp/data/variable_loader_implementations/dot_env_variable_loader.md +++ b/docs/api/core/utcp/data/variable_loader_implementations/dot_env_variable_loader.md @@ -61,7 +61,7 @@ Variable value if found in the file, None otherwise. ### class DotEnvVariableLoaderSerializer ([Serializer](./../../interfaces/serializer.md#serializer)[DotEnvVariableLoader]) {#dotenvvariableloaderserializer} -Variable loader implementation for reading variables from .env files. +*No class documentation available* #### Methods: diff --git a/docs/api/core/utcp/implementations/in_mem_tool_repository.md b/docs/api/core/utcp/implementations/in_mem_tool_repository.md index fe5085c..3b774bb 100644 --- a/docs/api/core/utcp/implementations/in_mem_tool_repository.md +++ b/docs/api/core/utcp/implementations/in_mem_tool_repository.md @@ -9,7 +9,7 @@ sidebar_label: in_mem_tool_repository ### class InMemToolRepository ([ConcurrentToolRepository](./../interfaces/concurrent_tool_repository.md#concurrenttoolrepository)) {#inmemtoolrepository} -In-memory implementation of tool repository for storing and managing UTCP tools. +*No class documentation available* #### Methods: @@ -164,7 +164,7 @@ A list of all manual call templates in the repository. ### class InMemToolRepositoryConfigSerializer ([Serializer](./../interfaces/serializer.md#serializer)[InMemToolRepository]) {#inmemtoolrepositoryconfigserializer} -Serializer for InMemToolRepository configuration to enable persistence and setup. +*No class documentation available* #### Methods: diff --git a/docs/api/core/utcp/utcp_client.md b/docs/api/core/utcp/utcp_client.md index dccd176..699279b 100644 --- a/docs/api/core/utcp/utcp_client.md +++ b/docs/api/core/utcp/utcp_client.md @@ -9,7 +9,7 @@ sidebar_label: utcp_client ### class UtcpClient {#utcpclient} -Main client for discovering and calling UTCP tools across multiple communication protocols. +*No class documentation available* #### Methods: diff --git a/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md b/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md index 37db0fa..abfe4aa 100644 --- a/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md +++ b/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md @@ -39,7 +39,7 @@ Supports environment variable injection and custom working directories. ### class CliCallTemplateSerializer ([Serializer](./../../../../../core/utcp/interfaces/serializer.md#serializer)[CliCallTemplate]) {#clicalltemplateserializer} -Call template for executing command-line tools and scripts. +*No class documentation available* #### Methods: diff --git a/docs/api/plugins/communication_protocols/http/src/utcp_http/http_call_template.md b/docs/api/plugins/communication_protocols/http/src/utcp_http/http_call_template.md index 9628946..2420d59 100644 --- a/docs/api/plugins/communication_protocols/http/src/utcp_http/http_call_template.md +++ b/docs/api/plugins/communication_protocols/http/src/utcp_http/http_call_template.md @@ -48,7 +48,7 @@ URL body, headers or query pattern parameters are passed as query parameters usi ### class HttpCallTemplateSerializer ([Serializer](./../../../../../core/utcp/interfaces/serializer.md#serializer)[HttpCallTemplate]) {#httpcalltemplateserializer} -Call template for HTTP-based tool calls with support for various HTTP methods and authentication. +*No class documentation available* #### Methods: diff --git a/docs/api/plugins/communication_protocols/http/src/utcp_http/sse_call_template.md b/docs/api/plugins/communication_protocols/http/src/utcp_http/sse_call_template.md index 29ce336..3dd6951 100644 --- a/docs/api/plugins/communication_protocols/http/src/utcp_http/sse_call_template.md +++ b/docs/api/plugins/communication_protocols/http/src/utcp_http/sse_call_template.md @@ -49,7 +49,7 @@ or query pattern parameters are passed as query parameters using '?arg_name=\{ar ### class SSECallTemplateSerializer ([Serializer](./../../../../../core/utcp/interfaces/serializer.md#serializer)[SseCallTemplate]) {#ssecalltemplateserializer} -Call template for Server-Sent Events (SSE) streaming connections. +*No class documentation available* #### Methods: diff --git a/docs/api/plugins/communication_protocols/text/src/utcp_text/text_call_template.md b/docs/api/plugins/communication_protocols/text/src/utcp_text/text_call_template.md index 0cb9ef3..3c5b062 100644 --- a/docs/api/plugins/communication_protocols/text/src/utcp_text/text_call_template.md +++ b/docs/api/plugins/communication_protocols/text/src/utcp_text/text_call_template.md @@ -35,7 +35,7 @@ static tool configurations or environments where manuals are distributed as file ### class TextCallTemplateSerializer ([Serializer](./../../../../../core/utcp/interfaces/serializer.md#serializer)[TextCallTemplate]) {#textcalltemplateserializer} -Call template for reading and processing text files from local or remote sources. +*No class documentation available* #### Methods: From b7c2ef3b953fc077b9f4b78d8e09daf81e64e7dc Mon Sep 17 00:00:00 2001 From: Luca Perrozzi Date: Fri, 5 Sep 2025 09:57:28 +0000 Subject: [PATCH 10/22] Update terminology from providers to protocol plugins - Rename docs/providers to docs/protocols for v1.0 terminology - Update sidebars.ts to reference protocols/ paths - Update protocols/index.md with correct plugin-based architecture description - Replace 'providers' with 'protocol plugins' throughout documentation - Align with v1.0 call template terminology --- docs/for-tool-providers.md | 4 +- docs/implementation.md | 4 +- docs/index.md | 22 ++--- docs/{providers => protocols}/cli.md | 0 docs/{providers => protocols}/http.md | 0 docs/protocols/index.md | 94 ++++++++++++++++++++++ docs/{providers => protocols}/mcp.md | 0 docs/{providers => protocols}/sse.md | 0 docs/{providers => protocols}/text.md | 0 docs/{providers => protocols}/websocket.md | 0 docs/providers/index.md | 92 --------------------- docs/security.md | 12 +-- docs/utcp-vs-mcp.md | 6 +- sidebars.ts | 14 ++-- 14 files changed, 125 insertions(+), 123 deletions(-) rename docs/{providers => protocols}/cli.md (100%) rename docs/{providers => protocols}/http.md (100%) create mode 100644 docs/protocols/index.md rename docs/{providers => protocols}/mcp.md (100%) rename docs/{providers => protocols}/sse.md (100%) rename docs/{providers => protocols}/text.md (100%) rename docs/{providers => protocols}/websocket.md (100%) delete mode 100644 docs/providers/index.md diff --git a/docs/for-tool-providers.md b/docs/for-tool-providers.md index cf5b80d..0d43f1c 100644 --- a/docs/for-tool-providers.md +++ b/docs/for-tool-providers.md @@ -121,7 +121,7 @@ Optional tool fields: | `tags` | array | Tags for categorization and search | | `examples` | array | Usage examples | -## Communication Protocols +## Communication Protocol Plugins ### HTTP Tools @@ -684,6 +684,6 @@ async def test_manual_integration(): 6. **Iterate**: Improve based on usage patterns and feedback For more information, see: -- [Communication Protocols](./providers/index.md) +- [Communication Protocol Plugins](./protocols/index.md) - [Implementation Guide](./implementation.md) - [Security Considerations](./security.md) diff --git a/docs/implementation.md b/docs/implementation.md index 06abbba..ff16d08 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -421,7 +421,7 @@ except UtcpError as e: ## Advanced Implementation Patterns (Python) -### Custom Communication Protocols +### Custom Communication Protocol Plugins ```python from utcp.interfaces.communication_protocol import CommunicationProtocol @@ -575,6 +575,6 @@ Check the [UTCP GitHub organization](https://github.com/universal-tool-calling-p 7. **Scale**: Plan for scaling as your tool ecosystem grows For more detailed information, see: -- [Communication Protocols](./providers/index.md) +- [Communication Protocol Plugins](./protocols/index.md) - [API Reference](./api/index.md) - [Security Considerations](./security.md) diff --git a/docs/index.md b/docs/index.md index ec2b57d..b46cf7b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -133,14 +133,14 @@ UTCP supports multiple communication protocols through plugins: | Protocol | Use Case | Plugin | Status | |----------|----------|--------|--------| -| **[HTTP](./providers/http.md)** | REST APIs, webhooks | `utcp-http` | ✅ Stable | -| **[WebSocket](./providers/websocket.md)** | Real-time communication | `utcp-websocket` | ✅ Stable | -| **[CLI](./providers/cli.md)** | Command-line tools | `utcp-cli` | ✅ Stable | -| **[Server-Sent Events](./providers/sse.md)** | Streaming data | `utcp-http` | ✅ Stable | -| **[Text Files](./providers/text.md)** | File reading | `utcp-text` | ✅ Stable | -| **[MCP](./providers/mcp.md)** | MCP interoperability | `utcp-mcp` | ✅ Stable | +| **[HTTP](./protocols/http.md)** | REST APIs, webhooks | `utcp-http` | ✅ Stable | +| **[WebSocket](./protocols/websocket.md)** | Real-time communication | `utcp-websocket` | ✅ Stable | +| **[CLI](./protocols/cli.md)** | Command-line tools | `utcp-cli` | ✅ Stable | +| **[Server-Sent Events](./protocols/sse.md)** | Streaming data | `utcp-http` | ✅ Stable | +| **[Text Files](./protocols/text.md)** | File reading | `utcp-text` | ✅ Stable | +| **[MCP](./protocols/mcp.md)** | MCP interoperability | `utcp-mcp` | ✅ Stable | -[View all protocols →](./providers/index.md) +[View all protocols →](./protocols/index.md) ## Architecture Overview @@ -194,19 +194,19 @@ You're building AI agents or applications that need to call external tools: ### For Tool Providers 1. **[Read the provider guide](./for-tool-providers.md)** - Learn how to expose your tools -2. **[Choose your protocol](./providers/index.md)** - Select the right communication method +2. **[Choose your protocol](./protocols/index.md)** - Select the right communication method 3. **[Implement your manual](./implementation.md)** - Add UTCP to your existing API 4. **[Secure your tools](./security.md)** - Implement proper authentication ### For Tool Consumers 1. **[Read the implementation guide](./implementation.md)** - Learn how to build UTCP clients -2. **[Explore protocols](./providers/index.md)** - Understand available communication options +2. **[Explore protocols](./protocols/index.md)** - Understand available communication options 3. **[Check examples](https://github.com/universal-tool-calling-protocol/python-utcp/tree/main/examples)** - See real-world implementations 4. **[Join the community](https://discord.gg/ZpMbQ8jRbD)** - Get help and share experiences ### Migration from Other Systems - **[From UTCP v0.1](./migration-v0.1-to-v1.0.md)** - Upgrade to the latest version -- **[From MCP](./providers/mcp.md)** - Migrate from Model Context Protocol +- **[From MCP](./protocols/mcp.md)** - Migrate from Model Context Protocol - **[From custom solutions](./implementation.md)** - Replace existing tool integrations ## Community & Support @@ -221,4 +221,4 @@ You're building AI agents or applications that need to call external tools: **Ready to get started?** Choose your path: - 🛠️ [**I want to expose my tools**](./for-tool-providers.md) - 🤖 [**I want to call tools**](./implementation.md) -- 📚 [**I want to learn more**](./providers/index.md) +- 📚 [**I want to learn more**](./protocols/index.md) diff --git a/docs/providers/cli.md b/docs/protocols/cli.md similarity index 100% rename from docs/providers/cli.md rename to docs/protocols/cli.md diff --git a/docs/providers/http.md b/docs/protocols/http.md similarity index 100% rename from docs/providers/http.md rename to docs/protocols/http.md diff --git a/docs/protocols/index.md b/docs/protocols/index.md new file mode 100644 index 0000000..01452a6 --- /dev/null +++ b/docs/protocols/index.md @@ -0,0 +1,94 @@ +--- +id: index +title: Communication Protocol Plugins +sidebar_position: 3 +--- + +# Communication Protocol Plugins + +UTCP v1.0 features a modular, plugin-based architecture where different communication protocols are implemented as separate plugins. Each protocol plugin provides call templates and communication handlers for specific transport methods. + +## Available Protocol Plugins + +| Protocol | Plugin Package | Call Template | Use Cases | +|----------|----------------|---------------|-----------| +| **[HTTP](./http.md)** | `utcp-http` | `HttpCallTemplate` | REST APIs, webhooks, web services | +| **[WebSocket](./websocket.md)** | `utcp-websocket` | `WebSocketCallTemplate` | Real-time communication, streaming | +| **[CLI](./cli.md)** | `utcp-cli` | `CliCallTemplate` | Command-line tools, scripts | +| **[Server-Sent Events](./sse.md)** | `utcp-http` | `SseCallTemplate` | Event streaming, live updates | +| **[Text Files](./text.md)** | `utcp-text` | `TextCallTemplate` | File reading, static content | +| **[MCP](./mcp.md)** | `utcp-mcp` | `McpCallTemplate` | Model Context Protocol interop | + +## Plugin Architecture + +Each protocol plugin consists of: + +### Call Templates +Define how to structure calls for the specific protocol: +```python +# Example: HTTP Call Template +{ + "call_template_type": "http", + "url": "https://api.example.com/endpoint", + "http_method": "POST", + "headers": {"Content-Type": "application/json"} +} +``` + +### Communication Protocols +Handle the actual communication logic: +```python +class HttpCommunicationProtocol: + async def call_tool(self, call_template, tool_args): + # Implementation for HTTP calls + pass +``` + +## Installing Protocol Plugins + +```bash +# Core UTCP library +pip install utcp + +# Install specific protocol plugins +pip install utcp-http # HTTP, SSE protocols +pip install utcp-cli # CLI protocol +pip install utcp-websocket # WebSocket protocol +pip install utcp-text # Text file protocol +pip install utcp-mcp # MCP interoperability +``` + +## Creating Custom Protocol Plugins + +You can extend UTCP with custom communication protocols: + +```python +from utcp.interfaces.communication_protocol import CommunicationProtocol +from utcp.data.call_template import CallTemplate + +class CustomCallTemplate(CallTemplate): + call_template_type: str = "custom" + custom_field: str + +class CustomCommunicationProtocol(CommunicationProtocol): + async def call_tool(self, call_template: CustomCallTemplate, tool_args: dict): + # Your custom protocol implementation + pass + +# Register the protocol +from utcp.plugins.discovery import register_communication_protocol +register_communication_protocol(CustomCommunicationProtocol()) +``` + +## Protocol Selection Guide + +Choose the right protocol plugin based on your needs: + +- **HTTP**: Most common for REST APIs and web services +- **WebSocket**: Real-time bidirectional communication +- **CLI**: Wrapping existing command-line tools +- **SSE**: Server-sent events for streaming data +- **Text**: Reading configuration files or static content +- **MCP**: Interoperability with Model Context Protocol tools + +For detailed information about each protocol plugin, see the individual protocol documentation pages. diff --git a/docs/providers/mcp.md b/docs/protocols/mcp.md similarity index 100% rename from docs/providers/mcp.md rename to docs/protocols/mcp.md diff --git a/docs/providers/sse.md b/docs/protocols/sse.md similarity index 100% rename from docs/providers/sse.md rename to docs/protocols/sse.md diff --git a/docs/providers/text.md b/docs/protocols/text.md similarity index 100% rename from docs/providers/text.md rename to docs/protocols/text.md diff --git a/docs/providers/websocket.md b/docs/protocols/websocket.md similarity index 100% rename from docs/providers/websocket.md rename to docs/protocols/websocket.md diff --git a/docs/providers/index.md b/docs/providers/index.md deleted file mode 100644 index 8f3f0f9..0000000 --- a/docs/providers/index.md +++ /dev/null @@ -1,92 +0,0 @@ ---- -id: index -title: Communication Protocols -sidebar_position: 3 ---- - -# Communication Protocols - -UTCP supports multiple communication protocols through its plugin architecture. Each protocol plugin provides the necessary implementation to call tools using that specific transport method. - -## Available Protocols - -### Core Protocols - -| Protocol | Plugin | Status | Description | -|----------|--------|--------|-------------| -| [HTTP](./http.md) | `utcp-http` | ✅ Stable | REST APIs, webhooks, and standard HTTP services | -| [Server-Sent Events](./sse.md) | `utcp-http` | ✅ Stable | Real-time streaming over HTTP | -| [WebSocket](./websocket.md) | `utcp-websocket` | ✅ Stable | Bidirectional real-time communication | -| [CLI](./cli.md) | `utcp-cli` | ✅ Stable | Command-line tools and scripts | -| [Text Files](./text.md) | `utcp-text` | ✅ Stable | Reading local and remote text files | - -### Integration Protocols - -| Protocol | Plugin | Status | Description | -|----------|--------|--------|-------------| -| [MCP](./mcp.md) | `utcp-mcp` | ✅ Stable | Model Context Protocol interoperability | - -### Experimental Protocols - -| Protocol | Plugin | Status | Description | -|----------|--------|--------|-------------| -| [GraphQL](./graphql.md) | `utcp-gql` | 🚧 Beta | GraphQL APIs and subscriptions | -| [gRPC](./grpc.md) | `utcp-grpc` | 🚧 Beta | High-performance RPC calls | -| [TCP](./tcp.md) | `utcp-socket` | 🚧 Beta | Raw TCP socket connections | -| [UDP](./udp.md) | `utcp-socket` | 🚧 Beta | UDP packet-based communication | - -## Protocol Selection Guide - -Choose the right protocol based on your tool's characteristics: - -### Use HTTP when: -- Your tool is a REST API -- You need simple request/response patterns -- You want maximum compatibility - -### Use WebSocket when: -- You need bidirectional communication -- Your tool provides real-time updates -- You want persistent connections - -### Use CLI when: -- Your tool is a command-line application -- You need to execute local scripts -- You're wrapping existing CLI tools - -### Use SSE when: -- You need server-to-client streaming -- You want real-time updates over HTTP -- You need simple event streaming - -## Plugin Architecture - -UTCP's plugin system allows you to: - -1. **Extend existing protocols** with custom authentication or data transformation -2. **Create new protocols** for specialized communication needs -3. **Combine protocols** for complex tool interactions - -### Creating Custom Protocols - -To create a custom protocol plugin, implement the [`CommunicationProtocol`](../api/core/utcp/interfaces/communication_protocol.md) interface: - -```python -from utcp.interfaces.communication_protocol import CommunicationProtocol -from utcp.data.call_template import CallTemplate - -class MyCustomProtocol(CommunicationProtocol): - def get_supported_call_template_types(self) -> List[str]: - return ["my_custom_protocol"] - - async def call_tool(self, call_template: CallTemplate, tool_args: Dict[str, Any]) -> Any: - # Implement your protocol logic here - pass -``` - -## Next Steps - -- Choose a protocol from the list above -- Read the specific protocol documentation -- Check out the [API Reference](../api/index.md) for implementation details -- See [Examples](../examples/index.md) for practical implementations diff --git a/docs/security.md b/docs/security.md index de5d007..b0172d9 100644 --- a/docs/security.md +++ b/docs/security.md @@ -644,9 +644,9 @@ def validate_manual_security(manual: dict) -> list: By following these security guidelines, you can safely implement UTCP while maintaining strong security posture across all communication protocols. For protocol-specific security details, see: -- [HTTP Security](./providers/http.md#security-considerations) -- [WebSocket Security](./providers/websocket.md#security-considerations) -- [CLI Security](./providers/cli.md#security-considerations) -- [SSE Security](./providers/sse.md#security-considerations) -- [Text Security](./providers/text.md#security-considerations) -- [MCP Security](./providers/mcp.md#security-considerations) +- [HTTP Security](./protocols/http.md#security-considerations) +- [WebSocket Security](./protocols/websocket.md#security-considerations) +- [CLI Security](./protocols/cli.md#security-considerations) +- [SSE Security](./protocols/sse.md#security-considerations) +- [Text Security](./protocols/text.md#security-considerations) +- [MCP Security](./protocols/mcp.md#security-considerations) diff --git a/docs/utcp-vs-mcp.md b/docs/utcp-vs-mcp.md index 099f0d9..72927ea 100644 --- a/docs/utcp-vs-mcp.md +++ b/docs/utcp-vs-mcp.md @@ -278,7 +278,7 @@ client = await UtcpClient.create(config={ # Phase 3: Deprecate MCP servers ``` -[**Complete migration guide →**](./providers/mcp.md) +[**Complete migration guide →**](./protocols/mcp.md) ### Hybrid Approach @@ -551,11 +551,11 @@ Both UTCP and MCP solve the tool integration problem, but with fundamentally dif ### To Get Started with UTCP: 1. **[Read the implementation guide](./implementation.md)** - Learn how to implement UTCP -2. **[Choose your protocols](./providers/index.md)** - Select communication methods +2. **[Choose your protocols](./protocols/index.md)** - Select communication methods 3. **[Check examples](https://github.com/universal-tool-calling-protocol/python-utcp/tree/main/examples)** - See real implementations ### To Migrate from MCP: -1. **[Read the MCP integration guide](./providers/mcp.md)** - Use MCP tools via UTCP +1. **[Read the MCP integration guide](./protocols/mcp.md)** - Use MCP tools via UTCP 2. **[Plan your migration](./migration-v0.1-to-v1.0.md)** - Step-by-step migration process 3. **[Join the community](https://discord.gg/ZpMbQ8jRbD)** - Get migration support diff --git a/sidebars.ts b/sidebars.ts index 28be80c..7ff5021 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -21,13 +21,13 @@ const sidebars: SidebarsConfig = { type: 'category', label: 'Communication Protocols', items: [ - 'providers/index', - 'providers/http', - 'providers/websocket', - 'providers/sse', - 'providers/cli', - 'providers/text', - 'providers/mcp', + 'protocols/index', + 'protocols/http', + 'protocols/websocket', + 'protocols/sse', + 'protocols/cli', + 'protocols/text', + 'protocols/mcp', ], }, 'implementation', From 68b77e85406454e7d1fa5b896ef608b9df15301c Mon Sep 17 00:00:00 2001 From: Luca Perrozzi Date: Fri, 5 Sep 2025 10:18:39 +0000 Subject: [PATCH 11/22] Phase 1: Make protocol documentation language-independent - Remove Python-specific examples from specification protocols - Replace with JSON/YAML configuration examples - Add references to language-specific documentation - Keep protocol concepts and architecture language-agnostic --- docs/protocols/http.md | 290 ++++++++++++-------------- docs/protocols/index.md | 63 +++--- docs/protocols/mcp.md | 446 +++++++++++++++++----------------------- 3 files changed, 350 insertions(+), 449 deletions(-) diff --git a/docs/protocols/http.md b/docs/protocols/http.md index 6f67741..74e4559 100644 --- a/docs/protocols/http.md +++ b/docs/protocols/http.md @@ -4,15 +4,9 @@ title: HTTP Protocol sidebar_position: 1 --- -# HTTP Protocol +# HTTP Protocol Plugin -The HTTP protocol plugin (`utcp-http`) enables UTCP to call REST APIs, webhooks, and any HTTP-based services. It's the most commonly used protocol and provides comprehensive support for modern web APIs. - -## Installation - -```bash -pip install utcp-http -``` +The HTTP protocol plugin enables UTCP to call tools via HTTP/HTTPS requests. This is the most commonly used protocol for REST APIs, webhooks, and web services. ## Call Template Structure @@ -20,51 +14,43 @@ pip install utcp-http { "call_template_type": "http", "url": "https://api.example.com/endpoint", - "http_method": "POST", + "http_method": "GET|POST|PUT|DELETE|PATCH", "headers": { "Content-Type": "application/json", - "Authorization": "Bearer ${API_TOKEN}" + "User-Agent": "UTCP-Client/1.0" + }, + "query_params": { + "param1": "${variable1}", + "param2": "static_value" }, "body": { - "query": "${query}", - "limit": 10 + "data": "${input_data}", + "timestamp": "${current_time}" }, + "timeout": 30, + "verify_ssl": true, "auth": { - "auth_type": "api_key", + "auth_type": "api_key|basic|oauth2", "api_key": "${API_KEY}", - "var_name": "X-API-Key", + "var_name": "Authorization", "location": "header" } } ``` -## Configuration Options - -### Required Fields - -| Field | Type | Description | -|-------|------|-------------| -| `call_template_type` | string | Must be `"http"` | -| `url` | string | The HTTP endpoint URL | -| `http_method` | string | HTTP method (GET, POST, PUT, DELETE, etc.) | - -### Optional Fields - -| Field | Type | Description | -|-------|------|-------------| -| `headers` | object | HTTP headers to include | -| `body` | object/string | Request body for POST/PUT requests | -| `query_params` | object | URL query parameters | -| `timeout` | number | Request timeout in seconds (default: 30) | -| `follow_redirects` | boolean | Whether to follow HTTP redirects (default: true) | -| `verify_ssl` | boolean | Whether to verify SSL certificates (default: true) | +## Supported HTTP Methods -## Authentication +| Method | Use Case | Body Support | +|--------|----------|--------------| +| `GET` | Retrieve data | No | +| `POST` | Create resources, submit data | Yes | +| `PUT` | Update/replace resources | Yes | +| `PATCH` | Partial updates | Yes | +| `DELETE` | Remove resources | Optional | -The HTTP protocol supports multiple authentication methods: +## Authentication Methods ### API Key Authentication - ```json { "auth": { @@ -76,13 +62,7 @@ The HTTP protocol supports multiple authentication methods: } ``` -**Locations:** -- `"header"`: Add as HTTP header -- `"query"`: Add as query parameter -- `"cookie"`: Add as cookie - ### Basic Authentication - ```json { "auth": { @@ -93,8 +73,7 @@ The HTTP protocol supports multiple authentication methods: } ``` -### OAuth2 Bearer Token - +### OAuth2 Authentication ```json { "auth": { @@ -102,157 +81,156 @@ The HTTP protocol supports multiple authentication methods: "client_id": "${CLIENT_ID}", "client_secret": "${CLIENT_SECRET}", "token_url": "https://auth.example.com/token", - "scope": "read:data" + "scope": "read write" } } ``` ## Variable Substitution -Use `${VARIABLE_NAME}` syntax to substitute values at runtime: +Variables in call templates are substituted with values from: +- Tool call arguments: `${argument_name}` +- Environment variables: `${ENV_VAR}` +- Configuration variables: `${config.variable}` +Example: ```json { "url": "https://api.example.com/users/${user_id}", "headers": { - "Authorization": "Bearer ${access_token}" + "Authorization": "Bearer ${ACCESS_TOKEN}" }, - "body": { - "name": "${user_name}", - "email": "${user_email}" + "query_params": { + "format": "${output_format}", + "limit": "${max_results}" } } ``` -Variables can come from: -- Tool arguments -- Environment variables -- Configuration files -- Runtime context - -## Examples +## OpenAPI Integration -### Simple GET Request +The HTTP protocol plugin can automatically generate UTCP manuals from OpenAPI/Swagger specifications: + +1. **Automatic Discovery**: Point to an OpenAPI spec URL +2. **Schema Conversion**: Converts OpenAPI paths to UTCP tools +3. **Authentication Mapping**: Maps OpenAPI security schemes to UTCP auth +4. **Parameter Mapping**: Converts OpenAPI parameters to UTCP inputs + +Example OpenAPI to UTCP conversion: +```yaml +# OpenAPI Specification +paths: + /users/{id}: + get: + parameters: + - name: id + in: path + required: true + schema: + type: string +``` +Becomes: ```json { - "name": "get_weather", - "description": "Get current weather for a location", - "inputs": { - "type": "object", - "properties": { - "location": {"type": "string"} - }, - "required": ["location"] - }, + "name": "get_user", "tool_call_template": { "call_template_type": "http", - "url": "https://api.weather.com/v1/current", - "http_method": "GET", - "query_params": { - "q": "${location}", - "appid": "${WEATHER_API_KEY}" - } - } -} -``` - -### POST with JSON Body - -```json -{ - "name": "create_user", - "description": "Create a new user account", + "url": "https://api.example.com/users/${id}", + "http_method": "GET" + }, "inputs": { "type": "object", "properties": { - "name": {"type": "string"}, - "email": {"type": "string"} - }, - "required": ["name", "email"] - }, - "tool_call_template": { - "call_template_type": "http", - "url": "https://api.example.com/users", - "http_method": "POST", - "headers": { - "Content-Type": "application/json", - "Authorization": "Bearer ${ACCESS_TOKEN}" + "id": {"type": "string"} }, - "body": { - "name": "${name}", - "email": "${email}", - "created_at": "{{now}}" - } + "required": ["id"] } } ``` -### File Upload +## Response Handling -```json -{ - "name": "upload_file", - "description": "Upload a file to the server", - "inputs": { - "type": "object", - "properties": { - "file_path": {"type": "string"}, - "description": {"type": "string"} - }, - "required": ["file_path"] - }, - "tool_call_template": { - "call_template_type": "http", - "url": "https://api.example.com/upload", - "http_method": "POST", - "headers": { - "Authorization": "Bearer ${ACCESS_TOKEN}" - }, - "body": { - "file": "@${file_path}", - "description": "${description}" - } - } -} -``` +HTTP responses are processed based on: +- **Status Codes**: 2xx considered success, others as errors +- **Content-Type**: JSON, XML, text, and binary content support +- **Headers**: Response headers available in tool output +- **Error Mapping**: HTTP errors mapped to UTCP exceptions + +## Configuration Options + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `timeout` | number | 30 | Request timeout in seconds | +| `verify_ssl` | boolean | true | Verify SSL certificates | +| `follow_redirects` | boolean | true | Follow HTTP redirects | +| `max_redirects` | number | 5 | Maximum redirect hops | +| `retry_count` | number | 0 | Number of retry attempts | +| `retry_delay` | number | 1 | Delay between retries (seconds) | + +## Security Considerations + +### SSL/TLS Verification +- Always verify SSL certificates in production +- Use `verify_ssl: false` only for testing/development +- Consider certificate pinning for high-security applications + +### Authentication Security +- Store credentials in environment variables, not in configuration files +- Use OAuth2 for user-facing applications +- Rotate API keys regularly +- Implement proper token refresh for OAuth2 + +### Input Validation +- Validate all input parameters before substitution +- Sanitize user inputs to prevent injection attacks +- Use allowlists for acceptable parameter values +- Implement rate limiting on the tool provider side + +### Network Security +- Use HTTPS for all production communications +- Implement proper firewall rules +- Consider using VPNs or private networks for sensitive tools +- Monitor and log all HTTP requests for security analysis ## Error Handling -The HTTP protocol handles various error conditions: +Common HTTP errors and their meanings: -| HTTP Status | Behavior | -|-------------|----------| -| 200-299 | Success - return response body | -| 400-499 | Client error - raise `HttpClientError` | -| 500-599 | Server error - raise `HttpServerError` | -| Timeout | Raise `HttpTimeoutError` | -| Connection | Raise `HttpConnectionError` | +| Status Code | Error Type | Description | +|-------------|------------|-------------| +| 400 | Bad Request | Invalid request parameters | +| 401 | Unauthorized | Authentication required or failed | +| 403 | Forbidden | Access denied | +| 404 | Not Found | Resource doesn't exist | +| 429 | Rate Limited | Too many requests | +| 500 | Server Error | Internal server error | +| 503 | Service Unavailable | Service temporarily unavailable | ## Best Practices -1. **Use HTTPS**: Always use secure connections for production APIs -2. **Set Timeouts**: Configure appropriate timeouts for your use case -3. **Handle Rate Limits**: Implement retry logic for rate-limited APIs -4. **Validate Inputs**: Use JSON Schema to validate tool inputs -5. **Secure Credentials**: Store API keys and tokens securely -6. **Monitor Usage**: Track API usage and performance metrics +### Performance +- Use connection pooling for multiple requests +- Implement appropriate timeouts +- Consider request/response compression +- Cache responses when appropriate -## OpenAPI Integration - -The HTTP protocol can automatically generate UTCP manuals from OpenAPI specifications: +### Reliability +- Implement retry logic with exponential backoff +- Handle network failures gracefully +- Use circuit breakers for unreliable services +- Monitor response times and error rates -```python -from utcp_http.openapi_converter import OpenApiConverter +### Maintainability +- Use descriptive tool names and descriptions +- Document all required parameters +- Provide usage examples +- Version your APIs and update call templates accordingly -converter = OpenApiConverter() -manual = await converter.convert_openapi_to_manual( - "https://api.example.com/openapi.json" -) -``` +## Language-Specific Implementation -## Related Protocols +For implementation details and examples in your programming language: -- [Server-Sent Events](./sse.md) - For streaming HTTP responses -- [Streamable HTTP](./streamable-http.md) - For chunked HTTP responses -- [WebSocket](./websocket.md) - For bidirectional real-time communication +- **Python**: [Python HTTP Protocol Documentation](https://github.com/universal-tool-calling-protocol/python-utcp/blob/main/docs/protocols/http.md) +- **TypeScript**: [TypeScript HTTP Protocol Documentation](https://github.com/universal-tool-calling-protocol/typescript-utcp/blob/main/docs/protocols/http.md) +- **Other languages**: Check respective repositories in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) diff --git a/docs/protocols/index.md b/docs/protocols/index.md index 01452a6..be58d3b 100644 --- a/docs/protocols/index.md +++ b/docs/protocols/index.md @@ -25,8 +25,7 @@ Each protocol plugin consists of: ### Call Templates Define how to structure calls for the specific protocol: -```python -# Example: HTTP Call Template +```json { "call_template_type": "http", "url": "https://api.example.com/endpoint", @@ -36,48 +35,34 @@ Define how to structure calls for the specific protocol: ``` ### Communication Protocols -Handle the actual communication logic: -```python -class HttpCommunicationProtocol: - async def call_tool(self, call_template, tool_args): - # Implementation for HTTP calls - pass -``` +Handle the actual communication logic for each protocol type. The implementation varies by programming language. ## Installing Protocol Plugins -```bash -# Core UTCP library -pip install utcp +Protocol plugins are available for different programming languages: -# Install specific protocol plugins -pip install utcp-http # HTTP, SSE protocols -pip install utcp-cli # CLI protocol -pip install utcp-websocket # WebSocket protocol -pip install utcp-text # Text file protocol -pip install utcp-mcp # MCP interoperability -``` +- **Python**: `pip install utcp-http utcp-cli utcp-websocket utcp-text utcp-mcp` +- **TypeScript**: `npm install @utcp/http @utcp/cli @utcp/websocket` +- **Other languages**: Check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) ## Creating Custom Protocol Plugins -You can extend UTCP with custom communication protocols: - -```python -from utcp.interfaces.communication_protocol import CommunicationProtocol -from utcp.data.call_template import CallTemplate +You can extend UTCP with custom communication protocols by implementing the protocol interface in your chosen language. Each implementation must: -class CustomCallTemplate(CallTemplate): - call_template_type: str = "custom" - custom_field: str +1. **Define Call Templates**: Specify the structure for protocol-specific calls +2. **Implement Communication Handler**: Handle the actual protocol communication +3. **Register the Protocol**: Make it available to the UTCP client -class CustomCommunicationProtocol(CommunicationProtocol): - async def call_tool(self, call_template: CustomCallTemplate, tool_args: dict): - # Your custom protocol implementation - pass - -# Register the protocol -from utcp.plugins.discovery import register_communication_protocol -register_communication_protocol(CustomCommunicationProtocol()) +Example call template structure: +```json +{ + "call_template_type": "custom", + "custom_field": "value", + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}" + } +} ``` ## Protocol Selection Guide @@ -91,4 +76,12 @@ Choose the right protocol plugin based on your needs: - **Text**: Reading configuration files or static content - **MCP**: Interoperability with Model Context Protocol tools +## Language-Specific Documentation + +For implementation details and examples in your programming language: + +- **Python**: [Python UTCP Documentation](https://github.com/universal-tool-calling-protocol/python-utcp/tree/main/docs) +- **TypeScript**: [TypeScript UTCP Documentation](https://github.com/universal-tool-calling-protocol/typescript-utcp/tree/main/docs) +- **Other languages**: Check respective repositories in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) + For detailed information about each protocol plugin, see the individual protocol documentation pages. diff --git a/docs/protocols/mcp.md b/docs/protocols/mcp.md index f7a1088..fc60697 100644 --- a/docs/protocols/mcp.md +++ b/docs/protocols/mcp.md @@ -1,18 +1,12 @@ --- id: mcp -title: Model Context Protocol (MCP) +title: MCP Protocol sidebar_position: 6 --- -# Model Context Protocol (MCP) +# MCP Protocol Plugin -The MCP protocol plugin (`utcp-mcp`) provides interoperability with the Model Context Protocol, allowing UTCP clients to call tools exposed through MCP servers. This enables gradual migration from MCP to UTCP or hybrid deployments. - -## Installation - -```bash -pip install utcp-mcp -``` +The MCP (Model Context Protocol) plugin provides interoperability between UTCP and existing MCP servers, enabling gradual migration from MCP to UTCP while maintaining compatibility with existing MCP tools. ## Call Template Structure @@ -21,316 +15,252 @@ pip install utcp-mcp "call_template_type": "mcp", "server_config": { "command": "node", - "args": ["/path/to/mcp-server.js"], + "args": ["mcp-server.js"], + "working_directory": "/app/mcp", "env": { - "API_KEY": "${API_KEY}" - } + "NODE_ENV": "production", + "LOG_LEVEL": "info" + }, + "timeout": 30 }, - "tool_name": "${tool_name}", - "connection_timeout": 30, - "call_timeout": 60 + "connection_timeout": 10, + "request_timeout": 30 } ``` -## Configuration Options +## Server Configuration -### Required Fields +### Command-based Servers +```json +{ + "server_config": { + "command": "python", + "args": ["-m", "mcp_server", "--config", "config.json"], + "working_directory": "/app", + "env": { + "PYTHONPATH": "/app/lib", + "API_KEY": "${MCP_API_KEY}" + } + } +} +``` -| Field | Type | Description | -|-------|------|-------------| -| `call_template_type` | string | Must be `"mcp"` | -| `server_config` | object | MCP server configuration | -| `tool_name` | string | Name of the MCP tool to call | +### HTTP-based Servers +```json +{ + "server_config": { + "transport": "http", + "url": "http://localhost:8080/mcp", + "headers": { + "Authorization": "Bearer ${MCP_TOKEN}" + } + } +} +``` -### Server Configuration +## Migration Strategy -| Field | Type | Description | -|-------|------|-------------| -| `command` | string | Command to start MCP server | -| `args` | array | Command arguments | -| `env` | object | Environment variables | -| `cwd` | string | Working directory | +The MCP protocol plugin enables a gradual migration path from MCP to native UTCP protocols: -### Optional Fields +### Phase 1: MCP Integration +- Use existing MCP servers through UTCP +- No changes to MCP server code required +- UTCP client can call MCP tools seamlessly -| Field | Type | Description | -|-------|------|-------------| -| `connection_timeout` | number | Server connection timeout (default: 30) | -| `call_timeout` | number | Tool call timeout (default: 60) | -| `server_name` | string | Friendly name for the server | -| `auto_restart` | boolean | Auto-restart server on failure (default: true) | +### Phase 2: Hybrid Approach +- Some tools use native UTCP protocols +- Legacy tools continue using MCP +- Gradual migration of high-value tools -## Examples +### Phase 3: Full Migration +- All tools use native UTCP protocols +- MCP servers deprecated +- Simplified architecture -### File System MCP Server +## Tool Discovery -```json -{ - "name": "read_file_mcp", - "description": "Read file content via MCP filesystem server", - "inputs": { - "type": "object", - "properties": { - "path": {"type": "string"} - }, - "required": ["path"] - }, - "tool_call_template": { - "call_template_type": "mcp", - "server_config": { - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-filesystem", "/allowed/path"], - "env": {} - }, - "tool_name": "read_file" - } -} -``` +MCP servers expose tools through the standard MCP protocol. The UTCP MCP plugin: -### Database MCP Server +1. **Connects** to the MCP server using stdio or HTTP transport +2. **Discovers** available tools via MCP's `tools/list` method +3. **Maps** MCP tool definitions to UTCP tool format +4. **Registers** tools in the UTCP client +## Request/Response Mapping + +### MCP to UTCP Tool Mapping ```json +// MCP Tool Definition { - "name": "query_database_mcp", - "description": "Query database via MCP server", - "inputs": { + "name": "read_file", + "description": "Read contents of a file", + "inputSchema": { "type": "object", "properties": { - "query": {"type": "string"}, - "params": {"type": "array"} - }, - "required": ["query"] - }, - "tool_call_template": { - "call_template_type": "mcp", - "server_config": { - "command": "python", - "args": ["-m", "mcp_server_sqlite", "--db-path", "/data/app.db"], - "env": { - "DATABASE_URL": "${DATABASE_URL}" - } + "path": {"type": "string"} }, - "tool_name": "execute_query", - "call_timeout": 120 + "required": ["path"] } } -``` - -### Custom MCP Server -```json +// UTCP Tool (after mapping) { - "name": "custom_mcp_tool", - "description": "Call custom MCP server tool", + "name": "read_file", + "description": "Read contents of a file", "inputs": { "type": "object", "properties": { - "input_data": {"type": "object"} + "path": {"type": "string"} }, - "required": ["input_data"] + "required": ["path"] }, "tool_call_template": { "call_template_type": "mcp", - "server_config": { - "command": "node", - "args": ["./custom-mcp-server.js"], - "cwd": "/app/servers", - "env": { - "NODE_ENV": "production", - "API_KEY": "${MCP_API_KEY}" - } - }, - "tool_name": "process_data", - "server_name": "custom_processor" + "server_config": {...} } } ``` -## Server Management - -### Automatic Server Lifecycle - -The MCP protocol plugin automatically manages server lifecycle: - -1. **Startup**: Launches MCP server when first tool is called -2. **Connection**: Establishes JSON-RPC connection -3. **Tool Discovery**: Retrieves available tools from server -4. **Call Routing**: Routes tool calls to appropriate server -5. **Shutdown**: Gracefully shuts down server when no longer needed - -### Server Pooling +### Request Flow +1. UTCP client receives tool call +2. MCP plugin formats request as MCP `tools/call` +3. Request sent to MCP server +4. MCP server processes and responds +5. Response mapped back to UTCP format -Multiple tools can share the same MCP server instance: +## Authentication and Security +### Server Authentication ```json { - "manual_version": "1.0.0", - "utcp_version": "1.0.1", - "tools": [ - { - "name": "list_files", - "tool_call_template": { - "call_template_type": "mcp", - "server_config": { - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-filesystem", "/data"] - }, - "tool_name": "list_directory" - } - }, - { - "name": "read_file", - "tool_call_template": { - "call_template_type": "mcp", - "server_config": { - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-filesystem", "/data"] - }, - "tool_name": "read_file" - } + "server_config": { + "command": "secure-mcp-server", + "env": { + "MCP_AUTH_TOKEN": "${MCP_SERVER_TOKEN}", + "MCP_CLIENT_ID": "${MCP_CLIENT_ID}" } - ] + } } ``` -## Error Handling - -| Error Type | Description | Handling | -|------------|-------------|----------| -| Server Start Failed | Cannot start MCP server | Raise `MCPServerStartError` | -| Connection Failed | Cannot connect to server | Raise `MCPConnectionError` | -| Tool Not Found | Tool doesn't exist on server | Raise `MCPToolNotFoundError` | -| Call Timeout | Tool call exceeded timeout | Raise `MCPTimeoutError` | -| Server Crashed | MCP server process died | Auto-restart if enabled | - -## Migration from MCP - -### Gradual Migration Strategy +### Transport Security +- **stdio**: Inherits process security model +- **HTTP**: Use HTTPS and proper authentication headers +- **WebSocket**: Use WSS and authentication tokens -1. **Wrap Existing MCP Servers**: Use UTCP-MCP plugin to call existing servers -2. **Identify High-Value Tools**: Prioritize frequently used tools for direct migration -3. **Migrate Tool by Tool**: Convert individual tools to native UTCP protocols -4. **Deprecate MCP Servers**: Remove MCP dependency once migration is complete - -### Migration Example - -**Before (Pure MCP):** -```javascript -// MCP Client -const client = new MCPClient(); -await client.connect("filesystem-server"); -const result = await client.callTool("read_file", {path: "/data/file.txt"}); -``` - -**During Migration (UTCP with MCP):** -```python -# UTCP Client with MCP plugin -client = await UtcpClient.create() -result = await client.call_tool("filesystem.read_file", { - "path": "/data/file.txt" -}) -``` +## Error Handling -**After Migration (Pure UTCP):** -```python -# UTCP Client with native protocol -client = await UtcpClient.create() -result = await client.call_tool("filesystem.read_file", { - "path": "/data/file.txt" -}) -``` +### Connection Errors +- Server startup failures +- Network connectivity issues +- Authentication failures +- Timeout errors -## Best Practices +### Protocol Errors +- Invalid MCP messages +- Unsupported MCP features +- Tool execution failures +- Resource access errors -1. **Server Reuse**: Share MCP servers across multiple tools when possible -2. **Timeout Configuration**: Set appropriate timeouts for server startup and calls -3. **Error Handling**: Implement retry logic for transient server failures -4. **Resource Management**: Monitor server resource usage and lifecycle -5. **Migration Planning**: Plan gradual migration to native UTCP protocols -6. **Testing**: Test MCP server compatibility thoroughly -7. **Documentation**: Document MCP server dependencies and requirements +### Error Mapping +MCP errors are mapped to UTCP exceptions: +- `InvalidRequest` → `ValidationError` +- `MethodNotFound` → `ToolNotFoundError` +- `InternalError` → `ToolCallError` ## Performance Considerations -### Overhead Comparison - -| Aspect | Native UTCP | UTCP-MCP | Pure MCP | -|--------|-------------|----------|----------| -| Latency | Low | Medium | Medium | -| Memory Usage | Low | Medium | High | -| Process Overhead | None | Medium | High | -| Network Hops | 1 | 2 | 2 | +### Connection Management +- Persistent connections for stdio transport +- Connection pooling for HTTP transport +- Automatic reconnection on failures +- Graceful shutdown handling -### Optimization Tips +### Request Optimization +- Batch multiple tool calls when possible +- Cache tool discovery results +- Implement request timeouts +- Monitor response times -1. **Server Pooling**: Reuse servers across multiple tools -2. **Connection Caching**: Cache server connections -3. **Batch Operations**: Group related tool calls when possible -4. **Resource Limits**: Set memory and CPU limits for MCP servers -5. **Health Monitoring**: Monitor server health and performance +## Limitations -## Compatibility +### MCP Feature Support +Not all MCP features are supported through UTCP: +- **Resources**: Not directly mapped to UTCP tools +- **Prompts**: Not supported in UTCP model +- **Sampling**: Not applicable to tool calling -### Supported MCP Versions +### Protocol Differences +- MCP's bidirectional communication vs UTCP's request/response +- MCP's resource model vs UTCP's tool-only model +- Different authentication mechanisms -- MCP 1.0.x: ✅ Full support -- MCP 0.x: ⚠️ Limited support +## Configuration Examples -### Known Limitations - -1. **Streaming**: MCP streaming not fully supported -2. **Resources**: MCP resources not mapped to UTCP -3. **Prompts**: MCP prompts not supported -4. **Sampling**: MCP sampling not supported - -## Common Use Cases - -- **Legacy Integration**: Calling existing MCP servers -- **Gradual Migration**: Transitioning from MCP to UTCP -- **Hybrid Deployments**: Using both MCP and UTCP tools -- **Third-party Tools**: Accessing MCP-only tools -- **Development**: Testing MCP compatibility - -## Troubleshooting - -### Server Won't Start - -```bash -# Check server command manually -npx -y @modelcontextprotocol/server-filesystem /path - -# Verify environment variables -echo $API_KEY - -# Check file permissions -ls -la /path/to/mcp-server.js -``` - -### Connection Issues - -```python -# Enable debug logging -import logging -logging.getLogger('utcp.mcp').setLevel(logging.DEBUG) - -# Test connection timeout +### Development Setup +```json { - "connection_timeout": 60, # Increase timeout - "auto_restart": true # Enable auto-restart + "manual_call_templates": [{ + "name": "dev_mcp", + "call_template_type": "mcp", + "server_config": { + "command": "node", + "args": ["dev-server.js"], + "env": {"NODE_ENV": "development"} + }, + "connection_timeout": 5, + "request_timeout": 10 + }] } ``` -### Tool Not Found - -```python -# List available tools from MCP server -server_tools = await mcp_client.list_tools() -print(f"Available tools: {[tool.name for tool in server_tools]}") +### Production Setup +```json +{ + "manual_call_templates": [{ + "name": "prod_mcp", + "call_template_type": "mcp", + "server_config": { + "transport": "http", + "url": "https://mcp.example.com/api", + "headers": { + "Authorization": "Bearer ${MCP_PROD_TOKEN}", + "X-Client-Version": "1.0.0" + } + }, + "connection_timeout": 30, + "request_timeout": 60 + }] +} ``` -## Related Documentation +## Best Practices -- [UTCP vs MCP Comparison](../utcp-vs-mcp.md) -- [Migration Guide](../migration/from-mcp.md) -- [HTTP Protocol](./http.md) - Alternative to MCP for REST APIs -- [CLI Protocol](./cli.md) - Alternative to MCP for command-line tools +### Migration Planning +1. **Inventory** existing MCP servers and tools +2. **Prioritize** tools for migration based on usage +3. **Test** MCP integration thoroughly +4. **Monitor** performance and reliability +5. **Migrate** incrementally to native UTCP protocols + +### Monitoring and Debugging +- Enable debug logging for MCP communication +- Monitor server health and response times +- Track tool usage patterns +- Log authentication failures +- Set up alerts for connection issues + +### Security +- Use secure transport methods (HTTPS, WSS) +- Implement proper authentication +- Validate all inputs and outputs +- Monitor for suspicious activity +- Keep MCP servers updated + +## Language-Specific Implementation + +For implementation details and examples in your programming language: + +- **Python**: [Python MCP Protocol Documentation](https://github.com/universal-tool-calling-protocol/python-utcp/blob/main/docs/protocols/mcp.md) +- **TypeScript**: [TypeScript MCP Protocol Documentation](https://github.com/universal-tool-calling-protocol/typescript-utcp/blob/main/docs/protocols/mcp.md) +- **Other languages**: Check respective repositories in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) From 53ddbdcc2c430d0a640516a79b8e34cd9c7662ed Mon Sep 17 00:00:00 2001 From: Luca Perrozzi Date: Fri, 5 Sep 2025 10:56:46 +0000 Subject: [PATCH 12/22] Continue Phase 1: Remove Python dependencies from core docs - Replace implementation.md with language-independent version - Remove Python examples from index.md quick start - Replace with JSON configuration and conceptual descriptions - Add references to language-specific documentation --- docs/for-tool-providers.md | 167 ++++----- docs/implementation.md | 710 ++++++++++++++----------------------- docs/index.md | 90 ++--- 3 files changed, 373 insertions(+), 594 deletions(-) diff --git a/docs/for-tool-providers.md b/docs/for-tool-providers.md index 0d43f1c..f975d0d 100644 --- a/docs/for-tool-providers.md +++ b/docs/for-tool-providers.md @@ -63,27 +63,25 @@ As a tool provider, you'll create a **UTCP Manual** - a standardized description ### 2. Expose via Discovery Endpoint -```python -from fastapi import FastAPI - -app = FastAPI() +Create an HTTP endpoint that returns your UTCP manual: -@app.get("/utcp") -def get_utcp_manual(): - return { - "manual_version": "1.0.0", - "utcp_version": "1.0.1", - "tools": [ - # ... your tools here - ] - } - -# Your existing API endpoints remain unchanged -@app.get("/users/{user_id}") -def get_user(user_id: str): - return {"id": user_id, "name": "John Doe", "email": "john@example.com"} +**Endpoint**: `GET /utcp` +**Response**: +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "tools": [ + // ... your tools here + ] +} ``` +Your existing API endpoints remain unchanged. For example: +- `GET /users/{user_id}` - Returns user data +- `POST /orders` - Creates new orders +- etc. + ## Manual Structure ### Required Fields @@ -467,44 +465,45 @@ app.listen(3000); ## OpenAPI Integration -Convert existing OpenAPI specifications to UTCP manuals: +If you already have an OpenAPI/Swagger specification, you can automatically convert it to a UTCP manual: -### Python with utcp-http +### Automatic Conversion -```python -from utcp_http.openapi_converter import OpenApiConverter +Many UTCP implementations provide OpenAPI converters that can: -# Convert OpenAPI spec to UTCP manual -converter = OpenApiConverter() -manual = await converter.convert_openapi_to_manual( - "https://api.example.com/openapi.json", - base_url="https://api.example.com" -) +1. **Parse OpenAPI specifications** from URLs or files +2. **Convert paths to UTCP tools** automatically +3. **Map authentication schemes** to UTCP auth types +4. **Generate proper input/output schemas** -# Serve the converted manual -@app.get("/utcp") -def get_utcp_manual(): - return manual.model_dump() -``` +### Conversion Configuration -### Customizing OpenAPI Conversion +You can customize the conversion process: -```python -# Convert with custom settings -manual = await converter.convert_openapi_to_manual( - "https://api.example.com/openapi.json", - base_url="https://api.example.com", - include_operations=["get", "post"], # Only include specific operations - exclude_paths=["/internal/*"], # Exclude internal paths - auth_template={ # Default auth for all tools - "auth_type": "api_key", - "api_key": "${API_KEY}", - "var_name": "X-API-Key", - "location": "header" - } -) +```json +{ + "source": "https://api.example.com/openapi.json", + "base_url": "https://api.example.com", + "include_operations": ["get", "post"], + "exclude_paths": ["/internal/*"], + "auth_template": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "X-API-Key", + "location": "header" + } +} ``` +### Manual Review + +After automatic conversion: +1. **Review generated tools** for accuracy +2. **Add missing descriptions** and examples +3. **Validate input/output schemas** +4. **Test with UTCP clients** +5. **Customize authentication** as needed + ## Best Practices ### Manual Design @@ -543,51 +542,33 @@ manual = await converter.convert_openapi_to_manual( ### Manual Validation -```python -from utcp.data.utcp_manual import UtcpManual -import json - -# Load and validate your manual -with open('manual.json', 'r') as f: - manual_data = json.load(f) - -try: - manual = UtcpManual(**manual_data) - print("Manual is valid!") -except Exception as e: - print(f"Manual validation failed: {e}") -``` +Validate your UTCP manual structure: + +1. **JSON Schema Validation**: Ensure your manual follows the UTCP schema +2. **Tool Definition Validation**: Check that all tools have required fields +3. **Call Template Validation**: Verify call templates are properly formatted +4. **Authentication Validation**: Test authentication configurations ### Integration Testing -```python -import pytest -from utcp.utcp_client import UtcpClient +Test your manual with UTCP clients: -@pytest.mark.asyncio -async def test_manual_integration(): - client = await UtcpClient.create(config={ - "manual_call_templates": [ - { - "name": "test_service", - "call_template_type": "http", - "url": "http://localhost:8000/utcp", - "http_method": "GET" - } - ] - }) - - # Test tool discovery - tools = await client.list_tools() - assert len(tools) > 0 - - # Test tool call - result = await client.call_tool( - "test_service.get_user", - tool_args={"user_id": "123"} - ) - assert "id" in result -``` +1. **Tool Discovery**: Verify clients can discover your tools +2. **Tool Execution**: Test actual tool calls with various inputs +3. **Error Handling**: Test error scenarios and responses +4. **Authentication**: Verify authentication works correctly +5. **Performance**: Test response times and reliability + +### Testing Checklist + +- [ ] Manual validates against UTCP schema +- [ ] All tools have unique names +- [ ] All required fields are present +- [ ] Call templates are correctly formatted +- [ ] Authentication works as expected +- [ ] Tools return expected outputs +- [ ] Error responses are properly formatted +- [ ] Performance meets requirements ## Migration Strategies @@ -687,3 +668,11 @@ For more information, see: - [Communication Protocol Plugins](./protocols/index.md) - [Implementation Guide](./implementation.md) - [Security Considerations](./security.md) + +## Language-Specific Implementation + +For detailed implementation examples and code samples in your programming language: + +- **Python**: [Python UTCP Documentation](https://github.com/universal-tool-calling-protocol/python-utcp/tree/main/docs) +- **TypeScript**: [TypeScript UTCP Documentation](https://github.com/universal-tool-calling-protocol/typescript-utcp/tree/main/docs) +- **Other languages**: Check respective repositories in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) diff --git a/docs/implementation.md b/docs/implementation.md index ff16d08..4e02fd5 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -4,575 +4,379 @@ title: Implementation Guide sidebar_position: 4 --- -# Implementation Guide +# UTCP Implementation Guide -:::info Language Note -This guide uses **Python** examples with the reference implementation. UTCP implementations are available in multiple languages - check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) for TypeScript, Go, and other language implementations. -::: +This guide covers the core concepts and patterns for implementing UTCP in any programming language, whether you're building tool providers or tool consumers. -This comprehensive guide walks you through implementing UTCP in your applications, whether you're creating a tool provider or building a client that consumes tools. +## Quick Start -## Quick Start (Python) +### 1. Install UTCP Library -### 1. Install UTCP +Choose the UTCP implementation for your programming language: -```bash -# Core UTCP library -pip install utcp - -# Protocol plugins (install as needed) -pip install utcp-http utcp-cli utcp-websocket utcp-text -``` +- **Python**: `pip install utcp utcp-http utcp-cli` +- **TypeScript**: `npm install @utcp/core @utcp/http @utcp/cli` +- **Other languages**: Check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) ### 2. Create Your First Tool Provider -```python -from fastapi import FastAPI - -app = FastAPI() - -@app.get("/utcp") -def get_utcp_manual(): - return { - "manual_version": "1.0.0", - "utcp_version": "1.0.1", - "tools": [ - { - "name": "get_weather", - "description": "Get current weather for a location", - "inputs": { - "type": "object", - "properties": { - "location": {"type": "string", "description": "City name"} - }, - "required": ["location"] - }, - "outputs": { - "type": "object", - "properties": { - "temperature": {"type": "number"}, - "conditions": {"type": "string"} - } - }, - "tool_call_template": { - "call_template_type": "http", - "url": "https://api.weather.com/v1/current", - "http_method": "GET", - "query_params": { - "q": "${location}", - "appid": "${WEATHER_API_KEY}" - } - } - } - ] +Create an HTTP endpoint that serves a UTCP manual: + +**Endpoint**: `GET /utcp` +**Response**: +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "info": { + "title": "Weather API", + "version": "1.0.0" + }, + "tools": [ + { + "name": "get_weather", + "description": "Get current weather for a location", + "inputs": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.weather.com/current", + "http_method": "GET", + "query_params": { + "q": "${location}", + "appid": "${WEATHER_API_KEY}" + } + } } + ] +} ``` ### 3. Create Your First Client -```python -import asyncio -from utcp.utcp_client import UtcpClient - -async def main(): - # Create client with manual discovery - client = await UtcpClient.create(config={ - "manual_call_templates": [ - { - "name": "weather_service", - "call_template_type": "http", - "url": "http://localhost:8000/utcp", - "http_method": "GET" - } - ] - }) - - # Call the tool - result = await client.call_tool( - "weather_service.get_weather", - tool_args={"location": "San Francisco"} - ) - print(f"Weather: {result}") - -if __name__ == "__main__": - asyncio.run(main()) -``` - -## Other Language Implementations - -UTCP is available in multiple programming languages: +Configure a UTCP client to discover and call tools: -| Language | Repository | Status | -|----------|------------|--------| -| **Python** | [python-utcp](https://github.com/universal-tool-calling-protocol/python-utcp) | ✅ Reference Implementation | -| **TypeScript** | [typescript-utcp](https://github.com/universal-tool-calling-protocol/typescript-utcp) | ✅ Stable | -| **Go** | [go-utcp](https://github.com/universal-tool-calling-protocol/go-utcp) | 🚧 In Development | -| **Rust** | [rust-utcp](https://github.com/universal-tool-calling-protocol/rust-utcp) | 🚧 In Development | +**Configuration**: +```json +{ + "manual_call_templates": [ + { + "name": "weather_service", + "call_template_type": "http", + "url": "https://api.weather.com/utcp", + "http_method": "GET" + } + ], + "variables": { + "WEATHER_API_KEY": "your-api-key" + } +} +``` -Visit the respective repositories for language-specific documentation and examples. +**Usage**: +1. Initialize UTCP client with configuration +2. Discover tools from the weather service +3. Call the `get_weather` tool with location parameter -## Tool Provider Implementation (Python) +## Core Concepts -### Manual Structure +### UTCP Manual -A UTCP Manual defines your tools and how to call them: +A UTCP manual is a JSON document that describes available tools and how to call them: ```json { "manual_version": "1.0.0", "utcp_version": "1.0.1", "info": { - "title": "Weather API", + "title": "API Name", "version": "1.0.0", - "description": "Weather data and forecasting tools" + "description": "API description" }, "tools": [ { - "name": "get_weather", - "description": "Get current weather conditions", + "name": "tool_name", + "description": "Tool description", "inputs": { "type": "object", "properties": { - "location": {"type": "string"}, - "units": {"type": "string", "enum": ["metric", "imperial"]} - }, - "required": ["location"] + "param": {"type": "string"} + } }, "outputs": { "type": "object", "properties": { - "temperature": {"type": "number"}, - "conditions": {"type": "string"}, - "humidity": {"type": "number"} + "result": {"type": "string"} } }, "tool_call_template": { "call_template_type": "http", - "url": "https://api.weather.com/v1/current", - "http_method": "GET", - "query_params": { - "q": "${location}", - "units": "${units}", - "appid": "${WEATHER_API_KEY}" - }, - "auth": { - "auth_type": "api_key", - "api_key": "${WEATHER_API_KEY}", - "var_name": "appid", - "location": "query" - } + "url": "https://api.example.com/endpoint", + "http_method": "POST" } } ] } ``` -### Discovery Endpoint (Python Examples) - -Expose your manual via a discovery endpoint: - -#### FastAPI Example - -```python -from fastapi import FastAPI -from utcp.data.utcp_manual import UtcpManual -from utcp.data.tool import Tool, JsonSchema -from utcp.data.call_template import CallTemplate - -app = FastAPI() - -# Define your manual -manual = UtcpManual( - manual_version="1.0.0", - utcp_version="1.0.1", - tools=[ - Tool( - name="get_weather", - description="Get current weather", - inputs=JsonSchema( - type="object", - properties={ - "location": JsonSchema(type="string") - }, - required=["location"] - ), - tool_call_template=CallTemplate( - call_template_type="http", - url="https://api.weather.com/v1/current", - http_method="GET", - query_params={ - "q": "${location}", - "appid": "${WEATHER_API_KEY}" - } - ) - ) - ] -) - -@app.get("/utcp") -def get_manual(): - return manual.model_dump() +### Call Templates + +Call templates define how to invoke tools using specific protocols: + +#### HTTP Call Template +```json +{ + "call_template_type": "http", + "url": "https://api.example.com/endpoint", + "http_method": "POST", + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer ${API_TOKEN}" + }, + "body": { + "data": "${input_data}" + } +} +``` + +#### CLI Call Template +```json +{ + "call_template_type": "cli", + "command": "python", + "args": ["script.py", "${input}"], + "working_directory": "/app", + "env": { + "PYTHONPATH": "/app/lib" + } +} ``` -#### Flask Example +### Variable Substitution -```python -from flask import Flask, jsonify +Variables in call templates are replaced with actual values: -app = Flask(__name__) +- **Tool arguments**: `${argument_name}` +- **Environment variables**: `${ENV_VAR}` +- **Configuration variables**: `${config.variable}` -@app.route('/utcp') -def get_manual(): - return jsonify({ - "manual_version": "1.0.0", - "utcp_version": "1.0.1", - "tools": [ - # ... tool definitions - ] - }) -``` +## Tool Provider Implementation + +### Manual Structure -### Authentication Patterns (Python) +Create a well-structured UTCP manual: -#### API Key in Header +1. **Info Section**: Describe your API +2. **Tools Array**: Define each available tool +3. **Input Schemas**: Specify required parameters +4. **Output Schemas**: Document return values +5. **Call Templates**: Define how to invoke each tool + +### Discovery Endpoint + +Expose your manual via HTTP: + +``` +GET /utcp +Content-Type: application/json -```json { - "auth": { - "auth_type": "api_key", - "api_key": "${API_KEY}", - "var_name": "X-API-Key", - "location": "header" - } + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "tools": [...] } ``` -#### Bearer Token +### Authentication + +Support various authentication methods: +#### API Key Authentication ```json { "auth": { "auth_type": "api_key", - "api_key": "${ACCESS_TOKEN}", - "var_name": "Authorization", + "api_key": "${API_KEY}", + "var_name": "X-API-Key", "location": "header" } } ``` -#### OAuth2 - +#### OAuth2 Authentication ```json { "auth": { "auth_type": "oauth2", "client_id": "${CLIENT_ID}", "client_secret": "${CLIENT_SECRET}", - "token_url": "https://auth.example.com/token", - "scope": "read:data" + "token_url": "https://auth.example.com/token" } } ``` -## Client Implementation (Python) +## Tool Consumer Implementation -### Basic Client Setup +### Client Configuration -```python -from utcp.utcp_client import UtcpClient - -# Create client with configuration -client = await UtcpClient.create(config={ - "manual_call_templates": [ - { - "name": "my_service", - "call_template_type": "http", - "url": "https://api.example.com/utcp", - "http_method": "GET" - } - ], - "variable_loaders": [ - { - "loader_type": "env", - "prefix": "UTCP_" - } - ] -}) -``` - -### Configuration Options (Python) +Configure your UTCP client: #### File-based Configuration - ```yaml # utcp-config.yaml manual_call_templates: - name: weather_service call_template_type: http - url: https://weather.example.com/utcp - http_method: GET - - name: database_service - call_template_type: http - url: https://db.example.com/utcp + url: https://api.weather.com/utcp http_method: GET +variables: + WEATHER_API_KEY: your-api-key + variable_loaders: - - loader_type: env - prefix: UTCP_ - - loader_type: dotenv + - type: dot_env file_path: .env ``` -```python -client = await UtcpClient.create(config="utcp-config.yaml") -``` - #### Programmatic Configuration - -```python -from utcp.data.utcp_client_config import UtcpClientConfig -from utcp.data.call_template import CallTemplate - -config = UtcpClientConfig( - manual_call_templates=[ - CallTemplate( - name="weather_service", - call_template_type="http", - url="https://weather.example.com/utcp", - http_method="GET" - ) - ] -) - -client = await UtcpClient.create(config=config) -``` - -### Tool Discovery and Search (Python) - -#### List Available Tools - -```python -# Get all tools -tools = await client.list_tools() -for tool in tools: - print(f"{tool.name}: {tool.description}") - -# Search tools by tag -weather_tools = await client.search_tools(tags=["weather"]) - -# Search tools by description -data_tools = await client.search_tools(description="data processing") -``` - -#### Tool Information - -```python -# Get detailed tool information -tool_info = await client.get_tool("weather_service.get_weather") -print(f"Inputs: {tool_info.inputs}") -print(f"Outputs: {tool_info.outputs}") +```json +{ + "manual_call_templates": [ + { + "name": "service_name", + "call_template_type": "http", + "url": "https://api.example.com/utcp" + } + ], + "variables": { + "API_KEY": "your-key" + } +} ``` -### Calling Tools (Python) +### Tool Discovery -#### Basic Tool Call +Discover available tools: -```python -result = await client.call_tool( - "weather_service.get_weather", - tool_args={"location": "New York"} -) -``` +1. **List Tools**: Get all available tools +2. **Tool Information**: Get detailed tool metadata +3. **Filter Tools**: Find tools by name, tags, or description -#### Tool Call with Context +### Tool Execution -```python -result = await client.call_tool( - "weather_service.get_weather", - tool_args={"location": "New York"}, - context={"user_id": "123", "session_id": "abc"} -) -``` +Execute tools with proper error handling: -#### Batch Tool Calls +1. **Basic Calls**: Simple tool invocation +2. **Batch Calls**: Execute multiple tools +3. **Context Passing**: Pass context between calls +4. **Error Handling**: Handle failures gracefully -```python -results = await client.call_tools([ - ("weather_service.get_weather", {"location": "New York"}), - ("weather_service.get_weather", {"location": "London"}), - ("weather_service.get_weather", {"location": "Tokyo"}) -]) -``` +## Advanced Implementation Patterns -### Error Handling (Python) +### Custom Protocol Plugins -```python -from utcp.exceptions import UtcpError, ToolNotFoundError, ToolCallError +Extend UTCP with custom communication protocols: -try: - result = await client.call_tool("nonexistent.tool", {}) -except ToolNotFoundError as e: - print(f"Tool not found: {e}") -except ToolCallError as e: - print(f"Tool call failed: {e}") -except UtcpError as e: - print(f"UTCP error: {e}") -``` +1. **Define Call Template**: Structure for your protocol +2. **Implement Communication Handler**: Protocol-specific logic +3. **Register Protocol**: Make it available to clients -## Advanced Implementation Patterns (Python) - -### Custom Communication Protocol Plugins - -```python -from utcp.interfaces.communication_protocol import CommunicationProtocol -from utcp.data.call_template import CallTemplate -from typing import Dict, Any, List - -class CustomProtocol(CommunicationProtocol): - def get_supported_call_template_types(self) -> List[str]: - return ["custom"] - - async def call_tool( - self, - call_template: CallTemplate, - tool_args: Dict[str, Any] - ) -> Any: - # Implement your custom protocol logic - return {"result": "custom protocol response"} - -# Register the protocol -from utcp.plugins.discovery import register_communication_protocol -register_communication_protocol(CustomProtocol()) +Example custom protocol structure: +```json +{ + "call_template_type": "custom", + "custom_field": "value", + "timeout": 30 +} ``` -### Custom Tool Repository - -```python -from utcp.interfaces.concurrent_tool_repository import ConcurrentToolRepository -from utcp.data.tool import Tool -from typing import List, Optional - -class DatabaseToolRepository(ConcurrentToolRepository): - async def add_tool(self, tool: Tool, manual_name: str) -> None: - # Store tool in database - pass - - async def get_tool(self, namespaced_name: str) -> Optional[Tool]: - # Retrieve tool from database - pass - - async def list_tools(self) -> List[Tool]: - # List all tools from database - pass - - # ... implement other methods - -# Use custom repository -client = await UtcpClient.create( - config=config, - tool_repository=DatabaseToolRepository() -) -``` +### Custom Tool Repositories -## Testing (Python) - -### Unit Testing Tool Providers - -```python -import pytest -from fastapi.testclient import TestClient -from your_app import app - -client = TestClient(app) - -def test_utcp_manual(): - response = client.get("/utcp") - assert response.status_code == 200 - - manual = response.json() - assert manual["manual_version"] == "1.0.0" - assert len(manual["tools"]) > 0 - - tool = manual["tools"][0] - assert "name" in tool - assert "description" in tool - assert "tool_call_template" in tool -``` +Implement custom tool storage: -### Integration Testing - -```python -import pytest -from utcp.utcp_client import UtcpClient - -@pytest.mark.asyncio -async def test_tool_call(): - client = await UtcpClient.create(config={ - "manual_call_templates": [ - { - "name": "test_service", - "call_template_type": "http", - "url": "http://localhost:8000/utcp", - "http_method": "GET" - } - ] - }) - - result = await client.call_tool( - "test_service.get_weather", - tool_args={"location": "Test City"} - ) - - assert "temperature" in result - assert "conditions" in result -``` +1. **Tool Storage**: How tools are stored and retrieved +2. **Search Functionality**: Tool discovery and filtering +3. **Caching**: Performance optimization +4. **Synchronization**: Multi-client coordination -## Language-Specific Resources +### Testing Strategies -### Python -- **Repository**: [python-utcp](https://github.com/universal-tool-calling-protocol/python-utcp) -- **Documentation**: [Python UTCP Docs](https://python-utcp.readthedocs.io/) -- **Examples**: [Python Examples](https://github.com/universal-tool-calling-protocol/python-utcp/tree/main/examples) +#### Unit Testing Tool Providers +- Test manual generation +- Validate tool definitions +- Test authentication +- Mock external dependencies -### TypeScript -- **Repository**: [typescript-utcp](https://github.com/universal-tool-calling-protocol/typescript-utcp) -- **Documentation**: [TypeScript UTCP Docs](https://typescript-utcp.readthedocs.io/) -- **Examples**: [TypeScript Examples](https://github.com/universal-tool-calling-protocol/typescript-utcp/tree/main/examples) +#### Integration Testing +- Test tool discovery +- Test tool execution +- Test error scenarios +- Test performance -### Other Languages -Check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) for the latest language implementations and their respective documentation. +#### End-to-End Testing +- Test complete workflows +- Test multiple protocols +- Test real-world scenarios +- Test scalability ## Best Practices -### Tool Provider Best Practices - -1. **Versioning**: Use semantic versioning for your manual -2. **Documentation**: Provide clear descriptions for all tools -3. **Validation**: Validate inputs using JSON Schema -4. **Error Handling**: Return meaningful error messages -5. **Rate Limiting**: Implement appropriate rate limits -6. **Monitoring**: Monitor tool usage and performance -7. **Security**: Implement proper authentication and authorization - -### Client Best Practices - -1. **Configuration Management**: Use external configuration files -2. **Error Handling**: Implement comprehensive error handling -3. **Retry Logic**: Implement retry logic for transient failures -4. **Caching**: Cache tool definitions and results when appropriate -5. **Monitoring**: Monitor client performance and errors -6. **Testing**: Write comprehensive tests for tool integrations -7. **Security**: Store credentials securely - -## Next Steps - -1. **Choose Your Language**: Select from available UTCP implementations -2. **Choose Your Protocols**: Select the communication protocols you need -3. **Design Your Tools**: Plan your tool structure and interfaces -4. **Implement Gradually**: Start with simple tools and expand -5. **Test Thoroughly**: Write comprehensive tests -6. **Monitor and Optimize**: Monitor performance and optimize as needed -7. **Scale**: Plan for scaling as your tool ecosystem grows +### Performance +- Use connection pooling +- Implement caching +- Optimize tool discovery +- Monitor response times + +### Security +- Validate all inputs +- Use secure authentication +- Implement rate limiting +- Log security events + +### Reliability +- Implement retry logic +- Handle network failures +- Use circuit breakers +- Monitor tool health + +### Maintainability +- Version your manuals +- Document all tools +- Use consistent naming +- Provide examples + +## Deployment Considerations + +### Scaling +- Load balance tool providers +- Cache tool discoveries +- Use async processing +- Monitor resource usage + +### Monitoring +- Track tool usage +- Monitor error rates +- Log performance metrics +- Set up alerts + +### Security +- Use HTTPS everywhere +- Implement proper authentication +- Validate all inputs +- Monitor for abuse + +## Language-Specific Implementation + +For detailed implementation examples and code samples in your programming language: + +- **Python**: [Python UTCP Documentation](https://github.com/universal-tool-calling-protocol/python-utcp/tree/main/docs) +- **TypeScript**: [TypeScript UTCP Documentation](https://github.com/universal-tool-calling-protocol/typescript-utcp/tree/main/docs) +- **Other languages**: Check respective repositories in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) For more detailed information, see: - [Communication Protocol Plugins](./protocols/index.md) diff --git a/docs/index.md b/docs/index.md index b46cf7b..a8dbbbe 100644 --- a/docs/index.md +++ b/docs/index.md @@ -41,63 +41,49 @@ pip install utcp utcp-http Add a discovery endpoint to your existing API: -```python -from fastapi import FastAPI - -app = FastAPI() - -# Your existing API endpoint (unchanged) -@app.get("/weather") -def get_weather(location: str): - return {"temperature": 22, "conditions": "Sunny"} - -# Add UTCP discovery endpoint -@app.get("/utcp") -def utcp_manual(): - return { - "manual_version": "1.0.0", - "utcp_version": "1.0.1", - "tools": [{ - "name": "get_weather", - "description": "Get current weather for a location", - "inputs": { - "type": "object", - "properties": {"location": {"type": "string"}}, - "required": ["location"] - }, - "tool_call_template": { - "call_template_type": "http", - "url": "http://localhost:8000/weather", - "http_method": "GET", - "query_params": {"location": "${location}"} - } - }] +**Add endpoint**: `GET /utcp` +**Return your UTCP manual**: +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "tools": [{ + "name": "get_weather", + "description": "Get current weather for a location", + "inputs": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"] + }, + "tool_call_template": { + "call_template_type": "http", + "url": "http://localhost:8000/weather", + "http_method": "GET", + "query_params": {"location": "${location}"} } + }] +} ``` ### 3. Call Your Tool -```python -import asyncio -from utcp.utcp_client import UtcpClient - -async def main(): - client = await UtcpClient.create(config={ - "manual_call_templates": [{ - "name": "weather_api", - "call_template_type": "http", - "url": "http://localhost:8000/utcp", - "http_method": "GET" - }] - }) - - result = await client.call_tool( - "weather_api.get_weather", - tool_args={"location": "San Francisco"} - ) - print(f"Weather: {result}") - -asyncio.run(main()) +**Configure UTCP client**: +```json +{ + "manual_call_templates": [{ + "name": "weather_api", + "call_template_type": "http", + "url": "http://localhost:8000/utcp", + "http_method": "GET" + }] +} +``` + +**Call the tool**: +1. Initialize UTCP client with configuration +2. Discover tools from weather API +3. Call `get_weather` tool with location parameter +4. Receive weather data response ``` **That's it!** Your tool is now discoverable and callable by any UTCP client. From 961da8a14790a28030f83659ca92b1ad0722c9e1 Mon Sep 17 00:00:00 2001 From: Luca Perrozzi Date: Fri, 5 Sep 2025 10:57:40 +0000 Subject: [PATCH 13/22] Complete Phase 1: Remove final Python dependencies - Replace remaining Python example in for-tool-providers.md - All core documentation now language-independent - Python-specific examples moved to python-utcp repository - JSON/YAML configuration examples throughout --- docs/for-tool-providers.md | 168 +++++++++++++++++-------------------- 1 file changed, 79 insertions(+), 89 deletions(-) diff --git a/docs/for-tool-providers.md b/docs/for-tool-providers.md index f975d0d..60b4eea 100644 --- a/docs/for-tool-providers.md +++ b/docs/for-tool-providers.md @@ -317,97 +317,87 @@ Use `${VARIABLE_NAME}` syntax for dynamic values: ## Implementation Examples -### FastAPI Implementation (Python) - -```python -from fastapi import FastAPI, HTTPException -from pydantic import BaseModel -from typing import List, Optional - -app = FastAPI() - -class User(BaseModel): - id: str - name: str - email: str - -# Your existing API endpoints -@app.get("/users/{user_id}") -def get_user(user_id: str) -> User: - # Your existing logic - return User(id=user_id, name="John Doe", email="john@example.com") - -@app.post("/users") -def create_user(user: User) -> User: - # Your existing logic - return user - -# UTCP Discovery endpoint -@app.get("/utcp") -def get_utcp_manual(): - return { - "manual_version": "1.0.0", - "utcp_version": "1.0.1", - "info": { - "title": "User Management API", - "version": "1.0.0", - "description": "API for managing user accounts" +### REST API Implementation + +For a typical REST API, you'll need to: + +1. **Keep your existing endpoints unchanged** +2. **Add a UTCP discovery endpoint** at `/utcp` +3. **Map your API operations to UTCP tools** + +Example API structure: +``` +GET /users/{user_id} # Your existing endpoint +POST /users # Your existing endpoint +GET /utcp # New UTCP discovery endpoint +``` + +The UTCP manual describes how to call your existing endpoints: + +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "info": { + "title": "User Management API", + "version": "1.0.0", + "description": "API for managing user accounts" + }, + "tools": [ + { + "name": "get_user", + "description": "Retrieve user information by ID", + "inputs": { + "type": "object", + "properties": { + "user_id": {"type": "string"} }, - "tools": [ - { - "name": "get_user", - "description": "Retrieve user information by ID", - "inputs": { - "type": "object", - "properties": { - "user_id": {"type": "string"} - }, - "required": ["user_id"] - }, - "outputs": { - "type": "object", - "properties": { - "id": {"type": "string"}, - "name": {"type": "string"}, - "email": {"type": "string"} - } - }, - "tool_call_template": { - "call_template_type": "http", - "url": f"{BASE_URL}/users/{{user_id}}", - "http_method": "GET", - "headers": { - "Authorization": "Bearer ${API_TOKEN}" - } - } - }, - { - "name": "create_user", - "description": "Create a new user account", - "inputs": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "email": {"type": "string", "format": "email"} - }, - "required": ["name", "email"] - }, - "tool_call_template": { - "call_template_type": "http", - "url": f"{BASE_URL}/users", - "http_method": "POST", - "headers": { - "Content-Type": "application/json", - "Authorization": "Bearer ${API_TOKEN}" - }, - "body": { - "name": "${name}", - "email": "${email}" - } - } - } - ] + "required": ["user_id"] + }, + "outputs": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + "email": {"type": "string"} + } + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/users/${user_id}", + "http_method": "GET", + "headers": { + "Authorization": "Bearer ${API_TOKEN}" + } + } + }, + { + "name": "create_user", + "description": "Create a new user account", + "inputs": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string"} + }, + "required": ["name", "email"] + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/users", + "http_method": "POST", + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer ${API_TOKEN}" + }, + "body": { + "name": "${name}", + "email": "${email}" + } + } } + ] +} ``` ### Express.js Implementation From 943e94b5200597f4b635a8bf09ac2240c7e631c8 Mon Sep 17 00:00:00 2001 From: Luca Perrozzi Date: Fri, 5 Sep 2025 11:02:10 +0000 Subject: [PATCH 14/22] Remove Python language bias from documentation - Remove Python-specific language from intro sections - Make language references neutral in index.md and for-tool-providers.md - Keep Python examples in CLI/MCP protocols as they're just examples - Update links to be language-neutral --- docs/for-tool-providers.md | 2 +- docs/index.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/for-tool-providers.md b/docs/for-tool-providers.md index 60b4eea..230614d 100644 --- a/docs/for-tool-providers.md +++ b/docs/for-tool-providers.md @@ -7,7 +7,7 @@ sidebar_position: 2 # For Tool Providers :::info Language Note -This guide uses **Python** examples with the reference implementation. UTCP implementations are available in multiple languages - check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) for TypeScript, Go, and other language implementations. +This guide covers how to expose your existing APIs and services as UTCP tools. UTCP implementations are available in multiple languages - check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) for Python, TypeScript, Go, and other language implementations. ::: This guide helps you expose your tools through UTCP so they can be discovered and used by AI agents and other applications. diff --git a/docs/index.md b/docs/index.md index a8dbbbe..349f66a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,7 +7,7 @@ sidebar_position: 1 # Universal Tool Calling Protocol (UTCP) :::info Language Examples -This documentation uses **Python** examples. UTCP is available in multiple languages - see [TypeScript](https://github.com/universal-tool-calling-protocol/typescript-utcp), [Go](https://github.com/universal-tool-calling-protocol/go-utcp), and other implementations in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol). +UTCP is available in multiple languages - see [Python](https://github.com/universal-tool-calling-protocol/python-utcp), [TypeScript](https://github.com/universal-tool-calling-protocol/typescript-utcp), [Go](https://github.com/universal-tool-calling-protocol/go-utcp), and other implementations in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol). ::: UTCP is a lightweight, secure, and scalable standard that enables AI agents and applications to discover and call tools directly using their native protocols - **no wrapper servers required**. @@ -187,7 +187,7 @@ You're building AI agents or applications that need to call external tools: ### For Tool Consumers 1. **[Read the implementation guide](./implementation.md)** - Learn how to build UTCP clients 2. **[Explore protocols](./protocols/index.md)** - Understand available communication options -3. **[Check examples](https://github.com/universal-tool-calling-protocol/python-utcp/tree/main/examples)** - See real-world implementations +3. **[Check examples](https://github.com/universal-tool-calling-protocol)** - See real-world implementations 4. **[Join the community](https://discord.gg/ZpMbQ8jRbD)** - Get help and share experiences ### Migration from Other Systems From 04d4d9373b21afd59c24dd91d33020b7c7c37d99 Mon Sep 17 00:00:00 2001 From: Luca Perrozzi Date: Fri, 5 Sep 2025 11:38:28 +0000 Subject: [PATCH 15/22] Start removing Python code from security.md while preserving JSON - Remove Python language bias from intro - Replace Python implementation examples with language-independent guidance - Keep all JSON configuration examples - Preserve security concepts and best practices --- docs/security.md | 70 +++++++++++------------------------------------- 1 file changed, 16 insertions(+), 54 deletions(-) diff --git a/docs/security.md b/docs/security.md index b0172d9..68dd1f3 100644 --- a/docs/security.md +++ b/docs/security.md @@ -6,10 +6,6 @@ sidebar_position: 6 # Security Considerations -:::info Language Examples -This guide uses **Python** examples for implementation code. UTCP security principles apply to all language implementations - check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) for language-specific security examples. -::: - Security is critical when enabling direct tool access through UTCP. This guide covers security considerations specific to UTCP's "manual" approach and provides practical guidance for secure implementations. ## UTCP Security Model @@ -33,30 +29,12 @@ UTCP's direct communication model has unique security characteristics: Secure your UTCP manual endpoints: -```python -from fastapi import FastAPI, HTTPException, Depends -from fastapi.security import HTTPBearer - -app = FastAPI() -security = HTTPBearer() - -@app.get("/utcp") -def get_manual(token: str = Depends(security)): - # Validate discovery access - if not validate_discovery_token(token.credentials): - raise HTTPException(401, "Invalid discovery token") - - return { - "manual_version": "1.0.0", - "utcp_version": "1.0.1", - "tools": get_authorized_tools(token.credentials) - } - -def get_authorized_tools(token: str): - # Return only tools the client is authorized to see - user_permissions = get_user_permissions(token) - return filter_tools_by_permissions(all_tools, user_permissions) -``` +**Best Practices:** +- Implement authentication for manual discovery endpoints +- Validate discovery access tokens +- Return only tools the client is authorized to see +- Filter tools based on user permissions +- Log all manual discovery attempts ## Authentication & Authorization @@ -76,21 +54,10 @@ def get_authorized_tools(token: str): ``` **Secure implementation:** -```python -import os -from datetime import datetime, timedelta - -class RotatingAPIKey: - def __init__(self): - self.current_key = os.getenv("API_KEY_CURRENT") - self.next_key = os.getenv("API_KEY_NEXT") - self.rotation_time = datetime.fromisoformat(os.getenv("KEY_ROTATION_TIME")) - - def get_valid_key(self): - if datetime.now() > self.rotation_time: - return self.next_key - return self.current_key -``` +- Use rotating API keys with current and next key support +- Implement automatic key rotation based on time +- Store keys securely in environment variables +- Validate keys before processing requests #### OAuth2 with Scope Validation @@ -205,17 +172,12 @@ CLI execution poses significant security risks. Use with extreme caution. - ✅ Validate exit codes - ✅ Use minimal user permissions -**Input sanitization example:** -```python -import re -import shlex - -def sanitize_cli_input(input_value: str) -> str: - # Remove dangerous characters - sanitized = re.sub(r'[;&|`$(){}[\]<>]', '', input_value) - # Escape for shell safety - return shlex.quote(sanitized) -``` +**Input sanitization requirements:** +- Remove dangerous shell metacharacters: `;`, `&`, `|`, `` ` ``, `$`, `()`, `{}`, `[]`, `<>` +- Escape inputs appropriately for shell execution +- Use parameterized command execution when possible +- Validate inputs against expected patterns +- Implement length limits for all inputs ### Server-Sent Events (SSE) Security From ac47d5830d6d23e5a1699f9cd43f89e13d48100d Mon Sep 17 00:00:00 2001 From: Luca Perrozzi Date: Fri, 5 Sep 2025 11:39:38 +0000 Subject: [PATCH 16/22] Continue removing Python code from security.md - Replace path validation Python example with requirements - Preserve JSON configuration examples - Keep security concepts language-independent --- docs/security.md | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/docs/security.md b/docs/security.md index 68dd1f3..f3d8f46 100644 --- a/docs/security.md +++ b/docs/security.md @@ -223,24 +223,13 @@ CLI execution poses significant security risks. Use with extreme caution. - ✅ Prevent directory traversal attacks - ✅ Use safe encoding handling -**Path validation example:** -```python -import os -from pathlib import Path - -def validate_file_path(file_path: str, allowed_dirs: list) -> bool: - try: - # Resolve to absolute path - abs_path = Path(file_path).resolve() - - # Check if path is within allowed directories - for allowed_dir in allowed_dirs: - if abs_path.is_relative_to(Path(allowed_dir).resolve()): - return True - return False - except (OSError, ValueError): - return False -``` +**Path validation requirements:** +- Resolve all paths to absolute paths +- Check if paths are within allowed directories +- Handle symbolic links by resolving them first +- Validate against directory traversal attacks (`../`) +- Return false for any path resolution errors +- Use allowlists of permitted directories ### MCP Security From 3a6d5dcba574535836444bf5290d17760c4e0ef7 Mon Sep 17 00:00:00 2001 From: Luca Perrozzi Date: Fri, 5 Sep 2025 12:57:34 +0000 Subject: [PATCH 17/22] Remove Python language bias from specification - Convert Python code examples to language-agnostic descriptions - Replace Python-specific documentation links with multi-language references - Maintain technical accuracy while ensuring language neutrality - Major files updated: utcp-vs-mcp.md, migration-v0.1-to-v1.0.md - Updated documentation links across all protocol files --- docs/for-tool-providers.md | 2 +- docs/implementation.md | 4 +- docs/index.md | 7 +- docs/migration-v0.1-to-v1.0.md | 289 ++++++++------------------------ docs/protocols/cli.md | 4 + docs/protocols/http.md | 2 +- docs/protocols/index.md | 14 +- docs/protocols/mcp.md | 2 +- docs/protocols/sse.md | 4 + docs/protocols/text.md | 4 + docs/protocols/websocket.md | 4 + docs/security.md | 298 ++++++++------------------------- docs/utcp-vs-mcp.md | 290 +++++++++++--------------------- 13 files changed, 278 insertions(+), 646 deletions(-) diff --git a/docs/for-tool-providers.md b/docs/for-tool-providers.md index 230614d..6e4a0aa 100644 --- a/docs/for-tool-providers.md +++ b/docs/for-tool-providers.md @@ -663,6 +663,6 @@ For more information, see: For detailed implementation examples and code samples in your programming language: -- **Python**: [Python UTCP Documentation](https://github.com/universal-tool-calling-protocol/python-utcp/tree/main/docs) +- **Multi-language**: [UTCP Implementation Examples](https://github.com/universal-tool-calling-protocol) - Examples across Python, TypeScript, Go, and other languages - **TypeScript**: [TypeScript UTCP Documentation](https://github.com/universal-tool-calling-protocol/typescript-utcp/tree/main/docs) - **Other languages**: Check respective repositories in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) diff --git a/docs/implementation.md b/docs/implementation.md index 4e02fd5..175a7ea 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -15,7 +15,7 @@ This guide covers the core concepts and patterns for implementing UTCP in any pr Choose the UTCP implementation for your programming language: - **Python**: `pip install utcp utcp-http utcp-cli` -- **TypeScript**: `npm install @utcp/core @utcp/http @utcp/cli` +- **Node.js**: `npm install @utcp/core @utcp/http @utcp/cli` - **Other languages**: Check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) ### 2. Create Your First Tool Provider @@ -374,7 +374,7 @@ Implement custom tool storage: For detailed implementation examples and code samples in your programming language: -- **Python**: [Python UTCP Documentation](https://github.com/universal-tool-calling-protocol/python-utcp/tree/main/docs) +- **Multi-language**: [UTCP Implementation Examples](https://github.com/universal-tool-calling-protocol) - Examples across Python, TypeScript, Go, and other languages - **TypeScript**: [TypeScript UTCP Documentation](https://github.com/universal-tool-calling-protocol/typescript-utcp/tree/main/docs) - **Other languages**: Check respective repositories in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) diff --git a/docs/index.md b/docs/index.md index 349f66a..c3860a1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -33,8 +33,13 @@ UTCP acts as a **"manual"** that tells agents how to call your tools directly: ### 1. Install UTCP ```bash -# Core library + HTTP support +# Example installation (Python) pip install utcp utcp-http + +# Example installation (Node.js) +npm install @utcp/core @utcp/http + +# See language-specific documentation for other implementations ``` ### 2. Expose Your First Tool diff --git a/docs/migration-v0.1-to-v1.0.md b/docs/migration-v0.1-to-v1.0.md index b220557..1306f9f 100644 --- a/docs/migration-v0.1-to-v1.0.md +++ b/docs/migration-v0.1-to-v1.0.md @@ -51,45 +51,20 @@ pip install utcp-http utcp-cli utcp-websocket utcp-text utcp-mcp ### v0.1 Client Code -```python -from utcp import UtcpClient - -# Old way -client = UtcpClient(config={ - "providers": [ - { - "name": "weather_service", - "provider_type": "http", - "url": "https://weather.example.com/utcp", - "http_method": "GET" - } - ] -}) - -# Call tool -result = client.call_tool("weather_service.get_weather", {"location": "NYC"}) -``` +**Legacy v0.1 client configuration:** +- Import UTCP client library +- Configure providers with provider-specific settings (name, type, URL, HTTP method) +- Call tools using provider.tool_name format +- Synchronous tool calling interface ### v1.0 Client Code -```python -from utcp.utcp_client import UtcpClient - -# New way - async factory method -client = await UtcpClient.create(config={ - "manual_call_templates": [ - { - "name": "weather_service", - "call_template_type": "http", - "url": "https://weather.example.com/utcp", - "http_method": "GET" - } - ] -}) - -# Call tool - now async -result = await client.call_tool("weather_service.get_weather", {"location": "NYC"}) -``` +**New v1.0 client configuration:** +- Import updated UTCP client library from new module path +- Use async factory method for client creation +- Configure manual call templates instead of providers +- Use async/await pattern for tool calling +- Enhanced error handling and response processing ## Configuration Migration @@ -302,34 +277,20 @@ variable_loaders: ### v0.1 Error Handling -```python -try: - result = client.call_tool("service.tool", args) -except Exception as e: - print(f"Error: {e}") -``` +**Basic error handling in v0.1:** +- Simple try/catch block with generic Exception handling +- Limited error information and context +- Basic error message printing ### v1.0 Error Handling -```python -from utcp.exceptions import ( - UtcpError, - ToolNotFoundError, - ToolCallError, - AuthenticationError -) - -try: - result = await client.call_tool("service.tool", args) -except ToolNotFoundError as e: - print(f"Tool not found: {e}") -except AuthenticationError as e: - print(f"Authentication failed: {e}") -except ToolCallError as e: - print(f"Tool call failed: {e}") -except UtcpError as e: - print(f"UTCP error: {e}") -``` +**Enhanced error handling in v1.0:** +- Import specific exception types from utcp.exceptions module +- Handle ToolNotFoundError for missing tools +- Handle AuthenticationError for auth failures +- Handle ToolCallError for tool execution failures +- Handle base UtcpError for general UTCP errors +- Use try/catch blocks with specific exception handling ## Step-by-Step Migration @@ -345,132 +306,46 @@ pip install utcp utcp-http utcp-cli utcp-websocket utcp-text ### Step 2: Update Client Code -```python -# Before -from utcp import UtcpClient - -def main(): - client = UtcpClient(config=config) - result = client.call_tool("service.tool", args) - return result - -# After -import asyncio -from utcp.utcp_client import UtcpClient - -async def main(): - client = await UtcpClient.create(config=config) - result = await client.call_tool("service.tool", args) - return result - -if __name__ == "__main__": - asyncio.run(main()) -``` +**Migration to async pattern:** +- **Before**: Synchronous client creation and tool calls +- **After**: Async client factory method and async tool calls +- Import asyncio module for async execution +- Use async/await keywords for client creation and tool calls +- Run async main function with asyncio.run() ### Step 3: Update Configuration -```python -# Migration helper function -def migrate_config_v0_to_v1(old_config): - new_config = { - "manual_call_templates": [], - "variable_loaders": [ - {"loader_type": "env", "prefix": "UTCP_"} - ] - } - - for provider in old_config.get("providers", []): - call_template = { - "name": provider["name"], - "call_template_type": provider["provider_type"], - } - - # Migrate HTTP providers - if provider["provider_type"] == "http": - call_template.update({ - "url": provider["url"], - "http_method": provider.get("method", "GET"), - "headers": provider.get("headers", {}), - "body": provider.get("body") - }) - - # Migrate CLI providers - elif provider["provider_type"] == "cli": - call_template.update({ - "command": provider["command"], - "args": provider.get("args", []), - "working_directory": provider.get("cwd") - }) - - new_config["manual_call_templates"].append(call_template) - - return new_config - -# Use migration helper -old_config = load_old_config() -new_config = migrate_config_v0_to_v1(old_config) -client = await UtcpClient.create(config=new_config) -``` +**Configuration migration helper:** +- Create migration function to convert v0.1 config to v1.0 format +- Transform providers array to manual_call_templates array +- Add variable_loaders configuration for environment variables +- Map provider_type to call_template_type +- Migrate HTTP provider settings (URL, method, headers, body) +- Migrate CLI provider settings (command, args, working_directory) +- Load old configuration and apply migration helper +- Create new client with migrated configuration ### Step 4: Update Manual Format -```python -# Migration helper for manuals -def migrate_manual_v0_to_v1(old_manual): - new_manual = { - "manual_version": "1.0.0", - "utcp_version": "1.0.1", - "info": { - "title": old_manual.get("provider_info", {}).get("name", "API"), - "version": old_manual.get("provider_info", {}).get("version", "1.0.0"), - "description": old_manual.get("provider_info", {}).get("description", "") - }, - "tools": [] - } - - for tool in old_manual.get("tools", []): - new_tool = { - "name": tool["name"], - "description": tool["description"], - "inputs": tool.get("parameters", {}), - "tool_call_template": {} - } - - # Migrate provider to call_template - provider = tool.get("provider", {}) - if provider.get("provider_type") == "http": - new_tool["tool_call_template"] = { - "call_template_type": "http", - "url": provider["url"], - "http_method": provider.get("method", "GET"), - "headers": provider.get("headers", {}), - "body": provider.get("body") - } - - new_manual["tools"].append(new_tool) - - return new_manual -``` +**Manual migration helper:** +- Create migration function to convert v0.1 manual to v1.0 format +- Update manual_version and utcp_version fields +- Transform provider_info to info structure (title, version, description) +- Migrate tools array with updated structure +- Convert tool parameters to inputs field +- Transform provider configuration to tool_call_template +- Map provider_type to call_template_type +- Migrate HTTP provider settings (URL, method, headers, body) ### Step 5: Test Migration -```python -import pytest -from utcp.utcp_client import UtcpClient - -@pytest.mark.asyncio -async def test_migrated_client(): - # Test with migrated configuration - client = await UtcpClient.create(config=migrated_config) - - # Test tool discovery - tools = await client.list_tools() - assert len(tools) > 0 - - # Test tool calls - result = await client.call_tool("service.tool", {"param": "value"}) - assert result is not None -``` +**Testing migrated client:** +- Import pytest and UtcpClient for async testing +- Create test function with pytest.mark.asyncio decorator +- Test client creation with migrated configuration +- Test tool discovery functionality (list_tools) +- Test tool calling with sample parameters +- Assert expected results and functionality ## Common Migration Issues @@ -479,26 +354,18 @@ async def test_migrated_client(): **Problem**: v1.0 client methods are async **Solution**: Add `async`/`await` keywords -```python -# Before -result = client.call_tool("tool", args) - -# After -result = await client.call_tool("tool", args) -``` +**Code changes:** +- **Before**: Synchronous tool calling (result = client.call_tool("tool", args)) +- **After**: Async tool calling (result = await client.call_tool("tool", args)) ### Issue 2: Configuration Format **Problem**: Configuration structure changed **Solution**: Use migration helper or update manually -```python -# Before -config = {"providers": [...]} - -# After -config = {"manual_call_templates": [...]} -``` +**Configuration changes:** +- **Before**: Use "providers" array in configuration +- **After**: Use "manual_call_templates" array in configuration ### Issue 3: Plugin Dependencies @@ -526,33 +393,21 @@ pip install utcp-http utcp-cli utcp-websocket ### Configuration Validator -```python -from utcp.data.utcp_client_config import UtcpClientConfig - -def validate_config(config_dict): - try: - config = UtcpClientConfig(**config_dict) - print("Configuration is valid!") - return config - except Exception as e: - print(f"Configuration validation failed: {e}") - return None -``` +**Configuration validation helper:** +- Import UtcpClientConfig from utcp.data.utcp_client_config +- Create validation function that accepts configuration dictionary +- Use UtcpClientConfig constructor to validate configuration structure +- Handle validation exceptions and provide error messages +- Return validated config object or None on failure ### Manual Validator -```python -from utcp.data.utcp_manual import UtcpManual - -def validate_manual(manual_dict): - try: - manual = UtcpManual(**manual_dict) - print("Manual is valid!") - return manual - except Exception as e: - print(f"Manual validation failed: {e}") - return None -``` +**Manual validation helper:** +- Import UtcpManual from utcp.data.utcp_manual +- Create validation function that accepts manual dictionary +- Use UtcpManual constructor to validate manual structure +- Handle validation exceptions and provide error messages +- Return validated manual object or None on failure ## Best Practices for Migration @@ -583,7 +438,7 @@ If you encounter issues during migration: 1. **Check Documentation**: Review the [Implementation Guide](./implementation.md) 2. **GitHub Issues**: Search existing issues or create new ones 3. **Discord Community**: Join the [UTCP Discord](https://discord.gg/ZpMbQ8jRbD) -4. **Examples**: Check the [examples repository](https://github.com/universal-tool-calling-protocol/python-utcp/tree/main/examples) +4. **Examples**: Check the [examples repository](https://github.com/universal-tool-calling-protocol) for implementations across multiple languages ## Rollback Plan diff --git a/docs/protocols/cli.md b/docs/protocols/cli.md index 05caf92..f6cc2f0 100644 --- a/docs/protocols/cli.md +++ b/docs/protocols/cli.md @@ -11,7 +11,11 @@ The CLI protocol plugin (`utcp-cli`) enables UTCP to execute command-line tools ## Installation ```bash +# Example installation (Python) pip install utcp-cli + +# Example installation (Node.js) +npm install @utcp/cli ``` ## Call Template Structure diff --git a/docs/protocols/http.md b/docs/protocols/http.md index 74e4559..6340aeb 100644 --- a/docs/protocols/http.md +++ b/docs/protocols/http.md @@ -231,6 +231,6 @@ Common HTTP errors and their meanings: For implementation details and examples in your programming language: -- **Python**: [Python HTTP Protocol Documentation](https://github.com/universal-tool-calling-protocol/python-utcp/blob/main/docs/protocols/http.md) +- **Multi-language**: [UTCP HTTP Protocol Examples](https://github.com/universal-tool-calling-protocol) - HTTP protocol examples across multiple languages - **TypeScript**: [TypeScript HTTP Protocol Documentation](https://github.com/universal-tool-calling-protocol/typescript-utcp/blob/main/docs/protocols/http.md) - **Other languages**: Check respective repositories in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) diff --git a/docs/protocols/index.md b/docs/protocols/index.md index be58d3b..ae85510 100644 --- a/docs/protocols/index.md +++ b/docs/protocols/index.md @@ -41,9 +41,15 @@ Handle the actual communication logic for each protocol type. The implementation Protocol plugins are available for different programming languages: -- **Python**: `pip install utcp-http utcp-cli utcp-websocket utcp-text utcp-mcp` -- **TypeScript**: `npm install @utcp/http @utcp/cli @utcp/websocket` -- **Other languages**: Check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) +```bash +# Example installation (Python) +pip install utcp-http utcp-cli utcp-websocket utcp-text utcp-mcp + +# Example installation (Node.js) +npm install @utcp/http @utcp/cli @utcp/websocket @utcp/text @utcp/mcp +``` + +For other languages, check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) ## Creating Custom Protocol Plugins @@ -80,7 +86,7 @@ Choose the right protocol plugin based on your needs: For implementation details and examples in your programming language: -- **Python**: [Python UTCP Documentation](https://github.com/universal-tool-calling-protocol/python-utcp/tree/main/docs) +- **Multi-language**: [UTCP Implementation Examples](https://github.com/universal-tool-calling-protocol) - Examples across Python, TypeScript, Go, and other languages - **TypeScript**: [TypeScript UTCP Documentation](https://github.com/universal-tool-calling-protocol/typescript-utcp/tree/main/docs) - **Other languages**: Check respective repositories in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) diff --git a/docs/protocols/mcp.md b/docs/protocols/mcp.md index fc60697..13da0f6 100644 --- a/docs/protocols/mcp.md +++ b/docs/protocols/mcp.md @@ -261,6 +261,6 @@ Not all MCP features are supported through UTCP: For implementation details and examples in your programming language: -- **Python**: [Python MCP Protocol Documentation](https://github.com/universal-tool-calling-protocol/python-utcp/blob/main/docs/protocols/mcp.md) +- **Multi-language**: [UTCP MCP Protocol Examples](https://github.com/universal-tool-calling-protocol) - MCP protocol examples across multiple languages - **TypeScript**: [TypeScript MCP Protocol Documentation](https://github.com/universal-tool-calling-protocol/typescript-utcp/blob/main/docs/protocols/mcp.md) - **Other languages**: Check respective repositories in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) diff --git a/docs/protocols/sse.md b/docs/protocols/sse.md index f39d964..74301e5 100644 --- a/docs/protocols/sse.md +++ b/docs/protocols/sse.md @@ -13,7 +13,11 @@ The Server-Sent Events protocol plugin (`utcp-http`) enables UTCP to receive rea SSE support is included with the HTTP plugin: ```bash +# Example installation (Python) pip install utcp-http + +# Example installation (Node.js) +npm install @utcp/http ``` ## Call Template Structure diff --git a/docs/protocols/text.md b/docs/protocols/text.md index e869124..a1a17e2 100644 --- a/docs/protocols/text.md +++ b/docs/protocols/text.md @@ -11,7 +11,11 @@ The Text protocol plugin (`utcp-text`) enables UTCP to read and process text fil ## Installation ```bash +# Example installation (Python) pip install utcp-text + +# Example installation (Node.js) +npm install @utcp/text ``` ## Call Template Structure diff --git a/docs/protocols/websocket.md b/docs/protocols/websocket.md index f71e3ad..8bd7da0 100644 --- a/docs/protocols/websocket.md +++ b/docs/protocols/websocket.md @@ -11,7 +11,11 @@ The WebSocket protocol plugin (`utcp-websocket`) enables UTCP to communicate wit ## Installation ```bash +# Example installation (Python) pip install utcp-websocket + +# Example installation (Node.js) +npm install @utcp/websocket ``` ## Call Template Structure diff --git a/docs/security.md b/docs/security.md index f3d8f46..c23f5c0 100644 --- a/docs/security.md +++ b/docs/security.md @@ -289,274 +289,112 @@ CLI execution poses significant security risks. Use with extreme caution. ### Parameter Sanitization -**Server-side validation:** -```python -from pydantic import BaseModel, validator, Field -import re - -class ToolInput(BaseModel): - user_id: str = Field(..., regex=r'^[a-zA-Z0-9_-]+$', max_length=50) - query: str = Field(..., max_length=1000) - - @validator('query') - def sanitize_query(cls, v): - # Remove potentially dangerous characters - return re.sub(r'[<>"\']', '', v).strip() -``` +**Server-side validation requirements:** +- Use regex patterns to validate input formats (e.g., `^[a-zA-Z0-9_-]+$` for user IDs) +- Implement maximum length limits for all string inputs +- Remove dangerous characters like `<>`, `"`, `'` from user inputs +- Validate all inputs against expected schemas +- Sanitize inputs before processing ## Secure Variable Handling ### Environment Variable Security -**Secure variable loading:** -```python -import os -from typing import Dict, Optional - -class SecureVariableLoader: - def __init__(self, allowed_prefixes: list = None): - self.allowed_prefixes = allowed_prefixes or ['UTCP_', 'API_'] - - def load_variable(self, var_name: str) -> Optional[str]: - # Only load variables with allowed prefixes - if not any(var_name.startswith(prefix) for prefix in self.allowed_prefixes): - raise ValueError(f"Variable {var_name} not allowed") - - return os.getenv(var_name) - - def substitute_variables(self, template: str, context: Dict[str, str]) -> str: - # Safely substitute variables - for var_name, value in context.items(): - if self.is_safe_variable(var_name, value): - template = template.replace(f"${{{var_name}}}", value) - return template - - def is_safe_variable(self, name: str, value: str) -> bool: - # Validate variable safety - if len(value) > 10000: # Prevent extremely long values - return False - if any(char in value for char in ['<', '>', '"', "'"]): # Basic XSS prevention - return False - return True -``` +**Secure variable loading requirements:** +- Only allow variables with approved prefixes (e.g., `UTCP_`, `API_`) +- Validate variable names against allowlists +- Implement length limits for variable values (e.g., max 10,000 characters) +- Check for dangerous characters in values (`<`, `>`, `"`, `'`) +- Use secure variable substitution methods +- Log all variable access attempts ### Runtime Variable Substitution -**Secure substitution:** -```python -import re -from typing import Dict - -def secure_substitute(template: str, variables: Dict[str, str]) -> str: - def replace_var(match): - var_name = match.group(1) - if var_name in variables: - value = variables[var_name] - # Validate and sanitize the value - if is_safe_value(value): - return value - else: - raise ValueError(f"Unsafe variable value for {var_name}") - else: - raise ValueError(f"Variable {var_name} not found") - - # Only replace variables with the expected pattern - return re.sub(r'\$\{([a-zA-Z_][a-zA-Z0-9_]*)\}', replace_var, template) - -def is_safe_value(value: str) -> bool: - # Implement your safety checks - return len(value) < 1000 and not any(c in value for c in ['<', '>', '"', "'", ';', '&']) -``` +**Secure substitution requirements:** + +Implement variable substitution with these security measures: +- Only substitute variables matching the pattern `${variable_name}` +- Validate variable names contain only alphanumeric characters and underscores +- Check that all variables exist before substitution +- Sanitize variable values to prevent injection attacks +- Reject values containing dangerous characters like `<`, `>`, `"`, `'`, `;`, `&` +- Limit variable value length to prevent buffer overflow attacks ## Network & Transport Security ### TLS Configuration **Minimum TLS requirements:** -```python -import ssl -import httpx - -# Configure secure HTTP client -client = httpx.AsyncClient( - verify=True, # Verify SSL certificates - timeout=30.0, - limits=httpx.Limits(max_connections=100, max_keepalive_connections=20) -) - -# For custom SSL context -ssl_context = ssl.create_default_context() -ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2 -ssl_context.check_hostname = True -ssl_context.verify_mode = ssl.CERT_REQUIRED -``` +**TLS configuration requirements:** + +Configure secure HTTP clients with these settings: +- Enable SSL certificate verification +- Set reasonable connection timeouts (e.g., 30 seconds) +- Limit maximum connections to prevent resource exhaustion +- Use TLS 1.2 or higher as minimum version +- Enable hostname verification +- Require certificate verification (CERT_REQUIRED mode) ### Certificate Validation **Enhanced certificate validation:** -```python -import ssl -import certifi - -def create_secure_context(): - context = ssl.create_default_context(cafile=certifi.where()) - context.check_hostname = True - context.verify_mode = ssl.CERT_REQUIRED - context.minimum_version = ssl.TLSVersion.TLSv1_2 - - # Disable weak ciphers - context.set_ciphers('ECDHE+AESGCM:ECDHE+CHACHA20:DHE+AESGCM:DHE+CHACHA20:!aNULL:!MD5:!DSS') - - return context -``` + +Implement robust certificate validation: +- Use trusted certificate authority bundles +- Enable hostname verification against certificate +- Require valid certificate chains +- Set minimum TLS version to 1.2 +- Configure strong cipher suites (ECDHE+AESGCM, CHACHA20, DHE+AESGCM) +- Reject weak algorithms (aNULL, MD5, DSS) ## Monitoring & Incident Response ### Security Logging **Comprehensive security logging:** -```python -import logging -import json -from datetime import datetime - -class SecurityLogger: - def __init__(self): - self.logger = logging.getLogger('utcp.security') - - def log_tool_call(self, tool_name: str, user_id: str, success: bool, **kwargs): - log_entry = { - 'timestamp': datetime.utcnow().isoformat(), - 'event_type': 'tool_call', - 'tool_name': tool_name, - 'user_id': user_id, - 'success': success, - 'metadata': kwargs - } - self.logger.info(json.dumps(log_entry)) - - def log_auth_failure(self, tool_name: str, reason: str, **kwargs): - log_entry = { - 'timestamp': datetime.utcnow().isoformat(), - 'event_type': 'auth_failure', - 'tool_name': tool_name, - 'reason': reason, - 'metadata': kwargs - } - self.logger.warning(json.dumps(log_entry)) -``` + +Implement security logging with these components: +- **Tool Call Logging**: Record all tool invocations with user ID, tool name, timestamp, success status, and parameters +- **Authentication Logging**: Log authentication attempts, failures, and reasons +- **Structured Format**: Use JSON format for easy parsing and analysis +- **Sensitive Data Protection**: Avoid logging sensitive information like passwords or tokens +- **Audit Trail**: Maintain immutable logs for compliance and forensic analysis ### Anomaly Detection **Basic anomaly detection:** -```python -from collections import defaultdict -from datetime import datetime, timedelta - -class AnomalyDetector: - def __init__(self): - self.call_counts = defaultdict(list) - self.rate_limits = { - 'calls_per_minute': 60, - 'calls_per_hour': 1000 - } - - def check_rate_limit(self, user_id: str) -> bool: - now = datetime.utcnow() - user_calls = self.call_counts[user_id] - - # Clean old entries - user_calls[:] = [call_time for call_time in user_calls - if now - call_time < timedelta(hours=1)] - - # Check limits - recent_calls = [call_time for call_time in user_calls - if now - call_time < timedelta(minutes=1)] - - if len(recent_calls) >= self.rate_limits['calls_per_minute']: - return False - if len(user_calls) >= self.rate_limits['calls_per_hour']: - return False - - # Record this call - user_calls.append(now) - return True -``` + +Implement anomaly detection with these features: +- **Rate Limiting**: Track requests per user with configurable limits (e.g., 60 calls/minute, 1000 calls/hour) +- **Time Window Management**: Clean old entries and maintain sliding time windows +- **Multi-tier Limits**: Enforce both short-term (per minute) and long-term (per hour) rate limits +- **Automatic Blocking**: Reject requests that exceed configured thresholds +- **Call Tracking**: Record timestamps of all user requests for analysis ## Security Testing & Validation ### Testing Methodologies **Security test examples:** -```python -import pytest -from utcp.utcp_client import UtcpClient - -@pytest.mark.asyncio -async def test_injection_prevention(): - client = await UtcpClient.create(config=test_config) - - # Test SQL injection attempt - malicious_input = "'; DROP TABLE users; --" - - with pytest.raises(ValueError, match="Invalid input"): - await client.call_tool("db.query", {"query": malicious_input}) - -@pytest.mark.asyncio -async def test_path_traversal_prevention(): - client = await UtcpClient.create(config=test_config) - - # Test directory traversal attempt - malicious_path = "../../../etc/passwd" - - with pytest.raises(ValueError, match="Path not allowed"): - await client.call_tool("file.read", {"path": malicious_path}) - -@pytest.mark.asyncio -async def test_rate_limiting(): - client = await UtcpClient.create(config=test_config) - - # Test rate limiting - for i in range(100): # Exceed rate limit - try: - await client.call_tool("api.test", {}) - except Exception as e: - assert "rate limit" in str(e).lower() - break - else: - pytest.fail("Rate limiting not enforced") -``` + +Implement comprehensive security testing: +- **Injection Prevention Tests**: Test for SQL injection, command injection, and other malicious inputs +- **Path Traversal Tests**: Verify protection against directory traversal attacks (../../../etc/passwd) +- **Rate Limiting Tests**: Confirm rate limiting enforcement under high load +- **Authentication Tests**: Validate proper authentication and authorization +- **Input Validation Tests**: Test boundary conditions and malformed inputs ### Security Automation **Automated security checks:** -```python -def validate_manual_security(manual: dict) -> list: - issues = [] - - for tool in manual.get('tools', []): - call_template = tool.get('tool_call_template', {}) - - # Check for HTTP over HTTPS - if call_template.get('call_template_type') == 'http': - url = call_template.get('url', '') - if url.startswith('http://'): - issues.append(f"Tool {tool['name']}: Uses insecure HTTP") - - # Check for hardcoded credentials - template_str = str(call_template) - if any(keyword in template_str.lower() for keyword in ['password', 'secret', 'key']): - if not any(var in template_str for var in ['${', '${']): - issues.append(f"Tool {tool['name']}: May contain hardcoded credentials") - - # Check for CLI security - if call_template.get('call_template_type') == 'cli': - command = call_template.get('command', '') - if not command.startswith('/'): - issues.append(f"Tool {tool['name']}: CLI command should use absolute path") - - return issues -``` + +Implement automated security validation: +- **Protocol Security**: Verify HTTPS usage instead of HTTP for web requests +- **Credential Detection**: Check for hardcoded passwords, secrets, or API keys +- **Variable Validation**: Ensure proper variable substitution patterns (${variable}) +- **CLI Security**: Validate command-line tools use absolute paths and safe commands +- **URL Validation**: Check for suspicious or malformed URLs +- **Configuration Review**: Automated scanning of UTCP manuals for security issues ## Security Checklist diff --git a/docs/utcp-vs-mcp.md b/docs/utcp-vs-mcp.md index 72927ea..275ab7d 100644 --- a/docs/utcp-vs-mcp.md +++ b/docs/utcp-vs-mcp.md @@ -56,23 +56,12 @@ UTCP provides a standardized way to describe how to call existing tools directly MCP requires building servers that wrap your tools: -```python -# MCP Server (required infrastructure) -from mcp.server import Server -from mcp.types import Tool - -server = Server("weather-server") - -@server.list_tools() -async def list_tools(): - return [Tool(name="get_weather", description="Get weather")] - -@server.call_tool() -async def call_tool(name: str, arguments: dict): - if name == "get_weather": - # Call actual weather API - return await weather_api.get(arguments["location"]) -``` +**MCP Server Implementation Requirements:** +- Create a dedicated server process for each tool provider +- Implement tool listing functionality to expose available tools +- Implement tool calling handlers that proxy requests to actual APIs +- Maintain server infrastructure and handle client connections +- Route all tool calls through the MCP server layer **Flow:** Agent → MCP Server → Tool → MCP Server → Agent @@ -81,55 +70,38 @@ async def call_tool(name: str, arguments: dict): ### Performance Impact #### UTCP Performance -```python -# Direct API call - no overhead -import httpx - -async def call_weather_tool(): - async with httpx.AsyncClient() as client: - response = await client.get( - "https://api.weather.com/current", - params={"location": "San Francisco"} - ) - return response.json() - -# Latency: ~100ms (API response time only) -``` + +**Direct API calls with no overhead:** +- Make HTTP requests directly to weather service endpoints +- No intermediate proxy servers or additional network hops +- Latency equals API response time only (~100ms) +- Native HTTP client performance with connection pooling #### MCP Performance -```python -# Requires MCP server proxy -import mcp -async def call_weather_tool(): - client = mcp.Client() - await client.connect("weather-server") - result = await client.call_tool("get_weather", {"location": "San Francisco"}) - return result - -# Latency: ~150ms (API + MCP server overhead) -``` +**Requires MCP server proxy:** +- Connect to MCP server before making tool calls +- Route requests through MCP server to actual weather API +- Additional network hop adds latency overhead +- Latency includes API response time plus MCP server processing (~150ms) ### Infrastructure Requirements #### UTCP Infrastructure -```python -# Add one endpoint to existing API -@app.get("/utcp") -def get_manual(): - return utcp_manual # Static JSON -# Total infrastructure: 0 additional servers -``` +**Minimal infrastructure requirements:** +- Add single discovery endpoint to existing API (e.g., GET /utcp) +- Return static JSON manual describing available tools +- No additional servers, processes, or infrastructure needed +- Total infrastructure: 0 additional servers #### MCP Infrastructure -```python -# Requires dedicated MCP server -# Plus process management, monitoring, scaling -# Plus client connection management -# Total infrastructure: N MCP servers (one per tool provider) -``` +**MCP infrastructure requirements:** +- Requires dedicated MCP server processes for each tool provider +- Process management, monitoring, and scaling infrastructure needed +- Client connection management and session handling required +- Total infrastructure: N MCP servers (one per tool provider) ### Protocol Support Comparison @@ -164,11 +136,12 @@ def get_manual(): ``` #### MCP Protocol Limitation -```python -# MCP only supports JSON-RPC over stdio/HTTP -# All tools must be wrapped in MCP servers -# Cannot directly call WebSocket, CLI, or other protocols -``` + +**MCP protocol constraints:** +- Only supports JSON-RPC over stdio/HTTP transport +- All tools must be wrapped in MCP server implementations +- Cannot directly call WebSocket, CLI, or other native protocols +- Requires protocol translation layer for non-HTTP tools ## Feature Comparison @@ -197,16 +170,12 @@ def get_manual(): - Existing security policies apply #### MCP: Server-Mediated Authentication -```python -# MCP server must handle auth translation -class WeatherMCPServer: - def __init__(self, api_key): - self.api_key = api_key # Server stores credentials - - async def call_tool(self, name, args): - # Server makes authenticated call - return await weather_api.get(args["location"], auth=self.api_key) -``` + +**MCP server authentication requirements:** +- MCP server must handle authentication translation between client and API +- Server stores and manages API credentials on behalf of clients +- Server makes authenticated calls to actual APIs using stored credentials +- Requires credential management and secure storage in MCP server **Challenges:** - Credential management in MCP servers @@ -228,32 +197,29 @@ class WeatherMCPServer: ``` #### MCP: Limited Streaming -```python -# MCP has basic streaming but requires server implementation -# More complex to set up and maintain -``` + +**MCP streaming limitations:** +- MCP has basic streaming capabilities but requires server implementation +- More complex to set up and maintain than native streaming protocols +- Additional abstraction layer between client and streaming data source ### Error Handling #### UTCP: Native Error Responses -```python -# Errors come directly from the tool -try: - result = await client.call_tool("api.get_data", {"id": "123"}) -except httpx.HTTPStatusError as e: - # Native HTTP error with full context - print(f"API returned {e.response.status_code}: {e.response.text}") -``` + +**Direct error handling:** +- Errors come directly from the tool without translation +- Native HTTP status codes and error messages preserved +- Full error context available including headers and response body +- No error translation or abstraction layer #### MCP: Wrapped Error Responses -```python -# Errors are wrapped by MCP server -try: - result = await mcp_client.call_tool("get_data", {"id": "123"}) -except MCPError as e: - # MCP error - original context may be lost - print(f"MCP error: {e.message}") -``` + +**Error abstraction layer:** +- Errors are wrapped and translated by MCP server +- Original error context may be lost in translation +- MCP-specific error format instead of native tool errors +- Additional debugging complexity due to error wrapping ## Migration & Interoperability @@ -261,22 +227,11 @@ except MCPError as e: UTCP provides an MCP plugin for gradual migration: -```python -# Phase 1: Use existing MCP servers via UTCP -client = await UtcpClient.create(config={ - "manual_call_templates": [{ - "name": "legacy_mcp_service", - "call_template_type": "mcp", - "server_config": { - "command": "node", - "args": ["existing-mcp-server.js"] - } - }] -}) - -# Phase 2: Migrate high-value tools to native UTCP -# Phase 3: Deprecate MCP servers -``` +**Migration Strategy:** +- **Phase 1**: Use existing MCP servers via UTCP's MCP protocol plugin +- Configure UTCP client to connect to legacy MCP servers using MCP call templates +- **Phase 2**: Migrate high-value tools to native UTCP protocols (HTTP, WebSocket, CLI) +- **Phase 3**: Deprecate MCP servers once migration is complete [**Complete migration guide →**](./protocols/mcp.md) @@ -284,28 +239,12 @@ client = await UtcpClient.create(config={ You can use both protocols simultaneously: -```python -client = await UtcpClient.create(config={ - "manual_call_templates": [ - { - "name": "native_api", - "call_template_type": "http", - "url": "https://api.example.com/utcp" - }, - { - "name": "legacy_mcp", - "call_template_type": "mcp", - "server_config": {"command": "mcp-server"} - } - ] -}) - -# Call native UTCP tool -result1 = await client.call_tool("native_api.get_data", {}) - -# Call MCP tool through UTCP -result2 = await client.call_tool("legacy_mcp.mcp_tool", {}) -``` +**Hybrid approach during migration:** +- Configure UTCP client with both native UTCP and legacy MCP call templates +- Native UTCP tools use direct HTTP/WebSocket/CLI protocols +- Legacy MCP tools continue using MCP protocol plugin +- Gradually migrate tools from MCP to native UTCP protocols +- Single client interface for both native and legacy tools ## Enterprise Decision Factors @@ -332,24 +271,21 @@ Monitoring: Additional monitoring stack needed ### Development Velocity #### UTCP Development Speed -```python -# Day 1: Add UTCP endpoint -@app.get("/utcp") -def get_manual(): - return {"tools": [...]} - -# Day 2: Tools are available to AI agents -# No additional infrastructure needed -``` + +**Rapid deployment timeline:** +- **Day 1**: Add UTCP discovery endpoint to existing API +- **Day 2**: Tools are immediately available to AI agents +- No additional infrastructure, servers, or deployment needed +- Minimal code changes to existing systems #### MCP Development Speed -```python -# Week 1-2: Build MCP server -# Week 3: Deploy and configure server -# Week 4: Set up monitoring and scaling -# Week 5: Handle production issues -# Ongoing: Server maintenance -``` + +**Extended development timeline:** +- **Week 1-2**: Build dedicated MCP server implementation +- **Week 3**: Deploy and configure server infrastructure +- **Week 4**: Set up monitoring, logging, and scaling +- **Week 5**: Handle production issues and debugging +- **Ongoing**: Server maintenance, updates, and operations ### Risk Assessment @@ -394,48 +330,24 @@ def get_manual(): ### E-commerce API Integration #### UTCP Approach -```python -# Existing e-commerce API -@app.get("/products/{product_id}") -def get_product(product_id: str): - return {"id": product_id, "name": "Widget", "price": 29.99} - -# Add UTCP discovery -@app.get("/utcp") -def get_manual(): - return { - "tools": [{ - "name": "get_product", - "tool_call_template": { - "call_template_type": "http", - "url": "https://api.shop.com/products/${product_id}", - "http_method": "GET" - } - }] - } -# Total additional code: ~10 lines -# Additional infrastructure: 0 servers -``` +**E-commerce API with UTCP:** +- Keep existing product API endpoints unchanged (GET /products/{product_id}) +- Add single UTCP discovery endpoint (GET /utcp) +- Return UTCP manual describing available tools and how to call them +- Tools directly reference existing API endpoints with proper parameters +- Total additional code: ~10 lines +- Additional infrastructure: 0 servers #### MCP Approach -```python -# Requires building MCP server -from mcp.server import Server - -server = Server("ecommerce-server") - -@server.call_tool() -async def call_tool(name: str, args: dict): - if name == "get_product": - # Call existing API - response = await httpx.get(f"https://api.shop.com/products/{args['product_id']}") - return response.json() - -# Plus: server deployment, monitoring, scaling -# Total additional code: ~50+ lines -# Additional infrastructure: 1+ servers -``` + +**E-commerce API with MCP:** +- Requires building dedicated MCP server wrapper +- Implement tool listing and calling handlers in MCP server +- MCP server calls existing API endpoints on behalf of clients +- Additional server deployment, monitoring, and scaling required +- Total additional code: ~50+ lines +- Additional infrastructure: 1+ servers ### Database Query Tool @@ -459,11 +371,11 @@ async def call_tool(name: str, args: dict): ``` #### MCP Approach -```python -# Requires MCP server with database connection -# Plus connection pooling, query validation, etc. -# Much more complex implementation -``` +**MCP database approach:** +- Requires MCP server with database connection management +- Connection pooling, query validation, and security in MCP server +- Much more complex implementation than direct database access +- Additional abstraction layer between client and database ## Performance Benchmarks @@ -552,7 +464,7 @@ Both UTCP and MCP solve the tool integration problem, but with fundamentally dif ### To Get Started with UTCP: 1. **[Read the implementation guide](./implementation.md)** - Learn how to implement UTCP 2. **[Choose your protocols](./protocols/index.md)** - Select communication methods -3. **[Check examples](https://github.com/universal-tool-calling-protocol/python-utcp/tree/main/examples)** - See real implementations +3. **[Check examples](https://github.com/universal-tool-calling-protocol)** - See real implementations across multiple languages ### To Migrate from MCP: 1. **[Read the MCP integration guide](./protocols/mcp.md)** - Use MCP tools via UTCP From ecc25738ffa424399052ca0672de501c93021e4b Mon Sep 17 00:00:00 2001 From: Luca Perrozzi Date: Fri, 5 Sep 2025 13:07:39 +0000 Subject: [PATCH 18/22] Remove UTCP specification field requirements from documentation - Remove field requirement tables (Required/Optional Fields) from protocol docs - Remove UTCP manual structure requirement tables from tool provider guide - Replace with references to API specification documentation - Keep API input/output schema examples as they describe tool interfaces - Eliminates duplication between user docs and API specification --- docs/for-tool-providers.md | 37 ++++++++----------------------------- docs/protocols/cli.md | 18 +----------------- docs/protocols/sse.md | 17 +---------------- docs/protocols/text.md | 17 +---------------- docs/protocols/websocket.md | 17 +---------------- 5 files changed, 12 insertions(+), 94 deletions(-) diff --git a/docs/for-tool-providers.md b/docs/for-tool-providers.md index 6e4a0aa..b0de272 100644 --- a/docs/for-tool-providers.md +++ b/docs/for-tool-providers.md @@ -84,40 +84,19 @@ Your existing API endpoints remain unchanged. For example: ## Manual Structure -### Required Fields +The UTCP manual follows a standardized structure that defines your tools and how to call them. For complete field specifications, data types, and validation rules, see the [UTCP Manual API Reference](../api/core/utcp/data/utcp_manual.md). -| Field | Type | Description | -|-------|------|-------------| -| `manual_version` | string | Version of your manual (semantic versioning) | -| `utcp_version` | string | UTCP specification version | -| `tools` | array | List of available tools | +### Key Components -### Optional Fields - -| Field | Type | Description | -|-------|------|-------------| -| `info` | object | Metadata about your API | -| `auth` | object | Default authentication for all tools | -| `variables` | object | Default variable values | +- **Manual metadata**: Version information and API details +- **Tool definitions**: Description of available tools and their capabilities +- **Call templates**: Instructions for invoking each tool +- **Authentication**: Security configuration for tool access +- **Variables**: Dynamic values for tool parameters ### Tool Definition -Each tool must include: - -| Field | Type | Description | -|-------|------|-------------| -| `name` | string | Unique tool identifier | -| `description` | string | Human-readable description | -| `inputs` | object | JSON Schema for input parameters | -| `tool_call_template` | object | How to call the tool | - -Optional tool fields: - -| Field | Type | Description | -|-------|------|-------------| -| `outputs` | object | JSON Schema for expected outputs | -| `tags` | array | Tags for categorization and search | -| `examples` | array | Usage examples | +Tools are defined in your UTCP manual with their input parameters, call instructions, and optional metadata. For complete field specifications, see the [Tool API Reference](../api/core/utcp/data/tool.md). ## Communication Protocol Plugins diff --git a/docs/protocols/cli.md b/docs/protocols/cli.md index f6cc2f0..2a4257e 100644 --- a/docs/protocols/cli.md +++ b/docs/protocols/cli.md @@ -39,23 +39,7 @@ npm install @utcp/cli ## Configuration Options -### Required Fields - -| Field | Type | Description | -|-------|------|-------------| -| `call_template_type` | string | Must be `"cli"` | -| `command` | string | The command to execute | - -### Optional Fields - -| Field | Type | Description | -|-------|------|-------------| -| `args` | array | Command arguments | -| `working_directory` | string | Working directory for command execution | -| `environment` | object | Environment variables | -| `timeout` | number | Execution timeout in seconds (default: 30) | -| `shell` | boolean | Whether to execute through shell (default: false) | -| `capture_output` | boolean | Whether to capture stdout/stderr (default: true) | +The CLI call template allows you to execute command-line tools and scripts. For complete field specifications and validation rules, see the [CLI Call Template API Reference](../api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md). ## Security Considerations diff --git a/docs/protocols/sse.md b/docs/protocols/sse.md index 74301e5..6493892 100644 --- a/docs/protocols/sse.md +++ b/docs/protocols/sse.md @@ -40,22 +40,7 @@ npm install @utcp/http ## Configuration Options -### Required Fields - -| Field | Type | Description | -|-------|------|-------------| -| `call_template_type` | string | Must be `"sse"` | -| `url` | string | SSE endpoint URL | - -### Optional Fields - -| Field | Type | Description | -|-------|------|-------------| -| `headers` | object | HTTP headers for the request | -| `query_params` | object | URL query parameters | -| `timeout` | number | Stream timeout in seconds (default: 60) | -| `max_events` | number | Maximum events to receive (default: unlimited) | -| `event_filter` | object | Filter events by type or data | +The Server-Sent Events (SSE) call template enables real-time streaming data from HTTP endpoints. For complete field specifications and validation rules, see the [SSE Call Template API Reference](../api/plugins/communication_protocols/sse/src/utcp_sse/sse_call_template.md). | `reconnect` | boolean | Auto-reconnect on connection loss (default: true) | | `reconnect_delay` | number | Delay between reconnection attempts (default: 3) | diff --git a/docs/protocols/text.md b/docs/protocols/text.md index a1a17e2..dcca388 100644 --- a/docs/protocols/text.md +++ b/docs/protocols/text.md @@ -35,22 +35,7 @@ npm install @utcp/text ## Configuration Options -### Required Fields - -| Field | Type | Description | -|-------|------|-------------| -| `call_template_type` | string | Must be `"text"` | -| `file_path` | string | Path to text file (local or URL) | - -### Optional Fields - -| Field | Type | Description | -|-------|------|-------------| -| `encoding` | string | File encoding (default: "utf-8") | -| `max_size` | number | Maximum file size in bytes (default: 1MB) | -| `line_range` | object | Specific line range to read | -| `pattern` | string | Regex pattern to filter content | -| `transform` | string | Content transformation ("upper", "lower", "strip") | +The Text call template enables reading and processing text files from local filesystem or URLs. For complete field specifications and validation rules, see the [Text Call Template API Reference](../api/plugins/communication_protocols/text/src/utcp_text/text_call_template.md). ## File Sources diff --git a/docs/protocols/websocket.md b/docs/protocols/websocket.md index 8bd7da0..2978390 100644 --- a/docs/protocols/websocket.md +++ b/docs/protocols/websocket.md @@ -41,22 +41,7 @@ npm install @utcp/websocket ## Configuration Options -### Required Fields - -| Field | Type | Description | -|-------|------|-------------| -| `call_template_type` | string | Must be `"websocket"` | -| `url` | string | WebSocket URL (ws:// or wss://) | -| `message` | object/string | Message to send to the WebSocket | - -### Optional Fields - -| Field | Type | Description | -|-------|------|-------------| -| `headers` | object | HTTP headers for WebSocket handshake | -| `connection_timeout` | number | Connection timeout in seconds (default: 10) | -| `response_timeout` | number | Response timeout in seconds (default: 30) | -| `close_after_response` | boolean | Close connection after receiving response (default: true) | +The WebSocket call template enables real-time communication with WebSocket servers. For complete field specifications and validation rules, see the [WebSocket Call Template API Reference](../api/plugins/communication_protocols/websocket/src/utcp_websocket/websocket_call_template.md). | `expected_responses` | number | Number of expected response messages (default: 1) | | `ping_interval` | number | Ping interval in seconds (default: 30) | From d098655f434a138c45eaad1c53862d8e06ba2523 Mon Sep 17 00:00:00 2001 From: Luca Perrozzi Date: Fri, 5 Sep 2025 13:10:50 +0000 Subject: [PATCH 19/22] Remove Express.js implementation section - Remove technology-specific implementation example - Keep language-agnostic conceptual guidance - Maintain focus on UTCP concepts rather than specific frameworks --- docs/for-tool-providers.md | 53 -------------------------------------- 1 file changed, 53 deletions(-) diff --git a/docs/for-tool-providers.md b/docs/for-tool-providers.md index b0de272..0174328 100644 --- a/docs/for-tool-providers.md +++ b/docs/for-tool-providers.md @@ -379,59 +379,6 @@ The UTCP manual describes how to call your existing endpoints: } ``` -### Express.js Implementation - -```javascript -const express = require('express'); -const app = express(); - -// Your existing API endpoints -app.get('/users/:userId', (req, res) => { - // Your existing logic - res.json({ - id: req.params.userId, - name: 'John Doe', - email: 'john@example.com' - }); -}); - -// UTCP Discovery endpoint -app.get('/utcp', (req, res) => { - res.json({ - manual_version: "1.0.0", - utcp_version: "1.0.1", - info: { - title: "User Management API", - version: "1.0.0", - description: "API for managing user accounts" - }, - tools: [ - { - name: "get_user", - description: "Retrieve user information by ID", - inputs: { - type: "object", - properties: { - user_id: { type: "string" } - }, - required: ["user_id"] - }, - tool_call_template: { - call_template_type: "http", - url: `${process.env.BASE_URL}/users/\${user_id}`, - http_method: "GET", - headers: { - "Authorization": "Bearer ${API_TOKEN}" - } - } - } - ] - }); -}); - -app.listen(3000); -``` - ## OpenAPI Integration If you already have an OpenAPI/Swagger specification, you can automatically convert it to a UTCP manual: From 91dee11a27d58c0318bf80eda5eb387a904f4e6e Mon Sep 17 00:00:00 2001 From: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Date: Sun, 7 Sep 2025 11:43:34 +0200 Subject: [PATCH 20/22] Updated API spec --- .../utcp_serializer_validation_error.md | 16 ++- .../filter_dict_post_processor.md | 49 ++++++++++ .../limit_strings_post_processor.md | 48 +++++++++ .../core/utcp/implementations/tag_search.md | 33 ++++++- docs/api/core/utcp/plugins/plugin_loader.md | 6 ++ docs/api/index.md | 16 ++- .../cli/src/utcp_cli/cli_call_template.md | 94 ++++++++++++++++-- .../utcp_cli/cli_communication_protocol.md | 86 +++++++++------- .../http/src/utcp_http/http_call_template.md | 75 ++++++++++++++ .../http/src/utcp_http/openapi_converter.md | 63 ++++++++++-- .../mcp/src/utcp_mcp/mcp_call_template.md | 98 +++++++++++++++++++ 11 files changed, 523 insertions(+), 61 deletions(-) create mode 100644 docs/api/core/utcp/implementations/post_processors/filter_dict_post_processor.md create mode 100644 docs/api/core/utcp/implementations/post_processors/limit_strings_post_processor.md diff --git a/docs/api/core/utcp/exceptions/utcp_serializer_validation_error.md b/docs/api/core/utcp/exceptions/utcp_serializer_validation_error.md index 4dcaaec..378f221 100644 --- a/docs/api/core/utcp/exceptions/utcp_serializer_validation_error.md +++ b/docs/api/core/utcp/exceptions/utcp_serializer_validation_error.md @@ -9,6 +9,20 @@ sidebar_label: utcp_serializer_validation_error ### class UtcpSerializerValidationError {#utcpserializervalidationerror} -*No class documentation available* +
+Documentation + +Exception raised when a serializer validation fails. + +Thrown by serializers when they cannot validate or convert data structures +due to invalid format, missing required fields, or type mismatches. +Contains the original validation error details for debugging. + + +**Usage** + +Typically caught when loading configuration files or processing +external data that doesn't conform to UTCP specifications. +
--- diff --git a/docs/api/core/utcp/implementations/post_processors/filter_dict_post_processor.md b/docs/api/core/utcp/implementations/post_processors/filter_dict_post_processor.md new file mode 100644 index 0000000..6193415 --- /dev/null +++ b/docs/api/core/utcp/implementations/post_processors/filter_dict_post_processor.md @@ -0,0 +1,49 @@ +--- +title: filter_dict_post_processor +sidebar_label: filter_dict_post_processor +--- + +# filter_dict_post_processor + +**File:** `core/src/utcp/implementations/post_processors/filter_dict_post_processor.py` + +### class FilterDictPostProcessor ([ToolPostProcessor](./../../interfaces/tool_post_processor.md#toolpostprocessor)) {#filterdictpostprocessor} + +
+Documentation + +Post-processor that filters dictionary keys from tool results. + +Provides flexible filtering capabilities to include or exclude specific keys +from dictionary results, with support for nested dictionaries and lists. +Can be configured to apply filtering only to specific tools or manuals. + + +**Attributes** + +- **`tool_post_processor_type`**: Always "filter_dict" for this processor. +- **`exclude_keys`**: List of keys to remove from dictionary results. +- **`only_include_keys`**: List of keys to keep in dictionary results (all others removed). +- **`exclude_tools`**: List of tool names to skip processing for. +- **`only_include_tools`**: List of tool names to process (all others skipped). +- **`exclude_manuals`**: List of manual names to skip processing for. +- **`only_include_manuals`**: List of manual names to process (all others skipped). +
+ +#### Fields: + +- tool_post_processor_type: Literal['filter_dict'] +- exclude_keys: Optional[List[str]] +- only_include_keys: Optional[List[str]] +- exclude_tools: Optional[List[str]] +- only_include_tools: Optional[List[str]] +- exclude_manuals: Optional[List[str]] +- only_include_manuals: Optional[List[str]] + +--- + +### class FilterDictPostProcessorConfigSerializer ([Serializer](./../../interfaces/serializer.md#serializer)[FilterDictPostProcessor]) {#filterdictpostprocessorconfigserializer} + +*No class documentation available* + +--- diff --git a/docs/api/core/utcp/implementations/post_processors/limit_strings_post_processor.md b/docs/api/core/utcp/implementations/post_processors/limit_strings_post_processor.md new file mode 100644 index 0000000..f524d48 --- /dev/null +++ b/docs/api/core/utcp/implementations/post_processors/limit_strings_post_processor.md @@ -0,0 +1,48 @@ +--- +title: limit_strings_post_processor +sidebar_label: limit_strings_post_processor +--- + +# limit_strings_post_processor + +**File:** `core/src/utcp/implementations/post_processors/limit_strings_post_processor.py` + +### class LimitStringsPostProcessor ([ToolPostProcessor](./../../interfaces/tool_post_processor.md#toolpostprocessor)) {#limitstringspostprocessor} + +
+Documentation + +Post-processor that limits the length of string values in tool results. + +Truncates string values to a specified maximum length to prevent +excessively large responses. Processes nested dictionaries and lists +recursively. Can be configured to apply limiting only to specific +tools or manuals. + + +**Attributes** + +- **`tool_post_processor_type`**: Always "limit_strings" for this processor. +- **`limit`**: Maximum length for string values (default: 10000 characters). +- **`exclude_tools`**: List of tool names to skip processing for. +- **`only_include_tools`**: List of tool names to process (all others skipped). +- **`exclude_manuals`**: List of manual names to skip processing for. +- **`only_include_manuals`**: List of manual names to process (all others skipped). +
+ +#### Fields: + +- tool_post_processor_type: Literal['limit_strings'] +- limit: int +- exclude_tools: Optional[List[str]] +- only_include_tools: Optional[List[str]] +- exclude_manuals: Optional[List[str]] +- only_include_manuals: Optional[List[str]] + +--- + +### class LimitStringsPostProcessorConfigSerializer ([Serializer](./../../interfaces/serializer.md#serializer)[LimitStringsPostProcessor]) {#limitstringspostprocessorconfigserializer} + +*No class documentation available* + +--- diff --git a/docs/api/core/utcp/implementations/tag_search.md b/docs/api/core/utcp/implementations/tag_search.md index af032ab..558045e 100644 --- a/docs/api/core/utcp/implementations/tag_search.md +++ b/docs/api/core/utcp/implementations/tag_search.md @@ -9,7 +9,38 @@ sidebar_label: tag_search ### class TagAndDescriptionWordMatchStrategy ([ToolSearchStrategy](./../interfaces/tool_search_strategy.md#toolsearchstrategy)) {#taganddescriptionwordmatchstrategy} -*No class documentation available* +
+Documentation + +Tag and description word match strategy. + + +**Implements A Weighted Scoring System That Matches Tools Based On** + +1. Tag matches (higher weight) +2. Description word matches (lower weight) + +The strategy normalizes queries to lowercase, extracts words using regex, +and calculates relevance scores for each tool. Results are sorted by +score in descending order. + + + +**Attributes** + +- **`tool_search_strategy_type`**: Always "tag_and_description_word_match". +- **`description_weight`**: Weight multiplier for description word matches (default: 1.0). +- **`tag_weight`**: Weight multiplier for tag matches (default: 3.0). + + + +**Scoring Algorithm** + +- Each matching tag contributes tag_weight points +- Each matching description word contributes description_weight points +- Tools with higher scores are ranked first +- Tools with zero score are included in results (ranked last) +
#### Fields: diff --git a/docs/api/core/utcp/plugins/plugin_loader.md b/docs/api/core/utcp/plugins/plugin_loader.md index 8e8e9c1..3ac6524 100644 --- a/docs/api/core/utcp/plugins/plugin_loader.md +++ b/docs/api/core/utcp/plugins/plugin_loader.md @@ -7,6 +7,12 @@ sidebar_label: plugin_loader **File:** `core/src/utcp/plugins/plugin_loader.py` +### Function _load_plugins() {#_load_plugins} + +*No function documentation available* + +--- + ### Function ensure_plugins_initialized() {#ensure_plugins_initialized} *No function documentation available* diff --git a/docs/api/index.md b/docs/api/index.md index 51585f8..4d60cfa 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -11,8 +11,8 @@ This specification is organized by module of the reference python implementation **Note:** The modules don't have to be implemented in the same way as in the reference implementation, but all of the functionality here needs to be provided. -**Total documented items:** 189 -**Modules documented:** 39 +**Total documented items:** 194 +**Modules documented:** 41 ## Core Modules @@ -93,6 +93,16 @@ Core UTCP framework components that define the fundamental interfaces and implem - **Contains:** 2 classes, 12 methods +### [utcp.implementations.post_processors.filter_dict_post_processor](./core\utcp\implementations\post_processors\filter_dict_post_processor.md) + +- **Contains:** 2 classes + + +### [utcp.implementations.post_processors.limit_strings_post_processor](./core\utcp\implementations\post_processors\limit_strings_post_processor.md) + +- **Contains:** 2 classes + + ### [utcp.implementations.tag_search](./core\utcp\implementations\tag_search.md) - **Contains:** 2 classes, 3 methods @@ -140,7 +150,7 @@ Core UTCP framework components that define the fundamental interfaces and implem ### [utcp.plugins.plugin_loader](./core\utcp\plugins\plugin_loader.md) -- **Contains:** 1 functions +- **Contains:** 2 functions ### [utcp.utcp_client](./core\utcp\utcp_client.md) diff --git a/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md b/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md index abfe4aa..9c15efd 100644 --- a/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md +++ b/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md @@ -12,19 +12,64 @@ sidebar_label: cli_call_template
Documentation -Call template configuration for Command Line Interface tools. +Call template configuration for Command Line Interface (CLI) tools. -Enables execution of command-line tools and programs as UTCP providers. -Supports environment variable injection and custom working directories. +This class defines the configuration for executing command-line tools and +programs as UTCP tool providers. It supports environment variable injection, +custom working directories, and defines the command to be executed. **Attributes** -- **`call_template_type`**: Always "cli" for CLI providers. -- **`command_name`**: The name or path of the command to execute. -- **`env_vars`**: Optional environment variables to set during command execution. -- **`working_dir`**: Optional custom working directory for command execution. -- **`auth`**: Always None - CLI providers don't support authentication. +- **`call_template_type`**: The type of the call template. Must be "cli". +- **`command_name`**: The command or path of the program to execute. It can + contain placeholders for arguments that will be substituted at + runtime (e.g., `${arg_name}`). +- **`env_vars`**: A dictionary of environment variables to set for the command's + execution context. Values can be static strings or placeholders for + variables from the UTCP client's variable substitutor. +- **`working_dir`**: The working directory from which to run the command. If not + provided, it defaults to the current process's working directory. +- **`auth`**: Authentication details. Not applicable to the CLI protocol, so it + is always None. + + + +**Basic Cli Command** + +```json + { + "name": "list_files_tool", + "call_template_type": "cli", + "command_name": "ls -la", + "working_dir": "/tmp" + } +``` + + + +**Command With Environment Variables And Argument Placeholders** + +```json + { + "name": "python_script_tool", + "call_template_type": "cli", + "command_name": "python script.py --input ${input_file}", + "env_vars": { + "PYTHONPATH": "/custom/path", + "API_KEY": "${API_KEY_VAR}" + } + } +``` + + + +**Security Considerations** + +- Commands are executed in a subprocess. Ensure that the commands +specified are from a trusted source. +- Avoid passing unsanitized user input directly into the command string. +Use tool argument validation where possible.
#### Fields: @@ -46,13 +91,42 @@ Supports environment variable injection and custom working directories.
to_dict(self, obj: CliCallTemplate) -> dict -*No method documentation available* +Converts a `CliCallTemplate` instance to its dictionary representation. + + +**Args** + +- **`obj`**: The `CliCallTemplate` instance to serialize. + + + +**Returns** + +A dictionary representing the `CliCallTemplate`.
validate_dict(self, obj: dict) -> CliCallTemplate -*No method documentation available* +Validates a dictionary and constructs a `CliCallTemplate` instance. + + +**Args** + +- **`obj`**: The dictionary to validate and deserialize. + + + +**Returns** + +A `CliCallTemplate` instance. + + + +**Raises** + +- **`[UtcpSerializerValidationError](./../../../../../core/utcp/exceptions/utcp_serializer_validation_error.md#utcpserializervalidationerror)`**: If the dictionary is not a valid + representation of a `CliCallTemplate`.
--- diff --git a/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_communication_protocol.md b/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_communication_protocol.md index ee2e071..6be94d7 100644 --- a/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_communication_protocol.md +++ b/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_communication_protocol.md @@ -9,87 +9,99 @@ sidebar_label: cli_communication_protocol ### class CliCommunicationProtocol ([CommunicationProtocol](./../../../../../core/utcp/interfaces/communication_protocol.md#communicationprotocol)) {#clicommunicationprotocol} -
-Documentation - -Transport implementation for CLI-based tool providers. +*No class documentation available* -Handles communication with command-line tools by executing processes -and managing their input/output. Supports both tool discovery and -execution phases with comprehensive error handling and timeout management. +#### Methods: +
+async register_manual(self, caller, manual_call_template: [CallTemplate](./../../../../../core/utcp/data/call_template.md#calltemplate)) -> [RegisterManualResult](./../../../../../core/utcp/data/register_manual_response.md#registermanualresult) -**Features** +Registers a CLI-based manual and discovers its tools. -- Asynchronous subprocess execution with proper cleanup -- [Tool](./../../../../../core/utcp/data/tool.md#tool) discovery through startup commands returning UTCP manuals -- Flexible argument formatting for various CLI conventions -- Environment variable injection for authentication -- JSON output parsing with graceful fallback to text -- Cross-platform command parsing and execution -- Configurable working directories and timeouts -- Process lifecycle management with proper termination +This method executes the command specified in the `[CliCallTemplate](./cli_call_template.md#clicalltemplate)`'s +`command_name` field. It then attempts to parse the command's output +(stdout) as a UTCP manual in JSON format. +**Args** -**Architecture** +- **`caller`**: The UTCP client instance that is calling this method. +- **`manual_call_template`**: The `[CliCallTemplate](./cli_call_template.md#clicalltemplate)` containing the details for + tool discovery, such as the command to run. -CLI tools are discovered by executing the provider's command_name -and parsing the output for UTCP manual JSON. [Tool](./../../../../../core/utcp/data/tool.md#tool) calls execute -the same command with formatted arguments and return processed output. +**Returns** -**Attributes** +A `[RegisterManualResult](./../../../../../core/utcp/data/register_manual_response.md#registermanualresult)` object indicating whether the registration +was successful and containing the discovered tools. -- **`_log`**: Logger function for debugging and error reporting. -
-#### Methods: -
-async register_manual(self, caller, manual_call_template: [CallTemplate](./../../../../../core/utcp/data/call_template.md#calltemplate)) -> [RegisterManualResult](./../../../../../core/utcp/data/register_manual_response.md#registermanualresult) +**Raises** -*No method documentation available* +- **`ValueError`**: If the `manual_call_template` is not an instance of + `[CliCallTemplate](./cli_call_template.md#clicalltemplate)` or if `command_name` is not set.
async deregister_manual(self, caller, manual_call_template: [CallTemplate](./../../../../../core/utcp/data/call_template.md#calltemplate)) -> None -*No method documentation available* +Deregisters a CLI manual. + +For the CLI protocol, this is a no-op as there are no persistent +connections to terminate. + + +**Args** + +- **`caller`**: The UTCP client instance that is calling this method. +- **`manual_call_template`**: The call template of the manual to deregister.
async call_tool(self, caller, tool_name: str, tool_args: Dict[str, Any], tool_call_template: [CallTemplate](./../../../../../core/utcp/data/call_template.md#calltemplate)) -> Any -Call a CLI tool. +Calls a CLI tool by executing its command. -Executes the command specified by provider.command_name with the provided arguments. +This method constructs and executes the command specified in the +`[CliCallTemplate](./cli_call_template.md#clicalltemplate)`. It formats the provided `tool_args` as command-line +arguments and runs the command in a subprocess. **Args** -- **`caller`**: The UTCP client that is calling this method. -- **`tool_name`**: Name of the tool to call -- **`tool_args`**: Arguments for the tool call -- **`tool_call_template`**: The [CliCallTemplate](./cli_call_template.md#clicalltemplate) for the tool +- **`caller`**: The UTCP client instance that is calling this method. +- **`tool_name`**: The name of the tool to call. +- **`tool_args`**: A dictionary of arguments for the tool call. +- **`tool_call_template`**: The `[CliCallTemplate](./cli_call_template.md#clicalltemplate)` for the tool. -**The Output From The Command Execution Based On Exit Code** +**Returns** +The result of the command execution. If the command exits with a code +of 0, it returns the content of stdout. If the exit code is non-zero, +it returns the content of stderr. The output is parsed as JSON if +possible; otherwise, it is returned as a raw string. **Raises** -- **`ValueError`**: If provider is not a CliProvider or command_name is not set +- **`ValueError`**: If `tool_call_template` is not an instance of + `[CliCallTemplate](./cli_call_template.md#clicalltemplate)` or if `command_name` is not set.
async call_tool_streaming(self, caller, tool_name: str, tool_args: Dict[str, Any], tool_call_template: [CallTemplate](./../../../../../core/utcp/data/call_template.md#calltemplate)) -> AsyncGenerator[Any, None] -*No method documentation available* +Streaming calls are not supported for the CLI protocol. + + +**Raises** + +- **`NotImplementedError`**: Always, as this functionality is not supported.
--- diff --git a/docs/api/plugins/communication_protocols/http/src/utcp_http/http_call_template.md b/docs/api/plugins/communication_protocols/http/src/utcp_http/http_call_template.md index 2420d59..98dda8c 100644 --- a/docs/api/plugins/communication_protocols/http/src/utcp_http/http_call_template.md +++ b/docs/api/plugins/communication_protocols/http/src/utcp_http/http_call_template.md @@ -20,6 +20,81 @@ parameters using \{parameter_name\} syntax. All tool arguments not mapped to URL body, headers or query pattern parameters are passed as query parameters using '?arg_name=\{arg_value\}'. +**Basic Http Get Request** + +```json + { + "name": "my_rest_api", + "call_template_type": "http", + "url": "https://api.example.com/users/{user_id}", + "http_method": "GET" + } +``` + + + +**Post With Authentication** + +```json + { + "name": "secure_api", + "call_template_type": "http", + "url": "https://api.example.com/users", + "http_method": "POST", + "content_type": "application/json", + "auth": { + "auth_type": "api_key", + "api_key": "Bearer ${API_KEY}", + "var_name": "Authorization", + "location": "header" + }, + "headers": { + "X-Custom-Header": "value" + }, + "body_field": "body", + "header_fields": ["user_id"] + } +``` + + + +**Oauth2 Authentication** + +```json + { + "name": "oauth_api", + "call_template_type": "http", + "url": "https://api.example.com/data", + "http_method": "GET", + "auth": { + "auth_type": "oauth2", + "client_id": "${CLIENT_ID}", + "client_secret": "${CLIENT_SECRET}", + "token_url": "https://auth.example.com/token" + } + } +``` + + + +**Basic Authentication** + +```json + { + "name": "basic_auth_api", + "call_template_type": "http", + "url": "https://api.example.com/secure", + "http_method": "GET", + "auth": { + "auth_type": "basic", + "username": "${USERNAME}", + "password": "${PASSWORD}" + } + } +``` + + + **Attributes** - **`call_template_type`**: Always "http" for HTTP providers. diff --git a/docs/api/plugins/communication_protocols/http/src/utcp_http/openapi_converter.md b/docs/api/plugins/communication_protocols/http/src/utcp_http/openapi_converter.md index 0a68759..dd41955 100644 --- a/docs/api/plugins/communication_protocols/http/src/utcp_http/openapi_converter.md +++ b/docs/api/plugins/communication_protocols/http/src/utcp_http/openapi_converter.md @@ -22,14 +22,50 @@ a UTCP tool with appropriate input/output schemas. **Features** -- Complete OpenAPI specification parsing -- Recursive JSON reference ($ref) resolution -- Authentication scheme conversion (API key, Basic, OAuth2) -- Input parameter and request body handling -- Response schema extraction -- URL template and path parameter support -- Provider name normalization -- Placeholder variable generation for configuration +- Complete OpenAPI specification parsing. +- Recursive JSON reference ($ref) resolution. +- Authentication scheme conversion (API key, Basic, OAuth2). +- Input parameter and request body handling. +- Response schema extraction. +- URL template and path parameter support. +- Call template name normalization. +- Placeholder variable generation for configuration. + + + +**Basic Openapi Conversion** + +```python + from utcp_http.openapi_converter import OpenApiConverter + + # Assuming you have a method to fetch and parse the spec + openapi_spec = fetch_and_parse_spec("https://api.example.com/openapi.json") + + converter = OpenApiConverter(openapi_spec) + manual = converter.convert() + + # Use the generated manual with a UTCP client + # client = await UtcpClient.create() + # await client.register_manual(manual) +``` + + + +**Converting Local Openapi File** + +```python + import yaml + + converter = OpenApiConverter() + + +**With Open("Api_Spec.Yaml", "R") As F** + +spec_content = yaml.safe_load(f) + +converter = OpenApiConverter(spec_content) +manual = converter.convert() +``` @@ -54,7 +90,16 @@ operation, and creating corresponding UTCP tools with HTTP call_templates.
convert(self) -> [UtcpManual](./../../../../../core/utcp/data/utcp_manual.md#utcpmanual) -*No method documentation available* +Converts the loaded OpenAPI specification into a [UtcpManual](./../../../../../core/utcp/data/utcp_manual.md#utcpmanual). + +This is the main entry point for the conversion process. It iterates through +the paths and operations in the specification, creating a UTCP tool for each +one. + + +**Returns** + +A [UtcpManual](./../../../../../core/utcp/data/utcp_manual.md#utcpmanual) object containing all the tools generated from the spec.
--- diff --git a/docs/api/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_call_template.md b/docs/api/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_call_template.md index 2b36fbb..8f5cc77 100644 --- a/docs/api/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_call_template.md +++ b/docs/api/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_call_template.md @@ -44,12 +44,109 @@ interfaces. Supports both stdio (local process) and HTTP (remote) transport methods. +**Basic Mcp Server With Stdio Transport** + +```json + { + "name": "mcp_server", + "call_template_type": "mcp", + "config": { + "mcpServers": { + "filesystem": { + "command": "node", + "args": ["mcp-server.js"], + "env": {"NODE_ENV": "production"} + } + } + } + } +``` + + + +**Mcp Server With Working Directory** + +```json + { + "name": "mcp_tools", + "call_template_type": "mcp", + "config": { + "mcpServers": { + "tools": { + "command": "python", + "args": ["-m", "mcp_server"], + "cwd": "/app/mcp", + "env": { + "PYTHONPATH": "/app", + "LOG_LEVEL": "INFO" + } + } + } + } + } +``` + + + +**Mcp Server With Oauth2 Authentication** + +```json + { + "name": "secure_mcp", + "call_template_type": "mcp", + "config": { + "mcpServers": { + "secure_server": { + "transport": "http", + "url": "https://mcp.example.com" + } + } + }, + "auth": { + "auth_type": "oauth2", + "token_url": "https://auth.example.com/token", + "client_id": "${CLIENT_ID}", + "client_secret": "${CLIENT_SECRET}", + "scope": "read:tools" + } + } +``` + + + +**During Migration (Utcp With Mcp)** + +```python + # UTCP Client with MCP plugin + client = await UtcpClient.create() + result = await client.call_tool("filesystem.read_file", { + "path": "/data/file.txt" + }) +``` + + + +**After Migration (Pure Utcp)** + +```python + # UTCP Client with native protocol + client = await UtcpClient.create() + result = await client.call_tool("filesystem.read_file", { + "path": "/data/file.txt" + }) +``` + + + **Attributes** - **`call_template_type`**: Always "mcp" for MCP providers. - **`config`**: Configuration object containing MCP server definitions. This follows the same format as the official MCP server configuration. - **`auth`**: Optional OAuth2 authentication for HTTP-based MCP servers. +- **`register_resources_as_tools`**: Whether to register MCP resources as callable tools. + When True, server resources are exposed as tools that can be called. + Default is False.
#### Fields: @@ -57,6 +154,7 @@ transport methods. - call_template_type: Literal['mcp'] - config: McpConfig - auth: Optional[[OAuth2Auth](./../../../../../core/utcp/data/auth_implementations/oauth2_auth.md#oauth2auth)] +- register_resources_as_tools: bool --- From d1426b21b890c068120d8612b37a88edb7453f99 Mon Sep 17 00:00:00 2001 From: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Date: Sun, 7 Sep 2025 17:37:30 +0200 Subject: [PATCH 21/22] Update docs --- docs/api/index.md | 4 +- .../cli/src/utcp_cli/cli_call_template.md | 126 +++++- .../utcp_cli/cli_communication_protocol.md | 3 +- docs/for-tool-providers.md | 110 +++-- docs/implementation.md | 48 ++- docs/index.md | 13 +- docs/migration-v0.1-to-v1.0.md | 37 +- docs/protocols/cli.md | 398 ++++++++++++------ docs/protocols/index.md | 6 +- docs/protocols/sse.md | 2 +- docs/protocols/streamable-http.md | 105 +++++ docs/protocols/websocket.md | 2 +- docs/security.md | 41 +- docs/utcp-vs-mcp.md | 10 +- sidebars.ts | 49 +-- 15 files changed, 645 insertions(+), 309 deletions(-) create mode 100644 docs/protocols/streamable-http.md diff --git a/docs/api/index.md b/docs/api/index.md index 4d60cfa..50ef8a5 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -11,7 +11,7 @@ This specification is organized by module of the reference python implementation **Note:** The modules don't have to be implemented in the same way as in the reference implementation, but all of the functionality here needs to be provided. -**Total documented items:** 194 +**Total documented items:** 195 **Modules documented:** 41 ## Core Modules @@ -164,7 +164,7 @@ Plugin implementations that extend UTCP with specific transport protocols and ca ### [communication_protocols.cli.src.utcp_cli.cli_call_template](./plugins\communication_protocols\cli\src\utcp_cli\cli_call_template.md) -- **Contains:** 2 classes, 2 methods +- **Contains:** 3 classes, 2 methods ### [communication_protocols.cli.src.utcp_cli.cli_communication_protocol](./plugins\communication_protocols\cli\src\utcp_cli\cli_communication_protocol.md) diff --git a/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md b/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md index 9c15efd..54c7dca 100644 --- a/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md +++ b/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md @@ -7,6 +7,53 @@ sidebar_label: cli_call_template **File:** `plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.py` +### class CommandStep {#commandstep} + +
+Documentation + +Configuration for a single command step in a CLI execution flow. + + +**Attributes** + +- **`command`**: The command string to execute. Can contain UTCP_ARG_argname_UTCP_END + placeholders that will be replaced with values from tool_args. Can also + reference previous command outputs using $CMD_0_OUTPUT, $CMD_1_OUTPUT, etc. +- **`append_to_final_output`**: Whether this command's output should be included + in the final result. If not specified, defaults to False for all + commands except the last one. + + + +**Basic Command Step** + +```json + { + "command": "git status", + "append_to_final_output": true + } +``` + + + +**Command With Argument Placeholders And Output Reference** + +```json + { + "command": "echo "Cloning to: UTCP_ARG_target_dir_UTCP_END, previous status: $CMD_0_OUTPUT"", + "append_to_final_output": true + } +``` +
+ +#### Fields: + +- command: str +- append_to_final_output: Optional[bool] + +--- + ### class CliCallTemplate ([CallTemplate](./../../../../../core/utcp/data/call_template.md#calltemplate)) {#clicalltemplate}
@@ -15,46 +62,92 @@ sidebar_label: cli_call_template Call template configuration for Command Line Interface (CLI) tools. This class defines the configuration for executing command-line tools and -programs as UTCP tool providers. It supports environment variable injection, -custom working directories, and defines the command to be executed. +programs as UTCP tool providers. Commands are executed in a single subprocess +to maintain state (like directory changes) between commands. + + + + +**You Can Reference The Output Of Previous Commands Using Variables** + + +- **`Example`**: `echo "Previous result: $CMD_0_OUTPUT"` + **Attributes** - **`call_template_type`**: The type of the call template. Must be "cli". -- **`command_name`**: The command or path of the program to execute. It can - contain placeholders for arguments that will be substituted at - runtime (e.g., `${arg_name}`). +- **`commands`**: A list of CommandStep objects defining the commands to execute + in order. Each command can contain UTCP_ARG_argname_UTCP_END placeholders + that will be replaced with values from tool_args during execution. - **`env_vars`**: A dictionary of environment variables to set for the command's execution context. Values can be static strings or placeholders for variables from the UTCP client's variable substitutor. -- **`working_dir`**: The working directory from which to run the command. If not +- **`working_dir`**: The working directory from which to run the commands. If not provided, it defaults to the current process's working directory. - **`auth`**: Authentication details. Not applicable to the CLI protocol, so it is always None. -**Basic Cli Command** +**Cross-Platform Directory Operations** + +```json + { + "name": "cross_platform_dir_tool", + "call_template_type": "cli", + "commands": [ + { + "command": "cd UTCP_ARG_target_dir_UTCP_END", + "append_to_final_output": false + }, + { + "command": "ls -la", + "append_to_final_output": true + } + ] + } +``` + + + +**Referencing Previous Command Output** ```json { - "name": "list_files_tool", + "name": "reference_previous_output_tool", "call_template_type": "cli", - "command_name": "ls -la", - "working_dir": "/tmp" + "commands": [ + { + "command": "git status --porcelain", + "append_to_final_output": false + }, + { + "command": "echo "Found changes: $CMD_0_OUTPUT"", + "append_to_final_output": true + } + ] } ``` -**Command With Environment Variables And Argument Placeholders** +**Command With Environment Variables And Placeholders** ```json { - "name": "python_script_tool", + "name": "python_multi_step_tool", "call_template_type": "cli", - "command_name": "python script.py --input ${input_file}", + "commands": [ + { + "command": "python setup.py install", + "append_to_final_output": false + }, + { + "command": "python script.py --input UTCP_ARG_input_file_UTCP_END --result "$CMD_0_OUTPUT"" + } + ], "env_vars": { "PYTHONPATH": "/custom/path", "API_KEY": "${API_KEY_VAR}" @@ -70,12 +163,17 @@ custom working directories, and defines the command to be executed. specified are from a trusted source. - Avoid passing unsanitized user input directly into the command string. Use tool argument validation where possible. +- All placeholders are replaced with string values from tool_args. +- Commands should use the appropriate syntax for the target platform +(PowerShell on Windows, Bash on Unix). +- Previous command outputs are available as variables but should be +used carefully to avoid command injection.
#### Fields: - call_template_type: Literal['cli'] -- command_name: str +- commands: List[CommandStep] - env_vars: Optional[Dict[str, str]] - working_dir: Optional[str] - auth: None diff --git a/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_communication_protocol.md b/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_communication_protocol.md index 6be94d7..5ed1941 100644 --- a/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_communication_protocol.md +++ b/docs/api/plugins/communication_protocols/cli/src/utcp_cli/cli_communication_protocol.md @@ -82,8 +82,7 @@ arguments and runs the command in a subprocess. The result of the command execution. If the command exits with a code of 0, it returns the content of stdout. If the exit code is non-zero, -it returns the content of stderr. The output is parsed as JSON if -possible; otherwise, it is returned as a raw string. +it returns the content of stderr. diff --git a/docs/for-tool-providers.md b/docs/for-tool-providers.md index 0174328..87ff739 100644 --- a/docs/for-tool-providers.md +++ b/docs/for-tool-providers.md @@ -24,11 +24,6 @@ As a tool provider, you'll create a **UTCP Manual** - a standardized description { "manual_version": "1.0.0", "utcp_version": "1.0.1", - "info": { - "title": "My API Tools", - "version": "1.0.0", - "description": "Collection of useful API tools" - }, "tools": [ { "name": "get_user", @@ -51,10 +46,7 @@ As a tool provider, you'll create a **UTCP Manual** - a standardized description "tool_call_template": { "call_template_type": "http", "url": "https://api.example.com/users/${user_id}", - "http_method": "GET", - "headers": { - "Authorization": "Bearer ${API_TOKEN}" - } + "http_method": "GET" } } ] @@ -84,7 +76,7 @@ Your existing API endpoints remain unchanged. For example: ## Manual Structure -The UTCP manual follows a standardized structure that defines your tools and how to call them. For complete field specifications, data types, and validation rules, see the [UTCP Manual API Reference](../api/core/utcp/data/utcp_manual.md). +The UTCP manual follows a standardized structure that defines your tools and how to call them. For complete field specifications, data types, and validation rules, see the [UTCP Manual API Reference](./api/core/utcp/data/utcp_manual.md). ### Key Components @@ -96,7 +88,7 @@ The UTCP manual follows a standardized structure that defines your tools and how ### Tool Definition -Tools are defined in your UTCP manual with their input parameters, call instructions, and optional metadata. For complete field specifications, see the [Tool API Reference](../api/core/utcp/data/tool.md). +Tools are defined in your UTCP manual with their input parameters, call instructions, and optional metadata. For complete field specifications, see the [Tool API Reference](./api/core/utcp/data/tool.md). ## Communication Protocol Plugins @@ -111,23 +103,28 @@ Most common for REST APIs: "inputs": { "type": "object", "properties": { - "name": {"type": "string"}, - "email": {"type": "string", "format": "email"} + "user_fields": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string", "format": "email"} + }, + "required": ["name", "email"] + } }, - "required": ["name", "email"] + "required": ["user_fields"] }, "tool_call_template": { "call_template_type": "http", "url": "https://api.example.com/users", "http_method": "POST", - "headers": { - "Content-Type": "application/json", - "Authorization": "Bearer ${API_TOKEN}" + "auth": { + "auth_type": "api_key", + "api_key": "Bearer ${API_TOKEN}", + "var_name": "Authorization", + "location": "header" }, - "body": { - "name": "${name}", - "email": "${email}" - } + "body_field": "user_fields" } } ``` @@ -138,8 +135,8 @@ For command-line applications: ```json { - "name": "git_status", - "description": "Get git repository status", + "name": "git_analysis", + "description": "Analyze git repository status and commit history", "inputs": { "type": "object", "properties": { @@ -149,35 +146,20 @@ For command-line applications: }, "tool_call_template": { "call_template_type": "cli", - "command": "git", - "args": ["status", "--porcelain"], - "working_directory": "${repo_path}" - } -} -``` - -### WebSocket Tools - -For real-time communication: - -```json -{ - "name": "subscribe_updates", - "description": "Subscribe to real-time updates", - "inputs": { - "type": "object", - "properties": { - "channel": {"type": "string"} - }, - "required": ["channel"] - }, - "tool_call_template": { - "call_template_type": "websocket", - "url": "wss://api.example.com/ws", - "message": { - "action": "subscribe", - "channel": "${channel}" - } + "commands": [ + { + "command": "cd UTCP_ARG_repo_path_UTCP_END", + "append_to_final_output": false + }, + { + "command": "git status --porcelain", + "append_to_final_output": false + }, + { + "command": "echo \"Status: $CMD_1_OUTPUT, Files changed: $(echo \"$CMD_1_OUTPUT\" | wc -l)\"", + "append_to_final_output": true + } + ] } } ``` @@ -199,11 +181,13 @@ For real-time communication: ### Bearer Token +To use a Bearer Token, you can use the `api_key` authentication type. + ```json { "auth": { "auth_type": "api_key", - "api_key": "${ACCESS_TOKEN}", + "api_key": "Bearer ${ACCESS_TOKEN}", "var_name": "Authorization", "location": "header" } @@ -345,8 +329,11 @@ The UTCP manual describes how to call your existing endpoints: "call_template_type": "http", "url": "https://api.example.com/users/${user_id}", "http_method": "GET", - "headers": { - "Authorization": "Bearer ${API_TOKEN}" + "auth": { + "auth_type": "api_key", + "api_key": "Bearer ${API_TOKEN}", + "var_name": "Authorization", + "location": "header" } } }, @@ -365,14 +352,13 @@ The UTCP manual describes how to call your existing endpoints: "call_template_type": "http", "url": "https://api.example.com/users", "http_method": "POST", - "headers": { - "Content-Type": "application/json", - "Authorization": "Bearer ${API_TOKEN}" + "auth": { + "auth_type": "api_key", + "api_key": "Bearer ${API_TOKEN}", + "var_name": "Authorization", + "location": "header" }, - "body": { - "name": "${name}", - "email": "${email}" - } + "body_field": "user_data" } } ] diff --git a/docs/implementation.md b/docs/implementation.md index 175a7ea..c1aee3a 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -47,9 +47,11 @@ Create an HTTP endpoint that serves a UTCP manual: "call_template_type": "http", "url": "https://api.weather.com/current", "http_method": "GET", - "query_params": { - "q": "${location}", - "appid": "${WEATHER_API_KEY}" + "auth": { + "auth_type": "api_key", + "api_key": "${WEATHER_API_KEY}", + "var_name": "appid", + "location": "query" } } } @@ -134,13 +136,16 @@ Call templates define how to invoke tools using specific protocols: "call_template_type": "http", "url": "https://api.example.com/endpoint", "http_method": "POST", - "headers": { - "Content-Type": "application/json", - "Authorization": "Bearer ${API_TOKEN}" + "auth": { + "auth_type": "api_key", + "api_key": "${API_TOKEN}", + "var_name": "Authorization", + "location": "header" }, - "body": { - "data": "${input_data}" - } + "body_field": "body" +} + +Tool arguments not used in the URL path or headers will be sent as query parameters for GET requests, or in the request body for POST/PUT/PATCH requests. The `body_field` specifies which tool argument contains the data for the request body. } ``` @@ -148,12 +153,20 @@ Call templates define how to invoke tools using specific protocols: ```json { "call_template_type": "cli", - "command": "python", - "args": ["script.py", "${input}"], - "working_directory": "/app", - "env": { + "commands": [ + { + "command": "cd /app", + "append_to_final_output": false + }, + { + "command": "python script.py UTCP_ARG_input_UTCP_END", + "append_to_final_output": true + } + ], + "env_vars": { "PYTHONPATH": "/app/lib" - } + }, + "working_dir": "/app" } ``` @@ -233,14 +246,13 @@ manual_call_templates: - name: weather_service call_template_type: http url: https://api.weather.com/utcp - http_method: GET variables: WEATHER_API_KEY: your-api-key -variable_loaders: - - type: dot_env - file_path: .env +load_variables_from: + - variable_loader_type: dotenv + env_file_path: .env ``` #### Programmatic Configuration diff --git a/docs/index.md b/docs/index.md index c3860a1..2355a3e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -63,8 +63,15 @@ Add a discovery endpoint to your existing API: "tool_call_template": { "call_template_type": "http", "url": "http://localhost:8000/weather", - "http_method": "GET", - "query_params": {"location": "${location}"} + "http_method": "GET" + } + }], + "auth": { + "auth_type": "api_key", + "api_key": "${WEATHER_API_KEY}", + "var_name": "appid", + "location": "query" + } } }] } @@ -127,7 +134,7 @@ UTCP supports multiple communication protocols through plugins: | **[HTTP](./protocols/http.md)** | REST APIs, webhooks | `utcp-http` | ✅ Stable | | **[WebSocket](./protocols/websocket.md)** | Real-time communication | `utcp-websocket` | ✅ Stable | | **[CLI](./protocols/cli.md)** | Command-line tools | `utcp-cli` | ✅ Stable | -| **[Server-Sent Events](./protocols/sse.md)** | Streaming data | `utcp-http` | ✅ Stable | +| **[Server-Sent Events](./protocols/streamable-http.md)** | Streaming data | `utcp-http` | ✅ Stable | | **[Text Files](./protocols/text.md)** | File reading | `utcp-text` | ✅ Stable | | **[MCP](./protocols/mcp.md)** | MCP interoperability | `utcp-mcp` | ✅ Stable | diff --git a/docs/migration-v0.1-to-v1.0.md b/docs/migration-v0.1-to-v1.0.md index 1306f9f..eb50de7 100644 --- a/docs/migration-v0.1-to-v1.0.md +++ b/docs/migration-v0.1-to-v1.0.md @@ -98,11 +98,9 @@ manual_call_templates: command: cat args: ["${filename}"] -variable_loaders: - - loader_type: env - prefix: UTCP_ - - loader_type: dotenv - file_path: .env +load_variables_from: + - variable_loader_type: dotenv + env_file_path: .env ``` ## Manual Format Migration @@ -169,9 +167,13 @@ variable_loaders: "call_template_type": "http", "url": "https://api.weather.com/current", "http_method": "GET", - "query_params": { - "location": "${location}" - } + }, + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "appid", + "location": "query" + } } } ] @@ -202,7 +204,7 @@ variable_loaders: "url": "https://api.example.com/endpoint", "http_method": "POST", "headers": {"Authorization": "Bearer ${TOKEN}"}, - "body": {"data": "${input}"}, + "body_field": "body", "auth": { "auth_type": "api_key", "api_key": "${TOKEN}", @@ -230,11 +232,18 @@ variable_loaders: ```json { "call_template_type": "cli", - "command": "python", - "args": ["script.py", "${input}"], - "working_directory": "/app", - "timeout": 30, - "environment": { + "commands": [ + { + "command": "cd /app", + "append_to_final_output": false + }, + { + "command": "python script.py UTCP_ARG_input_UTCP_END", + "append_to_final_output": true + } + ], + "working_dir": "/app", + "env_vars": { "PYTHONPATH": "/app/lib" } } diff --git a/docs/protocols/cli.md b/docs/protocols/cli.md index 2a4257e..f0ac213 100644 --- a/docs/protocols/cli.md +++ b/docs/protocols/cli.md @@ -6,7 +6,7 @@ sidebar_position: 4 # CLI Protocol -The CLI protocol plugin (`utcp-cli`) enables UTCP to execute command-line tools and scripts. This is particularly useful for wrapping existing CLI applications and making them available to AI agents. +The CLI protocol plugin (`utcp-cli`) enables UTCP to execute multi-command workflows and scripts. This protocol supports sequential command execution with state preservation and cross-platform compatibility. ## Installation @@ -18,28 +18,89 @@ pip install utcp-cli npm install @utcp/cli ``` +## Key Features + +- **Multi-Command Execution**: Execute multiple commands sequentially in a single subprocess +- **State Preservation**: Directory changes and environment persist between commands +- **Cross-Platform Script Generation**: PowerShell on Windows, Bash on Unix/Linux/macOS +- **Flexible Output Control**: Choose which command outputs to include in final result +- **Argument Substitution**: `UTCP_ARG_argname_UTCP_END` placeholder system +- **Output Referencing**: Access previous command outputs with `$CMD_0_OUTPUT`, `$CMD_1_OUTPUT` +- **Environment Variables**: Secure credential and configuration passing + ## Call Template Structure ```json { "call_template_type": "cli", - "command": "curl", - "args": [ - "-X", "GET", - "-H", "Authorization: Bearer ${API_TOKEN}", - "https://api.example.com/data" + "commands": [ + { + "command": "cd UTCP_ARG_target_dir_UTCP_END", + "append_to_final_output": false + }, + { + "command": "git status --porcelain", + "append_to_final_output": false + }, + { + "command": "echo \"Status: $CMD_1_OUTPUT, Files: $(echo \"$CMD_1_OUTPUT\" | wc -l)\"", + "append_to_final_output": true + } ], - "working_directory": "/tmp", - "environment": { - "API_TOKEN": "${API_TOKEN}" - }, - "timeout": 30 + "working_dir": "/tmp", + "env_vars": { + "GIT_AUTHOR_NAME": "UTCP Bot" + } } ``` ## Configuration Options -The CLI call template allows you to execute command-line tools and scripts. For complete field specifications and validation rules, see the [CLI Call Template API Reference](../api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md). +### Required Fields + +- **`call_template_type`**: Must be `"cli"` +- **`commands`**: Array of `CommandStep` objects defining the sequence of commands + +### Optional Fields + +- **`working_dir`**: Working directory for command execution +- **`env_vars`**: Environment variables to set for all commands + +### CommandStep Object + +- **`command`**: Command string with `UTCP_ARG_argname_UTCP_END` placeholders +- **`append_to_final_output`**: Whether to include this command's output in the final result (defaults to `false` for all except the last command) + +For complete field specifications and validation rules, see the [CLI Call Template API Reference](../api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md). + +## Argument Substitution + +Use `UTCP_ARG_argname_UTCP_END` placeholders in command strings: + +```json +{ + "command": "git clone UTCP_ARG_repo_url_UTCP_END UTCP_ARG_target_dir_UTCP_END" +} +``` + +## Output Referencing + +Reference previous command outputs using `$CMD_N_OUTPUT` variables: + +```json +{ + "commands": [ + { + "command": "git status --porcelain", + "append_to_final_output": false + }, + { + "command": "echo \"Changes detected: $CMD_0_OUTPUT\"", + "append_to_final_output": true + } + ] +} +``` ## Security Considerations @@ -67,9 +128,17 @@ Always validate and sanitize inputs to prevent command injection: }, "tool_call_template": { "call_template_type": "cli", - "command": "cat", - "args": ["${filename}"], - "working_directory": "/safe/directory" + "commands": [ + { + "command": "cd /safe/directory", + "append_to_final_output": false + }, + { + "command": "cat UTCP_ARG_filename_UTCP_END", + "append_to_final_output": true + } + ], + "working_dir": "/safe/directory" } } ``` @@ -81,12 +150,11 @@ Consider running CLI tools in sandboxed environments: ```json { "call_template_type": "cli", - "command": "docker", - "args": [ - "run", "--rm", "--read-only", - "-v", "/safe/data:/data:ro", - "alpine:latest", - "cat", "/data/${filename}" + "commands": [ + { + "command": "docker run --rm --read-only -v /safe/data:/data:ro alpine:latest cat /data/UTCP_ARG_filename_UTCP_END", + "append_to_final_output": true + } ] } ``` @@ -105,131 +173,182 @@ Consider running CLI tools in sandboxed environments: }, "tool_call_template": { "call_template_type": "cli", - "command": "uname", - "args": ["-a"] + "commands": [ + { + "command": "uname -a", + "append_to_final_output": true + } + ] } } ``` -### File Operations +### Multi-Step File Analysis ```json { - "name": "list_directory", - "description": "List files in a directory", + "name": "analyze_directory", + "description": "Analyze files in a directory", "inputs": { "type": "object", "properties": { "path": { "type": "string", - "description": "Directory path to list" + "description": "Directory path to analyze" } }, "required": ["path"] }, "tool_call_template": { "call_template_type": "cli", - "command": "ls", - "args": ["-la", "${path}"], - "timeout": 10 + "commands": [ + { + "command": "cd UTCP_ARG_path_UTCP_END", + "append_to_final_output": false + }, + { + "command": "find . -type f | wc -l", + "append_to_final_output": false + }, + { + "command": "du -sh .", + "append_to_final_output": false + }, + { + "command": "echo \"Directory Analysis: $CMD_2_OUTPUT total size, $CMD_1_OUTPUT files\"", + "append_to_final_output": true + } + ] } } ``` -### Script Execution +### Git Workflow ```json { - "name": "run_analysis", - "description": "Run data analysis script", + "name": "git_analysis", + "description": "Analyze git repository", "inputs": { "type": "object", "properties": { - "input_file": {"type": "string"}, - "output_format": {"type": "string", "enum": ["json", "csv"]} + "repo_url": {"type": "string"}, + "target_dir": {"type": "string"} }, - "required": ["input_file"] + "required": ["repo_url", "target_dir"] }, "tool_call_template": { "call_template_type": "cli", - "command": "python", - "args": [ - "/scripts/analyze.py", - "--input", "${input_file}", - "--format", "${output_format}" + "commands": [ + { + "command": "git clone UTCP_ARG_repo_url_UTCP_END UTCP_ARG_target_dir_UTCP_END", + "append_to_final_output": false + }, + { + "command": "cd UTCP_ARG_target_dir_UTCP_END", + "append_to_final_output": false + }, + { + "command": "git log --oneline -10", + "append_to_final_output": true + }, + { + "command": "find . -name '*.py' | wc -l", + "append_to_final_output": false + }, + { + "command": "echo \"Repository has $CMD_3_OUTPUT Python files\"", + "append_to_final_output": true + } ], - "working_directory": "/workspace", - "environment": { - "PYTHONPATH": "/workspace/lib" - }, - "timeout": 300 + "env_vars": { + "GIT_AUTHOR_NAME": "UTCP Bot", + "GIT_AUTHOR_EMAIL": "bot@utcp.dev" + } } } ``` -### Git Operations +### Python Development Pipeline ```json { - "name": "git_status", - "description": "Get git repository status", + "name": "python_pipeline", + "description": "Run Python development pipeline", "inputs": { "type": "object", "properties": { - "repo_path": {"type": "string"} + "project_dir": {"type": "string"}, + "script_name": {"type": "string"} }, - "required": ["repo_path"] + "required": ["project_dir", "script_name"] }, "tool_call_template": { "call_template_type": "cli", - "command": "git", - "args": ["status", "--porcelain"], - "working_directory": "${repo_path}" + "commands": [ + { + "command": "cd UTCP_ARG_project_dir_UTCP_END", + "append_to_final_output": false + }, + { + "command": "python -m pip install -r requirements.txt", + "append_to_final_output": false + }, + { + "command": "python UTCP_ARG_script_name_UTCP_END", + "append_to_final_output": true + } + ], + "working_dir": "/workspace", + "env_vars": { + "PYTHONPATH": "/workspace/lib", + "VIRTUAL_ENV": "/workspace/venv" + } } } ``` ## Output Handling -The CLI protocol captures and returns: +The CLI protocol executes all commands in a single subprocess and returns the combined output based on `append_to_final_output` settings: -- **stdout**: Standard output as the primary result -- **stderr**: Standard error (included in error cases) -- **return_code**: Process exit code -- **execution_time**: Time taken to execute +- Commands with `append_to_final_output: true` contribute to the final result +- Commands with `append_to_final_output: false` are executed for side effects only +- If no `append_to_final_output` is specified, only the last command's output is returned +- JSON output is automatically parsed if detected ### Success Response ```json -{ - "stdout": "file1.txt\nfile2.txt\n", - "stderr": "", - "return_code": 0, - "execution_time": 0.123 -} +"Directory Analysis: 2.1G total size, 1247 files" ``` -### Error Response +### JSON Output Detection + +If output starts with `{` or `[`, it's automatically parsed as JSON: ```json { - "stdout": "", - "stderr": "ls: cannot access '/invalid/path': No such file or directory\n", - "return_code": 2, - "execution_time": 0.045 + "files": 1247, + "size": "2.1G", + "analysis_time": "2023-12-01T10:30:00Z" } ``` ## Environment Variables -Set environment variables for command execution: +Set environment variables for all commands: ```json { "call_template_type": "cli", - "command": "node", - "args": ["app.js"], - "environment": { + "commands": [ + { + "command": "node app.js", + "append_to_final_output": true + } + ], + "env_vars": { "NODE_ENV": "production", "API_KEY": "${API_KEY}", "PORT": "3000" @@ -239,73 +358,114 @@ Set environment variables for command execution: ## Working Directory -Specify the working directory for command execution: +Specify the initial working directory: ```json { "call_template_type": "cli", - "command": "make", - "args": ["build"], - "working_directory": "/project/src" + "commands": [ + { + "command": "make build", + "append_to_final_output": true + } + ], + "working_dir": "/project/src" } ``` -## Best Practices +## Cross-Platform Considerations -1. **Validate Inputs**: Always validate and sanitize user inputs -2. **Use Absolute Paths**: Prefer absolute paths for commands and files -3. **Set Timeouts**: Configure appropriate timeouts to prevent hanging -4. **Limit Permissions**: Run with minimal necessary permissions -5. **Sandbox Execution**: Use containers or chroot when possible -6. **Log Execution**: Log all command executions for audit trails -7. **Handle Errors**: Properly handle and report command failures +### Command Syntax -## Common Use Cases - -- **Development Tools**: Git, npm, pip, docker commands -- **System Administration**: File operations, process management -- **Data Processing**: Scripts for ETL, analysis, reporting -- **Build Systems**: Make, gradle, webpack execution -- **Testing**: Running test suites and validation scripts - -## Error Handling - -| Error Type | Description | Handling | -|------------|-------------|----------| -| Command Not Found | Command doesn't exist | Raise `CommandNotFoundError` | -| Permission Denied | Insufficient permissions | Raise `PermissionError` | -| Timeout | Command exceeded timeout | Raise `TimeoutError` | -| Non-zero Exit | Command failed | Include stderr in error | - -## Platform Considerations - -### Windows +Commands should use appropriate syntax for the target platform: +**Windows (PowerShell):** ```json { - "call_template_type": "cli", - "command": "cmd", - "args": ["/c", "dir", "${path}"], - "shell": true + "commands": [ + {"command": "Get-ChildItem UTCP_ARG_path_UTCP_END"}, + {"command": "Set-Location UTCP_ARG_dir_UTCP_END"} + ] } ``` -### Unix/Linux - +**Unix/Linux/macOS (Bash):** ```json { - "call_template_type": "cli", - "command": "ls", - "args": ["-la", "${path}"] + "commands": [ + {"command": "ls -la UTCP_ARG_path_UTCP_END"}, + {"command": "cd UTCP_ARG_dir_UTCP_END"} + ] } ``` -### Cross-platform +### Universal Commands +Some commands work across platforms: ```json { - "call_template_type": "cli", - "command": "python", - "args": ["-c", "import os; print(os.listdir('${path}'))"] + "commands": [ + {"command": "git status"}, + {"command": "python --version"}, + {"command": "node -v"} + ] } ``` + +## Best Practices + +1. **Validate Inputs**: Always validate and sanitize user inputs using JSON Schema patterns +2. **Use Absolute Paths**: Prefer absolute paths for commands and files when possible +3. **Control Output**: Use `append_to_final_output` to control which command outputs are returned +4. **Reference Previous Output**: Use `$CMD_N_OUTPUT` to reference previous command results +5. **Limit Permissions**: Run with minimal necessary permissions +6. **Sandbox Execution**: Use containers or chroot when possible +7. **Handle Cross-Platform**: Consider platform-specific command syntax +8. **Environment Variables**: Use `env_vars` for secure credential passing + +## Common Use Cases + +- **Multi-step Builds**: setup → compile → test → package +- **Git Workflows**: clone → analyze → commit → push +- **Data Pipelines**: fetch → transform → validate → output +- **File Operations**: navigate → search → process → report +- **Development Tools**: install dependencies → run tests → generate docs +- **System Administration**: check status → backup → cleanup → verify + +## Error Handling + +| Error Type | Description | Handling | +|------------|-------------|---------| +| Missing Arguments | Required UTCP_ARG placeholder not provided | Validation error | +| Command Not Found | Command doesn't exist | Script execution error | +| Permission Denied | Insufficient permissions | Script execution error | +| Timeout | Script exceeded timeout | Async timeout error | +| Non-zero Exit | Script failed | Return stderr output | + +## Testing CLI Tools + +```python +import pytest +from utcp.utcp_client import UtcpClient + +@pytest.mark.asyncio +async def test_multi_command_cli_tool(): + client = await UtcpClient.create(config={ + "manual_call_templates": [{ + "name": "test_cli", + "call_template_type": "cli", + "commands": [ + { + "command": "echo UTCP_ARG_message_UTCP_END", + "append_to_final_output": false + }, + { + "command": "echo \"Previous: $CMD_0_OUTPUT\"" + } + ] + }] + }) + + result = await client.call_tool("test_cli.echo_chain", {"message": "hello"}) + assert "Previous: hello" in result +``` diff --git a/docs/protocols/index.md b/docs/protocols/index.md index ae85510..9a4f701 100644 --- a/docs/protocols/index.md +++ b/docs/protocols/index.md @@ -1,7 +1,7 @@ --- -id: index -title: Communication Protocol Plugins -sidebar_position: 3 +id: protocols +title: Protocols +sidebar_position: 4 --- # Communication Protocol Plugins diff --git a/docs/protocols/sse.md b/docs/protocols/sse.md index 6493892..93618e9 100644 --- a/docs/protocols/sse.md +++ b/docs/protocols/sse.md @@ -40,7 +40,7 @@ npm install @utcp/http ## Configuration Options -The Server-Sent Events (SSE) call template enables real-time streaming data from HTTP endpoints. For complete field specifications and validation rules, see the [SSE Call Template API Reference](../api/plugins/communication_protocols/sse/src/utcp_sse/sse_call_template.md). +The Server-Sent Events (SSE) call template enables real-time streaming data from HTTP endpoints. For complete field specifications and validation rules, see the [SSE Call Template API Reference](../api/plugins/communication_protocols/http/src/utcp_http/sse_call_template.md). | `reconnect` | boolean | Auto-reconnect on connection loss (default: true) | | `reconnect_delay` | number | Delay between reconnection attempts (default: 3) | diff --git a/docs/protocols/streamable-http.md b/docs/protocols/streamable-http.md new file mode 100644 index 0000000..3ed9e6e --- /dev/null +++ b/docs/protocols/streamable-http.md @@ -0,0 +1,105 @@ +--- +id: streamable-http +title: Streamable HTTP Protocol +sidebar_position: 2 +--- + +# Streamable HTTP Protocol + +The Streamable HTTP protocol plugin (`utcp-http`) enables UTCP to handle large HTTP responses by streaming them in chunks. This is ideal for tools that return large datasets, files, or progressive results that don't fit into a single response payload. It leverages HTTP Chunked Transfer Encoding. + +## Call Template Structure + +```json +{ + "call_template_type": "streamable_http", + "url": "https://api.example.com/download/large-file", + "http_method": "GET", + "headers": { + "Accept": "application/octet-stream" + }, + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "Authorization", + "location": "header" + }, + "chunk_size": 8192, + "timeout": 300000, + "body_field": "request_data", + "header_fields": ["custom_header_arg"] +} +``` + +## Configuration Options + +The Streamable HTTP call template provides a way to configure streaming from HTTP endpoints. + +| Option | Type | Default | Description | +|---|---|---|---| +| `url` | string | **Required** | The streaming HTTP endpoint URL. Supports path parameters like `/users/{user_id}`. | +| `http_method` | string | `GET` | The HTTP method to use. Supported methods are `GET` and `POST`. | +| `content_type` | string | `application/octet-stream` | The `Content-Type` header to set for the request, especially when `body_field` is used. | +| `chunk_size` | integer | `4096` | The size of each data chunk in bytes to read from the stream. | +| `timeout` | integer | `60000` | Request timeout in milliseconds. | +| `headers` | object | `null` | Optional static headers to include in every request. | +| `auth` | object | `null` | Optional authentication configuration. See [HTTP Authentication](./http.md#authentication-methods). | +| `body_field` | string | `null` | The name of a single tool argument to be sent as the HTTP request body. | +| `header_fields` | array | `null` | A list of tool argument names to be sent as request headers. | + +## Response Handling + +The protocol processes the incoming stream based on the `Content-Type` header of the response: + +- **`application/x-ndjson`**: The stream is parsed as Newline Delimited JSON. Each line is yielded as a separate JSON object. +- **`application/octet-stream`**: The stream is yielded in binary chunks of the specified `chunk_size`. +- **`application/json`**: The entire response is buffered and yielded as a single JSON object. This is for endpoints that stream a single, large JSON document. +- **Other Types**: For any other `Content-Type`, the response is treated as a binary stream and yielded in chunks of `chunk_size`. + +## Authentication + +Streamable HTTP supports the same authentication methods as the standard HTTP protocol, including API Key, Basic Auth, and OAuth2. The configuration is identical. + +For more details, see the [HTTP Authentication Methods](./http.md#authentication-methods) documentation. + +## Variable Substitution + +Path parameters, query parameters, headers, and authentication fields all support variable substitution from tool arguments and environment variables, following the same syntax as the standard HTTP protocol. + +Example: +```json +{ + "url": "https://api.example.com/files/{file_id}/download", + "headers": { + "Authorization": "Bearer ${ACCESS_TOKEN}" + } +} +``` +Here, `{file_id}` is substituted from a tool argument, and `${ACCESS_TOKEN}` is substituted from an environment or configuration variable. + +For more details, see the [HTTP Variable Substitution](./http.md#variable-substitution) documentation. + +## Security Considerations + +- **SSL/TLS**: It is strongly recommended to use `https://` endpoints to protect data in transit. The implementation enforces HTTPS or localhost connections by default. +- **Authentication**: Never hardcode credentials. Use variable substitution to inject secrets from a secure source (e.g., environment variables). +- **Input Sanitization**: Ensure that any arguments used in URL path parameters or query strings are properly validated and sanitized to prevent injection attacks. + +## Error Handling + +Errors are handled similarly to the standard HTTP protocol: + +| Status Code | Error Type | Description | +|---|---|---| +| 400 | Bad Request | Invalid request parameters or body. | +| 401 | Unauthorized | Authentication failed or is required. | +| 403 | Forbidden | The authenticated user does not have permission. | +| 404 | Not Found | The requested resource or endpoint does not exist. | +| 5xx | Server Error | An error occurred on the server side. | + +Connection errors, timeouts, and other network issues will also be raised as exceptions. + +## Related Protocols + +- [HTTP](./http.md) - For standard request/response interactions. +- [Server-Sent Events (SSE)](./sse.md) - For unidirectional, real-time event streams from server to client. \ No newline at end of file diff --git a/docs/protocols/websocket.md b/docs/protocols/websocket.md index 2978390..5ce7789 100644 --- a/docs/protocols/websocket.md +++ b/docs/protocols/websocket.md @@ -41,7 +41,7 @@ npm install @utcp/websocket ## Configuration Options -The WebSocket call template enables real-time communication with WebSocket servers. For complete field specifications and validation rules, see the [WebSocket Call Template API Reference](../api/plugins/communication_protocols/websocket/src/utcp_websocket/websocket_call_template.md). +The WebSocket call template enables real-time communication with WebSocket servers. For complete field specifications and validation rules, see the WebSocket Call Template API Reference (WIP). | `expected_responses` | number | Number of expected response messages (default: 1) | | `ping_interval` | number | Ping interval in seconds (default: 30) | diff --git a/docs/security.md b/docs/security.md index c23f5c0..fbf10f6 100644 --- a/docs/security.md +++ b/docs/security.md @@ -151,15 +151,21 @@ CLI execution poses significant security risks. Use with extreme caution. ```json { "call_template_type": "cli", - "command": "/usr/local/bin/safe-script", - "args": ["--input", "${sanitized_input}"], - "working_directory": "/safe/sandbox", - "environment": { + "commands": [ + { + "command": "cd /safe/sandbox", + "append_to_final_output": false + }, + { + "command": "/usr/local/bin/safe-script --input UTCP_ARG_sanitized_input_UTCP_END", + "append_to_final_output": true + } + ], + "working_dir": "/safe/sandbox", + "env_vars": { "PATH": "/usr/local/bin:/usr/bin", "HOME": "/tmp/sandbox" - }, - "timeout": 30, - "allowed_exit_codes": [0] + } } ``` @@ -209,10 +215,7 @@ CLI execution poses significant security risks. Use with extreme caution. ```json { "call_template_type": "text", - "file_path": "/safe/data/${filename}", - "max_size": 1048576, - "allowed_paths": ["/safe/data/"], - "encoding": "utf-8" + "file_path": "/safe/data/${filename}" } ``` @@ -237,13 +240,13 @@ CLI execution poses significant security risks. Use with extreme caution. ```json { "call_template_type": "mcp", - "server_config": { - "command": "/usr/local/bin/mcp-server", - "args": ["--config", "/safe/config.json"], - "env": { - "MCP_LOG_LEVEL": "INFO" - }, - "timeout": 60 + "config": { + "mcpServers": { + "server_name": { + "transport": "stdio", + "command": ["python", "-m", "my_mcp_server"] + } + } } } ``` @@ -391,7 +394,7 @@ Implement comprehensive security testing: Implement automated security validation: - **Protocol Security**: Verify HTTPS usage instead of HTTP for web requests - **Credential Detection**: Check for hardcoded passwords, secrets, or API keys -- **Variable Validation**: Ensure proper variable substitution patterns (${variable}) +- **Variable Validation**: Ensure proper variable substitution patterns ($\{variable\}) - **CLI Security**: Validate command-line tools use absolute paths and safe commands - **URL Validation**: Check for suspicious or malformed URLs - **Configuration Review**: Automated scanning of UTCP manuals for security issues diff --git a/docs/utcp-vs-mcp.md b/docs/utcp-vs-mcp.md index 275ab7d..bc670ff 100644 --- a/docs/utcp-vs-mcp.md +++ b/docs/utcp-vs-mcp.md @@ -127,8 +127,12 @@ MCP requires building servers that wrap your tools: "name": "cli_tool", "tool_call_template": { "call_template_type": "cli", - "command": "git", - "args": ["status"] + "commands": [ + { + "command": "git status --porcelain", + "append_to_final_output": true + } + ] } } ] @@ -332,7 +336,7 @@ Monitoring: Additional monitoring stack needed #### UTCP Approach **E-commerce API with UTCP:** -- Keep existing product API endpoints unchanged (GET /products/{product_id}) +- Keep existing product API endpoints unchanged (GET /products/\{product_id\}) - Add single UTCP discovery endpoint (GET /utcp) - Return UTCP manual describing available tools and how to call them - Tools directly reference existing API endpoints with proper parameters diff --git a/sidebars.ts b/sidebars.ts index 7ff5021..ea4924a 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -14,54 +14,7 @@ import type {SidebarsConfig} from '@docusaurus/plugin-content-docs'; */ const sidebars: SidebarsConfig = { // Main documentation sidebar - tutorialSidebar: [ - 'index', - 'for-tool-providers', - { - type: 'category', - label: 'Communication Protocols', - items: [ - 'protocols/index', - 'protocols/http', - 'protocols/websocket', - 'protocols/sse', - 'protocols/cli', - 'protocols/text', - 'protocols/mcp', - ], - }, - 'implementation', - 'security', - 'utcp-vs-mcp', - 'migration-v0.1-to-v1.0', - { - type: 'category', - label: 'API Reference', - items: [ - 'api/index', - { - type: 'category', - label: 'Core', - items: [ - { - type: 'autogenerated', - dirName: 'api/core', - }, - ], - }, - { - type: 'category', - label: 'Plugins', - items: [ - { - type: 'autogenerated', - dirName: 'api/plugins', - }, - ], - }, - ], - }, - ], + tutorialSidebar: [{type: 'autogenerated', dirName: '.'}], }; export default sidebars; From 466a790d43e2a198466e897bc548898a49c9f89a Mon Sep 17 00:00:00 2001 From: Razvan Radulescu <43811028+h3xxit@users.noreply.github.com> Date: Sun, 7 Sep 2025 17:42:16 +0200 Subject: [PATCH 22/22] Update 1.0 versioned docs --- .../utcp_serializer_validation_error.md | 16 +- .../filter_dict_post_processor.md | 49 ++ .../limit_strings_post_processor.md | 48 ++ .../core/utcp/implementations/tag_search.md | 33 +- .../api/core/utcp/plugins/plugin_loader.md | 6 + versioned_docs/version-1.0/api/index.md | 18 +- .../cli/src/utcp_cli/cli_call_template.md | 194 +++++- .../utcp_cli/cli_communication_protocol.md | 85 +-- .../http/src/utcp_http/http_call_template.md | 75 +++ .../http/src/utcp_http/openapi_converter.md | 63 +- .../mcp/src/utcp_mcp/mcp_call_template.md | 98 +++ .../version-1.0/for-tool-providers.md | 580 ++++++++++++++++++ versioned_docs/version-1.0/implementation.md | 396 ++++++++++++ versioned_docs/version-1.0/index.md | 323 ++++++---- .../version-1.0/migration-v0.1-to-v1.0.md | 465 ++++++++++++++ versioned_docs/version-1.0/protocols/cli.md | 471 ++++++++++++++ versioned_docs/version-1.0/protocols/http.md | 236 +++++++ versioned_docs/version-1.0/protocols/index.md | 93 +++ versioned_docs/version-1.0/protocols/mcp.md | 266 ++++++++ versioned_docs/version-1.0/protocols/sse.md | 386 ++++++++++++ .../version-1.0/protocols/streamable-http.md | 105 ++++ versioned_docs/version-1.0/protocols/text.md | 437 +++++++++++++ .../version-1.0/protocols/websocket.md | 355 +++++++++++ versioned_docs/version-1.0/security.md | 454 +++++++++++--- versioned_docs/version-1.0/utcp-vs-mcp.md | 507 +++++++++++++-- 25 files changed, 5435 insertions(+), 324 deletions(-) create mode 100644 versioned_docs/version-1.0/api/core/utcp/implementations/post_processors/filter_dict_post_processor.md create mode 100644 versioned_docs/version-1.0/api/core/utcp/implementations/post_processors/limit_strings_post_processor.md create mode 100644 versioned_docs/version-1.0/for-tool-providers.md create mode 100644 versioned_docs/version-1.0/implementation.md create mode 100644 versioned_docs/version-1.0/migration-v0.1-to-v1.0.md create mode 100644 versioned_docs/version-1.0/protocols/cli.md create mode 100644 versioned_docs/version-1.0/protocols/http.md create mode 100644 versioned_docs/version-1.0/protocols/index.md create mode 100644 versioned_docs/version-1.0/protocols/mcp.md create mode 100644 versioned_docs/version-1.0/protocols/sse.md create mode 100644 versioned_docs/version-1.0/protocols/streamable-http.md create mode 100644 versioned_docs/version-1.0/protocols/text.md create mode 100644 versioned_docs/version-1.0/protocols/websocket.md diff --git a/versioned_docs/version-1.0/api/core/utcp/exceptions/utcp_serializer_validation_error.md b/versioned_docs/version-1.0/api/core/utcp/exceptions/utcp_serializer_validation_error.md index 4dcaaec..378f221 100644 --- a/versioned_docs/version-1.0/api/core/utcp/exceptions/utcp_serializer_validation_error.md +++ b/versioned_docs/version-1.0/api/core/utcp/exceptions/utcp_serializer_validation_error.md @@ -9,6 +9,20 @@ sidebar_label: utcp_serializer_validation_error ### class UtcpSerializerValidationError {#utcpserializervalidationerror} -*No class documentation available* +
+Documentation + +Exception raised when a serializer validation fails. + +Thrown by serializers when they cannot validate or convert data structures +due to invalid format, missing required fields, or type mismatches. +Contains the original validation error details for debugging. + + +**Usage** + +Typically caught when loading configuration files or processing +external data that doesn't conform to UTCP specifications. +
--- diff --git a/versioned_docs/version-1.0/api/core/utcp/implementations/post_processors/filter_dict_post_processor.md b/versioned_docs/version-1.0/api/core/utcp/implementations/post_processors/filter_dict_post_processor.md new file mode 100644 index 0000000..6193415 --- /dev/null +++ b/versioned_docs/version-1.0/api/core/utcp/implementations/post_processors/filter_dict_post_processor.md @@ -0,0 +1,49 @@ +--- +title: filter_dict_post_processor +sidebar_label: filter_dict_post_processor +--- + +# filter_dict_post_processor + +**File:** `core/src/utcp/implementations/post_processors/filter_dict_post_processor.py` + +### class FilterDictPostProcessor ([ToolPostProcessor](./../../interfaces/tool_post_processor.md#toolpostprocessor)) {#filterdictpostprocessor} + +
+Documentation + +Post-processor that filters dictionary keys from tool results. + +Provides flexible filtering capabilities to include or exclude specific keys +from dictionary results, with support for nested dictionaries and lists. +Can be configured to apply filtering only to specific tools or manuals. + + +**Attributes** + +- **`tool_post_processor_type`**: Always "filter_dict" for this processor. +- **`exclude_keys`**: List of keys to remove from dictionary results. +- **`only_include_keys`**: List of keys to keep in dictionary results (all others removed). +- **`exclude_tools`**: List of tool names to skip processing for. +- **`only_include_tools`**: List of tool names to process (all others skipped). +- **`exclude_manuals`**: List of manual names to skip processing for. +- **`only_include_manuals`**: List of manual names to process (all others skipped). +
+ +#### Fields: + +- tool_post_processor_type: Literal['filter_dict'] +- exclude_keys: Optional[List[str]] +- only_include_keys: Optional[List[str]] +- exclude_tools: Optional[List[str]] +- only_include_tools: Optional[List[str]] +- exclude_manuals: Optional[List[str]] +- only_include_manuals: Optional[List[str]] + +--- + +### class FilterDictPostProcessorConfigSerializer ([Serializer](./../../interfaces/serializer.md#serializer)[FilterDictPostProcessor]) {#filterdictpostprocessorconfigserializer} + +*No class documentation available* + +--- diff --git a/versioned_docs/version-1.0/api/core/utcp/implementations/post_processors/limit_strings_post_processor.md b/versioned_docs/version-1.0/api/core/utcp/implementations/post_processors/limit_strings_post_processor.md new file mode 100644 index 0000000..f524d48 --- /dev/null +++ b/versioned_docs/version-1.0/api/core/utcp/implementations/post_processors/limit_strings_post_processor.md @@ -0,0 +1,48 @@ +--- +title: limit_strings_post_processor +sidebar_label: limit_strings_post_processor +--- + +# limit_strings_post_processor + +**File:** `core/src/utcp/implementations/post_processors/limit_strings_post_processor.py` + +### class LimitStringsPostProcessor ([ToolPostProcessor](./../../interfaces/tool_post_processor.md#toolpostprocessor)) {#limitstringspostprocessor} + +
+Documentation + +Post-processor that limits the length of string values in tool results. + +Truncates string values to a specified maximum length to prevent +excessively large responses. Processes nested dictionaries and lists +recursively. Can be configured to apply limiting only to specific +tools or manuals. + + +**Attributes** + +- **`tool_post_processor_type`**: Always "limit_strings" for this processor. +- **`limit`**: Maximum length for string values (default: 10000 characters). +- **`exclude_tools`**: List of tool names to skip processing for. +- **`only_include_tools`**: List of tool names to process (all others skipped). +- **`exclude_manuals`**: List of manual names to skip processing for. +- **`only_include_manuals`**: List of manual names to process (all others skipped). +
+ +#### Fields: + +- tool_post_processor_type: Literal['limit_strings'] +- limit: int +- exclude_tools: Optional[List[str]] +- only_include_tools: Optional[List[str]] +- exclude_manuals: Optional[List[str]] +- only_include_manuals: Optional[List[str]] + +--- + +### class LimitStringsPostProcessorConfigSerializer ([Serializer](./../../interfaces/serializer.md#serializer)[LimitStringsPostProcessor]) {#limitstringspostprocessorconfigserializer} + +*No class documentation available* + +--- diff --git a/versioned_docs/version-1.0/api/core/utcp/implementations/tag_search.md b/versioned_docs/version-1.0/api/core/utcp/implementations/tag_search.md index af032ab..558045e 100644 --- a/versioned_docs/version-1.0/api/core/utcp/implementations/tag_search.md +++ b/versioned_docs/version-1.0/api/core/utcp/implementations/tag_search.md @@ -9,7 +9,38 @@ sidebar_label: tag_search ### class TagAndDescriptionWordMatchStrategy ([ToolSearchStrategy](./../interfaces/tool_search_strategy.md#toolsearchstrategy)) {#taganddescriptionwordmatchstrategy} -*No class documentation available* +
+Documentation + +Tag and description word match strategy. + + +**Implements A Weighted Scoring System That Matches Tools Based On** + +1. Tag matches (higher weight) +2. Description word matches (lower weight) + +The strategy normalizes queries to lowercase, extracts words using regex, +and calculates relevance scores for each tool. Results are sorted by +score in descending order. + + + +**Attributes** + +- **`tool_search_strategy_type`**: Always "tag_and_description_word_match". +- **`description_weight`**: Weight multiplier for description word matches (default: 1.0). +- **`tag_weight`**: Weight multiplier for tag matches (default: 3.0). + + + +**Scoring Algorithm** + +- Each matching tag contributes tag_weight points +- Each matching description word contributes description_weight points +- Tools with higher scores are ranked first +- Tools with zero score are included in results (ranked last) +
#### Fields: diff --git a/versioned_docs/version-1.0/api/core/utcp/plugins/plugin_loader.md b/versioned_docs/version-1.0/api/core/utcp/plugins/plugin_loader.md index 8e8e9c1..3ac6524 100644 --- a/versioned_docs/version-1.0/api/core/utcp/plugins/plugin_loader.md +++ b/versioned_docs/version-1.0/api/core/utcp/plugins/plugin_loader.md @@ -7,6 +7,12 @@ sidebar_label: plugin_loader **File:** `core/src/utcp/plugins/plugin_loader.py` +### Function _load_plugins() {#_load_plugins} + +*No function documentation available* + +--- + ### Function ensure_plugins_initialized() {#ensure_plugins_initialized} *No function documentation available* diff --git a/versioned_docs/version-1.0/api/index.md b/versioned_docs/version-1.0/api/index.md index 51585f8..50ef8a5 100644 --- a/versioned_docs/version-1.0/api/index.md +++ b/versioned_docs/version-1.0/api/index.md @@ -11,8 +11,8 @@ This specification is organized by module of the reference python implementation **Note:** The modules don't have to be implemented in the same way as in the reference implementation, but all of the functionality here needs to be provided. -**Total documented items:** 189 -**Modules documented:** 39 +**Total documented items:** 195 +**Modules documented:** 41 ## Core Modules @@ -93,6 +93,16 @@ Core UTCP framework components that define the fundamental interfaces and implem - **Contains:** 2 classes, 12 methods +### [utcp.implementations.post_processors.filter_dict_post_processor](./core\utcp\implementations\post_processors\filter_dict_post_processor.md) + +- **Contains:** 2 classes + + +### [utcp.implementations.post_processors.limit_strings_post_processor](./core\utcp\implementations\post_processors\limit_strings_post_processor.md) + +- **Contains:** 2 classes + + ### [utcp.implementations.tag_search](./core\utcp\implementations\tag_search.md) - **Contains:** 2 classes, 3 methods @@ -140,7 +150,7 @@ Core UTCP framework components that define the fundamental interfaces and implem ### [utcp.plugins.plugin_loader](./core\utcp\plugins\plugin_loader.md) -- **Contains:** 1 functions +- **Contains:** 2 functions ### [utcp.utcp_client](./core\utcp\utcp_client.md) @@ -154,7 +164,7 @@ Plugin implementations that extend UTCP with specific transport protocols and ca ### [communication_protocols.cli.src.utcp_cli.cli_call_template](./plugins\communication_protocols\cli\src\utcp_cli\cli_call_template.md) -- **Contains:** 2 classes, 2 methods +- **Contains:** 3 classes, 2 methods ### [communication_protocols.cli.src.utcp_cli.cli_communication_protocol](./plugins\communication_protocols\cli\src\utcp_cli\cli_communication_protocol.md) diff --git a/versioned_docs/version-1.0/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md b/versioned_docs/version-1.0/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md index abfe4aa..54c7dca 100644 --- a/versioned_docs/version-1.0/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md +++ b/versioned_docs/version-1.0/api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md @@ -7,30 +7,173 @@ sidebar_label: cli_call_template **File:** `plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.py` +### class CommandStep {#commandstep} + +
+Documentation + +Configuration for a single command step in a CLI execution flow. + + +**Attributes** + +- **`command`**: The command string to execute. Can contain UTCP_ARG_argname_UTCP_END + placeholders that will be replaced with values from tool_args. Can also + reference previous command outputs using $CMD_0_OUTPUT, $CMD_1_OUTPUT, etc. +- **`append_to_final_output`**: Whether this command's output should be included + in the final result. If not specified, defaults to False for all + commands except the last one. + + + +**Basic Command Step** + +```json + { + "command": "git status", + "append_to_final_output": true + } +``` + + + +**Command With Argument Placeholders And Output Reference** + +```json + { + "command": "echo "Cloning to: UTCP_ARG_target_dir_UTCP_END, previous status: $CMD_0_OUTPUT"", + "append_to_final_output": true + } +``` +
+ +#### Fields: + +- command: str +- append_to_final_output: Optional[bool] + +--- + ### class CliCallTemplate ([CallTemplate](./../../../../../core/utcp/data/call_template.md#calltemplate)) {#clicalltemplate}
Documentation -Call template configuration for Command Line Interface tools. +Call template configuration for Command Line Interface (CLI) tools. + +This class defines the configuration for executing command-line tools and +programs as UTCP tool providers. Commands are executed in a single subprocess +to maintain state (like directory changes) between commands. + + + + +**You Can Reference The Output Of Previous Commands Using Variables** + + +- **`Example`**: `echo "Previous result: $CMD_0_OUTPUT"` -Enables execution of command-line tools and programs as UTCP providers. -Supports environment variable injection and custom working directories. **Attributes** -- **`call_template_type`**: Always "cli" for CLI providers. -- **`command_name`**: The name or path of the command to execute. -- **`env_vars`**: Optional environment variables to set during command execution. -- **`working_dir`**: Optional custom working directory for command execution. -- **`auth`**: Always None - CLI providers don't support authentication. +- **`call_template_type`**: The type of the call template. Must be "cli". +- **`commands`**: A list of CommandStep objects defining the commands to execute + in order. Each command can contain UTCP_ARG_argname_UTCP_END placeholders + that will be replaced with values from tool_args during execution. +- **`env_vars`**: A dictionary of environment variables to set for the command's + execution context. Values can be static strings or placeholders for + variables from the UTCP client's variable substitutor. +- **`working_dir`**: The working directory from which to run the commands. If not + provided, it defaults to the current process's working directory. +- **`auth`**: Authentication details. Not applicable to the CLI protocol, so it + is always None. + + + +**Cross-Platform Directory Operations** + +```json + { + "name": "cross_platform_dir_tool", + "call_template_type": "cli", + "commands": [ + { + "command": "cd UTCP_ARG_target_dir_UTCP_END", + "append_to_final_output": false + }, + { + "command": "ls -la", + "append_to_final_output": true + } + ] + } +``` + + + +**Referencing Previous Command Output** + +```json + { + "name": "reference_previous_output_tool", + "call_template_type": "cli", + "commands": [ + { + "command": "git status --porcelain", + "append_to_final_output": false + }, + { + "command": "echo "Found changes: $CMD_0_OUTPUT"", + "append_to_final_output": true + } + ] + } +``` + + + +**Command With Environment Variables And Placeholders** + +```json + { + "name": "python_multi_step_tool", + "call_template_type": "cli", + "commands": [ + { + "command": "python setup.py install", + "append_to_final_output": false + }, + { + "command": "python script.py --input UTCP_ARG_input_file_UTCP_END --result "$CMD_0_OUTPUT"" + } + ], + "env_vars": { + "PYTHONPATH": "/custom/path", + "API_KEY": "${API_KEY_VAR}" + } + } +``` + + + +**Security Considerations** + +- Commands are executed in a subprocess. Ensure that the commands +specified are from a trusted source. +- Avoid passing unsanitized user input directly into the command string. +Use tool argument validation where possible. +- All placeholders are replaced with string values from tool_args. +- Commands should use the appropriate syntax for the target platform +(PowerShell on Windows, Bash on Unix). +- Previous command outputs are available as variables but should be +used carefully to avoid command injection.
#### Fields: - call_template_type: Literal['cli'] -- command_name: str +- commands: List[CommandStep] - env_vars: Optional[Dict[str, str]] - working_dir: Optional[str] - auth: None @@ -46,13 +189,42 @@ Supports environment variable injection and custom working directories.
to_dict(self, obj: CliCallTemplate) -> dict -*No method documentation available* +Converts a `CliCallTemplate` instance to its dictionary representation. + + +**Args** + +- **`obj`**: The `CliCallTemplate` instance to serialize. + + + +**Returns** + +A dictionary representing the `CliCallTemplate`.
validate_dict(self, obj: dict) -> CliCallTemplate -*No method documentation available* +Validates a dictionary and constructs a `CliCallTemplate` instance. + + +**Args** + +- **`obj`**: The dictionary to validate and deserialize. + + + +**Returns** + +A `CliCallTemplate` instance. + + + +**Raises** + +- **`[UtcpSerializerValidationError](./../../../../../core/utcp/exceptions/utcp_serializer_validation_error.md#utcpserializervalidationerror)`**: If the dictionary is not a valid + representation of a `CliCallTemplate`.
--- diff --git a/versioned_docs/version-1.0/api/plugins/communication_protocols/cli/src/utcp_cli/cli_communication_protocol.md b/versioned_docs/version-1.0/api/plugins/communication_protocols/cli/src/utcp_cli/cli_communication_protocol.md index ee2e071..5ed1941 100644 --- a/versioned_docs/version-1.0/api/plugins/communication_protocols/cli/src/utcp_cli/cli_communication_protocol.md +++ b/versioned_docs/version-1.0/api/plugins/communication_protocols/cli/src/utcp_cli/cli_communication_protocol.md @@ -9,87 +9,98 @@ sidebar_label: cli_communication_protocol ### class CliCommunicationProtocol ([CommunicationProtocol](./../../../../../core/utcp/interfaces/communication_protocol.md#communicationprotocol)) {#clicommunicationprotocol} -
-Documentation - -Transport implementation for CLI-based tool providers. +*No class documentation available* -Handles communication with command-line tools by executing processes -and managing their input/output. Supports both tool discovery and -execution phases with comprehensive error handling and timeout management. +#### Methods: +
+async register_manual(self, caller, manual_call_template: [CallTemplate](./../../../../../core/utcp/data/call_template.md#calltemplate)) -> [RegisterManualResult](./../../../../../core/utcp/data/register_manual_response.md#registermanualresult) -**Features** +Registers a CLI-based manual and discovers its tools. -- Asynchronous subprocess execution with proper cleanup -- [Tool](./../../../../../core/utcp/data/tool.md#tool) discovery through startup commands returning UTCP manuals -- Flexible argument formatting for various CLI conventions -- Environment variable injection for authentication -- JSON output parsing with graceful fallback to text -- Cross-platform command parsing and execution -- Configurable working directories and timeouts -- Process lifecycle management with proper termination +This method executes the command specified in the `[CliCallTemplate](./cli_call_template.md#clicalltemplate)`'s +`command_name` field. It then attempts to parse the command's output +(stdout) as a UTCP manual in JSON format. +**Args** -**Architecture** +- **`caller`**: The UTCP client instance that is calling this method. +- **`manual_call_template`**: The `[CliCallTemplate](./cli_call_template.md#clicalltemplate)` containing the details for + tool discovery, such as the command to run. -CLI tools are discovered by executing the provider's command_name -and parsing the output for UTCP manual JSON. [Tool](./../../../../../core/utcp/data/tool.md#tool) calls execute -the same command with formatted arguments and return processed output. +**Returns** -**Attributes** +A `[RegisterManualResult](./../../../../../core/utcp/data/register_manual_response.md#registermanualresult)` object indicating whether the registration +was successful and containing the discovered tools. -- **`_log`**: Logger function for debugging and error reporting. -
-#### Methods: -
-async register_manual(self, caller, manual_call_template: [CallTemplate](./../../../../../core/utcp/data/call_template.md#calltemplate)) -> [RegisterManualResult](./../../../../../core/utcp/data/register_manual_response.md#registermanualresult) +**Raises** -*No method documentation available* +- **`ValueError`**: If the `manual_call_template` is not an instance of + `[CliCallTemplate](./cli_call_template.md#clicalltemplate)` or if `command_name` is not set.
async deregister_manual(self, caller, manual_call_template: [CallTemplate](./../../../../../core/utcp/data/call_template.md#calltemplate)) -> None -*No method documentation available* +Deregisters a CLI manual. + +For the CLI protocol, this is a no-op as there are no persistent +connections to terminate. + + +**Args** + +- **`caller`**: The UTCP client instance that is calling this method. +- **`manual_call_template`**: The call template of the manual to deregister.
async call_tool(self, caller, tool_name: str, tool_args: Dict[str, Any], tool_call_template: [CallTemplate](./../../../../../core/utcp/data/call_template.md#calltemplate)) -> Any -Call a CLI tool. +Calls a CLI tool by executing its command. -Executes the command specified by provider.command_name with the provided arguments. +This method constructs and executes the command specified in the +`[CliCallTemplate](./cli_call_template.md#clicalltemplate)`. It formats the provided `tool_args` as command-line +arguments and runs the command in a subprocess. **Args** -- **`caller`**: The UTCP client that is calling this method. -- **`tool_name`**: Name of the tool to call -- **`tool_args`**: Arguments for the tool call -- **`tool_call_template`**: The [CliCallTemplate](./cli_call_template.md#clicalltemplate) for the tool +- **`caller`**: The UTCP client instance that is calling this method. +- **`tool_name`**: The name of the tool to call. +- **`tool_args`**: A dictionary of arguments for the tool call. +- **`tool_call_template`**: The `[CliCallTemplate](./cli_call_template.md#clicalltemplate)` for the tool. -**The Output From The Command Execution Based On Exit Code** +**Returns** +The result of the command execution. If the command exits with a code +of 0, it returns the content of stdout. If the exit code is non-zero, +it returns the content of stderr. **Raises** -- **`ValueError`**: If provider is not a CliProvider or command_name is not set +- **`ValueError`**: If `tool_call_template` is not an instance of + `[CliCallTemplate](./cli_call_template.md#clicalltemplate)` or if `command_name` is not set.
async call_tool_streaming(self, caller, tool_name: str, tool_args: Dict[str, Any], tool_call_template: [CallTemplate](./../../../../../core/utcp/data/call_template.md#calltemplate)) -> AsyncGenerator[Any, None] -*No method documentation available* +Streaming calls are not supported for the CLI protocol. + + +**Raises** + +- **`NotImplementedError`**: Always, as this functionality is not supported.
--- diff --git a/versioned_docs/version-1.0/api/plugins/communication_protocols/http/src/utcp_http/http_call_template.md b/versioned_docs/version-1.0/api/plugins/communication_protocols/http/src/utcp_http/http_call_template.md index 2420d59..98dda8c 100644 --- a/versioned_docs/version-1.0/api/plugins/communication_protocols/http/src/utcp_http/http_call_template.md +++ b/versioned_docs/version-1.0/api/plugins/communication_protocols/http/src/utcp_http/http_call_template.md @@ -20,6 +20,81 @@ parameters using \{parameter_name\} syntax. All tool arguments not mapped to URL body, headers or query pattern parameters are passed as query parameters using '?arg_name=\{arg_value\}'. +**Basic Http Get Request** + +```json + { + "name": "my_rest_api", + "call_template_type": "http", + "url": "https://api.example.com/users/{user_id}", + "http_method": "GET" + } +``` + + + +**Post With Authentication** + +```json + { + "name": "secure_api", + "call_template_type": "http", + "url": "https://api.example.com/users", + "http_method": "POST", + "content_type": "application/json", + "auth": { + "auth_type": "api_key", + "api_key": "Bearer ${API_KEY}", + "var_name": "Authorization", + "location": "header" + }, + "headers": { + "X-Custom-Header": "value" + }, + "body_field": "body", + "header_fields": ["user_id"] + } +``` + + + +**Oauth2 Authentication** + +```json + { + "name": "oauth_api", + "call_template_type": "http", + "url": "https://api.example.com/data", + "http_method": "GET", + "auth": { + "auth_type": "oauth2", + "client_id": "${CLIENT_ID}", + "client_secret": "${CLIENT_SECRET}", + "token_url": "https://auth.example.com/token" + } + } +``` + + + +**Basic Authentication** + +```json + { + "name": "basic_auth_api", + "call_template_type": "http", + "url": "https://api.example.com/secure", + "http_method": "GET", + "auth": { + "auth_type": "basic", + "username": "${USERNAME}", + "password": "${PASSWORD}" + } + } +``` + + + **Attributes** - **`call_template_type`**: Always "http" for HTTP providers. diff --git a/versioned_docs/version-1.0/api/plugins/communication_protocols/http/src/utcp_http/openapi_converter.md b/versioned_docs/version-1.0/api/plugins/communication_protocols/http/src/utcp_http/openapi_converter.md index 0a68759..dd41955 100644 --- a/versioned_docs/version-1.0/api/plugins/communication_protocols/http/src/utcp_http/openapi_converter.md +++ b/versioned_docs/version-1.0/api/plugins/communication_protocols/http/src/utcp_http/openapi_converter.md @@ -22,14 +22,50 @@ a UTCP tool with appropriate input/output schemas. **Features** -- Complete OpenAPI specification parsing -- Recursive JSON reference ($ref) resolution -- Authentication scheme conversion (API key, Basic, OAuth2) -- Input parameter and request body handling -- Response schema extraction -- URL template and path parameter support -- Provider name normalization -- Placeholder variable generation for configuration +- Complete OpenAPI specification parsing. +- Recursive JSON reference ($ref) resolution. +- Authentication scheme conversion (API key, Basic, OAuth2). +- Input parameter and request body handling. +- Response schema extraction. +- URL template and path parameter support. +- Call template name normalization. +- Placeholder variable generation for configuration. + + + +**Basic Openapi Conversion** + +```python + from utcp_http.openapi_converter import OpenApiConverter + + # Assuming you have a method to fetch and parse the spec + openapi_spec = fetch_and_parse_spec("https://api.example.com/openapi.json") + + converter = OpenApiConverter(openapi_spec) + manual = converter.convert() + + # Use the generated manual with a UTCP client + # client = await UtcpClient.create() + # await client.register_manual(manual) +``` + + + +**Converting Local Openapi File** + +```python + import yaml + + converter = OpenApiConverter() + + +**With Open("Api_Spec.Yaml", "R") As F** + +spec_content = yaml.safe_load(f) + +converter = OpenApiConverter(spec_content) +manual = converter.convert() +``` @@ -54,7 +90,16 @@ operation, and creating corresponding UTCP tools with HTTP call_templates.
convert(self) -> [UtcpManual](./../../../../../core/utcp/data/utcp_manual.md#utcpmanual) -*No method documentation available* +Converts the loaded OpenAPI specification into a [UtcpManual](./../../../../../core/utcp/data/utcp_manual.md#utcpmanual). + +This is the main entry point for the conversion process. It iterates through +the paths and operations in the specification, creating a UTCP tool for each +one. + + +**Returns** + +A [UtcpManual](./../../../../../core/utcp/data/utcp_manual.md#utcpmanual) object containing all the tools generated from the spec.
--- diff --git a/versioned_docs/version-1.0/api/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_call_template.md b/versioned_docs/version-1.0/api/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_call_template.md index 2b36fbb..8f5cc77 100644 --- a/versioned_docs/version-1.0/api/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_call_template.md +++ b/versioned_docs/version-1.0/api/plugins/communication_protocols/mcp/src/utcp_mcp/mcp_call_template.md @@ -44,12 +44,109 @@ interfaces. Supports both stdio (local process) and HTTP (remote) transport methods. +**Basic Mcp Server With Stdio Transport** + +```json + { + "name": "mcp_server", + "call_template_type": "mcp", + "config": { + "mcpServers": { + "filesystem": { + "command": "node", + "args": ["mcp-server.js"], + "env": {"NODE_ENV": "production"} + } + } + } + } +``` + + + +**Mcp Server With Working Directory** + +```json + { + "name": "mcp_tools", + "call_template_type": "mcp", + "config": { + "mcpServers": { + "tools": { + "command": "python", + "args": ["-m", "mcp_server"], + "cwd": "/app/mcp", + "env": { + "PYTHONPATH": "/app", + "LOG_LEVEL": "INFO" + } + } + } + } + } +``` + + + +**Mcp Server With Oauth2 Authentication** + +```json + { + "name": "secure_mcp", + "call_template_type": "mcp", + "config": { + "mcpServers": { + "secure_server": { + "transport": "http", + "url": "https://mcp.example.com" + } + } + }, + "auth": { + "auth_type": "oauth2", + "token_url": "https://auth.example.com/token", + "client_id": "${CLIENT_ID}", + "client_secret": "${CLIENT_SECRET}", + "scope": "read:tools" + } + } +``` + + + +**During Migration (Utcp With Mcp)** + +```python + # UTCP Client with MCP plugin + client = await UtcpClient.create() + result = await client.call_tool("filesystem.read_file", { + "path": "/data/file.txt" + }) +``` + + + +**After Migration (Pure Utcp)** + +```python + # UTCP Client with native protocol + client = await UtcpClient.create() + result = await client.call_tool("filesystem.read_file", { + "path": "/data/file.txt" + }) +``` + + + **Attributes** - **`call_template_type`**: Always "mcp" for MCP providers. - **`config`**: Configuration object containing MCP server definitions. This follows the same format as the official MCP server configuration. - **`auth`**: Optional OAuth2 authentication for HTTP-based MCP servers. +- **`register_resources_as_tools`**: Whether to register MCP resources as callable tools. + When True, server resources are exposed as tools that can be called. + Default is False.
#### Fields: @@ -57,6 +154,7 @@ transport methods. - call_template_type: Literal['mcp'] - config: McpConfig - auth: Optional[[OAuth2Auth](./../../../../../core/utcp/data/auth_implementations/oauth2_auth.md#oauth2auth)] +- register_resources_as_tools: bool --- diff --git a/versioned_docs/version-1.0/for-tool-providers.md b/versioned_docs/version-1.0/for-tool-providers.md new file mode 100644 index 0000000..87ff739 --- /dev/null +++ b/versioned_docs/version-1.0/for-tool-providers.md @@ -0,0 +1,580 @@ +--- +id: for-tool-providers +title: For Tool Providers +sidebar_position: 2 +--- + +# For Tool Providers + +:::info Language Note +This guide covers how to expose your existing APIs and services as UTCP tools. UTCP implementations are available in multiple languages - check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) for Python, TypeScript, Go, and other language implementations. +::: + +This guide helps you expose your tools through UTCP so they can be discovered and used by AI agents and other applications. + +## Overview + +As a tool provider, you'll create a **UTCP Manual** - a standardized description of your tools that tells clients how to call them directly using their native protocols. This eliminates the need for wrapper servers and allows direct communication. + +## Quick Start + +### 1. Create a Simple Manual + +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "tools": [ + { + "name": "get_user", + "description": "Retrieve user information by ID", + "inputs": { + "type": "object", + "properties": { + "user_id": {"type": "string", "description": "User identifier"} + }, + "required": ["user_id"] + }, + "outputs": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + "email": {"type": "string"} + } + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/users/${user_id}", + "http_method": "GET" + } + } + ] +} +``` + +### 2. Expose via Discovery Endpoint + +Create an HTTP endpoint that returns your UTCP manual: + +**Endpoint**: `GET /utcp` +**Response**: +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "tools": [ + // ... your tools here + ] +} +``` + +Your existing API endpoints remain unchanged. For example: +- `GET /users/{user_id}` - Returns user data +- `POST /orders` - Creates new orders +- etc. + +## Manual Structure + +The UTCP manual follows a standardized structure that defines your tools and how to call them. For complete field specifications, data types, and validation rules, see the [UTCP Manual API Reference](./api/core/utcp/data/utcp_manual.md). + +### Key Components + +- **Manual metadata**: Version information and API details +- **Tool definitions**: Description of available tools and their capabilities +- **Call templates**: Instructions for invoking each tool +- **Authentication**: Security configuration for tool access +- **Variables**: Dynamic values for tool parameters + +### Tool Definition + +Tools are defined in your UTCP manual with their input parameters, call instructions, and optional metadata. For complete field specifications, see the [Tool API Reference](./api/core/utcp/data/tool.md). + +## Communication Protocol Plugins + +### HTTP Tools + +Most common for REST APIs: + +```json +{ + "name": "create_user", + "description": "Create a new user account", + "inputs": { + "type": "object", + "properties": { + "user_fields": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string", "format": "email"} + }, + "required": ["name", "email"] + } + }, + "required": ["user_fields"] + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/users", + "http_method": "POST", + "auth": { + "auth_type": "api_key", + "api_key": "Bearer ${API_TOKEN}", + "var_name": "Authorization", + "location": "header" + }, + "body_field": "user_fields" + } +} +``` + +### CLI Tools + +For command-line applications: + +```json +{ + "name": "git_analysis", + "description": "Analyze git repository status and commit history", + "inputs": { + "type": "object", + "properties": { + "repo_path": {"type": "string", "description": "Path to git repository"} + }, + "required": ["repo_path"] + }, + "tool_call_template": { + "call_template_type": "cli", + "commands": [ + { + "command": "cd UTCP_ARG_repo_path_UTCP_END", + "append_to_final_output": false + }, + { + "command": "git status --porcelain", + "append_to_final_output": false + }, + { + "command": "echo \"Status: $CMD_1_OUTPUT, Files changed: $(echo \"$CMD_1_OUTPUT\" | wc -l)\"", + "append_to_final_output": true + } + ] + } +} +``` + +## Authentication + +### API Key Authentication + +```json +{ + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "X-API-Key", + "location": "header" + } +} +``` + +### Bearer Token + +To use a Bearer Token, you can use the `api_key` authentication type. + +```json +{ + "auth": { + "auth_type": "api_key", + "api_key": "Bearer ${ACCESS_TOKEN}", + "var_name": "Authorization", + "location": "header" + } +} +``` + +### OAuth2 + +```json +{ + "auth": { + "auth_type": "oauth2", + "client_id": "${CLIENT_ID}", + "client_secret": "${CLIENT_SECRET}", + "token_url": "https://auth.example.com/token", + "scope": "read:users write:users" + } +} +``` + +### Per-Tool Authentication + +```json +{ + "name": "admin_action", + "description": "Perform admin action", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/admin/action", + "http_method": "POST", + "auth": { + "auth_type": "api_key", + "api_key": "${ADMIN_TOKEN}", + "var_name": "Authorization", + "location": "header" + } + } +} +``` + +## Variable Substitution + +Use `${VARIABLE_NAME}` syntax for dynamic values: + +### From Tool Arguments + +```json +{ + "url": "https://api.example.com/users/${user_id}", + "body": { + "name": "${name}", + "email": "${email}" + } +} +``` + +### From Environment Variables + +```json +{ + "headers": { + "Authorization": "Bearer ${API_TOKEN}", + "X-Client-ID": "${CLIENT_ID}" + } +} +``` + +### Default Values + +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "variables": { + "base_url": "https://api.example.com", + "timeout": 30 + }, + "tools": [ + { + "name": "get_data", + "tool_call_template": { + "call_template_type": "http", + "url": "${base_url}/data", + "timeout": "${timeout}" + } + } + ] +} +``` + +## Implementation Examples + +### REST API Implementation + +For a typical REST API, you'll need to: + +1. **Keep your existing endpoints unchanged** +2. **Add a UTCP discovery endpoint** at `/utcp` +3. **Map your API operations to UTCP tools** + +Example API structure: +``` +GET /users/{user_id} # Your existing endpoint +POST /users # Your existing endpoint +GET /utcp # New UTCP discovery endpoint +``` + +The UTCP manual describes how to call your existing endpoints: + +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "info": { + "title": "User Management API", + "version": "1.0.0", + "description": "API for managing user accounts" + }, + "tools": [ + { + "name": "get_user", + "description": "Retrieve user information by ID", + "inputs": { + "type": "object", + "properties": { + "user_id": {"type": "string"} + }, + "required": ["user_id"] + }, + "outputs": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + "email": {"type": "string"} + } + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/users/${user_id}", + "http_method": "GET", + "auth": { + "auth_type": "api_key", + "api_key": "Bearer ${API_TOKEN}", + "var_name": "Authorization", + "location": "header" + } + } + }, + { + "name": "create_user", + "description": "Create a new user account", + "inputs": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string"} + }, + "required": ["name", "email"] + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/users", + "http_method": "POST", + "auth": { + "auth_type": "api_key", + "api_key": "Bearer ${API_TOKEN}", + "var_name": "Authorization", + "location": "header" + }, + "body_field": "user_data" + } + } + ] +} +``` + +## OpenAPI Integration + +If you already have an OpenAPI/Swagger specification, you can automatically convert it to a UTCP manual: + +### Automatic Conversion + +Many UTCP implementations provide OpenAPI converters that can: + +1. **Parse OpenAPI specifications** from URLs or files +2. **Convert paths to UTCP tools** automatically +3. **Map authentication schemes** to UTCP auth types +4. **Generate proper input/output schemas** + +### Conversion Configuration + +You can customize the conversion process: + +```json +{ + "source": "https://api.example.com/openapi.json", + "base_url": "https://api.example.com", + "include_operations": ["get", "post"], + "exclude_paths": ["/internal/*"], + "auth_template": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "X-API-Key", + "location": "header" + } +} +``` + +### Manual Review + +After automatic conversion: +1. **Review generated tools** for accuracy +2. **Add missing descriptions** and examples +3. **Validate input/output schemas** +4. **Test with UTCP clients** +5. **Customize authentication** as needed + +## Best Practices + +### Manual Design + +1. **Clear Descriptions**: Write clear, concise tool descriptions +2. **Comprehensive Schemas**: Use detailed JSON schemas for inputs/outputs +3. **Consistent Naming**: Use consistent naming conventions +4. **Version Management**: Use semantic versioning for your manual +5. **Documentation**: Include examples and usage notes + +### Security + +1. **Authentication**: Always implement proper authentication +2. **Input Validation**: Validate all inputs on your API side +3. **Rate Limiting**: Implement rate limiting to prevent abuse +4. **HTTPS Only**: Use HTTPS for all production endpoints +5. **Credential Management**: Never hardcode credentials in manuals + +### Performance + +1. **Efficient Endpoints**: Design efficient API endpoints +2. **Caching**: Implement appropriate caching strategies +3. **Pagination**: Use pagination for large result sets +4. **Timeouts**: Set reasonable timeout values +5. **Monitoring**: Monitor API performance and usage + +### Maintenance + +1. **Versioning**: Version your manual and API together +2. **Backward Compatibility**: Maintain backward compatibility when possible +3. **Deprecation**: Provide clear deprecation notices +4. **Testing**: Test your manual with UTCP clients +5. **Documentation**: Keep documentation up to date + +## Testing Your Manual + +### Manual Validation + +Validate your UTCP manual structure: + +1. **JSON Schema Validation**: Ensure your manual follows the UTCP schema +2. **Tool Definition Validation**: Check that all tools have required fields +3. **Call Template Validation**: Verify call templates are properly formatted +4. **Authentication Validation**: Test authentication configurations + +### Integration Testing + +Test your manual with UTCP clients: + +1. **Tool Discovery**: Verify clients can discover your tools +2. **Tool Execution**: Test actual tool calls with various inputs +3. **Error Handling**: Test error scenarios and responses +4. **Authentication**: Verify authentication works correctly +5. **Performance**: Test response times and reliability + +### Testing Checklist + +- [ ] Manual validates against UTCP schema +- [ ] All tools have unique names +- [ ] All required fields are present +- [ ] Call templates are correctly formatted +- [ ] Authentication works as expected +- [ ] Tools return expected outputs +- [ ] Error responses are properly formatted +- [ ] Performance meets requirements + +## Migration Strategies + +### From OpenAPI + +1. **Automatic Conversion**: Use OpenAPI converter for initial conversion +2. **Manual Refinement**: Refine converted manual for better descriptions +3. **Authentication Setup**: Configure authentication properly +4. **Testing**: Test converted manual thoroughly + +### From MCP + +1. **Wrapper Approach**: Use UTCP-MCP plugin initially +2. **Gradual Migration**: Migrate tools one by one to native protocols +3. **Direct Implementation**: Implement tools using native UTCP protocols +4. **Deprecation**: Remove MCP dependency once migration is complete + +## Common Patterns + +### CRUD Operations + +```json +{ + "tools": [ + { + "name": "create_resource", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/resources", + "http_method": "POST" + } + }, + { + "name": "get_resource", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/resources/${id}", + "http_method": "GET" + } + }, + { + "name": "update_resource", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/resources/${id}", + "http_method": "PUT" + } + }, + { + "name": "delete_resource", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/resources/${id}", + "http_method": "DELETE" + } + } + ] +} +``` + +### Batch Operations + +```json +{ + "name": "batch_process", + "description": "Process multiple items in batch", + "inputs": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": {"type": "object"} + } + } + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/batch", + "http_method": "POST", + "body": { + "items": "${items}" + } + } +} +``` + +## Next Steps + +1. **Design Your Manual**: Plan your tool structure and descriptions +2. **Choose Protocols**: Select appropriate communication protocols +3. **Implement Discovery**: Add the `/utcp` endpoint to your API +4. **Test Integration**: Test with UTCP clients +5. **Monitor Usage**: Monitor how your tools are being used +6. **Iterate**: Improve based on usage patterns and feedback + +For more information, see: +- [Communication Protocol Plugins](./protocols/index.md) +- [Implementation Guide](./implementation.md) +- [Security Considerations](./security.md) + +## Language-Specific Implementation + +For detailed implementation examples and code samples in your programming language: + +- **Multi-language**: [UTCP Implementation Examples](https://github.com/universal-tool-calling-protocol) - Examples across Python, TypeScript, Go, and other languages +- **TypeScript**: [TypeScript UTCP Documentation](https://github.com/universal-tool-calling-protocol/typescript-utcp/tree/main/docs) +- **Other languages**: Check respective repositories in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) diff --git a/versioned_docs/version-1.0/implementation.md b/versioned_docs/version-1.0/implementation.md new file mode 100644 index 0000000..c1aee3a --- /dev/null +++ b/versioned_docs/version-1.0/implementation.md @@ -0,0 +1,396 @@ +--- +id: implementation +title: Implementation Guide +sidebar_position: 4 +--- + +# UTCP Implementation Guide + +This guide covers the core concepts and patterns for implementing UTCP in any programming language, whether you're building tool providers or tool consumers. + +## Quick Start + +### 1. Install UTCP Library + +Choose the UTCP implementation for your programming language: + +- **Python**: `pip install utcp utcp-http utcp-cli` +- **Node.js**: `npm install @utcp/core @utcp/http @utcp/cli` +- **Other languages**: Check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) + +### 2. Create Your First Tool Provider + +Create an HTTP endpoint that serves a UTCP manual: + +**Endpoint**: `GET /utcp` +**Response**: +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "info": { + "title": "Weather API", + "version": "1.0.0" + }, + "tools": [ + { + "name": "get_weather", + "description": "Get current weather for a location", + "inputs": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.weather.com/current", + "http_method": "GET", + "auth": { + "auth_type": "api_key", + "api_key": "${WEATHER_API_KEY}", + "var_name": "appid", + "location": "query" + } + } + } + ] +} +``` + +### 3. Create Your First Client + +Configure a UTCP client to discover and call tools: + +**Configuration**: +```json +{ + "manual_call_templates": [ + { + "name": "weather_service", + "call_template_type": "http", + "url": "https://api.weather.com/utcp", + "http_method": "GET" + } + ], + "variables": { + "WEATHER_API_KEY": "your-api-key" + } +} +``` + +**Usage**: +1. Initialize UTCP client with configuration +2. Discover tools from the weather service +3. Call the `get_weather` tool with location parameter + +## Core Concepts + +### UTCP Manual + +A UTCP manual is a JSON document that describes available tools and how to call them: + +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "info": { + "title": "API Name", + "version": "1.0.0", + "description": "API description" + }, + "tools": [ + { + "name": "tool_name", + "description": "Tool description", + "inputs": { + "type": "object", + "properties": { + "param": {"type": "string"} + } + }, + "outputs": { + "type": "object", + "properties": { + "result": {"type": "string"} + } + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/endpoint", + "http_method": "POST" + } + } + ] +} +``` + +### Call Templates + +Call templates define how to invoke tools using specific protocols: + +#### HTTP Call Template +```json +{ + "call_template_type": "http", + "url": "https://api.example.com/endpoint", + "http_method": "POST", + "auth": { + "auth_type": "api_key", + "api_key": "${API_TOKEN}", + "var_name": "Authorization", + "location": "header" + }, + "body_field": "body" +} + +Tool arguments not used in the URL path or headers will be sent as query parameters for GET requests, or in the request body for POST/PUT/PATCH requests. The `body_field` specifies which tool argument contains the data for the request body. +} +``` + +#### CLI Call Template +```json +{ + "call_template_type": "cli", + "commands": [ + { + "command": "cd /app", + "append_to_final_output": false + }, + { + "command": "python script.py UTCP_ARG_input_UTCP_END", + "append_to_final_output": true + } + ], + "env_vars": { + "PYTHONPATH": "/app/lib" + }, + "working_dir": "/app" +} +``` + +### Variable Substitution + +Variables in call templates are replaced with actual values: + +- **Tool arguments**: `${argument_name}` +- **Environment variables**: `${ENV_VAR}` +- **Configuration variables**: `${config.variable}` + +## Tool Provider Implementation + +### Manual Structure + +Create a well-structured UTCP manual: + +1. **Info Section**: Describe your API +2. **Tools Array**: Define each available tool +3. **Input Schemas**: Specify required parameters +4. **Output Schemas**: Document return values +5. **Call Templates**: Define how to invoke each tool + +### Discovery Endpoint + +Expose your manual via HTTP: + +``` +GET /utcp +Content-Type: application/json + +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "tools": [...] +} +``` + +### Authentication + +Support various authentication methods: + +#### API Key Authentication +```json +{ + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "X-API-Key", + "location": "header" + } +} +``` + +#### OAuth2 Authentication +```json +{ + "auth": { + "auth_type": "oauth2", + "client_id": "${CLIENT_ID}", + "client_secret": "${CLIENT_SECRET}", + "token_url": "https://auth.example.com/token" + } +} +``` + +## Tool Consumer Implementation + +### Client Configuration + +Configure your UTCP client: + +#### File-based Configuration +```yaml +# utcp-config.yaml +manual_call_templates: + - name: weather_service + call_template_type: http + url: https://api.weather.com/utcp + +variables: + WEATHER_API_KEY: your-api-key + +load_variables_from: + - variable_loader_type: dotenv + env_file_path: .env +``` + +#### Programmatic Configuration +```json +{ + "manual_call_templates": [ + { + "name": "service_name", + "call_template_type": "http", + "url": "https://api.example.com/utcp" + } + ], + "variables": { + "API_KEY": "your-key" + } +} +``` + +### Tool Discovery + +Discover available tools: + +1. **List Tools**: Get all available tools +2. **Tool Information**: Get detailed tool metadata +3. **Filter Tools**: Find tools by name, tags, or description + +### Tool Execution + +Execute tools with proper error handling: + +1. **Basic Calls**: Simple tool invocation +2. **Batch Calls**: Execute multiple tools +3. **Context Passing**: Pass context between calls +4. **Error Handling**: Handle failures gracefully + +## Advanced Implementation Patterns + +### Custom Protocol Plugins + +Extend UTCP with custom communication protocols: + +1. **Define Call Template**: Structure for your protocol +2. **Implement Communication Handler**: Protocol-specific logic +3. **Register Protocol**: Make it available to clients + +Example custom protocol structure: +```json +{ + "call_template_type": "custom", + "custom_field": "value", + "timeout": 30 +} +``` + +### Custom Tool Repositories + +Implement custom tool storage: + +1. **Tool Storage**: How tools are stored and retrieved +2. **Search Functionality**: Tool discovery and filtering +3. **Caching**: Performance optimization +4. **Synchronization**: Multi-client coordination + +### Testing Strategies + +#### Unit Testing Tool Providers +- Test manual generation +- Validate tool definitions +- Test authentication +- Mock external dependencies + +#### Integration Testing +- Test tool discovery +- Test tool execution +- Test error scenarios +- Test performance + +#### End-to-End Testing +- Test complete workflows +- Test multiple protocols +- Test real-world scenarios +- Test scalability + +## Best Practices + +### Performance +- Use connection pooling +- Implement caching +- Optimize tool discovery +- Monitor response times + +### Security +- Validate all inputs +- Use secure authentication +- Implement rate limiting +- Log security events + +### Reliability +- Implement retry logic +- Handle network failures +- Use circuit breakers +- Monitor tool health + +### Maintainability +- Version your manuals +- Document all tools +- Use consistent naming +- Provide examples + +## Deployment Considerations + +### Scaling +- Load balance tool providers +- Cache tool discoveries +- Use async processing +- Monitor resource usage + +### Monitoring +- Track tool usage +- Monitor error rates +- Log performance metrics +- Set up alerts + +### Security +- Use HTTPS everywhere +- Implement proper authentication +- Validate all inputs +- Monitor for abuse + +## Language-Specific Implementation + +For detailed implementation examples and code samples in your programming language: + +- **Multi-language**: [UTCP Implementation Examples](https://github.com/universal-tool-calling-protocol) - Examples across Python, TypeScript, Go, and other languages +- **TypeScript**: [TypeScript UTCP Documentation](https://github.com/universal-tool-calling-protocol/typescript-utcp/tree/main/docs) +- **Other languages**: Check respective repositories in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) + +For more detailed information, see: +- [Communication Protocol Plugins](./protocols/index.md) +- [API Reference](./api/index.md) +- [Security Considerations](./security.md) diff --git a/versioned_docs/version-1.0/index.md b/versioned_docs/version-1.0/index.md index 4f0a4e4..2355a3e 100644 --- a/versioned_docs/version-1.0/index.md +++ b/versioned_docs/version-1.0/index.md @@ -4,152 +4,219 @@ title: Introduction sidebar_position: 1 --- -# Introduction to UTCP 1.0 +# Universal Tool Calling Protocol (UTCP) -The Universal Tool Calling Protocol (UTCP) is a lightweight, secure, and scalable standard for defining and interacting with tools across a wide variety of communication protocols. Version 1.0 introduces a modular core with a plugin-based architecture, making it more extensible, testable, and easier to package. - -## Core Components +:::info Language Examples +UTCP is available in multiple languages - see [Python](https://github.com/universal-tool-calling-protocol/python-utcp), [TypeScript](https://github.com/universal-tool-calling-protocol/typescript-utcp), [Go](https://github.com/universal-tool-calling-protocol/go-utcp), and other implementations in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol). +::: -UTCP consists of four main components: +UTCP is a lightweight, secure, and scalable standard that enables AI agents and applications to discover and call tools directly using their native protocols - **no wrapper servers required**. -1. [**Manuals**](./api/core/utcp/data/utcp_manual.md): The standard tool provider description format that contains tool definitions -2. [**Tools**](./api/core/utcp/data/tool.md): The individual capabilities that can be called -3. [**Call Templates**](./api/core/utcp/data/call_template.md): The communication configurations that specify how tools are accessed. Concretely this maps a tool name and provided arguments to an actual API request in a communication protocol. -4. [**UtcpClient**](./api/core/utcp/utcp_client.md): The client that calls tools using the call templates. +## Why UTCP? -## The "Manual" Approach +### The Problem with Current Approaches +Most tool integration solutions force you to: +- Build and maintain wrapper servers for every tool +- Route all traffic through a middleman protocol +- Reimplement existing authentication and security +- Accept additional latency and complexity -UTCP's fundamental philosophy is to act as a descriptive manual rather than a prescriptive middleman: +### The UTCP Solution +UTCP acts as a **"manual"** that tells agents how to call your tools directly: -:::note -A UTCP Manual tells an agent: "Here is a tool. Here is its native endpoint (HTTP, WebSocket, CLI, etc.), and here is how to call it directly." +:::tip Core Philosophy +*"If a human can call your API, an AI agent should be able to call it too - with the same security and no additional infrastructure."* ::: -This approach eliminates the need for wrapper servers and allows direct communication between agents and tools. - -## New Architecture in 1.0 - -UTCP has been refactored into a core library and a set of optional plugins: - -### Core Package (`utcp`) -- **Data Models**: Pydantic models for [`Tool`](./api/core/utcp/data/tool.md), [`CallTemplate`](./api/core/utcp/data/call_template.md), [`UtcpManual`](./api/core/utcp/data/utcp_manual.md), and [`Auth`](./api/core/utcp/data/auth.md) -- **Pluggable Interfaces**: [`CommunicationProtocol`](./api/core/utcp/interfaces/communication_protocol.md), [`ConcurrentToolRepository`](./api/core/utcp/interfaces/concurrent_tool_repository.md), [`ToolSearchStrategy`](./api/core/utcp/interfaces/tool_search_strategy.md), [`VariableSubstitutor`](./api/core/utcp/interfaces/variable_substitutor.md), [`ToolPostProcessor`](./api/core/utcp/interfaces/tool_post_processor.md) -- **Default Implementations**: [`UtcpClient`](./api/core/utcp/utcp_client.md), [`InMemToolRepository`](./api/core/utcp/implementations/in_mem_tool_repository.md), [`TagAndDescriptionWordMatchStrategy`](./api/core/utcp/implementations/tag_search.md) - -### Protocol Plugins -- `utcp-http`: Supports HTTP, SSE, and streamable HTTP, plus an OpenAPI converter -- `utcp-cli`: For wrapping local command-line tools -- `utcp-mcp`: For interoperability with the Model Context Protocol (MCP) -- `utcp-text`: For reading text files -- `utcp-socket`: TCP and UDP protocols (work in progress) -- `utcp-gql`: GraphQL (work in progress) - -## Minimal Example - -Let's see how easy it is to use UTCP with a minimal example. - -### 1. Defining a Tool (Tool Provider) - -Create a simple HTTP endpoint that serves a UTCP Manual (JSON): - -```python -# app.py -from fastapi import FastAPI - -app = FastAPI() - -@app.get("/utcp") -def utcp_discovery(): - return { - "manual_version": "1.0.0", - "utcp_version": "1.0.1", - "tools": [ - { - "name": "get_weather", - "description": "Get current weather for a location", - "inputs": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City name" - } - }, - "required": ["location"] - }, - "outputs": { - "type": "object", - "properties": { - "temperature": {"type": "number"}, - "conditions": {"type": "string"} - } - }, - "tool_call_template": { - "call_template_type": "http", - "url": "https://example.com/api/weather", - "http_method": "GET" - } - } - ] - } +## Quick Start (5 Minutes) + +### 1. Install UTCP + +```bash +# Example installation (Python) +pip install utcp utcp-http + +# Example installation (Node.js) +npm install @utcp/core @utcp/http -# Implement the actual weather API endpoint -@app.get("/api/weather") -def get_weather(location: str): - # In a real app, you'd fetch actual weather data - return {"temperature": 22.5, "conditions": "Sunny"} +# See language-specific documentation for other implementations ``` -Run the server: +### 2. Expose Your First Tool + +Add a discovery endpoint to your existing API: + +**Add endpoint**: `GET /utcp` +**Return your UTCP manual**: +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "tools": [{ + "name": "get_weather", + "description": "Get current weather for a location", + "inputs": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"] + }, + "tool_call_template": { + "call_template_type": "http", + "url": "http://localhost:8000/weather", + "http_method": "GET" + } + }], + "auth": { + "auth_type": "api_key", + "api_key": "${WEATHER_API_KEY}", + "var_name": "appid", + "location": "query" + } + } + }] +} +``` -```bash -uvicorn app:app --reload +### 3. Call Your Tool + +**Configure UTCP client**: +```json +{ + "manual_call_templates": [{ + "name": "weather_api", + "call_template_type": "http", + "url": "http://localhost:8000/utcp", + "http_method": "GET" + }] +} ``` -### 2. Using the Tool (Client) - -```python -# client.py -import asyncio -from utcp.utcp_client import UtcpClient -from utcp_http.http_call_template import HttpCallTemplate - -async def main(): - # Create a UTCP client with configuration - client = await UtcpClient.create(config={ - "manual_call_templates": [ - { - "name": "weather_service", - "call_template_type": "http", - "http_method": "GET", - "url": "http://localhost:8000/utcp" - } - ] - }) - - # Tools are automatically registered from the manual call templates - # Call a tool by its namespaced name: {manual_name}.{tool_name} - result = await client.call_tool( - "weather_service.get_weather", - tool_args={"location": "San Francisco"} - ) - print(f"Weather: {result['temperature']}°C, {result['conditions']}") - -if __name__ == "__main__": - asyncio.run(main()) +**Call the tool**: +1. Initialize UTCP client with configuration +2. Discover tools from weather API +3. Call `get_weather` tool with location parameter +4. Receive weather data response ``` -Run the client: +**That's it!** Your tool is now discoverable and callable by any UTCP client. -```bash -python client.py +## Key Benefits + +| Benefit | Description | +|---------|-------------| +| **🚀 Zero Latency Overhead** | Direct tool calls, no proxy servers | +| **🔒 Native Security** | Use your existing authentication and authorization | +| **🌐 Protocol Flexibility** | HTTP, WebSocket, CLI, GraphQL, and more | +| **⚡ Easy Integration** | Add one endpoint, no infrastructure changes | +| **📈 Scalable** | Leverage your existing scaling and monitoring | + +## How It Works + +```mermaid +graph LR + A[AI Agent] -->|1. Discover| B[UTCP Manual] + B -->|2. Learn| C[Tool Definitions] + A -->|3. Call Directly| D[Your API] + D -->|4. Response| A ``` -## Benefits of the UTCP Approach +1. **Discovery**: Agent fetches your UTCP manual +2. **Learning**: Agent understands how to call your tools +3. **Direct Calling**: Agent calls your API directly using native protocols +4. **Response**: Your API responds normally + +## Supported Protocols + +UTCP supports multiple communication protocols through plugins: + +| Protocol | Use Case | Plugin | Status | +|----------|----------|--------|--------| +| **[HTTP](./protocols/http.md)** | REST APIs, webhooks | `utcp-http` | ✅ Stable | +| **[WebSocket](./protocols/websocket.md)** | Real-time communication | `utcp-websocket` | ✅ Stable | +| **[CLI](./protocols/cli.md)** | Command-line tools | `utcp-cli` | ✅ Stable | +| **[Server-Sent Events](./protocols/streamable-http.md)** | Streaming data | `utcp-http` | ✅ Stable | +| **[Text Files](./protocols/text.md)** | File reading | `utcp-text` | ✅ Stable | +| **[MCP](./protocols/mcp.md)** | MCP interoperability | `utcp-mcp` | ✅ Stable | + +[View all protocols →](./protocols/index.md) + +## Architecture Overview -1. **Direct Communication**: The client calls the tool's native endpoint directly -2. **No Wrapper Infrastructure**: No need to build and maintain wrapper servers -3. **Leverage Existing Systems**: Uses the tool's existing authentication, rate limiting, etc. -4. **Flexible Protocol Support**: Works with any communication protocol (HTTP, WebSockets, CLI, etc.) +UTCP v1.0 features a modular, plugin-based architecture: + +### Core Components +- **[Manuals](./api/core/utcp/data/utcp_manual.md)**: Tool definitions and metadata +- **[Tools](./api/core/utcp/data/tool.md)**: Individual callable capabilities +- **[Call Templates](./api/core/utcp/data/call_template.md)**: Protocol-specific call instructions +- **[UTCP Client](./api/core/utcp/utcp_client.md)**: Tool discovery and execution engine + +### Plugin System +- **Protocol Plugins**: HTTP, WebSocket, CLI, etc. +- **Custom Protocols**: Extend with your own communication methods +- **Tool Repositories**: Pluggable storage for tool definitions +- **Search Strategies**: Customizable tool discovery algorithms + +[Learn more about the architecture →](./api/index.md) + +## Who Should Use UTCP? + +### 🛠️ Tool Providers +You have APIs, services, or tools that you want AI agents to use: +- **Existing API owners** - Expose your REST APIs to AI agents +- **SaaS providers** - Make your services AI-accessible +- **Enterprise teams** - Enable internal tool usage by AI systems + +[**Get started as a tool provider →**](./for-tool-providers.md) + +### 🤖 Tool Consumers +You're building AI agents or applications that need to call external tools: +- **AI agent developers** - Give your agents access to external capabilities +- **Application builders** - Integrate third-party tools seamlessly +- **Enterprise developers** - Connect to internal and external services + +[**Get started as a tool consumer →**](./implementation.md) + +## UTCP vs Alternatives + +| Feature | UTCP | MCP | Custom Wrappers | +|---------|------|-----|-----------------| +| **Infrastructure** | None required | Wrapper servers | Custom servers | +| **Latency** | Direct calls | Double hop | Variable | +| **Security** | Native | Reimplemented | Custom | +| **Protocols** | Multiple | HTTP streaming | Single | +| **Maintenance** | Minimal | High | Very high | + +[**Detailed comparison with MCP →**](./utcp-vs-mcp.md) + +## Next Steps + +### For Tool Providers +1. **[Read the provider guide](./for-tool-providers.md)** - Learn how to expose your tools +2. **[Choose your protocol](./protocols/index.md)** - Select the right communication method +3. **[Implement your manual](./implementation.md)** - Add UTCP to your existing API +4. **[Secure your tools](./security.md)** - Implement proper authentication + +### For Tool Consumers +1. **[Read the implementation guide](./implementation.md)** - Learn how to build UTCP clients +2. **[Explore protocols](./protocols/index.md)** - Understand available communication options +3. **[Check examples](https://github.com/universal-tool-calling-protocol)** - See real-world implementations +4. **[Join the community](https://discord.gg/ZpMbQ8jRbD)** - Get help and share experiences + +### Migration from Other Systems +- **[From UTCP v0.1](./migration-v0.1-to-v1.0.md)** - Upgrade to the latest version +- **[From MCP](./protocols/mcp.md)** - Migrate from Model Context Protocol +- **[From custom solutions](./implementation.md)** - Replace existing tool integrations + +## Community & Support + +- **[GitHub Organization](https://github.com/universal-tool-calling-protocol)** - Source code and issues +- **[Discord Community](https://discord.gg/ZpMbQ8jRbD)** - Real-time help and discussions +- **[Tool Registry](https://utcp.io/registry)** - Discover available tools +- **[RFC Process](/about/RFC)** - Contribute to the specification + +--- -In the following sections, we'll explore the details of UTCP's components and how to implement them in your applications. +**Ready to get started?** Choose your path: +- 🛠️ [**I want to expose my tools**](./for-tool-providers.md) +- 🤖 [**I want to call tools**](./implementation.md) +- 📚 [**I want to learn more**](./protocols/index.md) diff --git a/versioned_docs/version-1.0/migration-v0.1-to-v1.0.md b/versioned_docs/version-1.0/migration-v0.1-to-v1.0.md new file mode 100644 index 0000000..eb50de7 --- /dev/null +++ b/versioned_docs/version-1.0/migration-v0.1-to-v1.0.md @@ -0,0 +1,465 @@ +--- +id: migration-v0.1-to-v1.0 +title: Migration Guide - v0.1 to v1.0 +sidebar_position: 7 +--- + +# Migration Guide: v0.1 to v1.0 + +This guide helps you migrate from UTCP v0.1 to v1.0, which introduces significant architectural improvements and new features. + +## Overview of Changes + +### Major Changes in v1.0 + +1. **Plugin Architecture**: Core functionality split into pluggable components +2. **Improved Data Models**: Enhanced Pydantic models with better validation +3. **New Protocol Support**: Additional communication protocols +4. **Better Error Handling**: More specific exception types +5. **Enhanced Authentication**: Expanded authentication options +6. **Performance Improvements**: Optimized client and protocol implementations + +### Breaking Changes + +| Component | v0.1 | v1.0 | Impact | +|-----------|------|------|--------| +| **Package Structure** | Single package | Core + plugins | High | +| **Client API** | `UtcpClient()` | `UtcpClient.create()` | Medium | +| **Configuration** | Simple dict | Structured config | Medium | +| **Protocol Registration** | Automatic | Plugin-based | High | +| **Error Types** | Generic exceptions | Specific exception types | Low | + +## Installation Changes + +### v0.1 Installation + +```bash +pip install utcp +``` + +### v1.0 Installation + +```bash +# Core package +pip install utcp + +# Protocol plugins (install as needed) +pip install utcp-http utcp-cli utcp-websocket utcp-text utcp-mcp +``` + +## Client Migration + +### v0.1 Client Code + +**Legacy v0.1 client configuration:** +- Import UTCP client library +- Configure providers with provider-specific settings (name, type, URL, HTTP method) +- Call tools using provider.tool_name format +- Synchronous tool calling interface + +### v1.0 Client Code + +**New v1.0 client configuration:** +- Import updated UTCP client library from new module path +- Use async factory method for client creation +- Configure manual call templates instead of providers +- Use async/await pattern for tool calling +- Enhanced error handling and response processing + +## Configuration Migration + +### v0.1 Configuration + +```yaml +providers: + - name: weather_service + provider_type: http + url: https://weather.example.com/utcp + http_method: GET + - name: file_service + provider_type: cli + command: cat + args: ["${filename}"] + +variables: + API_KEY: your_api_key +``` + +### v1.0 Configuration + +```yaml +manual_call_templates: + - name: weather_service + call_template_type: http + url: https://weather.example.com/utcp + http_method: GET + - name: file_service + call_template_type: cli + command: cat + args: ["${filename}"] + +load_variables_from: + - variable_loader_type: dotenv + env_file_path: .env +``` + +## Manual Format Migration + +### v0.1 Manual Format + +```json +{ + "utcp_version": "0.1.0", + "provider_info": { + "name": "weather_api", + "version": "1.0.0" + }, + "tools": [ + { + "name": "get_weather", + "description": "Get weather data", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + } + }, + "provider": { + "provider_type": "http", + "url": "https://api.weather.com/current", + "method": "GET" + } + } + ] +} +``` + +### v1.0 Manual Format + +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "info": { + "title": "Weather API", + "version": "1.0.0", + "description": "Weather data and forecasting tools" + }, + "tools": [ + { + "name": "get_weather", + "description": "Get weather data", + "inputs": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + }, + "outputs": { + "type": "object", + "properties": { + "temperature": {"type": "number"}, + "conditions": {"type": "string"} + } + }, + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.weather.com/current", + "http_method": "GET", + }, + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "appid", + "location": "query" + } + } + } + ] +} +``` + +## Protocol Migration + +### HTTP Protocol Changes + +#### v0.1 HTTP Provider + +```json +{ + "provider_type": "http", + "url": "https://api.example.com/endpoint", + "method": "POST", + "headers": {"Authorization": "Bearer ${TOKEN}"}, + "body": {"data": "${input}"} +} +``` + +#### v1.0 HTTP Call Template + +```json +{ + "call_template_type": "http", + "url": "https://api.example.com/endpoint", + "http_method": "POST", + "headers": {"Authorization": "Bearer ${TOKEN}"}, + "body_field": "body", + "auth": { + "auth_type": "api_key", + "api_key": "${TOKEN}", + "var_name": "Authorization", + "location": "header" + } +} +``` + +### CLI Protocol Changes + +#### v0.1 CLI Provider + +```json +{ + "provider_type": "cli", + "command": "python", + "args": ["script.py", "${input}"], + "cwd": "/app" +} +``` + +#### v1.0 CLI Call Template + +```json +{ + "call_template_type": "cli", + "commands": [ + { + "command": "cd /app", + "append_to_final_output": false + }, + { + "command": "python script.py UTCP_ARG_input_UTCP_END", + "append_to_final_output": true + } + ], + "working_dir": "/app", + "env_vars": { + "PYTHONPATH": "/app/lib" + } +} +``` + +## Authentication Migration + +### v0.1 Authentication + +```json +{ + "provider": { + "provider_type": "http", + "url": "https://api.example.com/data", + "headers": { + "Authorization": "Bearer ${API_TOKEN}" + } + } +} +``` + +### v1.0 Authentication + +```json +{ + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/data", + "auth": { + "auth_type": "api_key", + "api_key": "${API_TOKEN}", + "var_name": "Authorization", + "location": "header" + } + } +} +``` + +## Error Handling Migration + +### v0.1 Error Handling + +**Basic error handling in v0.1:** +- Simple try/catch block with generic Exception handling +- Limited error information and context +- Basic error message printing + +### v1.0 Error Handling + +**Enhanced error handling in v1.0:** +- Import specific exception types from utcp.exceptions module +- Handle ToolNotFoundError for missing tools +- Handle AuthenticationError for auth failures +- Handle ToolCallError for tool execution failures +- Handle base UtcpError for general UTCP errors +- Use try/catch blocks with specific exception handling + +## Step-by-Step Migration + +### Step 1: Update Dependencies + +```bash +# Uninstall old version +pip uninstall utcp + +# Install new version with plugins +pip install utcp utcp-http utcp-cli utcp-websocket utcp-text +``` + +### Step 2: Update Client Code + +**Migration to async pattern:** +- **Before**: Synchronous client creation and tool calls +- **After**: Async client factory method and async tool calls +- Import asyncio module for async execution +- Use async/await keywords for client creation and tool calls +- Run async main function with asyncio.run() + +### Step 3: Update Configuration + +**Configuration migration helper:** +- Create migration function to convert v0.1 config to v1.0 format +- Transform providers array to manual_call_templates array +- Add variable_loaders configuration for environment variables +- Map provider_type to call_template_type +- Migrate HTTP provider settings (URL, method, headers, body) +- Migrate CLI provider settings (command, args, working_directory) +- Load old configuration and apply migration helper +- Create new client with migrated configuration + +### Step 4: Update Manual Format + +**Manual migration helper:** +- Create migration function to convert v0.1 manual to v1.0 format +- Update manual_version and utcp_version fields +- Transform provider_info to info structure (title, version, description) +- Migrate tools array with updated structure +- Convert tool parameters to inputs field +- Transform provider configuration to tool_call_template +- Map provider_type to call_template_type +- Migrate HTTP provider settings (URL, method, headers, body) + +### Step 5: Test Migration + +**Testing migrated client:** +- Import pytest and UtcpClient for async testing +- Create test function with pytest.mark.asyncio decorator +- Test client creation with migrated configuration +- Test tool discovery functionality (list_tools) +- Test tool calling with sample parameters +- Assert expected results and functionality + +## Common Migration Issues + +### Issue 1: Async/Await + +**Problem**: v1.0 client methods are async +**Solution**: Add `async`/`await` keywords + +**Code changes:** +- **Before**: Synchronous tool calling (result = client.call_tool("tool", args)) +- **After**: Async tool calling (result = await client.call_tool("tool", args)) + +### Issue 2: Configuration Format + +**Problem**: Configuration structure changed +**Solution**: Use migration helper or update manually + +**Configuration changes:** +- **Before**: Use "providers" array in configuration +- **After**: Use "manual_call_templates" array in configuration + +### Issue 3: Plugin Dependencies + +**Problem**: Protocol implementations not found +**Solution**: Install required plugins + +```bash +pip install utcp-http utcp-cli utcp-websocket +``` + +### Issue 4: Manual Format + +**Problem**: Old manual format not recognized +**Solution**: Update manual structure + +```json +// Before +{"provider": {"provider_type": "http"}} + +// After +{"tool_call_template": {"call_template_type": "http"}} +``` + +## Validation Tools + +### Configuration Validator + +**Configuration validation helper:** +- Import UtcpClientConfig from utcp.data.utcp_client_config +- Create validation function that accepts configuration dictionary +- Use UtcpClientConfig constructor to validate configuration structure +- Handle validation exceptions and provide error messages +- Return validated config object or None on failure + +### Manual Validator + +**Manual validation helper:** +- Import UtcpManual from utcp.data.utcp_manual +- Create validation function that accepts manual dictionary +- Use UtcpManual constructor to validate manual structure +- Handle validation exceptions and provide error messages +- Return validated manual object or None on failure + +## Best Practices for Migration + +1. **Gradual Migration**: Migrate one component at a time +2. **Test Thoroughly**: Test each migrated component +3. **Backup Configurations**: Keep backups of v0.1 configurations +4. **Use Validation**: Validate configurations and manuals +5. **Monitor Performance**: Check for performance regressions +6. **Update Documentation**: Update internal documentation +7. **Train Team**: Ensure team understands new patterns + +## Post-Migration Checklist + +- [ ] All dependencies updated +- [ ] Client code uses async/await +- [ ] Configuration format updated +- [ ] Manual format updated +- [ ] Error handling updated +- [ ] Tests passing +- [ ] Performance acceptable +- [ ] Documentation updated +- [ ] Team trained on changes + +## Getting Help + +If you encounter issues during migration: + +1. **Check Documentation**: Review the [Implementation Guide](./implementation.md) +2. **GitHub Issues**: Search existing issues or create new ones +3. **Discord Community**: Join the [UTCP Discord](https://discord.gg/ZpMbQ8jRbD) +4. **Examples**: Check the [examples repository](https://github.com/universal-tool-calling-protocol) for implementations across multiple languages + +## Rollback Plan + +If migration issues occur, you can rollback: + +```bash +# Rollback to v0.1 +pip uninstall utcp utcp-http utcp-cli utcp-websocket utcp-text +pip install utcp==0.1.0 + +# Restore old configuration files +cp config-v0.1-backup.yaml config.yaml +``` + +Remember to test the rollback process in a non-production environment first. diff --git a/versioned_docs/version-1.0/protocols/cli.md b/versioned_docs/version-1.0/protocols/cli.md new file mode 100644 index 0000000..f0ac213 --- /dev/null +++ b/versioned_docs/version-1.0/protocols/cli.md @@ -0,0 +1,471 @@ +--- +id: cli +title: CLI Protocol +sidebar_position: 4 +--- + +# CLI Protocol + +The CLI protocol plugin (`utcp-cli`) enables UTCP to execute multi-command workflows and scripts. This protocol supports sequential command execution with state preservation and cross-platform compatibility. + +## Installation + +```bash +# Example installation (Python) +pip install utcp-cli + +# Example installation (Node.js) +npm install @utcp/cli +``` + +## Key Features + +- **Multi-Command Execution**: Execute multiple commands sequentially in a single subprocess +- **State Preservation**: Directory changes and environment persist between commands +- **Cross-Platform Script Generation**: PowerShell on Windows, Bash on Unix/Linux/macOS +- **Flexible Output Control**: Choose which command outputs to include in final result +- **Argument Substitution**: `UTCP_ARG_argname_UTCP_END` placeholder system +- **Output Referencing**: Access previous command outputs with `$CMD_0_OUTPUT`, `$CMD_1_OUTPUT` +- **Environment Variables**: Secure credential and configuration passing + +## Call Template Structure + +```json +{ + "call_template_type": "cli", + "commands": [ + { + "command": "cd UTCP_ARG_target_dir_UTCP_END", + "append_to_final_output": false + }, + { + "command": "git status --porcelain", + "append_to_final_output": false + }, + { + "command": "echo \"Status: $CMD_1_OUTPUT, Files: $(echo \"$CMD_1_OUTPUT\" | wc -l)\"", + "append_to_final_output": true + } + ], + "working_dir": "/tmp", + "env_vars": { + "GIT_AUTHOR_NAME": "UTCP Bot" + } +} +``` + +## Configuration Options + +### Required Fields + +- **`call_template_type`**: Must be `"cli"` +- **`commands`**: Array of `CommandStep` objects defining the sequence of commands + +### Optional Fields + +- **`working_dir`**: Working directory for command execution +- **`env_vars`**: Environment variables to set for all commands + +### CommandStep Object + +- **`command`**: Command string with `UTCP_ARG_argname_UTCP_END` placeholders +- **`append_to_final_output`**: Whether to include this command's output in the final result (defaults to `false` for all except the last command) + +For complete field specifications and validation rules, see the [CLI Call Template API Reference](../api/plugins/communication_protocols/cli/src/utcp_cli/cli_call_template.md). + +## Argument Substitution + +Use `UTCP_ARG_argname_UTCP_END` placeholders in command strings: + +```json +{ + "command": "git clone UTCP_ARG_repo_url_UTCP_END UTCP_ARG_target_dir_UTCP_END" +} +``` + +## Output Referencing + +Reference previous command outputs using `$CMD_N_OUTPUT` variables: + +```json +{ + "commands": [ + { + "command": "git status --porcelain", + "append_to_final_output": false + }, + { + "command": "echo \"Changes detected: $CMD_0_OUTPUT\"", + "append_to_final_output": true + } + ] +} +``` + +## Security Considerations + +:::danger Security Warning +CLI protocol executes commands on the local system. Always validate inputs and use with caution. +::: + +### Input Validation + +Always validate and sanitize inputs to prevent command injection: + +```json +{ + "name": "safe_file_read", + "description": "Safely read a file", + "inputs": { + "type": "object", + "properties": { + "filename": { + "type": "string", + "pattern": "^[a-zA-Z0-9._-]+$" + } + }, + "required": ["filename"] + }, + "tool_call_template": { + "call_template_type": "cli", + "commands": [ + { + "command": "cd /safe/directory", + "append_to_final_output": false + }, + { + "command": "cat UTCP_ARG_filename_UTCP_END", + "append_to_final_output": true + } + ], + "working_dir": "/safe/directory" + } +} +``` + +### Sandboxing + +Consider running CLI tools in sandboxed environments: + +```json +{ + "call_template_type": "cli", + "commands": [ + { + "command": "docker run --rm --read-only -v /safe/data:/data:ro alpine:latest cat /data/UTCP_ARG_filename_UTCP_END", + "append_to_final_output": true + } + ] +} +``` + +## Examples + +### Simple Command Execution + +```json +{ + "name": "get_system_info", + "description": "Get system information", + "inputs": { + "type": "object", + "properties": {} + }, + "tool_call_template": { + "call_template_type": "cli", + "commands": [ + { + "command": "uname -a", + "append_to_final_output": true + } + ] + } +} +``` + +### Multi-Step File Analysis + +```json +{ + "name": "analyze_directory", + "description": "Analyze files in a directory", + "inputs": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Directory path to analyze" + } + }, + "required": ["path"] + }, + "tool_call_template": { + "call_template_type": "cli", + "commands": [ + { + "command": "cd UTCP_ARG_path_UTCP_END", + "append_to_final_output": false + }, + { + "command": "find . -type f | wc -l", + "append_to_final_output": false + }, + { + "command": "du -sh .", + "append_to_final_output": false + }, + { + "command": "echo \"Directory Analysis: $CMD_2_OUTPUT total size, $CMD_1_OUTPUT files\"", + "append_to_final_output": true + } + ] + } +} +``` + +### Git Workflow + +```json +{ + "name": "git_analysis", + "description": "Analyze git repository", + "inputs": { + "type": "object", + "properties": { + "repo_url": {"type": "string"}, + "target_dir": {"type": "string"} + }, + "required": ["repo_url", "target_dir"] + }, + "tool_call_template": { + "call_template_type": "cli", + "commands": [ + { + "command": "git clone UTCP_ARG_repo_url_UTCP_END UTCP_ARG_target_dir_UTCP_END", + "append_to_final_output": false + }, + { + "command": "cd UTCP_ARG_target_dir_UTCP_END", + "append_to_final_output": false + }, + { + "command": "git log --oneline -10", + "append_to_final_output": true + }, + { + "command": "find . -name '*.py' | wc -l", + "append_to_final_output": false + }, + { + "command": "echo \"Repository has $CMD_3_OUTPUT Python files\"", + "append_to_final_output": true + } + ], + "env_vars": { + "GIT_AUTHOR_NAME": "UTCP Bot", + "GIT_AUTHOR_EMAIL": "bot@utcp.dev" + } + } +} +``` + +### Python Development Pipeline + +```json +{ + "name": "python_pipeline", + "description": "Run Python development pipeline", + "inputs": { + "type": "object", + "properties": { + "project_dir": {"type": "string"}, + "script_name": {"type": "string"} + }, + "required": ["project_dir", "script_name"] + }, + "tool_call_template": { + "call_template_type": "cli", + "commands": [ + { + "command": "cd UTCP_ARG_project_dir_UTCP_END", + "append_to_final_output": false + }, + { + "command": "python -m pip install -r requirements.txt", + "append_to_final_output": false + }, + { + "command": "python UTCP_ARG_script_name_UTCP_END", + "append_to_final_output": true + } + ], + "working_dir": "/workspace", + "env_vars": { + "PYTHONPATH": "/workspace/lib", + "VIRTUAL_ENV": "/workspace/venv" + } + } +} +``` + +## Output Handling + +The CLI protocol executes all commands in a single subprocess and returns the combined output based on `append_to_final_output` settings: + +- Commands with `append_to_final_output: true` contribute to the final result +- Commands with `append_to_final_output: false` are executed for side effects only +- If no `append_to_final_output` is specified, only the last command's output is returned +- JSON output is automatically parsed if detected + +### Success Response + +```json +"Directory Analysis: 2.1G total size, 1247 files" +``` + +### JSON Output Detection + +If output starts with `{` or `[`, it's automatically parsed as JSON: + +```json +{ + "files": 1247, + "size": "2.1G", + "analysis_time": "2023-12-01T10:30:00Z" +} +``` + +## Environment Variables + +Set environment variables for all commands: + +```json +{ + "call_template_type": "cli", + "commands": [ + { + "command": "node app.js", + "append_to_final_output": true + } + ], + "env_vars": { + "NODE_ENV": "production", + "API_KEY": "${API_KEY}", + "PORT": "3000" + } +} +``` + +## Working Directory + +Specify the initial working directory: + +```json +{ + "call_template_type": "cli", + "commands": [ + { + "command": "make build", + "append_to_final_output": true + } + ], + "working_dir": "/project/src" +} +``` + +## Cross-Platform Considerations + +### Command Syntax + +Commands should use appropriate syntax for the target platform: + +**Windows (PowerShell):** +```json +{ + "commands": [ + {"command": "Get-ChildItem UTCP_ARG_path_UTCP_END"}, + {"command": "Set-Location UTCP_ARG_dir_UTCP_END"} + ] +} +``` + +**Unix/Linux/macOS (Bash):** +```json +{ + "commands": [ + {"command": "ls -la UTCP_ARG_path_UTCP_END"}, + {"command": "cd UTCP_ARG_dir_UTCP_END"} + ] +} +``` + +### Universal Commands + +Some commands work across platforms: +```json +{ + "commands": [ + {"command": "git status"}, + {"command": "python --version"}, + {"command": "node -v"} + ] +} +``` + +## Best Practices + +1. **Validate Inputs**: Always validate and sanitize user inputs using JSON Schema patterns +2. **Use Absolute Paths**: Prefer absolute paths for commands and files when possible +3. **Control Output**: Use `append_to_final_output` to control which command outputs are returned +4. **Reference Previous Output**: Use `$CMD_N_OUTPUT` to reference previous command results +5. **Limit Permissions**: Run with minimal necessary permissions +6. **Sandbox Execution**: Use containers or chroot when possible +7. **Handle Cross-Platform**: Consider platform-specific command syntax +8. **Environment Variables**: Use `env_vars` for secure credential passing + +## Common Use Cases + +- **Multi-step Builds**: setup → compile → test → package +- **Git Workflows**: clone → analyze → commit → push +- **Data Pipelines**: fetch → transform → validate → output +- **File Operations**: navigate → search → process → report +- **Development Tools**: install dependencies → run tests → generate docs +- **System Administration**: check status → backup → cleanup → verify + +## Error Handling + +| Error Type | Description | Handling | +|------------|-------------|---------| +| Missing Arguments | Required UTCP_ARG placeholder not provided | Validation error | +| Command Not Found | Command doesn't exist | Script execution error | +| Permission Denied | Insufficient permissions | Script execution error | +| Timeout | Script exceeded timeout | Async timeout error | +| Non-zero Exit | Script failed | Return stderr output | + +## Testing CLI Tools + +```python +import pytest +from utcp.utcp_client import UtcpClient + +@pytest.mark.asyncio +async def test_multi_command_cli_tool(): + client = await UtcpClient.create(config={ + "manual_call_templates": [{ + "name": "test_cli", + "call_template_type": "cli", + "commands": [ + { + "command": "echo UTCP_ARG_message_UTCP_END", + "append_to_final_output": false + }, + { + "command": "echo \"Previous: $CMD_0_OUTPUT\"" + } + ] + }] + }) + + result = await client.call_tool("test_cli.echo_chain", {"message": "hello"}) + assert "Previous: hello" in result +``` diff --git a/versioned_docs/version-1.0/protocols/http.md b/versioned_docs/version-1.0/protocols/http.md new file mode 100644 index 0000000..6340aeb --- /dev/null +++ b/versioned_docs/version-1.0/protocols/http.md @@ -0,0 +1,236 @@ +--- +id: http +title: HTTP Protocol +sidebar_position: 1 +--- + +# HTTP Protocol Plugin + +The HTTP protocol plugin enables UTCP to call tools via HTTP/HTTPS requests. This is the most commonly used protocol for REST APIs, webhooks, and web services. + +## Call Template Structure + +```json +{ + "call_template_type": "http", + "url": "https://api.example.com/endpoint", + "http_method": "GET|POST|PUT|DELETE|PATCH", + "headers": { + "Content-Type": "application/json", + "User-Agent": "UTCP-Client/1.0" + }, + "query_params": { + "param1": "${variable1}", + "param2": "static_value" + }, + "body": { + "data": "${input_data}", + "timestamp": "${current_time}" + }, + "timeout": 30, + "verify_ssl": true, + "auth": { + "auth_type": "api_key|basic|oauth2", + "api_key": "${API_KEY}", + "var_name": "Authorization", + "location": "header" + } +} +``` + +## Supported HTTP Methods + +| Method | Use Case | Body Support | +|--------|----------|--------------| +| `GET` | Retrieve data | No | +| `POST` | Create resources, submit data | Yes | +| `PUT` | Update/replace resources | Yes | +| `PATCH` | Partial updates | Yes | +| `DELETE` | Remove resources | Optional | + +## Authentication Methods + +### API Key Authentication +```json +{ + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "X-API-Key", + "location": "header" + } +} +``` + +### Basic Authentication +```json +{ + "auth": { + "auth_type": "basic", + "username": "${USERNAME}", + "password": "${PASSWORD}" + } +} +``` + +### OAuth2 Authentication +```json +{ + "auth": { + "auth_type": "oauth2", + "client_id": "${CLIENT_ID}", + "client_secret": "${CLIENT_SECRET}", + "token_url": "https://auth.example.com/token", + "scope": "read write" + } +} +``` + +## Variable Substitution + +Variables in call templates are substituted with values from: +- Tool call arguments: `${argument_name}` +- Environment variables: `${ENV_VAR}` +- Configuration variables: `${config.variable}` + +Example: +```json +{ + "url": "https://api.example.com/users/${user_id}", + "headers": { + "Authorization": "Bearer ${ACCESS_TOKEN}" + }, + "query_params": { + "format": "${output_format}", + "limit": "${max_results}" + } +} +``` + +## OpenAPI Integration + +The HTTP protocol plugin can automatically generate UTCP manuals from OpenAPI/Swagger specifications: + +1. **Automatic Discovery**: Point to an OpenAPI spec URL +2. **Schema Conversion**: Converts OpenAPI paths to UTCP tools +3. **Authentication Mapping**: Maps OpenAPI security schemes to UTCP auth +4. **Parameter Mapping**: Converts OpenAPI parameters to UTCP inputs + +Example OpenAPI to UTCP conversion: +```yaml +# OpenAPI Specification +paths: + /users/{id}: + get: + parameters: + - name: id + in: path + required: true + schema: + type: string +``` + +Becomes: +```json +{ + "name": "get_user", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/users/${id}", + "http_method": "GET" + }, + "inputs": { + "type": "object", + "properties": { + "id": {"type": "string"} + }, + "required": ["id"] + } +} +``` + +## Response Handling + +HTTP responses are processed based on: +- **Status Codes**: 2xx considered success, others as errors +- **Content-Type**: JSON, XML, text, and binary content support +- **Headers**: Response headers available in tool output +- **Error Mapping**: HTTP errors mapped to UTCP exceptions + +## Configuration Options + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `timeout` | number | 30 | Request timeout in seconds | +| `verify_ssl` | boolean | true | Verify SSL certificates | +| `follow_redirects` | boolean | true | Follow HTTP redirects | +| `max_redirects` | number | 5 | Maximum redirect hops | +| `retry_count` | number | 0 | Number of retry attempts | +| `retry_delay` | number | 1 | Delay between retries (seconds) | + +## Security Considerations + +### SSL/TLS Verification +- Always verify SSL certificates in production +- Use `verify_ssl: false` only for testing/development +- Consider certificate pinning for high-security applications + +### Authentication Security +- Store credentials in environment variables, not in configuration files +- Use OAuth2 for user-facing applications +- Rotate API keys regularly +- Implement proper token refresh for OAuth2 + +### Input Validation +- Validate all input parameters before substitution +- Sanitize user inputs to prevent injection attacks +- Use allowlists for acceptable parameter values +- Implement rate limiting on the tool provider side + +### Network Security +- Use HTTPS for all production communications +- Implement proper firewall rules +- Consider using VPNs or private networks for sensitive tools +- Monitor and log all HTTP requests for security analysis + +## Error Handling + +Common HTTP errors and their meanings: + +| Status Code | Error Type | Description | +|-------------|------------|-------------| +| 400 | Bad Request | Invalid request parameters | +| 401 | Unauthorized | Authentication required or failed | +| 403 | Forbidden | Access denied | +| 404 | Not Found | Resource doesn't exist | +| 429 | Rate Limited | Too many requests | +| 500 | Server Error | Internal server error | +| 503 | Service Unavailable | Service temporarily unavailable | + +## Best Practices + +### Performance +- Use connection pooling for multiple requests +- Implement appropriate timeouts +- Consider request/response compression +- Cache responses when appropriate + +### Reliability +- Implement retry logic with exponential backoff +- Handle network failures gracefully +- Use circuit breakers for unreliable services +- Monitor response times and error rates + +### Maintainability +- Use descriptive tool names and descriptions +- Document all required parameters +- Provide usage examples +- Version your APIs and update call templates accordingly + +## Language-Specific Implementation + +For implementation details and examples in your programming language: + +- **Multi-language**: [UTCP HTTP Protocol Examples](https://github.com/universal-tool-calling-protocol) - HTTP protocol examples across multiple languages +- **TypeScript**: [TypeScript HTTP Protocol Documentation](https://github.com/universal-tool-calling-protocol/typescript-utcp/blob/main/docs/protocols/http.md) +- **Other languages**: Check respective repositories in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) diff --git a/versioned_docs/version-1.0/protocols/index.md b/versioned_docs/version-1.0/protocols/index.md new file mode 100644 index 0000000..9a4f701 --- /dev/null +++ b/versioned_docs/version-1.0/protocols/index.md @@ -0,0 +1,93 @@ +--- +id: protocols +title: Protocols +sidebar_position: 4 +--- + +# Communication Protocol Plugins + +UTCP v1.0 features a modular, plugin-based architecture where different communication protocols are implemented as separate plugins. Each protocol plugin provides call templates and communication handlers for specific transport methods. + +## Available Protocol Plugins + +| Protocol | Plugin Package | Call Template | Use Cases | +|----------|----------------|---------------|-----------| +| **[HTTP](./http.md)** | `utcp-http` | `HttpCallTemplate` | REST APIs, webhooks, web services | +| **[WebSocket](./websocket.md)** | `utcp-websocket` | `WebSocketCallTemplate` | Real-time communication, streaming | +| **[CLI](./cli.md)** | `utcp-cli` | `CliCallTemplate` | Command-line tools, scripts | +| **[Server-Sent Events](./sse.md)** | `utcp-http` | `SseCallTemplate` | Event streaming, live updates | +| **[Text Files](./text.md)** | `utcp-text` | `TextCallTemplate` | File reading, static content | +| **[MCP](./mcp.md)** | `utcp-mcp` | `McpCallTemplate` | Model Context Protocol interop | + +## Plugin Architecture + +Each protocol plugin consists of: + +### Call Templates +Define how to structure calls for the specific protocol: +```json +{ + "call_template_type": "http", + "url": "https://api.example.com/endpoint", + "http_method": "POST", + "headers": {"Content-Type": "application/json"} +} +``` + +### Communication Protocols +Handle the actual communication logic for each protocol type. The implementation varies by programming language. + +## Installing Protocol Plugins + +Protocol plugins are available for different programming languages: + +```bash +# Example installation (Python) +pip install utcp-http utcp-cli utcp-websocket utcp-text utcp-mcp + +# Example installation (Node.js) +npm install @utcp/http @utcp/cli @utcp/websocket @utcp/text @utcp/mcp +``` + +For other languages, check the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) + +## Creating Custom Protocol Plugins + +You can extend UTCP with custom communication protocols by implementing the protocol interface in your chosen language. Each implementation must: + +1. **Define Call Templates**: Specify the structure for protocol-specific calls +2. **Implement Communication Handler**: Handle the actual protocol communication +3. **Register the Protocol**: Make it available to the UTCP client + +Example call template structure: +```json +{ + "call_template_type": "custom", + "custom_field": "value", + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}" + } +} +``` + +## Protocol Selection Guide + +Choose the right protocol plugin based on your needs: + +- **HTTP**: Most common for REST APIs and web services +- **WebSocket**: Real-time bidirectional communication +- **CLI**: Wrapping existing command-line tools +- **SSE**: Server-sent events for streaming data +- **Text**: Reading configuration files or static content +- **MCP**: Interoperability with Model Context Protocol tools + +## Language-Specific Documentation + +For implementation details and examples in your programming language: + +- **Multi-language**: [UTCP Implementation Examples](https://github.com/universal-tool-calling-protocol) - Examples across Python, TypeScript, Go, and other languages +- **TypeScript**: [TypeScript UTCP Documentation](https://github.com/universal-tool-calling-protocol/typescript-utcp/tree/main/docs) +- **Other languages**: Check respective repositories in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) + +For detailed information about each protocol plugin, see the individual protocol documentation pages. diff --git a/versioned_docs/version-1.0/protocols/mcp.md b/versioned_docs/version-1.0/protocols/mcp.md new file mode 100644 index 0000000..13da0f6 --- /dev/null +++ b/versioned_docs/version-1.0/protocols/mcp.md @@ -0,0 +1,266 @@ +--- +id: mcp +title: MCP Protocol +sidebar_position: 6 +--- + +# MCP Protocol Plugin + +The MCP (Model Context Protocol) plugin provides interoperability between UTCP and existing MCP servers, enabling gradual migration from MCP to UTCP while maintaining compatibility with existing MCP tools. + +## Call Template Structure + +```json +{ + "call_template_type": "mcp", + "server_config": { + "command": "node", + "args": ["mcp-server.js"], + "working_directory": "/app/mcp", + "env": { + "NODE_ENV": "production", + "LOG_LEVEL": "info" + }, + "timeout": 30 + }, + "connection_timeout": 10, + "request_timeout": 30 +} +``` + +## Server Configuration + +### Command-based Servers +```json +{ + "server_config": { + "command": "python", + "args": ["-m", "mcp_server", "--config", "config.json"], + "working_directory": "/app", + "env": { + "PYTHONPATH": "/app/lib", + "API_KEY": "${MCP_API_KEY}" + } + } +} +``` + +### HTTP-based Servers +```json +{ + "server_config": { + "transport": "http", + "url": "http://localhost:8080/mcp", + "headers": { + "Authorization": "Bearer ${MCP_TOKEN}" + } + } +} +``` + +## Migration Strategy + +The MCP protocol plugin enables a gradual migration path from MCP to native UTCP protocols: + +### Phase 1: MCP Integration +- Use existing MCP servers through UTCP +- No changes to MCP server code required +- UTCP client can call MCP tools seamlessly + +### Phase 2: Hybrid Approach +- Some tools use native UTCP protocols +- Legacy tools continue using MCP +- Gradual migration of high-value tools + +### Phase 3: Full Migration +- All tools use native UTCP protocols +- MCP servers deprecated +- Simplified architecture + +## Tool Discovery + +MCP servers expose tools through the standard MCP protocol. The UTCP MCP plugin: + +1. **Connects** to the MCP server using stdio or HTTP transport +2. **Discovers** available tools via MCP's `tools/list` method +3. **Maps** MCP tool definitions to UTCP tool format +4. **Registers** tools in the UTCP client + +## Request/Response Mapping + +### MCP to UTCP Tool Mapping +```json +// MCP Tool Definition +{ + "name": "read_file", + "description": "Read contents of a file", + "inputSchema": { + "type": "object", + "properties": { + "path": {"type": "string"} + }, + "required": ["path"] + } +} + +// UTCP Tool (after mapping) +{ + "name": "read_file", + "description": "Read contents of a file", + "inputs": { + "type": "object", + "properties": { + "path": {"type": "string"} + }, + "required": ["path"] + }, + "tool_call_template": { + "call_template_type": "mcp", + "server_config": {...} + } +} +``` + +### Request Flow +1. UTCP client receives tool call +2. MCP plugin formats request as MCP `tools/call` +3. Request sent to MCP server +4. MCP server processes and responds +5. Response mapped back to UTCP format + +## Authentication and Security + +### Server Authentication +```json +{ + "server_config": { + "command": "secure-mcp-server", + "env": { + "MCP_AUTH_TOKEN": "${MCP_SERVER_TOKEN}", + "MCP_CLIENT_ID": "${MCP_CLIENT_ID}" + } + } +} +``` + +### Transport Security +- **stdio**: Inherits process security model +- **HTTP**: Use HTTPS and proper authentication headers +- **WebSocket**: Use WSS and authentication tokens + +## Error Handling + +### Connection Errors +- Server startup failures +- Network connectivity issues +- Authentication failures +- Timeout errors + +### Protocol Errors +- Invalid MCP messages +- Unsupported MCP features +- Tool execution failures +- Resource access errors + +### Error Mapping +MCP errors are mapped to UTCP exceptions: +- `InvalidRequest` → `ValidationError` +- `MethodNotFound` → `ToolNotFoundError` +- `InternalError` → `ToolCallError` + +## Performance Considerations + +### Connection Management +- Persistent connections for stdio transport +- Connection pooling for HTTP transport +- Automatic reconnection on failures +- Graceful shutdown handling + +### Request Optimization +- Batch multiple tool calls when possible +- Cache tool discovery results +- Implement request timeouts +- Monitor response times + +## Limitations + +### MCP Feature Support +Not all MCP features are supported through UTCP: +- **Resources**: Not directly mapped to UTCP tools +- **Prompts**: Not supported in UTCP model +- **Sampling**: Not applicable to tool calling + +### Protocol Differences +- MCP's bidirectional communication vs UTCP's request/response +- MCP's resource model vs UTCP's tool-only model +- Different authentication mechanisms + +## Configuration Examples + +### Development Setup +```json +{ + "manual_call_templates": [{ + "name": "dev_mcp", + "call_template_type": "mcp", + "server_config": { + "command": "node", + "args": ["dev-server.js"], + "env": {"NODE_ENV": "development"} + }, + "connection_timeout": 5, + "request_timeout": 10 + }] +} +``` + +### Production Setup +```json +{ + "manual_call_templates": [{ + "name": "prod_mcp", + "call_template_type": "mcp", + "server_config": { + "transport": "http", + "url": "https://mcp.example.com/api", + "headers": { + "Authorization": "Bearer ${MCP_PROD_TOKEN}", + "X-Client-Version": "1.0.0" + } + }, + "connection_timeout": 30, + "request_timeout": 60 + }] +} +``` + +## Best Practices + +### Migration Planning +1. **Inventory** existing MCP servers and tools +2. **Prioritize** tools for migration based on usage +3. **Test** MCP integration thoroughly +4. **Monitor** performance and reliability +5. **Migrate** incrementally to native UTCP protocols + +### Monitoring and Debugging +- Enable debug logging for MCP communication +- Monitor server health and response times +- Track tool usage patterns +- Log authentication failures +- Set up alerts for connection issues + +### Security +- Use secure transport methods (HTTPS, WSS) +- Implement proper authentication +- Validate all inputs and outputs +- Monitor for suspicious activity +- Keep MCP servers updated + +## Language-Specific Implementation + +For implementation details and examples in your programming language: + +- **Multi-language**: [UTCP MCP Protocol Examples](https://github.com/universal-tool-calling-protocol) - MCP protocol examples across multiple languages +- **TypeScript**: [TypeScript MCP Protocol Documentation](https://github.com/universal-tool-calling-protocol/typescript-utcp/blob/main/docs/protocols/mcp.md) +- **Other languages**: Check respective repositories in the [UTCP GitHub organization](https://github.com/universal-tool-calling-protocol) diff --git a/versioned_docs/version-1.0/protocols/sse.md b/versioned_docs/version-1.0/protocols/sse.md new file mode 100644 index 0000000..93618e9 --- /dev/null +++ b/versioned_docs/version-1.0/protocols/sse.md @@ -0,0 +1,386 @@ +--- +id: sse +title: Server-Sent Events (SSE) +sidebar_position: 3 +--- + +# Server-Sent Events (SSE) + +The Server-Sent Events protocol plugin (`utcp-http`) enables UTCP to receive real-time streaming data from HTTP servers. SSE is perfect for tools that need to stream live updates, notifications, or continuous data feeds. + +## Installation + +SSE support is included with the HTTP plugin: + +```bash +# Example installation (Python) +pip install utcp-http + +# Example installation (Node.js) +npm install @utcp/http +``` + +## Call Template Structure + +```json +{ + "call_template_type": "sse", + "url": "https://api.example.com/events", + "headers": { + "Authorization": "Bearer ${API_TOKEN}", + "Accept": "text/event-stream" + }, + "timeout": 60, + "max_events": 10, + "event_filter": { + "type": "data_update" + } +} +``` + +## Configuration Options + +The Server-Sent Events (SSE) call template enables real-time streaming data from HTTP endpoints. For complete field specifications and validation rules, see the [SSE Call Template API Reference](../api/plugins/communication_protocols/http/src/utcp_http/sse_call_template.md). +| `reconnect` | boolean | Auto-reconnect on connection loss (default: true) | +| `reconnect_delay` | number | Delay between reconnection attempts (default: 3) | + +## Authentication + +SSE uses standard HTTP authentication methods: + +### Bearer Token + +```json +{ + "headers": { + "Authorization": "Bearer ${ACCESS_TOKEN}" + } +} +``` + +### API Key + +```json +{ + "headers": { + "X-API-Key": "${API_KEY}" + } +} +``` + +### Query Parameter Auth + +```json +{ + "query_params": { + "token": "${API_TOKEN}", + "user_id": "${USER_ID}" + } +} +``` + +## Event Handling + +### Basic Event Stream + +```json +{ + "name": "stream_notifications", + "description": "Stream real-time notifications", + "inputs": { + "type": "object", + "properties": { + "user_id": {"type": "string"} + }, + "required": ["user_id"] + }, + "tool_call_template": { + "call_template_type": "sse", + "url": "https://api.example.com/notifications/stream", + "query_params": { + "user_id": "${user_id}" + }, + "headers": { + "Authorization": "Bearer ${ACCESS_TOKEN}" + }, + "timeout": 300 + } +} +``` + +### Filtered Events + +```json +{ + "call_template_type": "sse", + "url": "https://api.example.com/events", + "event_filter": { + "type": "order_update", + "status": ["completed", "cancelled"] + }, + "max_events": 5 +} +``` + +## Examples + +### Stock Price Stream + +```json +{ + "name": "stream_stock_prices", + "description": "Stream real-time stock price updates", + "inputs": { + "type": "object", + "properties": { + "symbols": { + "type": "array", + "items": {"type": "string"} + }, + "duration": {"type": "number", "default": 60} + }, + "required": ["symbols"] + }, + "tool_call_template": { + "call_template_type": "sse", + "url": "https://api.stocks.com/stream", + "query_params": { + "symbols": "${symbols}", + "format": "json" + }, + "headers": { + "Authorization": "Bearer ${STOCK_API_KEY}" + }, + "timeout": "${duration}", + "event_filter": { + "type": "price_update" + } + } +} +``` + +### Log Monitoring + +```json +{ + "name": "monitor_logs", + "description": "Monitor application logs in real-time", + "inputs": { + "type": "object", + "properties": { + "service": {"type": "string"}, + "level": {"type": "string", "enum": ["error", "warn", "info", "debug"]} + }, + "required": ["service"] + }, + "tool_call_template": { + "call_template_type": "sse", + "url": "https://logs.example.com/stream", + "query_params": { + "service": "${service}", + "level": "${level}" + }, + "headers": { + "X-API-Key": "${LOG_API_KEY}" + }, + "timeout": 600, + "max_events": 100 + } +} +``` + +### System Metrics Stream + +```json +{ + "name": "stream_metrics", + "description": "Stream system performance metrics", + "inputs": { + "type": "object", + "properties": { + "metrics": { + "type": "array", + "items": {"type": "string"} + }, + "interval": {"type": "number", "default": 5} + }, + "required": ["metrics"] + }, + "tool_call_template": { + "call_template_type": "sse", + "url": "https://monitoring.example.com/metrics/stream", + "query_params": { + "metrics": "${metrics}", + "interval": "${interval}" + }, + "headers": { + "Authorization": "Bearer ${MONITORING_TOKEN}" + }, + "timeout": 300, + "reconnect": true, + "reconnect_delay": 5 + } +} +``` + +## Event Format + +SSE events follow the standard format: + +``` +event: message +data: {"type": "update", "value": 123} +id: event-123 +retry: 3000 + +event: heartbeat +data: {"timestamp": "2024-01-15T10:30:00Z"} + +data: {"message": "Simple data without event type"} +``` + +### Parsed Event Structure + +```json +{ + "event": "message", + "data": {"type": "update", "value": 123}, + "id": "event-123", + "retry": 3000, + "timestamp": "2024-01-15T10:30:00Z" +} +``` + +## Response Handling + +### Single Event Response + +```json +{ + "events": [ + { + "event": "data", + "data": {"result": "success"}, + "id": "1", + "timestamp": "2024-01-15T10:30:00Z" + } + ], + "total_events": 1, + "duration": 2.5 +} +``` + +### Multiple Events Response + +```json +{ + "events": [ + { + "event": "start", + "data": {"status": "processing"}, + "id": "1" + }, + { + "event": "progress", + "data": {"percent": 50}, + "id": "2" + }, + { + "event": "complete", + "data": {"result": "success"}, + "id": "3" + } + ], + "total_events": 3, + "duration": 15.2 +} +``` + +## Error Handling + +| Error Type | Description | Handling | +|------------|-------------|----------| +| Connection Failed | Cannot connect to SSE endpoint | Raise `SSEConnectionError` | +| Stream Timeout | No events received within timeout | Return partial results | +| Parse Error | Invalid SSE event format | Skip malformed events | +| Authentication Failed | Invalid credentials | Raise `SSEAuthError` | +| Server Error | HTTP 5xx response | Raise `SSEServerError` | + +## Best Practices + +1. **Set Appropriate Timeouts**: Configure timeouts based on expected data frequency +2. **Handle Reconnections**: Enable auto-reconnect for long-running streams +3. **Filter Events**: Use event filters to reduce unnecessary data processing +4. **Monitor Performance**: Track event rates and processing times +5. **Validate Data**: Validate incoming event data against expected schemas +6. **Handle Backpressure**: Implement buffering for high-frequency events +7. **Graceful Shutdown**: Properly close streams when done + +## Advanced Features + +### Custom Event Parsing + +```json +{ + "call_template_type": "sse", + "url": "https://api.example.com/events", + "event_parser": { + "format": "json", + "extract_fields": ["timestamp", "level", "message"] + } +} +``` + +### Event Aggregation + +```json +{ + "call_template_type": "sse", + "url": "https://api.example.com/metrics", + "aggregation": { + "window": 10, + "function": "average", + "field": "value" + } +} +``` + +### Conditional Termination + +```json +{ + "call_template_type": "sse", + "url": "https://api.example.com/events", + "termination_condition": { + "event_type": "complete", + "data_field": "status", + "value": "finished" + } +} +``` + +## Common Use Cases + +- **Real-time Dashboards**: Live metrics, status updates +- **Notifications**: User alerts, system notifications +- **Log Streaming**: Application logs, audit trails +- **Progress Tracking**: Long-running task progress +- **Live Data Feeds**: News, social media, sensor data +- **Chat Applications**: Message streams, typing indicators + +## Protocol Comparison + +| Feature | SSE | WebSocket | HTTP Polling | +|---------|-----|-----------|--------------| +| Server-to-Client | ✅ | ✅ | ✅ | +| Client-to-Server | ❌ | ✅ | ✅ | +| Auto-Reconnect | ✅ | Manual | Manual | +| Overhead | Low | Low | High | +| Browser Support | ✅ | ✅ | ✅ | +| Simplicity | High | Medium | High | + +## Related Protocols + +- [HTTP](./http.md) - For request/response patterns +- [WebSocket](./websocket.md) - For bidirectional communication +- [Streamable HTTP](./streamable-http.md) - For chunked HTTP responses diff --git a/versioned_docs/version-1.0/protocols/streamable-http.md b/versioned_docs/version-1.0/protocols/streamable-http.md new file mode 100644 index 0000000..3ed9e6e --- /dev/null +++ b/versioned_docs/version-1.0/protocols/streamable-http.md @@ -0,0 +1,105 @@ +--- +id: streamable-http +title: Streamable HTTP Protocol +sidebar_position: 2 +--- + +# Streamable HTTP Protocol + +The Streamable HTTP protocol plugin (`utcp-http`) enables UTCP to handle large HTTP responses by streaming them in chunks. This is ideal for tools that return large datasets, files, or progressive results that don't fit into a single response payload. It leverages HTTP Chunked Transfer Encoding. + +## Call Template Structure + +```json +{ + "call_template_type": "streamable_http", + "url": "https://api.example.com/download/large-file", + "http_method": "GET", + "headers": { + "Accept": "application/octet-stream" + }, + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "Authorization", + "location": "header" + }, + "chunk_size": 8192, + "timeout": 300000, + "body_field": "request_data", + "header_fields": ["custom_header_arg"] +} +``` + +## Configuration Options + +The Streamable HTTP call template provides a way to configure streaming from HTTP endpoints. + +| Option | Type | Default | Description | +|---|---|---|---| +| `url` | string | **Required** | The streaming HTTP endpoint URL. Supports path parameters like `/users/{user_id}`. | +| `http_method` | string | `GET` | The HTTP method to use. Supported methods are `GET` and `POST`. | +| `content_type` | string | `application/octet-stream` | The `Content-Type` header to set for the request, especially when `body_field` is used. | +| `chunk_size` | integer | `4096` | The size of each data chunk in bytes to read from the stream. | +| `timeout` | integer | `60000` | Request timeout in milliseconds. | +| `headers` | object | `null` | Optional static headers to include in every request. | +| `auth` | object | `null` | Optional authentication configuration. See [HTTP Authentication](./http.md#authentication-methods). | +| `body_field` | string | `null` | The name of a single tool argument to be sent as the HTTP request body. | +| `header_fields` | array | `null` | A list of tool argument names to be sent as request headers. | + +## Response Handling + +The protocol processes the incoming stream based on the `Content-Type` header of the response: + +- **`application/x-ndjson`**: The stream is parsed as Newline Delimited JSON. Each line is yielded as a separate JSON object. +- **`application/octet-stream`**: The stream is yielded in binary chunks of the specified `chunk_size`. +- **`application/json`**: The entire response is buffered and yielded as a single JSON object. This is for endpoints that stream a single, large JSON document. +- **Other Types**: For any other `Content-Type`, the response is treated as a binary stream and yielded in chunks of `chunk_size`. + +## Authentication + +Streamable HTTP supports the same authentication methods as the standard HTTP protocol, including API Key, Basic Auth, and OAuth2. The configuration is identical. + +For more details, see the [HTTP Authentication Methods](./http.md#authentication-methods) documentation. + +## Variable Substitution + +Path parameters, query parameters, headers, and authentication fields all support variable substitution from tool arguments and environment variables, following the same syntax as the standard HTTP protocol. + +Example: +```json +{ + "url": "https://api.example.com/files/{file_id}/download", + "headers": { + "Authorization": "Bearer ${ACCESS_TOKEN}" + } +} +``` +Here, `{file_id}` is substituted from a tool argument, and `${ACCESS_TOKEN}` is substituted from an environment or configuration variable. + +For more details, see the [HTTP Variable Substitution](./http.md#variable-substitution) documentation. + +## Security Considerations + +- **SSL/TLS**: It is strongly recommended to use `https://` endpoints to protect data in transit. The implementation enforces HTTPS or localhost connections by default. +- **Authentication**: Never hardcode credentials. Use variable substitution to inject secrets from a secure source (e.g., environment variables). +- **Input Sanitization**: Ensure that any arguments used in URL path parameters or query strings are properly validated and sanitized to prevent injection attacks. + +## Error Handling + +Errors are handled similarly to the standard HTTP protocol: + +| Status Code | Error Type | Description | +|---|---|---| +| 400 | Bad Request | Invalid request parameters or body. | +| 401 | Unauthorized | Authentication failed or is required. | +| 403 | Forbidden | The authenticated user does not have permission. | +| 404 | Not Found | The requested resource or endpoint does not exist. | +| 5xx | Server Error | An error occurred on the server side. | + +Connection errors, timeouts, and other network issues will also be raised as exceptions. + +## Related Protocols + +- [HTTP](./http.md) - For standard request/response interactions. +- [Server-Sent Events (SSE)](./sse.md) - For unidirectional, real-time event streams from server to client. \ No newline at end of file diff --git a/versioned_docs/version-1.0/protocols/text.md b/versioned_docs/version-1.0/protocols/text.md new file mode 100644 index 0000000..dcca388 --- /dev/null +++ b/versioned_docs/version-1.0/protocols/text.md @@ -0,0 +1,437 @@ +--- +id: text +title: Text Protocol +sidebar_position: 5 +--- + +# Text Protocol + +The Text protocol plugin (`utcp-text`) enables UTCP to read and process text files from local filesystem or remote URLs. This is useful for tools that need to access documentation, configuration files, logs, or any text-based data. + +## Installation + +```bash +# Example installation (Python) +pip install utcp-text + +# Example installation (Node.js) +npm install @utcp/text +``` + +## Call Template Structure + +```json +{ + "call_template_type": "text", + "file_path": "/path/to/file.txt", + "encoding": "utf-8", + "max_size": 1048576, + "line_range": { + "start": 1, + "end": 100 + } +} +``` + +## Configuration Options + +The Text call template enables reading and processing text files from local filesystem or URLs. For complete field specifications and validation rules, see the [Text Call Template API Reference](../api/plugins/communication_protocols/text/src/utcp_text/text_call_template.md). + +## File Sources + +### Local Files + +```json +{ + "call_template_type": "text", + "file_path": "/var/log/application.log", + "encoding": "utf-8" +} +``` + +### Remote URLs + +```json +{ + "call_template_type": "text", + "file_path": "https://example.com/config.txt", + "max_size": 512000 +} +``` + +### Variable Substitution + +```json +{ + "call_template_type": "text", + "file_path": "/data/${filename}", + "encoding": "${file_encoding}" +} +``` + +## Examples + +### Read Configuration File + +```json +{ + "name": "read_config", + "description": "Read application configuration file", + "inputs": { + "type": "object", + "properties": { + "config_name": {"type": "string"} + }, + "required": ["config_name"] + }, + "tool_call_template": { + "call_template_type": "text", + "file_path": "/etc/app/${config_name}.conf", + "encoding": "utf-8", + "max_size": 65536 + } +} +``` + +### Read Log File with Line Range + +```json +{ + "name": "read_recent_logs", + "description": "Read recent log entries", + "inputs": { + "type": "object", + "properties": { + "log_file": {"type": "string"}, + "lines": {"type": "number", "default": 100} + }, + "required": ["log_file"] + }, + "tool_call_template": { + "call_template_type": "text", + "file_path": "/var/log/${log_file}", + "line_range": { + "start": -${lines}, + "end": -1 + } + } +} +``` + +### Read Remote Documentation + +```json +{ + "name": "fetch_documentation", + "description": "Fetch documentation from remote URL", + "inputs": { + "type": "object", + "properties": { + "doc_url": {"type": "string"}, + "section": {"type": "string"} + }, + "required": ["doc_url"] + }, + "tool_call_template": { + "call_template_type": "text", + "file_path": "${doc_url}", + "pattern": "(?s)## ${section}.*?(?=## |$)", + "max_size": 2097152 + } +} +``` + +### Search in File + +```json +{ + "name": "search_in_file", + "description": "Search for pattern in text file", + "inputs": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "search_pattern": {"type": "string"} + }, + "required": ["file_path", "search_pattern"] + }, + "tool_call_template": { + "call_template_type": "text", + "file_path": "${file_path}", + "pattern": "${search_pattern}", + "transform": "strip" + } +} +``` + +## Line Range Options + +### Absolute Line Numbers + +```json +{ + "line_range": { + "start": 10, + "end": 50 + } +} +``` + +### Relative to End (Tail) + +```json +{ + "line_range": { + "start": -100, + "end": -1 + } +} +``` + +### From Start (Head) + +```json +{ + "line_range": { + "start": 1, + "end": 100 + } +} +``` + +## Pattern Matching + +### Simple Text Search + +```json +{ + "pattern": "ERROR" +} +``` + +### Regex Pattern + +```json +{ + "pattern": "\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2} ERROR.*" +} +``` + +### Multi-line Pattern + +```json +{ + "pattern": "(?s)START.*?END" +} +``` + +## Content Transformations + +### Case Transformations + +```json +{ + "transform": "upper" // Convert to uppercase +} +``` + +```json +{ + "transform": "lower" // Convert to lowercase +} +``` + +### Whitespace Handling + +```json +{ + "transform": "strip" // Remove leading/trailing whitespace +} +``` + +### Custom Transformations + +```json +{ + "transform": "normalize_whitespace" // Normalize all whitespace +} +``` + +## Response Format + +### Successful Read + +```json +{ + "content": "File content here...", + "metadata": { + "file_path": "/path/to/file.txt", + "size": 1024, + "lines": 25, + "encoding": "utf-8", + "last_modified": "2024-01-15T10:30:00Z" + } +} +``` + +### Filtered Content + +```json +{ + "content": "Matching lines...", + "metadata": { + "file_path": "/path/to/file.txt", + "total_lines": 1000, + "matched_lines": 5, + "pattern": "ERROR", + "line_range": {"start": 1, "end": 100} + } +} +``` + +## Error Handling + +| Error Type | Description | Handling | +|------------|-------------|----------| +| File Not Found | File doesn't exist | Raise `FileNotFoundError` | +| Permission Denied | No read permission | Raise `PermissionError` | +| File Too Large | Exceeds max_size limit | Raise `FileSizeError` | +| Encoding Error | Invalid file encoding | Raise `EncodingError` | +| Network Error | URL fetch failed | Raise `NetworkError` | + +## Security Considerations + +### Path Traversal Prevention + +```json +{ + "call_template_type": "text", + "file_path": "/safe/directory/${filename}", + "allowed_paths": ["/safe/directory/"] +} +``` + +### File Size Limits + +```json +{ + "max_size": 1048576 // 1MB limit +} +``` + +### URL Restrictions + +```json +{ + "allowed_domains": ["example.com", "docs.company.com"] +} +``` + +## Best Practices + +1. **Set Size Limits**: Always set appropriate max_size limits +2. **Validate Paths**: Validate file paths to prevent directory traversal +3. **Handle Encoding**: Specify encoding explicitly for non-UTF-8 files +4. **Use Line Ranges**: Use line ranges for large files to improve performance +5. **Pattern Efficiency**: Use efficient regex patterns for content filtering +6. **Cache Results**: Cache frequently accessed files +7. **Monitor Access**: Log file access for security auditing + +## Advanced Features + +### Conditional Reading + +```json +{ + "call_template_type": "text", + "file_path": "/var/log/app.log", + "condition": { + "modified_since": "2024-01-15T00:00:00Z" + } +} +``` + +### Multi-file Reading + +```json +{ + "call_template_type": "text", + "file_paths": [ + "/etc/app/config1.txt", + "/etc/app/config2.txt" + ], + "merge_strategy": "concatenate" +} +``` + +### Streaming Large Files + +```json +{ + "call_template_type": "text", + "file_path": "/var/log/huge.log", + "streaming": true, + "chunk_size": 8192 +} +``` + +## Common Use Cases + +- **Configuration Management**: Reading config files, environment files +- **Log Analysis**: Processing application logs, system logs +- **Documentation**: Accessing README files, API docs, manuals +- **Data Processing**: Reading CSV, JSON, XML text files +- **Template Processing**: Reading template files for generation +- **Code Analysis**: Reading source code files for analysis +- **Monitoring**: Reading status files, health check files + +## Performance Considerations + +| File Size | Recommended Approach | +|-----------|---------------------| +| < 1MB | Read entire file | +| 1MB - 10MB | Use line ranges | +| 10MB - 100MB | Use streaming | +| > 100MB | Use external tools | + +## Integration Examples + +### With HTTP Protocol + +```json +{ + "name": "process_uploaded_file", + "description": "Process uploaded text file", + "inputs": { + "type": "object", + "properties": { + "file_url": {"type": "string"} + } + }, + "tool_call_template": { + "call_template_type": "text", + "file_path": "${file_url}", + "max_size": 5242880 + } +} +``` + +### With CLI Protocol + +```json +{ + "name": "analyze_log_file", + "description": "Analyze log file with external tool", + "inputs": { + "type": "object", + "properties": { + "log_path": {"type": "string"} + } + }, + "tool_call_template": { + "call_template_type": "cli", + "command": "log-analyzer", + "args": ["--file", "${log_path}", "--format", "json"] + } +} +``` diff --git a/versioned_docs/version-1.0/protocols/websocket.md b/versioned_docs/version-1.0/protocols/websocket.md new file mode 100644 index 0000000..5ce7789 --- /dev/null +++ b/versioned_docs/version-1.0/protocols/websocket.md @@ -0,0 +1,355 @@ +--- +id: websocket +title: WebSocket Protocol +sidebar_position: 2 +--- + +# WebSocket Protocol + +The WebSocket protocol plugin (`utcp-websocket`) enables UTCP to communicate with WebSocket servers for real-time, bidirectional communication. This is ideal for tools that require persistent connections or real-time updates. + +## Installation + +```bash +# Example installation (Python) +pip install utcp-websocket + +# Example installation (Node.js) +npm install @utcp/websocket +``` + +## Call Template Structure + +```json +{ + "call_template_type": "websocket", + "url": "wss://api.example.com/ws", + "message": { + "type": "request", + "action": "${action}", + "data": "${data}" + }, + "connection_timeout": 10, + "response_timeout": 30, + "auth": { + "auth_type": "api_key", + "api_key": "${WS_API_KEY}", + "location": "query" + } +} +``` + +## Configuration Options + +The WebSocket call template enables real-time communication with WebSocket servers. For complete field specifications and validation rules, see the WebSocket Call Template API Reference (WIP). +| `expected_responses` | number | Number of expected response messages (default: 1) | +| `ping_interval` | number | Ping interval in seconds (default: 30) | + +## Authentication + +WebSocket authentication can be handled in several ways: + +### Query Parameter Authentication + +```json +{ + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "token", + "location": "query" + } +} +``` + +### Header Authentication + +```json +{ + "auth": { + "auth_type": "api_key", + "api_key": "${API_KEY}", + "var_name": "Authorization", + "location": "header" + } +} +``` + +### Message-based Authentication + +```json +{ + "message": { + "type": "auth", + "token": "${API_KEY}" + } +} +``` + +## Message Formats + +### JSON Messages + +```json +{ + "call_template_type": "websocket", + "url": "wss://api.example.com/ws", + "message": { + "id": "{{uuid}}", + "method": "getData", + "params": { + "query": "${query}", + "limit": 10 + } + } +} +``` + +### Text Messages + +```json +{ + "call_template_type": "websocket", + "url": "wss://api.example.com/ws", + "message": "GET_DATA:${query}" +} +``` + +### Binary Messages + +```json +{ + "call_template_type": "websocket", + "url": "wss://api.example.com/ws", + "message": { + "type": "binary", + "data": "${base64_data}" + } +} +``` + +## Examples + +### Real-time Data Subscription + +```json +{ + "name": "subscribe_stock_price", + "description": "Subscribe to real-time stock price updates", + "inputs": { + "type": "object", + "properties": { + "symbol": {"type": "string"}, + "duration": {"type": "number", "default": 60} + }, + "required": ["symbol"] + }, + "tool_call_template": { + "call_template_type": "websocket", + "url": "wss://api.stocks.com/ws", + "message": { + "action": "subscribe", + "symbol": "${symbol}", + "type": "price" + }, + "response_timeout": "${duration}", + "expected_responses": -1, + "close_after_response": false + } +} +``` + +### Chat Bot Integration + +```json +{ + "name": "send_chat_message", + "description": "Send a message to the chat bot", + "inputs": { + "type": "object", + "properties": { + "message": {"type": "string"}, + "user_id": {"type": "string"} + }, + "required": ["message", "user_id"] + }, + "tool_call_template": { + "call_template_type": "websocket", + "url": "wss://chat.example.com/ws", + "message": { + "type": "message", + "user_id": "${user_id}", + "content": "${message}", + "timestamp": "{{now}}" + }, + "headers": { + "Authorization": "Bearer ${CHAT_TOKEN}" + } + } +} +``` + +### IoT Device Control + +```json +{ + "name": "control_device", + "description": "Send control commands to IoT device", + "inputs": { + "type": "object", + "properties": { + "device_id": {"type": "string"}, + "command": {"type": "string"}, + "value": {"type": "number"} + }, + "required": ["device_id", "command"] + }, + "tool_call_template": { + "call_template_type": "websocket", + "url": "wss://iot.example.com/device/${device_id}", + "message": { + "command": "${command}", + "value": "${value}", + "timestamp": "{{now}}" + }, + "connection_timeout": 5, + "response_timeout": 10 + } +} +``` + +## Connection Management + +### Persistent Connections + +For tools that need to maintain persistent connections: + +```json +{ + "call_template_type": "websocket", + "url": "wss://api.example.com/ws", + "message": {"action": "ping"}, + "close_after_response": false, + "ping_interval": 30 +} +``` + +### Connection Pooling + +The WebSocket protocol automatically manages connection pooling for efficiency: + +- Reuses connections to the same endpoint +- Handles connection lifecycle automatically +- Implements reconnection logic for dropped connections + +## Response Handling + +### Single Response + +```json +{ + "expected_responses": 1, + "close_after_response": true +} +``` + +### Multiple Responses + +```json +{ + "expected_responses": 5, + "response_timeout": 60 +} +``` + +### Streaming Responses + +```json +{ + "expected_responses": -1, + "response_timeout": 300, + "close_after_response": false +} +``` + +## Error Handling + +| Error Type | Description | Handling | +|------------|-------------|----------| +| Connection Failed | Cannot establish WebSocket connection | Raise `WebSocketConnectionError` | +| Authentication Failed | WebSocket handshake authentication failed | Raise `WebSocketAuthError` | +| Timeout | No response within timeout period | Raise `WebSocketTimeoutError` | +| Protocol Error | Invalid WebSocket protocol usage | Raise `WebSocketProtocolError` | +| Connection Closed | Server closed connection unexpectedly | Raise `WebSocketClosedError` | + +## Best Practices + +1. **Use Secure WebSockets**: Always use `wss://` for production +2. **Handle Reconnections**: Implement retry logic for connection failures +3. **Set Appropriate Timeouts**: Configure timeouts based on expected response times +4. **Validate Messages**: Validate both outgoing and incoming messages +5. **Monitor Connections**: Track connection health and performance +6. **Implement Heartbeats**: Use ping/pong for connection health checks +7. **Handle Backpressure**: Manage message queuing for high-throughput scenarios + +## Advanced Features + +### Message Filtering + +Filter incoming messages based on criteria: + +```json +{ + "call_template_type": "websocket", + "url": "wss://api.example.com/ws", + "message": {"subscribe": "all"}, + "message_filter": { + "type": "stock_price", + "symbol": "${symbol}" + } +} +``` + +### Custom Headers + +Include custom headers in the WebSocket handshake: + +```json +{ + "headers": { + "User-Agent": "UTCP-Client/1.0", + "X-Client-ID": "${CLIENT_ID}", + "Authorization": "Bearer ${TOKEN}" + } +} +``` + +### Compression + +Enable WebSocket compression: + +```json +{ + "compression": "deflate", + "compression_threshold": 1024 +} +``` + +## Common Use Cases + +- **Real-time Data**: Stock prices, sensor data, live metrics +- **Chat Applications**: Messaging, notifications, presence +- **Gaming**: Real-time game state, multiplayer coordination +- **IoT Control**: Device commands, status updates +- **Live Updates**: News feeds, social media streams +- **Collaborative Tools**: Document editing, shared whiteboards + +## Protocol Comparison + +| Feature | WebSocket | HTTP | SSE | +|---------|-----------|------|-----| +| Bidirectional | ✅ | ❌ | ❌ | +| Real-time | ✅ | ❌ | ✅ | +| Persistent | ✅ | ❌ | ✅ | +| Overhead | Low | High | Medium | +| Complexity | Medium | Low | Low | diff --git a/versioned_docs/version-1.0/security.md b/versioned_docs/version-1.0/security.md index b182938..fbf10f6 100644 --- a/versioned_docs/version-1.0/security.md +++ b/versioned_docs/version-1.0/security.md @@ -6,123 +6,439 @@ sidebar_position: 6 # Security Considerations -Security is a critical aspect of any protocol that enables tool access and execution. This section outlines key security considerations when implementing and using UTCP. +Security is critical when enabling direct tool access through UTCP. This guide covers security considerations specific to UTCP's "manual" approach and provides practical guidance for secure implementations. -## Authentication +## UTCP Security Model -UTCP supports several authentication methods across different communication protocol types: +### Direct Communication Implications -### API Key Authentication +UTCP's direct communication model has unique security characteristics: + +**Advantages:** +- No middleman to compromise +- Native security controls remain intact +- Reduced attack surface (no proxy servers) +- Existing monitoring and logging continue to work + +**Considerations:** +- Clients must handle multiple authentication methods +- Manual endpoints become discovery targets +- Variable substitution introduces injection risks + +### Manual Discovery Security + +Secure your UTCP manual endpoints: + +**Best Practices:** +- Implement authentication for manual discovery endpoints +- Validate discovery access tokens +- Return only tools the client is authorized to see +- Filter tools based on user permissions +- Log all manual discovery attempts + +## Authentication & Authorization + +### Enhanced Authentication Examples + +#### API Key with Rotation ```json { "auth": { "auth_type": "api_key", - "api_key": "YOUR_API_KEY", + "api_key": "${API_KEY}", "var_name": "X-API-Key", "location": "header" } } ``` -The `location` field specifies where the API key is placed, and can be `header`, `query`, or `cookie`. +**Secure implementation:** +- Use rotating API keys with current and next key support +- Implement automatic key rotation based on time +- Store keys securely in environment variables +- Validate keys before processing requests -### Basic Authentication +#### OAuth2 with Scope Validation ```json { "auth": { - "auth_type": "basic", - "username": "user", - "password": "pass" + "auth_type": "oauth2", + "client_id": "${CLIENT_ID}", + "client_secret": "${CLIENT_SECRET}", + "token_url": "https://auth.example.com/token", + "scope": "tools:read tools:execute" } } ``` -### OAuth2 Authentication +#### Per-Tool Authorization ```json { - "auth": { - "auth_type": "oauth2", - "client_id": "YOUR_CLIENT_ID", - "client_secret": "YOUR_CLIENT_SECRET", - "token_url": "https://auth.example.com/token", - "scope": "read:tools" + "name": "admin_tool", + "description": "Administrative operations", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/admin/action", + "http_method": "POST", + "auth": { + "auth_type": "api_key", + "api_key": "${ADMIN_TOKEN}", + "var_name": "Authorization", + "location": "header" + } } } ``` -The `scope` field is optional and specifies the level of access that the client is requesting. +## Protocol-Specific Security -## Tool Access Control +### HTTP/HTTPS Security -When exposing tools through UTCP, consider implementing these access controls: - -1. **Tool-Level Permissions**: Define which users/agents can access specific tools -2. **Parameter Constraints**: Restrict parameter values to prevent abuse -3. **Rate Limiting**: Implement per-user/per-tool rate limits -4. **Usage Quotas**: Set maximum usage quotas for tools -5. **Audit Logging**: Log all tool calls for security monitoring +**Required configurations:** +```json +{ + "call_template_type": "http", + "url": "https://api.example.com/endpoint", + "verify_ssl": true, + "timeout": 30, + "headers": { + "User-Agent": "UTCP-Client/1.0", + "X-Request-ID": "${request_id}" + } +} +``` -## Communication Protocol-Specific Considerations +**Security checklist:** +- ✅ Always use HTTPS in production +- ✅ Validate SSL certificates (`verify_ssl: true`) +- ✅ Set appropriate timeouts +- ✅ Include request tracking headers +- ✅ Implement retry limits -### HTTP +### WebSocket Security -- Always use HTTPS, never HTTP +**Secure WebSocket configuration:** +```json +{ + "call_template_type": "websocket", + "url": "wss://api.example.com/ws", + "headers": { + "Authorization": "Bearer ${WS_TOKEN}", + "Origin": "https://trusted-domain.com" + }, + "ping_interval": 30, + "connection_timeout": 10 +} +``` -### CLI +**Security measures:** +- ✅ Use WSS (secure WebSocket) only +- ✅ Validate Origin headers +- ✅ Implement connection timeouts +- ✅ Use heartbeat/ping for connection health +- ✅ Limit concurrent connections per client -:::important +### CLI Security -CLI poses a significant security risk as it executes commands on the local system. +:::danger High Risk Protocol +CLI execution poses significant security risks. Use with extreme caution. ::: -- Validate and sanitize all input parameters -- Run commands with the minimum necessary permissions -- Implement allow-lists for permitted commands -- Sandbox execution environments when possible +**Secure CLI implementation:** +```json +{ + "call_template_type": "cli", + "commands": [ + { + "command": "cd /safe/sandbox", + "append_to_final_output": false + }, + { + "command": "/usr/local/bin/safe-script --input UTCP_ARG_sanitized_input_UTCP_END", + "append_to_final_output": true + } + ], + "working_dir": "/safe/sandbox", + "env_vars": { + "PATH": "/usr/local/bin:/usr/bin", + "HOME": "/tmp/sandbox" + } +} +``` + +**Security requirements:** +- ✅ Use absolute paths for commands +- ✅ Sanitize all input parameters +- ✅ Run in sandboxed environments +- ✅ Limit environment variables +- ✅ Set strict timeouts +- ✅ Validate exit codes +- ✅ Use minimal user permissions + +**Input sanitization requirements:** +- Remove dangerous shell metacharacters: `;`, `&`, `|`, `` ` ``, `$`, `()`, `{}`, `[]`, `<>` +- Escape inputs appropriately for shell execution +- Use parameterized command execution when possible +- Validate inputs against expected patterns +- Implement length limits for all inputs + +### Server-Sent Events (SSE) Security + +**Secure SSE configuration:** +```json +{ + "call_template_type": "sse", + "url": "https://api.example.com/events", + "headers": { + "Authorization": "Bearer ${SSE_TOKEN}", + "Accept": "text/event-stream", + "Cache-Control": "no-cache" + }, + "timeout": 300, + "max_events": 1000 +} +``` + +**Security considerations:** +- ✅ Authenticate SSE connections +- ✅ Set maximum event limits +- ✅ Implement connection timeouts +- ✅ Validate event data format +- ✅ Monitor for event flooding + +### Text Protocol Security + +**Secure file access:** +```json +{ + "call_template_type": "text", + "file_path": "/safe/data/${filename}" +} +``` + +**Security measures:** +- ✅ Restrict file paths to safe directories +- ✅ Set maximum file size limits +- ✅ Validate file extensions +- ✅ Prevent directory traversal attacks +- ✅ Use safe encoding handling + +**Path validation requirements:** +- Resolve all paths to absolute paths +- Check if paths are within allowed directories +- Handle symbolic links by resolving them first +- Validate against directory traversal attacks (`../`) +- Return false for any path resolution errors +- Use allowlists of permitted directories + +### MCP Security + +**Secure MCP server configuration:** +```json +{ + "call_template_type": "mcp", + "config": { + "mcpServers": { + "server_name": { + "transport": "stdio", + "command": ["python", "-m", "my_mcp_server"] + } + } + } +} +``` + +**Security considerations:** +- ✅ Use trusted MCP server implementations +- ✅ Sandbox MCP server processes +- ✅ Limit server resource usage +- ✅ Monitor server health and logs +- ✅ Implement server restart policies + +## Input Validation & Sanitization + +### JSON Schema Validation + +**Comprehensive input validation:** +```json +{ + "inputs": { + "type": "object", + "properties": { + "email": { + "type": "string", + "format": "email", + "maxLength": 254 + }, + "age": { + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + "tags": { + "type": "array", + "items": {"type": "string", "pattern": "^[a-zA-Z0-9_-]+$"}, + "maxItems": 10 + } + }, + "required": ["email"], + "additionalProperties": false + } +} +``` + +### Parameter Sanitization + +**Server-side validation requirements:** +- Use regex patterns to validate input formats (e.g., `^[a-zA-Z0-9_-]+$` for user IDs) +- Implement maximum length limits for all string inputs +- Remove dangerous characters like `<>`, `"`, `'` from user inputs +- Validate all inputs against expected schemas +- Sanitize inputs before processing + +## Secure Variable Handling + +### Environment Variable Security + +**Secure variable loading requirements:** +- Only allow variables with approved prefixes (e.g., `UTCP_`, `API_`) +- Validate variable names against allowlists +- Implement length limits for variable values (e.g., max 10,000 characters) +- Check for dangerous characters in values (`<`, `>`, `"`, `'`) +- Use secure variable substitution methods +- Log all variable access attempts + +### Runtime Variable Substitution + +**Secure substitution requirements:** + +Implement variable substitution with these security measures: +- Only substitute variables matching the pattern `${variable_name}` +- Validate variable names contain only alphanumeric characters and underscores +- Check that all variables exist before substitution +- Sanitize variable values to prevent injection attacks +- Reject values containing dangerous characters like `<`, `>`, `"`, `'`, `;`, `&` +- Limit variable value length to prevent buffer overflow attacks + +## Network & Transport Security + +### TLS Configuration + +**Minimum TLS requirements:** +**TLS configuration requirements:** + +Configure secure HTTP clients with these settings: +- Enable SSL certificate verification +- Set reasonable connection timeouts (e.g., 30 seconds) +- Limit maximum connections to prevent resource exhaustion +- Use TLS 1.2 or higher as minimum version +- Enable hostname verification +- Require certificate verification (CERT_REQUIRED mode) + +### Certificate Validation + +**Enhanced certificate validation:** + +Implement robust certificate validation: +- Use trusted certificate authority bundles +- Enable hostname verification against certificate +- Require valid certificate chains +- Set minimum TLS version to 1.2 +- Configure strong cipher suites (ECDHE+AESGCM, CHACHA20, DHE+AESGCM) +- Reject weak algorithms (aNULL, MD5, DSS) + +## Monitoring & Incident Response + +### Security Logging + +**Comprehensive security logging:** + +Implement security logging with these components: +- **Tool Call Logging**: Record all tool invocations with user ID, tool name, timestamp, success status, and parameters +- **Authentication Logging**: Log authentication attempts, failures, and reasons +- **Structured Format**: Use JSON format for easy parsing and analysis +- **Sensitive Data Protection**: Avoid logging sensitive information like passwords or tokens +- **Audit Trail**: Maintain immutable logs for compliance and forensic analysis + +### Anomaly Detection + +**Basic anomaly detection:** -### WebSocket +Implement anomaly detection with these features: +- **Rate Limiting**: Track requests per user with configurable limits (e.g., 60 calls/minute, 1000 calls/hour) +- **Time Window Management**: Clean old entries and maintain sliding time windows +- **Multi-tier Limits**: Enforce both short-term (per minute) and long-term (per hour) rate limits +- **Automatic Blocking**: Reject requests that exceed configured thresholds +- **Call Tracking**: Record timestamps of all user requests for analysis -- Use secure WebSocket (WSS) connections +## Security Testing & Validation -## Data Protection +### Testing Methodologies -1. **Data in Transit**: Ensure all communications use TLS 1.2+ encryption -2. **Data at Rest**: Encrypt sensitive configuration data -3. **Sensitive Data in Logs**: Prevent logging of sensitive parameters -4. **PII Handling**: Implement proper controls for personal information +**Security test examples:** -## Secure Implementation Checklist +Implement comprehensive security testing: +- **Injection Prevention Tests**: Test for SQL injection, command injection, and other malicious inputs +- **Path Traversal Tests**: Verify protection against directory traversal attacks (../../../etc/passwd) +- **Rate Limiting Tests**: Confirm rate limiting enforcement under high load +- **Authentication Tests**: Validate proper authentication and authorization +- **Input Validation Tests**: Test boundary conditions and malformed inputs -- [ ] Use HTTPS/WSS for all network communications -- [ ] Implement proper authentication for all communication protocols -- [ ] Validate all input against schemas before processing -- [ ] Sanitize inputs to prevent injection attacks -- [ ] Implement rate limiting to prevent abuse -- [ ] Set appropriate timeouts for all operations -- [ ] Log security-relevant events -- [ ] Regularly update dependencies -- [ ] Implement proper error handling that doesn't leak sensitive information +### Security Automation -## Common Vulnerabilities to Avoid +**Automated security checks:** + +Implement automated security validation: +- **Protocol Security**: Verify HTTPS usage instead of HTTP for web requests +- **Credential Detection**: Check for hardcoded passwords, secrets, or API keys +- **Variable Validation**: Ensure proper variable substitution patterns ($\{variable\}) +- **CLI Security**: Validate command-line tools use absolute paths and safe commands +- **URL Validation**: Check for suspicious or malformed URLs +- **Configuration Review**: Automated scanning of UTCP manuals for security issues -| Vulnerability | Prevention | -|--------------|------------| -| Injection Attacks | Validate and sanitize all inputs | -| Credential Leakage | Use secure credential storage | -| Excessive Permissions | Follow the principle of least privilege | -| Man-in-the-Middle | Use certificate validation and pinning | -| Denial of Service | Implement rate limiting and timeouts | -| Information Disclosure | Ensure errors don't leak sensitive data | +## Security Checklist -## Secure Development Lifecycle +### Tool Provider Security + +- [ ] UTCP manual endpoint requires authentication +- [ ] All tool endpoints use HTTPS/WSS +- [ ] Input validation implemented for all tools +- [ ] Rate limiting configured per user/tool +- [ ] Security logging enabled +- [ ] Credentials stored securely (not hardcoded) +- [ ] SSL certificate validation enabled +- [ ] Appropriate timeouts configured +- [ ] Error messages don't leak sensitive information -1. **Design**: Conduct threat modeling during protocol design -2. **Implementation**: Follow secure coding practices -3. **Testing**: Perform security testing and code reviews -4. **Deployment**: Use secure deployment practices -5. **Maintenance**: Monitor for security issues and update regularly +### Tool Consumer Security + +- [ ] Variable substitution is sanitized +- [ ] SSL certificate verification enabled +- [ ] Connection timeouts configured +- [ ] Rate limiting respected +- [ ] Security events logged +- [ ] Credentials rotated regularly +- [ ] Network connections monitored +- [ ] Input validation before tool calls -By following these security considerations, UTCP implementations can minimize risks while enabling powerful tool integrations across various communication protocols. +### Protocol-Specific Security + +- [ ] **HTTP**: HTTPS only, certificate validation +- [ ] **WebSocket**: WSS only, origin validation +- [ ] **CLI**: Sandboxed execution, input sanitization +- [ ] **SSE**: Authenticated connections, event limits +- [ ] **Text**: Path validation, size limits +- [ ] **MCP**: Trusted servers, resource limits + +By following these security guidelines, you can safely implement UTCP while maintaining strong security posture across all communication protocols. + +For protocol-specific security details, see: +- [HTTP Security](./protocols/http.md#security-considerations) +- [WebSocket Security](./protocols/websocket.md#security-considerations) +- [CLI Security](./protocols/cli.md#security-considerations) +- [SSE Security](./protocols/sse.md#security-considerations) +- [Text Security](./protocols/text.md#security-considerations) +- [MCP Security](./protocols/mcp.md#security-considerations) diff --git a/versioned_docs/version-1.0/utcp-vs-mcp.md b/versioned_docs/version-1.0/utcp-vs-mcp.md index 3dec676..bc670ff 100644 --- a/versioned_docs/version-1.0/utcp-vs-mcp.md +++ b/versioned_docs/version-1.0/utcp-vs-mcp.md @@ -4,99 +4,478 @@ title: UTCP vs MCP sidebar_position: 5 --- -# UTCP vs MCP tool calling: A Comparison +# UTCP vs MCP: A Comprehensive Comparison -This page compares the Universal Tool Calling Protocol (UTCP) with the Model Context Protocol's (MCP) tool calling functionality, highlighting their different approaches to agent-tool integration. +:::info Language Examples +This comparison uses **Python** examples. Both UTCP and MCP have implementations in multiple languages - check respective GitHub organizations for language-specific examples. +::: + +This guide compares the Universal Tool Calling Protocol (UTCP) with the Model Context Protocol (MCP), helping you choose the right approach for your AI tool integration needs. ## Video Overview -## Architectural Differences - -| Aspect | MCP | UTCP | -|--------|-----|------| -| **Core Model** | Middleman | Manual | -| **Architecture** | Agents ↔ MCP Server ↔ Tool | Agent ↔ Tool (Direct) | -| **Integration Approach** | Wraps existing tools | Describes how to call existing tools | -| **Network Hops** | Double (Agent → MCP → Tool) | Single (Agent → Tool) | -| **Protocol Dependency** | Hard dependency on protocol for every call | Protocol only needed during discovery | +## Quick Comparison -## The Middleman vs Manual Philosophies +| Aspect | UTCP | MCP | +|--------|------|-----| +| **Philosophy** | Manual (describes how to call tools) | Middleman (wraps tools in protocol) | +| **Architecture** | Agent → Tool (Direct) | Agent → MCP Server → Tool | +| **Infrastructure** | None required | Wrapper servers needed | +| **Protocols** | HTTP, WebSocket, CLI, SSE, etc. | JSON-RPC over stdio/HTTP | +| **Performance** | Native tool performance | Additional proxy overhead | +| **Maintenance** | Minimal | High (server maintenance) | -### The MCP "Middleman" Approach +## Architectural Differences -MCP positions itself as the "USB-C for AI Agents" — a universal adapter that all tools must plug into. This approach: +### UTCP: The "Manual" Approach + +UTCP provides a standardized way to describe how to call existing tools directly: + +```json +{ + "manual_version": "1.0.0", + "utcp_version": "1.0.1", + "tools": [{ + "name": "get_weather", + "description": "Get current weather", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.weather.com/current", + "http_method": "GET", + "query_params": {"location": "${location}"} + } + }] +} +``` + +**Flow:** Agent discovers manual → Agent calls tool directly + +### MCP: The "Middleman" Approach + +MCP requires building servers that wrap your tools: + +**MCP Server Implementation Requirements:** +- Create a dedicated server process for each tool provider +- Implement tool listing functionality to expose available tools +- Implement tool calling handlers that proxy requests to actual APIs +- Maintain server infrastructure and handle client connections +- Route all tool calls through the MCP server layer + +**Flow:** Agent → MCP Server → Tool → MCP Server → Agent + +## Technical Comparison + +### Performance Impact + +#### UTCP Performance + +**Direct API calls with no overhead:** +- Make HTTP requests directly to weather service endpoints +- No intermediate proxy servers or additional network hops +- Latency equals API response time only (~100ms) +- Native HTTP client performance with connection pooling + +#### MCP Performance + +**Requires MCP server proxy:** +- Connect to MCP server before making tool calls +- Route requests through MCP server to actual weather API +- Additional network hop adds latency overhead +- Latency includes API response time plus MCP server processing (~150ms) + +### Infrastructure Requirements + +#### UTCP Infrastructure + +**Minimal infrastructure requirements:** +- Add single discovery endpoint to existing API (e.g., GET /utcp) +- Return static JSON manual describing available tools +- No additional servers, processes, or infrastructure needed +- Total infrastructure: 0 additional servers + +#### MCP Infrastructure + +**MCP infrastructure requirements:** +- Requires dedicated MCP server processes for each tool provider +- Process management, monitoring, and scaling infrastructure needed +- Client connection management and session handling required +- Total infrastructure: N MCP servers (one per tool provider) + +### Protocol Support Comparison + +#### UTCP Protocol Flexibility +```json +{ + "tools": [ + { + "name": "http_tool", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/data" + } + }, + { + "name": "websocket_tool", + "tool_call_template": { + "call_template_type": "websocket", + "url": "wss://api.example.com/ws" + } + }, + { + "name": "cli_tool", + "tool_call_template": { + "call_template_type": "cli", + "commands": [ + { + "command": "git status --porcelain", + "append_to_final_output": true + } + ] + } + } + ] +} +``` + +#### MCP Protocol Limitation + +**MCP protocol constraints:** +- Only supports JSON-RPC over stdio/HTTP transport +- All tools must be wrapped in MCP server implementations +- Cannot directly call WebSocket, CLI, or other native protocols +- Requires protocol translation layer for non-HTTP tools -- Forces all traffic through a new protocol layer -- Requires writing "wrappers" for existing tools -- Needs to reinvent solutions for auth, security, and other infrastructure concerns +## Feature Comparison -This creates what we call the "wrapper tax": the additional infrastructure, development, and maintenance overhead required to adapt existing tools to work with MCP. +### Authentication & Security + +#### UTCP: Native Authentication +```json +{ + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.example.com/data", + "auth": { + "auth_type": "oauth2", + "client_id": "${CLIENT_ID}", + "client_secret": "${CLIENT_SECRET}", + "token_url": "https://auth.example.com/token" + } + } +} +``` + +**Benefits:** +- Uses existing authentication systems +- No credential translation needed +- Native rate limiting and monitoring +- Existing security policies apply + +#### MCP: Server-Mediated Authentication + +**MCP server authentication requirements:** +- MCP server must handle authentication translation between client and API +- Server stores and manages API credentials on behalf of clients +- Server makes authenticated calls to actual APIs using stored credentials +- Requires credential management and secure storage in MCP server + +**Challenges:** +- Credential management in MCP servers +- Additional security layer to maintain +- Auth translation complexity + +### Streaming & Real-time Data + +#### UTCP: Native Streaming Support +```json +{ + "name": "stream_logs", + "tool_call_template": { + "call_template_type": "sse", + "url": "https://api.example.com/logs/stream", + "timeout": 300 + } +} +``` + +#### MCP: Limited Streaming + +**MCP streaming limitations:** +- MCP has basic streaming capabilities but requires server implementation +- More complex to set up and maintain than native streaming protocols +- Additional abstraction layer between client and streaming data source + +### Error Handling + +#### UTCP: Native Error Responses + +**Direct error handling:** +- Errors come directly from the tool without translation +- Native HTTP status codes and error messages preserved +- Full error context available including headers and response body +- No error translation or abstraction layer + +#### MCP: Wrapped Error Responses + +**Error abstraction layer:** +- Errors are wrapped and translated by MCP server +- Original error context may be lost in translation +- MCP-specific error format instead of native tool errors +- Additional debugging complexity due to error wrapping + +## Migration & Interoperability + +### Migrating from MCP to UTCP + +UTCP provides an MCP plugin for gradual migration: + +**Migration Strategy:** +- **Phase 1**: Use existing MCP servers via UTCP's MCP protocol plugin +- Configure UTCP client to connect to legacy MCP servers using MCP call templates +- **Phase 2**: Migrate high-value tools to native UTCP protocols (HTTP, WebSocket, CLI) +- **Phase 3**: Deprecate MCP servers once migration is complete + +[**Complete migration guide →**](./protocols/mcp.md) + +### Hybrid Approach + +You can use both protocols simultaneously: + +**Hybrid approach during migration:** +- Configure UTCP client with both native UTCP and legacy MCP call templates +- Native UTCP tools use direct HTTP/WebSocket/CLI protocols +- Legacy MCP tools continue using MCP protocol plugin +- Gradually migrate tools from MCP to native UTCP protocols +- Single client interface for both native and legacy tools + +## Enterprise Decision Factors + +### Total Cost of Ownership + +#### UTCP TCO +``` +Infrastructure: $0 (uses existing APIs) +Development: Low (add one endpoint) +Maintenance: Minimal (static JSON) +Scaling: Automatic (scales with existing API) +Monitoring: Existing tools work +``` + +#### MCP TCO +``` +Infrastructure: High (dedicated servers) +Development: High (build wrapper servers) +Maintenance: High (server management) +Scaling: Complex (scale MCP servers separately) +Monitoring: Additional monitoring stack needed +``` -### The UTCP "Manual" Approach +### Development Velocity -UTCP takes a different approach — it's a "manual" that describes how to call tools directly: +#### UTCP Development Speed + +**Rapid deployment timeline:** +- **Day 1**: Add UTCP discovery endpoint to existing API +- **Day 2**: Tools are immediately available to AI agents +- No additional infrastructure, servers, or deployment needed +- Minimal code changes to existing systems -- Provides all necessary information to call native APIs directly -- Gets out of the way after tool discovery -- Leverages existing infrastructure for auth, security, etc. +#### MCP Development Speed -This eliminates the wrapper tax and allows organizations to expose their existing APIs to AI agents without building and maintaining additional infrastructure. +**Extended development timeline:** +- **Week 1-2**: Build dedicated MCP server implementation +- **Week 3**: Deploy and configure server infrastructure +- **Week 4**: Set up monitoring, logging, and scaling +- **Week 5**: Handle production issues and debugging +- **Ongoing**: Server maintenance, updates, and operations -## Feature Comparison +### Risk Assessment -| Feature | MCP | UTCP | -|---------|-----|------| -| **Tool Discovery** | Via MCP Server | Via manual discovery endpoint | -| **Protocol Support** | HTTP Streaming | HTTP, WebSockets, gRPC, CLI, etc. | -| **Authentication** | Handled by MCP Server | Uses tool's native authentication | -| **Streaming** | Native support | Supported via appropriate communication protocol (SSE, WebSockets) | -| **Implementation Complexity** | High (requires wrapper servers) | Low (simple JSON definitions) | -| **Performance** | Additional overhead due to proxy | Direct, native performance | -| **Evolution Speed** | Slow (all participants must update) | Fast (individual communication protocols can evolve independently) | +| Risk Factor | UTCP | MCP | +|-------------|------|-----| +| **Single Point of Failure** | None (direct calls) | MCP servers | +| **Vendor Lock-in** | Low (standard protocols) | Medium (MCP-specific) | +| **Maintenance Burden** | Low | High | +| **Security Surface** | Minimal | Expanded | +| **Performance Risk** | Low | Medium | -## When to Choose Each Protocol - -### Choose MCP When: - -- You need strict standardization across all tools -- You're building a closed ecosystem where you control all components -- You're willing to invest in building and maintaining wrapper servers +## Decision Framework ### Choose UTCP When: -- You want to leverage existing APIs without building wrappers -- You need to support diverse communication protocols -- You value direct, efficient communication -- You prioritize low implementation overhead -- You want to minimize infrastructure costs +✅ **You have existing APIs** that work well +✅ **You want minimal infrastructure** overhead +✅ **You need multiple protocols** (HTTP, WebSocket, CLI, etc.) +✅ **You prioritize performance** and direct communication +✅ **You want to leverage existing** auth, monitoring, scaling +✅ **You have limited resources** for server maintenance +✅ **You need rapid deployment** of AI tool access + +### Choose MCP When: -## Real-World Example +✅ **You need strict protocol standardization** across all tools +✅ **You're building a closed ecosystem** with full control +✅ **You have resources** for building and maintaining servers +✅ **You need MCP-specific features** like resources and prompts +✅ **You're already invested** in MCP infrastructure +✅ **You prefer centralized control** over tool access + +### Hybrid Approach When: + +✅ **You're migrating from MCP** to UTCP gradually +✅ **You have mixed requirements** (some tools need MCP features) +✅ **You want to evaluate both** approaches in production +✅ **You have legacy MCP investments** to preserve + +## Real-World Examples + +### E-commerce API Integration + +#### UTCP Approach + +**E-commerce API with UTCP:** +- Keep existing product API endpoints unchanged (GET /products/\{product_id\}) +- Add single UTCP discovery endpoint (GET /utcp) +- Return UTCP manual describing available tools and how to call them +- Tools directly reference existing API endpoints with proper parameters +- Total additional code: ~10 lines +- Additional infrastructure: 0 servers + +#### MCP Approach + +**E-commerce API with MCP:** +- Requires building dedicated MCP server wrapper +- Implement tool listing and calling handlers in MCP server +- MCP server calls existing API endpoints on behalf of clients +- Additional server deployment, monitoring, and scaling required +- Total additional code: ~50+ lines +- Additional infrastructure: 1+ servers + +### Database Query Tool + +#### UTCP Approach +```json +{ + "name": "query_database", + "tool_call_template": { + "call_template_type": "http", + "url": "https://api.company.com/query", + "http_method": "POST", + "body": {"sql": "${query}"}, + "auth": { + "auth_type": "api_key", + "api_key": "${DB_API_KEY}", + "var_name": "Authorization", + "location": "header" + } + } +} +``` + +#### MCP Approach +**MCP database approach:** +- Requires MCP server with database connection management +- Connection pooling, query validation, and security in MCP server +- Much more complex implementation than direct database access +- Additional abstraction layer between client and database + +## Performance Benchmarks + +### Latency Comparison + +| Scenario | UTCP | MCP | Difference | +|----------|------|-----|------------| +| Simple API call | 50ms | 75ms | +50% overhead | +| Complex query | 200ms | 250ms | +25% overhead | +| File operation | 10ms | 20ms | +100% overhead | +| Streaming data | Real-time | Buffered | Significant delay | + +### Resource Usage + +| Resource | UTCP | MCP | +|----------|------|-----| +| Memory | 0MB (no servers) | 50-200MB per server | +| CPU | 0% (no processing) | 5-15% per server | +| Network | Direct | Double hops | +| Storage | 0GB | Logs, state, config | + +## Migration Timeline + +### From MCP to UTCP + +**Phase 1 (Week 1): Assessment** +- Inventory existing MCP servers +- Identify high-value tools for migration +- Plan migration strategy + +**Phase 2 (Week 2-3): Hybrid Setup** +- Install UTCP with MCP plugin +- Test existing MCP tools through UTCP +- Validate functionality + +**Phase 3 (Week 4-8): Gradual Migration** +- Migrate tools one by one to native UTCP +- Add `/utcp` endpoints to existing APIs +- Update client configurations + +**Phase 4 (Week 9+): Cleanup** +- Deprecate MCP servers +- Remove MCP infrastructure +- Monitor and optimize + +[**Detailed migration guide →**](./migration-v0.1-to-v1.0.md) + +## Community & Ecosystem + +### UTCP Ecosystem +- **Multiple language implementations**: Python, TypeScript, Go, Rust +- **Growing protocol support**: HTTP, WebSocket, CLI, SSE, Text, MCP +- **Active development**: Regular releases and improvements +- **Open governance**: RFC process for changes + +### MCP Ecosystem +- **Anthropic-led development**: Centralized development +- **Growing tool library**: Community-contributed servers +- **IDE integrations**: Claude Desktop, Cline, etc. +- **Established patterns**: Well-documented server patterns -Consider an organization with an existing REST API that they want to expose to AI agents: +## Conclusion -**With MCP:** -1. Build an MCP server that wraps the REST API -2. Translate all calls between MCP format and REST format -3. Maintain and scale this additional server infrastructure -4. Handle authentication translation between MCP and the API +Both UTCP and MCP solve the tool integration problem, but with fundamentally different approaches: -**With UTCP:** -1. Create a simple JSON definition describing the REST API -2. Expose this definition via a discovery endpoint (typically `/utcp`) -3. The AI agent can now call the REST API directly +**UTCP excels when you:** +- Want to leverage existing APIs without additional infrastructure +- Need support for multiple communication protocols +- Prioritize performance and direct communication +- Have limited resources for server maintenance +- Want rapid deployment and minimal complexity -## Code comparison +**MCP excels when you:** +- Need strict protocol standardization +- Are building a controlled ecosystem +- Have resources for server infrastructure +- Need MCP-specific features beyond tool calling +- Prefer centralized tool management -You can find a full typescript example detailing the MCP and UTCP approach [here](https://github.com/universal-tool-calling-protocol/typescript-utcp/tree/main/examples/src/concrete_example). +**For most organizations**, UTCP's "manual" approach offers significant advantages in terms of simplicity, performance, and cost-effectiveness. The ability to expose existing APIs to AI agents with minimal changes and no additional infrastructure makes it an attractive choice for rapid AI tool integration. -## Conclusion +**For gradual adoption**, consider starting with UTCP's MCP plugin to use existing MCP servers while migrating high-value tools to native UTCP protocols over time. -Both MCP and UTCP aim to solve the problem of standardizing tool calling for AI agents, but they take fundamentally different approaches. +## Next Steps -MCP acts as a middleman, requiring all tools to be wrapped in its protocol. This provides standardization but at the cost of additional infrastructure and development overhead. +### To Get Started with UTCP: +1. **[Read the implementation guide](./implementation.md)** - Learn how to implement UTCP +2. **[Choose your protocols](./protocols/index.md)** - Select communication methods +3. **[Check examples](https://github.com/universal-tool-calling-protocol)** - See real implementations across multiple languages -UTCP acts as a manual, describing how to call tools directly using their native interfaces. This eliminates the wrapper tax and leverages existing infrastructure, at the cost of requiring clients to handle different communication protocols. +### To Migrate from MCP: +1. **[Read the MCP integration guide](./protocols/mcp.md)** - Use MCP tools via UTCP +2. **[Plan your migration](./migration-v0.1-to-v1.0.md)** - Step-by-step migration process +3. **[Join the community](https://discord.gg/ZpMbQ8jRbD)** - Get migration support -The choice between them depends on your specific requirements, existing infrastructure, and development resources. However, UTCP's "manual" approach offers significant advantages in terms of simplicity, efficiency, and leveraging existing investments in API infrastructure. +### To Learn More: +- **[UTCP Architecture](./api/index.md)** - Technical deep dive +- **[Security Considerations](./security.md)** - Security best practices +- **[Tool Provider Guide](./for-tool-providers.md)** - Expose your tools