From 5c11a41ae21731daba94595e5aef2f836be00e86 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Thu, 6 Nov 2025 18:41:32 -0800 Subject: [PATCH 01/46] feat: implement segment telemetry --- README.md | 31 ++- mds/TELEMETRY.md | 573 ++++++++++++++++++++++++++++++++++++++++ package-lock.json | 127 ++++++++- package.json | 2 + src/mcp/server.ts | 70 ++++- src/stdio.ts | 76 +++++- src/telemetry.ts | 49 ++++ src/utils/user-cache.ts | 39 +++ src/utils/version.ts | 24 ++ tests/helpers.ts | 4 + 10 files changed, 981 insertions(+), 14 deletions(-) create mode 100644 mds/TELEMETRY.md create mode 100644 src/telemetry.ts create mode 100644 src/utils/user-cache.ts create mode 100644 src/utils/version.ts diff --git a/README.md b/README.md index 18d99f25..2cefdd30 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ The Apify Model Context Protocol (MCP) server at [**mcp.apify.com**](https://mcp ## Table of Contents - [🌐 Introducing the Apify MCP server](#-introducing-the-apify-mcp-server) - [πŸš€ Quickstart](#-quickstart) +- [πŸ“Š Telemetry](#telemetry) - [πŸ€– MCP clients and examples](#-mcp-clients-and-examples) - [πŸͺ„ Try Apify MCP instantly](#-try-apify-mcp-instantly) - [πŸ› οΈ Tools, resources, and prompts](#-tools-resources-and-prompts) @@ -258,13 +259,24 @@ The server provides a set of predefined example prompts to help you get started The server does not yet provide any resources. -### Debugging the NPM package +## πŸ“‘ Telemetry -To debug the server, use the [MCP Inspector](https://github.com/modelcontextprotocol/inspector) tool: +The Apify MCP Server collects telemetry data about tool calls to help Apify understand usage patterns and improve the service. By default, telemetry is **enabled** for all tool calls. -```shell -export APIFY_TOKEN="your-apify-token" -npx @modelcontextprotocol/inspector npx -y @apify/actors-mcp-server +### Opting Out of Telemetry + +You can disable telemetry in two ways: + +**For the hosted remote server (mcp.apify.com)**: +Add the `?telemetry=off` query parameter to the URL: +```text +https://mcp.apify.com?telemetry=off +``` + +**For the local stdio server**: +Use the `--telemetry off` CLI flag: +```bash +npx @apify/actors-mcp-server --telemetry off ``` # βš™οΈ Development @@ -325,6 +337,15 @@ The Apify MCP Server is also available on [Docker Hub](https://hub.docker.com/mc - Make sure the `APIFY_TOKEN` environment variable is set. - Always use the latest version of the MCP server by using `@apify/actors-mcp-server@latest`. +### Debugging the NPM package + +To debug the server, use the [MCP Inspector](https://github.com/modelcontextprotocol/inspector) tool: + +```shell +export APIFY_TOKEN="your-apify-token" +npx @modelcontextprotocol/inspector npx -y @apify/actors-mcp-server +``` + ## πŸ’‘ Limitations The Actor input schema is processed to be compatible with most MCP clients while adhering to [JSON Schema](https://json-schema.org/) standards. The processing includes: diff --git a/mds/TELEMETRY.md b/mds/TELEMETRY.md new file mode 100644 index 00000000..c9c6a20e --- /dev/null +++ b/mds/TELEMETRY.md @@ -0,0 +1,573 @@ +# Telemetry Implementation Plan + +## Overview + +This document outlines the implementation plan for analytics tracking in the Apify MCP Server using Segment. The goal is to track all tool calls to understand user behavior, tool usage patterns, and MCP client preferences. + +## Data to Be Collected + +### Required Fields per Tool Call + +```json +{ + "userId": "APIFY_USER_ID", + "event": "MCP Tool Call", + "properties": { + "app": "mcp_server", + "mcp_client": "Claude Desktop", + "connection_type": "stdio|remote", + "server_version": "VERSION", + "tool_name": "apify/instagram-scraper", + "reason": "REASON_FOR_TOOL_CALL" + }, + "timestamp": "ISO 8601 TIMESTAMP" +} +``` + +### Data Description + +- **userId**: Apify user ID from authenticated token + - Extracted from Apify API `/v2/users/me` endpoint using the token from `APIFY_TOKEN` env var or `~/.apify/auth.json` + - Cached in memory using SHA-256 hashed token as key (prevents storing raw tokens) + - Falls back to empty string if token is unavailable or API call fails + - Used to identify frequent Apify MCP server users and their use cases + - In Segment: Falls back to 'anonymous' if userId is empty string + +- **event**: "MCP Tool Call" - Event name for all tool calls + +- **MCP Client Name**: Which MCP client is being used (Claude Desktop, Cline, etc.) + - Extracted from `initializeRequestData.params.clientInfo.name` + - Falls back to 'unknown' if client name is unavailable + - Helps understand client distribution and preferences + - Informs which MCP spec features are most important + - Reference: https://modelcontextprotocol.io/clients + +- **Connection Type**: How server is accessed + - `stdio`: Local/direct connection (from `src/stdio.ts` entry point) + - `remote`: Remote/SSE connection (from `src/actor/server.ts` with SSE transport) + - Passed via `ActorsMcpServerOptions.connectionType` + - Differentiates between local and remote MCP server instances + +- **Server Version**: Apify MCP server version + - Dynamically read from package.json via `getPackageVersion()` function in `src/const.ts` + - Currently: '0.5.1' (from package.json) + - Automatically stays in sync with package.json version + - Safe fallback: '0.5.1' if package.json cannot be read + +- **Tool Name**: Name of the tool being called + - Critical for tool usage analytics + - Examples: `search-actors`, `apify/instagram-scraper`, `call-actor` + - For actor tools: uses full actor name (e.g., 'apify/instagram-scraper') + - For internal tools: uses tool name + +- **Reason**: Why the tool was called (LLM-provided reasoning) + - Currently: Empty string (TODO: extract from tool arguments when reason field is added to schemas) + - Originally proposed by @JiΕ™Γ­ Spilka + - Similar to implementation by mcpcat.io at MCP dev summit London + - Will be implemented via special `reason` input field in tool schemas + - Allows LLMs to explain their tool selection and usage context + +- **Timestamp**: When tool call occurred + - Handled automatically by Segment SDK + +## Analytics Use Cases + +### 1. Tool Usage Distribution +- Which tools are used most frequently? +- Which tools are rarely/never used? +- Create tool usage distribution charts +- Better than Prometheus counters (more reliable per Reddit discussions) + +### 2. Time-Series Tracking +- Tool call frequency over time +- Total tool calls per day/week/month +- Identify trends and peak usage periods + +### 3. MCP Client Distribution +- Which MCP clients are most popular? +- Which features of MCP spec are most relied upon? +- Identify implementation priorities based on client needs + +### 4. User Segmentation +- Identify frequent MCP server users +- Track use cases per user/organization +- Understand different user archetypes + +### 5. Tool Call Reasoning +- Understand why specific tools are called +- Group tool calls by context (e.g., "researching Instagram profile", "monitoring for new posts") +- Create dashboards showing tool calls with reasoning/context +- Group by MCP session ID for full interaction flows + +## Implementation Architecture + +### Tool Call Flow + +``` +Tool Call Request + ↓ +[CallToolRequestSchema Handler in src/mcp/server.ts] + ↓ +Tool Validation + ↓ +[TELEMETRY] Extract userId from token + ↓ +[TELEMETRY] Log debug info + ↓ +Tool Execution + ↓ +Return Response +``` + +### Telemetry Module Structure + +**File**: `src/telemetry.ts` + +- **Singleton Pattern**: Map-based singleton clients per environment + - Ensures only one Segment Analytics client per environment (`dev` or `prod`) + - Safe for multiple `ActorsMcpServer` instances to share the same client + - Lazy initialization on first `trackToolCall()` call + +- **Functions**: + - `getOrInitAnalyticsClient(env: 'dev' | 'prod'): Analytics` + - Gets or initializes the client for the specified environment + - Returns singleton instance + - Never called directly from server code + + - `trackToolCall(userId: string, env: 'dev' | 'prod', properties: Record): void` + - Sends tool call event to Segment + - Lazily initializes client if needed + - Converts empty userId to 'anonymous' to comply with Segment API + +### User Cache Module + +**File**: `src/utils/user-cache.ts` + +- **Caching Strategy**: In-memory cache with SHA-256 token hashing + - Caches the full User object returned by Apify API + - Uses token hash as cache key (not the raw token) + - Prevents repeated API calls for the same token + - Thread-safe Map-based storage + +- **Functions**: + - `getUserIdFromToken(token: string, apifyClient: ApifyClient): Promise` + - Fetches user info from `/v2/users/me` endpoint + - Returns cached User object if available + - Returns null if user not found or API call fails + - Hashes token before using as cache key + - Full User object is cached (contains id, username, email, etc.) + +- **Type Definition**: + - `CachedUserInfo` is a type alias for the User object returned by ApifyClient + - Inferred from `Awaited['get']>>` + - Provides type safety while staying DRY (no duplicate interface) + +### Server Integration + +**File**: `src/mcp/server.ts` + +- **ActorsMcpServerOptions Interface**: + - Added `telemetry?: null | 'dev' | 'prod'` option + - `null` (default): No telemetry + - `'dev'`: Use development Segment write key + - `'prod'`: Use production Segment write key + - Added `connectionType?: 'stdio' | 'remote'` option + - Specifies how the server is being accessed + - Passed to telemetry for connection type tracking + - Added `initializeRequestData?: InitializeRequest` option (from MCP SDK) + - Contains client info like `clientInfo.name`, capabilities, etc. + - Injected via message interception or HTTP request body + +- **Constructor** (lines 95-102): + - If telemetry is not explicitly set, reads from `ENVIRONMENT` env variable + - Falls back to undefined if neither provided nor env var available + - Supports environment-based telemetry control for hosted deployments + +- **Tool Call Handler** (lines 568-600, `CallToolRequestSchema`): + - After tool validation and before execution + - Extracts userId from token using `getUserIdFromToken()` + - Logs debug information about the operation: + - userId and whether user was found + - Token availability status + - Builds telemetry properties object with: + - `app`: 'mcp_server' (identifies this server) + - `mcp_client`: Client name from `initializeRequestData.params.clientInfo.name` or 'unknown' + - `connection_type`: 'stdio', 'remote', or 'unknown' + - `server_version`: From `getPackageVersion()` (package.json version) + - `tool_name`: Actor full name or internal tool name + - `reason`: Empty string (TODO: extract from tool args when implemented) + - Logs full telemetry payload before sending (debug level) + - Calls `trackToolCall()` with userId, telemetry environment, and properties + +**File**: `src/utils/version.ts` + +- **`getPackageVersion()` Function**: + - Dynamically reads version from package.json at runtime + - Used in telemetry to report current server version + - Safely falls back to null if file cannot be read + - Works in development and production environments (package.json included in npm files) + +**File**: `src/utils/user-cache.ts` + +- **`getUserIdFromToken(token: string, apifyClient: ApifyClient)` Function**: + - Fetches user info from `/v2/users/me` Apify API endpoint + - Caches full User object using token hash (SHA-256) as key + - Returns cached User object if available (prevents repeated API calls) + - Returns null if user not found or API call fails (safe fallback) + - Token is hashed before caching to avoid storing raw tokens in memory + +**File**: `src/stdio.ts` + +- **Token Resolution** (lines 128-139): + - First tries to read token from `APIFY_TOKEN` environment variable + - Falls back to reading from `~/.apify/auth.json` if env var not set + - Uses helper function `getTokenFromAuthFile()` to read auth file + - JSON file is parsed and token is extracted from `token` key + - Silently fails on file not found or parse errors (no error thrown) + +- **Server Initialization** (lines 143-147): + - Passes `connectionType: 'stdio'` when creating ActorsMcpServer + - Passes telemetry option from CLI `--telemetry` flag (`prod`/`dev`/`off`) + - Converts `'off'` to null, leaves `'prod'` and `'dev'` as-is + +- **Message Interception** (lines 162-176): + - Creates proxy for `transport.onmessage` to intercept MCP messages + - Captures initialize message (first message with `method: 'initialize'`) + - Extracts client information from initialize request data + - Updates mcpServer.options.initializeRequestData with captured data + - Comment explains this is a "hacky way to inject client information" + - Falls back to 'unknown' if client name not found in initialize data + +**File**: `src/telemetry.ts` + +- **`getOrInitAnalyticsClient(env: 'dev' | 'prod')` Function**: + - Singleton pattern ensures only one Segment Analytics client per environment + - Uses Map to store clients: `{ dev: Analytics, prod: Analytics }` + - Lazy initialization on first call + +- **`trackToolCall()` Function**: + - Takes userId (from user cache or empty string) + - Takes telemetry environment ('dev' or 'prod') + - Takes properties object with telemetry data + - Converts empty userId to 'anonymous' for Segment API compliance + - Sends event to Segment with event name: "MCP Tool Call" + +**File**: `package.json` + +- **Files Array**: + - Added `package.json` to `files` array + - Ensures package.json is included in npm publish + - Makes `getPackageVersion()` work in production environments + +### Remote Server Integration (apify-mcp-server-internal) + +The telemetry infrastructure is integrated into the remote server that hosts the MCP service at mcp.apify.com. + +**File**: `src/server/shared.ts` + +- **`injectRequestToolCallBodyParams()` Function**: + - Injects `apifyToken`, `userRentedActorIds`, and `mcpSessionId` into tool call request params + - Extracts session ID from two sources: + - `mcp-session-id` header (for Streamable HTTP transport) + - URL query parameters via `getURLSessionID()` (for legacy SSE transport) + - Enables telemetry to track tool calls across different transport types + - Provides fallback mechanism for session ID extraction + +**File**: `src/server/streamable.ts` + +- **Streamable HTTP Transport Handler**: + - Modern HTTP streaming transport for persistent sessions + - Session resumability support via `mcp-session-id` header + +- **`handleNewSession()` Handler**: + - Extracts `?telemetry=off` query parameter from request URL + - Converts parameter to option: `telemetryParam === 'off' ? null : undefined` + - Passes to ActorsMcpServer constructor: + - `connectionType: 'remote'` - identifies remote/HTTP connection + - `telemetry: telemetryOption` - per-session telemetry control + - `initializeRequestData: req.body` - client info from HTTP request + +- **`handleSessionRestore()` Handler**: + - Restores session from Redis state for resumable connections + - Same telemetry and connectionType handling as handleNewSession() + - Allows clients to resume sessions with telemetry disabled + +**File**: `src/server/legacy-sse.ts` + +- **Legacy SSE Transport Handler**: + - Server-Sent Events transport (deprecated but still supported for backward compatibility) + - Provides session resumability for older clients + +- **`initMCPSession()` Handler** (lines 153-210): + - Extracts `?telemetry=off` query parameter from response URL + - Converts parameter to option: `telemetryParam === 'off' ? null : undefined` + - Creates ActorsMcpServer with: + - `connectionType: 'remote'` - identifies remote/SSE connection + - `telemetry: telemetryOption` - per-session telemetry control + - Message interception proxy (lines 203-210): + - Proxies `transport.onmessage` to capture MCP initialize message + - Extracts client information from initialize request data + - Updates mcpServer.options.initializeRequestData with captured data + - Calls original onmessage handler to continue processing + +- **Per-Session Control**: + - Both transports support `?telemetry=off` query parameter + - Prevents test data from polluting production telemetry + - Example: `https://mcp.apify.com/?telemetry=off` for streamable + - Example: `https://mcp.apify.com/sse?telemetry=off` for SSE + +**Test Configuration**: + +- **`test/integration/tests/server-streamable.test.ts`**: + - mcpUrl configured with `/?telemetry=off` query parameter + - Prevents integration tests from sending telemetry events + +- **`test/integration/tests/server-sse.test.ts`**: + - mcpUrl configured with `/sse?telemetry=off` query parameter + - Prevents integration tests from sending telemetry events + +## Data Flow + +### Available Information at Tool Call Time + +From `CallToolRequestSchema` handler in `src/mcp/server.ts` (line 568+): +- `name`: Tool name (may have 'local__' prefix that is stripped) +- `args`: Validated input arguments (includes `reason` field when implemented) +- `apifyToken`: Apify API token (may be null in Skyfire mode) +- `userRentedActorIds`: List of rented actor IDs +- `progressToken`: Optional progress tracking token +- `meta`: Metadata including progressToken + +From `ActorsMcpServer` instance: +- `this.options.telemetry`: Telemetry environment configuration ('dev', 'prod', or null) +- `this.options.connectionType`: Connection type ('stdio' or 'remote') +- `this.options.initializeRequestData`: MCP client info and capabilities + - `params.clientInfo.name`: MCP client name (e.g., 'Claude Desktop', 'Cline') + - `params.capabilities`: Client capabilities + - `params.protocolVersion`: MCP protocol version + +From `tool` entry: +- `tool.type`: Tool type ('internal', 'actor', 'actor-mcp') +- `tool.tool.name`: Tool name (internal) +- For actor tools: `actorFullName` (e.g., 'apify/instagram-scraper') + +From `src/utils/version.ts`: +- `getPackageVersion()`: Current server version from package.json + +### Token Resolution Flow + +``` +Tool Call Request + ↓ +APIFY_TOKEN env var available? + β”œβ”€ YES β†’ Use env var + └─ NO β†’ Check ~/.apify/auth.json + β”œβ”€ YES β†’ Read and parse JSON + β”‚ Extract 'token' field + β”‚ Use that token + └─ NO β†’ No token (null) + ↓ +Token available? + β”œβ”€ YES β†’ Create ApifyClient with token + β”‚ Call getUserIdFromToken() + β”‚ Return cached or fetched User object + └─ NO β†’ userId stays empty string + ↓ +Track telemetry with userId +``` + +### Connection Type Detection + +Connection type is now passed via `ActorsMcpServerOptions`: +- **Stdio** (Local): When using `src/stdio.ts` entry point + - Passes `connectionType: 'stdio'` when creating ActorsMcpServer + - Message interception proxy captures initialize request data from MCP protocol + - Example: `npx @apify/actors-mcp-server --telemetry=dev` + +- **Streamable HTTP** (Remote): When using `src/server/streamable.ts` with Streamable HTTP transport + - Passes `connectionType: 'remote'` in both `handleSessionRestore()` and `handleNewSession()` handlers + - Extracts `?telemetry=off` query parameter to disable telemetry per-session + - Client info available via `req.body` (InitializeRequest passed as initializeRequestData) + - Connection: `https://mcp.apify.com/?telemetry=off` + +- **Legacy SSE** (Remote): When using `src/server/legacy-sse.ts` with SSE transport + - Passes `connectionType: 'remote'` in `initMCPSession()` handler + - Extracts `?telemetry=off` query parameter from URL + - Message interception proxy captures initialize request data from MCP JSON-RPC messages + - Connection: `https://mcp.apify.com/sse?telemetry=off` + +Future enhancement: Automatically detect connection type from transport type when not explicitly provided. + +## Implementation Notes + +### Current Implementation Status + +#### βœ… Completed +- Telemetry module with singleton Segment clients per environment +- Tool call tracking in `CallToolRequestSchema` handler at line 568 of `src/mcp/server.ts` +- Dynamic version reading from package.json via `getPackageVersion()` function in `src/utils/version.ts` +- Connection type option in ActorsMcpServerOptions interface +- Stdio transport passing `connectionType: 'stdio'` and telemetry CLI flag in `src/stdio.ts` +- package.json included in npm build files +- User cache module with token hashing and in-memory caching in `src/utils/user-cache.ts` +- Token resolution from env var and ~/.apify/auth.json file in `src/stdio.ts` +- Debug logging for telemetry operations in tool call handler +- Full User object caching (not custom wrapper interface) +- Message interception proxy to capture initialize request data in stdio (`src/stdio.ts`) +- Streamable HTTP transport with telemetry query parameter support (`src/server/streamable.ts`) + - `handleNewSession()`: Extracts `?telemetry=off` query param and passes to ActorsMcpServer + - `handleSessionRestore()`: Same telemetry and connectionType handling + - Both handlers pass `connectionType: 'remote'` +- Legacy SSE transport with telemetry query parameter support (`src/server/legacy-sse.ts`) + - `initMCPSession()`: Extracts `?telemetry=off` query param + - Message interception proxy to capture initialize request data from MCP protocol + - Passes `connectionType: 'remote'` and telemetry option +- Test configuration to prevent telemetry pollution + - `test/integration/tests/server-streamable.test.ts`: Uses `/?telemetry=off` + - `test/integration/tests/server-sse.test.ts`: Uses `/sse?telemetry=off` +- MCP session ID tracking and injection + - **Stdio transport** (`src/stdio.ts`): Manually generates UUID4 session ID using `randomUUID()` from `node:crypto` module + - Generated at startup (line 160 in stdio.ts) + - Represents a single session interaction since stdio doesn't have built-in session IDs + - Injected into all tool call messages via message interception proxy (lines 162-176 in stdio.ts) + - **Streamable HTTP transport**: Extracts `mcp-session-id` header from request + - **Legacy SSE transport**: Extracts session ID from URL query parameters via `getURLSessionID()` + - Session ID injected into tool call request params for telemetry tracking + - Supports cross-instance session correlation in distributed deployments + +#### πŸ”² Not Yet Implemented (TODOs) +- Extract reason from tool arguments (requires adding reason field to tool input schemas) +- Implement anonymousId tracking for device/session identification + +### Multi-Server Environment +- Multiple `ActorsMcpServer` instances may run simultaneously + - For Stdio (local): Each connection gets its own server instance + - For Streamable HTTP (remote): Sessions stored in memory and Redis + - For Legacy SSE (remote): Sessions stored in memory and Redis +- Telemetry clients are shared via singleton Map pattern per environment +- User cache is global (shared across all server instances) +- Session data is stored in Redis for cross-instance resumability +- Per-session telemetry control via `?telemetry=off` query parameter prevents test pollution + +### User Authentication +- Apify token extracted from `APIFY_TOKEN` env var first +- Falls back to `~/.apify/auth.json` if env var not set +- Token is hashed before caching (SHA-256) +- User ID cached using token hash as key +- May be empty in: + - Skyfire payment mode (uses `skyfire-pay-id` instead) + - Unauthenticated scenarios (future MCP documentation tools feature) + - If token is invalid or user fetch fails + +### Tool Input Schema Enhancement +- Currently: reason field is always empty string +- TODO: Add optional `reason` field to all tool input schemas +- LLMs will fill in reasoning for why they called the tool +- Enables dashboard and analytics on tool call context + +### Version Management +- Server version is dynamically read from package.json at runtime +- Function: `getPackageVersion()` in `src/utils/version.ts` +- Automatically stays in sync with package.json version +- Works in development, production, and packaged environments +- Fallback: null if package.json cannot be read (logged as 'unknown' in telemetry) + +### Debug Logging + +All telemetry operations emit debug logs including: +- User info fetching: `userId`, `userFound` flag, token availability status +- Full telemetry payload before sending: + - app ('mcp_server') + - mcp_client (client name from initialize data or 'unknown') + - connection_type ('stdio', 'remote', or 'unknown') + - server_version (from package.json or 'unknown') + - tool_name (actor full name or internal tool name) + - reason (empty string) + +Enable with `DEBUG=*` or `LOG_LEVEL=debug` to see telemetry details. + +Example debug output: +``` +Telemetry: fetched user info { userId: 'user-123', userFound: true } +Telemetry: tracking tool call { app: 'mcp_server', mcp_client: 'Claude Desktop', connection_type: 'stdio', server_version: '0.5.3', tool_name: 'apify/instagram-scraper', reason: '' } +``` + +### Future Enhancements + +1. **Device ID / Anonymous ID** + - Implement device ID tracking for session correlation + - Track unauthenticated users via anonymousId + - Link userId and anonymousId when user authenticates + +2. **Session Tracking** βœ… Implemented + - **Stdio** (`src/stdio.ts`): Manually generates UUID4 session ID using `randomUUID()` from `node:crypto` + - Generated at startup (line 160 in stdio.ts) to represent a single session interaction + - Since stdio doesn't have built-in session IDs, UUID4 is created for each stdio connection + - Injected into all tool call messages via message interception proxy (lines 162-176) + - Allows correlating multiple tool calls within the same session + - **Streamable HTTP**: Extracts `mcp-session-id` header from request + - Session ID extracted from request header and injected into tool call params + - **Legacy SSE**: Extracts session ID from URL query parameters + - Falls back to URL extraction when header not available + - Enables session tracking across distributed server instances + - Allows correlating multiple tool calls to a single user session + - Supports session-level analytics and debugging + +3. **Performance Metrics** + - Track tool call duration + - Monitor error rates by tool + - Identify slow tools + +4. **Custom Dashboards** + - Tool call distribution over time + - MCP client adoption trends + - Tool reasoning/context browser + - User journey analysis + +## Segment Configuration + +### Write Keys +- **Development**: `9rPHlMtxX8FJhilGEwkfUoZ0uzWxnzcT` +- **Production**: `cOkp5EIJaN69gYaN8bcp7KtaD0fGABwJ` + +### Event Names +- `MCP Tool Call`: Fired every time a tool is called + +### Integration +- Segment SDK: `@segment/analytics-node` v2.3.0 +- Node.js requirement: 18+ +- Batching: Default 20 messages per batch (SDK configuration) + +## Testing & Validation + +1. **Dev Environment** + - Initialize server with `telemetry: 'dev'` + - Verify events appear in Segment dev workspace + +2. **Production** + - Initialize server with `telemetry: 'prod'` + - Monitor Segment prod workspace for events + +3. **No Telemetry** + - Initialize server without telemetry option + - Verify no tracking occurs + - Verify no errors from missing telemetry + +4. **Token Resolution** + - Test with `APIFY_TOKEN` env var set + - Test with only `~/.apify/auth.json` file + - Test with both set (env var should take precedence) + - Test with neither (should still work but no userId) + +5. **User Cache** + - Same token should return cached result (no API call) + - Different token should trigger new API call + - Invalid token should return null safely + - Debug logs should show cache hits/misses + +## References + +- MCP Clients: https://modelcontextprotocol.io/clients +- mcpcat.io: Similar implementation with tool call reasoning +- Prometheus Discussion: https://www.reddit.com/r/PrometheusMonitoring/comments/1jyxnzv/prometheus_counters_very_unreliable_for_many/ +- Apify API: `/v2/users/me` endpoint for user info diff --git a/package-lock.json b/package-lock.json index ae8a8cd9..8b97783a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@apify/datastructures": "^2.0.3", "@apify/log": "^2.5.16", "@modelcontextprotocol/sdk": "^1.18.1", + "@segment/analytics-node": "^2.3.0", "@types/cheerio": "^0.22.35", "@types/turndown": "^5.0.5", "ajv": "^8.17.1", @@ -1479,6 +1480,27 @@ "dev": true, "license": "MIT" }, + "node_modules/@lukeed/csprng": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@lukeed/csprng/-/csprng-1.1.0.tgz", + "integrity": "sha512-Z7C/xXCiGWsg0KuKsHTKJxbWhpI3Vs5GwLfOean7MGyVFGqdRgBbAjOCh6u4bbjPc/8MJ2pZmK/0DLdCbivLDA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@lukeed/uuid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@lukeed/uuid/-/uuid-2.0.1.tgz", + "integrity": "sha512-qC72D4+CDdjGqJvkFMMEAtancHUQ7/d/tAiHf64z8MopFDmcrtbcJuerDtFceuAfQJ2pDSfCKCtbqoGBNnwg0w==", + "license": "MIT", + "dependencies": { + "@lukeed/csprng": "^1.1.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/@mixmark-io/domino": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz", @@ -2628,6 +2650,45 @@ "integrity": "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==", "license": "MIT" }, + "node_modules/@segment/analytics-core": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/@segment/analytics-core/-/analytics-core-1.8.2.tgz", + "integrity": "sha512-5FDy6l8chpzUfJcNlIcyqYQq4+JTUynlVoCeCUuVz+l+6W0PXg+ljKp34R4yLVCcY5VVZohuW+HH0VLWdwYVAg==", + "license": "MIT", + "dependencies": { + "@lukeed/uuid": "^2.0.0", + "@segment/analytics-generic-utils": "1.2.0", + "dset": "^3.1.4", + "tslib": "^2.4.1" + } + }, + "node_modules/@segment/analytics-generic-utils": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@segment/analytics-generic-utils/-/analytics-generic-utils-1.2.0.tgz", + "integrity": "sha512-DfnW6mW3YQOLlDQQdR89k4EqfHb0g/3XvBXkovH1FstUN93eL1kfW9CsDcVQyH3bAC5ZsFyjA/o/1Q2j0QeoWw==", + "license": "MIT", + "dependencies": { + "tslib": "^2.4.1" + } + }, + "node_modules/@segment/analytics-node": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@segment/analytics-node/-/analytics-node-2.3.0.tgz", + "integrity": "sha512-fOXLL8uY0uAWw/sTLmezze80hj8YGgXXlAfvSS6TUmivk4D/SP0C0sxnbpFdkUzWg2zT64qWIZj26afEtSnxUA==", + "license": "MIT", + "dependencies": { + "@lukeed/uuid": "^2.0.0", + "@segment/analytics-core": "1.8.2", + "@segment/analytics-generic-utils": "1.2.0", + "buffer": "^6.0.3", + "jose": "^5.1.0", + "node-fetch": "^2.6.7", + "tslib": "^2.4.1" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@sindresorhus/is": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.0.1.tgz", @@ -3744,6 +3805,26 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "license": "MIT" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/body-parser": { "version": "1.20.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", @@ -3860,6 +3941,30 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -4507,6 +4612,15 @@ "url": "https://dotenvx.com" } }, + "node_modules/dset": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.4.tgz", + "integrity": "sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -6637,6 +6751,15 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "license": "ISC" }, + "node_modules/jose": { + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/jose/-/jose-5.10.0.tgz", + "integrity": "sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -7040,7 +7163,6 @@ "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dev": true, "license": "MIT", "dependencies": { "whatwg-url": "^5.0.0" @@ -8748,7 +8870,6 @@ "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "dev": true, "license": "MIT" }, "node_modules/ts-api-utils": { @@ -9323,7 +9444,6 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "dev": true, "license": "BSD-2-Clause" }, "node_modules/whatwg-encoding": { @@ -9363,7 +9483,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dev": true, "license": "MIT", "dependencies": { "tr46": "~0.0.3", diff --git a/package.json b/package.json index 81366d42..a19fa237 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "files": [ "dist", "LICENSE", + "package.json", "server.json", "manifest.json" ], @@ -42,6 +43,7 @@ "@apify/datastructures": "^2.0.3", "@apify/log": "^2.5.16", "@modelcontextprotocol/sdk": "^1.18.1", + "@segment/analytics-node": "^2.3.0", "@types/cheerio": "^0.22.35", "@types/turndown": "^5.0.5", "ajv": "^8.17.1", diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 6b966d72..1f979670 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -35,6 +35,7 @@ import { SKYFIRE_TOOL_INSTRUCTIONS, } from '../const.js'; import { prompts } from '../prompts/index.js'; +import { trackToolCall } from '../telemetry.js'; import { callActorGetDataset, defaultTools, getActorsAsTools, toolCategories } from '../tools/index.js'; import { decodeDotPropertyNames } from '../tools/utils.js'; import type { ActorMcpTool, ActorTool, HelperTool, ToolEntry } from '../types.js'; @@ -42,6 +43,8 @@ import { buildActorResponseContent } from '../utils/actor-response.js'; import { buildMCPResponse } from '../utils/mcp.js'; import { createProgressTracker } from '../utils/progress.js'; import { cloneToolEntry, getToolPublicFieldOnly } from '../utils/tools.js'; +import { getUserIdFromToken } from '../utils/user-cache.js'; +import { getPackageVersion } from '../utils/version.js'; import { connectMCPClient } from './client.js'; import { EXTERNAL_TOOL_CALL_TIMEOUT_MSEC, LOG_LEVEL_MAP } from './const.js'; import { processParamsGetTools } from './utils.js'; @@ -55,6 +58,25 @@ interface ActorsMcpServerOptions { */ skyfireMode?: boolean; initializeRequestData?: InitializeRequest; + /** + * Enable telemetry tracking for tool calls. + * - null: No telemetry (default) + * - 'dev': Use development Segment write key + * - 'prod': Use production Segment write key + */ + telemetry?: null | 'dev' | 'prod'; + /** + * Connection type for telemetry tracking. + * - 'stdio': Direct/local connection + * - 'remote': Remote/HTTP streamble or SSE connection + */ + connectionType?: 'stdio' | 'remote'; + /** + * Apify API token for authentication + * Primarily used by stdio transport when token is read from ~/.apify/auth.json file + * instead of APIFY_TOKEN environment variable, so it can be passed to the server + */ + token?: string; } /** @@ -70,6 +92,15 @@ export class ActorsMcpServer { constructor(options: ActorsMcpServerOptions = {}) { this.options = options; + + // If telemetry is not explicitly set, try to read from ENVIRONMENT env variable, this is used in the mcp.apify.com deployment + if (this.options.telemetry === undefined) { + const envValue = process.env.ENVIRONMENT; + if (envValue === 'dev' || envValue === 'prod') { + this.options.telemetry = envValue; + } + } + const { setupSigintHandler = true } = options; this.server = new Server( { @@ -467,16 +498,21 @@ export class ActorsMcpServer { * @throws {McpError} - based on the McpServer class code from the typescript MCP SDK */ this.server.setRequestHandler(CallToolRequestSchema, async (request, extra) => { + log.debug('FINDME Received request to call tool', { request }); // eslint-disable-next-line prefer-const let { name, arguments: args, _meta: meta } = request.params; const { progressToken } = meta || {}; - const apifyToken = (request.params.apifyToken || process.env.APIFY_TOKEN) as string; + const apifyToken = (request.params.apifyToken || this.options.token || process.env.APIFY_TOKEN) as string; const userRentedActorIds = request.params.userRentedActorIds as string[] | undefined; + // Injected for telemetry purposes + const mcpSessionId = request.params.mcpSessionId as string | undefined; // Remove apifyToken from request.params just in case delete request.params.apifyToken; // Remove other custom params passed from apify-mcp-server delete request.params.userRentedActorIds; + // Remove mcpSessionId + delete request.params.mcpSessionId; // Validate token if (!apifyToken && !this.options.skyfireMode) { @@ -535,6 +571,38 @@ export class ActorsMcpServer { ); } + // Track telemetry if enabled + if (this.options.telemetry && (this.options.telemetry === 'dev' || this.options.telemetry === 'prod')) { + const toolFullName = tool.type === 'actor' ? (tool.tool as ActorTool).actorFullName : tool.tool.name; + + // Get userId from cache or fetch from API + let userId = ''; + // Use token from options (e.g., from stdio auth file) or from request + if (apifyToken) { + const apifyClient = new ApifyClient({ token: apifyToken }); + const userInfo = await getUserIdFromToken(apifyToken, apifyClient); + userId = userInfo?.id || ''; + log.debug('Telemetry: fetched user info', { userId, userFound: !!userInfo }); + } else { + log.debug('Telemetry: no API token provided'); + } + + const telemetryData = { + app: 'mcp_server', + mcp_client: this.options.initializeRequestData?.params?.clientInfo?.name || '', + mcp_session_id: mcpSessionId || '', + connection_type: this.options.connectionType || '', + // This is the version if the apify-mcp-server package + // this can be different from the internal remote server version + server_version: getPackageVersion() || '', + tool_name: toolFullName, + reason: '', + }; + + log.debug('Telemetry: tracking tool call', telemetryData); + trackToolCall(userId, this.options.telemetry, telemetryData); + } + try { // Handle internal tool if (tool.type === 'internal') { diff --git a/src/stdio.ts b/src/stdio.ts index d2e2621a..4a5511b9 100644 --- a/src/stdio.ts +++ b/src/stdio.ts @@ -13,7 +13,13 @@ * node stdio.js --actors=apify/google-search-scraper,apify/instagram-scraper */ +import { randomUUID } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; + import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; +import type { JSONRPCMessage } from '@modelcontextprotocol/sdk/types.js'; import yargs from 'yargs'; // Had to ignore the eslint import extension error for the yargs package. // Using .js or /index.js didn't resolve it due to the @types package issues. @@ -41,6 +47,23 @@ interface CliArgs { enableActorAutoLoading: boolean; /** Tool categories to include */ tools?: string; + /** Telemetry environment: 'prod', 'dev', or 'off' */ + telemetry: 'prod' | 'dev' | 'off'; +} + +/** + * Attempts to read Apify token from ~/.apify/auth.json file + * Returns the token if found, undefined otherwise + */ +function getTokenFromAuthFile(): string | undefined { + try { + const authPath = join(homedir(), '.apify', 'auth.json'); + const content = readFileSync(authPath, 'utf-8'); + const authData = JSON.parse(content); + return authData.token || undefined; + } catch { + return undefined; + } } // Configure logging, set to ERROR @@ -75,6 +98,15 @@ Deprecated: use tools experimental category instead.`, For more details visit https://mcp.apify.com`, example: 'actors,docs,apify/rag-web-browser', }) + .option('telemetry', { + type: 'string', + choices: ['prod', 'dev', 'off'], + default: 'prod', + describe: `Enable telemetry tracking for tool calls. Can also be set via TELEMETRY environment variable. +- 'prod': Send events to production Segment workspace (default) +- 'dev': Send events to development Segment workspace +- 'off': Disable telemetry`, + }) .help('help') .alias('h', 'help') .version(false) @@ -100,14 +132,21 @@ log.error = (...args: Parameters) => { console.error(...args); }; +// Get token from environment or auth file +const apifyToken = process.env.APIFY_TOKEN || getTokenFromAuthFile(); + // Validate environment -if (!process.env.APIFY_TOKEN) { - log.error('APIFY_TOKEN is required but not set in the environment variables.'); +if (!apifyToken) { + log.error('APIFY_TOKEN is required but not set in the environment variables or ~/.apify/auth.json'); process.exit(1); } async function main() { - const mcpServer = new ActorsMcpServer(); + const mcpServer = new ActorsMcpServer({ + connectionType: 'stdio', + telemetry: argv.telemetry === 'off' ? null : (argv.telemetry as 'dev' | 'prod'), + token: apifyToken, + }); // Create an Input object from CLI arguments const input: Input = { @@ -119,7 +158,7 @@ async function main() { // Normalize (merges actors into tools for backward compatibility) const normalizedInput = processInput(input); - const apifyClient = new ApifyClient({ token: process.env.APIFY_TOKEN }); + const apifyClient = new ApifyClient({ token: apifyToken }); // Use the shared tools loading logic const tools = await loadToolsFromInput(normalizedInput, apifyClient); @@ -127,6 +166,35 @@ async function main() { // Start server const transport = new StdioServerTransport(); + + // Generate a unique session ID for this stdio connection + // Note: stdio transport does not have a strict session ID concept like HTTP transports, + // so we generate a UUID4 to represent this single session interaction for telemetry tracking + const mcpSessionId = randomUUID(); + + // Create a proxy for transport.onmessage to intercept and capture initialize request data + // This is a hacky way to inject client information into the ActorsMcpServer class + const originalOnMessage = transport.onmessage; + + transport.onmessage = (message: JSONRPCMessage) => { + // Extract client information from initialize message + const msgRecord = message as Record; + if (msgRecord.method === 'initialize') { + // Update mcpServer options with initialize request data + (mcpServer.options as Record).initializeRequestData = msgRecord as Record; + } + // Inject session ID into tool call messages + if (msgRecord.method === 'tools/call' && msgRecord.params) { + const params = msgRecord.params as Record; + params.mcpSessionId = mcpSessionId; + } + + // Call the original onmessage handler + if (originalOnMessage) { + originalOnMessage(message); + } + }; + await mcpServer.connect(transport); } diff --git a/src/telemetry.ts b/src/telemetry.ts new file mode 100644 index 00000000..77f2e566 --- /dev/null +++ b/src/telemetry.ts @@ -0,0 +1,49 @@ +import { Analytics } from '@segment/analytics-node'; + +const DEV_WRITE_KEY = '9rPHlMtxX8FJhilGEwkfUoZ0uzWxnzcT'; +const PROD_WRITE_KEY = 'cOkp5EIJaN69gYaN8bcp7KtaD0fGABwJ'; + +const SEGMENT_EVENTS = { + TOOL_CALL: 'MCP Tool Call', +}; + +// Map to store singleton Segment Analytics clients per environment +const analyticsClients = new Map<'dev' | 'prod', Analytics>(); + +/** + * Gets or initializes a Segment Analytics client for the specified environment. + * This ensures that only one client is created per environment, even if multiple + * ActorsMcpServer instances are initialized with telemetry enabled. + * + * @param env - 'dev' for development, 'prod' for production + * @returns Analytics client instance + */ +export function getOrInitAnalyticsClient(env: 'dev' | 'prod'): Analytics { + if (!analyticsClients.has(env)) { + const writeKey = env === 'prod' ? PROD_WRITE_KEY : DEV_WRITE_KEY; + analyticsClients.set(env, new Analytics({ writeKey })); + } + return analyticsClients.get(env)!; +} + +/** + * Tracks a tool call event to Segment. + * + * @param userId - Apify user ID (TODO: extract from token when auth available) + * @param env - 'dev' for development, 'prod' for production + * @param properties - Additional event properties + */ +export function trackToolCall( + userId: string, + env: 'dev' | 'prod', + properties: Record, +): void { + const client = getOrInitAnalyticsClient(env); + + // TODO: Implement anonymousId tracking for device/session identification + client.track({ + userId: userId || 'anonymous', + event: SEGMENT_EVENTS.TOOL_CALL, + properties, + }); +} diff --git a/src/utils/user-cache.ts b/src/utils/user-cache.ts new file mode 100644 index 00000000..6a9aeec2 --- /dev/null +++ b/src/utils/user-cache.ts @@ -0,0 +1,39 @@ +import { createHash } from 'node:crypto'; + +import type { User } from 'apify-client'; + +import type { ApifyClient } from '../apify-client.js'; + +// Type for cached user info - stores the raw User object from API +const userCache = new Map(); + +/** + * Gets user info from token, using cache to avoid repeated API calls + * Token is hashed before caching to avoid storing raw tokens + * Returns the full User object from API or null if not found + */ +export async function getUserIdFromToken( + token: string, + apifyClient: ApifyClient, +): Promise { + // Hash token for cache key + const tokenHash = createHash('sha256').update(token).digest('hex'); + + // Check cache first + if (userCache.has(tokenHash)) { + return userCache.get(tokenHash)!; + } + + // Fetch from API + try { + const user = await apifyClient.user('me').get(); + if (!user || !user.id) { + return null; + } + + userCache.set(tokenHash, user); + return user; + } catch { + return null; + } +} diff --git a/src/utils/version.ts b/src/utils/version.ts new file mode 100644 index 00000000..127ff80f --- /dev/null +++ b/src/utils/version.ts @@ -0,0 +1,24 @@ +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +/** + * Gets the package version dynamically from package.json + * Returns null if the file cannot be read + */ +export function getPackageVersion(): string | null { + try { + // Read package.json and extract version + // In production, this will be replaced at build time + // eslint-disable-next-line no-underscore-dangle + const __filename = fileURLToPath(import.meta.url); + // eslint-disable-next-line no-underscore-dangle + const __dirname = dirname(__filename); + const packagePath = join(__dirname, '../../package.json'); + const packageData = JSON.parse(readFileSync(packagePath, 'utf-8')); + return packageData.version || null; + } catch { + // Return null if package.json cannot be read + return null; + } +} diff --git a/tests/helpers.ts b/tests/helpers.ts index 1fc5d2ac..2e1b0802 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -116,6 +116,8 @@ export async function createMcpStdioClient( if (tools !== undefined) { env.TOOLS = tools.join(','); } + // Disable telemetry for tests + env.TELEMETRY = 'off'; } else { // Use command line arguments as before if (actors !== undefined) { @@ -127,6 +129,8 @@ export async function createMcpStdioClient( if (tools !== undefined) { args.push('--tools', tools.join(',')); } + // Disable telemetry for tests + args.push('--telemetry', 'off'); } const transport = new StdioClientTransport({ From 3e58be67e81feb15ea8f3f74c7d8cf433d5c5e4a Mon Sep 17 00:00:00 2001 From: MQ37 Date: Thu, 6 Nov 2025 21:35:56 -0800 Subject: [PATCH 02/46] add reason input schema field for telemetry --- mds/TELEMETRY.md | 19 +++++-- src/actor/server.ts | 21 ++++++- src/mcp/server.ts | 49 +++++++++++----- src/tools/build.ts | 5 +- src/tools/dataset.ts | 15 ++++- src/tools/dataset_collection.ts | 5 +- src/tools/fetch-actor-details.ts | 5 +- src/tools/fetch-apify-docs.ts | 5 +- src/tools/get-actor-output.ts | 2 +- src/tools/get-html-skeleton.ts | 5 +- src/tools/helpers.ts | 5 +- src/tools/key_value_store.ts | 15 ++++- src/tools/key_value_store_collection.ts | 5 +- src/tools/run.ts | 6 +- src/tools/run_collection.ts | 5 +- src/tools/search-apify-docs.ts | 5 +- src/tools/store_collection.ts | 5 +- tests/helpers.ts | 20 +++++-- tests/integration/actor.server-sse.test.ts | 3 + .../actor.server-streamable.test.ts | 3 + tests/integration/suite.ts | 57 +++++++++++++++++++ 21 files changed, 213 insertions(+), 47 deletions(-) diff --git a/mds/TELEMETRY.md b/mds/TELEMETRY.md index c9c6a20e..a64173b1 100644 --- a/mds/TELEMETRY.md +++ b/mds/TELEMETRY.md @@ -61,11 +61,13 @@ This document outlines the implementation plan for analytics tracking in the Api - For internal tools: uses tool name - **Reason**: Why the tool was called (LLM-provided reasoning) - - Currently: Empty string (TODO: extract from tool arguments when reason field is added to schemas) - - Originally proposed by @JiΕ™Γ­ Spilka - - Similar to implementation by mcpcat.io at MCP dev summit London - - Will be implemented via special `reason` input field in tool schemas + - βœ… **IMPLEMENTED**: Added as optional `reason` field to all tool input schemas when telemetry is enabled + - Extracted from tool call arguments: `(args as Record).reason?.toString() || ''` + - When telemetry is `'off'`: reason field is NOT added to schemas (saves schema overhead for tests) + - When telemetry is `'dev'` or `'prod'`: reason field is dynamically added to ALL tool schemas + - Field description: "A brief explanation of why this tool is being called and what it will help you accomplish" - Allows LLMs to explain their tool selection and usage context + - Originally proposed by @JiΕ™Γ­ Spilka, similar to mcpcat.io implementation at MCP dev summit London - **Timestamp**: When tool call occurred - Handled automatically by Segment SDK @@ -434,9 +436,16 @@ Future enhancement: Automatically detect connection type from transport type whe - **Legacy SSE transport**: Extracts session ID from URL query parameters via `getURLSessionID()` - Session ID injected into tool call request params for telemetry tracking - Supports cross-instance session correlation in distributed deployments +- **Reason field implementation** βœ… + - Dynamically added to all tool input schemas when telemetry is enabled (`'dev'` or `'prod'`) + - Optional string field with title "Reason" and guidance description + - Extracted from tool arguments during tool call: `(args as Record).reason?.toString() || ''` + - Not added when telemetry is `'off'` (reduces schema overhead for tests) + - All AJV validators updated with `additionalProperties: true` to accept the field + - New tests verify: reason field presence/absence based on telemetry setting + - Works with `upsertTools()` conditional modification logic alongside Skyfire mode #### πŸ”² Not Yet Implemented (TODOs) -- Extract reason from tool arguments (requires adding reason field to tool input schemas) - Implement anonymousId tracking for device/session identification ### Multi-Server Environment diff --git a/src/actor/server.ts b/src/actor/server.ts index 5d14ac45..4875e03e 100644 --- a/src/actor/server.ts +++ b/src/actor/server.ts @@ -71,7 +71,12 @@ export function createExpressApp( rt: Routes.SSE, tr: TransportType.SSE, }); - const mcpServer = new ActorsMcpServer({ setupSigintHandler: false }); + // Extract telemetry query parameter - if 'off' disable, otherwise use ENVIRONMENT env variable + const urlParams = new URL(req.url, `http://${req.headers.host}`).searchParams; + const telemetryParam = urlParams.get('telemetry'); + const telemetry = telemetryParam === 'off' ? null : undefined; + + const mcpServer = new ActorsMcpServer({ setupSigintHandler: false, connectionType: 'remote', telemetry }); const transport = new SSEServerTransport(Routes.MESSAGE, res); // Load MCP server tools @@ -150,12 +155,22 @@ export function createExpressApp( // Reuse existing transport transport = transports[sessionId]; } else if (!sessionId && isInitializeRequest(req.body)) { - // New initialization request - use JSON response mode + // New initialization request transport = new StreamableHTTPServerTransport({ sessionIdGenerator: () => randomUUID(), enableJsonResponse: false, // Use SSE response mode }); - const mcpServer = new ActorsMcpServer({ setupSigintHandler: false, initializeRequestData: req.body as InitializeRequest }); + // Extract telemetry query parameter - if 'off' disable, otherwise use ENVIRONMENT env variable + const urlParams = new URL(req.url, `http://${req.headers.host}`).searchParams; + const telemetryParam = urlParams.get('telemetry'); + const telemetry = telemetryParam === 'off' ? null : undefined; + + const mcpServer = new ActorsMcpServer({ + setupSigintHandler: false, + initializeRequestData: req.body as InitializeRequest, + connectionType: 'remote', + telemetry, + }); // Load MCP server tools const apifyToken = process.env.APIFY_TOKEN as string; diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 1f979670..14f686f1 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -293,15 +293,18 @@ export class ActorsMcpServer { * @returns Array of added/updated tool wrappers */ public upsertTools(tools: ToolEntry[], shouldNotifyToolsChangedHandler = false) { - // Handle Skyfire mode modifications before storing tools - if (this.options.skyfireMode) { + const isTelemetryEnabled = this.options.telemetry === 'dev' || this.options.telemetry === 'prod'; + + if (this.options.skyfireMode || isTelemetryEnabled) { for (const wrap of tools) { - if (wrap.type === 'actor' - || (wrap.type === 'internal' && wrap.tool.name === HelperTools.ACTOR_CALL) - || (wrap.type === 'internal' && wrap.tool.name === HelperTools.ACTOR_OUTPUT_GET)) { - // Clone the tool before modifying it to avoid affecting shared objects - const clonedWrap = cloneToolEntry(wrap); + // Clone the tool before modifying it to avoid affecting shared objects + const clonedWrap = cloneToolEntry(wrap); + let modified = false; + // Handle Skyfire mode modifications + if (this.options.skyfireMode && (wrap.type === 'actor' + || (wrap.type === 'internal' && wrap.tool.name === HelperTools.ACTOR_CALL) + || (wrap.type === 'internal' && wrap.tool.name === HelperTools.ACTOR_OUTPUT_GET))) { // Add Skyfire instructions to description if not already present if (!clonedWrap.tool.description.includes(SKYFIRE_TOOL_INSTRUCTIONS)) { clonedWrap.tool.description += `\n\n${SKYFIRE_TOOL_INSTRUCTIONS}`; @@ -316,16 +319,30 @@ export class ActorsMcpServer { }; } } + modified = true; + } - // Store the cloned and modified tool - this.tools.set(clonedWrap.tool.name, clonedWrap); - } else { - // Store unmodified tools as-is - this.tools.set(wrap.tool.name, wrap); + // Handle telemetry modifications - add reason field to all tools when telemetry is enabled + if (isTelemetryEnabled) { + if (clonedWrap.tool.inputSchema && 'properties' in clonedWrap.tool.inputSchema) { + const props = clonedWrap.tool.inputSchema.properties as Record; + if (!props.reason) { + props.reason = { + type: 'string', + title: 'Reason', + description: 'A brief explanation of why this tool is being called and what it will help you accomplish. ' + + 'Keep it concise and do not include any personal identifiable information (PII) or sensitive data.', + }; + } + } + modified = true; } + + // Store the cloned and modified tool only if modifications were made + this.tools.set(clonedWrap.tool.name, modified ? clonedWrap : wrap); } } else { - // No skyfire mode - store tools as-is + // No skyfire mode and telemetry disabled - store tools as-is for (const wrap of tools) { this.tools.set(wrap.tool.name, wrap); } @@ -498,7 +515,6 @@ export class ActorsMcpServer { * @throws {McpError} - based on the McpServer class code from the typescript MCP SDK */ this.server.setRequestHandler(CallToolRequestSchema, async (request, extra) => { - log.debug('FINDME Received request to call tool', { request }); // eslint-disable-next-line prefer-const let { name, arguments: args, _meta: meta } = request.params; const { progressToken } = meta || {}; @@ -587,6 +603,9 @@ export class ActorsMcpServer { log.debug('Telemetry: no API token provided'); } + // Extract reason from tool arguments if provided + const reason = (args as Record).reason?.toString() || ''; + const telemetryData = { app: 'mcp_server', mcp_client: this.options.initializeRequestData?.params?.clientInfo?.name || '', @@ -596,7 +615,7 @@ export class ActorsMcpServer { // this can be different from the internal remote server version server_version: getPackageVersion() || '', tool_name: toolFullName, - reason: '', + reason, }; log.debug('Telemetry: tracking tool call', telemetryData); diff --git a/src/tools/build.ts b/src/tools/build.ts index f9b577fd..71d051c5 100644 --- a/src/tools/build.ts +++ b/src/tools/build.ts @@ -117,7 +117,10 @@ export const actorDefinitionTool: ToolEntry = { + 'Get details for an Actor with with Actor ID or Actor full name, i.e. username/name.' + `Limit the length of the README if needed.`, inputSchema: zodToJsonSchema(getActorDefinitionArgsSchema), - ajvValidate: ajv.compile(zodToJsonSchema(getActorDefinitionArgsSchema)), + ajvValidate: ajv.compile({ + ...zodToJsonSchema(getActorDefinitionArgsSchema), + additionalProperties: true, // Allow additional properties for telemetry reason field + }), call: async (toolArgs) => { const { args, apifyToken } = toolArgs; diff --git a/src/tools/dataset.ts b/src/tools/dataset.ts index a72ce760..3c5eccbe 100644 --- a/src/tools/dataset.ts +++ b/src/tools/dataset.ts @@ -58,7 +58,10 @@ USAGE EXAMPLES: - user_input: Show info for dataset xyz123 - user_input: What fields does username~my-dataset have?`, inputSchema: zodToJsonSchema(getDatasetArgs), - ajvValidate: ajv.compile(zodToJsonSchema(getDatasetArgs)), + ajvValidate: ajv.compile({ + ...zodToJsonSchema(getDatasetArgs), + additionalProperties: true, // Allow additional properties for telemetry reason field + }), call: async (toolArgs) => { const { args, apifyToken } = toolArgs; const parsed = getDatasetArgs.parse(args); @@ -93,7 +96,10 @@ USAGE EXAMPLES: - user_input: Get first 100 items from dataset abd123 - user_input: Get only metadata.url and title from dataset username~my-dataset (flatten metadata)`, inputSchema: zodToJsonSchema(getDatasetItemsArgs), - ajvValidate: ajv.compile(zodToJsonSchema(getDatasetItemsArgs)), + ajvValidate: ajv.compile({ + ...zodToJsonSchema(getDatasetItemsArgs), + additionalProperties: true, // Allow additional properties for telemetry reason field + }), call: async (toolArgs) => { const { args, apifyToken } = toolArgs; const parsed = getDatasetItemsArgs.parse(args); @@ -155,7 +161,10 @@ USAGE EXAMPLES: - user_input: Generate schema for dataset 34das2 using 10 items - user_input: Show schema of username~my-dataset (clean items only)`, inputSchema: zodToJsonSchema(getDatasetSchemaArgs), - ajvValidate: ajv.compile(zodToJsonSchema(getDatasetSchemaArgs)), + ajvValidate: ajv.compile({ + ...zodToJsonSchema(getDatasetSchemaArgs), + additionalProperties: true, // Allow additional properties for telemetry reason field + }), call: async (toolArgs) => { const { args, apifyToken } = toolArgs; const parsed = getDatasetSchemaArgs.parse(args); diff --git a/src/tools/dataset_collection.ts b/src/tools/dataset_collection.ts index 2f29f553..064747a9 100644 --- a/src/tools/dataset_collection.ts +++ b/src/tools/dataset_collection.ts @@ -43,7 +43,10 @@ USAGE EXAMPLES: - user_input: List my last 10 datasets (newest first) - user_input: List unnamed datasets`, inputSchema: zodToJsonSchema(getUserDatasetsListArgs), - ajvValidate: ajv.compile(zodToJsonSchema(getUserDatasetsListArgs)), + ajvValidate: ajv.compile({ + ...zodToJsonSchema(getUserDatasetsListArgs), + additionalProperties: true, // Allow additional properties for telemetry reason field + }), call: async (toolArgs) => { const { args, apifyToken } = toolArgs; const parsed = getUserDatasetsListArgs.parse(args); diff --git a/src/tools/fetch-actor-details.ts b/src/tools/fetch-actor-details.ts index 392e2237..b5fb8140 100644 --- a/src/tools/fetch-actor-details.ts +++ b/src/tools/fetch-actor-details.ts @@ -29,7 +29,10 @@ USAGE EXAMPLES: - user_input: What is the input schema for apify/rag-web-browser? - user_input: What is the pricing for apify/instagram-scraper?`, inputSchema: zodToJsonSchema(fetchActorDetailsToolArgsSchema), - ajvValidate: ajv.compile(zodToJsonSchema(fetchActorDetailsToolArgsSchema)), + ajvValidate: ajv.compile({ + ...zodToJsonSchema(fetchActorDetailsToolArgsSchema), + additionalProperties: true, // Allow additional properties for telemetry reason field + }), call: async (toolArgs) => { const { args, apifyToken } = toolArgs; const parsed = fetchActorDetailsToolArgsSchema.parse(args); diff --git a/src/tools/fetch-apify-docs.ts b/src/tools/fetch-apify-docs.ts index 7eb80b94..bb6c576e 100644 --- a/src/tools/fetch-apify-docs.ts +++ b/src/tools/fetch-apify-docs.ts @@ -30,7 +30,10 @@ USAGE EXAMPLES: - user_input: Fetch https://docs.apify.com/academy`, args: fetchApifyDocsToolArgsSchema, inputSchema: zodToJsonSchema(fetchApifyDocsToolArgsSchema), - ajvValidate: ajv.compile(zodToJsonSchema(fetchApifyDocsToolArgsSchema)), + ajvValidate: ajv.compile({ + ...zodToJsonSchema(fetchApifyDocsToolArgsSchema), + additionalProperties: true, // Allow additional properties for telemetry reason field + }), call: async (toolArgs) => { const { args } = toolArgs; diff --git a/src/tools/get-actor-output.ts b/src/tools/get-actor-output.ts index ba4aea81..303af495 100644 --- a/src/tools/get-actor-output.ts +++ b/src/tools/get-actor-output.ts @@ -87,7 +87,7 @@ USAGE EXAMPLES: Note: This tool is automatically included if the Apify MCP Server is configured with any Actor tools (e.g., "apify-slash-rag-web-browser") or tools that can interact with Actors (e.g., "call-actor", "add-actor").`, inputSchema: zodToJsonSchema(getActorOutputArgs), /** - * Allow additional properties for Skyfire mode to pass `skyfire-pay-id`. + * Allow additional properties for Skyfire mode to pass `skyfire-pay-id` and for telemetry to pass `reason` field. */ ajvValidate: ajv.compile({ ...zodToJsonSchema(getActorOutputArgs), additionalProperties: true }), call: async (toolArgs) => { diff --git a/src/tools/get-html-skeleton.ts b/src/tools/get-html-skeleton.ts index ba73cdae..852d08d8 100644 --- a/src/tools/get-html-skeleton.ts +++ b/src/tools/get-html-skeleton.ts @@ -53,7 +53,10 @@ USAGE EXAMPLES: - user_input: Get HTML skeleton for https://example.com - user_input: Get next chunk of HTML skeleton for https://example.com (chunk=2)`, inputSchema: zodToJsonSchema(getHtmlSkeletonArgs), - ajvValidate: ajv.compile(zodToJsonSchema(getHtmlSkeletonArgs)), + ajvValidate: ajv.compile({ + ...zodToJsonSchema(getHtmlSkeletonArgs), + additionalProperties: true, // Allow additional properties for telemetry reason field + }), call: async (toolArgs) => { const { args, apifyToken } = toolArgs; const parsed = getHtmlSkeletonArgs.parse(args); diff --git a/src/tools/helpers.ts b/src/tools/helpers.ts index f5f2f8de..275dfef0 100644 --- a/src/tools/helpers.ts +++ b/src/tools/helpers.ts @@ -27,7 +27,10 @@ USAGE EXAMPLES: - user_input: Add apify/rag-web-browser as a tool - user_input: Add apify/instagram-scraper as a tool`, inputSchema: zodToJsonSchema(addToolArgsSchema), - ajvValidate: ajv.compile(zodToJsonSchema(addToolArgsSchema)), + ajvValidate: ajv.compile({ + ...zodToJsonSchema(addToolArgsSchema), + additionalProperties: true, // Allow additional properties for telemetry reason field + }), // TODO: I don't like that we are passing apifyMcpServer and mcpServer to the tool call: async (toolArgs) => { const { apifyMcpServer, apifyToken, args, extra: { sendNotification } } = toolArgs; diff --git a/src/tools/key_value_store.ts b/src/tools/key_value_store.ts index eb9124e5..b781ff00 100644 --- a/src/tools/key_value_store.ts +++ b/src/tools/key_value_store.ts @@ -30,7 +30,10 @@ USAGE EXAMPLES: - user_input: Show info for key-value store username~my-store - user_input: Get details for store adb123`, inputSchema: zodToJsonSchema(getKeyValueStoreArgs), - ajvValidate: ajv.compile(zodToJsonSchema(getKeyValueStoreArgs)), + ajvValidate: ajv.compile({ + ...zodToJsonSchema(getKeyValueStoreArgs), + additionalProperties: true, // Allow additional properties for telemetry reason field + }), call: async (toolArgs) => { const { args, apifyToken } = toolArgs; const parsed = getKeyValueStoreArgs.parse(args); @@ -73,7 +76,10 @@ USAGE EXAMPLES: - user_input: List first 100 keys in store username~my-store - user_input: Continue listing keys in store a123 from key data.json`, inputSchema: zodToJsonSchema(getKeyValueStoreKeysArgs), - ajvValidate: ajv.compile(zodToJsonSchema(getKeyValueStoreKeysArgs)), + ajvValidate: ajv.compile({ + ...zodToJsonSchema(getKeyValueStoreKeysArgs), + additionalProperties: true, // Allow additional properties for telemetry reason field + }), call: async (toolArgs) => { const { args, apifyToken } = toolArgs; const parsed = getKeyValueStoreKeysArgs.parse(args); @@ -114,7 +120,10 @@ USAGE EXAMPLES: - user_input: Get record INPUT from store abc123 - user_input: Get record data.json from store username~my-store`, inputSchema: zodToJsonSchema(getKeyValueStoreRecordArgs), - ajvValidate: ajv.compile(zodToJsonSchema(getKeyValueStoreRecordArgs)), + ajvValidate: ajv.compile({ + ...zodToJsonSchema(getKeyValueStoreRecordArgs), + additionalProperties: true, // Allow additional properties for telemetry reason field + }), call: async (toolArgs) => { const { args, apifyToken } = toolArgs; const parsed = getKeyValueStoreRecordArgs.parse(args); diff --git a/src/tools/key_value_store_collection.ts b/src/tools/key_value_store_collection.ts index c62ed1ac..e38301c6 100644 --- a/src/tools/key_value_store_collection.ts +++ b/src/tools/key_value_store_collection.ts @@ -43,7 +43,10 @@ USAGE EXAMPLES: - user_input: List my last 10 key-value stores (newest first) - user_input: List unnamed key-value stores`, inputSchema: zodToJsonSchema(getUserKeyValueStoresListArgs), - ajvValidate: ajv.compile(zodToJsonSchema(getUserKeyValueStoresListArgs)), + ajvValidate: ajv.compile({ + ...zodToJsonSchema(getUserKeyValueStoresListArgs), + additionalProperties: true, // Allow additional properties for telemetry reason field + }), call: async (toolArgs) => { const { args, apifyToken } = toolArgs; const parsed = getUserKeyValueStoresListArgs.parse(args); diff --git a/src/tools/run.ts b/src/tools/run.ts index 1261b909..b96769bf 100644 --- a/src/tools/run.ts +++ b/src/tools/run.ts @@ -37,7 +37,7 @@ USAGE EXAMPLES: - user_input: Show details of run y2h7sK3Wc - user_input: What is the datasetId for run y2h7sK3Wc?`, inputSchema: zodToJsonSchema(getActorRunArgs), - ajvValidate: ajv.compile(zodToJsonSchema(getActorRunArgs)), + ajvValidate: ajv.compile({ ...zodToJsonSchema(getActorRunArgs), additionalProperties: true }), // Allow additional properties for telemetry reason field call: async (toolArgs) => { const { args, apifyToken } = toolArgs; const parsed = getActorRunArgs.parse(args); @@ -78,7 +78,7 @@ USAGE EXAMPLES: - user_input: Show last 20 lines of logs for run y2h7sK3Wc - user_input: Get logs for run y2h7sK3Wc`, inputSchema: zodToJsonSchema(GetRunLogArgs), - ajvValidate: ajv.compile(zodToJsonSchema(GetRunLogArgs)), + ajvValidate: ajv.compile({ ...zodToJsonSchema(GetRunLogArgs), additionalProperties: true }), // Allow additional properties for telemetry reason field call: async (toolArgs) => { const { args, apifyToken } = toolArgs; const parsed = GetRunLogArgs.parse(args); @@ -110,7 +110,7 @@ USAGE EXAMPLES: - user_input: Abort run y2h7sK3Wc - user_input: Gracefully abort run y2h7sK3Wc`, inputSchema: zodToJsonSchema(abortRunArgs), - ajvValidate: ajv.compile(zodToJsonSchema(abortRunArgs)), + ajvValidate: ajv.compile({ ...zodToJsonSchema(abortRunArgs), additionalProperties: true }), // Allow additional properties for telemetry reason field call: async (toolArgs) => { const { args, apifyToken } = toolArgs; const parsed = abortRunArgs.parse(args); diff --git a/src/tools/run_collection.ts b/src/tools/run_collection.ts index 7e211f0c..1d030c95 100644 --- a/src/tools/run_collection.ts +++ b/src/tools/run_collection.ts @@ -41,7 +41,10 @@ USAGE EXAMPLES: - user_input: List my last 10 runs (newest first) - user_input: Show only SUCCEEDED runs`, inputSchema: zodToJsonSchema(getUserRunsListArgs), - ajvValidate: ajv.compile(zodToJsonSchema(getUserRunsListArgs)), + ajvValidate: ajv.compile({ + ...zodToJsonSchema(getUserRunsListArgs), + additionalProperties: true, // Allow additional properties for telemetry reason field + }), call: async (toolArgs) => { const { args, apifyToken } = toolArgs; const parsed = getUserRunsListArgs.parse(args); diff --git a/src/tools/search-apify-docs.ts b/src/tools/search-apify-docs.ts index 2520956a..37e13bbd 100644 --- a/src/tools/search-apify-docs.ts +++ b/src/tools/search-apify-docs.ts @@ -51,7 +51,10 @@ export const searchApifyDocsTool: ToolEntry = { - query: How scrape with Crawlee?`, args: searchApifyDocsToolArgsSchema, inputSchema: zodToJsonSchema(searchApifyDocsToolArgsSchema), - ajvValidate: ajv.compile(zodToJsonSchema(searchApifyDocsToolArgsSchema)), + ajvValidate: ajv.compile({ + ...zodToJsonSchema(searchApifyDocsToolArgsSchema), + additionalProperties: true, // Allow additional properties for telemetry reason field + }), call: async (toolArgs) => { const { args } = toolArgs; diff --git a/src/tools/store_collection.ts b/src/tools/store_collection.ts index bbbec628..42eddbc0 100644 --- a/src/tools/store_collection.ts +++ b/src/tools/store_collection.ts @@ -92,7 +92,10 @@ USAGE EXAMPLES: - user_input: I need to scrape instagram profiles and comments - user_input: I need to get flights and airbnb data`, inputSchema: zodToJsonSchema(searchActorsArgsSchema), - ajvValidate: ajv.compile(zodToJsonSchema(searchActorsArgsSchema)), + ajvValidate: ajv.compile({ + ...zodToJsonSchema(searchActorsArgsSchema), + additionalProperties: true, // Allow additional properties for telemetry reason field + }), call: async (toolArgs) => { const { args, apifyToken, userRentedActorIds, apifyMcpServer } = toolArgs; const parsed = searchActorsArgsSchema.parse(args); diff --git a/tests/helpers.ts b/tests/helpers.ts index 2e1b0802..739a0097 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -13,6 +13,7 @@ export interface McpClientOptions { tools?: (ToolCategory | string)[]; // Tool categories, specific tool or Actor names to include useEnv?: boolean; // Use environment variables instead of command line arguments (stdio only) clientName?: string; // Client name for identification + telemetry?: 'dev' | 'prod' | 'off'; // Telemetry configuration (default: 'off' for tests) } export async function createMcpSseClient( @@ -24,6 +25,7 @@ export async function createMcpSseClient( } const url = new URL(serverUrl); const { actors, enableAddingActors, tools } = options || {}; + const telemetry = options?.telemetry ?? 'off'; if (actors !== undefined) { url.searchParams.append('actors', actors.join(',')); } @@ -33,6 +35,11 @@ export async function createMcpSseClient( if (tools !== undefined) { url.searchParams.append('tools', tools.join(',')); } + // Only append telemetry parameter if it's 'off' to disable telemetry + // Otherwise the server will use ENVIRONMENT env variable + if (telemetry === 'off') { + url.searchParams.append('telemetry', 'off'); + } const transport = new SSEClientTransport( url, @@ -63,6 +70,7 @@ export async function createMcpStreamableClient( } const url = new URL(serverUrl); const { actors, enableAddingActors, tools } = options || {}; + const telemetry = options?.telemetry ?? 'off'; if (actors !== undefined) { url.searchParams.append('actors', actors.join(',')); } @@ -72,6 +80,11 @@ export async function createMcpStreamableClient( if (tools !== undefined) { url.searchParams.append('tools', tools.join(',')); } + // Only append telemetry parameter if it's 'off' to disable telemetry + // Otherwise the server will use ENVIRONMENT env variable + if (telemetry === 'off') { + url.searchParams.append('telemetry', 'off'); + } const transport = new StreamableHTTPClientTransport( url, @@ -100,6 +113,7 @@ export async function createMcpStdioClient( throw new Error('APIFY_TOKEN environment variable is not set.'); } const { actors, enableAddingActors, tools, useEnv } = options || {}; + const telemetry = options?.telemetry ?? 'off'; const args = ['dist/stdio.js']; const env: Record = { APIFY_TOKEN: process.env.APIFY_TOKEN as string, @@ -116,8 +130,7 @@ export async function createMcpStdioClient( if (tools !== undefined) { env.TOOLS = tools.join(','); } - // Disable telemetry for tests - env.TELEMETRY = 'off'; + env.TELEMETRY = telemetry; } else { // Use command line arguments as before if (actors !== undefined) { @@ -129,8 +142,7 @@ export async function createMcpStdioClient( if (tools !== undefined) { args.push('--tools', tools.join(',')); } - // Disable telemetry for tests - args.push('--telemetry', 'off'); + args.push('--telemetry', telemetry); } const transport = new StdioClientTransport({ diff --git a/tests/integration/actor.server-sse.test.ts b/tests/integration/actor.server-sse.test.ts index a75408d7..971d7304 100644 --- a/tests/integration/actor.server-sse.test.ts +++ b/tests/integration/actor.server-sse.test.ts @@ -15,6 +15,9 @@ let httpServerPort: number; let httpServerHost: string; let mcpUrl: string; +// Set environment to dev for telemetry tests +process.env.ENVIRONMENT = 'dev'; + createIntegrationTestsSuite({ suiteName: 'Apify MCP Server SSE', transport: 'sse', diff --git a/tests/integration/actor.server-streamable.test.ts b/tests/integration/actor.server-streamable.test.ts index c21923b3..e0f7838c 100644 --- a/tests/integration/actor.server-streamable.test.ts +++ b/tests/integration/actor.server-streamable.test.ts @@ -15,6 +15,9 @@ let httpServerPort: number; let httpServerHost: string; let mcpUrl: string; +// Set environment to dev for telemetry tests +process.env.ENVIRONMENT = 'dev'; + createIntegrationTestsSuite({ suiteName: 'Apify MCP Server Streamable HTTP', transport: 'streamable-http', diff --git a/tests/integration/suite.ts b/tests/integration/suite.ts index 22eece0f..4d545c6d 100644 --- a/tests/integration/suite.ts +++ b/tests/integration/suite.ts @@ -1062,5 +1062,62 @@ export function createIntegrationTestsSuite( await client.close(); }); + + it('should NOT include reason field in tools when telemetry is off (default)', async () => { + client = await createClientFn({ telemetry: 'off' }); + const tools = await client.listTools(); + + // Check that tools don't have reason field in input schema + for (const tool of tools.tools) { + if (tool.inputSchema && typeof tool.inputSchema === 'object' && 'properties' in tool.inputSchema) { + const props = tool.inputSchema.properties as Record; + expect(props.reason).toBeUndefined(); + } + } + + await client.close(); + }); + + it('should include reason field in tools when telemetry is enabled (dev)', async () => { + client = await createClientFn({ telemetry: 'dev' }); + const tools = await client.listTools(); + + // Check that tools have reason field with proper description when telemetry is enabled + let reasonFieldFound = false; + for (const tool of tools.tools) { + if (tool.inputSchema && typeof tool.inputSchema === 'object' && 'properties' in tool.inputSchema) { + const props = tool.inputSchema.properties as Record; + if (props.reason !== undefined) { + reasonFieldFound = true; + const reasonField = props.reason as Record; + expect(reasonField.type).toBe('string'); + } + } + } + expect(reasonFieldFound).toBe(true); + + await client.close(); + }); + + it('should include reason field in tools when telemetry is enabled (prod)', async () => { + client = await createClientFn({ telemetry: 'prod' }); + const tools = await client.listTools(); + + // Check that tools have reason field with proper description when telemetry is enabled + let reasonFieldFound = false; + for (const tool of tools.tools) { + if (tool.inputSchema && typeof tool.inputSchema === 'object' && 'properties' in tool.inputSchema) { + const props = tool.inputSchema.properties as Record; + if (props.reason !== undefined) { + reasonFieldFound = true; + const reasonField = props.reason as Record; + expect(reasonField.type).toBe('string'); + } + } + } + expect(reasonFieldFound).toBe(true); + + await client.close(); + }); }); } From 5eb7f7092a543a915f72dbd3fde8f10dcb517095 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Thu, 6 Nov 2025 21:42:22 -0800 Subject: [PATCH 03/46] fix: reorder telemetry section in the table of contents --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2cefdd30..01a7303c 100644 --- a/README.md +++ b/README.md @@ -31,10 +31,10 @@ The Apify Model Context Protocol (MCP) server at [**mcp.apify.com**](https://mcp ## Table of Contents - [🌐 Introducing the Apify MCP server](#-introducing-the-apify-mcp-server) - [πŸš€ Quickstart](#-quickstart) -- [πŸ“Š Telemetry](#telemetry) - [πŸ€– MCP clients and examples](#-mcp-clients-and-examples) - [πŸͺ„ Try Apify MCP instantly](#-try-apify-mcp-instantly) - [πŸ› οΈ Tools, resources, and prompts](#-tools-resources-and-prompts) +- [πŸ“Š Telemetry](#telemetry) - [πŸ› Troubleshooting (local MCP server)](#-troubleshooting-local-mcp-server) - [βš™οΈ Development](#-development) - [🀝 Contributing](#-contributing) From 61480bc1ed1e5dca9bb7c5f362fb758f9d107f72 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Thu, 6 Nov 2025 21:43:21 -0800 Subject: [PATCH 04/46] fix: update default userId to an empty string in trackToolCall function --- src/telemetry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/telemetry.ts b/src/telemetry.ts index 77f2e566..002836ff 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -42,7 +42,7 @@ export function trackToolCall( // TODO: Implement anonymousId tracking for device/session identification client.track({ - userId: userId || 'anonymous', + userId: userId || '', event: SEGMENT_EVENTS.TOOL_CALL, properties, }); From 1b1215d2e7f9bda0a5d4a14d228b66296d648797 Mon Sep 17 00:00:00 2001 From: MQ37 Date: Thu, 6 Nov 2025 22:01:10 -0800 Subject: [PATCH 05/46] fix typo --- src/mcp/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 14f686f1..d5bf6c7e 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -611,7 +611,7 @@ export class ActorsMcpServer { mcp_client: this.options.initializeRequestData?.params?.clientInfo?.name || '', mcp_session_id: mcpSessionId || '', connection_type: this.options.connectionType || '', - // This is the version if the apify-mcp-server package + // This is the version of the apify-mcp-server package // this can be different from the internal remote server version server_version: getPackageVersion() || '', tool_name: toolFullName, From 2f5a4a3828a7cc6ad13bb886f30215968e49446a Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Wed, 19 Nov 2025 14:41:13 +0100 Subject: [PATCH 06/46] fix: add new variable telemetry-enabled instead of telemetry. Hide dev and prod settings --- README.md | 15 ++- mds/TELEMETRY.md | 254 ++++++++++++++++++++++++++----------- src/actor/server.ts | 28 ++-- src/mcp/server.ts | 43 ++++--- src/stdio.ts | 28 ++-- src/telemetry.ts | 2 +- tests/helpers.ts | 76 +++++------ tests/integration/suite.ts | 6 +- 8 files changed, 297 insertions(+), 155 deletions(-) diff --git a/README.md b/README.md index cb64f058..6bab1fe5 100644 --- a/README.md +++ b/README.md @@ -269,22 +269,23 @@ The server does not yet provide any resources. ## πŸ“‘ Telemetry -The Apify MCP Server collects telemetry data about tool calls to help Apify understand usage patterns and improve the service. By default, telemetry is **enabled** for all tool calls. +The Apify MCP Server collects telemetry data about tool calls to help Apify understand usage patterns and improve the service. +By default, telemetry is **enabled** for all tool calls. -### Opting Out of Telemetry +### Opting out of telemetry You can disable telemetry in two ways: -**For the hosted remote server (mcp.apify.com)**: -Add the `?telemetry=off` query parameter to the URL: +**For the remote server (mcp.apify.com)**: +Add the `?telemetry-enabled=false` query parameter to the URL: ```text -https://mcp.apify.com?telemetry=off +https://mcp.apify.com?telemetry-enabled=false ``` **For the local stdio server**: -Use the `--telemetry off` CLI flag: +Use the `--telemetry-enabled=false` CLI flag: ```bash -npx @apify/actors-mcp-server --telemetry off +npx @apify/actors-mcp-server --telemetry-enabled=false ``` # βš™οΈ Development diff --git a/mds/TELEMETRY.md b/mds/TELEMETRY.md index a64173b1..42aff948 100644 --- a/mds/TELEMETRY.md +++ b/mds/TELEMETRY.md @@ -4,6 +4,49 @@ This document outlines the implementation plan for analytics tracking in the Apify MCP Server using Segment. The goal is to track all tool calls to understand user behavior, tool usage patterns, and MCP client preferences. +**Note:** This document is intended for consumers of `ActorsMcpServer` in other repositories. It describes the telemetry API and how to configure it. + +## Quick Reference + +### ActorsMcpServerOptions + +```typescript +interface ActorsMcpServerOptions { + telemetryEnabled?: boolean; // Default: true (enabled) + telemetryEnv?: 'dev' | 'prod'; // Default: 'prod' + transportType?: 'stdio' | 'http' | 'sse'; +} +``` + +### Default Behavior +- **Telemetry**: Enabled by default (`telemetryEnabled: true`) +- **Environment**: Production by default (`telemetryEnv: 'prod'`) +- **ENVIRONMENT env var**: If set to 'dev' or 'prod', automatically enables telemetry with that environment + +### Usage Examples + +```typescript +// Enable telemetry with production environment (default) +const server = new ActorsMcpServer({ + telemetryEnabled: true, // or omit (defaults to true) + telemetryEnv: 'prod', // or omit (defaults to 'prod') + transportType: 'stdio', +}); + +// Enable telemetry with development environment (for debugging) +const server = new ActorsMcpServer({ + telemetryEnabled: true, + telemetryEnv: 'dev', + transportType: 'http', +}); + +// Disable telemetry +const server = new ActorsMcpServer({ + telemetryEnabled: false, + transportType: 'sse', +}); +``` + ## Data to Be Collected ### Required Fields per Tool Call @@ -15,7 +58,7 @@ This document outlines the implementation plan for analytics tracking in the Api "properties": { "app": "mcp_server", "mcp_client": "Claude Desktop", - "connection_type": "stdio|remote", + "transport_type": "stdio|http|sse", "server_version": "VERSION", "tool_name": "apify/instagram-scraper", "reason": "REASON_FOR_TOOL_CALL" @@ -42,11 +85,12 @@ This document outlines the implementation plan for analytics tracking in the Api - Informs which MCP spec features are most important - Reference: https://modelcontextprotocol.io/clients -- **Connection Type**: How server is accessed - - `stdio`: Local/direct connection (from `src/stdio.ts` entry point) - - `remote`: Remote/SSE connection (from `src/actor/server.ts` with SSE transport) - - Passed via `ActorsMcpServerOptions.connectionType` - - Differentiates between local and remote MCP server instances +- **Transport Type**: How server is accessed + - `stdio`: Local/direct stdio connection (from `src/stdio.ts` entry point) + - `http`: Remote HTTP streamable connection (from `src/actor/server.ts` with Streamable HTTP transport) + - `sse`: Remote Server-Sent Events (SSE) connection (from `src/actor/server.ts` with SSE transport) + - Passed via `ActorsMcpServerOptions.transportType` + - Differentiates between local and remote MCP server instances and transport types - **Server Version**: Apify MCP server version - Dynamically read from package.json via `getPackageVersion()` function in `src/const.ts` @@ -63,8 +107,8 @@ This document outlines the implementation plan for analytics tracking in the Api - **Reason**: Why the tool was called (LLM-provided reasoning) - βœ… **IMPLEMENTED**: Added as optional `reason` field to all tool input schemas when telemetry is enabled - Extracted from tool call arguments: `(args as Record).reason?.toString() || ''` - - When telemetry is `'off'`: reason field is NOT added to schemas (saves schema overhead for tests) - - When telemetry is `'dev'` or `'prod'`: reason field is dynamically added to ALL tool schemas + - When telemetry is disabled (`telemetryEnabled: false`): reason field is NOT added to schemas (saves schema overhead for tests) + - When telemetry is enabled (`telemetryEnabled: true`): reason field is dynamically added to ALL tool schemas - Field description: "A brief explanation of why this tool is being called and what it will help you accomplish" - Allows LLMs to explain their tool selection and usage context - Originally proposed by @JiΕ™Γ­ Spilka, similar to mcpcat.io implementation at MCP dev summit London @@ -135,7 +179,7 @@ Return Response - Gets or initializes the client for the specified environment - Returns singleton instance - Never called directly from server code - + - `trackToolCall(userId: string, env: 'dev' | 'prod', properties: Record): void` - Sends tool call event to Segment - Lazily initializes client if needed @@ -168,21 +212,70 @@ Return Response **File**: `src/mcp/server.ts` -- **ActorsMcpServerOptions Interface**: - - Added `telemetry?: null | 'dev' | 'prod'` option - - `null` (default): No telemetry +#### ActorsMcpServerOptions Interface + +When creating an `ActorsMcpServer` instance, use the following options: + +```typescript +interface ActorsMcpServerOptions { + telemetryEnabled?: boolean; // Default: true + telemetryEnv?: 'dev' | 'prod'; // Default: 'prod' + transportType?: 'stdio' | 'http' | 'sse'; + // ... other options +} +``` + +**Usage Examples:** + +```typescript +// Enable telemetry with production environment (default) +const server = new ActorsMcpServer({ + telemetryEnabled: true, // or omit (defaults to true) + telemetryEnv: 'prod', // or omit (defaults to 'prod') + transportType: 'stdio', +}); + +// Enable telemetry with development environment (for debugging) +const server = new ActorsMcpServer({ + telemetryEnabled: true, + telemetryEnv: 'dev', + transportType: 'http', +}); + +// Disable telemetry +const server = new ActorsMcpServer({ + telemetryEnabled: false, + transportType: 'sse', +}); +``` + +- **ActorsMcpServerOptions Interface Details**: + - Added `telemetryEnabled?: boolean` option + - `true` (default): Telemetry enabled + - `false`: Telemetry disabled + - If not explicitly set, reads from `ENVIRONMENT` env variable (if set to 'dev' or 'prod', enables telemetry) + - Defaults to `true` when not set + - Added `telemetryEnv?: 'dev' | 'prod'` option + - `'prod'` (default): Use production Segment write key - `'dev'`: Use development Segment write key - - `'prod'`: Use production Segment write key - - Added `connectionType?: 'stdio' | 'remote'` option + - Only used when `telemetryEnabled` is `true` + - If not explicitly set, reads from `ENVIRONMENT` env variable + - Defaults to `'prod'` when not set + - Added `transportType?: 'stdio' | 'http' | 'sse'` option + - `'stdio'`: Direct/local stdio connection + - `'http'`: Remote HTTP streamable connection + - `'sse'`: Remote Server-Sent Events (SSE) connection - Specifies how the server is being accessed - - Passed to telemetry for connection type tracking + - Passed to telemetry for transport type tracking - Added `initializeRequestData?: InitializeRequest` option (from MCP SDK) - Contains client info like `clientInfo.name`, capabilities, etc. - Injected via message interception or HTTP request body -- **Constructor** (lines 95-102): - - If telemetry is not explicitly set, reads from `ENVIRONMENT` env variable - - Falls back to undefined if neither provided nor env var available +- **Constructor** (lines 98-115): + - If `telemetryEnabled` is not explicitly set, reads from `ENVIRONMENT` env variable + - If `ENVIRONMENT === 'dev'` or `'prod'`: sets `telemetryEnabled = true` and `telemetryEnv = envValue` + - Otherwise, defaults to `telemetryEnabled = true` and `telemetryEnv = 'prod'` + - If `telemetryEnabled` is explicitly `true`, ensures `telemetryEnv` is set (defaults to 'prod') - Supports environment-based telemetry control for hosted deployments - **Tool Call Handler** (lines 568-600, `CallToolRequestSchema`): @@ -194,10 +287,10 @@ Return Response - Builds telemetry properties object with: - `app`: 'mcp_server' (identifies this server) - `mcp_client`: Client name from `initializeRequestData.params.clientInfo.name` or 'unknown' - - `connection_type`: 'stdio', 'remote', or 'unknown' + - `transport_type`: 'stdio', 'http', 'sse', or empty string - `server_version`: From `getPackageVersion()` (package.json version) - `tool_name`: Actor full name or internal tool name - - `reason`: Empty string (TODO: extract from tool args when implemented) + - `reason`: Extracted from tool arguments if provided, otherwise empty string - Logs full telemetry payload before sending (debug level) - Calls `trackToolCall()` with userId, telemetry environment, and properties @@ -227,10 +320,12 @@ Return Response - JSON file is parsed and token is extracted from `token` key - Silently fails on file not found or parse errors (no error thrown) -- **Server Initialization** (lines 143-147): - - Passes `connectionType: 'stdio'` when creating ActorsMcpServer - - Passes telemetry option from CLI `--telemetry` flag (`prod`/`dev`/`off`) - - Converts `'off'` to null, leaves `'prod'` and `'dev'` as-is +- **Server Initialization** (lines 154-159): + - Passes `transportType: 'stdio'` when creating ActorsMcpServer + - Passes telemetry options from CLI flags: + - `--telemetry-enabled` (boolean, default: true) - documented for end users + - `--telemetry-env` ('prod'|'dev', default: 'prod') - hidden flag for debugging only + - Converts CLI flags to `telemetryEnabled` (boolean) and `telemetryEnv` ('dev'|'prod') - **Message Interception** (lines 162-176): - Creates proxy for `transport.onmessage` to intercept MCP messages @@ -282,16 +377,18 @@ The telemetry infrastructure is integrated into the remote server that hosts the - Session resumability support via `mcp-session-id` header - **`handleNewSession()` Handler**: - - Extracts `?telemetry=off` query parameter from request URL - - Converts parameter to option: `telemetryParam === 'off' ? null : undefined` + - Extracts `?telemetry-enabled` and `?telemetry-env` query parameters from request URL + - Converts parameters to options: + - `telemetryEnabled = telemetryEnabledParam !== 'false'` (defaults to true) + - `telemetryEnv = telemetryEnvParam || 'prod'` (defaults to 'prod') - Passes to ActorsMcpServer constructor: - - `connectionType: 'remote'` - identifies remote/HTTP connection - - `telemetry: telemetryOption` - per-session telemetry control + - `transportType: 'http'` - identifies remote HTTP streamable connection + - `telemetryEnabled` and `telemetryEnv` - per-session telemetry control - `initializeRequestData: req.body` - client info from HTTP request - **`handleSessionRestore()` Handler**: - Restores session from Redis state for resumable connections - - Same telemetry and connectionType handling as handleNewSession() + - Same telemetry and transportType handling as handleNewSession() - Allows clients to resume sessions with telemetry disabled **File**: `src/server/legacy-sse.ts` @@ -301,11 +398,13 @@ The telemetry infrastructure is integrated into the remote server that hosts the - Provides session resumability for older clients - **`initMCPSession()` Handler** (lines 153-210): - - Extracts `?telemetry=off` query parameter from response URL - - Converts parameter to option: `telemetryParam === 'off' ? null : undefined` + - Extracts `?telemetry-enabled` and `?telemetry-env` query parameters from response URL + - Converts parameters to options: + - `telemetryEnabled = telemetryEnabledParam !== 'false'` (defaults to true) + - `telemetryEnv = telemetryEnvParam || 'prod'` (defaults to 'prod') - Creates ActorsMcpServer with: - - `connectionType: 'remote'` - identifies remote/SSE connection - - `telemetry: telemetryOption` - per-session telemetry control + - `transportType: 'sse'` - identifies remote SSE connection + - `telemetryEnabled` and `telemetryEnv` - per-session telemetry control - Message interception proxy (lines 203-210): - Proxies `transport.onmessage` to capture MCP initialize message - Extracts client information from initialize request data @@ -313,19 +412,21 @@ The telemetry infrastructure is integrated into the remote server that hosts the - Calls original onmessage handler to continue processing - **Per-Session Control**: - - Both transports support `?telemetry=off` query parameter + - Both transports support `?telemetry-enabled=false` query parameter to disable telemetry + - Optional `?telemetry-env=dev` parameter to use development workspace (for debugging only) - Prevents test data from polluting production telemetry - - Example: `https://mcp.apify.com/?telemetry=off` for streamable - - Example: `https://mcp.apify.com/sse?telemetry=off` for SSE + - Example: `https://mcp.apify.com/?telemetry-enabled=false` for streamable + - Example: `https://mcp.apify.com/sse?telemetry-enabled=false` for SSE + - Example: `https://mcp.apify.com/?telemetry-env=dev` for debugging (uses dev workspace) **Test Configuration**: - **`test/integration/tests/server-streamable.test.ts`**: - - mcpUrl configured with `/?telemetry=off` query parameter + - mcpUrl configured with `/?telemetry-enabled=false` query parameter - Prevents integration tests from sending telemetry events - **`test/integration/tests/server-sse.test.ts`**: - - mcpUrl configured with `/sse?telemetry=off` query parameter + - mcpUrl configured with `/sse?telemetry-enabled=false` query parameter - Prevents integration tests from sending telemetry events ## Data Flow @@ -341,8 +442,9 @@ From `CallToolRequestSchema` handler in `src/mcp/server.ts` (line 568+): - `meta`: Metadata including progressToken From `ActorsMcpServer` instance: -- `this.options.telemetry`: Telemetry environment configuration ('dev', 'prod', or null) -- `this.options.connectionType`: Connection type ('stdio' or 'remote') +- `this.options.telemetryEnabled`: Boolean indicating if telemetry is enabled (default: true) +- `this.options.telemetryEnv`: Telemetry environment ('dev' or 'prod', default: 'prod') +- `this.options.transportType`: Transport type ('stdio', 'http', or 'sse') - `this.options.initializeRequestData`: MCP client info and capabilities - `params.clientInfo.name`: MCP client name (e.g., 'Claude Desktop', 'Cline') - `params.capabilities`: Client capabilities @@ -378,27 +480,29 @@ Token available? Track telemetry with userId ``` -### Connection Type Detection +### Transport Type Detection -Connection type is now passed via `ActorsMcpServerOptions`: +Transport type is now passed via `ActorsMcpServerOptions`: - **Stdio** (Local): When using `src/stdio.ts` entry point - - Passes `connectionType: 'stdio'` when creating ActorsMcpServer + - Passes `transportType: 'stdio'` when creating ActorsMcpServer + - Passes `telemetryEnabled` and `telemetryEnv` from CLI flags - Message interception proxy captures initialize request data from MCP protocol - - Example: `npx @apify/actors-mcp-server --telemetry=dev` + - Example: `npx @apify/actors-mcp-server --telemetry-enabled=false` + - Example: `npx @apify/actors-mcp-server --telemetry-env=dev` (for debugging) -- **Streamable HTTP** (Remote): When using `src/server/streamable.ts` with Streamable HTTP transport - - Passes `connectionType: 'remote'` in both `handleSessionRestore()` and `handleNewSession()` handlers - - Extracts `?telemetry=off` query parameter to disable telemetry per-session +- **Streamable HTTP** (Remote): When using `src/actor/server.ts` with Streamable HTTP transport + - Passes `transportType: 'http'` when creating ActorsMcpServer + - Extracts `?telemetry-enabled` and `?telemetry-env` query parameters from URL - Client info available via `req.body` (InitializeRequest passed as initializeRequestData) - - Connection: `https://mcp.apify.com/?telemetry=off` + - Connection: `https://mcp.apify.com/?telemetry-enabled=false` + - Connection: `https://mcp.apify.com/?telemetry-env=dev` (for debugging) -- **Legacy SSE** (Remote): When using `src/server/legacy-sse.ts` with SSE transport - - Passes `connectionType: 'remote'` in `initMCPSession()` handler - - Extracts `?telemetry=off` query parameter from URL +- **Legacy SSE** (Remote): When using `src/actor/server.ts` with SSE transport + - Passes `transportType: 'sse'` when creating ActorsMcpServer + - Extracts `?telemetry-enabled` and `?telemetry-env` query parameters from URL - Message interception proxy captures initialize request data from MCP JSON-RPC messages - - Connection: `https://mcp.apify.com/sse?telemetry=off` - -Future enhancement: Automatically detect connection type from transport type when not explicitly provided. + - Connection: `https://mcp.apify.com/sse?telemetry-enabled=false` + - Connection: `https://mcp.apify.com/sse?telemetry-env=dev` (for debugging) ## Implementation Notes @@ -408,25 +512,24 @@ Future enhancement: Automatically detect connection type from transport type whe - Telemetry module with singleton Segment clients per environment - Tool call tracking in `CallToolRequestSchema` handler at line 568 of `src/mcp/server.ts` - Dynamic version reading from package.json via `getPackageVersion()` function in `src/utils/version.ts` -- Connection type option in ActorsMcpServerOptions interface -- Stdio transport passing `connectionType: 'stdio'` and telemetry CLI flag in `src/stdio.ts` +- Transport type option in ActorsMcpServerOptions interface +- Stdio transport passing `transportType: 'stdio'` and telemetry CLI flags in `src/stdio.ts` - package.json included in npm build files - User cache module with token hashing and in-memory caching in `src/utils/user-cache.ts` - Token resolution from env var and ~/.apify/auth.json file in `src/stdio.ts` - Debug logging for telemetry operations in tool call handler - Full User object caching (not custom wrapper interface) - Message interception proxy to capture initialize request data in stdio (`src/stdio.ts`) -- Streamable HTTP transport with telemetry query parameter support (`src/server/streamable.ts`) - - `handleNewSession()`: Extracts `?telemetry=off` query param and passes to ActorsMcpServer - - `handleSessionRestore()`: Same telemetry and connectionType handling - - Both handlers pass `connectionType: 'remote'` -- Legacy SSE transport with telemetry query parameter support (`src/server/legacy-sse.ts`) - - `initMCPSession()`: Extracts `?telemetry=off` query param +- Streamable HTTP transport with telemetry query parameter support (`src/actor/server.ts`) + - Extracts `?telemetry-enabled` and `?telemetry-env` query params + - Passes `transportType: 'http'` and telemetry options to ActorsMcpServer +- Legacy SSE transport with telemetry query parameter support (`src/actor/server.ts`) + - Extracts `?telemetry-enabled` and `?telemetry-env` query params - Message interception proxy to capture initialize request data from MCP protocol - - Passes `connectionType: 'remote'` and telemetry option + - Passes `transportType: 'sse'` and telemetry options - Test configuration to prevent telemetry pollution - - `test/integration/tests/server-streamable.test.ts`: Uses `/?telemetry=off` - - `test/integration/tests/server-sse.test.ts`: Uses `/sse?telemetry=off` + - `test/integration/tests/server-streamable.test.ts`: Uses `/?telemetry-enabled=false` + - `test/integration/tests/server-sse.test.ts`: Uses `/sse?telemetry-enabled=false` - MCP session ID tracking and injection - **Stdio transport** (`src/stdio.ts`): Manually generates UUID4 session ID using `randomUUID()` from `node:crypto` module - Generated at startup (line 160 in stdio.ts) @@ -437,10 +540,10 @@ Future enhancement: Automatically detect connection type from transport type whe - Session ID injected into tool call request params for telemetry tracking - Supports cross-instance session correlation in distributed deployments - **Reason field implementation** βœ… - - Dynamically added to all tool input schemas when telemetry is enabled (`'dev'` or `'prod'`) + - Dynamically added to all tool input schemas when telemetry is enabled (`telemetryEnabled: true`) - Optional string field with title "Reason" and guidance description - Extracted from tool arguments during tool call: `(args as Record).reason?.toString() || ''` - - Not added when telemetry is `'off'` (reduces schema overhead for tests) + - Not added when telemetry is disabled (`telemetryEnabled: false`) (reduces schema overhead for tests) - All AJV validators updated with `additionalProperties: true` to accept the field - New tests verify: reason field presence/absence based on telemetry setting - Works with `upsertTools()` conditional modification logic alongside Skyfire mode @@ -456,7 +559,7 @@ Future enhancement: Automatically detect connection type from transport type whe - Telemetry clients are shared via singleton Map pattern per environment - User cache is global (shared across all server instances) - Session data is stored in Redis for cross-instance resumability -- Per-session telemetry control via `?telemetry=off` query parameter prevents test pollution +- Per-session telemetry control via `?telemetry-enabled=false` query parameter prevents test pollution ### User Authentication - Apify token extracted from `APIFY_TOKEN` env var first @@ -488,7 +591,7 @@ All telemetry operations emit debug logs including: - Full telemetry payload before sending: - app ('mcp_server') - mcp_client (client name from initialize data or 'unknown') - - connection_type ('stdio', 'remote', or 'unknown') + - transport_type ('stdio', 'http', 'sse', or empty string) - server_version (from package.json or 'unknown') - tool_name (actor full name or internal tool name) - reason (empty string) @@ -498,7 +601,7 @@ Enable with `DEBUG=*` or `LOG_LEVEL=debug` to see telemetry details. Example debug output: ``` Telemetry: fetched user info { userId: 'user-123', userFound: true } -Telemetry: tracking tool call { app: 'mcp_server', mcp_client: 'Claude Desktop', connection_type: 'stdio', server_version: '0.5.3', tool_name: 'apify/instagram-scraper', reason: '' } +Telemetry: tracking tool call { app: 'mcp_server', mcp_client: 'Claude Desktop', transport_type: 'stdio', server_version: '0.5.3', tool_name: 'apify/instagram-scraper', reason: '' } ``` ### Future Enhancements @@ -550,15 +653,20 @@ Telemetry: tracking tool call { app: 'mcp_server', mcp_client: 'Claude Desktop', ## Testing & Validation 1. **Dev Environment** - - Initialize server with `telemetry: 'dev'` + - Initialize server with `telemetryEnabled: true, telemetryEnv: 'dev'` + - Or use CLI: `--telemetry-env=dev` (hidden flag for debugging) + - Or use URL: `?telemetry-env=dev` - Verify events appear in Segment dev workspace 2. **Production** - - Initialize server with `telemetry: 'prod'` + - Initialize server with `telemetryEnabled: true, telemetryEnv: 'prod'` (default) + - Or use CLI: `--telemetry-enabled` (default: true) - Monitor Segment prod workspace for events 3. **No Telemetry** - - Initialize server without telemetry option + - Initialize server with `telemetryEnabled: false` + - Or use CLI: `--telemetry-enabled=false` + - Or use URL: `?telemetry-enabled=false` - Verify no tracking occurs - Verify no errors from missing telemetry diff --git a/src/actor/server.ts b/src/actor/server.ts index e6bfa100..4af16d59 100644 --- a/src/actor/server.ts +++ b/src/actor/server.ts @@ -78,12 +78,19 @@ export function createExpressApp( rt: Routes.SSE, tr: TransportType.SSE, }); - // Extract telemetry query parameter - if 'off' disable, otherwise use ENVIRONMENT env variable + // Extract telemetry query parameters const urlParams = new URL(req.url, `http://${req.headers.host}`).searchParams; - const telemetryParam = urlParams.get('telemetry'); - const telemetry = telemetryParam === 'off' ? null : undefined; + const telemetryEnabledParam = urlParams.get('telemetry-enabled'); + const telemetryEnvParam = urlParams.get('telemetry-env'); + const telemetryEnabled = telemetryEnabledParam !== 'false'; // Default to true + const telemetryEnv: 'dev' | 'prod' = (telemetryEnvParam === 'dev' ? 'dev' : 'prod'); // Default to 'prod' - const mcpServer = new ActorsMcpServer({ setupSigintHandler: false, connectionType: 'remote', telemetry }); + const mcpServer = new ActorsMcpServer({ + setupSigintHandler: false, + transportType: 'sse', + telemetryEnabled, + telemetryEnv, + }); const transport = new SSEServerTransport(Routes.MESSAGE, res); // Load MCP server tools @@ -167,16 +174,19 @@ export function createExpressApp( sessionIdGenerator: () => randomUUID(), enableJsonResponse: false, // Use SSE response mode }); - // Extract telemetry query parameter - if 'off' disable, otherwise use ENVIRONMENT env variable + // Extract telemetry query parameters const urlParams = new URL(req.url, `http://${req.headers.host}`).searchParams; - const telemetryParam = urlParams.get('telemetry'); - const telemetry = telemetryParam === 'off' ? null : undefined; + const telemetryEnabledParam = urlParams.get('telemetry-enabled'); + const telemetryEnvParam = urlParams.get('telemetry-env'); + const telemetryEnabled = telemetryEnabledParam !== 'false'; // Default to true + const telemetryEnv: 'dev' | 'prod' = (telemetryEnvParam === 'dev' ? 'dev' : 'prod'); // Default to 'prod' const mcpServer = new ActorsMcpServer({ setupSigintHandler: false, initializeRequestData: req.body as InitializeRequest, - connectionType: 'remote', - telemetry, + transportType: 'http', + telemetryEnabled, + telemetryEnv, }); // Load MCP server tools diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 537dac04..02e78ab3 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -60,18 +60,23 @@ interface ActorsMcpServerOptions { skyfireMode?: boolean; initializeRequestData?: InitializeRequest; /** - * Enable telemetry tracking for tool calls. - * - null: No telemetry (default) + * Enable or disable telemetry tracking for tool calls. + * Defaults to true when not set, or can be set from ENVIRONMENT env variable. + */ + telemetryEnabled?: boolean; + /** + * Telemetry environment when telemetry is enabled. * - 'dev': Use development Segment write key - * - 'prod': Use production Segment write key + * - 'prod': Use production Segment write key (default) */ - telemetry?: null | 'dev' | 'prod'; + telemetryEnv?: 'dev' | 'prod'; /** - * Connection type for telemetry tracking. - * - 'stdio': Direct/local connection - * - 'remote': Remote/HTTP streamble or SSE connection + * Transport type for telemetry tracking. + * - 'stdio': Direct/local stdio connection + * - 'http': Remote HTTP streamable connection + * - 'sse': Remote Server-Sent Events (SSE) connection */ - connectionType?: 'stdio' | 'remote'; + transportType?: 'stdio' | 'http' | 'sse'; /** * Apify API token for authentication * Primarily used by stdio transport when token is read from ~/.apify/auth.json file @@ -94,12 +99,20 @@ export class ActorsMcpServer { constructor(options: ActorsMcpServerOptions = {}) { this.options = options; - // If telemetry is not explicitly set, try to read from ENVIRONMENT env variable, this is used in the mcp.apify.com deployment - if (this.options.telemetry === undefined) { + // If telemetryEnabled is not explicitly set, try to read from ENVIRONMENT env variable, this is used in the mcp.apify.com deployment + if (this.options.telemetryEnabled === undefined) { const envValue = process.env.ENVIRONMENT; if (envValue === 'dev' || envValue === 'prod') { - this.options.telemetry = envValue; + this.options.telemetryEnabled = true; + this.options.telemetryEnv = envValue; + } else { + // Default to enabled with prod environment + this.options.telemetryEnabled = true; + this.options.telemetryEnv = this.options.telemetryEnv || 'prod'; } + } else if (this.options.telemetryEnabled) { + // If telemetry is enabled, ensure telemetryEnv is set (default to 'prod') + this.options.telemetryEnv = this.options.telemetryEnv || 'prod'; } const { setupSigintHandler = true } = options; @@ -294,7 +307,7 @@ export class ActorsMcpServer { * @returns Array of added/updated tool wrappers */ public upsertTools(tools: ToolEntry[], shouldNotifyToolsChangedHandler = false) { - const isTelemetryEnabled = this.options.telemetry === 'dev' || this.options.telemetry === 'prod'; + const isTelemetryEnabled = this.options.telemetryEnabled === true; if (this.options.skyfireMode || isTelemetryEnabled) { for (const wrap of tools) { @@ -599,7 +612,7 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool } // Track telemetry if enabled - if (this.options.telemetry && (this.options.telemetry === 'dev' || this.options.telemetry === 'prod')) { + if (this.options.telemetryEnabled === true) { const toolFullName = tool.type === 'actor' ? tool.actorFullName : tool.name; // Get userId from cache or fetch from API @@ -621,7 +634,7 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool app: 'mcp_server', mcp_client: this.options.initializeRequestData?.params?.clientInfo?.name || '', mcp_session_id: mcpSessionId || '', - connection_type: this.options.connectionType || '', + transport_type: this.options.transportType || '', // This is the version of the apify-mcp-server package // this can be different from the internal remote server version server_version: getPackageVersion() || '', @@ -630,7 +643,7 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool }; log.debug('Telemetry: tracking tool call', telemetryData); - trackToolCall(userId, this.options.telemetry, telemetryData); + trackToolCall(userId, this.options.telemetryEnv || 'prod', telemetryData); } try { diff --git a/src/stdio.ts b/src/stdio.ts index 4a5511b9..02467800 100644 --- a/src/stdio.ts +++ b/src/stdio.ts @@ -47,8 +47,10 @@ interface CliArgs { enableActorAutoLoading: boolean; /** Tool categories to include */ tools?: string; - /** Telemetry environment: 'prod', 'dev', or 'off' */ - telemetry: 'prod' | 'dev' | 'off'; + /** Enable or disable telemetry tracking (default: true) */ + telemetryEnabled: boolean; + /** Telemetry environment: 'prod' or 'dev' (default: 'prod', only used when telemetry-enabled is true) */ + telemetryEnv: 'prod' | 'dev'; } /** @@ -98,14 +100,21 @@ Deprecated: use tools experimental category instead.`, For more details visit https://mcp.apify.com`, example: 'actors,docs,apify/rag-web-browser', }) - .option('telemetry', { + .option('telemetry-enabled', { + type: 'boolean', + default: true, + describe: `Enable or disable telemetry tracking for tool calls. Can also be set via TELEMETRY_ENABLED environment variable. +Default: true (enabled)`, + }) + .option('telemetry-env', { type: 'string', - choices: ['prod', 'dev', 'off'], + choices: ['prod', 'dev'], default: 'prod', - describe: `Enable telemetry tracking for tool calls. Can also be set via TELEMETRY environment variable. + hidden: true, + describe: `Telemetry environment when telemetry is enabled. Can also be set via TELEMETRY_ENV environment variable. - 'prod': Send events to production Segment workspace (default) - 'dev': Send events to development Segment workspace -- 'off': Disable telemetry`, +Only used when --telemetry-enabled is true`, }) .help('help') .alias('h', 'help') @@ -137,14 +146,15 @@ const apifyToken = process.env.APIFY_TOKEN || getTokenFromAuthFile(); // Validate environment if (!apifyToken) { - log.error('APIFY_TOKEN is required but not set in the environment variables or ~/.apify/auth.json'); + log.error('APIFY_TOKEN is required but not set in the environment variables or in ~/.apify/auth.json'); process.exit(1); } async function main() { const mcpServer = new ActorsMcpServer({ - connectionType: 'stdio', - telemetry: argv.telemetry === 'off' ? null : (argv.telemetry as 'dev' | 'prod'), + transportType: 'stdio', + telemetryEnabled: argv.telemetryEnabled, + telemetryEnv: argv.telemetryEnv || 'prod', token: apifyToken, }); diff --git a/src/telemetry.ts b/src/telemetry.ts index 002836ff..f88fb6af 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -4,7 +4,7 @@ const DEV_WRITE_KEY = '9rPHlMtxX8FJhilGEwkfUoZ0uzWxnzcT'; const PROD_WRITE_KEY = 'cOkp5EIJaN69gYaN8bcp7KtaD0fGABwJ'; const SEGMENT_EVENTS = { - TOOL_CALL: 'MCP Tool Call', + TOOL_CALL: 'MCP tool call', }; // Map to store singleton Segment Analytics clients per environment diff --git a/tests/helpers.ts b/tests/helpers.ts index 739a0097..23b95684 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -13,19 +13,18 @@ export interface McpClientOptions { tools?: (ToolCategory | string)[]; // Tool categories, specific tool or Actor names to include useEnv?: boolean; // Use environment variables instead of command line arguments (stdio only) clientName?: string; // Client name for identification - telemetry?: 'dev' | 'prod' | 'off'; // Telemetry configuration (default: 'off' for tests) + telemetryEnabled?: boolean; // Enable or disable telemetry (default: false for tests) + telemetryEnv?: 'dev' | 'prod'; // Telemetry environment (default: 'prod', only used when telemetryEnabled is true) } -export async function createMcpSseClient( - serverUrl: string, - options?: McpClientOptions, -): Promise { +function checkApifyToken(): void { if (!process.env.APIFY_TOKEN) { throw new Error('APIFY_TOKEN environment variable is not set.'); } - const url = new URL(serverUrl); - const { actors, enableAddingActors, tools } = options || {}; - const telemetry = options?.telemetry ?? 'off'; +} + +function appendSearchParams(url: URL, options?: McpClientOptions): void { + const { actors, enableAddingActors, tools, telemetryEnabled, telemetryEnv } = options || {}; if (actors !== undefined) { url.searchParams.append('actors', actors.join(',')); } @@ -35,11 +34,22 @@ export async function createMcpSseClient( if (tools !== undefined) { url.searchParams.append('tools', tools.join(',')); } - // Only append telemetry parameter if it's 'off' to disable telemetry - // Otherwise the server will use ENVIRONMENT env variable - if (telemetry === 'off') { - url.searchParams.append('telemetry', 'off'); + // Append telemetry parameters + if (telemetryEnabled === false) { + url.searchParams.append('telemetry-enabled', 'false'); } + if (telemetryEnv !== undefined && telemetryEnabled !== false) { + url.searchParams.append('telemetry-env', telemetryEnv); + } +} + +export async function createMcpSseClient( + serverUrl: string, + options?: McpClientOptions, +): Promise { + checkApifyToken(); + const url = new URL(serverUrl); + appendSearchParams(url, options); const transport = new SSEClientTransport( url, @@ -65,26 +75,9 @@ export async function createMcpStreamableClient( serverUrl: string, options?: McpClientOptions, ): Promise { - if (!process.env.APIFY_TOKEN) { - throw new Error('APIFY_TOKEN environment variable is not set.'); - } + checkApifyToken(); const url = new URL(serverUrl); - const { actors, enableAddingActors, tools } = options || {}; - const telemetry = options?.telemetry ?? 'off'; - if (actors !== undefined) { - url.searchParams.append('actors', actors.join(',')); - } - if (enableAddingActors !== undefined) { - url.searchParams.append('enableAddingActors', enableAddingActors.toString()); - } - if (tools !== undefined) { - url.searchParams.append('tools', tools.join(',')); - } - // Only append telemetry parameter if it's 'off' to disable telemetry - // Otherwise the server will use ENVIRONMENT env variable - if (telemetry === 'off') { - url.searchParams.append('telemetry', 'off'); - } + appendSearchParams(url, options); const transport = new StreamableHTTPClientTransport( url, @@ -109,11 +102,8 @@ export async function createMcpStreamableClient( export async function createMcpStdioClient( options?: McpClientOptions, ): Promise { - if (!process.env.APIFY_TOKEN) { - throw new Error('APIFY_TOKEN environment variable is not set.'); - } - const { actors, enableAddingActors, tools, useEnv } = options || {}; - const telemetry = options?.telemetry ?? 'off'; + checkApifyToken(); + const { actors, enableAddingActors, tools, useEnv, telemetryEnabled, telemetryEnv } = options || {}; const args = ['dist/stdio.js']; const env: Record = { APIFY_TOKEN: process.env.APIFY_TOKEN as string, @@ -130,7 +120,12 @@ export async function createMcpStdioClient( if (tools !== undefined) { env.TOOLS = tools.join(','); } - env.TELEMETRY = telemetry; + if (telemetryEnabled !== undefined) { + env.TELEMETRY_ENABLED = telemetryEnabled.toString(); + } + if (telemetryEnv !== undefined) { + env.TELEMETRY_ENV = telemetryEnv; + } } else { // Use command line arguments as before if (actors !== undefined) { @@ -142,7 +137,12 @@ export async function createMcpStdioClient( if (tools !== undefined) { args.push('--tools', tools.join(',')); } - args.push('--telemetry', telemetry); + if (telemetryEnabled === false) { + args.push('--telemetry-enabled', 'false'); + } + if (telemetryEnv !== undefined && telemetryEnabled !== false) { + args.push('--telemetry-env', telemetryEnv); + } } const transport = new StdioClientTransport({ diff --git a/tests/integration/suite.ts b/tests/integration/suite.ts index 58d7b514..4aef389a 100644 --- a/tests/integration/suite.ts +++ b/tests/integration/suite.ts @@ -1065,7 +1065,7 @@ export function createIntegrationTestsSuite( }); it('should NOT include reason field in tools when telemetry is off (default)', async () => { - client = await createClientFn({ telemetry: 'off' }); + client = await createClientFn({ telemetryEnabled: false }); const tools = await client.listTools(); // Check that tools don't have reason field in input schema @@ -1080,7 +1080,7 @@ export function createIntegrationTestsSuite( }); it('should include reason field in tools when telemetry is enabled (dev)', async () => { - client = await createClientFn({ telemetry: 'dev' }); + client = await createClientFn({ telemetryEnabled: true, telemetryEnv: 'dev' }); const tools = await client.listTools(); // Check that tools have reason field with proper description when telemetry is enabled @@ -1101,7 +1101,7 @@ export function createIntegrationTestsSuite( }); it('should include reason field in tools when telemetry is enabled (prod)', async () => { - client = await createClientFn({ telemetry: 'prod' }); + client = await createClientFn({ telemetryEnabled: true, telemetryEnv: 'prod' }); const tools = await client.listTools(); // Check that tools have reason field with proper description when telemetry is enabled From 92e80240e9f951d36d36bf4e476f43421f62ac79 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Wed, 19 Nov 2025 15:04:01 +0100 Subject: [PATCH 07/46] fix: add constants instead of prod and dev --- src/actor/server.ts | 6 ++++-- src/const.ts | 10 ++++++++++ src/mcp/server.ts | 16 +++++++++------- src/stdio.ts | 10 ++++++---- src/telemetry.ts | 17 +++++++++++++---- tests/helpers.ts | 4 ++-- 6 files changed, 44 insertions(+), 19 deletions(-) diff --git a/src/actor/server.ts b/src/actor/server.ts index 4af16d59..89478bad 100644 --- a/src/actor/server.ts +++ b/src/actor/server.ts @@ -13,7 +13,9 @@ import express from 'express'; import log from '@apify/log'; import { ApifyClient } from '../apify-client.js'; +import { type TelemetryEnv } from '../const.js'; import { ActorsMcpServer } from '../mcp/server.js'; +import { getTelemetryEnv } from '../telemetry.js'; import { getHelpMessage, HEADER_READINESS_PROBE, Routes, TransportType } from './const.js'; import { getActorRunData } from './utils.js'; @@ -83,7 +85,7 @@ export function createExpressApp( const telemetryEnabledParam = urlParams.get('telemetry-enabled'); const telemetryEnvParam = urlParams.get('telemetry-env'); const telemetryEnabled = telemetryEnabledParam !== 'false'; // Default to true - const telemetryEnv: 'dev' | 'prod' = (telemetryEnvParam === 'dev' ? 'dev' : 'prod'); // Default to 'prod' + const telemetryEnv: TelemetryEnv = getTelemetryEnv(telemetryEnvParam); const mcpServer = new ActorsMcpServer({ setupSigintHandler: false, @@ -179,7 +181,7 @@ export function createExpressApp( const telemetryEnabledParam = urlParams.get('telemetry-enabled'); const telemetryEnvParam = urlParams.get('telemetry-env'); const telemetryEnabled = telemetryEnabledParam !== 'false'; // Default to true - const telemetryEnv: 'dev' | 'prod' = (telemetryEnvParam === 'dev' ? 'dev' : 'prod'); // Default to 'prod' + const telemetryEnv: TelemetryEnv = getTelemetryEnv(telemetryEnvParam); const mcpServer = new ActorsMcpServer({ setupSigintHandler: false, diff --git a/src/const.ts b/src/const.ts index 346c274a..2d81515e 100644 --- a/src/const.ts +++ b/src/const.ts @@ -104,3 +104,13 @@ export const ALGOLIA = { export const PROGRESS_NOTIFICATION_INTERVAL_MS = 5_000; // 5 seconds export const APIFY_STORE_URL = 'https://apify.com'; + +// Telemetry +export type TelemetryEnv = 'dev' | 'prod'; + +export const TELEMETRY_ENV = { + DEV: 'dev', + PROD: 'prod', +} as const; + +export const DEFAULT_TELEMETRY_ENV: TelemetryEnv = TELEMETRY_ENV.PROD; diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 02e78ab3..03f7227c 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -34,8 +34,9 @@ import { SKYFIRE_README_CONTENT, SKYFIRE_TOOL_INSTRUCTIONS, } from '../const.js'; +import { TELEMETRY_ENV, type TelemetryEnv } from '../const.js'; import { prompts } from '../prompts/index.js'; -import { trackToolCall } from '../telemetry.js'; +import { getTelemetryEnv, trackToolCall } from '../telemetry.js'; import { callActorGetDataset, defaultTools, getActorsAsTools, toolCategories } from '../tools/index.js'; import { decodeDotPropertyNames } from '../tools/utils.js'; import type { ToolEntry } from '../types.js'; @@ -69,7 +70,7 @@ interface ActorsMcpServerOptions { * - 'dev': Use development Segment write key * - 'prod': Use production Segment write key (default) */ - telemetryEnv?: 'dev' | 'prod'; + telemetryEnv?: TelemetryEnv; /** * Transport type for telemetry tracking. * - 'stdio': Direct/local stdio connection @@ -102,17 +103,18 @@ export class ActorsMcpServer { // If telemetryEnabled is not explicitly set, try to read from ENVIRONMENT env variable, this is used in the mcp.apify.com deployment if (this.options.telemetryEnabled === undefined) { const envValue = process.env.ENVIRONMENT; - if (envValue === 'dev' || envValue === 'prod') { + const validEnv = getTelemetryEnv(envValue); + if (envValue === TELEMETRY_ENV.DEV || envValue === TELEMETRY_ENV.PROD) { this.options.telemetryEnabled = true; - this.options.telemetryEnv = envValue; + this.options.telemetryEnv = validEnv; } else { // Default to enabled with prod environment this.options.telemetryEnabled = true; - this.options.telemetryEnv = this.options.telemetryEnv || 'prod'; + this.options.telemetryEnv = getTelemetryEnv(this.options.telemetryEnv); } } else if (this.options.telemetryEnabled) { // If telemetry is enabled, ensure telemetryEnv is set (default to 'prod') - this.options.telemetryEnv = this.options.telemetryEnv || 'prod'; + this.options.telemetryEnv = getTelemetryEnv(this.options.telemetryEnv); } const { setupSigintHandler = true } = options; @@ -643,7 +645,7 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool }; log.debug('Telemetry: tracking tool call', telemetryData); - trackToolCall(userId, this.options.telemetryEnv || 'prod', telemetryData); + trackToolCall(userId, getTelemetryEnv(this.options.telemetryEnv), telemetryData); } try { diff --git a/src/stdio.ts b/src/stdio.ts index 02467800..3f12f0f8 100644 --- a/src/stdio.ts +++ b/src/stdio.ts @@ -29,8 +29,10 @@ import { hideBin } from 'yargs/helpers'; import log from '@apify/log'; import { ApifyClient } from './apify-client.js'; +import { DEFAULT_TELEMETRY_ENV, TELEMETRY_ENV, type TelemetryEnv } from './const.js'; import { processInput } from './input.js'; import { ActorsMcpServer } from './mcp/server.js'; +import { getTelemetryEnv } from './telemetry.js'; import type { Input, ToolSelector } from './types.js'; import { parseCommaSeparatedList } from './utils/generic.js'; import { loadToolsFromInput } from './utils/tools-loader.js'; @@ -50,7 +52,7 @@ interface CliArgs { /** Enable or disable telemetry tracking (default: true) */ telemetryEnabled: boolean; /** Telemetry environment: 'prod' or 'dev' (default: 'prod', only used when telemetry-enabled is true) */ - telemetryEnv: 'prod' | 'dev'; + telemetryEnv: TelemetryEnv; } /** @@ -108,8 +110,8 @@ Default: true (enabled)`, }) .option('telemetry-env', { type: 'string', - choices: ['prod', 'dev'], - default: 'prod', + choices: [TELEMETRY_ENV.PROD, TELEMETRY_ENV.DEV], + default: DEFAULT_TELEMETRY_ENV, hidden: true, describe: `Telemetry environment when telemetry is enabled. Can also be set via TELEMETRY_ENV environment variable. - 'prod': Send events to production Segment workspace (default) @@ -154,7 +156,7 @@ async function main() { const mcpServer = new ActorsMcpServer({ transportType: 'stdio', telemetryEnabled: argv.telemetryEnabled, - telemetryEnv: argv.telemetryEnv || 'prod', + telemetryEnv: getTelemetryEnv(argv.telemetryEnv), token: apifyToken, }); diff --git a/src/telemetry.ts b/src/telemetry.ts index f88fb6af..ddf39942 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -1,5 +1,7 @@ import { Analytics } from '@segment/analytics-node'; +import { DEFAULT_TELEMETRY_ENV, TELEMETRY_ENV, type TelemetryEnv } from './const.js'; + const DEV_WRITE_KEY = '9rPHlMtxX8FJhilGEwkfUoZ0uzWxnzcT'; const PROD_WRITE_KEY = 'cOkp5EIJaN69gYaN8bcp7KtaD0fGABwJ'; @@ -7,8 +9,15 @@ const SEGMENT_EVENTS = { TOOL_CALL: 'MCP tool call', }; +/** + * Gets the telemetry environment, defaulting to 'prod' if not provided or invalid + */ +export function getTelemetryEnv(env?: string | null): TelemetryEnv { + return (env === TELEMETRY_ENV.DEV || env === TELEMETRY_ENV.PROD) ? env : DEFAULT_TELEMETRY_ENV; +} + // Map to store singleton Segment Analytics clients per environment -const analyticsClients = new Map<'dev' | 'prod', Analytics>(); +const analyticsClients = new Map(); /** * Gets or initializes a Segment Analytics client for the specified environment. @@ -18,9 +27,9 @@ const analyticsClients = new Map<'dev' | 'prod', Analytics>(); * @param env - 'dev' for development, 'prod' for production * @returns Analytics client instance */ -export function getOrInitAnalyticsClient(env: 'dev' | 'prod'): Analytics { +export function getOrInitAnalyticsClient(env: TelemetryEnv): Analytics { if (!analyticsClients.has(env)) { - const writeKey = env === 'prod' ? PROD_WRITE_KEY : DEV_WRITE_KEY; + const writeKey = env === TELEMETRY_ENV.PROD ? PROD_WRITE_KEY : DEV_WRITE_KEY; analyticsClients.set(env, new Analytics({ writeKey })); } return analyticsClients.get(env)!; @@ -35,7 +44,7 @@ export function getOrInitAnalyticsClient(env: 'dev' | 'prod'): Analytics { */ export function trackToolCall( userId: string, - env: 'dev' | 'prod', + env: TelemetryEnv, properties: Record, ): void { const client = getOrInitAnalyticsClient(env); diff --git a/tests/helpers.ts b/tests/helpers.ts index 23b95684..50a4681d 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -4,7 +4,7 @@ import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js' import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; import { expect } from 'vitest'; -import { HelperTools } from '../src/const.js'; +import { HelperTools, type TelemetryEnv } from '../src/const.js'; import type { ToolCategory } from '../src/types.js'; export interface McpClientOptions { @@ -14,7 +14,7 @@ export interface McpClientOptions { useEnv?: boolean; // Use environment variables instead of command line arguments (stdio only) clientName?: string; // Client name for identification telemetryEnabled?: boolean; // Enable or disable telemetry (default: false for tests) - telemetryEnv?: 'dev' | 'prod'; // Telemetry environment (default: 'prod', only used when telemetryEnabled is true) + telemetryEnv?: TelemetryEnv; // Telemetry environment (default: 'prod', only used when telemetryEnabled is true) } function checkApifyToken(): void { From 40163520f7f4e0b45b8f489aacb1bbe62d4ff496 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Wed, 19 Nov 2025 15:46:33 +0100 Subject: [PATCH 08/46] fix: add capabilities to tracking --- src/mcp/server.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 03f7227c..9ea66315 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -632,14 +632,18 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool // Extract reason from tool arguments if provided const reason = (args as Record).reason?.toString() || ''; + const capabilities = this.options.initializeRequestData?.params?.capabilities; + const params = this.options.initializeRequestData?.params as InitializeRequest['params']; + const serverVersion = getPackageVersion() || ''; const telemetryData = { - app: 'mcp_server', - mcp_client: this.options.initializeRequestData?.params?.clientInfo?.name || '', + app_name: 'apify-mcp-server', + app_version: serverVersion, + mcp_client_name: params?.clientInfo?.name?.valueOf() || '', + mcp_client_version: params?.clientInfo?.version?.valueOf() || '', + mcp_protocol_version: params?.protocolVersion?.valueOf() || '', + mcp_capabilities: capabilities ? JSON.stringify(capabilities) : '', mcp_session_id: mcpSessionId || '', transport_type: this.options.transportType || '', - // This is the version of the apify-mcp-server package - // this can be different from the internal remote server version - server_version: getPackageVersion() || '', tool_name: toolFullName, reason, }; From 82eefe5ebbc0446b9603a50cd385a2c3eaa259c9 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Thu, 20 Nov 2025 08:57:38 +0100 Subject: [PATCH 09/46] fix: remove unused ENVIROMENT variable --- src/mcp/server.ts | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 9ea66315..ebf2655b 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -34,7 +34,7 @@ import { SKYFIRE_README_CONTENT, SKYFIRE_TOOL_INSTRUCTIONS, } from '../const.js'; -import { TELEMETRY_ENV, type TelemetryEnv } from '../const.js'; +import { type TelemetryEnv } from '../const.js'; import { prompts } from '../prompts/index.js'; import { getTelemetryEnv, trackToolCall } from '../telemetry.js'; import { callActorGetDataset, defaultTools, getActorsAsTools, toolCategories } from '../tools/index.js'; @@ -62,7 +62,7 @@ interface ActorsMcpServerOptions { initializeRequestData?: InitializeRequest; /** * Enable or disable telemetry tracking for tool calls. - * Defaults to true when not set, or can be set from ENVIRONMENT env variable. + * Defaults to true when not set. */ telemetryEnabled?: boolean; /** @@ -100,18 +100,11 @@ export class ActorsMcpServer { constructor(options: ActorsMcpServerOptions = {}) { this.options = options; - // If telemetryEnabled is not explicitly set, try to read from ENVIRONMENT env variable, this is used in the mcp.apify.com deployment + // Default telemetry configuration if (this.options.telemetryEnabled === undefined) { - const envValue = process.env.ENVIRONMENT; - const validEnv = getTelemetryEnv(envValue); - if (envValue === TELEMETRY_ENV.DEV || envValue === TELEMETRY_ENV.PROD) { - this.options.telemetryEnabled = true; - this.options.telemetryEnv = validEnv; - } else { - // Default to enabled with prod environment - this.options.telemetryEnabled = true; - this.options.telemetryEnv = getTelemetryEnv(this.options.telemetryEnv); - } + // Default to enabled with prod environment + this.options.telemetryEnabled = true; + this.options.telemetryEnv = getTelemetryEnv(this.options.telemetryEnv); } else if (this.options.telemetryEnabled) { // If telemetry is enabled, ensure telemetryEnv is set (default to 'prod') this.options.telemetryEnv = getTelemetryEnv(this.options.telemetryEnv); @@ -634,16 +627,17 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool const capabilities = this.options.initializeRequestData?.params?.capabilities; const params = this.options.initializeRequestData?.params as InitializeRequest['params']; - const serverVersion = getPackageVersion() || ''; const telemetryData = { - app_name: 'apify-mcp-server', - app_version: serverVersion, - mcp_client_name: params?.clientInfo?.name?.valueOf() || '', - mcp_client_version: params?.clientInfo?.version?.valueOf() || '', - mcp_protocol_version: params?.protocolVersion?.valueOf() || '', + app: 'mcp_server', + mcp_client: params?.clientInfo?.name || '', + mcp_client_version: params?.clientInfo?.version || '', + mcp_protocol_version: params?.protocolVersion || '', mcp_capabilities: capabilities ? JSON.stringify(capabilities) : '', mcp_session_id: mcpSessionId || '', transport_type: this.options.transportType || '', + // This is the version of the apify-mcp-server package + // this can be different from the internal remote server version + server_version: getPackageVersion() || '', tool_name: toolFullName, reason, }; From d7795064a48eba79caa178ec0404cdf7c75ccd04 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Thu, 20 Nov 2025 10:26:22 +0100 Subject: [PATCH 10/46] fix: add env variable to enable/disable telemetry --- README.md | 13 ++++++++++--- src/mcp/server.ts | 34 +++++++++++++++++++++++----------- src/stdio.ts | 4 ++-- src/utils/generic.ts | 19 +++++++++++++++++++ tests/helpers.ts | 4 ++-- 5 files changed, 56 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 6bab1fe5..2b1a97b2 100644 --- a/README.md +++ b/README.md @@ -274,18 +274,25 @@ By default, telemetry is **enabled** for all tool calls. ### Opting out of telemetry -You can disable telemetry in two ways: +You can opt out of telemetry by setting the `--telemetry-enabled` CLI flag to `false` or the `TELEMETRY_ENABLED` environment variable to `false`. +CLI flags take precedence over environment variables. + +#### Examples **For the remote server (mcp.apify.com)**: -Add the `?telemetry-enabled=false` query parameter to the URL: ```text +# Disable via URL parameter https://mcp.apify.com?telemetry-enabled=false ``` **For the local stdio server**: -Use the `--telemetry-enabled=false` CLI flag: ```bash +# Disable via CLI flag npx @apify/actors-mcp-server --telemetry-enabled=false + +# Or set environment variable +export TELEMETRY_ENABLED=false +npx @apify/actors-mcp-server ``` # βš™οΈ Development diff --git a/src/mcp/server.ts b/src/mcp/server.ts index ebf2655b..3f62374c 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -41,6 +41,7 @@ import { callActorGetDataset, defaultTools, getActorsAsTools, toolCategories } f import { decodeDotPropertyNames } from '../tools/utils.js'; import type { ToolEntry } from '../types.js'; import { buildActorResponseContent } from '../utils/actor-response.js'; +import { parseBooleanFromString } from '../utils/generic.js'; import { logHttpError } from '../utils/logging.js'; import { buildMCPResponse } from '../utils/mcp.js'; import { createProgressTracker } from '../utils/progress.js'; @@ -100,16 +101,6 @@ export class ActorsMcpServer { constructor(options: ActorsMcpServerOptions = {}) { this.options = options; - // Default telemetry configuration - if (this.options.telemetryEnabled === undefined) { - // Default to enabled with prod environment - this.options.telemetryEnabled = true; - this.options.telemetryEnv = getTelemetryEnv(this.options.telemetryEnv); - } else if (this.options.telemetryEnabled) { - // If telemetry is enabled, ensure telemetryEnv is set (default to 'prod') - this.options.telemetryEnv = getTelemetryEnv(this.options.telemetryEnv); - } - const { setupSigintHandler = true } = options; this.server = new Server( { @@ -120,7 +111,7 @@ export class ActorsMcpServer { capabilities: { tools: { listChanged: true }, /** - * Declaring prompts even though we are not using them + * Declaring resources even though we are not using them * to prevent clients like Claude desktop from failing. */ resources: { }, @@ -129,6 +120,7 @@ export class ActorsMcpServer { }, }, ); + this.setupTelemetry(); this.setupLoggingProxy(); this.tools = new Map(); this.setupErrorHandling(setupSigintHandler); @@ -141,6 +133,26 @@ export class ActorsMcpServer { this.setupResourceHandlers(); } + /** + * Telemetry configuration with precedence: explicit options > env vars > defaults + */ + private setupTelemetry() { + if (this.options.telemetryEnabled === undefined) { + // Check environment variable as fallback + const envEnabled = parseBooleanFromString(process.env.TELEMETRY_ENABLED); + this.options.telemetryEnabled = envEnabled !== undefined ? envEnabled : true; + } + // Set telemetryEnv: explicit option > env var > default ('prod') + const telemetryEnv = process.env.TELEMETRY_ENV; + const telemetryParam = this.options.telemetryEnv; + this.options.telemetryEnv = getTelemetryEnv(telemetryParam ?? telemetryEnv); + + // If telemetry is enabled, ensure telemetryEnv is set + if (this.options.telemetryEnabled && this.options.telemetryEnv === undefined) { + this.options.telemetryEnv = getTelemetryEnv(undefined); + } + } + /** * Returns an array of tool names. * @returns {string[]} - An array of tool names. diff --git a/src/stdio.ts b/src/stdio.ts index 3f12f0f8..0fdc4ab4 100644 --- a/src/stdio.ts +++ b/src/stdio.ts @@ -87,13 +87,13 @@ const argv = yargs(hideBin(process.argv)) type: 'boolean', default: false, describe: `Enable dynamically adding Actors as tools based on user requests. Can also be set via ENABLE_ADDING_ACTORS environment variable. -Deprecated: use tools experimental category instead.`, +Deprecated: use tools add-actor instead.`, }) .option('enableActorAutoLoading', { type: 'boolean', default: false, hidden: true, - describe: 'Deprecated: use enable-adding-actors instead.', + describe: 'Deprecated: Use tools add-actor instead.', }) .options('tools', { type: 'string', diff --git a/src/utils/generic.ts b/src/utils/generic.ts index 3757cb74..7302bbe4 100644 --- a/src/utils/generic.ts +++ b/src/utils/generic.ts @@ -65,3 +65,22 @@ export function isValidHttpUrl(urlString: string): boolean { return false; } } + +/** + * Parses a boolean value from a string. + * Accepts 'true', '1' as true, 'false', '0' as false. + * Returns undefined if the value is not a recognized boolean string. + */ +export function parseBooleanFromString(value: string | undefined | null): boolean | undefined { + if (value === undefined || value === null) { + return undefined; + } + const normalized = value.toLowerCase().trim(); + if (normalized === 'true' || normalized === '1') { + return true; + } + if (normalized === 'false' || normalized === '0') { + return false; + } + return undefined; +} diff --git a/tests/helpers.ts b/tests/helpers.ts index 50a4681d..441c8518 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -35,8 +35,8 @@ function appendSearchParams(url: URL, options?: McpClientOptions): void { url.searchParams.append('tools', tools.join(',')); } // Append telemetry parameters - if (telemetryEnabled === false) { - url.searchParams.append('telemetry-enabled', 'false'); + if (telemetryEnabled !== undefined) { + url.searchParams.append('telemetry-enabled', telemetryEnabled.toString()); } if (telemetryEnv !== undefined && telemetryEnabled !== false) { url.searchParams.append('telemetry-env', telemetryEnv); From 0e6cdfd38a84567c481c1dd0ac3b24925409c005 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Thu, 20 Nov 2025 10:26:26 +0100 Subject: [PATCH 11/46] fix: add env variable to enable/disable telemetry --- mds/TELEMETRY.md | 69 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/mds/TELEMETRY.md b/mds/TELEMETRY.md index 42aff948..c6c52af9 100644 --- a/mds/TELEMETRY.md +++ b/mds/TELEMETRY.md @@ -21,7 +21,19 @@ interface ActorsMcpServerOptions { ### Default Behavior - **Telemetry**: Enabled by default (`telemetryEnabled: true`) - **Environment**: Production by default (`telemetryEnv: 'prod'`) -- **ENVIRONMENT env var**: If set to 'dev' or 'prod', automatically enables telemetry with that environment + +### Configuration Precedence + +Telemetry configuration can be set via multiple methods with the following precedence (highest to lowest): + +1. **CLI arguments** (for stdio) or **URL query parameters** (for remote server) +2. **Environment variables** (`TELEMETRY_ENABLED`, `TELEMETRY_ENV`) +3. **Defaults** (`telemetryEnabled: true`, `telemetryEnv: 'prod'`) + +#### Environment Variables + +- `TELEMETRY_ENABLED`: Set to `true`, `1`, `false`, or `0` to enable/disable telemetry +- `TELEMETRY_ENV`: Set to `prod` or `dev` to specify the telemetry environment (only used when telemetry is enabled) ### Usage Examples @@ -253,13 +265,13 @@ const server = new ActorsMcpServer({ - Added `telemetryEnabled?: boolean` option - `true` (default): Telemetry enabled - `false`: Telemetry disabled - - If not explicitly set, reads from `ENVIRONMENT` env variable (if set to 'dev' or 'prod', enables telemetry) + - If not explicitly set, reads from `TELEMETRY_ENABLED` env variable - Defaults to `true` when not set - Added `telemetryEnv?: 'dev' | 'prod'` option - `'prod'` (default): Use production Segment write key - `'dev'`: Use development Segment write key - Only used when `telemetryEnabled` is `true` - - If not explicitly set, reads from `ENVIRONMENT` env variable + - If not explicitly set, reads from `TELEMETRY_ENV` env variable - Defaults to `'prod'` when not set - Added `transportType?: 'stdio' | 'http' | 'sse'` option - `'stdio'`: Direct/local stdio connection @@ -271,10 +283,13 @@ const server = new ActorsMcpServer({ - Contains client info like `clientInfo.name`, capabilities, etc. - Injected via message interception or HTTP request body -- **Constructor** (lines 98-115): - - If `telemetryEnabled` is not explicitly set, reads from `ENVIRONMENT` env variable - - If `ENVIRONMENT === 'dev'` or `'prod'`: sets `telemetryEnabled = true` and `telemetryEnv = envValue` - - Otherwise, defaults to `telemetryEnabled = true` and `telemetryEnv = 'prod'` +- **Constructor** (lines 100-120): + - If `telemetryEnabled` is not explicitly set, reads from `TELEMETRY_ENABLED` env variable + - Parses env var value: `'true'` or `'1'` = true, `'false'` or `'0'` = false + - If env var not set, defaults to `telemetryEnabled = true` + - If `telemetryEnv` is not explicitly set, reads from `TELEMETRY_ENV` env variable + - Validates env var value via `getTelemetryEnv()` (must be 'dev' or 'prod') + - If env var not set or invalid, defaults to `telemetryEnv = 'prod'` - If `telemetryEnabled` is explicitly `true`, ensures `telemetryEnv` is set (defaults to 'prod') - Supports environment-based telemetry control for hosted deployments @@ -320,11 +335,13 @@ const server = new ActorsMcpServer({ - JSON file is parsed and token is extracted from `token` key - Silently fails on file not found or parse errors (no error thrown) -- **Server Initialization** (lines 154-159): +- **Server Initialization** (lines 156-161): - Passes `transportType: 'stdio'` when creating ActorsMcpServer - - Passes telemetry options from CLI flags: + - Passes telemetry options from CLI flags (via yargs): - `--telemetry-enabled` (boolean, default: true) - documented for end users - `--telemetry-env` ('prod'|'dev', default: 'prod') - hidden flag for debugging only + - CLI flags take precedence over environment variables (via yargs `.env()`) + - Environment variables `TELEMETRY_ENABLED` and `TELEMETRY_ENV` are supported as fallback - Converts CLI flags to `telemetryEnabled` (boolean) and `telemetryEnv` ('dev'|'prod') - **Message Interception** (lines 162-176): @@ -337,6 +354,19 @@ const server = new ActorsMcpServer({ **File**: `src/telemetry.ts` +- **`parseBooleanFromString(value: string | undefined | null)` Function**: + - Parses boolean values from environment variable strings + - Accepts `'true'`, `'1'` as `true` + - Accepts `'false'`, `'0'` as `false` + - Returns `undefined` for unrecognized values + - Used to parse `TELEMETRY_ENABLED` environment variable + +- **`getTelemetryEnv(env?: string | null)` Function**: + - Validates and normalizes telemetry environment value + - Accepts `'dev'` or `'prod'` + - Returns default (`'prod'`) for invalid or missing values + - Used to parse `TELEMETRY_ENV` environment variable + - **`getOrInitAnalyticsClient(env: 'dev' | 'prod')` Function**: - Singleton pattern ensures only one Segment Analytics client per environment - Uses Map to store clients: `{ dev: Analytics, prod: Analytics }` @@ -378,9 +408,11 @@ The telemetry infrastructure is integrated into the remote server that hosts the - **`handleNewSession()` Handler**: - Extracts `?telemetry-enabled` and `?telemetry-env` query parameters from request URL + - URL parameters take precedence over environment variables + - Falls back to `TELEMETRY_ENABLED` and `TELEMETRY_ENV` env vars if URL params not provided - Converts parameters to options: - - `telemetryEnabled = telemetryEnabledParam !== 'false'` (defaults to true) - - `telemetryEnv = telemetryEnvParam || 'prod'` (defaults to 'prod') + - `telemetryEnabled`: URL param > env var > default (true) + - `telemetryEnv`: URL param > env var > default ('prod') - Passes to ActorsMcpServer constructor: - `transportType: 'http'` - identifies remote HTTP streamable connection - `telemetryEnabled` and `telemetryEnv` - per-session telemetry control @@ -399,9 +431,11 @@ The telemetry infrastructure is integrated into the remote server that hosts the - **`initMCPSession()` Handler** (lines 153-210): - Extracts `?telemetry-enabled` and `?telemetry-env` query parameters from response URL + - URL parameters take precedence over environment variables + - Falls back to `TELEMETRY_ENABLED` and `TELEMETRY_ENV` env vars if URL params not provided - Converts parameters to options: - - `telemetryEnabled = telemetryEnabledParam !== 'false'` (defaults to true) - - `telemetryEnv = telemetryEnvParam || 'prod'` (defaults to 'prod') + - `telemetryEnabled`: URL param > env var > default (true) + - `telemetryEnv`: URL param > env var > default ('prod') - Creates ActorsMcpServer with: - `transportType: 'sse'` - identifies remote SSE connection - `telemetryEnabled` and `telemetryEnv` - per-session telemetry control @@ -414,6 +448,8 @@ The telemetry infrastructure is integrated into the remote server that hosts the - **Per-Session Control**: - Both transports support `?telemetry-enabled=false` query parameter to disable telemetry - Optional `?telemetry-env=dev` parameter to use development workspace (for debugging only) + - URL parameters take precedence over `TELEMETRY_ENABLED` and `TELEMETRY_ENV` environment variables + - Environment variables can be used as fallback when URL parameters are not provided - Prevents test data from polluting production telemetry - Example: `https://mcp.apify.com/?telemetry-enabled=false` for streamable - Example: `https://mcp.apify.com/sse?telemetry-enabled=false` for SSE @@ -522,11 +558,18 @@ Transport type is now passed via `ActorsMcpServerOptions`: - Message interception proxy to capture initialize request data in stdio (`src/stdio.ts`) - Streamable HTTP transport with telemetry query parameter support (`src/actor/server.ts`) - Extracts `?telemetry-enabled` and `?telemetry-env` query params + - Falls back to `TELEMETRY_ENABLED` and `TELEMETRY_ENV` env vars when URL params not provided - Passes `transportType: 'http'` and telemetry options to ActorsMcpServer - Legacy SSE transport with telemetry query parameter support (`src/actor/server.ts`) - Extracts `?telemetry-enabled` and `?telemetry-env` query params + - Falls back to `TELEMETRY_ENABLED` and `TELEMETRY_ENV` env vars when URL params not provided - Message interception proxy to capture initialize request data from MCP protocol - Passes `transportType: 'sse'` and telemetry options +- Environment variable support for telemetry configuration + - `TELEMETRY_ENABLED`: Set to `'true'`, `'1'`, `'false'`, or `'0'` to enable/disable telemetry + - `TELEMETRY_ENV`: Set to `'prod'` or `'dev'` to specify telemetry environment + - Used as fallback when CLI/URL parameters are not provided + - Precedence: CLI/URL params > env vars > defaults - Test configuration to prevent telemetry pollution - `test/integration/tests/server-streamable.test.ts`: Uses `/?telemetry-enabled=false` - `test/integration/tests/server-sse.test.ts`: Uses `/sse?telemetry-enabled=false` From 72b9ed6fd255450588511f5bc81ff9bc7c88ea44 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Thu, 20 Nov 2025 11:06:25 +0100 Subject: [PATCH 12/46] fix: update unit-tests --- tests/integration/actor.server-sse.test.ts | 46 ----------- .../actor.server-streamable.test.ts | 46 ----------- tests/integration/suite.ts | 82 +++++++++---------- 3 files changed, 41 insertions(+), 133 deletions(-) delete mode 100644 tests/integration/actor.server-sse.test.ts delete mode 100644 tests/integration/actor.server-streamable.test.ts diff --git a/tests/integration/actor.server-sse.test.ts b/tests/integration/actor.server-sse.test.ts deleted file mode 100644 index 971d7304..00000000 --- a/tests/integration/actor.server-sse.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { Server as HttpServer } from 'node:http'; - -import type { Express } from 'express'; - -import log from '@apify/log'; - -import { createExpressApp } from '../../src/actor/server.js'; -import { createMcpSseClient } from '../helpers.js'; -import { createIntegrationTestsSuite } from './suite.js'; -import { getAvailablePort } from './utils/port.js'; - -let app: Express; -let httpServer: HttpServer; -let httpServerPort: number; -let httpServerHost: string; -let mcpUrl: string; - -// Set environment to dev for telemetry tests -process.env.ENVIRONMENT = 'dev'; - -createIntegrationTestsSuite({ - suiteName: 'Apify MCP Server SSE', - transport: 'sse', - createClientFn: async (options) => await createMcpSseClient(mcpUrl, options), - beforeAllFn: async () => { - log.setLevel(log.LEVELS.OFF); - - // Get an available port - httpServerPort = await getAvailablePort(); - httpServerHost = `http://localhost:${httpServerPort}`; - mcpUrl = `${httpServerHost}/sse`; - - // Create an express app - app = createExpressApp(httpServerHost); - - // Start a test server - await new Promise((resolve) => { - httpServer = app.listen(httpServerPort, () => resolve()); - }); - }, - afterAllFn: async () => { - await new Promise((resolve) => { - httpServer.close(() => resolve()); - }); - }, -}); diff --git a/tests/integration/actor.server-streamable.test.ts b/tests/integration/actor.server-streamable.test.ts deleted file mode 100644 index e0f7838c..00000000 --- a/tests/integration/actor.server-streamable.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { Server as HttpServer } from 'node:http'; - -import type { Express } from 'express'; - -import log from '@apify/log'; - -import { createExpressApp } from '../../src/actor/server.js'; -import { createMcpStreamableClient } from '../helpers.js'; -import { createIntegrationTestsSuite } from './suite.js'; -import { getAvailablePort } from './utils/port.js'; - -let app: Express; -let httpServer: HttpServer; -let httpServerPort: number; -let httpServerHost: string; -let mcpUrl: string; - -// Set environment to dev for telemetry tests -process.env.ENVIRONMENT = 'dev'; - -createIntegrationTestsSuite({ - suiteName: 'Apify MCP Server Streamable HTTP', - transport: 'streamable-http', - createClientFn: async (options) => await createMcpStreamableClient(mcpUrl, options), - beforeAllFn: async () => { - log.setLevel(log.LEVELS.OFF); - - // Get an available port - httpServerPort = await getAvailablePort(); - httpServerHost = `http://localhost:${httpServerPort}`; - mcpUrl = `${httpServerHost}/mcp`; - - // Create an express app - app = createExpressApp(httpServerHost); - - // Start a test server - await new Promise((resolve) => { - httpServer = app.listen(httpServerPort, () => resolve()); - }); - }, - afterAllFn: async () => { - await new Promise((resolve) => { - httpServer.close(() => resolve()); - }); - }, -}); diff --git a/tests/integration/suite.ts b/tests/integration/suite.ts index 4aef389a..adcc314b 100644 --- a/tests/integration/suite.ts +++ b/tests/integration/suite.ts @@ -31,6 +31,39 @@ function expectToolNamesToContain(names: string[], toolNames: string[] = []) { toolNames.forEach((name) => expect(names).toContain(name)); } +/** + * Checks that no tools have a reason field in their input schema + */ +function expectReasonFieldAbsent(tools: { tools: { inputSchema?: unknown }[] }) { + for (const tool of tools.tools) { + if (tool.inputSchema && typeof tool.inputSchema === 'object' && 'properties' in tool.inputSchema) { + const props = tool.inputSchema.properties as Record; + expect(props.reason).toBeUndefined(); + } + } +} + +/** + * Checks that at least one tool has a reason field in its input schema + * @param validateType - If true, also validates that the reason field type is 'string' + */ +function expectReasonFieldPresent(tools: { tools: { inputSchema?: unknown }[] }, validateType = false) { + let reasonFieldFound = false; + for (const tool of tools.tools) { + if (tool.inputSchema && typeof tool.inputSchema === 'object' && 'properties' in tool.inputSchema) { + const props = tool.inputSchema.properties as Record; + if (props.reason !== undefined) { + reasonFieldFound = true; + if (validateType) { + const reasonField = props.reason as Record; + expect(reasonField.type).toBe('string'); + } + } + } + } + expect(reasonFieldFound).toBe(true); +} + // eslint-disable-next-line @typescript-eslint/no-explicit-any function extractJsonFromMarkdown(text: string): any { // Handle markdown code blocks like ```json @@ -1064,60 +1097,27 @@ export function createIntegrationTestsSuite( await client.close(); }); - it('should NOT include reason field in tools when telemetry is off (default)', async () => { + it('should NOT include reason field in tools when telemetry is off', async () => { client = await createClientFn({ telemetryEnabled: false }); const tools = await client.listTools(); - - // Check that tools don't have reason field in input schema - for (const tool of tools.tools) { - if (tool.inputSchema && typeof tool.inputSchema === 'object' && 'properties' in tool.inputSchema) { - const props = tool.inputSchema.properties as Record; - expect(props.reason).toBeUndefined(); - } - } - + expectReasonFieldAbsent(tools); await client.close(); }); it('should include reason field in tools when telemetry is enabled (dev)', async () => { client = await createClientFn({ telemetryEnabled: true, telemetryEnv: 'dev' }); const tools = await client.listTools(); - - // Check that tools have reason field with proper description when telemetry is enabled - let reasonFieldFound = false; - for (const tool of tools.tools) { - if (tool.inputSchema && typeof tool.inputSchema === 'object' && 'properties' in tool.inputSchema) { - const props = tool.inputSchema.properties as Record; - if (props.reason !== undefined) { - reasonFieldFound = true; - const reasonField = props.reason as Record; - expect(reasonField.type).toBe('string'); - } - } - } - expect(reasonFieldFound).toBe(true); - + expectReasonFieldPresent(tools, true); await client.close(); }); - it('should include reason field in tools when telemetry is enabled (prod)', async () => { - client = await createClientFn({ telemetryEnabled: true, telemetryEnv: 'prod' }); + // Environment variable precedence tests + it.runIf(options.transport === 'stdio')('should use TELEMETRY_ENABLED env var when CLI arg is not provided', async () => { + // When useEnv=true, telemetryEnabled option translates to env.TELEMETRY_ENABLED in child process + client = await createClientFn({ useEnv: true, telemetryEnabled: false }); const tools = await client.listTools(); - // Check that tools have reason field with proper description when telemetry is enabled - let reasonFieldFound = false; - for (const tool of tools.tools) { - if (tool.inputSchema && typeof tool.inputSchema === 'object' && 'properties' in tool.inputSchema) { - const props = tool.inputSchema.properties as Record; - if (props.reason !== undefined) { - reasonFieldFound = true; - const reasonField = props.reason as Record; - expect(reasonField.type).toBe('string'); - } - } - } - expect(reasonFieldFound).toBe(true); - + expectReasonFieldAbsent(tools); await client.close(); }); }); From 53bead48438afae921f6e1a6d41e9a4951516cca Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Thu, 20 Nov 2025 11:49:41 +0100 Subject: [PATCH 13/46] fix: update telemetry properties --- src/mcp/server.ts | 12 +++----- src/telemetry.ts | 5 ++-- src/types.ts | 16 +++++++++++ tests/unit/telemetry.test.ts | 53 ++++++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 10 deletions(-) create mode 100644 tests/unit/telemetry.test.ts diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 3f62374c..4193367c 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -640,16 +640,14 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool const capabilities = this.options.initializeRequestData?.params?.capabilities; const params = this.options.initializeRequestData?.params as InitializeRequest['params']; const telemetryData = { - app: 'mcp_server', - mcp_client: params?.clientInfo?.name || '', + app_name: 'apify-mcp-server', + app_version: getPackageVersion() || '', + mcp_client_name: params?.clientInfo?.name || '', mcp_client_version: params?.clientInfo?.version || '', mcp_protocol_version: params?.protocolVersion || '', mcp_capabilities: capabilities ? JSON.stringify(capabilities) : '', mcp_session_id: mcpSessionId || '', transport_type: this.options.transportType || '', - // This is the version of the apify-mcp-server package - // this can be different from the internal remote server version - server_version: getPackageVersion() || '', tool_name: toolFullName, reason, }; @@ -731,9 +729,7 @@ Please verify the server URL is correct and accessible, and ensure you have a va // Handle actor tool if (tool.type === 'actor') { - if (this.options.skyfireMode - && args['skyfire-pay-id'] === undefined - ) { + if (this.options.skyfireMode && args['skyfire-pay-id'] === undefined) { return buildMCPResponse([SKYFIRE_TOOL_INSTRUCTIONS]); } diff --git a/src/telemetry.ts b/src/telemetry.ts index ddf39942..696d2533 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -1,6 +1,7 @@ import { Analytics } from '@segment/analytics-node'; import { DEFAULT_TELEMETRY_ENV, TELEMETRY_ENV, type TelemetryEnv } from './const.js'; +import type { ToolCallTelemetryProperties } from './types.js'; const DEV_WRITE_KEY = '9rPHlMtxX8FJhilGEwkfUoZ0uzWxnzcT'; const PROD_WRITE_KEY = 'cOkp5EIJaN69gYaN8bcp7KtaD0fGABwJ'; @@ -40,12 +41,12 @@ export function getOrInitAnalyticsClient(env: TelemetryEnv): Analytics { * * @param userId - Apify user ID (TODO: extract from token when auth available) * @param env - 'dev' for development, 'prod' for production - * @param properties - Additional event properties + * @param properties - Event properties for the tool call */ export function trackToolCall( userId: string, env: TelemetryEnv, - properties: Record, + properties: ToolCallTelemetryProperties, ): void { const client = getOrInitAnalyticsClient(env); diff --git a/src/types.ts b/src/types.ts index 2094d15d..a3e3f7a6 100644 --- a/src/types.ts +++ b/src/types.ts @@ -293,3 +293,19 @@ export type DatasetItem = Record; * Can be null or undefined in case of Skyfire requests. */ export type ApifyToken = string | null | undefined; + +/** + * Properties for tool call telemetry events sent to Segment. + */ +export interface ToolCallTelemetryProperties { + app_name: string; + app_version: string; + mcp_client_name: string; + mcp_client_version: string; + mcp_protocol_version: string; + mcp_capabilities: string; + mcp_session_id: string; + transport_type: string; + tool_name: string; + reason: string; +} diff --git a/tests/unit/telemetry.test.ts b/tests/unit/telemetry.test.ts new file mode 100644 index 00000000..aa30a082 --- /dev/null +++ b/tests/unit/telemetry.test.ts @@ -0,0 +1,53 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { TELEMETRY_ENV } from '../../src/const.js'; +import { trackToolCall } from '../../src/telemetry.js'; + +// Mock the Segment Analytics client +const mockTrack = vi.fn(); +vi.mock('@segment/analytics-node', () => ({ + Analytics: vi.fn().mockImplementation(() => ({ + track: mockTrack, + })), +})); + +describe('telemetry', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('should send correct payload structure to Segment', () => { + const userId = 'test-user-123'; + const properties = { + app_name: 'apify-mcp-server', + app_version: '0.5.6', + mcp_client_name: 'test-client', + mcp_client_version: '1.0.0', + mcp_protocol_version: '2024-11-05', + mcp_capabilities: '{}', + mcp_session_id: 'session-123', + transport_type: 'stdio', + tool_name: 'test-tool', + reason: 'test reason', + }; + + trackToolCall(userId, TELEMETRY_ENV.DEV, properties); + + expect(mockTrack).toHaveBeenCalledWith({ + userId: 'test-user-123', + event: 'MCP tool call', + properties: { + app_name: 'apify-mcp-server', + app_version: '0.5.6', + mcp_client_name: 'test-client', + mcp_client_version: '1.0.0', + mcp_protocol_version: '2024-11-05', + mcp_capabilities: '{}', + mcp_session_id: 'session-123', + transport_type: 'stdio', + tool_name: 'test-tool', + reason: 'test reason', + }, + }); + }); +}); From f46bff9c0e25753bd5e057144aff9537d5ca8551 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Thu, 20 Nov 2025 13:08:45 +0100 Subject: [PATCH 14/46] fix: add status and exec time to telemetry --- src/mcp/server.ts | 28 +++++++++++++++++++++------- src/types.ts | 2 ++ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 4193367c..a9ebe589 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -39,7 +39,7 @@ import { prompts } from '../prompts/index.js'; import { getTelemetryEnv, trackToolCall } from '../telemetry.js'; import { callActorGetDataset, defaultTools, getActorsAsTools, toolCategories } from '../tools/index.js'; import { decodeDotPropertyNames } from '../tools/utils.js'; -import type { ToolEntry } from '../types.js'; +import type { ToolCallTelemetryProperties, ToolEntry } from '../types.js'; import { buildActorResponseContent } from '../utils/actor-response.js'; import { parseBooleanFromString } from '../utils/generic.js'; import { logHttpError } from '../utils/logging.js'; @@ -618,12 +618,14 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool ); } - // Track telemetry if enabled + // Prepare telemetry data (but don't track yet) + let telemetryData: ToolCallTelemetryProperties | null = null; + let userId = ''; + if (this.options.telemetryEnabled === true) { const toolFullName = tool.type === 'actor' ? tool.actorFullName : tool.name; // Get userId from cache or fetch from API - let userId = ''; // Use token from options (e.g., from stdio auth file) or from request if (apifyToken) { const apifyClient = new ApifyClient({ token: apifyToken }); @@ -639,7 +641,7 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool const capabilities = this.options.initializeRequestData?.params?.capabilities; const params = this.options.initializeRequestData?.params as InitializeRequest['params']; - const telemetryData = { + telemetryData = { app_name: 'apify-mcp-server', app_version: getPackageVersion() || '', mcp_client_name: params?.clientInfo?.name || '', @@ -650,12 +652,14 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool transport_type: this.options.transportType || '', tool_name: toolFullName, reason, + tool_status: 'success', // Will be updated in finally + tool_exec_time_ms: 0, // Will be calculated in finally }; - - log.debug('Telemetry: tracking tool call', telemetryData); - trackToolCall(userId, getTelemetryEnv(this.options.telemetryEnv), telemetryData); } + const startTime = Date.now(); + let toolStatus: 'success' | 'failure' | 'cancelled' = 'success'; + try { // Handle internal tool if (tool.type === 'internal') { @@ -758,6 +762,7 @@ Please verify the server URL is correct and accessible, and ensure you have a va ); if (!callResult) { + toolStatus = 'cancelled'; // Receivers of cancellation notifications SHOULD NOT send a response for the cancelled request // https://modelcontextprotocol.io/specification/2025-06-18/basic/utilities/cancellation#behavior-requirements return { }; @@ -772,12 +777,21 @@ Please verify the server URL is correct and accessible, and ensure you have a va } } } catch (error) { + toolStatus = extra.signal?.aborted ? 'cancelled' : 'failure'; logHttpError(error, 'Error occurred while calling tool', { toolName: name }); const errorMessage = (error instanceof Error) ? error.message : 'Unknown error'; return buildMCPResponse([ `Error calling tool "${name}": ${errorMessage}. Please verify the tool name, input parameters, and ensure all required resources are available.`, ], true); + } finally { + // Track telemetry once at the end with determined status and execution time + if (telemetryData) { + const execTime = Date.now() - startTime; + telemetryData.tool_status = toolStatus; + telemetryData.tool_exec_time_ms = execTime; + trackToolCall(userId, getTelemetryEnv(this.options.telemetryEnv), telemetryData); + } } const availableTools = this.listToolNames(); diff --git a/src/types.ts b/src/types.ts index a3e3f7a6..bf4303d7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -308,4 +308,6 @@ export interface ToolCallTelemetryProperties { transport_type: string; tool_name: string; reason: string; + tool_status: 'success' | 'failure' | 'cancelled'; + tool_exec_time_ms: number; } From f3fb01c5a61857f1dd958a017bea2775cea5460e Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Thu, 20 Nov 2025 13:56:20 +0100 Subject: [PATCH 15/46] fix: add tool call number --- src/mcp/server.ts | 52 ++++++++++++++++++++++++++++++++++++ src/types.ts | 1 + tests/unit/telemetry.test.ts | 6 +++++ 3 files changed, 59 insertions(+) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index a9ebe589..30f29893 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -54,6 +54,19 @@ import { processParamsGetTools } from './utils.js'; type ToolsChangedHandler = (toolNames: string[]) => void; +/** + * Interface for storing and retrieving tool call counters per session. + * Used for tracking tool call sequence in user journeys. + */ +interface ToolCallCounterStore { + /** + * Gets and increments the tool call counter for a session atomically. + * @param sessionId - The session ID + * @returns Promise resolving to the new counter value (after increment) + */ + getAndIncrement(sessionId: string): Promise; +} + interface ActorsMcpServerOptions { setupSigintHandler?: boolean; /** @@ -85,6 +98,12 @@ interface ActorsMcpServerOptions { * instead of APIFY_TOKEN environment variable, so it can be passed to the server */ token?: string; + /** + * Optional store for tool call counters. + * If not provided, uses in-memory storage (suitable for stdio). + * For distributed deployments (HTTP/SSE), provide a Redis-backed implementation. + */ + toolCallCounterStore?: ToolCallCounterStore; } /** @@ -97,6 +116,8 @@ export class ActorsMcpServer { private sigintHandler: (() => Promise) | undefined; private currentLogLevel = 'info'; public readonly options: ActorsMcpServerOptions; + // In-memory storage for tool call counters (used when toolCallCounterStore is not provided) + private sessionToolCallCounters = new Map(); constructor(options: ActorsMcpServerOptions = {}) { this.options = options; @@ -133,6 +154,24 @@ export class ActorsMcpServer { this.setupResourceHandlers(); } + /** + * Gets and increments the tool call counter for a session. + * Uses external store if provided, otherwise uses in-memory Map. + * @param sessionId - The session ID + * @returns Promise resolving to the new counter value (after increment) + */ + private async getAndIncrementToolCallCounter(sessionId: string): Promise { + if (this.options.toolCallCounterStore) { + // Use external store (Redis for HTTP/SSE) + return await this.options.toolCallCounterStore.getAndIncrement(sessionId); + } + // Use in-memory storage (for stdio) + const current = this.sessionToolCallCounters.get(sessionId) || 0; + const newValue = current + 1; + this.sessionToolCallCounters.set(sessionId, newValue); + return newValue; + } + /** * Telemetry configuration with precedence: explicit options > env vars > defaults */ @@ -625,6 +664,18 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool if (this.options.telemetryEnabled === true) { const toolFullName = tool.type === 'actor' ? tool.actorFullName : tool.name; + // Get or increment tool call counter for this session + let toolCallNumber = 0; + const sessionId = mcpSessionId || 'unknown'; + if (sessionId !== 'unknown') { + try { + toolCallNumber = await this.getAndIncrementToolCallCounter(sessionId); + } catch (error) { + log.warning('Failed to get tool call counter', { sessionId, error: String(error) }); + // Continue with 0 if counter fails + } + } + // Get userId from cache or fetch from API // Use token from options (e.g., from stdio auth file) or from request if (apifyToken) { @@ -654,6 +705,7 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool reason, tool_status: 'success', // Will be updated in finally tool_exec_time_ms: 0, // Will be calculated in finally + tool_call_number: toolCallNumber, }; } diff --git a/src/types.ts b/src/types.ts index bf4303d7..66417a7f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -310,4 +310,5 @@ export interface ToolCallTelemetryProperties { reason: string; tool_status: 'success' | 'failure' | 'cancelled'; tool_exec_time_ms: number; + tool_call_number: number; } diff --git a/tests/unit/telemetry.test.ts b/tests/unit/telemetry.test.ts index aa30a082..10aaed2b 100644 --- a/tests/unit/telemetry.test.ts +++ b/tests/unit/telemetry.test.ts @@ -29,6 +29,9 @@ describe('telemetry', () => { transport_type: 'stdio', tool_name: 'test-tool', reason: 'test reason', + tool_status: 'success' as const, + tool_exec_time_ms: 100, + tool_call_number: 1, }; trackToolCall(userId, TELEMETRY_ENV.DEV, properties); @@ -47,6 +50,9 @@ describe('telemetry', () => { transport_type: 'stdio', tool_name: 'test-tool', reason: 'test reason', + tool_status: 'success', + tool_exec_time_ms: 100, + tool_call_number: 1, }, }); }); From 3f0d09d6adf767a74067b195b295f1991141180d Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Thu, 20 Nov 2025 14:54:20 +0100 Subject: [PATCH 16/46] fix: remove reason from telemetry --- mds/TELEMETRY.md | 39 ++----------------- src/mcp/server.ts | 20 ---------- src/tools/build.ts | 5 +-- src/tools/dataset.ts | 15 ++------ src/tools/dataset_collection.ts | 5 +-- src/tools/fetch-actor-details.ts | 5 +-- src/tools/fetch-apify-docs.ts | 5 +-- src/tools/get-html-skeleton.ts | 5 +-- src/tools/helpers.ts | 5 +-- src/tools/key_value_store.ts | 15 ++------ src/tools/key_value_store_collection.ts | 5 +-- src/tools/run.ts | 15 ++------ src/tools/run_collection.ts | 5 +-- src/tools/search-apify-docs.ts | 5 +-- src/tools/store_collection.ts | 5 +-- src/types.ts | 1 - tests/integration/suite.ts | 50 +------------------------ tests/unit/telemetry.test.ts | 2 - 18 files changed, 24 insertions(+), 183 deletions(-) diff --git a/mds/TELEMETRY.md b/mds/TELEMETRY.md index c6c52af9..8312bd08 100644 --- a/mds/TELEMETRY.md +++ b/mds/TELEMETRY.md @@ -72,8 +72,7 @@ const server = new ActorsMcpServer({ "mcp_client": "Claude Desktop", "transport_type": "stdio|http|sse", "server_version": "VERSION", - "tool_name": "apify/instagram-scraper", - "reason": "REASON_FOR_TOOL_CALL" + "tool_name": "apify/instagram-scraper" }, "timestamp": "ISO 8601 TIMESTAMP" } @@ -116,15 +115,6 @@ const server = new ActorsMcpServer({ - For actor tools: uses full actor name (e.g., 'apify/instagram-scraper') - For internal tools: uses tool name -- **Reason**: Why the tool was called (LLM-provided reasoning) - - βœ… **IMPLEMENTED**: Added as optional `reason` field to all tool input schemas when telemetry is enabled - - Extracted from tool call arguments: `(args as Record).reason?.toString() || ''` - - When telemetry is disabled (`telemetryEnabled: false`): reason field is NOT added to schemas (saves schema overhead for tests) - - When telemetry is enabled (`telemetryEnabled: true`): reason field is dynamically added to ALL tool schemas - - Field description: "A brief explanation of why this tool is being called and what it will help you accomplish" - - Allows LLMs to explain their tool selection and usage context - - Originally proposed by @JiΕ™Γ­ Spilka, similar to mcpcat.io implementation at MCP dev summit London - - **Timestamp**: When tool call occurred - Handled automatically by Segment SDK @@ -151,12 +141,6 @@ const server = new ActorsMcpServer({ - Track use cases per user/organization - Understand different user archetypes -### 5. Tool Call Reasoning -- Understand why specific tools are called -- Group tool calls by context (e.g., "researching Instagram profile", "monitoring for new posts") -- Create dashboards showing tool calls with reasoning/context -- Group by MCP session ID for full interaction flows - ## Implementation Architecture ### Tool Call Flow @@ -305,7 +289,6 @@ const server = new ActorsMcpServer({ - `transport_type`: 'stdio', 'http', 'sse', or empty string - `server_version`: From `getPackageVersion()` (package.json version) - `tool_name`: Actor full name or internal tool name - - `reason`: Extracted from tool arguments if provided, otherwise empty string - Logs full telemetry payload before sending (debug level) - Calls `trackToolCall()` with userId, telemetry environment, and properties @@ -471,7 +454,7 @@ The telemetry infrastructure is integrated into the remote server that hosts the From `CallToolRequestSchema` handler in `src/mcp/server.ts` (line 568+): - `name`: Tool name (may have 'local__' prefix that is stripped) -- `args`: Validated input arguments (includes `reason` field when implemented) +- `args`: Validated input arguments - `apifyToken`: Apify API token (may be null in Skyfire mode) - `userRentedActorIds`: List of rented actor IDs - `progressToken`: Optional progress tracking token @@ -582,14 +565,6 @@ Transport type is now passed via `ActorsMcpServerOptions`: - **Legacy SSE transport**: Extracts session ID from URL query parameters via `getURLSessionID()` - Session ID injected into tool call request params for telemetry tracking - Supports cross-instance session correlation in distributed deployments -- **Reason field implementation** βœ… - - Dynamically added to all tool input schemas when telemetry is enabled (`telemetryEnabled: true`) - - Optional string field with title "Reason" and guidance description - - Extracted from tool arguments during tool call: `(args as Record).reason?.toString() || ''` - - Not added when telemetry is disabled (`telemetryEnabled: false`) (reduces schema overhead for tests) - - All AJV validators updated with `additionalProperties: true` to accept the field - - New tests verify: reason field presence/absence based on telemetry setting - - Works with `upsertTools()` conditional modification logic alongside Skyfire mode #### πŸ”² Not Yet Implemented (TODOs) - Implement anonymousId tracking for device/session identification @@ -614,12 +589,6 @@ Transport type is now passed via `ActorsMcpServerOptions`: - Unauthenticated scenarios (future MCP documentation tools feature) - If token is invalid or user fetch fails -### Tool Input Schema Enhancement -- Currently: reason field is always empty string -- TODO: Add optional `reason` field to all tool input schemas -- LLMs will fill in reasoning for why they called the tool -- Enables dashboard and analytics on tool call context - ### Version Management - Server version is dynamically read from package.json at runtime - Function: `getPackageVersion()` in `src/utils/version.ts` @@ -637,14 +606,12 @@ All telemetry operations emit debug logs including: - transport_type ('stdio', 'http', 'sse', or empty string) - server_version (from package.json or 'unknown') - tool_name (actor full name or internal tool name) - - reason (empty string) - Enable with `DEBUG=*` or `LOG_LEVEL=debug` to see telemetry details. Example debug output: ``` Telemetry: fetched user info { userId: 'user-123', userFound: true } -Telemetry: tracking tool call { app: 'mcp_server', mcp_client: 'Claude Desktop', transport_type: 'stdio', server_version: '0.5.3', tool_name: 'apify/instagram-scraper', reason: '' } +Telemetry: tracking tool call { app: 'mcp_server', mcp_client: 'Claude Desktop', transport_type: 'stdio', server_version: '0.5.3', tool_name: 'apify/instagram-scraper' } ``` ### Future Enhancements diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 30f29893..683e2be9 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -382,22 +382,6 @@ export class ActorsMcpServer { modified = true; } - // Handle telemetry modifications - add reason field to all tools when telemetry is enabled - if (isTelemetryEnabled) { - if (clonedWrap.inputSchema && 'properties' in clonedWrap.inputSchema) { - const props = clonedWrap.inputSchema.properties as Record; - if (!props.reason) { - props.reason = { - type: 'string', - title: 'Reason', - description: 'A brief explanation of why this tool is being called and what it will help you accomplish. ' - + 'Keep it concise and do not include any personal identifiable information (PII) or sensitive data.', - }; - } - } - modified = true; - } - // Store the cloned and modified tool only if modifications were made this.tools.set(clonedWrap.name, modified ? clonedWrap : wrap); } @@ -687,9 +671,6 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool log.debug('Telemetry: no API token provided'); } - // Extract reason from tool arguments if provided - const reason = (args as Record).reason?.toString() || ''; - const capabilities = this.options.initializeRequestData?.params?.capabilities; const params = this.options.initializeRequestData?.params as InitializeRequest['params']; telemetryData = { @@ -702,7 +683,6 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool mcp_session_id: mcpSessionId || '', transport_type: this.options.transportType || '', tool_name: toolFullName, - reason, tool_status: 'success', // Will be updated in finally tool_exec_time_ms: 0, // Will be calculated in finally tool_call_number: toolCallNumber, diff --git a/src/tools/build.ts b/src/tools/build.ts index ee8792fb..3687df29 100644 --- a/src/tools/build.ts +++ b/src/tools/build.ts @@ -124,10 +124,7 @@ export const actorDefinitionTool: ToolEntry = { + 'Get details for an Actor with with Actor ID or Actor full name, i.e. username/name.' + `Limit the length of the README if needed.`, inputSchema: zodToJsonSchema(getActorDefinitionArgsSchema) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(getActorDefinitionArgsSchema), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(getActorDefinitionArgsSchema)), annotations: { title: 'Get Actor definition', readOnlyHint: true, diff --git a/src/tools/dataset.ts b/src/tools/dataset.ts index ced87429..b83ecefb 100644 --- a/src/tools/dataset.ts +++ b/src/tools/dataset.ts @@ -56,10 +56,7 @@ USAGE EXAMPLES: - user_input: Show info for dataset xyz123 - user_input: What fields does username~my-dataset have?`, inputSchema: zodToJsonSchema(getDatasetArgs) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(getDatasetArgs), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(getDatasetArgs)), annotations: { title: 'Get dataset', readOnlyHint: true, @@ -96,10 +93,7 @@ USAGE EXAMPLES: - user_input: Get first 100 items from dataset abd123 - user_input: Get only metadata.url and title from dataset username~my-dataset (flatten metadata)`, inputSchema: zodToJsonSchema(getDatasetItemsArgs) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(getDatasetItemsArgs), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(getDatasetItemsArgs)), annotations: { title: 'Get dataset items', readOnlyHint: true, @@ -163,10 +157,7 @@ USAGE EXAMPLES: - user_input: Generate schema for dataset 34das2 using 10 items - user_input: Show schema of username~my-dataset (clean items only)`, inputSchema: zodToJsonSchema(getDatasetSchemaArgs) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(getDatasetSchemaArgs), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(getDatasetSchemaArgs)), annotations: { title: 'Get dataset schema', readOnlyHint: true, diff --git a/src/tools/dataset_collection.ts b/src/tools/dataset_collection.ts index d15216f2..84ceb5fc 100644 --- a/src/tools/dataset_collection.ts +++ b/src/tools/dataset_collection.ts @@ -41,10 +41,7 @@ USAGE EXAMPLES: - user_input: List my last 10 datasets (newest first) - user_input: List unnamed datasets`, inputSchema: zodToJsonSchema(getUserDatasetsListArgs) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(getUserDatasetsListArgs), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(getUserDatasetsListArgs)), annotations: { title: 'Get user datasets list', readOnlyHint: true, diff --git a/src/tools/fetch-actor-details.ts b/src/tools/fetch-actor-details.ts index c3aa3872..97f09dab 100644 --- a/src/tools/fetch-actor-details.ts +++ b/src/tools/fetch-actor-details.ts @@ -29,10 +29,7 @@ USAGE EXAMPLES: - user_input: What is the input schema for apify/rag-web-browser? - user_input: What is the pricing for apify/instagram-scraper?`, inputSchema: zodToJsonSchema(fetchActorDetailsToolArgsSchema) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(fetchActorDetailsToolArgsSchema), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(fetchActorDetailsToolArgsSchema)), annotations: { title: 'Fetch Actor details', readOnlyHint: true, diff --git a/src/tools/fetch-apify-docs.ts b/src/tools/fetch-apify-docs.ts index a8f2bc8f..8b3c9729 100644 --- a/src/tools/fetch-apify-docs.ts +++ b/src/tools/fetch-apify-docs.ts @@ -28,10 +28,7 @@ USAGE EXAMPLES: - user_input: Fetch https://docs.apify.com/platform/actors/running#builds - user_input: Fetch https://docs.apify.com/academy`, inputSchema: zodToJsonSchema(fetchApifyDocsToolArgsSchema) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(fetchApifyDocsToolArgsSchema), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(fetchApifyDocsToolArgsSchema)), annotations: { title: 'Fetch Apify docs', readOnlyHint: true, diff --git a/src/tools/get-html-skeleton.ts b/src/tools/get-html-skeleton.ts index 5c766a43..76c1b661 100644 --- a/src/tools/get-html-skeleton.ts +++ b/src/tools/get-html-skeleton.ts @@ -51,10 +51,7 @@ USAGE EXAMPLES: - user_input: Get HTML skeleton for https://example.com - user_input: Get next chunk of HTML skeleton for https://example.com (chunk=2)`, inputSchema: zodToJsonSchema(getHtmlSkeletonArgs) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(getHtmlSkeletonArgs), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(getHtmlSkeletonArgs)), annotations: { title: 'Get HTML skeleton', readOnlyHint: true, diff --git a/src/tools/helpers.ts b/src/tools/helpers.ts index 8b02719e..a9d345f2 100644 --- a/src/tools/helpers.ts +++ b/src/tools/helpers.ts @@ -26,10 +26,7 @@ USAGE EXAMPLES: - user_input: Add apify/rag-web-browser as a tool - user_input: Add apify/instagram-scraper as a tool`, inputSchema: zodToJsonSchema(addToolArgsSchema) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(addToolArgsSchema), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(addToolArgsSchema)), annotations: { title: 'Add tool', openWorldHint: true, diff --git a/src/tools/key_value_store.ts b/src/tools/key_value_store.ts index 5fb7d757..260f6734 100644 --- a/src/tools/key_value_store.ts +++ b/src/tools/key_value_store.ts @@ -28,10 +28,7 @@ USAGE EXAMPLES: - user_input: Show info for key-value store username~my-store - user_input: Get details for store adb123`, inputSchema: zodToJsonSchema(getKeyValueStoreArgs) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(getKeyValueStoreArgs), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(getKeyValueStoreArgs)), annotations: { title: 'Get key-value store', readOnlyHint: true, @@ -76,10 +73,7 @@ USAGE EXAMPLES: - user_input: List first 100 keys in store username~my-store - user_input: Continue listing keys in store a123 from key data.json`, inputSchema: zodToJsonSchema(getKeyValueStoreKeysArgs) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(getKeyValueStoreKeysArgs), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(getKeyValueStoreKeysArgs)), annotations: { title: 'Get key-value store keys', readOnlyHint: true, @@ -122,10 +116,7 @@ USAGE EXAMPLES: - user_input: Get record INPUT from store abc123 - user_input: Get record data.json from store username~my-store`, inputSchema: zodToJsonSchema(getKeyValueStoreRecordArgs) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(getKeyValueStoreRecordArgs), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(getKeyValueStoreRecordArgs)), annotations: { title: 'Get key-value store record', readOnlyHint: true, diff --git a/src/tools/key_value_store_collection.ts b/src/tools/key_value_store_collection.ts index 3e6c748a..edf14ac6 100644 --- a/src/tools/key_value_store_collection.ts +++ b/src/tools/key_value_store_collection.ts @@ -41,10 +41,7 @@ USAGE EXAMPLES: - user_input: List my last 10 key-value stores (newest first) - user_input: List unnamed key-value stores`, inputSchema: zodToJsonSchema(getUserKeyValueStoresListArgs) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(getUserKeyValueStoresListArgs), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(getUserKeyValueStoresListArgs)), annotations: { title: 'Get user key-value stores list', readOnlyHint: true, diff --git a/src/tools/run.ts b/src/tools/run.ts index 17fbb20e..28a11564 100644 --- a/src/tools/run.ts +++ b/src/tools/run.ts @@ -35,10 +35,7 @@ USAGE EXAMPLES: - user_input: Show details of run y2h7sK3Wc - user_input: What is the datasetId for run y2h7sK3Wc?`, inputSchema: zodToJsonSchema(getActorRunArgs) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(getActorRunArgs), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(getActorRunArgs)), annotations: { title: 'Get Actor run', readOnlyHint: true, @@ -81,10 +78,7 @@ USAGE EXAMPLES: - user_input: Show last 20 lines of logs for run y2h7sK3Wc - user_input: Get logs for run y2h7sK3Wc`, inputSchema: zodToJsonSchema(GetRunLogArgs) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(GetRunLogArgs), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(GetRunLogArgs)), annotations: { title: 'Get Actor run log', readOnlyHint: true, @@ -118,10 +112,7 @@ USAGE EXAMPLES: - user_input: Abort run y2h7sK3Wc - user_input: Gracefully abort run y2h7sK3Wc`, inputSchema: zodToJsonSchema(abortRunArgs) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(abortRunArgs), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(abortRunArgs)), annotations: { title: 'Abort Actor run', openWorldHint: false, diff --git a/src/tools/run_collection.ts b/src/tools/run_collection.ts index 21358428..8f3d05d5 100644 --- a/src/tools/run_collection.ts +++ b/src/tools/run_collection.ts @@ -39,10 +39,7 @@ USAGE EXAMPLES: - user_input: List my last 10 runs (newest first) - user_input: Show only SUCCEEDED runs`, inputSchema: zodToJsonSchema(getUserRunsListArgs) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(getUserRunsListArgs), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(getUserRunsListArgs)), annotations: { title: 'Get user runs list', readOnlyHint: true, diff --git a/src/tools/search-apify-docs.ts b/src/tools/search-apify-docs.ts index 7f9066fc..38a21389 100644 --- a/src/tools/search-apify-docs.ts +++ b/src/tools/search-apify-docs.ts @@ -50,10 +50,7 @@ USAGE EXAMPLES: - query: How to define Actor input schema? - query: How scrape with Crawlee?`, inputSchema: zodToJsonSchema(searchApifyDocsToolArgsSchema) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(searchApifyDocsToolArgsSchema), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(searchApifyDocsToolArgsSchema)), annotations: { title: 'Search Apify docs', readOnlyHint: true, diff --git a/src/tools/store_collection.ts b/src/tools/store_collection.ts index 64c6b0b6..de35f412 100644 --- a/src/tools/store_collection.ts +++ b/src/tools/store_collection.ts @@ -112,10 +112,7 @@ Returns list of Actor cards with the following info: - **Rating:** Out of 5 (if available) `, inputSchema: zodToJsonSchema(searchActorsArgsSchema) as ToolInputSchema, - ajvValidate: ajv.compile({ - ...zodToJsonSchema(searchActorsArgsSchema), - additionalProperties: true, // Allow additional properties for telemetry reason field - }), + ajvValidate: ajv.compile(zodToJsonSchema(searchActorsArgsSchema)), annotations: { title: 'Search Actors', readOnlyHint: true, diff --git a/src/types.ts b/src/types.ts index 66417a7f..5301ea42 100644 --- a/src/types.ts +++ b/src/types.ts @@ -307,7 +307,6 @@ export interface ToolCallTelemetryProperties { mcp_session_id: string; transport_type: string; tool_name: string; - reason: string; tool_status: 'success' | 'failure' | 'cancelled'; tool_exec_time_ms: number; tool_call_number: number; diff --git a/tests/integration/suite.ts b/tests/integration/suite.ts index adcc314b..9a76ba2b 100644 --- a/tests/integration/suite.ts +++ b/tests/integration/suite.ts @@ -31,39 +31,6 @@ function expectToolNamesToContain(names: string[], toolNames: string[] = []) { toolNames.forEach((name) => expect(names).toContain(name)); } -/** - * Checks that no tools have a reason field in their input schema - */ -function expectReasonFieldAbsent(tools: { tools: { inputSchema?: unknown }[] }) { - for (const tool of tools.tools) { - if (tool.inputSchema && typeof tool.inputSchema === 'object' && 'properties' in tool.inputSchema) { - const props = tool.inputSchema.properties as Record; - expect(props.reason).toBeUndefined(); - } - } -} - -/** - * Checks that at least one tool has a reason field in its input schema - * @param validateType - If true, also validates that the reason field type is 'string' - */ -function expectReasonFieldPresent(tools: { tools: { inputSchema?: unknown }[] }, validateType = false) { - let reasonFieldFound = false; - for (const tool of tools.tools) { - if (tool.inputSchema && typeof tool.inputSchema === 'object' && 'properties' in tool.inputSchema) { - const props = tool.inputSchema.properties as Record; - if (props.reason !== undefined) { - reasonFieldFound = true; - if (validateType) { - const reasonField = props.reason as Record; - expect(reasonField.type).toBe('string'); - } - } - } - } - expect(reasonFieldFound).toBe(true); -} - // eslint-disable-next-line @typescript-eslint/no-explicit-any function extractJsonFromMarkdown(text: string): any { // Handle markdown code blocks like ```json @@ -1097,27 +1064,14 @@ export function createIntegrationTestsSuite( await client.close(); }); - it('should NOT include reason field in tools when telemetry is off', async () => { - client = await createClientFn({ telemetryEnabled: false }); - const tools = await client.listTools(); - expectReasonFieldAbsent(tools); - await client.close(); - }); - - it('should include reason field in tools when telemetry is enabled (dev)', async () => { - client = await createClientFn({ telemetryEnabled: true, telemetryEnv: 'dev' }); - const tools = await client.listTools(); - expectReasonFieldPresent(tools, true); - await client.close(); - }); - // Environment variable precedence tests it.runIf(options.transport === 'stdio')('should use TELEMETRY_ENABLED env var when CLI arg is not provided', async () => { // When useEnv=true, telemetryEnabled option translates to env.TELEMETRY_ENABLED in child process client = await createClientFn({ useEnv: true, telemetryEnabled: false }); const tools = await client.listTools(); - expectReasonFieldAbsent(tools); + // Verify tools are loaded correctly + expect(tools.tools.length).toBeGreaterThan(0); await client.close(); }); }); diff --git a/tests/unit/telemetry.test.ts b/tests/unit/telemetry.test.ts index 10aaed2b..3c56200b 100644 --- a/tests/unit/telemetry.test.ts +++ b/tests/unit/telemetry.test.ts @@ -28,7 +28,6 @@ describe('telemetry', () => { mcp_session_id: 'session-123', transport_type: 'stdio', tool_name: 'test-tool', - reason: 'test reason', tool_status: 'success' as const, tool_exec_time_ms: 100, tool_call_number: 1, @@ -49,7 +48,6 @@ describe('telemetry', () => { mcp_session_id: 'session-123', transport_type: 'stdio', tool_name: 'test-tool', - reason: 'test reason', tool_status: 'success', tool_exec_time_ms: 100, tool_call_number: 1, From a5fd7351877a2470c1f64f6bd97691effaa6598b Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Thu, 20 Nov 2025 15:09:56 +0100 Subject: [PATCH 17/46] fix: change tool status to match Actor run status --- src/mcp/server.ts | 8 ++++---- src/types.ts | 2 +- tests/unit/telemetry.test.ts | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 683e2be9..3433c254 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -683,14 +683,14 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool mcp_session_id: mcpSessionId || '', transport_type: this.options.transportType || '', tool_name: toolFullName, - tool_status: 'success', // Will be updated in finally + tool_status: 'succeeded', // Will be updated in finally tool_exec_time_ms: 0, // Will be calculated in finally tool_call_number: toolCallNumber, }; } const startTime = Date.now(); - let toolStatus: 'success' | 'failure' | 'cancelled' = 'success'; + let toolStatus: 'succeeded' | 'failed' | 'aborted' = 'succeeded'; try { // Handle internal tool @@ -794,7 +794,7 @@ Please verify the server URL is correct and accessible, and ensure you have a va ); if (!callResult) { - toolStatus = 'cancelled'; + toolStatus = 'aborted'; // Receivers of cancellation notifications SHOULD NOT send a response for the cancelled request // https://modelcontextprotocol.io/specification/2025-06-18/basic/utilities/cancellation#behavior-requirements return { }; @@ -809,7 +809,7 @@ Please verify the server URL is correct and accessible, and ensure you have a va } } } catch (error) { - toolStatus = extra.signal?.aborted ? 'cancelled' : 'failure'; + toolStatus = extra.signal?.aborted ? 'aborted' : 'failed'; logHttpError(error, 'Error occurred while calling tool', { toolName: name }); const errorMessage = (error instanceof Error) ? error.message : 'Unknown error'; return buildMCPResponse([ diff --git a/src/types.ts b/src/types.ts index 5301ea42..a9e9b596 100644 --- a/src/types.ts +++ b/src/types.ts @@ -307,7 +307,7 @@ export interface ToolCallTelemetryProperties { mcp_session_id: string; transport_type: string; tool_name: string; - tool_status: 'success' | 'failure' | 'cancelled'; + tool_status: 'succeeded' | 'failed' | 'aborted'; tool_exec_time_ms: number; tool_call_number: number; } diff --git a/tests/unit/telemetry.test.ts b/tests/unit/telemetry.test.ts index 3c56200b..32a75889 100644 --- a/tests/unit/telemetry.test.ts +++ b/tests/unit/telemetry.test.ts @@ -28,7 +28,7 @@ describe('telemetry', () => { mcp_session_id: 'session-123', transport_type: 'stdio', tool_name: 'test-tool', - tool_status: 'success' as const, + tool_status: 'succeeded' as const, tool_exec_time_ms: 100, tool_call_number: 1, }; @@ -48,7 +48,7 @@ describe('telemetry', () => { mcp_session_id: 'session-123', transport_type: 'stdio', tool_name: 'test-tool', - tool_status: 'success', + tool_status: 'succeeded', tool_exec_time_ms: 100, tool_call_number: 1, }, From a551686caf1688bd9c8aae849afe99de0ef8cb9a Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Thu, 20 Nov 2025 15:29:37 +0100 Subject: [PATCH 18/46] fix: change event name --- src/telemetry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/telemetry.ts b/src/telemetry.ts index 696d2533..208923cf 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -7,7 +7,7 @@ const DEV_WRITE_KEY = '9rPHlMtxX8FJhilGEwkfUoZ0uzWxnzcT'; const PROD_WRITE_KEY = 'cOkp5EIJaN69gYaN8bcp7KtaD0fGABwJ'; const SEGMENT_EVENTS = { - TOOL_CALL: 'MCP tool call', + TOOL_CALL: 'MCP Tool Call', }; /** From 0bf506b3b51772f80e500ba50ef250f4cb2cc393 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Thu, 20 Nov 2025 15:33:38 +0100 Subject: [PATCH 19/46] fix: tests --- tests/unit/telemetry.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/telemetry.test.ts b/tests/unit/telemetry.test.ts index 32a75889..efe44009 100644 --- a/tests/unit/telemetry.test.ts +++ b/tests/unit/telemetry.test.ts @@ -37,7 +37,7 @@ describe('telemetry', () => { expect(mockTrack).toHaveBeenCalledWith({ userId: 'test-user-123', - event: 'MCP tool call', + event: 'MCP Tool Call', properties: { app_name: 'apify-mcp-server', app_version: '0.5.6', From 2004c6f9ed582b50b301c9b0fb54ebe69969974d Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Thu, 20 Nov 2025 15:41:00 +0100 Subject: [PATCH 20/46] fix: remove TELEMETRY.md --- mds/TELEMETRY.md | 700 ----------------------------------------------- 1 file changed, 700 deletions(-) delete mode 100644 mds/TELEMETRY.md diff --git a/mds/TELEMETRY.md b/mds/TELEMETRY.md deleted file mode 100644 index 8312bd08..00000000 --- a/mds/TELEMETRY.md +++ /dev/null @@ -1,700 +0,0 @@ -# Telemetry Implementation Plan - -## Overview - -This document outlines the implementation plan for analytics tracking in the Apify MCP Server using Segment. The goal is to track all tool calls to understand user behavior, tool usage patterns, and MCP client preferences. - -**Note:** This document is intended for consumers of `ActorsMcpServer` in other repositories. It describes the telemetry API and how to configure it. - -## Quick Reference - -### ActorsMcpServerOptions - -```typescript -interface ActorsMcpServerOptions { - telemetryEnabled?: boolean; // Default: true (enabled) - telemetryEnv?: 'dev' | 'prod'; // Default: 'prod' - transportType?: 'stdio' | 'http' | 'sse'; -} -``` - -### Default Behavior -- **Telemetry**: Enabled by default (`telemetryEnabled: true`) -- **Environment**: Production by default (`telemetryEnv: 'prod'`) - -### Configuration Precedence - -Telemetry configuration can be set via multiple methods with the following precedence (highest to lowest): - -1. **CLI arguments** (for stdio) or **URL query parameters** (for remote server) -2. **Environment variables** (`TELEMETRY_ENABLED`, `TELEMETRY_ENV`) -3. **Defaults** (`telemetryEnabled: true`, `telemetryEnv: 'prod'`) - -#### Environment Variables - -- `TELEMETRY_ENABLED`: Set to `true`, `1`, `false`, or `0` to enable/disable telemetry -- `TELEMETRY_ENV`: Set to `prod` or `dev` to specify the telemetry environment (only used when telemetry is enabled) - -### Usage Examples - -```typescript -// Enable telemetry with production environment (default) -const server = new ActorsMcpServer({ - telemetryEnabled: true, // or omit (defaults to true) - telemetryEnv: 'prod', // or omit (defaults to 'prod') - transportType: 'stdio', -}); - -// Enable telemetry with development environment (for debugging) -const server = new ActorsMcpServer({ - telemetryEnabled: true, - telemetryEnv: 'dev', - transportType: 'http', -}); - -// Disable telemetry -const server = new ActorsMcpServer({ - telemetryEnabled: false, - transportType: 'sse', -}); -``` - -## Data to Be Collected - -### Required Fields per Tool Call - -```json -{ - "userId": "APIFY_USER_ID", - "event": "MCP Tool Call", - "properties": { - "app": "mcp_server", - "mcp_client": "Claude Desktop", - "transport_type": "stdio|http|sse", - "server_version": "VERSION", - "tool_name": "apify/instagram-scraper" - }, - "timestamp": "ISO 8601 TIMESTAMP" -} -``` - -### Data Description - -- **userId**: Apify user ID from authenticated token - - Extracted from Apify API `/v2/users/me` endpoint using the token from `APIFY_TOKEN` env var or `~/.apify/auth.json` - - Cached in memory using SHA-256 hashed token as key (prevents storing raw tokens) - - Falls back to empty string if token is unavailable or API call fails - - Used to identify frequent Apify MCP server users and their use cases - - In Segment: Falls back to 'anonymous' if userId is empty string - -- **event**: "MCP Tool Call" - Event name for all tool calls - -- **MCP Client Name**: Which MCP client is being used (Claude Desktop, Cline, etc.) - - Extracted from `initializeRequestData.params.clientInfo.name` - - Falls back to 'unknown' if client name is unavailable - - Helps understand client distribution and preferences - - Informs which MCP spec features are most important - - Reference: https://modelcontextprotocol.io/clients - -- **Transport Type**: How server is accessed - - `stdio`: Local/direct stdio connection (from `src/stdio.ts` entry point) - - `http`: Remote HTTP streamable connection (from `src/actor/server.ts` with Streamable HTTP transport) - - `sse`: Remote Server-Sent Events (SSE) connection (from `src/actor/server.ts` with SSE transport) - - Passed via `ActorsMcpServerOptions.transportType` - - Differentiates between local and remote MCP server instances and transport types - -- **Server Version**: Apify MCP server version - - Dynamically read from package.json via `getPackageVersion()` function in `src/const.ts` - - Currently: '0.5.1' (from package.json) - - Automatically stays in sync with package.json version - - Safe fallback: '0.5.1' if package.json cannot be read - -- **Tool Name**: Name of the tool being called - - Critical for tool usage analytics - - Examples: `search-actors`, `apify/instagram-scraper`, `call-actor` - - For actor tools: uses full actor name (e.g., 'apify/instagram-scraper') - - For internal tools: uses tool name - -- **Timestamp**: When tool call occurred - - Handled automatically by Segment SDK - -## Analytics Use Cases - -### 1. Tool Usage Distribution -- Which tools are used most frequently? -- Which tools are rarely/never used? -- Create tool usage distribution charts -- Better than Prometheus counters (more reliable per Reddit discussions) - -### 2. Time-Series Tracking -- Tool call frequency over time -- Total tool calls per day/week/month -- Identify trends and peak usage periods - -### 3. MCP Client Distribution -- Which MCP clients are most popular? -- Which features of MCP spec are most relied upon? -- Identify implementation priorities based on client needs - -### 4. User Segmentation -- Identify frequent MCP server users -- Track use cases per user/organization -- Understand different user archetypes - -## Implementation Architecture - -### Tool Call Flow - -``` -Tool Call Request - ↓ -[CallToolRequestSchema Handler in src/mcp/server.ts] - ↓ -Tool Validation - ↓ -[TELEMETRY] Extract userId from token - ↓ -[TELEMETRY] Log debug info - ↓ -Tool Execution - ↓ -Return Response -``` - -### Telemetry Module Structure - -**File**: `src/telemetry.ts` - -- **Singleton Pattern**: Map-based singleton clients per environment - - Ensures only one Segment Analytics client per environment (`dev` or `prod`) - - Safe for multiple `ActorsMcpServer` instances to share the same client - - Lazy initialization on first `trackToolCall()` call - -- **Functions**: - - `getOrInitAnalyticsClient(env: 'dev' | 'prod'): Analytics` - - Gets or initializes the client for the specified environment - - Returns singleton instance - - Never called directly from server code - - - `trackToolCall(userId: string, env: 'dev' | 'prod', properties: Record): void` - - Sends tool call event to Segment - - Lazily initializes client if needed - - Converts empty userId to 'anonymous' to comply with Segment API - -### User Cache Module - -**File**: `src/utils/user-cache.ts` - -- **Caching Strategy**: In-memory cache with SHA-256 token hashing - - Caches the full User object returned by Apify API - - Uses token hash as cache key (not the raw token) - - Prevents repeated API calls for the same token - - Thread-safe Map-based storage - -- **Functions**: - - `getUserIdFromToken(token: string, apifyClient: ApifyClient): Promise` - - Fetches user info from `/v2/users/me` endpoint - - Returns cached User object if available - - Returns null if user not found or API call fails - - Hashes token before using as cache key - - Full User object is cached (contains id, username, email, etc.) - -- **Type Definition**: - - `CachedUserInfo` is a type alias for the User object returned by ApifyClient - - Inferred from `Awaited['get']>>` - - Provides type safety while staying DRY (no duplicate interface) - -### Server Integration - -**File**: `src/mcp/server.ts` - -#### ActorsMcpServerOptions Interface - -When creating an `ActorsMcpServer` instance, use the following options: - -```typescript -interface ActorsMcpServerOptions { - telemetryEnabled?: boolean; // Default: true - telemetryEnv?: 'dev' | 'prod'; // Default: 'prod' - transportType?: 'stdio' | 'http' | 'sse'; - // ... other options -} -``` - -**Usage Examples:** - -```typescript -// Enable telemetry with production environment (default) -const server = new ActorsMcpServer({ - telemetryEnabled: true, // or omit (defaults to true) - telemetryEnv: 'prod', // or omit (defaults to 'prod') - transportType: 'stdio', -}); - -// Enable telemetry with development environment (for debugging) -const server = new ActorsMcpServer({ - telemetryEnabled: true, - telemetryEnv: 'dev', - transportType: 'http', -}); - -// Disable telemetry -const server = new ActorsMcpServer({ - telemetryEnabled: false, - transportType: 'sse', -}); -``` - -- **ActorsMcpServerOptions Interface Details**: - - Added `telemetryEnabled?: boolean` option - - `true` (default): Telemetry enabled - - `false`: Telemetry disabled - - If not explicitly set, reads from `TELEMETRY_ENABLED` env variable - - Defaults to `true` when not set - - Added `telemetryEnv?: 'dev' | 'prod'` option - - `'prod'` (default): Use production Segment write key - - `'dev'`: Use development Segment write key - - Only used when `telemetryEnabled` is `true` - - If not explicitly set, reads from `TELEMETRY_ENV` env variable - - Defaults to `'prod'` when not set - - Added `transportType?: 'stdio' | 'http' | 'sse'` option - - `'stdio'`: Direct/local stdio connection - - `'http'`: Remote HTTP streamable connection - - `'sse'`: Remote Server-Sent Events (SSE) connection - - Specifies how the server is being accessed - - Passed to telemetry for transport type tracking - - Added `initializeRequestData?: InitializeRequest` option (from MCP SDK) - - Contains client info like `clientInfo.name`, capabilities, etc. - - Injected via message interception or HTTP request body - -- **Constructor** (lines 100-120): - - If `telemetryEnabled` is not explicitly set, reads from `TELEMETRY_ENABLED` env variable - - Parses env var value: `'true'` or `'1'` = true, `'false'` or `'0'` = false - - If env var not set, defaults to `telemetryEnabled = true` - - If `telemetryEnv` is not explicitly set, reads from `TELEMETRY_ENV` env variable - - Validates env var value via `getTelemetryEnv()` (must be 'dev' or 'prod') - - If env var not set or invalid, defaults to `telemetryEnv = 'prod'` - - If `telemetryEnabled` is explicitly `true`, ensures `telemetryEnv` is set (defaults to 'prod') - - Supports environment-based telemetry control for hosted deployments - -- **Tool Call Handler** (lines 568-600, `CallToolRequestSchema`): - - After tool validation and before execution - - Extracts userId from token using `getUserIdFromToken()` - - Logs debug information about the operation: - - userId and whether user was found - - Token availability status - - Builds telemetry properties object with: - - `app`: 'mcp_server' (identifies this server) - - `mcp_client`: Client name from `initializeRequestData.params.clientInfo.name` or 'unknown' - - `transport_type`: 'stdio', 'http', 'sse', or empty string - - `server_version`: From `getPackageVersion()` (package.json version) - - `tool_name`: Actor full name or internal tool name - - Logs full telemetry payload before sending (debug level) - - Calls `trackToolCall()` with userId, telemetry environment, and properties - -**File**: `src/utils/version.ts` - -- **`getPackageVersion()` Function**: - - Dynamically reads version from package.json at runtime - - Used in telemetry to report current server version - - Safely falls back to null if file cannot be read - - Works in development and production environments (package.json included in npm files) - -**File**: `src/utils/user-cache.ts` - -- **`getUserIdFromToken(token: string, apifyClient: ApifyClient)` Function**: - - Fetches user info from `/v2/users/me` Apify API endpoint - - Caches full User object using token hash (SHA-256) as key - - Returns cached User object if available (prevents repeated API calls) - - Returns null if user not found or API call fails (safe fallback) - - Token is hashed before caching to avoid storing raw tokens in memory - -**File**: `src/stdio.ts` - -- **Token Resolution** (lines 128-139): - - First tries to read token from `APIFY_TOKEN` environment variable - - Falls back to reading from `~/.apify/auth.json` if env var not set - - Uses helper function `getTokenFromAuthFile()` to read auth file - - JSON file is parsed and token is extracted from `token` key - - Silently fails on file not found or parse errors (no error thrown) - -- **Server Initialization** (lines 156-161): - - Passes `transportType: 'stdio'` when creating ActorsMcpServer - - Passes telemetry options from CLI flags (via yargs): - - `--telemetry-enabled` (boolean, default: true) - documented for end users - - `--telemetry-env` ('prod'|'dev', default: 'prod') - hidden flag for debugging only - - CLI flags take precedence over environment variables (via yargs `.env()`) - - Environment variables `TELEMETRY_ENABLED` and `TELEMETRY_ENV` are supported as fallback - - Converts CLI flags to `telemetryEnabled` (boolean) and `telemetryEnv` ('dev'|'prod') - -- **Message Interception** (lines 162-176): - - Creates proxy for `transport.onmessage` to intercept MCP messages - - Captures initialize message (first message with `method: 'initialize'`) - - Extracts client information from initialize request data - - Updates mcpServer.options.initializeRequestData with captured data - - Comment explains this is a "hacky way to inject client information" - - Falls back to 'unknown' if client name not found in initialize data - -**File**: `src/telemetry.ts` - -- **`parseBooleanFromString(value: string | undefined | null)` Function**: - - Parses boolean values from environment variable strings - - Accepts `'true'`, `'1'` as `true` - - Accepts `'false'`, `'0'` as `false` - - Returns `undefined` for unrecognized values - - Used to parse `TELEMETRY_ENABLED` environment variable - -- **`getTelemetryEnv(env?: string | null)` Function**: - - Validates and normalizes telemetry environment value - - Accepts `'dev'` or `'prod'` - - Returns default (`'prod'`) for invalid or missing values - - Used to parse `TELEMETRY_ENV` environment variable - -- **`getOrInitAnalyticsClient(env: 'dev' | 'prod')` Function**: - - Singleton pattern ensures only one Segment Analytics client per environment - - Uses Map to store clients: `{ dev: Analytics, prod: Analytics }` - - Lazy initialization on first call - -- **`trackToolCall()` Function**: - - Takes userId (from user cache or empty string) - - Takes telemetry environment ('dev' or 'prod') - - Takes properties object with telemetry data - - Converts empty userId to 'anonymous' for Segment API compliance - - Sends event to Segment with event name: "MCP Tool Call" - -**File**: `package.json` - -- **Files Array**: - - Added `package.json` to `files` array - - Ensures package.json is included in npm publish - - Makes `getPackageVersion()` work in production environments - -### Remote Server Integration (apify-mcp-server-internal) - -The telemetry infrastructure is integrated into the remote server that hosts the MCP service at mcp.apify.com. - -**File**: `src/server/shared.ts` - -- **`injectRequestToolCallBodyParams()` Function**: - - Injects `apifyToken`, `userRentedActorIds`, and `mcpSessionId` into tool call request params - - Extracts session ID from two sources: - - `mcp-session-id` header (for Streamable HTTP transport) - - URL query parameters via `getURLSessionID()` (for legacy SSE transport) - - Enables telemetry to track tool calls across different transport types - - Provides fallback mechanism for session ID extraction - -**File**: `src/server/streamable.ts` - -- **Streamable HTTP Transport Handler**: - - Modern HTTP streaming transport for persistent sessions - - Session resumability support via `mcp-session-id` header - -- **`handleNewSession()` Handler**: - - Extracts `?telemetry-enabled` and `?telemetry-env` query parameters from request URL - - URL parameters take precedence over environment variables - - Falls back to `TELEMETRY_ENABLED` and `TELEMETRY_ENV` env vars if URL params not provided - - Converts parameters to options: - - `telemetryEnabled`: URL param > env var > default (true) - - `telemetryEnv`: URL param > env var > default ('prod') - - Passes to ActorsMcpServer constructor: - - `transportType: 'http'` - identifies remote HTTP streamable connection - - `telemetryEnabled` and `telemetryEnv` - per-session telemetry control - - `initializeRequestData: req.body` - client info from HTTP request - -- **`handleSessionRestore()` Handler**: - - Restores session from Redis state for resumable connections - - Same telemetry and transportType handling as handleNewSession() - - Allows clients to resume sessions with telemetry disabled - -**File**: `src/server/legacy-sse.ts` - -- **Legacy SSE Transport Handler**: - - Server-Sent Events transport (deprecated but still supported for backward compatibility) - - Provides session resumability for older clients - -- **`initMCPSession()` Handler** (lines 153-210): - - Extracts `?telemetry-enabled` and `?telemetry-env` query parameters from response URL - - URL parameters take precedence over environment variables - - Falls back to `TELEMETRY_ENABLED` and `TELEMETRY_ENV` env vars if URL params not provided - - Converts parameters to options: - - `telemetryEnabled`: URL param > env var > default (true) - - `telemetryEnv`: URL param > env var > default ('prod') - - Creates ActorsMcpServer with: - - `transportType: 'sse'` - identifies remote SSE connection - - `telemetryEnabled` and `telemetryEnv` - per-session telemetry control - - Message interception proxy (lines 203-210): - - Proxies `transport.onmessage` to capture MCP initialize message - - Extracts client information from initialize request data - - Updates mcpServer.options.initializeRequestData with captured data - - Calls original onmessage handler to continue processing - -- **Per-Session Control**: - - Both transports support `?telemetry-enabled=false` query parameter to disable telemetry - - Optional `?telemetry-env=dev` parameter to use development workspace (for debugging only) - - URL parameters take precedence over `TELEMETRY_ENABLED` and `TELEMETRY_ENV` environment variables - - Environment variables can be used as fallback when URL parameters are not provided - - Prevents test data from polluting production telemetry - - Example: `https://mcp.apify.com/?telemetry-enabled=false` for streamable - - Example: `https://mcp.apify.com/sse?telemetry-enabled=false` for SSE - - Example: `https://mcp.apify.com/?telemetry-env=dev` for debugging (uses dev workspace) - -**Test Configuration**: - -- **`test/integration/tests/server-streamable.test.ts`**: - - mcpUrl configured with `/?telemetry-enabled=false` query parameter - - Prevents integration tests from sending telemetry events - -- **`test/integration/tests/server-sse.test.ts`**: - - mcpUrl configured with `/sse?telemetry-enabled=false` query parameter - - Prevents integration tests from sending telemetry events - -## Data Flow - -### Available Information at Tool Call Time - -From `CallToolRequestSchema` handler in `src/mcp/server.ts` (line 568+): -- `name`: Tool name (may have 'local__' prefix that is stripped) -- `args`: Validated input arguments -- `apifyToken`: Apify API token (may be null in Skyfire mode) -- `userRentedActorIds`: List of rented actor IDs -- `progressToken`: Optional progress tracking token -- `meta`: Metadata including progressToken - -From `ActorsMcpServer` instance: -- `this.options.telemetryEnabled`: Boolean indicating if telemetry is enabled (default: true) -- `this.options.telemetryEnv`: Telemetry environment ('dev' or 'prod', default: 'prod') -- `this.options.transportType`: Transport type ('stdio', 'http', or 'sse') -- `this.options.initializeRequestData`: MCP client info and capabilities - - `params.clientInfo.name`: MCP client name (e.g., 'Claude Desktop', 'Cline') - - `params.capabilities`: Client capabilities - - `params.protocolVersion`: MCP protocol version - -From `tool` entry: -- `tool.type`: Tool type ('internal', 'actor', 'actor-mcp') -- `tool.tool.name`: Tool name (internal) -- For actor tools: `actorFullName` (e.g., 'apify/instagram-scraper') - -From `src/utils/version.ts`: -- `getPackageVersion()`: Current server version from package.json - -### Token Resolution Flow - -``` -Tool Call Request - ↓ -APIFY_TOKEN env var available? - β”œβ”€ YES β†’ Use env var - └─ NO β†’ Check ~/.apify/auth.json - β”œβ”€ YES β†’ Read and parse JSON - β”‚ Extract 'token' field - β”‚ Use that token - └─ NO β†’ No token (null) - ↓ -Token available? - β”œβ”€ YES β†’ Create ApifyClient with token - β”‚ Call getUserIdFromToken() - β”‚ Return cached or fetched User object - └─ NO β†’ userId stays empty string - ↓ -Track telemetry with userId -``` - -### Transport Type Detection - -Transport type is now passed via `ActorsMcpServerOptions`: -- **Stdio** (Local): When using `src/stdio.ts` entry point - - Passes `transportType: 'stdio'` when creating ActorsMcpServer - - Passes `telemetryEnabled` and `telemetryEnv` from CLI flags - - Message interception proxy captures initialize request data from MCP protocol - - Example: `npx @apify/actors-mcp-server --telemetry-enabled=false` - - Example: `npx @apify/actors-mcp-server --telemetry-env=dev` (for debugging) - -- **Streamable HTTP** (Remote): When using `src/actor/server.ts` with Streamable HTTP transport - - Passes `transportType: 'http'` when creating ActorsMcpServer - - Extracts `?telemetry-enabled` and `?telemetry-env` query parameters from URL - - Client info available via `req.body` (InitializeRequest passed as initializeRequestData) - - Connection: `https://mcp.apify.com/?telemetry-enabled=false` - - Connection: `https://mcp.apify.com/?telemetry-env=dev` (for debugging) - -- **Legacy SSE** (Remote): When using `src/actor/server.ts` with SSE transport - - Passes `transportType: 'sse'` when creating ActorsMcpServer - - Extracts `?telemetry-enabled` and `?telemetry-env` query parameters from URL - - Message interception proxy captures initialize request data from MCP JSON-RPC messages - - Connection: `https://mcp.apify.com/sse?telemetry-enabled=false` - - Connection: `https://mcp.apify.com/sse?telemetry-env=dev` (for debugging) - -## Implementation Notes - -### Current Implementation Status - -#### βœ… Completed -- Telemetry module with singleton Segment clients per environment -- Tool call tracking in `CallToolRequestSchema` handler at line 568 of `src/mcp/server.ts` -- Dynamic version reading from package.json via `getPackageVersion()` function in `src/utils/version.ts` -- Transport type option in ActorsMcpServerOptions interface -- Stdio transport passing `transportType: 'stdio'` and telemetry CLI flags in `src/stdio.ts` -- package.json included in npm build files -- User cache module with token hashing and in-memory caching in `src/utils/user-cache.ts` -- Token resolution from env var and ~/.apify/auth.json file in `src/stdio.ts` -- Debug logging for telemetry operations in tool call handler -- Full User object caching (not custom wrapper interface) -- Message interception proxy to capture initialize request data in stdio (`src/stdio.ts`) -- Streamable HTTP transport with telemetry query parameter support (`src/actor/server.ts`) - - Extracts `?telemetry-enabled` and `?telemetry-env` query params - - Falls back to `TELEMETRY_ENABLED` and `TELEMETRY_ENV` env vars when URL params not provided - - Passes `transportType: 'http'` and telemetry options to ActorsMcpServer -- Legacy SSE transport with telemetry query parameter support (`src/actor/server.ts`) - - Extracts `?telemetry-enabled` and `?telemetry-env` query params - - Falls back to `TELEMETRY_ENABLED` and `TELEMETRY_ENV` env vars when URL params not provided - - Message interception proxy to capture initialize request data from MCP protocol - - Passes `transportType: 'sse'` and telemetry options -- Environment variable support for telemetry configuration - - `TELEMETRY_ENABLED`: Set to `'true'`, `'1'`, `'false'`, or `'0'` to enable/disable telemetry - - `TELEMETRY_ENV`: Set to `'prod'` or `'dev'` to specify telemetry environment - - Used as fallback when CLI/URL parameters are not provided - - Precedence: CLI/URL params > env vars > defaults -- Test configuration to prevent telemetry pollution - - `test/integration/tests/server-streamable.test.ts`: Uses `/?telemetry-enabled=false` - - `test/integration/tests/server-sse.test.ts`: Uses `/sse?telemetry-enabled=false` -- MCP session ID tracking and injection - - **Stdio transport** (`src/stdio.ts`): Manually generates UUID4 session ID using `randomUUID()` from `node:crypto` module - - Generated at startup (line 160 in stdio.ts) - - Represents a single session interaction since stdio doesn't have built-in session IDs - - Injected into all tool call messages via message interception proxy (lines 162-176 in stdio.ts) - - **Streamable HTTP transport**: Extracts `mcp-session-id` header from request - - **Legacy SSE transport**: Extracts session ID from URL query parameters via `getURLSessionID()` - - Session ID injected into tool call request params for telemetry tracking - - Supports cross-instance session correlation in distributed deployments - -#### πŸ”² Not Yet Implemented (TODOs) -- Implement anonymousId tracking for device/session identification - -### Multi-Server Environment -- Multiple `ActorsMcpServer` instances may run simultaneously - - For Stdio (local): Each connection gets its own server instance - - For Streamable HTTP (remote): Sessions stored in memory and Redis - - For Legacy SSE (remote): Sessions stored in memory and Redis -- Telemetry clients are shared via singleton Map pattern per environment -- User cache is global (shared across all server instances) -- Session data is stored in Redis for cross-instance resumability -- Per-session telemetry control via `?telemetry-enabled=false` query parameter prevents test pollution - -### User Authentication -- Apify token extracted from `APIFY_TOKEN` env var first -- Falls back to `~/.apify/auth.json` if env var not set -- Token is hashed before caching (SHA-256) -- User ID cached using token hash as key -- May be empty in: - - Skyfire payment mode (uses `skyfire-pay-id` instead) - - Unauthenticated scenarios (future MCP documentation tools feature) - - If token is invalid or user fetch fails - -### Version Management -- Server version is dynamically read from package.json at runtime -- Function: `getPackageVersion()` in `src/utils/version.ts` -- Automatically stays in sync with package.json version -- Works in development, production, and packaged environments -- Fallback: null if package.json cannot be read (logged as 'unknown' in telemetry) - -### Debug Logging - -All telemetry operations emit debug logs including: -- User info fetching: `userId`, `userFound` flag, token availability status -- Full telemetry payload before sending: - - app ('mcp_server') - - mcp_client (client name from initialize data or 'unknown') - - transport_type ('stdio', 'http', 'sse', or empty string) - - server_version (from package.json or 'unknown') - - tool_name (actor full name or internal tool name) -Enable with `DEBUG=*` or `LOG_LEVEL=debug` to see telemetry details. - -Example debug output: -``` -Telemetry: fetched user info { userId: 'user-123', userFound: true } -Telemetry: tracking tool call { app: 'mcp_server', mcp_client: 'Claude Desktop', transport_type: 'stdio', server_version: '0.5.3', tool_name: 'apify/instagram-scraper' } -``` - -### Future Enhancements - -1. **Device ID / Anonymous ID** - - Implement device ID tracking for session correlation - - Track unauthenticated users via anonymousId - - Link userId and anonymousId when user authenticates - -2. **Session Tracking** βœ… Implemented - - **Stdio** (`src/stdio.ts`): Manually generates UUID4 session ID using `randomUUID()` from `node:crypto` - - Generated at startup (line 160 in stdio.ts) to represent a single session interaction - - Since stdio doesn't have built-in session IDs, UUID4 is created for each stdio connection - - Injected into all tool call messages via message interception proxy (lines 162-176) - - Allows correlating multiple tool calls within the same session - - **Streamable HTTP**: Extracts `mcp-session-id` header from request - - Session ID extracted from request header and injected into tool call params - - **Legacy SSE**: Extracts session ID from URL query parameters - - Falls back to URL extraction when header not available - - Enables session tracking across distributed server instances - - Allows correlating multiple tool calls to a single user session - - Supports session-level analytics and debugging - -3. **Performance Metrics** - - Track tool call duration - - Monitor error rates by tool - - Identify slow tools - -4. **Custom Dashboards** - - Tool call distribution over time - - MCP client adoption trends - - Tool reasoning/context browser - - User journey analysis - -## Segment Configuration - -### Write Keys -- **Development**: `9rPHlMtxX8FJhilGEwkfUoZ0uzWxnzcT` -- **Production**: `cOkp5EIJaN69gYaN8bcp7KtaD0fGABwJ` - -### Event Names -- `MCP Tool Call`: Fired every time a tool is called - -### Integration -- Segment SDK: `@segment/analytics-node` v2.3.0 -- Node.js requirement: 18+ -- Batching: Default 20 messages per batch (SDK configuration) - -## Testing & Validation - -1. **Dev Environment** - - Initialize server with `telemetryEnabled: true, telemetryEnv: 'dev'` - - Or use CLI: `--telemetry-env=dev` (hidden flag for debugging) - - Or use URL: `?telemetry-env=dev` - - Verify events appear in Segment dev workspace - -2. **Production** - - Initialize server with `telemetryEnabled: true, telemetryEnv: 'prod'` (default) - - Or use CLI: `--telemetry-enabled` (default: true) - - Monitor Segment prod workspace for events - -3. **No Telemetry** - - Initialize server with `telemetryEnabled: false` - - Or use CLI: `--telemetry-enabled=false` - - Or use URL: `?telemetry-enabled=false` - - Verify no tracking occurs - - Verify no errors from missing telemetry - -4. **Token Resolution** - - Test with `APIFY_TOKEN` env var set - - Test with only `~/.apify/auth.json` file - - Test with both set (env var should take precedence) - - Test with neither (should still work but no userId) - -5. **User Cache** - - Same token should return cached result (no API call) - - Different token should trigger new API call - - Invalid token should return null safely - - Debug logs should show cache hits/misses - -## References - -- MCP Clients: https://modelcontextprotocol.io/clients -- mcpcat.io: Similar implementation with tool call reasoning -- Prometheus Discussion: https://www.reddit.com/r/PrometheusMonitoring/comments/1jyxnzv/prometheus_counters_very_unreliable_for_many/ -- Apify API: `/v2/users/me` endpoint for user info From 336b2460602cfb4f5695450a1d85eaa9d85c1216 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Thu, 20 Nov 2025 15:50:46 +0100 Subject: [PATCH 21/46] fix: add user-cache (TTL) --- src/const.ts | 2 ++ src/utils/user-cache.ts | 11 +++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/const.ts b/src/const.ts index 2d81515e..23eee824 100644 --- a/src/const.ts +++ b/src/const.ts @@ -76,6 +76,8 @@ export const GET_HTML_SKELETON_CACHE_TTL_SECS = 5 * 60; // 5 minutes export const GET_HTML_SKELETON_CACHE_MAX_SIZE = 200; export const MCP_SERVER_CACHE_MAX_SIZE = 500; export const MCP_SERVER_CACHE_TTL_SECS = 30 * 60; // 30 minutes +export const USER_CACHE_MAX_SIZE = 200; +export const USER_CACHE_TTL_SECS = 60 * 60; // 1 hour export const ACTOR_PRICING_MODEL = { /** Rental Actors */ diff --git a/src/utils/user-cache.ts b/src/utils/user-cache.ts index 6a9aeec2..60af751d 100644 --- a/src/utils/user-cache.ts +++ b/src/utils/user-cache.ts @@ -3,9 +3,11 @@ import { createHash } from 'node:crypto'; import type { User } from 'apify-client'; import type { ApifyClient } from '../apify-client.js'; +import { USER_CACHE_MAX_SIZE, USER_CACHE_TTL_SECS } from '../const.js'; +import { TTLLRUCache } from './ttl-lru.js'; -// Type for cached user info - stores the raw User object from API -const userCache = new Map(); +// LRU cache with TTL for user info - stores the raw User object from API +const userCache = new TTLLRUCache(USER_CACHE_MAX_SIZE, USER_CACHE_TTL_SECS); /** * Gets user info from token, using cache to avoid repeated API calls @@ -20,8 +22,9 @@ export async function getUserIdFromToken( const tokenHash = createHash('sha256').update(token).digest('hex'); // Check cache first - if (userCache.has(tokenHash)) { - return userCache.get(tokenHash)!; + const cachedUser = userCache.get(tokenHash); + if (cachedUser) { + return cachedUser; } // Fetch from API From 92948ed06d91cfb3996f1c24246945fb48ad3b62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Spilka?= Date: Fri, 21 Nov 2025 09:44:07 +0100 Subject: [PATCH 22/46] Apply suggestions from code review Co-authored-by: Tomas Katz --- src/const.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/const.ts b/src/const.ts index 23eee824..7ab5c7d8 100644 --- a/src/const.ts +++ b/src/const.ts @@ -108,11 +108,10 @@ export const PROGRESS_NOTIFICATION_INTERVAL_MS = 5_000; // 5 seconds export const APIFY_STORE_URL = 'https://apify.com'; // Telemetry -export type TelemetryEnv = 'dev' | 'prod'; - export const TELEMETRY_ENV = { DEV: 'dev', PROD: 'prod', } as const; +export type TelemetryEnv = (typeof TELEMETRY_ENV)[keyof typeof TELEMETRY_ENV]; export const DEFAULT_TELEMETRY_ENV: TelemetryEnv = TELEMETRY_ENV.PROD; From 4c9592fc3710bf744e7a2d6022b9d5c1bce4a31a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Spilka?= Date: Fri, 21 Nov 2025 22:04:39 +0100 Subject: [PATCH 23/46] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: JiΕ™Γ­ Moravčík --- src/utils/user-cache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/user-cache.ts b/src/utils/user-cache.ts index 60af751d..24b23d65 100644 --- a/src/utils/user-cache.ts +++ b/src/utils/user-cache.ts @@ -14,7 +14,7 @@ const userCache = new TTLLRUCache(USER_CACHE_MAX_SIZE, USER_CACHE_TTL_SECS * Token is hashed before caching to avoid storing raw tokens * Returns the full User object from API or null if not found */ -export async function getUserIdFromToken( +export async function getUserInfoFromTokenCached( token: string, apifyClient: ApifyClient, ): Promise { From 944875a5cffe2563bbb99ea45159b19bb2f2a49a Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Fri, 21 Nov 2025 12:15:53 +0100 Subject: [PATCH 24/46] fix: remove map of clients (one client per deployment) --- src/actor/server.ts | 8 -------- src/mcp/server.ts | 2 +- src/telemetry.ts | 23 ++++++++++------------- tests/helpers.ts | 5 +---- tests/unit/telemetry.test.ts | 3 +-- 5 files changed, 13 insertions(+), 28 deletions(-) diff --git a/src/actor/server.ts b/src/actor/server.ts index 89478bad..32e97121 100644 --- a/src/actor/server.ts +++ b/src/actor/server.ts @@ -13,9 +13,7 @@ import express from 'express'; import log from '@apify/log'; import { ApifyClient } from '../apify-client.js'; -import { type TelemetryEnv } from '../const.js'; import { ActorsMcpServer } from '../mcp/server.js'; -import { getTelemetryEnv } from '../telemetry.js'; import { getHelpMessage, HEADER_READINESS_PROBE, Routes, TransportType } from './const.js'; import { getActorRunData } from './utils.js'; @@ -83,15 +81,12 @@ export function createExpressApp( // Extract telemetry query parameters const urlParams = new URL(req.url, `http://${req.headers.host}`).searchParams; const telemetryEnabledParam = urlParams.get('telemetry-enabled'); - const telemetryEnvParam = urlParams.get('telemetry-env'); const telemetryEnabled = telemetryEnabledParam !== 'false'; // Default to true - const telemetryEnv: TelemetryEnv = getTelemetryEnv(telemetryEnvParam); const mcpServer = new ActorsMcpServer({ setupSigintHandler: false, transportType: 'sse', telemetryEnabled, - telemetryEnv, }); const transport = new SSEServerTransport(Routes.MESSAGE, res); @@ -179,16 +174,13 @@ export function createExpressApp( // Extract telemetry query parameters const urlParams = new URL(req.url, `http://${req.headers.host}`).searchParams; const telemetryEnabledParam = urlParams.get('telemetry-enabled'); - const telemetryEnvParam = urlParams.get('telemetry-env'); const telemetryEnabled = telemetryEnabledParam !== 'false'; // Default to true - const telemetryEnv: TelemetryEnv = getTelemetryEnv(telemetryEnvParam); const mcpServer = new ActorsMcpServer({ setupSigintHandler: false, initializeRequestData: req.body as InitializeRequest, transportType: 'http', telemetryEnabled, - telemetryEnv, }); // Load MCP server tools diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 3433c254..d049f2ed 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -822,7 +822,7 @@ Please verify the tool name, input parameters, and ensure all required resources const execTime = Date.now() - startTime; telemetryData.tool_status = toolStatus; telemetryData.tool_exec_time_ms = execTime; - trackToolCall(userId, getTelemetryEnv(this.options.telemetryEnv), telemetryData); + trackToolCall(userId, telemetryData); } } diff --git a/src/telemetry.ts b/src/telemetry.ts index 208923cf..20f84a46 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -17,38 +17,35 @@ export function getTelemetryEnv(env?: string | null): TelemetryEnv { return (env === TELEMETRY_ENV.DEV || env === TELEMETRY_ENV.PROD) ? env : DEFAULT_TELEMETRY_ENV; } -// Map to store singleton Segment Analytics clients per environment -const analyticsClients = new Map(); +// Single Segment Analytics client (environment determined by process.env.TELEMETRY_ENV) +let analyticsClient: Analytics | null = null; /** - * Gets or initializes a Segment Analytics client for the specified environment. - * This ensures that only one client is created per environment, even if multiple - * ActorsMcpServer instances are initialized with telemetry enabled. + * Gets or initializes the Segment Analytics client. + * The environment is determined by the TELEMETRY_ENV environment variable. * - * @param env - 'dev' for development, 'prod' for production * @returns Analytics client instance */ -export function getOrInitAnalyticsClient(env: TelemetryEnv): Analytics { - if (!analyticsClients.has(env)) { +export function getOrInitAnalyticsClient(): Analytics { + if (!analyticsClient) { + const env = getTelemetryEnv(process.env.TELEMETRY_ENV); const writeKey = env === TELEMETRY_ENV.PROD ? PROD_WRITE_KEY : DEV_WRITE_KEY; - analyticsClients.set(env, new Analytics({ writeKey })); + analyticsClient = new Analytics({ writeKey }); } - return analyticsClients.get(env)!; + return analyticsClient; } /** * Tracks a tool call event to Segment. * * @param userId - Apify user ID (TODO: extract from token when auth available) - * @param env - 'dev' for development, 'prod' for production * @param properties - Event properties for the tool call */ export function trackToolCall( userId: string, - env: TelemetryEnv, properties: ToolCallTelemetryProperties, ): void { - const client = getOrInitAnalyticsClient(env); + const client = getOrInitAnalyticsClient(); // TODO: Implement anonymousId tracking for device/session identification client.track({ diff --git a/tests/helpers.ts b/tests/helpers.ts index 441c8518..8f0191e4 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -24,7 +24,7 @@ function checkApifyToken(): void { } function appendSearchParams(url: URL, options?: McpClientOptions): void { - const { actors, enableAddingActors, tools, telemetryEnabled, telemetryEnv } = options || {}; + const { actors, enableAddingActors, tools, telemetryEnabled } = options || {}; if (actors !== undefined) { url.searchParams.append('actors', actors.join(',')); } @@ -38,9 +38,6 @@ function appendSearchParams(url: URL, options?: McpClientOptions): void { if (telemetryEnabled !== undefined) { url.searchParams.append('telemetry-enabled', telemetryEnabled.toString()); } - if (telemetryEnv !== undefined && telemetryEnabled !== false) { - url.searchParams.append('telemetry-env', telemetryEnv); - } } export async function createMcpSseClient( diff --git a/tests/unit/telemetry.test.ts b/tests/unit/telemetry.test.ts index efe44009..47bb483e 100644 --- a/tests/unit/telemetry.test.ts +++ b/tests/unit/telemetry.test.ts @@ -1,6 +1,5 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { TELEMETRY_ENV } from '../../src/const.js'; import { trackToolCall } from '../../src/telemetry.js'; // Mock the Segment Analytics client @@ -33,7 +32,7 @@ describe('telemetry', () => { tool_call_number: 1, }; - trackToolCall(userId, TELEMETRY_ENV.DEV, properties); + trackToolCall(userId, properties); expect(mockTrack).toHaveBeenCalledWith({ userId: 'test-user-123', From 3b14e56077453790f2aacf424be4450b2160a19f Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Fri, 21 Nov 2025 13:35:56 +0100 Subject: [PATCH 25/46] fix: remove _initializeRequestData from loadToolsFromUrl --- src/mcp/server.ts | 2 +- src/mcp/utils.ts | 6 ++---- src/utils/tools-loader.ts | 3 --- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index d049f2ed..ee388285 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -324,7 +324,7 @@ export class ActorsMcpServer { * Used primarily for SSE. */ public async loadToolsFromUrl(url: string, apifyClient: ApifyClient) { - const tools = await processParamsGetTools(url, apifyClient, this.options.initializeRequestData); + const tools = await processParamsGetTools(url, apifyClient); if (tools.length > 0) { log.debug('Loading tools from query parameters'); this.upsertTools(tools, false); diff --git a/src/mcp/utils.ts b/src/mcp/utils.ts index 2aa8764c..78a773b2 100644 --- a/src/mcp/utils.ts +++ b/src/mcp/utils.ts @@ -1,7 +1,6 @@ import { createHash } from 'node:crypto'; import { parse } from 'node:querystring'; -import type { InitializeRequest } from '@modelcontextprotocol/sdk/types.js'; import type { ApifyClient } from 'apify-client'; import { processInput } from '../input.js'; @@ -41,11 +40,10 @@ export function getProxyMCPServerToolName(url: string, toolName: string): string * If URL contains query parameter `actors`, return tools from Actors otherwise return null. * @param url The URL to process * @param apifyClient The Apify client instance - * @param initializeRequestData Optional initialize request data */ -export async function processParamsGetTools(url: string, apifyClient: ApifyClient, initializeRequestData?: InitializeRequest) { +export async function processParamsGetTools(url: string, apifyClient: ApifyClient) { const input = parseInputParamsFromUrl(url); - return await loadToolsFromInput(input, apifyClient, initializeRequestData); + return await loadToolsFromInput(input, apifyClient); } export function parseInputParamsFromUrl(url: string): Input { diff --git a/src/utils/tools-loader.ts b/src/utils/tools-loader.ts index 57e5b563..90ccac64 100644 --- a/src/utils/tools-loader.ts +++ b/src/utils/tools-loader.ts @@ -3,7 +3,6 @@ * This eliminates duplication between stdio.ts and processParamsGetTools. */ -import type { InitializeRequest } from '@modelcontextprotocol/sdk/types.js'; import type { ValidateFunction } from 'ajv'; import type { ApifyClient } from 'apify'; @@ -35,13 +34,11 @@ function getInternalToolByNameMap(): Map { * * @param input The processed Input object * @param apifyClient The Apify client instance - * @param _initializeRequestData Optional initialize request data * @returns An array of tool entries */ export async function loadToolsFromInput( input: Input, apifyClient: ApifyClient, - _initializeRequestData?: InitializeRequest, ): Promise { // Helpers for readability const normalizeSelectors = (value: Input['tools']): (string | ToolCategory)[] | undefined => { From 4cc26d969e1f3d63809bcba60f0cea49f3dbd582 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Fri, 21 Nov 2025 14:07:22 +0100 Subject: [PATCH 26/46] fix: in the ActorsMcpServer use nested telemetry object --- src/actor/server.ts | 8 ++++-- src/mcp/server.ts | 61 ++++++++++++++++++++++++++------------------- src/stdio.ts | 6 +++-- 3 files changed, 45 insertions(+), 30 deletions(-) diff --git a/src/actor/server.ts b/src/actor/server.ts index 32e97121..d3e42f00 100644 --- a/src/actor/server.ts +++ b/src/actor/server.ts @@ -86,7 +86,9 @@ export function createExpressApp( const mcpServer = new ActorsMcpServer({ setupSigintHandler: false, transportType: 'sse', - telemetryEnabled, + telemetry: { + enabled: telemetryEnabled, + }, }); const transport = new SSEServerTransport(Routes.MESSAGE, res); @@ -180,7 +182,9 @@ export function createExpressApp( setupSigintHandler: false, initializeRequestData: req.body as InitializeRequest, transportType: 'http', - telemetryEnabled, + telemetry: { + enabled: telemetryEnabled, + }, }); // Load MCP server tools diff --git a/src/mcp/server.ts b/src/mcp/server.ts index ee388285..4a37b8eb 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -75,16 +75,27 @@ interface ActorsMcpServerOptions { skyfireMode?: boolean; initializeRequestData?: InitializeRequest; /** - * Enable or disable telemetry tracking for tool calls. - * Defaults to true when not set. + * Telemetry configuration options. */ - telemetryEnabled?: boolean; - /** - * Telemetry environment when telemetry is enabled. - * - 'dev': Use development Segment write key - * - 'prod': Use production Segment write key (default) - */ - telemetryEnv?: TelemetryEnv; + telemetry?: { + /** + * Enable or disable telemetry tracking for tool calls. + * Defaults to true when not set. + */ + enabled?: boolean; + /** + * Telemetry environment when telemetry is enabled. + * - 'dev': Use development Segment write key + * - 'prod': Use production Segment write key (default) + */ + env?: TelemetryEnv; + /** + * Optional store for tool call counters. + * If not provided, uses in-memory storage (suitable for stdio). + * For distributed deployments (HTTP/SSE), provide a Redis-backed implementation. + */ + toolCallCountStore?: ToolCallCounterStore; + }; /** * Transport type for telemetry tracking. * - 'stdio': Direct/local stdio connection @@ -98,12 +109,6 @@ interface ActorsMcpServerOptions { * instead of APIFY_TOKEN environment variable, so it can be passed to the server */ token?: string; - /** - * Optional store for tool call counters. - * If not provided, uses in-memory storage (suitable for stdio). - * For distributed deployments (HTTP/SSE), provide a Redis-backed implementation. - */ - toolCallCounterStore?: ToolCallCounterStore; } /** @@ -161,9 +166,9 @@ export class ActorsMcpServer { * @returns Promise resolving to the new counter value (after increment) */ private async getAndIncrementToolCallCounter(sessionId: string): Promise { - if (this.options.toolCallCounterStore) { + if (this.options.telemetry?.toolCallCountStore) { // Use external store (Redis for HTTP/SSE) - return await this.options.toolCallCounterStore.getAndIncrement(sessionId); + return await this.options.telemetry.toolCallCountStore.getAndIncrement(sessionId); } // Use in-memory storage (for stdio) const current = this.sessionToolCallCounters.get(sessionId) || 0; @@ -176,19 +181,24 @@ export class ActorsMcpServer { * Telemetry configuration with precedence: explicit options > env vars > defaults */ private setupTelemetry() { - if (this.options.telemetryEnabled === undefined) { + // Initialize telemetry object if not present + if (!this.options.telemetry) { + this.options.telemetry = {}; + } + + if (this.options.telemetry.enabled === undefined) { // Check environment variable as fallback const envEnabled = parseBooleanFromString(process.env.TELEMETRY_ENABLED); - this.options.telemetryEnabled = envEnabled !== undefined ? envEnabled : true; + this.options.telemetry.enabled = envEnabled !== undefined ? envEnabled : true; } // Set telemetryEnv: explicit option > env var > default ('prod') const telemetryEnv = process.env.TELEMETRY_ENV; - const telemetryParam = this.options.telemetryEnv; - this.options.telemetryEnv = getTelemetryEnv(telemetryParam ?? telemetryEnv); + const telemetryParam = this.options.telemetry.env; + this.options.telemetry.env = getTelemetryEnv(telemetryParam ?? telemetryEnv); // If telemetry is enabled, ensure telemetryEnv is set - if (this.options.telemetryEnabled && this.options.telemetryEnv === undefined) { - this.options.telemetryEnv = getTelemetryEnv(undefined); + if (this.options.telemetry.enabled && this.options.telemetry.env === undefined) { + this.options.telemetry.env = getTelemetryEnv(undefined); } } @@ -353,9 +363,8 @@ export class ActorsMcpServer { * @returns Array of added/updated tool wrappers */ public upsertTools(tools: ToolEntry[], shouldNotifyToolsChangedHandler = false) { - const isTelemetryEnabled = this.options.telemetryEnabled === true; - if (this.options.skyfireMode || isTelemetryEnabled) { + if (this.options.skyfireMode) { for (const wrap of tools) { // Clone the tool before modifying it to avoid affecting shared objects const clonedWrap = cloneToolEntry(wrap); @@ -645,7 +654,7 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool let telemetryData: ToolCallTelemetryProperties | null = null; let userId = ''; - if (this.options.telemetryEnabled === true) { + if (this.options.telemetry?.enabled === true) { const toolFullName = tool.type === 'actor' ? tool.actorFullName : tool.name; // Get or increment tool call counter for this session diff --git a/src/stdio.ts b/src/stdio.ts index 0fdc4ab4..5672d926 100644 --- a/src/stdio.ts +++ b/src/stdio.ts @@ -155,8 +155,10 @@ if (!apifyToken) { async function main() { const mcpServer = new ActorsMcpServer({ transportType: 'stdio', - telemetryEnabled: argv.telemetryEnabled, - telemetryEnv: getTelemetryEnv(argv.telemetryEnv), + telemetry: { + enabled: argv.telemetryEnabled, + env: getTelemetryEnv(argv.telemetryEnv), + }, token: apifyToken, }); From ab7f161015d428a7251f94298bf210c5e6c1484d Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Fri, 21 Nov 2025 14:18:53 +0100 Subject: [PATCH 27/46] fix: in the ActorsMcpServer use nested telemetry object --- tests/helpers.ts | 28 +++++++++++++++------------- tests/integration/suite.ts | 4 ++-- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/tests/helpers.ts b/tests/helpers.ts index 8f0191e4..d324e96b 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -13,8 +13,10 @@ export interface McpClientOptions { tools?: (ToolCategory | string)[]; // Tool categories, specific tool or Actor names to include useEnv?: boolean; // Use environment variables instead of command line arguments (stdio only) clientName?: string; // Client name for identification - telemetryEnabled?: boolean; // Enable or disable telemetry (default: false for tests) - telemetryEnv?: TelemetryEnv; // Telemetry environment (default: 'prod', only used when telemetryEnabled is true) + telemetry?: { + enabled?: boolean; // Enable or disable telemetry (default: false for tests) + env?: TelemetryEnv; // Telemetry environment (default: 'prod', only used when telemetry.enabled is true) + }; } function checkApifyToken(): void { @@ -24,7 +26,7 @@ function checkApifyToken(): void { } function appendSearchParams(url: URL, options?: McpClientOptions): void { - const { actors, enableAddingActors, tools, telemetryEnabled } = options || {}; + const { actors, enableAddingActors, tools, telemetry } = options || {}; if (actors !== undefined) { url.searchParams.append('actors', actors.join(',')); } @@ -35,8 +37,8 @@ function appendSearchParams(url: URL, options?: McpClientOptions): void { url.searchParams.append('tools', tools.join(',')); } // Append telemetry parameters - if (telemetryEnabled !== undefined) { - url.searchParams.append('telemetry-enabled', telemetryEnabled.toString()); + if (telemetry?.enabled !== undefined) { + url.searchParams.append('telemetry-enabled', telemetry.enabled.toString()); } } @@ -100,7 +102,7 @@ export async function createMcpStdioClient( options?: McpClientOptions, ): Promise { checkApifyToken(); - const { actors, enableAddingActors, tools, useEnv, telemetryEnabled, telemetryEnv } = options || {}; + const { actors, enableAddingActors, tools, useEnv, telemetry } = options || {}; const args = ['dist/stdio.js']; const env: Record = { APIFY_TOKEN: process.env.APIFY_TOKEN as string, @@ -117,11 +119,11 @@ export async function createMcpStdioClient( if (tools !== undefined) { env.TOOLS = tools.join(','); } - if (telemetryEnabled !== undefined) { - env.TELEMETRY_ENABLED = telemetryEnabled.toString(); + if (telemetry?.enabled !== undefined) { + env.TELEMETRY_ENABLED = telemetry.enabled.toString(); } - if (telemetryEnv !== undefined) { - env.TELEMETRY_ENV = telemetryEnv; + if (telemetry?.env !== undefined) { + env.TELEMETRY_ENV = telemetry.env; } } else { // Use command line arguments as before @@ -134,11 +136,11 @@ export async function createMcpStdioClient( if (tools !== undefined) { args.push('--tools', tools.join(',')); } - if (telemetryEnabled === false) { + if (telemetry?.enabled === false) { args.push('--telemetry-enabled', 'false'); } - if (telemetryEnv !== undefined && telemetryEnabled !== false) { - args.push('--telemetry-env', telemetryEnv); + if (telemetry?.env !== undefined && telemetry?.enabled !== false) { + args.push('--telemetry-env', telemetry.env); } } diff --git a/tests/integration/suite.ts b/tests/integration/suite.ts index 9a76ba2b..7e295a3d 100644 --- a/tests/integration/suite.ts +++ b/tests/integration/suite.ts @@ -1066,8 +1066,8 @@ export function createIntegrationTestsSuite( // Environment variable precedence tests it.runIf(options.transport === 'stdio')('should use TELEMETRY_ENABLED env var when CLI arg is not provided', async () => { - // When useEnv=true, telemetryEnabled option translates to env.TELEMETRY_ENABLED in child process - client = await createClientFn({ useEnv: true, telemetryEnabled: false }); + // When useEnv=true, telemetry.enabled option translates to env.TELEMETRY_ENABLED in child process + client = await createClientFn({ useEnv: true, telemetry: { enabled: false } }); const tools = await client.listTools(); // Verify tools are loaded correctly From 9da99b0a91d8ee153be4289734e7bff338965afa Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Fri, 21 Nov 2025 14:35:10 +0100 Subject: [PATCH 28/46] fix: log errors in telemetry --- src/mcp/server.ts | 1 - src/telemetry.ts | 46 +++++++++++++++++++++++++++++++++------------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 4a37b8eb..87fe30ee 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -363,7 +363,6 @@ export class ActorsMcpServer { * @returns Array of added/updated tool wrappers */ public upsertTools(tools: ToolEntry[], shouldNotifyToolsChangedHandler = false) { - if (this.options.skyfireMode) { for (const wrap of tools) { // Clone the tool before modifying it to avoid affecting shared objects diff --git a/src/telemetry.ts b/src/telemetry.ts index 20f84a46..154056df 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -1,14 +1,17 @@ import { Analytics } from '@segment/analytics-node'; +import log from '@apify/log'; + import { DEFAULT_TELEMETRY_ENV, TELEMETRY_ENV, type TelemetryEnv } from './const.js'; import type { ToolCallTelemetryProperties } from './types.js'; const DEV_WRITE_KEY = '9rPHlMtxX8FJhilGEwkfUoZ0uzWxnzcT'; const PROD_WRITE_KEY = 'cOkp5EIJaN69gYaN8bcp7KtaD0fGABwJ'; +// Event names following apify-core naming convention (Title Case) const SEGMENT_EVENTS = { TOOL_CALL: 'MCP Tool Call', -}; +} as const; /** * Gets the telemetry environment, defaulting to 'prod' if not provided or invalid @@ -24,13 +27,23 @@ let analyticsClient: Analytics | null = null; * Gets or initializes the Segment Analytics client. * The environment is determined by the TELEMETRY_ENV environment variable. * - * @returns Analytics client instance + * @returns Analytics client instance or null if initialization failed */ -export function getOrInitAnalyticsClient(): Analytics { +export function getOrInitAnalyticsClient(): Analytics | null { if (!analyticsClient) { - const env = getTelemetryEnv(process.env.TELEMETRY_ENV); - const writeKey = env === TELEMETRY_ENV.PROD ? PROD_WRITE_KEY : DEV_WRITE_KEY; - analyticsClient = new Analytics({ writeKey }); + try { + const env = getTelemetryEnv(process.env.TELEMETRY_ENV); + const writeKey = env === TELEMETRY_ENV.PROD ? PROD_WRITE_KEY : DEV_WRITE_KEY; + + analyticsClient = new Analytics({ + writeKey, + flushAt: 50, + flushInterval: 5000, + }); + } catch (error) { + log.error('Segment initialization failed', { error }); + return null; + } } return analyticsClient; } @@ -38,7 +51,7 @@ export function getOrInitAnalyticsClient(): Analytics { /** * Tracks a tool call event to Segment. * - * @param userId - Apify user ID (TODO: extract from token when auth available) + * @param userId - Apify user ID * @param properties - Event properties for the tool call */ export function trackToolCall( @@ -47,10 +60,17 @@ export function trackToolCall( ): void { const client = getOrInitAnalyticsClient(); - // TODO: Implement anonymousId tracking for device/session identification - client.track({ - userId: userId || '', - event: SEGMENT_EVENTS.TOOL_CALL, - properties, - }); + if (!client) { + return; + } + + try { + client.track({ + userId: userId || '', + event: SEGMENT_EVENTS.TOOL_CALL, + properties, + }); + } catch (error) { + log.error('Failed to track tool call event', { error, userId, toolName: properties.tool_name }); + } } From d6e7c6d0d5c69f1282b442a32ff64aa90ce9c137 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Fri, 21 Nov 2025 15:49:47 +0100 Subject: [PATCH 29/46] fix: move ActorsMcpServerOptions and ToolCallCounterStore to types.ts --- src/index-internals.ts | 3 +- src/mcp/server.ts | 60 +-------------------------------------- src/types.ts | 64 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 65 insertions(+), 62 deletions(-) diff --git a/src/index-internals.ts b/src/index-internals.ts index 0c805246..3e3130e5 100644 --- a/src/index-internals.ts +++ b/src/index-internals.ts @@ -8,7 +8,7 @@ import { processParamsGetTools } from './mcp/utils.js'; import { addTool } from './tools/helpers.js'; import { defaultTools, getActorsAsTools, toolCategories, toolCategoriesEnabledByDefault } from './tools/index.js'; import { actorNameToToolName } from './tools/utils.js'; -import type { ToolCategory } from './types.js'; +import type { ToolCallCounterStore, ToolCategory } from './types.js'; import { getExpectedToolNamesByCategories, getToolPublicFieldOnly } from './utils/tools.js'; import { TTLLRUCache } from './utils/ttl-lru.js'; @@ -24,6 +24,7 @@ export { toolCategories, toolCategoriesEnabledByDefault, type ToolCategory, + type ToolCallCounterStore, processParamsGetTools, getActorsAsTools, getToolPublicFieldOnly, diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 87fe30ee..ebe89985 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -34,12 +34,11 @@ import { SKYFIRE_README_CONTENT, SKYFIRE_TOOL_INSTRUCTIONS, } from '../const.js'; -import { type TelemetryEnv } from '../const.js'; import { prompts } from '../prompts/index.js'; import { getTelemetryEnv, trackToolCall } from '../telemetry.js'; import { callActorGetDataset, defaultTools, getActorsAsTools, toolCategories } from '../tools/index.js'; import { decodeDotPropertyNames } from '../tools/utils.js'; -import type { ToolCallTelemetryProperties, ToolEntry } from '../types.js'; +import type { ActorsMcpServerOptions, ToolCallTelemetryProperties, ToolEntry } from '../types.js'; import { buildActorResponseContent } from '../utils/actor-response.js'; import { parseBooleanFromString } from '../utils/generic.js'; import { logHttpError } from '../utils/logging.js'; @@ -54,63 +53,6 @@ import { processParamsGetTools } from './utils.js'; type ToolsChangedHandler = (toolNames: string[]) => void; -/** - * Interface for storing and retrieving tool call counters per session. - * Used for tracking tool call sequence in user journeys. - */ -interface ToolCallCounterStore { - /** - * Gets and increments the tool call counter for a session atomically. - * @param sessionId - The session ID - * @returns Promise resolving to the new counter value (after increment) - */ - getAndIncrement(sessionId: string): Promise; -} - -interface ActorsMcpServerOptions { - setupSigintHandler?: boolean; - /** - * Switch to enable Skyfire agentic payment mode. - */ - skyfireMode?: boolean; - initializeRequestData?: InitializeRequest; - /** - * Telemetry configuration options. - */ - telemetry?: { - /** - * Enable or disable telemetry tracking for tool calls. - * Defaults to true when not set. - */ - enabled?: boolean; - /** - * Telemetry environment when telemetry is enabled. - * - 'dev': Use development Segment write key - * - 'prod': Use production Segment write key (default) - */ - env?: TelemetryEnv; - /** - * Optional store for tool call counters. - * If not provided, uses in-memory storage (suitable for stdio). - * For distributed deployments (HTTP/SSE), provide a Redis-backed implementation. - */ - toolCallCountStore?: ToolCallCounterStore; - }; - /** - * Transport type for telemetry tracking. - * - 'stdio': Direct/local stdio connection - * - 'http': Remote HTTP streamable connection - * - 'sse': Remote Server-Sent Events (SSE) connection - */ - transportType?: 'stdio' | 'http' | 'sse'; - /** - * Apify API token for authentication - * Primarily used by stdio transport when token is read from ~/.apify/auth.json file - * instead of APIFY_TOKEN environment variable, so it can be passed to the server - */ - token?: string; -} - /** * Create Apify MCP server */ diff --git a/src/types.ts b/src/types.ts index a9e9b596..7ec2b9ee 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,11 +1,11 @@ import type { Server } from '@modelcontextprotocol/sdk/server/index.js'; import type { RequestHandlerExtra } from '@modelcontextprotocol/sdk/shared/protocol.js'; -import type { Notification, Prompt, Request, ToolSchema } from '@modelcontextprotocol/sdk/types.js'; +import type { InitializeRequest, Notification, Prompt, Request, ToolSchema } from '@modelcontextprotocol/sdk/types.js'; import type { ValidateFunction } from 'ajv'; import type { ActorDefaultRunOptions, ActorDefinition, ActorStoreList, PricingInfo } from 'apify-client'; import type z from 'zod'; -import type { ACTOR_PRICING_MODEL } from './const.js'; +import type { ACTOR_PRICING_MODEL, TelemetryEnv } from './const.js'; import type { ActorsMcpServer } from './mcp/server.js'; import type { toolCategories } from './tools/index.js'; import type { ProgressTracker } from './utils/progress.js'; @@ -311,3 +311,63 @@ export interface ToolCallTelemetryProperties { tool_exec_time_ms: number; tool_call_number: number; } + +/** + * Interface for storing and retrieving tool call counters per session. + * Used for tracking tool call sequence in user journeys. + */ +export interface ToolCallCounterStore { + /** + * Gets and increments the tool call counter for a session atomically. + * @param sessionId - The session ID + * @returns Promise resolving to the new counter value (after increment) + */ + getAndIncrement(sessionId: string): Promise; +} + +/** + * Options for configuring the ActorsMcpServer instance. + */ +export interface ActorsMcpServerOptions { + setupSigintHandler?: boolean; + /** + * Switch to enable Skyfire agentic payment mode. + */ + skyfireMode?: boolean; + initializeRequestData?: InitializeRequest; + /** + * Telemetry configuration options. + */ + telemetry?: { + /** + * Enable or disable telemetry tracking for tool calls. + * Defaults to true when not set. + */ + enabled?: boolean; + /** + * Telemetry environment when telemetry is enabled. + * - 'dev': Use development Segment write key + * - 'prod': Use production Segment write key (default) + */ + env?: TelemetryEnv; + /** + * Optional store for tool call counters. + * If not provided, uses in-memory storage (suitable for stdio). + * For distributed deployments (HTTP/SSE), provide a Redis-backed implementation. + */ + toolCallCountStore?: ToolCallCounterStore; + }; + /** + * Transport type for telemetry tracking. + * - 'stdio': Direct/local stdio connection + * - 'http': Remote HTTP streamable connection + * - 'sse': Remote Server-Sent Events (SSE) connection + */ + transportType?: 'stdio' | 'http' | 'sse'; + /** + * Apify API token for authentication + * Primarily used by stdio transport when token is read from ~/.apify/auth.json file + * instead of APIFY_TOKEN environment variable, so it can be passed to the server + */ + token?: string; +} From da55972b64618fd4e778a306642e8a0be8180804 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Fri, 21 Nov 2025 16:18:51 +0100 Subject: [PATCH 30/46] fix: Improve getAndIncrementToolCallCounter and add test --- src/mcp/server.ts | 23 +++--- .../unit/mcp.server.tool-call-counter.test.ts | 79 +++++++++++++++++++ 2 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 tests/unit/mcp.server.tool-call-counter.test.ts diff --git a/src/mcp/server.ts b/src/mcp/server.ts index ebe89985..c6327c47 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -103,20 +103,11 @@ export class ActorsMcpServer { /** * Gets and increments the tool call counter for a session. - * Uses external store if provided, otherwise uses in-memory Map. * @param sessionId - The session ID * @returns Promise resolving to the new counter value (after increment) */ private async getAndIncrementToolCallCounter(sessionId: string): Promise { - if (this.options.telemetry?.toolCallCountStore) { - // Use external store (Redis for HTTP/SSE) - return await this.options.telemetry.toolCallCountStore.getAndIncrement(sessionId); - } - // Use in-memory storage (for stdio) - const current = this.sessionToolCallCounters.get(sessionId) || 0; - const newValue = current + 1; - this.sessionToolCallCounters.set(sessionId, newValue); - return newValue; + return await this.options.telemetry!.toolCallCountStore!.getAndIncrement(sessionId); } /** @@ -142,6 +133,18 @@ export class ActorsMcpServer { if (this.options.telemetry.enabled && this.options.telemetry.env === undefined) { this.options.telemetry.env = getTelemetryEnv(undefined); } + + // Provide default in-memory store if not provided + if (!this.options.telemetry.toolCallCountStore) { + this.options.telemetry.toolCallCountStore = { + getAndIncrement: async (sessionId: string): Promise => { + const current = this.sessionToolCallCounters.get(sessionId) || 0; + const newValue = current + 1; + this.sessionToolCallCounters.set(sessionId, newValue); + return newValue; + }, + }; + } } /** diff --git a/tests/unit/mcp.server.tool-call-counter.test.ts b/tests/unit/mcp.server.tool-call-counter.test.ts new file mode 100644 index 00000000..d4eb00cf --- /dev/null +++ b/tests/unit/mcp.server.tool-call-counter.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from 'vitest'; + +import { ActorsMcpServer } from '../../src/index.js'; +import type { ToolCallCounterStore } from '../../src/types.js'; + +describe('ActorsMcpServer tool call counter', () => { + describe('default in-memory store', () => { + it('should increment counter correctly for multiple calls', async () => { + const server = new ActorsMcpServer({ setupSigintHandler: false }); + // Default store should be available after construction + const store = server.options.telemetry!.toolCallCountStore!; + expect(store).toBeDefined(); + + const sessionId = 'test-session-1'; + + // First call should return 1 + const firstCall = await store.getAndIncrement(sessionId); + expect(firstCall).toBe(1); + + // Second call should return 2 + const secondCall = await store.getAndIncrement(sessionId); + expect(secondCall).toBe(2); + + // Third call should return 3 + const thirdCall = await store.getAndIncrement(sessionId); + expect(thirdCall).toBe(3); + }); + + it('should have independent counters for different sessions', async () => { + const server = new ActorsMcpServer({ setupSigintHandler: false }); + const store = server.options.telemetry!.toolCallCountStore!; + const sessionId1 = 'session-1'; + const sessionId2 = 'session-2'; + + // Increment session 1 twice + const session1Call1 = await store.getAndIncrement(sessionId1); + expect(session1Call1).toBe(1); + const session1Call2 = await store.getAndIncrement(sessionId1); + expect(session1Call2).toBe(2); + + // Session 2 should start at 1 + const session2Call1 = await store.getAndIncrement(sessionId2); + expect(session2Call1).toBe(1); + + // Session 1 should still be at 2 + const session1Call3 = await store.getAndIncrement(sessionId1); + expect(session1Call3).toBe(3); + + // Session 2 should be at 2 + const session2Call2 = await store.getAndIncrement(sessionId2); + expect(session2Call2).toBe(2); + }); + }); + + describe('custom store', () => { + it('should use provided custom store instead of default', async () => { + const customStore: ToolCallCounterStore = { + getAndIncrement: async (sessionId: string) => { + // Custom implementation that returns a fixed value + // This is just for testing that custom store is used + return sessionId.length + 1; + }, + }; + + const server = new ActorsMcpServer({ + setupSigintHandler: false, + telemetry: { + toolCallCountStore: customStore, + }, + }); + + const store = server.options.telemetry!.toolCallCountStore!; + const result = await store.getAndIncrement('test-session'); + + // Should use custom store logic, not default + expect(result).toBe('test-session'.length + 1); + }); + }); +}); From 5ba57511a2355c1d3b2bc0ef0b34cf97b87ee230 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Fri, 21 Nov 2025 16:25:25 +0100 Subject: [PATCH 31/46] fix: Improve naming in setupTelemetry function --- src/mcp/server.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index c6327c47..626ced0c 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -119,15 +119,17 @@ export class ActorsMcpServer { this.options.telemetry = {}; } + // Configure telemetryEnabled: explicit option > env var > default (true) if (this.options.telemetry.enabled === undefined) { - // Check environment variable as fallback const envEnabled = parseBooleanFromString(process.env.TELEMETRY_ENABLED); this.options.telemetry.enabled = envEnabled !== undefined ? envEnabled : true; } - // Set telemetryEnv: explicit option > env var > default ('prod') - const telemetryEnv = process.env.TELEMETRY_ENV; - const telemetryParam = this.options.telemetry.env; - this.options.telemetry.env = getTelemetryEnv(telemetryParam ?? telemetryEnv); + + // Configure telemetryEnv: explicit option > env var > default ('prod') + // getTelemetryEnv always returns a value (defaults to 'prod'), so no need for undefined check + const envVarEnv = process.env.TELEMETRY_ENV; + const explicitEnv = this.options.telemetry.env; + this.options.telemetry.env = getTelemetryEnv(explicitEnv ?? envVarEnv); // If telemetry is enabled, ensure telemetryEnv is set if (this.options.telemetry.enabled && this.options.telemetry.env === undefined) { From a651486e663593ea8b76f45a2ade842bfcc1af26 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Fri, 21 Nov 2025 21:08:45 +0100 Subject: [PATCH 32/46] fix: comment --- src/mcp/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 626ced0c..46203354 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -341,7 +341,7 @@ export class ActorsMcpServer { this.tools.set(clonedWrap.name, modified ? clonedWrap : wrap); } } else { - // No skyfire mode and telemetry disabled - store tools as-is + // No skyfire mode - store tools as-is for (const tool of tools) { this.tools.set(tool.name, tool); } From 11e57f0e849f30c7b644b195aea7144dc7ee8263 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Fri, 21 Nov 2025 22:04:25 +0100 Subject: [PATCH 33/46] fix: refactor prepareTelemetryData --- src/mcp/server.ts | 150 ++++++++++++++++++++++++++++------------------ 1 file changed, 91 insertions(+), 59 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 46203354..ecca8ac7 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -38,7 +38,14 @@ import { prompts } from '../prompts/index.js'; import { getTelemetryEnv, trackToolCall } from '../telemetry.js'; import { callActorGetDataset, defaultTools, getActorsAsTools, toolCategories } from '../tools/index.js'; import { decodeDotPropertyNames } from '../tools/utils.js'; -import type { ActorsMcpServerOptions, ToolCallTelemetryProperties, ToolEntry } from '../types.js'; +import type { + ActorMcpTool, + ActorsMcpServerOptions, + ActorTool, + HelperTool, + ToolCallTelemetryProperties, + ToolEntry, +} from '../types.js'; import { buildActorResponseContent } from '../utils/actor-response.js'; import { parseBooleanFromString } from '../utils/generic.js'; import { logHttpError } from '../utils/logging.js'; @@ -519,9 +526,8 @@ export class ActorsMcpServer { const { progressToken } = meta || {}; const apifyToken = (request.params.apifyToken || this.options.token || process.env.APIFY_TOKEN) as string; const userRentedActorIds = request.params.userRentedActorIds as string[] | undefined; - // Injected for telemetry purposes - const mcpSessionId = request.params.mcpSessionId as string | undefined; - + // mcpSessionId was injected upstream by stdio; optional (for telemetry purposes only) + const mcpSessionId = typeof request.params.mcpSessionId === 'string' ? request.params.mcpSessionId : undefined; // Remove apifyToken from request.params just in case delete request.params.apifyToken; // Remove other custom params passed from apify-mcp-server @@ -595,54 +601,7 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool msg, ); } - - // Prepare telemetry data (but don't track yet) - let telemetryData: ToolCallTelemetryProperties | null = null; - let userId = ''; - - if (this.options.telemetry?.enabled === true) { - const toolFullName = tool.type === 'actor' ? tool.actorFullName : tool.name; - - // Get or increment tool call counter for this session - let toolCallNumber = 0; - const sessionId = mcpSessionId || 'unknown'; - if (sessionId !== 'unknown') { - try { - toolCallNumber = await this.getAndIncrementToolCallCounter(sessionId); - } catch (error) { - log.warning('Failed to get tool call counter', { sessionId, error: String(error) }); - // Continue with 0 if counter fails - } - } - - // Get userId from cache or fetch from API - // Use token from options (e.g., from stdio auth file) or from request - if (apifyToken) { - const apifyClient = new ApifyClient({ token: apifyToken }); - const userInfo = await getUserIdFromToken(apifyToken, apifyClient); - userId = userInfo?.id || ''; - log.debug('Telemetry: fetched user info', { userId, userFound: !!userInfo }); - } else { - log.debug('Telemetry: no API token provided'); - } - - const capabilities = this.options.initializeRequestData?.params?.capabilities; - const params = this.options.initializeRequestData?.params as InitializeRequest['params']; - telemetryData = { - app_name: 'apify-mcp-server', - app_version: getPackageVersion() || '', - mcp_client_name: params?.clientInfo?.name || '', - mcp_client_version: params?.clientInfo?.version || '', - mcp_protocol_version: params?.protocolVersion || '', - mcp_capabilities: capabilities ? JSON.stringify(capabilities) : '', - mcp_session_id: mcpSessionId || '', - transport_type: this.options.transportType || '', - tool_name: toolFullName, - tool_status: 'succeeded', // Will be updated in finally - tool_exec_time_ms: 0, // Will be calculated in finally - tool_call_number: toolCallNumber, - }; - } + const { telemetryData, userId } = await this.prepareTelemetryData(tool, mcpSessionId, apifyToken); const startTime = Date.now(); let toolStatus: 'succeeded' | 'failed' | 'aborted' = 'succeeded'; @@ -772,13 +731,7 @@ Please verify the server URL is correct and accessible, and ensure you have a va Please verify the tool name, input parameters, and ensure all required resources are available.`, ], true); } finally { - // Track telemetry once at the end with determined status and execution time - if (telemetryData) { - const execTime = Date.now() - startTime; - telemetryData.tool_status = toolStatus; - telemetryData.tool_exec_time_ms = execTime; - trackToolCall(userId, telemetryData); - } + this.finalizeAndTrackTelemetry(telemetryData, userId, startTime, toolStatus); } const availableTools = this.listToolNames(); @@ -797,6 +750,85 @@ Please verify the tool name and ensure the tool is properly registered.`; }); } + /** + * Finalizes and tracks telemetry for a tool call. + * Calculates execution time, sets final status, and sends the telemetry event. + * + * @param telemetryData - Telemetry data to finalize and track (null if telemetry is disabled) + * @param userId - Apify user ID + * @param startTime - Timestamp when the tool call started + * @param toolStatus - Final status of the tool call ('succeeded', 'failed', or 'aborted') + */ + private finalizeAndTrackTelemetry( + telemetryData: ToolCallTelemetryProperties | null, + userId: string, + startTime: number, + toolStatus: 'succeeded' | 'failed' | 'aborted', + ): void { + if (!telemetryData) { + return; + } + + const execTime = Date.now() - startTime; + const finalizedTelemetryData: ToolCallTelemetryProperties = { + ...telemetryData, + tool_status: toolStatus, + tool_exec_time_ms: execTime, + }; + trackToolCall(userId, finalizedTelemetryData); + } + + /* + * Creates telemetry data for a tool call. + */ + private async prepareTelemetryData( + tool: HelperTool | ActorTool | ActorMcpTool, mcpSessionId: string | undefined, apifyToken: string, + ): Promise<{ telemetryData: ToolCallTelemetryProperties | null; userId: string }> { + if (this.options.telemetry?.enabled !== true) { + return { telemetryData: null, userId: '' }; + } + + const toolFullName = tool.type === 'actor' ? tool.actorFullName : tool.name; + + // Get or increment tool call counter for this session + let toolCallNumber = 0; + if (mcpSessionId) { + try { + toolCallNumber = await this.getAndIncrementToolCallCounter(mcpSessionId); + } catch (error) { + log.warning('Failed to get tool call counter', { mcpSessionId, error: String(error) }); + } + } + + // Get userId from cache or fetch from API + let userId = ''; + if (apifyToken) { + const apifyClient = new ApifyClient({ token: apifyToken }); + const userInfo = await getUserIdFromToken(apifyToken, apifyClient); + userId = userInfo?.id || ''; + log.debug('Telemetry: fetched user info', { userId, userFound: !!userInfo }); + } + + const capabilities = this.options.initializeRequestData?.params?.capabilities; + const params = this.options.initializeRequestData?.params as InitializeRequest['params']; + const telemetryData: ToolCallTelemetryProperties = { + app_name: 'apify-mcp-server', + app_version: getPackageVersion() || '', + mcp_client_name: params?.clientInfo?.name || '', + mcp_client_version: params?.clientInfo?.version || '', + mcp_protocol_version: params?.protocolVersion || '', + mcp_capabilities: capabilities ? JSON.stringify(capabilities) : '', + mcp_session_id: mcpSessionId || '', + transport_type: this.options.transportType || '', + tool_name: toolFullName, + tool_status: 'succeeded', // Will be updated in finally + tool_exec_time_ms: 0, // Will be calculated in finally + tool_call_number: toolCallNumber, + }; + + return { telemetryData, userId }; + } + async connect(transport: Transport): Promise { await this.server.connect(transport); } From 571c08765bd35a0a45a2dda4206575b907fb24f6 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Fri, 21 Nov 2025 22:22:14 +0100 Subject: [PATCH 34/46] fix: simplify userId cache --- src/mcp/server.ts | 15 ++++++------ src/telemetry.ts | 4 ++-- src/utils/{user-cache.ts => userid-cache.ts} | 25 +++++++------------- 3 files changed, 17 insertions(+), 27 deletions(-) rename src/utils/{user-cache.ts => userid-cache.ts} (58%) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index ecca8ac7..09ad684a 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -52,7 +52,7 @@ import { logHttpError } from '../utils/logging.js'; import { buildMCPResponse } from '../utils/mcp.js'; import { createProgressTracker } from '../utils/progress.js'; import { cloneToolEntry, getToolPublicFieldOnly } from '../utils/tools.js'; -import { getUserIdFromToken } from '../utils/user-cache.js'; +import { getUserIdFromTokenCached } from '../utils/userid-cache.js'; import { getPackageVersion } from '../utils/version.js'; import { connectMCPClient } from './client.js'; import { EXTERNAL_TOOL_CALL_TIMEOUT_MSEC, LOG_LEVEL_MAP } from './const.js'; @@ -761,7 +761,7 @@ Please verify the tool name and ensure the tool is properly registered.`; */ private finalizeAndTrackTelemetry( telemetryData: ToolCallTelemetryProperties | null, - userId: string, + userId: string | null, startTime: number, toolStatus: 'succeeded' | 'failed' | 'aborted', ): void { @@ -783,9 +783,9 @@ Please verify the tool name and ensure the tool is properly registered.`; */ private async prepareTelemetryData( tool: HelperTool | ActorTool | ActorMcpTool, mcpSessionId: string | undefined, apifyToken: string, - ): Promise<{ telemetryData: ToolCallTelemetryProperties | null; userId: string }> { + ): Promise<{ telemetryData: ToolCallTelemetryProperties | null; userId: string | null }> { if (this.options.telemetry?.enabled !== true) { - return { telemetryData: null, userId: '' }; + return { telemetryData: null, userId: null }; } const toolFullName = tool.type === 'actor' ? tool.actorFullName : tool.name; @@ -801,12 +801,11 @@ Please verify the tool name and ensure the tool is properly registered.`; } // Get userId from cache or fetch from API - let userId = ''; + let userId: string | null = null; if (apifyToken) { const apifyClient = new ApifyClient({ token: apifyToken }); - const userInfo = await getUserIdFromToken(apifyToken, apifyClient); - userId = userInfo?.id || ''; - log.debug('Telemetry: fetched user info', { userId, userFound: !!userInfo }); + userId = await getUserIdFromTokenCached(apifyToken, apifyClient); + log.debug('Telemetry: fetched userId', { userId }); } const capabilities = this.options.initializeRequestData?.params?.capabilities; diff --git a/src/telemetry.ts b/src/telemetry.ts index 154056df..c5d5621b 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -51,11 +51,11 @@ export function getOrInitAnalyticsClient(): Analytics | null { /** * Tracks a tool call event to Segment. * - * @param userId - Apify user ID + * @param userId - Apify user ID (null if not available) * @param properties - Event properties for the tool call */ export function trackToolCall( - userId: string, + userId: string | null, properties: ToolCallTelemetryProperties, ): void { const client = getOrInitAnalyticsClient(); diff --git a/src/utils/user-cache.ts b/src/utils/userid-cache.ts similarity index 58% rename from src/utils/user-cache.ts rename to src/utils/userid-cache.ts index 24b23d65..4b1e3e08 100644 --- a/src/utils/user-cache.ts +++ b/src/utils/userid-cache.ts @@ -1,41 +1,32 @@ import { createHash } from 'node:crypto'; -import type { User } from 'apify-client'; - import type { ApifyClient } from '../apify-client.js'; import { USER_CACHE_MAX_SIZE, USER_CACHE_TTL_SECS } from '../const.js'; import { TTLLRUCache } from './ttl-lru.js'; // LRU cache with TTL for user info - stores the raw User object from API -const userCache = new TTLLRUCache(USER_CACHE_MAX_SIZE, USER_CACHE_TTL_SECS); +const userIdCache = new TTLLRUCache(USER_CACHE_MAX_SIZE, USER_CACHE_TTL_SECS); /** * Gets user info from token, using cache to avoid repeated API calls * Token is hashed before caching to avoid storing raw tokens - * Returns the full User object from API or null if not found + * Returns userId or null if not found */ -export async function getUserInfoFromTokenCached( +export async function getUserIdFromTokenCached( token: string, apifyClient: ApifyClient, -): Promise { - // Hash token for cache key +): Promise { const tokenHash = createHash('sha256').update(token).digest('hex'); + const cachedId = userIdCache.get(tokenHash); + if (cachedId) return cachedId; - // Check cache first - const cachedUser = userCache.get(tokenHash); - if (cachedUser) { - return cachedUser; - } - - // Fetch from API try { const user = await apifyClient.user('me').get(); if (!user || !user.id) { return null; } - - userCache.set(tokenHash, user); - return user; + userIdCache.set(tokenHash, user.id); + return user.id; } catch { return null; } From 421f1a38a1be6e176517647023e6457f3fe91b03 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Fri, 21 Nov 2025 22:28:22 +0100 Subject: [PATCH 35/46] fix: make telemetryEnv uppercase -> PROD, DEV --- src/const.ts | 4 ++-- src/mcp/server.ts | 4 ++-- src/stdio.ts | 6 +++--- src/telemetry.ts | 11 +++++++++-- src/types.ts | 4 ++-- tests/helpers.ts | 2 +- 6 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/const.ts b/src/const.ts index 7ab5c7d8..c597452a 100644 --- a/src/const.ts +++ b/src/const.ts @@ -109,8 +109,8 @@ export const APIFY_STORE_URL = 'https://apify.com'; // Telemetry export const TELEMETRY_ENV = { - DEV: 'dev', - PROD: 'prod', + DEV: 'DEV', + PROD: 'PROD', } as const; export type TelemetryEnv = (typeof TELEMETRY_ENV)[keyof typeof TELEMETRY_ENV]; diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 09ad684a..20d3a74c 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -132,8 +132,8 @@ export class ActorsMcpServer { this.options.telemetry.enabled = envEnabled !== undefined ? envEnabled : true; } - // Configure telemetryEnv: explicit option > env var > default ('prod') - // getTelemetryEnv always returns a value (defaults to 'prod'), so no need for undefined check + // Configure telemetryEnv: explicit option > env var > default ('PROD') + // getTelemetryEnv always returns a value (defaults to 'PROD'), so no need for undefined check const envVarEnv = process.env.TELEMETRY_ENV; const explicitEnv = this.options.telemetry.env; this.options.telemetry.env = getTelemetryEnv(explicitEnv ?? envVarEnv); diff --git a/src/stdio.ts b/src/stdio.ts index 5672d926..f322d617 100644 --- a/src/stdio.ts +++ b/src/stdio.ts @@ -51,7 +51,7 @@ interface CliArgs { tools?: string; /** Enable or disable telemetry tracking (default: true) */ telemetryEnabled: boolean; - /** Telemetry environment: 'prod' or 'dev' (default: 'prod', only used when telemetry-enabled is true) */ + /** Telemetry environment: 'PROD' or 'DEV' (default: 'PROD', only used when telemetry-enabled is true) */ telemetryEnv: TelemetryEnv; } @@ -114,8 +114,8 @@ Default: true (enabled)`, default: DEFAULT_TELEMETRY_ENV, hidden: true, describe: `Telemetry environment when telemetry is enabled. Can also be set via TELEMETRY_ENV environment variable. -- 'prod': Send events to production Segment workspace (default) -- 'dev': Send events to development Segment workspace +- 'PROD': Send events to production Segment workspace (default) +- 'DEV': Send events to development Segment workspace Only used when --telemetry-enabled is true`, }) .help('help') diff --git a/src/telemetry.ts b/src/telemetry.ts index c5d5621b..2839f0db 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -14,10 +14,17 @@ const SEGMENT_EVENTS = { } as const; /** - * Gets the telemetry environment, defaulting to 'prod' if not provided or invalid + * Gets the telemetry environment, defaulting to 'PROD' if not provided or invalid */ export function getTelemetryEnv(env?: string | null): TelemetryEnv { - return (env === TELEMETRY_ENV.DEV || env === TELEMETRY_ENV.PROD) ? env : DEFAULT_TELEMETRY_ENV; + if (!env) { + return DEFAULT_TELEMETRY_ENV; + } + const normalizedEnv = env.toUpperCase(); + if (normalizedEnv === TELEMETRY_ENV.DEV || normalizedEnv === TELEMETRY_ENV.PROD) { + return normalizedEnv as TelemetryEnv; + } + return DEFAULT_TELEMETRY_ENV; } // Single Segment Analytics client (environment determined by process.env.TELEMETRY_ENV) diff --git a/src/types.ts b/src/types.ts index 7ec2b9ee..2f7c2f1b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -346,8 +346,8 @@ export interface ActorsMcpServerOptions { enabled?: boolean; /** * Telemetry environment when telemetry is enabled. - * - 'dev': Use development Segment write key - * - 'prod': Use production Segment write key (default) + * - 'DEV': Use development Segment write key + * - 'PROD': Use production Segment write key (default) */ env?: TelemetryEnv; /** diff --git a/tests/helpers.ts b/tests/helpers.ts index d324e96b..c48a0750 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -15,7 +15,7 @@ export interface McpClientOptions { clientName?: string; // Client name for identification telemetry?: { enabled?: boolean; // Enable or disable telemetry (default: false for tests) - env?: TelemetryEnv; // Telemetry environment (default: 'prod', only used when telemetry.enabled is true) + env?: TelemetryEnv; // Telemetry environment (default: 'PROD', only used when telemetry.enabled is true) }; } From 8c8721fcf195fc0ed916fb06631b295d3b01140c Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Fri, 21 Nov 2025 22:44:14 +0100 Subject: [PATCH 36/46] fix: handle telemetry-enabled param in better way (also for null and empty string) --- src/actor/server.ts | 11 ++++++-- src/mcp/server.ts | 5 +++- src/utils/generic.ts | 16 +++++++++-- tests/unit/utils.generic.test.ts | 48 +++++++++++++++++++++++++++++++- 4 files changed, 73 insertions(+), 7 deletions(-) diff --git a/src/actor/server.ts b/src/actor/server.ts index d3e42f00..9dec2393 100644 --- a/src/actor/server.ts +++ b/src/actor/server.ts @@ -14,6 +14,7 @@ import log from '@apify/log'; import { ApifyClient } from '../apify-client.js'; import { ActorsMcpServer } from '../mcp/server.js'; +import { parseBooleanFromString } from '../utils/generic.js'; import { getHelpMessage, HEADER_READINESS_PROBE, Routes, TransportType } from './const.js'; import { getActorRunData } from './utils.js'; @@ -81,7 +82,10 @@ export function createExpressApp( // Extract telemetry query parameters const urlParams = new URL(req.url, `http://${req.headers.host}`).searchParams; const telemetryEnabledParam = urlParams.get('telemetry-enabled'); - const telemetryEnabled = telemetryEnabledParam !== 'false'; // Default to true + // URL param > env var > default (true) + const telemetryEnabled = parseBooleanFromString(telemetryEnabledParam) + ?? parseBooleanFromString(process.env.TELEMETRY_ENABLED) + ?? true; const mcpServer = new ActorsMcpServer({ setupSigintHandler: false, @@ -176,7 +180,10 @@ export function createExpressApp( // Extract telemetry query parameters const urlParams = new URL(req.url, `http://${req.headers.host}`).searchParams; const telemetryEnabledParam = urlParams.get('telemetry-enabled'); - const telemetryEnabled = telemetryEnabledParam !== 'false'; // Default to true + // URL param > env var > default (true) + const telemetryEnabled = parseBooleanFromString(telemetryEnabledParam) + ?? parseBooleanFromString(process.env.TELEMETRY_ENABLED) + ?? true; const mcpServer = new ActorsMcpServer({ setupSigintHandler: false, diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 20d3a74c..ade9a5b2 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -127,9 +127,12 @@ export class ActorsMcpServer { } // Configure telemetryEnabled: explicit option > env var > default (true) - if (this.options.telemetry.enabled === undefined) { + const telemetryEnabled = parseBooleanFromString(this.options.telemetry.enabled); + if (telemetryEnabled === undefined) { const envEnabled = parseBooleanFromString(process.env.TELEMETRY_ENABLED); this.options.telemetry.enabled = envEnabled !== undefined ? envEnabled : true; + } else { + this.options.telemetry.enabled = telemetryEnabled; } // Configure telemetryEnv: explicit option > env var > default ('PROD') diff --git a/src/utils/generic.ts b/src/utils/generic.ts index 7302bbe4..2f8e09d8 100644 --- a/src/utils/generic.ts +++ b/src/utils/generic.ts @@ -67,15 +67,25 @@ export function isValidHttpUrl(urlString: string): boolean { } /** - * Parses a boolean value from a string. + * Parses a boolean value from a string, boolean, null, or undefined. * Accepts 'true', '1' as true, 'false', '0' as false. - * Returns undefined if the value is not a recognized boolean string. + * If value is already a boolean, returns it directly. + * Returns undefined if the value is not a recognized boolean string or is null/undefined/empty string. */ -export function parseBooleanFromString(value: string | undefined | null): boolean | undefined { +export function parseBooleanFromString(value: string | boolean | undefined | null): boolean | undefined { + // If already a boolean, return it directly + if (typeof value === 'boolean') { + return value; + } + // Handle undefined/null if (value === undefined || value === null) { return undefined; } + // Handle empty string (after trim) const normalized = value.toLowerCase().trim(); + if (normalized === '') { + return undefined; + } if (normalized === 'true' || normalized === '1') { return true; } diff --git a/tests/unit/utils.generic.test.ts b/tests/unit/utils.generic.test.ts index 965712a5..dc1f9af2 100644 --- a/tests/unit/utils.generic.test.ts +++ b/tests/unit/utils.generic.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { getValuesByDotKeys, isValidHttpUrl, parseCommaSeparatedList } from '../../src/utils/generic.js'; +import { getValuesByDotKeys, isValidHttpUrl, parseBooleanFromString, parseCommaSeparatedList } from '../../src/utils/generic.js'; describe('getValuesByDotKeys', () => { it('should get value for a key without dot', () => { @@ -102,3 +102,49 @@ describe('isValidUrl', () => { expect(isValidHttpUrl('://example.com')).toBe(false); }); }); + +describe('parseBooleanFromString', () => { + it('should return boolean values directly', () => { + expect(parseBooleanFromString(true)).toBe(true); + expect(parseBooleanFromString(false)).toBe(false); + }); + + it('should parse "true" and "1" as true', () => { + expect(parseBooleanFromString('true')).toBe(true); + expect(parseBooleanFromString('TRUE')).toBe(true); + expect(parseBooleanFromString('True')).toBe(true); + expect(parseBooleanFromString('1')).toBe(true); + expect(parseBooleanFromString(' true ')).toBe(true); + expect(parseBooleanFromString(' 1 ')).toBe(true); + }); + + it('should parse "false" and "0" as false', () => { + expect(parseBooleanFromString('false')).toBe(false); + expect(parseBooleanFromString('FALSE')).toBe(false); + expect(parseBooleanFromString('False')).toBe(false); + expect(parseBooleanFromString('0')).toBe(false); + expect(parseBooleanFromString(' false ')).toBe(false); + expect(parseBooleanFromString(' 0 ')).toBe(false); + }); + + it('should return undefined for null and undefined', () => { + expect(parseBooleanFromString(null)).toBeUndefined(); + expect(parseBooleanFromString(undefined)).toBeUndefined(); + }); + + it('should return undefined for empty strings', () => { + expect(parseBooleanFromString('')).toBeUndefined(); + expect(parseBooleanFromString(' ')).toBeUndefined(); + expect(parseBooleanFromString('\t')).toBeUndefined(); + expect(parseBooleanFromString('\n')).toBeUndefined(); + }); + + it('should return undefined for unrecognized strings', () => { + expect(parseBooleanFromString('yes')).toBeUndefined(); + expect(parseBooleanFromString('no')).toBeUndefined(); + expect(parseBooleanFromString('2')).toBeUndefined(); + expect(parseBooleanFromString('maybe')).toBeUndefined(); + expect(parseBooleanFromString('on')).toBeUndefined(); + expect(parseBooleanFromString('off')).toBeUndefined(); + }); +}); From 209e41f32618f71fd35b32e01385b13afa21c523 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Mon, 24 Nov 2025 10:28:12 +0100 Subject: [PATCH 37/46] fix: simplify - get package version --- src/utils/version.ts | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/src/utils/version.ts b/src/utils/version.ts index 127ff80f..7ab1259a 100644 --- a/src/utils/version.ts +++ b/src/utils/version.ts @@ -1,24 +1,12 @@ -import { readFileSync } from 'node:fs'; -import { dirname, join } from 'node:path'; -import { fileURLToPath } from 'node:url'; +import { createRequire } from 'node:module'; + +const require = createRequire(import.meta.url); +const packageJson = require('../../package.json'); /** - * Gets the package version dynamically from package.json - * Returns null if the file cannot be read + * Gets the package version from package.json + * Returns null if version is not available */ export function getPackageVersion(): string | null { - try { - // Read package.json and extract version - // In production, this will be replaced at build time - // eslint-disable-next-line no-underscore-dangle - const __filename = fileURLToPath(import.meta.url); - // eslint-disable-next-line no-underscore-dangle - const __dirname = dirname(__filename); - const packagePath = join(__dirname, '../../package.json'); - const packageData = JSON.parse(readFileSync(packagePath, 'utf-8')); - return packageData.version || null; - } catch { - // Return null if package.json cannot be read - return null; - } + return packageJson.version || null; } From a3c5ece518a859703a48cdf244e54cc1601af379 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Mon, 24 Nov 2025 12:27:12 +0100 Subject: [PATCH 38/46] fix: rename app_name to app --- src/mcp/server.ts | 14 ++++++++++---- src/telemetry.ts | 4 ++-- src/types.ts | 2 +- tests/unit/telemetry.test.ts | 4 ++-- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index ade9a5b2..eb420290 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -2,6 +2,8 @@ * Model Context Protocol (MCP) server for Apify Actors */ +import * as crypto from 'node:crypto'; + import type { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { Server } from '@modelcontextprotocol/sdk/server/index.js'; import type { Transport } from '@modelcontextprotocol/sdk/shared/transport.js'; @@ -764,7 +766,7 @@ Please verify the tool name and ensure the tool is properly registered.`; */ private finalizeAndTrackTelemetry( telemetryData: ToolCallTelemetryProperties | null, - userId: string | null, + userId: string, startTime: number, toolStatus: 'succeeded' | 'failed' | 'aborted', ): void { @@ -786,9 +788,9 @@ Please verify the tool name and ensure the tool is properly registered.`; */ private async prepareTelemetryData( tool: HelperTool | ActorTool | ActorMcpTool, mcpSessionId: string | undefined, apifyToken: string, - ): Promise<{ telemetryData: ToolCallTelemetryProperties | null; userId: string | null }> { + ): Promise<{ telemetryData: ToolCallTelemetryProperties | null; userId: string }> { if (this.options.telemetry?.enabled !== true) { - return { telemetryData: null, userId: null }; + return { telemetryData: null, userId: crypto.randomUUID() }; } const toolFullName = tool.type === 'actor' ? tool.actorFullName : tool.name; @@ -810,11 +812,15 @@ Please verify the tool name and ensure the tool is properly registered.`; userId = await getUserIdFromTokenCached(apifyToken, apifyClient); log.debug('Telemetry: fetched userId', { userId }); } + if (!userId) { + userId = crypto.randomUUID(); + log.debug('Telemetry: using random userId', { userId }); + } const capabilities = this.options.initializeRequestData?.params?.capabilities; const params = this.options.initializeRequestData?.params as InitializeRequest['params']; const telemetryData: ToolCallTelemetryProperties = { - app_name: 'apify-mcp-server', + app: 'mcp', app_version: getPackageVersion() || '', mcp_client_name: params?.clientInfo?.name || '', mcp_client_version: params?.clientInfo?.version || '', diff --git a/src/telemetry.ts b/src/telemetry.ts index 2839f0db..7e53ab1a 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -62,7 +62,7 @@ export function getOrInitAnalyticsClient(): Analytics | null { * @param properties - Event properties for the tool call */ export function trackToolCall( - userId: string | null, + userId: string, properties: ToolCallTelemetryProperties, ): void { const client = getOrInitAnalyticsClient(); @@ -73,7 +73,7 @@ export function trackToolCall( try { client.track({ - userId: userId || '', + userId, event: SEGMENT_EVENTS.TOOL_CALL, properties, }); diff --git a/src/types.ts b/src/types.ts index 2f7c2f1b..7be33acc 100644 --- a/src/types.ts +++ b/src/types.ts @@ -298,7 +298,7 @@ export type ApifyToken = string | null | undefined; * Properties for tool call telemetry events sent to Segment. */ export interface ToolCallTelemetryProperties { - app_name: string; + app: 'mcp'; app_version: string; mcp_client_name: string; mcp_client_version: string; diff --git a/tests/unit/telemetry.test.ts b/tests/unit/telemetry.test.ts index 47bb483e..d7147564 100644 --- a/tests/unit/telemetry.test.ts +++ b/tests/unit/telemetry.test.ts @@ -18,7 +18,7 @@ describe('telemetry', () => { it('should send correct payload structure to Segment', () => { const userId = 'test-user-123'; const properties = { - app_name: 'apify-mcp-server', + app: 'mcp' as const, app_version: '0.5.6', mcp_client_name: 'test-client', mcp_client_version: '1.0.0', @@ -38,7 +38,7 @@ describe('telemetry', () => { userId: 'test-user-123', event: 'MCP Tool Call', properties: { - app_name: 'apify-mcp-server', + app: 'mcp', app_version: '0.5.6', mcp_client_name: 'test-client', mcp_client_version: '1.0.0', From 5165cc0f79d191a9d31dcb6192d88165289ab812 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Mon, 24 Nov 2025 13:22:23 +0100 Subject: [PATCH 39/46] fix: save mcp_client_capabilities --- src/mcp/server.ts | 2 +- src/types.ts | 2 +- tests/unit/telemetry.test.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index eb420290..538ff2d0 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -825,7 +825,7 @@ Please verify the tool name and ensure the tool is properly registered.`; mcp_client_name: params?.clientInfo?.name || '', mcp_client_version: params?.clientInfo?.version || '', mcp_protocol_version: params?.protocolVersion || '', - mcp_capabilities: capabilities ? JSON.stringify(capabilities) : '', + mcp_client_capabilities: capabilities ? JSON.stringify(capabilities) : '', mcp_session_id: mcpSessionId || '', transport_type: this.options.transportType || '', tool_name: toolFullName, diff --git a/src/types.ts b/src/types.ts index 7be33acc..3c04c739 100644 --- a/src/types.ts +++ b/src/types.ts @@ -303,7 +303,7 @@ export interface ToolCallTelemetryProperties { mcp_client_name: string; mcp_client_version: string; mcp_protocol_version: string; - mcp_capabilities: string; + mcp_client_capabilities: string; mcp_session_id: string; transport_type: string; tool_name: string; diff --git a/tests/unit/telemetry.test.ts b/tests/unit/telemetry.test.ts index d7147564..5a53559b 100644 --- a/tests/unit/telemetry.test.ts +++ b/tests/unit/telemetry.test.ts @@ -23,7 +23,7 @@ describe('telemetry', () => { mcp_client_name: 'test-client', mcp_client_version: '1.0.0', mcp_protocol_version: '2024-11-05', - mcp_capabilities: '{}', + mcp_client_capabilities: '{}', mcp_session_id: 'session-123', transport_type: 'stdio', tool_name: 'test-tool', @@ -43,7 +43,7 @@ describe('telemetry', () => { mcp_client_name: 'test-client', mcp_client_version: '1.0.0', mcp_protocol_version: '2024-11-05', - mcp_capabilities: '{}', + mcp_client_capabilities: '{}', mcp_session_id: 'session-123', transport_type: 'stdio', tool_name: 'test-tool', From dd5291e08396623ad6315eceb2ae12c18ab3f71d Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Mon, 24 Nov 2025 13:32:02 +0100 Subject: [PATCH 40/46] fix: set tool-status failed --- src/mcp/server.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 538ff2d0..021b60a7 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -633,7 +633,7 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool if (progressTracker) { progressTracker.stop(); } - + toolStatus = ('isError' in res && res.isError) ? 'failed' : 'succeeded'; return { ...res }; } @@ -646,6 +646,7 @@ Please check the tool's input schema using ${HelperTools.ACTOR_GET_DETAILS} tool Please verify the server URL is correct and accessible, and ensure you have a valid Apify token with appropriate permissions.`; log.softFail(msg, { statusCode: 408 }); // 408 Request Timeout await this.server.sendLoggingMessage({ level: 'error', data: msg }); + toolStatus = 'failed'; return buildMCPResponse([msg], true); } From e3a4bc0675fa7200e6ed7e483f5f0bfb2e8928f9 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Mon, 24 Nov 2025 21:57:48 +0100 Subject: [PATCH 41/46] fix: Segment requires either userId OR anonymousId, but not both When userId is available, use it; otherwise use anonymousId --- src/mcp/server.ts | 15 ++++----------- src/telemetry.ts | 8 ++++++-- tests/unit/telemetry.test.ts | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 021b60a7..46cbec53 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -2,8 +2,6 @@ * Model Context Protocol (MCP) server for Apify Actors */ -import * as crypto from 'node:crypto'; - import type { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { Server } from '@modelcontextprotocol/sdk/server/index.js'; import type { Transport } from '@modelcontextprotocol/sdk/shared/transport.js'; @@ -761,13 +759,13 @@ Please verify the tool name and ensure the tool is properly registered.`; * Calculates execution time, sets final status, and sends the telemetry event. * * @param telemetryData - Telemetry data to finalize and track (null if telemetry is disabled) - * @param userId - Apify user ID + * @param userId - Apify user ID (string or null if not available) * @param startTime - Timestamp when the tool call started * @param toolStatus - Final status of the tool call ('succeeded', 'failed', or 'aborted') */ private finalizeAndTrackTelemetry( telemetryData: ToolCallTelemetryProperties | null, - userId: string, + userId: string | null, startTime: number, toolStatus: 'succeeded' | 'failed' | 'aborted', ): void { @@ -789,9 +787,9 @@ Please verify the tool name and ensure the tool is properly registered.`; */ private async prepareTelemetryData( tool: HelperTool | ActorTool | ActorMcpTool, mcpSessionId: string | undefined, apifyToken: string, - ): Promise<{ telemetryData: ToolCallTelemetryProperties | null; userId: string }> { + ): Promise<{ telemetryData: ToolCallTelemetryProperties | null; userId: string | null }> { if (this.options.telemetry?.enabled !== true) { - return { telemetryData: null, userId: crypto.randomUUID() }; + return { telemetryData: null, userId: null }; } const toolFullName = tool.type === 'actor' ? tool.actorFullName : tool.name; @@ -813,11 +811,6 @@ Please verify the tool name and ensure the tool is properly registered.`; userId = await getUserIdFromTokenCached(apifyToken, apifyClient); log.debug('Telemetry: fetched userId', { userId }); } - if (!userId) { - userId = crypto.randomUUID(); - log.debug('Telemetry: using random userId', { userId }); - } - const capabilities = this.options.initializeRequestData?.params?.capabilities; const params = this.options.initializeRequestData?.params as InitializeRequest['params']; const telemetryData: ToolCallTelemetryProperties = { diff --git a/src/telemetry.ts b/src/telemetry.ts index 7e53ab1a..4f195280 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -1,3 +1,5 @@ +import * as crypto from 'node:crypto'; + import { Analytics } from '@segment/analytics-node'; import log from '@apify/log'; @@ -57,12 +59,14 @@ export function getOrInitAnalyticsClient(): Analytics | null { /** * Tracks a tool call event to Segment. + * Segment requires either userId OR anonymousId, but not both + * When userId is available, use it; otherwise use anonymousId * * @param userId - Apify user ID (null if not available) * @param properties - Event properties for the tool call */ export function trackToolCall( - userId: string, + userId: string | null, properties: ToolCallTelemetryProperties, ): void { const client = getOrInitAnalyticsClient(); @@ -73,7 +77,7 @@ export function trackToolCall( try { client.track({ - userId, + ...(userId ? { userId } : { anonymousId: crypto.randomUUID() }), event: SEGMENT_EVENTS.TOOL_CALL, properties, }); diff --git a/tests/unit/telemetry.test.ts b/tests/unit/telemetry.test.ts index 5a53559b..a252ecb2 100644 --- a/tests/unit/telemetry.test.ts +++ b/tests/unit/telemetry.test.ts @@ -15,7 +15,7 @@ describe('telemetry', () => { vi.clearAllMocks(); }); - it('should send correct payload structure to Segment', () => { + it('should send correct payload structure to Segment with userId', () => { const userId = 'test-user-123'; const properties = { app: 'mcp' as const, @@ -53,4 +53,35 @@ describe('telemetry', () => { }, }); }); + + it('should use anonymousId when userId is null', () => { + const properties = { + app: 'mcp' as const, + app_version: '0.5.6', + mcp_client_name: 'test-client', + mcp_client_version: '1.0.0', + mcp_protocol_version: '2024-11-05', + mcp_client_capabilities: '{}', + mcp_session_id: 'session-123', + transport_type: 'stdio', + tool_name: 'test-tool', + tool_status: 'succeeded' as const, + tool_exec_time_ms: 100, + tool_call_number: 1, + }; + + trackToolCall(null, properties); + + expect(mockTrack).toHaveBeenCalledTimes(1); + const callArgs = mockTrack.mock.calls[0][0]; + + // Should have anonymousId but not userId + expect(callArgs).toHaveProperty('anonymousId'); + expect(callArgs.anonymousId).toBeDefined(); + expect(typeof callArgs.anonymousId).toBe('string'); + expect(callArgs.anonymousId.length).toBeGreaterThan(0); + expect(callArgs).not.toHaveProperty('userId'); + expect(callArgs.event).toBe('MCP Tool Call'); + expect(callArgs.properties).toEqual(properties); + }); }); From 17c18607a240fafdd05eed5c80afb40e5b779c32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Spilka?= Date: Tue, 25 Nov 2025 09:58:49 +0100 Subject: [PATCH 42/46] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: JiΕ™Γ­ Moravčík --- src/telemetry.ts | 6 +----- src/utils/userid-cache.ts | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/telemetry.ts b/src/telemetry.ts index 4f195280..f360234f 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -71,12 +71,8 @@ export function trackToolCall( ): void { const client = getOrInitAnalyticsClient(); - if (!client) { - return; - } - try { - client.track({ + client?.track({ ...(userId ? { userId } : { anonymousId: crypto.randomUUID() }), event: SEGMENT_EVENTS.TOOL_CALL, properties, diff --git a/src/utils/userid-cache.ts b/src/utils/userid-cache.ts index 4b1e3e08..685ea326 100644 --- a/src/utils/userid-cache.ts +++ b/src/utils/userid-cache.ts @@ -8,7 +8,7 @@ import { TTLLRUCache } from './ttl-lru.js'; const userIdCache = new TTLLRUCache(USER_CACHE_MAX_SIZE, USER_CACHE_TTL_SECS); /** - * Gets user info from token, using cache to avoid repeated API calls + * Gets user ID from token, using cache to avoid repeated API calls * Token is hashed before caching to avoid storing raw tokens * Returns userId or null if not found */ From 8c05a447ae34fbdbb8361c051bd05cbed1af4054 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Tue, 25 Nov 2025 11:37:57 +0100 Subject: [PATCH 43/46] fix: add units to analyticsClient creation. Replace `sessionToolCallCounters` with LRUCache, add telemetry as class instance variables --- src/const.ts | 11 +++ src/mcp/server.ts | 84 +++++++++++-------- src/telemetry.ts | 22 +++-- .../unit/mcp.server.tool-call-counter.test.ts | 29 ++++--- tests/unit/telemetry.test.ts | 4 +- 5 files changed, 92 insertions(+), 58 deletions(-) diff --git a/src/const.ts b/src/const.ts index c597452a..99c5600c 100644 --- a/src/const.ts +++ b/src/const.ts @@ -78,6 +78,8 @@ export const MCP_SERVER_CACHE_MAX_SIZE = 500; export const MCP_SERVER_CACHE_TTL_SECS = 30 * 60; // 30 minutes export const USER_CACHE_MAX_SIZE = 200; export const USER_CACHE_TTL_SECS = 60 * 60; // 1 hour +export const SESSION_TOOL_CALL_COUNTER_CACHE_MAX_SIZE = 200; +export const SESSION_TOOL_CALL_COUNTER_CACHE_TTL_SECS = 60 * 60; // 1 hour export const ACTOR_PRICING_MODEL = { /** Rental Actors */ @@ -114,4 +116,13 @@ export const TELEMETRY_ENV = { } as const; export type TelemetryEnv = (typeof TELEMETRY_ENV)[keyof typeof TELEMETRY_ENV]; +export const DEFAULT_TELEMETRY_ENABLED = true; export const DEFAULT_TELEMETRY_ENV: TelemetryEnv = TELEMETRY_ENV.PROD; + +// We are using the same values as apify-core for consistency (despite that we ship events of different types). +// https://github.com/apify/apify-core/blob/2284766c122c6ac5bc4f27ec28051f4057d6f9c0/src/packages/analytics/src/server/segment.ts#L28 +// Reasoning from the apify-core: +// Flush at 50 events to avoid sending too many small requests (default is 15) +export const SEGMENT_FLUSH_AT_EVENTS = 50; +// Flush interval in milliseconds (default is 10000) +export const SEGMENT_FLUSH_INTERVAL_MS = 5_000; diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 46cbec53..1180d886 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -27,12 +27,17 @@ import log from '@apify/log'; import { ApifyClient } from '../apify-client.js'; import { + DEFAULT_TELEMETRY_ENABLED, + DEFAULT_TELEMETRY_ENV, HelperTools, SERVER_NAME, SERVER_VERSION, + SESSION_TOOL_CALL_COUNTER_CACHE_MAX_SIZE, + SESSION_TOOL_CALL_COUNTER_CACHE_TTL_SECS, SKYFIRE_PAY_ID_PROPERTY_DESCRIPTION, SKYFIRE_README_CONTENT, SKYFIRE_TOOL_INSTRUCTIONS, + type TelemetryEnv, } from '../const.js'; import { prompts } from '../prompts/index.js'; import { getTelemetryEnv, trackToolCall } from '../telemetry.js'; @@ -43,6 +48,7 @@ import type { ActorsMcpServerOptions, ActorTool, HelperTool, + ToolCallCounterStore, ToolCallTelemetryProperties, ToolEntry, } from '../types.js'; @@ -52,6 +58,7 @@ import { logHttpError } from '../utils/logging.js'; import { buildMCPResponse } from '../utils/mcp.js'; import { createProgressTracker } from '../utils/progress.js'; import { cloneToolEntry, getToolPublicFieldOnly } from '../utils/tools.js'; +import { TTLLRUCache } from '../utils/ttl-lru.js'; import { getUserIdFromTokenCached } from '../utils/userid-cache.js'; import { getPackageVersion } from '../utils/version.js'; import { connectMCPClient } from './client.js'; @@ -70,8 +77,17 @@ export class ActorsMcpServer { private sigintHandler: (() => Promise) | undefined; private currentLogLevel = 'info'; public readonly options: ActorsMcpServerOptions; - // In-memory storage for tool call counters (used when toolCallCounterStore is not provided) - private sessionToolCallCounters = new Map(); + + // Telemetry configuration (resolved from options and env vars in setupTelemetry) + private telemetryEnabled: boolean | null = null; + private telemetryEnv: TelemetryEnv = DEFAULT_TELEMETRY_ENV; + private toolCallCountStore: ToolCallCounterStore | undefined; + + // In-memory storage for tool call counters (used when toolCallCountStore is not provided) + private sessionToolCallCounters = new TTLLRUCache( + SESSION_TOOL_CALL_COUNTER_CACHE_MAX_SIZE, + SESSION_TOOL_CALL_COUNTER_CACHE_TTL_SECS, + ); constructor(options: ActorsMcpServerOptions = {}) { this.options = options; @@ -114,49 +130,47 @@ export class ActorsMcpServer { * @returns Promise resolving to the new counter value (after increment) */ private async getAndIncrementToolCallCounter(sessionId: string): Promise { - return await this.options.telemetry!.toolCallCountStore!.getAndIncrement(sessionId); + return await this.toolCallCountStore?.getAndIncrement(sessionId) ?? 0; } /** * Telemetry configuration with precedence: explicit options > env vars > defaults */ private setupTelemetry() { - // Initialize telemetry object if not present - if (!this.options.telemetry) { - this.options.telemetry = {}; - } - - // Configure telemetryEnabled: explicit option > env var > default (true) - const telemetryEnabled = parseBooleanFromString(this.options.telemetry.enabled); - if (telemetryEnabled === undefined) { - const envEnabled = parseBooleanFromString(process.env.TELEMETRY_ENABLED); - this.options.telemetry.enabled = envEnabled !== undefined ? envEnabled : true; + const explicitEnabled = parseBooleanFromString(this.options.telemetry?.enabled); + if (explicitEnabled !== undefined) { + this.telemetryEnabled = explicitEnabled; } else { - this.options.telemetry.enabled = telemetryEnabled; + const envEnabled = parseBooleanFromString(process.env.TELEMETRY_ENABLED); + this.telemetryEnabled = envEnabled ?? DEFAULT_TELEMETRY_ENABLED; } - // Configure telemetryEnv: explicit option > env var > default ('PROD') - // getTelemetryEnv always returns a value (defaults to 'PROD'), so no need for undefined check - const envVarEnv = process.env.TELEMETRY_ENV; - const explicitEnv = this.options.telemetry.env; - this.options.telemetry.env = getTelemetryEnv(explicitEnv ?? envVarEnv); + // Setup tool call counter store only if telemetry is enabled + if (this.telemetryEnabled) { + // Configure telemetryEnv: explicit option > env var > default ('PROD') + this.telemetryEnv = getTelemetryEnv(this.options.telemetry?.env ?? process.env.TELEMETRY_ENV); - // If telemetry is enabled, ensure telemetryEnv is set - if (this.options.telemetry.enabled && this.options.telemetry.env === undefined) { - this.options.telemetry.env = getTelemetryEnv(undefined); + if (this.options.telemetry?.toolCallCountStore) { + this.toolCallCountStore = this.options.telemetry.toolCallCountStore; + } else { + this.toolCallCountStore = { + getAndIncrement: async (sessionId: string): Promise => { + const current = this.sessionToolCallCounters.get(sessionId) ?? 0; + const newValue = current + 1; + this.sessionToolCallCounters.set(sessionId, newValue); + return newValue; + }, + }; + } } + } - // Provide default in-memory store if not provided - if (!this.options.telemetry.toolCallCountStore) { - this.options.telemetry.toolCallCountStore = { - getAndIncrement: async (sessionId: string): Promise => { - const current = this.sessionToolCallCounters.get(sessionId) || 0; - const newValue = current + 1; - this.sessionToolCallCounters.set(sessionId, newValue); - return newValue; - }, - }; - } + /** + * Gets the tool call counter store (for testing purposes) + * @internal + */ + public getToolCallCountStore(): ToolCallCounterStore | undefined { + return this.toolCallCountStore; } /** @@ -779,7 +793,7 @@ Please verify the tool name and ensure the tool is properly registered.`; tool_status: toolStatus, tool_exec_time_ms: execTime, }; - trackToolCall(userId, finalizedTelemetryData); + trackToolCall(userId, this.telemetryEnv, finalizedTelemetryData); } /* @@ -788,7 +802,7 @@ Please verify the tool name and ensure the tool is properly registered.`; private async prepareTelemetryData( tool: HelperTool | ActorTool | ActorMcpTool, mcpSessionId: string | undefined, apifyToken: string, ): Promise<{ telemetryData: ToolCallTelemetryProperties | null; userId: string | null }> { - if (this.options.telemetry?.enabled !== true) { + if (!this.telemetryEnabled) { return { telemetryData: null, userId: null }; } diff --git a/src/telemetry.ts b/src/telemetry.ts index f360234f..3e46964f 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -4,7 +4,13 @@ import { Analytics } from '@segment/analytics-node'; import log from '@apify/log'; -import { DEFAULT_TELEMETRY_ENV, TELEMETRY_ENV, type TelemetryEnv } from './const.js'; +import { + DEFAULT_TELEMETRY_ENV, + SEGMENT_FLUSH_AT_EVENTS, + SEGMENT_FLUSH_INTERVAL_MS, + TELEMETRY_ENV, + type TelemetryEnv, +} from './const.js'; import type { ToolCallTelemetryProperties } from './types.js'; const DEV_WRITE_KEY = '9rPHlMtxX8FJhilGEwkfUoZ0uzWxnzcT'; @@ -38,16 +44,14 @@ let analyticsClient: Analytics | null = null; * * @returns Analytics client instance or null if initialization failed */ -export function getOrInitAnalyticsClient(): Analytics | null { +export function getOrInitAnalyticsClient(telemetryEnv: TelemetryEnv): Analytics | null { if (!analyticsClient) { try { - const env = getTelemetryEnv(process.env.TELEMETRY_ENV); - const writeKey = env === TELEMETRY_ENV.PROD ? PROD_WRITE_KEY : DEV_WRITE_KEY; - + const writeKey = telemetryEnv === TELEMETRY_ENV.PROD ? PROD_WRITE_KEY : DEV_WRITE_KEY; analyticsClient = new Analytics({ writeKey, - flushAt: 50, - flushInterval: 5000, + flushAt: SEGMENT_FLUSH_AT_EVENTS, + flushInterval: SEGMENT_FLUSH_INTERVAL_MS, }); } catch (error) { log.error('Segment initialization failed', { error }); @@ -63,13 +67,15 @@ export function getOrInitAnalyticsClient(): Analytics | null { * When userId is available, use it; otherwise use anonymousId * * @param userId - Apify user ID (null if not available) + * @param telemetryEnv - Telemetry environment * @param properties - Event properties for the tool call */ export function trackToolCall( userId: string | null, + telemetryEnv: TelemetryEnv, properties: ToolCallTelemetryProperties, ): void { - const client = getOrInitAnalyticsClient(); + const client = getOrInitAnalyticsClient(telemetryEnv); try { client?.track({ diff --git a/tests/unit/mcp.server.tool-call-counter.test.ts b/tests/unit/mcp.server.tool-call-counter.test.ts index d4eb00cf..29fd7723 100644 --- a/tests/unit/mcp.server.tool-call-counter.test.ts +++ b/tests/unit/mcp.server.tool-call-counter.test.ts @@ -7,47 +7,49 @@ describe('ActorsMcpServer tool call counter', () => { describe('default in-memory store', () => { it('should increment counter correctly for multiple calls', async () => { const server = new ActorsMcpServer({ setupSigintHandler: false }); - // Default store should be available after construction - const store = server.options.telemetry!.toolCallCountStore!; + // Default store is available if telemetry is enabled + const store = server.getToolCallCountStore(); expect(store).toBeDefined(); const sessionId = 'test-session-1'; // First call should return 1 - const firstCall = await store.getAndIncrement(sessionId); + const firstCall = await store!.getAndIncrement(sessionId); expect(firstCall).toBe(1); // Second call should return 2 - const secondCall = await store.getAndIncrement(sessionId); + const secondCall = await store!.getAndIncrement(sessionId); expect(secondCall).toBe(2); // Third call should return 3 - const thirdCall = await store.getAndIncrement(sessionId); + const thirdCall = await store!.getAndIncrement(sessionId); expect(thirdCall).toBe(3); }); it('should have independent counters for different sessions', async () => { const server = new ActorsMcpServer({ setupSigintHandler: false }); - const store = server.options.telemetry!.toolCallCountStore!; + const store = server.getToolCallCountStore(); + expect(store).toBeDefined(); + const sessionId1 = 'session-1'; const sessionId2 = 'session-2'; // Increment session 1 twice - const session1Call1 = await store.getAndIncrement(sessionId1); + const session1Call1 = await store!.getAndIncrement(sessionId1); expect(session1Call1).toBe(1); - const session1Call2 = await store.getAndIncrement(sessionId1); + const session1Call2 = await store!.getAndIncrement(sessionId1); expect(session1Call2).toBe(2); // Session 2 should start at 1 - const session2Call1 = await store.getAndIncrement(sessionId2); + const session2Call1 = await store!.getAndIncrement(sessionId2); expect(session2Call1).toBe(1); // Session 1 should still be at 2 - const session1Call3 = await store.getAndIncrement(sessionId1); + const session1Call3 = await store!.getAndIncrement(sessionId1); expect(session1Call3).toBe(3); // Session 2 should be at 2 - const session2Call2 = await store.getAndIncrement(sessionId2); + const session2Call2 = await store!.getAndIncrement(sessionId2); expect(session2Call2).toBe(2); }); }); @@ -69,8 +71,9 @@ describe('ActorsMcpServer tool call counter', () => { }, }); - const store = server.options.telemetry!.toolCallCountStore!; - const result = await store.getAndIncrement('test-session'); + const store = server.getToolCallCountStore(); + expect(store).toBeDefined(); + const result = await store!.getAndIncrement('test-session'); // Should use custom store logic, not default expect(result).toBe('test-session'.length + 1); diff --git a/tests/unit/telemetry.test.ts b/tests/unit/telemetry.test.ts index a252ecb2..df530116 100644 --- a/tests/unit/telemetry.test.ts +++ b/tests/unit/telemetry.test.ts @@ -32,7 +32,7 @@ describe('telemetry', () => { tool_call_number: 1, }; - trackToolCall(userId, properties); + trackToolCall(userId, 'DEV', properties); expect(mockTrack).toHaveBeenCalledWith({ userId: 'test-user-123', @@ -70,7 +70,7 @@ describe('telemetry', () => { tool_call_number: 1, }; - trackToolCall(null, properties); + trackToolCall(null, 'DEV', properties); expect(mockTrack).toHaveBeenCalledTimes(1); const callArgs = mockTrack.mock.calls[0][0]; From 7bfe2351a6877f0b28f225b52f9ac9f3ae0c5f49 Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Tue, 25 Nov 2025 11:59:23 +0100 Subject: [PATCH 44/46] fix: types, makes telemetry.enabled required --- src/types.ts | 5 +- .../unit/mcp.server.tool-call-counter.test.ts | 47 +++++++++++++------ 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/src/types.ts b/src/types.ts index 3c04c739..50593a6c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -341,9 +341,10 @@ export interface ActorsMcpServerOptions { telemetry?: { /** * Enable or disable telemetry tracking for tool calls. - * Defaults to true when not set. + * Must be explicitly set when telemetry object is provided. + * When telemetry object is omitted entirely, defaults to true (via env var or default). */ - enabled?: boolean; + enabled: boolean; /** * Telemetry environment when telemetry is enabled. * - 'DEV': Use development Segment write key diff --git a/tests/unit/mcp.server.tool-call-counter.test.ts b/tests/unit/mcp.server.tool-call-counter.test.ts index 29fd7723..50ae856b 100644 --- a/tests/unit/mcp.server.tool-call-counter.test.ts +++ b/tests/unit/mcp.server.tool-call-counter.test.ts @@ -8,48 +8,58 @@ describe('ActorsMcpServer tool call counter', () => { it('should increment counter correctly for multiple calls', async () => { const server = new ActorsMcpServer({ setupSigintHandler: false }); // Default store is available if telemetry is enabled - const store = server.getToolCallCountStore(); - expect(store).toBeDefined(); + const toolCallCount = server.getToolCallCountStore(); + expect(toolCallCount).toBeDefined(); + + if (!toolCallCount) { + // TypeScript needs this for type narrowing + expect.fail('toolCallCount should be defined'); + } const sessionId = 'test-session-1'; // First call should return 1 - const firstCall = await store!.getAndIncrement(sessionId); + const firstCall = await toolCallCount.getAndIncrement(sessionId); expect(firstCall).toBe(1); // Second call should return 2 - const secondCall = await store!.getAndIncrement(sessionId); + const secondCall = await toolCallCount.getAndIncrement(sessionId); expect(secondCall).toBe(2); // Third call should return 3 - const thirdCall = await store!.getAndIncrement(sessionId); + const thirdCall = await toolCallCount.getAndIncrement(sessionId); expect(thirdCall).toBe(3); }); it('should have independent counters for different sessions', async () => { const server = new ActorsMcpServer({ setupSigintHandler: false }); - const store = server.getToolCallCountStore(); - expect(store).toBeDefined(); + const toolCallCount = server.getToolCallCountStore(); + expect(toolCallCount).toBeDefined(); + + if (!toolCallCount) { + // TypeScript needs this for type narrowing + expect.fail('toolCallCount should be defined'); + } const sessionId1 = 'session-1'; const sessionId2 = 'session-2'; // Increment session 1 twice - const session1Call1 = await store!.getAndIncrement(sessionId1); + const session1Call1 = await toolCallCount.getAndIncrement(sessionId1); expect(session1Call1).toBe(1); - const session1Call2 = await store!.getAndIncrement(sessionId1); + const session1Call2 = await toolCallCount.getAndIncrement(sessionId1); expect(session1Call2).toBe(2); // Session 2 should start at 1 - const session2Call1 = await store!.getAndIncrement(sessionId2); + const session2Call1 = await toolCallCount.getAndIncrement(sessionId2); expect(session2Call1).toBe(1); // Session 1 should still be at 2 - const session1Call3 = await store!.getAndIncrement(sessionId1); + const session1Call3 = await toolCallCount.getAndIncrement(sessionId1); expect(session1Call3).toBe(3); // Session 2 should be at 2 - const session2Call2 = await store!.getAndIncrement(sessionId2); + const session2Call2 = await toolCallCount.getAndIncrement(sessionId2); expect(session2Call2).toBe(2); }); }); @@ -67,13 +77,20 @@ describe('ActorsMcpServer tool call counter', () => { const server = new ActorsMcpServer({ setupSigintHandler: false, telemetry: { + enabled: true, toolCallCountStore: customStore, }, }); - const store = server.getToolCallCountStore(); - expect(store).toBeDefined(); - const result = await store!.getAndIncrement('test-session'); + const toolCallCount = server.getToolCallCountStore(); + expect(toolCallCount).toBeDefined(); + + if (!toolCallCount) { + // TypeScript needs this for type narrowing + expect.fail('toolCallCount should be defined'); + } + + const result = await toolCallCount.getAndIncrement('test-session'); // Should use custom store logic, not default expect(result).toBe('test-session'.length + 1); From 04482986dc0dd6f7ac349f2b8d9373ddf96e6b1f Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Tue, 25 Nov 2025 15:56:12 +0100 Subject: [PATCH 45/46] fix: disable telemetry for tests --- src/stdio.ts | 1 + tests/helpers.ts | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/stdio.ts b/src/stdio.ts index f322d617..ec9875c4 100644 --- a/src/stdio.ts +++ b/src/stdio.ts @@ -113,6 +113,7 @@ Default: true (enabled)`, choices: [TELEMETRY_ENV.PROD, TELEMETRY_ENV.DEV], default: DEFAULT_TELEMETRY_ENV, hidden: true, + coerce: (arg: string) => arg?.toUpperCase(), describe: `Telemetry environment when telemetry is enabled. Can also be set via TELEMETRY_ENV environment variable. - 'PROD': Send events to production Segment workspace (default) - 'DEV': Send events to development Segment workspace diff --git a/tests/helpers.ts b/tests/helpers.ts index c48a0750..5dc61142 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -36,10 +36,9 @@ function appendSearchParams(url: URL, options?: McpClientOptions): void { if (tools !== undefined) { url.searchParams.append('tools', tools.join(',')); } - // Append telemetry parameters - if (telemetry?.enabled !== undefined) { - url.searchParams.append('telemetry-enabled', telemetry.enabled.toString()); - } + // Append telemetry parameters (default to false for tests when not explicitly set) + const telemetryEnabled = telemetry?.enabled !== undefined ? telemetry.enabled : false; + url.searchParams.append('telemetry-enabled', telemetryEnabled.toString()); } export async function createMcpSseClient( From f703e9098307633250e9d9e2889eaf438f18aeaa Mon Sep 17 00:00:00 2001 From: Jiri Spilka Date: Wed, 26 Nov 2025 11:44:33 +0100 Subject: [PATCH 46/46] fix: remove tool_count_number, it is not required --- src/const.ts | 2 - src/index-internals.ts | 3 +- src/mcp/server.ts | 55 +---------- src/types.ts | 20 ---- .../unit/mcp.server.tool-call-counter.test.ts | 99 ------------------- tests/unit/telemetry.test.ts | 3 - 6 files changed, 2 insertions(+), 180 deletions(-) delete mode 100644 tests/unit/mcp.server.tool-call-counter.test.ts diff --git a/src/const.ts b/src/const.ts index 99c5600c..9887cc9c 100644 --- a/src/const.ts +++ b/src/const.ts @@ -78,8 +78,6 @@ export const MCP_SERVER_CACHE_MAX_SIZE = 500; export const MCP_SERVER_CACHE_TTL_SECS = 30 * 60; // 30 minutes export const USER_CACHE_MAX_SIZE = 200; export const USER_CACHE_TTL_SECS = 60 * 60; // 1 hour -export const SESSION_TOOL_CALL_COUNTER_CACHE_MAX_SIZE = 200; -export const SESSION_TOOL_CALL_COUNTER_CACHE_TTL_SECS = 60 * 60; // 1 hour export const ACTOR_PRICING_MODEL = { /** Rental Actors */ diff --git a/src/index-internals.ts b/src/index-internals.ts index 3e3130e5..0c805246 100644 --- a/src/index-internals.ts +++ b/src/index-internals.ts @@ -8,7 +8,7 @@ import { processParamsGetTools } from './mcp/utils.js'; import { addTool } from './tools/helpers.js'; import { defaultTools, getActorsAsTools, toolCategories, toolCategoriesEnabledByDefault } from './tools/index.js'; import { actorNameToToolName } from './tools/utils.js'; -import type { ToolCallCounterStore, ToolCategory } from './types.js'; +import type { ToolCategory } from './types.js'; import { getExpectedToolNamesByCategories, getToolPublicFieldOnly } from './utils/tools.js'; import { TTLLRUCache } from './utils/ttl-lru.js'; @@ -24,7 +24,6 @@ export { toolCategories, toolCategoriesEnabledByDefault, type ToolCategory, - type ToolCallCounterStore, processParamsGetTools, getActorsAsTools, getToolPublicFieldOnly, diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 1180d886..6f10614e 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -32,8 +32,6 @@ import { HelperTools, SERVER_NAME, SERVER_VERSION, - SESSION_TOOL_CALL_COUNTER_CACHE_MAX_SIZE, - SESSION_TOOL_CALL_COUNTER_CACHE_TTL_SECS, SKYFIRE_PAY_ID_PROPERTY_DESCRIPTION, SKYFIRE_README_CONTENT, SKYFIRE_TOOL_INSTRUCTIONS, @@ -48,7 +46,6 @@ import type { ActorsMcpServerOptions, ActorTool, HelperTool, - ToolCallCounterStore, ToolCallTelemetryProperties, ToolEntry, } from '../types.js'; @@ -58,7 +55,6 @@ import { logHttpError } from '../utils/logging.js'; import { buildMCPResponse } from '../utils/mcp.js'; import { createProgressTracker } from '../utils/progress.js'; import { cloneToolEntry, getToolPublicFieldOnly } from '../utils/tools.js'; -import { TTLLRUCache } from '../utils/ttl-lru.js'; import { getUserIdFromTokenCached } from '../utils/userid-cache.js'; import { getPackageVersion } from '../utils/version.js'; import { connectMCPClient } from './client.js'; @@ -81,13 +77,6 @@ export class ActorsMcpServer { // Telemetry configuration (resolved from options and env vars in setupTelemetry) private telemetryEnabled: boolean | null = null; private telemetryEnv: TelemetryEnv = DEFAULT_TELEMETRY_ENV; - private toolCallCountStore: ToolCallCounterStore | undefined; - - // In-memory storage for tool call counters (used when toolCallCountStore is not provided) - private sessionToolCallCounters = new TTLLRUCache( - SESSION_TOOL_CALL_COUNTER_CACHE_MAX_SIZE, - SESSION_TOOL_CALL_COUNTER_CACHE_TTL_SECS, - ); constructor(options: ActorsMcpServerOptions = {}) { this.options = options; @@ -124,15 +113,6 @@ export class ActorsMcpServer { this.setupResourceHandlers(); } - /** - * Gets and increments the tool call counter for a session. - * @param sessionId - The session ID - * @returns Promise resolving to the new counter value (after increment) - */ - private async getAndIncrementToolCallCounter(sessionId: string): Promise { - return await this.toolCallCountStore?.getAndIncrement(sessionId) ?? 0; - } - /** * Telemetry configuration with precedence: explicit options > env vars > defaults */ @@ -145,34 +125,12 @@ export class ActorsMcpServer { this.telemetryEnabled = envEnabled ?? DEFAULT_TELEMETRY_ENABLED; } - // Setup tool call counter store only if telemetry is enabled + // Configure telemetryEnv: explicit option > env var > default ('PROD') if (this.telemetryEnabled) { - // Configure telemetryEnv: explicit option > env var > default ('PROD') this.telemetryEnv = getTelemetryEnv(this.options.telemetry?.env ?? process.env.TELEMETRY_ENV); - - if (this.options.telemetry?.toolCallCountStore) { - this.toolCallCountStore = this.options.telemetry.toolCallCountStore; - } else { - this.toolCallCountStore = { - getAndIncrement: async (sessionId: string): Promise => { - const current = this.sessionToolCallCounters.get(sessionId) ?? 0; - const newValue = current + 1; - this.sessionToolCallCounters.set(sessionId, newValue); - return newValue; - }, - }; - } } } - /** - * Gets the tool call counter store (for testing purposes) - * @internal - */ - public getToolCallCountStore(): ToolCallCounterStore | undefined { - return this.toolCallCountStore; - } - /** * Returns an array of tool names. * @returns {string[]} - An array of tool names. @@ -808,16 +766,6 @@ Please verify the tool name and ensure the tool is properly registered.`; const toolFullName = tool.type === 'actor' ? tool.actorFullName : tool.name; - // Get or increment tool call counter for this session - let toolCallNumber = 0; - if (mcpSessionId) { - try { - toolCallNumber = await this.getAndIncrementToolCallCounter(mcpSessionId); - } catch (error) { - log.warning('Failed to get tool call counter', { mcpSessionId, error: String(error) }); - } - } - // Get userId from cache or fetch from API let userId: string | null = null; if (apifyToken) { @@ -839,7 +787,6 @@ Please verify the tool name and ensure the tool is properly registered.`; tool_name: toolFullName, tool_status: 'succeeded', // Will be updated in finally tool_exec_time_ms: 0, // Will be calculated in finally - tool_call_number: toolCallNumber, }; return { telemetryData, userId }; diff --git a/src/types.ts b/src/types.ts index 50593a6c..d660d460 100644 --- a/src/types.ts +++ b/src/types.ts @@ -309,20 +309,6 @@ export interface ToolCallTelemetryProperties { tool_name: string; tool_status: 'succeeded' | 'failed' | 'aborted'; tool_exec_time_ms: number; - tool_call_number: number; -} - -/** - * Interface for storing and retrieving tool call counters per session. - * Used for tracking tool call sequence in user journeys. - */ -export interface ToolCallCounterStore { - /** - * Gets and increments the tool call counter for a session atomically. - * @param sessionId - The session ID - * @returns Promise resolving to the new counter value (after increment) - */ - getAndIncrement(sessionId: string): Promise; } /** @@ -351,12 +337,6 @@ export interface ActorsMcpServerOptions { * - 'PROD': Use production Segment write key (default) */ env?: TelemetryEnv; - /** - * Optional store for tool call counters. - * If not provided, uses in-memory storage (suitable for stdio). - * For distributed deployments (HTTP/SSE), provide a Redis-backed implementation. - */ - toolCallCountStore?: ToolCallCounterStore; }; /** * Transport type for telemetry tracking. diff --git a/tests/unit/mcp.server.tool-call-counter.test.ts b/tests/unit/mcp.server.tool-call-counter.test.ts deleted file mode 100644 index 50ae856b..00000000 --- a/tests/unit/mcp.server.tool-call-counter.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { ActorsMcpServer } from '../../src/index.js'; -import type { ToolCallCounterStore } from '../../src/types.js'; - -describe('ActorsMcpServer tool call counter', () => { - describe('default in-memory store', () => { - it('should increment counter correctly for multiple calls', async () => { - const server = new ActorsMcpServer({ setupSigintHandler: false }); - // Default store is available if telemetry is enabled - const toolCallCount = server.getToolCallCountStore(); - expect(toolCallCount).toBeDefined(); - - if (!toolCallCount) { - // TypeScript needs this for type narrowing - expect.fail('toolCallCount should be defined'); - } - - const sessionId = 'test-session-1'; - - // First call should return 1 - const firstCall = await toolCallCount.getAndIncrement(sessionId); - expect(firstCall).toBe(1); - - // Second call should return 2 - const secondCall = await toolCallCount.getAndIncrement(sessionId); - expect(secondCall).toBe(2); - - // Third call should return 3 - const thirdCall = await toolCallCount.getAndIncrement(sessionId); - expect(thirdCall).toBe(3); - }); - - it('should have independent counters for different sessions', async () => { - const server = new ActorsMcpServer({ setupSigintHandler: false }); - const toolCallCount = server.getToolCallCountStore(); - expect(toolCallCount).toBeDefined(); - - if (!toolCallCount) { - // TypeScript needs this for type narrowing - expect.fail('toolCallCount should be defined'); - } - - const sessionId1 = 'session-1'; - const sessionId2 = 'session-2'; - - // Increment session 1 twice - const session1Call1 = await toolCallCount.getAndIncrement(sessionId1); - expect(session1Call1).toBe(1); - const session1Call2 = await toolCallCount.getAndIncrement(sessionId1); - expect(session1Call2).toBe(2); - - // Session 2 should start at 1 - const session2Call1 = await toolCallCount.getAndIncrement(sessionId2); - expect(session2Call1).toBe(1); - - // Session 1 should still be at 2 - const session1Call3 = await toolCallCount.getAndIncrement(sessionId1); - expect(session1Call3).toBe(3); - - // Session 2 should be at 2 - const session2Call2 = await toolCallCount.getAndIncrement(sessionId2); - expect(session2Call2).toBe(2); - }); - }); - - describe('custom store', () => { - it('should use provided custom store instead of default', async () => { - const customStore: ToolCallCounterStore = { - getAndIncrement: async (sessionId: string) => { - // Custom implementation that returns a fixed value - // This is just for testing that custom store is used - return sessionId.length + 1; - }, - }; - - const server = new ActorsMcpServer({ - setupSigintHandler: false, - telemetry: { - enabled: true, - toolCallCountStore: customStore, - }, - }); - - const toolCallCount = server.getToolCallCountStore(); - expect(toolCallCount).toBeDefined(); - - if (!toolCallCount) { - // TypeScript needs this for type narrowing - expect.fail('toolCallCount should be defined'); - } - - const result = await toolCallCount.getAndIncrement('test-session'); - - // Should use custom store logic, not default - expect(result).toBe('test-session'.length + 1); - }); - }); -}); diff --git a/tests/unit/telemetry.test.ts b/tests/unit/telemetry.test.ts index df530116..5a10535a 100644 --- a/tests/unit/telemetry.test.ts +++ b/tests/unit/telemetry.test.ts @@ -29,7 +29,6 @@ describe('telemetry', () => { tool_name: 'test-tool', tool_status: 'succeeded' as const, tool_exec_time_ms: 100, - tool_call_number: 1, }; trackToolCall(userId, 'DEV', properties); @@ -49,7 +48,6 @@ describe('telemetry', () => { tool_name: 'test-tool', tool_status: 'succeeded', tool_exec_time_ms: 100, - tool_call_number: 1, }, }); }); @@ -67,7 +65,6 @@ describe('telemetry', () => { tool_name: 'test-tool', tool_status: 'succeeded' as const, tool_exec_time_ms: 100, - tool_call_number: 1, }; trackToolCall(null, 'DEV', properties);