diff --git a/docs/tools/fetch.md b/docs/tools/fetch.md
new file mode 100644
index 0000000..612c993
--- /dev/null
+++ b/docs/tools/fetch.md
@@ -0,0 +1,102 @@
+# Fetch Tool
+
+The `fetch` tool allows MyCoder to make HTTP requests to external APIs. It uses the native Node.js fetch API and includes robust error handling capabilities.
+
+## Basic Usage
+
+```javascript
+const response = await fetch({
+ method: 'GET',
+ url: 'https://api.example.com/data',
+ headers: {
+ Authorization: 'Bearer token123',
+ },
+});
+
+console.log(response.status); // HTTP status code
+console.log(response.body); // Response body
+```
+
+## Parameters
+
+| Parameter | Type | Required | Description |
+| ---------- | ------- | -------- | ------------------------------------------------------------------------- |
+| method | string | Yes | HTTP method to use (GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS) |
+| url | string | Yes | URL to make the request to |
+| params | object | No | Query parameters to append to the URL |
+| body | object | No | Request body (for POST, PUT, PATCH requests) |
+| headers | object | No | Request headers |
+| maxRetries | number | No | Maximum number of retries for 4xx errors (default: 3, max: 5) |
+| retryDelay | number | No | Initial delay in ms before retrying (default: 1000, min: 100, max: 30000) |
+| slowMode | boolean | No | Enable slow mode to avoid rate limits (default: false) |
+
+## Error Handling
+
+The fetch tool includes sophisticated error handling for different types of HTTP errors:
+
+### 400 Bad Request Errors
+
+When a 400 Bad Request error occurs, the fetch tool will automatically retry the request with exponential backoff. This helps handle temporary issues or malformed requests.
+
+```javascript
+// Fetch with custom retry settings for Bad Request errors
+const response = await fetch({
+ method: 'GET',
+ url: 'https://api.example.com/data',
+ maxRetries: 2, // Retry up to 2 times (3 requests total)
+ retryDelay: 500, // Start with a 500ms delay, then increase exponentially
+});
+```
+
+### 429 Rate Limit Errors
+
+For 429 Rate Limit Exceeded errors, the fetch tool will:
+
+1. Automatically retry with exponential backoff
+2. Respect the `Retry-After` header if provided by the server
+3. Switch to "slow mode" to prevent further rate limit errors
+
+```javascript
+// Fetch with rate limit handling
+const response = await fetch({
+ method: 'GET',
+ url: 'https://api.example.com/data',
+ maxRetries: 5, // Retry up to 5 times for rate limit errors
+ retryDelay: 1000, // Start with a 1 second delay
+});
+
+// Check if slow mode was enabled due to rate limiting
+if (response.slowModeEnabled) {
+ console.log('Slow mode was enabled to handle rate limits');
+}
+```
+
+### Preemptive Slow Mode
+
+You can enable slow mode preemptively to avoid hitting rate limits in the first place:
+
+```javascript
+// Start with slow mode enabled
+const response = await fetch({
+ method: 'GET',
+ url: 'https://api.example.com/data',
+ slowMode: true, // Enable slow mode from the first request
+});
+```
+
+### Network Errors
+
+The fetch tool also handles network errors (such as connection issues) with the same retry mechanism.
+
+## Response Object
+
+The fetch tool returns an object with the following properties:
+
+| Property | Type | Description |
+| --------------- | ---------------- | ------------------------------------------------------------------ |
+| status | number | HTTP status code |
+| statusText | string | HTTP status text |
+| headers | object | Response headers |
+| body | string or object | Response body (parsed as JSON if content-type is application/json) |
+| retries | number | Number of retries performed (if any) |
+| slowModeEnabled | boolean | Whether slow mode was enabled |
diff --git a/packages/agent/src/tools/fetch/fetch.test.ts b/packages/agent/src/tools/fetch/fetch.test.ts
new file mode 100644
index 0000000..df4ec91
--- /dev/null
+++ b/packages/agent/src/tools/fetch/fetch.test.ts
@@ -0,0 +1,302 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+
+import { ToolContext } from '../../core/types.js';
+import { Logger } from '../../utils/logger.js';
+
+import { fetchTool } from './fetch.js';
+
+// Mock setTimeout to resolve immediately for all sleep calls
+vi.mock('node:timers', () => ({
+ setTimeout: (callback: () => void) => {
+ callback();
+ return { unref: vi.fn() };
+ },
+}));
+
+describe('fetchTool', () => {
+ // Create a mock logger
+ const mockLogger = {
+ debug: vi.fn(),
+ log: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ info: vi.fn(),
+ prefix: '',
+ logLevel: 'debug',
+ logLevelIndex: 0,
+ name: 'test-logger',
+ child: vi.fn(),
+ withPrefix: vi.fn(),
+ setLevel: vi.fn(),
+ nesting: 0,
+ listeners: [],
+ emitMessages: vi.fn(),
+ } as unknown as Logger;
+
+ // Create a mock ToolContext
+ const mockContext = {
+ logger: mockLogger,
+ workingDirectory: '/test',
+ headless: true,
+ userSession: false, // Use boolean as required by type
+ tokenTracker: { remaining: 1000, used: 0, total: 1000 },
+ abortSignal: new AbortController().signal,
+ shellManager: {} as any,
+ sessionManager: {} as any,
+ agentManager: {} as any,
+ history: [],
+ statusUpdate: vi.fn(),
+ captureOutput: vi.fn(),
+ isSubAgent: false,
+ parentAgentId: null,
+ subAgentMode: 'disabled',
+ } as unknown as ToolContext;
+
+ // Mock global fetch
+ let originalFetch: typeof global.fetch;
+ let mockFetch: ReturnType<typeof vi.fn>;
+
+ beforeEach(() => {
+ originalFetch = global.fetch;
+ mockFetch = vi.fn();
+ global.fetch = mockFetch as any;
+ vi.clearAllMocks();
+ });
+
+ afterEach(() => {
+ global.fetch = originalFetch;
+ });
+
+ it('should make a successful request', async () => {
+ const mockResponse = {
+ status: 200,
+ statusText: 'OK',
+ headers: new Headers({ 'content-type': 'application/json' }),
+ json: async () => ({ data: 'test' }),
+ text: async () => 'test',
+ ok: true,
+ };
+ mockFetch.mockResolvedValueOnce(mockResponse);
+
+ const result = await fetchTool.execute(
+ { method: 'GET', url: 'https://example.com' },
+ mockContext,
+ );
+
+ expect(result).toEqual({
+ status: 200,
+ statusText: 'OK',
+ headers: { 'content-type': 'application/json' },
+ body: { data: 'test' },
+ retries: 0,
+ slowModeEnabled: false,
+ });
+ expect(mockFetch).toHaveBeenCalledTimes(1);
+ });
+
+ it('should retry on 400 Bad Request error', async () => {
+ const mockErrorResponse = {
+ status: 400,
+ statusText: 'Bad Request',
+ headers: new Headers({}),
+ text: async () => 'Bad Request',
+ ok: false,
+ };
+
+ const mockSuccessResponse = {
+ status: 200,
+ statusText: 'OK',
+ headers: new Headers({ 'content-type': 'application/json' }),
+ json: async () => ({ data: 'success' }),
+ text: async () => 'success',
+ ok: true,
+ };
+
+ // First request fails, second succeeds
+ mockFetch.mockResolvedValueOnce(mockErrorResponse);
+ mockFetch.mockResolvedValueOnce(mockSuccessResponse);
+
+ const result = await fetchTool.execute(
+ {
+ method: 'GET',
+ url: 'https://example.com',
+ maxRetries: 2,
+ retryDelay: 100,
+ },
+ mockContext,
+ );
+
+ expect(result).toEqual({
+ status: 200,
+ statusText: 'OK',
+ headers: { 'content-type': 'application/json' },
+ body: { data: 'success' },
+ retries: 1,
+ slowModeEnabled: false,
+ });
+ expect(mockFetch).toHaveBeenCalledTimes(2);
+ expect(mockLogger.warn).toHaveBeenCalledWith(
+ expect.stringContaining('400 Bad Request Error'),
+ );
+ });
+
+ it('should implement exponential backoff for 429 Rate Limit errors', async () => {
+ const mockRateLimitResponse = {
+ status: 429,
+ statusText: 'Too Many Requests',
+ headers: new Headers({ 'retry-after': '2' }), // 2 seconds
+ text: async () => 'Rate Limit Exceeded',
+ ok: false,
+ };
+
+ const mockSuccessResponse = {
+ status: 200,
+ statusText: 'OK',
+ headers: new Headers({ 'content-type': 'application/json' }),
+ json: async () => ({ data: 'success after rate limit' }),
+ text: async () => 'success',
+ ok: true,
+ };
+
+ mockFetch.mockResolvedValueOnce(mockRateLimitResponse);
+ mockFetch.mockResolvedValueOnce(mockSuccessResponse);
+
+ const result = await fetchTool.execute(
+ {
+ method: 'GET',
+ url: 'https://example.com',
+ maxRetries: 2,
+ retryDelay: 100,
+ },
+ mockContext,
+ );
+
+ expect(result).toEqual({
+ status: 200,
+ statusText: 'OK',
+ headers: { 'content-type': 'application/json' },
+ body: { data: 'success after rate limit' },
+ retries: 1,
+ slowModeEnabled: true, // Slow mode should be enabled after a rate limit error
+ });
+ expect(mockFetch).toHaveBeenCalledTimes(2);
+ expect(mockLogger.warn).toHaveBeenCalledWith(
+ expect.stringContaining('429 Rate Limit Exceeded'),
+ );
+ });
+
+ it('should throw an error after maximum retries', async () => {
+ const mockErrorResponse = {
+ status: 400,
+ statusText: 'Bad Request',
+ headers: new Headers({}),
+ text: async () => 'Bad Request',
+ ok: false,
+ };
+
+ // All requests fail
+ mockFetch.mockResolvedValue(mockErrorResponse);
+
+ await expect(
+ fetchTool.execute(
+ {
+ method: 'GET',
+ url: 'https://example.com',
+ maxRetries: 2,
+ retryDelay: 100,
+ },
+ mockContext,
+ ),
+ ).rejects.toThrow('Failed after 2 retries');
+
+ expect(mockFetch).toHaveBeenCalledTimes(3); // Initial + 2 retries
+ expect(mockLogger.warn).toHaveBeenCalledTimes(2); // Two retry warnings
+ });
+
+ it('should respect retry-after header with timestamp', async () => {
+ const futureDate = new Date(Date.now() + 3000).toUTCString();
+ const mockRateLimitResponse = {
+ status: 429,
+ statusText: 'Too Many Requests',
+ headers: new Headers({ 'retry-after': futureDate }),
+ text: async () => 'Rate Limit Exceeded',
+ ok: false,
+ };
+
+ const mockSuccessResponse = {
+ status: 200,
+ statusText: 'OK',
+ headers: new Headers({ 'content-type': 'application/json' }),
+ json: async () => ({ data: 'success' }),
+ text: async () => 'success',
+ ok: true,
+ };
+
+ mockFetch.mockResolvedValueOnce(mockRateLimitResponse);
+ mockFetch.mockResolvedValueOnce(mockSuccessResponse);
+
+ const result = await fetchTool.execute(
+ {
+ method: 'GET',
+ url: 'https://example.com',
+ maxRetries: 2,
+ retryDelay: 100,
+ },
+ mockContext,
+ );
+
+ expect(result.status).toBe(200);
+ expect(result.slowModeEnabled).toBe(true);
+ expect(mockFetch).toHaveBeenCalledTimes(2);
+ });
+
+ it('should handle network errors with retries', async () => {
+ mockFetch.mockRejectedValueOnce(new Error('Network error'));
+ mockFetch.mockResolvedValueOnce({
+ status: 200,
+ statusText: 'OK',
+ headers: new Headers({ 'content-type': 'application/json' }),
+ json: async () => ({ data: 'success after network error' }),
+ text: async () => 'success',
+ ok: true,
+ });
+
+ const result = await fetchTool.execute(
+ {
+ method: 'GET',
+ url: 'https://example.com',
+ maxRetries: 2,
+ retryDelay: 100,
+ },
+ mockContext,
+ );
+
+ expect(result.status).toBe(200);
+ expect(result.retries).toBe(1);
+ expect(mockFetch).toHaveBeenCalledTimes(2);
+ expect(mockLogger.error).toHaveBeenCalledWith(
+ expect.stringContaining('Request failed'),
+ );
+ });
+
+ it('should use slow mode when explicitly enabled', async () => {
+ // First request succeeds
+ mockFetch.mockResolvedValueOnce({
+ status: 200,
+ statusText: 'OK',
+ headers: new Headers({ 'content-type': 'application/json' }),
+ json: async () => ({ data: 'success in slow mode' }),
+ text: async () => 'success',
+ ok: true,
+ });
+
+ const result = await fetchTool.execute(
+ { method: 'GET', url: 'https://example.com', slowMode: true },
+ mockContext,
+ );
+
+ expect(result.status).toBe(200);
+ expect(result.slowModeEnabled).toBe(true);
+ expect(mockFetch).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/packages/agent/src/tools/fetch/fetch.ts b/packages/agent/src/tools/fetch/fetch.ts
index 5757ad5..4372bae 100644
--- a/packages/agent/src/tools/fetch/fetch.ts
+++ b/packages/agent/src/tools/fetch/fetch.ts
@@ -19,6 +19,23 @@ const parameterSchema = z.object({
.optional()
.describe('Optional request body (for POST, PUT, PATCH requests)'),
headers: z.record(z.string()).optional().describe('Optional request headers'),
+ // New parameters for error handling
+ maxRetries: z
+ .number()
+ .min(0)
+ .max(5)
+ .optional()
+ .describe('Maximum number of retries for 4xx errors (default: 3)'),
+ retryDelay: z
+ .number()
+ .min(100)
+ .max(30000)
+ .optional()
+ .describe('Initial delay in ms before retrying (default: 1000)'),
+ slowMode: z
+ .boolean()
+ .optional()
+ .describe('Enable slow mode to avoid rate limits (default: false)'),
});
const returnSchema = z
@@ -27,12 +44,38 @@ const returnSchema = z
statusText: z.string(),
headers: z.record(z.string()),
body: z.union([z.string(), z.record(z.any())]),
+ retries: z.number().optional(),
+ slowModeEnabled: z.boolean().optional(),
})
.describe('HTTP response including status, headers, and body');
type Parameters = z.infer<typeof parameterSchema>;
type ReturnType = z.infer<typeof returnSchema>;
+/**
+ * Sleep for a specified number of milliseconds
+ * @param ms Milliseconds to sleep
+ * @internal
+ */
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Calculate exponential backoff delay with jitter
+ * @param attempt Current attempt number (0-based)
+ * @param baseDelay Base delay in milliseconds
+ * @returns Delay in milliseconds with jitter
+ */
+const calculateBackoff = (attempt: number, baseDelay: number): number => {
+ // Calculate exponential backoff: baseDelay * 2^attempt
+ const expBackoff = baseDelay * Math.pow(2, attempt);
+
+ // Add jitter (±20%) to avoid thundering herd problem
+ const jitter = expBackoff * 0.2 * (Math.random() * 2 - 1);
+
+ // Return backoff with jitter, capped at 30 seconds
+ return Math.min(expBackoff + jitter, 30000);
+};
+
export const fetchTool: Tool<Parameters, ReturnType> = {
name: 'fetch',
description:
@@ -43,65 +86,191 @@ export const fetchTool: Tool<Parameters, ReturnType> = {
parametersJsonSchema: zodToJsonSchema(parameterSchema),
returnsJsonSchema: zodToJsonSchema(returnSchema),
execute: async (
- { method, url, params, body, headers }: Parameters,
+ {
+ method,
+ url,
+ params,
+ body,
+ headers,
+ maxRetries = 3,
+ retryDelay = 1000,
+ slowMode = false,
+ }: Parameters,
{ logger },
): Promise<ReturnType> => {
- logger.debug(`Starting ${method} request to ${url}`);
- const urlObj = new URL(url);
-
- // Add query parameters
- if (params) {
- logger.debug('Adding query parameters:', params);
- Object.entries(params).forEach(([key, value]) =>
- urlObj.searchParams.append(key, value as string),
- );
- }
+ let retries = 0;
+ let slowModeEnabled = slowMode;
+ let lastError: Error | null = null;
- // Prepare request options
- const options = {
- method,
- headers: {
- ...(body &&
- !['GET', 'HEAD'].includes(method) && {
- 'content-type': 'application/json',
- }),
- ...headers,
- },
- ...(body &&
- !['GET', 'HEAD'].includes(method) && {
- body: JSON.stringify(body),
- }),
- };
-
- logger.debug('Request options:', options);
- const response = await fetch(urlObj.toString(), options);
- logger.debug(
- `Request completed with status ${response.status} ${response.statusText}`,
- );
+ while (retries <= maxRetries) {
+ try {
+ // If in slow mode, add a delay before making the request
+ if (slowModeEnabled && retries > 0) {
+ const slowModeDelay = 2000; // 2 seconds delay in slow mode
+ logger.debug(
+ `Slow mode enabled, waiting ${slowModeDelay}ms before request`,
+ );
+ await sleep(slowModeDelay);
+ }
+
+ logger.debug(
+ `Starting ${method} request to ${url}${retries > 0 ? ` (retry ${retries}/${maxRetries})` : ''}`,
+ );
+ const urlObj = new URL(url);
- const contentType = response.headers.get('content-type');
- const responseBody = contentType?.includes('application/json')
- ? await response.json()
- : await response.text();
+ // Add query parameters
+ if (params) {
+ logger.debug('Adding query parameters:', params);
+ Object.entries(params).forEach(([key, value]) =>
+ urlObj.searchParams.append(key, value as string),
+ );
+ }
- logger.debug('Response content-type:', contentType);
+ // Prepare request options
+ const options = {
+ method,
+ headers: {
+ ...(body &&
+ !['GET', 'HEAD'].includes(method) && {
+ 'content-type': 'application/json',
+ }),
+ ...headers,
+ },
+ ...(body &&
+ !['GET', 'HEAD'].includes(method) && {
+ body: JSON.stringify(body),
+ }),
+ };
- return {
- status: response.status,
- statusText: response.statusText,
- headers: Object.fromEntries(response.headers),
- body: responseBody as ReturnType['body'],
- };
+ logger.debug('Request options:', options);
+ const response = await fetch(urlObj.toString(), options);
+ logger.debug(
+ `Request completed with status ${response.status} ${response.statusText}`,
+ );
+
+ // Handle different 4xx errors
+ if (response.status >= 400 && response.status < 500) {
+ if (response.status === 400) {
+ // Bad Request - might be a temporary issue or problem with the request
+ if (retries < maxRetries) {
+ retries++;
+ const delay = calculateBackoff(retries, retryDelay);
+ logger.warn(
+ `400 Bad Request Error. Retrying in ${Math.round(delay)}ms (${retries}/${maxRetries})`,
+ );
+ await sleep(delay);
+ continue;
+ } else {
+ // Throw an error after max retries for bad request
+ throw new Error(
+ `Failed after ${maxRetries} retries: Bad Request (400)`,
+ );
+ }
+ } else if (response.status === 429) {
+ // Rate Limit Exceeded - implement exponential backoff
+ if (retries < maxRetries) {
+ retries++;
+ // Enable slow mode after the first rate limit error
+ slowModeEnabled = true;
+
+ // Get retry-after header if available, or use exponential backoff
+ const retryAfter = response.headers.get('retry-after');
+ let delay: number;
+
+ if (retryAfter) {
+ // If retry-after contains a timestamp
+ if (isNaN(Number(retryAfter))) {
+ const retryDate = new Date(retryAfter).getTime();
+ delay = retryDate - Date.now();
+ } else {
+ // If retry-after contains seconds
+ delay = parseInt(retryAfter, 10) * 1000;
+ }
+ } else {
+ // Use exponential backoff if no retry-after header
+ delay = calculateBackoff(retries, retryDelay);
+ }
+
+ logger.warn(
+ `429 Rate Limit Exceeded. Enabling slow mode and retrying in ${Math.round(delay)}ms (${retries}/${maxRetries})`,
+ );
+ await sleep(delay);
+ continue;
+ } else {
+ // Throw an error after max retries for rate limit
+ throw new Error(
+ `Failed after ${maxRetries} retries: Rate Limit Exceeded (429)`,
+ );
+ }
+ } else if (retries < maxRetries) {
+ // Other 4xx errors might be temporary, retry with backoff
+ retries++;
+ const delay = calculateBackoff(retries, retryDelay);
+ logger.warn(
+ `${response.status} Error. Retrying in ${Math.round(delay)}ms (${retries}/${maxRetries})`,
+ );
+ await sleep(delay);
+ continue;
+ } else {
+ // Throw an error after max retries for other 4xx errors
+ throw new Error(
+ `Failed after ${maxRetries} retries: HTTP ${response.status} (${response.statusText})`,
+ );
+ }
+ }
+
+ const contentType = response.headers.get('content-type');
+ const responseBody = contentType?.includes('application/json')
+ ? await response.json()
+ : await response.text();
+
+ logger.debug('Response content-type:', contentType);
+
+ return {
+ status: response.status,
+ statusText: response.statusText,
+ headers: Object.fromEntries(response.headers),
+ body: responseBody as ReturnType['body'],
+ retries,
+ slowModeEnabled,
+ };
+ } catch (error) {
+ lastError = error as Error;
+ logger.error(`Request failed: ${error}`);
+
+ if (retries < maxRetries) {
+ retries++;
+ const delay = calculateBackoff(retries, retryDelay);
+ logger.warn(
+ `Network error. Retrying in ${Math.round(delay)}ms (${retries}/${maxRetries})`,
+ );
+ await sleep(delay);
+ } else {
+ throw new Error(
+ `Failed after ${maxRetries} retries: ${lastError.message}`,
+ );
+ }
+ }
+ }
+
+ // This should never be reached due to the throw above, but TypeScript needs it
+ throw new Error(
+ `Failed after ${maxRetries} retries: ${lastError?.message || 'Unknown error'}`,
+ );
},
logParameters(params, { logger }) {
- const { method, url, params: queryParams } = params;
+ const { method, url, params: queryParams, maxRetries, slowMode } = params;
logger.log(
- `${method} ${url}${queryParams ? `?${new URLSearchParams(queryParams).toString()}` : ''}`,
+ `${method} ${url}${queryParams ? `?${new URLSearchParams(queryParams).toString()}` : ''}${
+ maxRetries !== undefined ? ` (max retries: ${maxRetries})` : ''
+ }${slowMode ? ' (slow mode)' : ''}`,
);
},
logReturns: (result, { logger }) => {
- const { status, statusText } = result;
- logger.log(`${status} ${statusText}`);
+ const { status, statusText, retries, slowModeEnabled } = result;
+ logger.log(
+ `${status} ${statusText}${retries ? ` after ${retries} retries` : ''}${slowModeEnabled ? ' (slow mode enabled)' : ''}`,
+ );
},
};