From 1aa817041d9cd399fa7d40c9e5c37a90f1398134 Mon Sep 17 00:00:00 2001 From: Prasenjit Sarkar Date: Wed, 22 Jul 2026 16:20:06 +0100 Subject: [PATCH] Generate OpenAPI spec from zod schemas + CI drift check swagger.yaml was hand-maintained and had drifted badly: it documented 17 of 55 endpoints (~31%), described a batch path the code doesn't serve (/result/{jobId} vs /download/{jobId}), and omitted request options the API accepts (onlyMainContent, cssSchema, changeTracking, contacts, maxAge). Make the spec derive from the code instead: - src/api/schemas: request zod schemas extracted from the route files into a pure module (imports only zod). The routes now validate with these, and the generator reads the same objects, so validation and docs share one source. Kept pure deliberately - importing a route pulls in Redis/BullMQ/browser-pool, which open handles at import time and would hang the generator. - src/api/openapi: builds the document via @asteasolutions/zod-to-openapi and writes swagger.yaml. Excluded from the tsc build so the devDependency never ships in dist. - routes-inventory: statically scans index.ts + routes/*.ts for the endpoints actually registered (parses source, no imports). - openapi.spec.ts: fails if any registered endpoint is undocumented, if the spec documents an endpoint that isn't served, or if swagger.yaml is stale. - .github/workflows/ci.yml: build + test + `npm run openapi:check`. Coverage is now 55/55 endpoints (50 paths, 55 operations). Verified the guard is not vacuous: adding an undocumented route fails the test by name, and editing a zod schema fails the sync check. Behaviour unchanged - validation semantics preserved (incl. .passthrough()). tsc, eslint and all 180 tests pass. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 38 + README.md | 16 + package-lock.json | 90 +- package.json | 9 +- src/api/openapi/document.ts | 246 ++ src/api/openapi/generate.ts | 64 + src/api/openapi/openapi.spec.ts | 72 + src/api/openapi/routes-inventory.ts | 117 + src/api/routes/agent.routes.ts | 13 +- src/api/routes/extract-auto.routes.ts | 20 +- src/api/routes/map.routes.ts | 21 +- src/api/routes/parse.routes.ts | 10 +- src/api/routes/phase2-tools.routes.ts | 17 +- src/api/routes/scraper.ts | 65 +- src/api/routes/search.routes.ts | 14 +- src/api/routes/session.routes.ts | 34 +- src/api/routes/sites.routes.ts | 36 +- src/api/routes/task.routes.ts | 17 +- src/api/schemas/index.ts | 326 +++ swagger.yaml | 3675 +++++++++++++++++-------- tsconfig.json | 6 +- 21 files changed, 3548 insertions(+), 1358 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 src/api/openapi/document.ts create mode 100644 src/api/openapi/generate.ts create mode 100644 src/api/openapi/openapi.spec.ts create mode 100644 src/api/openapi/routes-inventory.ts create mode 100644 src/api/schemas/index.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..37dc804 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: CI + +on: + push: + branches: [main, master] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + + - run: npm ci + + - name: Typecheck / build + run: npm run build + + - name: Unit tests + run: npm test + + # Fails if swagger.yaml drifted from the zod request schemas. + # Regenerate locally with `npm run openapi:generate` and commit. + - name: OpenAPI spec is in sync + run: npm run openapi:check diff --git a/README.md b/README.md index f7fbc8e..d2ff61b 100644 --- a/README.md +++ b/README.md @@ -682,6 +682,22 @@ curl -X POST https://deepscrapper.ai/api/crawl \ ## API Usage +### OpenAPI specification + +Every endpoint is described in [`swagger.yaml`](swagger.yaml) (OpenAPI 3.0) — paste it into +[editor.swagger.io](https://editor.swagger.io/), or generate a client with `openapi-generator`. + +The file is **generated from the zod request schemas** in `src/api/schemas`, which are the same +schemas the routes validate with — so the documented options can't drift from what the API +actually accepts: + +```bash +npm run openapi:generate # rewrite swagger.yaml after changing a schema +npm run openapi:check # verify it's in sync (CI runs this) +``` + +CI also fails if an endpoint is added without appearing in the spec, so coverage stays complete. + ### Basic Scraping ```bash diff --git a/package-lock.json b/package-lock.json index 0d2de9e..78ed2c0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -41,11 +41,13 @@ "zod": "^3.22.4" }, "devDependencies": { + "@asteasolutions/zod-to-openapi": "^7.3.4", "@types/archiver": "^6.0.3", "@types/cors": "^2.8.17", "@types/diff": "^5.2.1", "@types/express": "^4.17.21", "@types/jest": "^29.5.10", + "@types/js-yaml": "^4.0.9", "@types/morgan": "^1.9.9", "@types/node": "^20.10.0", "@types/pako": "^2.0.3", @@ -58,6 +60,7 @@ "@typescript-eslint/parser": "^8.31.0", "eslint": "^9.25.1", "jest": "^29.7.0", + "js-yaml": "^5.2.1", "nodemon": "^3.1.9", "ts-jest": "^29.1.1", "ts-node": "^10.9.2", @@ -78,6 +81,19 @@ "node": ">=6.0.0" } }, + "node_modules/@asteasolutions/zod-to-openapi": { + "version": "7.3.4", + "resolved": "https://registry.npmjs.org/@asteasolutions/zod-to-openapi/-/zod-to-openapi-7.3.4.tgz", + "integrity": "sha512-/2rThQ5zPi9OzVwes6U7lK1+Yvug0iXu25olp7S0XsYmOqnyMfxH7gdSQjn/+DSOHRg7wnotwGJSyL+fBKdnEA==", + "dev": true, + "license": "MIT", + "dependencies": { + "openapi3-ts": "^4.1.2" + }, + "peerDependencies": { + "zod": "^3.20.2" + } + }, "node_modules/@babel/code-frame": { "version": "7.26.2", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", @@ -781,6 +797,29 @@ "concat-map": "0.0.1" } }, + "node_modules/@eslint/eslintrc/node_modules/js-yaml": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.0.tgz", + "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -1727,6 +1766,13 @@ "pretty-format": "^29.0.0" } }, + "node_modules/@types/js-yaml": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", + "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -5741,16 +5787,26 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.1.tgz", + "integrity": "sha512-zfLtNfQqxVqq3uaTqSkh4x4hZw3KHobGUA0fJUj4wawW8bsQLTVqpHdXSIzidh7o+4lEW36tANuAGdaFx6Zgnw==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "license": "MIT", "dependencies": { "argparse": "^2.0.1" }, "bin": { - "js-yaml": "bin/js-yaml.js" + "js-yaml": "bin/js-yaml.mjs" } }, "node_modules/jsesc": { @@ -6690,6 +6746,16 @@ "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", "license": "MIT" }, + "node_modules/openapi3-ts": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/openapi3-ts/-/openapi3-ts-4.6.0.tgz", + "integrity": "sha512-a4sfn6L2sIShhtzJqmjGrARvxAW/3F2BJDdyRVvNF9VhAsZSh5hSyI3a9TNvmzBxXmq66nY5LNT5bQcBxYAZZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "yaml": "^2.9.0" + } + }, "node_modules/option": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/option/-/option-0.2.4.tgz", @@ -8637,6 +8703,22 @@ "dev": true, "license": "ISC" }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, "node_modules/yargs": { "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", diff --git a/package.json b/package.json index 0eb3d69..5616403 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,8 @@ "dev": "ts-node src/index.ts", "build": "tsc", "test": "jest", + "openapi:generate": "ts-node -T src/api/openapi/generate.ts", + "openapi:check": "ts-node -T src/api/openapi/generate.ts --check", "lint": "eslint src/**/*.ts", "lint:fix": "eslint src/**/*.ts --fix", "clean": "rm -rf dist cache", @@ -33,13 +35,13 @@ "cors": "^2.8.5", "diff": "^5.2.0", "dotenv": "^16.4.7", - "mammoth": "^1.8.0", "express": "^4.18.2", "express-rate-limit": "^7.4.1", "express-validator": "^7.2.1", "helmet": "^7.1.0", "ioredis": "^5.6.0", "ipaddr.js": "^2.2.0", + "mammoth": "^1.8.0", "morgan": "^1.10.0", "openai": "^4.89.1", "pako": "^2.1.0", @@ -56,15 +58,17 @@ "zod": "^3.22.4" }, "devDependencies": { + "@asteasolutions/zod-to-openapi": "^7.3.4", "@types/archiver": "^6.0.3", "@types/cors": "^2.8.17", "@types/diff": "^5.2.1", "@types/express": "^4.17.21", - "@types/pdf-parse": "^1.1.4", "@types/jest": "^29.5.10", + "@types/js-yaml": "^4.0.9", "@types/morgan": "^1.9.9", "@types/node": "^20.10.0", "@types/pako": "^2.0.3", + "@types/pdf-parse": "^1.1.4", "@types/turndown": "^5.0.4", "@types/user-agents": "^1.0.4", "@types/uuid": "^10.0.0", @@ -73,6 +77,7 @@ "@typescript-eslint/parser": "^8.31.0", "eslint": "^9.25.1", "jest": "^29.7.0", + "js-yaml": "^5.2.1", "nodemon": "^3.1.9", "ts-jest": "^29.1.1", "ts-node": "^10.9.2", diff --git a/src/api/openapi/document.ts b/src/api/openapi/document.ts new file mode 100644 index 0000000..af8c2e6 --- /dev/null +++ b/src/api/openapi/document.ts @@ -0,0 +1,246 @@ +import { + OpenAPIRegistry, + OpenApiGeneratorV3, + extendZodWithOpenApi, +} from '@asteasolutions/zod-to-openapi'; +import { z } from 'zod'; +import * as S from '../schemas'; + +/** + * Builds the OpenAPI document from the zod request schemas in `src/api/schemas`. + * + * The schemas here are the SAME objects the route handlers validate with, so a + * change to request validation shows up in the spec on the next generate. Run + * `npm run openapi:generate` to refresh `swagger.yaml`; `npm run openapi:check` + * (and the openapi.spec.ts test) fail if the committed file is stale or if an + * endpoint exists in the code but is missing here. + * + * NOTE: this module is excluded from the production `tsc` build (see tsconfig) + * because @asteasolutions/zod-to-openapi is a devDependency. + */ +extendZodWithOpenApi(z); + +const registry = new OpenAPIRegistry(); + +registry.registerComponent('securitySchemes', 'ApiKeyAuth', { + type: 'apiKey', + name: 'X-API-Key', + in: 'header', + description: 'API key. Required on every /api/* route.', +}); + +const AUTH = [{ ApiKeyAuth: [] as string[] }]; + +// --- common response shapes ------------------------------------------------- + +const errorResponse = z + .object({ success: z.literal(false), error: z.string() }) + .openapi('ErrorResponse'); + +const okResponse = z + .object({ success: z.literal(true) }) + .passthrough() + .openapi('SuccessResponse'); + +const asyncJobResponse = z + .object({ + success: z.literal(true), + id: z.string(), + url: z.string().describe('Polling URL for this job'), + status: z.string(), + }) + .openapi('AsyncJobAccepted'); + +const jsonBody = (schema: z.ZodTypeAny) => ({ + body: { required: true, content: { 'application/json': { schema } } }, +}); + +const pathParam = (name: string, description: string) => + z.object({ [name]: z.string().openapi({ description }) }); + +/** Standard response block: 200 + auth/validation/error codes. */ +const responses = (okSchema: z.ZodTypeAny, okDescription = 'Success') => ({ + 200: { + description: okDescription, + content: { 'application/json': { schema: okSchema } }, + }, + 400: { + description: 'Invalid request', + content: { 'application/json': { schema: errorResponse } }, + }, + 401: { + description: 'Missing or invalid API key', + content: { 'application/json': { schema: errorResponse } }, + }, + 429: { + description: 'Rate limit or quota exceeded', + content: { 'application/json': { schema: errorResponse } }, + }, +}); + +const withNotFound = (base: ReturnType) => ({ + ...base, + 404: { + description: 'Not found', + content: { 'application/json': { schema: errorResponse } }, + }, +}); + +type PathArgs = { + method: 'get' | 'post' | 'put' | 'delete' | 'patch'; + path: string; + tag: string; + summary: string; + description?: string; + body?: z.ZodTypeAny; + params?: z.AnyZodObject; + query?: z.AnyZodObject; + ok?: z.ZodTypeAny; + okDescription?: string; + notFound?: boolean; + public?: boolean; + rawContent?: { type: string; description: string }; +}; + +function add(a: PathArgs) { + const base = responses(a.ok ?? okResponse, a.okDescription); + const res: Record = a.notFound ? withNotFound(base) : base; + if (a.rawContent) { + res[200] = { + description: a.rawContent.description, + content: { [a.rawContent.type]: { schema: z.string() } }, + }; + } + registry.registerPath({ + method: a.method, + path: a.path, + tags: [a.tag], + summary: a.summary, + description: a.description, + security: a.public ? undefined : AUTH, + request: { + ...(a.body ? jsonBody(a.body) : {}), + ...(a.params ? { params: a.params } : {}), + ...(a.query ? { query: a.query } : {}), + }, + responses: res as never, + }); +} + +// --------------------------------------------------------------------------- +// Scrape +// --------------------------------------------------------------------------- +add({ method: 'post', path: '/api/scrape', tag: 'Scrape', summary: 'Scrape a URL to markdown, HTML, text or structured data', body: S.scrapeRequestSchema }); +add({ method: 'post', path: '/api/scrape/async', tag: 'Scrape', summary: 'Submit a scrape as an async job', body: S.scrapeAsyncRequestSchema, ok: asyncJobResponse, okDescription: 'Job accepted' }); +add({ method: 'get', path: '/api/scrape/job/{id}', tag: 'Scrape', summary: 'Poll an async scrape job', params: pathParam('id', 'Async scrape job id'), notFound: true }); +add({ method: 'post', path: '/api/extract-schema', tag: 'Scrape', summary: 'Extract structured data using a JSON Schema (LLM)', body: S.extractSchemaRequestSchema }); +add({ method: 'post', path: '/api/summarize', tag: 'Scrape', summary: 'Scrape a URL and generate an AI summary', body: S.summarizeRequestSchema }); +add({ method: 'delete', path: '/api/cache', tag: 'Scrape', summary: 'Invalidate the scrape cache (all, or a single URL)', body: S.cacheInvalidateSchema }); + +// --------------------------------------------------------------------------- +// Crawl +// --------------------------------------------------------------------------- +add({ method: 'post', path: '/api/crawl', tag: 'Crawl', summary: 'Start a multi-page crawl', body: S.crawlRequestSchema, ok: asyncJobResponse, okDescription: 'Crawl started' }); +add({ method: 'post', path: '/api/crawl/estimate', tag: 'Crawl', summary: 'Pre-run size/cost estimate for a crawl', body: S.crawlEstimateSchema }); +add({ method: 'get', path: '/api/crawl/active', tag: 'Crawl', summary: 'List currently-active crawls' }); +add({ method: 'get', path: '/api/crawl/{jobId}', tag: 'Crawl', summary: 'Get crawl status and exported files', params: pathParam('jobId', 'Crawl job id'), notFound: true }); +add({ method: 'get', path: '/api/crawl/{jobId}/errors', tag: 'Crawl', summary: 'List per-page failures for a crawl', params: pathParam('jobId', 'Crawl job id'), notFound: true }); +add({ method: 'get', path: '/api/crawl/{jobId}/stream', tag: 'Crawl', summary: 'Stream crawl pages as Server-Sent Events', params: pathParam('jobId', 'Crawl job id'), rawContent: { type: 'text/event-stream', description: 'SSE stream of pages as they complete' } }); +add({ method: 'get', path: '/api/crawl/{jobId}/download/zip', tag: 'Crawl', summary: 'Download all crawled pages as a ZIP', params: pathParam('jobId', 'Crawl job id'), query: z.object({ format: z.enum(['markdown', 'json']).optional() }), rawContent: { type: 'application/zip', description: 'ZIP archive of crawled pages' } }); +add({ method: 'get', path: '/api/crawl/{jobId}/download/json', tag: 'Crawl', summary: 'Download all crawled pages as one JSON array', params: pathParam('jobId', 'Crawl job id') }); +add({ method: 'delete', path: '/api/crawl/{jobId}', tag: 'Crawl', summary: 'Cancel a running crawl', params: pathParam('jobId', 'Crawl job id'), notFound: true }); + +// --------------------------------------------------------------------------- +// Batch scrape +// --------------------------------------------------------------------------- +const batchId = pathParam('batchId', 'Batch id (UUID)'); +add({ method: 'post', path: '/api/batch/scrape', tag: 'Batch', summary: 'Scrape many URLs concurrently', body: S.batchScrapeRequestSchema, ok: asyncJobResponse, okDescription: 'Batch accepted' }); +add({ method: 'get', path: '/api/batch/scrape/{batchId}/status', tag: 'Batch', summary: 'Batch progress and results', params: batchId, notFound: true }); +add({ method: 'get', path: '/api/batch/scrape/{batchId}/errors', tag: 'Batch', summary: 'Per-URL failures for a batch', params: batchId, notFound: true }); +add({ method: 'get', path: '/api/batch/scrape/{batchId}/download/zip', tag: 'Batch', summary: 'Download batch results as a ZIP', params: batchId, rawContent: { type: 'application/zip', description: 'ZIP archive of batch results' } }); +add({ method: 'get', path: '/api/batch/scrape/{batchId}/download/json', tag: 'Batch', summary: 'Download batch results as JSON', params: batchId, notFound: true }); +add({ method: 'get', path: '/api/batch/scrape/{batchId}/download/{jobId}', tag: 'Batch', summary: 'Download a single result from a batch', params: z.object({ batchId: z.string().openapi({ description: 'Batch id (UUID)' }), jobId: z.string().openapi({ description: 'Job id within the batch' }) }), notFound: true }); +add({ method: 'delete', path: '/api/batch/scrape/{batchId}', tag: 'Batch', summary: 'Cancel a batch', params: batchId, notFound: true }); +add({ method: 'post', path: '/api/batch/cleanup', tag: 'Batch', summary: 'Clean up batch records older than N days', query: z.object({ days: z.coerce.number().int().min(1).max(365).optional() }) }); + +// --------------------------------------------------------------------------- +// Map (URL discovery) +// --------------------------------------------------------------------------- +add({ method: 'post', path: '/api/map', tag: 'Map', summary: 'Discover all URLs on a site', body: S.mapRequestSchema }); +add({ method: 'get', path: '/api/map/health', tag: 'Map', summary: 'URL-discovery subsystem health' }); +add({ method: 'get', path: '/api/map/cache/stats', tag: 'Map', summary: 'URL-discovery cache statistics' }); +add({ method: 'post', path: '/api/map/cache/clear', tag: 'Map', summary: 'Clear the URL-discovery cache for a site', body: S.mapClearCacheSchema }); + +// --------------------------------------------------------------------------- +// Search / async tasks / tools +// --------------------------------------------------------------------------- +add({ method: 'post', path: '/api/search', tag: 'Search', summary: 'Web search, optionally scraping each result', body: S.searchRequestSchema }); +add({ method: 'post', path: '/api/extract', tag: 'Extract', summary: 'Async multi-URL LLM extraction', body: S.extractTaskSchema, ok: asyncJobResponse, okDescription: 'Job accepted' }); +add({ method: 'get', path: '/api/extract/{id}', tag: 'Extract', summary: 'Poll an extract job', params: pathParam('id', 'Extract job id'), notFound: true }); +add({ method: 'post', path: '/api/extract-auto', tag: 'Extract', summary: 'Self-healing extraction — derives and caches CSS selectors, re-derives on breakage', body: S.extractAutoSchema }); +add({ method: 'post', path: '/api/llmstxt', tag: 'Tools', summary: 'Generate an llms.txt for a site', body: S.llmstxtSchema, ok: asyncJobResponse, okDescription: 'Job accepted' }); +add({ method: 'get', path: '/api/llmstxt/{id}', tag: 'Tools', summary: 'Poll an llms.txt job', params: pathParam('id', 'llms.txt job id'), notFound: true }); +add({ method: 'post', path: '/api/parse', tag: 'Tools', summary: 'Parse a document (PDF/DOCX/…) to markdown', body: S.parseRequestSchema }); +add({ method: 'post', path: '/api/discover-apis', tag: 'Tools', summary: 'Surface a page’s underlying JSON/XHR endpoints', body: S.discoverApisSchema }); +add({ method: 'get', path: '/api/reader', tag: 'Tools', summary: 'Scrape a URL to markdown; honours Accept: text/markdown', query: z.object({ url: z.string().url().openapi({ description: 'URL to read' }) }) }); +add({ method: 'get', path: '/api/usage', tag: 'Ops', summary: 'API key usage and quota' }); +add({ method: 'get', path: '/api/proxies', tag: 'Ops', summary: 'Proxy pool health' }); + +// --------------------------------------------------------------------------- +// Agent +// --------------------------------------------------------------------------- +add({ method: 'post', path: '/api/agent', tag: 'Agent', summary: 'Autonomous navigation toward a natural-language goal', body: S.agentRequestSchema, ok: asyncJobResponse, okDescription: 'Job accepted' }); +add({ method: 'get', path: '/api/agent/{id}', tag: 'Agent', summary: 'Poll an agent run', params: pathParam('id', 'Agent task id'), notFound: true }); + +// --------------------------------------------------------------------------- +// Persistent browser sessions +// --------------------------------------------------------------------------- +const sessionId = pathParam('id', 'Session id'); +add({ method: 'post', path: '/api/sessions', tag: 'Sessions', summary: 'Create a persistent browser session', body: S.sessionCreateSchema }); +add({ method: 'get', path: '/api/sessions', tag: 'Sessions', summary: 'List active sessions' }); +add({ method: 'get', path: '/api/sessions/{id}', tag: 'Sessions', summary: 'Get a session', params: sessionId, notFound: true }); +add({ method: 'post', path: '/api/sessions/{id}/action', tag: 'Sessions', summary: 'Run an action in a session (navigate, click, scrape, …)', body: S.sessionActionSchema, params: sessionId, notFound: true }); +add({ method: 'delete', path: '/api/sessions/{id}', tag: 'Sessions', summary: 'Close a session', params: sessionId, notFound: true }); + +// --------------------------------------------------------------------------- +// Site specs (site -> MCP endpoint) +// --------------------------------------------------------------------------- +const siteId = pathParam('id', 'Site spec id'); +add({ method: 'post', path: '/api/sites', tag: 'Sites', summary: 'Create a reusable site spec (becomes an MCP tool)', body: S.siteCreateSchema }); +add({ method: 'get', path: '/api/sites', tag: 'Sites', summary: 'List site specs' }); +add({ method: 'get', path: '/api/sites/{id}', tag: 'Sites', summary: 'Get a site spec', params: siteId, notFound: true }); +add({ method: 'post', path: '/api/sites/{id}/run', tag: 'Sites', summary: 'Run a site spec by id', body: S.siteRunSchema, params: siteId, notFound: true }); +add({ method: 'post', path: '/api/sites/by-name/{name}/run', tag: 'Sites', summary: 'Run a site spec by name', body: S.siteRunSchema, params: pathParam('name', 'Site spec name (slug)'), notFound: true }); +add({ method: 'post', path: '/api/sites/{id}/verify', tag: 'Sites', summary: 'Verify a spec still extracts correctly (self-heals on drift)', params: siteId, notFound: true }); +add({ method: 'delete', path: '/api/sites/{id}', tag: 'Sites', summary: 'Delete a site spec', params: siteId, notFound: true }); + +// --------------------------------------------------------------------------- +// Ops (no API key) +// --------------------------------------------------------------------------- +add({ method: 'get', path: '/health', tag: 'Ops', summary: 'Liveness probe', public: true }); +add({ method: 'get', path: '/health/ready', tag: 'Ops', summary: 'Readiness probe (dependencies reachable)', public: true }); +add({ + method: 'get', path: '/metrics', tag: 'Ops', summary: 'Prometheus metrics', public: true, + rawContent: { type: 'text/plain', description: 'Prometheus exposition format' }, +}); + +/** Build the OpenAPI 3.0 document. */ +export function buildOpenApiDocument() { + const generator = new OpenApiGeneratorV3(registry.definitions); + return generator.generateDocument({ + openapi: '3.0.0', + info: { + title: 'DeepScraper API', + version: '1.0.0', + description: + 'Open-source web scraping API — scrape, crawl, map, search, extract structured data, ' + + 'drive persistent browser sessions, and expose any site as a reusable endpoint.\n\n' + + 'This file is GENERATED from the zod request schemas in `src/api/schemas`. ' + + 'Do not edit by hand — run `npm run openapi:generate`.', + license: { name: 'Apache-2.0', url: 'https://www.apache.org/licenses/LICENSE-2.0' }, + }, + servers: [{ url: 'http://localhost:3000', description: 'Local instance' }], + }); +} + +export { registry }; diff --git a/src/api/openapi/generate.ts b/src/api/openapi/generate.ts new file mode 100644 index 0000000..88bcd3f --- /dev/null +++ b/src/api/openapi/generate.ts @@ -0,0 +1,64 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import * as yaml from 'js-yaml'; +import { buildOpenApiDocument } from './document'; + +/** + * Writes (or verifies) `swagger.yaml` from the zod schemas. + * + * npm run openapi:generate # rewrite swagger.yaml + * npm run openapi:check # fail if swagger.yaml is stale (used by CI) + */ + +export const SPEC_PATH = path.resolve(__dirname, '../../../swagger.yaml'); + +const BANNER = [ + '# GENERATED FILE — DO NOT EDIT BY HAND.', + '#', + '# Produced from the zod request schemas in src/api/schemas by', + '# npm run openapi:generate', + '#', + '# CI runs `npm run openapi:check`, which fails if this file is out of sync with', + '# the code, so request validation and these docs cannot silently diverge.', + '', +].join('\n'); + +export function renderSpec(): string { + const doc = buildOpenApiDocument(); + const body = yaml.dump(doc, { noRefs: true, lineWidth: 100, sortKeys: false }); + return `${BANNER}${body}`; +} + +function main() { + const check = process.argv.includes('--check'); + const rendered = renderSpec(); + + if (!check) { + fs.writeFileSync(SPEC_PATH, rendered, 'utf8'); + const doc = buildOpenApiDocument(); + const count = Object.values(doc.paths ?? {}).reduce( + (n, item) => n + Object.keys(item as object).filter((k) => k !== 'parameters').length, + 0, + ); + console.log(`✅ wrote ${path.relative(process.cwd(), SPEC_PATH)} — ${Object.keys(doc.paths ?? {}).length} paths, ${count} operations`); + return; + } + + if (!fs.existsSync(SPEC_PATH)) { + console.error('❌ swagger.yaml is missing. Run: npm run openapi:generate'); + process.exit(1); + } + const onDisk = fs.readFileSync(SPEC_PATH, 'utf8'); + if (onDisk !== rendered) { + console.error( + '❌ swagger.yaml is out of date with the zod schemas.\n' + + ' Run `npm run openapi:generate` and commit the result.', + ); + process.exit(1); + } + console.log('✅ swagger.yaml is in sync with the zod schemas'); +} + +if (require.main === module) { + main(); +} diff --git a/src/api/openapi/openapi.spec.ts b/src/api/openapi/openapi.spec.ts new file mode 100644 index 0000000..8d0473d --- /dev/null +++ b/src/api/openapi/openapi.spec.ts @@ -0,0 +1,72 @@ +import * as fs from 'fs'; +import { buildOpenApiDocument } from './document'; +import { renderSpec, SPEC_PATH } from './generate'; +import { collectRoutes, routeKeys } from './routes-inventory'; + +/** + * Guards against the failure mode this spec was rewritten to fix: swagger.yaml + * silently drifting from the real API. + * + * 1. every route the app registers must be documented + * 2. nothing may be documented that the app does not serve + * 3. the committed swagger.yaml must match what the schemas generate + */ + +function documentedKeys(): string[] { + const doc = buildOpenApiDocument(); + const keys: string[] = []; + for (const [p, item] of Object.entries(doc.paths ?? {})) { + for (const method of Object.keys(item as Record)) { + if (method === 'parameters') continue; + keys.push(`${method.toUpperCase()} ${p}`); + } + } + return keys.sort(); +} + +describe('OpenAPI spec', () => { + const actual = routeKeys(collectRoutes()); + const documented = documentedKeys(); + + it('discovers the app routes it is meant to check (sanity)', () => { + // If the static scanner breaks, every other assertion here becomes vacuous. + expect(actual.length).toBeGreaterThan(40); + expect(actual).toContain('POST /api/scrape'); + expect(actual).toContain('GET /health'); + }); + + it('documents every endpoint the app registers', () => { + const missing = actual.filter((k) => !documented.includes(k)); + expect(missing).toEqual([]); + }); + + it('does not document endpoints the app does not serve', () => { + const extra = documented.filter((k) => !actual.includes(k)); + expect(extra).toEqual([]); + }); + + it('has swagger.yaml committed and in sync with the zod schemas', () => { + expect(fs.existsSync(SPEC_PATH)).toBe(true); + const onDisk = fs.readFileSync(SPEC_PATH, 'utf8'); + // If this fails: npm run openapi:generate && commit + expect(onDisk).toEqual(renderSpec()); + }); + + it('derives request options from the zod schemas (regression: stale options)', () => { + const doc = buildOpenApiDocument() as any; + const scrapeBody = + doc.paths['/api/scrape'].post.requestBody.content['application/json'].schema; + const options = scrapeBody.properties.options.properties; + // These were accepted by the code but absent from the old hand-written spec. + for (const opt of ['onlyMainContent', 'fitMarkdown', 'extractorFormat', 'useBrowser', 'stealthMode']) { + expect(options).toHaveProperty(opt); + } + }); + + it('requires an API key on /api routes but not on probes', () => { + const doc = buildOpenApiDocument() as any; + expect(doc.paths['/api/scrape'].post.security).toBeDefined(); + expect(doc.paths['/health'].get.security).toBeUndefined(); + expect(doc.components.securitySchemes.ApiKeyAuth.name).toBe('X-API-Key'); + }); +}); diff --git a/src/api/openapi/routes-inventory.ts b/src/api/openapi/routes-inventory.ts new file mode 100644 index 0000000..200d672 --- /dev/null +++ b/src/api/openapi/routes-inventory.ts @@ -0,0 +1,117 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +/** + * Statically discovers every route the Express app actually registers, by + * reading `src/index.ts` (mount points) and `src/api/routes/*.ts` (route + * definitions). + * + * It parses source text rather than importing the modules on purpose: importing + * a route file pulls in controllers -> services -> Redis, BullMQ and the browser + * pool, all of which open handles at import time. + * + * The OpenAPI test compares this inventory against the generated spec, so a new + * endpoint that nobody documented fails the build. + */ + +export interface RouteEntry { + method: string; + /** OpenAPI-style path, e.g. /api/crawl/{jobId} */ + path: string; +} + +const SRC = path.resolve(__dirname, '../../'); +const ROUTES_DIR = path.join(SRC, 'api', 'routes'); +const INDEX_FILE = path.join(SRC, 'index.ts'); + +const METHODS = 'get|post|put|delete|patch'; + +/** Express ":param" -> OpenAPI "{param}" */ +function toOpenApiPath(p: string): string { + return p.replace(/:([A-Za-z0-9_]+)/g, '{$1}'); +} + +function joinPath(mount: string, sub: string): string { + const joined = `${mount.replace(/\/+$/, '')}/${sub.replace(/^\/+/, '')}`; + const cleaned = joined.replace(/\/{2,}/g, '/').replace(/\/+$/, ''); + return cleaned === '' ? '/' : cleaned; +} + +/** + * Map a local identifier used in `app.use(...)` to the (file, routerVariable) + * it refers to. Default imports resolve to the file's `router` const; named + * imports keep their own name. + */ +function parseRouteImports(indexSrc: string): Map { + const map = new Map(); + + // import x from './api/routes/foo'; + for (const m of indexSrc.matchAll(/import\s+(\w+)\s+from\s+'\.\/api\/routes\/([\w.-]+)'/g)) { + map.set(m[1], { file: `${m[2]}.ts`, varName: 'router' }); + } + // import { a, b } from './api/routes/foo'; + for (const m of indexSrc.matchAll(/import\s+\{([^}]+)\}\s+from\s+'\.\/api\/routes\/([\w.-]+)'/g)) { + for (const raw of m[1].split(',')) { + const name = raw.trim().split(/\s+as\s+/).pop()!.trim(); + if (name) map.set(name, { file: `${m[2]}.ts`, varName: name }); + } + } + return map; +} + +/** Collect `.('')` definitions from one route file. */ +function parseRouteFile(fileSrc: string): Array<{ varName: string; method: string; sub: string }> { + const out: Array<{ varName: string; method: string; sub: string }> = []; + const re = new RegExp(`(\\w+)\\s*\\.\\s*(${METHODS})\\s*\\(\\s*(?:\\r?\\n\\s*)?['"\`]([^'"\`]*)['"\`]`, 'g'); + for (const m of fileSrc.matchAll(re)) { + out.push({ varName: m[1], method: m[2].toLowerCase(), sub: m[3] }); + } + return out; +} + +export function collectRoutes(): RouteEntry[] { + const indexSrc = fs.readFileSync(INDEX_FILE, 'utf8'); + const importMap = parseRouteImports(indexSrc); + const entries: RouteEntry[] = []; + + // Routes registered directly on the app (health, metrics, ...) + const appRe = new RegExp(`app\\s*\\.\\s*(${METHODS})\\s*\\(\\s*['"\`]([^'"\`]+)['"\`]`, 'g'); + for (const m of indexSrc.matchAll(appRe)) { + entries.push({ method: m[1].toLowerCase(), path: toOpenApiPath(m[2]) }); + } + + // Mounted routers: app.use('', ) + const fileCache = new Map>(); + for (const m of indexSrc.matchAll(/app\.use\(\s*'([^']+)'\s*,\s*(\w+)\s*\)/g)) { + const [, mount, ident] = m; + const target = importMap.get(ident); + if (!target) continue; // middleware (rate limiter, quota) — not a router + + const filePath = path.join(ROUTES_DIR, target.file); + if (!fs.existsSync(filePath)) continue; + + if (!fileCache.has(target.file)) { + fileCache.set(target.file, parseRouteFile(fs.readFileSync(filePath, 'utf8'))); + } + for (const r of fileCache.get(target.file)!) { + if (r.varName !== target.varName) continue; + entries.push({ method: r.method, path: toOpenApiPath(joinPath(mount, r.sub)) }); + } + } + + // De-duplicate and sort for stable comparison + const seen = new Set(); + return entries + .filter((e) => { + const k = `${e.method} ${e.path}`; + if (seen.has(k)) return false; + seen.add(k); + return true; + }) + .sort((a, b) => (a.path === b.path ? a.method.localeCompare(b.method) : a.path.localeCompare(b.path))); +} + +/** "GET /api/crawl/{jobId}" keys for set comparison. */ +export function routeKeys(entries: RouteEntry[]): string[] { + return entries.map((e) => `${e.method.toUpperCase()} ${e.path}`).sort(); +} diff --git a/src/api/routes/agent.routes.ts b/src/api/routes/agent.routes.ts index f2b4ccc..da1943e 100644 --- a/src/api/routes/agent.routes.ts +++ b/src/api/routes/agent.routes.ts @@ -1,28 +1,19 @@ import { Router, Request, Response } from 'express'; -import { z } from 'zod'; import { apiKeyAuth } from '../middleware/auth.middleware'; import { validateRequest } from '../middleware/validation'; import { expensiveLimiter, statusLimiter } from '../middleware/rate-limit.middleware'; import { createTask, getTask } from '../../services/task.service'; import { logger } from '../../utils/logger'; +import { agentRequestSchema } from '../schemas'; const router = Router(); -const agentSchema = z.object({ - url: z.string().url(), - prompt: z.string().min(1).max(5000), - schema: z.any().optional(), - maxSteps: z.number().int().positive().max(20).optional(), - onlyMainContent: z.boolean().optional(), - fitMarkdown: z.boolean().optional(), -}); - /** * @route POST /api/agent * @desc Start an autonomous navigation agent toward a natural-language goal. * Runs as an async task; poll GET /api/agent/:id for the result. */ -router.post('/', expensiveLimiter, apiKeyAuth, validateRequest(agentSchema), async (req: Request, res: Response) => { +router.post('/', expensiveLimiter, apiKeyAuth, validateRequest(agentRequestSchema), async (req: Request, res: Response) => { try { const id = await createTask('agent', req.body); const base = `${req.secure ? 'https' : 'http'}://${req.get('host')}`; diff --git a/src/api/routes/extract-auto.routes.ts b/src/api/routes/extract-auto.routes.ts index f70bcf4..6a98ea4 100644 --- a/src/api/routes/extract-auto.routes.ts +++ b/src/api/routes/extract-auto.routes.ts @@ -1,5 +1,4 @@ import { Router, Request, Response } from 'express'; -import { z } from 'zod'; import { apiKeyAuth } from '../middleware/auth.middleware'; import { validateRequest } from '../middleware/validation'; import { expensiveLimiter } from '../middleware/rate-limit.middleware'; @@ -7,25 +6,10 @@ import scraperManager from '../../scraper/scraper-manager'; import { selfHealExtract, DesiredField } from '../../services/self-heal-extractor.service'; import { CssExtractionSchema } from '../../transformers/css-extractor'; import { logger } from '../../utils/logger'; +import { extractAutoSchema } from '../schemas'; const router = Router(); -const fieldSchema = z.object({ - name: z.string().min(1).max(100), - description: z.string().max(500).optional(), - type: z.enum(['text', 'attribute', 'html', 'number', 'list', 'nested', 'nested_list']).optional(), - attribute: z.string().max(100).optional(), - required: z.boolean().optional(), -}); - -const bodySchema = z.object({ - url: z.string().url(), - fields: z.array(fieldSchema).min(1).max(50), - cssSchema: z.any().optional(), // optional bootstrap schema (skips first LLM derivation) - forceReheal: z.boolean().optional(), - scrapeOptions: z.record(z.any()).optional(), -}); - /** * @route POST /api/extract-auto * @desc Self-healing structured extraction. Derives CSS selectors with an LLM once, @@ -33,7 +17,7 @@ const bodySchema = z.object({ * the site changes and the selectors stop yielding data. * @access Private (API key required) */ -router.post('/', expensiveLimiter, apiKeyAuth, validateRequest(bodySchema), async (req: Request, res: Response) => { +router.post('/', expensiveLimiter, apiKeyAuth, validateRequest(extractAutoSchema), async (req: Request, res: Response) => { const { url, fields, cssSchema, forceReheal, scrapeOptions } = req.body as { url: string; fields: DesiredField[]; diff --git a/src/api/routes/map.routes.ts b/src/api/routes/map.routes.ts index 83af6a2..419acd6 100644 --- a/src/api/routes/map.routes.ts +++ b/src/api/routes/map.routes.ts @@ -3,27 +3,10 @@ import { MapController } from '../controllers/map.controller'; import { apiKeyAuth } from '../middleware/auth.middleware'; import { validateRequest } from '../middleware/validation'; import { expensiveLimiter } from '../middleware/rate-limit.middleware'; -import { z } from 'zod'; +import { mapRequestSchema, mapClearCacheSchema } from '../schemas'; // Validation schema for map request -const mapRequestSchema = z.object({ - url: z.string().url('Invalid URL format'), - maxUrls: z.number().int().min(1).max(30000).optional().default(5000), - includeSubdomains: z.boolean().optional().default(true), - searchQuery: z.string().optional(), - skipSitemaps: z.boolean().optional().default(false), - sitemapsOnly: z.boolean().optional().default(false), - useUrlIndex: z.boolean().optional().default(true), - timeoutMs: z.number().int().min(1000).max(300000).optional().default(30000), - includePatterns: z.array(z.string()).optional(), - excludePatterns: z.array(z.string()).optional() -}); - // Validation schema for cache clear request -const clearCacheSchema = z.object({ - url: z.string().url('Invalid URL format') -}); - const router = Router(); const mapController = new MapController(); @@ -243,7 +226,7 @@ router.get( router.post( '/cache/clear', apiKeyAuth, - validateRequest(clearCacheSchema), + validateRequest(mapClearCacheSchema), mapController.clearCache.bind(mapController) ); diff --git a/src/api/routes/parse.routes.ts b/src/api/routes/parse.routes.ts index 6ec4874..5fdd432 100644 --- a/src/api/routes/parse.routes.ts +++ b/src/api/routes/parse.routes.ts @@ -1,25 +1,19 @@ import { Router, Request, Response } from 'express'; -import { z } from 'zod'; import { apiKeyAuth } from '../middleware/auth.middleware'; import { validateRequest } from '../middleware/validation'; import { expensiveLimiter } from '../middleware/rate-limit.middleware'; import { parseDocument } from '../../services/document-parser.service'; import { logger } from '../../utils/logger'; +import { parseRequestSchema } from '../schemas'; const router = Router(); -const parseSchema = z.object({ - content: z.string().optional(), // base64 - url: z.string().url().optional(), - contentType: z.string().max(100).optional(), -}).refine(d => d.content || d.url, { message: 'Provide `content` (base64) or `url`' }); - /** * @route POST /api/parse * @desc Parse a PDF / DOCX / HTML document → markdown. * @access Private (API key required) */ -router.post('/', expensiveLimiter, apiKeyAuth, validateRequest(parseSchema), async (req: Request, res: Response) => { +router.post('/', expensiveLimiter, apiKeyAuth, validateRequest(parseRequestSchema), async (req: Request, res: Response) => { try { const result = await parseDocument(req.body); res.json({ success: true, ...result }); diff --git a/src/api/routes/phase2-tools.routes.ts b/src/api/routes/phase2-tools.routes.ts index 5dfe141..f5317ab 100644 --- a/src/api/routes/phase2-tools.routes.ts +++ b/src/api/routes/phase2-tools.routes.ts @@ -1,20 +1,15 @@ import { Router, Request, Response } from 'express'; -import { z } from 'zod'; import { apiKeyAuth } from '../middleware/auth.middleware'; import { validateRequest } from '../middleware/validation'; import { expensiveLimiter, statusLimiter } from '../middleware/rate-limit.middleware'; import scraperManager from '../../scraper/scraper-manager'; import { discoverApis } from '../../services/api-discovery.service'; import { logger } from '../../utils/logger'; +import { discoverApisSchema, crawlEstimateSchema } from '../schemas'; // ---- POST /api/discover-apis : surface a page's underlying JSON/XHR endpoints ---- export const discoverApisRouter = Router(); -const discoverSchema = z.object({ - url: z.string().url(), - timeout: z.number().int().min(1000).max(120000).optional(), - includeNonJson: z.boolean().optional(), -}); -discoverApisRouter.post('/', expensiveLimiter, apiKeyAuth, validateRequest(discoverSchema), async (req: Request, res: Response) => { +discoverApisRouter.post('/', expensiveLimiter, apiKeyAuth, validateRequest(discoverApisSchema), async (req: Request, res: Response) => { try { const { url, timeout, includeNonJson } = req.body; const result = await discoverApis(url, { timeout, includeNonJson }); @@ -55,13 +50,7 @@ readerRouter.get('/', expensiveLimiter, apiKeyAuth, async (req: Request, res: Re // ---- POST /api/crawl/estimate : pre-run cost/size estimate ---- export const crawlEstimateRouter = Router(); -const estimateSchema = z.object({ - url: z.string().url().optional(), - limit: z.number().int().positive().optional(), - maxDepth: z.number().int().min(0).optional(), - scrapeOptions: z.record(z.any()).optional(), -}); -crawlEstimateRouter.post('/', statusLimiter, apiKeyAuth, validateRequest(estimateSchema), (req: Request, res: Response) => { +crawlEstimateRouter.post('/', statusLimiter, apiKeyAuth, validateRequest(crawlEstimateSchema), (req: Request, res: Response) => { const { limit, scrapeOptions } = req.body as { limit?: number; scrapeOptions?: Record }; const maxLimit = Number(process.env.MAX_CRAWL_LIMIT ?? 1000); const maxPages = Math.min(limit ?? 100, maxLimit); diff --git a/src/api/routes/scraper.ts b/src/api/routes/scraper.ts index ee48807..ab93e68 100644 --- a/src/api/routes/scraper.ts +++ b/src/api/routes/scraper.ts @@ -1,5 +1,4 @@ import { Router, Request, Response } from 'express'; -import { z } from 'zod'; import scraperManager from '../../scraper/scraper-manager'; import { logger } from '../../utils/logger'; import { apiKeyAuth as auth } from '../middleware/auth.middleware'; @@ -7,6 +6,7 @@ import { validateRequest } from '../middleware/validation'; import { expensiveLimiter, statusLimiter } from '../middleware/rate-limit.middleware'; import { createTask, getTask } from '../../services/task.service'; import { ExtractionResult } from '../../types/schema'; +import { scrapeRequestSchema, extractSchemaRequestSchema, summarizeRequestSchema, cacheInvalidateSchema } from '../schemas'; // Extended ScraperResponse interface to include extraction results interface ExtendedScraperResponse { @@ -35,15 +35,6 @@ interface ExtendedScraperResponse { const router = Router(); // Browser action schema -const browserActionSchema = z.object({ - type: z.enum(['click', 'scroll', 'wait', 'fill', 'select']), - selector: z.string().optional(), - value: z.string().optional(), - position: z.number().optional(), - timeout: z.number().optional(), - optional: z.boolean().optional() -}); - /** * @route POST /api/scrape/async * @desc Submit a scrape as an async job (returns a job id to poll) @@ -87,25 +78,7 @@ router.post( expensiveLimiter, auth, validateRequest( - z.object({ - url: z.string().url(), - options: z.object({ - waitForSelector: z.string().optional(), - waitForTimeout: z.number().int().positive().optional(), - actions: z.array(browserActionSchema).optional(), - skipCache: z.boolean().optional(), - cacheTtl: z.number().int().positive().optional(), - extractorFormat: z.enum(['html', 'markdown', 'text']).optional(), - onlyMainContent: z.boolean().optional(), - fitMarkdown: z.boolean().optional(), - useBrowser: z.boolean().optional(), - stealthMode: z.boolean().optional(), - skipTlsVerification: z.boolean().optional(), - // Extraction options (LLM or deterministic CSS) — validated downstream. - extractionOptions: z.any().optional(), - // Allow forward-compatible scraper options through to the manager. - }).passthrough().optional() - }) + scrapeRequestSchema ), async (req: Request, res: Response) => { try { @@ -170,21 +143,7 @@ router.post( expensiveLimiter, auth, validateRequest( - z.object({ - url: z.string().url(), - schema: z.object({}).passthrough(), // Allow any schema object - options: z.object({ - waitForSelector: z.string().optional(), - waitForTimeout: z.number().int().positive().optional(), - actions: z.array(browserActionSchema).optional(), - skipCache: z.boolean().optional(), - cacheTtl: z.number().int().positive().optional(), - extractorFormat: z.enum(['html', 'markdown', 'text']).optional(), - temperature: z.number().min(0).max(2).optional(), - maxTokens: z.number().int().positive().optional(), - instructions: z.string().optional() - }).optional() - }) + extractSchemaRequestSchema ), async (req: Request, res: Response) => { try { @@ -403,19 +362,7 @@ router.post( expensiveLimiter, auth, validateRequest( - z.object({ - url: z.string().url(), - maxLength: z.number().int().positive().optional(), // Maximum length of summary in words - options: z.object({ - waitForSelector: z.string().optional(), - waitForTimeout: z.number().int().positive().optional(), - actions: z.array(browserActionSchema).optional(), - skipCache: z.boolean().optional(), - cacheTtl: z.number().int().positive().optional(), - extractorFormat: z.enum(['html', 'markdown', 'text']).optional(), - temperature: z.number().min(0).max(2).optional(), - }).optional() - }) + summarizeRequestSchema ), async (req: Request, res: Response) => { try { @@ -514,9 +461,7 @@ router.delete( '/cache', auth, validateRequest( - z.object({ - url: z.string().optional() // If provided, invalidate only this URL - }) + cacheInvalidateSchema ), async (req: Request, res: Response) => { try { diff --git a/src/api/routes/search.routes.ts b/src/api/routes/search.routes.ts index 700ecb9..5397020 100644 --- a/src/api/routes/search.routes.ts +++ b/src/api/routes/search.routes.ts @@ -1,24 +1,14 @@ import { Router, Request, Response } from 'express'; -import { z } from 'zod'; import { apiKeyAuth } from '../middleware/auth.middleware'; import { validateRequest } from '../middleware/validation'; import { expensiveLimiter } from '../middleware/rate-limit.middleware'; import { searchWeb } from '../../services/search.service'; import scraperManager from '../../scraper/scraper-manager'; import { logger } from '../../utils/logger'; +import { searchRequestSchema } from '../schemas'; const router = Router(); -const searchSchema = z.object({ - query: z.string().min(1).max(500), - limit: z.number().int().positive().max(50).optional(), - provider: z.enum(['duckduckgo', 'searxng', 'serper']).optional(), - lang: z.string().max(10).optional(), - // When true, scrape each result and attach markdown content. - scrapeResults: z.boolean().optional(), - scrapeOptions: z.record(z.any()).optional(), -}); - /** * @route POST /api/search * @desc Web search (+ optional scrape of each result). @@ -28,7 +18,7 @@ router.post( '/', expensiveLimiter, apiKeyAuth, - validateRequest(searchSchema), + validateRequest(searchRequestSchema), async (req: Request, res: Response) => { try { const { query, limit = 10, provider, lang, scrapeResults, scrapeOptions } = req.body; diff --git a/src/api/routes/session.routes.ts b/src/api/routes/session.routes.ts index 5a1325e..89cf6eb 100644 --- a/src/api/routes/session.routes.ts +++ b/src/api/routes/session.routes.ts @@ -1,5 +1,4 @@ import { Router, Request, Response } from 'express'; -import { z } from 'zod'; import { apiKeyAuth } from '../middleware/auth.middleware'; import { validateRequest } from '../middleware/validation'; import { expensiveLimiter, statusLimiter } from '../middleware/rate-limit.middleware'; @@ -10,42 +9,15 @@ import { SessionCapacityError, } from '../../services/session-manager.service'; import { logger } from '../../utils/logger'; +import { sessionCreateSchema, sessionActionSchema } from '../schemas'; const router = Router(); -const createSchema = z.object({ - userAgent: z.string().max(500).optional(), - viewport: z.object({ width: z.number().int().min(200).max(4000), height: z.number().int().min(200).max(4000) }).optional(), - initialUrl: z.string().url().optional(), - proxy: z.object({ server: z.string().max(300), username: z.string().max(200).optional(), password: z.string().max(200).optional() }).optional(), -}); - -const ACTION_TYPES = [ - 'navigate', 'click', 'type', 'fill', 'select', 'scroll', 'waitForSelector', - 'wait', 'screenshot', 'scrape', 'evaluate', 'back', 'forward', 'reload', 'content', -] as const; - -const actionSchema = z.object({ - type: z.enum(ACTION_TYPES), - url: z.string().url().optional(), - selector: z.string().max(2000).optional(), - value: z.string().optional(), - text: z.string().optional(), - position: z.number().optional(), - timeout: z.number().int().min(0).max(120000).optional(), - script: z.string().max(20000).optional(), - fullPage: z.boolean().optional(), - formats: z.array(z.string()).max(6).optional(), - onlyMainContent: z.boolean().optional(), - fitMarkdown: z.boolean().optional(), - waitUntil: z.enum(['load', 'domcontentloaded', 'networkidle', 'commit']).optional(), -}).passthrough(); - /** * @route POST /api/sessions * @desc Create a persistent interactive browser session. */ -router.post('/', expensiveLimiter, apiKeyAuth, validateRequest(createSchema), async (req: Request, res: Response) => { +router.post('/', expensiveLimiter, apiKeyAuth, validateRequest(sessionCreateSchema), async (req: Request, res: Response) => { try { const info = await sessionManager.createSession(req.body); res.status(201).json({ success: true, session: info }); @@ -80,7 +52,7 @@ router.get('/:id', statusLimiter, apiKeyAuth, (req: Request, res: Response) => { * @route POST /api/sessions/:id/action * @desc Run one action against the session (navigate/click/type/scrape/...). */ -router.post('/:id/action', expensiveLimiter, apiKeyAuth, validateRequest(actionSchema), async (req: Request, res: Response) => { +router.post('/:id/action', expensiveLimiter, apiKeyAuth, validateRequest(sessionActionSchema.passthrough()), async (req: Request, res: Response) => { try { const result = await sessionManager.runAction(req.params.id, req.body as SessionAction); const info = sessionManager.getSession(req.params.id); diff --git a/src/api/routes/sites.routes.ts b/src/api/routes/sites.routes.ts index 89f852f..b8998a4 100644 --- a/src/api/routes/sites.routes.ts +++ b/src/api/routes/sites.routes.ts @@ -1,5 +1,4 @@ import { Router, Request, Response } from 'express'; -import { z } from 'zod'; import { apiKeyAuth } from '../middleware/auth.middleware'; import { validateRequest } from '../middleware/validation'; import { expensiveLimiter, statusLimiter } from '../middleware/rate-limit.middleware'; @@ -16,39 +15,12 @@ import { } from '../../services/site-spec.service'; import { toSpecSummary } from '../../transformers/site-spec-core'; import { logger } from '../../utils/logger'; +import { siteCreateSchema, siteRunSchema } from '../schemas'; const router = Router(); -const paramSchema = z.object({ - name: z.string().min(1).max(60), - description: z.string().max(500).optional(), - required: z.boolean().optional(), -}); - -const fieldSchema = z.object({ - name: z.string().min(1).max(100), - description: z.string().max(500).optional(), - type: z.enum(['text', 'attribute', 'html', 'number', 'list', 'nested', 'nested_list']).optional(), - attribute: z.string().max(100).optional(), - required: z.boolean().optional(), -}); - -const createSchema = z.object({ - name: z.string().min(1).max(48), - description: z.string().max(500).optional(), - url: z.string().min(1).max(2000), - params: z.array(paramSchema).max(20).optional(), - fields: z.array(fieldSchema).min(1).max(50), - cssSchema: z.any().optional(), - sampleParams: z.record(z.any()).optional(), - sessionId: z.string().max(100).optional(), - verify: z.boolean().optional(), -}); - -const runSchema = z.object({ params: z.record(z.any()).optional() }); - /** @route POST /api/sites — create a saved, self-healing extraction spec. */ -router.post('/', expensiveLimiter, apiKeyAuth, validateRequest(createSchema), async (req: Request, res: Response) => { +router.post('/', expensiveLimiter, apiKeyAuth, validateRequest(siteCreateSchema), async (req: Request, res: Response) => { try { const { spec, sample, meta } = await createSpec(req.body); res.status(201).json({ success: true, spec: toSpecSummary(spec), sample, meta }); @@ -68,7 +40,7 @@ router.get('/', statusLimiter, apiKeyAuth, async (_req: Request, res: Response) }); /** @route POST /api/sites/by-name/:name/run — run a spec by its name (agent-friendly). */ -router.post('/by-name/:name/run', expensiveLimiter, apiKeyAuth, validateRequest(runSchema), async (req: Request, res: Response) => { +router.post('/by-name/:name/run', expensiveLimiter, apiKeyAuth, validateRequest(siteRunSchema), async (req: Request, res: Response) => { try { const result = await runSpecByName(req.params.name, req.body?.params ?? {}); res.status(result.success ? 200 : 400).json(result); @@ -88,7 +60,7 @@ router.get('/:id', statusLimiter, apiKeyAuth, async (req: Request, res: Response }); /** @route POST /api/sites/:id/run — execute the spec (with params) → fresh data. */ -router.post('/:id/run', expensiveLimiter, apiKeyAuth, validateRequest(runSchema), async (req: Request, res: Response) => { +router.post('/:id/run', expensiveLimiter, apiKeyAuth, validateRequest(siteRunSchema), async (req: Request, res: Response) => { const spec = await getSpec(req.params.id); if (!spec) return res.status(404).json({ success: false, error: 'spec not found' }); const result = await runSpec(spec, req.body?.params ?? {}); diff --git a/src/api/routes/task.routes.ts b/src/api/routes/task.routes.ts index 0d78a61..115aee4 100644 --- a/src/api/routes/task.routes.ts +++ b/src/api/routes/task.routes.ts @@ -1,10 +1,10 @@ import { Router, Request, Response } from 'express'; -import { z } from 'zod'; import { apiKeyAuth } from '../middleware/auth.middleware'; import { validateRequest } from '../middleware/validation'; import { expensiveLimiter, statusLimiter } from '../middleware/rate-limit.middleware'; import { createTask, getTask, TaskType } from '../../services/task.service'; import { logger } from '../../utils/logger'; +import { extractTaskSchema, llmstxtSchema } from '../schemas'; /** Shared status handler for async tasks. */ async function taskStatus(req: Request, res: Response): Promise { @@ -31,23 +31,10 @@ function makeCreateHandler(type: TaskType) { // ---- /api/extract (async multi-URL LLM extraction) ---- export const extractRouter = Router(); -const extractSchema = z.object({ - urls: z.array(z.string().url()).max(1000).optional(), - url: z.string().url().optional(), - prompt: z.string().max(5000).optional(), - schema: z.any().optional(), - limit: z.number().int().positive().max(1000).optional(), - scrapeOptions: z.record(z.any()).optional(), -}).refine(d => (d.urls && d.urls.length > 0) || d.url, { message: 'Provide `urls` or a `url`' }); -extractRouter.post('/', expensiveLimiter, apiKeyAuth, validateRequest(extractSchema), makeCreateHandler('extract')); +extractRouter.post('/', expensiveLimiter, apiKeyAuth, validateRequest(extractTaskSchema), makeCreateHandler('extract')); extractRouter.get('/:id', statusLimiter, apiKeyAuth, taskStatus); // ---- /api/llmstxt (generate llms.txt for a site) ---- export const llmstxtRouter = Router(); -const llmstxtSchema = z.object({ - url: z.string().url(), - maxUrls: z.number().int().positive().max(500).optional(), - includeFullText: z.boolean().optional(), -}); llmstxtRouter.post('/', expensiveLimiter, apiKeyAuth, validateRequest(llmstxtSchema), makeCreateHandler('llmstxt')); llmstxtRouter.get('/:id', statusLimiter, apiKeyAuth, taskStatus); diff --git a/src/api/schemas/index.ts b/src/api/schemas/index.ts new file mode 100644 index 0000000..5adf7b2 --- /dev/null +++ b/src/api/schemas/index.ts @@ -0,0 +1,326 @@ +import { z } from 'zod'; + +/** + * Request schemas for every documented endpoint — the single source of truth for + * BOTH runtime validation (the route handlers import these) and the generated + * OpenAPI spec (`src/api/openapi` imports these). + * + * This module must stay PURE: it may only import `zod`. The OpenAPI generator + * imports it directly, and pulling in a route/controller/service here would drag + * in Redis, BullMQ and the browser pool — all of which open handles at import + * time and would hang the generator. + * + * Adding a request field? Add it here and it appears in the spec automatically. + */ + +// --------------------------------------------------------------------------- +// Shared building blocks +// --------------------------------------------------------------------------- + +/** A scripted browser interaction performed before content is captured. */ +export const browserActionSchema = z.object({ + type: z.enum(['click', 'scroll', 'wait', 'fill', 'select']), + selector: z.string().optional(), + value: z.string().optional(), + position: z.number().optional(), + timeout: z.number().optional(), + optional: z.boolean().optional(), +}); + +/** Render/extraction options accepted by `POST /api/scrape`. */ +export const scrapeOptionsSchema = z.object({ + waitForSelector: z.string().optional(), + waitForTimeout: z.number().int().positive().optional(), + actions: z.array(browserActionSchema).optional(), + skipCache: z.boolean().optional(), + cacheTtl: z.number().int().positive().optional(), + extractorFormat: z.enum(['html', 'markdown', 'text']).optional(), + onlyMainContent: z.boolean().optional(), + fitMarkdown: z.boolean().optional(), + useBrowser: z.boolean().optional(), + stealthMode: z.boolean().optional(), + skipTlsVerification: z.boolean().optional(), + // Extraction options (LLM or deterministic CSS) — validated downstream. + extractionOptions: z.any().optional(), + // Allow forward-compatible scraper options through to the manager. +}); + +/** A field the caller wants extracted, used by extract-auto and site specs. */ +export const desiredFieldSchema = z.object({ + name: z.string().min(1).max(100), + description: z.string().max(500).optional(), + type: z.enum(['text', 'attribute', 'html', 'number', 'list', 'nested', 'nested_list']).optional(), + attribute: z.string().max(100).optional(), + required: z.boolean().optional(), +}); + +// --------------------------------------------------------------------------- +// Scrape +// --------------------------------------------------------------------------- + +export const scrapeRequestSchema = z.object({ + url: z.string().url(), + options: scrapeOptionsSchema.passthrough().optional(), +}); + +export const scrapeAsyncRequestSchema = z.object({ + url: z.string().url(), + options: scrapeOptionsSchema.passthrough().optional(), +}); + +export const extractSchemaRequestSchema = z.object({ + url: z.string().url(), + schema: z.object({}).passthrough(), // Allow any JSON Schema object + options: z + .object({ + waitForSelector: z.string().optional(), + waitForTimeout: z.number().int().positive().optional(), + actions: z.array(browserActionSchema).optional(), + skipCache: z.boolean().optional(), + cacheTtl: z.number().int().positive().optional(), + extractorFormat: z.enum(['html', 'markdown', 'text']).optional(), + temperature: z.number().min(0).max(2).optional(), + maxTokens: z.number().int().positive().optional(), + instructions: z.string().optional(), + }) + .optional(), +}); + +export const summarizeRequestSchema = z.object({ + url: z.string().url(), + maxLength: z.number().int().positive().optional(), // Maximum length of summary in words + options: z + .object({ + waitForSelector: z.string().optional(), + waitForTimeout: z.number().int().positive().optional(), + actions: z.array(browserActionSchema).optional(), + skipCache: z.boolean().optional(), + cacheTtl: z.number().int().positive().optional(), + extractorFormat: z.enum(['html', 'markdown', 'text']).optional(), + temperature: z.number().min(0).max(2).optional(), + }) + .optional(), +}); + +export const cacheInvalidateSchema = z.object({ + url: z.string().optional(), // If provided, invalidate only this URL +}); + +// --------------------------------------------------------------------------- +// Crawl +// --------------------------------------------------------------------------- + +/** + * Mirrors `validateCrawlRequest` (crawl-validation.middleware.ts), which also + * CLAMPS numeric values to the MAX_CRAWL_* env limits rather than rejecting. + */ +export const crawlRequestSchema = z.object({ + url: z.string().url(), + limit: z.number().int().positive().optional(), + maxUrls: z.number().int().positive().optional(), + maxDepth: z.number().int().min(0).optional(), + maxDiscoveryDepth: z.number().int().min(0).optional(), + webhook: z.string().url().optional(), + includePaths: z.array(z.string()).optional(), + excludePaths: z.array(z.string()).optional(), + includePatterns: z.array(z.string()).optional(), + excludePatterns: z.array(z.string()).optional(), + crawlOptions: z + .object({ + maxConcurrentCrawlers: z.number().int().positive().optional(), + browserPoolSize: z.number().int().positive().optional(), + }) + .passthrough() + .optional(), + scrapeOptions: z.record(z.any()).optional(), +}); + +export const crawlEstimateSchema = z.object({ + url: z.string().url().optional(), + limit: z.number().int().positive().optional(), + maxDepth: z.number().int().min(0).optional(), + scrapeOptions: z.record(z.any()).optional(), +}); + +// --------------------------------------------------------------------------- +// Batch scrape (validated at runtime by express-validator; mirrored here) +// --------------------------------------------------------------------------- + +export const batchScrapeRequestSchema = z.object({ + urls: z.array(z.string().url()).min(1).max(100), + concurrency: z.number().int().min(1).max(10).optional(), + webhook: z.string().url().optional(), + timeout: z.number().int().min(10000).optional(), + failFast: z.boolean().optional(), + maxRetries: z.number().int().min(0).max(10).optional(), + options: z + .object({ + timeout: z.number().int().min(1000).max(300000).optional(), + userAgent: z.string().max(500).optional(), + waitForTimeout: z.number().int().min(0).max(60000).optional(), + }) + .passthrough() + .optional(), +}); + +// --------------------------------------------------------------------------- +// Map (URL discovery) +// --------------------------------------------------------------------------- + +export const mapRequestSchema = z.object({ + url: z.string().url('Invalid URL format'), + maxUrls: z.number().int().min(1).max(30000).optional().default(5000), + includeSubdomains: z.boolean().optional().default(true), + searchQuery: z.string().optional(), + skipSitemaps: z.boolean().optional().default(false), + sitemapsOnly: z.boolean().optional().default(false), + useUrlIndex: z.boolean().optional().default(true), + timeoutMs: z.number().int().min(1000).max(300000).optional().default(30000), + includePatterns: z.array(z.string()).optional(), + excludePatterns: z.array(z.string()).optional(), +}); + +export const mapClearCacheSchema = z.object({ + url: z.string().url('Invalid URL format'), +}); + +// --------------------------------------------------------------------------- +// Search +// --------------------------------------------------------------------------- + +export const searchRequestSchema = z.object({ + query: z.string().min(1).max(500), + limit: z.number().int().positive().max(50).optional(), + provider: z.enum(['duckduckgo', 'searxng', 'serper']).optional(), + lang: z.string().max(10).optional(), + // When true, scrape each result and attach markdown content. + scrapeResults: z.boolean().optional(), + scrapeOptions: z.record(z.any()).optional(), +}); + +// --------------------------------------------------------------------------- +// Async tasks: extract + llms.txt +// --------------------------------------------------------------------------- + +export const extractTaskSchema = z.object({ + urls: z.array(z.string().url()).max(1000).optional(), + url: z.string().url().optional(), + prompt: z.string().max(5000).optional(), + schema: z.any().optional(), + limit: z.number().int().positive().max(1000).optional(), + scrapeOptions: z.record(z.any()).optional(), +}); + +export const llmstxtSchema = z.object({ + url: z.string().url(), + maxUrls: z.number().int().positive().max(500).optional(), + includeFullText: z.boolean().optional(), +}); + +// --------------------------------------------------------------------------- +// Self-healing extraction +// --------------------------------------------------------------------------- + +export const extractAutoSchema = z.object({ + url: z.string().url(), + fields: z.array(desiredFieldSchema).min(1).max(50), + cssSchema: z.any().optional(), // optional bootstrap schema (skips first LLM derivation) + forceReheal: z.boolean().optional(), + scrapeOptions: z.record(z.any()).optional(), +}); + +// --------------------------------------------------------------------------- +// Agent +// --------------------------------------------------------------------------- + +export const agentRequestSchema = z.object({ + url: z.string().url(), + prompt: z.string().min(1).max(5000), + schema: z.any().optional(), + maxSteps: z.number().int().positive().max(20).optional(), + onlyMainContent: z.boolean().optional(), + fitMarkdown: z.boolean().optional(), +}); + +// --------------------------------------------------------------------------- +// Persistent browser sessions +// --------------------------------------------------------------------------- + +export const SESSION_ACTION_TYPES = [ + 'navigate', 'click', 'type', 'fill', 'select', 'scroll', 'waitForSelector', + 'wait', 'screenshot', 'scrape', 'evaluate', 'back', 'forward', 'reload', 'content', +] as const; + +export const sessionCreateSchema = z.object({ + userAgent: z.string().max(500).optional(), + viewport: z + .object({ + width: z.number().int().min(200).max(4000), + height: z.number().int().min(200).max(4000), + }) + .optional(), + initialUrl: z.string().url().optional(), + proxy: z + .object({ + server: z.string().max(300), + username: z.string().max(200).optional(), + password: z.string().max(200).optional(), + }) + .optional(), +}); + +export const sessionActionSchema = z.object({ + type: z.enum(SESSION_ACTION_TYPES), + url: z.string().url().optional(), + selector: z.string().max(2000).optional(), + value: z.string().optional(), + text: z.string().optional(), + position: z.number().optional(), + timeout: z.number().int().min(0).max(120000).optional(), + script: z.string().max(20000).optional(), + fullPage: z.boolean().optional(), + formats: z.array(z.string()).max(6).optional(), + onlyMainContent: z.boolean().optional(), + fitMarkdown: z.boolean().optional(), + waitUntil: z.enum(['load', 'domcontentloaded', 'networkidle', 'commit']).optional(), +}); + +// --------------------------------------------------------------------------- +// Site specs (site -> MCP endpoint generator) +// --------------------------------------------------------------------------- + +export const siteParamSchema = z.object({ + name: z.string().min(1).max(60), + description: z.string().max(500).optional(), + required: z.boolean().optional(), +}); + +export const siteCreateSchema = z.object({ + name: z.string().min(1).max(48), + description: z.string().max(500).optional(), + url: z.string().min(1).max(2000), + params: z.array(siteParamSchema).max(20).optional(), + fields: z.array(desiredFieldSchema).min(1).max(50), + cssSchema: z.any().optional(), + sampleParams: z.record(z.any()).optional(), + sessionId: z.string().max(100).optional(), + verify: z.boolean().optional(), +}); + +export const siteRunSchema = z.object({ params: z.record(z.any()).optional() }); + +// --------------------------------------------------------------------------- +// Misc tools +// --------------------------------------------------------------------------- + +export const parseRequestSchema = z.object({ + content: z.string().optional(), // base64 + url: z.string().url().optional(), + contentType: z.string().max(100).optional(), +}); + +export const discoverApisSchema = z.object({ + url: z.string().url(), + timeout: z.number().int().min(1000).max(120000).optional(), + includeNonJson: z.boolean().optional(), +}); diff --git a/swagger.yaml b/swagger.yaml index ba4299f..9229c19 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -1,579 +1,82 @@ +# GENERATED FILE — DO NOT EDIT BY HAND. +# +# Produced from the zod request schemas in src/api/schemas by +# npm run openapi:generate +# +# CI runs `npm run openapi:check`, which fails if this file is out of sync with +# the code, so request validation and these docs cannot silently diverge. openapi: 3.0.0 info: title: DeepScraper API version: 1.0.0 - description: | - DeepScraper is an AI-powered web scraping API with intelligent extraction capabilities. - It provides comprehensive web scraping, crawling, batch processing, and URL discovery features. - - ## Key Features - - 🤖 LLM-powered data extraction using OpenAI - - 📦 Batch processing with concurrent scraping - - 🕷️ Multi-page crawling with configurable strategies - - 🗺️ High-performance URL discovery (5000+ URLs in seconds) - - 🎭 Browser automation with Playwright - - 📝 Multiple output formats (HTML, Markdown, Text) - - ⚡ Smart caching with configurable TTL - - 🔄 Background job processing with Redis + description: >- + Open-source web scraping API — scrape, crawl, map, search, extract structured data, drive + persistent browser sessions, and expose any site as a reusable endpoint. + + This file is GENERATED from the zod request schemas in `src/api/schemas`. Do not edit by hand — + run `npm run openapi:generate`. + license: + name: Apache-2.0 + url: https://www.apache.org/licenses/LICENSE-2.0 servers: - - url: https://app.extractr.ai - description: Production server - url: http://localhost:3000 - description: Local development server - + description: Local instance components: securitySchemes: ApiKeyAuth: type: apiKey - in: header name: X-API-Key - description: API key for authentication - + in: header + description: API key. Required on every /api/* route. schemas: - # Common schemas - BrowserAction: - type: object - properties: - type: - type: string - enum: [click, scroll, wait, fill, select] - description: Type of browser action to perform - selector: - type: string - description: CSS selector for the target element - value: - type: string - description: Value for fill/select actions - position: - type: number - description: Scroll position in pixels - timeout: - type: number - description: Wait timeout in milliseconds - optional: - type: boolean - description: If true, action failure won't stop scraping - - ScraperOptions: - type: object - properties: - timeout: - type: integer - description: Request timeout in milliseconds - minimum: 1000 - maximum: 300000 - default: 30000 - blockAds: - type: boolean - description: Block ad domains - default: false - blockResources: - type: boolean - description: Block images, fonts, and media - default: false - userAgent: - type: string - description: Custom user agent string - proxy: - type: string - description: Proxy URL - cookies: - type: object - additionalProperties: - type: string - description: Cookies to send with requests - headers: - type: object - additionalProperties: - type: string - description: Custom headers - waitForSelector: - type: string - description: CSS selector to wait for before extracting - waitForTimeout: - type: integer - description: Time to wait in milliseconds - minimum: 0 - maximum: 60000 - fullPage: - type: boolean - description: Capture full page screenshot - default: false - javascript: - type: boolean - description: Enable JavaScript execution - default: true - extractorFormat: - type: string - enum: [html, markdown, text] - default: html - description: Output format - actions: - type: array - items: - $ref: '#/components/schemas/BrowserAction' - description: Browser actions to perform - skipCache: - type: boolean - description: Skip cache for this request - default: false - cacheTtl: - type: integer - description: Custom cache TTL in seconds - minimum: 0 - skipTlsVerification: - type: boolean - description: Skip TLS certificate verification - default: false - useBrowser: - type: boolean - description: Use browser-based scraping with Playwright - default: false - stealthMode: - type: boolean - description: Enable stealth mode to avoid bot detection - default: false - - SchemaProperty: - type: object - properties: - type: - type: string - enum: [string, number, integer, boolean, array, object, null] - description: - type: string - required: - type: boolean - format: - type: string - items: - $ref: '#/components/schemas/SchemaProperty' - properties: - type: object - additionalProperties: - $ref: '#/components/schemas/SchemaProperty' - - Schema: - type: object - properties: - type: - type: string - enum: [object] - default: object - title: - type: string - description: - type: string - properties: - type: object - additionalProperties: - $ref: '#/components/schemas/SchemaProperty' - required: - type: array - items: - type: string - - ScraperResponse: - type: object - properties: - success: - type: boolean - url: - type: string - title: - type: string - content: - type: string - contentType: - type: string - enum: [html, markdown, text] - metadata: - type: object - properties: - timestamp: - type: string - format: date-time - status: - type: integer - headers: - type: object - processingTime: - type: integer - description: Processing time in milliseconds - fromCache: - type: boolean - - CrawlOptions: - type: object - properties: - includePaths: - type: array - items: - type: string - description: Regex patterns for paths to include - excludePaths: - type: array - items: - type: string - description: Regex patterns for paths to exclude - limit: - type: integer - minimum: 1 - maximum: 10000 - default: 100 - description: Maximum number of pages to crawl - maxDepth: - type: integer - minimum: 1 - maximum: 10 - default: 5 - description: Maximum crawl depth - allowBackwardCrawling: - type: boolean - default: false - allowExternalContentLinks: - type: boolean - default: false - allowSubdomains: - type: boolean - default: false - ignoreRobotsTxt: - type: boolean - default: false - regexOnFullURL: - type: boolean - default: false - scrapeOptions: - $ref: '#/components/schemas/ScraperOptions' - webhook: - type: string - format: uri - description: Webhook URL for completion notification - strategy: - type: string - enum: [bfs, dfs, best_first] - default: bfs - description: Crawl strategy - useBrowser: - type: boolean - default: false - description: Use browser-based crawling - useMapDiscovery: - type: boolean - default: false - description: Use high-performance URL discovery - - CrawlResponse: + SuccessResponse: type: object properties: success: type: boolean - id: - type: string - description: Crawl job ID - url: - type: string - description: Status endpoint URL - message: - type: string - outputDirectory: - type: string - description: Directory where files will be exported - crawlType: - type: string - streamingEnabled: - type: boolean - - CrawlStatus: + enum: + - true + required: + - success + ErrorResponse: type: object properties: success: type: boolean - status: + enum: + - false + error: type: string - enum: [scraping, completed, cancelled] - crawl: - type: object - jobs: - type: array - items: - type: object - properties: - id: - type: string - status: - type: string - document: - type: object - error: - type: string - count: - type: integer - exportedFiles: - type: object - properties: - count: - type: integer - outputDirectory: - type: string - files: - type: array - items: - type: string - - BatchScrapeRequest: - type: object required: - - urls - properties: - urls: - type: array - items: - type: string - format: uri - minItems: 1 - maxItems: 100 - description: URLs to scrape - options: - $ref: '#/components/schemas/ScraperOptions' - concurrency: - type: integer - minimum: 1 - maximum: 10 - default: 3 - description: Number of concurrent scraping operations - webhook: - type: string - format: uri - description: Webhook URL for notifications - timeout: - type: integer - minimum: 10000 - description: Overall timeout in milliseconds - failFast: - type: boolean - default: false - description: Stop on first error - maxRetries: - type: integer - minimum: 0 - maximum: 10 - default: 3 - - BatchScrapeResponse: + - success + - error + AsyncJobAccepted: type: object properties: success: type: boolean - batchId: - type: string - totalUrls: - type: integer - message: - type: string - statusUrl: - type: string - webhook: + enum: + - true + id: type: string - estimatedTime: - type: integer - - BatchStatus: - type: object - properties: - success: - type: boolean - batchId: + url: type: string + description: Polling URL for this job status: type: string - enum: [pending, processing, completed, completed_with_errors, failed, cancelled] - totalUrls: - type: integer - completedUrls: - type: integer - failedUrls: - type: integer - pendingUrls: - type: integer - progress: - type: number - format: float - minimum: 0 - maximum: 100 - startTime: - type: integer - endTime: - type: integer - processingTime: - type: integer - results: - type: array - items: - $ref: '#/components/schemas/ScraperResponse' - - MapRequest: - type: object required: + - success + - id - url - properties: - url: - type: string - format: uri - description: URL to discover links from - maxUrls: - type: integer - minimum: 1 - maximum: 30000 - default: 5000 - description: Maximum number of URLs to discover - includeSubdomains: - type: boolean - default: true - description: Include subdomain URLs - searchQuery: - type: string - description: Optional search query to filter URLs - skipSitemaps: - type: boolean - default: false - description: Skip sitemap discovery - sitemapsOnly: - type: boolean - default: false - description: Only use sitemap discovery - useUrlIndex: - type: boolean - default: true - description: Use URL index cache - timeoutMs: - type: integer - minimum: 1000 - maximum: 300000 - default: 30000 - description: Discovery timeout in milliseconds - includePatterns: - type: array - items: - type: string - description: Path patterns to include - excludePatterns: - type: array - items: - type: string - description: Path patterns to exclude - rateLimitingOptions: - type: object - properties: - minDelay: - type: integer - default: 500 - maxConcurrency: - type: integer - default: 2 - crawlOptions: - type: object - properties: - maxCrawlDepth: - type: integer - minimum: 1 - maximum: 5 - default: 3 - maxConcurrentCrawlers: - type: integer - minimum: 1 - maximum: 20 - default: 8 - - MapResponse: - type: object - properties: - success: - type: boolean - data: - type: object - properties: - links: - type: array - items: - type: string - total: - type: integer - discoveryMethods: - type: object - properties: - sitemap: - type: integer - search: - type: integer - crawling: - type: integer - commonPaths: - type: integer - robotsSitemaps: - type: integer - documents: - type: integer - timeTaken: - type: integer - fromCache: - type: boolean - metadata: - type: object - properties: - url: - type: string - includeSubdomains: - type: boolean - maxUrls: - type: integer - timestamp: - type: string - format: date-time - - ErrorResponse: - type: object - properties: - success: - type: boolean - example: false - error: - type: string - message: - type: string - + - status + parameters: {} paths: - # Health check - /health: - get: - summary: Health check endpoint - tags: - - System - responses: - '200': - description: Service is healthy - content: - application/json: - schema: - type: object - properties: - status: - type: string - example: UP - message: - type: string - example: Service is running - - # Scraping endpoints /api/scrape: post: - summary: Scrape a single URL - description: | - Scrapes content from a single URL with various options for extraction and formatting. - Supports browser automation, custom actions, and multiple output formats. tags: - - Scraping + - Scrape + summary: Scrape a URL to markdown, HTML, text or structured data security: - ApiKeyAuth: [] requestBody: @@ -582,44 +85,77 @@ paths: application/json: schema: type: object - required: - - url properties: url: type: string format: uri - description: URL to scrape options: - $ref: '#/components/schemas/ScraperOptions' - examples: - basic: - summary: Basic scraping - value: - url: "https://example.com" - markdown: - summary: Scrape as markdown - value: - url: "https://example.com" - options: - extractorFormat: "markdown" - withActions: - summary: With browser actions - value: - url: "https://example.com" - options: - extractorFormat: "markdown" + type: object + properties: + waitForSelector: + type: string + waitForTimeout: + type: integer + minimum: 0 + exclusiveMinimum: true actions: - - type: "click" - selector: ".load-more" - - type: "wait" - timeout: 2000 + type: array + items: + type: object + properties: + type: + type: string + enum: + - click + - scroll + - wait + - fill + - select + selector: + type: string + value: + type: string + position: + type: number + timeout: + type: number + optional: + type: boolean + required: + - type + skipCache: + type: boolean + cacheTtl: + type: integer + minimum: 0 + exclusiveMinimum: true + extractorFormat: + type: string + enum: + - html + - markdown + - text + onlyMainContent: + type: boolean + fitMarkdown: + type: boolean + useBrowser: + type: boolean + stealthMode: + type: boolean + skipTlsVerification: + type: boolean + extractionOptions: + nullable: true + required: + - url responses: '200': - description: Successfully scraped content + description: Success content: application/json: schema: - $ref: '#/components/schemas/ScraperResponse' + $ref: '#/components/schemas/SuccessResponse' '400': description: Invalid request content: @@ -627,26 +163,22 @@ paths: schema: $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized - Invalid API key + description: Missing or invalid API key content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - '500': - description: Server error + '429': + description: Rate limit or quota exceeded content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - - /api/extract-schema: + /api/scrape/async: post: - summary: Extract structured data using a schema - description: | - Uses LLM (OpenAI) to extract structured data from a webpage based on a provided schema. - Requires OpenAI API key configuration. tags: - - Scraping + - Scrape + summary: Submit a scrape as an async job security: - ApiKeyAuth: [] requestBody: @@ -655,70 +187,77 @@ paths: application/json: schema: type: object - required: - - url - - schema properties: url: type: string format: uri - description: URL to extract data from - schema: - $ref: '#/components/schemas/Schema' options: - allOf: - - $ref: '#/components/schemas/ScraperOptions' - - type: object - properties: - temperature: - type: number - minimum: 0 - maximum: 2 - default: 0.2 - maxTokens: - type: integer - minimum: 1 - instructions: - type: string - description: Additional extraction instructions - examples: - product: - summary: Extract product data - value: - url: "https://shop.example.com/product" - schema: - type: "object" - properties: - name: - type: "string" - description: "Product name" - price: - type: "string" - description: "Product price" - description: - type: "string" - description: "Product description" - required: ["name", "price"] + type: object + properties: + waitForSelector: + type: string + waitForTimeout: + type: integer + minimum: 0 + exclusiveMinimum: true + actions: + type: array + items: + type: object + properties: + type: + type: string + enum: + - click + - scroll + - wait + - fill + - select + selector: + type: string + value: + type: string + position: + type: number + timeout: + type: number + optional: + type: boolean + required: + - type + skipCache: + type: boolean + cacheTtl: + type: integer + minimum: 0 + exclusiveMinimum: true + extractorFormat: + type: string + enum: + - html + - markdown + - text + onlyMainContent: + type: boolean + fitMarkdown: + type: boolean + useBrowser: + type: boolean + stealthMode: + type: boolean + skipTlsVerification: + type: boolean + extractionOptions: + nullable: true + required: + - url responses: '200': - description: Successfully extracted structured data + description: Job accepted content: application/json: schema: - type: object - properties: - success: - type: boolean - url: - type: string - title: - type: string - extractedData: - type: object - contentType: - type: string - metadata: - type: object + $ref: '#/components/schemas/AsyncJobAccepted' '400': description: Invalid request content: @@ -726,75 +265,38 @@ paths: schema: $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized + description: Missing or invalid API key content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - '500': - description: Server error + '429': + description: Rate limit or quota exceeded content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - - /api/summarize: - post: - summary: Summarize webpage content - description: | - Uses LLM to generate a concise summary of webpage content. - Requires OpenAI API key configuration. + /api/scrape/job/{id}: + get: tags: - - Scraping + - Scrape + summary: Poll an async scrape job security: - ApiKeyAuth: [] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - url - properties: - url: - type: string - format: uri - description: URL to summarize - maxLength: - type: integer - minimum: 50 - maximum: 2000 - default: 500 - description: Maximum summary length in words - options: - allOf: - - $ref: '#/components/schemas/ScraperOptions' - - type: object - properties: - temperature: - type: number - minimum: 0 - maximum: 2 - default: 0.3 + parameters: + - schema: + type: string + description: Async scrape job id + required: true + name: id + in: path responses: '200': - description: Successfully generated summary + description: Success content: application/json: schema: - type: object - properties: - success: - type: boolean - url: - type: string - title: - type: string - summary: - type: string - metadata: - type: object + $ref: '#/components/schemas/SuccessResponse' '400': description: Invalid request content: @@ -802,27 +304,32 @@ paths: schema: $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized + description: Missing or invalid API key content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - '500': - description: Server error + '404': + description: Not found content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - - /api/cache: - delete: - summary: Clear cache - description: Clear the entire cache or invalidate a specific URL + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/extract-schema: + post: tags: - - Scraping + - Scrape + summary: Extract structured data using a JSON Schema (LLM) security: - ApiKeyAuth: [] requestBody: + required: true content: application/json: schema: @@ -831,42 +338,98 @@ paths: url: type: string format: uri - description: If provided, only invalidate cache for this URL + schema: + type: object + properties: {} + options: + type: object + properties: + waitForSelector: + type: string + waitForTimeout: + type: integer + minimum: 0 + exclusiveMinimum: true + actions: + type: array + items: + type: object + properties: + type: + type: string + enum: + - click + - scroll + - wait + - fill + - select + selector: + type: string + value: + type: string + position: + type: number + timeout: + type: number + optional: + type: boolean + required: + - type + skipCache: + type: boolean + cacheTtl: + type: integer + minimum: 0 + exclusiveMinimum: true + extractorFormat: + type: string + enum: + - html + - markdown + - text + temperature: + type: number + minimum: 0 + maximum: 2 + maxTokens: + type: integer + minimum: 0 + exclusiveMinimum: true + instructions: + type: string + required: + - url + - schema responses: '200': - description: Cache cleared successfully + description: Success content: application/json: schema: - type: object - properties: - success: - type: boolean - message: - type: string + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized + description: Missing or invalid API key content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - '500': - description: Server error + '429': + description: Rate limit or quota exceeded content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - - # Crawling endpoints - /api/crawl: + /api/summarize: post: - summary: Start a web crawl - description: | - Initiates a multi-page crawl starting from the given URL. - Supports traditional crawling and high-performance URL discovery mode. - Individual pages are exported as markdown files. tags: - - Crawling + - Scrape + summary: Scrape a URL and generate an AI summary security: - ApiKeyAuth: [] requestBody: @@ -875,100 +438,206 @@ paths: application/json: schema: type: object - required: - - url properties: url: type: string format: uri - description: Starting URL for the crawl - includePaths: - type: array - items: - type: string - description: Regex patterns for paths to include - excludePaths: - type: array - items: - type: string - description: Regex patterns for paths to exclude - limit: + maxLength: type: integer - minimum: 1 - maximum: 10000 - default: 100 - maxDepth: - type: integer - minimum: 1 - maximum: 10 - default: 5 - allowBackwardCrawling: - type: boolean - default: false - allowExternalContentLinks: - type: boolean - default: false - allowSubdomains: - type: boolean - default: false - ignoreRobotsTxt: - type: boolean - default: false - regexOnFullURL: - type: boolean - default: false - scrapeOptions: - $ref: '#/components/schemas/ScraperOptions' - webhook: + minimum: 0 + exclusiveMinimum: true + options: + type: object + properties: + waitForSelector: + type: string + waitForTimeout: + type: integer + minimum: 0 + exclusiveMinimum: true + actions: + type: array + items: + type: object + properties: + type: + type: string + enum: + - click + - scroll + - wait + - fill + - select + selector: + type: string + value: + type: string + position: + type: number + timeout: + type: number + optional: + type: boolean + required: + - type + skipCache: + type: boolean + cacheTtl: + type: integer + minimum: 0 + exclusiveMinimum: true + extractorFormat: + type: string + enum: + - html + - markdown + - text + temperature: + type: number + minimum: 0 + maximum: 2 + required: + - url + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/cache: + delete: + tags: + - Scrape + summary: Invalidate the scrape cache (all, or a single URL) + security: + - ApiKeyAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + url: type: string - format: uri - strategy: + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/crawl: + post: + tags: + - Crawl + summary: Start a multi-page crawl + security: + - ApiKeyAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + url: type: string - enum: [bfs, dfs, best_first] - default: bfs - useBrowser: - type: boolean - default: false - useMapDiscovery: - type: boolean - default: false - description: Use high-performance URL discovery + format: uri + limit: + type: integer + minimum: 0 + exclusiveMinimum: true maxUrls: type: integer - description: Override limit for discovery mode - timeoutMs: + minimum: 0 + exclusiveMinimum: true + maxDepth: + type: integer + minimum: 0 + maxDiscoveryDepth: type: integer - description: Discovery timeout + minimum: 0 + webhook: + type: string + format: uri + includePaths: + type: array + items: + type: string + excludePaths: + type: array + items: + type: string + includePatterns: + type: array + items: + type: string + excludePatterns: + type: array + items: + type: string crawlOptions: type: object properties: maxConcurrentCrawlers: type: integer - minimum: 1 - maximum: 20 - default: 3 - examples: - basic: - summary: Basic crawl - value: - url: "https://docs.example.com" - limit: 50 - withDiscovery: - summary: High-performance crawl - value: - url: "https://docs.example.com" - useMapDiscovery: true - maxUrls: 1000 - includePatterns: ["/api/", "/docs/"] - scrapeOptions: - extractorFormat: "markdown" + minimum: 0 + exclusiveMinimum: true + browserPoolSize: + type: integer + minimum: 0 + exclusiveMinimum: true + scrapeOptions: + type: object + additionalProperties: + nullable: true + required: + - url responses: '200': - description: Crawl initiated successfully + description: Crawl started content: application/json: schema: - $ref: '#/components/schemas/CrawlResponse' + $ref: '#/components/schemas/AsyncJobAccepted' '400': description: Invalid request content: @@ -976,124 +645,1898 @@ paths: schema: $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized + description: Missing or invalid API key content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - '500': - description: Server error + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/crawl/estimate: + post: + tags: + - Crawl + summary: Pre-run size/cost estimate for a crawl + security: + - ApiKeyAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + url: + type: string + format: uri + limit: + type: integer + minimum: 0 + exclusiveMinimum: true + maxDepth: + type: integer + minimum: 0 + scrapeOptions: + type: object + additionalProperties: + nullable: true + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/crawl/active: + get: + tags: + - Crawl + summary: List currently-active crawls + security: + - ApiKeyAuth: [] + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - /api/crawl/{jobId}: get: - summary: Get crawl status - description: Get the current status and results of a crawl job tags: - - Crawling + - Crawl + summary: Get crawl status and exported files security: - ApiKeyAuth: [] parameters: - - name: jobId + - schema: + type: string + description: Crawl job id + required: true + name: jobId in: path + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + delete: + tags: + - Crawl + summary: Cancel a running crawl + security: + - ApiKeyAuth: [] + parameters: + - schema: + type: string + description: Crawl job id required: true - description: Crawl job ID - schema: + name: jobId + in: path + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/crawl/{jobId}/errors: + get: + tags: + - Crawl + summary: List per-page failures for a crawl + security: + - ApiKeyAuth: [] + parameters: + - schema: type: string - - name: skip - in: query - description: Number of results to skip - schema: + description: Crawl job id + required: true + name: jobId + in: path + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/crawl/{jobId}/stream: + get: + tags: + - Crawl + summary: Stream crawl pages as Server-Sent Events + security: + - ApiKeyAuth: [] + parameters: + - schema: + type: string + description: Crawl job id + required: true + name: jobId + in: path + responses: + '200': + description: SSE stream of pages as they complete + content: + text/event-stream: + schema: + type: string + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/crawl/{jobId}/download/zip: + get: + tags: + - Crawl + summary: Download all crawled pages as a ZIP + security: + - ApiKeyAuth: [] + parameters: + - schema: + type: string + description: Crawl job id + required: true + name: jobId + in: path + - schema: + type: string + enum: + - markdown + - json + required: false + name: format + in: query + responses: + '200': + description: ZIP archive of crawled pages + content: + application/zip: + schema: + type: string + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/crawl/{jobId}/download/json: + get: + tags: + - Crawl + summary: Download all crawled pages as one JSON array + security: + - ApiKeyAuth: [] + parameters: + - schema: + type: string + description: Crawl job id + required: true + name: jobId + in: path + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/batch/scrape: + post: + tags: + - Batch + summary: Scrape many URLs concurrently + security: + - ApiKeyAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + urls: + type: array + items: + type: string + format: uri + minItems: 1 + maxItems: 100 + concurrency: + type: integer + minimum: 1 + maximum: 10 + webhook: + type: string + format: uri + timeout: + type: integer + minimum: 10000 + failFast: + type: boolean + maxRetries: + type: integer + minimum: 0 + maximum: 10 + options: + type: object + properties: + timeout: + type: integer + minimum: 1000 + maximum: 300000 + userAgent: + type: string + maxLength: 500 + waitForTimeout: + type: integer + minimum: 0 + maximum: 60000 + required: + - urls + responses: + '200': + description: Batch accepted + content: + application/json: + schema: + $ref: '#/components/schemas/AsyncJobAccepted' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/batch/scrape/{batchId}/status: + get: + tags: + - Batch + summary: Batch progress and results + security: + - ApiKeyAuth: [] + parameters: + - schema: + type: string + description: Batch id (UUID) + required: true + name: batchId + in: path + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/batch/scrape/{batchId}/errors: + get: + tags: + - Batch + summary: Per-URL failures for a batch + security: + - ApiKeyAuth: [] + parameters: + - schema: + type: string + description: Batch id (UUID) + required: true + name: batchId + in: path + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/batch/scrape/{batchId}/download/zip: + get: + tags: + - Batch + summary: Download batch results as a ZIP + security: + - ApiKeyAuth: [] + parameters: + - schema: + type: string + description: Batch id (UUID) + required: true + name: batchId + in: path + responses: + '200': + description: ZIP archive of batch results + content: + application/zip: + schema: + type: string + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/batch/scrape/{batchId}/download/json: + get: + tags: + - Batch + summary: Download batch results as JSON + security: + - ApiKeyAuth: [] + parameters: + - schema: + type: string + description: Batch id (UUID) + required: true + name: batchId + in: path + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/batch/scrape/{batchId}/download/{jobId}: + get: + tags: + - Batch + summary: Download a single result from a batch + security: + - ApiKeyAuth: [] + parameters: + - schema: + type: string + description: Batch id (UUID) + required: true + name: batchId + in: path + - schema: + type: string + description: Job id within the batch + required: true + name: jobId + in: path + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/batch/scrape/{batchId}: + delete: + tags: + - Batch + summary: Cancel a batch + security: + - ApiKeyAuth: [] + parameters: + - schema: + type: string + description: Batch id (UUID) + required: true + name: batchId + in: path + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/batch/cleanup: + post: + tags: + - Batch + summary: Clean up batch records older than N days + security: + - ApiKeyAuth: [] + parameters: + - schema: type: integer - minimum: 0 - - name: limit + minimum: 1 + maximum: 365 + required: false + name: days + in: query + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/map: + post: + tags: + - Map + summary: Discover all URLs on a site + security: + - ApiKeyAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + url: + type: string + format: uri + maxUrls: + type: integer + minimum: 1 + maximum: 30000 + default: 5000 + includeSubdomains: + type: boolean + default: true + searchQuery: + type: string + skipSitemaps: + type: boolean + default: false + sitemapsOnly: + type: boolean + default: false + useUrlIndex: + type: boolean + default: true + timeoutMs: + type: integer + minimum: 1000 + maximum: 300000 + default: 30000 + includePatterns: + type: array + items: + type: string + excludePatterns: + type: array + items: + type: string + required: + - url + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/map/health: + get: + tags: + - Map + summary: URL-discovery subsystem health + security: + - ApiKeyAuth: [] + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/map/cache/stats: + get: + tags: + - Map + summary: URL-discovery cache statistics + security: + - ApiKeyAuth: [] + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/map/cache/clear: + post: + tags: + - Map + summary: Clear the URL-discovery cache for a site + security: + - ApiKeyAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + url: + type: string + format: uri + required: + - url + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/search: + post: + tags: + - Search + summary: Web search, optionally scraping each result + security: + - ApiKeyAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + query: + type: string + minLength: 1 + maxLength: 500 + limit: + type: integer + minimum: 0 + exclusiveMinimum: true + maximum: 50 + provider: + type: string + enum: + - duckduckgo + - searxng + - serper + lang: + type: string + maxLength: 10 + scrapeResults: + type: boolean + scrapeOptions: + type: object + additionalProperties: + nullable: true + required: + - query + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/extract: + post: + tags: + - Extract + summary: Async multi-URL LLM extraction + security: + - ApiKeyAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + urls: + type: array + items: + type: string + format: uri + maxItems: 1000 + url: + type: string + format: uri + prompt: + type: string + maxLength: 5000 + schema: + nullable: true + limit: + type: integer + minimum: 0 + exclusiveMinimum: true + maximum: 1000 + scrapeOptions: + type: object + additionalProperties: + nullable: true + responses: + '200': + description: Job accepted + content: + application/json: + schema: + $ref: '#/components/schemas/AsyncJobAccepted' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/extract/{id}: + get: + tags: + - Extract + summary: Poll an extract job + security: + - ApiKeyAuth: [] + parameters: + - schema: + type: string + description: Extract job id + required: true + name: id + in: path + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/extract-auto: + post: + tags: + - Extract + summary: Self-healing extraction — derives and caches CSS selectors, re-derives on breakage + security: + - ApiKeyAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + url: + type: string + format: uri + fields: + type: array + items: + type: object + properties: + name: + type: string + minLength: 1 + maxLength: 100 + description: + type: string + maxLength: 500 + type: + type: string + enum: + - text + - attribute + - html + - number + - list + - nested + - nested_list + attribute: + type: string + maxLength: 100 + required: + type: boolean + required: + - name + minItems: 1 + maxItems: 50 + cssSchema: + nullable: true + forceReheal: + type: boolean + scrapeOptions: + type: object + additionalProperties: + nullable: true + required: + - url + - fields + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/llmstxt: + post: + tags: + - Tools + summary: Generate an llms.txt for a site + security: + - ApiKeyAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + url: + type: string + format: uri + maxUrls: + type: integer + minimum: 0 + exclusiveMinimum: true + maximum: 500 + includeFullText: + type: boolean + required: + - url + responses: + '200': + description: Job accepted + content: + application/json: + schema: + $ref: '#/components/schemas/AsyncJobAccepted' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/llmstxt/{id}: + get: + tags: + - Tools + summary: Poll an llms.txt job + security: + - ApiKeyAuth: [] + parameters: + - schema: + type: string + description: llms.txt job id + required: true + name: id + in: path + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/parse: + post: + tags: + - Tools + summary: Parse a document (PDF/DOCX/…) to markdown + security: + - ApiKeyAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + content: + type: string + url: + type: string + format: uri + contentType: + type: string + maxLength: 100 + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/discover-apis: + post: + tags: + - Tools + summary: Surface a page’s underlying JSON/XHR endpoints + security: + - ApiKeyAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + url: + type: string + format: uri + timeout: + type: integer + minimum: 1000 + maximum: 120000 + includeNonJson: + type: boolean + required: + - url + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/reader: + get: + tags: + - Tools + summary: 'Scrape a URL to markdown; honours Accept: text/markdown' + security: + - ApiKeyAuth: [] + parameters: + - schema: + type: string + format: uri + description: URL to read + required: true + name: url in: query - description: Maximum number of results to return - schema: - type: integer - minimum: 1 - maximum: 100 responses: '200': - description: Crawl status retrieved successfully + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/usage: + get: + tags: + - Ops + summary: API key usage and quota + security: + - ApiKeyAuth: [] + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/proxies: + get: + tags: + - Ops + summary: Proxy pool health + security: + - ApiKeyAuth: [] + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/agent: + post: + tags: + - Agent + summary: Autonomous navigation toward a natural-language goal + security: + - ApiKeyAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + url: + type: string + format: uri + prompt: + type: string + minLength: 1 + maxLength: 5000 + schema: + nullable: true + maxSteps: + type: integer + minimum: 0 + exclusiveMinimum: true + maximum: 20 + onlyMainContent: + type: boolean + fitMarkdown: + type: boolean + required: + - url + - prompt + responses: + '200': + description: Job accepted + content: + application/json: + schema: + $ref: '#/components/schemas/AsyncJobAccepted' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/agent/{id}: + get: + tags: + - Agent + summary: Poll an agent run + security: + - ApiKeyAuth: [] + parameters: + - schema: + type: string + description: Agent task id + required: true + name: id + in: path + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/sessions: + post: + tags: + - Sessions + summary: Create a persistent browser session + security: + - ApiKeyAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + userAgent: + type: string + maxLength: 500 + viewport: + type: object + properties: + width: + type: integer + minimum: 200 + maximum: 4000 + height: + type: integer + minimum: 200 + maximum: 4000 + required: + - width + - height + initialUrl: + type: string + format: uri + proxy: + type: object + properties: + server: + type: string + maxLength: 300 + username: + type: string + maxLength: 200 + password: + type: string + maxLength: 200 + required: + - server + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + get: + tags: + - Sessions + summary: List active sessions + security: + - ApiKeyAuth: [] + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/sessions/{id}: + get: + tags: + - Sessions + summary: Get a session + security: + - ApiKeyAuth: [] + parameters: + - schema: + type: string + description: Session id + required: true + name: id + in: path + responses: + '200': + description: Success + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + delete: + tags: + - Sessions + summary: Close a session + security: + - ApiKeyAuth: [] + parameters: + - schema: + type: string + description: Session id + required: true + name: id + in: path + responses: + '200': + description: Success content: application/json: schema: - $ref: '#/components/schemas/CrawlStatus' - '404': - description: Crawl not found + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized + description: Missing or invalid API key content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - '500': - description: Server error + '404': + description: Not found content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - - delete: - summary: Cancel a crawl - description: Cancel a running crawl job + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/sessions/{id}/action: + post: tags: - - Crawling + - Sessions + summary: Run an action in a session (navigate, click, scrape, …) security: - ApiKeyAuth: [] parameters: - - name: jobId - in: path - required: true - description: Crawl job ID - schema: + - schema: type: string + description: Session id + required: true + name: id + in: path + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + type: + type: string + enum: + - navigate + - click + - type + - fill + - select + - scroll + - waitForSelector + - wait + - screenshot + - scrape + - evaluate + - back + - forward + - reload + - content + url: + type: string + format: uri + selector: + type: string + maxLength: 2000 + value: + type: string + text: + type: string + position: + type: number + timeout: + type: integer + minimum: 0 + maximum: 120000 + script: + type: string + maxLength: 20000 + fullPage: + type: boolean + formats: + type: array + items: + type: string + maxItems: 6 + onlyMainContent: + type: boolean + fitMarkdown: + type: boolean + waitUntil: + type: string + enum: + - load + - domcontentloaded + - networkidle + - commit + required: + - type responses: '200': - description: Crawl cancelled successfully + description: Success content: application/json: schema: - type: object - properties: - success: - type: boolean - '404': - description: Crawl not found + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized + description: Missing or invalid API key content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - '500': - description: Server error + '404': + description: Not found content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - - # Batch scraping endpoints - /api/batch/scrape: + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/sites: post: - summary: Start batch scraping - description: | - Initiate batch scraping for multiple URLs with concurrent processing. - Progress and results can be monitored via the status endpoint. tags: - - Batch Operations + - Sites + summary: Create a reusable site spec (becomes an MCP tool) security: - ApiKeyAuth: [] requestBody: @@ -1101,25 +2544,89 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/BatchScrapeRequest' - examples: - basic: - summary: Basic batch scrape - value: - urls: - - "https://example1.com" - - "https://example2.com" - - "https://example3.com" - concurrency: 3 - options: - extractorFormat: "markdown" + type: object + properties: + name: + type: string + minLength: 1 + maxLength: 48 + description: + type: string + maxLength: 500 + url: + type: string + minLength: 1 + maxLength: 2000 + params: + type: array + items: + type: object + properties: + name: + type: string + minLength: 1 + maxLength: 60 + description: + type: string + maxLength: 500 + required: + type: boolean + required: + - name + maxItems: 20 + fields: + type: array + items: + type: object + properties: + name: + type: string + minLength: 1 + maxLength: 100 + description: + type: string + maxLength: 500 + type: + type: string + enum: + - text + - attribute + - html + - number + - list + - nested + - nested_list + attribute: + type: string + maxLength: 100 + required: + type: boolean + required: + - name + minItems: 1 + maxItems: 50 + cssSchema: + nullable: true + sampleParams: + type: object + additionalProperties: + nullable: true + sessionId: + type: string + maxLength: 100 + verify: + type: boolean + required: + - name + - url + - fields responses: - '202': - description: Batch scraping initiated + '200': + description: Success content: application/json: schema: - $ref: '#/components/schemas/BatchScrapeResponse' + $ref: '#/components/schemas/SuccessResponse' '400': description: Invalid request content: @@ -1127,319 +2634,270 @@ paths: schema: $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized + description: Missing or invalid API key content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - '500': - description: Server error + '429': + description: Rate limit or quota exceeded content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - - /api/batch/scrape/{batchId}/status: get: - summary: Get batch status - description: Get the current status and results of a batch scraping operation tags: - - Batch Operations + - Sites + summary: List site specs security: - ApiKeyAuth: [] - parameters: - - name: batchId - in: path - required: true - description: Batch operation ID - schema: - type: string responses: '200': - description: Batch status retrieved successfully + description: Success content: application/json: schema: - $ref: '#/components/schemas/BatchStatus' - '404': - description: Batch not found + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized + description: Missing or invalid API key content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - '500': - description: Server error + '429': + description: Rate limit or quota exceeded content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - - /api/batch/scrape/{batchId}: - delete: - summary: Cancel batch operation - description: Cancel a running batch scraping operation + /api/sites/{id}: + get: tags: - - Batch Operations + - Sites + summary: Get a site spec security: - ApiKeyAuth: [] parameters: - - name: batchId - in: path - required: true - description: Batch operation ID - schema: + - schema: type: string + description: Site spec id + required: true + name: id + in: path responses: '200': - description: Batch cancelled successfully + description: Success content: application/json: schema: - type: object - properties: - success: - type: boolean - message: - type: string - '404': - description: Batch not found + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized + description: Missing or invalid API key content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - '500': - description: Server error + '404': + description: Not found content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - - /api/batch/scrape/{batchId}/download/zip: - get: - summary: Download results as ZIP - description: Download all batch results as a ZIP archive + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + delete: tags: - - Batch Operations + - Sites + summary: Delete a site spec security: - ApiKeyAuth: [] parameters: - - name: batchId - in: path - required: true - description: Batch operation ID - schema: + - schema: type: string - - name: format - in: query - description: Content format for files in ZIP - schema: - type: string - enum: [json, markdown, html, text] - default: markdown + description: Site spec id + required: true + name: id + in: path responses: '200': - description: ZIP file generated successfully + description: Success content: - application/zip: + application/json: schema: - type: string - format: binary - '404': - description: Batch not found or no results + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized + description: Missing or invalid API key content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - '500': - description: Server error + '404': + description: Not found content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - - /api/batch/scrape/{batchId}/download/json: - get: - summary: Download results as JSON - description: Download all batch results in a single JSON file + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/sites/{id}/run: + post: tags: - - Batch Operations + - Sites + summary: Run a site spec by id security: - ApiKeyAuth: [] parameters: - - name: batchId - in: path - required: true - description: Batch operation ID - schema: + - schema: type: string + description: Site spec id + required: true + name: id + in: path + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + params: + type: object + additionalProperties: + nullable: true responses: '200': - description: JSON file generated successfully + description: Success content: application/json: schema: - type: object - properties: - batchId: - type: string - generatedAt: - type: string - format: date-time - summary: - type: object - results: - type: array - items: - $ref: '#/components/schemas/ScraperResponse' - '404': - description: Batch not found or no results + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized + description: Missing or invalid API key content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - '500': - description: Server error + '404': + description: Not found content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - - /api/batch/scrape/{batchId}/result/{jobId}: - get: - summary: Download individual result - description: Download a single result from a batch operation + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/sites/by-name/{name}/run: + post: tags: - - Batch Operations + - Sites + summary: Run a site spec by name security: - ApiKeyAuth: [] parameters: - - name: batchId - in: path - required: true - description: Batch operation ID - schema: + - schema: type: string - - name: jobId - in: path + description: Site spec name (slug) required: true - description: Individual job ID within the batch - schema: - type: string - - name: format - in: query - description: Download format - schema: - type: string - enum: [json, markdown, html, text] - default: json + name: name + in: path + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + params: + type: object + additionalProperties: + nullable: true responses: '200': - description: Result retrieved successfully + description: Success content: application/json: schema: - $ref: '#/components/schemas/ScraperResponse' - text/markdown: - schema: - type: string - text/html: - schema: - type: string - text/plain: - schema: - type: string - '404': - description: Result not found + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized + description: Missing or invalid API key content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - '500': - description: Server error + '404': + description: Not found content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - - # URL Discovery endpoints - /api/map: + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/sites/{id}/verify: post: - summary: Discover URLs from a website - description: | - High-performance URL discovery using multiple parallel methods: - - XML sitemap parsing (including sitemap indexes) - - Search engine discovery (site: queries) - - Browser-based crawling - - Common path discovery (/api, /docs, etc.) - - Robots.txt sitemap references - - Can discover 5000+ URLs in seconds with intelligent caching. tags: - - URL Discovery + - Sites + summary: Verify a spec still extracts correctly (self-heals on drift) security: - ApiKeyAuth: [] - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/MapRequest' - examples: - basic: - summary: Basic discovery - value: - url: "https://example.com" - maxUrls: 5000 - filtered: - summary: Filtered discovery - value: - url: "https://docs.example.com" - maxUrls: 1000 - includePatterns: ["/api/", "/guides/"] - excludePatterns: ["/archive/", "/old/"] - search: - summary: Search-based discovery - value: - url: "https://docs.example.com" - searchQuery: "authentication api" - maxUrls: 100 + parameters: + - schema: + type: string + description: Site spec id + required: true + name: id + in: path responses: '200': - description: URLs discovered successfully + description: Success content: application/json: schema: - $ref: '#/components/schemas/MapResponse' + $ref: '#/components/schemas/SuccessResponse' '400': description: Invalid request content: @@ -1447,84 +2905,65 @@ paths: schema: $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized + description: Missing or invalid API key content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - '500': - description: Server error + '404': + description: Not found content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - - /api/map/cache/stats: + '429': + description: Rate limit or quota exceeded + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /health: get: - summary: Get discovery cache statistics - description: Get statistics about the URL discovery cache tags: - - URL Discovery - security: - - ApiKeyAuth: [] + - Ops + summary: Liveness probe responses: '200': - description: Cache statistics retrieved successfully + description: Success content: application/json: schema: - type: object - properties: - success: - type: boolean - data: - type: object + $ref: '#/components/schemas/SuccessResponse' + '400': + description: Invalid request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized + description: Missing or invalid API key content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - '500': - description: Server error + '429': + description: Rate limit or quota exceeded content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - - /api/map/cache/clear: - post: - summary: Clear discovery cache - description: Clear the discovery cache for a specific URL + /health/ready: + get: tags: - - URL Discovery - security: - - ApiKeyAuth: [] - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - url - properties: - url: - type: string - format: uri - description: URL to clear from cache + - Ops + summary: Readiness probe (dependencies reachable) responses: '200': - description: Cache cleared successfully + description: Success content: application/json: schema: - type: object - properties: - success: - type: boolean - message: - type: string + $ref: '#/components/schemas/SuccessResponse' '400': description: Invalid request content: @@ -1532,70 +2971,44 @@ paths: schema: $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized + description: Missing or invalid API key content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - '500': - description: Server error + '429': + description: Rate limit or quota exceeded content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - - /api/map/health: + /metrics: get: - summary: Map service health check - description: Check the health of the URL discovery service tags: - - URL Discovery - security: - - ApiKeyAuth: [] + - Ops + summary: Prometheus metrics responses: '200': - description: Service is healthy + description: Prometheus exposition format content: - application/json: + text/plain: schema: - type: object - properties: - success: - type: boolean - status: - type: string - services: - type: object - cacheStats: - type: object - timestamp: - type: string - format: date-time - '503': - description: Service unhealthy + type: string + '400': + description: Invalid request content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' '401': - description: Unauthorized + description: Missing or invalid API key + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '429': + description: Rate limit or quota exceeded content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - -tags: - - name: System - description: System health and diagnostics - - name: Scraping - description: Single-page web scraping operations - - name: Crawling - description: Multi-page crawling operations - - name: Batch Operations - description: Batch scraping for multiple URLs - - name: URL Discovery - description: High-performance URL discovery and mapping - -externalDocs: - description: DeepScraper Documentation - url: https://docs.extractr.ai \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 7886ddd..06bfaf4 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,5 +12,9 @@ "resolveJsonModule": true }, "include": ["src/**/*"], - "exclude": ["node_modules", "**/*.spec.ts"] + "exclude": [ + "node_modules", + "**/*.spec.ts", + "src/api/openapi/**" + ] } \ No newline at end of file