diff --git a/.speakeasy/in.openapi.yaml b/.speakeasy/in.openapi.yaml index 74d2a197..3689f6cf 100644 --- a/.speakeasy/in.openapi.yaml +++ b/.speakeasy/in.openapi.yaml @@ -22836,6 +22836,7 @@ components: enum: - 'artificial-analysis' - 'design-arena' + - 'openrouter' - null example: 'artificial-analysis' type: @@ -22866,6 +22867,76 @@ components: - 'model_count' - 'task_type' type: 'object' + UnifiedBenchmarksORItem: + example: + accuracy: 0.72 + accuracy_stddev: 0.03 + avg_cost_per_task: 0.002 + benchmark_type: 'gpqa_diamond' + display_name: 'GPT-4o' + last_run_timestamp: '2026-06-03T12:00:00Z' + model_permaslug: 'openai/gpt-4o' + source: 'openrouter' + total_tasks: 300 + properties: + accuracy: + description: 'Aggregate accuracy score from 0 to 1. Higher is better.' + example: 0.72 + format: 'double' + type: 'number' + accuracy_stddev: + description: 'Standard deviation of run accuracy, or null for a single run.' + example: 0.03 + format: 'double' + type: + - 'number' + - 'null' + avg_cost_per_task: + description: 'Average cost per task in USD, or null if unavailable.' + example: 0.002 + format: 'double' + type: + - 'number' + - 'null' + benchmark_type: + description: 'OpenRouter benchmark evaluation type.' + enum: + - 'gpqa_diamond' + - 'tau_bench_verified_airline' + example: 'gpqa_diamond' + type: 'string' + display_name: + description: 'Human-readable model name.' + example: 'GPT-4o' + type: 'string' + last_run_timestamp: + description: 'Timestamp of the most recent public benchmark run.' + example: '2026-06-03T12:00:00Z' + type: 'string' + model_permaslug: + description: 'Stable OpenRouter model identifier.' + example: 'openai/gpt-4o' + type: 'string' + source: + description: 'Benchmark source discriminator.' + enum: + - 'openrouter' + type: 'string' + total_tasks: + description: 'Total benchmark tasks across runs.' + example: 300 + type: 'integer' + required: + - 'source' + - 'model_permaslug' + - 'display_name' + - 'benchmark_type' + - 'accuracy' + - 'accuracy_stddev' + - 'avg_cost_per_task' + - 'total_tasks' + - 'last_run_timestamp' + type: 'object' UnifiedBenchmarksResponse: example: data: @@ -22878,6 +22949,15 @@ components: completion: '0.00001' prompt: '0.0000025' source: 'artificial-analysis' + - accuracy: 0.72 + accuracy_stddev: 0.03 + avg_cost_per_task: 0.002 + benchmark_type: 'gpqa_diamond' + display_name: 'GPT-4o' + last_run_timestamp: '2026-06-03T12:00:00Z' + model_permaslug: 'openai/gpt-4o' + source: 'openrouter' + total_tasks: 300 meta: as_of: '2026-06-03T12:00:00Z' citation: null @@ -22893,10 +22973,12 @@ components: mapping: artificial-analysis: '#/components/schemas/UnifiedBenchmarksAAItem' design-arena: '#/components/schemas/UnifiedBenchmarksDAItem' + openrouter: '#/components/schemas/UnifiedBenchmarksORItem' propertyName: 'source' oneOf: - $ref: '#/components/schemas/UnifiedBenchmarksAAItem' - $ref: '#/components/schemas/UnifiedBenchmarksDAItem' + - $ref: '#/components/schemas/UnifiedBenchmarksORItem' type: 'array' meta: $ref: '#/components/schemas/UnifiedBenchmarksMeta' @@ -25620,7 +25702,7 @@ paths: x-speakeasy-name-override: 'createAuthCode' /benchmarks: get: - description: 'Unified benchmark endpoint that aggregates scores from multiple benchmark sources (Artificial Analysis, Design Arena). Filter by source to reproduce the exact shapes from the legacy per-source endpoints, or use task_type to find models suited for specific workloads. Authenticate with any valid OpenRouter API key. Rate-limited to 30 requests/minute per key and 500 requests/day per account.' + description: 'Unified benchmark endpoint that aggregates scores from multiple benchmark sources (Artificial Analysis, Design Arena, and OpenRouter''s own tau-bench and GPQA evals). Filter by source to reproduce the exact shapes from the legacy per-source endpoints, or use task_type to find models suited for specific workloads. Authenticate with any valid OpenRouter API key. Rate-limited to 30 requests/minute per key and 500 requests/day per account.' operationId: 'getBenchmarks' parameters: - description: 'Benchmark source to query. Determines the shape of the returned items. When omitted, returns results from all sources.' @@ -25632,6 +25714,7 @@ paths: enum: - 'artificial-analysis' - 'design-arena' + - 'openrouter' example: 'artificial-analysis' type: 'string' - description: 'Filter results by task type. For Artificial Analysis, maps to the corresponding index. For Design Arena, maps to the matching category.' @@ -25690,6 +25773,15 @@ paths: completion: '0.00001' prompt: '0.0000025' source: 'artificial-analysis' + - accuracy: 0.72 + accuracy_stddev: 0.03 + avg_cost_per_task: 0.002 + benchmark_type: 'gpqa_diamond' + display_name: 'GPT-4o' + last_run_timestamp: '2026-06-03T12:00:00Z' + model_permaslug: 'openai/gpt-4o' + source: 'openrouter' + total_tasks: 300 meta: as_of: '2026-06-03T12:00:00Z' citation: null