Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 93 additions & 1 deletion .speakeasy/in.openapi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22836,6 +22836,7 @@ components:
enum:
- 'artificial-analysis'
- 'design-arena'
- 'openrouter'
- null
example: 'artificial-analysis'
type:
Expand Down Expand Up @@ -22866,6 +22867,76 @@ components:
- 'model_count'
- 'task_type'
type: 'object'
UnifiedBenchmarksORItem:
example:
accuracy: 0.72
accuracy_stddev: 0.03
avg_cost_per_task: 0.002
benchmark_type: 'gpqa_diamond'
display_name: 'GPT-4o'
last_run_timestamp: '2026-06-03T12:00:00Z'
model_permaslug: 'openai/gpt-4o'
source: 'openrouter'
total_tasks: 300
properties:
accuracy:
description: 'Aggregate accuracy score from 0 to 1. Higher is better.'
example: 0.72
format: 'double'
type: 'number'
accuracy_stddev:
description: 'Standard deviation of run accuracy, or null for a single run.'
example: 0.03
format: 'double'
type:
- 'number'
- 'null'
avg_cost_per_task:
description: 'Average cost per task in USD, or null if unavailable.'
example: 0.002
format: 'double'
type:
- 'number'
- 'null'
benchmark_type:
description: 'OpenRouter benchmark evaluation type.'
enum:
- 'gpqa_diamond'
- 'tau_bench_verified_airline'
example: 'gpqa_diamond'
type: 'string'
display_name:
description: 'Human-readable model name.'
example: 'GPT-4o'
type: 'string'
last_run_timestamp:
description: 'Timestamp of the most recent public benchmark run.'
example: '2026-06-03T12:00:00Z'
type: 'string'
model_permaslug:
description: 'Stable OpenRouter model identifier.'
example: 'openai/gpt-4o'
type: 'string'
source:
description: 'Benchmark source discriminator.'
enum:
- 'openrouter'
type: 'string'
total_tasks:
description: 'Total benchmark tasks across runs.'
example: 300
type: 'integer'
required:
- 'source'
- 'model_permaslug'
- 'display_name'
- 'benchmark_type'
- 'accuracy'
- 'accuracy_stddev'
- 'avg_cost_per_task'
- 'total_tasks'
- 'last_run_timestamp'
type: 'object'
UnifiedBenchmarksResponse:
example:
data:
Expand All @@ -22878,6 +22949,15 @@ components:
completion: '0.00001'
prompt: '0.0000025'
source: 'artificial-analysis'
- accuracy: 0.72
accuracy_stddev: 0.03
avg_cost_per_task: 0.002
benchmark_type: 'gpqa_diamond'
display_name: 'GPT-4o'
last_run_timestamp: '2026-06-03T12:00:00Z'
model_permaslug: 'openai/gpt-4o'
source: 'openrouter'
total_tasks: 300
meta:
as_of: '2026-06-03T12:00:00Z'
citation: null
Expand All @@ -22893,10 +22973,12 @@ components:
mapping:
artificial-analysis: '#/components/schemas/UnifiedBenchmarksAAItem'
design-arena: '#/components/schemas/UnifiedBenchmarksDAItem'
openrouter: '#/components/schemas/UnifiedBenchmarksORItem'
propertyName: 'source'
oneOf:
- $ref: '#/components/schemas/UnifiedBenchmarksAAItem'
- $ref: '#/components/schemas/UnifiedBenchmarksDAItem'
- $ref: '#/components/schemas/UnifiedBenchmarksORItem'
type: 'array'
meta:
$ref: '#/components/schemas/UnifiedBenchmarksMeta'
Expand Down Expand Up @@ -25620,7 +25702,7 @@ paths:
x-speakeasy-name-override: 'createAuthCode'
/benchmarks:
get:
description: 'Unified benchmark endpoint that aggregates scores from multiple benchmark sources (Artificial Analysis, Design Arena). Filter by source to reproduce the exact shapes from the legacy per-source endpoints, or use task_type to find models suited for specific workloads. Authenticate with any valid OpenRouter API key. Rate-limited to 30 requests/minute per key and 500 requests/day per account.'
description: 'Unified benchmark endpoint that aggregates scores from multiple benchmark sources (Artificial Analysis, Design Arena, and OpenRouter''s own tau-bench and GPQA evals). Filter by source to reproduce the exact shapes from the legacy per-source endpoints, or use task_type to find models suited for specific workloads. Authenticate with any valid OpenRouter API key. Rate-limited to 30 requests/minute per key and 500 requests/day per account.'
operationId: 'getBenchmarks'
parameters:
- description: 'Benchmark source to query. Determines the shape of the returned items. When omitted, returns results from all sources.'
Expand All @@ -25632,6 +25714,7 @@ paths:
enum:
- 'artificial-analysis'
- 'design-arena'
- 'openrouter'
example: 'artificial-analysis'
type: 'string'
- description: 'Filter results by task type. For Artificial Analysis, maps to the corresponding index. For Design Arena, maps to the matching category.'
Expand Down Expand Up @@ -25690,6 +25773,15 @@ paths:
completion: '0.00001'
prompt: '0.0000025'
source: 'artificial-analysis'
- accuracy: 0.72
accuracy_stddev: 0.03
avg_cost_per_task: 0.002
benchmark_type: 'gpqa_diamond'
display_name: 'GPT-4o'
last_run_timestamp: '2026-06-03T12:00:00Z'
model_permaslug: 'openai/gpt-4o'
source: 'openrouter'
total_tasks: 300
meta:
as_of: '2026-06-03T12:00:00Z'
citation: null
Expand Down
Loading