Updated modalities structure

Author: fwang

README.md

@@ -51,25 +51,27 @@ Create a new TOML file in the provider's `models/` directory where the filename
 
 ```toml
 name = "Model Display Name"
-attachment = true     # or false - supports file attachments
-reasoning = false     # or true - supports reasoning / chain-of-thought
-tool_call = true      # or false - supports tool calling
-temperature = true    # or false - supports temperature control
-knowledge = "2024-04" # Knowledge-cutoff date
-input_modalities = ["text", "image"] # Supported input modalities
-output_modalities = ["text"]         # Supported output modalities
-release_date = "2025-02-19"          # First public release date
-last_updated = "2025-02-19"          # Most recent update date
+attachment = true           # or false - supports file attachments
+reasoning = false           # or true - supports reasoning / chain-of-thought
+tool_call = true            # or false - supports tool calling
+temperature = true          # or false - supports temperature control
+knowledge = "2024-04"       # Knowledge-cutoff date
+release_date = "2025-02-19" # First public release date
+last_updated = "2025-02-19" # Most recent update date
 
 [cost]
-input = 3.00          # Cost per million input tokens (USD)
-output = 15.00        # Cost per million output tokens (USD)
-cache_read = 0.30     # Cost per million cached read tokens (USD)
-cache_write = 3.75    # Cost per million cached write tokens (USD)
+input = 3.00                # Cost per million input tokens (USD)
+output = 15.00              # Cost per million output tokens (USD)
+cache_read = 0.30           # Cost per million cached read tokens (USD)
+cache_write = 3.75          # Cost per million cached write tokens (USD)
 
 [limit]
-context = 200_000     # Maximum context window (tokens)
-output = 8_192        # Maximum output tokens
+context = 200_000           # Maximum context window (tokens)
+output = 8_192              # Maximum output tokens
+
+[modalities]
+input = ["text", "image"]   # Supported input modalities
+output = ["text"]           # Supported output modalities
 ```
 
 #### 3. Submit a Pull Request
@@ -104,8 +106,6 @@ Models must conform to the following schema, as defined in `app/schemas.ts`.
 - `tool_call`: Boolean - Supports tool calling
 - `temperature`: Boolean — Supports temperature control
 - `knowledge` _(optional)_: String — Knowledge-cutoff date in `YYYY-MM` or `YYYY-MM-DD` format
-- `input_modalities`: Array of strings — Supported input modalities (e.g., ["text", "image"])
-- `output_modalities`: Array of strings — Supported output modalities (e.g., ["text"])
 - `release_date`: String — First public release date in `YYYY-MM` or `YYYY-MM-DD`
 - `last_updated`: String — Most recent update date in `YYYY-MM` or `YYYY-MM-DD`
 - `cost.input` _(optional)_: Number — Cost per million input tokens (USD)
@@ -114,6 +114,8 @@ Models must conform to the following schema, as defined in `app/schemas.ts`.
 - `cost.cache_write` _(optional)_: Number — Cost per million cached write tokens (USD)
 - `limit.context`: Number — Maximum context window (tokens)
 - `limit.output`: Number — Maximum output tokens
+- `modalities.input`: Array of strings — Supported input modalities (e.g., ["text", "image", "audio", "video", "pdf"])
+- `modalities.putput`: Array of strings — Supported output modalities (e.g., ["text"])
 
 ### Examples
 

packages/core/src/schema.ts

@@ -20,12 +20,10 @@ export const Model = z
     last_updated: z.string().regex(/^\d{4}-\d{2}(-\d{2})?$/, {
       message: "Must be in YYYY-MM or YYYY-MM-DD format",
     }),
-    input_modalities: z.array(
-      z.enum(["text", "audio", "image", "video", "pdf"])
-    ),
-    output_modalities: z.array(
-      z.enum(["text", "audio", "image", "video", "pdf"])
-    ),
+    modalities: z.object({
+      input: z.array(z.enum(["text", "audio", "image", "video", "pdf"])),
+      output: z.array(z.enum(["text", "audio", "image", "video", "pdf"])),
+    }),
     open_weights: z.boolean(),
     cost: z
       .object({

packages/web/src/render.tsx

@@ -309,14 +309,14 @@ export const Rendered = renderToString(
                   <td>{model.reasoning ? "Yes" : "No"}</td>
                   <td>
                     <div class="modalities">
-                      {model.input_modalities?.map((modality) =>
+                      {model.modalities.input.map((modality) =>
                         getModalityIcon(modality)
                       )}
                     </div>
                   </td>
                   <td>
                     <div class="modalities">
-                      {model.output_modalities?.map((modality) =>
+                      {model.modalities.output.map((modality) =>
                         getModalityIcon(modality)
                       )}
                     </div>

providers/amazon-bedrock/models/ai21.jamba-1-5-large-v1:0.toml

@@ -6,8 +6,6 @@ reasoning = false
 temperature = true
 knowledge = "2024-08"
 tool_call = true
-input_modalities = ["text"]
-output_modalities = ["text"]
 open_weights = true
 
 [cost]
@@ -16,4 +14,8 @@ output = 8.00
 
 [limit]
 context = 256_000
-output = 4_096
+output = 4_096
+
+[modalities]
+input = ["text"]
+output = ["text"]

providers/amazon-bedrock/models/ai21.jamba-1-5-mini-v1:0.toml

@@ -6,8 +6,6 @@ reasoning = false
 temperature = true
 knowledge = "2024-08"
 tool_call = true
-input_modalities = ["text"]
-output_modalities = ["text"]
 open_weights = true
 
 [cost]
@@ -16,4 +14,8 @@ output = 0.40
 
 [limit]
 context = 256_000
-output = 4_096
+output = 4_096
+
+[modalities]
+input = ["text"]
+output = ["text"]

providers/amazon-bedrock/models/amazon.nova-lite-v1:0.toml

@@ -6,8 +6,6 @@ reasoning = false
 temperature = true
 knowledge = "2024-10"
 tool_call = true
-input_modalities = ["text", "image", "video"]
-output_modalities = ["text"]
 open_weights = false
 
 [cost]
@@ -17,4 +15,8 @@ cache_read = 0.015
 
 [limit]
 context = 300_000
-output = 8_192
+output = 8_192
+
+[modalities]
+input = ["text", "image", "video"]
+output = ["text"]

providers/amazon-bedrock/models/amazon.nova-micro-v1:0.toml

@@ -6,8 +6,6 @@ reasoning = false
 temperature = true
 knowledge = "2024-10"
 tool_call = true
-input_modalities = ["text"]
-output_modalities = ["text"]
 open_weights = false
 
 [cost]
@@ -17,4 +15,8 @@ cache_read = 0.00875
 
 [limit]
 context = 128_000
-output = 8_192
+output = 8_192
+
+[modalities]
+input = ["text"]
+output = ["text"]

providers/amazon-bedrock/models/amazon.nova-premier-v1:0.toml

@@ -6,8 +6,6 @@ reasoning = true
 temperature = true
 knowledge = "2024-10"
 tool_call = true
-input_modalities = ["text", "image", "video"]
-output_modalities = ["text"]
 open_weights = false
 
 [cost]
@@ -16,4 +14,8 @@ output = 12.50
 
 [limit]
 context = 1_000_000
-output = 16_384
+output = 16_384
+
+[modalities]
+input = ["text", "image", "video"]
+output = ["text"]

providers/amazon-bedrock/models/amazon.nova-pro-v1:0.toml

@@ -6,8 +6,6 @@ reasoning = false
 temperature = true
 knowledge = "2024-10"
 tool_call = true
-input_modalities = ["text", "image", "video"]
-output_modalities = ["text"]
 open_weights = false
 
 [cost]
@@ -17,4 +15,8 @@ cache_read = 0.20
 
 [limit]
 context = 300_000
-output = 8_192
+output = 8_192
+
+[modalities]
+input = ["text", "image", "video"]
+output = ["text"]

providers/amazon-bedrock/models/anthropic.claude-3-5-haiku-20241022-v1:0.toml

@@ -6,8 +6,6 @@ reasoning = false
 temperature = true
 knowledge = "2024-07"
 tool_call = true
-input_modalities = ["text"]
-output_modalities = ["text"]
 open_weights = false
 
 [cost]
@@ -18,4 +16,8 @@ cache_write = 1.00
 
 [limit]
 context = 200_000
-output = 8_192
+output = 8_192
+
+[modalities]
+input = ["text"]
+output = ["text"]