Add tool support and tests

README.md

@@ -38,17 +38,17 @@ llm models list
 You should see a list that looks something like this:
 ```
 OpenRouter: openrouter/openai/gpt-3.5-turbo
-OpenRouter: openrouter/anthropic/claude-2
+OpenRouter: openrouter/anthropic/claude-sonnet-4
 OpenRouter: openrouter/meta-llama/llama-2-70b-chat
 ...
 ```
 To run a prompt against a model, pass its full model ID to the `-m` option, like this:
 ```bash
-llm -m openrouter/anthropic/claude-2 "Five spooky names for a pet tarantula"
+llm -m openrouter/anthropic/claude-sonnet-4 "Five spooky names for a pet tarantula"
 ```
 You can set a shorter alias for a model using the `llm aliases` command like so:
 ```bash
-llm aliases set claude openrouter/anthropic/claude-2
+llm aliases set claude openrouter/anthropic/claude-sonnet-4
 ```
 Now you can prompt Claude using:
 ```bash
@@ -266,3 +266,5 @@ To update recordings and snapshots, run:
 PYTEST_OPENROUTER_KEY="$(llm keys get openrouter)" \
   pytest --record-mode=rewrite --inline-snapshot=fix
 ```
+
+If tests against additional models are added, update `tests/models_persister.py` to preserve those model ids in the recordings.

llm_openrouter.py

@@ -16,20 +16,23 @@ def get_openrouter_models():
         path=llm.user_dir() / "openrouter_models.json",
         cache_timeout=3600,
     )["data"]
-    schema_supporting_ids = {
-        model["id"]
-        for model in fetch_cached_json(
-            url="https://openrouter.ai/api/v1/models?supported_parameters=structured_outputs",
-            path=llm.user_dir() / "openrouter_models_structured_outputs.json",
-            cache_timeout=3600,
-        )["data"]
-    }
-    # Annotate models with their schema support
-    for model in models:
-        model["supports_schema"] = model["id"] in schema_supporting_ids
     return models
 
 
+def get_supports_images(model_definition):
+    try:
+        return "image" in model_definition["architecture"]["input_modalities"]
+    except KeyError:
+        return False
+
+
+def has_parameter(model_definition, parameter):
+    try:
+        return parameter in model_definition["supported_parameters"]
+    except KeyError:
+        return False
+
+
 class ReasoningEffortEnum(str, Enum):
     low = "low"
     medium = "medium"
@@ -125,7 +128,8 @@ def register_models(register):
             model_id="openrouter/{}".format(model_definition["id"]),
             model_name=model_definition["id"],
             vision=supports_images,
-            supports_schema=model_definition["supports_schema"],
+            supports_schema=has_parameter(model_definition, "structured_outputs"),
+            supports_tools=has_parameter(model_definition, "tools"),
             api_base="https://openrouter.ai/api/v1",
             headers={"HTTP-Referer": "https://llm.datasette.io/", "X-Title": "LLM"},
         )
@@ -176,17 +180,6 @@ def fetch_cached_json(url, path, cache_timeout):
             )
 
 
-def get_supports_images(model_definition):
-    try:
-        # e.g. `text->text` or `text+image->text`
-        modality = model_definition["architecture"]["modality"]
-
-        input_modalities = modality.split("->")[0].split("+")
-        return "image" in input_modalities
-    except Exception:
-        return False
-
-
 @llm.hookimpl
 def register_commands(cli):
     @cli.group()
@@ -225,7 +218,8 @@ def models(free, json_):
                             + ": "
                             + (value if isinstance(value, str) else json.dumps(value))
                         )
-                bits.append(f"  supports_schema: {model['supports_schema']}")
+                bits.append(f"  supports_schema: {has_parameter(model, 'structured_outputs')}")
+                bits.append(f"  supports_tools: {has_parameter(model, 'tools')}")
                 pricing = format_pricing(model["pricing"])
                 if pricing:
                     bits.append("  pricing: " + pricing)

pyproject.toml

@@ -9,7 +9,7 @@ classifiers = [
     "License :: OSI Approved :: Apache Software License"
 ]
 dependencies = [
-    "llm>=0.23",
+    "llm>=0.27.1",
     "httpx",
     "openai>=1.57.0",
 ]

tests/conftest.py

@@ -1,12 +1,22 @@
 import pytest
 import os
+import vcr
+from models_persister import TruncatedModelsFilesystemPersister
 
 OPENROUTER_KEY = os.getenv("PYTEST_OPENROUTER_KEY", "sk-...")
 
 
+def pytest_recording_configure(config, vcr):
+    vcr.register_persister(TruncatedModelsFilesystemPersister)
+
+
 @pytest.fixture(scope="module")
 def vcr_config():
-    return {"filter_headers": ["authorization"]}
+    return {
+        "filter_headers": ["authorization"],
+        "decode_compressed_response": True,
+        "persister": TruncatedModelsFilesystemPersister,
+    }
 
 
 @pytest.fixture

tests/models_persister.py

@@ -0,0 +1,31 @@
+import json
+from vcr.persisters.filesystem import FilesystemPersister
+
+# Only save response data for a few models to keep the recording size down.
+TEST_MODEL_IDS = {
+    "openai/gpt-3.5-turbo",
+    "openai/gpt-4.1-mini",
+    "openai/gpt-4o",
+    "anthropic/claude-3.5-sonnet",
+    "anthropic/claude-sonnet-4",
+}
+
+
+class TruncatedModelsFilesystemPersister(FilesystemPersister):
+    @staticmethod
+    def save_cassette(cassette_path, cassette_dict, serializer):
+        for request, response in zip(
+            cassette_dict["requests"], cassette_dict["responses"]
+        ):
+            body = response.get("body", {})
+            if request.url.endswith("/api/v1/models") and "string" in body:
+                data = json.loads(body["string"])
+                if "data" in data:
+                    data["data"] = [
+                        model
+                        for model in data["data"]
+                        if model.get("id") in TEST_MODEL_IDS
+                    ]
+                    body["string"] = json.dumps(data)
+
+        FilesystemPersister.save_cassette(cassette_path, cassette_dict, serializer)

tests/test_llm_openrouter.py

@@ -19,25 +19,26 @@
 def test_prompt():
     model = llm.get_model("openrouter/openai/gpt-4o")
     response = model.prompt("Two names for a pet pelican, be brief")
-    assert str(response) == snapshot("Beaker or Zephyr.")
+    assert str(response) == snapshot("Pebbles and Skipper.")
     response_dict = dict(response.response_json)
     response_dict.pop("id")  # differs between requests
     assert response_dict == snapshot(
         {
-            "content": "Beaker or Zephyr.",
+            "content": "Pebbles and Skipper.",
             "role": "assistant",
             "finish_reason": "stop",
             "usage": {
-                "completion_tokens": 7,
+                "completion_tokens": 6,
                 "prompt_tokens": 17,
-                "total_tokens": 24,
+                "total_tokens": 23,
                 "completion_tokens_details": {"reasoning_tokens": 0},
                 "prompt_tokens_details": {"cached_tokens": 0},
-                "cost": 0.0001125,
+                "cost": 0.0001025,
+                "is_byok": False,
             },
             "object": "chat.completion.chunk",
             "model": "openai/gpt-4o",
-            "created": 1745444094,
+            "created": 1754441342,
         }
     )
 
@@ -49,7 +50,7 @@ def test_llm_models():
     assert result.exit_code == 0, result.output
     fragments = (
         "OpenRouter: openrouter/openai/gpt-3.5-turbo",
-        "OpenRouter: openrouter/anthropic/claude-2",
+        "OpenRouter: openrouter/anthropic/claude-sonnet-4",
     )
     for fragment in fragments:
         assert fragment in result.output
@@ -62,24 +63,83 @@ def test_image_prompt():
         "Describe image in three words",
         attachments=[llm.Attachment(content=TINY_PNG)],
     )
-    assert str(response) == snapshot("Bright Red Green")
+    assert str(response) == snapshot("Red green geometric shapes")
     response_dict = response.response_json
     response_dict.pop("id")  # differs between requests
     assert response_dict == snapshot(
         {
-            "content": "Bright Red Green",
+            "content": "Red green geometric shapes",
             "role": "assistant",
             "finish_reason": "stop",
             "usage": {
                 "completion_tokens": 7,
-                "prompt_tokens": 1682,
-                "total_tokens": 1689,
+                "prompt_tokens": 82,
+                "total_tokens": 89,
                 "completion_tokens_details": {"reasoning_tokens": 0},
                 "prompt_tokens_details": {"cached_tokens": 0},
-                "cost": 0.005151,
+                "cost": 0.000351,
+                "is_byok": False,
             },
             "object": "chat.completion.chunk",
             "model": "anthropic/claude-3.5-sonnet",
-            "created": 1745444099,
+            "created": 1754441344,
+        }
+    )
+
+
+@pytest.mark.vcr
+def test_tool_calls():
+    model = llm.get_model("openrouter/openai/gpt-4.1-mini")
+
+    def llm_version() -> str:
+        "Return the installed version of llm"
+        return "0.0+test"
+
+    chain = model.chain(
+        "What is the current llm version?",
+        tools=[llm_version],
+    )
+
+    responses = list(chain.responses())
+
+    responses[0].response_json.pop("id")  # differs between requests
+    responses[0].response_json.pop("created")  # differs between requests
+    assert responses[0].response_json == snapshot(
+        {
+            "content": "",
+            "role": "assistant",
+            "finish_reason": "tool_calls",
+            "usage": {
+                "completion_tokens": 11,
+                "prompt_tokens": 48,
+                "total_tokens": 59,
+                "completion_tokens_details": {"reasoning_tokens": 0},
+                "prompt_tokens_details": {"cached_tokens": 0},
+                "cost": 3.68e-05,
+                "is_byok": False,
+            },
+            "object": "chat.completion.chunk",
+            "model": "openai/gpt-4.1-mini",
+        }
+    )
+
+    responses[1].response_json.pop("id")  # differs between requests
+    responses[1].response_json.pop("created")  # differs between requests
+    assert responses[1].response_json == snapshot(
+        {
+            "content": "The current LLM version is 0.0+test.",
+            "role": "assistant",
+            "finish_reason": "stop",
+            "usage": {
+                "completion_tokens": 14,
+                "prompt_tokens": 73,
+                "total_tokens": 87,
+                "completion_tokens_details": {"reasoning_tokens": 0},
+                "prompt_tokens_details": {"cached_tokens": 0},
+                "cost": 5.16e-05,
+                "is_byok": False,
+            },
+            "object": "chat.completion.chunk",
+            "model": "openai/gpt-4.1-mini",
         }
     )