ci: schedule model metadata upkeep and check the snapshot against upstream

.github/workflows/model-metadata-upkeep.yml

@@ -0,0 +1,163 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Model metadata upkeep
+
+on:
+  schedule:
+    # Weekly, offset from the hour to reduce peak-time scheduling delays. The
+    # snapshot is a build input a human reviews, so a nightly cadence would
+    # only stack five near-identical pull requests against one week of upstream
+    # movement, on runners the whole foundation shares.
+    - cron: '41 6 * * 1'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: model-metadata-upkeep
+  cancel-in-progress: false
+
+jobs:
+  refresh:
+    # A fork inherits the schedule but owns neither the branch this pushes nor
+    # the pull request it opens.
+    if: github.repository == 'apache/maka'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: '24'
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci --ignore-scripts
+
+      - name: Report snapshot drift against models.dev
+        # Exit 2 is "upstream moved", which is the expected outcome and the
+        # reason this step exists. Any other non-zero status is the command
+        # itself failing, and the job stops on it.
+        run: |
+          npm run --silent check:model-metadata-drift > "$RUNNER_TEMP/drift.txt" || {
+            status=$?
+            cat "$RUNNER_TEMP/drift.txt"
+            [ "$status" -eq 2 ] || exit "$status"
+          }
+
+      - name: Publish the drift report
+        run: |
+          {
+            echo '### models.dev drift'
+            echo ''
+            echo '```text'
+            cat "$RUNNER_TEMP/drift.txt"
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      # --accept-upstream-removals, because the review seat this job is built
+      # around is the draft pull request below. A person still inspects every
+      # removal and still decides, in the diff, with the drift report in the
+      # body; refusing here would only make the job red every week, since it
+      # cannot rerun itself the way the acknowledgement asks a human to.
+      - name: Refresh the snapshot from models.dev
+        run: npm run refresh:model-metadata -- --accept-upstream-removals
+
+      - name: Verify the regenerated outputs
+        run: npm run check:model-metadata
+
+      - name: Detect a snapshot change
+        id: change
+        run: |
+          if git diff --quiet -- scripts/model-metadata/models-dev-api.snapshot.json; then
+            echo 'changed=false' >> "$GITHUB_OUTPUT"
+          else
+            echo 'changed=true' >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Open the review pull request
+        if: steps.change.outputs.changed == 'true'
+        env:
+          BRANCH: automation/model-metadata-refresh
+          GH_TOKEN: ${{ github.token }}
+          TITLE: 'chore(model-metadata): refresh the models.dev snapshot'
+        run: |
+          {
+            echo '## Summary'
+            echo ''
+            echo 'Scheduled `refresh:model-metadata` run. The snapshot is the build'
+            echo 'input for the bundled model catalog; this only moves it to what'
+            echo 'models.dev serves today.'
+            echo ''
+            echo 'Refs #4398'
+            echo ''
+            echo '## Verification'
+            echo ''
+            echo 'The workflow ran `refresh:model-metadata` and `check:model-metadata`'
+            echo 'before opening this. Drift against upstream at refresh time:'
+            echo ''
+            echo '```text'
+            cat "$RUNNER_TEMP/drift.txt"
+            echo '```'
+          } > "$RUNNER_TEMP/pr-body.md"
+          git config user.name 'Apache Maka'
+          git config user.email 'commits@maka.apache.org'
+          git switch -c "$BRANCH"
+          git add scripts/model-metadata/models-dev-api.snapshot.json
+          git commit -m "$TITLE"
+          # The token travels in a header rather than the remote URL, which git
+          # echoes back in its own error messages.
+          AUTH="$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 | tr -d '\n')"
+          REMOTE="https://github.com/${GITHUB_REPOSITORY}"
+          TIP="$(git -c "http.extraheader=Authorization: Basic $AUTH" \
+            ls-remote "$REMOTE" "refs/heads/$BRANCH" | cut -f1)"
+          if [ -n "$TIP" ]; then
+            # This branch only ever carries commits this job wrote. Anything
+            # else is a person working on the open pull request, and a force
+            # push would erase it.
+            git -c "http.extraheader=Authorization: Basic $AUTH" \
+              fetch --depth=1 "$REMOTE" "refs/heads/$BRANCH"
+            if [ "$(git log -1 --format=%s FETCH_HEAD)" != "$TITLE" ]; then
+              echo "::error::$BRANCH carries a commit this workflow did not write; refusing to overwrite it."
+              exit 1
+            fi
+            git -c "http.extraheader=Authorization: Basic $AUTH" push \
+              "--force-with-lease=refs/heads/$BRANCH:$TIP" "$REMOTE" "HEAD:refs/heads/$BRANCH"
+          else
+            git -c "http.extraheader=Authorization: Basic $AUTH" push \
+              "$REMOTE" "HEAD:refs/heads/$BRANCH"
+          fi
+          # A plain existence lookup also succeeds for a closed pull request,
+          # which would leave a maintainer's decision to close one silently
+          # disabling this job forever. Only an open one is one to update.
+          if [ "$(gh pr list --head "$BRANCH" --state open --json number --jq 'length')" -gt 0 ]; then
+            gh pr edit "$BRANCH" --body-file "$RUNNER_TEMP/pr-body.md"
+            echo "Updated the open pull request on $BRANCH."
+          else
+            gh pr create --draft --base main --head "$BRANCH" \
+              --title "$TITLE" --body-file "$RUNNER_TEMP/pr-body.md"
+          fi

package.json

@@ -62,7 +62,7 @@
     "release:asf:source": "node scripts/asf-source-release.mjs create",
     "release:asf:verify": "node scripts/asf-source-release.mjs verify",
     "release:asf:sign": "node scripts/asf-source-release.mjs sign",
-    "check:asf-source": "npm run check:model-metadata && node --test scripts/asf-source-release.test.mjs scripts/asf-source-workflow-policy.test.mjs scripts/asf-license-headers.test.mjs scripts/source-legal-inventory.test.mjs scripts/sync-model-metadata.test.mjs",
+    "check:asf-source": "npm run check:model-metadata && node --test scripts/asf-source-release.test.mjs scripts/asf-source-workflow-policy.test.mjs scripts/asf-license-headers.test.mjs scripts/model-metadata-upkeep-workflow-policy.test.mjs scripts/source-legal-inventory.test.mjs scripts/sync-model-metadata.test.mjs",
     "check:asf-npm": "node --test scripts/asf-npm-workflow-policy.test.mjs",
     "check:product-release-identity": "node scripts/product-release-identity.mjs",
     "package:cli:macos-arm64": "node scripts/package-macos-arm64-cli.mjs",
@@ -92,6 +92,7 @@
     "sync:model-metadata": "node scripts/sync-model-metadata.mjs",
     "refresh:model-metadata": "node scripts/sync-model-metadata.mjs --refresh",
     "check:model-metadata": "node scripts/sync-model-metadata.mjs --check",
+    "check:model-metadata-drift": "node scripts/sync-model-metadata.mjs --drift",
     "generate:bundled-skills": "node scripts/gen-bundled-skill-catalog.mjs",
     "computer-use": "node scripts/computer-use.mjs",
     "windows:inventory": "node --test scripts/windows-test-inventory.test.mjs && node scripts/windows-test-inventory.mjs --check",

packages/core/src/__tests__/model-catalog.test.ts

@@ -112,19 +112,28 @@ test('a declared output modality without text rules a model out of chat', () =>
     modelSource: 'fetched' as const,
   };
   assert.deepEqual(verdict(audioOnly), { ok: false });
-});
 
-test('an empty output modality list is not evidence against chat', () => {
-  // `modalities.output` is typed to text, image, and audio, so a video model's
-  // real output has no representation and serializes as `[]` — the same shape
-  // a generator bug would produce. Blocking on it would be guessing.
-  const video = {
+  // Video-only says exactly what the other two say. It could not be read at
+  // all until `modalities.output` could carry the value.
+  const videoOnly = {
     providerType: 'google' as const,
     defaultModel: 'gemini-omni-flash-preview',
     models: [{ id: 'gemini-omni-flash-preview' }],
     modelSource: 'fetched' as const,
   };
-  assert.deepEqual(verdict(video), { ok: true });
+  assert.deepEqual(verdict(videoOnly), { ok: false });
+});
+
+test('an empty output modality list is not evidence against chat', () => {
+  // A provider that declared no output modality and a generator bug that
+  // dropped them produce the same shape. Blocking on it would be guessing.
+  const undeclared = {
+    providerType: 'openai-compatible' as const,
+    defaultModel: 'relay-quiet',
+    models: [{ id: 'relay-quiet', modalities: { input: ['text' as const], output: [] } }],
+    modelSource: 'fetched' as const,
+  };
+  assert.deepEqual(verdict(undeclared), { ok: true });
 });
 
 test('an explicit chat capability outranks the declared output modality', () => {

packages/core/src/__tests__/model-metadata.test.ts

@@ -138,7 +138,6 @@ describe('deepseek v4 flash vision exp metadata regression', () => {
       metadata.description,
       'Experimental DeepSeek V4 Flash model for image understanding and multimodal agent tasks',
     );
-    assert.equal(metadata.docsUrl, 'https://api-docs.deepseek.com/guides/vision/');
     assert.equal(metadata.contextWindow, 1_000_000);
     assert.equal(metadata.maxOutputTokens, 384_000);
     assert.equal(metadata.structuredOutput, true);

packages/core/src/__tests__/runtime-policy-codec.test.ts

@@ -511,6 +511,19 @@ test('normalizes extended model facts used by the runtime host catalog', () => {
   });
 });
 
+test('carries the video and pdf modalities models.dev declares', () => {
+  const modalities = {
+    input: ['text', 'image', 'video'],
+    output: ['text', 'pdf', 'video'],
+  };
+  const result = normalizeConnectionModelDiscoveryResult({
+    models: [{ id: 'custom-model', modalities }],
+    source: 'fetched',
+    fetchedAt: 42,
+  });
+  assert.deepEqual(result.models[0], { id: 'custom-model', modalities });
+});
+
 test('rejects sparse model modality arrays', () => {
   assert.throws(
     () =>

packages/core/src/llm-connections.ts

@@ -86,6 +86,20 @@ export type ConnectionAuth =
   | { kind: 'oauth_token'; oauthToken: string; expiresAt?: number }
   | { kind: 'none' };
 
+/**
+ * The modalities a model may declare on either side. Every validator that
+ * admits a modality reads this one set: a decoder, an overlay normalizer, and
+ * a live-fetch reader each holding their own copy is how one of them stayed a
+ * catalog behind the others.
+ */
+export type ModelModality = 'text' | 'image' | 'audio' | 'pdf' | 'video';
+
+const MODEL_MODALITIES: readonly ModelModality[] = ['text', 'image', 'audio', 'pdf', 'video'];
+
+export function isModelModality(value: unknown): value is ModelModality {
+  return MODEL_MODALITIES.includes(value as ModelModality);
+}
+
 export interface ModelInfo {
   id: string;
   displayName?: string;
@@ -116,8 +130,8 @@ export interface ModelInfo {
   };
   /** Multimodal input/output support from provider catalog metadata. */
   modalities?: {
-    input: Array<'text' | 'image' | 'audio' | 'pdf'>;
-    output: Array<'text' | 'image' | 'audio'>;
+    input: ModelModality[];
+    output: ModelModality[];
   };
   /**
    * Read-time provenance for values overlaid from model-facts.json. This is

packages/core/src/model-catalog.ts

@@ -515,10 +515,9 @@ function displayNameForKnownModel(
  * never set `capabilities.imageGeneration` for any of them, so the capability
  * check below could not fire on bundled data.
  *
- * An EMPTY list is not evidence. `modalities.output` is typed to text, image,
- * and audio, so a video model's real output has no representation and
- * serializes as `[]` — the same shape a future generator bug would produce.
- * Only a non-empty list says something, and what it says is what it lists.
+ * An EMPTY list is not evidence. A provider that declared no output modality
+ * and a generator bug that dropped them produce the same shape. Only a
+ * non-empty list says something, and what it says is what it lists.
  */
 function declaresNoTextOutput(model: ModelInfo): boolean {
   const output = model.modalities?.output;

packages/core/src/model-facts.ts

@@ -18,6 +18,7 @@
  */
 
 import { providerDefaultsOf, type ProviderType } from './provider-registry.js';
+import { isModelModality } from './llm-connections.js';
 import type { ModelFactField, ModelInfo } from './llm-connections.js';
 import {
   CONNECTION_CATALOG_MAX_MODELS_PER_CONNECTION,
@@ -193,8 +194,8 @@ function normalizeModalities(value: unknown): NonNullable<ModelFactOverride['mod
   if (!isRecord(value)) throw new Error('Invalid modalities');
   if (value.input === undefined && value.output === undefined)
     throw new Error('Invalid modalities');
-  const input = normalizeModalityDirection(value.input, isModality);
-  const output = normalizeModalityDirection(value.output, isOutputModality);
+  const input = normalizeModalityDirection(value.input, isModelModality);
+  const output = normalizeModalityDirection(value.output, isModelModality);
   return {
     ...(input === undefined ? {} : { input }),
     ...(output === undefined ? {} : { output }),
@@ -212,12 +213,6 @@ function normalizeModalityDirection<T extends string>(
   return [...new Set(entries)];
 }
 
-function isModality(value: unknown): value is 'text' | 'image' | 'audio' | 'pdf' {
-  return value === 'text' || value === 'image' || value === 'audio' || value === 'pdf';
-}
-function isOutputModality(value: unknown): value is 'text' | 'image' | 'audio' {
-  return value === 'text' || value === 'image' || value === 'audio';
-}
 function isPositiveBoundedInteger(value: unknown): value is number {
   return (
     typeof value === 'number' &&

packages/core/src/model-metadata.ts

@@ -28,7 +28,6 @@ export interface ModelMetadata {
   displayName?: string;
   description?: string;
   lifecycle?: 'active' | 'beta' | 'alpha' | 'deprecated' | 'retired';
-  docsUrl?: string;
   contextWindow?: number;
   inputLimit?: number;
   maxOutputTokens?: number;
@@ -242,8 +241,6 @@ const SILICONFLOW_MODEL_OVERRIDES: Record<string, ModelMetadata> = Object.fromEn
     .map(([id]) => [id, { capabilities: { chat: true } }]),
 );
 
-const VOLCENGINE_CODING_PLAN_DOCS = 'https://www.volcengine.com/docs/82379/1925114';
-const VOLCENGINE_AGENT_PLAN_DOCS = 'https://www.volcengine.com/docs/82379/2366394';
 const VOLCENGINE_CODING_PLAN_MODEL_METADATA: Record<string, ModelMetadata> = {
   'ark-code-latest': planModel('Ark Code Latest', false),
   'doubao-seed-2.0-code': planModel('Doubao Seed 2.0 Code', true),
@@ -351,7 +348,6 @@ const STATIC_MODEL_METADATA: Partial<Record<ProviderType, Record<string, ModelMe
     'doubao-seed-2-0-pro-260215': {
       displayName: 'Doubao Seed 2.0 Pro',
       lifecycle: 'active',
-      docsUrl: 'https://www.volcengine.com/docs/82379',
       capabilities: { reasoning: true, functionCalling: true },
       thinkingOptions: {
         efforts: ['minimal', 'low', 'medium', 'high'],
@@ -413,7 +409,6 @@ const STATIC_MODEL_METADATA: Partial<Record<ProviderType, Record<string, ModelMe
       displayName: 'DeepSeek-V4-Flash-Vision-Exp',
       description:
         'Experimental DeepSeek V4 Flash model for image understanding and multimodal agent tasks',
-      docsUrl: 'https://api-docs.deepseek.com/guides/vision/',
       contextWindow: 1_000_000,
       maxOutputTokens: 384_000,
       structuredOutput: true,
@@ -443,7 +438,6 @@ function planModel(
   return {
     displayName,
     lifecycle: 'active',
-    docsUrl: VOLCENGINE_CODING_PLAN_DOCS,
     ...(contextWindow === undefined ? {} : { contextWindow }),
     ...(maxOutputTokens === undefined ? {} : { maxOutputTokens }),
     capabilities: { ...REASONING_FUNCTION_CALLING, vision },
@@ -462,7 +456,6 @@ function agentPlanModel(
   return {
     displayName,
     lifecycle: options.lifecycle ?? 'active',
-    docsUrl: VOLCENGINE_AGENT_PLAN_DOCS,
     contextWindow,
     maxOutputTokens,
     capabilities: {
@@ -483,7 +476,6 @@ function displayMetadataOnly(
         displayName: metadata.displayName,
         ...(metadata.description !== undefined ? { description: metadata.description } : {}),
         lifecycle: metadata.lifecycle,
-        docsUrl: metadata.docsUrl,
         ...(metadata.knowledgeCutoff !== undefined
           ? { knowledgeCutoff: metadata.knowledgeCutoff }
           : {}),