From c1f10a44a5211a5ca009a1f7ea395085d99137c4 Mon Sep 17 00:00:00 2001 From: "evlogai[bot]" Date: Tue, 11 Aug 2026 19:26:54 +0000 Subject: [PATCH 1/2] feat(evi): add a weekly cost and model watchdog --- apps/evi/agent/schedules/cost-watchdog.ts | 28 ++++++++ apps/evi/agent/skills/cost-watchdog/SKILL.md | 69 ++++++++++++++++++++ apps/evi/docs/notes.md | 6 ++ 3 files changed, 103 insertions(+) create mode 100644 apps/evi/agent/schedules/cost-watchdog.ts create mode 100644 apps/evi/agent/skills/cost-watchdog/SKILL.md diff --git a/apps/evi/agent/schedules/cost-watchdog.ts b/apps/evi/agent/schedules/cost-watchdog.ts new file mode 100644 index 00000000..3c0a0a41 --- /dev/null +++ b/apps/evi/agent/schedules/cost-watchdog.ts @@ -0,0 +1,28 @@ +import { defineSchedule } from 'eve/schedules' +import photon from '../channels/photon' +import { MAINTAINER_PHONE } from '../lib/trust' + +export default defineSchedule({ + // Weekly, Monday 08:00 UTC: after the daily digest (05:00) and the + // upstream-sync Monday run (07:00), and clear of self-review (Wed 08:00), so + // the four scheduled turns never contend for the thread. Monday keeps the + // report aligned to complete calendar weeks. Vercel evaluates cron in UTC. + cron: '0 8 * * 1', + // eslint-disable-next-line require-await + async run({ to, waitUntil, appAuth }) { + if (MAINTAINER_PHONE === undefined) { + throw new Error('MAINTAINER_PHONE is required for the cost-watchdog schedule.') + } + waitUntil( + to(photon, { + // Spectrum direct-chat guid: `any;-;
`, so the thread is + // derived from the phone number instead of a captured thread id. + adapterName: 'imessage', + threadId: `imessage:any;-;${MAINTAINER_PHONE}`, + }).send( + 'Load the cost-watchdog skill and run the weekly cost and model review for the last full week: pull the gateway spend by surface and model, flag drift against the previous week, check the current model landscape, and propose per-surface model adjustments (or state clearly there is nothing to improve). Write the report as a Linear document and post the top finding with the link here. This scheduled turn resumes a long-lived thread: ignore earlier conversation topics and stale pending requests, and do only this task.', + { auth: appAuth }, + ), + ) + }, +}) diff --git a/apps/evi/agent/skills/cost-watchdog/SKILL.md b/apps/evi/agent/skills/cost-watchdog/SKILL.md new file mode 100644 index 00000000..90f6a056 --- /dev/null +++ b/apps/evi/agent/skills/cost-watchdog/SKILL.md @@ -0,0 +1,69 @@ +--- +name: cost-watchdog +description: Weekly review of evlog's model cost and performance. Load this when the cost-watchdog schedule fires, or when Hugo asks for a cost check, a model review, a per-surface model analysis, or a spend/drift report for the gateway. +--- + +# Cost and model watchdog + +A recurring read of how evlog spends its model budget and whether the models in use are still the right ones. Run it weekly, on the last full week. Grounded in the AI Gateway report and the current model landscape on the web, never in your memory of prices. + +The core question: for every surface, is the model it runs still a sensible buy? The honest answer is often "yes, no change." A quiet week is a real result. + +## What the report gives you + +`ai_gateway__report` with `groupBy: 'tag'` returns one row per tag value scoped to the current environment: the `evi:env:*` row (the total) plus one `evi:surface:*` row per surface. Each row carries `total_cost`, `market_cost`, `input_tokens`, `output_tokens`, `cached_input_tokens`, `reasoning_tokens` and `request_count`. `groupBy: 'model'` returns one row per model. + +The surface list is whatever `evi:surface:*` rows the report actually returns. Do not assume the set; read it from the data. + +## Steps + +### 1. Define the window + +Run Monday morning. Cover the last 7 full days ending yesterday, and pull the 7 days before that as the comparison window, so every drift figure is period-over-period. + +### 2. Pull the numbers + +- `ai_gateway__report` for both windows, `groupBy: 'tag'`. That is the spend and token picture per surface and the total. +- `ai_gateway__report` for both windows, `groupBy: 'model'`. The per-model mix (today this is usually one model everywhere). +- For any surface worth a closer look, `ai_gateway__report` scoped to that surface (`tags: ['evi:env:', 'evi:surface:']`) with `groupBy: 'model'` to see what it runs and at what cost. + +Use the eval environment tag to keep benchmark and eval traffic out of the production read when the report lets you. + +### 3. Research the model landscape + +`web_search`/`web_fetch` for the current cost and quality picture of the models the report shows, and plausible alternatives. Look for: + +- Cost per 1M input and output tokens, and whether caching or reasoning tokens change the effective price. +- A quality or benchmark signal for each model in use and each candidate, so a swap is judged on quality and price together, not price alone. +- Whether a cheaper model has reached parity on the kind of work that surface does, or a pricier one is worth it for that surface only. + +Give every number you cite a source and a recency. Prices move; an unanchored figure is not an argument. + +### 4. Flag drift + +Compare the two windows and call out what moved, with a reason where one is visible: + +- Total or per-surface cost up or down, as a percent and a dollar figure. +- Model mix change: a model appearing, disappearing, or shifting share. +- Token shape change (input, output, cached, reasoning) that hints at a behavior or prompt drift, not just volume. +- A surface whose cost is out of proportion to its `request_count`. + +### 5. Propose per-surface model adjustments + +For each surface with nontrivial spend, judge the model it runs against what the report and the landscape research say. Recommend a swap only when there is a real, defensible win in cost, quality/task-fit, or both, with the projected effect. Otherwise state that the surface is fine as is. + +One constraint the report does not show: today the agent runs a single model everywhere, set by `EVI_MODEL` in `agent/lib/model.ts` (see `agent/lib/gateway.ts` for tagging). If a per-surface recommendation implies different models per surface, say that routing is currently global and the swap is one of two things: changing the global model, or adding surface-scoped routing as a follow-up decision. Never present a per-surface swap as a one-line config change when routing does not exist yet. + +## Deliver + +**The full report is a Linear document** on the evlog team, titled `Cost/model watchdog — YYYY-MM-DD`, with markdown sections: spend and model mix per surface, drift, landscape notes with sources, and the per-surface recommendations (or the explicit "nothing to improve"). + +**The thread get two or three lines**: the single most attention-worthy number or finding, and the document link. + +**A material, decision-worthy recommendation becomes a Linear issue** on the evlog team via `linear__save_issue`. Search first (`linear__list_issues`) for a covering issue, including your own from earlier runs; update rather than duplicate. File the strongest one or two, never a report's worth. A model change is Hugo's call, and the issue is where he makes it. + +If `linear__save_document` is unavailable or fails, fall back to posting the full report in the thread and say why. + +## When nothing is warranted + +One line. Spend flat, no drift, and the models in use still the sane choice means the report says so and stops. Never invent a drift or a swap to make the week look busy. diff --git a/apps/evi/docs/notes.md b/apps/evi/docs/notes.md index 2c98155e..e3874d51 100644 --- a/apps/evi/docs/notes.md +++ b/apps/evi/docs/notes.md @@ -77,6 +77,12 @@ deployments and promos move. Reconstructing an observed turn from it matched eve's reported `costUsd` to four decimals, which is how the overspend was found. +**`group_by: tag` on the report returns one row per tag value.** Scoped to +`evi:env:*`, that is the env total row plus one `evi:surface:*` row per surface, +each with cost, token and `request_count` columns, so the surface breakdown and +the per-model mix (`group_by: model`) are two calls away. The cost-watchdog +skill reads both; the surface set must be taken from the rows, never assumed. + ## github-tools **The `maintainer` preset ships gist tools that always 403 over Connect** — the From a0a766b4978df518145c477b6577bbbe077ba40f Mon Sep 17 00:00:00 2001 From: "evlogai[bot]" Date: Tue, 11 Aug 2026 19:38:16 +0000 Subject: [PATCH 2/2] feat(evi): point the cost watchdog at direct model and benchmark sources --- apps/evi/agent/skills/cost-watchdog/SKILL.md | 24 ++++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/apps/evi/agent/skills/cost-watchdog/SKILL.md b/apps/evi/agent/skills/cost-watchdog/SKILL.md index 90f6a056..820d8f8d 100644 --- a/apps/evi/agent/skills/cost-watchdog/SKILL.md +++ b/apps/evi/agent/skills/cost-watchdog/SKILL.md @@ -5,7 +5,7 @@ description: Weekly review of evlog's model cost and performance. Load this when # Cost and model watchdog -A recurring read of how evlog spends its model budget and whether the models in use are still the right ones. Run it weekly, on the last full week. Grounded in the AI Gateway report and the current model landscape on the web, never in your memory of prices. +A recurring read of how evlog spends its model budget and whether the models in use are still the right ones. Run it weekly, on the last full week. Grounded in the AI Gateway report and the current model landscape, never in your memory of prices. The core question: for every surface, is the model it runs still a sensible buy? The honest answer is often "yes, no change." A quiet week is a real result. @@ -15,6 +15,18 @@ The core question: for every surface, is the model it runs still a sensible buy? The surface list is whatever `evi:surface:*` rows the report actually returns. Do not assume the set; read it from the data. +## Resources + +Open these directly instead of searching; they are the stable home for everything the model-landscape step needs. + +- **AI Gateway model catalog, as JSON** (pricing and capabilities for every model in one fetch): `https://ai-gateway.vercel.sh/v1/models` +- **AI Gateway models browser** (human-readable, filter by provider, pricing, latency, throughput): `https://vercel.com/ai-gateway/models` +- **AI Gateway docs, models & providers**: `https://vercel.com/docs/ai-gateway/models-and-providers` +- **Model quality leaderboard** (ex-LMArena, blind A/B human preference Elo): `https://arena.ai/leaderboard` +- **Independent cost-efficiency and benchmarks** (Intelligence Index, Cost per Task, speed): `https://artificialanalysis.ai/` + +Use `web_search`/`web_fetch` only for what these do not cover, such as a candidate model's fit for a specific surface. Every figure cited still needs a source and a recency. + ## Steps ### 1. Define the window @@ -25,19 +37,17 @@ Run Monday morning. Cover the last 7 full days ending yesterday, and pull the 7 - `ai_gateway__report` for both windows, `groupBy: 'tag'`. That is the spend and token picture per surface and the total. - `ai_gateway__report` for both windows, `groupBy: 'model'`. The per-model mix (today this is usually one model everywhere). -- For any surface worth a closer look, `ai_gateway__report` scoped to that surface (`tags: ['evi:env:', 'evi:surface:']`) with `groupBy: 'model'` to see what it runs and at what cost. +- For any surface worth a closer look, `ai_gateway__report` scoped to that surface (`tags: ['evi:env:', 'evi:surface:'])` with `groupBy: 'model'` to see what it runs and at what cost. Use the eval environment tag to keep benchmark and eval traffic out of the production read when the report lets you. ### 3. Research the model landscape -`web_search`/`web_fetch` for the current cost and quality picture of the models the report shows, and plausible alternatives. Look for: +Open the Resources above and pull the current price and quality picture for the models the report shows, and plausible alternatives. Look for: -- Cost per 1M input and output tokens, and whether caching or reasoning tokens change the effective price. +- Cost per 1M input and output tokens (`/v1/models` gives per-token prices), and whether caching or reasoning tokens change the effective price. - A quality or benchmark signal for each model in use and each candidate, so a swap is judged on quality and price together, not price alone. -- Whether a cheaper model has reached parity on the kind of work that surface does, or a pricier one is worth it for that surface only. - -Give every number you cite a source and a recency. Prices move; an unanchored figure is not an argument. +- Whether a cheaper model has reached parity on the kind of work that surface does, or a pricier one is worth it for that surface only. Artificial Analysis' Cost per Task pairs best with the leaderboard for this call. ### 4. Flag drift