chore(deps): align knowledge with Eval 0.144.1

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Unreleased
 
+## 7.0.9
+
+### Changed
+
+- Updated `@tangle-network/agent-eval` to `0.144.1` so Knowledge and Runtime install the same official-optimizer callback contract.
+
 ## 7.0.8
 
 ### Changed

README.md

@@ -9,7 +9,7 @@ Supply application callbacks for those decisions, or use `@tangle-network/agent-
 ## Install
 
 ```bash
-pnpm add @tangle-network/agent-knowledge@7.0.8 @tangle-network/agent-eval@0.143.0
+pnpm add @tangle-network/agent-knowledge@7.0.9 @tangle-network/agent-eval@0.144.1
 ```
 
 Requires Node.js 20.19 or later.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-knowledge",
-  "version": "7.0.8",
+  "version": "7.0.9",
   "description": "Build, search, evaluate, and improve source-backed knowledge bases.",
   "homepage": "https://github.com/tangle-network/agent-knowledge#readme",
   "repository": {
@@ -75,7 +75,7 @@
     "verify:official-optimizers": "node scripts/verify-official-optimizers.mjs"
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.143.0",
+    "@tangle-network/agent-eval": "0.144.1",
     "@tangle-network/agent-interface": "0.43.0",
     "proper-lockfile": "4.1.2",
     "zod": "^4.4.3"

tests/contracts/agent-stack-cohort.ts

@@ -1,3 +1,4 @@
+import type { ExternalOptimizerModelCall } from '@tangle-network/agent-eval/campaign'
 import type { AgentCandidateKnowledgeRef } from '@tangle-network/agent-interface'
 
 import { fromAgentCandidateKnowledgeRef, toAgentCandidateKnowledgeRef } from '../../src/index'
@@ -8,3 +9,10 @@ export function roundTripCanonicalKnowledgeCandidate(
 ): AgentCandidateKnowledgeRef {
   return toAgentCandidateKnowledgeRef(fromAgentCandidateKnowledgeRef(candidate))
 }
+
+/** The installed Eval cohort exposes the caller-owned official optimizer model callback. */
+export function acceptExternalOptimizerModelCall(
+  call: ExternalOptimizerModelCall,
+): ExternalOptimizerModelCall {
+  return call
+}

tests/kb-improvement/optimization.test.ts

@@ -50,7 +50,12 @@ describe('optimizeKnowledgeBasePolicy', () => {
           expect('testScenarios' in input).toBe(false)
           return {
             winnerSurface: '{"mode":"grounded"}',
-            cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] },
+            cost: {
+              totalCostUsd: 0,
+              costProvenance: { kind: 'observed', usd: 0 },
+              accountingComplete: true,
+              incompleteReasons: [],
+            },
           }
         },
       }
@@ -156,7 +161,12 @@ describe('optimizeKnowledgeBasePolicy', () => {
           methodCalls += 1
           return {
             winnerSurface: '{"mode":"candidate"}',
-            cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] },
+            cost: {
+              totalCostUsd: 0,
+              costProvenance: { kind: 'observed', usd: 0 },
+              accountingComplete: true,
+              incompleteReasons: [],
+            },
           }
         },
       }
@@ -250,7 +260,12 @@ describe('optimizeKnowledgeBasePolicy', () => {
           expect('testScenarios' in input).toBe(false)
           return {
             winnerSurface: '{"evidence":"required","maxSources":4}',
-            cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] },
+            cost: {
+              totalCostUsd: 0,
+              costProvenance: { kind: 'observed', usd: 0 },
+              accountingComplete: true,
+              incompleteReasons: [],
+            },
           }
         },
       }
@@ -337,7 +352,12 @@ describe('optimizeKnowledgeBasePolicy', () => {
           await writeFile(join(root, 'knowledge', 'concurrent-change.md'), '# Concurrent change\n')
           return {
             winnerSurface: '{"evidence":"required","maxSources":2}',
-            cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] },
+            cost: {
+              totalCostUsd: 0,
+              costProvenance: { kind: 'observed', usd: 0 },
+              accountingComplete: true,
+              incompleteReasons: [],
+            },
           }
         },
       }

tests/memory/improvement.test.ts

@@ -188,7 +188,12 @@ describe('agent memory improvement', () => {
         await duplicateRuns
         return {
           winnerSurface: surface,
-          cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] },
+          cost: {
+            totalCostUsd: 0,
+            costProvenance: { kind: 'observed', usd: 0 },
+            accountingComplete: true,
+            incompleteReasons: [],
+          },
         }
       },
     }
@@ -386,6 +391,7 @@ describe('agent memory improvement', () => {
         runDir: '/runs/incomplete-method-cost',
         method: selectingMethod([{ visibility: 'private' }, { visibility: 'team' }], undefined, {
           totalCostUsd: 0,
+          costProvenance: { kind: 'uncaptured', usd: null },
           accountingComplete: false,
           incompleteReasons: ['external optimizer usage unavailable'],
         }),
@@ -439,7 +445,12 @@ function baseOptions(
 function selectingMethod<TConfig extends JsonValue>(
   configs: readonly TConfig[],
   inputs?: string[][],
-  cost = { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] },
+  cost = {
+    totalCostUsd: 0,
+    costProvenance: { kind: 'observed' as const, usd: 0 },
+    accountingComplete: true,
+    incompleteReasons: [],
+  },
 ): OptimizationMethod<MemoryConfigScenario, AgentMemorySequenceArtifact> {
   return {
     name: 'fixture-selection',

tests/official-optimization.integration.test.ts

@@ -1,10 +1,13 @@
 import { spawnSync } from 'node:child_process'
 import { mkdtemp, rm } from 'node:fs/promises'
-import { createServer, type Server } from 'node:http'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
-import type { JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign'
-import { afterEach, describe, expect, it } from 'vitest'
+import type {
+  ExternalOptimizerModelCall,
+  JudgeConfig,
+  Scenario,
+} from '@tangle-network/agent-eval/campaign'
+import { describe, expect, it } from 'vitest'
 import type { RetrievalEvalArtifact, RetrievalEvalScenario } from '../src'
 import { testExecutionRef } from './support/optimization'
 
@@ -19,25 +22,11 @@ const { runRetrievalImprovementLoop, runSerializedKnowledgeOptimization } = know
 
 const python = process.env.AGENT_EVAL_TEST_PYTHON
 const describeWithOfficialEngines = python ? describe : describe.skip
-const openServers: Server[] = []
-
-afterEach(async () => {
-  await Promise.all(
-    openServers.splice(0).map(
-      (server) =>
-        new Promise<void>((resolve, reject) => {
-          server.closeAllConnections?.()
-          server.close((error) => (error ? reject(error) : resolve()))
-        }),
-    ),
-  )
-})
 
 describeWithOfficialEngines('official optimizer integration', () => {
   it('runs official GEPA through retrieval optimization and final scoring', async () => {
     assertPythonModules(python!, ['agent_eval_rpc.gepa_bridge', 'gepa.optimize_anything'])
     const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-gepa-'))
-    const baseUrl = await startModelServer('```\n{"k":2}\n```')
     try {
       const method = gepaOptimizationMethod<RetrievalEvalScenario, RetrievalEvalArtifact>({
         name: 'official-gepa-retrieval',
@@ -67,7 +56,7 @@ describeWithOfficialEngines('official optimizer integration', () => {
             },
           },
         },
-        optimizer: optimizerModel(baseUrl),
+        optimizer: optimizerModel('```\n{"k":2}\n```', 'agent-knowledge-test:gepa-fixed-response'),
         describeScenario: (scenario) => ({
           query: scenario.query,
           expected: scenario.expected,
@@ -144,7 +133,6 @@ describeWithOfficialEngines('official optimizer integration', () => {
         reasoning: 'Add the missing response rule.',
       },
     })
-    const baseUrl = await startModelServer(modelResponse)
     try {
       const method = skillOptOptimizationMethod<SkillScenario, SkillArtifact>({
         name: 'official-skillopt-policy',
@@ -161,7 +149,7 @@ describeWithOfficialEngines('official optimizer integration', () => {
           maxAnalystRounds: 1,
           evaluationWorkers: 1,
         },
-        optimizer: optimizerModel(baseUrl),
+        optimizer: optimizerModel(modelResponse, 'agent-knowledge-test:skillopt-fixed-response'),
         maxEvaluations: 3,
         describeScenario: (scenario) => ({ prompt: scenario.prompt }),
         describeArtifact: (artifact) => ({ candidate: artifact.candidate }),
@@ -253,67 +241,65 @@ function pythonRunner(command: string, module: string) {
   return { command, args: ['-m', module] }
 }
 
-function optimizerModel(baseUrl: string) {
+function optimizerModel(content: string, callRef: string) {
+  const pricing = {
+    inputUsdPerMillion: 1,
+    outputUsdPerMillion: 2,
+  }
+  const call: ExternalOptimizerModelCall = async ({ callId, request }) =>
+    optimizerModelCall({ callId, request, content, pricing })
   return {
     model: 'local-optimizer',
-    baseUrl,
-    apiKey: 'local-test-key',
+    callRef,
+    call,
     budget: {
       maxCostUsd: 1,
-      maxRequests: 10,
+      maxRequests: 100,
       maxRequestBytes: 100_000,
       maxResponseBytes: 100_000,
       maxOutputTokensPerRequest: 2_000,
-      pricing: {
-        inputUsdPerMillion: 1,
-        outputUsdPerMillion: 2,
-      },
+      pricing,
     },
   }
 }
 
+async function optimizerModelCall({
+  callId,
+  request,
+  content,
+  pricing,
+}: {
+  callId: string
+  request: { model: string }
+  content: string
+  pricing: { inputUsdPerMillion: number; outputUsdPerMillion: number }
+}) {
+  return {
+    succeeded: true as const,
+    response: {
+      content,
+      usage: { promptTokens: 11, completionTokens: 13, totalTokens: 24 },
+      costUsd: 0.000037,
+      model: request.model,
+      durationMs: 0,
+      finishReason: 'stop',
+      contentEmpty: false,
+      raw: { id: callId, fixture: true },
+    },
+    receipt: {
+      model: request.model,
+      inputTokens: 11,
+      outputTokens: 13,
+      customTokenPricing: pricing,
+    },
+    execution: { kind: 'fixed-test-response', callId, model: request.model },
+  }
+}
+
 function assertPythonModules(command: string, modules: readonly string[]): void {
   const imports = modules.map((module) => `import ${module}`).join('; ')
   const result = spawnSync(command, ['-c', imports], { encoding: 'utf8' })
   if (result.status !== 0) {
     throw new Error(`official optimizer Python environment is unavailable: ${result.stderr.trim()}`)
   }
 }
-
-async function startModelServer(content: string): Promise<string> {
-  const server = createServer(async (request, response) => {
-    for await (const _chunk of request) {
-      // Drain the request before replying.
-    }
-    response.writeHead(200, { 'content-type': 'application/json' })
-    response.end(
-      JSON.stringify({
-        id: 'local-completion',
-        choices: [
-          {
-            finish_reason: 'stop',
-            index: 0,
-            message: { role: 'assistant', content },
-          },
-        ],
-        model: 'local-optimizer',
-        usage: {
-          prompt_tokens: 20,
-          completion_tokens: 20,
-          total_tokens: 40,
-        },
-      }),
-    )
-  })
-  openServers.push(server)
-  await new Promise<void>((resolve, reject) => {
-    server.once('error', reject)
-    server.listen(0, '127.0.0.1', () => {
-      server.off('error', reject)
-      resolve()
-    })
-  })
-  const address = server.address()
-  if (!address || typeof address === 'string') throw new Error('model server did not bind')
-  return `http://127.0.0.1:${address.port}/v1`
-}

tests/official-optimizer-resume.contract.test.ts

@@ -66,8 +66,19 @@ describe('official optimizer resume identity', () => {
           trainer: { epochs: 1, batchSize: 1 },
           optimizer: {
             model: 'unused-test-model',
-            baseUrl: 'http://127.0.0.1:1/v1',
-            apiKey: 'unused-test-key',
+            callRef: 'knowledge-test:never-invoked',
+            call: async ({ request }) => ({
+              succeeded: false,
+              error: 'the fake optimizer issued an unexpected model call',
+              receipt: {
+                model: request.model,
+                inputTokens: 0,
+                outputTokens: 0,
+                costUnknown: true,
+                usageUnknown: true,
+              },
+              execution: { kind: 'unexpected-test-call' },
+            }),
             budget: modelBudget,
           },
           maxEvaluations: 1,