feat(0.x): pluggable knowledge sources + freshness + change detection

.github/workflows/ci.yml

@@ -0,0 +1,37 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: pnpm/action-setup@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+
+      - name: Install deps
+        run: pnpm install --frozen-lockfile
+
+      - name: Lint (biome)
+        run: pnpm lint
+
+      - name: Typecheck
+        run: pnpm typecheck
+
+      - name: Test
+        run: pnpm test
+        env:
+          AGENT_KNOWLEDGE_RUN_NETWORK_TESTS: '1'
+
+      - name: Build
+        run: pnpm build

AGENTS.md

@@ -59,6 +59,20 @@ Use `knowledgeReleaseReportFromOptimization()` before promotion. It projects opt
 - Use `KnowledgeDiscoveryDispatcher` for research workers. Production apps should wire this to their own swarm/fleet runtime.
 - Do not bypass `lint` or `validate` before using generated knowledge in an agent.
 
+## Pluggable Sources + Freshness + Changes
+
+Agents that need to stay current against external authorities should compose:
+
+- `createCornellLiiSource({ selectors })` — US Code + Wex from law.cornell.edu.
+- `createIrsPublicationsSource({ publications, revenueProcedures })` — IRS index + named pubs.
+- `createStateSosSource({ state, baseUrl, entities })` — generic state SOS adapter.
+
+Every fetch returns `KnowledgeFragment[]` with `provenance.verifiable` indicating whether the authority was successfully authenticated. Refuse to cite fragments with `verifiable: false`.
+
+Track per-tenant freshness with `createFileSystemFreshnessStore({ root })` and re-fetch only when `stale({ workspaceId, sourceId, ttlMs })` returns true.
+
+Diff snapshots with `detectChanges(prev, next)`. Each `KnowledgeChange` carries `affectedDimensions` — pass those to your eval scheduler to re-run only the relevant campaigns.
+
 ## Authorship
 
 Do not add `Co-Authored-By:` trailers (or any other AI-attribution lines) to commits, PR descriptions, or other artifacts in this repo. Author = the human running the session. Applies to every contributor, including AI agents and subagents — do not include the default Claude Code template trailer.

README.md

@@ -198,3 +198,98 @@ await runAgentControlLoop({
   },
 })
 ```
+
+## Pluggable Knowledge Sources
+
+Static knowledge rots. Authorities like Cornell LII, the IRS, and state
+Secretaries of State change without warning — a ruling vacates an FTC
+non-compete rule, a CFR section renumbers, a state replaces Beverly-Killea
+with RULLCA. The `@tangle-network/agent-knowledge/sources` subpath ships
+three primitives that bridge "live authority" → "eval re-runs":
+
+- `KnowledgeSource` — pluggable contract (`fetch(opts) → KnowledgeFragment[]`).
+  Every fragment carries `provenance` (URL, source-attested timestamp,
+  jurisdiction, `verifiable` flag) and `dimensionHints` (which eval
+  dimensions a change in this fragment should re-score).
+- `KnowledgeFreshnessStore` — per-`(workspaceId, sourceId)` last-refresh
+  tracker. Filesystem adapter ships in-package; D1 / Postgres adapter
+  scaffold is shipped as `createD1FreshnessStoreStub(adapter)`.
+- `detectChanges(prev, next)` — diffs two fragment snapshots, emits
+  `KnowledgeChange[]` tagged with the affected eval dimensions so a cron
+  scheduler knows exactly which campaigns to re-run.
+
+Three concrete sources ship in-package:
+
+```ts
+import {
+  createCornellLiiSource,
+  createIrsPublicationsSource,
+  createStateSosSource,
+  createFileSystemFreshnessStore,
+  detectChanges,
+  type KnowledgeChange,
+  type KnowledgeFragment,
+} from '@tangle-network/agent-knowledge'
+
+const sources = [
+  // Federal statutes + Wex encyclopedia from law.cornell.edu.
+  createCornellLiiSource({
+    selectors: [
+      { kind: 'uscode', path: '18/1836' },               // DTSA
+      { kind: 'wex', path: 'restraint_of_trade', dimensionHints: ['jurisdictional_accuracy'] },
+    ],
+  }),
+  // IRS publications index + named publications + revenue procedures.
+  createIrsPublicationsSource({
+    publications: ['p15', 'p17', 'p463'],
+    revenueProcedures: [],
+  }),
+  // Generic state SOS adapter — one config per state you need tracked.
+  createStateSosSource({
+    state: 'CA',
+    baseUrl: 'https://www.sos.ca.gov',
+    entities: [{
+      id: 'business-entities-forms',
+      path: '/business-programs/business-entities/forms',
+      title: 'CA Business Entities Forms',
+      selector: { kind: 'whole' },
+    }],
+  }),
+]
+
+const freshness = createFileSystemFreshnessStore({ root: './kb' })
+
+// Worked example: Cornell LII updates the Wex `restraint_of_trade` entry
+// to reflect Ryan-LLC v. FTC. The cron tick below detects the change,
+// extracts the `jurisdictional_accuracy` dimension hint, and hands it to
+// the eval scheduler which re-runs only the campaigns tagged with that
+// dimension.
+async function tick({ workspaceId, prevSnapshots }: {
+  workspaceId: string
+  prevSnapshots: Record<string, KnowledgeFragment[]>
+}): Promise<KnowledgeChange[]> {
+  const allChanges: KnowledgeChange[] = []
+  for (const source of sources) {
+    const stale = await freshness.stale({
+      workspaceId,
+      sourceId: source.id,
+      ttlMs: 24 * 60 * 60 * 1000,
+    })
+    if (!stale) continue
+
+    const next = await source.fetch({ cacheDir: './.agent-knowledge/http-cache' })
+    const prev = prevSnapshots[source.id] ?? []
+    const { changes } = detectChanges(prev, next)
+    allChanges.push(...changes)
+
+    await freshness.mark({ workspaceId, sourceId: source.id, when: new Date() })
+    prevSnapshots[source.id] = next
+  }
+  return allChanges
+}
+```
+
+Polite-by-default: every HTTP fetch carries the package User-Agent, is
+throttled to 1 req/sec/origin, caches successful responses to disk, and
+marks `verifiable: false` on block pages / 4xx rather than promoting
+un-grounded content. See `src/sources/http.ts` for the invariants.

biome.json

@@ -0,0 +1,58 @@
+{
+  "$schema": "https://biomejs.dev/schemas/2.4.15/schema.json",
+  "files": {
+    "includes": ["src/**", "tests/**"],
+    "ignoreUnknown": true
+  },
+  "formatter": {
+    "enabled": true,
+    "indentStyle": "space",
+    "indentWidth": 2,
+    "lineWidth": 100,
+    "lineEnding": "lf"
+  },
+  "javascript": {
+    "formatter": {
+      "quoteStyle": "single",
+      "semicolons": "asNeeded",
+      "trailingCommas": "all",
+      "arrowParentheses": "always"
+    }
+  },
+  "linter": {
+    "enabled": true,
+    "rules": {
+      "recommended": true,
+      "suspicious": {
+        "noExplicitAny": "off",
+        "noConsole": "off",
+        "noAssignInExpressions": "warn",
+        "noImplicitAnyLet": "warn"
+      },
+      "style": {
+        "useImportType": "warn",
+        "useExportType": "warn",
+        "useNodejsImportProtocol": "error",
+        "noNonNullAssertion": "off",
+        "useTemplate": "warn",
+        "useExponentiationOperator": "warn",
+        "useShorthandFunctionType": "warn"
+      },
+      "complexity": {
+        "noUselessTypeConstraint": "warn",
+        "noBannedTypes": "warn"
+      },
+      "correctness": {
+        "noUnusedVariables": "off",
+        "noUnusedImports": "warn"
+      }
+    }
+  },
+  "assist": {
+    "actions": {
+      "source": {
+        "organizeImports": "on"
+      }
+    }
+  }
+}

package.json

@@ -28,6 +28,11 @@
       "types": "./dist/cli.d.ts",
       "import": "./dist/cli.js",
       "default": "./dist/cli.js"
+    },
+    "./sources": {
+      "types": "./dist/sources/index.d.ts",
+      "import": "./dist/sources/index.js",
+      "default": "./dist/sources/index.js"
     }
   },
   "bin": {
@@ -48,18 +53,25 @@
     "prepare": "tsup",
     "test": "vitest run",
     "test:watch": "vitest",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit",
+    "lint": "biome check src tests",
+    "format": "biome format --write src tests"
   },
   "dependencies": {
     "@tangle-network/agent-eval": "^0.23.0",
     "zod": "^4.3.6"
   },
   "devDependencies": {
+    "@biomejs/biome": "^2.4.15",
     "@types/node": "^25.6.0",
     "tsup": "^8.0.0",
     "typescript": "^5.7.0",
     "vitest": "^3.0.0"
   },
+  "pnpm": {
+    "minimumReleaseAge": 4320,
+    "minimumReleaseAgeExclude": []
+  },
   "engines": {
     "node": ">=20"
   },

src/adapters.ts

@@ -44,13 +44,18 @@ export function mediaTypeFor(uri: string): string {
 }
 
 function decodeText(input: SourceAdapterInput): string | undefined {
-  return input.text ?? (input.bytes ? new TextDecoder().decode(input.bytes).slice(0, 200_000) : undefined)
+  return (
+    input.text ??
+    (input.bytes ? new TextDecoder().decode(input.bytes).slice(0, 200_000) : undefined)
+  )
 }
 
 function anchorsForText(uri: string, text: string | undefined): SourceAdapterOutput['anchors'] {
   if (!text) return []
   const lines = text.split('\n')
-  const anchors: NonNullable<SourceAdapterOutput['anchors']> = [{ id: 'all', sourceId: '', label: 'Full source', lineStart: 1, lineEnd: lines.length }]
+  const anchors: NonNullable<SourceAdapterOutput['anchors']> = [
+    { id: 'all', sourceId: '', label: 'Full source', lineStart: 1, lineEnd: lines.length },
+  ]
   for (let i = 0; i < lines.length; i += 50) {
     anchors.push({
       id: `l${i + 1}`,