feat(core,mcp): binary cursor v2 — full-load ~2.7k chars (was ~5k) + maxCursorSeen knob

.changeset/cursor-v2-binary-encoding.md

@@ -0,0 +1,30 @@
+---
+"@refkit/core": minor
+---
+
+Shrink the load-more cursor to roughly half its size: `meta.nextCursor` is now
+a binary-packed base64url string (magic + version + page + raw fnv1a uint32
+seen-keys) instead of v1's JSON array of base36 hash strings — a full 500-entry
+cursor drops from ~5k to ~2.7k chars. Cursors ride inside LLM tool outputs
+downstream (and get replayed through conversation history), so every char
+counts; ~2.7k also clears consumers that clamp tool-output strings at 4k.
+
+The cursor stays opaque and self-contained: pass back `meta.nextCursor`, get
+the next deduped batch, no caller-side bookkeeping, no client instance state.
+Anything else — including a v1 JSON cursor from a previous release — still
+fails loudly with "invalid cursor" rather than quietly restarting from page 1
+(cursors are short-lived load-more state, not durable ids; there is no v1
+migration).
+
+New `createRefkit({ maxCursorSeen })` caps how many already-returned keys the
+cursor remembers (default unchanged at 500, most recent kept, ~5.4 chars each)
+for callers who want an even tighter cursor and can accept re-showing
+long-evicted results sooner. `Infinity` disables the cap; the effective floor
+is the batch just returned, so a too-small cap can never make load-more repeat
+the batch it just handed back.
+
+Hardening over v1, both restoring guarantees the removed zod schema provided:
+an out-of-uint32-range `controls.page` (negative, fractional, `NaN`, ≥ 2^32)
+encodes as a poison cursor that fails loudly on the next call instead of
+silently wrapping to a different page, and non-canonical base64url (tampered
+trailing bits) is rejected rather than silently aliased to a valid cursor.

.changeset/mcp-max-cursor-seen-env.md

@@ -0,0 +1,9 @@
+---
+"@refkit/mcp": minor
+---
+
+`REFKIT_MAX_CURSOR_SEEN` env var for the zero-config CLI: caps how many
+already-returned keys the load-more cursor remembers (core's `maxCursorSeen`),
+for hosts that clamp tool-output strings — the default 500-key cursor is ~2.7k
+chars; `REFKIT_MAX_CURSOR_SEEN=200` brings it near ~1.1k. Invalid values warn
+on stderr and fall back to the core default.

README.md

@@ -253,7 +253,7 @@ Agents can use refkit in two ways:
 npx -y @refkit/mcp
 ```
 
-It boots with the keyless sources (Met, Art Institute, Wikimedia, Openverse, Project Gutenberg, PoetryDB, Rijksmuseum, Poly Haven, ambientCG, Internet Archive) and auto-adds any BYOK source whose key is in the environment (`REFKIT_UNSPLASH_KEY`, `REFKIT_PEXELS_KEY`, `REFKIT_BRAVE_KEY`, … — legacy names like `UNSPLASH_KEY`, `PEXELS_KEY`, `BRAVE_TOKEN` still work as fallbacks). Pass `intent` to annotate each result with a use-verdict (may I use this, is attribution required); `gateFor` to return only allowed results; `rerank: true` for query-aware re-ranking (term coverage incl. CJK, resolution, source diversity); `cursor` (from the previous result's top-level `nextCursor` — always returned, no `explain` needed) to page through results without repeats. BYOK provider packages are `optionalDependencies` of `@refkit/mcp` — installed by default (zero-config `npx` keeps working), but an install with `--omit=optional` skips them, and a key whose package is missing just logs a stderr warning instead of crashing the server. Beyond search, `evaluate_use` and `build_attribution` expose the same license-verdict and attribution logic as standalone stateless tools, for when an agent already has a license id and just needs a verdict or a credit line. Or wire your own providers/keys via `serveStdio(createRefkit({ … }))` — see [`@refkit/mcp`](https://www.npmjs.com/package/@refkit/mcp).
+It boots with the keyless sources (Met, Art Institute, Wikimedia, Openverse, Project Gutenberg, PoetryDB, Rijksmuseum, Poly Haven, ambientCG, Internet Archive) and auto-adds any BYOK source whose key is in the environment (`REFKIT_UNSPLASH_KEY`, `REFKIT_PEXELS_KEY`, `REFKIT_BRAVE_KEY`, … — legacy names like `UNSPLASH_KEY`, `PEXELS_KEY`, `BRAVE_TOKEN` still work as fallbacks). Pass `intent` to annotate each result with a use-verdict (may I use this, is attribution required); `gateFor` to return only allowed results; `rerank: true` for query-aware re-ranking (term coverage incl. CJK, resolution, source diversity); `cursor` (from the previous result's top-level `nextCursor` — always returned, no `explain` needed) to page through results without repeats; `REFKIT_MAX_CURSOR_SEEN` shrinks the cursor for hosts that clamp tool-output strings. BYOK provider packages are `optionalDependencies` of `@refkit/mcp` — installed by default (zero-config `npx` keeps working), but an install with `--omit=optional` skips them, and a key whose package is missing just logs a stderr warning instead of crashing the server. Beyond search, `evaluate_use` and `build_attribution` expose the same license-verdict and attribution logic as standalone stateless tools, for when an agent already has a license id and just needs a verdict or a credit line. Or wire your own providers/keys via `serveStdio(createRefkit({ … }))` — see [`@refkit/mcp`](https://www.npmjs.com/package/@refkit/mcp).
 
 ## Not legal advice
 

packages/core/src/__tests__/client.test.ts

@@ -1,5 +1,6 @@
 import { describe, expect, it, vi } from 'vitest'
 import { createRefkit } from '../client'
+import { cursorSeenKey, decodeCursor } from '../cursor'
 import { defineProvider, type ReferenceProvider } from '../provider'
 import { lexicalReranker } from '../rerank'
 import type { Reference } from '../reference'
@@ -122,6 +123,53 @@ describe('createRefkit', () => {
     const rk = createRefkit({ providers: [provider('a', [ref('a-1', 'https://a/1')])] })
     await expect(rk.search({ query: 'x', modalities: ['image'], cursor: 'not-a-cursor' })).rejects.toThrow(/invalid cursor/)
     await expect(rk.search({ query: 'x', modalities: ['image'], cursor: '{"v":9}' })).rejects.toThrow(/invalid cursor/)
+    // Legacy v1 JSON cursors are short-lived load-more state, not durable ids —
+    // they fail like any other foreign string instead of being migrated.
+    await expect(rk.search({ query: 'x', modalities: ['image'], cursor: '{"v":1,"page":1,"seen":["abc"]}' })).rejects.toThrow(/invalid cursor/)
+  })
+
+  it('cursor: a bad caller-supplied controls.page fails loudly on the next call, not silently', async () => {
+    // v1's zod decode rejected out-of-range pages when the cursor came back;
+    // v2 must preserve that instead of wrapping to some other uint32 page.
+    const rk = createRefkit({ providers: [provider('a', [ref('a-1', 'https://a/1')])] })
+    const out = await rk.searchWithMeta({ query: 'x', modalities: ['image'], controls: { page: -1 } })
+    expect(out.references).toHaveLength(1)
+    await expect(rk.search({ query: 'x', modalities: ['image'], cursor: out.meta.nextCursor })).rejects.toThrow(/invalid cursor/)
+  })
+
+  it('cursor: seen never evicts the batch just returned, even when maxCursorSeen is smaller', async () => {
+    // A cap below the batch size would re-show this batch on the very next
+    // call and pagination would never converge.
+    const refs = [ref('a-1', 'https://a/1'), ref('a-2', 'https://a/2'), ref('a-3', 'https://a/3'), ref('a-4', 'https://a/4')]
+    const rk = createRefkit({ providers: [provider('a', refs)], maxCursorSeen: 1 })
+    const batch1 = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 2 })
+    expect(batch1.references.map(r => r.canonicalUrl)).toEqual(['https://a/1', 'https://a/2'])
+    const batch2 = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 2, cursor: batch1.meta.nextCursor })
+    expect(batch2.references.map(r => r.canonicalUrl)).toEqual(['https://a/3', 'https://a/4'])
+  })
+
+  it('cursor: maxCursorSeen Infinity disables the cap instead of falling back to the default', async () => {
+    // 501 results in one batch: the default cap would trim seen to 500.
+    const many = Array.from({ length: 501 }, (_, i) => ref(`a-${i}`, `https://a/${i}`))
+    const rk = createRefkit({ providers: [provider('a', many)], maxCursorSeen: Infinity })
+    const batch = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 501 })
+    expect(batch.references).toHaveLength(501)
+    expect(decodeCursor(batch.meta.nextCursor!).seen).toHaveLength(501)
+  })
+
+  it('cursor: maxCursorSeen caps remembered keys (oldest evicted first)', async () => {
+    const refs = [ref('a-1', 'https://a/1'), ref('a-2', 'https://a/2'), ref('a-3', 'https://a/3'), ref('a-4', 'https://a/4')]
+    const rk = createRefkit({ providers: [provider('a', refs)], maxCursorSeen: 2 })
+    const search = (cursor?: string) => rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 1, cursor })
+
+    const batch1 = await search()
+    const batch2 = await search(batch1.meta.nextCursor)
+    const batch3 = await search(batch2.meta.nextCursor)
+    expect(batch3.references.map(r => r.canonicalUrl)).toEqual(['https://a/3'])
+    // Only the 2 most recent keys survive; batch1's key was evicted.
+    expect(decodeCursor(batch3.meta.nextCursor!).seen).toEqual(
+      [cursorSeenKey('https://a/2'), cursorSeenKey('https://a/3')],
+    )
   })
 
   it('rejects a Promise passed as providers (un-awaited async factory)', () => {

packages/core/src/__tests__/cursor.test.ts

@@ -0,0 +1,81 @@
+import { describe, expect, it } from 'vitest'
+import { cursorSeenKey, decodeCursor, encodeCursor } from '../cursor'
+
+const INVALID = /invalid cursor/
+
+describe('cursor v2 encoding', () => {
+  it('roundtrips page + seen through an opaque base64url string', () => {
+    const state = { page: 7, seen: [0, 1, 0x7fffffff, 0xffffffff, 123456789] }
+    const encoded = encodeCursor(state)
+    expect(encoded).toMatch(/^[A-Za-z0-9_-]+$/)
+    expect(decodeCursor(encoded)).toEqual(state)
+  })
+
+  it('roundtrips an empty seen list', () => {
+    expect(decodeCursor(encodeCursor({ page: 1, seen: [] }))).toEqual({ page: 1, seen: [] })
+  })
+
+  it('cursorSeenKey is a deterministic uint32', () => {
+    const k = cursorSeenKey('https://a/1')
+    expect(Number.isInteger(k)).toBe(true)
+    expect(k).toBeGreaterThanOrEqual(0)
+    expect(k).toBeLessThan(2 ** 32)
+    expect(cursorSeenKey('https://a/1')).toBe(k)
+  })
+
+  it('stays under 2.8k chars at the 500-entry seen cap (v1 JSON was ~5k)', () => {
+    const seen = Array.from({ length: 500 }, (_, i) => (i * 2654435761) >>> 0)
+    const encoded = encodeCursor({ page: 9, seen })
+    expect(encoded.length).toBeLessThanOrEqual(2700)
+    expect(decodeCursor(encoded).seen).toEqual(seen)
+  })
+
+  it('throws "invalid cursor" on strings not produced by encodeCursor', () => {
+    for (const bad of [
+      '',
+      'not a cursor!', // characters outside the base64url alphabet
+      '{"v":1,"page":1,"seen":["abc"]}', // legacy v1 JSON cursor — no longer accepted
+      'not-a-cursor', // base64url alphabet but not our layout
+      'AAAAAAAA', // well-formed base64url, decodes to 6 bytes — shorter than the header
+      'abc', // too short to hold a header
+    ]) {
+      expect(() => decodeCursor(bad), JSON.stringify(bad)).toThrow(INVALID)
+    }
+  })
+
+  it('throws on wrong magic and wrong version (payload long enough to reach those checks)', () => {
+    const v2 = (bytes: number[]) => Buffer.from(bytes).toString('base64url')
+    // Sanity: Buffer's base64url of a genuine header matches our encoder.
+    expect(v2([0x52, 0x6b, 0x02, 1, 0, 0, 0])).toBe(encodeCursor({ page: 1, seen: [] }))
+    expect(() => decodeCursor(v2([0x00, 0x6b, 0x02, 1, 0, 0, 0]))).toThrow(INVALID) // bad magic byte 0
+    expect(() => decodeCursor(v2([0x52, 0x00, 0x02, 1, 0, 0, 0]))).toThrow(INVALID) // bad magic byte 1
+    expect(() => decodeCursor(v2([0x52, 0x6b, 0x01, 1, 0, 0, 0]))).toThrow(INVALID) // version 1
+    expect(() => decodeCursor(v2([0x52, 0x6b, 0x03, 1, 0, 0, 0]))).toThrow(INVALID) // version 3
+  })
+
+  it('throws on non-canonical trailing bits (single-char tamper the encoder could never emit)', () => {
+    // Both base64url remainder classes: rem-2 (4 unused bits) and rem-3 (2 unused bits).
+    for (const state of [{ page: 1, seen: [] }, { page: 3, seen: [42] }]) {
+      const encoded = encodeCursor(state)
+      expect(encoded.endsWith('A')).toBe(true) // unused trailing bits are zero
+      expect(() => decodeCursor(encoded.slice(0, -1) + 'B'), encoded).toThrow(INVALID)
+    }
+  })
+
+  it('throws on a truncated but otherwise genuine cursor', () => {
+    const encoded = encodeCursor({ page: 1, seen: [1, 2, 3] })
+    expect(() => decodeCursor(encoded.slice(0, -2))).toThrow(INVALID)
+  })
+
+  it('throws on a cursor carrying page 0', () => {
+    expect(() => decodeCursor(encodeCursor({ page: 0, seen: [] }))).toThrow(INVALID)
+  })
+
+  it('encodes an out-of-uint32-range page as a poison cursor that fails loudly on decode', () => {
+    // A bad caller-supplied controls.page must NOT silently wrap to a different
+    // page (v1 failed loudly on the next call; v2 must too).
+    for (const page of [-1, 2.5, Number.NaN, 2 ** 32, 2 ** 32 + 5]) {
+      expect(() => decodeCursor(encodeCursor({ page, seen: [7] })), String(page)).toThrow(INVALID)
+    }
+  })
+})

packages/core/src/client.ts

@@ -45,6 +45,15 @@ export interface RefkitOptions {
    *  sources at once — a provider's timeout only starts when its slot starts, so
    *  queueing never burns a queued provider's deadline. */
   concurrency?: number
+  /** Cap on already-returned keys remembered inside the load-more cursor (most
+   *  recent kept). Each key costs ~5.4 chars of cursor, so this bounds
+   *  `meta.nextCursor` length (~2.7k chars at the default 500). Lower it when
+   *  the cursor travels a size-sensitive channel (e.g. LLM tool output);
+   *  overflowing just risks re-showing results evicted long ago. `Infinity`
+   *  disables the cap. Effective floor is the batch just returned — evicting
+   *  keys the same call produced would repeat them immediately and load-more
+   *  would never converge. */
+  maxCursorSeen?: number
 }
 
 export interface ProviderError {
@@ -146,9 +155,10 @@ const DEFAULT_CACHE_TTL_MS = 300_000
 // Cursor: how many further provider pages one load-more call may try when the
 // current page's pool is fully consumed, before reporting an empty batch.
 const MAX_CURSOR_ADVANCES = 3
-// Cursor: cap on remembered already-returned keys (most recent kept). Bounds
-// cursor size (~7 bytes/key); overflowing just risks re-showing very old results.
-const MAX_CURSOR_SEEN = 500
+// Cursor: default cap on remembered already-returned keys (most recent kept;
+// see RefkitOptions.maxCursorSeen). Bounds cursor size (~5.4 chars/key packed);
+// overflowing just risks re-showing very old results.
+const DEFAULT_MAX_CURSOR_SEEN = 500
 
 function errorSummary(error: unknown): string {
   if (error instanceof Error) return error.message
@@ -336,13 +346,16 @@ export function createRefkit(options: RefkitOptions): RefkitClient {
     }
 
     const references = pass.refs.slice(0, limit)
+    // Never below this batch's size (evicting keys just returned would repeat
+    // them on the very next call); Infinity = uncapped, NaN falls back.
+    const rawMaxSeen = options.maxCursorSeen ?? DEFAULT_MAX_CURSOR_SEEN
+    const maxCursorSeen = Math.max(Number.isNaN(rawMaxSeen) ? DEFAULT_MAX_CURSOR_SEEN : rawMaxSeen, references.length)
     const nextCursor = references.length > 0
       ? encodeCursor({
-          v: 1,
           // Same page on purpose — its overfetched pool may still hold
           // unreturned results; the next call advances internally if not.
           page: page ?? 1,
-          seen: [...(cursorState?.seen ?? []), ...references.map(r => cursorSeenKey(r.canonicalUrl))].slice(-MAX_CURSOR_SEEN),
+          seen: [...(cursorState?.seen ?? []), ...references.map(r => cursorSeenKey(r.canonicalUrl))].slice(-maxCursorSeen),
         })
       : undefined
     const warnings: string[] = []