From aa4b04845c7b7d3d7c49fbbcfad3b41051e04b8d Mon Sep 17 00:00:00 2001 From: lizhixuan Date: Fri, 24 Jul 2026 20:21:30 +0800 Subject: [PATCH] feat(core,mcp): source-targeted search via SearchInput.sources Add `SearchInput.sources?: string[]` to @refkit/core: restrict a search to specific provider ids (intersected with modality matching), so a caller can scope a search-engine operator (e.g. `site:xiaohongshu.com` on Brave) to one web-discovery source without polluting the other providers' queries. Omit it to fan out to every configured source, unchanged. Selection stays fail-loud: a `sources` list matching no provider for the requested modalities throws (a typo must not read as "no results"); an id that resolves to nothing while others still match goes to `meta.warnings`. Providers excluded by an explicit `sources` filter report `reason: 'not-selected'` in `meta.providers`, distinct from `'unsupported-modality'`. @refkit/mcp `search_references` gains a `sources` param (its description enumerates the server's enabled source ids) and turns a source-selection miss into an agent-friendly tool error listing the valid ids. --- .changeset/search-sources.md | 20 +++++ README.md | 16 ++++ packages/core/src/__tests__/client.test.ts | 87 ++++++++++++++++++++++ packages/core/src/client.ts | 34 ++++++++- packages/mcp/src/__tests__/mcp.test.ts | 34 +++++++++ packages/mcp/src/index.ts | 33 +++++++- 6 files changed, 218 insertions(+), 6 deletions(-) create mode 100644 .changeset/search-sources.md diff --git a/.changeset/search-sources.md b/.changeset/search-sources.md new file mode 100644 index 0000000..caaf3ca --- /dev/null +++ b/.changeset/search-sources.md @@ -0,0 +1,20 @@ +--- +"@refkit/core": minor +"@refkit/mcp": minor +--- + +Add source-targeted search. `SearchInput.sources?: string[]` restricts a search +to specific provider ids (intersected with modality matching); omit it to fan out +to every configured source as before. This lets a caller scope a search-engine +operator — e.g. `site:xiaohongshu.com` against Brave's index — to one +web-discovery source without polluting the other providers' queries. + +Selection stays fail-loud: a `sources` list that matches no configured provider +for the requested modalities throws (a typo must not read as "no results"), while +an id that resolves to nothing when others still match is reported in +`meta.warnings`. Providers excluded by an explicit `sources` filter now report +`reason: 'not-selected'` in `meta.providers`, distinct from `'unsupported-modality'`. + +`@refkit/mcp`'s `search_references` tool gains a `sources` parameter (its +description enumerates the server's enabled source ids) and turns a +source-selection miss into an agent-friendly tool error that lists the valid ids. diff --git a/README.md b/README.md index ff17cb7..1356b59 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,22 @@ await refkit.search({ The provider package owns its native options surface, e.g. `UnsplashSearchOptions`, `FlickrSearchOptions`, `OpenverseImageSearchOptions`, `MetSearchOptions`, and `PoetryDbSearchOptions`. Response-format/debug parameters and auth-only knobs are intentionally omitted when they would break refkit's normalized `Reference` contract. +### Site-scoped discovery + +Pass `sources` to restrict a search to specific provider ids (still intersected with modality matching; omit to fan out to every configured source). This is the clean way to point a web-discovery source at another site's index with a `site:` operator — without that operator leaking into the other providers' queries: + +```ts +// Query Brave's index for Xiaohongshu nail-art notes — Brave only, so the +// `site:` operator never pollutes the other configured sources' queries. +const notes = await refkit.search({ + query: 'site:xiaohongshu.com 美甲', + modalities: ['image'], + sources: ['brave'], +}) +``` + +A `sources` list that matches nothing for the requested modalities throws (a source typo fails loudly instead of reading as "no results"); an id that resolves to nothing while others still match is reported in `meta.warnings`, and every excluded provider appears in `meta.providers` with `reason: 'not-selected'`. + When an agent or UI needs to explain what happened, use `searchWithMeta`: ```ts diff --git a/packages/core/src/__tests__/client.test.ts b/packages/core/src/__tests__/client.test.ts index 2575800..b0aa324 100644 --- a/packages/core/src/__tests__/client.test.ts +++ b/packages/core/src/__tests__/client.test.ts @@ -703,4 +703,91 @@ describe('createRefkit', () => { await rk.search({ query: 'x', modalities: ['image'], providerOptions: { c: { a: 1 } } }) expect(calls).toBe(1) // second search hits the same cache entry as the first }) + + describe('sources filter', () => { + it('restricts the fan-out to the requested source ids; others are skipped as not-selected', async () => { + let bCalled = false + const a = provider('a', [ref('a-1', 'https://a/1')]) + const b = defineProvider({ + id: 'b', modalities: ['image'], queryFeatures: ['keyword'], + search: async () => { bCalled = true; return [ref('b-1', 'https://b/1')] }, + }) + const rk = createRefkit({ providers: [a, b] }) + const out = await rk.searchWithMeta({ query: 'x', modalities: ['image'], sources: ['a'] }) + expect(out.references.map(r => r.canonicalUrl)).toEqual(['https://a/1']) + expect(bCalled).toBe(false) // never queried — its query is untouched + expect(out.meta.providers.find(s => s.providerId === 'b')).toEqual({ providerId: 'b', status: 'skipped', reason: 'not-selected' }) + }) + + it('a modality miss stays unsupported-modality; only a source exclusion is not-selected', async () => { + // b matches the modality but is filtered out by sources → not-selected. + // text never matches the modality → unsupported-modality, regardless of sources. + const a = provider('a', [ref('a-1', 'https://a/1')]) + const b = provider('b', [ref('b-1', 'https://b/1')]) + const textOnly = defineProvider({ id: 'text', modalities: ['text'], queryFeatures: [], search: async () => [] }) + const rk = createRefkit({ providers: [a, b, textOnly] }) + const out = await rk.searchWithMeta({ query: 'x', modalities: ['image'], sources: ['a'] }) + const byId = Object.fromEntries(out.meta.providers.map(s => [s.providerId, s])) + expect(byId.b).toMatchObject({ status: 'skipped', reason: 'not-selected' }) + expect(byId.text).toMatchObject({ status: 'skipped', reason: 'unsupported-modality' }) + }) + + it('throws a clear Error (not AggregateError) when sources match no configured provider', async () => { + const rk = createRefkit({ providers: [provider('a', [ref('a-1', 'https://a/1')])] }) + await expect(rk.search({ query: 'x', modalities: ['image'], sources: ['nope'] })).rejects.toThrow( + 'refkit.search: no configured provider matches source id(s) [nope] for modalities [image]', + ) + await expect(rk.search({ query: 'x', modalities: ['image'], sources: ['nope'] })).rejects.not.toBeInstanceOf(AggregateError) + }) + + it('throws the source-miss error when the requested source exists but not for this modality', async () => { + const imageOnly = provider('img', [ref('img-1', 'https://img/1')]) + const textOnly = defineProvider({ id: 'txt', modalities: ['text'], queryFeatures: [], search: async () => [] }) + const rk = createRefkit({ providers: [imageOnly, textOnly] }) + // txt is registered, but scoping an image search to [txt] has an empty intersection + await expect(rk.search({ query: 'x', modalities: ['image'], sources: ['txt'] })).rejects.toThrow( + /no configured provider matches source id\(s\) \[txt\]/, + ) + }) + + it('warns about unknown source ids while still searching the ones that resolved', async () => { + const rk = createRefkit({ providers: [provider('a', [ref('a-1', 'https://a/1')])] }) + const out = await rk.searchWithMeta({ query: 'x', modalities: ['image'], sources: ['a', 'ghost'] }) + expect(out.references).toHaveLength(1) + expect(out.meta.warnings).toContain('unknown source id(s) ignored: ghost.') + }) + + it('does not warn when every requested source id resolves', async () => { + const rk = createRefkit({ providers: [provider('a', [ref('a-1', 'https://a/1')]), provider('b', [ref('b-1', 'https://b/1')])] }) + const out = await rk.searchWithMeta({ query: 'x', modalities: ['image'], sources: ['a', 'b'] }) + expect(out.meta.warnings.some(w => w.includes('unknown source'))).toBe(false) + }) + + it('coexists with the load-more cursor across a round-trip (page/seen stay global)', async () => { + const pages: Record = { + 1: [ref('a-1', 'https://a/1'), ref('a-2', 'https://a/2'), ref('a-3', 'https://a/3'), ref('a-4', 'https://a/4')], + 2: [ref('a-5', 'https://a/5')], + } + const paging = defineProvider({ + id: 'a', modalities: ['image'], capabilities: { controls: ['page'] }, + search: async (q) => pages[q.controls?.page ?? 1] ?? [], + }) + let otherCalled = false + const other = defineProvider({ + id: 'b', modalities: ['image'], queryFeatures: ['keyword'], + search: async () => { otherCalled = true; return [ref('b-1', 'https://b/1')] }, + }) + const rk = createRefkit({ providers: [paging, other] }) + + const batch1 = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 2, sources: ['a'] }) + expect(batch1.references.map(r => r.canonicalUrl)).toEqual(['https://a/1', 'https://a/2']) + expect(batch1.meta.nextCursor).toBeDefined() + + // sources is re-supplied alongside the cursor (it is not encoded in the cursor) + const batch2 = await rk.searchWithMeta({ query: 'x', modalities: ['image'], limit: 2, sources: ['a'], cursor: batch1.meta.nextCursor }) + expect(batch2.references.map(r => r.canonicalUrl)).toEqual(['https://a/3', 'https://a/4']) + expect(batch2.references.every(r => !batch1.references.some(b => b.canonicalUrl === r.canonicalUrl))).toBe(true) + expect(otherCalled).toBe(false) // b stayed excluded across both pages + }) + }) }) diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index 64afe95..572705e 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -67,7 +67,7 @@ export interface ProviderSearchStatus { returned?: number accepted?: number rejected?: number - reason?: 'unsupported-modality' + reason?: 'unsupported-modality' | 'not-selected' error?: string latencyMs?: number cached?: boolean @@ -112,6 +112,16 @@ export interface SearchResult { export interface SearchInput { query: string modalities: Modality[] + /** Restrict this search to these provider ids (intersected with modality + * matching). Omit to fan out to every configured source. Lets the caller + * scope search-engine operators (e.g. `site:xiaohongshu.com`) to a + * web-discovery source without polluting other providers' queries. + * + * A total miss — no requested id matches a configured provider for the + * requested modalities — throws (a source typo must fail loudly, not read as + * "no results"); ids that resolve to nothing while others still match are + * reported in `meta.warnings`. */ + sources?: string[] /** @deprecated Compatibility alias for `controls.color` / `controls.orientation` * / `controls.language` (controls win on conflict). Use `controls`. */ filters?: SearchFilters @@ -193,10 +203,23 @@ export function createRefkit(options: RefkitOptions): RefkitClient { if (typeof doFetch !== 'function') { throw new Error('createRefkit: no fetch available — pass options.fetch') } - const chosen = options.providers.filter(p => p.modalities.some(m => input.modalities.includes(m))) + const matchesModality = (p: ReferenceProvider) => p.modalities.some(m => input.modalities.includes(m)) + const inSources = (p: ReferenceProvider) => input.sources == null || input.sources.includes(p.id) + const chosen = options.providers.filter(p => inSources(p) && matchesModality(p)) if (chosen.length === 0) { + // A source-scoped miss is a caller typo, not "no results" — fail loudly in + // the same spirit as the empty-providers guard, rather than silently + // returning an empty set that hides the mistake. + if (input.sources != null) { + throw new Error(`refkit.search: no configured provider matches source id(s) [${input.sources.join(', ')}] for modalities [${input.modalities.join(', ')}]`) + } throw new Error(`refkit.search: no registered provider supports modalities [${input.modalities.join(', ')}]`) } + // Individual unknown ids (while others still resolved) are tolerated but + // surfaced — routed into meta.warnings below, matching the soft-signal channel. + const unknownSources = input.sources + ? input.sources.filter(id => !options.providers.some(p => p.id === id)) + : [] const limit = input.limit ?? DEFAULT_LIMIT const poolFactor = Math.max(1, Number.isFinite(input.poolFactor) ? (input.poolFactor as number) : DEFAULT_POOL_FACTOR) // Overfetch a wider candidate pool per provider, then narrow to `limit` after @@ -238,7 +261,11 @@ export function createRefkit(options: RefkitOptions): RefkitClient { } : undefined const statusByProvider = new Map() for (const p of options.providers) { - if (!chosen.includes(p)) statusByProvider.set(p.id, { providerId: p.id, status: 'skipped', reason: 'unsupported-modality' }) + if (chosen.includes(p)) continue + // Distinguish a wrong-modality skip from one caused by an explicit sources + // filter, so meta explains WHY a provider sat this search out. + const reason = matchesModality(p) ? 'not-selected' : 'unsupported-modality' + statusByProvider.set(p.id, { providerId: p.id, status: 'skipped', reason }) } const runProvider = (p: ReferenceProvider) => { @@ -359,6 +386,7 @@ export function createRefkit(options: RefkitOptions): RefkitClient { }) : undefined const warnings: string[] = [] + if (unknownSources.length > 0) warnings.push(`unknown source id(s) ignored: ${unknownSources.join(', ')}.`) const failedCount = [...pass.statusByProvider.values()].filter(s => s.status === 'failed').length if (failedCount > 0) warnings.push(`${failedCount} provider(s) failed; returning partial results.`) for (const c of pass.rightsConflicts) { diff --git a/packages/mcp/src/__tests__/mcp.test.ts b/packages/mcp/src/__tests__/mcp.test.ts index d9f9714..f4e494e 100644 --- a/packages/mcp/src/__tests__/mcp.test.ts +++ b/packages/mcp/src/__tests__/mcp.test.ts @@ -219,6 +219,40 @@ describe('@refkit/mcp', () => { await client.close() }) + it('forwards sources to core, restricting which providers are searched', async () => { + let aCalled = false + let bCalled = false + const a = defineProvider({ id: 'a', modalities: ['image'], queryFeatures: ['keyword'], search: async () => { aCalled = true; return [] } }) + const b = defineProvider({ id: 'b', modalities: ['image'], queryFeatures: ['keyword'], search: async () => { bCalled = true; return [] } }) + const server = createRefkitMcpServer(createRefkit({ providers: [a, b] })) + const [clientT, serverT] = InMemoryTransport.createLinkedPair() + const client = new Client({ name: 'test', version: '1.0.0' }) + await Promise.all([client.connect(clientT), server.connect(serverT)]) + await client.callTool({ name: 'search_references', arguments: { query: 'x', modalities: ['image'], sources: ['a'] } }) + expect(aCalled).toBe(true) + expect(bCalled).toBe(false) + await client.close() + }) + + it('advertises the enabled source ids in the sources parameter description', async () => { + const client = await connectedClient() // openverse-only server + const { tools } = await client.listTools() + const tool = tools.find(t => t.name === 'search_references')! + const properties = (tool.inputSchema as { properties: Record }).properties + expect(properties.sources?.description).toContain('openverse') + await client.close() + }) + + it('surfaces a source-selection miss as an agent-friendly tool error listing valid ids', async () => { + const client = await connectedClient() // openverse-only server + const res = await client.callTool({ name: 'search_references', arguments: { query: 'x', modalities: ['image'], sources: ['nope'] } }) + expect(res.isError).toBe(true) + const text = (res.content as Array<{ type: string; text: string }>).map(c => c.text).join('\n') + expect(text).toContain('nope') // the offending request is echoed back + expect(text).toContain('openverse') // and the valid id is surfaced + await client.close() + }) + it('returns meta and use explanations when explain is true', async () => { const good = defineProvider({ id: 'good', diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index 122388f..f6a749d 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -137,7 +137,7 @@ const searchMetaSchema: z.ZodType = z.object({ returned: z.number().optional(), accepted: z.number().optional(), rejected: z.number().optional(), - reason: z.enum(['unsupported-modality']).optional(), + reason: z.enum(['unsupported-modality', 'not-selected']).optional(), error: z.string().optional(), latencyMs: z.number().optional(), cached: z.boolean().optional(), @@ -155,6 +155,10 @@ const searchMetaSchema: z.ZodType = z.object({ /** Wrap a configured RefkitClient as an MCP server exposing `search_references`. */ export function createRefkitMcpServer(refkit: RefkitClient): McpServer { const server = new McpServer({ name: 'refkit', version: VERSION }) + // Enumerated in the `sources` param description so an agent knows the valid ids + // up front (and reused to enrich a source-miss error). Providers are fixed for + // the server's lifetime, so this snapshot never drifts. + const enabledSourceIds = refkit.providers.map(p => p.id) server.registerTool( 'search_references', @@ -168,6 +172,11 @@ export function createRefkitMcpServer(refkit: RefkitClient): McpServer { inputSchema: { query: z.string().describe('what to search for, e.g. "cyberpunk alley at night"'), modalities: z.array(z.enum(MODALITIES)).optional().describe('default ["image"]'), + sources: z.array(z.string()).optional().describe( + `restrict the search to specific sources by id (omit to search every configured source).${ + enabledSourceIds.length ? ` Enabled source ids: ${enabledSourceIds.join(', ')}.` : '' + } Use to scope a search-engine operator (e.g. "site:example.com") to a web-discovery source without affecting other sources' queries.`, + ), filters: filtersSchema.optional().describe('compatibility alias for controls.orientation, controls.color, and controls.language'), controls: searchControlsSchema.optional().describe('provider-neutral search controls; providers translate supported controls and report ignored controls in explain metadata'), providerOptions: providerOptionsSchema.optional().describe('provider-specific search controls keyed by provider id; each provider whitelists supported keys'), @@ -184,10 +193,11 @@ export function createRefkitMcpServer(refkit: RefkitClient): McpServer { meta: searchMetaSchema.optional(), }, }, - async ({ query, modalities, filters, controls, providerOptions, explain, limit, cursor, rerank, intent, gateFor }) => { + async ({ query, modalities, filters, controls, providerOptions, explain, limit, cursor, rerank, intent, gateFor, sources }) => { const searchInput = { query, modalities: modalities ?? ['image'], + sources, filters: filters as SearchFilters | undefined, controls: controls as SearchControls | undefined, providerOptions: providerOptions as ProviderOptionsById | undefined, @@ -198,7 +208,24 @@ export function createRefkitMcpServer(refkit: RefkitClient): McpServer { } // Always searchWithMeta: the continuation token (meta.nextCursor) must not // depend on the explain diagnostics flag — only the meta DUMP is gated. - const result = await refkit.searchWithMeta(searchInput) + let result + try { + result = await refkit.searchWithMeta(searchInput) + } catch (err) { + // A source-selection miss (no requested id resolves for the modality) is a + // caller mistake, not an outage — turn core's throw into an agent-actionable + // tool error listing the valid ids. AggregateError (every chosen provider + // failed at fetch) is a genuine upstream fault, so let it propagate. + if (sources && sources.length > 0 && !(err instanceof AggregateError)) { + const available = enabledSourceIds.join(', ') || '(none)' + const detail = err instanceof Error ? err.message : String(err) + return { + isError: true, + content: [{ type: 'text', text: `${detail} Enabled source ids: ${available}.` }], + } + } + throw err + } const refs = result.references const assessIntent = intent ?? gateFor const references = refs.map(r =>