Skip to content

Commit 720ae2f

Browse files
committed
test(knowledge): add credential discovery tests + update QueryRouter docs
Add 9 unit tests validating that platform-corpus.json contains all FAQ entries for agentic credential setup (Gmail, extension credentials, file discovery, wunderland connect) and that they are reachable through KeywordFallback search. Update QUERY_ROUTER.md with agentic credential discovery documentation.
1 parent b5fd83c commit 720ae2f

2 files changed

Lines changed: 128 additions & 0 deletions

File tree

docs/QUERY_ROUTER.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,18 @@ User project docs (your ./docs, ./guides, etc.)
111111

112112
This means an agent can answer questions like "What vector stores does AgentOS support?" or "How do I set up a Bluesky channel?" without any project-specific documentation — the answer comes from the bundled platform knowledge.
113113

114+
### Agentic Credential Discovery
115+
116+
A key application of platform knowledge is **agentic credential setup**. The corpus includes dedicated FAQ entries for:
117+
118+
- **Gmail setup** (`faq:setup-gmail`): Step-by-step instructions for creating a Google Cloud project, enabling the Gmail API, downloading the `client_secret_*.json` file, and running the OAuth flow via `wunderland connect gmail --credentials`.
119+
- **General credential setup** (`faq:setup-credentials-general`): The universal pattern for helping users configure any extension — discover what's needed via `discover_capabilities`, find files via `shell_execute`, parse them via `file_read`, and persist credentials.
120+
- **Extension credentials reference** (`faq:extension-credentials`): Complete listing of what environment variables each extension requires (GITHUB_TOKEN, DISCORD_BOT_TOKEN, ELEVENLABS_API_KEY, etc.).
121+
- **File discovery** (`faq:find-credential-files`): How to locate downloaded credential files on the user's system (checking ~/Downloads, ~/Desktop, ~/.aws, ~/.ssh).
122+
- **wunderland connect reference** (`faq:wunderland-connect`): Command reference for the OAuth-based connect flow with all supported services and the `--credentials` flag.
123+
124+
When a user asks "help me set up Gmail" or "I downloaded a credentials file", the NL intent classifier routes to the connect flow, and the agent uses these knowledge entries combined with agentic tools (`shell_execute`, `file_read`) to guide the user through credential setup without any hard-coded logic in the CLI itself.
125+
114126
### Configuration
115127

116128
Platform knowledge is enabled by default. To disable it:
Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
/**
2+
* @fileoverview Tests for credential setup knowledge in the platform corpus.
3+
*
4+
* Validates that the bundled platform-corpus.json contains all FAQ entries
5+
* required for the agentic credential discovery flow (Gmail setup, extension
6+
* credentials, file discovery, wunderland connect reference) and that these
7+
* entries are surfaced through the KeywordFallback search engine.
8+
*
9+
* @module @framers/agentos/query-router/__tests__/credential-knowledge
10+
*/
11+
12+
import { describe, it, expect } from 'vitest';
13+
import { readFileSync } from 'node:fs';
14+
import { resolve } from 'node:path';
15+
import { KeywordFallback } from '../KeywordFallback.js';
16+
17+
// ---------------------------------------------------------------------------
18+
// Corpus loading
19+
// ---------------------------------------------------------------------------
20+
21+
/** Path to the bundled platform knowledge corpus. */
22+
const corpusPath = resolve(__dirname, '../../../knowledge/platform-corpus.json');
23+
24+
/** Parsed platform corpus entries. */
25+
const corpus: Array<{ id: string; heading: string; content: string; category: string }> =
26+
JSON.parse(readFileSync(corpusPath, 'utf-8'));
27+
28+
// ---------------------------------------------------------------------------
29+
// Helpers
30+
// ---------------------------------------------------------------------------
31+
32+
/**
33+
* Converts raw corpus entries into CorpusChunk-shaped objects suitable for
34+
* KeywordFallback. The sourcePath follows the convention `platform:<category>/<id>`.
35+
*/
36+
function toChunks() {
37+
return corpus.map((e) => ({
38+
id: e.id,
39+
heading: e.heading,
40+
content: e.content,
41+
sourcePath: `platform:${e.category}/${e.id}`,
42+
}));
43+
}
44+
45+
// ---------------------------------------------------------------------------
46+
// Tests — FAQ entry existence
47+
// ---------------------------------------------------------------------------
48+
49+
describe('Credential setup knowledge in platform corpus', () => {
50+
it('contains Gmail setup FAQ', () => {
51+
const gmail = corpus.find((e) => e.id === 'faq:setup-gmail');
52+
expect(gmail).toBeDefined();
53+
expect(gmail!.content).toContain('client_secret');
54+
expect(gmail!.content).toContain('GOOGLE_CLIENT_ID');
55+
});
56+
57+
it('contains general credential setup FAQ', () => {
58+
const general = corpus.find((e) => e.id === 'faq:setup-credentials-general');
59+
expect(general).toBeDefined();
60+
expect(general!.content).toContain('discover_capabilities');
61+
expect(general!.content).toContain('shell_execute');
62+
expect(general!.content).toContain('file_read');
63+
});
64+
65+
it('contains extension credentials reference', () => {
66+
const creds = corpus.find((e) => e.id === 'faq:extension-credentials');
67+
expect(creds).toBeDefined();
68+
expect(creds!.content).toContain('GITHUB_TOKEN');
69+
expect(creds!.content).toContain('DISCORD_BOT_TOKEN');
70+
expect(creds!.content).toContain('ELEVENLABS_API_KEY');
71+
});
72+
73+
it('contains file discovery FAQ', () => {
74+
const find = corpus.find((e) => e.id === 'faq:find-credential-files');
75+
expect(find).toBeDefined();
76+
expect(find!.content).toContain('Downloads');
77+
expect(find!.content).toContain('shell_execute');
78+
});
79+
80+
it('contains wunderland connect reference', () => {
81+
const connect = corpus.find((e) => e.id === 'faq:wunderland-connect');
82+
expect(connect).toBeDefined();
83+
expect(connect!.content).toContain('--credentials');
84+
});
85+
86+
// -------------------------------------------------------------------------
87+
// Tests — keyword search reachability
88+
// -------------------------------------------------------------------------
89+
90+
it('keyword search for "gmail setup" finds the Gmail FAQ', () => {
91+
const fallback = new KeywordFallback(toChunks());
92+
const results = fallback.search('gmail setup credentials', 5);
93+
expect(results.length).toBeGreaterThan(0);
94+
expect(results.some((r) => r.id.includes('gmail'))).toBe(true);
95+
});
96+
97+
it('keyword search for "API keys needed" finds extension credentials', () => {
98+
const fallback = new KeywordFallback(toChunks());
99+
const results = fallback.search('what API keys credentials needed', 5);
100+
expect(results.length).toBeGreaterThan(0);
101+
});
102+
103+
it('keyword search for "connect gmail" finds wunderland connect reference', () => {
104+
const fallback = new KeywordFallback(toChunks());
105+
const results = fallback.search('connect gmail wunderland', 5);
106+
expect(results.length).toBeGreaterThan(0);
107+
expect(results.some((r) => r.id.includes('connect') || r.id.includes('gmail'))).toBe(true);
108+
});
109+
110+
it('keyword search for "find credential files" finds the file discovery FAQ', () => {
111+
const fallback = new KeywordFallback(toChunks());
112+
const results = fallback.search('find credential files downloads', 5);
113+
expect(results.length).toBeGreaterThan(0);
114+
expect(results.some((r) => r.id.includes('credential') || r.id.includes('find'))).toBe(true);
115+
});
116+
});

0 commit comments

Comments
 (0)