Migrate FixieCorpus to the new (beta) corpus API. (#246)

The new corpus service is currently only deployed on beta.fixie.ai,
however, it should be more reliable and useful than the existing
app.fixie.ai API. Hence this PR updates AI.JSX's `FixieCorpus` component
to use the new API surface. I've also added a little demo program for
the component.

If we want to retain support for the old API, I'm happy to rename this
component to something like `FixieCorpusNew` or add a flag or something
to use the new API instead - feedback welcome!

---------

Co-authored-by: Nick Heiner <NickHeiner@users.noreply.github.com>

.hgignore

@@ -1,2 +1,3 @@
 node_modules
 *.map
+build

packages/ai-jsx/package.json

@@ -4,7 +4,7 @@
   "repository": "fixie-ai/ai-jsx",
   "bugs": "https://github.com/fixie-ai/ai-jsx/issues",
   "homepage": "https://ai-jsx.com",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "volta": {
     "extends": "../../package.json"
   },

packages/ai-jsx/src/batteries/docs.tsx

@@ -616,7 +616,7 @@ export class LocalCorpus<
 
 /** A fully mananged {@link Corpus} served by Fixie. */
 export class FixieCorpus<ChunkMetadata extends Jsonifiable = Jsonifiable> implements Corpus<ChunkMetadata> {
-  private static readonly DEFAULT_FIXIE_API_URL = 'https://app.fixie.ai/api';
+  private static readonly DEFAULT_FIXIE_API_URL = 'https://beta.fixie.ai/api/v1';
 
   private readonly fixieApiUrl: string;
 
@@ -625,7 +625,7 @@ export class FixieCorpus<ChunkMetadata extends Jsonifiable = Jsonifiable> implem
       this.fixieApiKey = getEnvVar('FIXIE_API_KEY', false);
       if (!this.fixieApiKey) {
         throw new AIJSXError(
-          'You must provide a Fixie API key to access Fixie corpora. Find yours at https://app.fixie.ai/profile.',
+          'You must provide a Fixie API key to access Fixie corpora. Find yours at https://beta.fixie.ai/profile.',
           ErrorCode.MissingFixieAPIKey,
           'user'
         );
@@ -635,27 +635,30 @@ export class FixieCorpus<ChunkMetadata extends Jsonifiable = Jsonifiable> implem
   }
 
   async search(query: string, params?: { limit?: number }): Promise<ScoredChunk<ChunkMetadata>[]> {
-    const response = await fetch(`${this.fixieApiUrl}/corpora/${this.corpusId}:query`, {
+    const url = `${this.fixieApiUrl}/corpora/${this.corpusId}:query`;
+    const response = await fetch(url, {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
         Authorization: `Bearer ${this.fixieApiKey}`,
       },
-      body: JSON.stringify({ query_string: query, chunk_limit: params?.limit }),
+      body: JSON.stringify({
+        corpus_id: this.corpusId,
+        query,
+        page_size: params?.limit ?? 10,
+      }),
     });
     if (response.status !== 200) {
       throw new AIJSXError(
-        `Fixie API returned status ${response.status}: ${await response.text()}`,
+        `Fixie API call to ${url} returned status ${response.status}: ${await response.text()}`,
         ErrorCode.FixieStatusNotOk,
         'runtime'
       );
     }
-    const apiResults = await response.json();
-    return apiResults.chunks.map((result: any) => ({
+    const apiResult = await response.json();
+    return apiResult.results.map((result: any) => ({
       chunk: {
-        content: result.content,
-        metadata: result.metadata,
-        documentName: result.document_name,
+        content: result.snippet,
       },
       score: result.score,
     }));

packages/docs/docs/changelog.md

@@ -1,6 +1,12 @@
 # Changelog
 
-## 0.10.0
+## 0.11.0
+
+- Updated the `<FixieCorpus>` component to use the new Fixie Corpus REST API.
+  This is currently only available to users on `beta.fixie.ai` but will be brought
+  to `app.fixie.ai` soon.
+
+## [0.10.0](https://github.com/fixie-ai/ai-jsx/commit/b758fe62c2d4c645e3a6271772d78116a97bc64a)
 
 - Memoized streaming elements no longer replay their entire stream with every render. Instead, they start with the last rendered frame.
 - Elements returned by partial rendering are automatically memoized to ensure they only render once.

packages/docs/docs/contributing/index.md

@@ -60,10 +60,18 @@ Follow these steps to make a new demo.
 To publish:
 
 1. During your PR:
+
    1. Make sure the `version` field in `packages/ai-jsx/package.json` has been incremented in accordance with [semver](https://semver.org/).
-   1. Update the [changelog](../changelog.md). Make sure to find the commit for the currently published version and add that URL to the version number.
-1. After your PR is merged:
+   1. Update the [changelog](../changelog.md). In addition to adding a section describing
+      the changes in the new version, be sure to update the heading for the previous version
+      with a link to the commit for that version, for example:
+
+   ```
+   ## [0.9.2](https://github.com/fixie-ai/ai-jsx/commit/219aebeb5e062bf3470a239443626915e0503ad9)
+   ```
+
+2. After your PR is merged:
    1. `cd packages/ai-jsx`
-   1. `npm publish`
+   2. `npm publish`
 
 This will create a bunch of temp files in your current working directory. To remove them, run `git clean -fd`.

packages/examples/package.json

@@ -11,6 +11,7 @@
     "@types/lodash": "^4.14.195",
     "@types/node": "^20.3.1",
     "@types/prompt-sync": "^4.2.0",
+    "@types/promptly": "^3.0.2",
     "@types/uuid": "^9.0.2",
     "@typescript-eslint/eslint-plugin": "^5.60.0",
     "@typescript-eslint/parser": "^5.60.0",
@@ -61,6 +62,7 @@
     "demo:validated-generation": "yarn build && node dist/src/validated-generation.js",
     "demo:image-generation": "yarn build && node dist/src/image-generation.js",
     "demo:shrink": "yarn build && node dist/src/conversation-shrinking.js",
+    "demo:fixie-corpus": "yarn build && node dist/src/fixie-corpus.js",
     "view-logs": "cat ai-jsx.log | pino-pretty",
     "lint": "eslint . --max-warnings 0",
     "lint:fix": "eslint . --fix",
@@ -96,6 +98,7 @@
     "pino": "^8.14.1",
     "pino-pretty": "^10.0.0",
     "prompt-sync": "^4.2.0",
+    "promptly": "^3.2.0",
     "react": "^18.2.0",
     "type-fest": "^3.12.0",
     "uuid": "^9.0.0",

packages/examples/src/fixie-corpus.tsx

@@ -0,0 +1,32 @@
+/**
+ * This is a simple demo showing how to query a corpus of documents through the Fixie service.
+ * The app only sends a single query to a corpus, and dumps out the raw text of the top 4 chunks.
+ */
+
+import { ScoredChunk, FixieCorpus } from 'ai-jsx/batteries/docs';
+import { showInspector } from 'ai-jsx/core/inspector';
+
+import promptly from 'promptly';
+const { prompt } = promptly;
+
+function ChunkFormatter({ doc }: { doc: ScoredChunk<any> }) {
+  return (
+    <>
+      {'\n'}```chunk{'\n'}
+      {doc.chunk.content.replaceAll('```', '\\`\\`\\`')}
+      {'\n'}```{'\n'}
+    </>
+  );
+}
+
+async function App({ corpusId }: { corpusId: string }) {
+  const corpus = new FixieCorpus(corpusId);
+
+  const query = 'How do I write a chatbot in AI.JSX?';
+  const results = await corpus.search(query, { limit: 4 });
+
+  return results.map((chunk) => <ChunkFormatter doc={chunk} />);
+}
+
+const corpusId = await prompt('Fixie Corpus ID: ');
+showInspector(<App corpusId={corpusId} />);

packages/fixie/.eslintrc.cjs

@@ -4,19 +4,24 @@ module.exports = {
   extends: ['eslint:recommended', 'plugin:@typescript-eslint/strict', 'nth'],
   parser: '@typescript-eslint/parser',
   parserOptions: {
-    project: [path.join(__dirname, 'tsconfig.json')],
+    project: [
+      path.join(__dirname, 'tsconfig.json'),
+      path.join(__dirname, 'scripts', 'tsconfig.json'),
+      path.join(__dirname, 'test', 'tsconfig.json'),
+    ],
   },
   plugins: ['@typescript-eslint'],
   root: true,
 
   env: {
     node: true,
+    es6: true,
   },
 
   rules: {
+    // Disable eslint rules to let their TS equivalents take over.
     'no-unused-vars': 'off',
     '@typescript-eslint/no-unused-vars': ['warn', { ignoreRestSiblings: true, argsIgnorePattern: '^_' }],
-
     'no-undef': 'off',
     'no-magic-numbers': 'off',
     '@typescript-eslint/no-magic-numbers': 'off',
@@ -27,8 +32,9 @@ module.exports = {
     'no-use-before-define': 'off',
     '@typescript-eslint/no-use-before-define': ['error', { functions: false, variables: true }],
 
-    'no-trailing-spaces': 'off',
+    'no-trailing-spaces': 'warn',
     'no-else-return': ['warn', { allowElseIf: false }],
+    'no-constant-condition': ['error', { checkLoops: false }],
 
     // Disable style rules to let prettier own it
     'object-curly-spacing': 'off',
@@ -41,6 +47,7 @@ module.exports = {
     'generator-star-spacing': 'off',
     'space-before-function-paren': 'off',
     'jsx-quotes': 'off',
+    'brace-style': 'off',
 
     // Add additional strictness beyond the recommended set
     '@typescript-eslint/parameter-properties': ['warn', { prefer: 'parameter-properties' }],

yarn.lock

@@ -5940,6 +5940,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/promptly@npm:^3.0.2":
+  version: 3.0.2
+  resolution: "@types/promptly@npm:3.0.2"
+  dependencies:
+    "@types/node": "*"
+  checksum: dd431c6a7a9d0306254a4435eb7cf9526689846c8b65920c1f56bb79f47e94d0498a95bfc9c7df6002a6bcfb5c4b527b0710e408b9097e8c19c92ea79970bc13
+  languageName: node
+  linkType: hard
+
 "@types/prop-types@npm:*":
   version: 15.7.5
   resolution: "@types/prop-types@npm:15.7.5"
@@ -11579,6 +11588,7 @@ __metadata:
     "@types/lodash": ^4.14.195
     "@types/node": ^20.3.1
     "@types/prompt-sync": ^4.2.0
+    "@types/promptly": ^3.0.2
     "@types/uuid": ^9.0.2
     "@typescript-eslint/eslint-plugin": ^5.60.0
     "@typescript-eslint/parser": ^5.60.0
@@ -11605,6 +11615,7 @@ __metadata:
     pino: ^8.14.1
     pino-pretty: ^10.0.0
     prompt-sync: ^4.2.0
+    promptly: ^3.2.0
     react: ^18.2.0
     ts-jest: ^29.1.0
     ts-node: ^10.9.1
@@ -17602,6 +17613,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"mute-stream@npm:~0.0.4":
+  version: 0.0.8
+  resolution: "mute-stream@npm:0.0.8"
+  checksum: ff48d251fc3f827e5b1206cda0ffdaec885e56057ee86a3155e1951bc940fd5f33531774b1cc8414d7668c10a8907f863f6561875ee6e8768931a62121a531a1
+  languageName: node
+  linkType: hard
+
 "mz@npm:^2.4.0, mz@npm:^2.7.0":
   version: 2.7.0
   resolution: "mz@npm:2.7.0"
@@ -19970,6 +19988,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"promptly@npm:^3.2.0":
+  version: 3.2.0
+  resolution: "promptly@npm:3.2.0"
+  dependencies:
+    read: ^1.0.4
+  checksum: 8bf4f346726c36ac96fbc0f52d810172b53dc61607ed884fa78bb7b57164070900a72e6525a480721d339001615450c37656caba37e4e7f1f9ec9b2c0bda7e19
+  languageName: node
+  linkType: hard
+
 "prompts@npm:^2.0.1, prompts@npm:^2.4.2":
   version: 2.4.2
   resolution: "prompts@npm:2.4.2"
@@ -20564,6 +20591,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"read@npm:^1.0.4":
+  version: 1.0.7
+  resolution: "read@npm:1.0.7"
+  dependencies:
+    mute-stream: ~0.0.4
+  checksum: 2777c254e5732cac96f5d0a1c0f6b836c89ae23d8febd405b206f6f24d5de1873420f1a0795e0e3721066650d19adf802c7882c4027143ee0acf942a4f34f97b
+  languageName: node
+  linkType: hard
+
 "readable-stream@npm:^2.0.1":
   version: 2.3.8
   resolution: "readable-stream@npm:2.3.8"