Upgrade js-tiktoken, use standard interface (#500)

js-tiktoken 1.0.8 broke the import we were using to pull in just the
cl100k_base encoding. Switch to the standard `getEncoding` import
instead.

packages/ai-jsx/package.json

@@ -4,7 +4,7 @@
   "repository": "fixie-ai/ai-jsx",
   "bugs": "https://github.com/fixie-ai/ai-jsx/issues",
   "homepage": "https://ai-jsx.com",
-  "version": "0.26.0",
+  "version": "0.26.1",
   "volta": {
     "extends": "../../package.json"
   },
@@ -385,7 +385,7 @@
     "cli-highlight": "^2.1.11",
     "cli-spinners": "^2.9.0",
     "ink": "^4.2.0",
-    "js-tiktoken": "^1.0.7",
+    "js-tiktoken": "^1.0.8",
     "js-yaml": "^4.1.0",
     "langchain": "^0.0.143",
     "lodash": "^4.17.21",

packages/ai-jsx/src/lib/openai.tsx

@@ -22,8 +22,7 @@ import { OpenAI as OpenAIClient } from 'openai';
 export { OpenAI as OpenAIClient } from 'openai';
 import { FinalRequestOptions } from 'openai/core';
 import { debugRepresentation } from '../core/debug.js';
-import cl100k_base from 'js-tiktoken/ranks/cl100k_base';
-import { Tiktoken } from 'js-tiktoken/lite';
+import { getEncoding } from 'js-tiktoken';
 import _ from 'lodash';
 
 // https://platform.openai.com/docs/models/model-endpoint-compatibility
@@ -136,7 +135,7 @@ export function OpenAI({
 }
 
 // Preload the tokenizer to avoid a large delay on first use.
-const cl100kTokenizer = new Tiktoken(cl100k_base);
+const cl100kTokenizer = getEncoding('cl100k_base');
 export const tokenizer = {
   encode: (text: string) => cl100kTokenizer.encode(text),
   decode: (tokens: number[]) => cl100kTokenizer.decode(tokens),

packages/docs/docs/changelog.md

@@ -1,6 +1,10 @@
 # Changelog
 
-## 0.26.0
+## 0.26.1
+
+- Fix `js-tiktoken` import that fails on 1.0.8.
+
+## [0.26.0](https://github.com/fixie-ai/ai-jsx/commit/c5501a1d1016a3cb27259a79bc38b8d68942f761)
 
 - In the `Sidekick` component:
   - Remove the MDX repair attempt.

yarn.lock

@@ -7306,7 +7306,7 @@ __metadata:
     globby: ^13.1.4
     ink: ^4.2.0
     jest: ^29.5.0
-    js-tiktoken: ^1.0.7
+    js-tiktoken: ^1.0.8
     js-yaml: ^4.1.0
     langchain: ^0.0.143
     load-json-file: ^7.0.1
@@ -16413,6 +16413,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"js-tiktoken@npm:^1.0.8":
+  version: 1.0.8
+  resolution: "js-tiktoken@npm:1.0.8"
+  dependencies:
+    base64-js: ^1.5.1
+  checksum: ac6e666f14661b4e744bd60987e35275668863a686413bb35baca2d9a503bc95fcfa907f3c02b8e8abe4a4a70abe622c06cbb72ce3574b3b929d22f9146c3f85
+  languageName: node
+  linkType: hard
+
 "js-tokens@npm:^3.0.0 || ^4.0.0, js-tokens@npm:^4.0.0":
   version: 4.0.0
   resolution: "js-tokens@npm:4.0.0"