knowpro improvements

ts/examples/chat/README.md

@@ -48,6 +48,16 @@ You can list all commands matching a prefix by typing the prefix: e.g. @kpSearch
 @kpAnswer --query "List the names of all books"
 ```
 
+### Batching commands
+
+You can place commands in a text file and run them as a batch by using the **batch** command.
+
+```
+@batch --filePath <>
+```
+
+Type batch --? for options.
+
 ### Notes
 
 - Requires file system access. Creates test directories under **/data/testChat**

ts/examples/chat/src/memory/knowproMemory.ts

@@ -111,6 +111,7 @@ export async function createKnowproCommands(
     commands.kpTopics = topics;
     commands.kpMessages = showMessages;
     commands.kpAbstractMessage = abstract;
+    commands.kpRelatedTerms = addRelatedTerm;
 
     /*----------------
      * COMMANDS
@@ -581,6 +582,45 @@ export async function createKnowproCommands(
         }
     }
 
+    function addRelatedTermDef(): CommandMetadata {
+        return {
+            description: "Add an alias",
+            args: {
+                term: arg("Value for which to add an alias"),
+            },
+            options: {
+                relatedTerm: arg("Alias to add"),
+                weight: argNum("Relationship weight", 0.9),
+            },
+        };
+    }
+    commands.kpRelatedTerms.metadata = addRelatedTermDef();
+    async function addRelatedTerm(args: string[]) {
+        if (!ensureConversationLoaded()) {
+            return;
+        }
+        const namedArgs = parseNamedArguments(args, addRelatedTermDef());
+        const aliases =
+            context.conversation!.secondaryIndexes?.termToRelatedTermsIndex
+                ?.aliases;
+        if (!aliases) {
+            context.printer.writeLine(
+                "No aliases available on this conversation",
+            );
+            return;
+        }
+        if (aliases instanceof kp.TermToRelatedTermsMap) {
+            if (namedArgs.relatedTerm) {
+                const relatedTerm: kp.Term = { text: namedArgs.relatedTerm };
+                aliases.addRelatedTerm(namedArgs.term, relatedTerm);
+            }
+        }
+        const relatedTerms = aliases.lookupTerm(namedArgs.term);
+        if (relatedTerms && relatedTerms.length > 0) {
+            context.printer.writeList(relatedTerms.map((rt) => rt.text));
+        }
+    }
+
     /*---------- 
       End COMMANDS
     ------------*/

ts/packages/aiclient/src/models.ts

@@ -125,3 +125,15 @@ export type GeneratedImage = {
     revised_prompt: string;
     image_url: string;
 };
+
+export type EmbeddingModelMetadata = {
+    modelName?: string | undefined;
+    embeddingSize: number;
+};
+
+export function modelMetadata_ada002(): EmbeddingModelMetadata {
+    return {
+        modelName: "ada-002",
+        embeddingSize: 1536,
+    };
+}

ts/packages/interactiveApp/src/interactiveApp.ts

@@ -828,7 +828,7 @@ export function addBatchHandler(app: InteractiveApp) {
 
     function batchDef(): CommandMetadata {
         return {
-            description: "Run a batch file",
+            description: "Run a batch file of commands",
             args: {
                 filePath: {
                     description: "Batch file path.",

ts/packages/knowPro/src/answerGenerator.ts

@@ -126,16 +126,81 @@ export type AnswerGeneratorSettings = {
 
 /**
  * Generate a natural language answer for question about a conversation using the provided search results as context
+ *  - Each search result is first turned into an answer individually
+ *  - If more than one search result provided, then individual answers are combined into a single answer
  * If the context exceeds the generator.setting.maxCharsInBudget, will break up the context into
  * chunks, run them in parallel, and then merge the answers found in individual chunks
  * @param conversation conversation about which this is a question
- * @param generator answer generator to use
+ * @param generator answer generator to use to turn search results onto language answers: @see AnswerGenerator
  * @param question question that was asked
- * @param searchResult the results of running a search query for the question on the conversation
+ * @param searchResults the results of running a search query for the question on the conversation
  * @param progress Progress callback
  * @returns Answers
  */
 export async function generateAnswer(
+    conversation: IConversation,
+    generator: IAnswerGenerator,
+    question: string,
+    searchResults: ConversationSearchResult | ConversationSearchResult[],
+    progress?: asyncArray.ProcessProgress<
+        contextSchema.AnswerContext,
+        Result<answerSchema.AnswerResponse>
+    >,
+    contextOptions?: AnswerContextOptions,
+): Promise<Result<answerSchema.AnswerResponse>> {
+    let answerResponse: Result<answerSchema.AnswerResponse>;
+    if (!Array.isArray(searchResults)) {
+        answerResponse = await generateAnswerFromSearchResult(
+            conversation,
+            generator,
+            question,
+            searchResults,
+            progress,
+        );
+    } else {
+        if (searchResults.length === 0) {
+            return error("No search results");
+        }
+        if (searchResults.length === 1) {
+            answerResponse = await generateAnswerFromSearchResult(
+                conversation,
+                generator,
+                question,
+                searchResults[0],
+                progress,
+            );
+        } else {
+            // Get answers for individual searches in parallel
+            const partialResults = await asyncArray.mapAsync(
+                searchResults,
+                generator.settings.concurrency,
+                (sr) =>
+                    generateAnswerFromSearchResult(
+                        conversation,
+                        generator,
+                        question,
+                        sr,
+                        progress,
+                    ),
+            );
+            // Use partial responses to build a complete answer
+            const partialResponses: answerSchema.AnswerResponse[] = [];
+            for (const result of partialResults) {
+                if (!result.success) {
+                    return result;
+                }
+                partialResponses.push(result.data);
+            }
+            answerResponse = await generator.combinePartialAnswers(
+                question,
+                partialResponses,
+            );
+        }
+    }
+    return answerResponse;
+}
+
+async function generateAnswerFromSearchResult(
     conversation: IConversation,
     generator: IAnswerGenerator,
     question: string,

ts/packages/knowPro/src/serialization.ts

@@ -11,13 +11,15 @@ import {
 import { deserializeEmbeddings, serializeEmbeddings } from "./fuzzyIndex.js";
 import path from "path";
 import { IConversationDataWithIndexes } from "./secondaryIndexes.js";
+import { EmbeddingModelMetadata, modelMetadata_ada002 } from "aiclient";
 
 export async function writeConversationDataToFile(
     conversationData: IConversationDataWithIndexes,
     dirPath: string,
     baseFileName: string,
+    modelMeta?: EmbeddingModelMetadata,
 ): Promise<void> {
-    const fileData = toConversationFileData(conversationData);
+    const fileData = toConversationFileData(conversationData, modelMeta);
     if (fileData.binaryData) {
         if (
             fileData.binaryData.embeddings &&
@@ -41,28 +43,29 @@ export async function writeConversationDataToFile(
 export async function readConversationDataFromFile(
     dirPath: string,
     baseFileName: string,
-    embeddingSize: number | undefined,
+    embeddingSize?: number,
 ): Promise<IConversationDataWithIndexes | undefined> {
     const jsonData = await readJsonFile<ConversationJsonData>(
         path.join(dirPath, baseFileName + DataFileSuffix),
     );
     if (!jsonData) {
         return undefined;
     }
+    let fileData: ConversationFileData = {
+        jsonData: jsonData,
+        binaryData: {},
+    };
+    validateFileData(fileData, embeddingSize);
     let embeddings: Float32Array[] | undefined;
     if (embeddingSize && embeddingSize > 0) {
         const embeddingsBuffer = await readFile(
             path.join(dirPath, baseFileName + EmbeddingFileSuffix),
         );
         if (embeddingsBuffer) {
             embeddings = deserializeEmbeddings(embeddingsBuffer, embeddingSize);
+            fileData.binaryData.embeddings = embeddings;
         }
     }
-    let fileData: ConversationFileData = {
-        jsonData: jsonData,
-        binaryData: { embeddings },
-    };
-    validateFileData(fileData);
     fileData.jsonData.fileHeader ??= createFileHeader();
     return fromConversationFileData(fileData);
 }
@@ -115,6 +118,8 @@ type FileHeader = {
 type EmbeddingFileHeader = {
     relatedCount?: number | undefined;
     messageCount?: number | undefined;
+    // The V 0.1 file format requires that all embeddings are the same size
+    modelMetadata?: EmbeddingModelMetadata | undefined;
 };
 
 type EmbeddingData = {
@@ -131,14 +136,28 @@ type ConversationBinaryData = {
     embeddings?: Float32Array[] | undefined;
 };
 
-function validateFileData(fileData: ConversationFileData): void {
+function validateFileData(
+    fileData: ConversationFileData,
+    expectedEmbeddingSize?: number | undefined,
+): void {
     if (fileData.jsonData === undefined) {
         throw new Error(`${Error_FileCorrupt}: Missing json data`);
     }
+    if (expectedEmbeddingSize && fileData.jsonData.embeddingFileHeader) {
+        const actualEmbeddingSize =
+            fileData.jsonData.embeddingFileHeader.modelMetadata
+                ?.embeddingSize ?? modelMetadata_ada002().embeddingSize;
+        if (expectedEmbeddingSize !== actualEmbeddingSize) {
+            throw new Error(
+                `File has embeddings of size ${expectedEmbeddingSize}`,
+            );
+        }
+    }
 }
 
 function toConversationFileData(
     conversationData: IConversationDataWithIndexes,
+    modelMeta?: EmbeddingModelMetadata,
 ): ConversationFileData {
     let fileData: ConversationFileData = {
         jsonData: {
@@ -157,7 +176,14 @@ function toConversationFileData(
         fileData.binaryData,
         conversationData.messageIndexData?.indexData,
     );
-
+    const embeddingSize = checkEmbeddingSize(
+        fileData,
+        modelMeta?.embeddingSize,
+    );
+    modelMeta ??= { embeddingSize };
+    if (modelMeta !== undefined) {
+        embeddingFileHeader.modelMetadata = modelMeta;
+    }
     return fileData;
 }
 
@@ -219,6 +245,7 @@ function getEmbeddingsFromBinaryData(
     embeddingData: EmbeddingData | undefined,
     startAt: number,
     length?: number | undefined,
+    modelMetadata?: EmbeddingModelMetadata,
 ): number {
     if (binaryData.embeddings && embeddingData && length && length > 0) {
         embeddingData.embeddings = binaryData.embeddings.slice(
@@ -230,7 +257,30 @@ function getEmbeddingsFromBinaryData(
                 `${Error_FileCorrupt}: expected ${length}, got ${embeddingData.embeddings.length}`,
             );
         }
+        if (modelMetadata) {
+        }
         return length;
     }
     return 0;
 }
+
+function checkEmbeddingSize(
+    fileData: ConversationFileData,
+    embeddingSize?: number,
+): number {
+    if (fileData.binaryData) {
+        const embeddings = fileData.binaryData.embeddings;
+        if (embeddings && embeddings.length > 0) {
+            embeddingSize ??= embeddings[0].length;
+            for (let i = 1; i < embeddings.length; ++i) {
+                if (embeddingSize !== embeddings[i].length) {
+                    throw new Error(
+                        `Embeddings not of same size ${embeddingSize}`,
+                    );
+                }
+            }
+            return embeddingSize;
+        }
+    }
+    return 0;
+}