diff --git a/ts/examples/chat/README.md b/ts/examples/chat/README.md index 58c6f3a90..63825d6db 100644 --- a/ts/examples/chat/README.md +++ b/ts/examples/chat/README.md @@ -48,6 +48,16 @@ You can list all commands matching a prefix by typing the prefix: e.g. @kpSearch @kpAnswer --query "List the names of all books" ``` +### Batching commands + +You can place commands in a text file and run them as a batch by using the **batch** command. + +``` +@batch --filePath <> +``` + +Type batch --? for options. + ### Notes - Requires file system access. Creates test directories under **/data/testChat** diff --git a/ts/examples/chat/src/memory/knowproMemory.ts b/ts/examples/chat/src/memory/knowproMemory.ts index e661377de..e7798c7cb 100644 --- a/ts/examples/chat/src/memory/knowproMemory.ts +++ b/ts/examples/chat/src/memory/knowproMemory.ts @@ -111,6 +111,7 @@ export async function createKnowproCommands( commands.kpTopics = topics; commands.kpMessages = showMessages; commands.kpAbstractMessage = abstract; + commands.kpRelatedTerms = addRelatedTerm; /*---------------- * COMMANDS @@ -581,6 +582,45 @@ export async function createKnowproCommands( } } + function addRelatedTermDef(): CommandMetadata { + return { + description: "Add an alias", + args: { + term: arg("Value for which to add an alias"), + }, + options: { + relatedTerm: arg("Alias to add"), + weight: argNum("Relationship weight", 0.9), + }, + }; + } + commands.kpRelatedTerms.metadata = addRelatedTermDef(); + async function addRelatedTerm(args: string[]) { + if (!ensureConversationLoaded()) { + return; + } + const namedArgs = parseNamedArguments(args, addRelatedTermDef()); + const aliases = + context.conversation!.secondaryIndexes?.termToRelatedTermsIndex + ?.aliases; + if (!aliases) { + context.printer.writeLine( + "No aliases available on this conversation", + ); + return; + } + if (aliases instanceof kp.TermToRelatedTermsMap) { + if (namedArgs.relatedTerm) { + const relatedTerm: kp.Term = { text: namedArgs.relatedTerm }; + aliases.addRelatedTerm(namedArgs.term, relatedTerm); + } + } + const relatedTerms = aliases.lookupTerm(namedArgs.term); + if (relatedTerms && relatedTerms.length > 0) { + context.printer.writeList(relatedTerms.map((rt) => rt.text)); + } + } + /*---------- End COMMANDS ------------*/ diff --git a/ts/packages/aiclient/src/models.ts b/ts/packages/aiclient/src/models.ts index 860c0da78..102f3f4fd 100644 --- a/ts/packages/aiclient/src/models.ts +++ b/ts/packages/aiclient/src/models.ts @@ -125,3 +125,15 @@ export type GeneratedImage = { revised_prompt: string; image_url: string; }; + +export type EmbeddingModelMetadata = { + modelName?: string | undefined; + embeddingSize: number; +}; + +export function modelMetadata_ada002(): EmbeddingModelMetadata { + return { + modelName: "ada-002", + embeddingSize: 1536, + }; +} diff --git a/ts/packages/interactiveApp/src/interactiveApp.ts b/ts/packages/interactiveApp/src/interactiveApp.ts index ebe4ce1c5..d77e6fe4d 100644 --- a/ts/packages/interactiveApp/src/interactiveApp.ts +++ b/ts/packages/interactiveApp/src/interactiveApp.ts @@ -828,7 +828,7 @@ export function addBatchHandler(app: InteractiveApp) { function batchDef(): CommandMetadata { return { - description: "Run a batch file", + description: "Run a batch file of commands", args: { filePath: { description: "Batch file path.", diff --git a/ts/packages/knowPro/src/answerGenerator.ts b/ts/packages/knowPro/src/answerGenerator.ts index fa390e8b5..d192990ee 100644 --- a/ts/packages/knowPro/src/answerGenerator.ts +++ b/ts/packages/knowPro/src/answerGenerator.ts @@ -126,16 +126,81 @@ export type AnswerGeneratorSettings = { /** * Generate a natural language answer for question about a conversation using the provided search results as context + * - Each search result is first turned into an answer individually + * - If more than one search result provided, then individual answers are combined into a single answer * If the context exceeds the generator.setting.maxCharsInBudget, will break up the context into * chunks, run them in parallel, and then merge the answers found in individual chunks * @param conversation conversation about which this is a question - * @param generator answer generator to use + * @param generator answer generator to use to turn search results onto language answers: @see AnswerGenerator * @param question question that was asked - * @param searchResult the results of running a search query for the question on the conversation + * @param searchResults the results of running a search query for the question on the conversation * @param progress Progress callback * @returns Answers */ export async function generateAnswer( + conversation: IConversation, + generator: IAnswerGenerator, + question: string, + searchResults: ConversationSearchResult | ConversationSearchResult[], + progress?: asyncArray.ProcessProgress< + contextSchema.AnswerContext, + Result + >, + contextOptions?: AnswerContextOptions, +): Promise> { + let answerResponse: Result; + if (!Array.isArray(searchResults)) { + answerResponse = await generateAnswerFromSearchResult( + conversation, + generator, + question, + searchResults, + progress, + ); + } else { + if (searchResults.length === 0) { + return error("No search results"); + } + if (searchResults.length === 1) { + answerResponse = await generateAnswerFromSearchResult( + conversation, + generator, + question, + searchResults[0], + progress, + ); + } else { + // Get answers for individual searches in parallel + const partialResults = await asyncArray.mapAsync( + searchResults, + generator.settings.concurrency, + (sr) => + generateAnswerFromSearchResult( + conversation, + generator, + question, + sr, + progress, + ), + ); + // Use partial responses to build a complete answer + const partialResponses: answerSchema.AnswerResponse[] = []; + for (const result of partialResults) { + if (!result.success) { + return result; + } + partialResponses.push(result.data); + } + answerResponse = await generator.combinePartialAnswers( + question, + partialResponses, + ); + } + } + return answerResponse; +} + +async function generateAnswerFromSearchResult( conversation: IConversation, generator: IAnswerGenerator, question: string, diff --git a/ts/packages/knowPro/src/serialization.ts b/ts/packages/knowPro/src/serialization.ts index ccc31cb24..1dcbb2f6c 100644 --- a/ts/packages/knowPro/src/serialization.ts +++ b/ts/packages/knowPro/src/serialization.ts @@ -11,13 +11,15 @@ import { import { deserializeEmbeddings, serializeEmbeddings } from "./fuzzyIndex.js"; import path from "path"; import { IConversationDataWithIndexes } from "./secondaryIndexes.js"; +import { EmbeddingModelMetadata, modelMetadata_ada002 } from "aiclient"; export async function writeConversationDataToFile( conversationData: IConversationDataWithIndexes, dirPath: string, baseFileName: string, + modelMeta?: EmbeddingModelMetadata, ): Promise { - const fileData = toConversationFileData(conversationData); + const fileData = toConversationFileData(conversationData, modelMeta); if (fileData.binaryData) { if ( fileData.binaryData.embeddings && @@ -41,7 +43,7 @@ export async function writeConversationDataToFile( export async function readConversationDataFromFile( dirPath: string, baseFileName: string, - embeddingSize: number | undefined, + embeddingSize?: number, ): Promise { const jsonData = await readJsonFile( path.join(dirPath, baseFileName + DataFileSuffix), @@ -49,6 +51,11 @@ export async function readConversationDataFromFile( if (!jsonData) { return undefined; } + let fileData: ConversationFileData = { + jsonData: jsonData, + binaryData: {}, + }; + validateFileData(fileData, embeddingSize); let embeddings: Float32Array[] | undefined; if (embeddingSize && embeddingSize > 0) { const embeddingsBuffer = await readFile( @@ -56,13 +63,9 @@ export async function readConversationDataFromFile( ); if (embeddingsBuffer) { embeddings = deserializeEmbeddings(embeddingsBuffer, embeddingSize); + fileData.binaryData.embeddings = embeddings; } } - let fileData: ConversationFileData = { - jsonData: jsonData, - binaryData: { embeddings }, - }; - validateFileData(fileData); fileData.jsonData.fileHeader ??= createFileHeader(); return fromConversationFileData(fileData); } @@ -115,6 +118,8 @@ type FileHeader = { type EmbeddingFileHeader = { relatedCount?: number | undefined; messageCount?: number | undefined; + // The V 0.1 file format requires that all embeddings are the same size + modelMetadata?: EmbeddingModelMetadata | undefined; }; type EmbeddingData = { @@ -131,14 +136,28 @@ type ConversationBinaryData = { embeddings?: Float32Array[] | undefined; }; -function validateFileData(fileData: ConversationFileData): void { +function validateFileData( + fileData: ConversationFileData, + expectedEmbeddingSize?: number | undefined, +): void { if (fileData.jsonData === undefined) { throw new Error(`${Error_FileCorrupt}: Missing json data`); } + if (expectedEmbeddingSize && fileData.jsonData.embeddingFileHeader) { + const actualEmbeddingSize = + fileData.jsonData.embeddingFileHeader.modelMetadata + ?.embeddingSize ?? modelMetadata_ada002().embeddingSize; + if (expectedEmbeddingSize !== actualEmbeddingSize) { + throw new Error( + `File has embeddings of size ${expectedEmbeddingSize}`, + ); + } + } } function toConversationFileData( conversationData: IConversationDataWithIndexes, + modelMeta?: EmbeddingModelMetadata, ): ConversationFileData { let fileData: ConversationFileData = { jsonData: { @@ -157,7 +176,14 @@ function toConversationFileData( fileData.binaryData, conversationData.messageIndexData?.indexData, ); - + const embeddingSize = checkEmbeddingSize( + fileData, + modelMeta?.embeddingSize, + ); + modelMeta ??= { embeddingSize }; + if (modelMeta !== undefined) { + embeddingFileHeader.modelMetadata = modelMeta; + } return fileData; } @@ -219,6 +245,7 @@ function getEmbeddingsFromBinaryData( embeddingData: EmbeddingData | undefined, startAt: number, length?: number | undefined, + modelMetadata?: EmbeddingModelMetadata, ): number { if (binaryData.embeddings && embeddingData && length && length > 0) { embeddingData.embeddings = binaryData.embeddings.slice( @@ -230,7 +257,30 @@ function getEmbeddingsFromBinaryData( `${Error_FileCorrupt}: expected ${length}, got ${embeddingData.embeddings.length}`, ); } + if (modelMetadata) { + } return length; } return 0; } + +function checkEmbeddingSize( + fileData: ConversationFileData, + embeddingSize?: number, +): number { + if (fileData.binaryData) { + const embeddings = fileData.binaryData.embeddings; + if (embeddings && embeddings.length > 0) { + embeddingSize ??= embeddings[0].length; + for (let i = 1; i < embeddings.length; ++i) { + if (embeddingSize !== embeddings[i].length) { + throw new Error( + `Embeddings not of same size ${embeddingSize}`, + ); + } + } + return embeddingSize; + } + } + return 0; +} diff --git a/ts/packages/knowPro/test/answerGenerator.spec.ts b/ts/packages/knowPro/test/answerGenerator.spec.ts new file mode 100644 index 000000000..baf4ccfca --- /dev/null +++ b/ts/packages/knowPro/test/answerGenerator.spec.ts @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + describeIf, + hasTestKeys, + parseCommandArgs, + verifyResult, +} from "test-lib"; +import { + emptyConversation, + getTestChatModel, + loadTestConversationForOnline, + loadTestQueries, +} from "./testCommon.js"; +import { IConversation } from "../src/interfaces.js"; +import { createSearchQueryTranslator } from "../src/searchQueryTranslator.js"; +import { + createLanguageSearchOptions, + searchConversationWithLanguage, +} from "../src/searchLang.js"; +import { verifyAnswerResponse, verifySearchResults } from "./verify.js"; +import { + AnswerGenerator, + createAnswerGeneratorSettings, + generateAnswer, +} from "../src/answerGenerator.js"; + +describeIf( + "answerGenerator.online", + () => hasTestKeys(), + () => { + const testTimeout = 5 * 60 * 1000; + let conversation: IConversation = emptyConversation(); + let knowledgeModel = getTestChatModel(); + let queryTranslator = createSearchQueryTranslator(knowledgeModel); + beforeAll(async () => { + conversation = await loadTestConversationForOnline(); + }); + + test("answer.createGenerator", () => { + expect( + () => + new AnswerGenerator( + createAnswerGeneratorSettings(knowledgeModel), + ), + ).not.toThrow(); + }); + + test( + "answer.generate", + async () => { + let testQueries = loadTestQueries( + "./test/data/Episode_53_nlpAnswer.txt", + ); + let searchOptions = createLanguageSearchOptions(); + const answerGenerator = new AnswerGenerator( + createAnswerGeneratorSettings(knowledgeModel), + ); + for (const testQuery of testQueries) { + const cmd = parseCommandArgs(testQuery); + const question = cmd.namedArgs?.query; + if (question) { + // Get search result first + const results = await searchConversationWithLanguage( + conversation!, + question, + queryTranslator, + searchOptions, + ); + verifyResult(results, (data) => + verifySearchResults(data), + ); + if (results.success) { + const response = await generateAnswer( + conversation, + answerGenerator, + question, + results.data, + ); + verifyResult(response, (data) => + verifyAnswerResponse(data), + ); + } + } + } + }, + testTimeout, + ); + }, +); diff --git a/ts/packages/knowPro/test/data/Episode_53_nlpAnswer.txt b/ts/packages/knowPro/test/data/Episode_53_nlpAnswer.txt new file mode 100644 index 000000000..13cf9e100 --- /dev/null +++ b/ts/packages/knowPro/test/data/Episode_53_nlpAnswer.txt @@ -0,0 +1,8 @@ +--query "List all books" + +--query "List all books and also list all movies" + +--query "List all movies" + +--query "List all books and movies" + diff --git a/ts/packages/knowPro/test/verify.ts b/ts/packages/knowPro/test/verify.ts index 5372b62c4..cbeced487 100644 --- a/ts/packages/knowPro/test/verify.ts +++ b/ts/packages/knowPro/test/verify.ts @@ -24,6 +24,7 @@ import { matchSearchTermToEntity, } from "../src/query.js"; import { isPropertyTerm, isSearchGroupTerm } from "../src/compileLib.js"; +import { AnswerResponse } from "../src/answerResponseSchema.js"; export function expectHasEntities( semanticRefs: SemanticRef[], @@ -175,3 +176,25 @@ export function verifyDidMatchSearchGroup( break; } } + +export function verifySearchResult(result: ConversationSearchResult) { + expect(result.rawSearchQuery).toBeDefined(); + expect(result.knowledgeMatches.size).toBeGreaterThan(0); + expect(result.messageMatches.length).toBeGreaterThan(0); +} + +export function verifySearchResults(results: ConversationSearchResult[]) { + expect(results.length).toBeGreaterThan(0); + for (let i = 0; i < results.length; ++i) { + verifySearchResult(results[i]); + } +} + +export function verifyAnswerResponse(response: AnswerResponse) { + expect(response.type).toBeDefined(); + if (response.type === "Answered") { + expect(response.answer).toBeDefined(); + } else { + expect(response.whyNoAnswer).toBeDefined(); + } +} diff --git a/ts/packages/knowProTest/src/knowproCommands.ts b/ts/packages/knowProTest/src/knowproCommands.ts index 1d3c492a1..cb31fc16d 100644 --- a/ts/packages/knowProTest/src/knowproCommands.ts +++ b/ts/packages/knowProTest/src/knowproCommands.ts @@ -139,7 +139,6 @@ export async function execGetAnswerRequest( /** * Multiple query expressions can produce multiple search results * Currently, we take each individual search result and generate a separate answer - * TODO: combine answers * @param context * @param request * @param searchResults @@ -155,6 +154,7 @@ async function getAnswersForSearchResults( question: string, answer: Result, ) => void, + retryNoAnswer: boolean = true, ): Promise> { let answerResponses: kp.AnswerResponse[] = []; if (!request.messages) { @@ -171,21 +171,52 @@ async function getAnswersForSearchResults( if (choices && choices.length > 0) { question = kp.createMultipleChoiceQuestion(question, choices); } - const answerResult = await getAnswerFromSearchResult( - context, - request, - searchResult, - choices, - options, - ); + let answerResult = await async.getResultWithRetry(() => { + return getAnswerFromSearchResult( + context, + request, + searchResult, + choices, + options, + ); + }); + if ( + retryNoAnswer && + answerResult.success && + answerResult.data.type === "NoAnswer" + ) { + answerResult = await async.getResultWithRetry(() => { + return getAnswerFromSearchResult( + context, + request, + searchResult, + choices, + options, + ); + }); + } if (!answerResult.success) { return answerResult; } answerResponses.push(answerResult.data); - if (progressCallback) { + if (progressCallback && request.combineAnswer !== true) { progressCallback(i, question, answerResult); } } + if (request.combineAnswer === true) { + const answerResult = + await context.answerGenerator.combinePartialAnswers( + request.query, + answerResponses, + ); + if (!answerResult.success) { + return answerResult; + } + answerResponses = [answerResult.data]; + if (progressCallback) { + progressCallback(0, request.query, answerResult); + } + } return success(answerResponses); } @@ -195,7 +226,6 @@ async function getAnswerFromSearchResult( searchResult: kp.ConversationSearchResult, choices: string[] | undefined, options: kp.AnswerContextOptions, - retryNoAnswer: boolean = true, ): Promise> { const conversation = context.ensureConversationLoaded(); const fastStopSav = context.answerGenerator.settings.fastStop; @@ -207,43 +237,17 @@ async function getAnswerFromSearchResult( if (choices && choices.length > 0) { question = kp.createMultipleChoiceQuestion(question, choices); } - const maxAttempts = 2; - let answerResult = await async.getResultWithRetry( - () => - // - // Generate an answer from search results - // - kp.generateAnswer( - conversation, - context.answerGenerator, - question, - searchResult, - undefined, - options, - ), - maxAttempts, + // + // Generate an answer from search results + // + let answerResult = await kp.generateAnswer( + conversation, + context.answerGenerator, + question, + searchResult, + undefined, + options, ); - if ( - retryNoAnswer && - answerResult.success && - answerResult.data.type === "NoAnswer" - ) { - answerResult = await async.getResultWithRetry( - () => - // - // Generate an answer from search results - // - kp.generateAnswer( - conversation, - context.answerGenerator, - question, - searchResult, - undefined, - options, - ), - maxAttempts, - ); - } return answerResult; } finally { context.answerGenerator.settings.fastStop = fastStopSav; diff --git a/ts/packages/knowProTest/src/knowproContext.ts b/ts/packages/knowProTest/src/knowproContext.ts index afb3041b8..7a57d7c67 100644 --- a/ts/packages/knowProTest/src/knowproContext.ts +++ b/ts/packages/knowProTest/src/knowproContext.ts @@ -14,12 +14,12 @@ export class KnowproContext { constructor(basePath?: string) { this.basePath = basePath ?? "/data/testChat/knowpro"; this.knowledgeModel = createKnowledgeModel(); - (this.queryTranslator = kp.createSearchQueryTranslator( + this.queryTranslator = kp.createSearchQueryTranslator( this.knowledgeModel, - )), - (this.answerGenerator = new kp.AnswerGenerator( - kp.createAnswerGeneratorSettings(this.knowledgeModel), - )); + ); + this.answerGenerator = new kp.AnswerGenerator( + kp.createAnswerGeneratorSettings(this.knowledgeModel), + ); } public ensureConversationLoaded(): kp.IConversation { diff --git a/ts/packages/knowProTest/src/types.ts b/ts/packages/knowProTest/src/types.ts index 7b74b4b6d..52f8100ec 100644 --- a/ts/packages/knowProTest/src/types.ts +++ b/ts/packages/knowProTest/src/types.ts @@ -56,6 +56,11 @@ export interface GetAnswerRequest extends SearchRequest { knowledgeTopK?: number | undefined; choices?: string | undefined; searchResponse?: SearchResponse | undefined; + /** + * If searchResponse.searchResults.length > 1, use answers for individual + * search results as partial answers... and then combine them using the LLM + */ + combineAnswer?: boolean | undefined; } export interface GetAnswerResponse { @@ -80,6 +85,10 @@ export function getAnswerRequestDef( knowledgeTopK, ); def.options.choices = arg("Answer choices, separated by ';'"); + def.options.combineAnswer = argBool( + "Combine results of multiple search results into a single answer", + false, + ); return def; } diff --git a/ts/packages/testLib/src/test.ts b/ts/packages/testLib/src/test.ts index 5b7af6d1a..67cbc72ff 100644 --- a/ts/packages/testLib/src/test.ts +++ b/ts/packages/testLib/src/test.ts @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +import { Result } from "typechat"; import { hasTestKeys } from "./models.js"; export function testIf( @@ -33,3 +34,13 @@ export function describeIf( export function shouldSkip() { return !hasTestKeys(); } + +export function verifyResult( + result: Result, + cb?: (data: T) => void, +): void { + expect(result.success); + if (result.success && cb) { + cb(result.data); + } +}