From 184810557df891aa635442a25fc08d97af895b44 Mon Sep 17 00:00:00 2001
From: Umesh Madan <umeshma@microsoft.com>
Date: Fri, 23 Aug 2024 17:07:54 -0700
Subject: [PATCH] Action Terms: verbs and indirect object support

---
 .../src/conversation/actions.ts               | 48 ++++++++++++++-----
 .../src/conversation/conversation.ts          |  3 ++
 .../conversation/knowledgeTermSearchSchema.ts |  8 ++--
 3 files changed, 44 insertions(+), 15 deletions(-)

diff --git a/ts/packages/knowledgeProcessor/src/conversation/actions.ts b/ts/packages/knowledgeProcessor/src/conversation/actions.ts
index 24207ec3e7..73ee24b434 100644
--- a/ts/packages/knowledgeProcessor/src/conversation/actions.ts
+++ b/ts/packages/knowledgeProcessor/src/conversation/actions.ts
@@ -17,7 +17,7 @@ import {
     createKnowledgeStore,
     createTextIndex,
 } from "../knowledgeIndex.js";
-import { Action } from "./knowledgeSchema.js";
+import { Action, VerbTense } from "./knowledgeSchema.js";
 import path from "path";
 import { ActionFilter } from "./knowledgeSearchSchema.js";
 import {
@@ -280,13 +280,25 @@ export async function createActionIndex<TSourceId = any>(
         }
 
         const names = await getNameIndex();
-        const [subjectToActionIds, objectToActionIds] = await Promise.all([
+        const [
+            subjectToActionIds,
+            objectToActionIds,
+            indirectToObjectIds,
+            verbToActionIds,
+        ] = await Promise.all([
             matchTerms(names, subjectIndex, filter.terms, options),
             matchTerms(names, objectIndex, filter.terms, options),
+            matchTerms(names, indirectObjectIndex, filter.terms, options),
+            matchVerbTerms(filter.verbs, undefined, options),
         ]);
         results.actionIds = [
             ...intersectMultiple(
-                intersectUnionMultiple(subjectToActionIds, objectToActionIds),
+                intersectUnionMultiple(
+                    subjectToActionIds,
+                    objectToActionIds,
+                    indirectToObjectIds,
+                    verbToActionIds,
+                ),
                 itemsFromTemporalSequence(results.temporalSequence),
             ),
         ];
@@ -338,16 +350,11 @@ export async function createActionIndex<TSourceId = any>(
         options: ActionSearchOptions,
     ): Promise<ActionId[] | undefined> {
         if (filter.verbFilter && filter.verbFilter.verbs.length > 0) {
-            const verbOptions = options.verbSearchOptions ?? options;
-            const matches = await verbIndex.getNearest(
-                actionVerbsToString(
-                    filter.verbFilter.verbs,
-                    filter.verbFilter.verbTense,
-                ),
-                verbOptions.maxMatches,
-                verbOptions.minScore,
+            return matchVerbTerms(
+                filter.verbFilter.verbs,
+                filter.verbFilter.verbTense,
+                options,
             );
-            return matches;
         }
         return undefined;
     }
@@ -366,6 +373,23 @@ export async function createActionIndex<TSourceId = any>(
         return intersectUnionMultiple(...matches);
     }
 
+    async function matchVerbTerms(
+        verbs: string[] | undefined,
+        verbTense: VerbTense | undefined,
+        options: ActionSearchOptions,
+    ): Promise<ActionId[] | undefined> {
+        if (verbs && verbs.length > 0) {
+            const verbOptions = options.verbSearchOptions ?? options;
+            const matches = await verbIndex.getNearest(
+                actionVerbsToString(verbs, verbTense),
+                verbOptions.maxMatches,
+                verbOptions.minScore,
+            );
+            return matches;
+        }
+        return undefined;
+    }
+
     async function matchTimeRange(timeRange: DateTimeRange | undefined) {
         if (timeRange) {
             return await actionStore.sequence.getEntriesInRange(
diff --git a/ts/packages/knowledgeProcessor/src/conversation/conversation.ts b/ts/packages/knowledgeProcessor/src/conversation/conversation.ts
index 82f5d9f5e6..b53be816db 100644
--- a/ts/packages/knowledgeProcessor/src/conversation/conversation.ts
+++ b/ts/packages/knowledgeProcessor/src/conversation/conversation.ts
@@ -703,16 +703,19 @@ export async function createConversation(
         ]);
         const results = createSearchResponse<MessageId, TopicId, EntityId>();
         for (const filter of filters) {
+            // Only search actions if (a) actions are enabled (b) we have an action filter
             const topicResult = await topicIndex.searchTerms(
                 filter,
                 options.topic,
             );
             results.topics.push(topicResult);
+
             const entityResult = await entityIndex.searchTerms(
                 filter,
                 options.entity,
             );
             results.entities.push(entityResult);
+
             if (options.action) {
                 const actionResult = await actionIndex.searchTerms(
                     filter,
diff --git a/ts/packages/knowledgeProcessor/src/conversation/knowledgeTermSearchSchema.ts b/ts/packages/knowledgeProcessor/src/conversation/knowledgeTermSearchSchema.ts
index 5236f6f383..5e1cf8f69d 100644
--- a/ts/packages/knowledgeProcessor/src/conversation/knowledgeTermSearchSchema.ts
+++ b/ts/packages/knowledgeProcessor/src/conversation/knowledgeTermSearchSchema.ts
@@ -8,8 +8,10 @@ import { DateTimeRange } from "./dateTimeSchema.js";
  The message sequence, and any entities, actions and topics in each message are indexed.
  */
 
-// Search indexes for following terms
+// Search indexes for following "bag of words" terms
 export type TermFilter = {
+    // action verb terms to look for
+    verbs?: string[];
     // Terms are one of the following:
     // Entity Terms:
     // - the name of an entity or thing such as "Bach", "Great Gatsby", "frog" or "piano"
@@ -17,8 +19,8 @@ export type TermFilter = {
     //   An entity can have multiple types; entity types should be single words
     // - facets: specific, inherent, defining, or non-immediate facet of an entity such as "blue", "old", "famous", "sister", "aunt_of", "weight: 4 kg"
     // Action Terms:
-    // - verbs
-    // - subject, object, indirectObject
+    // - subject, object and indirectObject associated with the verb
+    // verbs are not duplicated
     terms: string[];
     // Use only if request explicitly asks for time range
     timeRange?: DateTimeRange | undefined; // in this time range