Merge pull request #1426 from botpress/f_fix-intent-election

fix(nlu): intent election

modules/nlu/src/backend/pipelines/intents/utils.test.ts

@@ -11,22 +11,36 @@ describe('NLU intent utils', () => {
     const set3 = mapSet([])
     const set4 = mapSet([0.45, 0.12, 0.11, 0.08, 0.0002])
     const set5 = mapSet([0.2, 0.12, 0.11, 0.08, 0.0002])
+    const set6 = mapSet([0.2, 0.1, 0.08, 0.6])
+    const set7 = mapSet([0.28, 0.22, 0.14, 0.12])
+    const set8 = mapSet([0.8])
+    const set9 = mapSet([0.6])
+    const set10 = mapSet([0.2, 0.1])
+    const set11 = mapSet([0.45, 0.1, 0.05, 0.05, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002])
 
-    const res1 = findMostConfidentIntentMeanStd(set1, 0.8, 4)
-    const res2 = findMostConfidentIntentMeanStd(set2, 0.8, 4)
-    const res3 = findMostConfidentIntentMeanStd(set3, 0.8, 4)
-    const res4 = findMostConfidentIntentMeanStd(set4, 0.8, 4)
-    const res4b = findMostConfidentIntentMeanStd(set4, 0.8, 5)
-    const res5 = findMostConfidentIntentMeanStd(set5, 0.8, 4)
-    const res5b = findMostConfidentIntentMeanStd(set5, 0.8, 2)
+    const res1 = findMostConfidentIntentMeanStd(set1, 0.8)
+    const res2 = findMostConfidentIntentMeanStd(set2, 0.8)
+    const res3 = findMostConfidentIntentMeanStd(set3, 0.8)
+    const res4 = findMostConfidentIntentMeanStd(set4, 0.8)
+    const res5 = findMostConfidentIntentMeanStd(set5, 0.8)
+    const res6 = findMostConfidentIntentMeanStd(set6, 0.8)
+    const res7 = findMostConfidentIntentMeanStd(set7, 0.8)
+    const res8 = findMostConfidentIntentMeanStd(set8, 0.8)
+    const res9 = findMostConfidentIntentMeanStd(set9, 0.8)
+    const res10 = findMostConfidentIntentMeanStd(set10, 0.8)
+    const res11 = findMostConfidentIntentMeanStd(set11, 0.8)
 
     expect(res1.name).toBe('0')
     expect(res2.name).toBe('0')
     expect(res3.name).toBe('none')
     expect(res4.name).toBe('0')
-    expect(res4b.name).toBe('none')
     expect(res5.name).toBe('none')
-    expect(res5b.name).toBe('0')
+    expect(res6.name).toBe('3')
+    expect(res7.name).toBe('none')
+    expect(res8.name).toBe('0')
+    expect(res9.name).toBe('none')
+    expect(res10.name).toBe('none')
+    expect(res11.name).toBe('0')
   })
 
   describe('matches', () => {

modules/nlu/src/backend/pipelines/intents/utils.ts

@@ -10,16 +10,16 @@ export const NoneIntent: sdk.NLU.Intent = {
 }
 
 /**
- * Finds the most confident intent, either by the intent being above a fixed threshold, or else if an intent is more than {@param std} standard deviation (outlier method).
+ * Finds the most confident intent, either by the intent being above a fixed threshold, or else if an intent is more than {@param zThresh} standard deviation (outlier method) from the mean.
  * NOTE: If you ever need this in another context, we could move this into tools and replace the "intent" concept for a more generic "prediction"
  * @param intents
  * @param fixedThreshold
- * @param std number of standard deviation away. normally between 2 and 5
+ * @param zThresh number of standard deviation between 2 furthest from the mean
  */
 export function findMostConfidentIntentMeanStd(
   intents: sdk.NLU.Intent[],
   fixedThreshold: number,
-  std: number = 3
+  zThresh: number = 1.25
 ): sdk.NLU.Intent {
   if (!intents.length) {
     return NoneIntent
@@ -31,14 +31,19 @@ export function findMostConfidentIntentMeanStd(
     return best
   }
 
-  const mean = _.meanBy<sdk.NLU.Intent>(intents, 'confidence')
-  const stdErr =
-    Math.sqrt(intents.reduce((a, c) => a + Math.pow(c.confidence - mean, 2), 0) / intents.length) /
-    Math.sqrt(intents.length)
-
-  const dominant = intents.find(x => x.confidence >= stdErr * std + mean)
+  if (intents.length < 3) {
+    return NoneIntent
+  }
 
-  return dominant || NoneIntent
+  const mean = _.meanBy<sdk.NLU.Intent>(intents, 'confidence')
+  const stdDeviation = Math.sqrt(
+    intents.reduce((a, c) => a + Math.pow(c.confidence - mean, 2), 0) / (intents.length - 1)
+  )
+  const zintents = intents
+    .map(intent => ({ intent, z: (intent.confidence - mean) / stdDeviation }))
+    .sort((a, b) => (a.z > b.z ? -1 : 1))
+
+  return zintents[0].z - zintents[1].z > zThresh ? zintents[0].intent : NoneIntent
 }
 
 export const createIntentMatcher = (intentName: string): ((pattern: string) => boolean) => {