fix(nlu): patterns tests

botpress · Mar 18, 2019 · 927a9fa · 927a9fa
1 parent 2ad52bf
commit 927a9fa
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 6 deletions.
diff --git a/modules/nlu/src/backend/pipelines/entities/pattern_extractor.test.ts b/modules/nlu/src/backend/pipelines/entities/pattern_extractor.test.ts
@@ -59,13 +59,13 @@ describe('Custom entity extraction', () => {
 
     const userInput = `
 I'm riding my mercedes-benz to the dealership then I will take my BM to buy an other mercedes because we need merchandise for the shop BMW!` /*
-              [============]                                                         [======]                                          [=]
+              [============]                                      ==                 [======]                                          [=]
 */
 
     const extractor = new PatternExtractor(Toolkit)
     const entities = await extractor.extractLists(userInput, 'en', [entityDef])
 
-    expect(entities.length).toEqual(3)
+    expect(entities.length).toEqual(4)
 
     expect(entities[0].name).toEqual(entityDef.name)
     expect(entities[0].meta.start).toEqual(15)
@@ -80,10 +80,16 @@ I'm riding my mercedes-benz to the dealership then I will take my BM to buy an o
     expect(entities[1].data.value).toEqual('Mercedes-Benz')
 
     expect(entities[2].name).toEqual(entityDef.name)
-    expect(entities[2].meta.start).toEqual(136)
-    expect(entities[2].meta.end).toEqual(140)
-    expect(entities[2].meta.source).toEqual('BMW!')
+    expect(entities[2].meta.start).toEqual(67)
+    expect(entities[2].meta.end).toEqual(69)
+    expect(entities[2].meta.source).toEqual('BM')
     expect(entities[2].data.value).toEqual('BMW')
+
+    expect(entities[3].name).toEqual(entityDef.name)
+    expect(entities[3].meta.start).toEqual(136)
+    expect(entities[3].meta.end).toEqual(140)
+    expect(entities[3].meta.source).toEqual('BMW!')
+    expect(entities[3].data.value).toEqual('BMW')
   })
 
   test('Extract exact list entitites', async () => {

diff --git a/modules/nlu/src/backend/pipelines/entities/pattern_extractor.ts b/modules/nlu/src/backend/pipelines/entities/pattern_extractor.ts
@@ -10,6 +10,8 @@ const debug = DEBUG('nlu').sub('entities')
 const debugLists = debug.sub('lists')
 const MIN_LENGTH_FUZZY_MATCH = 4
 
+const stripSpecialChars = str => str.replace(/!|@|#|$|%|\^|&|\*|(|)|_|-|\+|`|~|<|>|\?|\/|\.|,|;|:|"|'|[|]|{|}/gi, '')
+
 export default class PatternExtractor {
   constructor(private toolkit: typeof sdk.MLToolkit) {}
 
@@ -63,7 +65,7 @@ export default class PatternExtractor {
             distance = Math.min(1, distance * (0.1 * (4 - diffLen) + 1))
           }
         } else {
-          distance = partOfPhrase.toLowerCase() === occ.toLowerCase() ? 1 : 0
+          distance = stripSpecialChars(partOfPhrase.toLowerCase()) === stripSpecialChars(occ.toLowerCase()) ? 1 : 0
         }
 
         // if is closer OR if the match found is longer