fix(core.gbapp): Fix in NLP handling of acents.

GeneralBots · Jan 3, 2021 · 2a5ecd4 · 2a5ecd4
1 parent e7d7a1a
commit 2a5ecd4
Show file tree

Hide file tree

Showing 8 changed files with 70 additions and 52 deletions.
diff --git a/packages/basic.gblib/strings.ts b/packages/basic.gblib/strings.ts
@@ -1,34 +1,9 @@
 
 export const Messages = {
-  global_quit: /^(sair|sai|chega|exit|quit|finish|end|ausfahrt|verlassen)/i,
   'en-US': {
-    show_video: 'I will show you a video, please wait...',
-    good_morning: 'good morning',
-    good_evening: 'good evening',
-    good_night: 'good night',
-    hi: (msg) => `Hello, ${msg}.`,
-    very_sorry_about_error: `I'm sorry to inform that there was an error which was recorded to be solved.`,
-    canceled: 'Canceled. If I can be useful, let me know how',
-    whats_email: 'What\'s your E-mail address?',
-    which_language: 'Please, type the language name you would like to talk through.',
-    validation_enter_valid_email: 'Please enter a valid e-mail.'   ,
-    language_chosen: 'Very good, so let\'s go...'   ,
-    affirmative_sentences: /^(sim|s|positivo|afirmativo|claro|evidente|sem dúvida|confirmo|confirmar|confirmado|uhum|si|y|yes|sure)/i
 
   },
   'pt-BR': {
-    show_video: 'Vou te mostrar um vídeo. Por favor, aguarde...',
-    good_morning: 'bom dia',
-    good_evening: 'boa tarde',
-    good_night: 'boa noite',
-    hi: (msg) => `Oi, ${msg}.`,
-    very_sorry_about_error: `Lamento, ocorreu um erro que já foi registrado para ser tratado.`,
-    canceled: 'Cancelado, avise como posso ser útil novamente.',
-    whats_email: 'Qual seu e-mail?',
-    which_language: 'Por favor, digite o idioma que você gostaria de usar para conversarmos.',
-    validation_enter_valid_email: 'Por favor digite um email válido.',
-    language_chosen: 'Muito bem, então vamos lá...'   ,
-    affirmative_sentences: /^(sim|s|positivo|afirmativo|claro|evidente|sem dúvida|confirmo|confirmar|confirmado|uhum|si|y|yes|sure)/i
 
   }
 };
diff --git a/packages/core.gbapp/services/GBConversationalService.ts b/packages/core.gbapp/services/GBConversationalService.ts
@@ -46,10 +46,10 @@ import { SecService } from '../../security.gbapp/services/SecService';
 import { AnalyticsService } from '../../analytics.gblib/services/AnalyticsService';
 import { MicrosoftAppCredentials } from 'botframework-connector';
 import { GBConfigService } from './GBConfigService';
+import { CollectionUtil, AzureText } from 'pragmatismo-io-framework';
 const urlJoin = require('url-join');
 const PasswordGenerator = require('strict-password-generator').default;
 const Nexmo = require('nexmo');
-import { CollectionUtil, AzureText } from 'pragmatismo-io-framework';
 const { join } = require('path');
 const shell = require('any-shell-escape');
 const { exec } = require('child_process');
@@ -60,23 +60,25 @@ const fs = require('fs');
 const SpeechToTextV1 = require('ibm-watson/speech-to-text/v1');
 const { IamAuthenticator } = require('ibm-watson/auth');
 
-export interface LanguagePickerSettings {
-  defaultLocale?: string;
-  supportedLocales?: string[];
-}
-
 /**
  * Provides basic services for handling messages and dispatching to back-end
  * services like NLP or Search.
  */
 export class GBConversationalService {
+
+  /**
+   * Reference to the core service.
+   */
   public coreService: IGBCoreService;
 
+  /**
+   * 
+   * @param coreService 
+   */
   constructor(coreService: IGBCoreService) {
     this.coreService = coreService;
   }
 
-
   static defaultDiacriticsRemovalMap = [
     { 'base': 'A', 'letters': '\u0041\u24B6\uFF21\u00C0\u00C1\u00C2\u1EA6\u1EA4\u1EAA\u1EA8\u00C3\u0100\u0102\u1EB0\u1EAE\u1EB4\u1EB2\u0226\u01E0\u00C4\u01DE\u1EA2\u00C5\u01FA\u01CD\u0200\u0202\u1EA0\u1EAC\u1EB6\u1E00\u0104\u023A\u2C6F' },
     { 'base': 'AA', 'letters': '\uA732' },

diff --git a/packages/core.gbapp/services/GBCoreService.ts b/packages/core.gbapp/services/GBCoreService.ts
@@ -100,7 +100,7 @@ export class GBCoreService implements IGBCoreService {
   private dialect: string;
 
   /**
-   * Constructor retrieves default values.
+   * 
    */
   constructor() {
     this.adminService = new GBAdminService(this);

diff --git a/packages/core.gbapp/services/GBDeployer.ts b/packages/core.gbapp/services/GBDeployer.ts
@@ -800,6 +800,7 @@ export class GBDeployer implements IGBDeployer {
       'whatsapp.gblib',
       'sharepoint.gblib',
       'core.gbapp',
+      'basic.gblib',
       'admin.gbapp',
       'azuredeployer.gbapp',
       'customer-satisfaction.gbapp',

diff --git a/packages/core.gbapp/services/GBMinService.ts b/packages/core.gbapp/services/GBMinService.ts
@@ -62,7 +62,7 @@ import {
   IGBInstance,
   IGBPackage
 } from 'botlib';
-import { AzureText, CollectionUtil } from 'pragmatismo-io-framework';
+import { CollectionUtil } from 'pragmatismo-io-framework';
 
 import { MicrosoftAppCredentials } from 'botframework-connector';
 import fs = require('fs');
@@ -112,8 +112,6 @@ export class GBMinService {
 
   /**
    * Static initialization of minimal instance.
-   *
-   * @param core Basic database services to identify instance, for example.
    */
   constructor(
     core: IGBCoreService,
@@ -582,7 +580,7 @@ export class GBMinService {
     const userState = new UserState(storage);
     adapter.use(new AutoSaveStateMiddleware(conversationState, userState));
     MicrosoftAppCredentials.trustServiceUrl('https://directline.botframework.com',
-                                            new Date(new Date().setFullYear(new Date().getFullYear() + 10))
+      new Date(new Date().setFullYear(new Date().getFullYear() + 10))
     );
 
     // The minimal bot is built here.
@@ -612,7 +610,7 @@ export class GBMinService {
     await CollectionUtil.asyncForEach(min.appPackages, async (e: IGBPackage) => {
       let services: ConcatArray<never>;
       if ((services = await e.onExchangeData(min, 'getServices', null))) {
-        min.gbappServices = {...min.gbappServices, ...services};
+        min.gbappServices = { ...min.gbappServices, ...services };
       }
     });
 
@@ -958,7 +956,7 @@ export class GBMinService {
       process.env.GBKB_ENABLE_AUTO_PUBLISH === 'true'
     ) {
       await min.conversationalService.sendText(min, step,
-                                               `Oi, ainda não possuo pacotes de conhecimento publicados. Por favor, aguarde alguns segundos enquanto eu auto-publico alguns pacotes.`
+        `Oi, ainda não possuo pacotes de conhecimento publicados. Por favor, aguarde alguns segundos enquanto eu auto-publico alguns pacotes.`
       );
       await step.beginDialog('/publish', { confirm: true, firstTime: true });
     } else {
@@ -984,18 +982,36 @@ export class GBMinService {
           keepTextList = keepTextList.concat(result);
         }
       });
-      let textProcessed = GBConversationalService.removeDiacritics(text);
+
+      const getNormalizedRegExp = (value) => {
+        var chars = [
+          { letter: 'a', reg: '[aáàãäâ]' },
+          { letter: 'e', reg: '[eéèëê]' },
+          { letter: 'i', reg: '[iíìïî]' },
+          { letter: 'o', reg: '[oóòõöô]' },
+          { letter: 'u', reg: '[uúùüû]' },
+          { letter: 'c', reg: '[cç]' }
+        ];
+
+        for (var i in chars) {
+          value = value.replace(new RegExp(chars[i].letter, 'gi'), chars[i].reg);
+        };
+        return value;
+      };
+
+      let textProcessed = text;
       if (keepTextList) {
         keepTextList = keepTextList.filter(p => p.trim() !== '');
         let i = 0;
         await CollectionUtil.asyncForEach(keepTextList, item => {
           const it = GBConversationalService.removeDiacritics(item);
+          const noAccentText = GBConversationalService.removeDiacritics(textProcessed);
 
-          if (textProcessed.toLowerCase().indexOf(it.toLowerCase()) != -1) {
+          if (noAccentText.toLowerCase().indexOf(it.toLowerCase()) != -1) {
             const replacementToken = 'X' + GBAdminService.getNumberIdentifier().substr(0, 4);
             replacements[i] = { text: item, replacementToken: replacementToken };
             i++;
-            textProcessed = textProcessed.replace(new RegExp(`\\b${it.trim()}\\b`, 'gi'), `${replacementToken}`);
+            textProcessed = textProcessed.replace(new RegExp(`\\b${getNormalizedRegExp(it.trim())}\\b`, 'gi'), `${replacementToken}`);
           }
         });
       }
@@ -1008,10 +1024,10 @@ export class GBMinService {
       // Detects user typed language and updates their locale profile if applies.
 
       let locale = min.core.getParam<string>(min.instance, 'Default User Language',
-                                             GBConfigService.get('DEFAULT_USER_LANGUAGE')
+        GBConfigService.get('DEFAULT_USER_LANGUAGE')
       );
       const detectLanguage = min.core.getParam<boolean>(min.instance, 'Language Detector',
-                                                        GBConfigService.getBoolean('LANGUAGE_DETECTOR')
+        GBConfigService.getBoolean('LANGUAGE_DETECTOR')
       ) === 'true';
       const systemUser = user.systemUser;
       locale = systemUser.locale;

diff --git a/packages/core.gbapp/strings.ts b/packages/core.gbapp/strings.ts
@@ -2,9 +2,33 @@
 export const Messages = {
   global_quit: /^(sair|sai|chega|exit|quit|finish|end|ausfahrt|verlassen)/i,
   'en-US': {
+    show_video: 'I will show you a video, please wait...',
+    good_morning: 'good morning',
+    good_evening: 'good evening',
+    good_night: 'good night',
+    hi: (msg) => `Hello, ${msg}.`,
+    very_sorry_about_error: `I'm sorry to inform that there was an error which was recorded to be solved.`,
+    canceled: 'Canceled. If I can be useful, let me know how',
+    whats_email: 'What\'s your E-mail address?',
+    which_language: 'Please, type the language name you would like to talk through.',
+    validation_enter_valid_email: 'Please enter a valid e-mail.',
+    language_chosen: 'Very good, so let\'s go...',
+    affirmative_sentences: /^(sim|s|positivo|afirmativo|claro|evidente|sem dúvida|confirmo|confirmar|confirmado|uhum|si|y|yes|sure)/i
 
   },
   'pt-BR': {
+    show_video: 'Vou te mostrar um vídeo. Por favor, aguarde...',
+    good_morning: 'bom dia',
+    good_evening: 'boa tarde',
+    good_night: 'boa noite',
+    hi: (msg) => `Oi, ${msg}.`,
+    very_sorry_about_error: `Lamento, ocorreu um erro que já foi registrado para ser tratado.`,
+    canceled: 'Cancelado, avise como posso ser útil novamente.',
+    whats_email: 'Qual seu e-mail?',
+    which_language: 'Por favor, digite o idioma que você gostaria de usar para conversarmos.',
+    validation_enter_valid_email: 'Por favor digite um email válido.',
+    language_chosen: 'Muito bem, então vamos lá...',
+    affirmative_sentences: /^(sim|s|positivo|afirmativo|claro|evidente|sem dúvida|confirmo|confirmar|confirmado|uhum|si|y|yes|sure)/i
 
   }
 };
diff --git a/packages/default.gbtest/first-test.xlsx b/packages/default.gbtest/first-test.xlsx
diff --git a/packages/kb.gbapp/dialogs/AskDialog.ts b/packages/kb.gbapp/dialogs/AskDialog.ts
@@ -116,9 +116,11 @@ export class AskDialog extends IGBDialog {
           const user = await sec.ensureUser(min.instance.instanceId, member.id, member.name, '', 'web', member.name);
 
           let handled = false;
+          let nextDialog = null;
+
           await CollectionUtil.asyncForEach(min.appPackages, async (e: IGBPackage) => {
             if (
-              await e.onExchangeData(min, 'handleAnswer', {
+              nextDialog = await e.onExchangeData(min, 'handleAnswer', {
                 query: text,
                 step: step,
                 message: text,
@@ -129,13 +131,11 @@ export class AskDialog extends IGBDialog {
             }
           });
 
-          if (!handled) {
-            return await step.beginDialog('/answer', {
-              query: text
-            });
-          } else {
-            return await step.next();
-          }
+          await step.beginDialog(nextDialog ? nextDialog : '/answer', {
+            query: text,
+            user: user ? user['dataValues'] : null,
+            message: text
+          });
         } else {
           return await step.next();
         }