Skip to content

Commit

Permalink
fix(kb.gbapp): Importing improved.
Browse files Browse the repository at this point in the history
  • Loading branch information
@rodrigorodriguez
rodrigorodriguez committed Aug 26, 2020
1 parent 31588a2 commit 4cb9d5b
Show file tree
Hide file tree
Showing 4 changed files with 95 additions and 66 deletions.
10 changes: 5 additions & 5 deletions packages/default.gbdialog/bot.vbs.gbignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ if installments > 60 then
talk "The maximum number of payments is 60"
else
talk "What is the amount requested?"
hear ammount
hear amount

if ammount >100000 then
if amount >100000 then
talk "We are sorry, we can only accept proposals bellow 100k"
else

Expand Down Expand Up @@ -57,12 +57,12 @@ else
' TODO: This must be reviewed in terms of financing logic.

nInstallments = parseInt(installments)
vAmmount = parseFloat(ammount)
initialPayment = vAmmount * 0.3 ' 30% of the value
vamount = parseFloat(amount)
initialPayment = vamount * 0.3 ' 30% of the value
tac = 800
adjustment = 1.3

totalValue = ammount - initialPayment + tac
totalValue = amount - initialPayment + tac
paymentValue = totalValue * adjustment
finalValue = paymentValue * nInstallments + initialPayment

Expand Down
6 changes: 3 additions & 3 deletions packages/default.gbdialog/delivery.vbs.gbignore
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
' General Bots Copyright (c) Pragmatismo.io. All rights reserved. Licensed under the AGPL-3.0.

talk "Quer pagar quanto?"
hear ammount
hear amount

talk "Para onde?"
hear address

if ammount < 5 then
if amount < 5 then
talk "O mínimo que vendo este produto é 5."
else

if address is in "Rio" then
get payment ammount
get payment amount
delivery to address
else
talk "Vou ver se tenho um parceiro para entregar aí e te falo. Eu só entrego no Rio."
Expand Down
6 changes: 3 additions & 3 deletions packages/default.gbdialog/get-payment.vbs.gbignore
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
' General Bots Copyright (c) Pragmatismo.io. All rights reserved. Licensed under the AGPL-3.0.

talk "Quer pagar quanto?"
hear ammount
hear amount

if ammount < 5 then
if amount < 5 then
talk "O mínimo que vendo este produto é 5."
else
get payment ammount
get payment amount
end if

talk "Valeu!"
139 changes: 84 additions & 55 deletions packages/kb.gbapp/services/KBService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,15 @@ const walkPromise = require('walk-promise');
const { SearchService } = require('azure-search-client');
var Excel = require('exceljs');
import { GBServer } from '../../../src/app';
import { IGBKBService, GBDialogStep, GBLog, IGBConversationalService, IGBCoreService, IGBInstance, GBMinInstance } from 'botlib';
import {
IGBKBService,
GBDialogStep,
GBLog,
IGBConversationalService,
IGBCoreService,
IGBInstance,
GBMinInstance
} from 'botlib';
import { Op } from 'sequelize';
import { Sequelize } from 'sequelize-typescript';
import { AzureDeployerService } from '../../azuredeployer.gbapp/services/AzureDeployerService';
Expand Down Expand Up @@ -90,7 +98,9 @@ export class KBService implements IGBKBService {

public static getSubjectItemsSeparatedBySpaces(subjects: GuaribasSubject[]) {
const out = [];
if (subjects === undefined) { return ''; }
if (subjects === undefined) {
return '';
}
subjects.forEach(subject => {
out.push(subject.internalId);
});
Expand Down Expand Up @@ -128,7 +138,6 @@ export class KBService implements IGBKBService {
}

public async getAnswerByText(instanceId: number, text: string): Promise<any> {

text = text.trim();
const service = new CSService();
let question = await service.getQuestionFromAlternateText(instanceId, text);
Expand Down Expand Up @@ -185,9 +194,10 @@ export class KBService implements IGBKBService {
// tslint:disable:no-unsafe-any
if (instance.searchKey !== null && GBConfigService.get('STORAGE_DIALECT') === 'mssql') {
const client = new SearchService(instance.searchHost.split('.')[0], instance.searchKey);
const results = await client.indexes.use(instance.searchIndex)
const results = await client.indexes
.use(instance.searchIndex)
.buildQuery()
.filter((f) => f.eq('instanceId', instance.instanceId))
.filter(f => f.eq('instanceId', instance.instanceId))
.search(query)
.top(1)
.executeQuery();
Expand Down Expand Up @@ -265,25 +275,39 @@ export class KBService implements IGBKBService {
instanceId: number,
packageId: number
): Promise<GuaribasQuestion[]> {

GBLog.info(`Now reading file ${filePath}...`);
var workbook = new Excel.Workbook();
let data = await workbook.xlsx.readFile(filePath);

let lastQuestionId: number;
let lastAnswer: GuaribasAnswer;
let rows = data._worksheets[1]._rows;

GBLog.info(`Now importing ${rows.length} rows from tabular file ${filePath}...`);
// Finds a valid worksheet because Excel returns empty slots
// when loading worksheets collection.

let worksheet: any;
for (let t = 0; t < data._worksheets.length; t++) {
worksheet = data._worksheets[t];
if (worksheet)
{
break;
}
}

return asyncPromise.eachSeries(rows, async line => {
let rows = worksheet._rows;

GBLog.info(`Now importing ${rows.length} rows from tabular file ${filePath}...`);
return asyncPromise.eachSeries(rows, async line => {
// Skips the first line.

if (line._cells[0] !== undefined &&
if (
line != undefined &&
line._cells[0] !== undefined &&
line._cells[1] !== undefined &&
line._cells[2] !== undefined &&
line._cells[3] !== undefined &&
line._cells[4] !== undefined) {
line._cells[4] !== undefined
) {
// Extracts values from columns in the current line.

const subjectsText = line._cells[0].text;
Expand All @@ -292,18 +316,17 @@ export class KBService implements IGBKBService {
const question = line._cells[3].text;
let answer = line._cells[4].text;

if (!(subjectsText === 'subjects' && from === 'from')
&& (answer !== null && question !== null)) {

if (!(subjectsText === 'subjects' && from === 'from') && answer !== null && question !== null) {
let format = '.txt';

// Extracts answer from external media if any.

let media = null;

if (typeof (answer) !== "string") {
if (typeof answer !== 'string') {
GBLog.info(`[GBImporter] Answer is NULL related to Question '${question}'.`);
answer = 'Existe um problema na base de conhecimento. Fui treinado para entender sua pergunta, avise a quem me criou que a resposta não foi informada para esta pergunta.';
answer =
'Existe um problema na base de conhecimento. Fui treinado para entender sua pergunta, avise a quem me criou que a resposta não foi informada para esta pergunta.';
} else if (answer.indexOf('.md') > -1) {
const mediaFilename = urlJoin(path.dirname(filePath), '..', 'articles', answer);
if (Fs.existsSync(mediaFilename)) {
Expand Down Expand Up @@ -381,35 +404,42 @@ export class KBService implements IGBKBService {
public async sendAnswer(min: GBMinInstance, channel: string, step: GBDialogStep, answer: GuaribasAnswer) {
if (answer.content.endsWith('.mp4')) {
await this.playVideo(min, min.conversationalService, step, answer, channel);
}
else if (answer.format === '.md') {

} else if (answer.format === '.md') {
await this.playMarkdown(min, answer, channel, step, min.conversationalService);

} else if (answer.content.endsWith('.ogg') && process.env.AUDIO_DISABLED !== "true") {

} else if (answer.content.endsWith('.ogg') && process.env.AUDIO_DISABLED !== 'true') {
await this.playAudio(min, answer, channel, step, min.conversationalService);
} else {
await min.conversationalService.sendText(min, step, answer.content);
await min.conversationalService.sendEvent(min, step, 'stop', undefined);
}
}

private async playAudio(min: GBMinInstance, answer: GuaribasAnswer, channel: string, step: GBDialogStep, conversationalService: IGBConversationalService) {
private async playAudio(
min: GBMinInstance,
answer: GuaribasAnswer,
channel: string,
step: GBDialogStep,
conversationalService: IGBConversationalService
) {
conversationalService.sendAudio(min, step, answer.content);
}

private async playMarkdown(min: GBMinInstance, answer: GuaribasAnswer, channel: string, step: GBDialogStep, conversationalService: IGBConversationalService) {

private async playMarkdown(
min: GBMinInstance,
answer: GuaribasAnswer,
channel: string,
step: GBDialogStep,
conversationalService: IGBConversationalService
) {
let sec = new SecService();
const member = step.context.activity.from;
const user = await sec.ensureUser(min.instance.instanceId, member.id,
member.name, "", "web", member.name);
const user = await sec.ensureUser(min.instance.instanceId, member.id, member.name, '', 'web', member.name);
const minBoot = GBServer.globals.minBoot as any;

// Calls language translator.

let text = await min.conversationalService.translate(min,
let text = await min.conversationalService.translate(
min,
min.instance.translatorKey ? min.instance.translatorKey : minBoot.instance.translatorKey,
min.instance.translatorEndpoint ? min.instance.translatorEndpoint : minBoot.instance.translatorEndpoint,
answer.content,
Expand Down Expand Up @@ -438,29 +468,30 @@ export class KBService implements IGBKBService {

// According to the channel, formats the output optimized to it.

if (channel === 'webchat' &&
GBConfigService.get('DISABLE_WEB') !== 'true') {
if (channel === 'webchat' && GBConfigService.get('DISABLE_WEB') !== 'true') {
html = marked(text);
await this.sendMarkdownToWeb(min, step, conversationalService, html, answer);
}
else if (channel === 'whatsapp') {

} else if (channel === 'whatsapp') {
await conversationalService.sendMarkdownToMobile(min, step, user.userSystemId, text);
}
else {
} else {
html = marked(text);
await min.conversationalService.sendText(min, step, html);
}
}

private async sendMarkdownToWeb(min, step: GBDialogStep, conversationalService: IGBConversationalService, html: string, answer: GuaribasAnswer) {

private async sendMarkdownToWeb(
min,
step: GBDialogStep,
conversationalService: IGBConversationalService,
html: string,
answer: GuaribasAnswer
) {
let sec = new SecService();
const member = step.context.activity.from;
const user = await sec.ensureUser(min.instance.instanceId, member.id,
member.name, "", "web", member.name);
const user = await sec.ensureUser(min.instance.instanceId, member.id, member.name, '', 'web', member.name);
const minBoot = GBServer.globals.minBoot as any;
html = await min.conversationalService.translate(min,
html = await min.conversationalService.translate(
min,
min.instance.translatorKey ? min.instance.translatorKey : minBoot.instance.translatorKey,
min.instance.translatorEndpoint ? min.instance.translatorEndpoint : minBoot.instance.translatorEndpoint,
html,
Expand All @@ -481,11 +512,15 @@ export class KBService implements IGBKBService {
});
}


private async playVideo(min, conversationalService: IGBConversationalService,
step: GBDialogStep, answer: GuaribasAnswer, channel: string) {
if (channel === "whatsapp") {
await min.conversationalService.sendFile(min, step, null, answer.content, "");
private async playVideo(
min,
conversationalService: IGBConversationalService,
step: GBDialogStep,
answer: GuaribasAnswer,
channel: string
) {
if (channel === 'whatsapp') {
await min.conversationalService.sendFile(min, step, null, answer.content, '');
} else {
await conversationalService.sendEvent(min, step, 'play', {
playerType: 'video',
Expand All @@ -499,7 +534,6 @@ export class KBService implements IGBKBService {
packageStorage: GuaribasPackage,
instance: IGBInstance
): Promise<any> {

// Imports subjects tree into database and return it.

const subjectFile = urlJoin(localPath, 'subjects.json');
Expand All @@ -525,18 +559,16 @@ export class KBService implements IGBKBService {

await CollectionUtil.asyncForEach(files, async file => {
if (file !== null && file.name.endsWith('.md')) {

let content = await this.getAnswerTextByMediaName(instance.instanceId, file.name);

if (content === null) {

const fullFilename = urlJoin(file.root, file.name);
content = Fs.readFileSync(fullFilename, 'utf-8');

await GuaribasAnswer.create({
instanceId: instance.instanceId,
content: content,
format: ".md",
format: '.md',
media: file.name,
packageId: packageId,
prevId: 0 // TODO: Calculate total rows and increment.
Expand All @@ -552,8 +584,7 @@ export class KBService implements IGBKBService {
if (file !== null && file.name.endsWith('.xlsx')) {
return await this.importKbTabularFile(urlJoin(file.root, file.name), instance.instanceId, packageId);
}
})

});
}

public async importSubjectFile(packageId: number, filename: string, instance: IGBInstance): Promise<any> {
Expand Down Expand Up @@ -595,8 +626,7 @@ export class KBService implements IGBKBService {
});
await this.undeployPackageFromStorage(instance, packageId);

GBLog.info("Remember to call rebuild index manually after package removal.");

GBLog.info('Remember to call rebuild index manually after package removal.');
}

private async undeployPackageFromStorage(instance: any, packageId: number) {
Expand All @@ -614,12 +644,11 @@ export class KBService implements IGBKBService {
const packageName = Path.basename(localPath);
GBLog.info(`[GBDeployer] Opening package: ${localPath}`);


const instance = await core.loadInstanceByBotId(min.botId);
GBLog.info(`[GBDeployer] Importing: ${localPath}`);
const p = await deployer.deployPackageToStorage(instance.instanceId, packageName);
await this.importKbPackage(localPath, p, instance);
deployer.mountGBKBAssets(packageName,min.botId, localPath);
deployer.mountGBKBAssets(packageName, min.botId, localPath);

await deployer.rebuildIndex(instance, new AzureDeployerService(deployer).getKBSearchSchema(instance.searchIndex));
GBLog.info(`[GBDeployer] Finished import of ${localPath}`);
Expand Down

0 comments on commit 4cb9d5b

Please sign in to comment.