Skip to content

Commit

Permalink
fix(core.gbapp): Fixes in a blank environment creation.
Browse files Browse the repository at this point in the history
  • Loading branch information
@rodrigorodriguez
rodrigorodriguez committed Apr 17, 2021
1 parent 8de9777 commit b0d45c3
Show file tree
Hide file tree
Showing 9 changed files with 85 additions and 50 deletions.
2 changes: 1 addition & 1 deletion packages/admin.gbapp/services/GBAdminService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ export class GBAdminService implements IGBAdminService {
maximumLength: 14
};
let password = passwordGenerator.generatePassword(options);
password = password.replace(/[\@\[\=\:\;\?]/gi, '#');
password = password.replace(/[\@\[\=\:\;\?\"\'\#]/gi, '1');

return password;
}
Expand Down
6 changes: 6 additions & 0 deletions packages/azuredeployer.gbapp/dialogs/StartDialog.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,11 @@ cannot start or end with or contain consecutive dashes and having 4 to 42 charac
return botId;
}


/**
*
* Update Manifest in Azure: "signInAudience": "AzureADandPersonalMicrosoftAccount" and "accessTokenAcceptedVersion": 2.
*/
private static retrieveAppId() {
let appId = GBConfigService.get('MARKETPLACE_ID');
if (appId === undefined) {
Expand All @@ -154,6 +159,7 @@ cannot start or end with or contain consecutive dashes and having 4 to 42 charac
please go to https://apps.dev.microsoft.com/portal/register-app to
generate manually an App ID and App Secret.\n`
);

process.stdout.write('Generated Application Id (MARKETPLACE_ID):');
appId = scanf('%s').replace(/(\n|\r)+$/, '');
}
Expand Down
75 changes: 46 additions & 29 deletions packages/azuredeployer.gbapp/services/AzureDeployerService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ import { SqlManagementClient } from 'azure-arm-sql';
import { WebSiteManagementClient } from 'azure-arm-website';
//tslint:disable-next-line:no-submodule-imports
import { AppServicePlan, Site, SiteConfigResource, SiteLogsConfig, SiteSourceControl } from 'azure-arm-website/lib/models';
import { GBLog, IGBInstallationDeployer, IGBInstance, IGBDeployer } from 'botlib';
import { GBLog, IGBInstallationDeployer, IGBInstance, IGBDeployer, IGBCoreService } from 'botlib';
import { GBAdminService } from '../../../packages/admin.gbapp/services/GBAdminService';
import { GBCorePackage } from '../../../packages/core.gbapp';
import { GBConfigService } from '../../../packages/core.gbapp/services/GBConfigService';
Expand Down Expand Up @@ -82,13 +82,18 @@ export class AzureDeployerService implements IGBInstallationDeployer {
public subscriptionId: string;
public farmName: any;
public deployer: IGBDeployer;
public core: IGBCoreService;
private freeTier: boolean;

constructor(deployer: IGBDeployer, freeTier: boolean = true) {
constructor(deployer: IGBDeployer, freeTier: boolean = false) {
this.deployer = deployer;
this.freeTier = freeTier;
}

public async runSearch(instance: IGBInstance) {
await this.deployer.rebuildIndex(instance, this.getKBSearchSchema(instance.searchIndex));
}

public static async createInstance(deployer: GBDeployer): Promise<AzureDeployerService> {

const username = GBConfigService.get('CLOUD_USERNAME');
Expand All @@ -97,6 +102,7 @@ export class AzureDeployerService implements IGBInstallationDeployer {
const subscriptionId = GBConfigService.get('CLOUD_SUBSCRIPTIONID');

const service = new AzureDeployerService(deployer);
service.core = deployer.core;
service.initServices(credentials, subscriptionId);

return service;
Expand Down Expand Up @@ -350,15 +356,6 @@ export class AzureDeployerService implements IGBInstallationDeployer {
};
await storageClient.firewallRules.createOrUpdate(groupName, serverName, 'gb', params);

// AllowAllWindowsAzureIps must be created that way, so the Azure Search can
// access SQL Database to index its contents.

params = {
startIpAddress: '0.0.0.0',
endIpAddress: '0.0.0.0'
};
await storageClient.firewallRules.createOrUpdate(groupName, serverName, 'AllowAllWindowsAzureIps', params);

}

public async deployFarm(
Expand Down Expand Up @@ -396,10 +393,19 @@ export class AzureDeployerService implements IGBInstallationDeployer {
await this.createStorage(name, storageServer, storageName, instance.cloudLocation);
instance.storageUsername = administratorLogin;
instance.storagePassword = administratorPassword;
instance.storageName = storageServer;
instance.storageName = storageName;
instance.storageDialect = 'mssql';
instance.storageServer = `${storageServer}.database.windows.net`;

GBLog.info(`Deploying Search...`);
const searchName = `${name}-search`.toLowerCase();
await this.createSearch(name, searchName, instance.cloudLocation);
const searchKeys = await this.searchClient.adminKeys.get(name, searchName);
instance.searchHost = `${searchName}.search.windows.net`;
instance.searchIndex = 'azuresql-index';
instance.searchIndexer = 'azuresql-indexer';
instance.searchKey = searchKeys.primaryKey;

GBLog.info(`Deploying Speech...`);
const speech = await this.createSpeech(name, `${name}-speech`, instance.cloudLocation);
keys = await this.cognitiveClient.accounts.listKeys(name, speech.name);
Expand All @@ -409,7 +415,6 @@ export class AzureDeployerService implements IGBInstallationDeployer {
GBLog.info(`Deploying Text Analytics...`);
const textAnalytics = await this.createTextAnalytics(name, `${name}-textanalytics`, instance.cloudLocation);
keys = await this.cognitiveClient.accounts.listKeys(name, textAnalytics.name);

instance.textAnalyticsEndpoint = textAnalytics.endpoint.replace(`/text/analytics/v2.0`, '');
instance.textAnalyticsKey = keys.key1;

Expand All @@ -419,17 +424,6 @@ export class AzureDeployerService implements IGBInstallationDeployer {
instance.spellcheckerKey = keys.key1;
instance.spellcheckerEndpoint = spellChecker.endpoint;


GBLog.info(`Deploying Search...`);
const searchName = `${name}-search`.toLowerCase();
await this.createSearch(name, searchName, instance.cloudLocation);
const searchKeys = await this.searchClient.adminKeys.get(name, searchName);
instance.searchHost = `${searchName}.search.windows.net`;
instance.searchIndex = 'azuresql-index';
instance.searchIndexer = 'azuresql-indexer';
instance.searchKey = searchKeys.primaryKey;
await this.deployer.rebuildIndex(instance, this.getKBSearchSchema(instance.searchIndex));

GBLog.info(`Deploying NLP...`);
const nlp = await this.createNLP(name, `${name}-nlp`, instance.cloudLocation);
const nlpa = await this.createNLPAuthoring(name, `${name}-nlpa`, instance.cloudLocation);
Expand All @@ -441,7 +435,7 @@ export class AzureDeployerService implements IGBInstallationDeployer {
instance.nlpEndpoint = nlp.endpoint;
instance.nlpKey = keys.key1;
instance.nlpAppId = nlpAppId;

GBLog.info(`Deploying Bot...`);
instance.botEndpoint = this.defaultEndPoint;

Expand All @@ -459,7 +453,7 @@ export class AzureDeployerService implements IGBInstallationDeployer {

GBLog.info('Opening your browser with default.gbui...');
const opn = require('opn');
opn(`https://${serverName}.azurewebsites.net`);
opn(`http://localhost:4242`);

return instance;
}
Expand Down Expand Up @@ -591,7 +585,18 @@ export class AzureDeployerService implements IGBInstallationDeployer {
fullyQualifiedDomainName: serverName
};

return await this.storageClient.servers.createOrUpdate(group, name, params);
const database = await this.storageClient.servers.createOrUpdate(group, name, params);

// AllowAllWindowsAzureIps must be created that way, so the Azure Search can
// access SQL Database to index its contents.

const paramsFirewall = {
startIpAddress: '0.0.0.0',
endIpAddress: '0.0.0.0'
};
await this.storageClient.firewallRules.createOrUpdate(group, name, 'AllowAllWindowsAzureIps', paramsFirewall);

return database;
}

public async createApplication(token: string, name: string) {
Expand Down Expand Up @@ -668,7 +673,19 @@ export class AzureDeployerService implements IGBInstallationDeployer {

const body = JSON.stringify(parameters);
const apps = await this.makeNlpRequest(location, authoringKey, undefined, 'GET', 'apps');
const app = JSON.parse(apps.bodyAsText).filter(x => x.name === name)[0];

let app = null;
if (apps.bodyAsText && apps.bodyAsText !== '[]') {
const result = JSON.parse(apps.bodyAsText)
if (result.error) {
if (result.error.code !== "401") {
throw new Error(result.error);
}
}
else {
app = result.filter(x => x.name === name)[0];
}
}
let id: string;
if (!app) {
const res = await this.makeNlpRequest(location, authoringKey, body, 'POST', 'apps');
Expand Down Expand Up @@ -847,7 +864,7 @@ export class AzureDeployerService implements IGBInstallationDeployer {
}
};
const server = await this.webSiteClient.webApps.createOrUpdate(group, name, parameters);

const siteLogsConfig: SiteLogsConfig = {
applicationLogs: {
fileSystem: { level: 'Error' }
Expand Down
2 changes: 1 addition & 1 deletion packages/core.gbapp/services/GBConfigService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ export class GBConfigService {
value = undefined;
break;
case 'STORAGE_SYNC':
value = 'false';
value = 'true';
break;
case 'STORAGE_SYNC_ALTER':
value = 'false';
Expand Down
35 changes: 21 additions & 14 deletions packages/core.gbapp/services/GBCoreService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,8 @@ export class GBCoreService implements IGBCoreService {
constructor() {
this.adminService = new GBAdminService(this);
}
public async ensureInstances(instances: IGBInstance[], bootInstance: any, core: IGBCoreService) {}
public async ensureInstances(instances: IGBInstance[], bootInstance: any, core: IGBCoreService) { }

/**
* Gets database config and connect to storage. Currently two databases
* are available: SQL Server and SQLite.
Expand Down Expand Up @@ -134,8 +134,8 @@ export class GBCoreService implements IGBCoreService {
const logging: boolean | Function =
GBConfigService.get('STORAGE_LOGGING') === 'true'
? (str: string): void => {
GBLog.info(str);
}
GBLog.info(str);
}
: false;

const encrypt: boolean = GBConfigService.get('STORAGE_ENCRYPT') === 'true';
Expand Down Expand Up @@ -168,7 +168,7 @@ export class GBCoreService implements IGBCoreService {
this.sequelize = new Sequelize(database, username, password, sequelizeOptions);

// Specifies custom setup for MSFT...

if (this.dialect === 'mssql') {
this.queryGenerator = this.sequelize.getQueryInterface().QueryGenerator;
// tslint:disable:no-unsafe-any
Expand Down Expand Up @@ -201,15 +201,16 @@ export class GBCoreService implements IGBCoreService {
}
}


/**
* Syncronizes structure between model and tables in storage.
*/
* Syncronizes structure between model and tables in storage.
*/
public async syncDatabaseStructure() {
if (GBConfigService.get('STORAGE_SYNC') === 'true') {
const alter = GBConfigService.get('STORAGE_SYNC_ALTER') === 'true';
GBLog.info('Syncing database...');

return this.sequelize.sync({
return await this.sequelize.sync({
alter: alter,
force: false // Keep it false this due to data loss danger.
});
Expand Down Expand Up @@ -274,7 +275,8 @@ export class GBCoreService implements IGBCoreService {
* full base environment.
*/
public async writeEnv(instance: IGBInstance) {
const env = `ADDITIONAL_DEPLOY_PATH=
const env = `
ADDITIONAL_DEPLOY_PATH=
ADMIN_PASS=${instance.adminPass}
BOT_ID=${instance.botId}
CLOUD_SUBSCRIPTIONID=${instance.cloudSubscriptionId}
Expand All @@ -290,6 +292,7 @@ STORAGE_NAME=${instance.storageName}
STORAGE_USERNAME=${instance.storageUsername}
STORAGE_PASSWORD=${instance.storagePassword}
STORAGE_SYNC=true
ENDPOINT_UPDATE=true
`;

fs.writeFileSync('.env', env);
Expand Down Expand Up @@ -499,15 +502,19 @@ STORAGE_SYNC=true
GBLog.info(`Deploying cognitive infrastructure (on the cloud / on premises)...`);
try {
const { instance, credentials, subscriptionId } = await StartDialog.createBaseInstance(installationDeployer);
installationDeployer['core'] = this;
const changedInstance = await installationDeployer.deployFarm(
proxyAddress,
instance,
credentials,
subscriptionId
);
core.writeEnv(changedInstance);
GBLog.info(`File .env written, starting General Bots...`);
await this.writeEnv(changedInstance);
GBConfigService.init();

GBLog.info(`File .env written. Preparing storage and search for the first time...`);
await this.openStorageFrontier(installationDeployer);
await this.initStorage();

return changedInstance;
} catch (error) {
Expand Down Expand Up @@ -639,9 +646,9 @@ STORAGE_SYNC=true
value = instance['dataValues'][name];
if (value === null) {
const minBoot = GBServer.globals.minBoot as any;
if (minBoot.instance && minBoot.instance.datavalues){
if (minBoot.instance && minBoot.instance.datavalues) {
value = minBoot.instance.datavalues[name];
}
}
}
}

Expand Down Expand Up @@ -678,7 +685,7 @@ STORAGE_SYNC=true
options
);
GBLog.info(`Running .gbdialog word ${item.name} on:${item.schedule}...`);
} catch (error) {}
} catch (error) { }
});
}
} catch (error) {
Expand Down
2 changes: 1 addition & 1 deletion packages/core.gbapp/services/GBDeployer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -738,7 +738,7 @@ export class GBDeployer implements IGBDeployer {

// If it is a 404 there is nothing to delete as it is the first creation.

if (err.code !== 404 || err.code !== "OperationNotAllowed") {
if (err.code !== 404 && err.code !== "OperationNotAllowed") {
throw err;
}
}
Expand Down
4 changes: 2 additions & 2 deletions packages/core.gbapp/services/GBMinService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -635,7 +635,7 @@ export class GBMinService {
// If there is WhatsApp configuration specified, initialize
// infrastructure objects.

if (min.instance.whatsappServiceKey !== null) {
if (min.instance.whatsappServiceUrl !== null) {
min.whatsAppDirectLine = new WhatsappDirectLine(
min,
min.botId,
Expand All @@ -647,7 +647,7 @@ export class GBMinService {
await min.whatsAppDirectLine.setup(true);
} else {
const minBoot = GBServer.globals.minBoot as any;
if (minBoot.instance.whatsappServiceKey) {
if (minBoot.instance.whatsappServiceUrl) {
min.whatsAppDirectLine = new WhatsappDirectLine(
min,
min.botId,
Expand Down
1 change: 1 addition & 0 deletions packages/whatsapp.gblib/services/WhatsappDirectLine.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ export class WhatsappDirectLine extends GBService {
}

public async setup(setUrl) {

this.directLineClient =
new Swagger({
spec: JSON.parse(fs.readFileSync('directline-3.0.json', 'utf8')),
Expand Down
8 changes: 6 additions & 2 deletions src/app.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ export class GBServer {
GBLog.verbose(`Error initializing storage: ${error}`);
GBServer.globals.bootInstance =
await core.createBootInstance(core, azureDeployer, GBServer.globals.publicAddress);
await core.initStorage();

}

core.ensureAdminIsSecured();
Expand All @@ -150,6 +150,7 @@ export class GBServer {
GBLog.info(`Deploying packages...`);
GBServer.globals.sysPackages = await core.loadSysPackages(core);
await core.checkStorage(azureDeployer);
await core.syncDatabaseStructure();
await deployer.deployPackages(core, server, GBServer.globals.appPackages);

GBLog.info(`Publishing instances...`);
Expand All @@ -160,6 +161,7 @@ export class GBServer {
);

if (instances.length === 0) {

const instance = await importer.importIfNotExistsBotPackage(
GBConfigService.get('BOT_ID'),
'boot.gbot',
Expand All @@ -168,6 +170,8 @@ export class GBServer {
);
await deployer.deployBotFull(instance, GBServer.globals.publicAddress);
instances.push(instance);

await azureDeployer['runSearch'](instance);
}

GBServer.globals.bootInstance = instances[0];
Expand All @@ -186,7 +190,7 @@ export class GBServer {
}

GBLog.info(`The Bot Server is in RUNNING mode...`);

// Opens Navigator.

// TODO: Config: core.openBrowserInDevelopment();
Expand Down

0 comments on commit b0d45c3

Please sign in to comment.