From 726bde596fed649ec17da93de7517c830dc05e90 Mon Sep 17 00:00:00 2001 From: Justin Wilaby Date: Wed, 19 Dec 2018 13:26:55 -0800 Subject: [PATCH] #1186 - Integrated APIs to retrieve cosmos dbs from azure --- .../src/data/sagas/servicesExplorerSagas.ts | 1 - .../getStartedWithCSDialog.tsx | 2 +- .../src/commands/connectedServiceCommands.ts | 5 + packages/app/main/src/main.ts | 2 +- .../src/services/azureManagementApiService.ts | 10 +- .../main/src/services/cosmosDbApiService.ts | 125 ++++++++++++++++++ .../app/main/src/services/qnaApiService.ts | 3 +- .../src/services/storageAccountApiService.ts | 2 +- 8 files changed, 141 insertions(+), 9 deletions(-) create mode 100644 packages/app/main/src/services/cosmosDbApiService.ts diff --git a/packages/app/client/src/data/sagas/servicesExplorerSagas.ts b/packages/app/client/src/data/sagas/servicesExplorerSagas.ts index 56aad624f..a265cd3c7 100644 --- a/packages/app/client/src/data/sagas/servicesExplorerSagas.ts +++ b/packages/app/client/src/data/sagas/servicesExplorerSagas.ts @@ -248,7 +248,6 @@ function* openAddConnectedServiceContextMenu(action: ConnectedServiceAction

{ `You have not signed up for a QnA Maker account under ${ this.props.authenticatedUser }. ` } - Get started with QnA Maker + Get started with QnA Maker

{ ' Alternatively, you can ' } diff --git a/packages/app/main/src/commands/connectedServiceCommands.ts b/packages/app/main/src/commands/connectedServiceCommands.ts index 20580bea4..516c0511d 100644 --- a/packages/app/main/src/commands/connectedServiceCommands.ts +++ b/packages/app/main/src/commands/connectedServiceCommands.ts @@ -1,6 +1,7 @@ import { CommandRegistry } from '@bfemulator/sdk-shared'; import { IConnectedService, ServiceTypes } from 'botframework-config/lib/schema'; import { SharedConstants } from '@bfemulator/app-shared'; +import { CosmosDbApiService } from '../services/cosmosDbApiService'; import { StorageAccountApiService } from '../services/storageAccountApiService'; import { LuisApi } from '../services/luisApiService'; import { QnaApiService } from '../services/qnaApiService'; @@ -27,6 +28,10 @@ export function registerCommands(commandRegistry: CommandRegistry) { it = StorageAccountApiService.getBlobStorageServices(armToken); break; + case ServiceTypes.CosmosDB: + it = CosmosDbApiService.getCosmosDbServices(armToken); + break; + default: throw new TypeError(`The ServiceTypes ${serviceType} is not a known service type`); } diff --git a/packages/app/main/src/main.ts b/packages/app/main/src/main.ts index e1a1db6cc..78234d9a2 100644 --- a/packages/app/main/src/main.ts +++ b/packages/app/main/src/main.ts @@ -161,7 +161,7 @@ AppUpdater.on('download-progress', async (info: ProgressInfo) => { } }); -AppUpdater.on('error', async (err: Error, message: string) => { +AppUpdater.on('error', async (err: Error, message: string = '') => { // TODO - localization AppMenuBuilder.refreshAppUpdateMenu(); // TODO - Send to debug.txt / error dump file diff --git a/packages/app/main/src/services/azureManagementApiService.ts b/packages/app/main/src/services/azureManagementApiService.ts index 609a6cc40..69f2b9005 100644 --- a/packages/app/main/src/services/azureManagementApiService.ts +++ b/packages/app/main/src/services/azureManagementApiService.ts @@ -73,7 +73,7 @@ export enum Provider { ApplicationInsights = 'microsoft.insights', BotService = 'microsoft.botservice', CognitiveServices = 'Microsoft.CognitiveServices', - CosmosDB = 'microsoft.documentdb', + CosmosDB = 'Microsoft.DocumentDB', Storage = 'Microsoft.Storage' } @@ -99,7 +99,8 @@ export class AzureManagementApiService { return { headers: { Authorization: `Bearer ${ armToken }`, - Accept: 'application/json, text/plain, */*' + Accept: 'application/json, text/plain, */*', + 'x-ms-date': new Date().toUTCString() } }; } @@ -177,9 +178,10 @@ export class AzureManagementApiService { * @param armToken * @param accounts * @param apiVersion + * @param responseProperty */ public static async getKeysForAccounts - (armToken: string, accounts: AzureResource[], apiVersion: string): Promise { + (armToken: string, accounts: AzureResource[], apiVersion: string, responseProperty: string): Promise { const keys: any[] = []; const req = AzureManagementApiService.getRequestInit(armToken); const url = `${ baseUrl }{id}/listKeys?api-version=${ apiVersion }`; @@ -191,7 +193,7 @@ export class AzureManagementApiService { const keyResponse: Response = keyResponses[i]; if (keyResponse.ok) { const keyResponseJson = await keyResponse.json(); - const key = (keyResponseJson.keys || keyResponseJson.key1); + const key = keyResponseJson[responseProperty]; if (key && '' + key) { // Excludes empty strings and empty arrays keys[i] = key; // maintain index position - do not "push" } diff --git a/packages/app/main/src/services/cosmosDbApiService.ts b/packages/app/main/src/services/cosmosDbApiService.ts new file mode 100644 index 000000000..28266558d --- /dev/null +++ b/packages/app/main/src/services/cosmosDbApiService.ts @@ -0,0 +1,125 @@ +import { ServiceCodes } from '@bfemulator/app-shared/built'; +import { CosmosDbService } from 'botframework-config'; +import * as crypto from 'crypto'; +import { AccountIdentifier, AzureManagementApiService, AzureResource, Provider } from './azureManagementApiService'; + +export class CosmosDbApiService { + + public static* getCosmosDbServices(armToken: string): IterableIterator { + const payload = { services: [], code: ServiceCodes.OK }; + + // 1. get a list of subscriptions for the user + yield { label: 'Retrieving subscriptions from Azure…', progress: 10 }; + const subs = yield AzureManagementApiService.getSubscriptions(armToken); + if (!subs) { + payload.code = ServiceCodes.AccountNotFound; + return payload; + } + + // 2. Retrieve a list of database accounts + yield { label: 'Retrieving account data from Azure…', progress: 25 }; + const databaseAccounts: AzureResource[] = yield AzureManagementApiService + .getAzureResource(armToken, subs, Provider.CosmosDB, AccountIdentifier.CosmosDb); + if (!databaseAccounts) { + payload.code = ServiceCodes.AccountNotFound; + return payload; + } + + // 3. Retrieve a list of keys + yield { label: 'Retrieving access keys from Azure…', progress: 45 }; + const keys: string[] = yield AzureManagementApiService + .getKeysForAccounts(armToken, databaseAccounts, '2015-04-08', 'primaryMasterKey'); + if (!keys) { + payload.code = ServiceCodes.Error; + return payload; + } + + // 4. retrieve a list of CosmosDBs + yield { label: 'Retrieving Cosmos DBs from Azure…', progress: 65 }; + const cosmosDbRequests = databaseAccounts.map((account, index) => { + const req = AzureManagementApiService.getRequestInit(armToken); + req.headers['x-ms-version'] = '2017-02-22'; + (req.headers as any).Authorization = getAuthorizationTokenUsingMasterKey(keys[index]); + return fetch(`https://${ account.name }.documents.azure.com/dbs`, req); + }); + const cosmosDbResponses: Response[] = yield Promise.all(cosmosDbRequests); + const cosmosDbs = []; + let i = cosmosDbResponses.length; + while (i--) { + const response = cosmosDbResponses[i]; + if (!response.ok) { + continue; + } + const responseJson = yield response.json(); + if ((responseJson.Databases || [].length)) { + responseJson.Databases.forEach(db => cosmosDbs.push({ db, account: databaseAccounts[i] })); + } + } + + // 5. Retrieve a list of collections - please note that this is + // an endpoint used in the azure portal for retrieving collections + // It does not appear to be documented anywhere and was used because + // the documented API was returning a 401 no matter what params and + // auth headers where used. + yield { label: 'Retrieving collections from Azure…', progress: 85 }; + const collectionRequests = cosmosDbs.map(info => { + const { db, account } = info; + const { id, name, properties, subscriptionId } = account as AzureResource; + const req = AzureManagementApiService.getRequestInit(armToken); + const resourceGroup = id.split('/')[4]; + const params = [ + `resourceUrl=${ properties.documentEndpoint }dbs/${ db._rid }/colls/`, + `rid=${ db._rid }`, + 'rtype=colls', + `sid=${ subscriptionId }`, + `rg=${ resourceGroup }`, + `dba=${ name }` + ]; + const proxyUrl = `https://main.documentdb.ext.azure.com/api/RuntimeProxy?${ params.join('&') }`; + return fetch(proxyUrl, req); + }); + + const collectionResponses = yield Promise.all(collectionRequests); + i = collectionResponses.length; + while (i--) { + const collectionResponse: Response = collectionResponses[i]; + if (!collectionResponse.ok) { + continue; + } + const { db, account } = cosmosDbs[i]; + const collectionResponseJson = yield collectionResponse.json(); + (collectionResponseJson.DocumentCollections || []).forEach(collection => { + payload.services.push(buildServiceModel(account, db, collection)); + }); + } + return payload; + } +} + +function buildServiceModel +(account: AzureResource, cosmosDb: AzureResource, collection: { id: string }): CosmosDbService { + const service = new CosmosDbService(); + service.database = cosmosDb.id; + service.collection = collection.id; + service.endpoint = account.properties.documentEndpoint; + service.serviceName = service.name = collection.id; + service.resourceGroup = account.id.split('/')[4]; + service.subscriptionId = account.subscriptionId; + service.tenantId = account.tenantId; + + return service; +} + +function getAuthorizationTokenUsingMasterKey(masterKey: string = '', resourceId: string = ''): string { + const key = Buffer.from(masterKey, 'base64'); + const text = 'get\n' + + 'dbs\n' + + resourceId + '\n' + + new Date().toUTCString().toLowerCase() + '\n' + + '' + '\n'; + + const body = Buffer.from(text); + const signature = crypto.createHmac('sha256', key).update(body).digest('base64'); + + return encodeURIComponent('type=master&ver=1.0&sig=' + signature); +} diff --git a/packages/app/main/src/services/qnaApiService.ts b/packages/app/main/src/services/qnaApiService.ts index dfdc6de85..f132db7e7 100644 --- a/packages/app/main/src/services/qnaApiService.ts +++ b/packages/app/main/src/services/qnaApiService.ts @@ -43,7 +43,8 @@ export class QnaApiService { // 3. Retrieve the keys for each account yield { label: 'Retrieving keys from Azure…', progress: 65 }; - const keys: string[] = yield AzureManagementApiService.getKeysForAccounts(armToken, accounts, '2017-04-18'); + const keys: string[] = yield AzureManagementApiService + .getKeysForAccounts(armToken, accounts, '2017-04-18', 'key1'); if (!keys) { payload.code = ServiceCodes.Error; return payload; diff --git a/packages/app/main/src/services/storageAccountApiService.ts b/packages/app/main/src/services/storageAccountApiService.ts index 729a39b9d..c40d35bc9 100644 --- a/packages/app/main/src/services/storageAccountApiService.ts +++ b/packages/app/main/src/services/storageAccountApiService.ts @@ -86,7 +86,7 @@ export class StorageAccountApiService { yield { label: 'Retrieving Access Keys from Azure…', progress: 95 }; // Do not retrieve keys for accounts without blob containers const keys: KeyEntry[][] = yield AzureManagementApiService - .getKeysForAccounts(armToken, blobContainerInfos.map(info => info.account), '2018-07-01'); + .getKeysForAccounts(armToken, blobContainerInfos.map(info => info.account), '2018-07-01', 'keys'); // Build the BlobStorageService objects i = keys.length; while (i--) {