#1186 - Integrated APIs to retrieve cosmos dbs from azure

packages/app/client/src/data/sagas/servicesExplorerSagas.ts

@@ -248,7 +248,6 @@ function* openAddConnectedServiceContextMenu(action: ConnectedServiceAction<Conn
   const { id: serviceType } = response;
   action.payload.serviceType = serviceType;
   if (serviceType === ServiceTypes.Generic ||
-    serviceType === ServiceTypes.CosmosDB ||
     serviceType === ServiceTypes.AppInsights) {
     yield* launchConnectedServiceEditor(action);
   } else {

packages/app/client/src/ui/dialogs/getStartedWithCSDialog/getStartedWithCSDialog.tsx

@@ -183,7 +183,7 @@ export class GetStartedWithCSDialog extends Component<GetStartedWithCSDialogProp
         </p>
         <p>
           { `You have not signed up for a QnA Maker account under ${ this.props.authenticatedUser }. ` }
-          <a href="javascript:void(0)">Get started with QnA Maker</a>
+          <a href="https://aka.ms/bot-framework-emulator-qna-docs-home">Get started with QnA Maker</a>
         </p>
         <p>
           { ' Alternatively, you can ' }

packages/app/main/src/commands/connectedServiceCommands.ts

@@ -1,6 +1,7 @@
 import { CommandRegistry } from '@bfemulator/sdk-shared';
 import { IConnectedService, ServiceTypes } from 'botframework-config/lib/schema';
 import { SharedConstants } from '@bfemulator/app-shared';
+import { CosmosDbApiService } from '../services/cosmosDbApiService';
 import { StorageAccountApiService } from '../services/storageAccountApiService';
 import { LuisApi } from '../services/luisApiService';
 import { QnaApiService } from '../services/qnaApiService';
@@ -27,6 +28,10 @@ export function registerCommands(commandRegistry: CommandRegistry) {
           it = StorageAccountApiService.getBlobStorageServices(armToken);
           break;
 
+        case ServiceTypes.CosmosDB:
+          it = CosmosDbApiService.getCosmosDbServices(armToken);
+          break;
+
         default:
           throw new TypeError(`The ServiceTypes ${serviceType} is not a known service type`);
       }

packages/app/main/src/main.ts

@@ -161,7 +161,7 @@ AppUpdater.on('download-progress', async (info: ProgressInfo) => {
   }
 });
 
-AppUpdater.on('error', async (err: Error, message: string) => {
+AppUpdater.on('error', async (err: Error, message: string = '') => {
   // TODO - localization
   AppMenuBuilder.refreshAppUpdateMenu();
   // TODO - Send to debug.txt / error dump file

packages/app/main/src/services/azureManagementApiService.ts

@@ -73,7 +73,7 @@ export enum Provider {
   ApplicationInsights = 'microsoft.insights',
   BotService = 'microsoft.botservice',
   CognitiveServices = 'Microsoft.CognitiveServices',
-  CosmosDB = 'microsoft.documentdb',
+  CosmosDB = 'Microsoft.DocumentDB',
   Storage = 'Microsoft.Storage'
 }
 
@@ -99,7 +99,8 @@ export class AzureManagementApiService {
     return {
       headers: {
         Authorization: `Bearer ${ armToken }`,
-        Accept: 'application/json, text/plain, */*'
+        Accept: 'application/json, text/plain, */*',
+        'x-ms-date': new Date().toUTCString()
       }
     };
   }
@@ -177,9 +178,10 @@ export class AzureManagementApiService {
    * @param armToken
    * @param accounts
    * @param apiVersion
+   * @param responseProperty
    */
   public static async getKeysForAccounts
-  (armToken: string, accounts: AzureResource[], apiVersion: string): Promise<string[]> {
+  (armToken: string, accounts: AzureResource[], apiVersion: string, responseProperty: string): Promise<string[]> {
     const keys: any[] = [];
     const req = AzureManagementApiService.getRequestInit(armToken);
     const url = `${ baseUrl }{id}/listKeys?api-version=${ apiVersion }`;
@@ -191,7 +193,7 @@ export class AzureManagementApiService {
       const keyResponse: Response = keyResponses[i];
       if (keyResponse.ok) {
         const keyResponseJson = await keyResponse.json();
-        const key = (keyResponseJson.keys || keyResponseJson.key1);
+        const key = keyResponseJson[responseProperty];
         if (key && '' + key) { // Excludes empty strings and empty arrays
           keys[i] = key; // maintain index position - do not "push"
         }

packages/app/main/src/services/cosmosDbApiService.ts

@@ -0,0 +1,125 @@
+import { ServiceCodes } from '@bfemulator/app-shared/built';
+import { CosmosDbService } from 'botframework-config';
+import * as crypto from 'crypto';
+import { AccountIdentifier, AzureManagementApiService, AzureResource, Provider } from './azureManagementApiService';
+
+export class CosmosDbApiService {
+
+  public static* getCosmosDbServices(armToken: string): IterableIterator<any> {
+    const payload = { services: [], code: ServiceCodes.OK };
+
+    // 1. get a list of subscriptions for the user
+    yield { label: 'Retrieving subscriptions from Azure…', progress: 10 };
+    const subs = yield AzureManagementApiService.getSubscriptions(armToken);
+    if (!subs) {
+      payload.code = ServiceCodes.AccountNotFound;
+      return payload;
+    }
+
+    // 2. Retrieve a list of database accounts
+    yield { label: 'Retrieving account data from Azure…', progress: 25 };
+    const databaseAccounts: AzureResource[] = yield AzureManagementApiService
+      .getAzureResource(armToken, subs, Provider.CosmosDB, AccountIdentifier.CosmosDb);
+    if (!databaseAccounts) {
+      payload.code = ServiceCodes.AccountNotFound;
+      return payload;
+    }
+
+    // 3. Retrieve a list of keys
+    yield { label: 'Retrieving access keys from Azure…', progress: 45 };
+    const keys: string[] = yield AzureManagementApiService
+      .getKeysForAccounts(armToken, databaseAccounts, '2015-04-08', 'primaryMasterKey');
+    if (!keys) {
+      payload.code = ServiceCodes.Error;
+      return payload;
+    }
+
+    // 4. retrieve a list of CosmosDBs
+    yield { label: 'Retrieving Cosmos DBs from Azure…', progress: 65 };
+    const cosmosDbRequests = databaseAccounts.map((account, index) => {
+      const req = AzureManagementApiService.getRequestInit(armToken);
+      req.headers['x-ms-version'] = '2017-02-22';
+      (req.headers as any).Authorization = getAuthorizationTokenUsingMasterKey(keys[index]);
+      return fetch(`https://${ account.name }.documents.azure.com/dbs`, req);
+    });
+    const cosmosDbResponses: Response[] = yield Promise.all(cosmosDbRequests);
+    const cosmosDbs = [];
+    let i = cosmosDbResponses.length;
+    while (i--) {
+      const response = cosmosDbResponses[i];
+      if (!response.ok) {
+        continue;
+      }
+      const responseJson = yield response.json();
+      if ((responseJson.Databases || [].length)) {
+        responseJson.Databases.forEach(db => cosmosDbs.push({ db, account: databaseAccounts[i] }));
+      }
+    }
+
+    // 5. Retrieve a list of collections - please note that this is
+    // an endpoint used in the azure portal for retrieving collections
+    // It does not appear to be documented anywhere and was used because
+    // the documented API was returning a 401 no matter what params and
+    // auth headers where used.
+    yield { label: 'Retrieving collections from Azure…', progress: 85 };
+    const collectionRequests = cosmosDbs.map(info => {
+      const { db, account } = info;
+      const { id, name, properties, subscriptionId } = account as AzureResource;
+      const req = AzureManagementApiService.getRequestInit(armToken);
+      const resourceGroup = id.split('/')[4];
+      const params = [
+        `resourceUrl=${ properties.documentEndpoint }dbs/${ db._rid }/colls/`,
+        `rid=${ db._rid }`,
+        'rtype=colls',
+        `sid=${ subscriptionId }`,
+        `rg=${ resourceGroup }`,
+        `dba=${ name }`
+      ];
+      const proxyUrl = `https://main.documentdb.ext.azure.com/api/RuntimeProxy?${ params.join('&') }`;
+      return fetch(proxyUrl, req);
+    });
+
+    const collectionResponses = yield Promise.all(collectionRequests);
+    i = collectionResponses.length;
+    while (i--) {
+      const collectionResponse: Response = collectionResponses[i];
+      if (!collectionResponse.ok) {
+        continue;
+      }
+      const { db, account } = cosmosDbs[i];
+      const collectionResponseJson = yield collectionResponse.json();
+      (collectionResponseJson.DocumentCollections || []).forEach(collection => {
+        payload.services.push(buildServiceModel(account, db, collection));
+      });
+    }
+    return payload;
+  }
+}
+
+function buildServiceModel
+(account: AzureResource, cosmosDb: AzureResource, collection: { id: string }): CosmosDbService {
+  const service = new CosmosDbService();
+  service.database = cosmosDb.id;
+  service.collection = collection.id;
+  service.endpoint = account.properties.documentEndpoint;
+  service.serviceName = service.name = collection.id;
+  service.resourceGroup = account.id.split('/')[4];
+  service.subscriptionId = account.subscriptionId;
+  service.tenantId = account.tenantId;
+
+  return service;
+}
+
+function getAuthorizationTokenUsingMasterKey(masterKey: string = '', resourceId: string = ''): string {
+  const key = Buffer.from(masterKey, 'base64');
+  const text = 'get\n' +
+    'dbs\n' +
+    resourceId + '\n' +
+    new Date().toUTCString().toLowerCase() + '\n' +
+    '' + '\n';
+
+  const body = Buffer.from(text);
+  const signature = crypto.createHmac('sha256', key).update(body).digest('base64');
+
+  return encodeURIComponent('type=master&ver=1.0&sig=' + signature);
+}

packages/app/main/src/services/qnaApiService.ts

@@ -43,7 +43,8 @@ export class QnaApiService {
 
     // 3. Retrieve the keys for each account
     yield { label: 'Retrieving keys from Azure…', progress: 65 };
-    const keys: string[] = yield AzureManagementApiService.getKeysForAccounts(armToken, accounts, '2017-04-18');
+    const keys: string[] = yield AzureManagementApiService
+      .getKeysForAccounts(armToken, accounts, '2017-04-18', 'key1');
     if (!keys) {
       payload.code = ServiceCodes.Error;
       return payload;

packages/app/main/src/services/storageAccountApiService.ts

@@ -86,7 +86,7 @@ export class StorageAccountApiService {
     yield { label: 'Retrieving Access Keys from Azure…', progress: 95 };
     // Do not retrieve keys for accounts without blob containers
     const keys: KeyEntry[][] = yield AzureManagementApiService
-      .getKeysForAccounts(armToken, blobContainerInfos.map(info => info.account), '2018-07-01');
+      .getKeysForAccounts(armToken, blobContainerInfos.map(info => info.account), '2018-07-01', 'keys');
     // Build the BlobStorageService objects
     i = keys.length;
     while (i--) {