fix: composer proxy support (#6931)

* add proxy setting for all request

* update node-fetch

* update the package manage

* update the pav yarn.lock

* update the bf-dialog

* fix the package version

* update the package

* udpate the axios proxy support

* remove node-fetch-with-proxy

* update template readme api

* update the orchestrator version

* fix unit tests

* fix unit tests

* update bf-orchestrator

* use shared

* update the package.json

* use fetch to replace axios

* fix conflict

* fix unit test

Composer/packages/electron-server/package.json

@@ -70,6 +70,7 @@
     "format-message-generate-id": "^6.2.3",
     "fs-extra": "^9.0.0",
     "lodash": "^4.17.19",
+    "node-fetch": "2.6.1",
     "semver": "7.3.2",
     "uuid": "^8.3.1"
   },

Composer/packages/electron-server/src/auth/oneAuthService.ts

@@ -4,8 +4,8 @@ import path from 'path';
 
 import { AzureTenant, ElectronAuthParameters } from '@botframework-composer/types';
 import { app } from 'electron';
-import fetch from 'node-fetch';
 
+import fetch from '../utility/fetch';
 import ElectronWindow from '../electronWindow';
 import { isLinux, isMac } from '../utility/platform';
 import logger from '../utility/logger';

Composer/packages/electron-server/src/utility/fetch.ts

@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+import fetch, { RequestInfo, RequestInit, Response } from 'node-fetch';
+import { proxyAgent } from '@bfc/shared';
+
+const fetchWithProxy = (url: RequestInfo, init?: RequestInit): Promise<Response> => {
+  const agent = proxyAgent();
+  const options = agent ? { agent, ...init } : init;
+  return fetch(url, options);
+};
+
+export default fetchWithProxy;

Composer/packages/lib/shared/package.json

@@ -38,6 +38,7 @@
   "dependencies": {
     "@botframework-composer/types": "*",
     "format-message": "6.2.3",
+    "https-proxy-agent": "^5.0.0",
     "json-schema": "^0.2.5",
     "multimatch": "^5.0.0",
     "nanoid": "^3.1.3",

Composer/packages/lib/shared/src/httpsProxy.ts

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import httpsProxyAgent, { HttpsProxyAgentOptions } from 'https-proxy-agent';
+
+export const proxyAgent = () => {
+  const envProxy = process.env.HTTPS_PROXY || process.env.https_proxy;
+
+  if (!envProxy) return undefined;
+
+  const parsed = new URL(envProxy);
+  const proxyOpt: HttpsProxyAgentOptions = {
+    hostname: parsed.hostname,
+    port: parsed.port,
+  };
+
+  return httpsProxyAgent(proxyOpt);
+};
+
+export const httpsProxy = (config) => {
+  const parsed = new URL(config.url);
+  const protocol = parsed.protocol;
+  if (protocol !== 'https:') {
+    return config;
+  }
+
+  const agent = proxyAgent();
+  if (agent) {
+    config.httpsAgent = agent;
+    //Disable direct proxy
+    config.proxy = false;
+  }
+
+  return config;
+};

Composer/packages/lib/shared/src/index.ts

@@ -28,4 +28,5 @@ export * from './settings';
 export * from './skillsUtils';
 export * from './viewUtils';
 export * from './walkerUtils';
+export * from './httpsProxy';
 export const DialogUtils = dialogUtils;

Composer/packages/server-workers/package.json

@@ -15,7 +15,7 @@
   "author": "benbro@microsoft.com",
   "license": "MIT",
   "dependencies": {
-    "@microsoft/bf-dialog": "4.13.1",
+    "@microsoft/bf-dialog": "4.14.0-dev.20210415.161c029",
     "debug": "^4.3.1",
     "yeoman-environment": "^2.10.3"
   },

Composer/packages/server/package.json

@@ -52,14 +52,12 @@
     "@types/mock-fs": "^3.6.30",
     "@types/morgan": "^1.7.35",
     "@types/node": "^11.9.6",
-    "@types/node-fetch": "^2.5.7",
     "@types/rimraf": "^2.0.2",
     "@types/ws": "^7.4.0",
     "@types/yeoman-environment": "^2.10.2",
     "cross-env": "^7.0.2",
     "eslint": "7.0.0",
     "fs-extra": "7.0.1",
-    "jest-fetch-mock": "^3.0.3",
     "mock-fs": "^4.10.1",
     "nodemon": "^2.0.3",
     "prettier": "2.0.5",
@@ -77,7 +75,7 @@
     "@bfc/server-workers": "*",
     "@bfc/shared": "*",
     "@botframework-composer/types": "*",
-    "@microsoft/bf-dialog": "4.13.1",
+    "@microsoft/bf-dialog": "4.14.0-dev.20210415.161c029",
     "@microsoft/bf-dispatcher": "^4.11.0-beta.20201016.393c6b2",
     "@microsoft/bf-generate-library": "^4.14.0-preview.20210702.31a9d59",
     "@microsoft/bf-lu": "4.14.0-dev.20210602.be805a8",

Composer/packages/server/src/controllers/__tests__/asset.test.ts

@@ -2,7 +2,6 @@
 // Licensed under the MIT License.
 
 import { Request, Response } from 'express';
-import { enableFetchMocks } from 'jest-fetch-mock';
 
 import { AssetController } from '../asset';
 
@@ -28,6 +27,17 @@ describe('get bot project templates', () => {
   });
 });
 
+const mockFetchResponse = {
+  id: 'generator-conversational-core',
+  _rev: '9-82a6c228b16649a143a49974b0adc022',
+  name: 'generator-conversational-core',
+  readme: '# Mock readme markdown',
+};
+
+jest.mock('../../utility/fetch', () => () => {
+  return Promise.resolve({ json: () => mockFetchResponse });
+});
+
 describe('getTemplateReadMe', () => {
   let mockRes: any;
 
@@ -51,16 +61,7 @@ describe('getTemplateReadMe', () => {
     body: {},
   } as Request;
 
-  const mockFetchResponse = {
-    id: 'generator-conversational-core',
-    _rev: '9-82a6c228b16649a143a49974b0adc022',
-    name: 'generator-conversational-core',
-    readme: '# Mock readme markdown',
-  };
-
   it('should return a readMe', async () => {
-    enableFetchMocks();
-    fetchMock.mockResponseOnce(JSON.stringify(mockFetchResponse));
     await AssetController.getTemplateReadMe(mockReq, mockRes);
     expect(mockRes.status).toHaveBeenCalledWith(200);
     expect(mockRes.json).toHaveBeenCalledWith('# Mock readme markdown');

Composer/packages/server/src/controllers/asset.ts

@@ -4,6 +4,7 @@
 import { BotTemplate, emptyBotNpmTemplateName, localTemplateId, QnABotTemplateId } from '@bfc/shared';
 import formatMessage from 'format-message';
 
+import fetch from '../utility/fetch';
 import AssetService from '../services/asset';
 import { getNpmTemplates } from '../utility/npm';
 import log from '../logger';

Composer/packages/server/src/externalContentProvider/__tests__/powerVirtualAgentsProvider.test.ts

@@ -34,7 +34,7 @@ jest.mock('../../services/auth/auth', () => ({
 }));
 
 const mockFetch = jest.fn();
-jest.mock('node-fetch', () => async (...args) => await mockFetch(...args));
+jest.mock('../../utility/fetch', () => async (...args) => await mockFetch(...args));
 
 describe('Power Virtual Agents provider', () => {
   const envBackup = { ...process.env };

Composer/packages/server/src/externalContentProvider/powerVirtualAgentsProvider.ts

@@ -5,8 +5,9 @@ import { createWriteStream } from 'fs';
 import { join } from 'path';
 
 import { ensureDirSync, remove } from 'fs-extra';
-import fetch, { RequestInit } from 'node-fetch';
+import { RequestInit } from 'node-fetch';
 
+import fetch from '../utility/fetch';
 import logger from '../logger';
 import { authService } from '../services/auth/auth';
 

Composer/packages/server/src/models/asset/__tests__/assetManager.test.ts

@@ -2,7 +2,6 @@
 // Licensed under the MIT License.
 
 import rimraf from 'rimraf';
-import { enableFetchMocks } from 'jest-fetch-mock';
 import { BotTemplate } from '@bfc/shared';
 
 import { ExtensionContext } from '../../../models/extension/extensionContext';
@@ -38,6 +37,40 @@ jest.mock('@bfc/server-workers', () => {
   };
 });
 
+const mockFeedResponse1 = {
+  objects: [
+    {
+      package: {
+        name: 'generator-conversational-core',
+        version: '1.0.3',
+        description: 'Preview conversational core package for TESTING ONLY',
+        keywords: ['conversationalcore', 'yeoman-generator'],
+      },
+    },
+  ],
+};
+
+const mockFeedResponse2 = {
+  versions: {
+    '0.0.0': {
+      name: '@microsoft/generator-bot-core-language',
+      version: '0.0.0',
+    },
+    '1.0.0': {
+      name: '@microsoft/generator-bot-core-language',
+      version: '1.0.0',
+    },
+  },
+};
+
+jest.mock('../../../utility/fetch', () => (url: string) => {
+  if (url.includes('conversationalcore')) {
+    return Promise.resolve({ json: () => mockFeedResponse1 });
+  } else {
+    return Promise.resolve({ json: () => mockFeedResponse2 });
+  }
+});
+
 const mockSampleBotPath = Path.join(__dirname, '../../../__mocks__/asset/projects/SampleBot');
 const mockCopyToPath = Path.join(__dirname, '../../../__mocks__/new');
 const locationRef = {
@@ -124,21 +157,6 @@ describe('assetManager', () => {
   });
 
   describe('getFeedContents', () => {
-    const mockFeedResponse = {
-      objects: [
-        {
-          package: {
-            name: 'generator-conversational-core',
-            version: '1.0.3',
-            description: 'Preview conversational core package for TESTING ONLY',
-            keywords: ['conversationalcore', 'yeoman-generator'],
-          },
-        },
-      ],
-    };
-
-    enableFetchMocks();
-    fetchMock.mockResponseOnce(JSON.stringify(mockFeedResponse));
     it('Get contents of a feed and return template array', async () => {
       const assetManager = new AssetManager();
 
@@ -163,21 +181,6 @@ describe('assetManager', () => {
   });
 
   describe('getNpmPackageVersions', () => {
-    const mockFeedResponse = {
-      versions: {
-        '0.0.0': {
-          name: '@microsoft/generator-bot-core-language',
-          version: '0.0.0',
-        },
-        '1.0.0': {
-          name: '@microsoft/generator-bot-core-language',
-          version: '1.0.0',
-        },
-      },
-    };
-
-    enableFetchMocks();
-    fetchMock.mockResponseOnce(JSON.stringify(mockFeedResponse));
     it('Get available versions for a given npm package', async () => {
       const assetManager = new AssetManager();
 

Composer/packages/server/src/models/asset/assetManager.ts

@@ -11,6 +11,7 @@ import { BotTemplate, emptyBotNpmTemplateName, FeedName, QnABotTemplateId } from
 import { ServerWorker } from '@bfc/server-workers';
 import isArray from 'lodash/isArray';
 
+import fetch from '../../utility/fetch';
 import { ExtensionContext } from '../extension/extensionContext';
 import log from '../../logger';
 import { LocalDiskStorage } from '../storage/localDiskStorage';

Composer/packages/server/src/models/bot/feedManager.ts

@@ -1,14 +1,11 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
-import axios from 'axios';
+import fetch from '../../utility/fetch';
 
 // TODO: update feed url
 const feedUrl = 'https://aka.ms/bf-composer-home-feed-v1';
 
 export const getFeedUrl = async (): Promise<any> => {
-  const { data: content } = await axios({
-    method: 'get',
-    url: feedUrl,
-  });
-  return content && typeof content === 'string' ? JSON.parse(content) : content;
+  const response = await fetch(feedUrl);
+  return await response.json();
 };

Composer/packages/server/src/utility/__tests__/npm.test.ts

@@ -3,10 +3,10 @@
 /* eslint-disable no-underscore-dangle */
 import { Readable, Writable } from 'stream';
 
-import fetch from 'node-fetch';
 import { mkdir, remove } from 'fs-extra';
 import tar from 'tar';
 
+import fetch from '../fetch';
 import { search, downloadPackage } from '../npm';
 
 class MockBody extends Readable {
@@ -26,7 +26,7 @@ class MockExtractor extends Writable {
   }
 }
 
-jest.mock('node-fetch', () => jest.fn());
+jest.mock('../fetch', () => jest.fn());
 jest.mock('fs-extra', () => ({
   mkdir: jest.fn(),
   remove: jest.fn(),

Composer/packages/server/src/utility/fetch.ts

@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+import fetch, { RequestInfo, RequestInit, Response } from 'node-fetch';
+import { proxyAgent } from '@bfc/shared';
+
+const fetchWithProxy = (url: RequestInfo, init?: RequestInit): Promise<Response> => {
+  const agent = proxyAgent();
+  const options = agent ? { agent, ...init } : init;
+  return fetch(url, options);
+};
+
+export default fetchWithProxy;

Composer/packages/server/src/utility/httpClient.ts

@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import axios from 'axios';
+import { httpsProxy } from '@bfc/shared';
+
+axios.interceptors.request.use(httpsProxy);
+
+export default axios;

Composer/packages/server/src/utility/npm.ts

@@ -4,10 +4,10 @@
 import { promisify } from 'util';
 
 import { mkdir, remove } from 'fs-extra';
-import fetch from 'node-fetch';
 import tar from 'tar';
 import { BotTemplate, ExtensionSearchResult } from '@botframework-composer/types';
 
+import fetch from '../utility/fetch';
 import logger from '../logger';
 
 const streamPipeline = promisify(require('stream').pipeline);

Composer/packages/tools/language-servers/language-understanding/package.json

@@ -20,7 +20,7 @@
   "dependencies": {
     "@bfc/indexers": "*",
     "@bfc/shared": "*",
-    "@microsoft/bf-cli-command": "^4.11.1",
+    "@microsoft/bf-cli-command": "4.14.0-dev.20210604.3f9ee15",
     "@microsoft/bf-lu": "4.14.0-dev.20210602.be805a8",
     "express": "^4.15.2",
     "monaco-languageclient": "^0.10.0",