build: add script to fetch plugin types (#711)

PR-URL: #711

.circleci/config.yml

@@ -3,6 +3,7 @@
 
 node_modules_cache_key: &node_modules_cache_key node-modules-cache-{{ .Environment.CIRCLE_JOB }}-{{ arch }}-{{ checksum "package.json" }}-{{ checksum "package-lock.json" }}
 test_fixtures_cache_key: &test_fixtures_cache_key test-fixtures-cache-{{ .Environment.CIRCLE_JOB }}-{{ arch }}-{{ checksum "test/fixtures/plugin-fixtures.json" }}
+plugin_types_cache_key: &plugin_types_cache_key plugin-types-cache-{{ .Environment.CIRCLE_JOB }}-{{ arch }}-{{ checksum "src/plugins/types/index.d.ts" }}
 test_env: &test_env
   GCLOUD_TRACE_NEW_CONTEXT: 1
 release_tags: &release_tags
@@ -19,13 +20,19 @@ unit_tests: &unit_tests
         command: npm config set unsafe-perm true
     - restore_cache:
         key: *node_modules_cache_key
+    - restore_cache:
+        key: *plugin_types_cache_key
     - run:
         name: Install modules and dependencies
         command: npm install
     - save_cache:
         key: *node_modules_cache_key
         paths:
           - node_modules
+    - save_cache:
+        key: *plugin_types_cache_key
+        paths:
+          - src/plugins/types
     - run:
         name: Check code style and linting
         command: npm run check

.gitignore

@@ -6,3 +6,5 @@ npm-debug.log
 .vscode/
 build/
 *.tgz
+src/plugins/types/*
+!src/plugins/types/index.d.ts

appveyor.yml

@@ -7,6 +7,7 @@ environment:
   - nodejs_version: "8"
 
 cache:
+  - src/plugins/types -> src/plugins/types/index.d.ts
   - node_modules -> package-lock.json, package.json
 
 # Install scripts. (runs after repo cloning)

package.json

@@ -13,15 +13,16 @@
     "system-test": "npm run script decrypt-service-account-credentials run-system-tests",
     "changelog": "./bin/run-changelog.sh",
     "check-install": "npm run script check-install",
+    "get-plugin-types": "npm run script get-plugin-types",
     "coverage": "npm run script run-unit-tests-with-coverage report-coverage",
     "bump": "./bin/run-bump.sh",
     "check": "gts check",
     "clean": "rimraf build/src",
     "compile-all": "npm run script compile-auto",
     "compile-strict": "npm run script compile-auto-strict",
-    "compile": "npm run script compile-auto compile-auto-strict",
+    "compile": "npm run script get-plugin-types compile-auto compile-auto-strict",
     "fix": "gts fix",
-    "prepare": "npm run script npm-clean compile-es2015 compile-es2015-strict",
+    "prepare": "npm run script npm-clean get-plugin-types compile-es2015 compile-es2015-strict",
     "script": "ts-node -P ./scripts/tsconfig.json ./scripts",
     "license-check": "jsgl --local ."
   },
@@ -50,25 +51,19 @@
   },
   "devDependencies": {
     "@types/builtin-modules": "^2.0.0",
-    "@types/connect": "^3.4.31",
     "@types/continuation-local-storage": "^3.2.1",
-    "@types/express": "^4.11.0",
     "@types/extend": "^3.0.0",
     "@types/glob": "^5.0.32",
-    "@types/hapi": "^16.1.12",
     "@types/is": "0.0.19",
-    "@types/koa": "^2.0.43",
     "@types/methods": "^1.1.0",
     "@types/mocha": "^5.0.0",
     "@types/ncp": "^2.0.1",
     "@types/nock": "^9.1.2",
     "@types/node": "^9.4.6",
     "@types/once": "^1.4.0",
-    "@types/pg": "^7.4.5",
     "@types/pify": "^3.0.0",
     "@types/proxyquire": "^1.3.28",
     "@types/request": "^2.0.8",
-    "@types/restify": "^5.0.7",
     "@types/semver": "^5.4.0",
     "@types/shimmer": "^1.0.1",
     "@types/tmp": "0.0.33",

scripts/get-plugin-types.ts

@@ -0,0 +1,41 @@
+import { flatten, globP, mkdirP, ncpP, readFileP, spawnP, tmpDirP, writeFileP } from './utils';
+
+const TYPES_DIRECTORY = 'src/plugins/types';
+
+async function mkdirSafeP(dir: string) {
+  try {
+    await mkdirP(dir);
+    return true;
+  } catch (e) {
+    if (e.code !== 'EEXIST') {
+      throw new Error(`Error creating directory ${dir}`);
+    }
+    return false;
+  }
+}
+
+export async function getPluginTypes() {
+  await mkdirSafeP(TYPES_DIRECTORY);
+
+  const indexTs = (await readFileP(`${TYPES_DIRECTORY}/index.d.ts`, 'utf8') as string)
+    .split('\n');
+  for (const line of indexTs) {
+    const matches = line.match(/^import \* as .* from '\.\/(.+)';\s*\/\/\s*(.+)@(.+)$/);
+    if (!matches) {
+      continue;
+    }
+    console.log(matches);
+    const [, packageName, name, version] = matches;
+    const installDir = `${TYPES_DIRECTORY}/${packageName}`;
+    if (await mkdirSafeP(installDir)) {
+      await spawnP('npm', ['init', '-y'], {
+        cwd: installDir
+      });
+      await spawnP('npm', ['install', `@types/${name}@${version}`], {
+        cwd: installDir
+      });
+      await writeFileP(`${installDir}/index.ts`,
+        `import * as _ from '${name}'; export = _;\n`, 'utf8');
+    }
+  }
+}

scripts/index.ts

@@ -17,6 +17,7 @@ const {
 import { checkInstall } from './check-install';
 import { compile } from './compile';
 import { encryptCredentials, decryptCredentials } from './credentials';
+import { getPluginTypes } from './get-plugin-types';
 import { initTestFixtures } from './init-test-fixtures';
 import { reportCoverage } from './report-coverage';
 import { runTests } from './run-tests';
@@ -81,6 +82,9 @@ async function run(steps: string[]) {
 
           await decryptCredentials({ key, iv }, `node-team-test-${keyID}.json`);
           break;
+        case 'get-plugin-types':
+          await getPluginTypes();
+          break;
         case 'init-test-fixtures':
           await initTestFixtures(!TRACE_TEST_EXCLUDE_INTEGRATION);
           break;

src/plugins/types/index.d.ts

@@ -1,9 +1,40 @@
-import * as connect_3 from 'connect'; // connect@3
-import * as express_4 from 'express'; // express@4
-import * as hapi_16 from 'hapi'; // hapi@16
-import * as koa_2 from 'koa'; // koa@2
-import * as pg_7 from 'pg'; // pg@7
-import * as restify_5 from 'restify'; // restify@5
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//---module type imports---//
+
+/**
+ * NOTE: The lines in this section are parsed by scripts/get-plugin-types.ts
+ * and therefore must have a specific format
+ * 
+ * import * as X_Y from './X_Y'; // X@Y
+ * 
+ * where X is the module name and Y is the module version string.
+ * Ideally, Y is just the module's major version, since variable names cannot
+ * contain dots.
+ */
+
+import * as connect_3 from './connect_3'; // connect@3
+import * as express_4 from './express_4'; // express@4
+import * as hapi_16 from './hapi_16'; // hapi@16
+import * as koa_2 from './koa_2'; // koa@2
+import * as pg_7 from './pg_7'; // pg@7
+import * as restify_5 from './restify_5'; // restify@5
+
+//---other imports---//
 
 import { EventEmitter } from 'events';
 import { Server } from 'http';