Bundle extension files using esbuild

.gitignore

@@ -3,6 +3,7 @@
 
 # Generated files
 /dist/
+/dist-test/
 out/
 build/
 server/

.vscode/launch.json

@@ -56,15 +56,15 @@
       "request": "launch",
       "runtimeExecutable": "${execPath}",
       "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
-        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/no-workspace/index",
+        "--extensionDevelopmentPath=${workspaceRoot}/dist-test/vscode-codeql",
+        "--extensionTestsPath=${workspaceRoot}/dist-test/vscode-codeql/out/vscode-tests/no-workspace/index",
         "--disable-workspace-trust",
         "--disable-extensions",
         "--disable-gpu"
       ],
       "sourceMaps": true,
       "outFiles": [
-        "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
+        "${workspaceRoot}/dist-test/vscode-codeql/out/**/*.js",
       ],
     },
     {
@@ -73,16 +73,16 @@
       "request": "launch",
       "runtimeExecutable": "${execPath}",
       "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
-        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/minimal-workspace/index",
+        "--extensionDevelopmentPath=${workspaceRoot}/dist-test/vscode-codeql",
+        "--extensionTestsPath=${workspaceRoot}/dist-test/vscode-codeql/out/vscode-tests/minimal-workspace/index",
         "--disable-workspace-trust",
         "--disable-extensions",
         "--disable-gpu",
-        "${workspaceRoot}/extensions/ql-vscode/test/data"
+        "${workspaceRoot}/dist-test/vscode-codeql/test/data"
       ],
       "sourceMaps": true,
       "outFiles": [
-        "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
+        "${workspaceRoot}/dist-test/vscode-codeql/out/**/*.js",
       ],
     },
     {
@@ -91,8 +91,8 @@
       "request": "launch",
       "runtimeExecutable": "${execPath}",
       "args": [
-        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
-        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/cli-integration/index",
+        "--extensionDevelopmentPath=${workspaceRoot}/dist-test/vscode-codeql",
+        "--extensionTestsPath=${workspaceRoot}/dist-test/vscode-codeql/out/vscode-tests/cli-integration/index",
         "--disable-workspace-trust",
         "--disable-gpu",
         "--disable-extension",
@@ -101,7 +101,7 @@
         "github.codespaces",
         "--disable-extension",
         "github.copilot",
-        "${workspaceRoot}/extensions/ql-vscode/src/vscode-tests/cli-integration/data",
+        "${workspaceRoot}/dist-test/vscode-codeql/out/vscode-tests/cli-integration/data",
         // Uncomment the last line and modify the path to a checked out 
         // instance of the codeql repository so the libraries are
         // available in the workspace for the tests.
@@ -122,7 +122,7 @@
       },
       "sourceMaps": true,
       "outFiles": [
-        "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
+        "${workspaceRoot}/dist-test/vscode-codeql/out/**/*.js",
       ],
     }
   ]

extensions/ql-vscode/.vscodeignore

@@ -3,7 +3,7 @@
 .vscode-test/**
 typings/**
 out/test/**
-out/vscode-tests/**
+out/test-run/**
 **/@types/**
 **/*.ts
 test/**
@@ -14,3 +14,4 @@ gulpfile.js/**
 tsconfig.json
 tsfmt.json
 vsc-extension-quickstart.md
+node_modules/**

extensions/ql-vscode/gulpfile.ts/appInsights.ts

@@ -11,7 +11,7 @@ export function injectAppInsightsKey() {
   }
 
   // replace the key
-  return gulp.src(['out/telemetry.js'])
+  return gulp.src(['out/extension.js'])
     .pipe(replace(/REPLACE-APP-INSIGHTS-KEY/, process.env.APP_INSIGHTS_KEY))
     .pipe(gulp.dest('out/'));
 }

extensions/ql-vscode/gulpfile.ts/deploy.ts

@@ -14,15 +14,16 @@ const packageFiles = [
   'language-configuration.json',
   'snippets.json',
   'media',
-  'node_modules',
-  'out'
+  'out',
 ];
 
+async function copyDirectory(sourcePath: string, destPath: string): Promise<void> {
+  console.log(`copying ${sourcePath} to ${destPath}`);
+  await fs.copy(sourcePath, destPath);
+}
+
 async function copyPackage(sourcePath: string, destPath: string): Promise<void> {
-  for (const file of packageFiles) {
-    console.log(`copying ${path.resolve(sourcePath, file)} to ${path.resolve(destPath, file)}`);
-    await fs.copy(path.resolve(sourcePath, file), path.resolve(destPath, file));
-  }
+  await Promise.all(packageFiles.map(file => copyDirectory(path.resolve(sourcePath, file), path.resolve(destPath, file))));
 }
 
 export async function deployPackage(packageJsonPath: string): Promise<DeployedPackage> {
@@ -58,6 +59,7 @@ export async function deployPackage(packageJsonPath: string): Promise<DeployedPa
     const sourcePath = path.join(__dirname, '..');
     console.log(`Copying package '${packageJson.name}' and its dependencies to '${distPath}'...`);
     await copyPackage(sourcePath, distPath);
+    await copyDirectory(path.resolve(sourcePath, 'node_modules'), path.resolve(distPath, 'node_modules'));
 
     return {
       distPath: distPath,
@@ -70,3 +72,27 @@ export async function deployPackage(packageJsonPath: string): Promise<DeployedPa
     throw e;
   }
 }
+
+export async function copyTestPackageToDist(distDir: string, packageJsonPath: string, copyNodeModules = true) {
+  const packageJson: any = JSON.parse(await fs.readFile(packageJsonPath, 'utf8'));
+
+  const now = new Date();
+  packageJson.version = packageJson.version +
+    `-test.${now.getUTCFullYear()}.${now.getUTCMonth() + 1}.${now.getUTCDate()}` +
+    `.${now.getUTCHours()}.${now.getUTCMinutes()}.${now.getUTCSeconds()}`;
+
+  const distPath = path.join(distDir, packageJson.name);
+  await fs.mkdirs(distPath);
+
+  await fs.writeFile(path.join(distPath, 'package.json'), JSON.stringify(packageJson, null, 2));
+
+  const sourcePath = path.join(__dirname, '..');
+  console.log(`Copying package '${packageJson.name}' and its dependencies to '${distPath}'...`);
+  await Promise.all([
+    copyPackage(sourcePath, distPath),
+    copyNodeModules ? copyDirectory(path.resolve(sourcePath, 'node_modules'), path.resolve(distPath, 'node_modules')) : Promise.resolve(),
+    copyDirectory(path.resolve(sourcePath, 'test', 'data'), path.resolve(distPath, 'test', 'data'))
+  ]);
+
+  await copyDirectory(path.resolve(sourcePath, 'out', 'test-run'), path.resolve(distPath, 'out'));
+}

extensions/ql-vscode/gulpfile.ts/index.ts

@@ -1,27 +1,38 @@
 import * as gulp from 'gulp';
-import { compileTypeScript, watchTypeScript, cleanOutput } from './typescript';
+import { compileEsbuild, watchEsbuild, cleanOutput, checkTypeScript, watchCheckTypeScript, compileTypeScriptTests, watchTypeScriptTests } from './typescript';
 import { compileTextMateGrammar } from './textmate';
-import { copyTestData } from './tests';
+import { copyTestData, copyTests, watchCopyTests } from './tests';
 import { compileView, watchView } from './webpack';
 import { packageExtension } from './package';
 import { injectAppInsightsKey } from './appInsights';
 
 export const buildWithoutPackage =
   gulp.series(
     cleanOutput,
-    gulp.parallel(
-      compileTypeScript, compileTextMateGrammar, compileView, copyTestData
-    )
+    gulp.parallel(compileEsbuild, compileTextMateGrammar, compileView, copyTestData, checkTypeScript, compileTypeScriptTests)
   );
 
+export const watch = gulp.parallel(
+  watchEsbuild, watchTypeScriptTests, watchCheckTypeScript, watchView, watchCopyTests
+);
+
+export const buildTests = gulp.series(
+  buildWithoutPackage,
+  copyTests
+);
+
 export {
   cleanOutput,
   compileTextMateGrammar,
-  watchTypeScript,
+  watchEsbuild,
   watchView,
-  compileTypeScript,
+  compileEsbuild,
   copyTestData,
   injectAppInsightsKey,
   compileView,
+  checkTypeScript,
+  watchCheckTypeScript,
+  compileTypeScriptTests,
+  watchTypeScriptTests,
 };
-export default gulp.series(buildWithoutPackage, injectAppInsightsKey, packageExtension);
+export default gulp.series(buildWithoutPackage, injectAppInsightsKey, packageExtension, copyTests);

extensions/ql-vscode/gulpfile.ts/tests.ts

@@ -1,4 +1,9 @@
 import * as gulp from 'gulp';
+import * as path from 'path';
+import del from 'del';
+import { copyTestPackageToDist } from './deploy';
+
+const testDistDir = path.join(__dirname, '../../../dist-test');
 
 export function copyTestData() {
   copyNoWorkspaceData();
@@ -8,10 +13,29 @@ export function copyTestData() {
 
 function copyNoWorkspaceData() {
   return gulp.src('src/vscode-tests/no-workspace/data/**/*')
-    .pipe(gulp.dest('out/vscode-tests/no-workspace/data'));
+    .pipe(gulp.dest('out/test-run/vscode-tests/no-workspace/data'));
 }
 
 function copyCliIntegrationData() {
-  return gulp.src('src/vscode-tests/cli-integration/data/**/*')
-    .pipe(gulp.dest('out/vscode-tests/cli-integration/data'));
+  return gulp.src('src/vscode-tests/cli-integration/data*/**/*')
+    .pipe(gulp.dest('out/test-run/vscode-tests/cli-integration'));
+}
+
+function cleanTestsOutput() {
+  // The path to the test dist dir is outside of the working directory, so we need to force it
+  return del(testDistDir, { force: true });
+}
+
+async function copyToTests(): Promise<void> {
+  await copyTestPackageToDist(testDistDir, path.resolve('package.json'));
+}
+
+async function copyTestsWithoutNodeModules(): Promise<void> {
+  await copyTestPackageToDist(testDistDir, path.resolve('package.json'), false);
+}
+
+export const copyTests = gulp.series(cleanTestsOutput, copyToTests);
+
+export function watchCopyTests() {
+  gulp.watch('out/**/*', copyTestsWithoutNodeModules);
 }

extensions/ql-vscode/gulpfile.ts/textmate.ts

@@ -1,7 +1,7 @@
 import * as gulp from 'gulp';
 import * as jsYaml from 'js-yaml';
 import * as through from 'through2';
-import * as PluginError from 'plugin-error';
+import PluginError from 'plugin-error';
 import * as Vinyl from 'vinyl';
 
 /**

extensions/ql-vscode/gulpfile.ts/typescript.ts

@@ -1,8 +1,9 @@
 import * as colors from 'ansi-colors';
 import * as gulp from 'gulp';
-import * as sourcemaps from 'gulp-sourcemaps';
-import * as ts from 'gulp-typescript';
-import * as del from 'del';
+import esbuild from 'gulp-esbuild';
+import ts from 'gulp-typescript';
+import sourcemaps from 'gulp-sourcemaps';
+import del from 'del';
 
 function goodReporter(): ts.reporter.Reporter {
   return {
@@ -20,22 +21,57 @@ function goodReporter(): ts.reporter.Reporter {
 }
 
 const tsProject = ts.createProject('tsconfig.json');
+const testsTsProject = ts.createProject('tsconfig.json', {
+  esModuleInterop: false,
+});
 
 export function cleanOutput() {
   return tsProject.projectDirectory ? del(tsProject.projectDirectory + '/out/*') : Promise.resolve();
 }
 
-export function compileTypeScript() {
-  return tsProject.src()
+export function compileEsbuild() {
+  return gulp.src('./src/extension.ts')
+    .pipe(esbuild({
+      outfile: 'extension.js',
+      bundle: true,
+      external: ['vscode'],
+      format: 'cjs',
+      platform: 'node',
+      target: 'es2020',
+      sourcemap: 'linked',
+    }))
+    .pipe(gulp.dest('out'));
+}
+
+export function compileTypeScriptTests() {
+  // Unfortunately we cannot use ESBuild for the test compilation because it compiles
+  // to ES6 modules which have additional restrictions compared to the code generated
+  // by the TypeScript compiler.
+  return testsTsProject.src()
     .pipe(sourcemaps.init())
-    .pipe(tsProject(goodReporter()))
+    .pipe(testsTsProject(goodReporter()))
     .pipe(sourcemaps.write('.', {
       includeContent: false,
       sourceRoot: '.',
     }))
-    .pipe(gulp.dest('out'));
+    .pipe(gulp.dest('out/test-run'));
+}
+
+export function watchEsbuild() {
+  gulp.watch('src/**/*.ts', compileEsbuild);
+}
+
+export function watchTypeScriptTests() {
+  gulp.watch('src/**/*.ts', compileTypeScriptTests);
+}
+
+export function checkTypeScript() {
+  // This doesn't actually output the TypeScript files, it just
+  // runs the TypeScript compiler and reports any errors.
+  return tsProject.src();
+  // .pipe(tsProject(goodReporter()));
 }
 
-export function watchTypeScript() {
-  gulp.watch('src/**/*.ts', compileTypeScript);
+export function watchCheckTypeScript() {
+  gulp.watch('src/**/*.ts', checkTypeScript);
 }

extensions/ql-vscode/gulpfile.ts/webpack.config.ts

@@ -1,6 +1,6 @@
 import * as path from 'path';
 import * as webpack from 'webpack';
-import * as MiniCssExtractPlugin from 'mini-css-extract-plugin';
+import MiniCssExtractPlugin from 'mini-css-extract-plugin';
 
 export const config: webpack.Configuration = {
   mode: 'development',

extensions/ql-vscode/gulpfile.ts/webpack.ts

@@ -1,4 +1,4 @@
-import * as webpack from 'webpack';
+import webpack from 'webpack';
 import { config } from './webpack.config';
 
 export function compileView(cb: (err?: Error) => void) {