Convert scripts/Gulpfile to checked mjs/cjs so they can run without compilation (microsoft#50988)

Author: jakebailey

.dockerignore

@@ -17,20 +17,8 @@ build.json
 *.config
 scripts/debug.bat
 scripts/run.bat
-scripts/word2md.js
-scripts/buildProtocol.js
-scripts/ior.js
-scripts/configurePrerelease.js
-scripts/open-user-pr.js
-scripts/open-cherry-pick-pr.js
-scripts/processDiagnosticMessages.d.ts
-scripts/processDiagnosticMessages.js
-scripts/produceLKG.js
-scripts/importDefinitelyTypedTests/importDefinitelyTypedTests.js
-scripts/generateLocalizedDiagnosticMessages.js
-scripts/configureLanguageServiceBuild.js
-scripts/*.js.map
-scripts/typings/
+scripts/**/*.js
+scripts/**/*.js.map
 coverage/
 internal/
 **/.DS_Store

.eslintignore

@@ -4,7 +4,8 @@
 /lib/**
 /src/lib/*.generated.d.ts
 # Ignore all compiled script outputs
-/scripts/*.js
+/scripts/**/*.js
+/scripts/**/*.d.*
 # But, not the ones that are hand-written.
 # TODO: remove once scripts are pure JS
 !/scripts/browserIntegrationTest.js

.eslintplugin.js

@@ -2,7 +2,7 @@ const fs = require("fs");
 const path = require("path");
 
 const rulesDir = path.join(__dirname, "scripts", "eslint", "rules");
-const ext = ".js";
+const ext = ".cjs";
 const ruleFiles = fs.readdirSync(rulesDir).filter((p) => p.endsWith(ext));
 
 module.exports = {

.eslintrc.json

@@ -12,17 +12,6 @@
     "plugins": [
         "@typescript-eslint", "jsdoc", "no-null", "import", "eslint-plugin-local"
     ],
-    "overrides": [
-        // By default, the ESLint CLI only looks at .js files. But, it will also look at
-        // any files which are referenced in an override config. Most users of typescript-eslint
-        // get this behavior by default by extending a recommended typescript-eslint config, which 
-        // just so happens to override some core ESLint rules. We don't extend from any config, so
-        // explicitly reference TS files here so the CLI picks them up.
-        //
-        // ESLint in VS Code will lint any opened file (so long as it's not eslintignore'd), so
-        // that will work regardless of the below.
-        { "files": ["*.ts", "*.mts", "*.cts", "*.mjs", "*.cjs"] }
-    ],
     "rules": {
         "@typescript-eslint/adjacent-overload-signatures": "error",
         "@typescript-eslint/array-type": "error",
@@ -151,5 +140,31 @@
         "no-prototype-builtins": "error",
         "no-self-assign": "error",
         "no-dupe-else-if": "error"
-    }
+    },
+    "overrides": [
+        // By default, the ESLint CLI only looks at .js files. But, it will also look at
+        // any files which are referenced in an override config. Most users of typescript-eslint
+        // get this behavior by default by extending a recommended typescript-eslint config, which 
+        // just so happens to override some core ESLint rules. We don't extend from any config, so
+        // explicitly reference TS files here so the CLI picks them up.
+        //
+        // ESLint in VS Code will lint any opened file (so long as it's not eslintignore'd), so
+        // that will work regardless of the below.
+        //
+        // The same applies to mjs files; ESLint appears to not scan those either.
+        { "files": ["*.ts", "*.mts", "*.cts", "*.mjs", "*.cjs"] },
+        {
+            "files": ["*.mjs", "*.mts"],
+            "rules": {
+                // These globals don't exist outside of CJS files.
+                "no-restricted-globals": ["error",
+                    { "name": "__filename" },
+                    { "name": "__dirname" },
+                    { "name": "require" },
+                    { "name": "module" },
+                    { "name": "exports" }
+                ]
+            }
+        }
+    ]
 }

.github/workflows/ci.yml

@@ -67,3 +67,20 @@ jobs:
 
     - name: Validate the browser can import TypeScript
       run: gulp test-browser-integration
+
+  misc:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-node@v3
+      with:
+        node-version: "*"
+        check-latest: true
+    - run:  npm ci
+
+    - name: Build scripts
+      run: gulp scripts
+
+    - name: ESLint tests
+      run: gulp run-eslint-rules-tests

.gitignore

@@ -40,22 +40,8 @@ tests/cases/**/*.js.map
 scripts/eslint/built/
 scripts/debug.bat
 scripts/run.bat
-scripts/word2md.js
-scripts/buildProtocol.js
-scripts/ior.js
-scripts/configurePrerelease.js
-scripts/configureLanguageServiceBuild.js
-scripts/open-user-pr.js
-scripts/open-cherry-pick-pr.js
-scripts/processDiagnosticMessages.d.ts
-scripts/processDiagnosticMessages.js
-scripts/produceLKG.js
-scripts/importDefinitelyTypedTests/importDefinitelyTypedTests.js
-scripts/generateLocalizedDiagnosticMessages.js
-scripts/request-pr-review.js
-scripts/errorCheck.js
-scripts/*.js.map
-scripts/typings/
+scripts/**/*.js
+scripts/**/*.js.map
 coverage/
 internal/
 **/.DS_Store

Gulpfile.js Gulpfile.mjsGulpfile.js renamed to Gulpfile.mjs

@@ -1,20 +1,22 @@
 // @ts-check
-const path = require("path");
-const fs = require("fs");
-const log = require("fancy-log");
-const newer = require("gulp-newer");
-const sourcemaps = require("gulp-sourcemaps");
-const del = require("del");
-const rename = require("gulp-rename");
-const concat = require("gulp-concat");
-const merge2 = require("merge2");
-const { src, dest, task, parallel, series, watch } = require("gulp");
-const { append, transform } = require("gulp-insert");
-const { prependFile } = require("./scripts/build/prepend");
-const { exec, readJson, needsUpdate, getDiffTool, getDirSize, rm } = require("./scripts/build/utils");
-const { runConsoleTests, refBaseline, localBaseline, refRwcBaseline, localRwcBaseline } = require("./scripts/build/tests");
-const { buildProject, cleanProject, watchProject } = require("./scripts/build/projects");
-const cmdLineOptions = require("./scripts/build/options");
+import path from "path";
+import fs from "fs";
+import log from "fancy-log";
+import newer from "gulp-newer";
+import sourcemaps from "gulp-sourcemaps";
+import del from "del";
+import rename from "gulp-rename";
+import concat from "gulp-concat";
+import merge2 from "merge2";
+import gulp from "gulp";
+import { append, transform } from "gulp-insert";
+import { prependFile } from "./scripts/build/prepend.mjs";
+import { exec, readJson, needsUpdate, getDiffTool, getDirSize, rm } from "./scripts/build/utils.mjs";
+import { runConsoleTests, refBaseline, localBaseline, refRwcBaseline, localRwcBaseline } from "./scripts/build/tests.mjs";
+import { buildProject, cleanProject, watchProject } from "./scripts/build/projects.mjs";
+import cmdLineOptions from "./scripts/build/options.mjs";
+
+const { src, dest, task, parallel, series, watch } = gulp;
 
 const copyright = "CopyrightNotice.txt";
 const cleanTasks = [];
@@ -23,9 +25,6 @@ const buildScripts = () => buildProject("scripts");
 task("scripts", buildScripts);
 task("scripts").description = "Builds files in the 'scripts' folder.";
 
-const cleanScripts = () => cleanProject("scripts");
-cleanTasks.push(cleanScripts);
-
 /** @type {{ libs: string[]; paths: Record<string, string | undefined>; }} */
 const libraries = readJson("./src/lib/libs.json");
 const libs = libraries.libs.map(lib => {
@@ -56,10 +55,10 @@ const diagnosticMessagesJson = "src/compiler/diagnosticMessages.json";
 const diagnosticMessagesGeneratedJson = "src/compiler/diagnosticMessages.generated.json";
 const generateDiagnostics = async () => {
     if (needsUpdate(diagnosticMessagesJson, [diagnosticMessagesGeneratedJson, diagnosticInformationMapTs])) {
-        await exec(process.execPath, ["scripts/processDiagnosticMessages.js", diagnosticMessagesJson]);
+        await exec(process.execPath, ["scripts/processDiagnosticMessages.mjs", diagnosticMessagesJson]);
     }
 };
-task("generate-diagnostics", series(buildScripts, generateDiagnostics));
+task("generate-diagnostics", generateDiagnostics);
 task("generate-diagnostics").description = "Generates a diagnostic file in TypeScript based on an input JSON file";
 
 const cleanDiagnostics = () => del([diagnosticInformationMapTs, diagnosticMessagesGeneratedJson]);
@@ -88,7 +87,7 @@ const localizationTargets = ["cs", "de", "es", "fr", "it", "ja", "ko", "pl", "pt
 
 const localize = async () => {
     if (needsUpdate(diagnosticMessagesGeneratedJson, generatedLCGFile)) {
-        return exec(process.execPath, ["scripts/generateLocalizedDiagnosticMessages.js", "src/loc/lcl", "built/local", diagnosticMessagesGeneratedJson], { ignoreExitCode: true });
+        return exec(process.execPath, ["scripts/generateLocalizedDiagnosticMessages.mjs", "src/loc/lcl", "built/local", diagnosticMessagesGeneratedJson], { ignoreExitCode: true });
     }
 };
 
@@ -97,7 +96,7 @@ const cleanDebugTools = () => cleanProject("src/debug");
 cleanTasks.push(cleanDebugTools);
 
 // Pre-build steps when targeting the LKG compiler
-const lkgPreBuild = parallel(generateLibs, series(buildScripts, generateDiagnostics, buildDebugTools));
+const lkgPreBuild = parallel(generateLibs, series(generateDiagnostics, buildDebugTools));
 
 const buildTsc = () => buildProject("src/tsc");
 task("tsc", series(lkgPreBuild, buildTsc));
@@ -113,7 +112,7 @@ task("watch-tsc", series(lkgPreBuild, parallel(watchLib, watchDiagnostics, watch
 task("watch-tsc").description = "Watch for changes and rebuild the command-line compiler only.";
 
 // Pre-build steps when targeting the built/local compiler.
-const localPreBuild = parallel(generateLibs, series(buildScripts, generateDiagnostics, buildDebugTools, buildTsc));
+const localPreBuild = parallel(generateLibs, series(generateDiagnostics, buildDebugTools, buildTsc));
 
 // Pre-build steps to use based on supplied options.
 const preBuild = cmdLineOptions.lkg ? lkgPreBuild : localPreBuild;
@@ -335,17 +334,8 @@ task("clean-tests").description = "Cleans the outputs for the test infrastructur
 
 const watchTests = () => watchProject("src/testRunner", cmdLineOptions);
 
-const buildEslintRules = () => buildProject("scripts/eslint");
-task("build-eslint-rules", buildEslintRules);
-task("build-eslint-rules").description = "Compiles eslint rules to js";
-
-const cleanEslintRules = () => cleanProject("scripts/eslint");
-cleanTasks.push(cleanEslintRules);
-task("clean-eslint-rules", cleanEslintRules);
-task("clean-eslint-rules").description = "Cleans the outputs for the eslint rules";
-
-const runEslintRulesTests = () => runConsoleTests("scripts/eslint/built/tests", "mocha-fivemat-progress-reporter", /*runInParallel*/ false, /*watchMode*/ false);
-task("run-eslint-rules-tests", series(buildEslintRules, runEslintRulesTests));
+const runEslintRulesTests = () => runConsoleTests("scripts/eslint/tests", "mocha-fivemat-progress-reporter", /*runInParallel*/ false, /*watchMode*/ false);
+task("run-eslint-rules-tests", runEslintRulesTests);
 task("run-eslint-rules-tests").description = "Runs the eslint rule tests";
 
 /** @type { (folder: string) => { (): Promise<any>; displayName?: string } } */
@@ -459,8 +449,8 @@ task("runtests-parallel").flags = {
 };
 
 
-task("test-browser-integration", () => exec(process.execPath, ["scripts/browserIntegrationTest.js"]));
-task("test-browser-integration").description = "Runs scripts/browserIntegrationTest.ts which tests that typescript.js loads in a browser";
+task("test-browser-integration", () => exec(process.execPath, ["scripts/browserIntegrationTest.mjs"]));
+task("test-browser-integration").description = "Runs scripts/browserIntegrationTest.mjs which tests that typescript.js loads in a browser";
 
 
 task("diff", () => exec(getDiffTool(), [refBaseline, localBaseline], { ignoreExitCode: true, waitForExit: false }));
@@ -493,13 +483,9 @@ const updateSublime = () => src(["built/local/tsserver.js", "built/local/tsserve
 task("update-sublime", updateSublime);
 task("update-sublime").description = "Updates the sublime plugin's tsserver";
 
-const buildImportDefinitelyTypedTests = () => buildProject("scripts/importDefinitelyTypedTests");
-const cleanImportDefinitelyTypedTests = () => cleanProject("scripts/importDefinitelyTypedTests");
-cleanTasks.push(cleanImportDefinitelyTypedTests);
-
 // TODO(rbuckton): Should the path to DefinitelyTyped be configurable via an environment variable?
-const importDefinitelyTypedTests = () => exec(process.execPath, ["scripts/importDefinitelyTypedTests/importDefinitelyTypedTests.js", "./", "../DefinitelyTyped"]);
-task("importDefinitelyTypedTests", series(buildImportDefinitelyTypedTests, importDefinitelyTypedTests));
+const importDefinitelyTypedTests = () => exec(process.execPath, ["scripts/importDefinitelyTypedTests.mjs", "./", "../DefinitelyTyped"]);
+task("importDefinitelyTypedTests", importDefinitelyTypedTests);
 task("importDefinitelyTypedTests").description = "Runs the importDefinitelyTypedTests script to copy DT's tests to the TS-internal RWC tests";
 
 const buildReleaseTsc = () => buildProject("src/tsc/tsconfig.release.json");
@@ -529,7 +515,7 @@ const produceLKG = async () => {
         throw new Error("Cannot replace the LKG unless all built targets are present in directory 'built/local/'. The following files are missing:\n" + missingFiles.join("\n"));
     }
     const sizeBefore = getDirSize("lib");
-    await exec(process.execPath, ["scripts/produceLKG.js"]);
+    await exec(process.execPath, ["scripts/produceLKG.mjs"]);
     const sizeAfter = getDirSize("lib");
     if (sizeAfter > (sizeBefore * 1.10)) {
         throw new Error("The lib folder increased by 10% or more. This likely indicates a bug.");
@@ -543,8 +529,8 @@ task("LKG").flags = {
 };
 task("lkg", series("LKG"));
 
-const generateSpec = () => exec("cscript", ["//nologo", "scripts/word2md.js", path.resolve("doc/TypeScript Language Specification - ARCHIVED.docx"), path.resolve("doc/spec-ARCHIVED.md")]);
-task("generate-spec", series(buildScripts, generateSpec));
+const generateSpec = () => exec("cscript", ["//nologo", "scripts/word2md.mjs", path.resolve("doc/TypeScript Language Specification - ARCHIVED.docx"), path.resolve("doc/spec-ARCHIVED.md")]);
+task("generate-spec", generateSpec);
 task("generate-spec").description = "Generates a Markdown version of the Language Specification";
 
 task("clean", series(parallel(cleanTasks), cleanBuilt));
@@ -554,13 +540,13 @@ const configureNightly = () => exec(process.execPath, ["scripts/configurePrerele
 task("configure-nightly", series(buildScripts, configureNightly));
 task("configure-nightly").description = "Runs scripts/configurePrerelease.ts to prepare a build for nightly publishing";
 
-const configureInsiders = () => exec(process.execPath, ["scripts/configurePrerelease.js", "insiders", "package.json", "src/compiler/corePublic.ts"]);
-task("configure-insiders", series(buildScripts, configureInsiders));
-task("configure-insiders").description = "Runs scripts/configurePrerelease.ts to prepare a build for insiders publishing";
+const configureInsiders = () => exec(process.execPath, ["scripts/configurePrerelease.mjs", "insiders", "package.json", "src/compiler/corePublic.ts"]);
+task("configure-insiders", configureInsiders);
+task("configure-insiders").description = "Runs scripts/configurePrerelease.mjs to prepare a build for insiders publishing";
 
-const configureExperimental = () => exec(process.execPath, ["scripts/configurePrerelease.js", "experimental", "package.json", "src/compiler/corePublic.ts"]);
-task("configure-experimental", series(buildScripts, configureExperimental));
-task("configure-experimental").description = "Runs scripts/configurePrerelease.ts to prepare a build for experimental publishing";
+const configureExperimental = () => exec(process.execPath, ["scripts/configurePrerelease.mjs", "experimental", "package.json", "src/compiler/corePublic.ts"]);
+task("configure-experimental", configureExperimental);
+task("configure-experimental").description = "Runs scripts/configurePrerelease.mjs to prepare a build for experimental publishing";
 
 const publishNightly = () => exec("npm", ["publish", "--tag", "next"]);
 task("publish-nightly", series(task("clean"), task("LKG"), task("clean"), task("runtests-parallel"), publishNightly));