Provide per-project build progress in tsc -b

[Knagis]

Suggestion

🔍 Search Terms

tsc diagnostics tsc progress

✅ Viability Checklist

My suggestion meets these guidelines:

• This wouldn't be a breaking change in existing TypeScript/JavaScript code
• This wouldn't change the runtime behavior of existing JavaScript code
• This could be implemented without emitting different JS based on the types of the expressions
• This isn't a runtime feature (e.g. library functionality, non-ECMAScript syntax with JavaScript output, new syntax sugar for JS, etc.)
• This feature would agree with the rest of TypeScript's Design Goals.

⭐ Suggestion

Please consider printing by default (in TTY mode) progress about solution compilation. It would be a minimalistic version of --diagnostics flag.

Also note that currently --diagnostics is not easily usable for tsc -b because it does not say which project each statistics belong to.

📃 Motivating Example

I patched tsc to print out every project that it will build and that prints time spent for each project. It can be done via simple xterm commands, no need for any dependencies.

💻 Use Cases

This has helped us to:

• realize that some packages were never cached but always rebuilt (seems because of overlapping included files)
• see just how long some common reused packages compile vs the consumers
• visualize the dependency graph, allowing developers to see that they are including references they really don't need.

Overall this has been eye opening in regards to what contributes to our build times.

As mentioned above --diagnostics could be used to achieve this (though it is bit too verbose to get an overview and it is optional) however it not printing the project name is a blocker to use it.

[jasonaden]

@Knagis Could you provide this patch? I would love to see this info on my build. It will help considerably on debugging and determining where to focus performance improvement work.

[Knagis]

@jasonaden There you go. This has both the build output and locking (to support parallel compilation described in #42216). Should be straight forward to remove the locking part if you don't need it (it is done with proper-lockfile that creates empty folder and no files). I originally wrote the build status just so that we could see when the lock was pausing the build, didn't expect that the progress output itself will turn out to be so useful.

diff --git a/lib/tsc.js b/lib/tsc.js
index d3d0ee328b3406f4a1315beb45bed7a25693b07b..f2770c36476ce62f65b153714d595e78569ced5a 100644
--- a/lib/tsc.js
+++ b/lib/tsc.js
@@ -121380,26 +121380,119 @@ function queueReferencingProjects(state, project, projectPath, projectIndex, con
     }
   }
 }
-function build(state, project, cancellationToken, writeFile2, getCustomTransformers, onlyReferences) {
+async function build(state, project, cancellationToken, writeFile2, getCustomTransformers, onlyReferences) {
   mark("SolutionBuilder::beforeBuild");
-  const result = buildWorker(state, project, cancellationToken, writeFile2, getCustomTransformers, onlyReferences);
+  const result = await buildWorker(state, project, cancellationToken, writeFile2, getCustomTransformers, onlyReferences);
   mark("SolutionBuilder::afterBuild");
   measure("SolutionBuilder::Build", "SolutionBuilder::beforeBuild", "SolutionBuilder::afterBuild");
   return result;
 }
-function buildWorker(state, project, cancellationToken, writeFile2, getCustomTransformers, onlyReferences) {
+function readableProjectPaths(arr) {
+  if (!arr.length) {
+    return arr;
+  }
+  const parts = arr[0].split(`/`);
+  let prefix = "";
+  for (const p of parts) {
+    const prefixCand = prefix + p + "/";
+    if (arr.some(o => !o.startsWith(prefixCand))) {
+      break;
+    }
+    prefix = prefixCand;
+  }
+  return arr.map(o => o.substring(prefix.length));
+}
+function getProjectStatusStr(readablePaths, idx, status, statusDescr) {
+  let color;
+  if (status === "running") {
+      color = "33"; // dark yellow
+  } else if (status === "done") {
+      color = "32"; // dark green
+  } else if (status === "error") {
+      color = "31"; // dark red
+  } else if (status === "locked") {
+      color = "96"; // bright cyan
+  } else {
+      color = "97"; // white
+  }
+  return `\x1b[${color};1m${readablePaths[idx]}\x1b[0m: ${statusDescr || status}`;
+}
+function printProjectStatus(readablePaths, idx, status, statusDescr) {
+  if (process.stderr.isTTY) {
+    process.stderr.write("\x1b7"); // save cursor position
+    process.stderr.write(`\x1b[${readablePaths.length - idx}A`); // move up to the line it was previously written to
+    process.stderr.write(getProjectStatusStr(readablePaths, idx, status, statusDescr));
+    process.stderr.write("\x1b[K"); // erase to the end of line
+    process.stderr.write("\x1b8"); // restore cursor position
+    process.stderr.write("\x1b[1A\n"); // move up one line and add newline to flush node buffer
+  } else {
+    process.stderr.write(getProjectStatusStr(readablePaths, idx, status, statusDescr) + "\n");
+  }
+}
+async function buildWorker(state, project, cancellationToken, writeFile2, getCustomTransformers, onlyReferences) {
+  const lockfile = require("proper-lockfile");
   const buildOrder = getBuildOrderFor(state, project, onlyReferences);
   if (!buildOrder)
     return 3 /* InvalidProject_OutputsSkipped */;
   setupInitialBuild(state, cancellationToken);
   let reportQueue = true;
   let successfulProjects = 0;
+
+  const readablePaths = readableProjectPaths(buildOrder);
+  if (process.stderr.isTTY)
+    for (let i = 0; i < buildOrder.length; i++) {
+      process.stderr.write(getProjectStatusStr(readablePaths, i, "pending") + '\n');
+    }
+  let lastIdx = 0;
   while (true) {
     const invalidatedProject = getNextInvalidatedProject(state, buildOrder, reportQueue);
+    const idx = invalidatedProject ? buildOrder.indexOf(invalidatedProject.project) : buildOrder.length;
+    while (lastIdx < idx) {
+      printProjectStatus(readablePaths, lastIdx, "done", "done (cached)");
+      lastIdx++;
+    }
     if (!invalidatedProject)
       break;
+
+    let release;
+    try {
+      // unfortunately need big enough stale time for any project to build because the mtime is updated on timer
+      // but compilation is done sync, so the timer does not have time to fire.
+      release = await lockfile.lock(invalidatedProject.projectPath, { stale: 60000 });
+    } catch (e) {
+      printProjectStatus(readablePaths, idx, "locked", "waiting for lock")
+      while (await lockfile.check(invalidatedProject.projectPath, { stale: 60000 })) {
+        await new Promise(resolve => setTimeout(resolve, 1000));
+      }
+      state.projectStatus.delete(invalidatedProject.projectPath);
+      state.buildInfoCache.delete(invalidatedProject.projectPath);
+      continue;
+    }
+
+    lastIdx++;
+    const started = process.hrtime.bigint();
+    printProjectStatus(readablePaths, idx, "running")
+    try {
+
     reportQueue = false;
     invalidatedProject.done(cancellationToken, writeFile2, getCustomTransformers == null ? void 0 : getCustomTransformers(invalidatedProject.project));
+
+        if (state.projectStatus.get(invalidatedProject.projectPath)?.type !== 1) {
+          for (let i = idx; i < buildOrder.length; i++) {
+            // if there were errors, likely stuff has been written to stderr so the positions of our status report has been broken.
+            process.stderr.write(getProjectStatusStr(readablePaths, i, "pending") + '\n');
+          }
+          printProjectStatus(readablePaths, idx, "error", "error: " + state.projectStatus.get(invalidatedProject.projectPath)?.reason);
+        } else {
+          printProjectStatus(readablePaths, idx, "done", "done in " + (process.hrtime.bigint() - started) / BigInt(1e6) + " ms");
+        }
+      } finally {
+        try {
+          await release();
+        } catch (e) {
+          console.error("failed to release lock", e);
+        }
+    }
     if (!state.diagnostics.has(invalidatedProject.projectPath))
       successfulProjects++;
   }
@@ -122554,7 +122634,7 @@ function reportWatchModeWithoutSysSupport(sys2, reportDiagnostic) {
   }
   return false;
 }
-function performBuild(sys2, cb, buildOptions, watchOptions, projects, errors) {
+async function performBuild(sys2, cb, buildOptions, watchOptions, projects, errors) {
   const reportDiagnostic = updateReportDiagnostic(
     sys2,
     createDiagnosticReporter(sys2),
@@ -122603,7 +122683,7 @@ function performBuild(sys2, cb, buildOptions, watchOptions, projects, errors) {
       }
     };
     const builder2 = createSolutionBuilderWithWatch(buildHost2, projects, buildOptions, watchOptions);
-    builder2.build();
+    await builder2.build();
     reportSolutionBuilderTimes(builder2, solutionPerformance2);
     reportBuildStatistics = true;
     return builder2;
@@ -122619,7 +122699,7 @@ function performBuild(sys2, cb, buildOptions, watchOptions, projects, errors) {
   const solutionPerformance = enableSolutionPerformance(sys2, buildOptions);
   updateSolutionBuilderHost(sys2, cb, buildHost, solutionPerformance);
   const builder = createSolutionBuilder(buildHost, projects, buildOptions);
-  const exitStatus = buildOptions.clean ? builder.clean() : builder.build();
+  const exitStatus = buildOptions.clean ? builder.clean() : await builder.build();
   reportSolutionBuilderTimes(builder, solutionPerformance);
   dumpTracingLegend();
   return sys2.exit(exitStatus);