fix(jsii-reflect): don't load same assembly multiple times (#461)

deCDK tests were calling `loadModule()` for every package.
`loadModule()` did have some recursion avoidance INSIDE one call,
but not across multiple load calls on the same type system.

This brings down the deCDK tests from 80s to 10s.

Add an option to disable validation, which would bring the full CDK
typesystem loading down from 10s to 600ms (validation is enabled
by default).

Some more type exposure updates so that we can update awslint
and friends in the CDK repository to take advantage of the new
jsii model.

Also: change some JVM settings in pacmak to speed up Java build
from around 30s to around 10s per package (on my machine, on
my Oracle JVM, YYMV on OpenJDK).

Author: rix0rrr

packages/jsii-diff/bin/jsii-diff.ts

@@ -91,7 +91,7 @@ async function loadFromFilesystem(name: string) {
   if (stat.isDirectory()) {
     return await ts.loadModule(name);
   } else {
-    return await ts.loadFile(name, true);
+    return await ts.loadFile(name);
   }
 }
 

packages/jsii-pacmak/bin/jsii-pacmak.ts

@@ -7,6 +7,7 @@ import process = require('process');
 import yargs = require('yargs');
 import logging = require('../lib/logging');
 import { Target } from '../lib/target';
+import { Timers } from '../lib/timer';
 import { resolveDependencyDirectory, shell } from '../lib/util';
 import { VERSION_DESC } from '../lib/version';
 
@@ -128,22 +129,34 @@ import { VERSION_DESC } from '../lib/version';
             await updateNpmIgnore(packageDir, npmIgnoreExclude);
         }
 
+        const timers = new Timers();
+
         const tmpdir = await fs.mkdtemp(path.join(os.tmpdir(), 'npm-pack'));
         try {
-            const tarball = await npmPack(packageDir, tmpdir);
+            const tarball = await timers.recordAsync('npm pack', () => {
+                return npmPack(packageDir, tmpdir);
+            });
             for (const targetName of targets) {
                 // if we are targeting a single language, output to outdir, otherwise outdir/<target>
                 const targetOutputDir = (targets.length > 1 || forceSubdirectory)
                                       ? path.join(outDir, targetName.toString())
                                       : outDir;
                 logging.debug(`Building ${pkg.name}/${targetName}: ${targetOutputDir}`);
-                await generateTarget(packageDir, targetName.toString(), targetOutputDir, tarball);
+
+                await timers.recordAsync(targetName.toString(), () =>
+                    generateTarget(packageDir, targetName.toString(), targetOutputDir, tarball)
+                );
             }
         } finally {
-            logging.debug(`Removing ${tmpdir}`);
+            if (argv.clean) {
+                logging.debug(`Removing ${tmpdir}`);
+            } else {
+                logging.debug(`Temporary directory retained (--no-clean): ${tmpdir}`);
+            }
             await fs.remove(tmpdir);
         }
 
+        logging.info(`Packaged. ${timers.display()}`);
     }
 
     async function generateTarget(packageDir: string, targetName: string, targetOutputDir: string, tarball: string) {
@@ -221,7 +234,7 @@ async function updateNpmIgnore(packageDir: string, excludeOutdir: string | undef
 
     if (modified) {
         await fs.writeFile(npmIgnorePath, lines.join('\n') + '\n');
-        logging.info('Updated .npmignre');
+        logging.info('Updated .npmignore');
     }
 
     function includePattern(comment: string, ...patterns: string[]) {

packages/jsii-pacmak/lib/targets/java.ts

@@ -65,7 +65,14 @@ export default class Java extends Target {
         await shell(
             'mvn',
             [...mvnArguments, 'deploy', `-D=altDeploymentRepository=local::default::${url}`, `--settings=${userXml}`],
-            { cwd: sourceDir }
+            {
+                cwd: sourceDir,
+                env: {
+                    // Twiddle the JVM settings a little for Maven. Delaying JIT compilation
+                    // brings down Maven execution time by about 1/3rd (15->10s, 30->20s)
+                    MAVEN_OPTS: `${process.env.MAVEN_OPTS || ''} -XX:+TieredCompilation -XX:TieredStopAtLevel=1`
+                }
+             }
         );
     }
 
@@ -433,7 +440,10 @@ class JavaGenerator extends Generator {
                                 },
                                 configuration: {
                                     failOnError: false,
-                                    show: 'protected'
+                                    show: 'protected',
+                                    // Adding these makes JavaDoc generation about a 3rd faster (which is far and away the most
+                                    // expensive part of the build)
+                                    additionalJOption: ['-J-XX:+TieredCompilation</additionalJOption', '-J-XX:TieredStopAtLevel=1</additionalJOption']
                                 }
                             }]
                         }

packages/jsii-pacmak/lib/timer.ts

@@ -0,0 +1,82 @@
+/**
+ * A single timer
+ */
+export class Timer {
+  public timeMs?: number;
+  private startTime: number;
+
+  constructor(public readonly label: string) {
+    this.startTime = Date.now();
+  }
+
+  public start() {
+    this.startTime = Date.now();
+  }
+
+  public end() {
+    this.timeMs = (Date.now() - this.startTime) / 1000;
+  }
+
+  public isSet() {
+    return this.timeMs !== undefined;
+  }
+
+  public humanTime() {
+    if (!this.timeMs) { return '???'; }
+
+    const parts = [];
+
+    let time = this.timeMs;
+    if (time > 60) {
+      const mins = Math.floor(time / 60);
+      parts.push(mins + 'm');
+      time -= mins * 60;
+    }
+    parts.push(time.toFixed(1) + 's');
+
+    return parts.join('');
+  }
+}
+
+/**
+ * A collection of Timers
+ */
+export class Timers {
+  private readonly timers: Timer[] = [];
+
+  public record<T>(label: string, operation: () => T): T {
+    const timer = this.start(label);
+    try {
+      const x = operation();
+      timer.end();
+      return x;
+    } catch (e) {
+      timer.end();
+      throw e;
+    }
+  }
+
+  public async recordAsync<T>(label: string, operation: () => Promise<T>) {
+    const timer = this.start(label);
+    try {
+      const x = await operation();
+      timer.end();
+      return x;
+    } catch (e) {
+      timer.end();
+      throw e;
+    }
+  }
+
+  public start(label: string) {
+    const timer = new Timer(label);
+    this.timers.push(timer);
+    return timer;
+  }
+
+  public display(): string {
+    const timers = this.timers.filter(t => t.isSet());
+    timers.sort((a: Timer, b: Timer) => b.timeMs! - a.timeMs!);
+    return timers.map(t => `${t.label} (${t.humanTime()})`).join(' | ');
+  }
+}

packages/jsii-pacmak/lib/util.ts

@@ -17,7 +17,8 @@ export function resolveDependencyDirectory(packageDir: string, dependencyName: s
 
 export function shell(cmd: string, args: string[], options: SpawnOptions): Promise<string> {
     return new Promise<string>((resolve, reject) => {
-        const child = spawn(cmd, args, { ...options, shell: true, stdio: ['ignore', 'pipe', 'pipe'] });
+        logging.debug(cmd, args.join(' '), JSON.stringify(options));
+        const child = spawn(cmd, args, { ...options, shell: true, env: { ...process.env, ...options.env || {} }, stdio: ['ignore', 'pipe', 'pipe'] });
         const stdout = new Array<Buffer>();
         const stderr = new Array<Buffer>();
         child.stdout.on('data', chunk => {

packages/jsii-pacmak/test/expected.jsii-calc-base/java/pom.xml

@@ -103,6 +103,8 @@
         <configuration>
           <failOnError>false</failOnError>
           <show>protected</show>
+          <additionalJOption>-J-XX:+TieredCompilation&lt;/additionalJOption</additionalJOption>
+          <additionalJOption>-J-XX:TieredStopAtLevel=1&lt;/additionalJOption</additionalJOption>
         </configuration>
       </plugin>
     </plugins>

packages/jsii-pacmak/test/expected.jsii-calc-lib/java/pom.xml

@@ -103,6 +103,8 @@
         <configuration>
           <failOnError>false</failOnError>
           <show>protected</show>
+          <additionalJOption>-J-XX:+TieredCompilation&lt;/additionalJOption</additionalJOption>
+          <additionalJOption>-J-XX:TieredStopAtLevel=1&lt;/additionalJOption</additionalJOption>
         </configuration>
       </plugin>
     </plugins>

packages/jsii-pacmak/test/expected.jsii-calc/java/pom.xml

@@ -134,6 +134,8 @@
         <configuration>
           <failOnError>false</failOnError>
           <show>protected</show>
+          <additionalJOption>-J-XX:+TieredCompilation&lt;/additionalJOption</additionalJOption>
+          <additionalJOption>-J-XX:TieredStopAtLevel=1&lt;/additionalJOption</additionalJOption>
         </configuration>
       </plugin>
     </plugins>

packages/jsii-reflect/lib/assembly.ts

@@ -154,6 +154,16 @@ export class Assembly {
     return this._types[fqn];
   }
 
+  /**
+   * Validate an assembly after loading
+   *
+   * If the assembly was loaded without validation, call this to validate
+   * it after all. Throws an exception if validation fails.
+   */
+  public validate() {
+    jsii.validateAssembly(this.spec);
+  }
+
   private get _dependencies() {
     if (!this._dependencyCache) {
       this._dependencyCache = { };

packages/jsii-reflect/lib/index.ts

@@ -1,5 +1,6 @@
 export * from './assembly';
 export * from './class';
+export * from './callable';
 export * from './dependency';
 export * from './docs';
 export * from './enum';