From 0e99622f2769f8dcaeac41ac3847372c990146e1 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Tue, 16 Apr 2019 13:58:21 +0200 Subject: [PATCH 1/5] fix(jsii-reflect): don't load same assembly multiple times deCDK tests were calling `loadModule()` for every package. `loadModule()` did have some recursion avoidance INSIDE one call, but not across multiple load calls on the same type system. This brings down the deCDK tests from 80s to 10s. Add an option to disable validation, which would bring the full CDK typesystem loading down from 10s to 600ms (validation is enabled by default). Some more type exposure updates so that we can update awslint and friends in the CDK repository to take advantage of the new jsii model. --- packages/jsii-reflect/lib/assembly.ts | 10 +++++ packages/jsii-reflect/lib/index.ts | 1 + packages/jsii-reflect/lib/initializer.ts | 4 ++ packages/jsii-reflect/lib/method.ts | 4 ++ packages/jsii-reflect/lib/type-system.ts | 54 ++++++++++++++++++------ 5 files changed, 59 insertions(+), 14 deletions(-) diff --git a/packages/jsii-reflect/lib/assembly.ts b/packages/jsii-reflect/lib/assembly.ts index 861b31c08c..4303bf4dc1 100644 --- a/packages/jsii-reflect/lib/assembly.ts +++ b/packages/jsii-reflect/lib/assembly.ts @@ -154,6 +154,16 @@ export class Assembly { return this._types[fqn]; } + /** + * Validate an assembly after loading + * + * If the assembly was loaded without validation, call this to validate + * it after all. Throws an exception if validation fails. + */ + public validate() { + jsii.validateAssembly(this.spec); + } + private get _dependencies() { if (!this._dependencyCache) { this._dependencyCache = { }; diff --git a/packages/jsii-reflect/lib/index.ts b/packages/jsii-reflect/lib/index.ts index 362d998800..211c90b020 100644 --- a/packages/jsii-reflect/lib/index.ts +++ b/packages/jsii-reflect/lib/index.ts @@ -1,5 +1,6 @@ export * from './assembly'; export * from './class'; +export * from './callable'; export * from './dependency'; export * from './docs'; export * from './enum'; diff --git a/packages/jsii-reflect/lib/initializer.ts b/packages/jsii-reflect/lib/initializer.ts index 4a64026285..515efd8033 100644 --- a/packages/jsii-reflect/lib/initializer.ts +++ b/packages/jsii-reflect/lib/initializer.ts @@ -5,6 +5,10 @@ import { SourceLocatable } from './source'; import { MemberKind, TypeMember } from './type-member'; export class Initializer extends Callable implements Documentable, Overridable, TypeMember, SourceLocatable { + public static isInitializer(x: Callable): x is Initializer { + return x instanceof Initializer; + } + public readonly kind = MemberKind.Initializer; public readonly name = ''; public readonly abstract = false; diff --git a/packages/jsii-reflect/lib/method.ts b/packages/jsii-reflect/lib/method.ts index 843eac7507..38c59f5897 100644 --- a/packages/jsii-reflect/lib/method.ts +++ b/packages/jsii-reflect/lib/method.ts @@ -15,6 +15,10 @@ import { TypeSystem } from './type-system'; export const INITIALIZER_NAME = ''; export class Method extends Callable implements Documentable, Overridable, TypeMember, SourceLocatable { + public static isMethod(x: Callable): x is Method { + return x instanceof Method; + } + public readonly kind = MemberKind.Method; constructor( diff --git a/packages/jsii-reflect/lib/type-system.ts b/packages/jsii-reflect/lib/type-system.ts index 2927eb5ec3..33c8de26c2 100644 --- a/packages/jsii-reflect/lib/type-system.ts +++ b/packages/jsii-reflect/lib/type-system.ts @@ -32,18 +32,21 @@ export class TypeSystem { * If `fileOrDirectory` is a file, it will be treated as a single .jsii file. * If `fileOrDirectory` is a directory, it will be treated as a jsii npm module. * + * Not validating makes the difference between loading assemblies with lots + * of dependencies (such as app-delivery) in 90ms vs 3500ms. + * * @param fileOrDirectory A .jsii file path or a module directory + * @param validate Whether or not to validate the assembly while loading it. */ - public async load(fileOrDirectory: string) { + public async load(fileOrDirectory: string, validate = true) { if ((await stat(fileOrDirectory)).isDirectory()) { - return await this.loadModule(fileOrDirectory); + return await this.loadModule(fileOrDirectory, validate); } else { - return await this.loadFile(fileOrDirectory); + return await this.loadFile(fileOrDirectory, true, validate); } } - public async loadModule(dir: string): Promise { - const visited = new Set(); + public async loadModule(dir: string, validate = true): Promise { const self = this; const out = await _loadModule(dir, true); @@ -54,18 +57,29 @@ export class TypeSystem { return out; async function _loadModule(moduleDirectory: string, isRoot = false) { - if (visited.has(moduleDirectory)) { - return; - } - visited.add(moduleDirectory); - const filePath = path.join(moduleDirectory, 'package.json'); const pkg = JSON.parse((await readFile(filePath)).toString()); if (!pkg.jsii) { throw new Error(`No "jsii" section in ${filePath}`); } - const root = await self.loadFile(path.join(moduleDirectory, '.jsii'), isRoot); + // Load the assembly, but don't recurse if we already have an assembly with the same name. + // Validation is not an insignificant time sink, and loading IS insignificant, so do a + // load without validation first. This saves about 2/3rds of processing time. + const ass = await self.loadAssembly(path.join(moduleDirectory, '.jsii'), false); + if (self.includesAssembly(ass.name)) { + const existing = self.findAssembly(ass.name); + if (existing.version !== ass.version) { + throw new Error(`Conflicting versions of ${ass.name} in type system: previously loaded ${existing.version}, trying to load ${ass.version}`); + } + return existing; + } + + if (validate) { + ass.validate(); + } + + const root = await self.addAssembly(ass, isRoot); const bundled: string[] = pkg.bundledDependencies || pkg.bundleDependencies || []; const loadDependencies = async (deps: { [name: string]: string }) => { @@ -87,9 +101,9 @@ export class TypeSystem { } } - public async loadFile(file: string, isRoot = true) { - const spec = JSON.parse((await readFile(file)).toString()); - return this.addAssembly(new Assembly(this, jsii.validateAssembly(spec)), isRoot); + public async loadFile(file: string, isRoot = true, validate = true) { + const assembly = await this.loadAssembly(file, validate); + return this.addAssembly(assembly, isRoot); } public addAssembly(asm: Assembly, isRoot = true) { @@ -205,4 +219,16 @@ export class TypeSystem { }); return out; } + + /** + * Load an assembly without adding it to the typesystem + * @param file Assembly file to load + * @param validate Whether to validate the assembly or just assume it matches the schema + */ + private async loadAssembly(file: string, validate = true) { + const spec = JSON.parse((await readFile(file)).toString()); + const ass = validate ? jsii.validateAssembly(spec) : spec as jsii.Assembly; + return new Assembly(this, ass); + } + } From b77a5cab37cb9bab60a68cba8f2602abd1a688c5 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Tue, 16 Apr 2019 15:49:02 +0200 Subject: [PATCH 2/5] Speed up Java build from around 30s to around 10s per package --- packages/jsii-pacmak/bin/jsii-pacmak.ts | 21 ++++-- packages/jsii-pacmak/lib/targets/java.ts | 14 +++- packages/jsii-pacmak/lib/timer.ts | 82 ++++++++++++++++++++++++ packages/jsii-pacmak/lib/util.ts | 3 +- 4 files changed, 113 insertions(+), 7 deletions(-) create mode 100644 packages/jsii-pacmak/lib/timer.ts diff --git a/packages/jsii-pacmak/bin/jsii-pacmak.ts b/packages/jsii-pacmak/bin/jsii-pacmak.ts index dfcb913018..429477fa64 100644 --- a/packages/jsii-pacmak/bin/jsii-pacmak.ts +++ b/packages/jsii-pacmak/bin/jsii-pacmak.ts @@ -7,6 +7,7 @@ import process = require('process'); import yargs = require('yargs'); import logging = require('../lib/logging'); import { Target } from '../lib/target'; +import { Timers } from '../lib/timer'; import { resolveDependencyDirectory, shell } from '../lib/util'; import { VERSION_DESC } from '../lib/version'; @@ -128,22 +129,34 @@ import { VERSION_DESC } from '../lib/version'; await updateNpmIgnore(packageDir, npmIgnoreExclude); } + const timers = new Timers(); + const tmpdir = await fs.mkdtemp(path.join(os.tmpdir(), 'npm-pack')); try { - const tarball = await npmPack(packageDir, tmpdir); + const tarball = await timers.recordAsync('npm pack', () => { + return npmPack(packageDir, tmpdir); + }); for (const targetName of targets) { // if we are targeting a single language, output to outdir, otherwise outdir/ const targetOutputDir = (targets.length > 1 || forceSubdirectory) ? path.join(outDir, targetName.toString()) : outDir; logging.debug(`Building ${pkg.name}/${targetName}: ${targetOutputDir}`); - await generateTarget(packageDir, targetName.toString(), targetOutputDir, tarball); + + await timers.recordAsync(targetName.toString(), () => + generateTarget(packageDir, targetName.toString(), targetOutputDir, tarball) + ); } } finally { - logging.debug(`Removing ${tmpdir}`); + if (argv.clean) { + logging.debug(`Removing ${tmpdir}`); + } else { + logging.debug(`Temporary directory retained (--no-clean): ${tmpdir}`); + } await fs.remove(tmpdir); } + logging.info(`Packaged. ${timers.display()}`); } async function generateTarget(packageDir: string, targetName: string, targetOutputDir: string, tarball: string) { @@ -221,7 +234,7 @@ async function updateNpmIgnore(packageDir: string, excludeOutdir: string | undef if (modified) { await fs.writeFile(npmIgnorePath, lines.join('\n') + '\n'); - logging.info('Updated .npmignre'); + logging.info('Updated .npmignore'); } function includePattern(comment: string, ...patterns: string[]) { diff --git a/packages/jsii-pacmak/lib/targets/java.ts b/packages/jsii-pacmak/lib/targets/java.ts index 97eac91fa8..2855f4ca38 100644 --- a/packages/jsii-pacmak/lib/targets/java.ts +++ b/packages/jsii-pacmak/lib/targets/java.ts @@ -65,7 +65,14 @@ export default class Java extends Target { await shell( 'mvn', [...mvnArguments, 'deploy', `-D=altDeploymentRepository=local::default::${url}`, `--settings=${userXml}`], - { cwd: sourceDir } + { + cwd: sourceDir, + env: { + // Twiddle the JVM settings a little for Maven. Delaying JIT compilation + // brings down Maven execution time by about 1/3rd (15->10s, 30->20s) + MAVEN_OPTS: `${process.env.MAVEN_OPTS || ''} -XX:+TieredCompilation -XX:TieredStopAtLevel=1` + } + } ); } @@ -433,7 +440,10 @@ class JavaGenerator extends Generator { }, configuration: { failOnError: false, - show: 'protected' + show: 'protected', + // Adding these makes JavaDoc generation about a 3rd faster (which is far and away the most + // expensive part of the build) + additionalJOption: ['-J-XX:+TieredCompilation 60) { + const mins = Math.floor(time / 60); + parts.push(mins + 'm'); + time -= mins * 60; + } + parts.push(time.toFixed(1) + 's'); + + return parts.join(''); + } +} + +/** + * A collection of Timers + */ +export class Timers { + private readonly timers: Timer[] = []; + + public record(label: string, operation: () => T): T { + const timer = this.start(label); + try { + const x = operation(); + timer.end(); + return x; + } catch (e) { + timer.end(); + throw e; + } + } + + public async recordAsync(label: string, operation: () => Promise) { + const timer = this.start(label); + try { + const x = await operation(); + timer.end(); + return x; + } catch (e) { + timer.end(); + throw e; + } + } + + public start(label: string) { + const timer = new Timer(label); + this.timers.push(timer); + return timer; + } + + public display(): string { + const timers = this.timers.filter(t => t.isSet()); + timers.sort((a: Timer, b: Timer) => b.timeMs! - a.timeMs!); + return timers.map(t => `${t.label} (${t.humanTime()})`).join(' | '); + } +} diff --git a/packages/jsii-pacmak/lib/util.ts b/packages/jsii-pacmak/lib/util.ts index 7193382200..1eb5719f37 100644 --- a/packages/jsii-pacmak/lib/util.ts +++ b/packages/jsii-pacmak/lib/util.ts @@ -17,7 +17,8 @@ export function resolveDependencyDirectory(packageDir: string, dependencyName: s export function shell(cmd: string, args: string[], options: SpawnOptions): Promise { return new Promise((resolve, reject) => { - const child = spawn(cmd, args, { ...options, shell: true, stdio: ['ignore', 'pipe', 'pipe'] }); + logging.debug(cmd, args.join(' '), JSON.stringify(options)); + const child = spawn(cmd, args, { ...options, shell: true, env: { ...process.env, ...options.env || {} }, stdio: ['ignore', 'pipe', 'pipe'] }); const stdout = new Array(); const stderr = new Array(); child.stdout.on('data', chunk => { From d00e36a91c35bafc10e8d7592cb1e65c62b39c51 Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Tue, 16 Apr 2019 16:00:15 +0200 Subject: [PATCH 3/5] Fix roots test, make keyword args --- packages/jsii-reflect/lib/type-system.ts | 50 +++++++++++++++--------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/packages/jsii-reflect/lib/type-system.ts b/packages/jsii-reflect/lib/type-system.ts index 33c8de26c2..45e64cea64 100644 --- a/packages/jsii-reflect/lib/type-system.ts +++ b/packages/jsii-reflect/lib/type-system.ts @@ -38,15 +38,15 @@ export class TypeSystem { * @param fileOrDirectory A .jsii file path or a module directory * @param validate Whether or not to validate the assembly while loading it. */ - public async load(fileOrDirectory: string, validate = true) { + public async load(fileOrDirectory: string, options: { validate?: boolean } = {}) { if ((await stat(fileOrDirectory)).isDirectory()) { - return await this.loadModule(fileOrDirectory, validate); + return await this.loadModule(fileOrDirectory, options); } else { - return await this.loadFile(fileOrDirectory, true, validate); + return await this.loadFile(fileOrDirectory, { ...options, isRoot: true }); } } - public async loadModule(dir: string, validate = true): Promise { + public async loadModule(dir: string, options: { validate?: boolean } = {}): Promise { const self = this; const out = await _loadModule(dir, true); @@ -66,20 +66,25 @@ export class TypeSystem { // Load the assembly, but don't recurse if we already have an assembly with the same name. // Validation is not an insignificant time sink, and loading IS insignificant, so do a // load without validation first. This saves about 2/3rds of processing time. - const ass = await self.loadAssembly(path.join(moduleDirectory, '.jsii'), false); - if (self.includesAssembly(ass.name)) { - const existing = self.findAssembly(ass.name); - if (existing.version !== ass.version) { - throw new Error(`Conflicting versions of ${ass.name} in type system: previously loaded ${existing.version}, trying to load ${ass.version}`); + const asm = await self.loadAssembly(path.join(moduleDirectory, '.jsii'), false); + if (self.includesAssembly(asm.name)) { + const existing = self.findAssembly(asm.name); + if (existing.version !== asm.version) { + throw new Error(`Conflicting versions of ${asm.name} in type system: previously loaded ${existing.version}, trying to load ${asm.version}`); } + // Make sure that we mark this thing as root after all if it wasn't yet. + if (isRoot) { + self.addRoot(asm); + } + return existing; } - if (validate) { - ass.validate(); + if (options.validate !== false) { + asm.validate(); } - const root = await self.addAssembly(ass, isRoot); + const root = await self.addAssembly(asm, { isRoot }); const bundled: string[] = pkg.bundledDependencies || pkg.bundleDependencies || []; const loadDependencies = async (deps: { [name: string]: string }) => { @@ -101,12 +106,12 @@ export class TypeSystem { } } - public async loadFile(file: string, isRoot = true, validate = true) { - const assembly = await this.loadAssembly(file, validate); - return this.addAssembly(assembly, isRoot); + public async loadFile(file: string, options: { isRoot?: boolean, validate?: boolean } = {}) { + const assembly = await this.loadAssembly(file, options.validate !== false); + return this.addAssembly(assembly, options); } - public addAssembly(asm: Assembly, isRoot = true) { + public addAssembly(asm: Assembly, options: { isRoot?: boolean } = {}) { if (asm.system !== this) { throw new Error('Assembly has been created for different typesystem'); } @@ -116,8 +121,8 @@ export class TypeSystem { this.assemblies.push(asm); } - if (isRoot && !this.roots.includes(asm)) { - this.roots.push(asm); + if (options.isRoot !== false) { + this.addRoot(asm); } return asm; @@ -132,6 +137,10 @@ export class TypeSystem { return name in this._assemblyLookup; } + public isRoot(name: string) { + return this.roots.map(r => r.name).includes(name); + } + public findAssembly(name: string) { if (!(name in this._assemblyLookup)) { throw new Error(`Assembly "${name}" not found`); @@ -231,4 +240,9 @@ export class TypeSystem { return new Assembly(this, ass); } + private addRoot(asm: Assembly) { + if (!this.roots.map(r => r.name).includes(asm.name)) { + this.roots.push(asm); + } + } } From 9e06f58be97ee8652c6d8daf420852e77684483b Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Tue, 16 Apr 2019 16:06:40 +0200 Subject: [PATCH 4/5] Fix breaking API --- packages/jsii-diff/bin/jsii-diff.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-diff/bin/jsii-diff.ts b/packages/jsii-diff/bin/jsii-diff.ts index e4b77ee481..41ba874a29 100644 --- a/packages/jsii-diff/bin/jsii-diff.ts +++ b/packages/jsii-diff/bin/jsii-diff.ts @@ -91,7 +91,7 @@ async function loadFromFilesystem(name: string) { if (stat.isDirectory()) { return await ts.loadModule(name); } else { - return await ts.loadFile(name, true); + return await ts.loadFile(name); } } From 24a790f982adb0094272d9bc8c38a1027684c07c Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Tue, 16 Apr 2019 16:15:04 +0200 Subject: [PATCH 5/5] Update expectations --- packages/jsii-pacmak/test/expected.jsii-calc-base/java/pom.xml | 2 ++ packages/jsii-pacmak/test/expected.jsii-calc-lib/java/pom.xml | 2 ++ packages/jsii-pacmak/test/expected.jsii-calc/java/pom.xml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/packages/jsii-pacmak/test/expected.jsii-calc-base/java/pom.xml b/packages/jsii-pacmak/test/expected.jsii-calc-base/java/pom.xml index d20a88fd89..94416e0818 100644 --- a/packages/jsii-pacmak/test/expected.jsii-calc-base/java/pom.xml +++ b/packages/jsii-pacmak/test/expected.jsii-calc-base/java/pom.xml @@ -103,6 +103,8 @@ false protected + -J-XX:+TieredCompilation</additionalJOption + -J-XX:TieredStopAtLevel=1</additionalJOption diff --git a/packages/jsii-pacmak/test/expected.jsii-calc-lib/java/pom.xml b/packages/jsii-pacmak/test/expected.jsii-calc-lib/java/pom.xml index d76f149d07..46510ef13a 100644 --- a/packages/jsii-pacmak/test/expected.jsii-calc-lib/java/pom.xml +++ b/packages/jsii-pacmak/test/expected.jsii-calc-lib/java/pom.xml @@ -103,6 +103,8 @@ false protected + -J-XX:+TieredCompilation</additionalJOption + -J-XX:TieredStopAtLevel=1</additionalJOption diff --git a/packages/jsii-pacmak/test/expected.jsii-calc/java/pom.xml b/packages/jsii-pacmak/test/expected.jsii-calc/java/pom.xml index 56369d78ad..2906433539 100644 --- a/packages/jsii-pacmak/test/expected.jsii-calc/java/pom.xml +++ b/packages/jsii-pacmak/test/expected.jsii-calc/java/pom.xml @@ -134,6 +134,8 @@ false protected + -J-XX:+TieredCompilation</additionalJOption + -J-XX:TieredStopAtLevel=1</additionalJOption