feat(rosetta): extract and compile samples into "tablets" (#925)

Version 2 of the "sampiler" is called "Rosetta". It now provides
more control over the compilation of samples found in the source code.

It integrates into an existing build by running `jsii-rosetta extract`,
which will extract sample code from a jsii assembly, compile it, convert
it to all supported languages, and storing the result in a a "tablet
file" (effectively, a sample dictionary).

Tablet files can then be used by `jsii-pacmak` to look up translations
for the code samples it encounters as it is generating language-specific
sources.

In case the build does not contain a Rosetta step, Pacmak will try to
convert samples that are not found in the tablet on the fly. However,
the samples will not benefit from compilation, type checking and fixture
support.

This change also fixes the `jsii-pacmak` all-at-once builder, and will
properly copy out binaries for the .NET and Java builds back to the
declared output directories, and properly use output directories of
packages not included in the megabuild as local package repositories.

.gitignore

@@ -7,3 +7,4 @@ tsconfig.tsbuildinfo
 dist/
 .vscode
 *.tsbuildinfo
+*.tabl.json

packages/jsii-calc-base-of-base/package.json

@@ -24,12 +24,13 @@
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "scripts": {
-    "build": "jsii",
+    "build": "jsii && jsii-rosetta",
     "test": "diff-test test/assembly.jsii .jsii",
     "test:update": "npm run build && UPDATE_DIFF=1 npm run test"
   },
   "devDependencies": {
     "jsii": "^0.20.5",
+    "jsii-rosetta": "^0.20.5",
     "jsii-build-tools": "^0.20.5"
   },
   "jsii": {

packages/jsii-calc-base/package.json

@@ -24,7 +24,7 @@
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "scripts": {
-    "build": "jsii",
+    "build": "jsii && jsii-rosetta",
     "test": "diff-test test/assembly.jsii .jsii",
     "test:update": "npm run build && UPDATE_DIFF=1 npm run test"
   },
@@ -36,6 +36,7 @@
   },
   "devDependencies": {
     "jsii": "^0.20.5",
+    "jsii-rosetta": "^0.20.5",
     "jsii-build-tools": "^0.20.5"
   },
   "jsii": {
@@ -59,4 +60,4 @@
     },
     "versionFormat": "short"
   }
-}
+}

packages/jsii-calc-lib/.npmignore

@@ -6,3 +6,7 @@
 
 # Include .jsii
 !.jsii
+
+
+# Exclude jsii outdir
+dist

packages/jsii-calc-lib/package.json

@@ -26,7 +26,7 @@
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "scripts": {
-    "build": "jsii",
+    "build": "jsii && jsii-rosetta",
     "test": "diff-test test/assembly.jsii .jsii",
     "test:update": "npm run build && UPDATE_DIFF=1 npm run test"
   },
@@ -38,6 +38,7 @@
   },
   "devDependencies": {
     "jsii": "^0.20.5",
+    "jsii-rosetta": "^0.20.5",
     "jsii-build-tools": "^0.20.5"
   },
   "jsii": {
@@ -63,4 +64,4 @@
     },
     "versionFormat": "short"
   }
-}
+}

packages/jsii-calc/README.md

@@ -2,13 +2,20 @@
 
 This library is used to demonstrate and test the features of JSII
 
-## Sphinx
+## How to use running sum API:
 
-This file will be incorporated into the sphinx documentation.
+First, create a calculator:
 
-If this file starts with an "H1" line (in our case `# jsii Calculator`), this
-heading will be used as the Sphinx topic name. Otherwise, the name of the module
-(`jsii-calc`) will be used instead.
+```ts
+const calculator = new calc.Calculator();
+```
+
+Then call some operations:
+
+
+```ts fixture=with-calculator
+calculator.add(10);
+```
 
 ## Code Samples
 

packages/jsii-calc/package.json

@@ -25,7 +25,7 @@
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "scripts": {
-    "build": "jsii",
+    "build": "jsii && jsii-rosetta --compile",
     "watch": "jsii -w",
     "test": "node test/test.calc.js && diff-test test/assembly.jsii .jsii",
     "test:update": "npm run build && UPDATE_DIFF=1 npm run test"
@@ -43,6 +43,7 @@
   },
   "devDependencies": {
     "jsii": "^0.20.5",
+    "jsii-rosetta": "^0.20.5",
     "jsii-build-tools": "^0.20.5"
   },
   "jsii": {
@@ -100,4 +101,4 @@
       ]
     }
   ]
-}
+}

packages/jsii-calc/rosetta/default.ts-fixture

@@ -0,0 +1,3 @@
+import calc = require('.');
+
+/// here

packages/jsii-calc/rosetta/with-calculator.ts-fixture

@@ -0,0 +1,4 @@
+import calc = require('.');
+const calculator = new calc.Calculator();
+
+/// here

packages/jsii-calc/test/assembly.jsii

@@ -195,7 +195,7 @@
   },
   "name": "jsii-calc",
   "readme": {
-    "markdown": "# jsii Calculator\n\nThis library is used to demonstrate and test the features of JSII\n\n## Sphinx\n\nThis file will be incorporated into the sphinx documentation.\n\nIf this file starts with an \"H1\" line (in our case `# jsii Calculator`), this\nheading will be used as the Sphinx topic name. Otherwise, the name of the module\n(`jsii-calc`) will be used instead.\n\n## Code Samples\n\n```ts\n/* This is totes a magic comment in here, just you wait! */\nconst foo = 'bar';\n```\n"
+    "markdown": "# jsii Calculator\n\nThis library is used to demonstrate and test the features of JSII\n\n## How to use running sum API:\n\nFirst, create a calculator:\n\n```ts\nconst calculator = new calc.Calculator();\n```\n\nThen call some operations:\n\n\n```ts fixture=with-calculator\ncalculator.add(10);\n```\n\n## Code Samples\n\n```ts\n/* This is totes a magic comment in here, just you wait! */\nconst foo = 'bar';\n```\n"
   },
   "repository": {
     "directory": "packages/jsii-calc",
@@ -11128,5 +11128,5 @@
     }
   },
   "version": "0.20.5",
-  "fingerprint": "g9C1lL8c+vgxBjOWVBFMMPlcwkF3Z81xxTAGfc73x9o="
+  "fingerprint": "/MRTbTnRC1UWxsPIrca+9Yo1IBKsEueT75P22pQoV1o="
 }

packages/jsii-pacmak/bin/jsii-pacmak.ts

@@ -2,6 +2,7 @@
 import path = require('path');
 import process = require('process');
 import yargs = require('yargs');
+import { Rosetta } from 'jsii-rosetta';
 import logging = require('../lib/logging');
 import { Timers } from '../lib/timer';
 import { VERSION_DESC } from '../lib/version';
@@ -78,6 +79,15 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets';
       desc: 'Auto-update .npmignore to exclude the output directory and include the .jsii file',
       default: true
     })
+    .option('rosetta-tablet', {
+      type: 'string',
+      desc: 'Location of a jsii-rosetta tablet with sample translations (created using \'jsii-rosetta extract\')'
+    })
+    .option('rosetta-translate-live', {
+      type: 'boolean',
+      desc: 'Translate code samples on-the-fly if they can\'t be found in the samples tablet',
+      default: true
+    })
     .version(VERSION_DESC)
     .strict()
     .argv;
@@ -89,6 +99,11 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets';
 
   const timers = new Timers();
 
+  const rosetta = new Rosetta({ liveConversion: argv['rosetta-translate-live'] });
+  if (argv['rosetta-tablet']) {
+    await rosetta.loadTabletFromFile(argv['rosetta-tablet']);
+  }
+
   const modulesToPackage = await findJsiiModules(argv._, argv.recurse);
   logging.info(`Found ${modulesToPackage.length} modules to package`);
   if (modulesToPackage.length === 0) {
@@ -114,9 +129,12 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets';
   });
 
   await timers.recordAsync('load jsii', () => {
-    logging.info('Loading jsii assemblies');
+    logging.info('Loading jsii assemblies and translations');
     return Promise.all(modulesToPackage
-      .map(m => m.load()));
+      .map(async m => {
+        await m.load();
+        await rosetta.addAssembly(m.assembly.spec, m.moduleDirectory);
+      }));
   });
 
   try {
@@ -155,19 +173,20 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets';
 
   async function buildTargetsForLanguage(targetLanguage: string, modules: JsiiModule[], perLanguageDirectory: boolean) {
     // ``argv.target`` is guaranteed valid by ``yargs`` through the ``choices`` directive.
-    const builder = ALL_BUILDERS[targetLanguage as TargetName];
-    if (!builder) {
+    const factory = ALL_BUILDERS[targetLanguage as TargetName];
+    if (!factory) {
       throw new Error(`Unsupported target: '${targetLanguage}'`);
     }
 
-    await builder.buildModules(modules, {
+    await factory(modules, {
       clean: argv.clean,
       codeOnly: argv['code-only'],
+      rosetta,
       force: argv.force,
       fingerprint: argv.fingerprint,
       arguments: argv,
       languageSubdirectory: perLanguageDirectory,
-    });
+    }).buildModules();
   }
 })().catch(err => {
   process.stderr.write(`${err.stack}\n`);

packages/jsii-pacmak/lib/builder.ts

@@ -3,6 +3,7 @@ import logging = require('./logging');
 import { JsiiModule } from './packaging';
 import { TargetConstructor, Target } from './target';
 import { Scratch } from './util';
+import { Rosetta } from 'jsii-rosetta';
 
 export interface BuildOptions {
   /**
@@ -36,6 +37,11 @@ export interface BuildOptions {
    * Whether to add an additional subdirectory for the target language
    */
   languageSubdirectory?: boolean;
+
+  /**
+   * The Rosetta instance to load examples from
+   */
+  rosetta: Rosetta;
 }
 
 /**
@@ -44,32 +50,27 @@ export interface BuildOptions {
  * Building can happen one target at a time, or multiple targets at a time.
  */
 export interface TargetBuilder {
-  buildModules(modules: JsiiModule[], options: BuildOptions): Promise<void>;
-}
-
-/**
- * Return the output directory if all modules have the same directory
- */
-export function allOutputDirectoriesTheSame(modules: JsiiModule[]): boolean {
-  if (modules.length === 0) { return true; }
-  const ret = modules[0].outputDirectory;
-  return modules.every(m => m.outputDirectory === ret);
+  buildModules(): Promise<void>;
 }
 
 /**
  * Builds the targets for the given language sequentially
  */
 export class OneByOneBuilder implements TargetBuilder {
-  public constructor(private readonly targetName: string, private readonly targetConstructor: TargetConstructor) {
+  public constructor(
+    private readonly targetName: string,
+    private readonly targetConstructor: TargetConstructor,
+    private readonly modules: JsiiModule[],
+    private readonly options: BuildOptions) {
 
   }
 
-  public async buildModules(modules: JsiiModule[], options: BuildOptions): Promise<void> {
-    for (const module of modules) {
-      if (options.codeOnly) {
-        await this.generateModuleCode(module, options);
+  public async buildModules(): Promise<void> {
+    for (const module of this.modules) {
+      if (this.options.codeOnly) {
+        await this.generateModuleCode(module, this.options);
       } else {
-        await this.buildModule(module, options);
+        await this.buildModule(module, this.options);
       }
     }
   }
@@ -110,7 +111,8 @@ export class OneByOneBuilder implements TargetBuilder {
       assembly: module.assembly,
       fingerprint: options.fingerprint,
       force: options.force,
-      arguments: options.arguments
+      arguments: options.arguments,
+      rosetta: options.rosetta,
     });
   }