From eec44e106ee1e3d2e3d03f70e4d87a4d7ee0bbba Mon Sep 17 00:00:00 2001 From: Rico Huijbers Date: Wed, 13 Nov 2019 15:47:17 +0100 Subject: [PATCH] feat(rosetta): extract and compile samples into "tablets" (#925) Version 2 of the "sampiler" is called "Rosetta". It now provides more control over the compilation of samples found in the source code. It integrates into an existing build by running `jsii-rosetta extract`, which will extract sample code from a jsii assembly, compile it, convert it to all supported languages, and storing the result in a a "tablet file" (effectively, a sample dictionary). Tablet files can then be used by `jsii-pacmak` to look up translations for the code samples it encounters as it is generating language-specific sources. In case the build does not contain a Rosetta step, Pacmak will try to convert samples that are not found in the tablet on the fly. However, the samples will not benefit from compilation, type checking and fixture support. This change also fixes the `jsii-pacmak` all-at-once builder, and will properly copy out binaries for the .NET and Java builds back to the declared output directories, and properly use output directories of packages not included in the megabuild as local package repositories. --- .gitignore | 1 + packages/jsii-calc-base-of-base/package.json | 3 +- packages/jsii-calc-base/package.json | 5 +- packages/jsii-calc-lib/.npmignore | 4 + packages/jsii-calc-lib/package.json | 5 +- packages/jsii-calc/README.md | 17 +- packages/jsii-calc/package.json | 5 +- packages/jsii-calc/rosetta/default.ts-fixture | 3 + .../rosetta/with-calculator.ts-fixture | 4 + packages/jsii-calc/test/assembly.jsii | 4 +- packages/jsii-pacmak/bin/jsii-pacmak.ts | 31 +- packages/jsii-pacmak/lib/builder.ts | 36 ++- packages/jsii-pacmak/lib/target.ts | 70 ++-- packages/jsii-pacmak/lib/targets/dotnet.ts | 250 +++++++++++---- .../lib/targets/dotnet/dotnetgenerator.ts | 3 +- .../lib/targets/dotnet/dotnettyperesolver.ts | 19 +- .../lib/targets/dotnet/filegenerator.ts | 27 +- packages/jsii-pacmak/lib/targets/index.ts | 19 +- packages/jsii-pacmak/lib/targets/java.ts | 302 +++++++++++------- packages/jsii-pacmak/lib/targets/python.ts | 287 +++++++++-------- packages/jsii-pacmak/lib/util.ts | 24 ++ packages/jsii-pacmak/package.json | 2 +- packages/jsii-pacmak/test/build-test.sh | 33 +- .../.jsii | 4 +- .../jsii/tests/calculator/package-info.java | 12 +- .../test/expected.jsii-calc/python/README.md | 17 +- .../python/src/jsii_calc/__init__.py | 18 +- packages/jsii-pacmak/tsconfig.json | 2 +- .../jsii-python-runtime/bin/generate-calc | 5 + packages/jsii-python-runtime/package.json | 2 +- .../.gitignore | 0 .../.npmignore | 0 packages/jsii-rosetta/README.md | 193 +++++++++++ packages/jsii-rosetta/bin/jsii-rosetta | 2 + packages/jsii-rosetta/bin/jsii-rosetta.ts | 152 +++++++++ .../examples/controlflow.ts | 0 .../examples/incomplete.ts | 0 packages/jsii-rosetta/lib/commands/convert.ts | 36 +++ packages/jsii-rosetta/lib/commands/extract.ts | 128 ++++++++ .../lib/commands/extract_worker.ts | 37 +++ packages/jsii-rosetta/lib/commands/read.ts | 72 +++++ packages/jsii-rosetta/lib/fixtures.ts | 57 ++++ packages/jsii-rosetta/lib/index.ts | 6 + packages/jsii-rosetta/lib/jsii/assemblies.ts | 121 +++++++ .../lib/jsii/jsii-utils.ts | 4 +- packages/jsii-rosetta/lib/jsii/packages.ts | 26 ++ .../lib/languages/default.ts | 86 ++--- packages/jsii-rosetta/lib/languages/index.ts | 9 + .../lib/languages/python.ts | 12 +- .../lib/languages/visualize.ts | 86 ++--- packages/jsii-rosetta/lib/logging.ts | 35 ++ .../lib/markdown/extract-snippets.ts | 21 ++ .../lib/markdown/markdown-renderer.ts | 4 +- .../lib/markdown/markdown.ts | 0 .../lib/markdown/replace-code-renderer.ts | 2 +- .../markdown/replace-typescript-transform.ts | 37 +++ .../lib/markdown/structure-renderer.ts | 0 .../lib/o-tree.ts | 75 ++++- .../lib/renderer.ts} | 106 +++--- packages/jsii-rosetta/lib/rosetta.ts | 152 +++++++++ packages/jsii-rosetta/lib/snippet.ts | 122 +++++++ packages/jsii-rosetta/lib/tablets/key.ts | 11 + packages/jsii-rosetta/lib/tablets/schema.ts | 63 ++++ packages/jsii-rosetta/lib/tablets/tablets.ts | 175 ++++++++++ packages/jsii-rosetta/lib/translate.ts | 132 ++++++++ .../lib/typescript/ast-utils.ts | 68 +++- .../lib/typescript/imports.ts | 6 +- .../lib/typescript/ts-compiler.ts | 12 +- packages/jsii-rosetta/lib/util.ts | 52 +++ .../package.json | 17 +- .../jsii-rosetta/test/jsii/assemblies.test.ts | 152 +++++++++ .../jsii-rosetta/test/jsii/astutils.test.ts | 19 ++ .../jsii/fixtures/rosetta/explicit.ts-fixture | 2 + .../test/markdown/roundtrip.test.ts | 9 + .../test/otree.test.ts | 0 .../test/python/calls.test.ts | 0 .../test/python/classes.test.ts | 0 .../test/python/comments.test.ts | 0 .../test/python/expressions.test.ts | 0 .../test/python/hiding.test.ts | 21 ++ .../test/python/imports.test.ts | 0 .../test/python/misc.test.ts | 0 .../test/python/python.ts | 7 +- .../test/python/statements.test.ts | 6 +- packages/jsii-rosetta/test/rosetta.test.ts | 151 +++++++++ .../tsconfig.json | 3 + packages/jsii-sampiler/CHANGELOG.md | 64 ---- packages/jsii-sampiler/README.md | 147 --------- packages/jsii-sampiler/bin/jsii-sampiler | 2 - packages/jsii-sampiler/bin/jsii-sampiler.ts | 86 ----- packages/jsii-sampiler/lib/index.ts | 3 - packages/jsii-sampiler/lib/translate.ts | 118 ------- packages/jsii-sampiler/lib/util.ts | 26 -- packages/jsii/lib/docs.ts | 69 +++- packages/jsii/lib/literate.ts | 20 +- packages/jsii/test/docs.test.ts | 18 ++ packages/jsii/test/literate.test.ts | 6 +- 97 files changed, 3174 insertions(+), 1086 deletions(-) create mode 100644 packages/jsii-calc/rosetta/default.ts-fixture create mode 100644 packages/jsii-calc/rosetta/with-calculator.ts-fixture rename packages/{jsii-sampiler => jsii-rosetta}/.gitignore (100%) rename packages/{jsii-sampiler => jsii-rosetta}/.npmignore (100%) create mode 100644 packages/jsii-rosetta/README.md create mode 100755 packages/jsii-rosetta/bin/jsii-rosetta create mode 100644 packages/jsii-rosetta/bin/jsii-rosetta.ts rename packages/{jsii-sampiler => jsii-rosetta}/examples/controlflow.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/examples/incomplete.ts (100%) create mode 100644 packages/jsii-rosetta/lib/commands/convert.ts create mode 100644 packages/jsii-rosetta/lib/commands/extract.ts create mode 100644 packages/jsii-rosetta/lib/commands/extract_worker.ts create mode 100644 packages/jsii-rosetta/lib/commands/read.ts create mode 100644 packages/jsii-rosetta/lib/fixtures.ts create mode 100644 packages/jsii-rosetta/lib/index.ts create mode 100644 packages/jsii-rosetta/lib/jsii/assemblies.ts rename packages/{jsii-sampiler => jsii-rosetta}/lib/jsii/jsii-utils.ts (92%) create mode 100644 packages/jsii-rosetta/lib/jsii/packages.ts rename packages/{jsii-sampiler => jsii-rosetta}/lib/languages/default.ts (72%) create mode 100644 packages/jsii-rosetta/lib/languages/index.ts rename packages/{jsii-sampiler => jsii-rosetta}/lib/languages/python.ts (97%) rename packages/{jsii-sampiler => jsii-rosetta}/lib/languages/visualize.ts (71%) create mode 100644 packages/jsii-rosetta/lib/logging.ts create mode 100644 packages/jsii-rosetta/lib/markdown/extract-snippets.ts rename packages/{jsii-sampiler => jsii-rosetta}/lib/markdown/markdown-renderer.ts (97%) rename packages/{jsii-sampiler => jsii-rosetta}/lib/markdown/markdown.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/lib/markdown/replace-code-renderer.ts (94%) create mode 100644 packages/jsii-rosetta/lib/markdown/replace-typescript-transform.ts rename packages/{jsii-sampiler => jsii-rosetta}/lib/markdown/structure-renderer.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/lib/o-tree.ts (70%) rename packages/{jsii-sampiler/lib/converter.ts => jsii-rosetta/lib/renderer.ts} (82%) create mode 100644 packages/jsii-rosetta/lib/rosetta.ts create mode 100644 packages/jsii-rosetta/lib/snippet.ts create mode 100644 packages/jsii-rosetta/lib/tablets/key.ts create mode 100644 packages/jsii-rosetta/lib/tablets/schema.ts create mode 100644 packages/jsii-rosetta/lib/tablets/tablets.ts create mode 100644 packages/jsii-rosetta/lib/translate.ts rename packages/{jsii-sampiler => jsii-rosetta}/lib/typescript/ast-utils.ts (85%) rename packages/{jsii-sampiler => jsii-rosetta}/lib/typescript/imports.ts (93%) rename packages/{jsii-sampiler => jsii-rosetta}/lib/typescript/ts-compiler.ts (84%) create mode 100644 packages/jsii-rosetta/lib/util.ts rename packages/{jsii-sampiler => jsii-rosetta}/package.json (76%) create mode 100644 packages/jsii-rosetta/test/jsii/assemblies.test.ts create mode 100644 packages/jsii-rosetta/test/jsii/astutils.test.ts create mode 100644 packages/jsii-rosetta/test/jsii/fixtures/rosetta/explicit.ts-fixture rename packages/{jsii-sampiler => jsii-rosetta}/test/markdown/roundtrip.test.ts (95%) rename packages/{jsii-sampiler => jsii-rosetta}/test/otree.test.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/calls.test.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/classes.test.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/comments.test.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/expressions.test.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/hiding.test.ts (76%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/imports.test.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/misc.test.ts (100%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/python.ts (87%) rename packages/{jsii-sampiler => jsii-rosetta}/test/python/statements.test.ts (88%) create mode 100644 packages/jsii-rosetta/test/rosetta.test.ts rename packages/{jsii-sampiler => jsii-rosetta}/tsconfig.json (96%) delete mode 100644 packages/jsii-sampiler/CHANGELOG.md delete mode 100644 packages/jsii-sampiler/README.md delete mode 100755 packages/jsii-sampiler/bin/jsii-sampiler delete mode 100644 packages/jsii-sampiler/bin/jsii-sampiler.ts delete mode 100644 packages/jsii-sampiler/lib/index.ts delete mode 100644 packages/jsii-sampiler/lib/translate.ts delete mode 100644 packages/jsii-sampiler/lib/util.ts diff --git a/.gitignore b/.gitignore index fb9f4cf428..6b8d2a7121 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ tsconfig.tsbuildinfo dist/ .vscode *.tsbuildinfo +*.tabl.json diff --git a/packages/jsii-calc-base-of-base/package.json b/packages/jsii-calc-base-of-base/package.json index 8e0eda7f7a..2f56d55b6e 100644 --- a/packages/jsii-calc-base-of-base/package.json +++ b/packages/jsii-calc-base-of-base/package.json @@ -24,12 +24,13 @@ "main": "lib/index.js", "types": "lib/index.d.ts", "scripts": { - "build": "jsii", + "build": "jsii && jsii-rosetta", "test": "diff-test test/assembly.jsii .jsii", "test:update": "npm run build && UPDATE_DIFF=1 npm run test" }, "devDependencies": { "jsii": "^0.20.5", + "jsii-rosetta": "^0.20.5", "jsii-build-tools": "^0.20.5" }, "jsii": { diff --git a/packages/jsii-calc-base/package.json b/packages/jsii-calc-base/package.json index cb8903a1b6..12bcf89f7f 100644 --- a/packages/jsii-calc-base/package.json +++ b/packages/jsii-calc-base/package.json @@ -24,7 +24,7 @@ "main": "lib/index.js", "types": "lib/index.d.ts", "scripts": { - "build": "jsii", + "build": "jsii && jsii-rosetta", "test": "diff-test test/assembly.jsii .jsii", "test:update": "npm run build && UPDATE_DIFF=1 npm run test" }, @@ -36,6 +36,7 @@ }, "devDependencies": { "jsii": "^0.20.5", + "jsii-rosetta": "^0.20.5", "jsii-build-tools": "^0.20.5" }, "jsii": { @@ -59,4 +60,4 @@ }, "versionFormat": "short" } -} \ No newline at end of file +} diff --git a/packages/jsii-calc-lib/.npmignore b/packages/jsii-calc-lib/.npmignore index d2284ef6ef..5a7c6524f1 100644 --- a/packages/jsii-calc-lib/.npmignore +++ b/packages/jsii-calc-lib/.npmignore @@ -6,3 +6,7 @@ # Include .jsii !.jsii + + +# Exclude jsii outdir +dist diff --git a/packages/jsii-calc-lib/package.json b/packages/jsii-calc-lib/package.json index 19e543f6c3..e6891677e7 100644 --- a/packages/jsii-calc-lib/package.json +++ b/packages/jsii-calc-lib/package.json @@ -26,7 +26,7 @@ "main": "lib/index.js", "types": "lib/index.d.ts", "scripts": { - "build": "jsii", + "build": "jsii && jsii-rosetta", "test": "diff-test test/assembly.jsii .jsii", "test:update": "npm run build && UPDATE_DIFF=1 npm run test" }, @@ -38,6 +38,7 @@ }, "devDependencies": { "jsii": "^0.20.5", + "jsii-rosetta": "^0.20.5", "jsii-build-tools": "^0.20.5" }, "jsii": { @@ -63,4 +64,4 @@ }, "versionFormat": "short" } -} \ No newline at end of file +} diff --git a/packages/jsii-calc/README.md b/packages/jsii-calc/README.md index 9044031f0a..619c3478d4 100644 --- a/packages/jsii-calc/README.md +++ b/packages/jsii-calc/README.md @@ -2,13 +2,20 @@ This library is used to demonstrate and test the features of JSII -## Sphinx +## How to use running sum API: -This file will be incorporated into the sphinx documentation. +First, create a calculator: -If this file starts with an "H1" line (in our case `# jsii Calculator`), this -heading will be used as the Sphinx topic name. Otherwise, the name of the module -(`jsii-calc`) will be used instead. +```ts +const calculator = new calc.Calculator(); +``` + +Then call some operations: + + +```ts fixture=with-calculator +calculator.add(10); +``` ## Code Samples diff --git a/packages/jsii-calc/package.json b/packages/jsii-calc/package.json index 60e275e194..da6f72a435 100644 --- a/packages/jsii-calc/package.json +++ b/packages/jsii-calc/package.json @@ -25,7 +25,7 @@ "main": "lib/index.js", "types": "lib/index.d.ts", "scripts": { - "build": "jsii", + "build": "jsii && jsii-rosetta --compile", "watch": "jsii -w", "test": "node test/test.calc.js && diff-test test/assembly.jsii .jsii", "test:update": "npm run build && UPDATE_DIFF=1 npm run test" @@ -43,6 +43,7 @@ }, "devDependencies": { "jsii": "^0.20.5", + "jsii-rosetta": "^0.20.5", "jsii-build-tools": "^0.20.5" }, "jsii": { @@ -100,4 +101,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/jsii-calc/rosetta/default.ts-fixture b/packages/jsii-calc/rosetta/default.ts-fixture new file mode 100644 index 0000000000..0dd13f8603 --- /dev/null +++ b/packages/jsii-calc/rosetta/default.ts-fixture @@ -0,0 +1,3 @@ +import calc = require('.'); + +/// here diff --git a/packages/jsii-calc/rosetta/with-calculator.ts-fixture b/packages/jsii-calc/rosetta/with-calculator.ts-fixture new file mode 100644 index 0000000000..1eddf79bd3 --- /dev/null +++ b/packages/jsii-calc/rosetta/with-calculator.ts-fixture @@ -0,0 +1,4 @@ +import calc = require('.'); +const calculator = new calc.Calculator(); + +/// here diff --git a/packages/jsii-calc/test/assembly.jsii b/packages/jsii-calc/test/assembly.jsii index f5f5942eda..7d85c69166 100644 --- a/packages/jsii-calc/test/assembly.jsii +++ b/packages/jsii-calc/test/assembly.jsii @@ -195,7 +195,7 @@ }, "name": "jsii-calc", "readme": { - "markdown": "# jsii Calculator\n\nThis library is used to demonstrate and test the features of JSII\n\n## Sphinx\n\nThis file will be incorporated into the sphinx documentation.\n\nIf this file starts with an \"H1\" line (in our case `# jsii Calculator`), this\nheading will be used as the Sphinx topic name. Otherwise, the name of the module\n(`jsii-calc`) will be used instead.\n\n## Code Samples\n\n```ts\n/* This is totes a magic comment in here, just you wait! */\nconst foo = 'bar';\n```\n" + "markdown": "# jsii Calculator\n\nThis library is used to demonstrate and test the features of JSII\n\n## How to use running sum API:\n\nFirst, create a calculator:\n\n```ts\nconst calculator = new calc.Calculator();\n```\n\nThen call some operations:\n\n\n```ts fixture=with-calculator\ncalculator.add(10);\n```\n\n## Code Samples\n\n```ts\n/* This is totes a magic comment in here, just you wait! */\nconst foo = 'bar';\n```\n" }, "repository": { "directory": "packages/jsii-calc", @@ -11128,5 +11128,5 @@ } }, "version": "0.20.5", - "fingerprint": "g9C1lL8c+vgxBjOWVBFMMPlcwkF3Z81xxTAGfc73x9o=" + "fingerprint": "/MRTbTnRC1UWxsPIrca+9Yo1IBKsEueT75P22pQoV1o=" } diff --git a/packages/jsii-pacmak/bin/jsii-pacmak.ts b/packages/jsii-pacmak/bin/jsii-pacmak.ts index 028af791e8..97817f2d5d 100644 --- a/packages/jsii-pacmak/bin/jsii-pacmak.ts +++ b/packages/jsii-pacmak/bin/jsii-pacmak.ts @@ -2,6 +2,7 @@ import path = require('path'); import process = require('process'); import yargs = require('yargs'); +import { Rosetta } from 'jsii-rosetta'; import logging = require('../lib/logging'); import { Timers } from '../lib/timer'; import { VERSION_DESC } from '../lib/version'; @@ -78,6 +79,15 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets'; desc: 'Auto-update .npmignore to exclude the output directory and include the .jsii file', default: true }) + .option('rosetta-tablet', { + type: 'string', + desc: 'Location of a jsii-rosetta tablet with sample translations (created using \'jsii-rosetta extract\')' + }) + .option('rosetta-translate-live', { + type: 'boolean', + desc: 'Translate code samples on-the-fly if they can\'t be found in the samples tablet', + default: true + }) .version(VERSION_DESC) .strict() .argv; @@ -89,6 +99,11 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets'; const timers = new Timers(); + const rosetta = new Rosetta({ liveConversion: argv['rosetta-translate-live'] }); + if (argv['rosetta-tablet']) { + await rosetta.loadTabletFromFile(argv['rosetta-tablet']); + } + const modulesToPackage = await findJsiiModules(argv._, argv.recurse); logging.info(`Found ${modulesToPackage.length} modules to package`); if (modulesToPackage.length === 0) { @@ -114,9 +129,12 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets'; }); await timers.recordAsync('load jsii', () => { - logging.info('Loading jsii assemblies'); + logging.info('Loading jsii assemblies and translations'); return Promise.all(modulesToPackage - .map(m => m.load())); + .map(async m => { + await m.load(); + await rosetta.addAssembly(m.assembly.spec, m.moduleDirectory); + })); }); try { @@ -155,19 +173,20 @@ import { ALL_BUILDERS, TargetName } from '../lib/targets'; async function buildTargetsForLanguage(targetLanguage: string, modules: JsiiModule[], perLanguageDirectory: boolean) { // ``argv.target`` is guaranteed valid by ``yargs`` through the ``choices`` directive. - const builder = ALL_BUILDERS[targetLanguage as TargetName]; - if (!builder) { + const factory = ALL_BUILDERS[targetLanguage as TargetName]; + if (!factory) { throw new Error(`Unsupported target: '${targetLanguage}'`); } - await builder.buildModules(modules, { + await factory(modules, { clean: argv.clean, codeOnly: argv['code-only'], + rosetta, force: argv.force, fingerprint: argv.fingerprint, arguments: argv, languageSubdirectory: perLanguageDirectory, - }); + }).buildModules(); } })().catch(err => { process.stderr.write(`${err.stack}\n`); diff --git a/packages/jsii-pacmak/lib/builder.ts b/packages/jsii-pacmak/lib/builder.ts index 19d3c140f8..33d4e15357 100644 --- a/packages/jsii-pacmak/lib/builder.ts +++ b/packages/jsii-pacmak/lib/builder.ts @@ -3,6 +3,7 @@ import logging = require('./logging'); import { JsiiModule } from './packaging'; import { TargetConstructor, Target } from './target'; import { Scratch } from './util'; +import { Rosetta } from 'jsii-rosetta'; export interface BuildOptions { /** @@ -36,6 +37,11 @@ export interface BuildOptions { * Whether to add an additional subdirectory for the target language */ languageSubdirectory?: boolean; + + /** + * The Rosetta instance to load examples from + */ + rosetta: Rosetta; } /** @@ -44,32 +50,27 @@ export interface BuildOptions { * Building can happen one target at a time, or multiple targets at a time. */ export interface TargetBuilder { - buildModules(modules: JsiiModule[], options: BuildOptions): Promise; -} - -/** - * Return the output directory if all modules have the same directory - */ -export function allOutputDirectoriesTheSame(modules: JsiiModule[]): boolean { - if (modules.length === 0) { return true; } - const ret = modules[0].outputDirectory; - return modules.every(m => m.outputDirectory === ret); + buildModules(): Promise; } /** * Builds the targets for the given language sequentially */ export class OneByOneBuilder implements TargetBuilder { - public constructor(private readonly targetName: string, private readonly targetConstructor: TargetConstructor) { + public constructor( + private readonly targetName: string, + private readonly targetConstructor: TargetConstructor, + private readonly modules: JsiiModule[], + private readonly options: BuildOptions) { } - public async buildModules(modules: JsiiModule[], options: BuildOptions): Promise { - for (const module of modules) { - if (options.codeOnly) { - await this.generateModuleCode(module, options); + public async buildModules(): Promise { + for (const module of this.modules) { + if (this.options.codeOnly) { + await this.generateModuleCode(module, this.options); } else { - await this.buildModule(module, options); + await this.buildModule(module, this.options); } } } @@ -110,7 +111,8 @@ export class OneByOneBuilder implements TargetBuilder { assembly: module.assembly, fingerprint: options.fingerprint, force: options.force, - arguments: options.arguments + arguments: options.arguments, + rosetta: options.rosetta, }); } diff --git a/packages/jsii-pacmak/lib/target.ts b/packages/jsii-pacmak/lib/target.ts index 43100b2010..6c62e39799 100644 --- a/packages/jsii-pacmak/lib/target.ts +++ b/packages/jsii-pacmak/lib/target.ts @@ -6,6 +6,7 @@ import path = require('path'); import { IGenerator } from './generator'; import logging = require('./logging'); import { resolveDependencyDirectory } from './util'; +import { Rosetta } from 'jsii-rosetta'; export abstract class Target { @@ -15,12 +16,14 @@ export abstract class Target { protected readonly arguments: { [name: string]: any }; protected readonly targetName: string; protected readonly assembly: reflect.Assembly; + protected readonly rosetta: Rosetta; protected abstract readonly generator: IGenerator; public constructor(options: TargetOptions) { this.packageDir = options.packageDir; this.assembly = options.assembly; + this.rosetta = options.rosetta; this.fingerprint = options.fingerprint != null ? options.fingerprint : true; this.force = options.force != null ? options.force : false; this.arguments = options.arguments; @@ -69,35 +72,45 @@ export abstract class Target { * @param packageDir The directory of the package to resolve from. */ protected async findLocalDepsOutput(rootPackageDir: string) { - const results = new Set(); - - async function recurse(this: Target, packageDir: string, isRoot: boolean) { - const pkg = await fs.readJson(path.join(packageDir, 'package.json')); - - // no jsii or jsii.outdir - either a misconfigured jsii package or a non-jsii dependency. either way, we are done here. - if (!pkg.jsii || !pkg.jsii.outdir) { - return; - } - - // if an output directory exists for this module, then we add it to our - // list of results (unless it's the root package, which we are currently building) - const outdir = path.join(packageDir, pkg.jsii.outdir, this.targetName); - if (results.has(outdir)) { return; } // Already visited, don't recurse again - - if (!isRoot && await fs.pathExists(outdir)) { - logging.debug(`Found ${outdir} as a local dependency output`); - results.add(outdir); - } - - // now descend to dependencies - await Promise.all(Object.keys(pkg.dependencies || {}).map(dependencyName => { - const dependencyDir = resolveDependencyDirectory(packageDir, dependencyName); - return recurse.call(this, dependencyDir, false); - })); + return findLocalBuildDirs(rootPackageDir, this.targetName); + } +} + +/** + * Traverses the dep graph and returns a list of pacmak output directories + * available locally for this specific target. This allows target builds to + * take local dependencies in case a dependency is checked-out. + * + * @param packageDir The directory of the package to resolve from. + */ +export async function findLocalBuildDirs(rootPackageDir: string, targetName: string) { + const results = new Set(); + await recurse(rootPackageDir, true); + return Array.from(results); + + async function recurse(packageDir: string, isRoot: boolean) { + const pkg = await fs.readJson(path.join(packageDir, 'package.json')); + + // no jsii or jsii.outdir - either a misconfigured jsii package or a non-jsii dependency. either way, we are done here. + if (!pkg.jsii || !pkg.jsii.outdir) { + return; } - await recurse.call(this, rootPackageDir, true); - return Array.from(results); + // if an output directory exists for this module, then we add it to our + // list of results (unless it's the root package, which we are currently building) + const outdir = path.join(packageDir, pkg.jsii.outdir, targetName); + if (results.has(outdir)) { return; } // Already visited, don't recurse again + + if (!isRoot && await fs.pathExists(outdir)) { + logging.debug(`Found ${outdir} as a local dependency output`); + results.add(outdir); + } + + // now descend to dependencies + await Promise.all(Object.keys(pkg.dependencies || {}).map(dependencyName => { + const dependencyDir = resolveDependencyDirectory(packageDir, dependencyName); + return recurse(dependencyDir, false); + })); } } @@ -173,6 +186,9 @@ export interface TargetOptions { /** The JSII-reflect assembly for this JSII assembly */ assembly: reflect.Assembly; + /** The Rosetta instance */ + rosetta: Rosetta; + /** * Whether to fingerprint the produced artifacts. * @default true diff --git a/packages/jsii-pacmak/lib/targets/dotnet.ts b/packages/jsii-pacmak/lib/targets/dotnet.ts index 176e157c1a..1e8b401f5f 100644 --- a/packages/jsii-pacmak/lib/targets/dotnet.ts +++ b/packages/jsii-pacmak/lib/targets/dotnet.ts @@ -1,85 +1,133 @@ import * as fs from 'fs-extra'; import * as spec from 'jsii-spec'; import * as path from 'path'; -import * as xmlbuilder from 'xmlbuilder'; import * as logging from '../logging'; -import { PackageInfo, Target } from '../target'; -import { shell } from '../util'; +import xmlbuilder = require('xmlbuilder'); +import { PackageInfo, Target, TargetOptions, findLocalBuildDirs } from '../target'; +import { shell, Scratch, setExtend, filterAsync } from '../util'; import { DotNetGenerator } from './dotnet/dotnetgenerator'; +import { TargetBuilder, BuildOptions } from '../builder'; +import { JsiiModule } from '../packaging'; -export default class Dotnet extends Target { - public static toPackageInfos(assm: spec.Assembly): { [language: string]: PackageInfo } { - const packageId = assm.targets!.dotnet!.packageId; - const version = assm.version; - const packageInfo: PackageInfo = { - repository: 'Nuget', - url: `https://www.nuget.org/packages/${packageId}/${version}`, - usage: { - 'csproj': { - language: 'xml', - code: `` - }, - 'dotnet': { - language: 'console', - code: `dotnet add package ${packageId} --version ${version}` - }, - 'packages.config': { - language: 'xml', - code: `` - } +/** + * Build .NET packages all together, by generating an aggregate solution file + */ +export class DotnetBuilder implements TargetBuilder { + private readonly targetName = 'dotnet'; + + public constructor(private readonly modules: JsiiModule[], private readonly options: BuildOptions) { + } + + public async buildModules(): Promise { + if (this.modules.length === 0) { return; } + + if (this.options.codeOnly) { + // Simple, just generate code to respective output dirs + for (const module of this.modules) { + await this.generateModuleCode(module, this.outputDir(module.outputDirectory)); } - }; - return { 'C#': packageInfo }; + return; + } + + // Otherwise make a single tempdir to hold all sources, build them together and copy them back out + const scratchDirs: Array> = []; + try { + const tempSourceDir = await this.generateAggregateSourceDir(this.modules); + scratchDirs.push(tempSourceDir); + + // Build solution + logging.debug('Building .NET'); + await shell('dotnet', ['build', '-c', 'Release'], { cwd: tempSourceDir.directory }); + + await this.copyOutArtifacts(tempSourceDir.object); + if (this.options.clean) { + await Scratch.cleanupAll(scratchDirs); + } + } catch(e) { + logging.warn(`Exception occurred, not cleaning up ${scratchDirs.map(s => s.directory)}`); + throw e; + } } - public static toNativeReference(_type: spec.Type, options: any) { - return { - 'c#': `using ${options.namespace};` - }; + private async generateAggregateSourceDir(modules: JsiiModule[]): Promise> { + return Scratch.make(async (tmpDir: string) => { + logging.debug(`Generating aggregate .NET source dir at ${tmpDir}`); + + const csProjs = []; + const ret: TemporaryDotnetPackage[] = []; + + for (const module of modules) { + // Code generator will make its own subdirectory + await this.generateModuleCode(module, tmpDir); + const loc = projectLocation(module); + csProjs.push(loc.projectFile); + ret.push({ + outputTargetDirectory: module.outputDirectory, + artifactsDir: path.join(tmpDir, loc.projectDir, 'bin', 'Release') + }); + } + + // Use 'dotnet' command line tool to build a solution file from these csprojs + await shell('dotnet', ['new', 'sln', '-n', 'JsiiBuild'], { cwd: tmpDir }); + await shell('dotnet', ['sln', 'add', ...csProjs], { cwd: tmpDir }); + + await this.generateNuGetConfigForLocalDeps(tmpDir); + + return ret; + }); } - protected readonly generator = new DotNetGenerator(); + private async copyOutArtifacts(packages: TemporaryDotnetPackage[]) { + logging.debug('Copying out .NET artifacts'); + for (const pkg of packages) { + const targetDirectory = this.outputDir(pkg.outputTargetDirectory); - public async build(sourceDir: string, outDir: string): Promise { - await this.generateNuGetConfigForLocalDeps(sourceDir, outDir); - const pkg = await fs.readJson(path.join(this.packageDir, 'package.json')); - const packageId: string = pkg.jsii.targets.dotnet.packageId; - const project: string = path.join(packageId, `${packageId}.csproj`); + await fs.mkdirp(targetDirectory); + await fs.copy(pkg.artifactsDir, targetDirectory, { recursive: true }); - await shell( - 'dotnet', - ['build', project, '-c', 'Release'], - { cwd: sourceDir } - ); + // This copies more than we need, remove the directory with the bare assembly again + await fs.remove(path.join(targetDirectory, 'netcoreapp3.0')); + } + } - await this.copyFiles( - path.join(sourceDir, packageId, 'bin', 'Release'), - outDir); - await fs.remove(path.join(outDir, 'netcoreapp3.0')); + private async generateModuleCode(module: JsiiModule, where: string): Promise { + const target = this.makeTarget(module); + logging.debug(`Generating ${this.targetName} code into ${where}`); + await target.generateCode(where, module.tarball); } - private async generateNuGetConfigForLocalDeps(sourceDirectory: string, currentOutputDirectory: string): Promise { + /** + * Decide whether or not to append 'dotnet' to the given output directory + */ + private outputDir(declaredDir: string) { + return this.options.languageSubdirectory ? path.join(declaredDir, this.targetName) : declaredDir; + } + + /** + * Write a NuGet.config that will include build directories for local packages not in the current build + * + */ + private async generateNuGetConfigForLocalDeps(where: string): Promise { // Traverse the dependency graph of this module and find all modules that have // an /dotnet directory. We will add those as local NuGet repositories. // This enables building against local modules. - const localRepos = await this.findLocalDepsOutput(this.packageDir); + const allDepsOutputDirs = new Set(); + for (const module of this.modules) { + setExtend(allDepsOutputDirs, await findLocalBuildDirs(module.moduleDirectory, this.targetName)); - // Add the current output directory as a local repo for the case where we build multiple packages - // into the same output. NuGet throws an error if a source directory doesn't exist, so we check - // before adding it to the list. - if (await fs.pathExists(currentOutputDirectory)) { - localRepos.push(path.resolve(process.cwd(), currentOutputDirectory)); + // Also include output directory where we're building to, in case we build multiple packages into + // the same output directory. + allDepsOutputDirs.add(this.outputDir(module.outputDirectory)); } + const localRepos = Array.from(allDepsOutputDirs); + // If dotnet-jsonmodel is checked-out and we can find a local repository, add it to the list. try { /* eslint-disable @typescript-eslint/no-var-requires */ const jsiiDotNetJsonModel = require('jsii-dotnet-jsonmodel'); /* eslint-enable @typescript-eslint/no-var-requires */ - const localDotNetJsonModel = jsiiDotNetJsonModel.repository; - if (await fs.pathExists(localDotNetJsonModel)) { - localRepos.push(localDotNetJsonModel); - } + localRepos.push(jsiiDotNetJsonModel.repository); } catch { // Couldn't locate jsii-dotnet-jsonmodel, which is owkay! } @@ -89,15 +137,15 @@ export default class Dotnet extends Target { /* eslint-disable @typescript-eslint/no-var-requires */ const jsiiDotNetRuntime = require('jsii-dotnet-runtime'); /* eslint-enable @typescript-eslint/no-var-requires */ - const localDotNetRuntime = jsiiDotNetRuntime.repository; - if (await fs.pathExists(localDotNetRuntime)) { - localRepos.push(localDotNetRuntime); - } + localRepos.push(jsiiDotNetRuntime.repository); } catch { // Couldn't locate jsii-dotnet-runtime, which is owkay! } - logging.debug('local NuGet repos:', localRepos); + // Filter out nonexistant directories, .NET will be unhappy if paths don't exist + const existingLocalRepos = await filterAsync(localRepos, fs.pathExists); + + logging.debug('local NuGet repos:', existingLocalRepos); // Construct XML content. const configuration = xmlbuilder.create('configuration', { encoding: 'UTF-8' }); @@ -108,7 +156,7 @@ export default class Dotnet extends Target { nugetOrgAdd.att('value', 'https://api.nuget.org/v3/index.json'); nugetOrgAdd.att('protocolVersion', '3'); - localRepos.forEach((repo, index) => { + existingLocalRepos.forEach((repo, index) => { const add = packageSources.ele('add'); add.att('key', `local-${index}`); add.att('value', path.join(repo)); @@ -117,8 +165,86 @@ export default class Dotnet extends Target { const xml = configuration.end({ pretty: true }); // Write XML content to NuGet.config. - const filePath = path.join(sourceDirectory, 'NuGet.config'); + const filePath = path.join(where, 'NuGet.config'); logging.debug(`Generated ${filePath}`); await fs.writeFile(filePath, xml); } + + private makeTarget(module: JsiiModule): Dotnet { + return new Dotnet({ + targetName: this.targetName, + packageDir: module.moduleDirectory, + assembly: module.assembly, + fingerprint: this.options.fingerprint, + force: this.options.force, + arguments: this.options.arguments, + rosetta: this.options.rosetta, + }, this.modules.map(m => m.name)); + } +} + +interface TemporaryDotnetPackage { + /** + * Where the artifacts will be stored after build (relative to build dir) + */ + artifactsDir: string; + + /** + * Where the artifacts ought to go for this particular module + */ + outputTargetDirectory: string; +} + +function projectLocation(module: JsiiModule) { + const packageId: string = module.assembly.targets!.dotnet!.packageId; + return { + projectDir: packageId, + projectFile: path.join(packageId, `${packageId}.csproj`) + }; } + +export default class Dotnet extends Target { + public static toPackageInfos(assm: spec.Assembly): { [language: string]: PackageInfo } { + const packageId = assm.targets!.dotnet!.packageId; + const version = assm.version; + const packageInfo: PackageInfo = { + repository: 'Nuget', + url: `https://www.nuget.org/packages/${packageId}/${version}`, + usage: { + 'csproj': { + language: 'xml', + code: `` + }, + 'dotnet': { + language: 'console', + code: `dotnet add package ${packageId} --version ${version}` + }, + 'packages.config': { + language: 'xml', + code: `` + } + } + }; + return { 'C#': packageInfo }; + } + + public static toNativeReference(_type: spec.Type, options: any) { + return { + 'c#': `using ${options.namespace};` + }; + } + + protected readonly generator: DotNetGenerator; + + public constructor(options: TargetOptions, assembliesCurrentlyBeingCompiled: string[]) { + super(options); + + this.generator = new DotNetGenerator(assembliesCurrentlyBeingCompiled); + } + + /* eslint-disable @typescript-eslint/require-await */ + public async build(_sourceDir: string, _outDir: string): Promise { + throw new Error('Should not be called; use builder instead'); + } + /* eslint-enable @typescript-eslint/require-await */ +} \ No newline at end of file diff --git a/packages/jsii-pacmak/lib/targets/dotnet/dotnetgenerator.ts b/packages/jsii-pacmak/lib/targets/dotnet/dotnetgenerator.ts index 1811ba1df8..d556e50293 100644 --- a/packages/jsii-pacmak/lib/targets/dotnet/dotnetgenerator.ts +++ b/packages/jsii-pacmak/lib/targets/dotnet/dotnetgenerator.ts @@ -28,7 +28,7 @@ export class DotNetGenerator extends Generator { private dotnetDocGenerator!: DotNetDocGenerator; - public constructor() { + public constructor(private readonly assembliesCurrentlyBeingCompiled: string[]) { super(); // Override the openBlock to get a correct C# looking code block with the curly brace after the line @@ -50,6 +50,7 @@ export class DotNetGenerator extends Generator { this.typeresolver = new DotNetTypeResolver(this.assembly, (fqn: string) => this.findModule(fqn), (fqn: string) => this.findType(fqn), + this.assembliesCurrentlyBeingCompiled ); this.dotnetRuntimeGenerator = new DotNetRuntimeGenerator(this.code, this.typeresolver); diff --git a/packages/jsii-pacmak/lib/targets/dotnet/dotnettyperesolver.ts b/packages/jsii-pacmak/lib/targets/dotnet/dotnettyperesolver.ts index d8e9c1702b..20e33536b3 100644 --- a/packages/jsii-pacmak/lib/targets/dotnet/dotnettyperesolver.ts +++ b/packages/jsii-pacmak/lib/targets/dotnet/dotnettyperesolver.ts @@ -17,7 +17,9 @@ export class DotNetTypeResolver { public constructor(assembly: spec.Assembly, findModule: FindModuleCallback, - findType: FindTypeCallback) { + findType: FindTypeCallback, + private readonly assembliesCurrentlyBeingCompiled: string[] + ) { this.assembly = assembly; this.findModule = findModule; this.findType = findType; @@ -58,10 +60,10 @@ export class DotNetTypeResolver { return `${actualNamespace}.${typeName}`; } return `${dotnetNamespace}.${type.namespace}.${typeName}`; - } + } // When undefined, the type is located at the root of the assembly return `${dotnetNamespace}.${typeName}`; - + } @@ -82,7 +84,12 @@ export class DotNetTypeResolver { // suffix is guaranteed to start with a leading `-` version = `${depInfo.version}${suffix}`; } - this.namespaceDependencies.set(depName, new DotNetDependency(namespace, packageId, depName, version)); + this.namespaceDependencies.set(depName, new DotNetDependency( + namespace, + packageId, + depName, + version, + this.assembliesCurrentlyBeingCompiled.includes(depName))); } } } @@ -115,9 +122,9 @@ export class DotNetTypeResolver { return this.toNativeFqn(typeref.fqn); } else if (typeref.union) { return 'object'; - } + } throw new Error(`Invalid type reference: ${JSON.stringify(typeref)}`); - + } /** diff --git a/packages/jsii-pacmak/lib/targets/dotnet/filegenerator.ts b/packages/jsii-pacmak/lib/targets/dotnet/filegenerator.ts index e9c96281b5..4e2147a613 100644 --- a/packages/jsii-pacmak/lib/targets/dotnet/filegenerator.ts +++ b/packages/jsii-pacmak/lib/targets/dotnet/filegenerator.ts @@ -8,16 +8,12 @@ import { nextMajorVersion } from '../../util'; // Represents a dependency in the dependency tree. export class DotNetDependency { - public namespace: string; - public packageId: string; - public fqn: string; - public version: string; - - public constructor(namespace: string, packageId: string, fqn: string, version: string) { - this.namespace = namespace; - this.packageId = packageId; - this.fqn = fqn; - this.version = version; + public constructor( + public readonly namespace: string, + public readonly packageId: string, + public readonly fqn: string, + public readonly version: string, + public readonly partOfCompilation: boolean) { } } @@ -106,9 +102,14 @@ export class FileGenerator { packageReference.att('Version', `[${jsiiVersion},${jsiiVersionNextMajor})`); dependencies.forEach((value: DotNetDependency) => { - const dependencyReference = itemGroup2.ele('PackageReference'); - dependencyReference.att('Include', value.packageId); - dependencyReference.att('Version', value.version); + if (value.partOfCompilation) { + const dependencyReference = itemGroup2.ele('ProjectReference'); + dependencyReference.att('Include', `../${value.packageId}/${value.packageId}.csproj`); + } else { + const dependencyReference = itemGroup2.ele('PackageReference'); + dependencyReference.att('Include', value.packageId); + dependencyReference.att('Version', value.version); + } }); const xml = rootNode.end({ pretty: true, spaceBeforeSlash: true }); diff --git a/packages/jsii-pacmak/lib/targets/index.ts b/packages/jsii-pacmak/lib/targets/index.ts index 263a9d405d..94ca1c82bc 100644 --- a/packages/jsii-pacmak/lib/targets/index.ts +++ b/packages/jsii-pacmak/lib/targets/index.ts @@ -1,19 +1,22 @@ -import { OneByOneBuilder, TargetBuilder } from '../builder'; +import { OneByOneBuilder, TargetBuilder, BuildOptions } from '../builder'; -import Dotnet from './dotnet'; +import { DotnetBuilder } from './dotnet'; import { JavaBuilder } from './java'; import JavaScript from './js'; import Python from './python'; import Ruby from './ruby'; +import { JsiiModule } from '../packaging'; export type TargetName = 'dotnet' | 'java' | 'js' | 'python' | 'ruby'; +export type BuilderFactory = (modules: JsiiModule[], options: BuildOptions) => TargetBuilder; -export const ALL_BUILDERS: {[key in TargetName]: TargetBuilder} = { - dotnet: new OneByOneBuilder('dotnet', Dotnet), - java: new JavaBuilder(), - js: new OneByOneBuilder('js', JavaScript), - python: new OneByOneBuilder('python', Python), - ruby: new OneByOneBuilder('ruby', Ruby), + +export const ALL_BUILDERS: {[key in TargetName]: BuilderFactory} = { + dotnet: (ms, o) => new DotnetBuilder(ms, o), + java: (ms, o) => new JavaBuilder(ms, o), + js: (ms, o) => new OneByOneBuilder('js', JavaScript, ms, o), + python: (ms, o) => new OneByOneBuilder('python', Python, ms, o), + ruby: (ms, o) => new OneByOneBuilder('ruby', Ruby, ms, o), }; diff --git a/packages/jsii-pacmak/lib/targets/java.ts b/packages/jsii-pacmak/lib/targets/java.ts index 62df1231a4..5d92a6de63 100644 --- a/packages/jsii-pacmak/lib/targets/java.ts +++ b/packages/jsii-pacmak/lib/targets/java.ts @@ -8,10 +8,10 @@ import xmlbuilder = require('xmlbuilder'); import { Generator } from '../generator'; import logging = require('../logging'); import { md2html } from '../markdown'; -import { PackageInfo, Target } from '../target'; -import { shell, Scratch } from '../util'; +import { PackageInfo, Target, findLocalBuildDirs } from '../target'; +import { shell, Scratch, slugify, setExtend } from '../util'; import { VERSION, VERSION_DESC } from '../version'; -import { TargetBuilder, BuildOptions, allOutputDirectoriesTheSame, OneByOneBuilder } from '../builder'; +import { TargetBuilder, BuildOptions } from '../builder'; import { JsiiModule } from '../packaging'; /* eslint-disable @typescript-eslint/no-var-requires */ @@ -21,59 +21,87 @@ const spdxLicenseList = require('spdx-license-list'); const BUILDER_CLASS_NAME = 'Builder'; /** - * Build Java packages in parallel, by generating an aggregate POM + * Build Java packages all together, by generating an aggregate POM + * + * This will make the Java build a lot more efficient (~300%). + * + * Do this by copying the code into a temporary directory, generating an aggregate + * POM there, and then copying the artifacts back into the respective output + * directories. */ export class JavaBuilder implements TargetBuilder { private readonly targetName = 'java'; - public async buildModules(modules: JsiiModule[], options: BuildOptions): Promise { - if (modules.length === 0) { return; } + public constructor(private readonly modules: JsiiModule[], private readonly options: BuildOptions) { + } + + public async buildModules(): Promise { + if (this.modules.length === 0) { return; } - // We can only do the optimized build if '-o' was specified, which we will notice - // as all module outputdirectories being the same. (Maybe we can build to per-package - // dist dirs as well, but this is the smallest delta from what we had and we will - // always specify the output dir anyway). - if (!allOutputDirectoriesTheSame(modules)) { - logging.warn('Single output directory not specified, doing (slower) one-by-one build for Java'); - await new OneByOneBuilder(this.targetName, Java).buildModules(modules, options); + if (this.options.codeOnly) { + // Simple, just generate code to respective output dirs + for (const module of this.modules) { + await this.generateModuleCode(module, this.options, this.outputDir(module.outputDirectory)); + } return; } - const singleOutputDir = this.finalOutputDir(modules[0], options); + // Otherwise make a single tempdir to hold all sources, build them together and copy them back out + const scratchDirs: Array> = []; + try { + const tempSourceDir = await this.generateAggregateSourceDir(this.modules, this.options); + scratchDirs.push(tempSourceDir); - const moduleDirectories = []; - - for (const module of modules) { - moduleDirectories.push(await this.generateModuleCode(module, options, options.codeOnly)); - } + // Need any old module object to make a target to be able to invoke build, though none of its settings + // will be used. + const target = this.makeTarget(this.modules[0], this.options); + const tempOutputDir = await Scratch.make(async dir => { + logging.debug(`Building Java code to ${dir}`); + await target.build(tempSourceDir.directory, dir); + }); + scratchDirs.push(tempOutputDir); - if (!options.codeOnly && modules.length > 0) { - // Need a module to get a target - const pomDirectory = modules.length > 1 - ? await this.generateAggregatePom(moduleDirectories) - : moduleDirectories[0].directory; - const target = this.makeTarget(modules[0], options); + await this.copyOutArtifacts(tempOutputDir.directory, tempSourceDir.object); - await target.build(pomDirectory, singleOutputDir); + if (this.options.clean) { + await Scratch.cleanupAll(scratchDirs); + } + } catch(e) { + logging.warn(`Exception occurred, not cleaning up ${scratchDirs.map(s => s.directory)}`); + throw e; } } - private async generateModuleCode(module: JsiiModule, options: BuildOptions, finalDirectory?: boolean): Promise> { + private async generateModuleCode(module: JsiiModule, options: BuildOptions, where: string): Promise { const target = this.makeTarget(module, options); + logging.debug(`Generating Java code into ${where}`); + await target.generateCode(where, module.tarball); + } + + private async generateAggregateSourceDir(modules: JsiiModule[], options: BuildOptions): Promise> { + return Scratch.make(async (tmpDir: string) => { + logging.debug(`Generating aggregate Java source dir at ${tmpDir}`); + const ret: TemporaryJavaPackage[] = []; + for (const module of modules) { + const relativeName = slugify(module.name); + const sourceDir = path.join(tmpDir, relativeName); + await this.generateModuleCode(module, options, sourceDir); + + ret.push({ + relativeSourceDir: relativeName, + relativeArtifactsDir: moduleArtifactsSubdir(module), + outputTargetDirectory: module.outputDirectory + }); + } - const srcDir = finalDirectory - ? Scratch.fake(this.finalOutputDir(module, options), undefined) - : await Scratch.make(_ => undefined); - - logging.debug(`Generating ${this.targetName} code into ${srcDir.directory}`); - await target.generateCode(srcDir.directory, module.tarball); + await this.generateAggregatePom(tmpDir, ret.map(m => m.relativeSourceDir)); + await this.generateMavenSettingsForLocalDeps(tmpDir); - return srcDir; + return ret; + }); } - private async generateAggregatePom(sourceDirectories: Array>) { - const parentDir = this.findSharedParentDirectory(sourceDirectories.map(s => s.directory)); - + private async generateAggregatePom(where: string, moduleNames: string[]) { const aggregatePom = xmlbuilder.create({ project: { '@xmlns': 'http://maven.apache.org/POM/4.0.0', @@ -91,33 +119,104 @@ export class JavaBuilder implements TargetBuilder { 'version': '1.0.0', 'modules': { - module: sourceDirectories.map(s => path.relative(parentDir, s.directory)) + module: moduleNames, } } }, { encoding: 'UTF-8' }).end({ pretty: true }); - logging.debug(`Generated ${parentDir}/pom.xml`); - await fs.writeFile(path.join(parentDir, 'pom.xml'), aggregatePom); - return parentDir; + logging.debug(`Generated ${where}/pom.xml`); + await fs.writeFile(path.join(where, 'pom.xml'), aggregatePom); + } + + private async copyOutArtifacts(artifactsRoot: string, packages: TemporaryJavaPackage[]) { + logging.debug('Copying out Java artifacts'); + // The artifacts directory looks like this: + // /tmp/XXX/software/amazon/awscdk/something/v1.2.3 + // /else/v1.2.3 + // /entirely/v1.2.3 + // + // We get the 'software/amazon/awscdk/something' path from the package, identifying + // the files we need to copy, including Maven metadata. But we need to recreate + // the whole path in the target directory. + + for (const pkg of packages) { + const artifactsSource = path.join(artifactsRoot, pkg.relativeArtifactsDir); + const artifactsDest = path.join(this.outputDir(pkg.outputTargetDirectory), pkg.relativeArtifactsDir); + + await fs.mkdirp(artifactsDest); + await fs.copy(artifactsSource, artifactsDest, { recursive: true }); + } + } + + /** + * Decide whether or not to append 'java' to the given output directory + */ + private outputDir(declaredDir: string) { + return this.options.languageSubdirectory ? path.join(declaredDir, this.targetName) : declaredDir; } /** - * Find the longest shared given a set of directories + * Generates maven settings file for this build. + * @param where The generated sources directory. This is where user.xml will be placed. + * @param currentOutputDirectory The current output directory. Will be added as a local maven repo. */ - private findSharedParentDirectory(dirs: string[]) { - if (dirs.length === 0) { return ''; } - const dirParts = dirs.map(dir => dir.split(path.sep)); + private async generateMavenSettingsForLocalDeps(where: string) { + const filePath = path.join(where, 'user.xml'); - return dirParts.reduce(longestPrefix).join(path.sep); + // traverse the dep graph of this module and find all modules that have + // an /java directory. we will add those as local maven + // repositories which will resolve instead of Maven Central for those + // module. this enables building against local modules (i.e. in lerna + // repositories or linked modules). + const allDepsOutputDirs = new Set(); + for (const module of this.modules) { + setExtend(allDepsOutputDirs, await findLocalBuildDirs(module.moduleDirectory, this.targetName)); - function longestPrefix(accumulator: string[], current: string[]) { - const len = Math.min(accumulator.length, current.length); - let i = 0; - while (i < len && accumulator[i] === current[i]) { - i++; - } - return accumulator.slice(0, i); + // Also include output directory where we're building to, in case we build multiple packages into + // the same output directory. + allDepsOutputDirs.add(path.join(module.outputDirectory, this.options.languageSubdirectory ? this.targetName : '')); } + + const localRepos = Array.from(allDepsOutputDirs); + + // if java-runtime is checked-out and we can find a local repository, + // add it to the list. + const localJavaRuntime = await findJavaRuntimeLocalRepository(); + if (localJavaRuntime) { + localRepos.push(localJavaRuntime); + } + + logging.debug('local maven repos:', localRepos); + + const profileName = 'local-jsii-modules'; + const settings = xmlbuilder.create({ + settings: { + '@xmlns': 'http://maven.apache.org/POM/4.0.0', + '@xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', + '@xsi:schemaLocation': 'http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd', + '#comment': [ + `Generated by jsii-pacmak@${VERSION_DESC} on ${new Date().toISOString()}`, + ], + 'profiles': { + profile: { + id: profileName, + repositories: { + repository: localRepos.map((repo, index) => ({ + id: `local${index}`, + url: `file://${repo}` + })) + } + } + }, + 'activeProfiles': { + activeProfile: profileName + } + } + }, { encoding: 'UTF-8' }).end({ pretty: true }); + + logging.debug(`Generated ${filePath}`); + await fs.writeFile(filePath, settings); + return filePath; } private makeTarget(module: JsiiModule, options: BuildOptions): Target { @@ -127,16 +226,36 @@ export class JavaBuilder implements TargetBuilder { assembly: module.assembly, fingerprint: options.fingerprint, force: options.force, - arguments: options.arguments + arguments: options.arguments, + rosetta: options.rosetta, }); } +} - private finalOutputDir(module: JsiiModule, options: BuildOptions): string { - if (options.languageSubdirectory) { - return path.join(module.outputDirectory, this.targetName); - } - return module.outputDirectory; - } +interface TemporaryJavaPackage { + /** + * Where the sources are (relative to the source root) + */ + relativeSourceDir: string; + + /** + * Where the artifacts will be stored after build (relative to build dir) + */ + relativeArtifactsDir: string; + + /** + * Where the artifacts ought to go for this particular module + */ + outputTargetDirectory: string; +} + +/** + * Return the subdirectory of the output directory where the artifacts for this particular package are produced + */ +function moduleArtifactsSubdir(module: JsiiModule) { + const groupId = module.assembly.targets!.java!.maven.groupId; + const artifactId = module.assembly.targets!.java!.maven.artifactId; + return `${groupId.replace(/\./g, '/')}/${artifactId}`; } export default class Java extends Target { @@ -185,10 +304,9 @@ export default class Java extends Target { mvnArguments.push(this.arguments[arg].toString()); } - const userXml = await this.generateMavenSettingsForLocalDeps(sourceDir, outDir); await shell( 'mvn', - [...mvnArguments, 'deploy', `-D=altDeploymentRepository=local::default::${url}`, `--settings=${userXml}`], + [...mvnArguments, 'deploy', `-D=altDeploymentRepository=local::default::${url}`, '--settings=user.xml'], { cwd: sourceDir, env: { @@ -199,66 +317,6 @@ export default class Java extends Target { } ); } - - /** - * Generates maven settings file for this build. - * @param sourceDir The generated sources directory. This is where user.xml will be placed. - * @param currentOutputDirectory The current output directory. Will be added as a local maven repo. - */ - private async generateMavenSettingsForLocalDeps(sourceDir: string, currentOutputDirectory: string) { - const filePath = path.join(sourceDir, 'user.xml'); - - // traverse the dep graph of this module and find all modules that have - // an /java directory. we will add those as local maven - // repositories which will resolve instead of Maven Central for those - // module. this enables building against local modules (i.e. in lerna - // repositories or linked modules). - const localRepos = await this.findLocalDepsOutput(this.packageDir); - - // add the current output directory as a local repo as well for the case - // where we build multiple packages into the same output. - localRepos.push(currentOutputDirectory); - - // if java-runtime is checked-out and we can find a local repository, - // add it to the list. - const localJavaRuntime = await findJavaRuntimeLocalRepository(); - if (localJavaRuntime) { - localRepos.push(localJavaRuntime); - } - - logging.debug('local maven repos:', localRepos); - - const profileName = 'local-jsii-modules'; - const settings = xmlbuilder.create({ - settings: { - '@xmlns': 'http://maven.apache.org/POM/4.0.0', - '@xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', - '@xsi:schemaLocation': 'http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd', - '#comment': [ - `Generated by jsii-pacmak@${VERSION_DESC} on ${new Date().toISOString()}`, - ], - 'profiles': { - profile: { - id: profileName, - repositories: { - repository: localRepos.map((repo, index) => ({ - id: `local${index}`, - url: `file://${repo}` - })) - } - } - }, - 'activeProfiles': { - activeProfile: profileName - } - } - }, { encoding: 'UTF-8' }).end({ pretty: true }); - - logging.debug(`Generated ${filePath}`); - await fs.writeFile(filePath, settings); - return filePath; - } - } // ################## diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 075e8c8fd7..27f170ff27 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -3,21 +3,30 @@ import path = require('path'); import { CodeMaker, toSnakeCase } from 'codemaker'; import * as escapeStringRegexp from 'escape-string-regexp'; import * as reflect from 'jsii-reflect'; -import * as sampiler from 'jsii-sampiler'; import * as spec from 'jsii-spec'; import { Stability } from 'jsii-spec'; import { Generator, GeneratorOptions } from '../generator'; import { warn } from '../logging'; import { md2rst } from '../markdown'; -import { Target } from '../target'; +import { Target, TargetOptions } from '../target'; import { shell } from '../util'; +import { Translation, Rosetta, typeScriptSnippetFromSource } from 'jsii-rosetta'; + + +const INCOMPLETE_DISCLAIMER = '# Example automatically generated. See https://github.com/aws/jsii/issues/826'; /* eslint-disable @typescript-eslint/no-var-requires */ const spdxLicenseList = require('spdx-license-list'); /* eslint-enable @typescript-eslint/no-var-requires */ export default class Python extends Target { - protected readonly generator = new PythonGenerator(); + protected readonly generator: PythonGenerator; + + public constructor(options: TargetOptions) { + super(options); + + this.generator = new PythonGenerator(options.rosetta); + } public async build(sourceDir: string, outDir: string): Promise { // Format our code to make it easier to read, we do this here instead of trying @@ -51,6 +60,7 @@ export default class Python extends Target { } } } + } // ################## @@ -241,7 +251,7 @@ abstract class BasePythonClassType implements PythonType, ISortableType { const bases = classParams.length > 0 ? `(${classParams.join(', ')})` : ''; code.openBlock(`class ${this.pythonName}${bases}`); - emitDocString(code, this.docs, { documentableItem: `class-${this.pythonName}` }); + this.generator.emitDocString(code, this.docs, { documentableItem: `class-${this.pythonName}` }); this.emitPreamble(code, resolver); @@ -398,7 +408,7 @@ abstract class BaseMethod implements PythonBase { } code.openBlock(`def ${this.pythonName}(${pythonParams.join(', ')}) -> ${returnType}`); - emitDocString(code, this.docs, { arguments: documentableArgs, documentableItem: `method-${this.pythonName}` }); + this.generator.emitDocString(code, this.docs, { arguments: documentableArgs, documentableItem: `method-${this.pythonName}` }); this.emitBody(code, resolver, renderAbstract, forceEmitBody); code.closeBlock(); } @@ -422,7 +432,7 @@ abstract class BaseMethod implements PythonBase { // We need to build up a list of properties, which are mandatory, these are the // ones we will specifiy to start with in our dictionary literal. - const liftedProps = this.getLiftedProperties(resolver).map(p => new StructField(p)); + const liftedProps = this.getLiftedProperties(resolver).map(p => new StructField(this.generator, p)); const assignments = liftedProps .map(p => p.pythonName) .map(v => `${v}=${v}`); @@ -502,7 +512,9 @@ abstract class BaseProperty implements PythonBase { private readonly immutable: boolean; - public constructor(public readonly pythonName: string, + public constructor( + private readonly generator: PythonGenerator, + public readonly pythonName: string, private readonly jsName: string, private readonly type: spec.OptionalValue, private readonly docs: spec.Docs | undefined, @@ -526,7 +538,7 @@ abstract class BaseProperty implements PythonBase { code.line('@abc.abstractmethod'); } code.openBlock(`def ${this.pythonName}(${this.implicitParameter}) -> ${pythonType}`); - emitDocString(code, this.docs, { documentableItem: `prop-${this.pythonName}` }); + this.generator.emitDocString(code, this.docs, { documentableItem: `prop-${this.pythonName}` }); if ((this.shouldEmitBody || forceEmitBody) && (!renderAbstract || !this.abstract)) { code.line(`return jsii.${this.jsiiGetMethod}(${this.implicitParameter}, "${this.jsName}")`); } else { @@ -562,7 +574,7 @@ class Interface extends BasePythonClassType { resolver = this.fqn ? resolver.bind(this.fqn) : resolver; const proxyBases: string[] = this.bases.map(b => `jsii.proxy_for(${resolver.resolve({ type: b })})`); code.openBlock(`class ${this.getProxyClassName()}(${proxyBases.join(', ')})`); - emitDocString(code, this.docs, { documentableItem: `class-${this.pythonName}` }); + this.generator.emitDocString(code, this.docs, { documentableItem: `class-${this.pythonName}` }); code.line(`__jsii_type__ = "${this.fqn}"`); if (this.members.length > 0) { @@ -647,7 +659,7 @@ class Struct extends BasePythonClassType { * Find all fields (inherited as well) */ private get allMembers(): StructField[] { - return this.thisInterface.allProperties.map(x => new StructField(x.spec)); + return this.thisInterface.allProperties.map(x => new StructField(this.generator, x.spec)); } private get thisInterface() { @@ -692,7 +704,7 @@ class Struct extends BasePythonClassType { name: m.pythonName, docs: m.docs, })); - emitDocString(code, this.docs, { arguments: args, documentableItem: `class-${this.pythonName}` }); + this.generator.emitDocString(code, this.docs, { arguments: args, documentableItem: `class-${this.pythonName}` }); } private emitGetter(member: StructField, code: CodeMaker, resolver: TypeResolver) { @@ -732,7 +744,7 @@ class StructField implements PythonBase { public readonly docs?: spec.Docs; public readonly type: spec.OptionalValue; - public constructor(public readonly prop: spec.Property) { + public constructor(private readonly generator: PythonGenerator, public readonly prop: spec.Property) { this.pythonName = toPythonPropertyName(prop.name); this.jsiiName = prop.name; this.type = prop; @@ -763,7 +775,7 @@ class StructField implements PythonBase { } public emitDocString(code: CodeMaker) { - emitDocString(code, this.docs, { documentableItem: `prop-${this.pythonName}` }); + this.generator.emitDocString(code, this.docs, { documentableItem: `prop-${this.pythonName}` }); } public emit(code: CodeMaker, resolver: TypeResolver) { @@ -941,14 +953,18 @@ class Enum extends BasePythonClassType { } class EnumMember implements PythonBase { - public constructor(public readonly pythonName: string, private readonly value: string, private readonly docs: spec.Docs | undefined) { + public constructor( + private readonly generator: PythonGenerator, + public readonly pythonName: string, + private readonly value: string, + private readonly docs: spec.Docs | undefined) { this.pythonName = pythonName; this.value = value; } public emit(code: CodeMaker, _resolver: TypeResolver) { code.line(`${this.pythonName} = "${this.value}"`); - emitDocString(code, this.docs, { documentableItem: `enum-${this.pythonName}` }); + this.generator.emitDocString(code, this.docs, { documentableItem: `enum-${this.pythonName}` }); } } @@ -1088,7 +1104,7 @@ class Package { private readonly modules: Map; private readonly data: Map; - public constructor(name: string, version: string, metadata: spec.Assembly) { + public constructor(private readonly generator: PythonGenerator, name: string, version: string, metadata: spec.Assembly) { this.name = name; this.version = version; this.metadata = metadata; @@ -1112,7 +1128,7 @@ class Package { public write(code: CodeMaker, resolver: TypeResolver) { if (this.metadata.readme) { // Conversion is expensive, so cache the result in a variable (we need it twice) - this.convertedReadme = convertSnippetsInMarkdown(this.metadata.readme.markdown, 'README.md').trim(); + this.convertedReadme = this.generator.convertMarkdown(this.metadata.readme.markdown).trim(); } const modules = [...this.modules.values()].sort((a, b) => a.pythonName.localeCompare(b.pythonName)); @@ -1474,7 +1490,7 @@ class PythonGenerator extends Generator { private package!: Package; private readonly types: Map; - public constructor(options: GeneratorOptions = {}) { + public constructor(private readonly rosetta: Rosetta, options: GeneratorOptions = {}) { super(options); this.code.openBlockFormatter = s => `${s}:`; @@ -1483,6 +1499,118 @@ class PythonGenerator extends Generator { this.types = new Map(); } + public emitDocString(code: CodeMaker, docs: spec.Docs | undefined, options: { + arguments?: DocumentableArgument[]; + documentableItem?: string; + } = {}) { + if ((!docs || Object.keys(docs).length === 0) && !options.arguments) { return; } + if (!docs) { docs = {}; } + + const lines = new Array(); + + if (docs.summary) { + lines.push(md2rst(docs.summary)); + brk(); + } else { + lines.push(''); + } + + function brk() { + if (lines.length > 0 && lines[lines.length - 1].trim() !== '') { lines.push(''); } + } + + function block(heading: string, content: string, doBrk = true) { + if (doBrk) { brk(); } + lines.push(heading); + const contentLines = md2rst(content).split('\n'); + if (contentLines.length <= 1) { + lines.push(`:${heading}: ${contentLines.join('')}`); + } else { + lines.push(`:${heading}:`); + brk(); + for (const line of contentLines) { + lines.push(`${line}`); + } + } + if (doBrk) { brk(); } + } + + if (docs.remarks) { + brk(); + lines.push(...md2rst(this.convertMarkdown(docs.remarks || '')).split('\n')); + brk(); + } + + if (options.arguments && options.arguments.length > 0) { + brk(); + for (const param of options.arguments) { + // Add a line for every argument. Even if there is no description, we need + // the docstring so that the Sphinx extension can add the type annotations. + lines.push(`:param ${toPythonParameterName(param.name)}: ${onelineDescription(param.docs)}`); + } + brk(); + } + + if (docs.default) { block('default', docs.default); } + if (docs.returns) { block('return', docs.returns); } + if (docs.deprecated) { block('deprecated', docs.deprecated); } + if (docs.see) { block('see', docs.see, false); } + if (docs.stability && shouldMentionStability(docs.stability)) { block('stability', docs.stability, false); } + if (docs.subclassable) { block('subclassable', 'Yes'); } + + for (const [k, v] of Object.entries(docs.custom || {})) { + block(`${k}:`, v, false); + } + + if (docs.example) { + brk(); + lines.push('Example::'); + const exampleText = this.convertExample(docs.example); + + for (const line of exampleText.split('\n')) { + lines.push(` ${line}`); + } + brk(); + } + + while (lines.length > 0 && lines[lines.length - 1] === '') { lines.pop(); } + + if (lines.length === 0) { return; } + + if (lines.length === 1) { + code.line(`"""${lines[0]}"""`); + return; + } + + code.line(`"""${lines[0]}`); + lines.splice(0, 1); + + for (const line of lines) { + code.line(line); + } + + code.line('"""'); + } + + public convertExample(example: string): string { + const snippet = typeScriptSnippetFromSource(example, 'example'); + const translated = this.rosetta.translateSnippet(snippet, 'python'); + if (!translated) { return example; } + return this.prefixDisclaimer(translated); + } + + public convertMarkdown(markdown: string): string { + return this.rosetta.translateSnippetsInMarkdown(markdown, 'python', trans => ({ + language: trans.language, + source: this.prefixDisclaimer(trans) + })); + } + + private prefixDisclaimer(translated: Translation) { + if (translated.didCompile) { return translated.source; } + return `${INCOMPLETE_DISCLAIMER}\n${translated.source}`; + } + public getPythonType(fqn: string): PythonType { const type = this.types.get(fqn); @@ -1499,6 +1627,7 @@ class PythonGenerator extends Generator { protected onBeginAssembly(assm: spec.Assembly, _fingerprint: boolean) { this.package = new Package( + this, assm.targets!.python!.distName, assm.version, assm, @@ -1619,6 +1748,7 @@ class PythonGenerator extends Generator { protected onStaticProperty(cls: spec.ClassType, prop: spec.Property) { this.getPythonType(cls.fqn).addMember( new StaticProperty( + this, toPythonPropertyName(prop.name, prop.const), prop.name, prop, @@ -1661,6 +1791,7 @@ class PythonGenerator extends Generator { protected onProperty(cls: spec.ClassType, prop: spec.Property) { this.getPythonType(cls.fqn).addMember( new Property( + this, toPythonPropertyName(prop.name, prop.const, prop.protected), prop.name, prop, @@ -1720,9 +1851,10 @@ class PythonGenerator extends Generator { let ifaceProperty: InterfaceProperty | StructField; if (ifc.datatype) { - ifaceProperty = new StructField(prop); + ifaceProperty = new StructField(this, prop); } else { ifaceProperty = new InterfaceProperty( + this, toPythonPropertyName(prop.name, prop.const, prop.protected), prop.name, prop, @@ -1741,6 +1873,7 @@ class PythonGenerator extends Generator { protected onEnumMember(enm: spec.EnumType, member: spec.EnumMember) { this.getPythonType(enm.fqn).addMember( new EnumMember( + this, toPythonIdentifier(member.name), member.name, member.docs, @@ -1831,99 +1964,6 @@ interface DocumentableArgument { docs?: spec.Docs; } -function emitDocString(code: CodeMaker, docs: spec.Docs | undefined, options: { - arguments?: DocumentableArgument[]; - documentableItem?: string; -} = {}) { - if ((!docs || Object.keys(docs).length === 0) && !options.arguments) { return; } - if (!docs) { docs = {}; } - - const lines = new Array(); - - if (docs.summary) { - lines.push(md2rst(docs.summary)); - brk(); - } else { - lines.push(''); - } - - function brk() { - if (lines.length > 0 && lines[lines.length - 1].trim() !== '') { lines.push(''); } - } - - function block(heading: string, content: string, doBrk = true) { - if (doBrk) { brk(); } - lines.push(heading); - const contentLines = md2rst(content).split('\n'); - if (contentLines.length <= 1) { - lines.push(`:${heading}: ${contentLines.join('')}`); - } else { - lines.push(`:${heading}:`); - brk(); - for (const line of contentLines) { - lines.push(`${line}`); - } - } - if (doBrk) { brk(); } - } - - if (docs.remarks) { - brk(); - lines.push(...md2rst(convertSnippetsInMarkdown(docs.remarks || '', options.documentableItem || 'docstring')).split('\n')); - brk(); - } - - if (options.arguments && options.arguments.length > 0) { - brk(); - for (const param of options.arguments) { - // Add a line for every argument. Even if there is no description, we need - // the docstring so that the Sphinx extension can add the type annotations. - lines.push(`:param ${toPythonParameterName(param.name)}: ${onelineDescription(param.docs)}`); - } - brk(); - } - - if (docs.default) { block('default', docs.default); } - if (docs.returns) { block('return', docs.returns); } - if (docs.deprecated) { block('deprecated', docs.deprecated); } - if (docs.see) { block('see', docs.see, false); } - if (docs.stability && shouldMentionStability(docs.stability)) { block('stability', docs.stability, false); } - if (docs.subclassable) { block('subclassable', 'Yes'); } - - for (const [k, v] of Object.entries(docs.custom || {})) { - block(`${k}:`, v, false); - } - - if (docs.example) { - brk(); - lines.push('Example::'); - const exampleText = convertExample(docs.example, options.documentableItem || 'example'); - - for (const line of exampleText.split('\n')) { - lines.push(` ${line}`); - } - brk(); - } - - while (lines.length > 0 && lines[lines.length - 1] === '') { lines.pop(); } - - if (lines.length === 0) { return; } - - if (lines.length === 1) { - code.line(`"""${lines[0]}"""`); - return; - } - - code.line(`"""${lines[0]}`); - lines.splice(0, 1); - - for (const line of lines) { - code.line(line); - } - - code.line('"""'); -} - /** * Render a one-line description of the given docs, used for method arguments and inlined properties */ @@ -1946,25 +1986,4 @@ function isStruct(typeSystem: reflect.TypeSystem, ref: spec.TypeReference): bool if (!spec.isNamedTypeReference(ref)) { return false; } const type = typeSystem.tryFindFqn(ref.fqn); return type !== undefined && type.isInterfaceType() && type.isDataType(); -} - -const pythonTranslator = new sampiler.PythonVisitor({ - disclaimer: 'Example may have issues. See https://github.com/aws/jsii/issues/826' -}); - -function convertExample(example: string, filename: string): string { - const source = new sampiler.LiteralSource(example, filename); - const result = sampiler.translateTypeScript(source, pythonTranslator); - sampiler.printDiagnostics(result.diagnostics, process.stderr); - return sampiler.renderTree(result.tree); -} - -function convertSnippetsInMarkdown(markdown: string, filename: string): string { - const source = new sampiler.LiteralSource(markdown, filename); - const result = sampiler.translateMarkdown(source, pythonTranslator, { - languageIdentifier: 'python' - }); - // FIXME: This should translate into an exit code somehow - sampiler.printDiagnostics(result.diagnostics, process.stderr); - return sampiler.renderTree(result.tree); -} +} \ No newline at end of file diff --git a/packages/jsii-pacmak/lib/util.ts b/packages/jsii-pacmak/lib/util.ts index 0dbdafe0a9..33c78ea52b 100644 --- a/packages/jsii-pacmak/lib/util.ts +++ b/packages/jsii-pacmak/lib/util.ts @@ -96,6 +96,13 @@ export async function loadAssembly(modulePath: string): Promise { return spec.validateAssembly(await fs.readJson(assmPath)); } +/** + * Strip filesystem unsafe characters from a string + */ +export function slugify(x: string) { + return x.replace(/[^a-zA-Z0-9_-]/g, '_'); +} + /** * Class that makes a temporary directory and holds on to an operation object */ @@ -147,3 +154,20 @@ export function nextMajorVersion(version: string): string { } return v.inc('patch').version; } + + +export function setExtend(xs: Set, els: Iterable) { + for (const el of els) { + xs.add(el); + } +} + +export async function filterAsync(xs: A[], pred: (x: A) => Promise): Promise { + const ret = new Array(); + for (const x of xs) { + if (await pred(x)) { + ret.push(x); + } + } + return ret; +} \ No newline at end of file diff --git a/packages/jsii-pacmak/package.json b/packages/jsii-pacmak/package.json index cc6f581fe4..1963921d02 100644 --- a/packages/jsii-pacmak/package.json +++ b/packages/jsii-pacmak/package.json @@ -40,7 +40,7 @@ "escape-string-regexp": "^2.0.0", "fs-extra": "^8.1.0", "jsii-reflect": "^0.20.5", - "jsii-sampiler": "^0.20.5", + "jsii-rosetta": "^0.20.5", "jsii-spec": "^0.20.5", "spdx-license-list": "^6.1.0", "xmlbuilder": "^13.0.2", diff --git a/packages/jsii-pacmak/test/build-test.sh b/packages/jsii-pacmak/test/build-test.sh index 2eab08894d..8585c49838 100755 --- a/packages/jsii-pacmak/test/build-test.sh +++ b/packages/jsii-pacmak/test/build-test.sh @@ -2,7 +2,36 @@ set -euo pipefail cd $(dirname $0) -outdir=$(mktemp -d) -../bin/jsii-pacmak -o ${outdir} --recurse ../../jsii-calc -vv +# Test various build modes for jsii-pacmak (these are all ways in +# which users can decide to run jsii-pacmak) +# +# The following list of packages must be toposorted, like a monorepo +# manager would order the individual builds. +packagedirs="\ + ../../jsii-calc-base-of-base \ + ../../jsii-calc-base \ + ../../jsii-calc-lib \ + ../../jsii-calc \ + " +clean_dists() { + for dir in $packagedirs; do rm -rf $dir/dist; done +} +# Single target, recursive build to a certain location +outdir=$(mktemp -d) +clean_dists +echo "Testing SINGLE TARGET, RECURSIVE build." +../bin/jsii-pacmak -o ${outdir} --recurse ../../jsii-calc rm -rf ${outdir} + +# Multiple targets, build one-by-one into own directory +clean_dists +echo "Testing ONE-BY-ONE build." +for dir in $packagedirs; do + ../bin/jsii-pacmak $dir -vv +done + +# Multiple targets, build all at once into own directory +clean_dists +echo "Testing ALL-AT-ONCE build." +../bin/jsii-pacmak $packagedirs diff --git a/packages/jsii-pacmak/test/expected.jsii-calc/dotnet/Amazon.JSII.Tests.CalculatorPackageId/.jsii b/packages/jsii-pacmak/test/expected.jsii-calc/dotnet/Amazon.JSII.Tests.CalculatorPackageId/.jsii index f5f5942eda..7d85c69166 100644 --- a/packages/jsii-pacmak/test/expected.jsii-calc/dotnet/Amazon.JSII.Tests.CalculatorPackageId/.jsii +++ b/packages/jsii-pacmak/test/expected.jsii-calc/dotnet/Amazon.JSII.Tests.CalculatorPackageId/.jsii @@ -195,7 +195,7 @@ }, "name": "jsii-calc", "readme": { - "markdown": "# jsii Calculator\n\nThis library is used to demonstrate and test the features of JSII\n\n## Sphinx\n\nThis file will be incorporated into the sphinx documentation.\n\nIf this file starts with an \"H1\" line (in our case `# jsii Calculator`), this\nheading will be used as the Sphinx topic name. Otherwise, the name of the module\n(`jsii-calc`) will be used instead.\n\n## Code Samples\n\n```ts\n/* This is totes a magic comment in here, just you wait! */\nconst foo = 'bar';\n```\n" + "markdown": "# jsii Calculator\n\nThis library is used to demonstrate and test the features of JSII\n\n## How to use running sum API:\n\nFirst, create a calculator:\n\n```ts\nconst calculator = new calc.Calculator();\n```\n\nThen call some operations:\n\n\n```ts fixture=with-calculator\ncalculator.add(10);\n```\n\n## Code Samples\n\n```ts\n/* This is totes a magic comment in here, just you wait! */\nconst foo = 'bar';\n```\n" }, "repository": { "directory": "packages/jsii-calc", @@ -11128,5 +11128,5 @@ } }, "version": "0.20.5", - "fingerprint": "g9C1lL8c+vgxBjOWVBFMMPlcwkF3Z81xxTAGfc73x9o=" + "fingerprint": "/MRTbTnRC1UWxsPIrca+9Yo1IBKsEueT75P22pQoV1o=" } diff --git a/packages/jsii-pacmak/test/expected.jsii-calc/java/src/main/java/software/amazon/jsii/tests/calculator/package-info.java b/packages/jsii-pacmak/test/expected.jsii-calc/java/src/main/java/software/amazon/jsii/tests/calculator/package-info.java index 0fd07a89d1..be50e425e6 100644 --- a/packages/jsii-pacmak/test/expected.jsii-calc/java/src/main/java/software/amazon/jsii/tests/calculator/package-info.java +++ b/packages/jsii-pacmak/test/expected.jsii-calc/java/src/main/java/software/amazon/jsii/tests/calculator/package-info.java @@ -1,11 +1,13 @@ /** *

jsii Calculator

*

This library is used to demonstrate and test the features of JSII

- *

Sphinx

- *

This file will be incorporated into the sphinx documentation.

- *

If this file starts with an "H1" line (in our case # jsii Calculator), this - * heading will be used as the Sphinx topic name. Otherwise, the name of the module - * (jsii-calc) will be used instead.

+ *

How to use running sum API:

+ *

First, create a calculator:

+ *
const calculator = new calc.Calculator();
+ * 
+ *

Then call some operations:

+ *
calculator.add(10);
+ * 
*

Code Samples

*
/* This is totes a magic comment in here, just you wait! *{@literal /}
  * const foo = 'bar';
diff --git a/packages/jsii-pacmak/test/expected.jsii-calc/python/README.md b/packages/jsii-pacmak/test/expected.jsii-calc/python/README.md
index e90486664b..1622efcd58 100644
--- a/packages/jsii-pacmak/test/expected.jsii-calc/python/README.md
+++ b/packages/jsii-pacmak/test/expected.jsii-calc/python/README.md
@@ -2,18 +2,23 @@
 
 This library is used to demonstrate and test the features of JSII
 
-## Sphinx
+## How to use running sum API:
 
-This file will be incorporated into the sphinx documentation.
+First, create a calculator:
 
-If this file starts with an "H1" line (in our case `# jsii Calculator`), this
-heading will be used as the Sphinx topic name. Otherwise, the name of the module
-(`jsii-calc`) will be used instead.
+```python
+calculator = calc.Calculator()
+```
+
+Then call some operations:
+
+```python
+calculator.add(10)
+```
 
 ## Code Samples
 
 ```python
-# Example may have issues. See https://github.com/aws/jsii/issues/826
 # This is totes a magic comment in here, just you wait!
 foo = "bar"
 ```
diff --git a/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/__init__.py b/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/__init__.py
index f599cf01c2..4bbb0b68b8 100644
--- a/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/__init__.py
+++ b/packages/jsii-pacmak/test/expected.jsii-calc/python/src/jsii_calc/__init__.py
@@ -3,18 +3,23 @@
 
 This library is used to demonstrate and test the features of JSII
 
-## Sphinx
+## How to use running sum API:
 
-This file will be incorporated into the sphinx documentation.
+First, create a calculator:
 
-If this file starts with an "H1" line (in our case `# jsii Calculator`), this
-heading will be used as the Sphinx topic name. Otherwise, the name of the module
-(`jsii-calc`) will be used instead.
+```python
+calculator = calc.Calculator()
+```
+
+Then call some operations:
+
+```python
+calculator.add(10)
+```
 
 ## Code Samples
 
 ```python
-# Example may have issues. See https://github.com/aws/jsii/issues/826
 # This is totes a magic comment in here, just you wait!
 foo = "bar"
 ```
@@ -793,7 +798,6 @@ class ClassWithDocs(metaclass=jsii.JSIIMeta, jsii_type="jsii-calc.ClassWithDocs"
     :customAttribute:: hasAValue
 
     Example::
-        # Example may have issues. See https://github.com/aws/jsii/issues/826
         def an_example():
             pass
     """
diff --git a/packages/jsii-pacmak/tsconfig.json b/packages/jsii-pacmak/tsconfig.json
index 9c9053dbe9..75e3b8ad29 100644
--- a/packages/jsii-pacmak/tsconfig.json
+++ b/packages/jsii-pacmak/tsconfig.json
@@ -22,7 +22,7 @@
   "references": [
     { "path": "../jsii-spec" },
     { "path": "../codemaker" },
-    { "path": "../jsii-sampiler" },
+    { "path": "../jsii-rosetta" },
     { "path": "../jsii-reflect" }
   ]
 }
diff --git a/packages/jsii-python-runtime/bin/generate-calc b/packages/jsii-python-runtime/bin/generate-calc
index 990e02fcbb..f4605e14ef 100755
--- a/packages/jsii-python-runtime/bin/generate-calc
+++ b/packages/jsii-python-runtime/bin/generate-calc
@@ -3,6 +3,11 @@ import os
 import subprocess
 import sys
 
+# Clean out this directory, as it otherwise may
+# accumuluate multiple versions of the same library
+# and pip will complain.
+subprocess.run(['rm', '-rf', '.env/jsii-calc'], check=True)
+
 subprocess.run(
     [
         "jsii-pacmak",
diff --git a/packages/jsii-python-runtime/package.json b/packages/jsii-python-runtime/package.json
index 2b24f4061a..44ea856f28 100644
--- a/packages/jsii-python-runtime/package.json
+++ b/packages/jsii-python-runtime/package.json
@@ -24,7 +24,7 @@
   "scripts": {
     "generate": "python3 bin/generate",
     "deps": "python3 -m venv .env && .env/bin/pip install pip==19.0.1 setuptools==40.7.0 wheel==0.32.3 && .env/bin/pip install -r requirements.txt",
-    "build": "cp ../../README.md . && npm run generate && npm run deps && .env/bin/python setup.py sdist -d . bdist_wheel -d . && rm -rf build",
+    "build": "cp ../../README.md . && rm -f jsii-*.whl && npm run generate && npm run deps && .env/bin/python setup.py sdist -d . bdist_wheel -d . && rm -rf build",
     "package": "package-python",
     "test": ".env/bin/python bin/generate-calc && .env/bin/py.test -v --mypy",
     "test:update": "UPDATE_DIFF=1 .env/bin/python bin/generate-calc && .env/bin/py.test -v --mypy"
diff --git a/packages/jsii-sampiler/.gitignore b/packages/jsii-rosetta/.gitignore
similarity index 100%
rename from packages/jsii-sampiler/.gitignore
rename to packages/jsii-rosetta/.gitignore
diff --git a/packages/jsii-sampiler/.npmignore b/packages/jsii-rosetta/.npmignore
similarity index 100%
rename from packages/jsii-sampiler/.npmignore
rename to packages/jsii-rosetta/.npmignore
diff --git a/packages/jsii-rosetta/README.md b/packages/jsii-rosetta/README.md
new file mode 100644
index 0000000000..d6e541102a
--- /dev/null
+++ b/packages/jsii-rosetta/README.md
@@ -0,0 +1,193 @@
+# jsii-rosetta: a transpiler for code samples
+
+Utility to transcribe example code snippets from TypeScript to other
+jsii languages.
+
+Has knowledge about jsii language translation conventions to do the
+translations. Only supports a limited set of TypeScript language features.
+
+## Compilability
+
+The translator can translate both code that completely compiles and typechecks,
+as well as code that doesn't.
+
+In case of non-compiling samples the translations will be based off of
+grammatical parsing only. This has the downside that we do not have the type
+information available to the exact right thing in all instances.
+
+If the samples don't compile or don't have full type information:
+
+- No way to declare typed variables for Java and C#.
+- Can only "see" the fields of structs as far as they are declared in the same
+  snippet. Inherited fields or structs declared not in the same snippet are
+  invisible.
+- When we explode a struct parameter into keyword parameters and we pass it on
+  to another callable, we can't know which keyword arguments the called function
+  actually takes so we just pass all of them (might be too many).
+- When structs contain nested structs, Python and other languages need to know
+  the types of these fields to generate the right calls.
+- Object literals are used both to represent structs as well as to represent
+  dictionaries, and without type information it's impossible to determine
+  which is which.
+
+## Hiding code from samples
+
+In order to make examples compile, boilerplate code may need to be added
+that detracts from the example at hand (such as variable declarations
+and imports).
+
+This package supports hiding parts of the original source after
+translation.
+
+To mark special locations in the source tree, we can use one of three mechanisms:
+
+* Use a `void` expression statement to mark statement locations in the AST.
+* Use the `comma` operator combined with a `void` expression to mark expression
+  locations in the AST.
+* Use special directive comments (`/// !hide`, `/// !show`) to mark locations
+  that span AST nodes. This is less reliable (because the source location of
+  translated syntax sometimes will have to be estimated) but the only option if
+  you want to mark non-contiguous nodes (such as hide part of a class
+  declaration but show statements inside the constructor).
+
+The `void` expression keyword and or the `comma` operator feature are
+little-used JavaScript features that are reliably parsed by TypeScript and do
+not affect the semantics of the application in which they appear (so the program
+executes the same with or without them).
+
+A handy mnemonic for this feature is that you can use it to "send your
+code into the void".
+
+### Hiding statements
+
+Statement hiding looks like this:
+
+```ts
+before();    // will be shown
+
+void 0;      // start hiding (the argument to 'void' doesn't matter)
+middle();    // will not be shown
+void 'show'; // stop hiding
+
+after();     // will be shown again
+```
+
+### Hiding expressions
+
+For hiding expressions, we use `comma` expressions to attach a `void`
+statement to an expression value without changing the meaning of the
+code.
+
+Example:
+
+```ts
+foo(1, 2, (void 1, 3));
+```
+
+Will render as
+
+```
+foo(1, 2)
+```
+
+Also supports a visible ellipsis:
+
+```ts
+const x = (void '...', 3);
+```
+
+Renders to:
+
+```
+x = ...
+```
+
+### Hiding across AST nodes
+
+Use special comment directives:
+
+```ts
+before();
+/// !hide
+notShown();
+/// !show
+after();
+```
+
+(May also start with `/// !show` and `/// !hide`).
+
+## Fixtures
+
+To avoid having to repeat common setup every time, code samples can use
+"fixtures": a source template where the example is inserted. A fixture
+must contain the text `/// here` and may look like this:
+
+```ts
+const module = require('@some/dependency');
+
+class MyClass {
+    constructor() {
+        const obj = new MyObject();
+
+        /// here
+    }
+}
+```
+
+The example will be inserted at the location marked as `/// here` and
+will have access to `module`, `obj` and `this`.
+
+The default file loaded as a fixture is called `rosetta/default.ts-fixture`
+in the package directory (if it exists).
+
+Examples can request an alternative fixture by specifying a `fixture` parameter
+as part of the code block fence:
+
+    ` ` `ts fixture=some-fixture
+    ...
+
+Or opt out of using the default fixture by specifying `nofixture`.
+
+## Build integration
+
+Because we want to control the compilation environment when compiling examples,
+extracting and compiling all samples can be run as an external build step in a
+monorepo. This allows you to set up an environment with all desired packages and
+compile all samples in one go.
+
+The `jsii-rosetta extract` command will take one or more jsii assemblies,
+extract the snippets from them, will try to compile them with respect to a
+given home directory, and finally store all translations in something called
+a "tablet" (which is a lookup map from the original snippet to all of its
+translations).
+
+A translation tablet will automatically be used by `jsii-pacmak` if present, so
+it can subsitute the translated examples into the converted source code when
+writing out the converted code. When not given a tablet, `jsii-pacmak` can still
+live-convert samples, but you will not have the fine control over the
+compilation environment that you would have if you were to use the `extract`
+command.
+
+Works like this:
+
+```
+$ jsii-rosetta extract --compile $(find . -name .jsii) --directory some/dir
+$ jsii-pacmak --samples-tablet .jsii-samples.tbl
+```
+
+(The `extract` command is the default and may be omitted, but if you're passing
+assembliess as arguments you should terminate the option list by passing `--`).
+
+### Running in parallel
+
+Since TypeScript compilation takes a lot of time, much time can be gained
+by using the CPUs in your system effectively. `jsii-rosetta extract` will
+run the compilations in parallel if support for NodeJS Worker Threads is
+detected.
+
+Worker threads are enabled by default on NodeJS 12.x, and can be enabled on
+NodeJS 10.x by using a flag:
+
+```
+$ node --experimental-worker /path/to/jsii-rosetta extract ...
+```
diff --git a/packages/jsii-rosetta/bin/jsii-rosetta b/packages/jsii-rosetta/bin/jsii-rosetta
new file mode 100755
index 0000000000..1ddc3571aa
--- /dev/null
+++ b/packages/jsii-rosetta/bin/jsii-rosetta
@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+require('./jsii-rosetta.js');
diff --git a/packages/jsii-rosetta/bin/jsii-rosetta.ts b/packages/jsii-rosetta/bin/jsii-rosetta.ts
new file mode 100644
index 0000000000..852fd4b4b9
--- /dev/null
+++ b/packages/jsii-rosetta/bin/jsii-rosetta.ts
@@ -0,0 +1,152 @@
+import fs = require('fs-extra');
+import yargs = require('yargs');
+import { TranslateResult, DEFAULT_TABLET_NAME, translateTypeScript } from '../lib';
+import { PythonVisitor } from '../lib/languages/python';
+import { VisualizeAstVisitor } from '../lib/languages/visualize';
+import { extractSnippets } from '../lib/commands/extract';
+import logging = require('../lib/logging');
+import path = require('path');
+import { readTablet as readTablet } from '../lib/commands/read';
+import { translateMarkdown } from '../lib/commands/convert';
+import { File, printDiagnostics, isErrorDiagnostic } from '../lib/util';
+
+async function main() {
+  const argv = yargs
+    .usage('$0  [args]')
+    .option('verbose', {
+      alias: 'v',
+      type: 'boolean',
+      desc: 'Increase logging verbosity',
+      count: true,
+      default: 0
+    })
+    .command('snippet FILE', 'Translate a single snippet', command => command
+        .positional('file', { type: 'string', describe: 'The file to translate (leave out for stdin)' })
+        .option('python', { alias: 'p', boolean: true, description: 'Translate snippets to Python' })
+    , wrapHandler(async args => {
+      const result = translateTypeScript(
+        await makeFileSource(args.file || '-', 'stdin.ts'),
+        makeVisitor(args));
+      renderResult(result);
+    }))
+    .command('markdown FILE', 'Translate a MarkDown file', command => command
+        .positional('file', { type: 'string', describe: 'The file to translate (leave out for stdin)' })
+        .option('python', { alias: 'p', boolean: true, description: 'Translate snippets to Python' })
+    , wrapHandler(async args => {
+      const result = translateMarkdown(
+        await makeFileSource(args.file || '-', 'stdin.md'),
+        makeVisitor(args));
+      renderResult(result);
+    }))
+    .command(['extract [ASSEMBLY..]', '$0 [ASSEMBLY..]'], 'Extract code snippets from one or more assemblies into a language tablets', command => command
+      .positional('ASSEMBLY', { type: 'string', string: true, default: new Array(), describe: 'Assembly or directory to extract from' })
+      .option('output', { alias: 'o', type: 'string', describe: 'Output file where to store the sample tablets', default: DEFAULT_TABLET_NAME })
+      .option('compile', { alias: 'c', type: 'boolean', describe: 'Try compiling', default: false })
+      .option('directory', { alias: 'd', type: 'string', describe: 'Working directory (for require() etc)' })
+      .option('fail', { alias: 'f', type: 'boolean', describe: 'Fail if there are compilation errors', default: false })
+    , wrapHandler(async args => {
+
+      // Easiest way to get a fixed working directory (for sources) in is to
+      // chdir, since underneath the in-memory layer we're using a regular TS
+      // compilerhost. Have to make all file references absolute before we chdir
+      // though.
+      const absAssemblies = (args.ASSEMBLY.length > 0 ? args.ASSEMBLY : ['.']).map(x => path.resolve(x));
+      const absOutput = path.resolve(args.output);
+      if (args.directory) {
+        process.chdir(args.directory);
+      }
+
+      const result = await extractSnippets(absAssemblies, absOutput, args.compile);
+
+      printDiagnostics(result.diagnostics, process.stderr);
+
+      if (result.diagnostics.length > 0) {
+        logging.warn(`${result.diagnostics.length} diagnostics encountered in ${result.tablet.count} snippets`);
+      }
+
+      if (result.diagnostics.some(isErrorDiagnostic) && args.fail) {
+        process.exit(1);
+      }
+    }))
+    .command('read  [KEY] [LANGUAGE]', 'Display snippets in a language tablet file', command => command
+      .positional('TABLET', { type: 'string', required: true, describe: 'Language tablet to read' })
+      .positional('KEY', { type: 'string', describe: 'Snippet key to read' })
+      .positional('LANGUAGE', { type: 'string', describe: 'Language ID to read' })
+      .demandOption('TABLET')
+    , wrapHandler(async args => {
+      await readTablet(args.TABLET, args.KEY, args.LANGUAGE);
+    }))
+    .demandCommand()
+    .help()
+    .strict()  // Error on wrong command
+    .version(require('../package.json').version)
+    .showHelpOnFail(false)
+    .argv;
+
+  // Evaluating .argv triggers the parsing but the command gets implicitly executed,
+  // so we don't need the output.
+  Array.isArray(argv);
+}
+
+/**
+ * Wrap a command's handler with standard pre- and post-work
+ */
+function wrapHandler(handler: (x: A) => Promise) {
+  return (argv: A) => {
+    logging.level = argv.verbose !== undefined ? argv.verbose : 0;
+    return handler(argv);
+  };
+}
+
+function makeVisitor(args: { python?: boolean }) {
+  if (args.python) { return new PythonVisitor(); }
+  // Default to visualizing AST, including nodes we don't recognize yet
+  return new VisualizeAstVisitor();
+}
+
+async function makeFileSource(fileName: string, stdinName: string): Promise {
+  if (fileName === '-') {
+    return {
+      contents: await readStdin(),
+      fileName: stdinName
+    };
+  }
+  return {
+    contents: await fs.readFile(fileName, { encoding: 'utf-8' }),
+    fileName: fileName
+  };
+}
+
+async function readStdin(): Promise {
+  process.stdin.setEncoding('utf8');
+
+  const parts: string[] = [];
+
+  return new Promise((resolve, reject) => {
+    process.stdin.on('readable', () => {
+      const chunk = process.stdin.read();
+      if (chunk !== null) { parts.push(`${chunk}`); }
+    });
+
+    process.stdin.on('error', reject);
+    process.stdin.on('end', () => resolve(parts.join('')));
+  });
+}
+
+function renderResult(result: TranslateResult) {
+  process.stdout.write(result.translation + '\n');
+
+  if (result.diagnostics.length > 0) {
+    printDiagnostics(result.diagnostics, process.stderr);
+
+    if (result.diagnostics.some(isErrorDiagnostic)) {
+      process.exit(1);
+    }
+  }
+}
+
+main().catch(e => {
+  // tslint:disable-next-line:no-console
+  console.error(e);
+  process.exit(1);
+});
\ No newline at end of file
diff --git a/packages/jsii-sampiler/examples/controlflow.ts b/packages/jsii-rosetta/examples/controlflow.ts
similarity index 100%
rename from packages/jsii-sampiler/examples/controlflow.ts
rename to packages/jsii-rosetta/examples/controlflow.ts
diff --git a/packages/jsii-sampiler/examples/incomplete.ts b/packages/jsii-rosetta/examples/incomplete.ts
similarity index 100%
rename from packages/jsii-sampiler/examples/incomplete.ts
rename to packages/jsii-rosetta/examples/incomplete.ts
diff --git a/packages/jsii-rosetta/lib/commands/convert.ts b/packages/jsii-rosetta/lib/commands/convert.ts
new file mode 100644
index 0000000000..97b0773c87
--- /dev/null
+++ b/packages/jsii-rosetta/lib/commands/convert.ts
@@ -0,0 +1,36 @@
+import { AstHandler, AstRendererOptions } from '../renderer';
+import { TranslateResult, Translator } from '../translate';
+import { MarkdownRenderer } from '../markdown/markdown-renderer';
+import { transformMarkdown } from '../markdown/markdown';
+import { File } from '../util';
+import { ReplaceTypeScriptTransform } from '../markdown/replace-typescript-transform';
+
+export interface TranslateMarkdownOptions extends AstRendererOptions {
+  /**
+   * What language to put in the returned markdown blocks
+   */
+  languageIdentifier?: string;
+}
+
+
+export function translateMarkdown(markdown: File, visitor: AstHandler, options: TranslateMarkdownOptions = {}): TranslateResult {
+  const translator = new Translator(false);
+
+  const translatedMarkdown = transformMarkdown(
+      markdown.contents,
+      new MarkdownRenderer(),
+      new ReplaceTypeScriptTransform(markdown.fileName, tsSnippet => {
+        const translated = translator.translatorFor(tsSnippet).renderUsing(visitor);
+        return {
+          language: options.languageIdentifier || '',
+          source: translated,
+        };
+      })
+  );
+
+  return {
+    translation: translatedMarkdown,
+    diagnostics: translator.diagnostics,
+  };
+}
+
diff --git a/packages/jsii-rosetta/lib/commands/extract.ts b/packages/jsii-rosetta/lib/commands/extract.ts
new file mode 100644
index 0000000000..7bd2b4c052
--- /dev/null
+++ b/packages/jsii-rosetta/lib/commands/extract.ts
@@ -0,0 +1,128 @@
+import { loadAssemblies, allTypeScriptSnippets } from '../jsii/assemblies';
+import logging = require('../logging');
+import os = require('os');
+import path = require('path');
+import ts = require('typescript');
+import { LanguageTablet, TranslatedSnippet } from '../tablets/tablets';
+import { Translator } from '../translate';
+import { TypeScriptSnippet } from '../snippet';
+import { divideEvenly } from '../util';
+
+export interface ExtractResult {
+  diagnostics: ts.Diagnostic[];
+  tablet: LanguageTablet;
+}
+
+/**
+ * Extract all samples from the given assemblies into a tablet
+ */
+export async function extractSnippets(assemblyLocations: string[], outputFile: string, includeCompilerDiagnostics: boolean): Promise {
+  logging.info(`Loading ${assemblyLocations.length} assemblies`);
+  const assemblies = await loadAssemblies(assemblyLocations);
+
+  const snippets = allTypeScriptSnippets(assemblies);
+
+  const tablet = new LanguageTablet();
+
+  logging.info(`Translating`);
+  const startTime = Date.now();
+
+  const result = await translateAll(snippets, includeCompilerDiagnostics);
+
+  for (const snippet of result.translatedSnippets) {
+    tablet.addSnippet(snippet);
+  }
+
+  const delta =  (Date.now() - startTime) / 1000;
+  logging.info(`Converted ${tablet.count} snippets in ${delta} seconds (${(delta / tablet.count).toPrecision(3)}s/snippet)`);
+  logging.info(`Saving language tablet to ${outputFile}`);
+  await tablet.save(outputFile);
+
+  return { diagnostics: result.diagnostics, tablet };
+}
+
+interface TranslateAllResult {
+  translatedSnippets: TranslatedSnippet[];
+  diagnostics: ts.Diagnostic[];
+}
+
+/**
+ * Translate all snippets
+ *
+ * Uses a worker-based parallel translation if available, falling back to a single-threaded workflow if not.
+ */
+async function translateAll(snippets: IterableIterator, includeCompilerDiagnostics: boolean): Promise {
+  try {
+    const worker = await import('worker_threads');
+
+    return workerBasedTranslateAll(worker, snippets, includeCompilerDiagnostics);
+  } catch(e) {
+    if (e.code !== 'MODULE_NOT_FOUND') { throw e; }
+    logging.warn('Worker threads not available (use NodeJS >= 10.5 and --experimental-worker). Working sequentially.');
+
+    return singleThreadedTranslateAll(snippets, includeCompilerDiagnostics);
+  }
+}
+
+/**
+ * Translate the given snippets using a single compiler
+ *
+ * Used both here (directly) and via extract_worker to translate a batch of
+ * snippets in parallel.
+ */
+export function singleThreadedTranslateAll(snippets: IterableIterator, includeCompilerDiagnostics: boolean): TranslateAllResult {
+  const translatedSnippets = new Array();
+
+  const translator = new Translator(includeCompilerDiagnostics);
+  for (const block of snippets) {
+    translatedSnippets.push(translator.translate(block));
+  }
+
+  return { translatedSnippets, diagnostics: translator.diagnostics };
+}
+
+/**
+ * Divide the work evenly over all processors by running 'extract_worker' in Worker Threads, then combine results
+ *
+ * Never include 'extract_worker' directly, only do TypeScript type references (so that in
+ * the script we may assume that 'worker_threads' successfully imports).
+ */
+async function workerBasedTranslateAll(worker: typeof import('worker_threads'), snippets: IterableIterator, includeCompilerDiagnostics: boolean): Promise {
+  // Use about half the advertised cores because hyperthreading doesn't seem to help that
+  // much (on my machine, using more than half the cores actually makes it slower).
+  const N = Math.max(1, Math.ceil(os.cpus().length / 2));
+  const snippetArr = Array.from(snippets);
+  const groups = divideEvenly(N, snippetArr);
+  logging.info(`Translating ${snippetArr.length} snippets using ${groups.length} workers`);
+
+  // Run workers
+  const responses = await Promise.all(groups
+    .map(snippets => ({ snippets, includeCompilerDiagnostics }))
+    .map(runWorker));
+
+  // Combine results
+  const x = responses.reduce((acc, current) => {
+    // Modifying 'acc' in place to not incur useless copying
+    acc.translatedSnippetSchemas.push(...current.translatedSnippetSchemas);
+    acc.diagnostics.push(...current.diagnostics);
+    return acc;
+  }, { translatedSnippetSchemas: [], diagnostics: [] })
+  // Hydrate TranslatedSnippets from data back to objects
+  return { diagnostics: x.diagnostics, translatedSnippets: x.translatedSnippetSchemas.map(s => TranslatedSnippet.fromSchema(s)) };
+
+  /**
+   * Turn running the worker into a nice Promise.
+   */
+  function runWorker(request: import('./extract_worker').TranslateRequest): Promise {
+    return new Promise((resolve, reject) => {
+      const wrk = new worker.Worker(path.join(__dirname, 'extract_worker.js'), { workerData: request });
+      wrk.on('message', resolve);
+      wrk.on('error', reject);
+      wrk.on('exit', code => {
+        if (code !== 0) {
+          reject(new Error(`Worker exited with code ${code}`));
+        }
+      });
+    });
+  }
+}
\ No newline at end of file
diff --git a/packages/jsii-rosetta/lib/commands/extract_worker.ts b/packages/jsii-rosetta/lib/commands/extract_worker.ts
new file mode 100644
index 0000000000..c2194b0d58
--- /dev/null
+++ b/packages/jsii-rosetta/lib/commands/extract_worker.ts
@@ -0,0 +1,37 @@
+/**
+ * Pool worker for extract.ts
+ */
+import { TypeScriptSnippet } from '../snippet';
+import ts = require('typescript');
+import { singleThreadedTranslateAll } from './extract';
+import worker = require('worker_threads');
+import { TranslatedSnippetSchema } from '../tablets/schema';
+
+export interface TranslateRequest {
+  includeCompilerDiagnostics: boolean;
+  snippets: TypeScriptSnippet[];
+}
+
+export interface TranslateResponse {
+  diagnostics: ts.Diagnostic[];
+  // Cannot be 'TranslatedSnippet' because needs to be serializable
+  translatedSnippetSchemas: TranslatedSnippetSchema[];
+}
+
+function translateSnippet(request: TranslateRequest): TranslateResponse {
+  const result = singleThreadedTranslateAll(request.snippets[Symbol.iterator](), request.includeCompilerDiagnostics);
+
+  return { diagnostics: result.diagnostics, translatedSnippetSchemas: result.translatedSnippets.map(s => s.toSchema()) };
+}
+
+if (worker.isMainThread) {
+  // Throw an error to prevent accidental require() of this module. In principle not a big
+  // deal, but we want to be compatible with run modes where 'worker_threads' is not available
+  // and by doing this people on platforms where 'worker_threads' is available don't accidentally
+  // add a require().
+  throw new Error('This script should be run as a worker, not included directly.');
+}
+
+const request = worker.workerData;
+const response = translateSnippet(request);
+worker.parentPort!.postMessage(response);
\ No newline at end of file
diff --git a/packages/jsii-rosetta/lib/commands/read.ts b/packages/jsii-rosetta/lib/commands/read.ts
new file mode 100644
index 0000000000..c1592b529e
--- /dev/null
+++ b/packages/jsii-rosetta/lib/commands/read.ts
@@ -0,0 +1,72 @@
+import { LanguageTablet, TranslatedSnippet, Translation } from "../tablets/tablets";
+import { TargetLanguage } from "../languages";
+
+export async function readTablet(tabletFile: string, key?: string, lang?: string) {
+  const tab = new LanguageTablet();
+  await tab.load(tabletFile);
+
+  if (key !== undefined) {
+    const snippet = tab.tryGetSnippet(key);
+    if (snippet === undefined) {
+      throw new Error(`No such snippet: ${key}`);
+    }
+    displaySnippet(snippet);
+  } else {
+    listSnippets();
+  }
+
+  function listSnippets() {
+    for (const key of tab.snippetKeys) {
+      process.stdout.write(snippetHeader(key) + '\n');
+      displaySnippet(tab.tryGetSnippet(key)!);
+      process.stdout.write('\n');
+    }
+  }
+
+  function displaySnippet(snippet: TranslatedSnippet) {
+    if (snippet.didCompile !== undefined) {
+      process.stdout.write(`Compiled: ${snippet.didCompile}\n`);
+    }
+
+    if (lang !== undefined) {
+      const translation = snippet.get(lang as TargetLanguage);
+      if (translation === undefined) {
+        throw new Error(`No translation for ${lang} in snippet ${snippet.key}`);
+      }
+      displayTranslation(translation);
+    } else {
+      listTranslations(snippet);
+    }
+  }
+
+  function listTranslations(snippet: TranslatedSnippet) {
+    const original = snippet.originalSource;
+    if (original !== undefined) {
+      displayTranslation(original);
+    }
+
+    for (const lang of snippet.languages) {
+      process.stdout.write(languageHeader(lang) + '\n');
+      displayTranslation(snippet.get(lang)!);
+    }
+  }
+
+  function displayTranslation(translation: Translation) {
+    process.stdout.write(translation.source + '\n');
+  }
+}
+
+function snippetHeader(key: string) {
+  return center(` ${key} `, 100, '=');
+}
+
+function languageHeader(key: string) {
+  return center(` ${key} `, 30, '-');
+}
+
+function center(str: string, n: number, fill: string) {
+  const before = Math.floor((n - str.length) / 2);
+  const after = Math.ceil((n - str.length) / 2);
+
+  return fill.repeat(Math.max(before, 0)) + str + fill.repeat(Math.max(after, 0));
+}
\ No newline at end of file
diff --git a/packages/jsii-rosetta/lib/fixtures.ts b/packages/jsii-rosetta/lib/fixtures.ts
new file mode 100644
index 0000000000..ce0566e1f9
--- /dev/null
+++ b/packages/jsii-rosetta/lib/fixtures.ts
@@ -0,0 +1,57 @@
+import fs = require('fs-extra');
+import path = require('path');
+import { TypeScriptSnippet, SnippetParameters } from './snippet';
+
+/**
+ * Complete snippets with fixtures, if required
+ */
+export function fixturize(snippet: TypeScriptSnippet): TypeScriptSnippet {
+  let source = snippet.visibleSource;
+  const parameters = snippet.parameters || {};
+
+  const directory = parameters[SnippetParameters.$PROJECT_DIRECTORY];
+  if (!directory) { return snippet; }
+
+  const literateSource = parameters[SnippetParameters.LITERATE_SOURCE];
+  if (literateSource) {
+    // Compatibility with the "old school" example inclusion mechanism.
+    // Completely load this file and attach a parameter with its directory.
+    source = loadLiterateSource(directory, literateSource);
+    parameters[SnippetParameters.$COMPILATION_DIRECTORY] = path.join(directory, path.dirname(literateSource));
+  } else if (parameters[SnippetParameters.FIXTURE]) {
+    // Explicitly request a fixture
+    source = loadAndSubFixture(directory, parameters.fixture, source, true);
+  } else if (parameters[SnippetParameters.NO_FIXTURE] === undefined) {
+    // Don't explicitly request no fixture
+    source = loadAndSubFixture(directory, 'default', source, false);
+  }
+
+  return { visibleSource: snippet.visibleSource, completeSource: source, where: snippet.where, parameters };
+}
+
+function loadLiterateSource(directory: string, literateFileName: string) {
+  const fullPath = path.join(directory, literateFileName);
+  const exists = fs.existsSync(fullPath);
+  if (!exists) {
+    // This couldn't really happen in practice, but do the check anyway
+    throw new Error(`Sample uses literate source ${literateFileName}, but not found: ${fullPath}`);
+  }
+  return fs.readFileSync(fullPath, { encoding: 'utf-8' });
+}
+
+function loadAndSubFixture(directory: string, fixtureName: string, source: string, mustExist: boolean) {
+  const fixtureFileName = path.join(directory, `rosetta/${fixtureName}.ts-fixture`);
+  const exists = fs.existsSync(fixtureFileName);
+  if (!exists && mustExist) {
+    throw new Error(`Sample uses fixture ${fixtureName}, but not found: ${fixtureFileName}`);
+  }
+  if (!exists) { return source; }
+  const fixtureContents = fs.readFileSync(fixtureFileName, { encoding: 'utf-8' });
+
+  const subRegex = /\/\/\/ here/i;
+  if (!subRegex.test(fixtureContents)) {
+    throw new Error(`Fixture does not contain '/// here': ${fixtureFileName}`);
+  }
+
+  return fixtureContents.replace(subRegex, `/// !show\n${source}\n/// !hide`);
+}
diff --git a/packages/jsii-rosetta/lib/index.ts b/packages/jsii-rosetta/lib/index.ts
new file mode 100644
index 0000000000..773380721a
--- /dev/null
+++ b/packages/jsii-rosetta/lib/index.ts
@@ -0,0 +1,6 @@
+export * from './translate';
+export { renderTree } from './o-tree';
+export { PythonVisitor } from './languages/python';
+export * from './tablets/tablets'
+export * from './rosetta';
+export * from './snippet';
\ No newline at end of file
diff --git a/packages/jsii-rosetta/lib/jsii/assemblies.ts b/packages/jsii-rosetta/lib/jsii/assemblies.ts
new file mode 100644
index 0000000000..4100c044ab
--- /dev/null
+++ b/packages/jsii-rosetta/lib/jsii/assemblies.ts
@@ -0,0 +1,121 @@
+import spec = require('jsii-spec');
+import fs = require('fs-extra');
+import path = require('path');
+import { TypeScriptSnippet, typeScriptSnippetFromSource, updateParameters, SnippetParameters } from '../snippet';
+import { extractTypescriptSnippetsFromMarkdown } from '../markdown/extract-snippets';
+import { fixturize } from '../fixtures';
+
+export interface LoadedAssembly {
+  assembly: spec.Assembly;
+  directory: string;
+}
+
+/**
+ * Load assemblies by filename or directory
+ */
+export async function loadAssemblies(assemblyLocations: string[]) {
+  const ret: LoadedAssembly[] = [];
+  for (const loc of assemblyLocations) {
+    const stat = await fs.stat(loc);
+    if (stat.isDirectory()) {
+      ret.push({
+        assembly: await loadAssemblyFromFile(path.join(loc, '.jsii')),
+        directory: loc
+      });
+    } else {
+      ret.push({
+        assembly: await loadAssemblyFromFile(loc),
+        directory: path.dirname(loc),
+      });
+    }
+  }
+  return ret;
+}
+
+async function loadAssemblyFromFile(filename: string) {
+  const contents = await fs.readJSON(filename, { encoding: 'utf-8' });
+  return spec.validateAssembly(contents);
+}
+
+export type AssemblySnippetSource = { type: 'markdown'; markdown: string; where: string } | { type: 'literal'; source: string; where: string };
+
+/**
+ * Return all markdown and example snippets from the given assembly
+ */
+export function allSnippetSources(assembly: spec.Assembly): AssemblySnippetSource[] {
+  const ret: AssemblySnippetSource[] = [];
+
+  if (assembly.readme) {
+    ret.push({ type: 'markdown', markdown: assembly.readme.markdown, where: removeSlashes(`${assembly.name}-README`) });
+  }
+
+  if (assembly.types) {
+    Object.values(assembly.types).forEach(type => {
+      emitDocs(type.docs, `${assembly.name}.${type.name}`);
+
+      if (spec.isEnumType(type)) {
+        type.members.forEach(m => emitDocs(m.docs, `${assembly.name}.${type.name}.${m.name}`));
+      }
+      if (spec.isClassOrInterfaceType(type)) {
+        (type.methods || []).forEach(m => emitDocs(m.docs, `${assembly.name}.${type.name}#${m.name}`));
+        (type.properties || []).forEach(m => emitDocs(m.docs, `${assembly.name}.${type.name}#${m.name}`));
+      }
+    });
+  }
+
+  return ret;
+
+  function emitDocs(docs: spec.Docs | undefined, where: string) {
+    if (!docs) { return; }
+
+    if (docs.remarks) { ret.push({ 'type': 'markdown', markdown: docs.remarks, where: removeSlashes(where) }); }
+    if (docs.example && exampleLooksLikeSource(docs.example)) {
+      ret.push({ 'type': 'literal', source: docs.example, where: removeSlashes(`${where}-example`) });
+    }
+  }
+}
+
+/**
+ * Remove slashes from a "where" description, as the TS compiler will interpret it as a directory
+ * and we can't have that for compiling literate files
+ */
+function removeSlashes(x: string) {
+  return x.replace(/\//g, '.');
+}
+
+export function* allTypeScriptSnippets(assemblies: Array<{ assembly: spec.Assembly, directory: string }>): IterableIterator {
+  for (const assembly of assemblies) {
+    for (const source of allSnippetSources(assembly.assembly)) {
+      switch (source.type) {
+        case 'literal':
+          const snippet = updateParameters(typeScriptSnippetFromSource(source.source, source.where), {
+            [SnippetParameters.$PROJECT_DIRECTORY]: assembly.directory
+          });
+          yield fixturize(snippet);
+          break;
+        case 'markdown':
+          for (const snippet of extractTypescriptSnippetsFromMarkdown(source.markdown, source.where)) {
+            const withDirectory = updateParameters(snippet, {
+              [SnippetParameters.$PROJECT_DIRECTORY]: assembly.directory
+            });
+            yield fixturize(withDirectory);
+          }
+      }
+    }
+  }
+}
+
+/**
+ * See if the given source text looks like a code sample
+ *
+ * Many @examples for properties are examples of values (ARNs, formatted strings)
+ * not code samples, which should not be translated
+ *
+ * If the value contains whitespace (newline, space) then we'll assume it's a code
+ * sample.
+ */
+function exampleLooksLikeSource(text: string) {
+  return !!text.trim().match(WHITESPACE);
+}
+
+const WHITESPACE = new RegExp('\\s');
\ No newline at end of file
diff --git a/packages/jsii-sampiler/lib/jsii/jsii-utils.ts b/packages/jsii-rosetta/lib/jsii/jsii-utils.ts
similarity index 92%
rename from packages/jsii-sampiler/lib/jsii/jsii-utils.ts
rename to packages/jsii-rosetta/lib/jsii/jsii-utils.ts
index 278e90cace..b8cdf8c4d1 100644
--- a/packages/jsii-sampiler/lib/jsii/jsii-utils.ts
+++ b/packages/jsii-rosetta/lib/jsii/jsii-utils.ts
@@ -1,5 +1,5 @@
 import ts = require('typescript');
-import { AstConverter } from '../converter';
+import { AstRenderer } from '../renderer';
 
 export function isStructInterface(name: string) {
   return !name.startsWith('I');
@@ -22,7 +22,7 @@ export interface StructProperty {
   questionMark: boolean;
 }
 
-export function propertiesOfStruct(type: ts.Type, context: AstConverter): StructProperty[] {
+export function propertiesOfStruct(type: ts.Type, context: AstRenderer): StructProperty[] {
   return type.isClassOrInterface() ? type.getProperties().map(s => {
     let propType;
     let questionMark = false;
diff --git a/packages/jsii-rosetta/lib/jsii/packages.ts b/packages/jsii-rosetta/lib/jsii/packages.ts
new file mode 100644
index 0000000000..c3f1c6c233
--- /dev/null
+++ b/packages/jsii-rosetta/lib/jsii/packages.ts
@@ -0,0 +1,26 @@
+
+
+/**
+ * Resolve a package name in an example to a JSII assembly
+ *
+ * We assume we've changed directory to the directory where we need to resolve from.
+ */
+export function resolvePackage(packageName: string) {
+  try {
+    const resolved = require.resolve(`${packageName}/package.json`, { paths: [process.cwd()] });
+    return require(resolved);
+  } catch(e) {
+    return undefined;
+  }
+}
+
+export function jsiiTargetParam(packageName: string, field: string) {
+  const pkgJson = resolvePackage(packageName);
+
+  const path = ['jsii', 'targets', ...field.split('.')];
+  let r = pkgJson;
+  while (path.length > 0 && typeof r === 'object' && r !== null) {
+    r = r[path.splice(0, 1)[0]];
+  }
+  return r;
+}
\ No newline at end of file
diff --git a/packages/jsii-sampiler/lib/languages/default.ts b/packages/jsii-rosetta/lib/languages/default.ts
similarity index 72%
rename from packages/jsii-sampiler/lib/languages/default.ts
rename to packages/jsii-rosetta/lib/languages/default.ts
index 0c405ded07..0d778fdb63 100644
--- a/packages/jsii-sampiler/lib/languages/default.ts
+++ b/packages/jsii-rosetta/lib/languages/default.ts
@@ -1,5 +1,5 @@
 import ts = require('typescript');
-import { AstConverter, AstHandler, nimpl } from "../converter";
+import { AstRenderer, AstHandler, nimpl } from "../renderer";
 import { OTree } from '../o-tree';
 import { ImportStatement } from '../typescript/imports';
 
@@ -11,58 +11,58 @@ export abstract class DefaultVisitor implements AstHandler {
 
   public abstract mergeContext(old: C, update: C): C;
 
-  public commentRange(node: ts.CommentRange, context: AstConverter): OTree {
+  public commentRange(node: ts.CommentRange, context: AstRenderer): OTree {
     return new OTree([
       context.textAt(node.pos, node.end),
       node.hasTrailingNewLine ? '\n' : ''
     ]);
   }
 
-  public sourceFile(node: ts.SourceFile, context: AstConverter): OTree {
+  public sourceFile(node: ts.SourceFile, context: AstRenderer): OTree {
     return new OTree(context.convertAll(node.statements));
   }
 
-  public jsDoc(_node: ts.JSDoc, _context: AstConverter): OTree {
+  public jsDoc(_node: ts.JSDoc, _context: AstRenderer): OTree {
     // Already handled by other doc handlers
     return new OTree([]);
   }
 
-  public importStatement(node: ImportStatement, context: AstConverter): OTree {
+  public importStatement(node: ImportStatement, context: AstRenderer): OTree {
     return this.notImplemented(node.node, context);
   }
 
-  public functionDeclaration(node: ts.FunctionDeclaration, children: AstConverter): OTree {
+  public functionDeclaration(node: ts.FunctionDeclaration, children: AstRenderer): OTree {
     return this.notImplemented(node, children);
   }
 
-  public stringLiteral(node: ts.StringLiteral, _children: AstConverter): OTree {
+  public stringLiteral(node: ts.StringLiteral, _children: AstRenderer): OTree {
     return new OTree([JSON.stringify(node.text)]);
   }
 
-  public noSubstitutionTemplateLiteral(node: ts.NoSubstitutionTemplateLiteral, _context: AstConverter): OTree {
+  public noSubstitutionTemplateLiteral(node: ts.NoSubstitutionTemplateLiteral, _context: AstRenderer): OTree {
     return new OTree([JSON.stringify(node.text)]);
   }
 
-  public identifier(node: ts.Identifier, _children: AstConverter): OTree {
+  public identifier(node: ts.Identifier, _children: AstRenderer): OTree {
     return new OTree([node.text]);
   }
 
-  public block(node: ts.Block, children: AstConverter): OTree {
+  public block(node: ts.Block, children: AstRenderer): OTree {
     return new OTree(['{'], ['\n', ...children.convertAll(node.statements)], {
       indent: 4,
       suffix: '}',
     });
   }
 
-  public parameterDeclaration(node: ts.ParameterDeclaration, children: AstConverter): OTree {
+  public parameterDeclaration(node: ts.ParameterDeclaration, children: AstRenderer): OTree {
     return this.notImplemented(node, children);
   }
 
-  public returnStatement(node: ts.ReturnStatement, children: AstConverter): OTree {
+  public returnStatement(node: ts.ReturnStatement, children: AstRenderer): OTree {
     return new OTree(['return ', children.convert(node.expression)]);
   }
 
-  public binaryExpression(node: ts.BinaryExpression, context: AstConverter): OTree {
+  public binaryExpression(node: ts.BinaryExpression, context: AstRenderer): OTree {
     return new OTree([
       context.convert(node.left),
       ' ',
@@ -72,7 +72,7 @@ export abstract class DefaultVisitor implements AstHandler {
     ]);
   }
 
-  public prefixUnaryExpression(node: ts.PrefixUnaryExpression, context: AstConverter): OTree {
+  public prefixUnaryExpression(node: ts.PrefixUnaryExpression, context: AstRenderer): OTree {
 
     return new OTree([
       UNARY_OPS[node.operator],
@@ -80,15 +80,15 @@ export abstract class DefaultVisitor implements AstHandler {
     ]);
   }
 
-  public ifStatement(node: ts.IfStatement, context: AstConverter): OTree {
+  public ifStatement(node: ts.IfStatement, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public propertyAccessExpression(node: ts.PropertyAccessExpression, context: AstConverter): OTree {
+  public propertyAccessExpression(node: ts.PropertyAccessExpression, context: AstRenderer): OTree {
     return new OTree([context.convert(node.expression), '.', context.convert(node.name)]);
   }
 
-  public callExpression(node: ts.CallExpression, context: AstConverter): OTree {
+  public callExpression(node: ts.CallExpression, context: AstRenderer): OTree {
     return new OTree([
       context.convert(node.expression),
       '(',
@@ -96,112 +96,112 @@ export abstract class DefaultVisitor implements AstHandler {
       ')']);
   }
 
-  public expressionStatement(node: ts.ExpressionStatement, context: AstConverter): OTree {
+  public expressionStatement(node: ts.ExpressionStatement, context: AstRenderer): OTree {
     return new OTree([context.convert(node.expression)], [], { canBreakLine: true });
   }
 
-  public token(node: ts.Token, context: AstConverter): OTree {
+  public token(node: ts.Token, context: AstRenderer): OTree {
     return new OTree([context.textOf(node)]);
   }
 
-  public objectLiteralExpression(node: ts.ObjectLiteralExpression, context: AstConverter): OTree {
+  public objectLiteralExpression(node: ts.ObjectLiteralExpression, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public newExpression(node: ts.NewExpression, context: AstConverter): OTree {
+  public newExpression(node: ts.NewExpression, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public propertyAssignment(node: ts.PropertyAssignment, context: AstConverter): OTree {
+  public propertyAssignment(node: ts.PropertyAssignment, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public variableStatement(node: ts.VariableStatement, context: AstConverter): OTree {
+  public variableStatement(node: ts.VariableStatement, context: AstRenderer): OTree {
     return new OTree([context.convert(node.declarationList)], [], { canBreakLine: true });
   }
 
-  public variableDeclarationList(node: ts.VariableDeclarationList, context: AstConverter): OTree {
+  public variableDeclarationList(node: ts.VariableDeclarationList, context: AstRenderer): OTree {
     return new OTree([], context.convertAll(node.declarations));
   }
 
-  public variableDeclaration(node: ts.VariableDeclaration, context: AstConverter): OTree {
+  public variableDeclaration(node: ts.VariableDeclaration, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public arrayLiteralExpression(node: ts.ArrayLiteralExpression, context: AstConverter): OTree {
+  public arrayLiteralExpression(node: ts.ArrayLiteralExpression, context: AstRenderer): OTree {
     return new OTree(['['], context.convertAll(node.elements), {
       separator: ', ',
       suffix: ']',
     });
   }
 
-  public shorthandPropertyAssignment(node: ts.ShorthandPropertyAssignment, context: AstConverter): OTree {
+  public shorthandPropertyAssignment(node: ts.ShorthandPropertyAssignment, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public forOfStatement(node: ts.ForOfStatement, context: AstConverter): OTree {
+  public forOfStatement(node: ts.ForOfStatement, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public classDeclaration(node: ts.ClassDeclaration, context: AstConverter): OTree {
+  public classDeclaration(node: ts.ClassDeclaration, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public constructorDeclaration(node: ts.ConstructorDeclaration, context: AstConverter): OTree {
+  public constructorDeclaration(node: ts.ConstructorDeclaration, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public propertyDeclaration(node: ts.PropertyDeclaration, context: AstConverter): OTree {
+  public propertyDeclaration(node: ts.PropertyDeclaration, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public methodDeclaration(node: ts.MethodDeclaration, context: AstConverter): OTree {
+  public methodDeclaration(node: ts.MethodDeclaration, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public interfaceDeclaration(node: ts.InterfaceDeclaration, context: AstConverter): OTree {
+  public interfaceDeclaration(node: ts.InterfaceDeclaration, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public propertySignature(node: ts.PropertySignature, context: AstConverter): OTree {
+  public propertySignature(node: ts.PropertySignature, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public asExpression(node: ts.AsExpression, context: AstConverter): OTree {
+  public asExpression(node: ts.AsExpression, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public spreadElement(node: ts.SpreadElement, context: AstConverter): OTree {
+  public spreadElement(node: ts.SpreadElement, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public spreadAssignment(node: ts.SpreadAssignment, context: AstConverter): OTree {
+  public spreadAssignment(node: ts.SpreadAssignment, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public ellipsis(_node: ts.SpreadElement | ts.SpreadAssignment, _context: AstConverter): OTree {
+  public ellipsis(_node: ts.SpreadElement | ts.SpreadAssignment, _context: AstRenderer): OTree {
     return new OTree(['...']);
   }
 
-  public templateExpression(node: ts.TemplateExpression, context: AstConverter): OTree {
+  public templateExpression(node: ts.TemplateExpression, context: AstRenderer): OTree {
     return this.notImplemented(node, context);
   }
 
-  public nonNullExpression(node: ts.NonNullExpression, context: AstConverter): OTree {
+  public nonNullExpression(node: ts.NonNullExpression, context: AstRenderer): OTree {
     // We default we drop the non-null assertion
     return context.convert(node.expression);
   }
 
-  public parenthesizedExpression(node: ts.ParenthesizedExpression, context: AstConverter): OTree {
+  public parenthesizedExpression(node: ts.ParenthesizedExpression, context: AstRenderer): OTree {
     return new OTree(['(', context.convert(node.expression), ')']);
   }
 
-  public maskingVoidExpression(_node: ts.VoidExpression, _context: AstConverter): OTree {
+  public maskingVoidExpression(_node: ts.VoidExpression, _context: AstRenderer): OTree {
     // Don't render anything by default when nodes are masked
     return new OTree([]);
   }
 
-  private notImplemented(node: ts.Node, context: AstConverter) {
+  private notImplemented(node: ts.Node, context: AstRenderer) {
     context.reportUnsupported(node);
     return nimpl(node, context);
   }
diff --git a/packages/jsii-rosetta/lib/languages/index.ts b/packages/jsii-rosetta/lib/languages/index.ts
new file mode 100644
index 0000000000..c6a5860811
--- /dev/null
+++ b/packages/jsii-rosetta/lib/languages/index.ts
@@ -0,0 +1,9 @@
+import { PythonVisitor } from "./python";
+import { AstHandler } from "../renderer";
+
+export type TargetLanguage = 'python';
+export type VisitorFactory = () => AstHandler;
+
+export const TARGET_LANGUAGES: {[key in TargetLanguage]: VisitorFactory} = {
+  'python': () => new PythonVisitor()
+};
\ No newline at end of file
diff --git a/packages/jsii-sampiler/lib/languages/python.ts b/packages/jsii-rosetta/lib/languages/python.ts
similarity index 97%
rename from packages/jsii-sampiler/lib/languages/python.ts
rename to packages/jsii-rosetta/lib/languages/python.ts
index 4c871d3291..d234823060 100644
--- a/packages/jsii-sampiler/lib/languages/python.ts
+++ b/packages/jsii-rosetta/lib/languages/python.ts
@@ -1,11 +1,12 @@
 import ts = require('typescript');
-import { AstConverter, nimpl } from "../converter";
+import { AstRenderer, nimpl } from "../renderer";
 import { isStructType, parameterAcceptsUndefined, propertiesOfStruct, StructProperty, structPropertyAcceptsUndefined } from '../jsii/jsii-utils';
 import { NO_SYNTAX, OTree, renderTree } from "../o-tree";
 import { matchAst, nodeOfType, stripCommentMarkers, voidExpressionString } from '../typescript/ast-utils';
 import { ImportStatement } from '../typescript/imports';
 import { startsWithUppercase } from "../util";
 import { DefaultVisitor } from './default';
+import { jsiiTargetParam } from '../jsii/packages';
 
 interface StructVar {
   variableName: string;
@@ -60,7 +61,7 @@ interface PythonLanguageContext {
   readonly currentMethodName?: string;
 }
 
-type PythonVisitorContext = AstConverter;
+type PythonVisitorContext = AstRenderer;
 
 export interface PythonVisitorOptions {
   disclaimer?: string;
@@ -458,7 +459,12 @@ export class PythonVisitor extends DefaultVisitor {
   }
 
   protected convertModuleReference(ref: string) {
-    return ref.replace(/^@/, '').replace(/\//g, '.').replace(/-/g, '_');
+    // Get the Python target name from the referenced package (if available)
+    const resolvedPackage = jsiiTargetParam(ref, 'python.module');
+
+    // Return that or some default-derived module name representation
+
+    return resolvedPackage || ref.replace(/^@/, '').replace(/\//g, '.').replace(/-/g, '_');
   }
 
   /**
diff --git a/packages/jsii-sampiler/lib/languages/visualize.ts b/packages/jsii-rosetta/lib/languages/visualize.ts
similarity index 71%
rename from packages/jsii-sampiler/lib/languages/visualize.ts
rename to packages/jsii-rosetta/lib/languages/visualize.ts
index 837b0f3c74..ed95eca390 100644
--- a/packages/jsii-sampiler/lib/languages/visualize.ts
+++ b/packages/jsii-rosetta/lib/languages/visualize.ts
@@ -1,5 +1,5 @@
 import ts = require('typescript');
-import { AstConverter, AstHandler, nimpl } from "../converter";
+import { AstRenderer, AstHandler, nimpl } from "../renderer";
 import { OTree } from '../o-tree';
 import { ImportStatement } from '../typescript/imports';
 
@@ -13,172 +13,172 @@ export class VisualizeAstVisitor implements AstHandler {
     return undefined;
   }
 
-  public commentRange(node: ts.CommentRange, context: AstConverter): OTree {
+  public commentRange(node: ts.CommentRange, context: AstRenderer): OTree {
     return new OTree(['(Comment', context.textAt(node.pos, node.end)], [], { suffix: ')' });
   }
 
-  public jsDoc(_node: ts.JSDoc, _context: AstConverter): OTree {
+  public jsDoc(_node: ts.JSDoc, _context: AstRenderer): OTree {
     // Already handled by other doc handlers
     return new OTree([]);
   }
 
-  public sourceFile(node: ts.SourceFile, context: AstConverter): OTree {
+  public sourceFile(node: ts.SourceFile, context: AstRenderer): OTree {
     return new OTree(context.convertAll(node.statements));
   }
 
-  public importStatement(node: ImportStatement, context: AstConverter): OTree {
+  public importStatement(node: ImportStatement, context: AstRenderer): OTree {
     return this.defaultNode('importStatement', node.node, context);
   }
 
-  public functionDeclaration(node: ts.FunctionDeclaration, children: AstConverter): OTree {
+  public functionDeclaration(node: ts.FunctionDeclaration, children: AstRenderer): OTree {
     return this.defaultNode('functionDeclaration', node, children);
   }
 
-  public stringLiteral(node: ts.StringLiteral, children: AstConverter): OTree {
+  public stringLiteral(node: ts.StringLiteral, children: AstRenderer): OTree {
     return this.defaultNode('stringLiteral', node, children);
   }
 
-  public identifier(node: ts.Identifier, children: AstConverter): OTree {
+  public identifier(node: ts.Identifier, children: AstRenderer): OTree {
     return this.defaultNode('identifier', node, children);
   }
 
-  public block(node: ts.Block, children: AstConverter): OTree {
+  public block(node: ts.Block, children: AstRenderer): OTree {
     return this.defaultNode('block', node, children);
   }
 
-  public parameterDeclaration(node: ts.ParameterDeclaration, children: AstConverter): OTree {
+  public parameterDeclaration(node: ts.ParameterDeclaration, children: AstRenderer): OTree {
     return this.defaultNode('parameterDeclaration', node, children);
   }
 
-  public returnStatement(node: ts.ReturnStatement, children: AstConverter): OTree {
+  public returnStatement(node: ts.ReturnStatement, children: AstRenderer): OTree {
     return this.defaultNode('returnStatement', node, children);
   }
 
-  public binaryExpression(node: ts.BinaryExpression, children: AstConverter): OTree {
+  public binaryExpression(node: ts.BinaryExpression, children: AstRenderer): OTree {
     return this.defaultNode('binaryExpression', node, children);
   }
 
-  public ifStatement(node: ts.IfStatement, context: AstConverter): OTree {
+  public ifStatement(node: ts.IfStatement, context: AstRenderer): OTree {
     return this.defaultNode('ifStatement', node, context);
   }
 
-  public propertyAccessExpression(node: ts.PropertyAccessExpression, context: AstConverter): OTree {
+  public propertyAccessExpression(node: ts.PropertyAccessExpression, context: AstRenderer): OTree {
     return this.defaultNode('propertyAccessExpression', node, context);
   }
 
-  public callExpression(node: ts.CallExpression, context: AstConverter): OTree {
+  public callExpression(node: ts.CallExpression, context: AstRenderer): OTree {
     return this.defaultNode('callExpression', node, context);
   }
 
-  public expressionStatement(node: ts.ExpressionStatement, context: AstConverter): OTree {
+  public expressionStatement(node: ts.ExpressionStatement, context: AstRenderer): OTree {
     return this.defaultNode('expressionStatement', node, context);
   }
 
-  public token(node: ts.Token, context: AstConverter): OTree {
+  public token(node: ts.Token, context: AstRenderer): OTree {
     return this.defaultNode('token', node, context);
   }
 
-  public objectLiteralExpression(node: ts.ObjectLiteralExpression, context: AstConverter): OTree {
+  public objectLiteralExpression(node: ts.ObjectLiteralExpression, context: AstRenderer): OTree {
     return this.defaultNode('objectLiteralExpression', node, context);
   }
 
-  public newExpression(node: ts.NewExpression, context: AstConverter): OTree {
+  public newExpression(node: ts.NewExpression, context: AstRenderer): OTree {
     return this.defaultNode('newExpression', node, context);
   }
 
-  public propertyAssignment(node: ts.PropertyAssignment, context: AstConverter): OTree {
+  public propertyAssignment(node: ts.PropertyAssignment, context: AstRenderer): OTree {
     return this.defaultNode('propertyAssignment', node, context);
   }
 
-  public variableStatement(node: ts.VariableStatement, context: AstConverter): OTree {
+  public variableStatement(node: ts.VariableStatement, context: AstRenderer): OTree {
     return this.defaultNode('variableStatement', node, context);
   }
 
-  public variableDeclarationList(node: ts.VariableDeclarationList, context: AstConverter): OTree {
+  public variableDeclarationList(node: ts.VariableDeclarationList, context: AstRenderer): OTree {
     return this.defaultNode('variableDeclarationList', node, context);
   }
 
-  public variableDeclaration(node: ts.VariableDeclaration, context: AstConverter): OTree {
+  public variableDeclaration(node: ts.VariableDeclaration, context: AstRenderer): OTree {
     return this.defaultNode('variableDeclaration', node, context);
   }
 
-  public arrayLiteralExpression(node: ts.ArrayLiteralExpression, context: AstConverter): OTree {
+  public arrayLiteralExpression(node: ts.ArrayLiteralExpression, context: AstRenderer): OTree {
     return this.defaultNode('arrayLiteralExpression', node, context);
   }
 
-  public shorthandPropertyAssignment(node: ts.ShorthandPropertyAssignment, context: AstConverter): OTree {
+  public shorthandPropertyAssignment(node: ts.ShorthandPropertyAssignment, context: AstRenderer): OTree {
     return this.defaultNode('shorthandPropertyAssignment', node, context);
   }
 
-  public forOfStatement(node: ts.ForOfStatement, context: AstConverter): OTree {
+  public forOfStatement(node: ts.ForOfStatement, context: AstRenderer): OTree {
     return this.defaultNode('forOfStatement', node, context);
   }
 
-  public classDeclaration(node: ts.ClassDeclaration, context: AstConverter): OTree {
+  public classDeclaration(node: ts.ClassDeclaration, context: AstRenderer): OTree {
     return this.defaultNode('classDeclaration', node, context);
   }
 
-  public constructorDeclaration(node: ts.ConstructorDeclaration, context: AstConverter): OTree {
+  public constructorDeclaration(node: ts.ConstructorDeclaration, context: AstRenderer): OTree {
     return this.defaultNode('constructorDeclaration', node, context);
   }
 
-  public propertyDeclaration(node: ts.PropertyDeclaration, context: AstConverter): OTree {
+  public propertyDeclaration(node: ts.PropertyDeclaration, context: AstRenderer): OTree {
     return this.defaultNode('propertyDeclaration', node, context);
   }
 
-  public methodDeclaration(node: ts.MethodDeclaration, context: AstConverter): OTree {
+  public methodDeclaration(node: ts.MethodDeclaration, context: AstRenderer): OTree {
     return this.defaultNode('methodDeclaration', node, context);
   }
 
-  public interfaceDeclaration(node: ts.InterfaceDeclaration, context: AstConverter): OTree {
+  public interfaceDeclaration(node: ts.InterfaceDeclaration, context: AstRenderer): OTree {
     return this.defaultNode('interfaceDeclaration', node, context);
   }
 
-  public propertySignature(node: ts.PropertySignature, context: AstConverter): OTree {
+  public propertySignature(node: ts.PropertySignature, context: AstRenderer): OTree {
     return this.defaultNode('propertySignature', node, context);
   }
 
-  public asExpression(node: ts.AsExpression, context: AstConverter): OTree {
+  public asExpression(node: ts.AsExpression, context: AstRenderer): OTree {
     return this.defaultNode('asExpression', node, context);
   }
 
-  public prefixUnaryExpression(node: ts.PrefixUnaryExpression, context: AstConverter): OTree {
+  public prefixUnaryExpression(node: ts.PrefixUnaryExpression, context: AstRenderer): OTree {
     return this.defaultNode('prefixUnaryExpression', node, context);
   }
 
-  public spreadElement(node: ts.SpreadElement, context: AstConverter): OTree {
+  public spreadElement(node: ts.SpreadElement, context: AstRenderer): OTree {
     return this.defaultNode('spreadElement', node, context);
   }
 
-  public spreadAssignment(node: ts.SpreadAssignment, context: AstConverter): OTree {
+  public spreadAssignment(node: ts.SpreadAssignment, context: AstRenderer): OTree {
     return this.defaultNode('spreadAssignment', node, context);
   }
 
-  public ellipsis(node: ts.SpreadAssignment | ts.SpreadElement, context: AstConverter): OTree {
+  public ellipsis(node: ts.SpreadAssignment | ts.SpreadElement, context: AstRenderer): OTree {
     return this.defaultNode('ellipsis', node, context);
   }
 
-  public templateExpression(node: ts.TemplateExpression, context: AstConverter): OTree {
+  public templateExpression(node: ts.TemplateExpression, context: AstRenderer): OTree {
     return this.defaultNode('templateExpression', node, context);
   }
 
-  public nonNullExpression(node: ts.NonNullExpression, context: AstConverter): OTree {
+  public nonNullExpression(node: ts.NonNullExpression, context: AstRenderer): OTree {
     return this.defaultNode('nonNullExpression', node, context);
   }
 
-  public parenthesizedExpression(node: ts.ParenthesizedExpression, context: AstConverter): OTree {
+  public parenthesizedExpression(node: ts.ParenthesizedExpression, context: AstRenderer): OTree {
     return this.defaultNode('parenthesizedExpression', node, context);
   }
 
-  public maskingVoidExpression(node: ts.VoidExpression, context: AstConverter): OTree {
+  public maskingVoidExpression(node: ts.VoidExpression, context: AstRenderer): OTree {
     return this.defaultNode('maskingVoidExpression', node, context);
   }
 
-  public noSubstitutionTemplateLiteral(node: ts.NoSubstitutionTemplateLiteral, context: AstConverter): OTree {
+  public noSubstitutionTemplateLiteral(node: ts.NoSubstitutionTemplateLiteral, context: AstRenderer): OTree {
     return this.defaultNode('noSubstitutionTemplateLiteral', node, context);
   }
 
-  private defaultNode(handlerName: string, node: ts.Node, context: AstConverter): OTree {
+  private defaultNode(handlerName: string, node: ts.Node, context: AstRenderer): OTree {
     return nimpl(node, context, {
       additionalInfo: this.includeHandlerNames ? handlerName : ''
     });
diff --git a/packages/jsii-rosetta/lib/logging.ts b/packages/jsii-rosetta/lib/logging.ts
new file mode 100644
index 0000000000..7740acab0a
--- /dev/null
+++ b/packages/jsii-rosetta/lib/logging.ts
@@ -0,0 +1,35 @@
+import util = require('util');
+
+export enum Level {
+  WARN = -1,
+  QUIET = 0,
+  INFO = 1,
+  VERBOSE = 2,
+}
+
+export const LEVEL_INFO: number = Level.INFO;
+export const LEVEL_VERBOSE: number = Level.VERBOSE;
+
+/** The minimal logging level for messages to be emitted. */
+/* eslint-disable prefer-const */
+export let level = Level.QUIET;
+/* eslint-enable prefer-const */
+
+export function warn(fmt: string, ...args: any[]) {
+  log(Level.WARN, fmt, ...args);
+}
+
+export function info(fmt: string, ...args: any[]) {
+  log(Level.INFO, fmt, ...args);
+}
+
+export function debug(fmt: string, ...args: any[]) {
+  log(Level.VERBOSE, fmt, ...args);
+}
+
+function log(messageLevel: Level, fmt: string, ...args: any[]) {
+  if (level >= messageLevel) {
+    const levelName = Level[messageLevel];
+    process.stderr.write(`[jsii-rosetta] [${levelName}] ${util.format(fmt, ...args)}\n`);
+  }
+}
\ No newline at end of file
diff --git a/packages/jsii-rosetta/lib/markdown/extract-snippets.ts b/packages/jsii-rosetta/lib/markdown/extract-snippets.ts
new file mode 100644
index 0000000000..e10033eb45
--- /dev/null
+++ b/packages/jsii-rosetta/lib/markdown/extract-snippets.ts
@@ -0,0 +1,21 @@
+import cm = require('commonmark');
+import { visitCommonMarkTree } from '../markdown/markdown';
+import { CodeBlock } from '../markdown/replace-code-renderer';
+import { TypeScriptSnippet } from '../snippet';
+import { ReplaceTypeScriptTransform } from './replace-typescript-transform';
+
+export type TypeScriptReplacer = (code: TypeScriptSnippet) => CodeBlock | undefined;
+
+export function extractTypescriptSnippetsFromMarkdown(markdown: string, wherePrefix: string): TypeScriptSnippet[] {
+  const parser = new cm.Parser();
+  const doc = parser.parse(markdown);
+
+  const ret: TypeScriptSnippet[] = [];
+
+  visitCommonMarkTree(doc, new ReplaceTypeScriptTransform(wherePrefix, (ts) => {
+    ret.push(ts);
+    return undefined;
+  }));
+
+  return ret;
+}
\ No newline at end of file
diff --git a/packages/jsii-sampiler/lib/markdown/markdown-renderer.ts b/packages/jsii-rosetta/lib/markdown/markdown-renderer.ts
similarity index 97%
rename from packages/jsii-sampiler/lib/markdown/markdown-renderer.ts
rename to packages/jsii-rosetta/lib/markdown/markdown-renderer.ts
index c876a38b86..c7873d831d 100644
--- a/packages/jsii-sampiler/lib/markdown/markdown-renderer.ts
+++ b/packages/jsii-rosetta/lib/markdown/markdown-renderer.ts
@@ -41,8 +41,8 @@ export class MarkdownRenderer implements CommonMarkRenderer {
     return node.literal || '';
   }
 
-  public html_block(_node: cm.Node, context: RendererContext) {
-    return `${context.content()}`;
+  public html_block(node: cm.Node, _context: RendererContext) {
+    return node.literal || '';
   }
 
   public link(node: cm.Node, context: RendererContext) {
diff --git a/packages/jsii-sampiler/lib/markdown/markdown.ts b/packages/jsii-rosetta/lib/markdown/markdown.ts
similarity index 100%
rename from packages/jsii-sampiler/lib/markdown/markdown.ts
rename to packages/jsii-rosetta/lib/markdown/markdown.ts
diff --git a/packages/jsii-sampiler/lib/markdown/replace-code-renderer.ts b/packages/jsii-rosetta/lib/markdown/replace-code-renderer.ts
similarity index 94%
rename from packages/jsii-sampiler/lib/markdown/replace-code-renderer.ts
rename to packages/jsii-rosetta/lib/markdown/replace-code-renderer.ts
index 4229e4bd7b..fb93793fd8 100644
--- a/packages/jsii-sampiler/lib/markdown/replace-code-renderer.ts
+++ b/packages/jsii-rosetta/lib/markdown/replace-code-renderer.ts
@@ -18,7 +18,7 @@ export class ReplaceCodeTransform implements CommonMarkVisitor {
   public code_block(node: cm.Node) {
     const ret = this.replacer({ language: node.info || '', source: node.literal || '' });
     node.info = ret.language;
-    node.literal = ret.source;
+    node.literal = ret.source + (!ret.source || ret.source.endsWith('\n') ? '' : '\n');
   }
 
   public block_quote(): void { /* nothing */ }
diff --git a/packages/jsii-rosetta/lib/markdown/replace-typescript-transform.ts b/packages/jsii-rosetta/lib/markdown/replace-typescript-transform.ts
new file mode 100644
index 0000000000..d89ba5a89a
--- /dev/null
+++ b/packages/jsii-rosetta/lib/markdown/replace-typescript-transform.ts
@@ -0,0 +1,37 @@
+import { ReplaceCodeTransform, CodeBlock } from "./replace-code-renderer";
+import { TypeScriptSnippet, typeScriptSnippetFromSource, parseKeyValueList } from "../snippet";
+
+export type TypeScriptReplacer = (code: TypeScriptSnippet) => CodeBlock | undefined;
+
+/**
+ * A specialization of ReplaceCodeTransform that maintains state about TypeScript snippets
+ */
+export class ReplaceTypeScriptTransform extends ReplaceCodeTransform {
+  private readonly wherePrefix: string;
+
+  constructor(wherePrefix: string, replacer: TypeScriptReplacer) {
+    let count = 0;
+    super(block => {
+      const languageParts = block.language ? block.language.split(' ') : [];
+      if (languageParts[0] !== 'typescript' && languageParts[0] !== 'ts') { return block; }
+
+      count += 1;
+      const tsSnippet = typeScriptSnippetFromSource(
+        block.source,
+        this.addSnippetNumber(count),
+        parseKeyValueList(languageParts.slice(1))
+      );
+
+      return replacer(tsSnippet) || block;
+    });
+
+    this.wherePrefix = wherePrefix;
+  }
+
+  private addSnippetNumber(snippetNumber: number) {
+    // First snippet (most cases) will not be numbered
+    if (snippetNumber === 1) { return this.wherePrefix; }
+
+    return `${this.wherePrefix}-snippet${snippetNumber}`;
+  }
+}
diff --git a/packages/jsii-sampiler/lib/markdown/structure-renderer.ts b/packages/jsii-rosetta/lib/markdown/structure-renderer.ts
similarity index 100%
rename from packages/jsii-sampiler/lib/markdown/structure-renderer.ts
rename to packages/jsii-rosetta/lib/markdown/structure-renderer.ts
diff --git a/packages/jsii-sampiler/lib/o-tree.ts b/packages/jsii-rosetta/lib/o-tree.ts
similarity index 70%
rename from packages/jsii-sampiler/lib/o-tree.ts
rename to packages/jsii-rosetta/lib/o-tree.ts
index a81ae58be5..6ca3feb8f1 100644
--- a/packages/jsii-sampiler/lib/o-tree.ts
+++ b/packages/jsii-rosetta/lib/o-tree.ts
@@ -1,4 +1,3 @@
-
 export interface OTreeOptions {
   /**
    * Adjust indentation with the given number
@@ -48,9 +47,9 @@ export interface OTreeOptions {
  * "Output" Tree
  *
  * Tree-like structure that holds sequences of trees and strings, which
- * can be rendered to an output stream.
+ * can be rendered to an output sink.
  */
-export class OTree {
+export class OTree implements OTree {
   public static simplify(xs: Array): Array {
     return xs.filter(notUndefined).filter(notEmpty);
   }
@@ -59,6 +58,7 @@ export class OTree {
 
   private readonly prefix: Array;
   private readonly children: Array;
+  private span?: Span;
 
   constructor(
     prefix: Array,
@@ -70,25 +70,36 @@ export class OTree {
     this.attachComment = !!options.canBreakLine;
   }
 
+  /**
+   * Set the span in the source file this tree node relates to
+   */
+  public setSpan(start: number, end: number) {
+    this.span = { start, end };
+  }
+
   public write(sink: OTreeSink) {
     if (!sink.tagOnce(this.options.renderOnce)) { return; }
 
+    const meVisible = sink.renderingForSpan(this.span);
+
     for (const x of this.prefix) {
       sink.write(x);
     }
 
-    const popIndent = sink.requestIndentChange(this.options.indent || 0);
+    const popIndent = sink.requestIndentChange(meVisible ? this.options.indent || 0 : 0);
     let mark = sink.mark();
     for (const child of this.children || []) {
-      if (this.options.separator && mark.wroteNonWhitespaceSinceMark) { sink.write(this.options.separator); }
+      if (this.options.separator && mark.wroteNonWhitespaceSinceMark) {
+        sink.write(this.options.separator);
+      }
       mark = sink.mark();
 
       sink.write(child);
     }
-
     popIndent();
 
     if (this.options.suffix) {
+      sink.renderingForSpan(this.span);
       sink.write(this.options.suffix);
     }
   }
@@ -111,11 +122,28 @@ export interface SinkMark {
   readonly wroteNonWhitespaceSinceMark: boolean;
 }
 
+export interface OTreeSinkOptions {
+  visibleSpans?: Span[];
+}
+
+/**
+ * Output sink for OTree objects
+ *
+ * Maintains state about what has been rendered supports suppressing code
+ * fragments based on their tagged source location.
+ *
+ * Basically: manages the state that was too hard to manage in the
+ * tree :).
+ */
 export class OTreeSink {
   private readonly indentLevels: number[] = [0];
   private readonly fragments = new Array();
   private singletonsRendered = new Set();
   private pendingIndentChange = 0;
+  private rendering = true;
+
+  constructor(private readonly options: OTreeSinkOptions = {}) {
+  }
 
   public tagOnce(key: string | undefined): boolean {
     if (key === undefined) { return true; }
@@ -124,6 +152,11 @@ export class OTreeSink {
     return true;
   }
 
+  /**
+   * Get a mark for the current sink output location
+   *
+   * Marks can be used to query about things that have been written to output.
+   */
   public mark(): SinkMark {
     const self = this;
     const markIndex = this.fragments.length;
@@ -139,6 +172,8 @@ export class OTreeSink {
     if (text instanceof OTree) {
       text.write(this);
     } else {
+      if (!this.rendering) { return; }
+
       if (containsNewline(text)) {
         this.applyPendingIndentChange();
       }
@@ -146,6 +181,13 @@ export class OTreeSink {
     }
   }
 
+  public renderingForSpan(span?: Span): boolean {
+    if (span && this.options.visibleSpans) {
+      this.rendering = this.options.visibleSpans.some(v => spanInside(span, v));
+    }
+    return this.rendering;
+  }
+
   public requestIndentChange(x: number): () => void {
     if (x === 0) { return () => undefined; }
 
@@ -162,8 +204,8 @@ export class OTreeSink {
   }
 
   public toString() {
-    // Strip trailing whitespace from every line
-    return this.fragments.join('').replace(/[ \t]+$/gm, '');
+    // Strip trailing whitespace from every line, and empty lines from the start and end
+    return this.fragments.join('').replace(/[ \t]+$/gm, '').replace(/^\n+/, '').replace(/\n+$/, '');
   }
 
   private append(x: string) {
@@ -190,8 +232,8 @@ function notEmpty(x: OTree | string) {
   return x instanceof OTree ? !x.isEmpty : x !== '';
 }
 
-export function renderTree(tree: OTree): string {
-  const sink = new OTreeSink();
+export function renderTree(tree: OTree, options?: OTreeSinkOptions): string {
+  const sink = new OTreeSink(options);
   tree.write(sink);
   return sink.toString();
 }
@@ -199,3 +241,16 @@ export function renderTree(tree: OTree): string {
 function containsNewline(x: string) {
   return x.indexOf('\n') !== -1;
 }
+
+export interface Span {
+  start: number;
+  end: number
+}
+
+export function spanInside(a: Span, b: Span) {
+  return b.start <= a.start && a.end <= b.end;
+}
+
+export function spanContains(a: Span, position: number) {
+  return a.start <= position && position < a.end;
+}
\ No newline at end of file
diff --git a/packages/jsii-sampiler/lib/converter.ts b/packages/jsii-rosetta/lib/renderer.ts
similarity index 82%
rename from packages/jsii-sampiler/lib/converter.ts
rename to packages/jsii-rosetta/lib/renderer.ts
index 42c540b6b8..e4f3d9897c 100644
--- a/packages/jsii-sampiler/lib/converter.ts
+++ b/packages/jsii-rosetta/lib/renderer.ts
@@ -5,14 +5,14 @@ import { commentRangeFromTextRange, extractMaskingVoidExpression, extractShowing
 import { analyzeImportDeclaration, analyzeImportEquals, ImportStatement } from './typescript/imports';
 
 /**
- * AST conversion operation
+ * Render a TypeScript AST to some other representation (encoded in OTrees)
  *
  * Dispatch the actual conversion to a specific handler which will get the
  * appropriate method called for particular AST nodes. The handler may use
  * context to modify its own operations when traversing the tree hierarchy,
  * the type of which should be expressed via the C parameter.
  */
-export class AstConverter {
+export class AstRenderer {
   public readonly diagnostics = new Array();
   public readonly currentContext: C;
 
@@ -20,7 +20,7 @@ export class AstConverter {
     private readonly sourceFile: ts.SourceFile,
     private readonly typeChecker: ts.TypeChecker,
     private readonly handler: AstHandler,
-    private readonly options: ConvertOptions = {}) {
+    private readonly options: AstRendererOptions = {}) {
 
     this.currentContext = handler.defaultContext;
   }
@@ -28,7 +28,7 @@ export class AstConverter {
   /**
    * Merge the new context with the current context and create a new Converter from it
    */
-  public updateContext(contextUpdate: C): AstConverter {
+  public updateContext(contextUpdate: C): AstRenderer {
     const newContext = this.handler.mergeContext(this.currentContext, contextUpdate);
 
     // Use prototypal inheritance to create a version of 'this' in which only
@@ -46,9 +46,12 @@ export class AstConverter {
 
     // Basic transform of node
     const transformed = this.dispatch(node);
+    transformed.setSpan(node.getStart(this.sourceFile), node.getEnd());
     if (!transformed.attachComment) { return transformed; }
 
-    return this.attachLeadingTrivia(node, transformed);
+    const withTrivia = this.attachLeadingTrivia(node, transformed);
+    withTrivia.setSpan(node.getStart(this.sourceFile), node.getEnd());
+    return withTrivia;
   }
 
   /**
@@ -102,7 +105,9 @@ export class AstConverter {
 
   public report(node: ts.Node, messageText: string, category: ts.DiagnosticCategory = ts.DiagnosticCategory.Error) {
     this.diagnostics.push({
-      category, code: 0,
+      category,
+      code: 0,
+      source: 'rosetta',
       messageText,
       file: this.sourceFile,
       start: node.getStart(this.sourceFile),
@@ -236,6 +241,9 @@ export class AstConverter {
         case 'blockcomment':
           precede.push(this.handler.commentRange(commentRangeFromTextRange(range), this));
           break;
+
+        case 'directive':
+          break;
       }
     }
 
@@ -263,53 +271,53 @@ export interface AstHandler {
   readonly defaultContext: C;
   mergeContext(old: C, update: C): C;
 
-  sourceFile(node: ts.SourceFile, context: AstConverter): OTree;
-  commentRange(node: ts.CommentRange, context: AstConverter): OTree;
-  importStatement(node: ImportStatement, context: AstConverter): OTree;
-  stringLiteral(node: ts.StringLiteral, children: AstConverter): OTree;
-  functionDeclaration(node: ts.FunctionDeclaration, children: AstConverter): OTree;
-  identifier(node: ts.Identifier, children: AstConverter): OTree;
-  block(node: ts.Block, children: AstConverter): OTree;
-  parameterDeclaration(node: ts.ParameterDeclaration, children: AstConverter): OTree;
-  returnStatement(node: ts.ReturnStatement, context: AstConverter): OTree;
-  binaryExpression(node: ts.BinaryExpression, context: AstConverter): OTree;
-  ifStatement(node: ts.IfStatement, context: AstConverter): OTree;
-  propertyAccessExpression(node: ts.PropertyAccessExpression, context: AstConverter): OTree;
-  callExpression(node: ts.CallExpression, context: AstConverter): OTree;
-  expressionStatement(node: ts.ExpressionStatement, context: AstConverter): OTree;
-  token(node: ts.Token, context: AstConverter): OTree;
-  objectLiteralExpression(node: ts.ObjectLiteralExpression, context: AstConverter): OTree;
-  newExpression(node: ts.NewExpression, context: AstConverter): OTree;
-  propertyAssignment(node: ts.PropertyAssignment, context: AstConverter): OTree;
-  variableStatement(node: ts.VariableStatement, context: AstConverter): OTree;
-  variableDeclarationList(node: ts.VariableDeclarationList, context: AstConverter): OTree;
-  variableDeclaration(node: ts.VariableDeclaration, context: AstConverter): OTree;
-  jsDoc(node: ts.JSDoc, context: AstConverter): OTree;
-  arrayLiteralExpression(node: ts.ArrayLiteralExpression, context: AstConverter): OTree;
-  shorthandPropertyAssignment(node: ts.ShorthandPropertyAssignment, context: AstConverter): OTree;
-  forOfStatement(node: ts.ForOfStatement, context: AstConverter): OTree;
-  classDeclaration(node: ts.ClassDeclaration, context: AstConverter): OTree;
-  constructorDeclaration(node: ts.ConstructorDeclaration, context: AstConverter): OTree;
-  propertyDeclaration(node: ts.PropertyDeclaration, context: AstConverter): OTree;
-  methodDeclaration(node: ts.MethodDeclaration, context: AstConverter): OTree;
-  interfaceDeclaration(node: ts.InterfaceDeclaration, context: AstConverter): OTree;
-  propertySignature(node: ts.PropertySignature, context: AstConverter): OTree;
-  asExpression(node: ts.AsExpression, context: AstConverter): OTree;
-  prefixUnaryExpression(node: ts.PrefixUnaryExpression, context: AstConverter): OTree;
-  spreadElement(node: ts.SpreadElement, context: AstConverter): OTree;
-  spreadAssignment(node: ts.SpreadAssignment, context: AstConverter): OTree;
-  templateExpression(node: ts.TemplateExpression, context: AstConverter): OTree;
-  nonNullExpression(node: ts.NonNullExpression, context: AstConverter): OTree;
-  parenthesizedExpression(node: ts.ParenthesizedExpression, context: AstConverter): OTree;
-  maskingVoidExpression(node: ts.VoidExpression, context: AstConverter): OTree;
-  noSubstitutionTemplateLiteral(node: ts.NoSubstitutionTemplateLiteral, context: AstConverter): OTree;
+  sourceFile(node: ts.SourceFile, context: AstRenderer): OTree;
+  commentRange(node: ts.CommentRange, context: AstRenderer): OTree;
+  importStatement(node: ImportStatement, context: AstRenderer): OTree;
+  stringLiteral(node: ts.StringLiteral, children: AstRenderer): OTree;
+  functionDeclaration(node: ts.FunctionDeclaration, children: AstRenderer): OTree;
+  identifier(node: ts.Identifier, children: AstRenderer): OTree;
+  block(node: ts.Block, children: AstRenderer): OTree;
+  parameterDeclaration(node: ts.ParameterDeclaration, children: AstRenderer): OTree;
+  returnStatement(node: ts.ReturnStatement, context: AstRenderer): OTree;
+  binaryExpression(node: ts.BinaryExpression, context: AstRenderer): OTree;
+  ifStatement(node: ts.IfStatement, context: AstRenderer): OTree;
+  propertyAccessExpression(node: ts.PropertyAccessExpression, context: AstRenderer): OTree;
+  callExpression(node: ts.CallExpression, context: AstRenderer): OTree;
+  expressionStatement(node: ts.ExpressionStatement, context: AstRenderer): OTree;
+  token(node: ts.Token, context: AstRenderer): OTree;
+  objectLiteralExpression(node: ts.ObjectLiteralExpression, context: AstRenderer): OTree;
+  newExpression(node: ts.NewExpression, context: AstRenderer): OTree;
+  propertyAssignment(node: ts.PropertyAssignment, context: AstRenderer): OTree;
+  variableStatement(node: ts.VariableStatement, context: AstRenderer): OTree;
+  variableDeclarationList(node: ts.VariableDeclarationList, context: AstRenderer): OTree;
+  variableDeclaration(node: ts.VariableDeclaration, context: AstRenderer): OTree;
+  jsDoc(node: ts.JSDoc, context: AstRenderer): OTree;
+  arrayLiteralExpression(node: ts.ArrayLiteralExpression, context: AstRenderer): OTree;
+  shorthandPropertyAssignment(node: ts.ShorthandPropertyAssignment, context: AstRenderer): OTree;
+  forOfStatement(node: ts.ForOfStatement, context: AstRenderer): OTree;
+  classDeclaration(node: ts.ClassDeclaration, context: AstRenderer): OTree;
+  constructorDeclaration(node: ts.ConstructorDeclaration, context: AstRenderer): OTree;
+  propertyDeclaration(node: ts.PropertyDeclaration, context: AstRenderer): OTree;
+  methodDeclaration(node: ts.MethodDeclaration, context: AstRenderer): OTree;
+  interfaceDeclaration(node: ts.InterfaceDeclaration, context: AstRenderer): OTree;
+  propertySignature(node: ts.PropertySignature, context: AstRenderer): OTree;
+  asExpression(node: ts.AsExpression, context: AstRenderer): OTree;
+  prefixUnaryExpression(node: ts.PrefixUnaryExpression, context: AstRenderer): OTree;
+  spreadElement(node: ts.SpreadElement, context: AstRenderer): OTree;
+  spreadAssignment(node: ts.SpreadAssignment, context: AstRenderer): OTree;
+  templateExpression(node: ts.TemplateExpression, context: AstRenderer): OTree;
+  nonNullExpression(node: ts.NonNullExpression, context: AstRenderer): OTree;
+  parenthesizedExpression(node: ts.ParenthesizedExpression, context: AstRenderer): OTree;
+  maskingVoidExpression(node: ts.VoidExpression, context: AstRenderer): OTree;
+  noSubstitutionTemplateLiteral(node: ts.NoSubstitutionTemplateLiteral, context: AstRenderer): OTree;
 
   // Not a node, called when we recognize a spread element/assignment that is only
   // '...' and nothing else.
-  ellipsis(node: ts.SpreadElement | ts.SpreadAssignment, context: AstConverter): OTree;
+  ellipsis(node: ts.SpreadElement | ts.SpreadAssignment, context: AstRenderer): OTree;
 }
 
-export function nimpl(node: ts.Node, context: AstConverter, options: { additionalInfo?: string} = {}) {
+export function nimpl(node: ts.Node, context: AstRenderer, options: { additionalInfo?: string} = {}) {
   const children = nodeChildren(node).map(c => context.convert(c));
 
   let syntaxKind = ts.SyntaxKind[node.kind];
@@ -330,7 +338,7 @@ export function nimpl(node: ts.Node, context: AstConverter, options: { add
   });
 }
 
-export interface ConvertOptions {
+export interface AstRendererOptions {
   /**
    * If enabled, don't translate the text of unknown nodes
    *
diff --git a/packages/jsii-rosetta/lib/rosetta.ts b/packages/jsii-rosetta/lib/rosetta.ts
new file mode 100644
index 0000000000..fd19e57f4e
--- /dev/null
+++ b/packages/jsii-rosetta/lib/rosetta.ts
@@ -0,0 +1,152 @@
+import fs = require('fs-extra');
+import path = require('path');
+import spec = require('jsii-spec');
+import { DEFAULT_TABLET_NAME, LanguageTablet, Translation } from "./tablets/tablets";
+import { allTypeScriptSnippets } from './jsii/assemblies';
+import { TargetLanguage } from './languages';
+import { Translator } from './translate';
+import { isError } from 'util';
+import { transformMarkdown } from './markdown/markdown';
+import { MarkdownRenderer } from './markdown/markdown-renderer';
+import { CodeBlock } from './markdown/replace-code-renderer';
+import { TypeScriptSnippet } from './snippet';
+import { printDiagnostics } from './util';
+import { ReplaceTypeScriptTransform } from './markdown/replace-typescript-transform';
+
+export interface RosettaOptions {
+  /**
+   * Whether or not to live-convert samples
+   *
+   * @default false
+   */
+  liveConversion?: boolean;
+
+  /**
+   * Target languages to use for live conversion
+   *
+   * @default All languages
+   */
+  targetLanguages?: TargetLanguage[];
+}
+
+/**
+ * Entry point class for consumers for Rosetta functionality
+ *
+ * Rosetta can work in one of two modes:
+ *
+ * 1. Live translation of snippets.
+ * 2. Read translations from a pre-translated tablet (prepared using `jsii-rosetta extract` command).
+ *
+ * The second method affords more control over the precise circumstances of
+ * sample compilation and is recommended, but the first method will do
+ * when the second one is not necessary.
+ */
+export class Rosetta {
+  private readonly loadedTablets: LanguageTablet[] = [];
+  private readonly liveTablet = new LanguageTablet();
+  private readonly extractedSnippets: Record = {};
+  private readonly translator = new Translator(false);
+
+  constructor(private readonly options: RosettaOptions = {}) {
+  }
+
+  /**
+   * Diagnostics encountered while doing live translation
+   */
+  public get diagnostics() {
+    return this.translator.diagnostics;
+  }
+
+  /**
+   * Load a tablet as a source for translateable snippets
+   */
+  public async loadTabletFromFile(tabletFile: string) {
+    const tablet = new LanguageTablet();
+    await tablet.load(tabletFile);
+    this.addTablet(tablet);
+  }
+
+  /**
+   * Directly add a tablet
+   *
+   * Should only be needed for testing, use `loadTabletFromFile` and `addAssembly` instead.
+   */
+  public addTablet(tablet: LanguageTablet) {
+    this.loadedTablets.push(tablet);
+  }
+
+  /**
+   * Add an assembly
+   *
+   * If a default tablet file is found in the assembly's directory, it will be
+   * loaded.
+   *
+   * Otherwise, if live conversion is enabled, the snippets in the assembly
+   * become available for live translation later.
+   *
+   * (We do it like this to centralize the logic around the "where" calculation,
+   * otherwise each language generator has to reimplement a way to describe API
+   * elements while spidering the jsii assembly).
+   */
+  public async addAssembly(assembly: spec.Assembly, assemblyDir: string) {
+    if (await fs.pathExists(path.join(assemblyDir, DEFAULT_TABLET_NAME))) {
+      await this.loadTabletFromFile(path.join(assemblyDir, DEFAULT_TABLET_NAME));
+      return;
+    }
+
+    if (this.options.liveConversion) {
+      for (const tsnip of allTypeScriptSnippets([{ assembly, directory: assemblyDir }])) {
+        this.extractedSnippets[tsnip.visibleSource] = tsnip;
+      }
+    }
+  }
+
+  public translateSnippet(source: TypeScriptSnippet, targetLang: TargetLanguage): Translation | undefined {
+    // Look for it in loaded tablets
+    for (const tab of this.allTablets) {
+      const ret = tab.lookup(source, targetLang);
+      if (ret !== undefined) { return ret; }
+    }
+
+    if (!this.options.liveConversion) { return undefined; }
+    if (this.options.targetLanguages && !this.options.targetLanguages.includes(targetLang)) {
+      throw new Error(`Rosetta configured for live conversion to ${this.options.targetLanguages}, but requested ${targetLang}`);
+    }
+
+    // See if we're going to live-convert it with full source information
+    const extracted = this.extractedSnippets[source.visibleSource];
+    if (extracted !== undefined) {
+      const snippet = this.translator.translate(extracted, this.options.targetLanguages);
+      this.liveTablet.addSnippet(snippet);
+      return snippet.get(targetLang);
+    }
+
+    // Try to live-convert it on the spot (we won't have "where" information or fixtures)
+    const snippet = this.translator.translate(source, this.options.targetLanguages);
+    return snippet.get(targetLang);
+  }
+
+  public translateSnippetsInMarkdown(markdown: string, targetLang: TargetLanguage, translationToCodeBlock: (x: Translation) => CodeBlock = id): string {
+    return transformMarkdown(markdown, new MarkdownRenderer(), new ReplaceTypeScriptTransform('markdown', tsSnip => {
+      const translated = this.translateSnippet(tsSnip, targetLang);
+      if (!translated) { return undefined; }
+
+      return translationToCodeBlock(translated);
+    }));
+  }
+
+  public printDiagnostics(stream: NodeJS.WritableStream) {
+    printDiagnostics(this.diagnostics, stream);
+  }
+
+  public get hasErrors() {
+    return this.diagnostics.some(isError);
+  };
+
+  private get allTablets(): LanguageTablet[]  {
+    return [...this.loadedTablets, this.liveTablet];
+  }
+}
+
+
+function id(x: Translation) { return x; }
\ No newline at end of file
diff --git a/packages/jsii-rosetta/lib/snippet.ts b/packages/jsii-rosetta/lib/snippet.ts
new file mode 100644
index 0000000000..186664e0ee
--- /dev/null
+++ b/packages/jsii-rosetta/lib/snippet.ts
@@ -0,0 +1,122 @@
+/**
+ * A piece of TypeScript code found in an assembly, ready to be translated
+ */
+export interface TypeScriptSnippet {
+  /**
+   * The snippet code that ends up in the JSII assembly
+   */
+  readonly visibleSource: string;
+
+  /**
+   * A human-readable description of where this snippet was found in the assembly
+   */
+  readonly where: string;
+
+  /**
+   * When enhanced with a fixture, the snippet's complete source code
+   */
+  readonly completeSource?: string;
+
+  /**
+   * Parameters for the conversion
+   */
+  readonly parameters?: Record;
+}
+
+/**
+ * Construct a TypeScript snippet from literal source
+ *
+ * Will parse parameters from a directive in the given source.
+ */
+export function typeScriptSnippetFromSource(typeScriptSource: string, where: string, parameters: Record = {}) {
+  const [source, sourceParameters] = parametersFromSourceDirectives(typeScriptSource);
+  return {
+    visibleSource: source.trimRight(),
+    where,
+    parameters: Object.assign({}, parameters, sourceParameters)
+  };
+}
+
+export function updateParameters(snippet: TypeScriptSnippet, params: Record): TypeScriptSnippet {
+  return {
+    ...snippet,
+    parameters: Object.assign({}, snippet.parameters || {}, params)
+  };
+}
+
+/**
+ * Get the complete (compilable) source of a snippet
+ */
+export function completeSource(snippet: TypeScriptSnippet) {
+  return snippet.completeSource || snippet.visibleSource;
+}
+
+/**
+ * Extract snippet parameters from the first line of the source if it's a compiler directive
+ */
+function parametersFromSourceDirectives(source: string): [string, Record] {
+  const [firstLine, rest] = source.split('\n', 2);
+  // Also extract parameters from an initial line starting with '/// ' (getting rid of that line).
+  const m = /\/\/\/(.*)$/.exec(firstLine);
+  if (m) {
+    const paramClauses = m[1].trim().split(' ').map(s => s.trim()).filter(s => s !== '');
+    return [rest, parseKeyValueList(paramClauses)];
+  }
+
+  return [source, {}];
+}
+
+/**
+ * Parse a set of 'param param=value' directives into an object
+ */
+export function parseKeyValueList(parameters: string[]): Record {
+  const ret: Record  = {};
+  for (const param of parameters) {
+    const parts = param.split('=', 2);
+    if (parts.length === 2) {
+      ret[parts[0]] = parts[1];
+    } else {
+      ret[parts[0]] = '';
+    }
+  }
+
+  return ret;
+}
+
+/**
+ * Recognized snippet parameters
+ */
+export enum SnippetParameters {
+  /**
+   * Use fixture with the given name (author parameter)
+   */
+  FIXTURE = 'fixture',
+
+  /**
+   * Don't use a fixture (author parameter)
+   */
+  NO_FIXTURE = 'nofixture',
+
+  /**
+   * Snippet was extracted from this literate file (backwards compatibility)
+   *
+   * Parameter attached by 'jsii'; load the given file instead of any fixture,
+   * process as usual.
+   */
+  LITERATE_SOURCE = 'lit',
+
+  /**
+   * What directory to resolve fixtures in for this snippet (system parameter)
+   *
+   * Attached during processing, should not be used by authors.
+   */
+  $PROJECT_DIRECTORY = '$directory',
+
+  /**
+   * What directory to pretend the file is in (system parameter)
+   *
+   * Attached when compiling a literate file, as they compile in
+   * the location where they are stored.
+   */
+  $COMPILATION_DIRECTORY = '$compilation',
+};
\ No newline at end of file
diff --git a/packages/jsii-rosetta/lib/tablets/key.ts b/packages/jsii-rosetta/lib/tablets/key.ts
new file mode 100644
index 0000000000..1d2329859b
--- /dev/null
+++ b/packages/jsii-rosetta/lib/tablets/key.ts
@@ -0,0 +1,11 @@
+import crypto = require('crypto');
+import { TypeScriptSnippet } from '../snippet';
+
+/**
+ * Determine the key for a code block
+ */
+export function snippetKey(snippet: TypeScriptSnippet) {
+  const h = crypto.createHash('sha256');
+  h.update(snippet.visibleSource);
+  return h.digest('hex');
+}
\ No newline at end of file
diff --git a/packages/jsii-rosetta/lib/tablets/schema.ts b/packages/jsii-rosetta/lib/tablets/schema.ts
new file mode 100644
index 0000000000..7287d3ecdb
--- /dev/null
+++ b/packages/jsii-rosetta/lib/tablets/schema.ts
@@ -0,0 +1,63 @@
+/**
+ * Tablet file schema
+ */
+export interface TabletSchema {
+  /**
+   * Schema version
+   */
+  version: string;
+
+  /**
+   * What version of the tool this schema was generated with
+   *
+   * Hashing algorithms may depend on the tool version, and the tool
+   * will reject tablets generated by different versions.
+   *
+   * Since tablets are designed to be used as scratch space during a build, not
+   * designed to be stored long-term, this limitation does not impact
+   * usability.
+   */
+  toolVersion: string;
+
+  /**
+   * All the snippets in the tablet
+   */
+  snippets: {[key: string]: TranslatedSnippetSchema};
+}
+
+export const ORIGINAL_SNIPPET_KEY = '$';
+
+/**
+ * Schema for a snippet
+ */
+export interface TranslatedSnippetSchema {
+  /**
+   * Translations for each individual language
+   *
+   * Since TypeScript is a valid output translation, the original will be
+   * listed under the key '$'.
+   */
+  translations: {[key: string]: TranslationSchema};
+
+  /**
+   * A human-readable description of the location this code snippet was found
+   */
+  where: string;
+
+  /**
+   * Whether this was compiled without errors
+   *
+   * Undefined means no compilation was not attempted.
+   */
+  didCompile?: boolean;
+}
+
+/**
+ * A single snippet's translation
+ */
+export interface TranslationSchema {
+  /**
+   * The source of a single translation
+   */
+  source: string;
+}
\ No newline at end of file
diff --git a/packages/jsii-rosetta/lib/tablets/tablets.ts b/packages/jsii-rosetta/lib/tablets/tablets.ts
new file mode 100644
index 0000000000..77f766d8a4
--- /dev/null
+++ b/packages/jsii-rosetta/lib/tablets/tablets.ts
@@ -0,0 +1,175 @@
+import fs = require('fs-extra');
+import { TabletSchema, TranslatedSnippetSchema, TranslationSchema, ORIGINAL_SNIPPET_KEY } from './schema';
+import { snippetKey } from './key';
+import { TargetLanguage } from '../languages';
+import { TypeScriptSnippet } from '../snippet';
+
+const TOOL_VERSION = require('../../package.json').version;
+
+export const DEFAULT_TABLET_NAME = '.jsii.tabl.json';
+
+/**
+ * A tablet containing various snippets in multiple languages
+ */
+export class LanguageTablet {
+  private readonly snippets: Record = {};
+
+  public addSnippet(snippet: TranslatedSnippet) {
+    const existingSnippet = this.snippets[snippet.key];
+    this.snippets[snippet.key] = existingSnippet ? existingSnippet.merge(snippet) : snippet;
+  }
+
+  public get snippetKeys() {
+    return Object.keys(this.snippets);
+  }
+
+  public tryGetSnippet(key: string): TranslatedSnippet | undefined {
+    return this.snippets[key];
+  }
+
+  public lookup(typeScriptSource: TypeScriptSnippet, language: TargetLanguage): Translation | undefined {
+    const snippet = this.snippets[snippetKey(typeScriptSource)];
+    return snippet && snippet.get(language);
+}
+
+  public async load(filename: string) {
+    const obj = await fs.readJson(filename, { encoding: 'utf-8' });
+
+    if (!obj.toolVersion || !obj.snippets) {
+      throw new Error(`File '${filename}' does not seem to be a Tablet file`);
+    }
+    if (obj.toolVersion !== TOOL_VERSION) {
+      throw new Error(`Tablet file '${filename}' has been created with version '${obj.toolVersion}', cannot read with current version '${TOOL_VERSION}'`);
+    }
+
+    Object.assign(this.snippets, mapValues(obj.snippets, (schema: TranslatedSnippetSchema) => TranslatedSnippet.fromSchema(schema)));
+  }
+
+  public get count() {
+    return Object.keys(this.snippets).length;
+  }
+
+  public async save(filename: string) {
+    await fs.writeJson(filename, this.toSchema(), { encoding: 'utf-8', spaces: 2 });
+  }
+
+  private toSchema(): TabletSchema {
+    return {
+      version: '1',
+      toolVersion: TOOL_VERSION,
+      snippets: mapValues(this.snippets, s => s.toSchema())
+    };
+  }
+}
+
+export class TranslatedSnippet {
+  public static fromSchema(schema: TranslatedSnippetSchema) {
+    const ret = new TranslatedSnippet();
+    Object.assign(ret.translations, schema.translations);
+    ret._didCompile = schema.didCompile;
+    ret._where = schema.where;
+    return ret;
+  }
+
+  public static fromSnippet(original: TypeScriptSnippet, didCompile?: boolean) {
+    const ret = new TranslatedSnippet();
+    Object.assign(ret.translations, { [ORIGINAL_SNIPPET_KEY]: { source: original.visibleSource }});
+    ret._didCompile = didCompile;
+    ret._where = original.where;
+    return ret;
+  }
+
+  private readonly translations: Record = {};
+  private _key?: string;
+  private _didCompile?: boolean;
+  private _where: string = '';
+
+  private constructor() {
+  }
+
+  public get didCompile() {
+    return this._didCompile;
+  }
+
+  public get where() {
+    return this._where;
+  }
+
+  public get key() {
+    if (this._key === undefined) {
+      this._key = snippetKey(this.asTypescriptSnippet());
+    }
+    return this._key;
+  }
+
+  public asTypescriptSnippet(): TypeScriptSnippet {
+    return {
+      visibleSource: this.translations[ORIGINAL_SNIPPET_KEY].source,
+      where: this.where,
+    };
+  }
+
+  public get originalSource(): Translation {
+    return {
+      source: this.translations[ORIGINAL_SNIPPET_KEY].source,
+      language: 'typescript',
+      didCompile: this.didCompile
+    };
+  }
+
+  public addTranslatedSource(language: TargetLanguage, translation: string): Translation {
+    this.translations[language] = { source: translation };
+
+    return {
+      source: translation,
+      language,
+      didCompile: this.didCompile
+    };
+  }
+
+  public get languages(): TargetLanguage[] {
+    return Object.keys(this.translations).filter(x => x !== ORIGINAL_SNIPPET_KEY) as TargetLanguage[];
+  }
+
+  public get(language: TargetLanguage): Translation | undefined {
+    const t = this.translations[language];
+    return t && { source: t.source, language, didCompile: this.didCompile };
+  }
+
+  public merge(other: TranslatedSnippet) {
+    const ret = new TranslatedSnippet();
+    Object.assign(ret.translations, this.translations, other.translations);
+    ret._didCompile = this.didCompile;
+    ret._where = this.where;
+    return ret;
+  }
+
+  public toTypeScriptSnippet() {
+    return {
+      source: this.originalSource,
+      where: this.where
+    };
+  }
+
+  public toSchema(): TranslatedSnippetSchema {
+    return {
+      translations: this.translations,
+      didCompile: this.didCompile,
+      where: this.where
+    }
+  }
+}
+
+export interface Translation {
+  source: string;
+  language: string;
+  didCompile?: boolean;
+}
+
+function mapValues(xs: Record, fn: (x: A) => B): Record {
+  const ret: Record = {};
+  for (const [key, value] of Object.entries(xs)) {
+    ret[key] = fn(value);
+  }
+  return ret;
+}
\ No newline at end of file
diff --git a/packages/jsii-rosetta/lib/translate.ts b/packages/jsii-rosetta/lib/translate.ts
new file mode 100644
index 0000000000..5f0f76c156
--- /dev/null
+++ b/packages/jsii-rosetta/lib/translate.ts
@@ -0,0 +1,132 @@
+import logging = require('./logging');
+import ts = require('typescript');
+import { AstRenderer, AstHandler, AstRendererOptions } from './renderer';
+import { renderTree, Span, spanContains } from './o-tree';
+import { TypeScriptCompiler, CompilationResult } from './typescript/ts-compiler';
+import { TranslatedSnippet } from './tablets/tablets';
+import { TARGET_LANGUAGES, TargetLanguage } from './languages';
+import { calculateVisibleSpans } from './typescript/ast-utils';
+import { File } from './util';
+import { TypeScriptSnippet, completeSource, SnippetParameters } from './snippet';
+import { snippetKey } from './tablets/key';
+
+export function translateTypeScript(source: File, visitor: AstHandler, options: SnippetTranslatorOptions = {}): TranslateResult {
+  const translator = new SnippetTranslator({ visibleSource: source.contents, where: source.fileName }, options);
+  const translated = translator.renderUsing(visitor);
+
+  return {
+    translation: translated,
+    diagnostics: translator.diagnostics,
+  };
+}
+
+
+/**
+ * Translate one or more TypeScript snippets into other languages
+ *
+ * Can be configured to fully typecheck the samples, or perform only syntactical
+ * translation.
+ */
+export class Translator {
+  private readonly compiler = new TypeScriptCompiler();
+  public readonly diagnostics: ts.Diagnostic[] = [];
+
+  constructor(private readonly includeCompilerDiagnostics: boolean) {
+  }
+
+  public translate(snip: TypeScriptSnippet, languages = Object.keys(TARGET_LANGUAGES) as TargetLanguage[]) {
+    logging.debug(`Translating ${snippetKey(snip)} ${Object.entries(snip.parameters || {})}`);
+    const translator = this.translatorFor(snip);
+    const snippet = TranslatedSnippet.fromSnippet(snip, this.includeCompilerDiagnostics ? translator.compileDiagnostics.length === 0 : undefined);
+
+    for (const lang of languages) {
+      const languageConverterFactory = TARGET_LANGUAGES[lang];
+      const translated = translator.renderUsing(languageConverterFactory());
+      snippet.addTranslatedSource(lang, translated);
+    }
+
+    this.diagnostics.push(...translator.diagnostics);
+
+    return snippet;
+  }
+
+  /**
+   * Return the snippet translator for the given snippet
+   *
+   * We used to cache these, but each translator holds on to quite a bit of memory,
+   * so we don't do that anymore.
+   */
+  public translatorFor(snippet: TypeScriptSnippet) {
+    const translator = new SnippetTranslator(snippet, {
+      compiler: this.compiler,
+      includeCompilerDiagnostics: this.includeCompilerDiagnostics,
+    });
+    return translator;
+  }
+}
+
+export interface SnippetTranslatorOptions extends AstRendererOptions {
+  /**
+   * Re-use the given compiler if given
+   */
+  readonly compiler?: TypeScriptCompiler;
+
+  /**
+   * Include compiler errors in return diagnostics
+   *
+   * If false, only translation diagnostics will be returned.
+   *
+   * @default false
+   */
+  readonly includeCompilerDiagnostics?: boolean;
+}
+
+export interface TranslateResult {
+  translation: string;
+  diagnostics: ts.Diagnostic[];
+}
+
+/**
+ * Translate a single TypeScript snippet
+ */
+export class SnippetTranslator {
+  public readonly translateDiagnostics: ts.Diagnostic[] = [];
+  public readonly compileDiagnostics: ts.Diagnostic[] = [];
+  private readonly visibleSpans: Span[];
+  private compilation!: CompilationResult;
+
+  constructor(snippet: TypeScriptSnippet, private readonly options: SnippetTranslatorOptions = {}) {
+    const compiler = options.compiler || new TypeScriptCompiler();
+    const source = completeSource(snippet);
+
+    const fakeCurrentDirectory = snippet.parameters && snippet.parameters[SnippetParameters.$COMPILATION_DIRECTORY];
+    this.compilation = compiler.compileInMemory(snippet.where, source, fakeCurrentDirectory);
+
+    // Respect '/// !hide' and '/// !show' directives
+    this.visibleSpans = calculateVisibleSpans(source);
+
+    // This makes it about 5x slower, so only do it on demand
+    if (options.includeCompilerDiagnostics) {
+      const program = this.compilation.program;
+      this.compileDiagnostics.push(...program.getGlobalDiagnostics(), ...program.getSyntacticDiagnostics(), ...program.getDeclarationDiagnostics(), ...program.getSemanticDiagnostics());
+    }
+  }
+
+  public renderUsing(visitor: AstHandler) {
+    const converter = new AstRenderer(this.compilation.rootFile, this.compilation.program.getTypeChecker(), visitor, this.options);
+    const converted = converter.convert(this.compilation.rootFile);
+    this.translateDiagnostics.push(...filterVisibleDiagnostics(converter.diagnostics, this.visibleSpans));
+    return renderTree(converted, { visibleSpans: this.visibleSpans });
+  }
+
+  public get diagnostics() {
+    return [...this.compileDiagnostics, ...this.translateDiagnostics];
+  }
+}
+
+/**
+ * Hide diagnostics that are rosetta-sourced if they are reported against a non-visible span
+ */
+function filterVisibleDiagnostics(diags: ts.Diagnostic[], visibleSpans: Span[]): ts.Diagnostic[] {
+  return diags.filter(d => d.source !== 'rosetta' || d.start === undefined || visibleSpans.some(s => spanContains(s, d.start!)));
+}
\ No newline at end of file
diff --git a/packages/jsii-sampiler/lib/typescript/ast-utils.ts b/packages/jsii-rosetta/lib/typescript/ast-utils.ts
similarity index 85%
rename from packages/jsii-sampiler/lib/typescript/ast-utils.ts
rename to packages/jsii-rosetta/lib/typescript/ast-utils.ts
index 95cefee736..8b95a81373 100644
--- a/packages/jsii-sampiler/lib/typescript/ast-utils.ts
+++ b/packages/jsii-rosetta/lib/typescript/ast-utils.ts
@@ -1,4 +1,46 @@
 import ts = require('typescript');
+import { Span } from '../o-tree';
+
+export interface MarkedSpan {
+  start: number;
+  end: number;
+  visible: boolean;
+}
+
+export function calculateVisibleSpans(source: string): Span[] {
+  return calculateMarkedSpans(source).filter(s => s.visible);
+}
+
+export function calculateMarkedSpans(source: string): MarkedSpan[] {
+  const regEx = /\/\/\/ (.*)(\r?\n)?$/gm;
+
+  const ret = new Array();
+  let match;
+  let spanStart;
+  let visible = true;
+  while ((match = regEx.exec(source)) != null) {
+    const directiveStart = match.index;
+    const directive = match[1].trim();
+    if (['!hide', '!show'].includes(directive)) {
+      const isShow = directive === '!show';
+      if (spanStart === undefined) {
+        // Add a span at the start which is the reverse of the actual first directive
+        ret.push({ start: 0, end: directiveStart, visible: !isShow });
+      } else {
+        // Else add a span for the current directive
+        ret.push({ start: spanStart, end: directiveStart, visible });
+      }
+      visible = isShow;
+      spanStart = match.index + match[0].length;
+    }
+  }
+
+  // Add the remainder under the last visibility
+  ret.push({ start: spanStart || 0, end: source.length, visible });
+
+  // Filter empty spans and return
+  return ret.filter(s => s.start !== s.end);
+}
 
 export function stripCommentMarkers(comment: string, multiline: boolean) {
   if (multiline) {
@@ -187,7 +229,7 @@ export function commentRangeFromTextRange(rng: TextRange): ts.CommentRange {
 interface TextRange {
   pos: number;
   end: number;
-  type: 'linecomment' | 'blockcomment' | 'other';
+  type: 'linecomment' | 'blockcomment' | 'other' | 'directive';
   hasTrailingNewLine: boolean;
 }
 
@@ -247,12 +289,24 @@ export function scanText(text: string, start: number, end?: number): TextRange[]
 
   function scanSinglelineComment() {
     const nl = Math.min(findNext('\r', pos + 2), findNext('\n', pos + 2));
-    ret.push({
-      type: 'linecomment',
-      hasTrailingNewLine: true,
-      pos,
-      end: nl
-    });
+
+    if (text[pos + 2] === '/') {
+      // Special /// comment
+      ret.push({
+        type: 'directive',
+        hasTrailingNewLine: true,
+        pos: pos + 1,
+        end: nl
+      });
+    } else {
+      // Regular // comment
+      ret.push({
+        type: 'linecomment',
+        hasTrailingNewLine: true,
+        pos,
+        end: nl
+      });
+    }
     pos = nl + 1;
     start = pos;
   }
diff --git a/packages/jsii-sampiler/lib/typescript/imports.ts b/packages/jsii-rosetta/lib/typescript/imports.ts
similarity index 93%
rename from packages/jsii-sampiler/lib/typescript/imports.ts
rename to packages/jsii-rosetta/lib/typescript/imports.ts
index 55970f355c..f3a5aa7c17 100644
--- a/packages/jsii-sampiler/lib/typescript/imports.ts
+++ b/packages/jsii-rosetta/lib/typescript/imports.ts
@@ -1,5 +1,5 @@
 import ts = require('typescript');
-import { AstConverter } from '../converter';
+import { AstRenderer } from '../renderer';
 import { allOfType, matchAst, nodeOfType, stringFromLiteral } from "./ast-utils";
 
 /**
@@ -19,7 +19,7 @@ export interface ImportBinding {
   alias?: string;
 }
 
-export function analyzeImportEquals(node: ts.ImportEqualsDeclaration, context: AstConverter): ImportStatement {
+export function analyzeImportEquals(node: ts.ImportEqualsDeclaration, context: AstRenderer): ImportStatement {
   let moduleName = '???';
   matchAst(node.moduleReference,
     nodeOfType('ref', ts.SyntaxKind.ExternalModuleReference),
@@ -34,7 +34,7 @@ export function analyzeImportEquals(node: ts.ImportEqualsDeclaration, context: A
   };
 }
 
-export function analyzeImportDeclaration(node: ts.ImportDeclaration, context: AstConverter): ImportStatement {
+export function analyzeImportDeclaration(node: ts.ImportDeclaration, context: AstRenderer): ImportStatement {
   const packageName = stringFromLiteral(node.moduleSpecifier);
 
   const starBindings = matchAst(node,
diff --git a/packages/jsii-sampiler/lib/typescript/ts-compiler.ts b/packages/jsii-rosetta/lib/typescript/ts-compiler.ts
similarity index 84%
rename from packages/jsii-sampiler/lib/typescript/ts-compiler.ts
rename to packages/jsii-rosetta/lib/typescript/ts-compiler.ts
index 74397c1250..4889f72824 100644
--- a/packages/jsii-sampiler/lib/typescript/ts-compiler.ts
+++ b/packages/jsii-rosetta/lib/typescript/ts-compiler.ts
@@ -7,14 +7,14 @@ export class TypeScriptCompiler {
     this.realHost = ts.createCompilerHost(STANDARD_COMPILER_OPTIONS, true);
   }
 
-  public createInMemoryCompilerHost(sourcePath: string, sourceContents: string): ts.CompilerHost {
+  public createInMemoryCompilerHost(sourcePath: string, sourceContents: string, currentDirectory?: string): ts.CompilerHost {
     const realHost = this.realHost;
     const sourceFile = ts.createSourceFile(sourcePath, sourceContents, ts.ScriptTarget.Latest);
 
     return {
       fileExists: filePath => filePath === sourcePath || realHost.fileExists(filePath),
       directoryExists: realHost.directoryExists && realHost.directoryExists.bind(realHost),
-      getCurrentDirectory: realHost.getCurrentDirectory.bind(realHost),
+      getCurrentDirectory: () => currentDirectory || realHost.getCurrentDirectory(),
       getDirectories: realHost.getDirectories && realHost.getDirectories.bind(realHost),
       getCanonicalFileName: fileName => realHost.getCanonicalFileName(fileName),
       getNewLine: realHost.getNewLine.bind(realHost),
@@ -30,7 +30,7 @@ export class TypeScriptCompiler {
     };
   }
 
-  public compileInMemory(filename: string, contents: string): CompilationResult {
+  public compileInMemory(filename: string, contents: string, currentDirectory?: string): CompilationResult {
     if (!filename.endsWith('.ts')) {
       // Necessary or the TypeScript compiler won't compile the file.
       filename += '.ts';
@@ -39,7 +39,7 @@ export class TypeScriptCompiler {
     const program = ts.createProgram({
       rootNames: [filename],
       options: STANDARD_COMPILER_OPTIONS,
-      host: this.createInMemoryCompilerHost(filename, contents),
+      host: this.createInMemoryCompilerHost(filename, contents, currentDirectory),
     });
 
     const rootFiles = program.getSourceFiles().filter(f => f.fileName === filename);
@@ -71,8 +71,8 @@ export const STANDARD_COMPILER_OPTIONS: ts.CompilerOptions = {
   noImplicitAny: true,
   noImplicitReturns: true,
   noImplicitThis: true,
-  noUnusedLocals: true,
-  noUnusedParameters: true,
+  noUnusedLocals: false,  // Important, becomes super annoying without this
+  noUnusedParameters: false, // Important, becomes super annoying without this
   resolveJsonModule: true,
   strict: true,
   strictNullChecks: true,
diff --git a/packages/jsii-rosetta/lib/util.ts b/packages/jsii-rosetta/lib/util.ts
new file mode 100644
index 0000000000..ee9d8c5a0a
--- /dev/null
+++ b/packages/jsii-rosetta/lib/util.ts
@@ -0,0 +1,52 @@
+import { VisualizeAstVisitor } from './languages/visualize';
+import ts = require('typescript');
+import { translateTypeScript } from './translate';
+
+export function startsWithUppercase(x: string) {
+  return x.match(/^[A-Z]/);
+}
+
+export interface File {
+  readonly contents: string;
+  readonly fileName: string;
+}
+
+export function visualizeTypeScriptAst(source: File) {
+  const vis = translateTypeScript(source, new VisualizeAstVisitor(true), {
+    bestEffort: false
+  });
+  return vis.translation + '\n';
+}
+
+export function printDiagnostics(diags: ts.Diagnostic[], stream: NodeJS.WritableStream) {
+  diags.forEach(d => printDiagnostic(d, stream));
+}
+
+export function printDiagnostic(diag: ts.Diagnostic, stream: NodeJS.WritableStream) {
+  const host = {
+    getCurrentDirectory() { return '.'; },
+    getCanonicalFileName(fileName: string) { return fileName; },
+    getNewLine() { return '\n'; }
+  };
+
+  const message = ts.formatDiagnosticsWithColorAndContext([diag], host);
+  stream.write(message);
+}
+
+export function isErrorDiagnostic(diag: ts.Diagnostic) {
+  return diag.category === ts.DiagnosticCategory.Error;
+}
+
+/**
+ * Chunk an array of elements into approximately equal groups
+ */
+export function divideEvenly(groups: number, xs: A[]): A[][] {
+  const chunkSize = Math.ceil(xs.length / groups);
+  const ret: A[][] = [];
+
+  for (let i = 0; i < groups; i++) {
+    ret.push(xs.slice(i * chunkSize, (i + 1) * chunkSize));
+  }
+
+  return ret;
+}
\ No newline at end of file
diff --git a/packages/jsii-sampiler/package.json b/packages/jsii-rosetta/package.json
similarity index 76%
rename from packages/jsii-sampiler/package.json
rename to packages/jsii-rosetta/package.json
index a39e4c68f0..b7b1e6c01e 100644
--- a/packages/jsii-sampiler/package.json
+++ b/packages/jsii-rosetta/package.json
@@ -1,10 +1,10 @@
 {
-  "name": "jsii-sampiler",
+  "name": "jsii-rosetta",
   "version": "0.20.5",
   "description": "Translate TypeScript code snippets to other languages",
   "main": "lib/index.js",
   "bin": {
-    "jsii-sampiler": "bin/jsii-sampiler"
+    "jsii-rosetta": "bin/jsii-rosetta"
   },
   "scripts": {
     "build": "tsc --build",
@@ -21,12 +21,16 @@
     "@types/yargs": "^13.0.3",
     "jest": "^24.9.0",
     "jsii-build-tools": "^0.20.5",
-    "memory-streams": "^0.1.3"
+    "jsii": "^0.20.5",
+    "memory-streams": "^0.1.3",
+    "mock-fs": "^4.10.2",
+    "@types/mock-fs": "^4.10.0"
   },
   "dependencies": {
     "commonmark": "^0.29.0",
     "fs-extra": "^8.1.0",
     "typescript": "~3.6.4",
+    "jsii-spec": "^0.20.0",
     "yargs": "^14.2.0"
   },
   "jest": {
@@ -37,10 +41,11 @@
       "lib/**/*.js"
     ],
     "collectCoverage": true,
+    "coverageReporters": ["json", "lcov", "text", "clover", "html"],
     "coverageThreshold": {
       "global": {
         "branches": 70,
-        "statements": 75
+        "statements": 70
       }
     }
   },
@@ -54,9 +59,9 @@
   "repository": {
     "type": "git",
     "url": "https://github.com/aws/jsii.git",
-    "directory": "packages/jsii-sampiler"
+    "directory": "packages/jsii-rosetta"
   },
   "engines": {
-    "node": ">= 10.3.0"
+    "node": ">= 10.5.0"
   }
 }
diff --git a/packages/jsii-rosetta/test/jsii/assemblies.test.ts b/packages/jsii-rosetta/test/jsii/assemblies.test.ts
new file mode 100644
index 0000000000..37731c0d58
--- /dev/null
+++ b/packages/jsii-rosetta/test/jsii/assemblies.test.ts
@@ -0,0 +1,152 @@
+import mockfs = require('mock-fs');
+import spec = require('jsii-spec');
+import { allTypeScriptSnippets } from '../../lib/jsii/assemblies';
+import path = require('path');
+import { SnippetParameters } from '../../lib/snippet';
+
+test('Extract snippet from README', () => {
+  const snippets = Array.from(allTypeScriptSnippets([{
+    assembly: fakeAssembly({
+      readme: {
+        markdown: [
+          'Before the example.',
+          '```ts',
+          'someExample();',
+          '```',
+          'After the example.'
+        ].join('\n')
+      }
+    }),
+    directory: path.join(__dirname, 'fixtures'),
+  }]));
+
+  expect(snippets[0].visibleSource).toEqual('someExample();');
+});
+
+test('Extract snippet from type docstring', () => {
+  const snippets = Array.from(allTypeScriptSnippets([{
+    assembly: fakeAssembly({
+      types: {
+        'asm.MyType': {
+          kind: spec.TypeKind.Class,
+          assembly: 'asm',
+          fqn: 'asm.MyType',
+          name: 'MyType',
+          docs: {
+            summary: 'My Type',
+            remarks: [
+              'Before the example.',
+              '```ts',
+              'someExample();',
+              '```',
+              'After the example.'
+            ].join('\n'),
+          }
+        },
+      }
+    }),
+    directory: path.join(__dirname, 'fixtures'),
+  }]));
+
+  expect(snippets[0].visibleSource).toEqual('someExample();');
+});
+
+test('Snippet can include fixture', () => {
+  const snippets = Array.from(allTypeScriptSnippets([{
+    assembly: fakeAssembly({
+      readme: {
+        markdown: [
+          'Before the example.',
+          '```ts fixture=explicit',
+          'someExample();',
+          '```',
+          'After the example.'
+        ].join('\n')
+      }
+    }),
+    directory: path.join(__dirname, 'fixtures'),
+  }]));
+
+  expect(snippets[0].visibleSource).toEqual('someExample();');
+  expect(snippets[0].completeSource).toEqual([
+    '// This is a fixture',
+    '/// !show',
+    'someExample();',
+    '/// !hide',
+  ].join('\n'));
+});
+
+test('Use fixture from example', () => {
+  const snippets = Array.from(allTypeScriptSnippets([{
+    assembly: fakeAssembly({
+      types: {
+        'asm.MyType': {
+          kind: spec.TypeKind.Class,
+          assembly: 'asm',
+          fqn: 'asm.MyType',
+          name: 'MyType',
+          docs: {
+            example: [
+              '/// fixture=explicit',
+              'someExample();',
+            ].join('\n'),
+          }
+        },
+      }
+    }),
+    directory: path.join(__dirname, 'fixtures'),
+  }]));
+
+  expect(snippets[0].visibleSource).toEqual('someExample();');
+  expect(snippets[0].completeSource).toEqual([
+    '// This is a fixture',
+    '/// !show',
+    'someExample();',
+    '/// !hide',
+  ].join('\n'));
+});
+
+
+test('Backwards compatibility with literate integ tests', () => {
+  mockfs({
+    '/package/test/integ.example.lit.ts': '# Some literate source file'
+  });
+
+  try {
+    const snippets = Array.from(allTypeScriptSnippets([{
+      assembly: fakeAssembly({
+        readme: {
+          markdown: [
+            'Before the example.',
+            '```ts lit=test/integ.example.lit.ts',
+            'someExample();',
+            '```',
+            'After the example.'
+          ].join('\n')
+        }
+      }),
+      directory: '/package'
+    }]));
+
+    expect(snippets[0].visibleSource).toEqual('someExample();');
+    expect(snippets[0].completeSource).toEqual('# Some literate source file');
+    expect(snippets[0].parameters && snippets[0].parameters[SnippetParameters.$COMPILATION_DIRECTORY]).toEqual('/package/test');
+  } finally {
+    mockfs.restore();
+  }
+});
+
+export function fakeAssembly(parts: Partial): spec.Assembly {
+  return Object.assign({
+    schema: spec.SchemaVersion.LATEST,
+    name: '',
+    description: '',
+    homepage: '',
+    repository: { directory: '', type: '', url: '' },
+    author: { email: '', name: '', organization: false, roles: [], url: '' },
+    fingerprint: '',
+    version: '',
+    jsiiVersion: '',
+    license: '',
+  }, parts);
+}
\ No newline at end of file
diff --git a/packages/jsii-rosetta/test/jsii/astutils.test.ts b/packages/jsii-rosetta/test/jsii/astutils.test.ts
new file mode 100644
index 0000000000..43cc2d0635
--- /dev/null
+++ b/packages/jsii-rosetta/test/jsii/astutils.test.ts
@@ -0,0 +1,19 @@
+import { calculateVisibleSpans } from "../../lib/typescript/ast-utils";
+
+test('full text visible by default', () => {
+  expect(calculateVisibleSpans('asdf')).toEqual([
+    { start: 0, end: 4, visible: true }
+  ]);
+});
+
+test('initial span visible if directive is hiding', () => {
+  expect(calculateVisibleSpans('asdf\n/// !hide\nxyz')).toEqual([
+    { start: 0, end: 5, visible: true }
+  ]);
+});
+
+test('initial span invisible if directive is showing', () => {
+  expect(calculateVisibleSpans('asdf\n/// !show\nxyz')).toEqual([
+    { start: 14, end: 18, visible: true }
+  ]);
+});
diff --git a/packages/jsii-rosetta/test/jsii/fixtures/rosetta/explicit.ts-fixture b/packages/jsii-rosetta/test/jsii/fixtures/rosetta/explicit.ts-fixture
new file mode 100644
index 0000000000..d87a43e2f4
--- /dev/null
+++ b/packages/jsii-rosetta/test/jsii/fixtures/rosetta/explicit.ts-fixture
@@ -0,0 +1,2 @@
+// This is a fixture
+/// here
\ No newline at end of file
diff --git a/packages/jsii-sampiler/test/markdown/roundtrip.test.ts b/packages/jsii-rosetta/test/markdown/roundtrip.test.ts
similarity index 95%
rename from packages/jsii-sampiler/test/markdown/roundtrip.test.ts
rename to packages/jsii-rosetta/test/markdown/roundtrip.test.ts
index 5307b2576f..b23ca8f8e2 100644
--- a/packages/jsii-sampiler/test/markdown/roundtrip.test.ts
+++ b/packages/jsii-rosetta/test/markdown/roundtrip.test.ts
@@ -152,6 +152,15 @@ test('headings', () => {
   `);
 });
 
+test('HTML comments', () => {
+  expectOutput(`
+
+  `, `
+
+  `);
+});
+
+
 function expectOutput(source: string, expected: string) {
   if (DEBUG) {
     const struct = new StructureRenderer();
diff --git a/packages/jsii-sampiler/test/otree.test.ts b/packages/jsii-rosetta/test/otree.test.ts
similarity index 100%
rename from packages/jsii-sampiler/test/otree.test.ts
rename to packages/jsii-rosetta/test/otree.test.ts
diff --git a/packages/jsii-sampiler/test/python/calls.test.ts b/packages/jsii-rosetta/test/python/calls.test.ts
similarity index 100%
rename from packages/jsii-sampiler/test/python/calls.test.ts
rename to packages/jsii-rosetta/test/python/calls.test.ts
diff --git a/packages/jsii-sampiler/test/python/classes.test.ts b/packages/jsii-rosetta/test/python/classes.test.ts
similarity index 100%
rename from packages/jsii-sampiler/test/python/classes.test.ts
rename to packages/jsii-rosetta/test/python/classes.test.ts
diff --git a/packages/jsii-sampiler/test/python/comments.test.ts b/packages/jsii-rosetta/test/python/comments.test.ts
similarity index 100%
rename from packages/jsii-sampiler/test/python/comments.test.ts
rename to packages/jsii-rosetta/test/python/comments.test.ts
diff --git a/packages/jsii-sampiler/test/python/expressions.test.ts b/packages/jsii-rosetta/test/python/expressions.test.ts
similarity index 100%
rename from packages/jsii-sampiler/test/python/expressions.test.ts
rename to packages/jsii-rosetta/test/python/expressions.test.ts
diff --git a/packages/jsii-sampiler/test/python/hiding.test.ts b/packages/jsii-rosetta/test/python/hiding.test.ts
similarity index 76%
rename from packages/jsii-sampiler/test/python/hiding.test.ts
rename to packages/jsii-rosetta/test/python/hiding.test.ts
index 76dd0e1a7b..cfd32ac0ca 100644
--- a/packages/jsii-sampiler/test/python/hiding.test.ts
+++ b/packages/jsii-rosetta/test/python/hiding.test.ts
@@ -57,4 +57,25 @@ test('hide statements with explicit ellipsis', () => {
   # ...
   after()
   `);
+});
+
+test('hide halfway into class using comments', () => {
+  expectPython(`
+  prepare();
+
+  /// !hide
+  class Something {
+    constructor() {
+
+      /// !show
+      console.log(this, 'it seems to work');
+      /// !hide
+    }
+  }
+  `, `
+  prepare()
+
+  print(self, "it seems to work")
+  `
+  );
 });
\ No newline at end of file
diff --git a/packages/jsii-sampiler/test/python/imports.test.ts b/packages/jsii-rosetta/test/python/imports.test.ts
similarity index 100%
rename from packages/jsii-sampiler/test/python/imports.test.ts
rename to packages/jsii-rosetta/test/python/imports.test.ts
diff --git a/packages/jsii-sampiler/test/python/misc.test.ts b/packages/jsii-rosetta/test/python/misc.test.ts
similarity index 100%
rename from packages/jsii-sampiler/test/python/misc.test.ts
rename to packages/jsii-rosetta/test/python/misc.test.ts
diff --git a/packages/jsii-sampiler/test/python/python.ts b/packages/jsii-rosetta/test/python/python.ts
similarity index 87%
rename from packages/jsii-sampiler/test/python/python.ts
rename to packages/jsii-rosetta/test/python/python.ts
index 4a92989a10..6ec30d230e 100644
--- a/packages/jsii-sampiler/test/python/python.ts
+++ b/packages/jsii-rosetta/test/python/python.ts
@@ -1,11 +1,11 @@
-import { LiteralSource, renderTree, translateTypeScript } from "../../lib";
+import { translateTypeScript } from "../../lib";
 import { PythonVisitor } from "../../lib/languages/python";
 import { visualizeTypeScriptAst } from "../../lib/util";
 
 const DEBUG = false;
 
 export function ts2python(source: string): string {
-  const src = new LiteralSource(source, 'test.ts');
+  const src = { contents: source, fileName: 'test.ts' };
 
   if (DEBUG) {
     // tslint:disable-next-line:no-console
@@ -15,8 +15,7 @@ export function ts2python(source: string): string {
 
   // Very debug. Much print.
   // console.log(JSON.stringify(result.tree, undefined, 2));
-
-  return renderTree(result.tree) + '\n';
+  return result.translation + '\n';
 }
 
 export function expectPython(source: string, expected: string) {
diff --git a/packages/jsii-sampiler/test/python/statements.test.ts b/packages/jsii-rosetta/test/python/statements.test.ts
similarity index 88%
rename from packages/jsii-sampiler/test/python/statements.test.ts
rename to packages/jsii-rosetta/test/python/statements.test.ts
index 99d1b610bd..015ad3e12f 100644
--- a/packages/jsii-sampiler/test/python/statements.test.ts
+++ b/packages/jsii-rosetta/test/python/statements.test.ts
@@ -1,5 +1,5 @@
 import { expectPython } from "./python";
-import { LiteralSource, PythonVisitor, translateTypeScript, renderTree } from "../../lib";
+import { PythonVisitor, translateTypeScript } from "../../lib";
 
 test('if', () => {
   expectPython(`
@@ -101,13 +101,13 @@ test('whitespace between statements in a block', () => {
 });
 
 test('prepend disclaimer', () => {
-  const src = new LiteralSource('console.log("hello");', 'test.ts');
+  const src = { contents: 'console.log("hello");', fileName: 'test.ts' };
 
   const result = translateTypeScript(src, new PythonVisitor({
     disclaimer: 'Do not write this code'
   }));
 
-  expect(renderTree(result.tree)).toEqual(
+  expect(result.translation).toEqual(
 `# Do not write this code
 print("hello")`);
 });
\ No newline at end of file
diff --git a/packages/jsii-rosetta/test/rosetta.test.ts b/packages/jsii-rosetta/test/rosetta.test.ts
new file mode 100644
index 0000000000..7aacb2765b
--- /dev/null
+++ b/packages/jsii-rosetta/test/rosetta.test.ts
@@ -0,0 +1,151 @@
+import { Rosetta, LanguageTablet, TranslatedSnippet, TypeScriptSnippet, DEFAULT_TABLET_NAME } from '../lib';
+import mockfs = require('mock-fs');
+import { TargetLanguage } from '../lib/languages';
+import { fakeAssembly } from './jsii/assemblies.test';
+
+const SAMPLE_CODE: TypeScriptSnippet = {
+  visibleSource: 'callThisFunction();',
+  where: 'sample',
+};
+
+test('Rosetta object can do live translation', () => {
+  // GIVEN
+  const rosetta = new Rosetta({
+    liveConversion: true,
+    targetLanguages: ["python"]
+  });
+
+  // WHEN
+  const translated = rosetta.translateSnippet(SAMPLE_CODE, "python");
+
+  // THEN
+  expect(translated).toMatchObject({
+    source: "call_this_function()",
+    language: "python"
+  });
+});
+
+test('Can use preloaded tablet', () => {
+  // GIVEN
+  const rosetta = new Rosetta();
+
+  const tablet = new LanguageTablet();
+  tablet.addSnippet(makeSnippet(SAMPLE_CODE, {
+    python: 'Not Really Translated'
+  }));
+  rosetta.addTablet(tablet);
+
+  // WHEN
+  const translated = rosetta.translateSnippet(SAMPLE_CODE, "python");
+
+  // THEN
+  expect(translated).toMatchObject({
+    source: "Not Really Translated",
+    language: "python"
+  });
+});
+
+test('Rosetta object can do live translation', () => {
+  // GIVEN
+  const rosetta = new Rosetta({
+    liveConversion: true,
+    targetLanguages: ["python"]
+  });
+
+  // WHEN
+  const translated = rosetta.translateSnippet(SAMPLE_CODE, "python");
+
+  // THEN
+  expect(translated).toMatchObject({
+    source: "call_this_function()",
+    language: "python"
+  });
+});
+
+
+test('Rosetta object can do translation and annotation of snippets in MarkDown', () => {
+  // GIVEN
+  const rosetta = new Rosetta({
+    liveConversion: true,
+    targetLanguages: ["python"]
+  });
+
+  // WHEN
+  const translated = rosetta.translateSnippetsInMarkdown([
+    '# MarkDown Translation',
+    '',
+    'Now follows a snippet:',
+    '```ts',
+    SAMPLE_CODE.visibleSource,
+    '```',
+    'That was it, thank you for your attention.'
+  ].join('\n'), 'python', trans => {
+    return { ...trans, source: '# We translated something!\n' + trans.source };
+  });
+
+  // THEN
+  expect(translated).toEqual([
+    '# MarkDown Translation',
+    '',
+    'Now follows a snippet:',
+    '',
+    '```python',
+    '# We translated something!',
+    'call_this_function()',
+    '```',
+    '',
+    'That was it, thank you for your attention.'
+  ].join('\n'));
+});
+
+describe('with mocked filesystem', () => {
+  beforeEach(() => { mockfs(); });
+  afterEach(() => { mockfs.restore(); });
+
+  const tablet = new LanguageTablet();
+  tablet.addSnippet(makeSnippet(SAMPLE_CODE, {
+    python: 'My Stored Translation'
+  }));
+
+  test('Can save language tablet and load it in Rosetta class', async () => {
+    // GIVEN
+    await tablet.save('/test.tablet');
+
+    // WHEN
+    const rosetta = new Rosetta();
+    await rosetta.loadTabletFromFile('/test.tablet');
+    const translated = rosetta.translateSnippet(SAMPLE_CODE, "python");
+
+    // THEN
+    expect(translated).toMatchObject({
+      source: "My Stored Translation",
+      language: "python"
+    });
+  });
+
+  test('Rosetta class automatically loads default-named tablets in same directory as assembly', async () => {
+    // GIVEN
+    await tablet.save('/' + DEFAULT_TABLET_NAME);
+
+    // WHEN
+    const rosetta = new Rosetta();
+    await rosetta.addAssembly(fakeAssembly({}), '/');
+    const translated = rosetta.translateSnippet(SAMPLE_CODE, "python");
+
+    // THEN
+    expect(translated).toMatchObject({
+      source: "My Stored Translation",
+      language: "python"
+    });
+  });
+
+});
+
+
+function makeSnippet(original: TypeScriptSnippet, translations: Record) {
+  const snippet = TranslatedSnippet.fromSnippet(original);
+  for (const [key, value] of Object.entries(translations)) {
+    snippet.addTranslatedSource(key as TargetLanguage, value);
+  }
+  return snippet;
+}
\ No newline at end of file
diff --git a/packages/jsii-sampiler/tsconfig.json b/packages/jsii-rosetta/tsconfig.json
similarity index 96%
rename from packages/jsii-sampiler/tsconfig.json
rename to packages/jsii-rosetta/tsconfig.json
index beb48c6fe6..77418e6ad0 100644
--- a/packages/jsii-sampiler/tsconfig.json
+++ b/packages/jsii-rosetta/tsconfig.json
@@ -21,5 +21,8 @@
   ],
   "exclude": [
     "examples"
+  ],
+  "references": [
+      { "path": "../jsii-spec" }
   ]
 }
diff --git a/packages/jsii-sampiler/CHANGELOG.md b/packages/jsii-sampiler/CHANGELOG.md
deleted file mode 100644
index 42e928587b..0000000000
--- a/packages/jsii-sampiler/CHANGELOG.md
+++ /dev/null
@@ -1,64 +0,0 @@
-# Change Log
-
-All notable changes to this project will be documented in this file.
-See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
-
-## [0.20.5](https://github.com/aws/jsii/compare/v0.20.4...v0.20.5) (2019-11-13)
-
-**Note:** Version bump only for package jsii-sampiler
-
-
-
-
-
-## [0.20.4](https://github.com/aws/jsii/compare/v0.20.3...v0.20.4) (2019-11-12)
-
-**Note:** Version bump only for package jsii-sampiler
-
-
-
-
-
-## [0.20.3](https://github.com/aws/jsii/compare/v0.20.2...v0.20.3) (2019-11-11)
-
-**Note:** Version bump only for package jsii-sampiler
-
-
-
-
-
-## [0.20.2](https://github.com/aws/jsii/compare/v0.20.1...v0.20.2) (2019-11-08)
-
-**Note:** Version bump only for package jsii-sampiler
-
-
-
-
-
-## [0.20.1](https://github.com/aws/jsii/compare/v0.20.0...v0.20.1) (2019-11-06)
-
-**Note:** Version bump only for package jsii-sampiler
-
-
-
-
-
-## [0.20.0](https://github.com/aws/jsii/compare/v0.19.0...v0.20.0) (2019-10-30)
-
-**Note:** Version bump only for package jsii-sampiler
-
-
-
-
-
-# [0.19.0](https://github.com/aws/jsii/compare/v0.18.0...v0.19.0) (2019-10-14)
-
-
-### Bug Fixes
-
-* **sampiler:** Add missing .npmignore ([#875](https://github.com/aws/jsii/issues/875)) ([b16fc6b](https://github.com/aws/jsii/commit/b16fc6bdaf1825d53629c2a44b769f924ffb91d0))
-
-
-### Features
-
-* **sampiler:** translate code samples to Python ([#827](https://github.com/aws/jsii/issues/827)) ([c9a7002](https://github.com/aws/jsii/commit/c9a7002431c0db6224d595eb5555b916036d4575))
diff --git a/packages/jsii-sampiler/README.md b/packages/jsii-sampiler/README.md
deleted file mode 100644
index c0ddf4580a..0000000000
--- a/packages/jsii-sampiler/README.md
+++ /dev/null
@@ -1,147 +0,0 @@
-# jsii-sampiler: a transpiler for code samples
-
-Utility to transcribe example code snippets from TypeScript to other
-jsii languages.
-
-Has knowledge about jsii language translation conventions to do the
-translations. Only supports a limited set of TypeScript language features.
-
-## Compilability
-
-The sampiler can translate both code that completely compiles and typechecks,
-as well as code that doesn't.
-
-In case of non-compiling samples the translations will be based off of
-grammatical parsing only. This has the downside that we do not have the type
-information available to the exact right thing in all instances.
-
-If the samples don't compile or don't have full type information:
-
-- No way to declare typed variables for Java and C#.
-- Can only "see" the fields of structs as far as they are declared in the same
-  snippet. Inherited fields or structs declared not in the same snippet are
-  invisible.
-- When we explode a struct parameter into keyword parameters and we pass it on
-  to another callable, we can't know which keyword arguments the called function
-  actually takes so we just pass all of them (might be too many).
-- When structs contain nested structs, Python and other languages need to know
-  the types of these fields to generate the right calls.
-- Object literals are used both to represent structs as well as to represent
-  dictionaries, and without type information it's impossible to determine
-  which is which.
-
-## Void masking
-
-In order to make examples compile, boilerplate code may need to be added
-that detracts from the example at hand (such as variable declarations
-and imports).
-
-This package supports hiding parts of the original source after
-translation.
-
-To mark special locations in the source tree, we use the `void`
-expression keyword and or the `comma` operator feature to attach this
-expression to another expression.  Both are little-used JavaScript
-features that are reliably parsed by TypeScript and do not affect the
-semantics of the application in which they appear (so the program
-executes the same with or without them).
-
-A handy mnemonic for this feature is that you can use it to "send your
-code into the void".
-
-### Hiding statements
-
-Statement hiding looks like this:
-
-```ts
-before();    // will be shown
-
-void 0;      // start hiding (the argument to 'void' doesn't matter)
-middle();    // will not be shown
-void 'show'; // stop hiding
-
-after();     // will be shown again
-```
-
-Void masking only works to the end of the enclosing scope, so in some
-cases you can omit the `void 'show'` directive to turn hiding back off.
-
-To explicit show that code was hidden, pass `'block'` to the void
-statement:
-
-
-```ts
-before();
-void 'block'; // start hiding, will render a '# ...'
-middle();
-```
-
-### Hiding expressions
-
-For hiding expressions, we use `comma` expressions to attach a `void`
-statement to an expression value without changing the meaning of the
-code.
-
-Example:
-
-```ts
-foo(1, 2, (void 1, 3));
-```
-
-Will render as
-
-```
-foo(1, 2)
-```
-
-Also supports a visible ellipsis:
-
-```ts
-const x = (void '...', 3);
-```
-
-Renders to:
-
-```
-x = ...
-```
-
-## Build integration
-
-This tool has the ability to hide irrelevant parts of the generated code
-snippet (see the section called "void masking" below). Because the samples
-should be compilable to extract all necessary type information, and because
-they could depend on any package, the following steps need to happen:
-
-* All packages need to be built (by `jsii`). Ideally, the reduced example ends
-  up in the assembly.
-* After all packages have been built, sample snippets should be checked
-  for compilability and supported language constructs (not all language
-  features can be translated to other languages). This requires the full
-  snippets (before reducing).
-* After the full samples have been type-checked, their reduced version
-  can be translated and inserted into the various generated packages by
-  `jsii-pacmak`.
-
-To avoid an additional dependency of `jsii` on the `jsii-samples` mechanism,
-what we'll do instead is mutating the assembly in-place. So simplified,
-the workflow looks like this:
-
-* All packages get compiled by `jsii`.
-* We postprocess all assemblies using `jsii-samples`, extracting code to
-  a side-archive (`.jsii.samples`) and replacing the original version in the
-  assembly, and generating all other language versions. This becomes a
-  translation table, with the key being a hash of the reduced snippet.
-* `jsii-pacmak` replaces snippets from the translation table.
-
-In this process, `jsii-samples` is as much self-contained as possible. It
-works on an assembly to produce a lookup file, which `jsii-pacmak` reads.
-`jsii-pacmak` has a simple fallback, which is use the unsubtituted example in
-case the right example is not available.
-
-Alternatively, since `jsii` doesn't really provide any facilities to mutate
-an assembly in-place, we leave the unreduced examples in the source assembly,
-and force all downstream renderers (such as the doc renderer and API tooling)
-to use `jsii-samples` to reduce the snippets before presenting them. This is
-not ideal but probably the way we're going to go because `jsii` doesn't provide
-any tooling to mutate an assembly in-place.
diff --git a/packages/jsii-sampiler/bin/jsii-sampiler b/packages/jsii-sampiler/bin/jsii-sampiler
deleted file mode 100755
index d778d67448..0000000000
--- a/packages/jsii-sampiler/bin/jsii-sampiler
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/usr/bin/env node
-require('./jsii-sampiler.js');
diff --git a/packages/jsii-sampiler/bin/jsii-sampiler.ts b/packages/jsii-sampiler/bin/jsii-sampiler.ts
deleted file mode 100644
index 7e57c7221d..0000000000
--- a/packages/jsii-sampiler/bin/jsii-sampiler.ts
+++ /dev/null
@@ -1,86 +0,0 @@
-import yargs = require('yargs');
-import { FileSource, isErrorDiagnostic, LiteralSource, printDiagnostics,
-  renderTree, translateMarkdown, TranslateResult, translateTypeScript } from '../lib';
-import { PythonVisitor } from '../lib/languages/python';
-import { VisualizeAstVisitor } from '../lib/languages/visualize';
-
-async function main() {
-  const argv = yargs
-    .usage('$0  [args]')
-    .command('snippet [file]', 'Translate a single snippet', command => command
-        .positional('file', { type: 'string', describe: 'The file to translate (leave out for stdin)' })
-        .option('python', { alias: 'p', boolean: true, description: 'Translate snippets to Python' })
-    , async args => {
-      const result = translateTypeScript(
-        await makeFileSource(args.file || '-', 'stdin.ts'),
-        makeVisitor(args));
-      renderResult(result);
-    })
-    .command('markdown ', 'Translate a MarkDown file', command => command
-        .positional('file', { type: 'string', describe: 'The file to translate (leave out for stdin)' })
-        .option('python', { alias: 'p', boolean: true, description: 'Translate snippets to Python' })
-    , async args => {
-      const result = translateMarkdown(
-        await makeFileSource(args.file || '-', 'stdin.md'),
-        makeVisitor(args));
-      renderResult(result);
-      return 5;
-    })
-    .demandCommand()
-    .help()
-    .strict()  // Error on wrong command
-    .version(require('../package.json').version)
-    .showHelpOnFail(false)
-    .argv;
-
-  // Evaluating .argv triggers the parsing but the command gets implicitly executed,
-  // so we don't need the output.
-  Array.isArray(argv);
-}
-
-function makeVisitor(args: { python?: boolean }) {
-  if (args.python) { return new PythonVisitor(); }
-  // Default to visualizing AST, including nodes we don't recognize yet
-  return new VisualizeAstVisitor();
-}
-
-async function makeFileSource(fileName: string, stdinName: string) {
-  if (fileName === '-') {
-    return new LiteralSource(await readStdin(), stdinName);
-  }
-  return new FileSource(fileName);
-}
-
-async function readStdin(): Promise {
-  process.stdin.setEncoding('utf8');
-
-  const parts: string[] = [];
-
-  return new Promise((resolve, reject) => {
-    process.stdin.on('readable', () => {
-      const chunk = process.stdin.read();
-      if (chunk !== null) { parts.push(`${chunk}`); }
-    });
-
-    process.stdin.on('error', reject);
-    process.stdin.on('end', () => resolve(parts.join('')));
-  });
-}
-
-function renderResult(result: TranslateResult) {
-  process.stdout.write(renderTree(result.tree) + '\n');
-
-  if (result.diagnostics.length > 0) {
-    printDiagnostics(result.diagnostics, process.stderr);
-
-    if (result.diagnostics.some(isErrorDiagnostic)) {
-      process.exit(1);
-    }
-  }
-}
-
-main().catch(e => {
-  // tslint:disable-next-line:no-console
-  console.error(e);
-  process.exit(1);
-});
\ No newline at end of file
diff --git a/packages/jsii-sampiler/lib/index.ts b/packages/jsii-sampiler/lib/index.ts
deleted file mode 100644
index 16b185ccbe..0000000000
--- a/packages/jsii-sampiler/lib/index.ts
+++ /dev/null
@@ -1,3 +0,0 @@
-export * from './translate';
-export { renderTree } from './o-tree';
-export { PythonVisitor } from './languages/python';
\ No newline at end of file
diff --git a/packages/jsii-sampiler/lib/translate.ts b/packages/jsii-sampiler/lib/translate.ts
deleted file mode 100644
index c9bc42a9ce..0000000000
--- a/packages/jsii-sampiler/lib/translate.ts
+++ /dev/null
@@ -1,118 +0,0 @@
-import fs = require('fs-extra');
-import ts = require('typescript');
-import { AstConverter, AstHandler, ConvertOptions } from './converter';
-import { transformMarkdown } from './markdown/markdown';
-import { MarkdownRenderer } from './markdown/markdown-renderer';
-import { ReplaceCodeTransform } from './markdown/replace-code-renderer';
-import { OTree, renderTree } from './o-tree';
-import { TypeScriptCompiler } from './typescript/ts-compiler';
-import { inTempDir } from './util';
-
-export interface Source {
-  withFile(fn: (fileName: string) => A): A;
-  withContents(fn: (fileName: string, contents: string) => A): A;
-}
-
-export class FileSource implements Source {
-  constructor(private readonly fileName: string) { }
-
-  public withFile(fn: (fileName: string) => A): A {
-    return fn(this.fileName);
-  }
-
-  public withContents(fn: (fileName: string, contents: string) => A): A {
-    const contents = fs.readFileSync(this.fileName, 'utf-8');
-    return fn(this.fileName, contents);
-  }
-}
-
-export class LiteralSource implements Source {
-  constructor(private readonly source: string, private readonly filenameHint = 'index.ts') { }
-
-  public withFile(fn: (fileName: string) => A): A {
-    return inTempDir(() => {
-      fs.writeFileSync(this.filenameHint, this.source);
-      return fn(this.filenameHint);
-    });
-  }
-
-  public withContents(fn: (fileName: string, contents: string) => A): A {
-    return fn(this.filenameHint, this.source);
-  }
-}
-
-export interface TranslateMarkdownOptions extends ConvertOptions {
-  /**
-   * What language to put in the returned markdown blocks
-   */
-  languageIdentifier?: string;
-}
-
-export function translateMarkdown(markdown: Source, visitor: AstHandler, options: TranslateMarkdownOptions = {}): TranslateResult {
-  const compiler = new TypeScriptCompiler();
-
-  let index = 0;
-  const diagnostics = new Array();
-
-  const translatedMarkdown = markdown.withContents((filename, contents) => {
-    return transformMarkdown(contents, new MarkdownRenderer(), new ReplaceCodeTransform(code => {
-      if (code.language !== 'typescript' && code.language !== 'ts') { return code; }
-
-      index += 1;
-      const snippetSource = new LiteralSource(code.source, `${filename}-snippet${index}.ts`);
-      const snippetTranslation = translateSnippet(snippetSource, compiler, visitor, options);
-
-      diagnostics.push(...snippetTranslation.diagnostics);
-
-      return { language: options.languageIdentifier || '', source: renderTree(snippetTranslation.tree) + '\n' };
-    }));
-  });
-
-  return { tree: new OTree([translatedMarkdown]), diagnostics };
-}
-
-export type TranslateOptions = ConvertOptions;
-
-export function translateTypeScript(source: Source, visitor: AstHandler, options: TranslateOptions = {}): TranslateResult {
-  const compiler = new TypeScriptCompiler();
-
-  return translateSnippet(source, compiler, visitor, options);
-}
-
-function translateSnippet(source: Source, compiler: TypeScriptCompiler, visitor: AstHandler, options: TranslateOptions = {}): TranslateResult {
-  return source.withContents((filename, contents) => {
-    const result = compiler.compileInMemory(filename, contents);
-
-    const converter = new AstConverter(result.rootFile, result.program.getTypeChecker(), visitor, options);
-    const converted = converter.convert(result.rootFile);
-
-    return {
-      tree: converted,
-      diagnostics: converter.diagnostics
-    };
-  });
-}
-
-export function printDiagnostics(diags: ts.Diagnostic[], stream: NodeJS.WritableStream) {
-  diags.forEach(d => printDiagnostic(d, stream));
-}
-
-export function printDiagnostic(diag: ts.Diagnostic, stream: NodeJS.WritableStream) {
-  const host = {
-    getCurrentDirectory() { return '.'; },
-    getCanonicalFileName(fileName: string) { return fileName; },
-    getNewLine() { return '\n'; }
-  };
-
-  const message = ts.formatDiagnosticsWithColorAndContext([diag], host);
-  stream.write(message);
-}
-
-export function isErrorDiagnostic(diag: ts.Diagnostic) {
-  return diag.category === ts.DiagnosticCategory.Error;
-}
-
-export interface TranslateResult {
-  tree: OTree;
-  diagnostics: ts.Diagnostic[];
-}
diff --git a/packages/jsii-sampiler/lib/util.ts b/packages/jsii-sampiler/lib/util.ts
deleted file mode 100644
index f4c79948a2..0000000000
--- a/packages/jsii-sampiler/lib/util.ts
+++ /dev/null
@@ -1,26 +0,0 @@
-import fs = require('fs-extra');
-import os = require('os');
-import path = require('path');
-import { renderTree, Source, translateTypeScript } from '.';
-import { VisualizeAstVisitor } from './languages/visualize';
-
-export function startsWithUppercase(x: string) {
-  return x.match(/^[A-Z]/);
-}
-
-export function inTempDir(block: () => T): T {
-  const origDir = process.cwd();
-  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'jsii'));
-  process.chdir(tmpDir);
-  const ret = block();
-  process.chdir(origDir);
-  fs.removeSync(tmpDir);
-  return ret;
-}
-
-export function visualizeTypeScriptAst(source: Source) {
-  const vis = translateTypeScript(source, new VisualizeAstVisitor(true), {
-    bestEffort: false
-  });
-  return renderTree(vis.tree) + '\n';
-}
diff --git a/packages/jsii/lib/docs.ts b/packages/jsii/lib/docs.ts
index 7fce0d3650..34b7cbe980 100644
--- a/packages/jsii/lib/docs.ts
+++ b/packages/jsii/lib/docs.ts
@@ -33,9 +33,27 @@
 import spec = require('jsii-spec');
 import ts = require('typescript');
 
+/**
+ * Tags that we recognize
+ */
+enum DocTag {
+  PARAM = 'param',
+  DEFAULT = 'default',
+  DEFAULT_VALUE = 'defaultValue',
+  DEPRECATED = 'deprecated',
+  RETURNS = 'returns',
+  RETURN = 'return',
+  STABLE = 'stable',
+  EXPERIMENTAL = 'experimental',
+  SEE = 'see',
+  SUBCLASSABLE = 'subclassable',
+  EXAMPLE = 'example',
+  STABILITY = 'stability',
+}
+
 export function parseSymbolDocumentation(sym: ts.Symbol, typeChecker: ts.TypeChecker): DocsParsingResult {
   const comment = ts.displayPartsToString(sym.getDocumentationComment(typeChecker)).trim();
-  const tags = sym.getJsDocTags();
+  const tags = reabsorbExampleTags(sym.getJsDocTags());
 
   // Right here we'll just guess that the first declaration site is the most important one.
   return parseDocParts(comment, tags);
@@ -47,7 +65,7 @@ export function parseSymbolDocumentation(sym: ts.Symbol, typeChecker: ts.TypeChe
 export function getReferencedDocParams(sym: ts.Symbol): string[] {
   const ret = new Array();
   for (const tag of sym.getJsDocTags()) {
-    if (tag.name === 'param') {
+    if (tag.name === DocTag.PARAM) {
       const parts = (tag.text || '').split(' ');
       ret.push(parts[0]);
     }
@@ -64,7 +82,7 @@ function parseDocParts(comments: string | undefined, tags: ts.JSDocTagInfo[]): D
   const tagNames = new Map();
   for (const tag of tags) {
     // 'param' gets parsed as a tag and as a comment for a method
-    if (tag.name !== 'param') { tagNames.set(tag.name, tag.text); }
+    if (tag.name !== DocTag.PARAM) { tagNames.set(tag.name, tag.text); }
   }
 
   function eatTag(...names: string[]): string | undefined {
@@ -78,17 +96,17 @@ function parseDocParts(comments: string | undefined, tags: ts.JSDocTagInfo[]): D
     return undefined;
   }
 
-  docs.default = eatTag('default', 'defaultValue');
-  docs.deprecated = eatTag('deprecated');
-  docs.example = eatTag('example');
-  docs.returns = eatTag('returns', 'return');
-  docs.see = eatTag('see');
-  docs.subclassable = eatTag('subclassable') !== undefined ? true : undefined;
+  docs.default = eatTag(DocTag.DEFAULT, DocTag.DEFAULT_VALUE);
+  docs.deprecated = eatTag(DocTag.DEPRECATED);
+  docs.example = eatTag(DocTag.EXAMPLE);
+  docs.returns = eatTag(DocTag.RETURNS, DocTag.RETURN);
+  docs.see = eatTag(DocTag.SEE);
+  docs.subclassable = eatTag(DocTag.SUBCLASSABLE) !== undefined ? true : undefined;
 
-  docs.stability = parseStability(eatTag('stability'), diagnostics);
+  docs.stability = parseStability(eatTag(DocTag.STABILITY), diagnostics);
   //  @experimental is a shorthand for '@stability experimental', same for '@stable'
-  const experimental = eatTag('experimental') !== undefined;
-  const stable = eatTag('stable') !== undefined;
+  const experimental = eatTag(DocTag.EXPERIMENTAL) !== undefined;
+  const stable = eatTag(DocTag.STABLE) !== undefined;
   // Can't combine them
   if (countBools(docs.stability !== undefined, experimental, stable) > 1) {
     diagnostics.push('Use only one of @stability, @experimental or @stable');
@@ -205,3 +223,30 @@ function parseStability(s: string | undefined, diagnostics: string[]): spec.Stab
   diagnostics.push(`Unrecognized @stability: '${s}'`);
   return undefined;
 }
+
+
+/**
+ * Unrecognized tags that follow an '@ example' tag will be absorbed back into the example value
+ *
+ * The TypeScript parser by itself is naive and will start parsing a new tag there.
+ *
+ * We do this until we encounter a supported @ keyword.
+ */
+function reabsorbExampleTags(tags: ts.JSDocTagInfo[]): ts.JSDocTagInfo[] {
+  const recognizedTags: string[] = Object.values(DocTag);
+  const ret = [...tags];
+
+  let i = 0;
+  while (i < ret.length) {
+    if (ret[i].name === 'example') {
+      while (i + 1 < ret.length && !recognizedTags.includes(ret[i + 1].name)) {
+        // Incorrectly classified as @tag, absorb back into example
+        ret[i].text += `@${ret[i + 1].name}${ret[i + 1].text}`;
+        ret.splice(i + 1, 1);
+      }
+    }
+    i++;
+  }
+
+  return ret;
+}
diff --git a/packages/jsii/lib/literate.ts b/packages/jsii/lib/literate.ts
index 1eaeef642f..3026d1b0aa 100644
--- a/packages/jsii/lib/literate.ts
+++ b/packages/jsii/lib/literate.ts
@@ -62,9 +62,9 @@ import path = require('path');
 /**
  * Convert an annotated TypeScript source file to MarkDown
  */
-export function typescriptSourceToMarkdown(lines: string[]): string[] {
+export function typescriptSourceToMarkdown(lines: string[], codeBlockAnnotations: string[]): string[] {
   const relevantLines = findRelevantLines(lines);
-  const markdownLines = markdownify(relevantLines);
+  const markdownLines = markdownify(relevantLines, codeBlockAnnotations);
   return markdownLines;
 }
 
@@ -87,9 +87,11 @@ export async function includeAndRenderExamples(lines: string[], loader: FileLoad
     if (m) {
       // Found an include
       /* eslint-disable no-await-in-loop */
-      const source = await loader(m[2]);
+      const filename = m[2];
+      const source = await loader(filename);
       /* eslint-enable no-await-in-loop */
-      const imported = typescriptSourceToMarkdown(source);
+      // 'lit' source attribute will make snippet compiler know to extract the same source
+      const imported = typescriptSourceToMarkdown(source, [`lit=${filename}`]);
       ret.push(...imported);
     } else {
       ret.push(line);
@@ -165,7 +167,7 @@ function stripCommonIndent(lines: string[]): string[] {
 /**
  * Turn source lines into Markdown, starting in TypeScript mode
  */
-function markdownify(lines: string[]): string[] {
+function markdownify(lines: string[], codeBlockAnnotations: string[]): string[] {
   const typescriptLines: string[] = [];
   const ret: string[] = [];
 
@@ -185,11 +187,13 @@ function markdownify(lines: string[]): string[] {
   return ret;
 
   /**
-     * Flush typescript lines with a triple-backtick-ts block around it.
-     */
+   * Flush typescript lines with a triple-backtick-ts block around it.
+   */
   function flushTS() {
     if (typescriptLines.length !== 0) {
-      ret.push('```ts', ...typescriptLines, '```');
+      /* eslint-disable prefer-template */
+      ret.push('```ts' + (codeBlockAnnotations.length > 0 ? ' ' + codeBlockAnnotations.join(' ') : ''), ...typescriptLines, '```');
+      /* eslint-enable prefer-template */
       typescriptLines.splice(0); // Clear
     }
   }
diff --git a/packages/jsii/test/docs.test.ts b/packages/jsii/test/docs.test.ts
index d14f184d8e..b1a7f7782d 100644
--- a/packages/jsii/test/docs.test.ts
+++ b/packages/jsii/test/docs.test.ts
@@ -333,3 +333,21 @@ test('stability is inherited from parent type', async () => {
     expect(method.docs!.stability).toBe(stability);
   }
 });
+
+// ----------------------------------------------------------------------
+test('@example can contain @ sign', async () => {
+  const assembly = await compile(`
+    /**
+     * An IAM role to associate with the instance profile assigned to this Auto Scaling Group.
+     *
+     * @example
+     *
+     * import x = require('@banana');
+     */
+    export class Foo {
+    }
+  `);
+
+  const classType = assembly.types!['testpkg.Foo'] as spec.ClassType;
+  expect(classType.docs!.example).toBe('import x = require(\'@banana\');');
+});
\ No newline at end of file
diff --git a/packages/jsii/test/literate.test.ts b/packages/jsii/test/literate.test.ts
index 7855a942c5..ddd17554d9 100644
--- a/packages/jsii/test/literate.test.ts
+++ b/packages/jsii/test/literate.test.ts
@@ -115,11 +115,11 @@ test('can do example inclusion', async () => {
 
   expect(rendered).toEqual([
     'This is a preamble',
-    '```ts',
+    '```ts lit=test/something.lit.ts',
     'const x = 1;',
     '```',
     'This is how we print x',
-    '```ts',
+    '```ts lit=test/something.lit.ts',
     'console.log(x);',
     '```',
     'This is a postamble'
@@ -127,6 +127,6 @@ test('can do example inclusion', async () => {
 });
 
 function assertRendersTo(source: string[], expected: string[]) {
-  const rendered = typescriptSourceToMarkdown(source);
+  const rendered = typescriptSourceToMarkdown(source, []);
   expect(expected).toEqual(rendered);
 }