Introduce Broccoli Plugin For Bundle Compiler

packages/@glimmer/app-compiler/README.md

@@ -0,0 +1,72 @@
+# @glimmer/app-compiler
+
+A broccoli plugin that wraps Glimmer's optimizing compiler for producing the required pre-computed objects for Glimmer VM based applications.
+
+## Basic Usage
+
+For most apps the plugin can be used as the following:
+
+```ts
+import { BundleCompiler } from '@glimmer/app-compiler';
+...
+
+let optimizedApp = BundleCompiler(app, {
+  projectPath: 'my-app',
+  mode: 'module-unification'
+});
+
+return optimizedApp;
+```
+
+## Advanced Usage
+
+If you need to do more advanced things like registering AST plugins or using custom project layout you can do something like the following:
+
+```ts
+import { BundleCompiler } from '@glimmer/app-compiler';
+import { BundleCompilerDelegate } from '@glimmer/compiler-delegates'
+...
+
+class MyCustomCompilerDelegate implements BundleCompilerDelegate {
+  ...
+}
+
+let optimizedApp = BundleCompiler(app, {
+  projectPath: 'my-app',
+  delegate: MyCustomCompilerDelegate,
+  outputFiles: {
+    heapFile: 'snowflake/location/template.gbx',
+    dataSegment: 'snowflake/data.js'
+  },
+  bundleCompiler: {
+    plugins: [MyASTPlugin]
+  }
+});
+
+return optimizedApp;
+```
+
+### Options
+
+```ts
+interface GlimmerBundleCompilerOptions {
+  projectPath: string;                      // where the project is at
+  bundleCompiler: BundleCompilerOptions;    // Options specifically for the compiler
+  outputFiles?: OutputFiles;                // Where to write the output
+  delegate?: BundleCompilerDelegateConstructor; // Delegate to discover information about templates
+  mode?: 'module-unification';              // Builtin delegate
+}
+
+interface BundleCompilerDelegateConstructor {
+  new(): BundleCompilerDelegate;
+}
+
+export interface OutputFiles {
+  dataSegment: Option<string>;
+  heapFile: Option<string>;
+}
+
+interface BundleCompilerOptions {
+  plugins?: ASTPluginBuilder[];
+}
+```

packages/@glimmer/app-compiler/index.ts

@@ -0,0 +1 @@
+export { default as GlimmerBundleCompiler } from './src/bundle-compiler';

packages/@glimmer/app-compiler/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@glimmer/app-compiler",
+  "version": "0.8.0-alpha.9",
+  "description": "Broccoli tooling for Glimmer-VM based applications",
+  "repository": "https://github.com/glimmerjs/glimmer.js",
+  "license": "MIT",
+  "files": [
+    "dist"
+  ],
+  "main": "dist/commonjs/es5/index.js",
+  "module": "dist/modules/es2017/index.js",
+  "types": "dist/types/index.d.ts",
+  "dependencies": {
+    "@glimmer/application": "0.8.0-alpha.10",
+    "@glimmer/bundle-compiler": "^0.29.8",
+    "@glimmer/compiler-delegates": "0.8.0-alpha.9",
+    "@glimmer/interfaces": "^0.29.8",
+    "@glimmer/opcode-compiler": "^0.29.8",
+    "broccoli-plugin": "^1.3.0",
+    "debug": "^3.1.0",
+    "walk-sync": "^0.3.2"
+  },
+  "devDependencies": {
+    "broccoli-stew": "^1.5.0",
+    "broccoli-test-helper": "^1.2.0",
+    "co": "^4.6.0"
+  }
+}

packages/@glimmer/app-compiler/src/bundle-compiler.ts

@@ -0,0 +1,111 @@
+import { BundleCompiler, BundleCompilerOptions, specifierFor } from '@glimmer/bundle-compiler';
+import { ModuleUnificationCompilerDelegate, BundleCompilerDelegate, OutputFiles } from '@glimmer/compiler-delegates';
+import Plugin from 'broccoli-plugin';
+import walkSync from 'walk-sync';
+import { readFileSync, writeFileSync, mkdirSync } from 'fs';
+import { join, extname } from 'path';
+import { mainLayout } from '@glimmer/application';
+import { CompilableTemplate } from '@glimmer/opcode-compiler';
+
+export type CompilerMode = 'module-unification';
+
+export interface BundleCompilerDelegateConstructor {
+  new(projectPath: string, outputFiles: OutputFiles): BundleCompilerDelegate;
+}
+
+export interface GlimmerBundleCompilerOptions {
+  projectPath: string;
+  bundleCompiler: BundleCompilerOptions;
+  outputFiles?: OutputFiles;
+  delegate?: BundleCompilerDelegateConstructor;
+  mode?: CompilerMode;
+}
+
+export default class GlimmerBundleCompiler extends Plugin {
+  options: GlimmerBundleCompilerOptions;
+  inputPaths: string[];
+  outputPath: string;
+  compiler: BundleCompiler;
+  private delegate: BundleCompilerDelegate;
+  constructor(inputNode, options) {
+    super([inputNode], options);
+    this.options = this.defaultOptions(options);
+  }
+
+  private defaultOptions(options: GlimmerBundleCompilerOptions) {
+    if (!options.projectPath) {
+      throw new Error('Must supply a projectPath');
+    }
+
+    if (!options.mode && !options.delegate) {
+      throw new Error('Must pass a bundle compiler mode or pass a custom compiler delegate.');
+    }
+
+    return Object.assign({
+      outputFiles: {
+        heapFile: 'src/templates.gbx',
+        dataSegment: 'src/data-segment.js'
+      }
+    }, options);
+  }
+
+  listEntries() {
+    let [srcPath] = this.inputPaths;
+    return walkSync.entries(srcPath);
+  }
+
+  _readFile(file) {
+    return readFileSync(join(this.inputPaths[0], file), 'UTF-8');
+  }
+
+  createBundleCompiler() {
+    let delegate;
+    let { options } = this;
+    if (options.mode && options.mode === 'module-unification') {
+      delegate = this.delegate = new ModuleUnificationCompilerDelegate(options.projectPath, options.outputFiles);
+    } else if (options.delegate) {
+      delegate = this.delegate = new options.delegate(options.projectPath, options.outputFiles);
+    }
+
+    this.compiler = new BundleCompiler(delegate, options.bundleCompiler = {});
+  }
+
+  build() {
+    if (!this.compiler && !this.delegate) {
+      this.createBundleCompiler();
+    }
+
+    let { outputPath } = this;
+
+    let specifier = specifierFor('__BUILTIN__', 'default');
+    let compilable = CompilableTemplate.topLevel(JSON.parse(mainLayout.block), this.compiler.compileOptions(specifier));
+
+    this.compiler.addCustom(specifier, compilable);
+
+    this.listEntries().forEach(entry => {
+      let { relativePath } = entry;
+      if (entry.isDirectory()) {
+        mkdirSync(join(outputPath, relativePath));
+      } else {
+        let content = this._readFile(relativePath);
+        if (extname(relativePath) === '.hbs') {
+          let normalizedPath = this.delegate.normalizePath(relativePath);
+          let specifier = this.delegate.specifierFor(normalizedPath);
+          this.compiler.add(specifier, content);
+        } else {
+          writeFileSync(join(outputPath, relativePath), content);
+        }
+      }
+    });
+
+    let { heap, pool } = this.compiler.compile();
+    let map = this.compiler.getSpecifierMap();
+    let { compiledBlocks } = this.compiler;
+    let dataSegment = this.delegate.generateDataSegment(map, pool, heap.table, heap.handle, compiledBlocks);
+
+    let { outputFiles } = this.options;
+
+    writeFileSync(join(outputPath, outputFiles.dataSegment), dataSegment);
+    writeFileSync(join(outputPath, outputFiles.heapFile), new Buffer(heap.buffer));
+  }
+}

packages/@glimmer/app-compiler/test/node/bundle-compiler-test.ts

@@ -0,0 +1,134 @@
+import { module, test } from 'qunitjs';
+import { GlimmerBundleCompiler } from '@glimmer/app-compiler';
+import { createTempDir, buildOutput } from 'broccoli-test-helper';
+import co from 'co';
+import { ModuleUnificationCompilerDelegate } from '@glimmer/compiler-delegates';
+
+class TestModuleUnificationDelegate extends ModuleUnificationCompilerDelegate {
+  normalizePath(modulePath: string): string {
+    return modulePath.replace('my-app/', '');
+  }
+}
+
+module('Broccol Glimmer Bundle Compiler', function(hooks) {
+  let input = null;
+
+  hooks.beforeEach(() => createTempDir().then(tempDir => (input = tempDir)));
+
+  hooks.afterEach(() => {
+    input.dispose();
+  });
+
+  test('requires a mode or delegate', function (assert) {
+    assert.throws(() => {
+      new GlimmerBundleCompiler(input.path(), { projectPath: 'src' });
+    }, /Must pass a bundle compiler mode or pass a custom compiler delegate\./);
+  });
+
+  test('requires a project path', function (assert) {
+    assert.throws(() => {
+      new GlimmerBundleCompiler(input.path(), {
+        mode: 'module-unification'
+      });
+    }, /Must supply a projectPath/);
+  });
+
+  test('syncs forward all files', co.wrap(function *(assert) {
+    input.write({
+      'my-app': {
+        'package.json': JSON.stringify({name: 'my-app'}),
+        src: {
+          ui: {
+            components: {
+              A: {
+                'template.hbs': '<div>Hello</div>',
+                'component.ts': 'export default class A {}'
+              },
+
+              B: {
+                'template.hbs': 'From B: <A @foo={{bar}} /> {{@bar}}',
+                'component.ts': 'export default class B {}'
+              },
+
+              C: {
+                'template.hbs': 'From C',
+                'component.ts': 'export default class C {}'
+              },
+
+              D: {
+                'template.hbs': '{{component C}}',
+                'component.ts': 'export default class D {}'
+              }
+            }
+          }
+        }
+      }
+    });
+
+    let compiler = new GlimmerBundleCompiler(input.path(), {
+      projectPath: `${input.path()}/my-app`,
+      delegate: TestModuleUnificationDelegate,
+      outputFiles: {
+        dataSegment: 'my-app/src/data.js',
+        heapFile: 'my-app/src/templates.gbx'
+      }
+    });
+
+    let output = yield buildOutput(compiler);
+    let files = output.read();
+
+    assert.deepEqual(Object.keys(files), ['my-app']);
+    assert.deepEqual(Object.keys(files['my-app']).sort(), ['src', 'package.json'].sort());
+    assert.deepEqual(Object.keys(files['my-app'].src).sort(), ['ui', 'templates.gbx', 'data.js'].sort());
+    assert.deepEqual(Object.keys(files['my-app'].src.ui), ['components']);
+
+    Object.keys(files['my-app'].src.ui.components).forEach((component) => {
+      assert.deepEqual(Object.keys(files['my-app'].src.ui.components[component]), ['component.ts']);
+    });
+  }));
+
+  test('[MU] compiles the gbx and data segment', co.wrap(function *(assert) {
+    input.write({
+      'my-app': {
+        'package.json': JSON.stringify({name: 'my-app'}),
+        src: {
+          ui: {
+            components: {
+              A: {
+                'template.hbs': '<div>Hello</div>'
+              },
+
+              B: {
+                'template.hbs': 'From B: <A @foo={{bar}} /> {{@bar}}'
+              },
+
+              C: {
+                'template.hbs': 'From C'
+              },
+
+              D: {
+                'template.hbs': '{{component C}}'
+              }
+            }
+          }
+        }
+      }
+    });
+
+    let compiler = new GlimmerBundleCompiler(input.path(), {
+      projectPath: `${input.path()}/my-app`,
+      delegate: TestModuleUnificationDelegate,
+      outputFiles: {
+        dataSegment: 'my-app/src/data.js',
+        heapFile: 'my-app/src/templates.gbx'
+      }
+    });
+
+    let output = yield buildOutput(compiler);
+    let files = output.read();
+
+    let buffer = new Uint16Array(files['my-app'].src['templates.gbx']);
+
+    assert.ok(buffer, 'Buffer is aligned');
+  }));
+});

packages/@glimmer/application/index.ts

@@ -4,3 +4,4 @@ export { default as ApplicationRegistry } from './src/application-registry';
 export { RuntimeResolver } from './src/runtime-resolver';
 export { default as Iterable } from './src/iterable';
 export { debugInfoForReference } from './src/helpers/action';
+export { default as mainLayout } from './src/templates/main';

packages/@glimmer/compiler-delegates/index.ts

@@ -1 +1,3 @@
 export { default as ModuleUnificationCompilerDelegate } from './src/delegates/module-unification';
+export { BundleCompilerDelegate } from './src/bundle';
+export { OutputFiles } from './src/utils/code-gen';

packages/@glimmer/compiler-delegates/package.json

@@ -11,13 +11,15 @@
   "module": "dist/modules/es2017/index.js",
   "types": "dist/types/index.d.ts",
   "dependencies": {
-    "debug": "^3.1.0",
     "@glimmer/bundle-compiler": "^0.29.8",
+    "@glimmer/component": "^0.8.0",
+    "@glimmer/interfaces": "^0.29.8",
     "@glimmer/opcode-compiler": "^0.29.8",
+    "@glimmer/program": "^0.29.8",
     "@glimmer/syntax": "^0.29.8",
     "@glimmer/util": "^0.29.8",
-    "@glimmer/program": "^0.29.8",
-    "@glimmer/interfaces": "^0.29.8",
+    "@types/node": "^8.0.46",
+    "debug": "^3.1.0",
     "glimmer-analyzer": "^0.3.2",
     "resolve": "^1.4.0"
   }

packages/@glimmer/compiler-delegates/src/bundle.ts

@@ -1,8 +1,15 @@
-import { CompilerDelegate, SpecifierMap } from '@glimmer/bundle-compiler';
+import { CompilerDelegate, SpecifierMap, Specifier } from '@glimmer/bundle-compiler';
 import { ConstantPool } from '@glimmer/program';
+import { SerializedTemplateBlock } from '@glimmer/wire-format';
+import { ICompilableTemplate } from '@glimmer/opcode-compiler';
+import { ProgramSymbolTable } from '@glimmer/interfaces';
 
 export type Metadata = {};
 
+export type AddedTemplate = SerializedTemplateBlock | ICompilableTemplate<ProgramSymbolTable>;
+
 export interface BundleCompilerDelegate extends CompilerDelegate {
-  generateDataSegment(map: SpecifierMap, pool: ConstantPool, heapTable: number[]): string;
+  normalizePath(absolutePath: string): string;
+  specifierFor(relativePath: string): Specifier;
+  generateDataSegment(map: SpecifierMap, pool: ConstantPool, heapTable: number[], nextFreeHandle: number, blocks: Map<Specifier, AddedTemplate>): string;
 }