feat(broccoli): Add naive caching strategy for Broccoli. (#190)

 - Aggregator Broc plugin now lives in Broccoli package.
 - Use fs-tree-diff to better keep input and output dirs in sync with minimal changes.
 - Don't do any work if input directory has not changed.
 - Add more robust tests for Analyze and Aggregate plugins with broccoli-test-helper.
 - Use typed versions of walk-sync and fs-tree-diff.
 - TODO: If input directory has changed, we still re-build the world. This can be improved.

package.json

@@ -44,7 +44,7 @@
     "tslint": "^5.9.1",
     "typedoc": "^0.11.0",
     "typedoc-plugin-monorepo": "^0.1.0",
-    "typescript": "~2.8.0",
+    "typescript": "2.8",
     "watch": "^1.0.2"
   },
   "workspaces": [

packages/@css-blocks/broccoli/package.json

@@ -29,23 +29,26 @@
   "devDependencies": {
     "@css-blocks/code-style": "^0.18.0",
     "@css-blocks/glimmer": "^0.19.0",
+    "@types/glob": "^5.0.35",
+    "broccoli-test-helper": "^1.4.0",
     "watch": "^1.0.2"
   },
   "dependencies": {
     "@css-blocks/core": "^0.19.0",
     "@glimmer/compiler": "^0.33.0",
     "@glimmer/syntax": "^0.33.0",
     "@opticss/template-api": "^0.3.0",
-    "@types/recursive-readdir": "^2.2.0",
     "broccoli-funnel": "^2.0.1",
     "broccoli-merge-trees": "^3.0.0",
     "broccoli-plugin": "^1.3.0",
     "broccoli-test-helper": "^1.2.0",
     "colors": "^1.2.1",
     "debug": "^3.1.0",
     "fs-extra": "^5.0.0",
+    "fs-tree-diff": "^0.5.9",
+    "glob": "^7.1.2",
     "opticss": "^0.3.0",
-    "recursive-readdir": "^2.2.2",
-    "walk-sync": "^0.3.2"
+    "symlink-or-copy": "^1.2.0",
+    "walk-sync": "^0.3.3"
   }
 }

packages/@css-blocks/broccoli/src/Aggregate.ts

@@ -0,0 +1,85 @@
+
+import * as path from "path";
+
+import * as fs from "fs-extra";
+import * as FSTree from "fs-tree-diff";
+import * as walkSync from "walk-sync";
+
+import { Transport } from "./Transport";
+import { BroccoliPlugin } from "./utils";
+
+// Common CSS preprocessor file endings to auto-discover
+const COMMON_FILE_ENDINGS = [".scss", ".sass", ".less", ".stylus"];
+
+/**
+ * Process-global dumping zone for CSS output as it comes through the pipeline 🤮
+ * This will disappear once we have a functional language server and replaced
+ * with a post-build step.
+ */
+export class CSSBlocksAggregate extends BroccoliPlugin {
+
+  private transport: Transport;
+  private out: string;
+  private _out = "";
+  private previousCSS = "";
+  private previous = new FSTree();
+
+  /**
+   * Initialize this new instance with the app tree, transport, and analysis options.
+   * @param inputNodes Broccoli trees who's output we depend on. First node must be the tree where stylesheets are placed.
+   * @param transport Magical shared-memory Transport object shared with the aggregator and Template transformer.
+   * @param out Output file name.
+   */
+  // tslint:disable-next-line:prefer-whatever-to-any
+  constructor(inputNodes: any[], transport: Transport, out: string) {
+    super(inputNodes, {
+      name: "broccoli-css-blocks-aggregate",
+      persistentOutput: true,
+    });
+    this.transport = transport;
+    this.out = out;
+  }
+
+  /**
+   * Re-run the broccoli build over supplied inputs.
+   */
+  build() {
+    let output = this.outputPath;
+    let input = this.inputPaths[0];
+    let { id, css } = this.transport;
+
+    // Test if anything has changed since last time. If not, skip trying to update tree.
+    let newFsTree = FSTree.fromEntries(walkSync.entries(input));
+    let diff = this.previous.calculatePatch(newFsTree);
+    if (diff.length) {
+      this.previous = newFsTree;
+      FSTree.applyPatch(input, output, diff);
+    }
+
+    // Auto-discover common preprocessor extensions.
+    if (!this._out) {
+      let prev = path.parse(path.join(input, this.out));
+      let origExt = prev.ext;
+      prev.base = ""; // Needed for path.format to register ext change
+      for (let ending of COMMON_FILE_ENDINGS) {
+        prev.ext = ending;
+        if (fs.existsSync(path.format(prev))) { break; }
+        prev.ext = origExt;
+      }
+      let out = path.parse(this.out);
+      out.base = ""; // Needed for path.format to register ext change
+      out.ext = prev.ext;
+      this._out = path.format(out);
+    }
+
+    let outHasChanged = !!diff.find((o) => o[1] === this._out);
+    if (outHasChanged || this.previousCSS !== css) {
+      let prev = path.join(input, this._out);
+      let out = path.join(output, this._out);
+      prev = fs.existsSync(prev) ? fs.readFileSync(prev).toString() : "";
+      if (fs.existsSync(out)) { fs.unlinkSync(out); }
+      fs.writeFileSync(out, `${prev}\n/* CSS Blocks Start: "${id}" */\n${css}\n/* CSS Blocks End: "${id}" */\n`);
+      this.previousCSS = css;
+    }
+  }
+}

packages/@css-blocks/broccoli/src/Analyze.ts

@@ -0,0 +1,142 @@
+import * as path from "path";
+
+import { Analyzer, BlockCompiler, StyleMapping } from "@css-blocks/core";
+import { TemplateTypes } from "@opticss/template-api";
+
+import * as debugGenerator from "debug";
+import * as fs from "fs-extra";
+import * as FSTree from "fs-tree-diff";
+import * as glob from "glob";
+import { OptiCSSOptions, Optimizer, postcss } from "opticss";
+import * as walkSync from "walk-sync";
+
+import { Transport } from "./Transport";
+import { BroccoliPlugin, symlinkOrCopy } from "./utils";
+
+const debug = debugGenerator("css-blocks:broccoli");
+
+export interface BroccoliOptions {
+  entry: string[];
+  output: string;
+  root: string;
+  analyzer: Analyzer<keyof TemplateTypes>;
+  optimization?: Partial<OptiCSSOptions>;
+}
+
+/**
+ * Runs analysis on an `inputNode` that represents the entire
+ * application. `options.transport` will be populated with
+ * analysis results. Output is the same application tree
+ * with all Block files removed.
+ */
+export class CSSBlocksAnalyze extends BroccoliPlugin {
+
+  private analyzer: Analyzer<keyof TemplateTypes>;
+  private entries: string[];
+  private output: string;
+  private root: string;
+  private transport: Transport;
+  private optimizationOptions: Partial<OptiCSSOptions>;
+  private previous: FSTree = new FSTree();
+
+  /**
+   * Initialize this new instance with the app tree, transport, and analysis options.
+   * @param inputNode Single Broccoli tree node containing *entire* app.
+   * @param transport Magical shared-memory Transport object shared with the aggregator and Template transformer.
+   * @param options Analysis options.
+   */
+  // tslint:disable-next-line:prefer-whatever-to-any
+  constructor(inputNode: any, transport: Transport, options: BroccoliOptions) {
+    super([inputNode], {
+      name: "broccoli-css-blocks-analyze",
+      persistentOutput: true,
+    });
+    this.transport = transport;
+    this.entries = options.entry.slice(0);
+    this.output = options.output || "css-blocks.css";
+    this.optimizationOptions = options.optimization || {};
+    this.analyzer = options.analyzer;
+    this.root = options.root || process.cwd();
+    this.transport.css = this.transport.css ? this.transport.css : "";
+  }
+
+  /**
+   * Re-run the broccoli build over supplied inputs.
+   */
+  async build() {
+    let input = this.inputPaths[0];
+    let output = this.outputPath;
+    let options = this.analyzer.cssBlocksOptions;
+    let blockCompiler = new BlockCompiler(postcss, options);
+    let optimizer = new Optimizer(this.optimizationOptions, this.analyzer.optimizationOptions);
+
+    // Test if anything has changed since last time. If not, skip all analysis work.
+    let newFsTree = FSTree.fromEntries(walkSync.entries(input));
+    let diff = this.previous.calculatePatch(newFsTree);
+    if (!diff.length) { return; }
+    this.previous = newFsTree;
+    FSTree.applyPatch(input, output, diff);
+
+    // When no entry points are passed, we treat *every* template as an entry point.
+    this.entries = this.entries.length ? this.entries : glob.sync("**/*.hbs", { cwd: input });
+
+    // The glimmer-analyzer package tries to require() package.json
+    // in the root of the directory it is passed. We pass it our broccoli
+    // tree, so it needs to contain package.json too.
+    // TODO: Ideally this is configurable in glimmer-analyzer. We can
+    //       contribute that option back to the project. However,
+    //       other template integrations may want this available too...
+    let pjsonLink = path.join(input, "package.json");
+    if (!fs.existsSync(pjsonLink)) {
+      symlinkOrCopy(path.join(this.root, "package.json"), pjsonLink);
+    }
+
+    // Oh hey look, we're analyzing.
+    this.analyzer.reset();
+    this.transport.reset();
+    await this.analyzer.analyze(input, this.entries);
+
+    // Compile all Blocks and add them as sources to the Optimizer.
+    // TODO: handle a sourcemap from compiling the block file via a preprocessor.
+    let blocks = this.analyzer.transitiveBlockDependencies();
+    for (let block of blocks) {
+      if (block.stylesheet) {
+        let root = blockCompiler.compile(block, block.stylesheet, this.analyzer);
+        let result = root.toResult({ to: this.output, map: { inline: false, annotation: false } });
+        let filesystemPath = options.importer.filesystemPath(block.identifier, options);
+        let filename = filesystemPath || options.importer.debugIdentifier(block.identifier, options);
+
+        // If this Block has a representation on disk, remove it from our output tree.
+        if (filesystemPath) {
+          let outputStylesheet = path.join(output, path.relative(input, filesystemPath));
+          debug(`Removing block file ${outputStylesheet} from output.`);
+          if (fs.existsSync(outputStylesheet)) { fs.removeSync(outputStylesheet); }
+        }
+
+        // Add the compiled Block file to the optimizer.
+        optimizer.addSource({
+          content: result.css,
+          filename,
+          sourceMap: result.map.toJSON(),
+        });
+      }
+    }
+
+    // Add each Analysis to the Optimizer.
+    this.analyzer.eachAnalysis((a) => optimizer.addAnalysis(a.forOptimizer(options)));
+
+    // Run optimization and compute StyleMapping.
+    let optimized = await optimizer.optimize(this.output);
+    let styleMapping = new StyleMapping<keyof TemplateTypes>(optimized.styleMapping, blocks, options, this.analyzer.analyses());
+
+    // Attach all computed data to our magic shared memory transport object...
+    this.transport.mapping = styleMapping;
+    this.transport.blocks = blocks;
+    this.transport.analyzer = this.analyzer;
+    this.transport.css += optimized.output.content.toString();
+
+    debug(`Compilation Finished: ${this.transport.id}`);
+
+  }
+
+}

packages/@css-blocks/broccoli/src/Transport.ts

@@ -0,0 +1,24 @@
+import { Analyzer, Block, StyleMapping } from "@css-blocks/core";
+import { TemplateTypes } from "@opticss/template-api";
+
+// Magic shared memory transport object 🤮
+// This will disappear once we have a functional language server.
+export class Transport {
+  id: string;
+  css = "";
+  blocks: Set<Block> = new Set();
+  mapping?: StyleMapping<keyof TemplateTypes>;
+  analyzer?: Analyzer<keyof TemplateTypes>;
+
+  constructor(id: string) {
+    this.id = id;
+    this.reset();
+  }
+
+  reset() {
+    this.css = "";
+    this.blocks = new Set();
+    this.mapping = undefined;
+    this.analyzer = undefined;
+  }
+}