From 8a95f70ff04da6cc32c92278155b112cb37105f0 Mon Sep 17 00:00:00 2001 From: Devon Govett Date: Mon, 26 Mar 2018 22:40:27 -0700 Subject: [PATCH] Implement pipelines to compose multiple asset types together (#1065) --- src/Asset.js | 13 +++- src/Pipeline.js | 102 ++++++++++++++++++++++++++++++++ src/assets/CSSAsset.js | 12 +++- src/assets/CoffeeScriptAsset.js | 29 ++++++--- src/assets/HTMLAsset.js | 3 +- src/assets/JSAsset.js | 1 + src/assets/LESSAsset.js | 23 +++++-- src/assets/ReasonAsset.js | 16 ++--- src/assets/SASSAsset.js | 25 ++++++-- src/assets/StylusAsset.js | 20 +++++-- src/assets/TypeScriptAsset.js | 35 +++++++---- src/assets/WebManifestAsset.js | 4 +- src/worker.js | 21 ++----- 13 files changed, 239 insertions(+), 65 deletions(-) create mode 100644 src/Pipeline.js diff --git a/src/Asset.js b/src/Asset.js index df22e1edff7..ad3df555267 100644 --- a/src/Asset.js +++ b/src/Asset.js @@ -26,7 +26,7 @@ class Asset { this.type = path.extname(this.name).slice(1); this.processed = false; - this.contents = null; + this.contents = options.rendition ? options.rendition.value : null; this.ast = null; this.generated = null; this.hash = null; @@ -58,6 +58,13 @@ class Asset { } async getDependencies() { + if ( + this.options.rendition && + this.options.rendition.hasDependencies === false + ) { + return; + } + await this.loadIfNeeded(); if (this.contents && this.mightHaveDependencies()) { @@ -154,6 +161,10 @@ class Asset { return this.generated; } + async postProcess(generated) { + return generated; + } + generateHash() { return objectHash(this.generated); } diff --git a/src/Pipeline.js b/src/Pipeline.js new file mode 100644 index 00000000000..26ea70ba07c --- /dev/null +++ b/src/Pipeline.js @@ -0,0 +1,102 @@ +const Parser = require('./Parser'); +const path = require('path'); +const md5 = require('./utils/md5'); + +/** + * A Pipeline composes multiple Asset types together. + */ +class Pipeline { + constructor(options) { + this.options = options; + this.parser = new Parser(options); + } + + async process(path, pkg, options) { + let asset = this.parser.getAsset(path, pkg, options); + let generated = await this.processAsset(asset); + let generatedMap = {}; + for (let rendition of generated) { + generatedMap[rendition.type] = rendition.value; + } + + return { + dependencies: Array.from(asset.dependencies.values()), + generated: generatedMap, + hash: asset.hash, + cacheData: asset.cacheData + }; + } + + async processAsset(asset) { + try { + await asset.process(); + } catch (err) { + throw asset.generateErrorMessage(err); + } + + let inputType = path.extname(asset.name).slice(1); + let generated = []; + + for (let rendition of this.iterateRenditions(asset)) { + let {type, value} = rendition; + if (typeof value !== 'string' || rendition.final) { + generated.push(rendition); + continue; + } + + // Find an asset type for the rendition type. + // If the asset is not already an instance of this asset type, process it. + let AssetType = this.parser.findParser( + asset.name.slice(0, -inputType.length) + type + ); + if (!(asset instanceof AssetType)) { + let opts = Object.assign({rendition}, asset.options); + let subAsset = new AssetType(asset.name, asset.package, opts); + subAsset.contents = value; + subAsset.dependencies = asset.dependencies; + + let processed = await this.processAsset(subAsset); + generated = generated.concat(processed); + Object.assign(asset.cacheData, subAsset.cacheData); + asset.hash = md5(asset.hash + subAsset.hash); + } else { + generated.push(rendition); + } + } + + // Post process. This allows assets a chance to modify the output produced by sub-asset types. + asset.generated = generated; + try { + generated = await asset.postProcess(generated); + } catch (err) { + throw asset.generateErrorMessage(err); + } + + return generated; + } + + *iterateRenditions(asset) { + if (Array.isArray(asset.generated)) { + return yield* asset.generated; + } + + if (typeof asset.generated === 'string') { + return yield { + type: asset.type, + value: asset.generated + }; + } + + // Backward compatibility support for the old API. + // Assume all renditions are final - don't compose asset types together. + for (let type in asset.generated) { + yield { + type, + value: asset.generated[type], + final: true + }; + } + } +} + +module.exports = Pipeline; diff --git a/src/assets/CSSAsset.js b/src/assets/CSSAsset.js index 06c1d182805..e27a79303e7 100644 --- a/src/assets/CSSAsset.js +++ b/src/assets/CSSAsset.js @@ -116,7 +116,17 @@ class CSSAsset extends Asset { 'module.exports = ' + JSON.stringify(this.cssModules, false, 2) + ';'; } - return {css, js}; + return [ + { + type: 'css', + value: css + }, + { + type: 'js', + value: js, + final: true + } + ]; } generateErrorMessage(err) { diff --git a/src/assets/CoffeeScriptAsset.js b/src/assets/CoffeeScriptAsset.js index 9b5573e9d3f..64c39c3a6c2 100644 --- a/src/assets/CoffeeScriptAsset.js +++ b/src/assets/CoffeeScriptAsset.js @@ -1,24 +1,35 @@ -const JSAsset = require('./JSAsset'); +const Asset = require('../Asset'); const localRequire = require('../utils/localRequire'); -class CoffeeScriptAsset extends JSAsset { - async parse(code) { +class CoffeeScriptAsset extends Asset { + constructor(name, pkg, options) { + super(name, pkg, options); + this.type = 'js'; + } + + async generate() { // require coffeescript, installed locally in the app let coffee = await localRequire('coffeescript', this.name); // Transpile Module using CoffeeScript and parse result as ast format through babylon - let transpiled = coffee.compile(code, { + let transpiled = coffee.compile(this.contents, { sourceMap: this.options.sourceMaps }); + let sourceMap; if (transpiled.sourceMap) { - this.sourceMap = transpiled.sourceMap.generate(); - this.sourceMap.sources = [this.relativeName]; - this.sourceMap.sourcesContent = [this.contents]; + sourceMap = transpiled.sourceMap.generate(); + sourceMap.sources = [this.relativeName]; + sourceMap.sourcesContent = [this.contents]; } - this.contents = this.options.sourceMaps ? transpiled.js : transpiled; - return await super.parse(this.contents); + return [ + { + type: 'js', + value: this.options.sourceMaps ? transpiled.js : transpiled, + sourceMap + } + ]; } } diff --git a/src/assets/HTMLAsset.js b/src/assets/HTMLAsset.js index 73c5233ba24..5b742356dda 100644 --- a/src/assets/HTMLAsset.js +++ b/src/assets/HTMLAsset.js @@ -160,8 +160,7 @@ class HTMLAsset extends Asset { } generate() { - let html = this.isAstDirty ? render(this.ast) : this.contents; - return {html}; + return this.isAstDirty ? render(this.ast) : this.contents; } } diff --git a/src/assets/JSAsset.js b/src/assets/JSAsset.js index 1f6be95af2d..7261c359f74 100644 --- a/src/assets/JSAsset.js +++ b/src/assets/JSAsset.js @@ -27,6 +27,7 @@ class JSAsset extends Asset { this.isES6Module = false; this.outputCode = null; this.cacheData.env = {}; + this.sourceMap = options.rendition ? options.rendition.sourceMap : null; } shouldInvalidate(cacheData) { diff --git a/src/assets/LESSAsset.js b/src/assets/LESSAsset.js index a2eb4a97042..8669bc1dd68 100644 --- a/src/assets/LESSAsset.js +++ b/src/assets/LESSAsset.js @@ -1,8 +1,13 @@ -const CSSAsset = require('./CSSAsset'); +const Asset = require('../Asset'); const localRequire = require('../utils/localRequire'); const promisify = require('../utils/promisify'); -class LESSAsset extends CSSAsset { +class LESSAsset extends Asset { + constructor(name, pkg, options) { + super(name, pkg, options); + this.type = 'css'; + } + async parse(code) { // less should be installed locally in the module that's being required let less = await localRequire('less', this.name); @@ -15,9 +20,7 @@ class LESSAsset extends CSSAsset { opts.filename = this.name; opts.plugins = (opts.plugins || []).concat(urlPlugin(this)); - let res = await render(code, opts); - res.render = () => res.css; - return res; + return await render(code, opts); } collectDependencies() { @@ -25,6 +28,16 @@ class LESSAsset extends CSSAsset { this.addDependency(dep, {includedInParent: true}); } } + + generate() { + return [ + { + type: 'css', + value: this.ast.css, + hasDependencies: false + } + ]; + } } function urlPlugin(asset) { diff --git a/src/assets/ReasonAsset.js b/src/assets/ReasonAsset.js index 0c72c475b63..c205234cd62 100644 --- a/src/assets/ReasonAsset.js +++ b/src/assets/ReasonAsset.js @@ -1,9 +1,14 @@ -const JSAsset = require('./JSAsset'); +const Asset = require('../Asset'); const fs = require('../utils/fs'); const localRequire = require('../utils/localRequire'); -class ReasonAsset extends JSAsset { - async parse() { +class ReasonAsset extends Asset { + constructor(name, pkg, options) { + super(name, pkg, options); + this.type = 'js'; + } + + async generate() { const bsb = await localRequire('bsb-js', this.name); // This runs BuckleScript - the Reason to JS compiler. @@ -18,10 +23,7 @@ class ReasonAsset extends JSAsset { // BuckleScript configuration to simplify the file processing. const outputFile = this.name.replace(/\.(re|ml)$/, '.bs.js'); const outputContent = await fs.readFile(outputFile); - this.contents = outputContent.toString(); - - // After loading the compiled JS source, use the normal JS behavior. - return await super.parse(this.contents); + return outputContent.toString(); } } diff --git a/src/assets/SASSAsset.js b/src/assets/SASSAsset.js index b8824ecd943..a7e0d0ffae4 100644 --- a/src/assets/SASSAsset.js +++ b/src/assets/SASSAsset.js @@ -1,10 +1,15 @@ -const CSSAsset = require('./CSSAsset'); +const Asset = require('../Asset'); const localRequire = require('../utils/localRequire'); const promisify = require('../utils/promisify'); const path = require('path'); const os = require('os'); -class SASSAsset extends CSSAsset { +class SASSAsset extends Asset { + constructor(name, pkg, options) { + super(name, pkg, options); + this.type = 'css'; + } + async parse(code) { // node-sass should be installed locally in the module that's being required let sass = await localRequire('node-sass', this.name); @@ -17,7 +22,7 @@ class SASSAsset extends CSSAsset { opts.includePaths = (opts.includePaths || []).concat( path.dirname(this.name) ); - opts.data = opts.data ? (opts.data + os.EOL + code) : code; + opts.data = opts.data ? opts.data + os.EOL + code : code; opts.indentedSyntax = typeof opts.indentedSyntax === 'boolean' ? opts.indentedSyntax @@ -30,9 +35,7 @@ class SASSAsset extends CSSAsset { } }); - let res = await render(opts); - res.render = () => res.css.toString(); - return res; + return await render(opts); } collectDependencies() { @@ -40,6 +43,16 @@ class SASSAsset extends CSSAsset { this.addDependency(dep, {includedInParent: true}); } } + + generate() { + return [ + { + type: 'css', + value: this.ast.css.toString(), + hasDependencies: false + } + ]; + } } module.exports = SASSAsset; diff --git a/src/assets/StylusAsset.js b/src/assets/StylusAsset.js index dd5d8e0f817..0f45c47044f 100644 --- a/src/assets/StylusAsset.js +++ b/src/assets/StylusAsset.js @@ -1,11 +1,17 @@ -const CSSAsset = require('./CSSAsset'); +// const CSSAsset = require('./CSSAsset'); +const Asset = require('../Asset'); const localRequire = require('../utils/localRequire'); const Resolver = require('../Resolver'); const syncPromise = require('../utils/syncPromise'); const URL_RE = /^(?:url\s*\(\s*)?['"]?(?:[#/]|(?:https?:)?\/\/)/i; -class StylusAsset extends CSSAsset { +class StylusAsset extends Asset { + constructor(name, pkg, options) { + super(name, pkg, options); + this.type = 'css'; + } + async parse(code) { // stylus should be installed locally in the module that's being required let stylus = await localRequire('stylus', this.name); @@ -26,8 +32,14 @@ class StylusAsset extends CSSAsset { return style; } - collectDependencies() { - // Do nothing. Dependencies are collected by our custom evaluator. + generate() { + return [ + { + type: 'css', + value: this.ast.render(), + hasDependencies: false + } + ]; } generateErrorMessage(err) { diff --git a/src/assets/TypeScriptAsset.js b/src/assets/TypeScriptAsset.js index ef41d84d522..71d92a2ce04 100644 --- a/src/assets/TypeScriptAsset.js +++ b/src/assets/TypeScriptAsset.js @@ -1,8 +1,13 @@ -const JSAsset = require('./JSAsset'); +const Asset = require('../Asset'); const localRequire = require('../utils/localRequire'); -class TypeScriptAsset extends JSAsset { - async parse(code) { +class TypeScriptAsset extends Asset { + constructor(name, pkg, options) { + super(name, pkg, options); + this.type = 'js'; + } + + async generate() { // require typescript, installed locally in the app let typescript = await localRequire('typescript', this.name); let transpilerOptions = { @@ -30,13 +35,16 @@ class TypeScriptAsset extends JSAsset { transpilerOptions.compilerOptions.sourceMap = this.options.sourceMaps; // Transpile Module using TypeScript and parse result as ast format through babylon - let transpiled = typescript.transpileModule(code, transpilerOptions); - this.sourceMap = transpiled.sourceMapText; + let transpiled = typescript.transpileModule( + this.contents, + transpilerOptions + ); + let sourceMap = transpiled.sourceMapText; - if (this.sourceMap) { - this.sourceMap = JSON.parse(this.sourceMap); - this.sourceMap.sources = [this.relativeName]; - this.sourceMap.sourcesContent = [this.contents]; + if (sourceMap) { + sourceMap = JSON.parse(sourceMap); + sourceMap.sources = [this.relativeName]; + sourceMap.sourcesContent = [this.contents]; // Remove the source map URL let content = transpiled.outputText; @@ -46,8 +54,13 @@ class TypeScriptAsset extends JSAsset { ); } - this.contents = transpiled.outputText; - return await super.parse(this.contents); + return [ + { + type: 'js', + value: transpiled.outputText, + sourceMap + } + ]; } } diff --git a/src/assets/WebManifestAsset.js b/src/assets/WebManifestAsset.js index 670f7c782ad..f149857b7a0 100644 --- a/src/assets/WebManifestAsset.js +++ b/src/assets/WebManifestAsset.js @@ -31,9 +31,7 @@ class WebManifestAsset extends Asset { } generate() { - return { - webmanifest: JSON.stringify(this.ast) - }; + return JSON.stringify(this.ast); } } diff --git a/src/worker.js b/src/worker.js index 22253a66457..6a5e8d069d4 100644 --- a/src/worker.js +++ b/src/worker.js @@ -1,10 +1,10 @@ require('v8-compile-cache'); -const Parser = require('./Parser'); +const Pipeline = require('./Pipeline'); -let parser; +let pipeline; exports.init = function(options, callback) { - parser = new Parser(options || {}); + pipeline = new Pipeline(options || {}); Object.assign(process.env, options.env || {}); process.env.HMR_PORT = options.hmrPort; process.env.HMR_HOSTNAME = options.hmrHostname; @@ -14,22 +14,11 @@ exports.init = function(options, callback) { exports.run = async function(path, pkg, options, isWarmUp, callback) { try { options.isWarmUp = isWarmUp; - var asset = parser.getAsset(path, pkg, options); - await asset.process(); + var result = await pipeline.process(path, pkg, options); - callback(null, { - dependencies: Array.from(asset.dependencies.values()), - generated: asset.generated, - hash: asset.hash, - cacheData: asset.cacheData - }); + callback(null, result); } catch (err) { let returned = err; - - if (asset) { - returned = asset.generateErrorMessage(returned); - } - returned.fileName = path; callback(returned); }