Permalink
Browse files

feat(bundler): build.options.cache turn on tracing cache and transpil…

…e cache

Cache is centralized in OS temp dir.
Tracing cache can be shared among apps.
Transpile cache is only for esnext. Scoped to appName-env, to respect env based babelrc.
No transpile cache for TS project, gulp-typescript does incremental build, not working with gulp-cache.
New au command "au clear-cache" clear both caches.
Reduce consecutive build time to about 1/5 for esnext app, based on preliminary benchmark.
TODO benchmark on TS app.
  • Loading branch information...
huochunpeng committed Sep 11, 2018
1 parent ea005fe commit 15af83f53e8307344e9e52cb51f6475fce19e0d8
View
@@ -307,7 +307,7 @@ exports.Bundle = class {
if (buildOptions.isApplicable('rev')) {
//Generate a unique hash based off of the bundle contents
//Must generate hash after we write the loader config so that any other bundle changes (hash changes) can cause a new hash for the vendor file
this.hash = generateHash(concat.content);
this.hash = generateHash(concat.content).slice(0, 10);
bundleFileName = generateHashedPath(this.config.name, this.hash);
}
@@ -319,7 +319,7 @@ exports.Bundle = class {
} else if (buildOptions.isApplicable('rev')) {
//Generate a unique hash based off of the bundle contents
//Must generate hash after we write the loader config so that any other bundle changes (hash changes) can cause a new hash for the vendor file
this.hash = generateHash(concat.content);
this.hash = generateHash(concat.content).slice(0, 10);
bundleFileName = generateHashedPath(this.config.name, this.hash);
}
@@ -4,7 +4,7 @@ const findDeps = require('./find-deps').findDeps;
const cjsTransform = require('./amodro-trace/read/cjs');
const esTransform = require('./amodro-trace/read/es');
const allWriteTransforms = require('./amodro-trace/write/all');
const {moduleIdWithPlugin} = require('./utils');
const Utils = require('./utils');
const logger = require('aurelia-logging').getLogger('BundledSource');
exports.BundledSource = class {
@@ -58,6 +58,10 @@ exports.BundledSource = class {
return this.bundler.loaderConfig;
}
_getUseCache() {
return this.bundler.buildOptions.isApplicable('cache');
}
get moduleId() {
if (this._moduleId) return this._moduleId;
@@ -123,27 +127,9 @@ exports.BundledSource = class {
// support Node.js's json module
let contents = `define(\'${moduleId}\',[],function(){return JSON.parse(${JSON.stringify(this.contents)});});\n`;
// be nice to requirejs json plugin users, add json! prefix
contents += `define(\'${moduleIdWithPlugin(moduleId, 'json', loaderType)}\',[\'${moduleId}\'],function(m){return m;});\n`;
contents += `define(\'${Utils.moduleIdWithPlugin(moduleId, 'json', loaderType)}\',[\'${moduleId}\'],function(m){return m;});\n`;
this.contents = contents;
} else {
// forceCjsWrap bypasses a r.js parse bug.
// See lib/amodro-trace/read/cjs.js for more info.
let forceCjsWrap = !!modulePath.match(/\/(cjs|commonjs)\//i);
let contents;
try {
contents = cjsTransform(modulePath, this.contents, forceCjsWrap);
} catch (ignore) {
// file is not in amd/cjs format, try native es module
try {
contents = esTransform(modulePath, this.contents);
} catch (e) {
logger.error('Could not convert to AMD module, skipping ' + modulePath);
logger.error('Error was: ' + e);
contents = this.contents;
}
}
deps = [];
let context = {pkgsMainMap: {}, config: {shim: {}}};
@@ -197,24 +183,72 @@ exports.BundledSource = class {
}
}
const writeTransform = allWriteTransforms({
const opts = {
stubModules: loaderConfig.stubModules,
wrapShim: wrapShim || loaderConfig.wrapShim,
replacement: replacement
});
};
// Use cache for js files to avoid expensive parsing and transform.
let cache;
let hash;
const useCache = this._getUseCache();
if (useCache) {
// Only hash on moduleId, opts and contents.
// This ensures cache on npm packages can be shared
// among different apps.
const key = [
moduleId,
JSON.stringify(context),
JSON.stringify(opts),
this.contents // contents here is after gulp transpile task
].join('|');
hash = Utils.generateHash(key);
cache = Utils.getCache(hash);
}
contents = writeTransform(context, moduleId, modulePath, contents);
if (cache) {
this.contents = cache.contents;
deps = cache.deps;
} else {
let contents;
// forceCjsWrap bypasses a r.js parse bug.
// See lib/amodro-trace/read/cjs.js for more info.
let forceCjsWrap = !!modulePath.match(/\/(cjs|commonjs)\//i);
const tracedDeps = findDeps(modulePath, contents);
if (tracedDeps && tracedDeps.length) {
deps.push.apply(deps, tracedDeps);
try {
contents = cjsTransform(modulePath, this.contents, forceCjsWrap);
} catch (ignore) {
// file is not in amd/cjs format, try native es module
try {
contents = esTransform(modulePath, this.contents);
} catch (e) {
logger.error('Could not convert to AMD module, skipping ' + modulePath);
logger.error('Error was: ' + e);
contents = this.contents;
}
}
const writeTransform = allWriteTransforms(opts);
contents = writeTransform(context, moduleId, modulePath, contents);
const tracedDeps = findDeps(modulePath, contents);
if (tracedDeps && tracedDeps.length) {
deps.push.apply(deps, tracedDeps);
}
this.contents = contents;
// write cache
if (useCache && hash) {
Utils.setCache(hash, {
deps: deps,
contents: this.contents
});
}
}
this.contents = contents;
}
this.requiresTransform = false;
// return deps
if (!deps || deps.length === 0) return;
let moduleIds = Array.from(new Set(deps)) // unique
View
@@ -2,6 +2,8 @@
const Bundler = require('./bundler').Bundler;
const PackageAnalyzer = require('./package-analyzer').PackageAnalyzer;
const PackageInstaller = require('./package-installer').PackageInstaller;
const cacheDir = require('./utils').cacheDir;
const del = require('del');
let bundler;
let project;
@@ -55,6 +57,11 @@ exports.dest = function(opts) {
.then(() => bundler.write());
};
exports.clearCache = function() {
// delete cache folder outside of cwd
return del(cacheDir, {force: true});
};
function buildLoaderConfig(p) {
project = p || project;
through = require('through2'); //dep of vinyl-fs
View
@@ -2,6 +2,7 @@
const path = require('path');
const crypto = require('crypto');
const fs = require('../file-system');
const tmpDir = require('os').tmpdir();
exports.knownExtensions = ['.js', '.json', '.css', '.svg', '.html'];
@@ -38,14 +39,28 @@ exports.moduleIdWithPlugin = function(moduleId, pluginName, type) {
}
};
exports.generateBundleName = function(contents, fileName, rev) {
let hash;
if (rev === true) {
hash = exports.generateHash(new Buffer(contents, 'utf-8'));
} else {
hash = rev;
const CACHE_DIR = path.resolve(tmpDir, 'aurelia-cli-cache');
exports.cacheDir = CACHE_DIR;
function cachedFilePath(hash) {
const folder = hash.substr(0, 2);
const fileName = hash.substr(2);
return path.resolve(CACHE_DIR, folder, fileName);
}
exports.getCache = function(hash) {
const filePath = cachedFilePath(hash);
try {
return JSON.parse(fs.readFileSync(filePath));
} catch (e) {
// ignore
}
return rev ? exports.generateHashedPath(fileName, hash) : fileName;
};
exports.setCache = function(hash, object) {
const filePath = cachedFilePath(hash);
// async write
fs.writeFile(filePath, JSON.stringify(object));
};
exports.runSequentially = function(tasks, cb) {
@@ -85,11 +100,8 @@ exports.revertHashedPath = function(pth, hash) {
});
};
exports.generateHash = function(buf) {
if (!Buffer.isBuffer(buf)) {
throw new TypeError('Expected a buffer');
}
return crypto.createHash('md5').update(buf).digest('hex').slice(0, 10);
exports.generateHash = function(bufOrStr) {
return crypto.createHash('md5').update(bufOrStr).digest('hex');
};
exports.escapeForRegex = function(str) {
@@ -40,7 +40,9 @@ module.exports = function(project, options) {
],
options: {
minify: 'stage & prod',
sourcemaps: 'dev & stage'
sourcemaps: 'dev & stage',
rev: false,
cache: 'dev & stage'
},
bundles: [
{
@@ -106,6 +108,8 @@ module.exports = function(project, options) {
).addToTasks(
ProjectItem.resource('build.ext', 'tasks/build.ext', model.transpiler),
ProjectItem.resource('build.json', 'tasks/build.json'),
ProjectItem.resource('clear-cache.ext', 'tasks/clear-cache.ext', model.transpiler),
ProjectItem.resource('clear-cache.json', 'tasks/clear-cache.json'),
ProjectItem.resource('copy-files.ext', 'tasks/copy-files.ext', model.transpiler),
ProjectItem.resource('run.ext', 'tasks/run.ext', model.transpiler),
ProjectItem.resource('run.json', 'tasks/run.json'),
@@ -25,6 +25,7 @@ module.exports = function(project) {
'babel-polyfill',
'babel-register',
'gulp-babel',
'gulp-cache',
'gulp-eslint'
);
};
View
@@ -44,6 +44,7 @@
"file-loader": "^1.1.11",
"gulp": "^4.0.0",
"gulp-babel": "^7.0.1",
"gulp-cache": "^1.0.2",
"gulp-changed-in-place": "^2.3.0",
"gulp-eslint": "^4.0.2",
"gulp-htmlmin": "^4.0.0",
@@ -0,0 +1,16 @@
import gulp from 'gulp';
import {build} from 'aurelia-cli';
import cache from 'gulp-cache';
function clearTraceCache() {
return build.clearCache();
}
function clearTranspileCache() {
return cache.clearAll();
}
export default gulp.series(
clearTraceCache,
clearTranspileCache
);
@@ -0,0 +1,4 @@
{
"name": "clear-cache",
"description": "Clear both transpile cache (only for esnext), and tracing-cache (for CLI built-in tracer)."
}
@@ -0,0 +1,6 @@
import * as gulp from 'gulp';
import {build} from 'aurelia-cli';
export default function clearCache() {
return build.clearCache();
}
@@ -5,24 +5,34 @@ import babel from 'gulp-babel';
import sourcemaps from 'gulp-sourcemaps';
import notify from 'gulp-notify';
import rename from 'gulp-rename';
import cache from 'gulp-cache';
import project from '../aurelia.json';
import {CLIOptions, build} from 'aurelia-cli';
import {CLIOptions, build, Configuration} from 'aurelia-cli';
function configureEnvironment() {
let env = CLIOptions.getEnvironment();
let env = CLIOptions.getEnvironment();
const buildOptions = new Configuration(project.build.options);
const useCache = buildOptions.isApplicable('cache');
function configureEnvironment() {
return gulp.src(`aurelia_project/environments/${env}.js`)
.pipe(changedInPlace({firstPass: true}))
.pipe(rename('environment.js'))
.pipe(gulp.dest(project.paths.root));
}
function buildJavaScript() {
let transpile = babel(project.transpiler.options);
if (useCache) {
// the cache directory is "gulp-cache/projName-env" inside folder require('os').tmpdir()
// use command 'au clear-cache' to purge all caches
transpile = cache(transpile, {name: project.name + '-' + env});
}
return gulp.src(project.transpiler.source)
.pipe(plumber({errorHandler: notify.onError('Error: <%= error.message %>')}))
.pipe(changedInPlace({firstPass: true}))
.pipe(sourcemaps.init())
.pipe(babel(project.transpiler.options))
.pipe(transpile)
.pipe(build.bundle());
}
Oops, something went wrong.

0 comments on commit 15af83f

Please sign in to comment.