Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

[api refactor] Add caching to bundler. In process, moved bundler into…

… ./lib/bundler and added a common library of utility functions.
  • Loading branch information...
commit 960d0a00f9fe4c40469d92ba34f896d1b420fbe6 1 parent 737917f
@jfhbrook authored
View
15 app.js
@@ -1,17 +1,16 @@
var path = require('path'),
- utile = require('utile');
+ utile = require('utile'),
+ colors = require('colors');
utile.inspect = require('util').inspect;
var flatiron = require('flatiron'),
app = flatiron.app;
-// This is a new library used for running browserify bundle jobs for us.
-var bundler = require('./bundler');
+var bundler = require('./lib/bundler');
app.config.file({ file: path.join(__dirname, 'config.json') });
-// Note that 'bundler' is written to work as a flatiron plugin.
app.use(flatiron.plugins.http);
app.use(bundler);
@@ -19,20 +18,16 @@ app.router.path('/', function () {
this.get(function () {
this.res.writeHead(200, { 'content-type': 'text/plain' });
this.res.write('Welcome to the Browserify CDN! You probably want to post. Ex:\n\n');
- this.res.end(' curl -X POST -d \'var traverse = require("traverse");\' address:3600\n');
+ this.res.end(' curl -X POST -d \'var traverse = require("traverse");\' localhost:3600\n');
});
this.post(function () {
- // If the request body doesn't have the property we expect, it's assumed
- // to be raw javascript. Note that the raw unparsed body is buffered into
- // req.chunks (as a Buffer).
var req = this.req,
res = this.res,
js = req.body.js ? req.body.js : req.chunks.toString();
- // 'app.bundler' was created when we attached 'bundler'.
- app.bundler.bundle(js, function (err, data) {
+ app.bundler.get(js, function (err, data) {
if (err) {
return res.json(500, {
success: false,
View
3  config.json
@@ -1,3 +0,0 @@
-{
- "port": "3600"
-}
View
43 bundler.js → lib/bundler/bundler.js
@@ -2,21 +2,17 @@ var EventEmitter2 = require('eventemitter2').EventEmitter2,
browserify = require('browserify'),
detective = require('detective'),
npm = require('npm'),
- crypto = require('crypto'),
- util = require('utile');
+ uglify = require('uglify-js'),
+ util = require('utile'),
+ common = require('../common');
-// I wrote the Bundler as a constructor with prototype methods.
-// I find that it's a good fit for stateful problems.
var Bundler = module.exports = function (opts) {
-
- // This lets you create a new Bundler without the 'new' keyword.
if (!(this instanceof Bundler)) {
return new Bundler;
}
var bundler = this;
- // Set the bundler's persistent options here. Also handle defaults.
bundler.options = opts || {};
bundler.options.npm = bundler.options.npm || {};
@@ -25,16 +21,17 @@ var Bundler = module.exports = function (opts) {
bundler.options.cache = true;
}
- // Bundler inherits from an EE2 with wildcards and the :: delimiter.
+ // Add filtering with uglify
+ if (!bundler.options.filter) {
+ bundler.options.filter = uglify;
+ }
+
EventEmitter2.call(this, {
wildcard: true,
delimiter: '::',
maxListeners: 0
})
- // Bundler requires a loaded npm in order to work properly.
- // The rest of the code in the constructor sets this up and emits events
- // to signal when it's ready.
bundler.ready = false;
npm.load({}, function (err) {
@@ -51,46 +48,40 @@ var Bundler = module.exports = function (opts) {
util.inherits(Bundler, EventEmitter2);
-// The 'bundle' method is what actually attempts to bundle your project.
Bundler.prototype.bundle = function (src, cb) {
var bundler = this,
modules = detective(src);
- // We used 'detective' to get a list of needed modules, so that we can make
- // sure they're installed before trying to browserify.
npm.commands.install(modules, function (err) {
if (err) {
cb(err);
}
- var bundle;
+ var bundle, doc;
- // Attempt to browserify the passed-in javascript source
try {
- bundle = browserify(this.options)
+ bundle = browserify(bundler.options)
.addEntry('index.js', { body: src || '' })
.bundle();
- // Hit the callback with our complete bundle object.
- cb(null, {
- src: src,
- md5: crypto.createHash('md5').update(src).digest('base64'),
+ doc = {
+ source: src,
+ md5: common.md5(src),
bundle: bundle
- });
+ };
}
catch (err) {
- cb(err);
+ return cb(err);
}
+
+ cb(null, doc);
});
};
-// Bundler can be used as a broadway plugin.
Bundler.attach = function (opts) {
this.bundler = new Bundler(opts);
}
-// The major win we get from making Bundler attachable is that we can integrate
-// its initialization step with our app.
Bundler.init = function (done) {
var npm = 'npm'.red.inverse;
View
157 lib/bundler/cache.js
@@ -0,0 +1,157 @@
+var Bundler = require('./bundler'),
+ common = require('../common'),
+ resourceful = require('resourceful'),
+ errs = require('errs'),
+ colors = require('colors'),
+ utile = require('utile');
+
+// Cached bundle resources.
+var CachedBundle = resourceful.define('cachedBundle', function () {
+ this.string('md5');
+ this.string('source');
+ this.string('bundle');
+});
+
+// A version of the bundler that caches. Use `.get(text)`.
+var CachingBundler = module.exports = function (opts) {
+ // This lets you create a new caching Bundler without the 'new' keyword.
+ if (!(this instanceof CachingBundler)) {
+ return new CachingBundler;
+ }
+
+ opts = opts || {};
+ Bundler.call(this, opts);
+ // Set up resourceful to use couchdb.
+ resourceful.use('couchdb', opts);
+}
+
+utile.inherits(CachingBundler, Bundler);
+
+CachingBundler.prototype.get = function (src, cb) {
+ var self = this,
+ md5 = common.md5(src);
+
+ // This logging function simply adds some extra text to the message and
+ // then emits it with the event 'log::lvl' (ex: 'log::info').
+ function log (lvl, msg) {
+ var restOf = [].slice.call(arguments, 2),
+ data = utile.format.apply(
+ null,
+ [ '(md5 `%s`) '.cyan + msg, md5 ].concat(restOf)
+ );
+
+ self.emit('log::' + lvl, data);
+ }
+
+ log('info', 'Trying to find bundle...');
+
+ // Check to see if the file already exists in the cache, based on md5 hash of
+ // the source text. This logic accounts for the possibility of hash collisions
+ // and multiple versions of the same bundle.
+ CachedBundle.find({ md5: md5 }, function (err, bundles) {
+ if (err) {
+ return cb(realError(err));
+ }
+
+ var matches;
+
+ if (!bundles.length) {
+ // No cached bundles found. We should bundle.
+ log('warn', 'Not found.');
+ return bundle(src);
+ }
+
+ // Log the document id's for found bundles. If there is more than one
+ // bundle, it means there was either an md5 hash collision or multiple
+ // documents for the same bundle exist.
+ log('info', 'Found bundle(s): %j', bundles.map(function (doc) {
+ return doc._id;
+ }));
+
+ // Try to find one that has the same text.
+ matches = bundles.filter(function (bundle) {
+ return bundle.source == src;
+ });
+
+ // Any answer other than 1 means an actual md5 hash collison.
+ switch (matches.length) {
+ case 1:
+ log('info', 'Match found.');
+ finish(null, matches[0]);
+ break;
+ case 0:
+ log('info', 'Hash collision with existing document(s) %j', matches);
+ bundle(src);
+ break;
+ default:
+ log('Multiple documents describing this document exist.');
+ log('Showing the first one.');
+ finish(null, matches[0]);
+ break;
+ }
+ });
+
+ // This function is called when we need to bundle. It addition to logging, it
+ // also attempts to cache the resulting bundle.
+ function bundle (src) {
+ log('info', 'Bundling.');
+ self.bundle(src, function (err, doc) {
+ if (err) {
+ return cb(err);
+ }
+
+ // Here, we create a new CachedBundle, which maps to our data store.
+ var bundle = new CachedBundle(doc);
+
+ // This is how you save resources with resourceful.
+ log('info', 'Caching bundle for next time...')
+ CachedBundle.save(bundle, function (err, bundle) {
+ if (err) {
+ log('error', 'Error while caching bundle');
+ }
+ return finish(realError(err), bundle);
+ });
+
+ });
+ }
+
+ // This is pretty much the last step, whether the doc came from the cache
+ // or from a bundling.
+ function finish (err, bundle) {
+ if (err) {
+ return cb(err);
+ }
+
+ log('info', 'Returning bundle.');
+ cb(null, bundle);
+ }
+
+};
+
+
+// Attach the caching bundler, passing in our config options for the couch.
+// Also emits logging events to the app.
+CachingBundler.attach = function (options) {
+ var app = this;
+
+ this.bundler = new CachingBundler(utile.mixin(
+ options,
+ this.config.get('couch')
+ ));
+
+ // Little known fact about flatiron: The logging plugin will log messages
+ // emitted on the `log` namespace of the core app, which is an instance of
+ // EventEmitter2.
+ app.bundler.on('log::**', function (msg) {
+ app.emit(this.event, msg);
+ });
+
+};
+
+CachingBundler.init = Bundler.init;
+
+// A helper because, unfortunately, sometimes cradle returns errors that
+// aren't of type Error and look like [object Object] when thrown.
+function realError(err) {
+ return err ? errs.merge(new Error(err.message || err.reason), err) : err;
+}
View
1  lib/bundler/index.js
@@ -0,0 +1 @@
+module.exports = require('./cache');
View
5 lib/common.js
@@ -0,0 +1,5 @@
+var crypto = require('crypto');
+
+exports.md5 = function md5 (text) {
+ return crypto.createHash('md5').update(text).digest('base64');
+};
View
21 package.json
@@ -1,16 +1,27 @@
{
"name": "browserify-cdn",
- "version": "0.0.0",
- "scripts": {
- "start": "node ./app.js"
- },
+ "version": "0.0.0-3",
+ "start": "node ./app.js",
+ "private": true,
"dependencies": {
"flatiron": "0.1.16",
"union": "0.3.0",
+ "resourceful": "0.1.10",
+ "ecstatic": "0.1.6",
"utile": "0.0.10",
+ "errs": "0.2.x",
"eventemitter2": "0.4.x",
"browserify": "1.10.6",
"detective": "0.1.x",
- "npm": "1.1.x"
+ "npm": "1.1.x",
+ "uglify-js": "1.2.x",
+ "colors": "*"
+ },
+ "subdomain": "browserify",
+ "scripts": {
+ "start": "node ./app.js"
+ },
+ "engines": {
+ "node": "0.6.x"
}
}
Please sign in to comment.
Something went wrong with that request. Please try again.