Permalink
Browse files

Merge pull request #20 from danwrong/translations

Translations & Esprima
  • Loading branch information...
2 parents ddf032f + faff313 commit 45043535372e0facdbe432bfd716e293a3430685 @kennethkufluk kennethkufluk committed Jun 18, 2012
Showing with 320 additions and 100 deletions.
  1. +25 −24 lib/loadbuilder/analyzer.js
  2. +32 −29 lib/loadbuilder/asset.js
  3. +65 −6 lib/loadbuilder/builder.js
  4. +135 −0 lib/loadbuilder/file.js
  5. +5 −2 package.json
  6. +48 −29 test/analyzer.js
  7. +6 −7 test/asset.js
  8. +3 −3 test/builder.js
  9. +1 −0 test/modules/named.js
View
49 lib/loadbuilder/analyzer.js
@@ -1,21 +1,26 @@
-var parser = require('uglify-js').parser;
+var parser = require('esprima'),
+ file = require('./file');
function match(fragment, tree) {
var matches = [];
+ if (fragment && tree && Object.keys(fragment).every(function(fragmentKey, i) {
+ var item = fragment[fragmentKey],
+ subMatches;
- if ((fragment.length <= tree.length) && fragment.every(function(item, i) {
- var subMatches;
+ if ((typeof tree[fragmentKey] == 'undefined') || (tree[fragmentKey] === null)) {
+ return false;
+ }
if (item === null) {
- matches.push(tree[i]);
+ matches.push(tree[fragmentKey]);
return true;
}
- if (item === tree[i]) {
+ if (item === tree[fragmentKey]) {
return true;
}
- if (Array.isArray(tree[i]) && Array.isArray(item) && (subMatches = match(item, tree[i]))) {
+ if (typeof tree[fragmentKey] === 'object' && typeof item === 'object' && (subMatches = match(item, tree[fragmentKey]))) {
matches = matches.concat(subMatches);
return true;
}
@@ -30,32 +35,28 @@ function match(fragment, tree) {
function walk(matcher, tree, parent, index) {
var matches = [], m;
-
- if (Array.isArray(tree)) {
- if (m = match(matcher, tree)) {
- matches.push({
- parent: parent || null,
- index: index || 0,
- values: m
- });
- }
-
- tree.forEach(function(node, i) {
- matches = matches.concat(walk(matcher, node, tree, i));
+ if (m = match(matcher, tree)) {
+ matches.push({
+ parent: parent || null,
+ index: index || 0,
+ values: m
+ });
+ }
+ if (typeof tree == 'object') {
+ Object.keys(tree).forEach(function(node, i) {
+ if (tree[node]===null) return;
+ matches = matches.concat(walk(matcher, tree[node], tree, i));
});
}
return matches;
}
function analyze(matcher, source) {
// treeish can be source string, file object (from asset.js) or ast
- if (source.mtime && !source.ast) {
- source.ast = parser.parse(source.source);
- }
var tree;
- if (source.ast) {
- tree = source.ast;
- } else if (Array.isArray(source)) {
+ if (file.isFile(source)) {
+ tree = source.ast();
+ } else if (typeof source == 'object') {
tree = source;
} else {
tree = parser.parse(source);
View
61 lib/loadbuilder/asset.js
@@ -1,15 +1,17 @@
var util = require('./util'),
analyzer = require('./analyzer'),
- fs = require('fs'),
path = require('path'),
jshint = require('jshint').JSHINT,
- uglify = require("uglify-js");
+ file = require('./file');
-var USING = [ 'call', [ 'name', 'using' ], null ];
-var ARRAY_ARG = [ 'array', null ];
-var STRING_ARG = [ 'string', null ];
-var PROVIDE = [ 'call', [ 'name', 'provide' ], null ];
-var REQUIRE = [ 'call', [ 'name', 'require' ], [ [ 'string', null] ] ];
+var ARRAY_ARG = {
+ type: "ArrayExpression",
+ elements: null
+};
+var STRING_ARG = {
+ type: "Literal",
+ value: null
+};
var dependencyCache = {};
var dependencyCacheLastUpdate = {};
@@ -20,10 +22,9 @@ function Script(id) {
util.extend(Script.prototype, {
dependencies: function() {
-
var usings, dependencies = [];
- usings = analyzer.analyze(USING, this.fromFile());
+ usings = this.fromFile().usings();
usings.forEach(function(call) {
var args = call.values[0];
@@ -48,7 +49,6 @@ util.extend(Script.prototype, {
}
}, this);
}, this);
-
return dependencies;
},
lint: function(options) {
@@ -60,20 +60,20 @@ util.extend(Script.prototype, {
process.exit(1);
}
},
- toSource: function() {
- return this.deferWrapper(this.fromFile().source);
+ ast: function() {
+ return this.fromFile().ast();
},
- preProcess: function(data) {
- return this.builder.preProcessor ? this.builder.preProcessor(data) : data;
+ addTranslationMarkers: function(fnName) {
+ return this.fromFile().addTranslationMarkers(fnName);
+ },
+ toSource: function() {
+ return this.deferWrapper(this.fromFile().generate());
},
fromFile: function() {
- var fileInfo = fs.statSync(this.fullPath());
- if (!this.file || this.file.mtime != fileInfo.mtime.getTime()) {
- this.file = {
- mtime: fileInfo.mtime.getTime(),
- source: this.preProcess(fs.readFileSync(this.fullPath(), 'utf8'))
- };
+ if (!this.file) {
+ this.file = file(this.fullPath(), this.builder.preProcessor);
}
+ this.file.load();
return this.file;
},
fullPath: function() {
@@ -116,15 +116,15 @@ util.extend(Module.prototype, {
}
},
isCJS: function() {
- return !analyzer.analyze(PROVIDE, this.fromFile()).length;
+ return !this.fromFile().provides().length;
},
amdWrappedSource: function() {
var deps = ['module', 'require', 'exports'].concat(this.dependencies().map(function(d) { return d.id; })),
preamble = "define(" + JSON.stringify(this.id) + "," +
JSON.stringify(deps) + ",function(module, require, exports) {\n",
postamble = "\n});"
- return preamble + this.fromFile().source + postamble;
+ return preamble + this.fromFile().generate() + postamble;
},
dependencies: function() {
@@ -135,28 +135,31 @@ util.extend(Module.prototype, {
}
},
dependenciesFromRequire: function() {
- var requires = analyzer.analyze(REQUIRE, this.fromFile());
+ var requires = this.fromFile().requires();
return requires.map(function(r) {
- var asset = this.builder.matchAsset(r.values[0], false);
+ var asset = this.builder.matchAsset(r.values[0][0].value, false);
return (asset.length==0) ? null : asset;
}, this).filter(function(item){ return item!=null; });
},
addId: function() {
- var provides = analyzer.analyze(PROVIDE, this.fromFile()),
- tree = this.fromFile().ast,
+ var provides = this.fromFile().provides(),
+ tree = this.ast(),
provide;
if (provides.length > 0) {
provide = provides[0];
// TODO make this nice - maybe have a transform function?
if (analyzer.match(STRING_ARG, provide.values[0][0]) == null) {
- provide.parent[provide.index][2].unshift(['string', this.id]);
+ provide.parent.expression.arguments.unshift({
+ type: "Literal",
+ value: this.id
+ });
+ this.fromFile().taintAst();
}
}
-
- return uglify.uglify.gen_code(tree, { beautify: true });
+ return this.fromFile().generate();
}
});
View
71 lib/loadbuilder/builder.js
@@ -2,15 +2,23 @@ var util = require('./util'),
asset = require('./asset'),
path = require('path'),
fs = require('fs'),
- uglify = require("uglify-js");
+ uglify = require("uglify-js"),
+ crypto = require('crypto'),
+ mkdirp = require('mkdirp');
+
+function hashString(string) {
+ var md5sum = crypto.createHash('md5');
+ md5sum.update(string);
+ return md5sum.digest('hex');
+}
function collect(excluded, assets, includeDependencies) {
var collected = [];
assets.forEach(function(asset) {
if(!asset) {
- console.warn('Undefined asset in assets list');
- console.log(new Error().stack);
+ this.warn('Undefined asset in assets list');
+ this.log(new Error().stack);
return;
}
var deps = [];
@@ -24,7 +32,7 @@ function collect(excluded, assets, includeDependencies) {
collect(excluded, deps, includeDependencies)
).concat(asset);
}
- });
+ }, this);
return collected;
}
@@ -113,15 +121,13 @@ util.extend(Builder.prototype, {
},
mapAssets: function(assets) {
var mapped = [];
-
for (var i=0, asset; asset = assets[i]; i++) {
if (typeof asset == 'string') {
asset = this.matchAsset(asset);
}
mapped.push(asset);
}
-
return mapped;
},
lint: function(options) {
@@ -155,6 +161,10 @@ util.extend(Builder.prototype, {
toSource: function() {
var source = this.collectedAssets().map(function(a) {
this.log('* ' + a.id);
+ if (this.i18nFn && !a.i18n) {
+ a.addTranslationMarkers(this.i18nFn);
+ a.i18n = true;
+ }
return a.toSource();
}, this).join('\n');
@@ -190,6 +200,55 @@ util.extend(Builder.prototype, {
},
collectedAssets: function() {
return dedupe(collect(this.excludes, this.assets, this.options.includeDependencies));
+ },
+ publishi18n: function(folder, outputName, translations, cb){
+ var bundle = this;
+ // Collect the bundle dependencies and build
+ this.addI18nMarkers('_').minify({ except: ['_', '$'] }).translate(translations).writei18n(folder, outputName, cb);
+ },
+ addI18nMarkers: function(underscore) {
+ // replace _() strings with something that's easy to regex for
+ this.i18nFn = underscore; // name of i18n function
+ return this;
+ },
+ translate: function(translations) {
+ this.translations = translations;
+ return this;
+ },
+ writei18n: function(folder, fileName, success) {
+ var manifest = this.manifest(),
+ versions = {},
+ source = this.toSource();
+
+ if (!this.i18nFn || !this.translations) {
+ throw new Error('did not translate - check i18n function name and translation table');
+ }
+
+ Object.keys(this.translations).forEach(function(langCode) {
+ var fullPath = fileName.replace('<lang>', langCode);
+ var langSrc = source.replace(/____i18n____(.*?)____\/i18n____/g, function(fullMatch, subMatch) {
+ if (this.translations[langCode].hasOwnProperty(subMatch)) {
+ console.log('matched for', langCode, this.translations[langCode][subMatch]);
+ return this.translations[langCode][subMatch];
+ }
+ // revert to English if we have no string
+ console.log('using english', langCode, subMatch);
+ return subMatch;
+ }.bind(this));
+ var hash = hashString(langSrc);
+ fullPath = fullPath.replace('<hash>', hash);
+ versions[langCode] = fullPath;
+
+ fullPath = path.join(folder, langCode, fullPath);
+ mkdirp.sync(path.dirname(fullPath));
+ fs.writeFileSync(fullPath, langSrc, 'utf8');
+ }, this);
+
+ success(manifest, versions);
+
+ this.log('> ' + fileName);
+
+ return this;
}
});
View
135 lib/loadbuilder/file.js
@@ -0,0 +1,135 @@
+var util = require('./util'),
+ fs = require('fs'),
+ esprima = require('esprima'),
+ escodegen= require('escodegen');
+
+var USING = {
+ type: "CallExpression",
+ callee: {
+ type: "Identifier",
+ name: "using"
+ },
+ arguments: null
+};
+var REQUIRE = {
+ type: "CallExpression",
+ callee: {
+ type: "Identifier",
+ name: "require"
+ },
+ arguments: null
+};
+var PROVIDE = {
+ type: "CallExpression",
+ callee: {
+ type: "Identifier",
+ name: "provide"
+ },
+ arguments: null
+};
+
+var lbAnalyzer; // solution for circular ref
+
+var files = {};
+
+function File(fullPath, preProcessor) {
+ this.fullPath = fullPath;
+ this.preProcessor = preProcessor;
+ this.derived = {};
+ this.load();
+}
+
+util.extend(File.prototype, {
+ load: function() {
+ var fileInfo = fs.statSync(this.fullPath),
+ mtime = fileInfo.mtime.getTime();
+ if (!this.source || this.mtime != mtime) {
+ this.mtime = mtime;
+ this.source = this.preProcess(fs.readFileSync(this.fullPath, 'utf8'));
+ this.taint();
+ }
+ },
+ taint: function() {
+ this.derived = {};
+ },
+ taintAst: function() {
+ delete this.derived.generated;
+ },
+ preProcess: function(data) {
+ return this.preProcessor ? this.preProcessor(data) : data;
+ },
+ ast: function() {
+ if (!this.derived.esprimaAst) {
+ var ast = esprima.parse(this.source, {
+ range: true,
+ tokens: true,
+ comment: true
+ });
+ ast = escodegen.attachComments(ast, ast.comments, ast.tokens);
+ this.derived.esprimaAst = ast;
+ }
+ return this.derived.esprimaAst;
+ },
+ addTranslationMarkers: function(fnName) {
+ if (!this.derived.i18nAdded) {
+ var UNDERSCORE = {
+ type: "CallExpression",
+ callee: {
+ type: "Identifier",
+ name: fnName
+ },
+ arguments: null
+ },
+ tree = this.ast(),
+ underscores = this.analyzer().analyze(UNDERSCORE, tree);
+ // replace the underscore functions
+ underscores.forEach(function(underscore) {
+ var string = underscore.values[0];
+ string[0].value = '____i18n____' + string[0].value + '____/i18n____';
+ this.taintAst();
+ }, this);
+ this.derived.i18nAdded = true;
+ }
+ },
+ generate: function(options) {
+ if (!this.derived.generated) {
+ options = options || {};
+ var tree = this.ast();
+ this.source = this.derived.generated = escodegen.generate(tree, util.extend({ comment: true }, options));
+ }
+ return this.derived.generated;
+ },
+ usings: function() {
+ if (!this.derived.usings) {
+ this.derived.usings = this.analyzer().analyze(USING, this.ast());
+ }
+ return this.derived.usings;
+ },
+ requires: function() {
+ if (!this.derived.requires) {
+ this.derived.requires = this.analyzer().analyze(REQUIRE, this.ast());
+ }
+ return this.derived.requires;
+ },
+ provides: function() {
+ if (!this.derived.provides) {
+ this.derived.provides = this.analyzer().analyze(PROVIDE, this.ast());
+ }
+ return this.derived.provides;
+ },
+ analyzer: function() {
+ if (!lbAnalyzer) lbAnalyzer = require('./analyzer');
+ return lbAnalyzer;
+ }
+});
+
+function file(fullPath, preProcessor) {
+ if (files[fullPath]) return files[fullPath];
+ return files[fullPath] = new File(fullPath, preProcessor);
+}
+
+file.isFile = function(obj) {
+ return obj instanceof File;
+}
+
+module.exports = file;
View
7 package.json
@@ -1,18 +1,21 @@
{
"name": "loadbuilder",
- "version": "0.2.12",
+ "version": "0.3.0",
"description": "Combine and compress dependency chains created by Loadrunner",
"contributors": [{ "name": "Dan Webb", "email": "dan@danwebb.net" }, { "name": "Kenneth Kufluk", "email": "kenneth@kufluk.com" }],
"homepage": "https://github.com/danwrong/loadbuilder",
"directories" : { "lib" : "./lib" },
"main" : "./lib/loadbuilder/builder",
"engines": { "node": ">= 0.4.0" },
"dependencies": {
+ "esprima": "0.9.9",
+ "escodegen": "0.0.4",
"jshint": "0.5.9",
"opts": "1.2.2",
"uglify-js": "git://github.com/kennethkufluk/UglifyJS.git#loadbuilder",
"colors": "0.6.0-1",
- "glob": "3.1.9"
+ "glob": "3.1.9",
+ "mkdirp": "0.3.0"
},
"devDependencies": {
"expresso": ">=0.9.2"
View
77 test/analyzer.js
@@ -1,19 +1,34 @@
var analyzer = require('loadbuilder/analyzer'),
assert = require('assert');
-var tree = [1, 2, [3], [4, [5]], [5]], match, matches;
+var tree = {
+ a: 1,
+ b: 2,
+ c: [
+ 3,
+ 4,
+ {
+ d: 7
+ }
+ ],
+ e: {
+ f: 8
+ },
+ f: 9
+ },
+ match, matches;
module.exports = {
testMatchShouldReturnNullIfNoMatches: function() {
- match = analyzer.match(['not', 'here', 3556], tree);
+ match = analyzer.match({ not: 'here' }, tree);
assert.isNull(match);
},
testMatchShouldReturnArrayIfMatches: function() {
- match = analyzer.match([1, 2], tree);
+ match = analyzer.match({ a: 1, b: 2 }, tree);
assert.eql(match, []);
- match = analyzer.match([1, 2, [3]], tree);
+ match = analyzer.match({ a: 1, b: 2, c: [ 3 ] }, tree);
assert.eql(match, []);
match = analyzer.match(tree, tree);
@@ -22,55 +37,59 @@ module.exports = {
testMatchShouldReturnMatchedWildcardsInArray: function() {
- match = analyzer.match([null, 2], tree);
+ match = analyzer.match({ a: null }, tree);
assert.eql(match, [1]);
- match = analyzer.match([1, 2, [null]], tree);
- assert.eql(match, [3]);
+ match = analyzer.match({
+ a: 1,
+ b: 2,
+ c: null
+ }, tree);
+ assert.eql(match, [[3,4,{"d":7}]]);
- match = analyzer.match([null, 2, [null]], tree);
- assert.eql(match, [1, 3]);
- },
-
- testMatchShouldMatchArraysWithWildcard: function() {
-
- match = analyzer.match([1, 2, null], tree);
- assert.eql(match, [[3]]);
+ match = analyzer.match({ a: null, b: null }, tree);
+ assert.eql(match, [1, 2]);
},
testWalkShouldReturnArrayOfMatches: function() {
- matches = analyzer.walk([4], tree);
+ matches = analyzer.walk({ f: 8 }, tree);
assert.eql(matches, [
{ parent: tree, index: 3, values: [] }
]);
- matches = analyzer.walk([1, null], tree);
+ matches = analyzer.walk({ a: 1, b: null }, tree);
assert.eql(matches, [
{ parent: null, index: 0, values: [2] }
]);
- matches = analyzer.walk([4, [null]], tree);
+ matches = analyzer.walk({ b: 2, c: [null] }, tree);
assert.eql(matches, [
- { parent: tree, index: 3, values: [5] }
+ { parent: null, index: 0, values: [3] }
]);
- matches = analyzer.walk([5], tree);
+ matches = analyzer.walk({ f: null }, tree);
assert.eql(matches, [
- { parent: tree[3], index: 1, values: [] },
- { parent: tree, index: 4, values: [] }
+ { parent: null, index: 0, values: [9] },
+ { parent: tree, index: 3, values: [8] }
]);
},
testAnalyzeShouldWalkJSSource: function() {
var src = "function pow() { var thing = require('blah'); }";
- matches = analyzer.analyze(
- [ 'call',
- [ 'name', 'require' ],
- [
- ['string', null]
- ]
- ], src);
+ matches = analyzer.analyze({
+ "type": "CallExpression",
+ "callee": {
+ "type": "Identifier",
+ "name": "require"
+ },
+ "arguments": [
+ {
+ "type": "Literal",
+ "value": null
+ }
+ ]
+ }, src);
assert.eql(matches[0].values, ['blah']);
}
View
13 test/asset.js
@@ -18,13 +18,12 @@ module.exports = {
});
assert.equal("deferred('fixtures/simple.js', function() {\nalert('hello world');\n});", a.toSource());
},
- testShouldFindDependenciesForModule: function() {
+ testShouldFindDependenciesForScript: function() {
var a = new asset.Script('fixtures/dep1.js');
a.builder = builder({
docroot: __dirname
});
-
- assert.equal('dep1dep', a.dependencies()[0].id);
+ assert.equal('fixtures/dep1dep.js', a.dependencies()[0].id);
},
testShouldAddNameToAnonModule: function() {
var a = new asset.Module('anon');
@@ -33,7 +32,7 @@ module.exports = {
path: '/modules'
});
- assert.equal('provide("anon", function(exports) {\n exports("hi");\n});', a.toSource());
+ assert.equal("provide('anon', function (exports) {\n exports('hi');\n});", a.toSource());
},
testShouldNotAddNameToNamedModule: function() {
var a = new asset.Module('named');
@@ -42,7 +41,7 @@ module.exports = {
path: '/modules'
});
- assert.equal('provide("named", function(exports) {\n exports("hi");\n});', a.toSource());
+ assert.equal("/*! license */\nprovide('named', function (exports) {\n exports('hi');\n});", a.toSource());
},
testShouldFindDependenciesForModule: function() {
var a = new asset.Module('has_dep');
@@ -53,7 +52,7 @@ module.exports = {
assert.equal('anon', a.dependencies()[0].id);
},
- testShouldFindDependenciesForModule: function() {
+ testShouldFindDependenciesForModule2: function() {
var a = new asset.Module('common');
a.builder = builder({
docroot: __dirname,
@@ -71,6 +70,6 @@ module.exports = {
});
assert.equal('define("common",["module","require","exports","anon"],' +
- 'function(module, require, exports) {\nvar a = require("anon");\n});', a.toSource());
+ 'function(module, require, exports) {\nvar a = require(\'anon\');\n});', a.toSource());
}
}
View
6 test/builder.js
@@ -30,7 +30,7 @@ module.exports = {
testShouldBeAbleToExcludeALowLevelDep: function() {
assert.equal(
builder(opts).include('mod_with_dep').exclude('named').toSource(),
- "provide(\"mod_with_dep\", function(exports) {\n using(\"named\", function() {\n exports(3);\n });\n});"
+ "provide('mod_with_dep', function (exports) {\n using('named', function () {\n exports(3);\n });\n});"
);
},
testShouldCollectDependencies: function() {
@@ -69,7 +69,7 @@ module.exports = {
var a = builder(opts).include('mod_with_dep'),
result = builder(opts).include('mod_with_same_dep').exclude(a).toSource();
assert.equal(
- "provide(\"mod_with_same_dep\", function(exports) {\n using(\"named\", function() {\n exports(3);\n });\n});",
+ "provide('mod_with_same_dep', function (exports) {\n using('named', function () {\n exports(3);\n });\n});",
result
);
},
@@ -90,7 +90,7 @@ module.exports = {
},
testShouldBeAbleToSuccessFullyLoadANamedModule: function() {
assert.equal(
- "provide(\"named\", function(exports) {\n exports(\"hi\");\n});",
+ "/*! license */\nprovide('named', function (exports) {\n exports('hi');\n});",
builder(opts).include('named').toSource()
);
}
View
1 test/modules/named.js
@@ -1,3 +1,4 @@
+/*! license */
provide('named', function(exports) {
exports('hi');
});

0 comments on commit 4504353

Please sign in to comment.