Skip to content

Commit

Permalink
Convert all esprima.parse calls to go through an adapter so xpconnect…
Browse files Browse the repository at this point in the history 
… can use native parser for all ops
  • Loading branch information
@jrburke
jrburke committed Jun 29, 2013
1 parent 210eef0 commit 633962d
Show file tree
Hide file tree
Showing 9 changed files with 168 additions and 106 deletions.
6 changes: 5 additions & 1 deletion build/example.build.js
View file
Expand Up @@ -472,11 +472,15 @@
fileExclusionRegExp: /^\./,

//By default, comments that have a license in them are preserved in the
//output. However, for a larger built files there could be a lot of
//output when a minifier is used in the "optimize" option.
//However, for a larger built files there could be a lot of
//comment files that may be better served by having a smaller comment
//at the top of the file that points to the list of all the licenses.
//This option will turn off the auto-preservation, but you will need
//work out how best to surface the license information.
//NOTE: As of 2.1.7, if using xpcshell to run the optimizer, it cannot
//parse out comments since its native Reflect parser is used, and does
//not have the same comments option support as esprima.
preserveLicenseComments: true,

//Sets the logging level. It is a number. If you want "silent" running,
Expand Down
21 changes: 21 additions & 0 deletions build/jslib/esprimaAdapter.js
View file
@@ -0,0 +1,21 @@
/**
* @license Copyright (c) 2012, The Dojo Foundation All Rights Reserved.
* Available via the MIT or new BSD license.
* see: http://github.com/jrburke/requirejs for details
*/

/*global define, Reflect */

/*
* xpcshell has a smaller stack on linux and windows (1MB vs 9MB on mac),
* and the recursive nature of esprima can cause it to overflow pretty
* quickly. So favor it built in Reflect parser:
* https://developer.mozilla.org/en-US/docs/SpiderMonkey/Parser_API
*/
define(['./esprima', 'env'], function (esprima, env) {
if (env.get() === 'xpconnect' && typeof Reflect !== 'undefined') {
return Reflect;
} else {
return esprima;
}
});
2 changes: 0 additions & 2 deletions build/jslib/optimize.js
View file
Expand Up @@ -239,8 +239,6 @@ function (lang, logger, envOptimize, file, parse,
throw new Error('Cannot parse file: ' + fileName + ' for comments. Skipping it. Error is:\n' + e.toString());
}
}
if (fileName.indexOf('sourcemap') !== -1)
debugger;

fileContents = licenseContents + optFunc(fileName,
fileContents,
Expand Down
77 changes: 56 additions & 21 deletions build/jslib/parse.js
View file
Expand Up @@ -7,7 +7,7 @@
/*jslint plusplus: true */
/*global define: false */

define(['./esprima', 'lang'], function (esprima, lang) {
define(['./esprimaAdapter', 'lang'], function (esprima, lang) {
'use strict';

function arrayToString(ary) {
Expand Down Expand Up @@ -355,10 +355,10 @@ define(['./esprima', 'lang'], function (esprima, lang) {
*/
parse.findConfig = function (fileContents) {
/*jslint evil: true */
var jsConfig, foundRange, foundConfig, quote, quoteMatch,
var jsConfig, foundConfig, stringData, foundRange, quote, quoteMatch,
quoteRegExp = /(:\s|\[\s*)(['"])/,
astRoot = esprima.parse(fileContents, {
range: true
loc: true
});

traverse(astRoot, function (node) {
Expand All @@ -373,21 +373,24 @@ define(['./esprima', 'lang'], function (esprima, lang) {
arg = node[argPropName] && node[argPropName][0];

if (arg && arg.type === 'ObjectExpression') {
jsConfig = parse.nodeToString(fileContents, arg);
foundRange = arg.range;
stringData = parse.nodeToString(fileContents, arg);
jsConfig = stringData.value;
foundRange = stringData.range;
return false;
}
} else {
arg = parse.getRequireObjectLiteral(node);
if (arg) {
jsConfig = parse.nodeToString(fileContents, arg);
foundRange = arg.range;
stringData = parse.nodeToString(fileContents, arg);
jsConfig = stringData.value;
foundRange = stringData.range;
return false;
}
}
});

if (jsConfig) {
// Eval the config
quoteMatch = quoteRegExp.exec(jsConfig);
quote = (quoteMatch && quoteMatch[2]) || '"';
foundConfig = eval('(' + jsConfig + ')');
Expand Down Expand Up @@ -422,26 +425,39 @@ define(['./esprima', 'lang'], function (esprima, lang) {
* @return {String} the fileContents with the namespace applied
*/
parse.renameNamespace = function (fileContents, ns) {
var ranges = [],
var lines,
locs = [],
astRoot = esprima.parse(fileContents, {
range: true
loc: true
});

parse.recurse(astRoot, function (callName, config, name, deps, node) {
ranges.push(node.range);
locs.push(node.loc);
//Do not recurse into define functions, they should be using
//local defines.
return callName !== 'define';
}, {});

//Go backwards through the found ranges, adding in the namespace name
//in front.
ranges.reverse();
ranges.forEach(function (range) {
fileContents = fileContents.substring(0, range[0]) +
ns + '.' +
fileContents.substring(range[0]);
});
if (locs.length) {
lines = fileContents.split('\n');

//Go backwards through the found locs, adding in the namespace name
//in front.
locs.reverse();
locs.forEach(function (loc) {
var startIndex = loc.start.column,
//start.line is 1-based, not 0 based.
lineIndex = loc.start.line - 1,
line = lines[lineIndex];

lines[lineIndex] = line.substring(0, startIndex) +
ns + '.' +
line.substring(startIndex,
line.length);
});

fileContents = lines.join('\n');
}

return fileContents;
};
Expand Down Expand Up @@ -762,14 +778,29 @@ define(['./esprima', 'lang'], function (esprima, lang) {
/**
* Converts an AST node into a JS source string by extracting
* the node's location from the given contents string. Assumes
* esprima.parse() with ranges was done.
* esprima.parse() with loc was done.
* @param {String} contents
* @param {Object} node
* @returns {String} a JS source string.
*/
parse.nodeToString = function (contents, node) {
var range = node.range;
return contents.substring(range[0], range[1]);
var loc = node.loc,
lines = contents.split('\n'),
preamble = lines.slice(0, loc.start.line - 1).join('\n') + '\n' +
lines[loc.start.line - 1].substring(0, loc.start.column),
extracted = lines[loc.start.line - 1].substring(loc.start.column) +
'\n' +
lines.slice(loc.start.line, loc.end.line - 1).join('\n') +
'\n' +
lines[loc.end.line - 1].substring(0, loc.end.column);

return {
value: extracted,
range: [
preamble.length,
preamble.length + extracted.length
]
};
};

/**
Expand All @@ -780,6 +811,10 @@ define(['./esprima', 'lang'], function (esprima, lang) {
*/
parse.getLicenseComments = function (fileName, contents) {
var commentNode, refNode, subNode, value, i, j,
//xpconnect's Reflect does not support comment or range, but
//prefer continued operation vs strict parity of operation,
//as license comments can be expressed in other ways, like
//via wrap args, or linked via sourcemaps.
ast = esprima.parse(contents, {
comment: true,
range: true
Expand Down
12 changes: 5 additions & 7 deletions build/jslib/transform.js
View file
Expand Up @@ -4,10 +4,10 @@
* see: http://github.com/jrburke/requirejs for details
*/

/*global define, Reflect */
/*global define */

define([ './esprima', './parse', 'logger', 'lang', 'env'],
function (esprima, parse, logger, lang, env) {
define([ './esprimaAdapter', './parse', 'logger', 'lang'],
function (esprima, parse, logger, lang) {
'use strict';
var transform,
baseIndentRegExp = /^([ \t]+)/,
Expand All @@ -27,16 +27,14 @@ function (esprima, parse, logger, lang, env) {
toTransport: function (namespace, moduleName, path, contents, onFound, options) {
options = options || {};

var parser, astRoot, contentLines, modLine,
var astRoot, contentLines, modLine,
foundAnon,
scanCount = 0,
scanReset = false,
defineInfos = [];

try {
parser = env.get() === 'xpconnect' && typeof Reflect !== 'undefined' ?
Reflect : esprima;
astRoot = parser.parse(contents, {
astRoot = esprima.parse(contents, {
loc: true
});
} catch (e) {
Expand Down
89 changes: 46 additions & 43 deletions build/tests/builds.js
View file
@@ -1,7 +1,7 @@
/*jslint plusplus: true, nomen: true */
/*global define: false, require: false, doh: false */

define(['build', 'env!env/file'], function (build, file) {
define(['build', 'env!env/file', 'env'], function (build, file, env) {
'use strict';

//Remove any old builds
Expand Down Expand Up @@ -442,62 +442,65 @@ define(['build', 'env!env/file'], function (build, file) {
);
doh.run();

doh.register("preserveLicense",
[
function preserveLicense(t) {
file.deleteFile("lib/comments/built.js");
//Skip in xpconnect's since Reflect's parser cannot maintain comments.
if (env.get() !== 'xpconnect') {
doh.register("preserveLicense",
[
function preserveLicense(t) {
file.deleteFile("lib/comments/built.js");

build(["lib/comments/build.js"]);
build(["lib/comments/build.js"]);

t.is(nol(c("lib/comments/expected.js")),
nol(c("lib/comments/built.js")));
t.is(nol(c("lib/comments/expected.js")),
nol(c("lib/comments/built.js")));

require._buildReset();
}
require._buildReset();
}

]
);
doh.run();
]
);
doh.run();

//Only keep unique comments
//https://github.com/jrburke/r.js/issues/251
doh.register("preserveLicenseUnique",
[
function preserveLicenseUnique(t) {
file.deleteFile("lib/comments/unique/built.js");
//Only keep unique comments
//https://github.com/jrburke/r.js/issues/251
doh.register("preserveLicenseUnique",
[
function preserveLicenseUnique(t) {
file.deleteFile("lib/comments/unique/built.js");

build(["lib/comments/unique/build.js"]);
build(["lib/comments/unique/build.js"]);

t.is(nol(c("lib/comments/unique/expected.js")),
nol(c("lib/comments/unique/built.js")));
t.is(nol(c("lib/comments/unique/expected.js")),
nol(c("lib/comments/unique/built.js")));

require._buildReset();
}
require._buildReset();
}

]
);
doh.run();
]
);
doh.run();

//Do not dupe parts of comments when at the end of the file.
//https://github.com/jrburke/r.js/issues/264
doh.register("preserveLicenseNoPartialDupe",
[
function preserveLicenseNoPartialDupe(t) {
file.deleteFile("lib/comments/noPartialDupe/built");
//Do not dupe parts of comments when at the end of the file.
//https://github.com/jrburke/r.js/issues/264
doh.register("preserveLicenseNoPartialDupe",
[
function preserveLicenseNoPartialDupe(t) {
file.deleteFile("lib/comments/noPartialDupe/built");

//Run two builds, this profile has keepBuildDir set to true.
build(["lib/comments/noPartialDupe/build.js"]);
build(["lib/comments/noPartialDupe/build.js"]);
//Run two builds, this profile has keepBuildDir set to true.
build(["lib/comments/noPartialDupe/build.js"]);
build(["lib/comments/noPartialDupe/build.js"]);

t.is(nol(c("lib/comments/noPartialDupe/expected.js")),
nol(c("lib/comments/noPartialDupe/built/underscore.js")));
t.is(nol(c("lib/comments/noPartialDupe/expected.js")),
nol(c("lib/comments/noPartialDupe/built/underscore.js")));

require._buildReset();
}
require._buildReset();
}

]
);
doh.run();
]
);
doh.run();
}


doh.register("nestedFind",
Expand Down

0 comments on commit 633962d

Please sign in to comment.