diff --git a/build/example.build.js b/build/example.build.js index 91e45e6f..9e9d41c5 100644 --- a/build/example.build.js +++ b/build/example.build.js @@ -472,11 +472,15 @@ fileExclusionRegExp: /^\./, //By default, comments that have a license in them are preserved in the - //output. However, for a larger built files there could be a lot of + //output when a minifier is used in the "optimize" option. + //However, for a larger built files there could be a lot of //comment files that may be better served by having a smaller comment //at the top of the file that points to the list of all the licenses. //This option will turn off the auto-preservation, but you will need //work out how best to surface the license information. + //NOTE: As of 2.1.7, if using xpcshell to run the optimizer, it cannot + //parse out comments since its native Reflect parser is used, and does + //not have the same comments option support as esprima. preserveLicenseComments: true, //Sets the logging level. It is a number. If you want "silent" running, diff --git a/build/jslib/esprimaAdapter.js b/build/jslib/esprimaAdapter.js new file mode 100644 index 00000000..6e78f015 --- /dev/null +++ b/build/jslib/esprimaAdapter.js @@ -0,0 +1,21 @@ +/** + * @license Copyright (c) 2012, The Dojo Foundation All Rights Reserved. + * Available via the MIT or new BSD license. + * see: http://github.com/jrburke/requirejs for details + */ + +/*global define, Reflect */ + +/* + * xpcshell has a smaller stack on linux and windows (1MB vs 9MB on mac), + * and the recursive nature of esprima can cause it to overflow pretty + * quickly. So favor it built in Reflect parser: + * https://developer.mozilla.org/en-US/docs/SpiderMonkey/Parser_API + */ +define(['./esprima', 'env'], function (esprima, env) { + if (env.get() === 'xpconnect' && typeof Reflect !== 'undefined') { + return Reflect; + } else { + return esprima; + } +}); diff --git a/build/jslib/optimize.js b/build/jslib/optimize.js index 6a964bda..9919fe3f 100644 --- a/build/jslib/optimize.js +++ b/build/jslib/optimize.js @@ -239,8 +239,6 @@ function (lang, logger, envOptimize, file, parse, throw new Error('Cannot parse file: ' + fileName + ' for comments. Skipping it. Error is:\n' + e.toString()); } } -if (fileName.indexOf('sourcemap') !== -1) - debugger; fileContents = licenseContents + optFunc(fileName, fileContents, diff --git a/build/jslib/parse.js b/build/jslib/parse.js index 67183ce3..4bf89ae3 100644 --- a/build/jslib/parse.js +++ b/build/jslib/parse.js @@ -7,7 +7,7 @@ /*jslint plusplus: true */ /*global define: false */ -define(['./esprima', 'lang'], function (esprima, lang) { +define(['./esprimaAdapter', 'lang'], function (esprima, lang) { 'use strict'; function arrayToString(ary) { @@ -355,10 +355,10 @@ define(['./esprima', 'lang'], function (esprima, lang) { */ parse.findConfig = function (fileContents) { /*jslint evil: true */ - var jsConfig, foundRange, foundConfig, quote, quoteMatch, + var jsConfig, foundConfig, stringData, foundRange, quote, quoteMatch, quoteRegExp = /(:\s|\[\s*)(['"])/, astRoot = esprima.parse(fileContents, { - range: true + loc: true }); traverse(astRoot, function (node) { @@ -373,21 +373,24 @@ define(['./esprima', 'lang'], function (esprima, lang) { arg = node[argPropName] && node[argPropName][0]; if (arg && arg.type === 'ObjectExpression') { - jsConfig = parse.nodeToString(fileContents, arg); - foundRange = arg.range; + stringData = parse.nodeToString(fileContents, arg); + jsConfig = stringData.value; + foundRange = stringData.range; return false; } } else { arg = parse.getRequireObjectLiteral(node); if (arg) { - jsConfig = parse.nodeToString(fileContents, arg); - foundRange = arg.range; + stringData = parse.nodeToString(fileContents, arg); + jsConfig = stringData.value; + foundRange = stringData.range; return false; } } }); if (jsConfig) { + // Eval the config quoteMatch = quoteRegExp.exec(jsConfig); quote = (quoteMatch && quoteMatch[2]) || '"'; foundConfig = eval('(' + jsConfig + ')'); @@ -422,26 +425,39 @@ define(['./esprima', 'lang'], function (esprima, lang) { * @return {String} the fileContents with the namespace applied */ parse.renameNamespace = function (fileContents, ns) { - var ranges = [], + var lines, + locs = [], astRoot = esprima.parse(fileContents, { - range: true + loc: true }); parse.recurse(astRoot, function (callName, config, name, deps, node) { - ranges.push(node.range); + locs.push(node.loc); //Do not recurse into define functions, they should be using //local defines. return callName !== 'define'; }, {}); - //Go backwards through the found ranges, adding in the namespace name - //in front. - ranges.reverse(); - ranges.forEach(function (range) { - fileContents = fileContents.substring(0, range[0]) + - ns + '.' + - fileContents.substring(range[0]); - }); + if (locs.length) { + lines = fileContents.split('\n'); + + //Go backwards through the found locs, adding in the namespace name + //in front. + locs.reverse(); + locs.forEach(function (loc) { + var startIndex = loc.start.column, + //start.line is 1-based, not 0 based. + lineIndex = loc.start.line - 1, + line = lines[lineIndex]; + + lines[lineIndex] = line.substring(0, startIndex) + + ns + '.' + + line.substring(startIndex, + line.length); + }); + + fileContents = lines.join('\n'); + } return fileContents; }; @@ -762,14 +778,29 @@ define(['./esprima', 'lang'], function (esprima, lang) { /** * Converts an AST node into a JS source string by extracting * the node's location from the given contents string. Assumes - * esprima.parse() with ranges was done. + * esprima.parse() with loc was done. * @param {String} contents * @param {Object} node * @returns {String} a JS source string. */ parse.nodeToString = function (contents, node) { - var range = node.range; - return contents.substring(range[0], range[1]); + var loc = node.loc, + lines = contents.split('\n'), + preamble = lines.slice(0, loc.start.line - 1).join('\n') + '\n' + + lines[loc.start.line - 1].substring(0, loc.start.column), + extracted = lines[loc.start.line - 1].substring(loc.start.column) + + '\n' + + lines.slice(loc.start.line, loc.end.line - 1).join('\n') + + '\n' + + lines[loc.end.line - 1].substring(0, loc.end.column); + + return { + value: extracted, + range: [ + preamble.length, + preamble.length + extracted.length + ] + }; }; /** @@ -780,6 +811,10 @@ define(['./esprima', 'lang'], function (esprima, lang) { */ parse.getLicenseComments = function (fileName, contents) { var commentNode, refNode, subNode, value, i, j, + //xpconnect's Reflect does not support comment or range, but + //prefer continued operation vs strict parity of operation, + //as license comments can be expressed in other ways, like + //via wrap args, or linked via sourcemaps. ast = esprima.parse(contents, { comment: true, range: true diff --git a/build/jslib/transform.js b/build/jslib/transform.js index 73acdf2f..34c0bb95 100644 --- a/build/jslib/transform.js +++ b/build/jslib/transform.js @@ -4,10 +4,10 @@ * see: http://github.com/jrburke/requirejs for details */ -/*global define, Reflect */ +/*global define */ -define([ './esprima', './parse', 'logger', 'lang', 'env'], -function (esprima, parse, logger, lang, env) { +define([ './esprimaAdapter', './parse', 'logger', 'lang'], +function (esprima, parse, logger, lang) { 'use strict'; var transform, baseIndentRegExp = /^([ \t]+)/, @@ -27,16 +27,14 @@ function (esprima, parse, logger, lang, env) { toTransport: function (namespace, moduleName, path, contents, onFound, options) { options = options || {}; - var parser, astRoot, contentLines, modLine, + var astRoot, contentLines, modLine, foundAnon, scanCount = 0, scanReset = false, defineInfos = []; try { - parser = env.get() === 'xpconnect' && typeof Reflect !== 'undefined' ? - Reflect : esprima; - astRoot = parser.parse(contents, { + astRoot = esprima.parse(contents, { loc: true }); } catch (e) { diff --git a/build/tests/builds.js b/build/tests/builds.js index d1bdd492..424d357f 100644 --- a/build/tests/builds.js +++ b/build/tests/builds.js @@ -1,7 +1,7 @@ /*jslint plusplus: true, nomen: true */ /*global define: false, require: false, doh: false */ -define(['build', 'env!env/file'], function (build, file) { +define(['build', 'env!env/file', 'env'], function (build, file, env) { 'use strict'; //Remove any old builds @@ -442,62 +442,65 @@ define(['build', 'env!env/file'], function (build, file) { ); doh.run(); - doh.register("preserveLicense", - [ - function preserveLicense(t) { - file.deleteFile("lib/comments/built.js"); + //Skip in xpconnect's since Reflect's parser cannot maintain comments. + if (env.get() !== 'xpconnect') { + doh.register("preserveLicense", + [ + function preserveLicense(t) { + file.deleteFile("lib/comments/built.js"); - build(["lib/comments/build.js"]); + build(["lib/comments/build.js"]); - t.is(nol(c("lib/comments/expected.js")), - nol(c("lib/comments/built.js"))); + t.is(nol(c("lib/comments/expected.js")), + nol(c("lib/comments/built.js"))); - require._buildReset(); - } + require._buildReset(); + } - ] - ); - doh.run(); + ] + ); + doh.run(); - //Only keep unique comments - //https://github.com/jrburke/r.js/issues/251 - doh.register("preserveLicenseUnique", - [ - function preserveLicenseUnique(t) { - file.deleteFile("lib/comments/unique/built.js"); + //Only keep unique comments + //https://github.com/jrburke/r.js/issues/251 + doh.register("preserveLicenseUnique", + [ + function preserveLicenseUnique(t) { + file.deleteFile("lib/comments/unique/built.js"); - build(["lib/comments/unique/build.js"]); + build(["lib/comments/unique/build.js"]); - t.is(nol(c("lib/comments/unique/expected.js")), - nol(c("lib/comments/unique/built.js"))); + t.is(nol(c("lib/comments/unique/expected.js")), + nol(c("lib/comments/unique/built.js"))); - require._buildReset(); - } + require._buildReset(); + } - ] - ); - doh.run(); + ] + ); + doh.run(); - //Do not dupe parts of comments when at the end of the file. - //https://github.com/jrburke/r.js/issues/264 - doh.register("preserveLicenseNoPartialDupe", - [ - function preserveLicenseNoPartialDupe(t) { - file.deleteFile("lib/comments/noPartialDupe/built"); + //Do not dupe parts of comments when at the end of the file. + //https://github.com/jrburke/r.js/issues/264 + doh.register("preserveLicenseNoPartialDupe", + [ + function preserveLicenseNoPartialDupe(t) { + file.deleteFile("lib/comments/noPartialDupe/built"); - //Run two builds, this profile has keepBuildDir set to true. - build(["lib/comments/noPartialDupe/build.js"]); - build(["lib/comments/noPartialDupe/build.js"]); + //Run two builds, this profile has keepBuildDir set to true. + build(["lib/comments/noPartialDupe/build.js"]); + build(["lib/comments/noPartialDupe/build.js"]); - t.is(nol(c("lib/comments/noPartialDupe/expected.js")), - nol(c("lib/comments/noPartialDupe/built/underscore.js"))); + t.is(nol(c("lib/comments/noPartialDupe/expected.js")), + nol(c("lib/comments/noPartialDupe/built/underscore.js"))); - require._buildReset(); - } + require._buildReset(); + } - ] - ); - doh.run(); + ] + ); + doh.run(); + } doh.register("nestedFind", diff --git a/build/tests/parse.js b/build/tests/parse.js index 40b3f545..08d19325 100644 --- a/build/tests/parse.js +++ b/build/tests/parse.js @@ -1,7 +1,6 @@ -/*jslint */ /*global doh, define */ -define(['parse', 'env!env/file'], function (parse, file) { +define(['parse', 'env!env/file', 'env'], function (parse, file, env) { 'use strict'; function c(fileName) { @@ -278,33 +277,37 @@ define(['parse', 'env!env/file'], function (parse, file) { ); doh.run(); - doh.register('parseLicenseComments', - [ - function parseLicenseComments(t) { - var contents, - manyCommentsName = 'parse/comments/manyComments.js', - multiLineName = 'parse/comments/multiLine.js', - multiSingleLineName = 'parse/comments/multiSingleLine.js', - expectedManyCommentsName = 'parse/comments/expected/manyComments.js', - outputManyCommentsName = 'parse/comments/output/manyComments.js', - expectedMultiLineName = 'parse/comments/expected/multiLine.js', - outputMultiLineName = 'parse/comments/output/multiLine.js', - expectedMultiSingleLineName = 'parse/comments/expected/multiSingleLine.js', - outputMultiSingleLineName = 'parse/comments/output/multiSingleLine.js'; - - contents = parse.getLicenseComments(manyCommentsName, file.readFile(manyCommentsName)).trim(); - //file.saveFile(outputManyCommentsName, contents); - t.is(file.readFile(expectedManyCommentsName).trim(), contents); - - contents = parse.getLicenseComments(multiLineName, file.readFile(multiLineName)).trim(); - //file.saveFile(outputMultiLineName, contents); - t.is(file.readFile(expectedMultiLineName).trim(), contents); - - contents = parse.getLicenseComments(multiSingleLineName, file.readFile(multiSingleLineName)).trim(); - //file.saveFile(outputMultiSingleLineName, contents); - t.is(file.readFile(expectedMultiSingleLineName).trim(), contents); - } - ] - ); - doh.run(); + + //Skip in xpconnect's since Reflect's parser cannot maintain comments. + if (env.get() !== 'xpconnect') { + doh.register('parseLicenseComments', + [ + function parseLicenseComments(t) { + var contents, + manyCommentsName = 'parse/comments/manyComments.js', + multiLineName = 'parse/comments/multiLine.js', + multiSingleLineName = 'parse/comments/multiSingleLine.js', + expectedManyCommentsName = 'parse/comments/expected/manyComments.js', + outputManyCommentsName = 'parse/comments/output/manyComments.js', + expectedMultiLineName = 'parse/comments/expected/multiLine.js', + outputMultiLineName = 'parse/comments/output/multiLine.js', + expectedMultiSingleLineName = 'parse/comments/expected/multiSingleLine.js', + outputMultiSingleLineName = 'parse/comments/output/multiSingleLine.js'; + + contents = parse.getLicenseComments(manyCommentsName, file.readFile(manyCommentsName)).trim(); + //file.saveFile(outputManyCommentsName, contents); + t.is(file.readFile(expectedManyCommentsName).trim(), contents); + + contents = parse.getLicenseComments(multiLineName, file.readFile(multiLineName)).trim(); + //file.saveFile(outputMultiLineName, contents); + t.is(file.readFile(expectedMultiLineName).trim(), contents); + + contents = parse.getLicenseComments(multiSingleLineName, file.readFile(multiSingleLineName)).trim(); + //file.saveFile(outputMultiSingleLineName, contents); + t.is(file.readFile(expectedMultiSingleLineName).trim(), contents); + } + ] + ); + doh.run(); + } }); \ No newline at end of file diff --git a/build/tests/transform.js b/build/tests/transform.js index 0dcd1a7b..55ef3ad8 100644 --- a/build/tests/transform.js +++ b/build/tests/transform.js @@ -1,4 +1,3 @@ -/*jslint */ /*global doh, define, java, Packages, Components */ define(['transform', 'env!env/file'], function (transform, file) { diff --git a/dist.js b/dist.js index fbccbe33..6380aab7 100644 --- a/dist.js +++ b/dist.js @@ -42,6 +42,7 @@ var fs = require('fs'), 'build/jslib/logger.js', 'build/jslib/blank.js', 'build/jslib/esprima.js', + 'build/jslib/esprimaAdapter.js', 'build/jslib/uglifyjs/consolidator.js', 'build/jslib/uglifyjs/parse-js.js', 'build/jslib/uglifyjs/squeeze-more.js',