Permalink
Browse files

adding support for optimizing all plugin resources. Still a bit exper…

…imental.
  • Loading branch information...
1 parent 5acd1bd commit 0f5c86c12d399439d4f464cf060c2f9f4b443841 @jrburke jrburke committed May 26, 2011
Showing with 234 additions and 51 deletions.
  1. +8 −0 build/example.build.js
  2. +73 −3 build/jslib/build.js
  3. +31 −7 build/jslib/optimize.js
  4. +77 −21 build/jslib/parse.js
  5. +4 −4 build/tests/parse.js
  6. +2 −1 build/tests/text.build.js
  7. +38 −15 require.js
  8. +1 −0 tasks.txt
@@ -124,6 +124,14 @@
//together.
skipModuleInsertion: false,
+ //If it is not a one file optimization, scan through all .js files in the
+ //output directory for any plugin resource dependencies, and if the plugin
+ //supports optimizing them as separate files, optimize them. Can be a
+ //slower optimization. Only use if there are some plugins that use things
+ //like XMLHttpRequest that do not work across domains, but the built code
+ //will be placed on another domain.
+ optimizeAllPluginResources: false,
+
//List the modules that will be optimized. All their immediate and deep
//dependencies will be included in the module's file when the build is
//done. If that module or any of its dependencies includes i18n bundles,
View
@@ -22,7 +22,8 @@ function (lang, logger, file, parse, optimize, pragma,
optimize: "uglify",
optimizeCss: "standard.keepLines",
inlineText: true,
- isBuild: true
+ isBuild: true,
+ optimizeAllPluginResources: false
};
/**
@@ -47,6 +48,13 @@ function (lang, logger, file, parse, optimize, pragma,
return dirName;
}
+ //Method used by plugin writeFile calls, defined up here to avoid
+ //jslint warning about "making a function in a loop".
+ function writeFile(name, contents) {
+ logger.trace('Saving plugin-optimized file: ' + name);
+ file.saveUtf8File(name, contents);
+ }
+
/**
* Main API entry point into the build. The args argument can either be
* an array of arguments (like the onese passed on a command-line),
@@ -105,11 +113,13 @@ function (lang, logger, file, parse, optimize, pragma,
build._run = function (cmdConfig) {
var buildFileContents = "",
+ pluginCollector = {},
buildPaths, fileName, fileNames,
prop, paths, i,
baseConfig, config,
modules, builtModule, srcPath, buildContext,
- destPath;
+ destPath, moduleName, moduleMap, parentModuleMap, context,
+ resources, resource, pluginProcessed = {}, plugin;
//Can now run the patches to require.js to allow it to be used for
//build generation. Do it here instead of at the top of the module
@@ -294,9 +304,69 @@ function (lang, logger, file, parse, optimize, pragma,
//JS optimizations.
fileNames = file.getFilteredFileList(config.dir, /\.js$/, true);
for (i = 0; (fileName = fileNames[i]); i++) {
- optimize.jsFile(fileName, fileName, config);
+ //Generate the module name from the config.dir root.
+ moduleName = fileName.replace(config.dir, '');
+ //Get rid of the extension
+ moduleName = moduleName.substring(0, moduleName.length - 3);
+ optimize.jsFile(fileName, fileName, config, moduleName, pluginCollector);
}
+ //Normalize all the plugin resources.
+ context = require.s.contexts._;
+
+ for (moduleName in pluginCollector) {
+ if (pluginCollector.hasOwnProperty(moduleName)) {
+ parentModuleMap = context.makeModuleMap(moduleName);
+ resources = pluginCollector[moduleName];
+ for (i = 0; (resource = resources[i]); i++) {
+ moduleMap = context.makeModuleMap(resource, parentModuleMap);
+ if (!context.plugins[moduleMap.prefix]) {
+ //Set the value in context.plugins so it
+ //will be evaluated as a full plugin.
+ context.plugins[moduleMap.prefix] = true;
+
+ //Do not bother if the plugin is not available.
+ if (!file.exists(require.toUrl(moduleMap.prefix + '.js'))) {
+ continue;
+ }
+
+ //Rely on the require in the build environment
+ //to be synchronous
+ context.require([moduleMap.prefix]);
+
+ //Now that the plugin is loaded, redo the moduleMap
+ //since the plugin will need to normalize part of the path.
+ moduleMap = context.makeModuleMap(resource, parentModuleMap);
+ }
+
+ //Only bother with plugin resources that can be handled
+ //processed by the plugin, via support of the writeFile
+ //method.
+ if (!pluginProcessed[moduleMap.fullName]) {
+ //Only do the work if the plugin was really loaded.
+ //Using an internal access because the file may
+ //not really be loaded.
+ plugin = context.defined[moduleMap.prefix];
+ if (plugin && plugin.writeFile) {
+ plugin.writeFile(
+ moduleMap.prefix,
+ moduleMap.name,
+ require,
+ writeFile,
+ context.config
+ );
+ }
+
+ pluginProcessed[moduleMap.fullName] = true;
+ }
+ }
+
+ }
+ }
+
+ //console.log('PLUGIN COLLECTOR: ' + JSON.stringify(pluginCollector, null, " "));
+
+
//All module layers are done, write out the build.txt file.
file.saveUtf8File(config.dir + "build.txt", buildFileContents);
}
@@ -7,8 +7,10 @@
/*jslint plusplus: false, nomen: false, regexp: false, strict: false */
/*global define: false */
-define([ 'lang', 'logger', 'env!env/optimize', 'env!env/file', 'uglifyjs/index'],
-function (lang, logger, envOptimize, file, uglify) {
+define([ 'lang', 'logger', 'env!env/optimize', 'env!env/file', 'parse',
+ 'uglifyjs/index'],
+function (lang, logger, envOptimize, file, parse,
+ uglify) {
var optimize,
cssImportRegExp = /\@import\s+(url\()?\s*([^);]+)\s*(\))?([\w, ]*)(;)?/g,
@@ -127,23 +129,45 @@ function (lang, logger, envOptimize, file, uglify) {
optimize = {
/**
- * Optimizes a file that contains JavaScript content. It will inline
- * text plugin files and run it through Google Closure Compiler
- * minification, if the config options specify it.
+ * Optimizes a file that contains JavaScript content. Optionally collects
+ * plugin resources mentioned in a file, and then passes the content
+ * through an minifier if one is specified via config.optimize.
*
* @param {String} fileName the name of the file to optimize
* @param {String} outFileName the name of the file to use for the
* saved optimized content.
* @param {Object} config the build config object.
+ * @param {String} [moduleName] the module name to use for the file.
+ * Used for plugin resource collection.
+ * @param {Array} [pluginCollector] storage for any plugin resources
+ * found.
*/
- jsFile: function (fileName, outFileName, config) {
+ jsFile: function (fileName, outFileName, config, moduleName, pluginCollector) {
var parts = (config.optimize + "").split('.'),
optimizerName = parts[0],
keepLines = parts[1] === 'keepLines',
- fileContents, optFunc;
+ fileContents, optFunc, deps, i, dep;
fileContents = file.readFile(fileName);
+ //If there is a plugin collector, scan the file for plugin resources.
+ if (config.optimizeAllPluginResources && pluginCollector) {
+ try {
+ deps = parse.findDependencies(fileName, fileContents);
+ if (deps.length) {
+ for (i = 0; (dep = deps[i]); i++) {
+ if (dep.indexOf('!') !== -1) {
+ (pluginCollector[moduleName] ||
+ (pluginCollector[moduleName] = [])).push(dep);
+ }
+ }
+ }
+ } catch (e) {
+ logger.error('Parse error looking for plugin resources in ' +
+ fileName + ', skipping.');
+ }
+ }
+
//Optimize the JS files if asked.
if (optimizerName && optimizerName !== 'none') {
optFunc = envOptimize[optimizerName] || optimize.optimizers[optimizerName];
View
@@ -37,31 +37,46 @@ define(['uglifyjs/index'], function (uglify) {
/**
* Validates a node as being an object literal (like for i18n bundles)
- * or an array literal with just string members.
+ * or an array literal with just string members. If an array literal,
+ * only return array members that are full strings. So the caller of
+ * this function should use the return value as the new value for the
+ * node.
+ *
* This function does not need to worry about comments, they are not
* present in this AST.
+ *
+ * @param {Node} node an AST node.
+ *
+ * @returns {Node} an AST node to use for the valid dependencies.
+ * If null is returned, then it means the input node was not a valid
+ * dependency.
*/
function validateDeps(node) {
- var arrayArgs, i, dep;
+ var newDeps = ['array', []],
+ arrayArgs, i, dep;
+
+ if (!node) {
+ return null;
+ }
if (isObjectLiteral(node) || node[0] === 'function') {
- return true;
+ return node;
}
//Dependencies can be an object literal or an array.
if (!isArrayLiteral(node)) {
- return false;
+ return null;
}
arrayArgs = node[1];
for (i = 0; i < arrayArgs.length; i++) {
dep = arrayArgs[i];
- if (dep[0] !== 'string') {
- return false;
+ if (dep[0] === 'string') {
+ newDeps[1].push(dep);
}
}
- return true;
+ return newDeps[1].length ? newDeps : null;
}
/**
@@ -77,7 +92,12 @@ define(['uglifyjs/index'], function (uglify) {
var matches = [], result = null,
astRoot = parser.parse(fileContents);
- parse.recurse(astRoot, matches);
+ parse.recurse(astRoot, function () {
+ var parsed = parse.callToString.apply(parse, arguments);
+ if (parsed) {
+ matches.push(parsed);
+ }
+ });
if (matches.length) {
result = matches.join("\n");
@@ -94,19 +114,16 @@ define(['uglifyjs/index'], function (uglify) {
/**
* Handles parsing a file recursively for require calls.
* @param {Array} parentNode the AST node to start with.
- * @param {Array} matches where to store the string matches
+ * @param {Function} onMatch function to call on a parse match.
*/
- parse.recurse = function (parentNode, matches) {
- var i, node, parsed;
+ parse.recurse = function (parentNode, onMatch) {
+ var i, node;
if (isArray(parentNode)) {
for (i = 0; i < parentNode.length; i++) {
node = parentNode[i];
if (isArray(node)) {
- parsed = this.parseNode(node);
- if (parsed) {
- matches.push(parsed);
- }
- this.recurse(node, matches);
+ this.parseNode(node, onMatch);
+ this.recurse(node, onMatch);
}
}
}
@@ -192,6 +209,42 @@ define(['uglifyjs/index'], function (uglify) {
return null;
};
+ /**
+ * Finds all dependencies specified in dependency arrays and inside
+ * simplified commonjs wrappers.
+ * @param {String} fileName
+ * @param {String} fileContents
+ *
+ * @returns {Array} an array of dependency strings. The dependencies
+ * have not been normalized, they may be relative IDs.
+ */
+ parse.findDependencies = function (fileName, fileContents) {
+ //This is a litle bit inefficient, it ends up with two uglifyjs parser
+ //calls. Can revisit later, but trying to build out larger functional
+ //pieces first.
+ var dependencies = parse.getAnonDeps(fileName, fileContents),
+ astRoot = parser.parse(fileContents),
+ i, dep;
+
+ parse.recurse(astRoot, function (callName, config, name, deps) {
+ //Normalize the input args.
+ if (name && isArrayLiteral(name)) {
+ deps = name;
+ name = null;
+ }
+
+ if (!(deps = validateDeps(deps)) || !isArrayLiteral(deps)) {
+ return;
+ }
+
+ for (i = 0; (dep = deps[1][i]); i++) {
+ dependencies.push(dep[1]);
+ }
+ });
+
+ return dependencies;
+ };
+
parse.findRequireDepNames = function (node, deps) {
var moduleName, i, n, call, args;
@@ -291,7 +344,7 @@ define(['uglifyjs/index'], function (uglify) {
name = null;
}
- if (deps && !validateDeps(deps)) {
+ if (!(deps = validateDeps(deps))) {
return null;
}
@@ -311,11 +364,14 @@ define(['uglifyjs/index'], function (uglify) {
/**
* Determines if a specific node is a valid require or define/require.def call.
* @param {Array} node
+ * @param {Function} onMatch a function to call when a match is found.
+ * It is passed the match name, and the config, name, deps possible args.
+ * The config, name and deps args are not normalized.
*
* @returns {String} a JS source string with the valid require/define call.
* Otherwise null.
*/
- parse.parseNode = function (node) {
+ parse.parseNode = function (node, onMatch) {
var call, name, config, deps, args;
if (!isArray(node)) {
@@ -337,11 +393,11 @@ define(['uglifyjs/index'], function (uglify) {
config = null;
}
- if (!deps || !validateDeps(deps)) {
+ if (!(deps = validateDeps(deps))) {
return null;
}
- return this.callToString("require", null, null, deps);
+ return onMatch("require", null, null, deps);
} else if ((call[0] === 'name' && call[1] === 'define') ||
(call[0] === 'dot' && call[1][1] === 'require' &&
@@ -364,7 +420,7 @@ define(['uglifyjs/index'], function (uglify) {
name[0] === 'function' || isObjectLiteral(name))) &&
(!deps || isArrayLiteral(deps) ||
deps[0] === 'function' || isObjectLiteral(deps))) {
- return this.callToString("define", null, name, deps);
+ return onMatch("define", null, name, deps);
}
}
}
Oops, something went wrong.

0 comments on commit 0f5c86c

Please sign in to comment.