Permalink
Browse files

Add support for module groups to iOS Random Access Bundle format

Summary:
Adds the possibility to specify an array of files (group roots) that are used to bundle modules outside of the “startup section” into bigger groups by colocating their code.
A require call for any grouped module will load all modules in that group.
Files contained by multiple groups are deoptimized (i.e. bundled as individual script)

Reviewed By: martinbigio

Differential Revision: D3841780

fbshipit-source-id: 8d37782792efd66b5f557c7567489f68c9b229d8
  • Loading branch information...
1 parent 9ff4d31 commit b62ed2f291ad86d3d8192fe360fab4bfd5774576 @davidaurelio davidaurelio committed with Facebook Github Bot 5 Sep 16, 2016
@@ -32,19 +32,20 @@ function saveAsIndexedFile(bundle, options, log) {
} = options;
log('start');
- const {startupModules, lazyModules} = bundle.getUnbundle();
+ const {startupModules, lazyModules, groups} = bundle.getUnbundle();
log('finish');
+ const moduleGroups = ModuleGroups(groups, lazyModules);
const startupCode = joinModules(startupModules);
log('Writing unbundle output to:', bundleOutput);
const writeUnbundle = writeBuffers(
fs.createWriteStream(bundleOutput),
- buildTableAndContents(startupCode, lazyModules, encoding)
+ buildTableAndContents(startupCode, lazyModules, moduleGroups, encoding)
).then(() => log('Done writing unbundle output'));
const sourceMap =
- buildSourceMapWithMetaData({startupModules, lazyModules});
+ buildSourceMapWithMetaData({startupModules, lazyModules, moduleGroups});
return Promise.all([
writeUnbundle,
@@ -84,7 +85,7 @@ function entryOffset(n) {
return (2 + n * 2) * SIZEOF_UINT32;
}
-function buildModuleTable(startupCode, buffers) {
+function buildModuleTable(startupCode, buffers, moduleGroups) {
// table format:
// - num_entries: uint_32 number of entries
// - startup_code_len: uint_32 length of the startup section
@@ -94,7 +95,8 @@ function buildModuleTable(startupCode, buffers) {
// - module_offset: uint_32 offset into the modules blob
// - module_length: uint_32 length of the module code in bytes
- const maxId = buffers.reduce((max, {id}) => Math.max(max, id), 0);
+ const moduleIds = Array.from(moduleGroups.modulesById.keys());
+ const maxId = moduleIds.reduce((max, id) => Math.max(max, id));
const numEntries = maxId + 1;
const table = new Buffer(entryOffset(numEntries)).fill(0);
@@ -107,32 +109,59 @@ function buildModuleTable(startupCode, buffers) {
// entries
let codeOffset = startupCode.length;
buffers.forEach(({id, buffer}) => {
- const offset = entryOffset(id);
- // module_offset
- table.writeUInt32LE(codeOffset, offset);
- // module_length
- table.writeUInt32LE(buffer.length, offset + SIZEOF_UINT32);
+ const idsInGroup = moduleGroups.groups.has(id)
+ ? [id].concat(Array.from(moduleGroups.groups.get(id)))
+ : [id];
+
+ idsInGroup.forEach(moduleId => {
+ const offset = entryOffset(moduleId);
+ // module_offset
+ table.writeUInt32LE(codeOffset, offset);
+ // module_length
+ table.writeUInt32LE(buffer.length, offset + SIZEOF_UINT32);
+ });
codeOffset += buffer.length;
});
return table;
}
-function buildModuleBuffers(modules, encoding) {
- return modules.map(
- module => moduleToBuffer(module.id, module.code, encoding));
+function groupCode(rootCode, moduleGroup, modulesById) {
+ if (!moduleGroup || !moduleGroup.size) {
+ return rootCode;
+ }
+ const code = [rootCode];
+ for (const id of moduleGroup) {
+ code.push(modulesById.get(id).code);
+ }
+
+ return code.join('\n');
+}
+
+function buildModuleBuffers(modules, moduleGroups, encoding) {
+ return modules
+ .filter(m => !moduleGroups.modulesInGroups.has(m.id))
+ .map(({id, code}) => moduleToBuffer(
+ id,
+ groupCode(
+ code,
+ moduleGroups.groups.get(id),
+ moduleGroups.modulesById,
+ ),
+ encoding
+ ));
}
-function buildTableAndContents(startupCode, modules, encoding) {
+function buildTableAndContents(startupCode, modules, moduleGroups, encoding) {
// file contents layout:
// - magic number char[4] 0xE5 0xD1 0x0B 0xFB (0xFB0BD1E5 uint32 LE)
// - offset table table see `buildModuleTables`
// - code blob char[] null-terminated code strings, starting with
// the startup code
const startupCodeBuffer = nullTerminatedBuffer(startupCode, encoding);
- const moduleBuffers = buildModuleBuffers(modules, encoding);
- const table = buildModuleTable(startupCodeBuffer, moduleBuffers);
+ const moduleBuffers = buildModuleBuffers(modules, moduleGroups, encoding);
+ const table = buildModuleTable(startupCodeBuffer, moduleBuffers, moduleGroups);
return [
fileHeader,
@@ -141,4 +170,18 @@ function buildTableAndContents(startupCode, modules, encoding) {
].concat(moduleBuffers.map(({buffer}) => buffer));
}
+function ModuleGroups(groups, modules) {
+ return {
+ groups,
+ modulesById: new Map(modules.map(m => [m.id, m])),
+ modulesInGroups: new Set(concat(groups.values())),
+ };
+}
+
+function * concat(iterators) {
+ for (const it of iterators) {
+ yield * it;
+ }
+}
+
module.exports = saveAsIndexedFile;
@@ -10,13 +10,14 @@
const {combineSourceMaps, joinModules} = require('./util');
-module.exports = ({startupModules, lazyModules}) => {
+module.exports = ({startupModules, lazyModules, moduleGroups}) => {
const startupModule = {
code: joinModules(startupModules),
map: combineSourceMaps({modules: startupModules}),
};
return combineSourceMaps({
modules: [startupModule].concat(lazyModules),
+ moduleGroups,
withCustomOffsets: true,
});
};
@@ -31,7 +31,7 @@ const wrapperEnd = wrappedCode => wrappedCode.indexOf('{') + 1;
const Section = (line, column, map) => ({map, offset: {line, column}});
-function combineSourceMaps({modules, withCustomOffsets}) {
+function combineSourceMaps({modules, withCustomOffsets, moduleGroups}) {
let offsets;
const sections = [];
const sourceMap = {
@@ -45,13 +45,41 @@ function combineSourceMaps({modules, withCustomOffsets}) {
let line = 0;
modules.forEach(({code, id, map, name}) => {
- const hasOffset = withCustomOffsets && id != null;
- const column = hasOffset ? wrapperEnd(code) : 0;
+ let column = 0;
+ let hasOffset = false;
+ let group;
+ let groupLines = 0;
+
+ if (withCustomOffsets) {
+ if (moduleGroups && moduleGroups.modulesInGroups.has(id)) {
+ // this is a module appended to another module
+ return;
+ }
+
+ if (moduleGroups && moduleGroups.groups.has(id)) {
+ group = moduleGroups.groups.get(id);
+ const otherModules = Array.from(group).map(
+ moduleId => moduleGroups.modulesById.get(moduleId));
+ otherModules.forEach(m => {
+ groupLines += countLines(m.code);
+ });
+ map = combineSourceMaps({
+ modules: [{code, id, map, name}].concat(otherModules),
+ });
+ }
+
+ hasOffset = id != null;
+ column = wrapperEnd(code);
+ }
+
sections.push(Section(line, column, map || lineToLineSourceMap(code, name)));
if (hasOffset) {
offsets[id] = line;
+ for (const moduleId of group || []) {
+ offsets[moduleId] = line;
+ }
}
- line += countLines(code);
+ line += countLines(code) + groupLines;
});
return sourceMap;
@@ -17,7 +17,7 @@ const crypto = require('crypto');
const SOURCEMAPPING_URL = '\n\/\/# sourceMappingURL=';
class Bundle extends BundleBase {
- constructor({sourceMapUrl, dev, minify} = {}) {
+ constructor({sourceMapUrl, dev, minify, ramGroups} = {}) {
super();
this._sourceMap = false;
this._sourceMapUrl = sourceMapUrl;
@@ -26,6 +26,7 @@ class Bundle extends BundleBase {
this._dev = dev;
this._minify = minify;
+ this._ramGroups = ramGroups;
this._ramBundle = null; // cached RAM Bundle
}
@@ -111,9 +112,17 @@ class Bundle extends BundleBase {
// separate modules we need to preload from the ones we don't
const [startupModules, lazyModules] = partition(modules, shouldPreload);
+ const ramGroups = this._ramGroups;
+ let groups;
this._ramBundle = {
startupModules,
lazyModules,
+ get groups() {
+ if (!groups) {
+ groups = createGroups(ramGroups || [], lazyModules);
+ }
+ return groups;
+ }
};
}
@@ -265,6 +274,10 @@ class Bundle extends BundleBase {
].join('\n');
}
+ setRamGroups(ramGroups) {
+ this._ramGroups = ramGroups;
+ }
+
toJSON() {
this.assertFinalized('Cannot serialize bundle unless finalized');
@@ -318,4 +331,89 @@ function partition(array, predicate) {
return [included, excluded];
}
+function * filter(iterator, predicate) {
+ for (const value of iterator) {
+ if (predicate(value)) {
+ yield value;
+ }
+ }
+}
+
+function * subtree(moduleTransport, moduleTransportsByPath, seen = new Set()) {
+ seen.add(moduleTransport.id);
+ for (const [, {path}] of moduleTransport.meta.dependencyPairs) {
+ const dependency = moduleTransportsByPath.get(path);
+ if (dependency && !seen.has(dependency.id)) {
+ yield dependency.id;
+ yield * subtree(dependency, moduleTransportsByPath, seen);
+ }
+ }
+}
+
+class ArrayMap extends Map {
+ get(key) {
+ let array = super.get(key);
+ if (!array) {
+ array = [];
+ this.set(key, array);
+ }
+ return array;
+ }
+}
+
+function createGroups(ramGroups, lazyModules) {
+ // build two maps that allow to lookup module data
+ // by path or (numeric) module id;
+ const byPath = new Map();
+ const byId = new Map();
+ lazyModules.forEach(m => {
+ byPath.set(m.sourcePath, m);
+ byId.set(m.id, m.sourcePath);
+ });
+
+ // build a map of group root IDs to an array of module IDs in the group
+ const result = new Map(
+ ramGroups
+ .map(modulePath => {
+ const root = byPath.get(modulePath);
+ if (!root) {
+ throw Error(`Group root ${modulePath} is not part of the bundle`);
+ }
+ return [
+ root.id,
+ // `subtree` yields the IDs of all transitive dependencies of a module
+ new Set(subtree(byPath.get(root.sourcePath), byPath)),
+ ];
+ })
+ );
+
+ if (ramGroups.length > 1) {
+ // build a map of all grouped module IDs to an array of group root IDs
+ const all = new ArrayMap();
+ for (const [parent, children] of result) {
+ for (const module of children) {
+ all.get(module).push(parent);
+ }
+ }
+
+ // find all module IDs that are part of more than one group
+ const doubles = filter(all, ([, parents]) => parents.length > 1);
+ for (const [moduleId, parents] of doubles) {
+ // remove them from their groups
+ parents.forEach(p => result.get(p).delete(moduleId));
+
+ // print a warning for each removed module
+ const parentNames = parents.map(byId.get, byId);
+ const lastName = parentNames.pop();
+ console.warn(
+ `Module ${byId.get(moduleId)} belongs to groups ${
+ parentNames.join(', ')}, and ${lastName
+ }. Removing it from all groups.`
+ );
+ }
+ }
+
+ return result;
+}
+
module.exports = Bundle;
Oops, something went wrong.

0 comments on commit b62ed2f

Please sign in to comment.