Skip to content

Commit

Permalink
Merge pull request #15515 from webpack/feat/concatenate-assets
Browse files Browse the repository at this point in the history 
add asset modules concatenation
  • Loading branch information
@sokra
sokra committed Apr 4, 2022
2 parents 05ebf5b + 19d1a93 commit c38caa2
Show file tree
Hide file tree
Showing 30 changed files with 373 additions and 76 deletions.
99 changes: 93 additions & 6 deletions lib/ChunkGraph.js
View file
Expand Up @@ -85,14 +85,17 @@ const getModuleRuntimes = chunks => {
};

/**
* @param {SortableSet<Module>} set the set
* @returns {Map<string, SortableSet<Module>>} modules by source type
* @param {WeakMap<Module, Set<string>> | undefined} sourceTypesByModule sourceTypesByModule
* @returns {function (SortableSet<Module>): Map<string, SortableSet<Module>>} modules by source type
*/
const modulesBySourceType = set => {
const modulesBySourceType = sourceTypesByModule => set => {
/** @type {Map<string, SortableSet<Module>>} */
const map = new Map();
for (const module of set) {
for (const sourceType of module.getSourceTypes()) {
const sourceTypes =
(sourceTypesByModule && sourceTypesByModule.get(module)) ||
module.getSourceTypes();
for (const sourceType of sourceTypes) {
let innerSet = map.get(sourceType);
if (innerSet === undefined) {
innerSet = new SortableSet();
Expand All @@ -110,6 +113,7 @@ const modulesBySourceType = set => {
}
return map;
};
const defaultModulesBySourceType = modulesBySourceType(undefined);

/** @type {WeakMap<Function, any>} */
const createOrderedArrayFunctionMap = new WeakMap();
Expand Down Expand Up @@ -201,6 +205,8 @@ class ChunkGraphChunk {
constructor() {
/** @type {SortableSet<Module>} */
this.modules = new SortableSet();
/** @type {WeakMap<Module, Set<string>> | undefined} */
this.sourceTypesByModule = undefined;
/** @type {Map<Module, Entrypoint>} */
this.entryModules = new Map();
/** @type {SortableSet<RuntimeModule>} */
Expand All @@ -213,6 +219,8 @@ class ChunkGraphChunk {
this.runtimeRequirements = undefined;
/** @type {Set<string>} */
this.runtimeRequirementsInTree = new Set();

this._modulesBySourceType = defaultModulesBySourceType;
}
}

Expand Down Expand Up @@ -315,6 +323,8 @@ class ChunkGraph {
const cgm = this._getChunkGraphModule(module);
const cgc = this._getChunkGraphChunk(chunk);
cgc.modules.delete(module);
// No need to invalidate cgc._modulesBySourceType because we modified cgc.modules anyway
if (cgc.sourceTypesByModule) cgc.sourceTypesByModule.delete(module);
cgm.chunks.delete(chunk);
}

Expand Down Expand Up @@ -568,11 +578,84 @@ class ChunkGraph {
getChunkModulesIterableBySourceType(chunk, sourceType) {
const cgc = this._getChunkGraphChunk(chunk);
const modulesWithSourceType = cgc.modules
.getFromUnorderedCache(modulesBySourceType)
.getFromUnorderedCache(cgc._modulesBySourceType)
.get(sourceType);
return modulesWithSourceType;
}

/**
* @param {Chunk} chunk chunk
* @param {Module} module chunk module
* @param {Set<string>} sourceTypes source types
*/
setChunkModuleSourceTypes(chunk, module, sourceTypes) {
const cgc = this._getChunkGraphChunk(chunk);
if (cgc.sourceTypesByModule === undefined) {
cgc.sourceTypesByModule = new WeakMap();
}
cgc.sourceTypesByModule.set(module, sourceTypes);
// Update cgc._modulesBySourceType to invalidate the cache
cgc._modulesBySourceType = modulesBySourceType(cgc.sourceTypesByModule);
}

/**
* @param {Chunk} chunk chunk
* @param {Module} module chunk module
* @returns {Set<string>} source types
*/
getChunkModuleSourceTypes(chunk, module) {
const cgc = this._getChunkGraphChunk(chunk);
if (cgc.sourceTypesByModule === undefined) {
return module.getSourceTypes();
}
return cgc.sourceTypesByModule.get(module) || module.getSourceTypes();
}

/**
* @param {Module} module module
* @returns {Set<string>} source types
*/
getModuleSourceTypes(module) {
return (
this._getOverwrittenModuleSourceTypes(module) || module.getSourceTypes()
);
}

/**
* @param {Module} module module
* @returns {Set<string> | undefined} source types
*/
_getOverwrittenModuleSourceTypes(module) {
let newSet = false;
let sourceTypes;
for (const chunk of this.getModuleChunksIterable(module)) {
const cgc = this._getChunkGraphChunk(chunk);
if (cgc.sourceTypesByModule === undefined) return;
const st = cgc.sourceTypesByModule.get(module);
if (st === undefined) return;
if (!sourceTypes) {
sourceTypes = st;
continue;
} else if (!newSet) {
for (const type of st) {
if (!newSet) {
if (!sourceTypes.has(type)) {
newSet = true;
sourceTypes = new Set(sourceTypes);
sourceTypes.add(type);
}
} else {
sourceTypes.add(type);
}
}
} else {
for (const type of st) sourceTypes.add(type);
}
}

return sourceTypes;
}

/**
* @param {Chunk} chunk the chunk
* @param {function(Module, Module): -1|0|1} comparator comparator function
Expand All @@ -593,7 +676,7 @@ class ChunkGraph {
getOrderedChunkModulesIterableBySourceType(chunk, sourceType, comparator) {
const cgc = this._getChunkGraphChunk(chunk);
const modulesWithSourceType = cgc.modules
.getFromUnorderedCache(modulesBySourceType)
.getFromUnorderedCache(cgc._modulesBySourceType)
.get(sourceType);
if (modulesWithSourceType === undefined) return undefined;
modulesWithSourceType.sortWith(comparator);
Expand Down Expand Up @@ -1473,6 +1556,10 @@ Caller might not support runtime-dependent code generation (opt-out via optimiza
const graphHash = cgm.graphHashes.provide(runtime, () => {
const hash = createHash(this._hashFunction);
hash.update(`${cgm.id}${this.moduleGraph.isAsync(module)}`);
const sourceTypes = this._getOverwrittenModuleSourceTypes(module);
if (sourceTypes !== undefined) {
for (const type of sourceTypes) hash.update(type);
}
this.moduleGraph.getExportsInfo(module).updateHash(hash, runtime);
return BigInt(`0x${/** @type {string} */ (hash.digest("hex"))}`);
});
Expand Down
1 change: 1 addition & 0 deletions lib/Module.js
View file
Expand Up @@ -60,6 +60,7 @@ const makeSerializable = require("./util/makeSerializable");
* @property {ConcatenationScope=} concatenationScope when in concatenated module, information about other concatenated modules
* @property {CodeGenerationResults} codeGenerationResults code generation results of other modules (need to have a codeGenerationDependency to use that)
* @property {Compilation=} compilation the compilation
* @property {ReadonlySet<string>=} sourceTypes source types
*/

/**
Expand Down
5 changes: 3 additions & 2 deletions lib/NormalModule.js
View file
Expand Up @@ -1176,7 +1176,8 @@ class NormalModule extends Module {
chunkGraph,
runtime,
concatenationScope,
codeGenerationResults
codeGenerationResults,
sourceTypes
}) {
/** @type {Set<string>} */
const runtimeRequirements = new Set();
Expand All @@ -1195,7 +1196,7 @@ class NormalModule extends Module {
};

const sources = new Map();
for (const type of this.generator.getTypes(this)) {
for (const type of sourceTypes || chunkGraph.getModuleSourceTypes(this)) {
const source = this.error
? new RawSource(
"throw new Error(" + JSON.stringify(this.error.message) + ");"
Expand Down
45 changes: 36 additions & 9 deletions lib/asset/AssetGenerator.js
View file
Expand Up @@ -8,6 +8,7 @@
const mimeTypes = require("mime-types");
const path = require("path");
const { RawSource } = require("webpack-sources");
const ConcatenationScope = require("../ConcatenationScope");
const Generator = require("../Generator");
const RuntimeGlobals = require("../RuntimeGlobals");
const createHash = require("../util/createHash");
Expand All @@ -23,6 +24,7 @@ const nonNumericOnlyHash = require("../util/nonNumericOnlyHash");
/** @typedef {import("../Generator").GenerateContext} GenerateContext */
/** @typedef {import("../Generator").UpdateHashContext} UpdateHashContext */
/** @typedef {import("../Module")} Module */
/** @typedef {import("../Module").ConcatenationBailoutReasonContext} ConcatenationBailoutReasonContext */
/** @typedef {import("../NormalModule")} NormalModule */
/** @typedef {import("../RuntimeTemplate")} RuntimeTemplate */
/** @typedef {import("../util/Hash")} Hash */
Expand Down Expand Up @@ -145,6 +147,15 @@ class AssetGenerator extends Generator {
).replace(/^\.\//, "");
}

/**
* @param {NormalModule} module module for which the bailout reason should be determined
* @param {ConcatenationBailoutReasonContext} context context
* @returns {string | undefined} reason why this module can't be concatenated, undefined when it can be concatenated
*/
getConcatenationBailoutReason(module, context) {
return undefined;
}

/**
* @param {NormalModule} module module
* @returns {string} mime type
Expand Down Expand Up @@ -198,14 +209,21 @@ class AssetGenerator extends Generator {
*/
generate(
module,
{ runtime, chunkGraph, runtimeTemplate, runtimeRequirements, type, getData }
{
runtime,
concatenationScope,
chunkGraph,
runtimeTemplate,
runtimeRequirements,
type,
getData
}
) {
switch (type) {
case "asset":
return module.originalSource();
default: {
runtimeRequirements.add(RuntimeGlobals.module);

let content;
const originalSource = module.originalSource();
if (module.buildInfo.dataUrl) {
let encodedSource;
Expand Down Expand Up @@ -255,11 +273,7 @@ class AssetGenerator extends Generator {
}
const data = getData();
data.set("url", Buffer.from(encodedSource));
return new RawSource(
`${RuntimeGlobals.module}.exports = ${JSON.stringify(
encodedSource
)};`
);
content = JSON.stringify(encodedSource);
} else {
const assetModuleFilename =
this.filename || runtimeTemplate.outputOptions.assetModuleFilename;
Expand Down Expand Up @@ -343,9 +357,22 @@ class AssetGenerator extends Generator {
data.set("filename", filename);
data.set("assetInfo", assetInfo);
}
content = assetPath;
}

if (concatenationScope) {
concatenationScope.registerNamespaceExport(
ConcatenationScope.NAMESPACE_OBJECT_EXPORT
);
return new RawSource(
`${runtimeTemplate.supportsConst() ? "const" : "var"} ${
ConcatenationScope.NAMESPACE_OBJECT_EXPORT
} = ${content};`
);
} else {
runtimeRequirements.add(RuntimeGlobals.module);
return new RawSource(
`${RuntimeGlobals.module}.exports = ${assetPath};`
`${RuntimeGlobals.module}.exports = ${content};`
);
}
}
Expand Down
1 change: 1 addition & 0 deletions lib/asset/AssetParser.js
View file
Expand Up @@ -31,6 +31,7 @@ class AssetParser extends Parser {
}
state.module.buildInfo.strict = true;
state.module.buildMeta.exportsType = "default";
state.module.buildMeta.defaultObject = false;

if (typeof this.dataUrlCondition === "function") {
state.module.buildInfo.dataUrl = this.dataUrlCondition(source, {
Expand Down
37 changes: 31 additions & 6 deletions lib/asset/AssetSourceGenerator.js
View file
Expand Up @@ -6,11 +6,13 @@
"use strict";

const { RawSource } = require("webpack-sources");
const ConcatenationScope = require("../ConcatenationScope");
const Generator = require("../Generator");
const RuntimeGlobals = require("../RuntimeGlobals");

/** @typedef {import("webpack-sources").Source} Source */
/** @typedef {import("../Generator").GenerateContext} GenerateContext */
/** @typedef {import("../Module").ConcatenationBailoutReasonContext} ConcatenationBailoutReasonContext */
/** @typedef {import("../NormalModule")} NormalModule */

const TYPES = new Set(["javascript"]);
Expand All @@ -21,9 +23,10 @@ class AssetSourceGenerator extends Generator {
* @param {GenerateContext} generateContext context for generate
* @returns {Source} generated code
*/
generate(module, { chunkGraph, runtimeTemplate, runtimeRequirements }) {
runtimeRequirements.add(RuntimeGlobals.module);

generate(
module,
{ concatenationScope, chunkGraph, runtimeTemplate, runtimeRequirements }
) {
const originalSource = module.originalSource();

if (!originalSource) {
Expand All @@ -38,9 +41,31 @@ class AssetSourceGenerator extends Generator {
} else {
encodedSource = content.toString("utf-8");
}
return new RawSource(
`${RuntimeGlobals.module}.exports = ${JSON.stringify(encodedSource)};`
);

let sourceContent;
if (concatenationScope) {
concatenationScope.registerNamespaceExport(
ConcatenationScope.NAMESPACE_OBJECT_EXPORT
);
sourceContent = `${runtimeTemplate.supportsConst() ? "const" : "var"} ${
ConcatenationScope.NAMESPACE_OBJECT_EXPORT
} = ${JSON.stringify(encodedSource)};`;
} else {
runtimeRequirements.add(RuntimeGlobals.module);
sourceContent = `${RuntimeGlobals.module}.exports = ${JSON.stringify(
encodedSource
)};`;
}
return new RawSource(sourceContent);
}

/**
* @param {NormalModule} module module for which the bailout reason should be determined
* @param {ConcatenationBailoutReasonContext} context context
* @returns {string | undefined} reason why this module can't be concatenated, undefined when it can be concatenated
*/
getConcatenationBailoutReason(module, context) {
return undefined;
}

/**
Expand Down
1 change: 1 addition & 0 deletions lib/asset/AssetSourceParser.js
View file
Expand Up @@ -23,6 +23,7 @@ class AssetSourceParser extends Parser {
const { module } = state;
module.buildInfo.strict = true;
module.buildMeta.exportsType = "default";
state.module.buildMeta.defaultObject = false;

return state;
}
Expand Down
3 changes: 2 additions & 1 deletion lib/optimize/ConcatenatedModule.js
View file
Expand Up @@ -1669,7 +1669,8 @@ ${defineGetters}`
chunkGraph,
runtime,
concatenationScope,
codeGenerationResults
codeGenerationResults,
sourceTypes: TYPES
});
const source = codeGenResult.sources.get("javascript");
const data = codeGenResult.data;
Expand Down
16 changes: 15 additions & 1 deletion lib/optimize/ModuleConcatenationPlugin.js
View file
Expand Up @@ -425,7 +425,21 @@ class ModuleConcatenationPlugin {
for (const chunk of chunkGraph.getModuleChunksIterable(
rootModule
)) {
chunkGraph.disconnectChunkAndModule(chunk, m);
const sourceTypes = chunkGraph.getChunkModuleSourceTypes(
chunk,
m
);
if (sourceTypes.size === 1) {
chunkGraph.disconnectChunkAndModule(chunk, m);
} else {
const newSourceTypes = new Set(sourceTypes);
newSourceTypes.delete("javascript");
chunkGraph.setChunkModuleSourceTypes(
chunk,
m,
newSourceTypes
);
}
}
}
}
Expand Down

0 comments on commit c38caa2

Please sign in to comment.