From 875a462ed8e46637a50f3efee1767d6c1a80a7a5 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Tue, 29 Jun 2021 16:33:58 -0400 Subject: [PATCH 01/73] WIP step 1 and 2 bundler --- .../bundlers/default/src/DefaultBundler.js | 516 ++++-------------- packages/core/core/src/BundleGraph.js | 3 +- packages/core/core/src/public/BundleGraph.js | 2 + .../test/integration/commonjs/index.js | 2 +- .../core/integration-tests/test/javascript.js | 2 +- packages/core/types/index.js | 5 +- 6 files changed, 114 insertions(+), 416 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 16c706633b2..1de26e0e0d5 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -2,14 +2,16 @@ import type { Asset, - Bundle, BundleGroup, Config, MutableBundleGraph, PluginOptions, } from '@parcel/types'; +import type {NodeId} from '@parcel/core/src/types'; import type {SchemaEntity} from '@parcel/utils'; +import Graph from '@parcel/core/src/Graph'; + import invariant from 'assert'; import {Bundler} from '@parcel/plugin'; import {validateSchema} from '@parcel/utils'; @@ -38,441 +40,115 @@ const HTTP_OPTIONS = { }, }; -let skipOptimize = false; +type BundleId = string; +type AssetId = string; +type Bundle = {| + assetIds: Array, + size: number, + sourceBundles: Array, +|}; +type BundleNode = {| + id: BundleId, + +type: '', + value: Bundle, +|}; export default (new Bundler({ - // RULES: - // 1. If dep.isAsync or dep.isEntry, start a new bundle group. - // 2. If an asset is a different type than the current bundle, make a parallel bundle in the same bundle group. - // 3. If an asset is already in a parent bundle in the same entry point, exclude from child bundles. - // 4. If an asset is only in separate isolated entry points (e.g. workers, different HTML pages), duplicate it. - loadConfig({config, options}) { return loadBundlerConfig(config, options); }, - bundle({bundleGraph, config}) { - let bundleRoots: Map> = new Map(); - let bundlesByEntryAsset: Map = new Map(); - - // Step 1: create bundles for each of the explicit code split points. - bundleGraph.traverse({ - enter: (node, context, actions) => { - if (node.type !== 'dependency') { - return { - ...context, - bundleGroup: context?.bundleGroup, - bundleByType: context?.bundleByType, - parentNode: node, - parentBundle: - bundlesByEntryAsset.get(node.value) ?? context?.parentBundle, - }; - } - - let dependency = node.value; - if (bundleGraph.isDependencySkipped(dependency)) { - actions.skipChildren(); - return; - } - - let assets = bundleGraph.getDependencyAssets(dependency); - let resolution = bundleGraph.getDependencyResolution(dependency); - let bundleGroup = context?.bundleGroup; - // Create a new bundle for entries, lazy/parallel dependencies, isolated/inline assets. - if ( - resolution && - (!bundleGroup || - dependency.priority === 'lazy' || - dependency.priority === 'parallel' || - resolution.bundleBehavior === 'isolated' || - resolution.bundleBehavior === 'inline') - ) { - let bundleByType: Map = - context?.bundleByType ?? new Map(); - - // Only create a new bundle group for entries, lazy dependencies, and isolated assets. - // Otherwise, the bundle is loaded together with the parent bundle. - if ( - !bundleGroup || - dependency.priority === 'lazy' || - resolution.bundleBehavior === 'isolated' - ) { - bundleGroup = bundleGraph.createBundleGroup( - dependency, - nullthrows(dependency.target ?? context?.bundleGroup?.target), - ); - - bundleByType = new Map(); - } - - for (let asset of assets) { - let bundle = bundleGraph.createBundle({ - entryAsset: asset, - needsStableName: - dependency.bundleBehavior === 'inline' || - asset.bundleBehavior === 'inline' - ? false - : dependency.isEntry || dependency.needsStableName, - bundleBehavior: dependency.bundleBehavior ?? asset.bundleBehavior, - target: bundleGroup.target, - }); - bundleByType.set(bundle.type, bundle); - bundlesByEntryAsset.set(asset, bundle); - bundleGraph.addBundleToBundleGroup(bundle, bundleGroup); - - // The bundle may have already been created, and the graph gave us back the original one... - if (!bundleRoots.has(bundle)) { - bundleRoots.set(bundle, [asset]); - } - - // If the bundle is in the same bundle group as the parent, create an asset reference - // between the dependency, the asset, and the target bundle. - if (bundleGroup === context?.bundleGroup) { - bundleGraph.createAssetReference(dependency, asset, bundle); - } - } - - return { - bundleGroup, - bundleByType, - parentNode: node, - parentBundle: context?.parentBundle, - }; - } - - invariant(context != null); - invariant(context.parentNode.type === 'asset'); - invariant(context.parentBundle != null); - invariant(bundleGroup != null); - let parentAsset = context.parentNode.value; - let parentBundle = context.parentBundle; - let bundleByType = nullthrows(context.bundleByType); - - for (let asset of assets) { - if (parentAsset.type === asset.type) { - continue; - } - - let existingBundle = bundleByType.get(asset.type); - if (existingBundle) { - // If a bundle of this type has already been created in this group, - // merge this subgraph into it. - nullthrows(bundleRoots.get(existingBundle)).push(asset); - bundlesByEntryAsset.set(asset, existingBundle); - bundleGraph.createAssetReference(dependency, asset, existingBundle); - } else { - let bundle = bundleGraph.createBundle({ - uniqueKey: asset.id, - env: asset.env, - type: asset.type, - target: bundleGroup.target, - needsStableName: - asset.bundleBehavior === 'inline' || - dependency.bundleBehavior === 'inline' || - (dependency.priority === 'parallel' && - !dependency.needsStableName) - ? false - : parentBundle.needsStableName, - bundleBehavior: dependency.bundleBehavior ?? asset.bundleBehavior, - isSplittable: asset.isBundleSplittable ?? true, - pipeline: asset.pipeline, - }); - bundleByType.set(bundle.type, bundle); - bundlesByEntryAsset.set(asset, bundle); - bundleGraph.createAssetReference(dependency, asset, bundle); - - // The bundle may have already been created, and the graph gave us back the original one... - if (!bundleRoots.has(bundle)) { - bundleRoots.set(bundle, [asset]); - } - } - } - - return { - ...context, - parentNode: node, - }; - }, - }); + bundle({bundleGraph: assetGraph, config}) { + //assetgraph here is a MutableBundleGraph - for (let [bundle, rootAssets] of bundleRoots) { - for (let asset of rootAssets) { - bundleGraph.addEntryToBundle(asset, bundle); - } - } + let bundleRoots: Map = new Map(); //asset to tuple of bundle Ids + let reachableBundles: Map> = new Map(); - // If there's only one bundle, we can skip the rest of the steps. - skipOptimize = bundleRoots.size === 1; - if (skipOptimize) { - return; - } + let bundleGraph: Graph = new Graph(); - invariant(config != null); + let stack: Array<[AssetId, NodeId]> = []; - // Step 2: Remove asset graphs that begin with entries to other bundles. - bundleGraph.traverseBundles(bundle => { - if ( - bundle.bundleBehavior === 'inline' || - bundle.bundleBehavior === 'isolated' || - !bundle.isSplittable || - bundle.env.isIsolated() - ) { - return; - } - - // Skip bundles where the entry is reachable in a parent bundle. This can occur when both synchronously and - // asynchronously importing an asset from a bundle. This asset will later be internalized into the parent. - let entries = bundle.getEntryAssets(); - let mainEntry = entries[0]; - if ( - mainEntry == null || - entries.length !== 1 || - bundleGraph.isAssetReachableFromBundle(mainEntry, bundle) - ) { - return; + // Step 1: Create bundles at the explicit split points in the graph. + // Create bundles for each entry. + let entries: Array = []; + assetGraph.traverse((node, context, actions) => { + if (node.type !== 'asset') { + return node; } - let candidates = bundleGraph.findBundlesWithAsset(mainEntry).filter( - containingBundle => - containingBundle.id !== bundle.id && - // Don't add to BundleGroups for entry bundles, as that would require - // another entry bundle depending on these conditions, making it difficult - // to predict and reference. - // TODO: reconsider this. This is only true for the global output format. - !containingBundle.needsStableName && - containingBundle.bundleBehavior !== 'inline' && - containingBundle.bundleBehavior !== 'isolated' && - containingBundle.isSplittable, + invariant( + context != null && + context.type === 'dependency' && + context.value.isEntry, ); - - for (let candidate of candidates) { - let bundleGroups = bundleGraph.getBundleGroupsContainingBundle( - candidate, - ); - if ( - Array.from(bundleGroups).every( - group => - bundleGraph - .getBundlesInBundleGroup(group) - .filter(b => b.bundleBehavior !== 'inline').length < - config.maxParallelRequests, - ) - ) { - bundleGraph.createBundleReference(candidate, bundle); - bundleGraph.removeAssetGraphFromBundle(mainEntry, candidate); - } - } + entries.push(node.value); + actions.skipChildren(); }); - - // Step 3: Remove assets that are duplicated in a parent bundle. - deduplicate(bundleGraph); - - // Step 4: Mark async dependencies on assets that are already available in - // the bundle as internally resolvable. This removes the dependency between - // the bundle and the bundle group providing that asset. If all connections - // to that bundle group are removed, remove that bundle group. - let asyncBundleGroups: Set = new Set(); - bundleGraph.traverse((node, _, actions) => { - if ( - node.type !== 'dependency' || - node.value.isEntry || - node.value.priority !== 'lazy' - ) { - return; - } - - if (bundleGraph.isDependencySkipped(node.value)) { - actions.skipChildren(); - return; - } - - let dependency = node.value; - if (dependency.specifierType === 'url') { - // Don't internalize dependencies on URLs, e.g. `new Worker('foo.js')` - return; - } - - let resolution = bundleGraph.getDependencyResolution(dependency); - if (resolution == null) { - return; - } - - let externalResolution = bundleGraph.resolveAsyncDependency(dependency); - if (externalResolution?.type === 'bundle_group') { - asyncBundleGroups.add(externalResolution.value); - } - - for (let bundle of bundleGraph.findBundlesWithDependency(dependency)) { - if ( - bundle.hasAsset(resolution) || - bundleGraph.isAssetReachableFromBundle(resolution, bundle) - ) { - bundleGraph.internalizeAsyncDependency(bundle, dependency); - } - } - }); - - // Remove any bundle groups that no longer have any parent bundles. - for (let bundleGroup of asyncBundleGroups) { - if (bundleGraph.getParentBundlesOfBundleGroup(bundleGroup).length === 0) { - bundleGraph.removeBundleGroup(bundleGroup); - } - } - }, - optimize({bundleGraph, config}) { - // if only one bundle, no need to optimize - if (skipOptimize) { - return; + // console.log( + // 'entries are', + // entries.map(value => value.filePath), + // ); + + for (let entry of entries) { + let nodeId = bundleGraph.addNode(createBundleNode(createBundle(entry))); + bundleRoots.set(entry, [nodeId, nodeId]); } + // Traverse the asset graph and create bundles for asset type changes and async dependencies. + // This only adds the entry asset of each bundle, not the subgraph. + assetGraph.traverse({ + enter(node, context, actions) { + //Discover + if (node.type === 'asset') { + let bundleIdTuple = bundleRoots.get(node.value); + if (bundleIdTuple) { + // Push to the stack when a new bundle is created. + stack.unshift([node.value.id, bundleIdTuple[1]]); // TODO: switch this to be push/pop instead of unshift + } + } else if (node.type === 'dependency') { + let dependency = node.value; + //TreeEdge Event + invariant(context?.type === 'asset'); + let parentAsset = context.value; + + let assets = assetGraph.getDependencyAssets(dependency); + invariant(assets.length === 1); + let childAsset = assets[0]; + + // Create a new bundle when the asset type changes. + if (parentAsset.type !== childAsset.type) { + let [_, bundleGroupNodeId] = nullthrows(stack[0]); + let bundleId = bundleGraph.addNode( + createBundleNode(createBundle(childAsset)), + ); + bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); - invariant(config != null); - - // Step 5: Find duplicated assets in different bundle groups, and separate them into their own parallel bundles. - // If multiple assets are always seen together in the same bundles, combine them together. - // If the sub-graph from an asset is >= 30kb, and the number of parallel requests in the bundle group is < 5, create a new bundle containing the sub-graph. - let candidateBundles: Map< - string, - {| - assets: Array, - sourceBundles: Set, - size: number, - |}, - > = new Map(); - - bundleGraph.traverse((node, ctx, actions) => { - if (node.type !== 'asset') { - return; - } - - let asset = node.value; - let containingBundles = bundleGraph - .findBundlesWithAsset(asset) - // Don't create shared bundles from entry bundles, as that would require - // another entry bundle depending on these conditions, making it difficult - // to predict and reference. - // TODO: reconsider this. This is only true for the global output format. - // This also currently affects other bundles with stable names, e.g. service workers. - .filter(b => { - let entries = b.getEntryAssets(); - - return ( - !b.needsStableName && - b.isSplittable && - entries.every(entry => entry.id !== asset.id) - ); - }); - - if (containingBundles.length > config.minBundles) { - let id = containingBundles - .map(b => b.id) - .sort() - .join(':'); - - let candidate = candidateBundles.get(id); - if (candidate) { - candidate.assets.push(asset); - for (let bundle of containingBundles) { - candidate.sourceBundles.add(bundle); + // Add an edge from the bundle group entry to the new bundle. + // This indicates that the bundle is loaded together with the entry + bundleGraph.addEdge(bundleGroupNodeId, bundleId); + return node; } - candidate.size += bundleGraph.getTotalSize(asset); - } else { - candidateBundles.set(id, { - assets: [asset], - sourceBundles: new Set(containingBundles), - size: bundleGraph.getTotalSize(asset), - }); + // Create a new bundle as well as a new bundle group if the dependency is async. + // TODO: add create new bundlegroup on async deps } - - // Skip children from consideration since we added a parent already. - actions.skipChildren(); - } - }); - - // Sort candidates by size (consider larger bundles first), and ensure they meet the size threshold - let sortedCandidates: Array<{| - assets: Array, - sourceBundles: Set, - size: number, - |}> = Array.from(candidateBundles.values()) - .filter(bundle => bundle.size >= config.minBundleSize) - .sort((a, b) => b.size - a.size); - - for (let {assets, sourceBundles} of sortedCandidates) { - let eligibleSourceBundles = new Set(); - - for (let bundle of sourceBundles) { - // Find all bundle groups connected to the original bundles - let bundleGroups = bundleGraph.getBundleGroupsContainingBundle(bundle); - // Check if all bundle groups are within the parallel request limit - if ( - bundleGroups.every( - group => - bundleGraph - .getBundlesInBundleGroup(group) - .filter(b => b.bundleBehavior !== 'inline').length < - config.maxParallelRequests, - ) - ) { - eligibleSourceBundles.add(bundle); + return node; + }, + exit(node, context, actions) { + if (stack[0] === node.value) { + stack.shift(); } - } - - // Do not create a shared bundle unless there are at least 2 source bundles - if (eligibleSourceBundles.size < 2) { - continue; - } - - let [firstBundle] = [...eligibleSourceBundles]; - let sharedBundle = bundleGraph.createBundle({ - uniqueKey: hashString( - [...eligibleSourceBundles].map(b => b.id).join(':'), - ), - // Allow this bundle to be deduplicated. It shouldn't be further split. - // TODO: Reconsider bundle/asset flags. - isSplittable: true, - env: firstBundle.env, - target: firstBundle.target, - type: firstBundle.type, - }); - - // Remove all of the root assets from each of the original bundles - // and reference the new shared bundle. - for (let asset of assets) { - bundleGraph.addAssetGraphToBundle(asset, sharedBundle); + }, + }); - for (let bundle of eligibleSourceBundles) { - { - bundleGraph.createBundleReference(bundle, sharedBundle); - bundleGraph.removeAssetGraphFromBundle(asset, bundle); - } - } - } + // Step 2: Determine reachability for every asset from each bundle root. + // This is later used to determine which bundles to place each asset in. + let reachableNodes: Map> = new Map(); + for (let [root] of bundleRoots) { + assetGraph.traverse(); } - - // Remove assets that are duplicated between shared bundles. - deduplicate(bundleGraph); }, + optimize() {}, }): Bundler); -function deduplicate(bundleGraph: MutableBundleGraph) { - bundleGraph.traverse(node => { - if (node.type === 'asset') { - let asset = node.value; - // Search in reverse order, so bundles that are loaded keep the duplicated asset, not later ones. - // This ensures that the earlier bundle is able to execute before the later one. - let bundles = bundleGraph.findBundlesWithAsset(asset).reverse(); - for (let bundle of bundles) { - if ( - bundle.hasAsset(asset) && - bundleGraph.isAssetReachableFromBundle(asset, bundle) - ) { - bundleGraph.removeAssetGraphFromBundle(asset, bundle); - } - } - } - }); -} - const CONFIG_SCHEMA: SchemaEntity = { type: 'object', properties: { @@ -493,6 +169,22 @@ const CONFIG_SCHEMA: SchemaEntity = { additionalProperties: false, }; +function createBundle(asset: Asset): Bundle { + return { + assetIds: [asset.id], + size: asset.stats.size, + sourceBundles: [], + }; +} + +function createBundleNode(bundle: Bundle): BundleNode { + return { + id: '', + type: '', + value: bundle, + }; +} + async function loadBundlerConfig(config: Config, options: PluginOptions) { let conf = await config.getConfig([], { packageKey: '@parcel/bundler-default', diff --git a/packages/core/core/src/BundleGraph.js b/packages/core/core/src/BundleGraph.js index b4864fa71ac..0d8a6e689c0 100644 --- a/packages/core/core/src/BundleGraph.js +++ b/packages/core/core/src/BundleGraph.js @@ -973,6 +973,7 @@ export default class BundleGraph { traverse( visit: GraphVisitor, + start?: Asset, ): ?TContext { return this._graph.filteredTraverse( nodeId => { @@ -982,7 +983,7 @@ export default class BundleGraph { } }, visit, - undefined, // start with root + start ? this._graph.getNodeIdByContentKey(start.id) : undefined, // start with root // $FlowFixMe ALL_EDGE_TYPES, ); diff --git a/packages/core/core/src/public/BundleGraph.js b/packages/core/core/src/public/BundleGraph.js index c561ab17a74..03fc6d8e49c 100644 --- a/packages/core/core/src/public/BundleGraph.js +++ b/packages/core/core/src/public/BundleGraph.js @@ -273,6 +273,7 @@ export default class BundleGraph traverse( visit: GraphVisitor, + start?: ?IAsset, ): ?TContext { return this.#graph.traverse( mapVisitor( @@ -285,6 +286,7 @@ export default class BundleGraph }, visit, ), + start ? assetToAssetValue(start) : undefined, ); } diff --git a/packages/core/integration-tests/test/integration/commonjs/index.js b/packages/core/integration-tests/test/integration/commonjs/index.js index 9a6fc97209e..99a30d0574a 100644 --- a/packages/core/integration-tests/test/integration/commonjs/index.js +++ b/packages/core/integration-tests/test/integration/commonjs/index.js @@ -1,6 +1,6 @@ var local = require('./local'); // eslint-disable-next-line no-unused-vars -var url = require('url'); +//var url = require('url'); module.exports = function () { return local.a + local.b; diff --git a/packages/core/integration-tests/test/javascript.js b/packages/core/integration-tests/test/javascript.js index 2c18ad94c24..3ff989819f6 100644 --- a/packages/core/integration-tests/test/javascript.js +++ b/packages/core/integration-tests/test/javascript.js @@ -22,7 +22,7 @@ describe('javascript', function() { await removeDistDirectory(); }); - it('should produce a basic JS bundle with CommonJS requires', async function() { + it.only('should produce a basic JS bundle with CommonJS requires', async function() { let b = await bundle( path.join(__dirname, '/integration/commonjs/index.js'), ); diff --git a/packages/core/types/index.js b/packages/core/types/index.js index 4c231426e64..e5480187f78 100644 --- a/packages/core/types/index.js +++ b/packages/core/types/index.js @@ -1375,7 +1375,10 @@ export interface BundleGraph { asset: Asset, boundary: ?Bundle, ): Array; - traverse(GraphVisitor): ?TContext; + traverse( + GraphVisitor, + ?Asset, + ): ?TContext; traverseBundles( visit: GraphVisitor, startBundle: ?Bundle, From b6f5cc8e0b842aa2004eda70a90671652c4ebb8b Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Tue, 29 Jun 2021 17:16:19 -0400 Subject: [PATCH 02/73] WIP done step 2 bundle --- .../bundlers/default/src/DefaultBundler.js | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 1de26e0e0d5..d35003aab26 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -14,7 +14,7 @@ import Graph from '@parcel/core/src/Graph'; import invariant from 'assert'; import {Bundler} from '@parcel/plugin'; -import {validateSchema} from '@parcel/utils'; +import {validateSchema, DefaultMap} from '@parcel/utils'; import {hashString} from '@parcel/hash'; import nullthrows from 'nullthrows'; import {encodeJSONKeyComponent} from '@parcel/diagnostic'; @@ -141,10 +141,34 @@ export default (new Bundler({ // Step 2: Determine reachability for every asset from each bundle root. // This is later used to determine which bundles to place each asset in. - let reachableNodes: Map> = new Map(); + let reachableRoots: DefaultMap> = new DefaultMap( + () => new Set(), + ); for (let [root] of bundleRoots) { - assetGraph.traverse(); + assetGraph.traverse((node, _, actions) => { + if (node.type !== 'asset') { + return; + } + if (node.value === root) { + return; + } + + if (bundleRoots.has(root)) { + actions.skipChildren(); + return; + } + reachableRoots.get(node.value).add(root); + }, root); } + + // Step 3: Place all assets into bundles. Each asset is placed into a single + // bundle based on the bundle entries it is reachable from. This creates a + // maximally code split bundle graph with no duplication. + + // Create a mapping from entry asset ids to bundle ids + + let bundles: Map = new Map(); + //TODO Step 3, some mapping from multiple entry asset ids to a bundle Id }, optimize() {}, }): Bundler); From fbec82f0bde71a85b8a68d1a74328bb375ca082c Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Wed, 30 Jun 2021 12:47:57 -0700 Subject: [PATCH 03/73] Implement Step 3 --- .../bundlers/default/src/DefaultBundler.js | 131 +++++++++++++++--- packages/core/core/src/dumpGraphToGraphViz.js | 4 + 2 files changed, 112 insertions(+), 23 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index d35003aab26..97a2a09220d 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -11,6 +11,7 @@ import type {NodeId} from '@parcel/core/src/types'; import type {SchemaEntity} from '@parcel/utils'; import Graph from '@parcel/core/src/Graph'; +import dumpGraphToGraphViz from '@parcel/core/src/dumpGraphToGraphViz'; import invariant from 'assert'; import {Bundler} from '@parcel/plugin'; @@ -49,7 +50,7 @@ type Bundle = {| |}; type BundleNode = {| id: BundleId, - +type: '', + +type: 'mybundle', value: Bundle, |}; @@ -59,14 +60,16 @@ export default (new Bundler({ }, bundle({bundleGraph: assetGraph, config}) { - //assetgraph here is a MutableBundleGraph - - let bundleRoots: Map = new Map(); //asset to tuple of bundle Ids - let reachableBundles: Map> = new Map(); - + // Asset to the bundle it's an entry of + let bundleRoots: Map = new Map(); + // + let reachableBundles: DefaultMap> = new DefaultMap( + () => new Set(), + ); + // let bundleGraph: Graph = new Graph(); - - let stack: Array<[AssetId, NodeId]> = []; + // + let stack: Array<[Asset, NodeId]> = []; // Step 1: Create bundles at the explicit split points in the graph. // Create bundles for each entry. @@ -84,27 +87,31 @@ export default (new Bundler({ entries.push(node.value); actions.skipChildren(); }); - // console.log( - // 'entries are', - // entries.map(value => value.filePath), - // ); for (let entry of entries) { let nodeId = bundleGraph.addNode(createBundleNode(createBundle(entry))); bundleRoots.set(entry, [nodeId, nodeId]); } + + let assets = []; // Traverse the asset graph and create bundles for asset type changes and async dependencies. // This only adds the entry asset of each bundle, not the subgraph. assetGraph.traverse({ - enter(node, context, actions) { + enter(node, context) { //Discover if (node.type === 'asset') { + assets.push(node.value); + let bundleIdTuple = bundleRoots.get(node.value); if (bundleIdTuple) { // Push to the stack when a new bundle is created. - stack.unshift([node.value.id, bundleIdTuple[1]]); // TODO: switch this to be push/pop instead of unshift + stack.push([node.value, bundleIdTuple[1]]); // TODO: switch this to be push/pop instead of unshift } } else if (node.type === 'dependency') { + if (context == null) { + return node; + } + let dependency = node.value; //TreeEdge Event invariant(context?.type === 'asset'); @@ -116,7 +123,7 @@ export default (new Bundler({ // Create a new bundle when the asset type changes. if (parentAsset.type !== childAsset.type) { - let [_, bundleGroupNodeId] = nullthrows(stack[0]); + let [, bundleGroupNodeId] = nullthrows(stack[stack.length - 1]); let bundleId = bundleGraph.addNode( createBundleNode(createBundle(childAsset)), ); @@ -127,14 +134,30 @@ export default (new Bundler({ bundleGraph.addEdge(bundleGroupNodeId, bundleId); return node; } + // Create a new bundle as well as a new bundle group if the dependency is async. - // TODO: add create new bundlegroup on async deps + if (dependency.priority === 'lazy') { + let bundleId = bundleGraph.addNode( + createBundleNode(createBundle(childAsset)), + ); + bundleRoots.set(childAsset, [bundleId, bundleId]); + + // Walk up the stack until we hit a different asset type + // and mark each bundle as reachable from every parent bundle + for (let i = stack.length - 1; i >= 0; i--) { + let [stackAsset] = stack[i]; + if (stackAsset.type !== childAsset.type) { + break; + } + reachableBundles.get(stackAsset).add(childAsset); + } + } } return node; }, - exit(node, context, actions) { - if (stack[0] === node.value) { - stack.shift(); + exit(node) { + if (stack[stack.length - 1] === node.value) { + stack.pop(); } }, }); @@ -149,11 +172,12 @@ export default (new Bundler({ if (node.type !== 'asset') { return; } + if (node.value === root) { return; } - if (bundleRoots.has(root)) { + if (bundleRoots.has(node.value)) { actions.skipChildren(); return; } @@ -167,8 +191,61 @@ export default (new Bundler({ // Create a mapping from entry asset ids to bundle ids - let bundles: Map = new Map(); //TODO Step 3, some mapping from multiple entry asset ids to a bundle Id + let bundles: Map = new Map(); + for (let asset of assets) { + // Find bundle entries reachable from the asset. + let reachable = [...reachableRoots.get(asset)]; + + // Filter out bundles when the asset is reachable in a parent bundle. + reachable = reachable.filter(b => + reachable.every(a => !reachableBundles.get(a).has(b)), + ); + + let rootBundle = bundleRoots.get(asset); + if (rootBundle != null) { + // If the asset is a bundle root, add the bundle to every other reachable bundle group. + if (!bundles.has(asset.id)) { + bundles.set(asset.id, rootBundle[0]); + } + for (let reachableAsset of reachable) { + if (reachableAsset !== asset) { + bundleGraph.addEdge( + nullthrows(bundleRoots.get(reachableAsset))[1], + rootBundle[0], + ); + } + } + } else if (reachable.length > 0) { + // If the asset is reachable from more than one entry, find or create + // a bundle for that combination of entries, and add the asset to it. + let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); + let key = reachable.map(a => a.id).join(','); + + let bundleId = bundles.get(key); + if (bundleId == null) { + let bundle = createBundle(); + bundle.sourceBundles = sourceBundles; + bundleId = bundleGraph.addNode(createBundleNode(bundle)); + bundles.set(key, bundleId); + } + + let bundle = nullthrows(bundleGraph.getNode(bundleId)).value; + bundle.assetIds.push(asset.id); + bundle.size += asset.stats.size; + + // Add the bundle to each reachable bundle group. + for (let reachableAsset of reachable) { + let reachableRoot = nullthrows(bundleRoots.get(reachableAsset))[1]; + if (reachableRoot !== bundleId) { + bundleGraph.addEdge(reachableRoot, bundleId); + } + } + } + } + + // $FlowFixMe + dumpGraphToGraphViz(bundleGraph, 'NewBundleGraph'); }, optimize() {}, }): Bundler); @@ -193,7 +270,15 @@ const CONFIG_SCHEMA: SchemaEntity = { additionalProperties: false, }; -function createBundle(asset: Asset): Bundle { +function createBundle(asset?: Asset): Bundle { + if (asset == null) { + return { + assetIds: [], + size: 0, + sourceBundles: [], + }; + } + return { assetIds: [asset.id], size: asset.stats.size, @@ -204,7 +289,7 @@ function createBundle(asset: Asset): Bundle { function createBundleNode(bundle: Bundle): BundleNode { return { id: '', - type: '', + type: 'mybundle', value: bundle, }; } diff --git a/packages/core/core/src/dumpGraphToGraphViz.js b/packages/core/core/src/dumpGraphToGraphViz.js index a3d6bf6382e..f1614813ddc 100644 --- a/packages/core/core/src/dumpGraphToGraphViz.js +++ b/packages/core/core/src/dumpGraphToGraphViz.js @@ -104,6 +104,10 @@ export default async function dumpGraphToGraphViz( label += '\\nusedSymbols: ' + [...node.usedSymbols].join(','); } } + } else if (node.type === 'mybundle') { + label += `(assetIds: ${node.value.assetIds.join( + ', ', + )}) (sourceBundles: ${node.value.sourceBundles.join(', ')})`; } else if (node.type === 'asset_group') { if (node.deferred) label += '(deferred)'; // $FlowFixMe From 8342a9347ed2fc1f56772e8b0c56eaec8b1bdcd0 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Wed, 30 Jun 2021 14:07:22 -0700 Subject: [PATCH 04/73] Create a legacy graph from an ideal graph --- .../bundlers/default/src/DefaultBundler.js | 380 ++++++++++-------- .../core/src/public/MutableBundleGraph.js | 17 + .../core/integration-tests/test/javascript.js | 2 +- packages/core/types/index.js | 1 + 4 files changed, 235 insertions(+), 165 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 97a2a09220d..ea0c7062c1f 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -2,7 +2,8 @@ import type { Asset, - BundleGroup, + Bundle as LegacyBundle, + Dependency, Config, MutableBundleGraph, PluginOptions, @@ -54,201 +55,252 @@ type BundleNode = {| value: Bundle, |}; +type IdealGraph = {| + dependencyLoadsBundle: Map, + bundleGraph: Graph, +|}; + export default (new Bundler({ loadConfig({config, options}) { return loadBundlerConfig(config, options); }, - bundle({bundleGraph: assetGraph, config}) { - // Asset to the bundle it's an entry of - let bundleRoots: Map = new Map(); - // - let reachableBundles: DefaultMap> = new DefaultMap( - () => new Set(), + bundle({bundleGraph, config}) { + decorateLegacyGraph(createIdealGraph(bundleGraph), bundleGraph); + }, + optimize() {}, +}): Bundler); + +function decorateLegacyGraph( + idealGraph: IdealGraph, + bundleGraph: MutableBundleGraph, +): void { + let idealBundleToLegacyBundle: Map = new Map(); + + let {bundleGraph: idealBundleGraph, dependencyLoadsBundle} = idealGraph; + for (let [dependency, bundleNodeId] of dependencyLoadsBundle) { + let bundleGroup = bundleGraph.createBundleGroup( + dependency, + // TODO: don't nullthrows? + nullthrows(dependency.target), ); - // - let bundleGraph: Graph = new Graph(); - // - let stack: Array<[Asset, NodeId]> = []; - - // Step 1: Create bundles at the explicit split points in the graph. - // Create bundles for each entry. - let entries: Array = []; - assetGraph.traverse((node, context, actions) => { - if (node.type !== 'asset') { - return node; - } - invariant( - context != null && - context.type === 'dependency' && - context.value.isEntry, - ); - entries.push(node.value); - actions.skipChildren(); + // add the main bundle in the group + let mainIdealBundle = nullthrows(idealBundleGraph.getNode(bundleNodeId)) + .value; + let mainBundle = bundleGraph.createBundle({ + entryAsset: bundleGraph.getAssetById(mainIdealBundle.assetIds[0]), + target: nullthrows(dependency.target), + needsStableName: dependency.isEntry, }); + idealBundleToLegacyBundle.set(mainIdealBundle, mainBundle); + + bundleGraph.addBundleToBundleGroup(mainBundle, bundleGroup); + } + + for (let {value: bundle} of idealBundleGraph.nodes.values()) { + let assets = bundle.assetIds.map(a => bundleGraph.getAssetById(a)); - for (let entry of entries) { - let nodeId = bundleGraph.addNode(createBundleNode(createBundle(entry))); - bundleRoots.set(entry, [nodeId, nodeId]); + for (let asset of assets) { + bundleGraph.addAssetToBundle( + asset, + nullthrows(idealBundleToLegacyBundle.get(bundle)), + ); } + } +} - let assets = []; - // Traverse the asset graph and create bundles for asset type changes and async dependencies. - // This only adds the entry asset of each bundle, not the subgraph. - assetGraph.traverse({ - enter(node, context) { - //Discover - if (node.type === 'asset') { - assets.push(node.value); - - let bundleIdTuple = bundleRoots.get(node.value); - if (bundleIdTuple) { - // Push to the stack when a new bundle is created. - stack.push([node.value, bundleIdTuple[1]]); // TODO: switch this to be push/pop instead of unshift - } - } else if (node.type === 'dependency') { - if (context == null) { - return node; - } +function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { + // Asset to the bundle it's an entry of + let bundleRoots: Map = new Map(); + let dependencyLoadsBundle: Map = new Map(); + // + let reachableBundles: DefaultMap> = new DefaultMap( + () => new Set(), + ); + // + let bundleGraph: Graph = new Graph(); + // + let stack: Array<[Asset, NodeId]> = []; + + // Step 1: Create bundles at the explicit split points in the graph. + // Create bundles for each entry. + let entries: Array<[Dependency, Asset]> = []; + assetGraph.traverse((node, context, actions) => { + if (node.type !== 'asset') { + return node; + } - let dependency = node.value; - //TreeEdge Event - invariant(context?.type === 'asset'); - let parentAsset = context.value; - - let assets = assetGraph.getDependencyAssets(dependency); - invariant(assets.length === 1); - let childAsset = assets[0]; - - // Create a new bundle when the asset type changes. - if (parentAsset.type !== childAsset.type) { - let [, bundleGroupNodeId] = nullthrows(stack[stack.length - 1]); - let bundleId = bundleGraph.addNode( - createBundleNode(createBundle(childAsset)), - ); - bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); - - // Add an edge from the bundle group entry to the new bundle. - // This indicates that the bundle is loaded together with the entry - bundleGraph.addEdge(bundleGroupNodeId, bundleId); - return node; - } + invariant( + context != null && context.type === 'dependency' && context.value.isEntry, + ); + entries.push([context.value, node.value]); + actions.skipChildren(); + }); - // Create a new bundle as well as a new bundle group if the dependency is async. - if (dependency.priority === 'lazy') { - let bundleId = bundleGraph.addNode( - createBundleNode(createBundle(childAsset)), - ); - bundleRoots.set(childAsset, [bundleId, bundleId]); - - // Walk up the stack until we hit a different asset type - // and mark each bundle as reachable from every parent bundle - for (let i = stack.length - 1; i >= 0; i--) { - let [stackAsset] = stack[i]; - if (stackAsset.type !== childAsset.type) { - break; - } - reachableBundles.get(stackAsset).add(childAsset); - } - } + for (let [dependency, asset] of entries) { + let nodeId = bundleGraph.addNode(createBundleNode(createBundle(asset))); + bundleRoots.set(asset, [nodeId, nodeId]); + dependencyLoadsBundle.set(dependency, nodeId); + } + + let assets = []; + // Traverse the asset graph and create bundles for asset type changes and async dependencies. + // This only adds the entry asset of each bundle, not the subgraph. + assetGraph.traverse({ + enter(node, context) { + //Discover + if (node.type === 'asset') { + assets.push(node.value); + + let bundleIdTuple = bundleRoots.get(node.value); + if (bundleIdTuple) { + // Push to the stack when a new bundle is created. + stack.push([node.value, bundleIdTuple[1]]); // TODO: switch this to be push/pop instead of unshift } - return node; - }, - exit(node) { - if (stack[stack.length - 1] === node.value) { - stack.pop(); + } else if (node.type === 'dependency') { + if (context == null) { + return node; } - }, - }); - // Step 2: Determine reachability for every asset from each bundle root. - // This is later used to determine which bundles to place each asset in. - let reachableRoots: DefaultMap> = new DefaultMap( - () => new Set(), - ); - for (let [root] of bundleRoots) { - assetGraph.traverse((node, _, actions) => { - if (node.type !== 'asset') { - return; + let dependency = node.value; + //TreeEdge Event + invariant(context?.type === 'asset'); + let parentAsset = context.value; + + let assets = assetGraph.getDependencyAssets(dependency); + invariant(assets.length === 1); + let childAsset = assets[0]; + + // Create a new bundle when the asset type changes. + if (parentAsset.type !== childAsset.type) { + let [, bundleGroupNodeId] = nullthrows(stack[stack.length - 1]); + let bundleId = bundleGraph.addNode( + createBundleNode(createBundle(childAsset)), + ); + bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); + + // Add an edge from the bundle group entry to the new bundle. + // This indicates that the bundle is loaded together with the entry + bundleGraph.addEdge(bundleGroupNodeId, bundleId); + return node; } - if (node.value === root) { - return; + // Create a new bundle as well as a new bundle group if the dependency is async. + if (dependency.priority === 'lazy') { + let bundleId = bundleGraph.addNode( + createBundleNode(createBundle(childAsset)), + ); + bundleRoots.set(childAsset, [bundleId, bundleId]); + dependencyLoadsBundle.set(dependency, bundleId); + + // Walk up the stack until we hit a different asset type + // and mark each bundle as reachable from every parent bundle + for (let i = stack.length - 1; i >= 0; i--) { + let [stackAsset] = stack[i]; + if (stackAsset.type !== childAsset.type) { + break; + } + reachableBundles.get(stackAsset).add(childAsset); + } } + } + return node; + }, + exit(node) { + if (stack[stack.length - 1] === node.value) { + stack.pop(); + } + }, + }); - if (bundleRoots.has(node.value)) { - actions.skipChildren(); - return; - } - reachableRoots.get(node.value).add(root); - }, root); - } + // Step 2: Determine reachability for every asset from each bundle root. + // This is later used to determine which bundles to place each asset in. + let reachableRoots: DefaultMap> = new DefaultMap( + () => new Set(), + ); + for (let [root] of bundleRoots) { + assetGraph.traverse((node, _, actions) => { + if (node.type !== 'asset') { + return; + } - // Step 3: Place all assets into bundles. Each asset is placed into a single - // bundle based on the bundle entries it is reachable from. This creates a - // maximally code split bundle graph with no duplication. + if (node.value === root) { + return; + } - // Create a mapping from entry asset ids to bundle ids + if (bundleRoots.has(node.value)) { + actions.skipChildren(); + return; + } + reachableRoots.get(node.value).add(root); + }, root); + } - //TODO Step 3, some mapping from multiple entry asset ids to a bundle Id - let bundles: Map = new Map(); - for (let asset of assets) { - // Find bundle entries reachable from the asset. - let reachable = [...reachableRoots.get(asset)]; + // Step 3: Place all assets into bundles. Each asset is placed into a single + // bundle based on the bundle entries it is reachable from. This creates a + // maximally code split bundle graph with no duplication. - // Filter out bundles when the asset is reachable in a parent bundle. - reachable = reachable.filter(b => - reachable.every(a => !reachableBundles.get(a).has(b)), - ); + // Create a mapping from entry asset ids to bundle ids - let rootBundle = bundleRoots.get(asset); - if (rootBundle != null) { - // If the asset is a bundle root, add the bundle to every other reachable bundle group. - if (!bundles.has(asset.id)) { - bundles.set(asset.id, rootBundle[0]); - } - for (let reachableAsset of reachable) { - if (reachableAsset !== asset) { - bundleGraph.addEdge( - nullthrows(bundleRoots.get(reachableAsset))[1], - rootBundle[0], - ); - } - } - } else if (reachable.length > 0) { - // If the asset is reachable from more than one entry, find or create - // a bundle for that combination of entries, and add the asset to it. - let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); - let key = reachable.map(a => a.id).join(','); - - let bundleId = bundles.get(key); - if (bundleId == null) { - let bundle = createBundle(); - bundle.sourceBundles = sourceBundles; - bundleId = bundleGraph.addNode(createBundleNode(bundle)); - bundles.set(key, bundleId); + //TODO Step 3, some mapping from multiple entry asset ids to a bundle Id + let bundles: Map = new Map(); + for (let asset of assets) { + // Find bundle entries reachable from the asset. + let reachable = [...reachableRoots.get(asset)]; + + // Filter out bundles when the asset is reachable in a parent bundle. + reachable = reachable.filter(b => + reachable.every(a => !reachableBundles.get(a).has(b)), + ); + + let rootBundle = bundleRoots.get(asset); + if (rootBundle != null) { + // If the asset is a bundle root, add the bundle to every other reachable bundle group. + if (!bundles.has(asset.id)) { + bundles.set(asset.id, rootBundle[0]); + } + for (let reachableAsset of reachable) { + if (reachableAsset !== asset) { + bundleGraph.addEdge( + nullthrows(bundleRoots.get(reachableAsset))[1], + rootBundle[0], + ); } + } + } else if (reachable.length > 0) { + // If the asset is reachable from more than one entry, find or create + // a bundle for that combination of entries, and add the asset to it. + let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); + let key = reachable.map(a => a.id).join(','); + + let bundleId = bundles.get(key); + if (bundleId == null) { + let bundle = createBundle(); + bundle.sourceBundles = sourceBundles; + bundleId = bundleGraph.addNode(createBundleNode(bundle)); + bundles.set(key, bundleId); + } - let bundle = nullthrows(bundleGraph.getNode(bundleId)).value; - bundle.assetIds.push(asset.id); - bundle.size += asset.stats.size; + let bundle = nullthrows(bundleGraph.getNode(bundleId)).value; + bundle.assetIds.push(asset.id); + bundle.size += asset.stats.size; - // Add the bundle to each reachable bundle group. - for (let reachableAsset of reachable) { - let reachableRoot = nullthrows(bundleRoots.get(reachableAsset))[1]; - if (reachableRoot !== bundleId) { - bundleGraph.addEdge(reachableRoot, bundleId); - } + // Add the bundle to each reachable bundle group. + for (let reachableAsset of reachable) { + let reachableRoot = nullthrows(bundleRoots.get(reachableAsset))[1]; + if (reachableRoot !== bundleId) { + bundleGraph.addEdge(reachableRoot, bundleId); } } } + } - // $FlowFixMe - dumpGraphToGraphViz(bundleGraph, 'NewBundleGraph'); - }, - optimize() {}, -}): Bundler); + // $FlowFixMe + dumpGraphToGraphViz(bundleGraph, 'NewBundleGraph'); + + return {bundleGraph, dependencyLoadsBundle}; +} const CONFIG_SCHEMA: SchemaEntity = { type: 'object', diff --git a/packages/core/core/src/public/MutableBundleGraph.js b/packages/core/core/src/public/MutableBundleGraph.js index a0a0143adc0..5432a1fc57d 100644 --- a/packages/core/core/src/public/MutableBundleGraph.js +++ b/packages/core/core/src/public/MutableBundleGraph.js @@ -52,6 +52,23 @@ export default class MutableBundleGraph extends BundleGraph ); } + addAssetToBundle(asset: IAsset, bundle: IBundle) { + this.#graph._graph.addEdge( + this.#graph._graph.getNodeIdByContentKey(bundle.id), + this.#graph._graph.getNodeIdByContentKey(asset.id), + 'contains', + ); + + let dependencies = this.#graph.getDependencies(assetToAssetValue(asset)); + for (let dependency of dependencies) { + this.#graph._graph.addEdge( + this.#graph._graph.getNodeIdByContentKey(bundle.id), + this.#graph._graph.getNodeIdByContentKey(dependency.id), + 'contains', + ); + } + } + addEntryToBundle( asset: IAsset, bundle: IBundle, diff --git a/packages/core/integration-tests/test/javascript.js b/packages/core/integration-tests/test/javascript.js index 3ff989819f6..2c18ad94c24 100644 --- a/packages/core/integration-tests/test/javascript.js +++ b/packages/core/integration-tests/test/javascript.js @@ -22,7 +22,7 @@ describe('javascript', function() { await removeDistDirectory(); }); - it.only('should produce a basic JS bundle with CommonJS requires', async function() { + it('should produce a basic JS bundle with CommonJS requires', async function() { let b = await bundle( path.join(__dirname, '/integration/commonjs/index.js'), ); diff --git a/packages/core/types/index.js b/packages/core/types/index.js index e5480187f78..9ccec391a19 100644 --- a/packages/core/types/index.js +++ b/packages/core/types/index.js @@ -1285,6 +1285,7 @@ export interface MutableBundleGraph extends BundleGraph { Bundle, shouldSkipDependency?: (Dependency) => boolean, ): void; + addAssetToBundle(Asset, Bundle): void; addEntryToBundle( Asset, Bundle, From ea2413e3bb4db3b152eb15ce6f7f1ca45ce551bf Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Wed, 30 Jun 2021 15:15:47 -0700 Subject: [PATCH 05/73] Work around dependency targets and connect bundles to groups --- .../bundlers/default/src/DefaultBundler.js | 52 +++++++++++-------- .../core/src/public/MutableBundleGraph.js | 18 +++++-- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index ea0c7062c1f..5f9433d23e3 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -3,6 +3,7 @@ import type { Asset, Bundle as LegacyBundle, + BundleGroup, Dependency, Config, MutableBundleGraph, @@ -78,19 +79,22 @@ function decorateLegacyGraph( let idealBundleToLegacyBundle: Map = new Map(); let {bundleGraph: idealBundleGraph, dependencyLoadsBundle} = idealGraph; + let lastTarget; + let dependencyToBundleGroup: Map = new Map(); for (let [dependency, bundleNodeId] of dependencyLoadsBundle) { - let bundleGroup = bundleGraph.createBundleGroup( - dependency, - // TODO: don't nullthrows? - nullthrows(dependency.target), - ); + let target = nullthrows(dependency.target ?? lastTarget); + let bundleGroup = bundleGraph.createBundleGroup(dependency, target); + if (dependency.target != null) { + lastTarget = dependency.target; + } // add the main bundle in the group let mainIdealBundle = nullthrows(idealBundleGraph.getNode(bundleNodeId)) .value; + let entryAsset = bundleGraph.getAssetById(mainIdealBundle.assetIds[0]); let mainBundle = bundleGraph.createBundle({ - entryAsset: bundleGraph.getAssetById(mainIdealBundle.assetIds[0]), - target: nullthrows(dependency.target), + entryAsset, + target, needsStableName: dependency.isEntry, }); idealBundleToLegacyBundle.set(mainIdealBundle, mainBundle); @@ -172,22 +176,11 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { invariant(assets.length === 1); let childAsset = assets[0]; - // Create a new bundle when the asset type changes. - if (parentAsset.type !== childAsset.type) { - let [, bundleGroupNodeId] = nullthrows(stack[stack.length - 1]); - let bundleId = bundleGraph.addNode( - createBundleNode(createBundle(childAsset)), - ); - bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); - - // Add an edge from the bundle group entry to the new bundle. - // This indicates that the bundle is loaded together with the entry - bundleGraph.addEdge(bundleGroupNodeId, bundleId); - return node; - } - // Create a new bundle as well as a new bundle group if the dependency is async. - if (dependency.priority === 'lazy') { + if ( + dependency.priority === 'lazy' || + childAsset.bundleBehavior === 'isolated' + ) { let bundleId = bundleGraph.addNode( createBundleNode(createBundle(childAsset)), ); @@ -203,6 +196,21 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { } reachableBundles.get(stackAsset).add(childAsset); } + return node; + } + + // Create a new bundle when the asset type changes. + if (parentAsset.type !== childAsset.type) { + let [, bundleGroupNodeId] = nullthrows(stack[stack.length - 1]); + let bundleId = bundleGraph.addNode( + createBundleNode(createBundle(childAsset)), + ); + bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); + + // Add an edge from the bundle group entry to the new bundle. + // This indicates that the bundle is loaded together with the entry + bundleGraph.addEdge(bundleGroupNodeId, bundleId); + return node; } } return node; diff --git a/packages/core/core/src/public/MutableBundleGraph.js b/packages/core/core/src/public/MutableBundleGraph.js index 5432a1fc57d..db15aa8ce7f 100644 --- a/packages/core/core/src/public/MutableBundleGraph.js +++ b/packages/core/core/src/public/MutableBundleGraph.js @@ -53,19 +53,27 @@ export default class MutableBundleGraph extends BundleGraph } addAssetToBundle(asset: IAsset, bundle: IBundle) { + let bundleNodeId = this.#graph._graph.getNodeIdByContentKey(bundle.id); this.#graph._graph.addEdge( - this.#graph._graph.getNodeIdByContentKey(bundle.id), + bundleNodeId, this.#graph._graph.getNodeIdByContentKey(asset.id), 'contains', ); let dependencies = this.#graph.getDependencies(assetToAssetValue(asset)); for (let dependency of dependencies) { - this.#graph._graph.addEdge( - this.#graph._graph.getNodeIdByContentKey(bundle.id), - this.#graph._graph.getNodeIdByContentKey(dependency.id), - 'contains', + let dependencyNodeId = this.#graph._graph.getNodeIdByContentKey( + dependency.id, ); + this.#graph._graph.addEdge(bundleNodeId, dependencyNodeId, 'contains'); + + for (let [bundleGroupNodeId, bundleGroupNode] of this.#graph._graph + .getNodeIdsConnectedFrom(dependencyNodeId) + .map(id => [id, nullthrows(this.#graph._graph.getNode(id))]) + .filter(([, node]) => node.type === 'bundle_group')) { + invariant(bundleGroupNode.type === 'bundle_group'); + this.#graph._graph.addEdge(bundleNodeId, bundleGroupNodeId, 'bundle'); + } } } From 3c2d5086fff209b40fa7000ed4aed467049874ae Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Wed, 30 Jun 2021 16:40:56 -0700 Subject: [PATCH 06/73] Begin to handle context changes --- packages/bundlers/default/src/DefaultBundler.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 5f9433d23e3..783552ded2d 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -80,7 +80,6 @@ function decorateLegacyGraph( let {bundleGraph: idealBundleGraph, dependencyLoadsBundle} = idealGraph; let lastTarget; - let dependencyToBundleGroup: Map = new Map(); for (let [dependency, bundleNodeId] of dependencyLoadsBundle) { let target = nullthrows(dependency.target ?? lastTarget); let bundleGroup = bundleGraph.createBundleGroup(dependency, target); @@ -117,6 +116,7 @@ function decorateLegacyGraph( function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // Asset to the bundle it's an entry of let bundleRoots: Map = new Map(); + let bundles: Map = new Map(); let dependencyLoadsBundle: Map = new Map(); // let reachableBundles: DefaultMap> = new DefaultMap( @@ -144,6 +144,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { for (let [dependency, asset] of entries) { let nodeId = bundleGraph.addNode(createBundleNode(createBundle(asset))); + bundles.set(asset.id, nodeId); bundleRoots.set(asset, [nodeId, nodeId]); dependencyLoadsBundle.set(dependency, nodeId); } @@ -173,6 +174,10 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { let parentAsset = context.value; let assets = assetGraph.getDependencyAssets(dependency); + if (assets.length === 0) { + return node; + } + invariant(assets.length === 1); let childAsset = assets[0]; @@ -184,6 +189,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { let bundleId = bundleGraph.addNode( createBundleNode(createBundle(childAsset)), ); + bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); dependencyLoadsBundle.set(dependency, bundleId); @@ -191,7 +197,10 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // and mark each bundle as reachable from every parent bundle for (let i = stack.length - 1; i >= 0; i--) { let [stackAsset] = stack[i]; - if (stackAsset.type !== childAsset.type) { + if ( + stackAsset.type !== childAsset.type || + stackAsset.env.context !== childAsset.env.context + ) { break; } reachableBundles.get(stackAsset).add(childAsset); @@ -205,6 +214,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { let bundleId = bundleGraph.addNode( createBundleNode(createBundle(childAsset)), ); + bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); // Add an edge from the bundle group entry to the new bundle. @@ -252,7 +262,6 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // Create a mapping from entry asset ids to bundle ids //TODO Step 3, some mapping from multiple entry asset ids to a bundle Id - let bundles: Map = new Map(); for (let asset of assets) { // Find bundle entries reachable from the asset. let reachable = [...reachableRoots.get(asset)]; From 10b82e811ee388ea7f7c823f58b2392ed268c6da Mon Sep 17 00:00:00 2001 From: Joey Slater Date: Thu, 1 Jul 2021 11:54:26 -0500 Subject: [PATCH 07/73] Updated DefaultBundler away from BundleNode to just Bundle --- .../bundlers/default/src/DefaultBundler.js | 35 +++++-------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 783552ded2d..a37b658275d 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -43,18 +43,12 @@ const HTTP_OPTIONS = { }, }; -type BundleId = string; type AssetId = string; type Bundle = {| assetIds: Array, size: number, sourceBundles: Array, |}; -type BundleNode = {| - id: BundleId, - +type: 'mybundle', - value: Bundle, -|}; type IdealGraph = {| dependencyLoadsBundle: Map, @@ -88,8 +82,7 @@ function decorateLegacyGraph( } // add the main bundle in the group - let mainIdealBundle = nullthrows(idealBundleGraph.getNode(bundleNodeId)) - .value; + let mainIdealBundle = nullthrows(idealBundleGraph.getNode(bundleNodeId)); let entryAsset = bundleGraph.getAssetById(mainIdealBundle.assetIds[0]); let mainBundle = bundleGraph.createBundle({ entryAsset, @@ -101,7 +94,7 @@ function decorateLegacyGraph( bundleGraph.addBundleToBundleGroup(mainBundle, bundleGroup); } - for (let {value: bundle} of idealBundleGraph.nodes.values()) { + for (let bundle of idealBundleGraph.nodes.values()) { let assets = bundle.assetIds.map(a => bundleGraph.getAssetById(a)); for (let asset of assets) { @@ -123,7 +116,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { () => new Set(), ); // - let bundleGraph: Graph = new Graph(); + let bundleGraph: Graph = new Graph(); // let stack: Array<[Asset, NodeId]> = []; @@ -143,7 +136,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { }); for (let [dependency, asset] of entries) { - let nodeId = bundleGraph.addNode(createBundleNode(createBundle(asset))); + let nodeId = bundleGraph.addNode(createBundle(asset)); bundles.set(asset.id, nodeId); bundleRoots.set(asset, [nodeId, nodeId]); dependencyLoadsBundle.set(dependency, nodeId); @@ -186,9 +179,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { dependency.priority === 'lazy' || childAsset.bundleBehavior === 'isolated' ) { - let bundleId = bundleGraph.addNode( - createBundleNode(createBundle(childAsset)), - ); + let bundleId = bundleGraph.addNode(createBundle(childAsset)); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); dependencyLoadsBundle.set(dependency, bundleId); @@ -211,9 +202,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // Create a new bundle when the asset type changes. if (parentAsset.type !== childAsset.type) { let [, bundleGroupNodeId] = nullthrows(stack[stack.length - 1]); - let bundleId = bundleGraph.addNode( - createBundleNode(createBundle(childAsset)), - ); + let bundleId = bundleGraph.addNode(createBundle(childAsset)); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); @@ -295,11 +284,11 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { if (bundleId == null) { let bundle = createBundle(); bundle.sourceBundles = sourceBundles; - bundleId = bundleGraph.addNode(createBundleNode(bundle)); + bundleId = bundleGraph.addNode(bundle); bundles.set(key, bundleId); } - let bundle = nullthrows(bundleGraph.getNode(bundleId)).value; + let bundle = nullthrows(bundleGraph.getNode(bundleId)); bundle.assetIds.push(asset.id); bundle.size += asset.stats.size; @@ -355,14 +344,6 @@ function createBundle(asset?: Asset): Bundle { }; } -function createBundleNode(bundle: Bundle): BundleNode { - return { - id: '', - type: 'mybundle', - value: bundle, - }; -} - async function loadBundlerConfig(config: Config, options: PluginOptions) { let conf = await config.getConfig([], { packageKey: '@parcel/bundler-default', From c3fc80f8ff15aa33229bc85c3c0155b913d477a0 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Thu, 1 Jul 2021 16:29:51 -0400 Subject: [PATCH 08/73] Don't extract shared bundles from entries --- .../bundlers/default/src/DefaultBundler.js | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index a37b658275d..49b60ccae1c 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -52,7 +52,7 @@ type Bundle = {| type IdealGraph = {| dependencyLoadsBundle: Map, - bundleGraph: Graph, + bundleGraph: Graph, |}; export default (new Bundler({ @@ -122,7 +122,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // Step 1: Create bundles at the explicit split points in the graph. // Create bundles for each entry. - let entries: Array<[Dependency, Asset]> = []; + let entries: Map = new Map(); assetGraph.traverse((node, context, actions) => { if (node.type !== 'asset') { return node; @@ -131,11 +131,11 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { invariant( context != null && context.type === 'dependency' && context.value.isEntry, ); - entries.push([context.value, node.value]); + entries.set(node.value, context.value); actions.skipChildren(); }); - for (let [dependency, asset] of entries) { + for (let [asset, dependency] of entries) { let nodeId = bundleGraph.addNode(createBundle(asset)); bundles.set(asset.id, nodeId); bundleRoots.set(asset, [nodeId, nodeId]); @@ -249,15 +249,27 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // maximally code split bundle graph with no duplication. // Create a mapping from entry asset ids to bundle ids - - //TODO Step 3, some mapping from multiple entry asset ids to a bundle Id for (let asset of assets) { // Find bundle entries reachable from the asset. - let reachable = [...reachableRoots.get(asset)]; + let reachable: Array = [...reachableRoots.get(asset)]; + + let reachableEntries = reachable.filter(a => entries.has(a)); + + for (let entry of reachableEntries) { + //add asset ids to entry bundle + let entryAssetBundle = nullthrows( + bundleGraph.getNode(nullthrows(bundleRoots.get(entry))[0]), + ); + + entryAssetBundle.assetIds.push(asset.id); + entryAssetBundle.size += asset.stats.size; + } // Filter out bundles when the asset is reachable in a parent bundle. - reachable = reachable.filter(b => - reachable.every(a => !reachableBundles.get(a).has(b)), + reachable = reachable.filter( + b => + !entries.has(b) && + reachable.every(a => !reachableBundles.get(a).has(b)), ); let rootBundle = bundleRoots.get(asset); @@ -267,7 +279,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { bundles.set(asset.id, rootBundle[0]); } for (let reachableAsset of reachable) { - if (reachableAsset !== asset) { + if (reachableAsset !== asset && !entries.has(reachableAsset)) { bundleGraph.addEdge( nullthrows(bundleRoots.get(reachableAsset))[1], rootBundle[0], @@ -276,19 +288,21 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { } } else if (reachable.length > 0) { // If the asset is reachable from more than one entry, find or create - // a bundle for that combination of entries, and add the asset to it. + // a bundle for that combination of bundles, and add the asset to it. let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); let key = reachable.map(a => a.id).join(','); let bundleId = bundles.get(key); + let bundle; if (bundleId == null) { - let bundle = createBundle(); + bundle = createBundle(); bundle.sourceBundles = sourceBundles; bundleId = bundleGraph.addNode(bundle); bundles.set(key, bundleId); + } else { + bundle = nullthrows(bundleGraph.getNode(bundleId)); } - let bundle = nullthrows(bundleGraph.getNode(bundleId)); bundle.assetIds.push(asset.id); bundle.size += asset.stats.size; From 211905e2bbb89fd851b7325fca4a170961e7d01f Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Thu, 1 Jul 2021 17:32:38 -0400 Subject: [PATCH 09/73] WIP single traversal --- .../bundlers/default/src/DefaultBundler.js | 70 ++++++++++++------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 49b60ccae1c..0214f1fb8d2 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -51,8 +51,9 @@ type Bundle = {| |}; type IdealGraph = {| - dependencyLoadsBundle: Map, + bundleLoadedByDependency: Map, bundleGraph: Graph, + entryBundles: Array, |}; export default (new Bundler({ @@ -70,28 +71,45 @@ function decorateLegacyGraph( idealGraph: IdealGraph, bundleGraph: MutableBundleGraph, ): void { + let entryBundles = idealGraph.entryBundles; let idealBundleToLegacyBundle: Map = new Map(); + //TODO Single traversal of bundles/bundlegroups - let {bundleGraph: idealBundleGraph, dependencyLoadsBundle} = idealGraph; + let {bundleGraph: idealBundleGraph, bundleLoadedByDependency} = idealGraph; + let visited: Set = new Set(); let lastTarget; - for (let [dependency, bundleNodeId] of dependencyLoadsBundle) { - let target = nullthrows(dependency.target ?? lastTarget); - let bundleGroup = bundleGraph.createBundleGroup(dependency, target); - if (dependency.target != null) { - lastTarget = dependency.target; - } + for (let entryBundle of entryBundles) { + idealBundleGraph.traverse((bundleNodeId, _, actions) => { + if (visited.has(bundleNodeId)) { + actions.skipChildren(); + return; + } + visited.add(bundleNodeId); + + let dependency = bundleLoadedByDependency.get(bundleNodeId); + if (dependency) { + let target = nullthrows(dependency.target ?? lastTarget); + let bundleGroup = bundleGraph.createBundleGroup(dependency, target); + if (dependency.target != null) { + lastTarget = dependency.target; + } + + // add the main bundle in the group + let mainIdealBundle = nullthrows( + idealBundleGraph.getNode(bundleNodeId), + ); - // add the main bundle in the group - let mainIdealBundle = nullthrows(idealBundleGraph.getNode(bundleNodeId)); - let entryAsset = bundleGraph.getAssetById(mainIdealBundle.assetIds[0]); - let mainBundle = bundleGraph.createBundle({ - entryAsset, - target, - needsStableName: dependency.isEntry, - }); - idealBundleToLegacyBundle.set(mainIdealBundle, mainBundle); - - bundleGraph.addBundleToBundleGroup(mainBundle, bundleGroup); + let entryAsset = bundleGraph.getAssetById(mainIdealBundle.assetIds[0]); + let mainBundle = bundleGraph.createBundle({ + entryAsset, + target, + needsStableName: dependency.isEntry, + }); + idealBundleToLegacyBundle.set(mainIdealBundle, mainBundle); + + bundleGraph.addBundleToBundleGroup(mainBundle, bundleGroup); + } + }, entryBundle); } for (let bundle of idealBundleGraph.nodes.values()) { @@ -110,14 +128,13 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // Asset to the bundle it's an entry of let bundleRoots: Map = new Map(); let bundles: Map = new Map(); - let dependencyLoadsBundle: Map = new Map(); + let bundleLoadedByDependency: Map = new Map(); // let reachableBundles: DefaultMap> = new DefaultMap( () => new Set(), ); // let bundleGraph: Graph = new Graph(); - // let stack: Array<[Asset, NodeId]> = []; // Step 1: Create bundles at the explicit split points in the graph. @@ -139,7 +156,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { let nodeId = bundleGraph.addNode(createBundle(asset)); bundles.set(asset.id, nodeId); bundleRoots.set(asset, [nodeId, nodeId]); - dependencyLoadsBundle.set(dependency, nodeId); + bundleLoadedByDependency.set(nodeId, dependency); } let assets = []; @@ -182,7 +199,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { let bundleId = bundleGraph.addNode(createBundle(childAsset)); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); - dependencyLoadsBundle.set(dependency, bundleId); + bundleLoadedByDependency.set(bundleId, dependency); // Walk up the stack until we hit a different asset type // and mark each bundle as reachable from every parent bundle @@ -302,7 +319,6 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { } else { bundle = nullthrows(bundleGraph.getNode(bundleId)); } - bundle.assetIds.push(asset.id); bundle.size += asset.stats.size; @@ -319,7 +335,11 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // $FlowFixMe dumpGraphToGraphViz(bundleGraph, 'NewBundleGraph'); - return {bundleGraph, dependencyLoadsBundle}; + return { + bundleGraph, + bundleLoadedByDependency, + entryBundles: [...bundleRoots.values()].map(v => v[0]), + }; } const CONFIG_SCHEMA: SchemaEntity = { From 6ee81add02b9dda3de210f21d6a380a3b36c416a Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Fri, 2 Jul 2021 14:28:02 -0400 Subject: [PATCH 10/73] fix missing edge from sibling bundle to main bundle group --- .../bundlers/default/src/DefaultBundler.js | 74 ++++++++++--------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 0214f1fb8d2..5df89bc4189 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -44,7 +44,7 @@ const HTTP_OPTIONS = { }; type AssetId = string; -type Bundle = {| +export type Bundle = {| assetIds: Array, size: number, sourceBundles: Array, @@ -78,48 +78,50 @@ function decorateLegacyGraph( let {bundleGraph: idealBundleGraph, bundleLoadedByDependency} = idealGraph; let visited: Set = new Set(); let lastTarget; - for (let entryBundle of entryBundles) { - idealBundleGraph.traverse((bundleNodeId, _, actions) => { - if (visited.has(bundleNodeId)) { - actions.skipChildren(); - return; + + let entryBundleToBundleGroup: Map = new Map(); + + for (let [bundleNodeId, idealBundle] of idealBundleGraph.nodes) { + let dependency = bundleLoadedByDependency.get(bundleNodeId); + let target = lastTarget; + let bundleGroup; + if (dependency) { + target = nullthrows(dependency.target ?? lastTarget); + bundleGroup = bundleGraph.createBundleGroup(dependency, target); + if (dependency.target != null) { + lastTarget = dependency.target; } - visited.add(bundleNodeId); - - let dependency = bundleLoadedByDependency.get(bundleNodeId); - if (dependency) { - let target = nullthrows(dependency.target ?? lastTarget); - let bundleGroup = bundleGraph.createBundleGroup(dependency, target); - if (dependency.target != null) { - lastTarget = dependency.target; - } + entryBundleToBundleGroup.set(bundleNodeId, bundleGroup); + } - // add the main bundle in the group - let mainIdealBundle = nullthrows( - idealBundleGraph.getNode(bundleNodeId), - ); + let entryAsset = bundleGraph.getAssetById(idealBundle.assetIds[0]); + let bundle = nullthrows( + bundleGraph.createBundle({ + entryAsset, + target: nullthrows(target), + needsStableName: dependency?.isEntry, + }), + ); + if (bundleGroup) { + bundleGraph.addBundleToBundleGroup(bundle, bundleGroup); + } + idealBundleToLegacyBundle.set(idealBundle, bundle); - let entryAsset = bundleGraph.getAssetById(mainIdealBundle.assetIds[0]); - let mainBundle = bundleGraph.createBundle({ - entryAsset, - target, - needsStableName: dependency.isEntry, - }); - idealBundleToLegacyBundle.set(mainIdealBundle, mainBundle); + let assets = idealBundle.assetIds.map(a => bundleGraph.getAssetById(a)); - bundleGraph.addBundleToBundleGroup(mainBundle, bundleGroup); - } - }, entryBundle); + for (let asset of assets) { + bundleGraph.addAssetToBundle(asset, bundle); + } } - for (let bundle of idealBundleGraph.nodes.values()) { - let assets = bundle.assetIds.map(a => bundleGraph.getAssetById(a)); - - for (let asset of assets) { - bundleGraph.addAssetToBundle( - asset, - nullthrows(idealBundleToLegacyBundle.get(bundle)), + for (let [bundleId, bundleGroup] of entryBundleToBundleGroup) { + let outboundNodeIds = idealBundleGraph.getNodeIdsConnectedFrom(bundleId); + for (let id of outboundNodeIds) { + let siblingBundle = nullthrows(idealBundleGraph.getNode(id)); + let legacySiblingBundle = nullthrows( + idealBundleToLegacyBundle.get(siblingBundle), ); + bundleGraph.addBundleToBundleGroup(legacySiblingBundle, bundleGroup); } } } From 02edbc5c42b745f256921fa578d29602c57953d1 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Fri, 2 Jul 2021 16:11:54 -0400 Subject: [PATCH 11/73] alter dumpgraph to not expect node --- packages/core/core/src/dumpGraphToGraphViz.js | 148 ++++++++++-------- 1 file changed, 79 insertions(+), 69 deletions(-) diff --git a/packages/core/core/src/dumpGraphToGraphViz.js b/packages/core/core/src/dumpGraphToGraphViz.js index f1614813ddc..c49fca433da 100644 --- a/packages/core/core/src/dumpGraphToGraphViz.js +++ b/packages/core/core/src/dumpGraphToGraphViz.js @@ -5,6 +5,7 @@ import type {Environment} from './types'; import type Graph from './Graph'; import type {AssetGraphNode, BundleGraphNode} from './types'; import {SpecifierType, Priority} from './types'; +import type {Bundle} from '@parcel/bundler-default'; import path from 'path'; @@ -31,7 +32,7 @@ const TYPE_COLORS = { export default async function dumpGraphToGraphViz( // $FlowFixMe - graph: Graph | Graph, + graph: Graph, name: string, ): Promise { if ( @@ -53,81 +54,90 @@ export default async function dumpGraphToGraphViz( n.set('color', COLORS[node.type || 'default']); n.set('shape', 'box'); n.set('style', 'filled'); - let label = `${node.type || 'No Type'}: [${node.id}]: `; - if (node.type === 'dependency') { - label += node.value.specifier; - let parts = []; - if (node.value.priority !== Priority.sync) - parts.push(node.value.priority); - if (node.value.isOptional) parts.push('optional'); - if (node.value.specifierType === SpecifierType.url) parts.push('url'); - if (node.hasDeferred) parts.push('deferred'); - if (node.excluded) parts.push('excluded'); - if (parts.length) label += ' (' + parts.join(', ') + ')'; - if (node.value.env) label += ` (${getEnvDescription(node.value.env)})`; - let depSymbols = node.value.symbols; - if (detailedSymbols) { - if (depSymbols) { - if (depSymbols.size) { + let label; + if (node.type) { + label = `${node.type || 'No Type'}: [${node.id}]: `; + if (node.type === 'dependency') { + label += node.value.specifier; + let parts = []; + if (node.value.priority !== Priority.sync) + parts.push(node.value.priority); + if (node.value.isOptional) parts.push('optional'); + if (node.value.specifierType === SpecifierType.url) parts.push('url'); + if (node.hasDeferred) parts.push('deferred'); + if (node.excluded) parts.push('excluded'); + if (parts.length) label += ' (' + parts.join(', ') + ')'; + if (node.value.env) label += ` (${getEnvDescription(node.value.env)})`; + let depSymbols = node.value.symbols; + if (detailedSymbols) { + if (depSymbols) { + if (depSymbols.size) { + label += + '\\nsymbols: ' + + [...depSymbols].map(([e, {local}]) => [e, local]).join(';'); + } + let weakSymbols = [...depSymbols] + .filter(([, {isWeak}]) => isWeak) + .map(([s]) => s); + if (weakSymbols.length) { + label += '\\nweakSymbols: ' + weakSymbols.join(','); + } + if (node.usedSymbolsUp.size > 0) { + label += '\\nusedSymbolsUp: ' + [...node.usedSymbolsUp].join(','); + } + if (node.usedSymbolsDown.size > 0) { + label += + '\\nusedSymbolsDown: ' + [...node.usedSymbolsDown].join(','); + } + } else { + label += '\\nsymbols: cleared'; + } + } + } else if (node.type === 'asset') { + label += path.basename(node.value.filePath) + '#' + node.value.type; + if (detailedSymbols) { + if (!node.value.symbols) { + label += '\\nsymbols: cleared'; + } else if (node.value.symbols.size) { label += '\\nsymbols: ' + - [...depSymbols].map(([e, {local}]) => [e, local]).join(';'); - } - let weakSymbols = [...depSymbols] - .filter(([, {isWeak}]) => isWeak) - .map(([s]) => s); - if (weakSymbols.length) { - label += '\\nweakSymbols: ' + weakSymbols.join(','); - } - if (node.usedSymbolsUp.size > 0) { - label += '\\nusedSymbolsUp: ' + [...node.usedSymbolsUp].join(','); + [...node.value.symbols] + .map(([e, {local}]) => [e, local]) + .join(';'); } - if (node.usedSymbolsDown.size > 0) { - label += - '\\nusedSymbolsDown: ' + [...node.usedSymbolsDown].join(','); + if (node.usedSymbols.size) { + label += '\\nusedSymbols: ' + [...node.usedSymbols].join(','); } - } else { - label += '\\nsymbols: cleared'; - } - } - } else if (node.type === 'asset') { - label += path.basename(node.value.filePath) + '#' + node.value.type; - if (detailedSymbols) { - if (!node.value.symbols) { - label += '\\nsymbols: cleared'; - } else if (node.value.symbols.size) { - label += - '\\nsymbols: ' + - [...node.value.symbols].map(([e, {local}]) => [e, local]).join(';'); - } - if (node.usedSymbols.size) { - label += '\\nusedSymbols: ' + [...node.usedSymbols].join(','); } + } else if (node.type === 'mybundle') { + label += `(assetIds: ${node.value.assetIds.join( + ', ', + )}) (sourceBundles: ${node.value.sourceBundles.join(', ')})`; + } else if (node.type === 'asset_group') { + if (node.deferred) label += '(deferred)'; + // $FlowFixMe + } else if (node.type === 'file') { + label += path.basename(node.value.filePath); + // $FlowFixMe + } else if (node.type === 'transformer_request') { + label += + path.basename(node.value.filePath) + + ` (${getEnvDescription(node.value.env)})`; + // $FlowFixMe + } else if (node.type === 'bundle') { + let parts = []; + if (node.value.needsStableName) parts.push('stable name'); + if (node.value.bundleBehavior) parts.push(node.value.bundleBehavior); + if (parts.length) label += ' (' + parts.join(', ') + ')'; + if (node.value.env) label += ` (${getEnvDescription(node.value.env)})`; + // $FlowFixMe + } else if (node.type === 'request') { + label = node.value.type + ':' + node.id; } - } else if (node.type === 'mybundle') { - label += `(assetIds: ${node.value.assetIds.join( + } else { + label = `(${nodeId(id)}), (assetIds: ${node.assetIds.join( ', ', - )}) (sourceBundles: ${node.value.sourceBundles.join(', ')})`; - } else if (node.type === 'asset_group') { - if (node.deferred) label += '(deferred)'; - // $FlowFixMe - } else if (node.type === 'file') { - label += path.basename(node.value.filePath); - // $FlowFixMe - } else if (node.type === 'transformer_request') { - label += - path.basename(node.value.filePath) + - ` (${getEnvDescription(node.value.env)})`; - // $FlowFixMe - } else if (node.type === 'bundle') { - let parts = []; - if (node.value.needsStableName) parts.push('stable name'); - if (node.value.bundleBehavior) parts.push(node.value.bundleBehavior); - if (parts.length) label += ' (' + parts.join(', ') + ')'; - if (node.value.env) label += ` (${getEnvDescription(node.value.env)})`; - // $FlowFixMe - } else if (node.type === 'request') { - label = node.value.type + ':' + node.id; + )}) (sourceBundles: ${node.sourceBundles.join(', ')})`; } n.set('label', label); } From 3b91ac35faee20fbc9a027aa525c8d6803f5dd06 Mon Sep 17 00:00:00 2001 From: Joey Slater Date: Fri, 2 Jul 2021 17:31:00 -0500 Subject: [PATCH 12/73] Handles bundle references and better handling of Target --- .../bundlers/default/src/DefaultBundler.js | 116 +++++++++++++----- packages/core/core/src/public/Environment.js | 7 ++ .../core/src/public/MutableBundleGraph.js | 4 + packages/core/core/src/public/Target.js | 7 ++ 4 files changed, 106 insertions(+), 28 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 5df89bc4189..a83aa1eedb4 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -5,9 +5,11 @@ import type { Bundle as LegacyBundle, BundleGroup, Dependency, + Environment, Config, MutableBundleGraph, PluginOptions, + Target, } from '@parcel/types'; import type {NodeId} from '@parcel/core/src/types'; import type {SchemaEntity} from '@parcel/utils'; @@ -48,6 +50,9 @@ export type Bundle = {| assetIds: Array, size: number, sourceBundles: Array, + target: Target, + env: Environment, + type: string, |}; type IdealGraph = {| @@ -71,40 +76,46 @@ function decorateLegacyGraph( idealGraph: IdealGraph, bundleGraph: MutableBundleGraph, ): void { - let entryBundles = idealGraph.entryBundles; let idealBundleToLegacyBundle: Map = new Map(); - //TODO Single traversal of bundles/bundlegroups let {bundleGraph: idealBundleGraph, bundleLoadedByDependency} = idealGraph; - let visited: Set = new Set(); - let lastTarget; - let entryBundleToBundleGroup: Map = new Map(); for (let [bundleNodeId, idealBundle] of idealBundleGraph.nodes) { let dependency = bundleLoadedByDependency.get(bundleNodeId); - let target = lastTarget; + + let entryAsset = bundleGraph.getAssetById(idealBundle.assetIds[0]); let bundleGroup; + let bundle; if (dependency) { - target = nullthrows(dependency.target ?? lastTarget); - bundleGroup = bundleGraph.createBundleGroup(dependency, target); - if (dependency.target != null) { - lastTarget = dependency.target; - } + bundleGroup = bundleGraph.createBundleGroup( + dependency, + idealBundle.target, + ); entryBundleToBundleGroup.set(bundleNodeId, bundleGroup); - } - let entryAsset = bundleGraph.getAssetById(idealBundle.assetIds[0]); - let bundle = nullthrows( - bundleGraph.createBundle({ - entryAsset, - target: nullthrows(target), - needsStableName: dependency?.isEntry, - }), - ); - if (bundleGroup) { + bundle = nullthrows( + bundleGraph.createBundle({ + entryAsset, + needsStableName: dependency?.isEntry, + target: idealBundle.target, + }), + ); + bundleGraph.addBundleToBundleGroup(bundle, bundleGroup); + } else { + bundle = nullthrows( + bundleGraph.createBundle({ + uniqueKey: + idealBundle.assetIds.join(',') + + idealBundle.sourceBundles.join(','), + type: idealBundle.type, + target: idealBundle.target, + env: idealBundle.env, + }), + ); } + idealBundleToLegacyBundle.set(idealBundle, bundle); let assets = idealBundle.assetIds.map(a => bundleGraph.getAssetById(a)); @@ -116,12 +127,23 @@ function decorateLegacyGraph( for (let [bundleId, bundleGroup] of entryBundleToBundleGroup) { let outboundNodeIds = idealBundleGraph.getNodeIdsConnectedFrom(bundleId); + let mainBundleOfBundleGroup = nullthrows( + idealBundleGraph.getNode(bundleId), + ); + let legacyMainBundleOfBundleGroup = nullthrows( + idealBundleToLegacyBundle.get(mainBundleOfBundleGroup), + ); + for (let id of outboundNodeIds) { let siblingBundle = nullthrows(idealBundleGraph.getNode(id)); let legacySiblingBundle = nullthrows( idealBundleToLegacyBundle.get(siblingBundle), ); bundleGraph.addBundleToBundleGroup(legacySiblingBundle, bundleGroup); + bundleGraph.createBundleReference( + legacyMainBundleOfBundleGroup, + legacySiblingBundle, + ); } } } @@ -155,7 +177,9 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { }); for (let [asset, dependency] of entries) { - let nodeId = bundleGraph.addNode(createBundle(asset)); + let nodeId = bundleGraph.addNode( + createBundle({asset, target: nullthrows(dependency.target)}), + ); bundles.set(asset.id, nodeId); bundleRoots.set(asset, [nodeId, nodeId]); bundleLoadedByDependency.set(nodeId, dependency); @@ -198,7 +222,12 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { dependency.priority === 'lazy' || childAsset.bundleBehavior === 'isolated' ) { - let bundleId = bundleGraph.addNode(createBundle(childAsset)); + let bundleId = bundleGraph.addNode( + createBundle({ + asset: childAsset, + target: nullthrows(bundleGraph.getNode(stack[0][1])).target, + }), + ); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); bundleLoadedByDependency.set(bundleId, dependency); @@ -209,7 +238,8 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { let [stackAsset] = stack[i]; if ( stackAsset.type !== childAsset.type || - stackAsset.env.context !== childAsset.env.context + stackAsset.env.context !== childAsset.env.context || + stackAsset.env.isIsolated() ) { break; } @@ -221,7 +251,10 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // Create a new bundle when the asset type changes. if (parentAsset.type !== childAsset.type) { let [, bundleGroupNodeId] = nullthrows(stack[stack.length - 1]); - let bundleId = bundleGraph.addNode(createBundle(childAsset)); + let bundleGroup = nullthrows(bundleGraph.getNode(bundleGroupNodeId)); + let bundleId = bundleGraph.addNode( + createBundle({asset: childAsset, target: bundleGroup.target}), + ); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); @@ -314,7 +347,14 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { let bundleId = bundles.get(key); let bundle; if (bundleId == null) { - bundle = createBundle(); + let firstSourceBundle = nullthrows( + bundleGraph.getNode(sourceBundles[0]), + ); + bundle = createBundle({ + target: firstSourceBundle.target, + type: firstSourceBundle.type, + env: firstSourceBundle.env, + }); bundle.sourceBundles = sourceBundles; bundleId = bundleGraph.addNode(bundle); bundles.set(key, bundleId); @@ -364,19 +404,39 @@ const CONFIG_SCHEMA: SchemaEntity = { additionalProperties: false, }; -function createBundle(asset?: Asset): Bundle { - if (asset == null) { +function createBundle( + opts: + | {| + target: Target, + env: Environment, + type: string, + |} + | {| + target: Target, + asset: Asset, + env?: Environment, + type?: string, + |}, +): Bundle { + if (opts.asset == null) { return { assetIds: [], size: 0, sourceBundles: [], + target: opts.target, + type: nullthrows(opts.type), + env: nullthrows(opts.env), }; } + let asset = nullthrows(opts.asset); return { assetIds: [asset.id], size: asset.stats.size, sourceBundles: [], + target: opts.target, + type: opts.type ?? asset.type, + env: opts.env ?? asset.env, }; } diff --git a/packages/core/core/src/public/Environment.js b/packages/core/core/src/public/Environment.js index 377bbe815cd..1edfd425fbe 100644 --- a/packages/core/core/src/public/Environment.js +++ b/packages/core/core/src/public/Environment.js @@ -16,6 +16,8 @@ import nullthrows from 'nullthrows'; import browserslist from 'browserslist'; import semver from 'semver'; +const inspect = Symbol.for('nodejs.util.inspect.custom'); + export const BROWSER_ENVS: Set = new Set([ 'browser', 'web-worker', @@ -162,6 +164,11 @@ export default class Environment implements IEnvironment { return this.#environment.loc; } + // $FlowFixMe[unsupported-syntax] + [inspect](): string { + return `Env(${this.#environment.context})`; + } + isBrowser(): boolean { return BROWSER_ENVS.has(this.#environment.context); } diff --git a/packages/core/core/src/public/MutableBundleGraph.js b/packages/core/core/src/public/MutableBundleGraph.js index db15aa8ce7f..a3dce629e90 100644 --- a/packages/core/core/src/public/MutableBundleGraph.js +++ b/packages/core/core/src/public/MutableBundleGraph.js @@ -59,6 +59,10 @@ export default class MutableBundleGraph extends BundleGraph this.#graph._graph.getNodeIdByContentKey(asset.id), 'contains', ); + this.#graph._graph.addEdge( + bundleNodeId, + this.#graph._graph.getNodeIdByContentKey(asset.id), + ); let dependencies = this.#graph.getDependencies(assetToAssetValue(asset)); for (let dependency of dependencies) { diff --git a/packages/core/core/src/public/Target.js b/packages/core/core/src/public/Target.js index a04635737c5..08808911cfe 100644 --- a/packages/core/core/src/public/Target.js +++ b/packages/core/core/src/public/Target.js @@ -9,6 +9,8 @@ import type {Target as TargetValue} from '../types'; import Environment from './Environment'; import nullthrows from 'nullthrows'; +const inspect = Symbol.for('nodejs.util.inspect.custom'); + const internalTargetToTarget: WeakMap = new WeakMap(); const _targetToInternalTarget: WeakMap = new WeakMap(); export function targetToInternalTarget(target: ITarget): TargetValue { @@ -53,4 +55,9 @@ export default class Target implements ITarget { get loc(): ?SourceLocation { return this.#target.loc; } + + // $FlowFixMe[unsupported-syntax] + [inspect](): string { + return `Target(${this.name} - ${this.env[inspect]()})`; + } } From feb3c260255fa00eadda1c1fcb0d49c1047601c5 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 6 Jul 2021 12:52:01 -0700 Subject: [PATCH 13/73] Add needsStableName and bundleBehavior --- .../bundlers/default/src/DefaultBundler.js | 30 ++++++++++++++++--- packages/core/core/src/dumpGraphToGraphViz.js | 19 ++++++------ .../core/utils/src/replaceBundleReferences.js | 4 +-- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index a83aa1eedb4..1bc6938aae3 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -3,6 +3,7 @@ import type { Asset, Bundle as LegacyBundle, + BundleBehavior, BundleGroup, Dependency, Environment, @@ -48,6 +49,8 @@ const HTTP_OPTIONS = { type AssetId = string; export type Bundle = {| assetIds: Array, + bundleBehavior?: ?BundleBehavior, + needsStableName: boolean, size: number, sourceBundles: Array, target: Target, @@ -97,7 +100,8 @@ function decorateLegacyGraph( bundle = nullthrows( bundleGraph.createBundle({ entryAsset, - needsStableName: dependency?.isEntry, + needsStableName: idealBundle.needsStableName, + bundleBehavior: idealBundle.bundleBehavior, target: idealBundle.target, }), ); @@ -109,6 +113,8 @@ function decorateLegacyGraph( uniqueKey: idealBundle.assetIds.join(',') + idealBundle.sourceBundles.join(','), + needsStableName: idealBundle.needsStableName, + bundleBehavior: idealBundle.bundleBehavior, type: idealBundle.type, target: idealBundle.target, env: idealBundle.env, @@ -178,7 +184,11 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { for (let [asset, dependency] of entries) { let nodeId = bundleGraph.addNode( - createBundle({asset, target: nullthrows(dependency.target)}), + createBundle({ + asset, + target: nullthrows(dependency.target), + needsStableName: dependency.isEntry, + }), ); bundles.set(asset.id, nodeId); bundleRoots.set(asset, [nodeId, nodeId]); @@ -249,11 +259,18 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { } // Create a new bundle when the asset type changes. - if (parentAsset.type !== childAsset.type) { + if ( + parentAsset.type !== childAsset.type || + childAsset.bundleBehavior === 'inline' + ) { let [, bundleGroupNodeId] = nullthrows(stack[stack.length - 1]); let bundleGroup = nullthrows(bundleGraph.getNode(bundleGroupNodeId)); let bundleId = bundleGraph.addNode( - createBundle({asset: childAsset, target: bundleGroup.target}), + createBundle({ + asset: childAsset, + target: bundleGroup.target, + needsStableName: dependency.bundleBehavior === 'inline', + }), ); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); @@ -410,12 +427,14 @@ function createBundle( target: Target, env: Environment, type: string, + needsStableName?: boolean, |} | {| target: Target, asset: Asset, env?: Environment, type?: string, + needsStableName?: boolean, |}, ): Bundle { if (opts.asset == null) { @@ -426,6 +445,7 @@ function createBundle( target: opts.target, type: nullthrows(opts.type), env: nullthrows(opts.env), + needsStableName: Boolean(opts.needsStableName), }; } @@ -437,6 +457,8 @@ function createBundle( target: opts.target, type: opts.type ?? asset.type, env: opts.env ?? asset.env, + needsStableName: Boolean(opts.needsStableName), + bundleBehavior: asset.bundleBehavior, }; } diff --git a/packages/core/core/src/dumpGraphToGraphViz.js b/packages/core/core/src/dumpGraphToGraphViz.js index c49fca433da..323b6adbe53 100644 --- a/packages/core/core/src/dumpGraphToGraphViz.js +++ b/packages/core/core/src/dumpGraphToGraphViz.js @@ -55,7 +55,13 @@ export default async function dumpGraphToGraphViz( n.set('shape', 'box'); n.set('style', 'filled'); let label; - if (node.type) { + if (node.assetIds) { + label = `(${nodeId(id)}), (assetIds: ${node.assetIds.join( + ', ', + )}) (sourceBundles: ${node.sourceBundles.join( + ', ', + )}) (bb ${node.bundleBehavior ?? 'none'})`; + } else if (node.type) { label = `${node.type || 'No Type'}: [${node.id}]: `; if (node.type === 'dependency') { label += node.value.specifier; @@ -109,10 +115,6 @@ export default async function dumpGraphToGraphViz( label += '\\nusedSymbols: ' + [...node.usedSymbols].join(','); } } - } else if (node.type === 'mybundle') { - label += `(assetIds: ${node.value.assetIds.join( - ', ', - )}) (sourceBundles: ${node.value.sourceBundles.join(', ')})`; } else if (node.type === 'asset_group') { if (node.deferred) label += '(deferred)'; // $FlowFixMe @@ -127,17 +129,14 @@ export default async function dumpGraphToGraphViz( } else if (node.type === 'bundle') { let parts = []; if (node.value.needsStableName) parts.push('stable name'); - if (node.value.bundleBehavior) parts.push(node.value.bundleBehavior); + if (node.value.name) parts.push('[name ', node.value.name, ']'); + parts.push('bb-', node.value.bundleBehavior); if (parts.length) label += ' (' + parts.join(', ') + ')'; if (node.value.env) label += ` (${getEnvDescription(node.value.env)})`; // $FlowFixMe } else if (node.type === 'request') { label = node.value.type + ':' + node.id; } - } else { - label = `(${nodeId(id)}), (assetIds: ${node.assetIds.join( - ', ', - )}) (sourceBundles: ${node.sourceBundles.join(', ')})`; } n.set('label', label); } diff --git a/packages/core/utils/src/replaceBundleReferences.js b/packages/core/utils/src/replaceBundleReferences.js index 1fcf520df7c..f0d69892560 100644 --- a/packages/core/utils/src/replaceBundleReferences.js +++ b/packages/core/utils/src/replaceBundleReferences.js @@ -53,7 +53,7 @@ export function replaceURLReferences({ continue; } - let resolved = bundleGraph.getReferencedBundle(dependency, bundle); + const resolved = bundleGraph.getReferencedBundle(dependency, bundle); if (resolved == null) { replacements.set(dependency.id, { from: dependency.id, @@ -62,7 +62,7 @@ export function replaceURLReferences({ continue; } - if (!resolved || resolved.bundleBehavior === 'inline') { + if (resolved.bundleBehavior === 'inline') { // If a bundle is inline, it should be replaced with inline contents, // not a URL. continue; From 26011324d9135b100e1915d4967d3d67ccff0bd3 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Tue, 6 Jul 2021 18:08:46 -0400 Subject: [PATCH 14/73] WIP for fixing reference edges --- .../bundlers/default/src/DefaultBundler.js | 43 ++++++++++++++----- packages/core/core/src/BundleGraph.js | 14 ++++++ .../core/src/public/MutableBundleGraph.js | 18 +++++++- 3 files changed, 63 insertions(+), 12 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 1bc6938aae3..03c247f4ec0 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -62,6 +62,7 @@ type IdealGraph = {| bundleLoadedByDependency: Map, bundleGraph: Graph, entryBundles: Array, + assetReference: DefaultMap>, |}; export default (new Bundler({ @@ -79,6 +80,7 @@ function decorateLegacyGraph( idealGraph: IdealGraph, bundleGraph: MutableBundleGraph, ): void { + //TODO add in reference edges based on stored assets from create ideal graph let idealBundleToLegacyBundle: Map = new Map(); let {bundleGraph: idealBundleGraph, bundleLoadedByDependency} = idealGraph; @@ -146,10 +148,23 @@ function decorateLegacyGraph( idealBundleToLegacyBundle.get(siblingBundle), ); bundleGraph.addBundleToBundleGroup(legacySiblingBundle, bundleGroup); - bundleGraph.createBundleReference( - legacyMainBundleOfBundleGroup, - legacySiblingBundle, - ); + //TODO Put this back for shared bundles + // bundleGraph.createBundleReference( + // legacyMainBundleOfBundleGroup, + // legacySiblingBundle, + // ); + } + } + + /** + * TODO: Create all bundles, bundlegroups, without adding anything to them + * Draw connections to bundles + * Add references to bundles + */ + for (let [asset, references] of idealGraph.assetReference) { + for (let [dependency, bundle] of references) { + let legacyBundle = nullthrows(idealBundleToLegacyBundle.get(bundle)); + bundleGraph.createAssetReference(dependency, asset, legacyBundle); } } } @@ -159,6 +174,10 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { let bundleRoots: Map = new Map(); let bundles: Map = new Map(); let bundleLoadedByDependency: Map = new Map(); + let assetReference: DefaultMap< + Asset, + Array<[Dependency, Bundle]>, + > = new DefaultMap(() => []); // let reachableBundles: DefaultMap> = new DefaultMap( () => new Set(), @@ -265,19 +284,20 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { ) { let [, bundleGroupNodeId] = nullthrows(stack[stack.length - 1]); let bundleGroup = nullthrows(bundleGraph.getNode(bundleGroupNodeId)); - let bundleId = bundleGraph.addNode( - createBundle({ - asset: childAsset, - target: bundleGroup.target, - needsStableName: dependency.bundleBehavior === 'inline', - }), - ); + let bundle = createBundle({ + asset: childAsset, + target: bundleGroup.target, + needsStableName: dependency.bundleBehavior === 'inline', + }); + let bundleId = bundleGraph.addNode(bundle); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); // Add an edge from the bundle group entry to the new bundle. // This indicates that the bundle is loaded together with the entry bundleGraph.addEdge(bundleGroupNodeId, bundleId); + assetReference.get(childAsset).push([dependency, bundle]); + return node; } } @@ -398,6 +418,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { bundleGraph, bundleLoadedByDependency, entryBundles: [...bundleRoots.values()].map(v => v[0]), + assetReference, }; } diff --git a/packages/core/core/src/BundleGraph.js b/packages/core/core/src/BundleGraph.js index 0d8a6e689c0..88f2e9e2217 100644 --- a/packages/core/core/src/BundleGraph.js +++ b/packages/core/core/src/BundleGraph.js @@ -257,12 +257,24 @@ export default class BundleGraph { .some(node => node.type === 'bundle') ) { this._graph.addEdge(bundleNodeId, nodeId, 'references'); + this.markDependencyReferenceable(node.value); + //all bundles that have this dependency need to have an edge from bundle to that dependency } } }, assetNodeId); this._bundleContentHashes.delete(bundle.id); } + markDependencyReferenceable(dependency: Dependency) { + for (let bundle of this.findBundlesWithDependency(dependency)) { + this._graph.addEdge( + this._graph.getNodeIdByContentKey(bundle.id), + this._graph.getNodeIdByContentKey(dependency.id), + 'references', + ); + } + } + addEntryToBundle( asset: Asset, bundle: Bundle, @@ -449,6 +461,7 @@ export default class BundleGraph { this.removeExternalDependency(bundle, node.value); if (this._graph.hasEdge(bundleNodeId, nodeId, 'references')) { this._graph.addEdge(bundleNodeId, nodeId, 'references'); + this.markDependencyReferenceable(node.value); } } }, assetNodeId); @@ -580,6 +593,7 @@ export default class BundleGraph { this._graph.addEdge(dependencyId, assetId, 'references'); this._graph.addEdge(dependencyId, bundleId, 'references'); + this.markDependencyReferenceable(dependency); if (this._graph.hasEdge(dependencyId, assetId)) { this._graph.removeEdge(dependencyId, assetId); } diff --git a/packages/core/core/src/public/MutableBundleGraph.js b/packages/core/core/src/public/MutableBundleGraph.js index a3dce629e90..f2ebf19f81f 100644 --- a/packages/core/core/src/public/MutableBundleGraph.js +++ b/packages/core/core/src/public/MutableBundleGraph.js @@ -51,7 +51,7 @@ export default class MutableBundleGraph extends BundleGraph : undefined, ); } - + //TODO: Lift function to bundlegraph.js addAssetToBundle(asset: IAsset, bundle: IBundle) { let bundleNodeId = this.#graph._graph.getNodeIdByContentKey(bundle.id); this.#graph._graph.addEdge( @@ -78,6 +78,21 @@ export default class MutableBundleGraph extends BundleGraph invariant(bundleGroupNode.type === 'bundle_group'); this.#graph._graph.addEdge(bundleNodeId, bundleGroupNodeId, 'bundle'); } + // If the dependency references a target bundle, add a reference edge from + // the source bundle to the dependency for easy traversal. + //TODO Consider bundle being created from dependency + if ( + this.#graph._graph + .getNodeIdsConnectedFrom(dependencyNodeId, 'references') + .map(id => nullthrows(this.#graph._graph.getNode(id))) + .some(node => node.type === 'bundle') + ) { + this.#graph._graph.addEdge( + bundleNodeId, + dependencyNodeId, + 'references', + ); + } } } @@ -136,6 +151,7 @@ export default class MutableBundleGraph extends BundleGraph this.#graph._graph.addEdge(dependencyNodeId, bundleGroupNodeId); this.#graph._graph.replaceNodeIdsConnectedTo(bundleGroupNodeId, assetNodes); this.#graph._graph.addEdge(dependencyNodeId, resolvedNodeId, 'references'); + this.#graph.markDependencyReferenceable(dependencyNode.value); this.#graph._graph.removeEdge(dependencyNodeId, resolvedNodeId); if (dependency.isEntry) { From 98122fa5abcde78492d1ee4cd3197b69f286ccdb Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Wed, 7 Jul 2021 15:28:16 -0400 Subject: [PATCH 15/73] Add in entry asset for all bundles except shared --- .../bundlers/default/src/DefaultBundler.js | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 03c247f4ec0..7d3eb4c3d9a 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -76,6 +76,10 @@ export default (new Bundler({ optimize() {}, }): Bundler); +/** + Test: does not create bundles for dynamic imports when assets are available up the graph + Issue: Ideal bundlegraph creates dynamic import bundle & will not place asset in both bundle groups/bundles even if asset is present statically "up the tree" + */ function decorateLegacyGraph( idealGraph: IdealGraph, bundleGraph: MutableBundleGraph, @@ -90,6 +94,7 @@ function decorateLegacyGraph( let dependency = bundleLoadedByDependency.get(bundleNodeId); let entryAsset = bundleGraph.getAssetById(idealBundle.assetIds[0]); + // This entry asset is the first asset of the bundle (not entry file asset) let bundleGroup; let bundle; if (dependency) { @@ -109,7 +114,9 @@ function decorateLegacyGraph( ); bundleGraph.addBundleToBundleGroup(bundle, bundleGroup); - } else { + } else if (idealBundle.sourceBundles.length > 0) { + //TODO this should be > 1 + //this should only happen for shared bundles bundle = nullthrows( bundleGraph.createBundle({ uniqueKey: @@ -122,6 +129,15 @@ function decorateLegacyGraph( env: idealBundle.env, }), ); + } else { + bundle = nullthrows( + bundleGraph.createBundle({ + entryAsset, + needsStableName: idealBundle.needsStableName, + bundleBehavior: idealBundle.bundleBehavior, + target: idealBundle.target, + }), + ); } idealBundleToLegacyBundle.set(idealBundle, bundle); @@ -341,7 +357,6 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { for (let asset of assets) { // Find bundle entries reachable from the asset. let reachable: Array = [...reachableRoots.get(asset)]; - let reachableEntries = reachable.filter(a => entries.has(a)); for (let entry of reachableEntries) { @@ -412,7 +427,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { } // $FlowFixMe - dumpGraphToGraphViz(bundleGraph, 'NewBundleGraph'); + dumpGraphToGraphViz(bundleGraph, 'IdealBundleGraph'); return { bundleGraph, From 740ed313950a696c8614302f1f16aaa6a8b88767 Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Tue, 13 Jul 2021 17:02:58 -0700 Subject: [PATCH 16/73] wip for dynamic import already available up the graph --- .../bundlers/default/src/DefaultBundler.js | 30 +++++++++---------- .../core/integration-tests/test/javascript.js | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 7d3eb4c3d9a..b057e290029 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -340,7 +340,16 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { if (node.value === root) { return; } + //if its an entry bundle + // collect all of its async and sync dependencies + // ----set intersection (both sync and async)--- + // if thre is a sync dep on a bundle root from an entry bundle + + // put them in the reachablebroots + // if (entries.has(node)) { + + // } if (bundleRoots.has(node.value)) { actions.skipChildren(); return; @@ -357,25 +366,16 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { for (let asset of assets) { // Find bundle entries reachable from the asset. let reachable: Array = [...reachableRoots.get(asset)]; - let reachableEntries = reachable.filter(a => entries.has(a)); - - for (let entry of reachableEntries) { - //add asset ids to entry bundle - let entryAssetBundle = nullthrows( - bundleGraph.getNode(nullthrows(bundleRoots.get(entry))[0]), - ); - entryAssetBundle.assetIds.push(asset.id); - entryAssetBundle.size += asset.stats.size; - } + console.log('b4 filter. asset to reachable:', asset.filePath, reachable); // Filter out bundles when the asset is reachable in a parent bundle. - reachable = reachable.filter( - b => - !entries.has(b) && - reachable.every(a => !reachableBundles.get(a).has(b)), + reachable = reachable.filter(b => + reachable.every(a => !reachableBundles.get(a).has(b)), ); + console.log('asset to reachable:', asset.filePath, reachable); + let rootBundle = bundleRoots.get(asset); if (rootBundle != null) { // If the asset is a bundle root, add the bundle to every other reachable bundle group. @@ -383,7 +383,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { bundles.set(asset.id, rootBundle[0]); } for (let reachableAsset of reachable) { - if (reachableAsset !== asset && !entries.has(reachableAsset)) { + if (reachableAsset !== asset) { bundleGraph.addEdge( nullthrows(bundleRoots.get(reachableAsset))[1], rootBundle[0], diff --git a/packages/core/integration-tests/test/javascript.js b/packages/core/integration-tests/test/javascript.js index 8eb1723c898..888eab61fe1 100644 --- a/packages/core/integration-tests/test/javascript.js +++ b/packages/core/integration-tests/test/javascript.js @@ -1848,7 +1848,7 @@ describe('javascript', function() { assert.deepEqual(await Promise.all((await run(b)).default), [5, 4]); }); - it('does not create bundles for dynamic imports when assets are available up the graph', async () => { + it.only('does not create bundles for dynamic imports when assets are available up the graph', async () => { let b = await bundle( path.join(__dirname, '/integration/internalize-no-bundle-split/index.js'), ); From b7545d9726532fc2b95776b0959feccb5d9031c2 Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Wed, 14 Jul 2021 14:13:32 -0700 Subject: [PATCH 17/73] Duplicate assets in entries after the maximally deduplicated graph --- .../bundlers/default/src/DefaultBundler.js | 65 ++++++++++++++----- packages/core/core/src/Graph.js | 9 +-- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index b057e290029..ff247fe8729 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -267,6 +267,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { dependency.priority === 'lazy' || childAsset.bundleBehavior === 'isolated' ) { + // TODO: This bundle can be "created" by multiple dependencies? let bundleId = bundleGraph.addNode( createBundle({ asset: childAsset, @@ -331,29 +332,36 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { let reachableRoots: DefaultMap> = new DefaultMap( () => new Set(), ); - for (let [root] of bundleRoots) { - assetGraph.traverse((node, _, actions) => { - if (node.type !== 'asset') { - return; - } + let reachableAsyncRoots: DefaultMap> = new DefaultMap( + () => new Set(), + ); + for (let [root] of bundleRoots) { + assetGraph.traverse((node, isAsync, actions) => { if (node.value === root) { return; } - //if its an entry bundle - // collect all of its async and sync dependencies - // ----set intersection (both sync and async)--- - // if thre is a sync dep on a bundle root from an entry bundle + if (node.type === 'dependency') { + if (node.value.priority !== 'sync') { + let assets = assetGraph.getDependencyAssets(node.value); + if (assets.length === 0) { + return node; + } + + invariant(assets.length === 1); + let bundleRoot = assets[0]; + invariant(bundleRoots.has(bundleRoot)); - // put them in the reachablebroots - // if (entries.has(node)) { + reachableAsyncRoots + .get(nullthrows(bundles.get(bundleRoot.id))) + .add(root); + actions.skipChildren(); + } - // } - if (bundleRoots.has(node.value)) { - actions.skipChildren(); return; } + reachableRoots.get(node.value).add(root); }, root); } @@ -367,15 +375,11 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // Find bundle entries reachable from the asset. let reachable: Array = [...reachableRoots.get(asset)]; - console.log('b4 filter. asset to reachable:', asset.filePath, reachable); - // Filter out bundles when the asset is reachable in a parent bundle. reachable = reachable.filter(b => reachable.every(a => !reachableBundles.get(a).has(b)), ); - console.log('asset to reachable:', asset.filePath, reachable); - let rootBundle = bundleRoots.get(asset); if (rootBundle != null) { // If the asset is a bundle root, add the bundle to every other reachable bundle group. @@ -426,6 +430,31 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { } } + // Step 4: Merge any sibling bundles required by entry bundles back into the entry bundle. + // Entry bundles must be predictable, so cannot have unpredictable siblings. + for (let entryAsset of entries.keys()) { + let entryBundleId = nullthrows(bundleRoots.get(entryAsset)?.[0]); + let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); + for (let siblingId of bundleGraph.getNodeIdsConnectedFrom(entryBundleId)) { + let sibling = nullthrows(bundleGraph.getNode(siblingId)); + if (sibling.type !== entryBundle.type) { + continue; + } + + for (let assetId of sibling.assetIds) { + entryBundle.assetIds.push(assetId); + } + bundleGraph.removeEdge(entryBundleId, siblingId); + reachableAsyncRoots.get(siblingId).delete(entryAsset); + } + } + + for (let [asyncBundleRoot, dependentRoots] of reachableAsyncRoots) { + if (dependentRoots.size === 0) { + bundleGraph.removeNode(asyncBundleRoot); + } + } + // $FlowFixMe dumpGraphToGraphViz(bundleGraph, 'IdealBundleGraph'); diff --git a/packages/core/core/src/Graph.js b/packages/core/core/src/Graph.js index fe37bfa5023..fdf5bad176b 100644 --- a/packages/core/core/src/Graph.js +++ b/packages/core/core/src/Graph.js @@ -249,14 +249,7 @@ export default class Graph { if (this.rootNodeId == null) { // If the graph does not have a root, and there are inbound edges, // this node should not be considered orphaned. - // return false; - for (let [, inboundNodeIds] of this.inboundEdges.getEdgesByType(nodeId)) { - if (inboundNodeIds.size > 0) { - return false; - } - } - - return true; + return false; } // Otherwise, attempt to traverse backwards to the root. If there is a path, From 6a016aa8d036d89a4af42f2e6ce969e2fbf19d5e Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Wed, 14 Jul 2021 15:19:24 -0700 Subject: [PATCH 18/73] WIP internalization --- .../bundlers/default/src/DefaultBundler.js | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index ff247fe8729..69436d5bd62 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -49,6 +49,7 @@ const HTTP_OPTIONS = { type AssetId = string; export type Bundle = {| assetIds: Array, + internalizedAssetIds: Array, bundleBehavior?: ?BundleBehavior, needsStableName: boolean, size: number, @@ -370,7 +371,6 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // bundle based on the bundle entries it is reachable from. This creates a // maximally code split bundle graph with no duplication. - // Create a mapping from entry asset ids to bundle ids for (let asset of assets) { // Find bundle entries reachable from the asset. let reachable: Array = [...reachableRoots.get(asset)]; @@ -394,6 +394,31 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { ); } } + + let reachableAsync = [ + ...(reachableAsyncRoots.has(rootBundle[0]) + ? reachableAsyncRoots.get(rootBundle[0]) + : []), + ]; + + // TODO: is this correct? + reachableAsync = reachableAsync.filter(b => + reachableAsync.every(a => !reachableBundles.get(a).has(b)), + ); + for (let reachableAsset of reachableAsync) { + if (reachableAsset !== asset) { + let bundle = nullthrows( + bundleGraph.getNode(nullthrows(bundles.get(reachableAsset.id))), + ); + console.log( + 'PUSHING', + asset.id, + 'into bundle', + nullthrows(bundles.get(reachableAsset.id)), + ); + bundle.internalizedAssetIds.push(asset.id); + } + } } else if (reachable.length > 0) { // If the asset is reachable from more than one entry, find or create // a bundle for that combination of bundles, and add the asset to it. @@ -505,6 +530,7 @@ function createBundle( if (opts.asset == null) { return { assetIds: [], + internalizedAssetIds: [], size: 0, sourceBundles: [], target: opts.target, @@ -517,6 +543,7 @@ function createBundle( let asset = nullthrows(opts.asset); return { assetIds: [asset.id], + internalizedAssetIds: [], size: asset.stats.size, sourceBundles: [], target: opts.target, From be902f4fafcb3da36c8f541a4bd9950c14d08d84 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Thu, 15 Jul 2021 16:09:05 -0400 Subject: [PATCH 19/73] internalization wip 2 --- .../bundlers/default/src/DefaultBundler.js | 63 ++++++++++++++----- packages/core/core/src/BundleGraph.js | 16 +++++ packages/core/core/src/dumpGraphToGraphViz.js | 2 +- .../core/src/public/MutableBundleGraph.js | 1 + .../core/integration-tests/test/javascript.js | 4 +- packages/runtimes/js/src/JSRuntime.js | 2 + 6 files changed, 71 insertions(+), 17 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 69436d5bd62..a53938726e7 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -47,6 +47,7 @@ const HTTP_OPTIONS = { }; type AssetId = string; +type BundleRoot = Asset; export type Bundle = {| assetIds: Array, internalizedAssetIds: Array, @@ -148,6 +149,29 @@ function decorateLegacyGraph( for (let asset of assets) { bundleGraph.addAssetToBundle(asset, bundle); } + + console.log('INTERNALIZED', idealBundle.internalizedAssetIds); + for (let internalized of idealBundle.internalizedAssetIds) { + let incomingDeps = bundleGraph.getIncomingDependencies( + bundleGraph.getAssetById(internalized), + ); + for (let incomingDep of incomingDeps) { + if ( + incomingDep.priority === 'lazy' && + bundle.hasDependency(incomingDep) + ) { + console.log('INTERNALIZING DEP', incomingDep); + bundleGraph.internalizeAsyncDependency(bundle, incomingDep); + } else { + console.log( + 'NOT INTERNALIZING DEP', + incomingDep, + incomingDep.priority, + bundle.hasDependency(incomingDep), + ); + } + } + } } for (let [bundleId, bundleGroup] of entryBundleToBundleGroup) { @@ -188,7 +212,7 @@ function decorateLegacyGraph( function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // Asset to the bundle it's an entry of - let bundleRoots: Map = new Map(); + let bundleRoots: Map = new Map(); let bundles: Map = new Map(); let bundleLoadedByDependency: Map = new Map(); let assetReference: DefaultMap< @@ -196,9 +220,10 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { Array<[Dependency, Bundle]>, > = new DefaultMap(() => []); // - let reachableBundles: DefaultMap> = new DefaultMap( - () => new Set(), - ); + let reachableBundles: DefaultMap< + BundleRoot, + Set, + > = new DefaultMap(() => new Set()); // let bundleGraph: Graph = new Graph(); let stack: Array<[Asset, NodeId]> = []; @@ -330,10 +355,10 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // Step 2: Determine reachability for every asset from each bundle root. // This is later used to determine which bundles to place each asset in. - let reachableRoots: DefaultMap> = new DefaultMap( + let reachableRoots: DefaultMap> = new DefaultMap( () => new Set(), ); - let reachableAsyncRoots: DefaultMap> = new DefaultMap( + let reachableAsyncRoots: DefaultMap> = new DefaultMap( () => new Set(), ); @@ -373,13 +398,18 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { for (let asset of assets) { // Find bundle entries reachable from the asset. - let reachable: Array = [...reachableRoots.get(asset)]; + let reachable: Array = [...reachableRoots.get(asset)]; // Filter out bundles when the asset is reachable in a parent bundle. reachable = reachable.filter(b => reachable.every(a => !reachableBundles.get(a).has(b)), ); + // BundleRoot = Root Asset of a bundle + // reachableRoots = any asset => all BundleRoots that require it synchronously + // reachableBundles = Some BundleRoot => all BundleRoot decendants + // reachable = all bundle root assets that cant always have that asset reliably on page (so they need to be pulled in by shared bundle or other) + let rootBundle = bundleRoots.get(asset); if (rootBundle != null) { // If the asset is a bundle root, add the bundle to every other reachable bundle group. @@ -394,7 +424,8 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { ); } } - + // reachableAsyncRoots = all bundleNodeId => all BundleRoots that require it asynchronously + // reachableAsync = for one bundleRoot => all let reachableAsync = [ ...(reachableAsyncRoots.has(rootBundle[0]) ? reachableAsyncRoots.get(rootBundle[0]) @@ -402,19 +433,23 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { ]; // TODO: is this correct? - reachableAsync = reachableAsync.filter(b => - reachableAsync.every(a => !reachableBundles.get(a).has(b)), + let willInternalizeRoots = reachableAsync.filter( + b => + ![...reachableRoots.get(asset)].every( + a => !(a === b || reachableBundles.get(a).has(b)), + ), ); - for (let reachableAsset of reachableAsync) { - if (reachableAsset !== asset) { + + for (let bundleRoot of willInternalizeRoots) { + if (bundleRoot !== asset) { let bundle = nullthrows( - bundleGraph.getNode(nullthrows(bundles.get(reachableAsset.id))), + bundleGraph.getNode(nullthrows(bundles.get(bundleRoot.id))), ); console.log( 'PUSHING', asset.id, 'into bundle', - nullthrows(bundles.get(reachableAsset.id)), + nullthrows(bundles.get(bundleRoot.id)), ); bundle.internalizedAssetIds.push(asset.id); } diff --git a/packages/core/core/src/BundleGraph.js b/packages/core/core/src/BundleGraph.js index b26e0365f6e..6eb3a6d980e 100644 --- a/packages/core/core/src/BundleGraph.js +++ b/packages/core/core/src/BundleGraph.js @@ -291,6 +291,22 @@ export default class BundleGraph { throw new Error('Expected an async dependency'); } + // It's possible for internalized async dependencies to not have + // reference edges and still have untyped edges. + // TODO: Maybe don't use internalized async edges at all? + let dependencyNodeId = this._graph.getNodeIdByContentKey(dependency.id); + let resolved = this.getDependencyResolution(dependency); + if (resolved) { + let resolvedNodeId = this._graph.getNodeIdByContentKey(resolved.id); + + if ( + !this._graph.hasEdge(dependencyNodeId, resolvedNodeId, 'references') + ) { + this._graph.addEdge(dependencyNodeId, resolvedNodeId, 'references'); + this._graph.removeEdge(dependencyNodeId, resolvedNodeId); + } + } + this._graph.addEdge( this._graph.getNodeIdByContentKey(bundle.id), this._graph.getNodeIdByContentKey(dependency.id), diff --git a/packages/core/core/src/dumpGraphToGraphViz.js b/packages/core/core/src/dumpGraphToGraphViz.js index 1814a227f1b..08cf979428f 100644 --- a/packages/core/core/src/dumpGraphToGraphViz.js +++ b/packages/core/core/src/dumpGraphToGraphViz.js @@ -61,7 +61,7 @@ export default async function dumpGraphToGraphViz( ', ', )}) (bb ${node.bundleBehavior ?? 'none'})`; } else if (node.type) { - label = `${node.type || 'No Type'}: [${node.id}]: `; + label = `[${id}] ${node.type || 'No Type'}: [${node.id}]: `; if (node.type === 'dependency') { label += node.value.specifier; let parts = []; diff --git a/packages/core/core/src/public/MutableBundleGraph.js b/packages/core/core/src/public/MutableBundleGraph.js index df71bfaf101..4fc1b26a96c 100644 --- a/packages/core/core/src/public/MutableBundleGraph.js +++ b/packages/core/core/src/public/MutableBundleGraph.js @@ -152,6 +152,7 @@ export default class MutableBundleGraph extends BundleGraph this.#graph._graph.replaceNodeIdsConnectedTo(bundleGroupNodeId, assetNodes); this.#graph._graph.addEdge(dependencyNodeId, resolvedNodeId, 'references'); this.#graph.markDependencyReferenceable(dependencyNode.value); + console.log('removing edge from', dependencyNodeId, 'to', resolvedNodeId); this.#graph._graph.removeEdge(dependencyNodeId, resolvedNodeId); if (dependency.isEntry) { diff --git a/packages/core/integration-tests/test/javascript.js b/packages/core/integration-tests/test/javascript.js index 888eab61fe1..f102150c402 100644 --- a/packages/core/integration-tests/test/javascript.js +++ b/packages/core/integration-tests/test/javascript.js @@ -1847,8 +1847,8 @@ describe('javascript', function() { assert.deepEqual(await Promise.all((await run(b)).default), [5, 4]); }); - - it.only('does not create bundles for dynamic imports when assets are available up the graph', async () => { + //TODO: Fixed + it('does not create bundles for dynamic imports when assets are available up the graph', async () => { let b = await bundle( path.join(__dirname, '/integration/internalize-no-bundle-split/index.js'), ); diff --git a/packages/runtimes/js/src/JSRuntime.js b/packages/runtimes/js/src/JSRuntime.js index 640518cadaf..47d14e8fdb4 100644 --- a/packages/runtimes/js/src/JSRuntime.js +++ b/packages/runtimes/js/src/JSRuntime.js @@ -83,6 +83,7 @@ export default (new Runtime({ let assets = []; for (let dependency of asyncDependencies) { let resolved = bundleGraph.resolveAsyncDependency(dependency, bundle); + console.log('resolved', dependency, dependency.priority, 'to', resolved); if (resolved == null) { continue; } @@ -92,6 +93,7 @@ export default (new Runtime({ // If this bundle already has the asset this dependency references, // return a simple runtime of `Promise.resolve(internalRequire(assetId))`. // The linker handles this for scope-hoisting. + console.log('creating promise.resolve runtime'); assets.push({ filePath: __filename, code: `module.exports = Promise.resolve(module.bundle.root(${JSON.stringify( From 9365a48b7ec207a106e61e4e9d45357a5c570ba3 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Thu, 15 Jul 2021 15:05:59 -0700 Subject: [PATCH 20/73] Ideas from our sync --- .../bundlers/default/src/DefaultBundler.js | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index a53938726e7..89c9363e6af 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -226,10 +226,11 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { > = new DefaultMap(() => new Set()); // let bundleGraph: Graph = new Graph(); - let stack: Array<[Asset, NodeId]> = []; + let stack: Array<[BundleRoot, NodeId]> = []; - // Step 1: Create bundles at the explicit split points in the graph. - // Create bundles for each entry. + // Step 1: Create bundles for each entry. + // TODO: Try to not create bundles during this first path, only annotate + // BundleRoots let entries: Map = new Map(); assetGraph.traverse((node, context, actions) => { if (node.type !== 'asset') { @@ -369,7 +370,10 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { } if (node.type === 'dependency') { - if (node.value.priority !== 'sync') { + // Handle other situations in which bundles are created and maybe + // centralize that in a function + // if (isABundleRoot and dependency is not sync) { + if (node.value.priority === 'lazy') { let assets = assetGraph.getDependencyAssets(node.value); if (assets.length === 0) { return node; @@ -400,9 +404,13 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // Find bundle entries reachable from the asset. let reachable: Array = [...reachableRoots.get(asset)]; - // Filter out bundles when the asset is reachable in a parent bundle. + // Filter out bundles when the asset is reachable in every parent bundle. + // (Only keep a bundle if all of the others are not descendents of it) reachable = reachable.filter(b => - reachable.every(a => !reachableBundles.get(a).has(b)), + reachable.every(a => { + const isAncestorOfB = reachableBundles.get(a).has(b); + return !isAncestorOfB; + }), ); // BundleRoot = Root Asset of a bundle From 57cfb0d0f993f78f348418db3c23d88aa781e85d Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Wed, 21 Jul 2021 17:02:49 -0400 Subject: [PATCH 21/73] WIP missing edge --- packages/bundlers/default/src/DefaultBundler.js | 4 ++++ packages/core/core/src/Graph.js | 2 ++ packages/core/core/src/public/MutableBundleGraph.js | 2 +- packages/core/test-utils/src/utils.js | 2 ++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 89c9363e6af..b8b1534eafb 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -151,6 +151,10 @@ function decorateLegacyGraph( } console.log('INTERNALIZED', idealBundle.internalizedAssetIds); + } + + for (let [, idealBundle] of idealBundleGraph.nodes) { + let bundle = nullthrows(idealBundleToLegacyBundle.get(idealBundle)); for (let internalized of idealBundle.internalizedAssetIds) { let incomingDeps = bundleGraph.getIncomingDependencies( bundleGraph.getAssetById(internalized), diff --git a/packages/core/core/src/Graph.js b/packages/core/core/src/Graph.js index fdf5bad176b..9c9f7f6897d 100644 --- a/packages/core/core/src/Graph.js +++ b/packages/core/core/src/Graph.js @@ -235,6 +235,8 @@ export default class Graph { ); } + console.trace('removeEdge: removing edge from', from, 'to', to, 'on', type); + this.outboundEdges.removeEdge(from, to, type); this.inboundEdges.removeEdge(to, from, type); diff --git a/packages/core/core/src/public/MutableBundleGraph.js b/packages/core/core/src/public/MutableBundleGraph.js index 4fc1b26a96c..e8809597866 100644 --- a/packages/core/core/src/public/MutableBundleGraph.js +++ b/packages/core/core/src/public/MutableBundleGraph.js @@ -152,8 +152,8 @@ export default class MutableBundleGraph extends BundleGraph this.#graph._graph.replaceNodeIdsConnectedTo(bundleGroupNodeId, assetNodes); this.#graph._graph.addEdge(dependencyNodeId, resolvedNodeId, 'references'); this.#graph.markDependencyReferenceable(dependencyNode.value); - console.log('removing edge from', dependencyNodeId, 'to', resolvedNodeId); this.#graph._graph.removeEdge(dependencyNodeId, resolvedNodeId); + console.log('removed edge from', dependencyNodeId, 'to', resolvedNodeId); if (dependency.isEntry) { this.#graph._graph.addEdge( diff --git a/packages/core/test-utils/src/utils.js b/packages/core/test-utils/src/utils.js index ca52b05ac34..88501f61557 100644 --- a/packages/core/test-utils/src/utils.js +++ b/packages/core/test-utils/src/utils.js @@ -505,7 +505,9 @@ export function assertBundles( assets, }); }); + console.log('Actual Bundles are', actualBundles); + console.log('Expected Bundles are', expectedBundles); for (let bundle of expectedBundles) { if (!Array.isArray(bundle.assets)) { throw new Error( From 6c715d5442d25f00b17e0e41d9586d80c8e790f0 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Thu, 29 Jul 2021 16:35:20 -0400 Subject: [PATCH 22/73] Convert reachableRoots to ContentGraph and WIP needed duplication of assets in bundles --- .../bundlers/default/src/DefaultBundler.js | 54 ++++++++++++++++--- packages/core/core/src/Graph.js | 2 +- .../core/integration-tests/test/javascript.js | 2 +- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index b8b1534eafb..c505dfea8d1 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -16,6 +16,7 @@ import type {NodeId} from '@parcel/core/src/types'; import type {SchemaEntity} from '@parcel/utils'; import Graph from '@parcel/core/src/Graph'; +import ContentGraph from '@parcel/core/src/ContentGraph'; import dumpGraphToGraphViz from '@parcel/core/src/dumpGraphToGraphViz'; import invariant from 'assert'; @@ -360,14 +361,16 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // Step 2: Determine reachability for every asset from each bundle root. // This is later used to determine which bundles to place each asset in. - let reachableRoots: DefaultMap> = new DefaultMap( - () => new Set(), - ); + let reachableRoots: ContentGraph = new ContentGraph(); + let reachableAsyncRoots: DefaultMap> = new DefaultMap( () => new Set(), ); for (let [root] of bundleRoots) { + let rootNodeId = reachableRoots.hasContentKey(root.id) + ? reachableRoots.getNodeIdByContentKey(root.id) + : reachableRoots.addNodeByContentKey(root.id, root); assetGraph.traverse((node, isAsync, actions) => { if (node.value === root) { return; @@ -396,18 +399,46 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { return; } - reachableRoots.get(node.value).add(root); + let nodeId = reachableRoots.hasContentKey(node.value.id) + ? reachableRoots.getNodeIdByContentKey(node.value.id) + : reachableRoots.addNodeByContentKey(node.value.id, node.value); + reachableRoots.addEdge(rootNodeId, nodeId); }, root); } + // Step 2.5 + //IDEA 2: Somehow store all assets available (guarenteed to be loaded at this bundles load in time) at a certain point, for an asset/ bundleRoot, and do a lookup to + // determine what MUST be duplicated. + + // PART 1 (located in STEP 1) + // Make bundlegraph that models bundleRoots and async deps only + // Turn reachableRoots into graph so that we have sync deps (Bidirectional) + + // PART 2 + // traverse PART 2 BundleGraph (BFS) + // Maintain a MAP BundleRoot => Set of assets loaded thus far + + // At BundleRoot X + // Ask for children [Z..] + + // get all assets guarenteed to be loaded when bundle X is loaded + // map.set(Z, {all assets gurenteed to be loaded at this point (by ancestors (X)) INTERSECTION WITH current map.get(z) }) + + let ancestorAssets: Map> = new Map(); + //set difference + // Step 3: Place all assets into bundles. Each asset is placed into a single // bundle based on the bundle entries it is reachable from. This creates a // maximally code split bundle graph with no duplication. for (let asset of assets) { // Find bundle entries reachable from the asset. - let reachable: Array = [...reachableRoots.get(asset)]; + let reachable: Array = getReachableBundleRoots( + asset, + reachableRoots, + ); + console.log('Reachable before for', asset.filePath, 'is ', reachable); // Filter out bundles when the asset is reachable in every parent bundle. // (Only keep a bundle if all of the others are not descendents of it) reachable = reachable.filter(b => @@ -415,7 +446,10 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { const isAncestorOfB = reachableBundles.get(a).has(b); return !isAncestorOfB; }), - ); + ); //don't want to filter out bundle if 'b' is not "reachable" from all of its (a) immediate parents + + console.log('Reachable for', asset.filePath, 'is ', reachable); + //IDEA: reachableBundles as a graph so we can query an assets ancestors and/or decendants // BundleRoot = Root Asset of a bundle // reachableRoots = any asset => all BundleRoots that require it synchronously @@ -447,7 +481,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // TODO: is this correct? let willInternalizeRoots = reachableAsync.filter( b => - ![...reachableRoots.get(asset)].every( + !getReachableBundleRoots(asset, reachableRoots).every( a => !(a === b || reachableBundles.get(a).has(b)), ), ); @@ -633,3 +667,9 @@ async function loadBundlerConfig(config: Config, options: PluginOptions) { conf.contents.maxParallelRequests ?? defaults.maxParallelRequests, }; } + +function getReachableBundleRoots(asset, graph): Array { + return graph + .getNodeIdsConnectedTo(graph.getNodeIdByContentKey(asset.id)) + .map(nodeId => nullthrows(graph.getNode(nodeId))); +} diff --git a/packages/core/core/src/Graph.js b/packages/core/core/src/Graph.js index 9c9f7f6897d..42bf6d5f062 100644 --- a/packages/core/core/src/Graph.js +++ b/packages/core/core/src/Graph.js @@ -235,7 +235,7 @@ export default class Graph { ); } - console.trace('removeEdge: removing edge from', from, 'to', to, 'on', type); + //console.trace('removeEdge: removing edge from', from, 'to', to, 'on', type); this.outboundEdges.removeEdge(from, to, type); this.inboundEdges.removeEdge(to, from, type); diff --git a/packages/core/integration-tests/test/javascript.js b/packages/core/integration-tests/test/javascript.js index f102150c402..953304e55ae 100644 --- a/packages/core/integration-tests/test/javascript.js +++ b/packages/core/integration-tests/test/javascript.js @@ -2005,7 +2005,7 @@ describe('javascript', function() { await run(b); }); - + //TODO it('should dynamic import files which import raw files', async function() { let b = await bundle( path.join(__dirname, '/integration/dynamic-references-raw/index.js'), From 2f933d580505e8901c0119f3da42f2ba8645531a Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Thu, 29 Jul 2021 16:56:08 -0400 Subject: [PATCH 23/73] commenting out consoles --- .../bundlers/default/src/DefaultBundler.js | 32 +++++++++---------- .../core/src/public/MutableBundleGraph.js | 2 +- packages/core/test-utils/src/utils.js | 4 +-- packages/runtimes/js/src/JSRuntime.js | 2 +- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index c505dfea8d1..f7d68ba43f5 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -151,7 +151,7 @@ function decorateLegacyGraph( bundleGraph.addAssetToBundle(asset, bundle); } - console.log('INTERNALIZED', idealBundle.internalizedAssetIds); + //console.log('INTERNALIZED', idealBundle.internalizedAssetIds); } for (let [, idealBundle] of idealBundleGraph.nodes) { @@ -165,15 +165,15 @@ function decorateLegacyGraph( incomingDep.priority === 'lazy' && bundle.hasDependency(incomingDep) ) { - console.log('INTERNALIZING DEP', incomingDep); + //console.log('INTERNALIZING DEP', incomingDep); bundleGraph.internalizeAsyncDependency(bundle, incomingDep); } else { - console.log( - 'NOT INTERNALIZING DEP', - incomingDep, - incomingDep.priority, - bundle.hasDependency(incomingDep), - ); + // console.log( + // 'NOT INTERNALIZING DEP', + // incomingDep, + // incomingDep.priority, + // bundle.hasDependency(incomingDep), + // ); } } } @@ -438,7 +438,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { reachableRoots, ); - console.log('Reachable before for', asset.filePath, 'is ', reachable); + //console.log('Reachable before for', asset.filePath, 'is ', reachable); // Filter out bundles when the asset is reachable in every parent bundle. // (Only keep a bundle if all of the others are not descendents of it) reachable = reachable.filter(b => @@ -448,7 +448,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { }), ); //don't want to filter out bundle if 'b' is not "reachable" from all of its (a) immediate parents - console.log('Reachable for', asset.filePath, 'is ', reachable); + //console.log('Reachable for', asset.filePath, 'is ', reachable); //IDEA: reachableBundles as a graph so we can query an assets ancestors and/or decendants // BundleRoot = Root Asset of a bundle @@ -491,12 +491,12 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { let bundle = nullthrows( bundleGraph.getNode(nullthrows(bundles.get(bundleRoot.id))), ); - console.log( - 'PUSHING', - asset.id, - 'into bundle', - nullthrows(bundles.get(bundleRoot.id)), - ); + // console.log( + // 'PUSHING', + // asset.id, + // 'into bundle', + // nullthrows(bundles.get(bundleRoot.id)), + // ); bundle.internalizedAssetIds.push(asset.id); } } diff --git a/packages/core/core/src/public/MutableBundleGraph.js b/packages/core/core/src/public/MutableBundleGraph.js index e8809597866..7973a589359 100644 --- a/packages/core/core/src/public/MutableBundleGraph.js +++ b/packages/core/core/src/public/MutableBundleGraph.js @@ -153,7 +153,7 @@ export default class MutableBundleGraph extends BundleGraph this.#graph._graph.addEdge(dependencyNodeId, resolvedNodeId, 'references'); this.#graph.markDependencyReferenceable(dependencyNode.value); this.#graph._graph.removeEdge(dependencyNodeId, resolvedNodeId); - console.log('removed edge from', dependencyNodeId, 'to', resolvedNodeId); + //console.log('removed edge from', dependencyNodeId, 'to', resolvedNodeId); if (dependency.isEntry) { this.#graph._graph.addEdge( diff --git a/packages/core/test-utils/src/utils.js b/packages/core/test-utils/src/utils.js index 88501f61557..3126989bad5 100644 --- a/packages/core/test-utils/src/utils.js +++ b/packages/core/test-utils/src/utils.js @@ -505,9 +505,9 @@ export function assertBundles( assets, }); }); - console.log('Actual Bundles are', actualBundles); + //console.log('Actual Bundles are', actualBundles); - console.log('Expected Bundles are', expectedBundles); + //console.log('Expected Bundles are', expectedBundles); for (let bundle of expectedBundles) { if (!Array.isArray(bundle.assets)) { throw new Error( diff --git a/packages/runtimes/js/src/JSRuntime.js b/packages/runtimes/js/src/JSRuntime.js index 47d14e8fdb4..10631022e10 100644 --- a/packages/runtimes/js/src/JSRuntime.js +++ b/packages/runtimes/js/src/JSRuntime.js @@ -83,7 +83,7 @@ export default (new Runtime({ let assets = []; for (let dependency of asyncDependencies) { let resolved = bundleGraph.resolveAsyncDependency(dependency, bundle); - console.log('resolved', dependency, dependency.priority, 'to', resolved); + //console.log('resolved', dependency, dependency.priority, 'to', resolved); if (resolved == null) { continue; } From d9f4e265cac432e55414f4aedf8034d2c9541645 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Thu, 29 Jul 2021 17:51:58 -0400 Subject: [PATCH 24/73] support multiple dependencies on same bundle FIX --- .../bundlers/default/src/DefaultBundler.js | 51 ++++++++++++------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index f7d68ba43f5..3e9fca02b49 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -62,7 +62,7 @@ export type Bundle = {| |}; type IdealGraph = {| - bundleLoadedByDependency: Map, + bundleLoadedByDependency: DefaultMap>, bundleGraph: Graph, entryBundles: Array, assetReference: DefaultMap>, @@ -94,17 +94,22 @@ function decorateLegacyGraph( let entryBundleToBundleGroup: Map = new Map(); for (let [bundleNodeId, idealBundle] of idealBundleGraph.nodes) { - let dependency = bundleLoadedByDependency.get(bundleNodeId); + let dependencies = bundleLoadedByDependency.get(bundleNodeId); let entryAsset = bundleGraph.getAssetById(idealBundle.assetIds[0]); // This entry asset is the first asset of the bundle (not entry file asset) let bundleGroup; let bundle; - if (dependency) { - bundleGroup = bundleGraph.createBundleGroup( - dependency, - idealBundle.target, - ); + + if (dependencies && dependencies.size > 0) { + //console.log('deps are', dependencies); + for (let dependency of dependencies) { + bundleGroup = bundleGraph.createBundleGroup( + dependency, + idealBundle.target, + ); + } + invariant(bundleGroup); entryBundleToBundleGroup.set(bundleNodeId, bundleGroup); bundle = nullthrows( @@ -219,7 +224,10 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // Asset to the bundle it's an entry of let bundleRoots: Map = new Map(); let bundles: Map = new Map(); - let bundleLoadedByDependency: Map = new Map(); + let bundleLoadedByDependency: DefaultMap< + NodeId, + Set, + > = new DefaultMap(() => new Set()); let assetReference: DefaultMap< Asset, Array<[Dependency, Bundle]>, @@ -232,6 +240,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // let bundleGraph: Graph = new Graph(); let stack: Array<[BundleRoot, NodeId]> = []; + let asyncBundleRootGraph: ContentGraph = new ContentGraph(); // Step 1: Create bundles for each entry. // TODO: Try to not create bundles during this first path, only annotate @@ -259,7 +268,8 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { ); bundles.set(asset.id, nodeId); bundleRoots.set(asset, [nodeId, nodeId]); - bundleLoadedByDependency.set(nodeId, dependency); + bundleLoadedByDependency.get(nodeId).add(dependency); + asyncBundleRootGraph.addNodeByContentKey(asset.id, asset); } let assets = []; @@ -300,15 +310,18 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { childAsset.bundleBehavior === 'isolated' ) { // TODO: This bundle can be "created" by multiple dependencies? - let bundleId = bundleGraph.addNode( - createBundle({ - asset: childAsset, - target: nullthrows(bundleGraph.getNode(stack[0][1])).target, - }), - ); - bundles.set(childAsset.id, bundleId); - bundleRoots.set(childAsset, [bundleId, bundleId]); - bundleLoadedByDependency.set(bundleId, dependency); + let bundleId = bundles.get(childAsset.id); + if (bundleId == null) { + bundleId = bundleGraph.addNode( + createBundle({ + asset: childAsset, + target: nullthrows(bundleGraph.getNode(stack[0][1])).target, + }), + ); + bundles.set(childAsset.id, bundleId); + bundleRoots.set(childAsset, [bundleId, bundleId]); + } + bundleLoadedByDependency.get(bundleId).add(dependency); // Walk up the stack until we hit a different asset type // and mark each bundle as reachable from every parent bundle @@ -412,7 +425,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // PART 1 (located in STEP 1) // Make bundlegraph that models bundleRoots and async deps only - // Turn reachableRoots into graph so that we have sync deps (Bidirectional) + // Turn reachableRoots into graph so that we have sync deps (Bidirectional) [x] // PART 2 // traverse PART 2 BundleGraph (BFS) From 2fffa45f3b9b6f1c2d9809efec608138e1645ec2 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Fri, 30 Jul 2021 13:49:11 -0400 Subject: [PATCH 25/73] AsyncBundleRootGraph WIP --- .../bundlers/default/src/DefaultBundler.js | 29 ++++++++++++++++--- .../core/integration-tests/test/javascript.js | 2 +- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 3e9fca02b49..ea42dcf84a2 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -241,6 +241,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { let bundleGraph: Graph = new Graph(); let stack: Array<[BundleRoot, NodeId]> = []; let asyncBundleRootGraph: ContentGraph = new ContentGraph(); + //TODO of asyncBundleRootGraph: we should either add a root node or use bundleGraph which has a root automatically // Step 1: Create bundles for each entry. // TODO: Try to not create bundles during this first path, only annotate @@ -321,6 +322,21 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); } + //Add child and connection from parent to child bundleRoot + let childNodeId = asyncBundleRootGraph.hasContentKey(childAsset.id) + ? asyncBundleRootGraph.getNodeIdByContentKey(childAsset.id) + : asyncBundleRootGraph.addNodeByContentKey( + childAsset.id, + childAsset, + ); + if (asyncBundleRootGraph.hasContentKey(parentAsset.id)) { + let parentNodeId = asyncBundleRootGraph.getNodeIdByContentKey( + parentAsset.id, + ); + asyncBundleRootGraph.addEdge(parentNodeId, childNodeId); + } else { + //add edge from root to node ? + } bundleLoadedByDependency.get(bundleId).add(dependency); // Walk up the stack until we hit a different asset type @@ -359,7 +375,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // This indicates that the bundle is loaded together with the entry bundleGraph.addEdge(bundleGroupNodeId, bundleId); assetReference.get(childAsset).push([dependency, bundle]); - + asyncBundleRootGraph.addNodeByContentKey(childAsset.id, childAsset); return node; } } @@ -420,7 +436,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { } // Step 2.5 - //IDEA 2: Somehow store all assets available (guarenteed to be loaded at this bundles load in time) at a certain point, for an asset/ bundleRoot, and do a lookup to + // IDEA 2: Somehow store all assets available (guarenteed to be loaded at this bundles load in time) at a certain point, for an asset/ bundleRoot, and do a lookup to // determine what MUST be duplicated. // PART 1 (located in STEP 1) @@ -432,7 +448,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // Maintain a MAP BundleRoot => Set of assets loaded thus far // At BundleRoot X - // Ask for children [Z..] + // Peek/Ask for children [Z..] // get all assets guarenteed to be loaded when bundle X is loaded // map.set(Z, {all assets gurenteed to be loaded at this point (by ancestors (X)) INTERSECTION WITH current map.get(z) }) @@ -576,7 +592,12 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // $FlowFixMe dumpGraphToGraphViz(bundleGraph, 'IdealBundleGraph'); - + for (let [nodeId, node] of asyncBundleRootGraph.nodes) { + console.log('node id is', nodeId, 'and node is', node); + for (let edge of asyncBundleRootGraph.getNodeIdsConnectedFrom(nodeId)) { + console.log('Edge from ', edge, ' to ', nodeId); + } + } return { bundleGraph, bundleLoadedByDependency, diff --git a/packages/core/integration-tests/test/javascript.js b/packages/core/integration-tests/test/javascript.js index 953304e55ae..eb99b21001d 100644 --- a/packages/core/integration-tests/test/javascript.js +++ b/packages/core/integration-tests/test/javascript.js @@ -2143,7 +2143,7 @@ describe('javascript', function() { assert.equal(await output(), 5); }); - it('should duplicate an asset if it is not present in every parent bundle', async function() { + it.only('should duplicate an asset if it is not present in every parent bundle', async function() { let b = await bundle( ['a.js', 'b.js'].map(entry => path.join(__dirname, 'integration/dynamic-hoist-no-dedupe', entry), From dfe1e80a2987ba8ec65cadfd62a377caa848b4db Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Fri, 30 Jul 2021 15:41:43 -0700 Subject: [PATCH 26/73] topoSort --- .../bundlers/default/src/DefaultBundler.js | 123 ++++++++++++++---- packages/core/core/src/Graph.js | 10 ++ packages/core/utils/src/collection.js | 14 ++ packages/core/utils/src/index.js | 2 + 4 files changed, 122 insertions(+), 27 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index ea42dcf84a2..4af410834b3 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -21,7 +21,12 @@ import dumpGraphToGraphViz from '@parcel/core/src/dumpGraphToGraphViz'; import invariant from 'assert'; import {Bundler} from '@parcel/plugin'; -import {validateSchema, DefaultMap} from '@parcel/utils'; +import { + validateSchema, + DefaultMap, + setIntersection, + setUnion, +} from '@parcel/utils'; import {hashString} from '@parcel/hash'; import nullthrows from 'nullthrows'; import {encodeJSONKeyComponent} from '@parcel/diagnostic'; @@ -240,7 +245,9 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // let bundleGraph: Graph = new Graph(); let stack: Array<[BundleRoot, NodeId]> = []; - let asyncBundleRootGraph: ContentGraph = new ContentGraph(); + let asyncBundleRootGraph: ContentGraph< + BundleRoot | 'root', + > = new ContentGraph(); //TODO of asyncBundleRootGraph: we should either add a root node or use bundleGraph which has a root automatically // Step 1: Create bundles for each entry. @@ -259,6 +266,9 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { actions.skipChildren(); }); + let rootNodeId = nullthrows(asyncBundleRootGraph.addNode('root')); + asyncBundleRootGraph.setRootNodeId(rootNodeId); + for (let [asset, dependency] of entries) { let nodeId = bundleGraph.addNode( createBundle({ @@ -269,8 +279,11 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { ); bundles.set(asset.id, nodeId); bundleRoots.set(asset, [nodeId, nodeId]); + asyncBundleRootGraph.addEdge( + rootNodeId, + asyncBundleRootGraph.addNodeByContentKey(asset.id, asset), + ); bundleLoadedByDependency.get(nodeId).add(dependency); - asyncBundleRootGraph.addNodeByContentKey(asset.id, asset); } let assets = []; @@ -322,21 +335,6 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); } - //Add child and connection from parent to child bundleRoot - let childNodeId = asyncBundleRootGraph.hasContentKey(childAsset.id) - ? asyncBundleRootGraph.getNodeIdByContentKey(childAsset.id) - : asyncBundleRootGraph.addNodeByContentKey( - childAsset.id, - childAsset, - ); - if (asyncBundleRootGraph.hasContentKey(parentAsset.id)) { - let parentNodeId = asyncBundleRootGraph.getNodeIdByContentKey( - parentAsset.id, - ); - asyncBundleRootGraph.addEdge(parentNodeId, childNodeId); - } else { - //add edge from root to node ? - } bundleLoadedByDependency.get(bundleId).add(dependency); // Walk up the stack until we hit a different asset type @@ -351,6 +349,29 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { break; } reachableBundles.get(stackAsset).add(childAsset); + + if (i === stack.length - 1) { + //Add child and connection from parent to child bundleRoot + let childNodeId = asyncBundleRootGraph.hasContentKey( + childAsset.id, + ) + ? asyncBundleRootGraph.getNodeIdByContentKey(childAsset.id) + : asyncBundleRootGraph.addNodeByContentKey( + childAsset.id, + childAsset, + ); + + let parentNodeId = asyncBundleRootGraph.hasContentKey( + stackAsset.id, + ) + ? asyncBundleRootGraph.getNodeIdByContentKey(stackAsset.id) + : asyncBundleRootGraph.addNodeByContentKey( + stackAsset.id, + stackAsset, + ); + + asyncBundleRootGraph.addEdge(parentNodeId, childNodeId); + } } return node; } @@ -375,7 +396,6 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // This indicates that the bundle is loaded together with the entry bundleGraph.addEdge(bundleGroupNodeId, bundleId); assetReference.get(childAsset).push([dependency, bundle]); - asyncBundleRootGraph.addNodeByContentKey(childAsset.id, childAsset); return node; } } @@ -440,7 +460,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // determine what MUST be duplicated. // PART 1 (located in STEP 1) - // Make bundlegraph that models bundleRoots and async deps only + // Make bundlegraph that models bundleRoots and async deps only [x] // Turn reachableRoots into graph so that we have sync deps (Bidirectional) [x] // PART 2 @@ -454,8 +474,48 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // map.set(Z, {all assets gurenteed to be loaded at this point (by ancestors (X)) INTERSECTION WITH current map.get(z) }) let ancestorAssets: Map> = new Map(); - //set difference + //set difference //Map + console.log('toposort:', asyncBundleRootGraph.topoSort()); + for (let nodeId of asyncBundleRootGraph.topoSort()) { + let bundleRoot = asyncBundleRootGraph.getNode(nodeId); + if (bundleRoot === 'root') continue; + invariant(bundleRoot != null); + + let syncAssetsLoaded = reachableRoots + .getNodeIdsConnectedFrom( + reachableRoots.getNodeIdByContentKey(bundleRoot.id), + ) + .map(id => nullthrows(reachableRoots.getNode(id))); //assets synchronously loaded when a is loaded + + console.log({syncAssetsLoaded}); + + let ancestors = ancestorAssets.get(bundleRoot); + + let combined = ancestors + ? setUnion(ancestors, syncAssetsLoaded) + : new Set(syncAssetsLoaded); + let children = asyncBundleRootGraph.getNodeIdsConnectedFrom(nodeId); + console.log({children}); + + for (let childId of children) { + let child = asyncBundleRootGraph.getNode(childId); + invariant(child !== 'root' && child != null); + const availableAssets = ancestorAssets.get(child); + + if (availableAssets == null) { + ancestorAssets.set(child, combined); + } else if ( + asyncBundleRootGraph.getNodeIdsConnectedTo(childId).length > 1 + ) { + ancestorAssets.set(child, setIntersection(combined, availableAssets)); + console.log({combined}); + } else { + ancestorAssets.set(child, setUnion(combined, availableAssets)); + } + } + } + console.log(ancestorAssets); // Step 3: Place all assets into bundles. Each asset is placed into a single // bundle based on the bundle entries it is reachable from. This creates a // maximally code split bundle graph with no duplication. @@ -470,12 +530,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { //console.log('Reachable before for', asset.filePath, 'is ', reachable); // Filter out bundles when the asset is reachable in every parent bundle. // (Only keep a bundle if all of the others are not descendents of it) - reachable = reachable.filter(b => - reachable.every(a => { - const isAncestorOfB = reachableBundles.get(a).has(b); - return !isAncestorOfB; - }), - ); //don't want to filter out bundle if 'b' is not "reachable" from all of its (a) immediate parents + reachable = reachable.filter(b => !ancestorAssets.get(b)?.has(asset)); //don't want to filter out bundle if 'b' is not "reachable" from all of its (a) immediate parents //console.log('Reachable for', asset.filePath, 'is ', reachable); //IDEA: reachableBundles as a graph so we can query an assets ancestors and/or decendants @@ -581,6 +636,20 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { } bundleGraph.removeEdge(entryBundleId, siblingId); reachableAsyncRoots.get(siblingId).delete(entryAsset); + if (sibling.sourceBundles.length > 1) { + let entryBundleIndex = sibling.sourceBundles.indexOf(entryBundleId); + invariant(entryBundleIndex >= 0); + sibling.sourceBundles.splice(entryBundleIndex, 1); + + if (sibling.sourceBundles.length === 1) { + let id = sibling.sourceBundles.pop(); + let bundle = nullthrows(bundleGraph.getNode(id)); + for (let assetId of sibling.assetIds) { + bundle.assetIds.push(assetId); + } + bundleGraph.removeEdge(id, siblingId); + } + } } } diff --git a/packages/core/core/src/Graph.js b/packages/core/core/src/Graph.js index 42bf6d5f062..24e97d6bdfb 100644 --- a/packages/core/core/src/Graph.js +++ b/packages/core/core/src/Graph.js @@ -469,6 +469,16 @@ export default class Graph { return null; } + topoSort(): Array { + let sorted: Array = []; + this.traverse({ + exit: nodeId => { + sorted.push(nodeId); + }, + }); + return sorted.reverse(); + } + findAncestor(nodeId: NodeId, fn: (nodeId: NodeId) => boolean): ?NodeId { let res = null; this.traverseAncestors(nodeId, (nodeId, ctx, traversal) => { diff --git a/packages/core/utils/src/collection.js b/packages/core/utils/src/collection.js index 21f2751d486..08826aa3825 100644 --- a/packages/core/utils/src/collection.js +++ b/packages/core/utils/src/collection.js @@ -43,3 +43,17 @@ export function setDifference(a: Set, b: Set): Set { } return difference; } + +export function setIntersection(a: Set, b: Set): Set { + let intersection = new Set(); + for (let e of a) { + if (b.has(e)) { + intersection.add(e); + } + } + return intersection; +} + +export function setUnion(a: Iterable, b: Iterable): Set { + return new Set([...a, ...b]); +} diff --git a/packages/core/utils/src/index.js b/packages/core/utils/src/index.js index be4d656b0df..6cedef6b6de 100644 --- a/packages/core/utils/src/index.js +++ b/packages/core/utils/src/index.js @@ -35,6 +35,8 @@ export { objectSortedEntries, objectSortedEntriesDeep, setDifference, + setIntersection, + setUnion, } from './collection'; export {resolveConfig, resolveConfigSync, loadConfig} from './config'; export {DefaultMap, DefaultWeakMap} from './DefaultMap'; From 2a5dbb136e88e0a818c66a47858dce78f541832b Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Mon, 2 Aug 2021 12:33:50 -0700 Subject: [PATCH 27/73] wip --- .../bundlers/default/src/DefaultBundler.js | 23 ++++++++----------- .../core/integration-tests/test/javascript.js | 6 ++--- packages/core/test-utils/src/utils.js | 4 ++-- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 4af410834b3..39ae8848273 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -402,7 +402,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { return node; }, exit(node) { - if (stack[stack.length - 1] === node.value) { + if (stack[stack.length - 1]?.[0] === node.value) { stack.pop(); } }, @@ -475,7 +475,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { let ancestorAssets: Map> = new Map(); //set difference //Map - console.log('toposort:', asyncBundleRootGraph.topoSort()); + //console.log('toposort:', asyncBundleRootGraph.topoSort()); for (let nodeId of asyncBundleRootGraph.topoSort()) { let bundleRoot = asyncBundleRootGraph.getNode(nodeId); if (bundleRoot === 'root') continue; @@ -487,7 +487,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { ) .map(id => nullthrows(reachableRoots.getNode(id))); //assets synchronously loaded when a is loaded - console.log({syncAssetsLoaded}); + //console.log({syncAssetsLoaded}); let ancestors = ancestorAssets.get(bundleRoot); @@ -495,7 +495,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { ? setUnion(ancestors, syncAssetsLoaded) : new Set(syncAssetsLoaded); let children = asyncBundleRootGraph.getNodeIdsConnectedFrom(nodeId); - console.log({children}); + //console.log({children}); for (let childId of children) { let child = asyncBundleRootGraph.getNode(childId); @@ -508,14 +508,14 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { asyncBundleRootGraph.getNodeIdsConnectedTo(childId).length > 1 ) { ancestorAssets.set(child, setIntersection(combined, availableAssets)); - console.log({combined}); + //console.log({combined}); } else { ancestorAssets.set(child, setUnion(combined, availableAssets)); } } } - console.log(ancestorAssets); + //console.log(ancestorAssets); // Step 3: Place all assets into bundles. Each asset is placed into a single // bundle based on the bundle entries it is reachable from. This creates a // maximally code split bundle graph with no duplication. @@ -611,11 +611,8 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { bundle.size += asset.stats.size; // Add the bundle to each reachable bundle group. - for (let reachableAsset of reachable) { - let reachableRoot = nullthrows(bundleRoots.get(reachableAsset))[1]; - if (reachableRoot !== bundleId) { - bundleGraph.addEdge(reachableRoot, bundleId); - } + for (let sourceBundleId of sourceBundles) { + bundleGraph.addEdge(sourceBundleId, bundleId); } } } @@ -662,9 +659,9 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // $FlowFixMe dumpGraphToGraphViz(bundleGraph, 'IdealBundleGraph'); for (let [nodeId, node] of asyncBundleRootGraph.nodes) { - console.log('node id is', nodeId, 'and node is', node); + //console.log('node id is', nodeId, 'and node is', node); for (let edge of asyncBundleRootGraph.getNodeIdsConnectedFrom(nodeId)) { - console.log('Edge from ', edge, ' to ', nodeId); + //console.log('Edge from ', edge, ' to ', nodeId); } } return { diff --git a/packages/core/integration-tests/test/javascript.js b/packages/core/integration-tests/test/javascript.js index eb99b21001d..37353be0849 100644 --- a/packages/core/integration-tests/test/javascript.js +++ b/packages/core/integration-tests/test/javascript.js @@ -1890,7 +1890,7 @@ describe('javascript', function() { ]); }); - it('should create a shared bundle between browser and worker contexts', async () => { + it.only('should create a shared bundle between browser and worker contexts', async () => { let b = await bundle( path.join(__dirname, '/integration/html-shared-worker/index.html'), {mode: 'production', defaultTargetOptions: {shouldScopeHoist: false}}, @@ -2055,7 +2055,7 @@ describe('javascript', function() { assert.equal(await output(), 3); }); - it('should duplicate small modules across multiple bundles', async function() { + it.only('should duplicate small modules across multiple bundles', async function() { let b = await bundle( path.join(__dirname, '/integration/dynamic-common-small/index.js'), ); @@ -2143,7 +2143,7 @@ describe('javascript', function() { assert.equal(await output(), 5); }); - it.only('should duplicate an asset if it is not present in every parent bundle', async function() { + it('should duplicate an asset if it is not present in every parent bundle', async function() { let b = await bundle( ['a.js', 'b.js'].map(entry => path.join(__dirname, 'integration/dynamic-hoist-no-dedupe', entry), diff --git a/packages/core/test-utils/src/utils.js b/packages/core/test-utils/src/utils.js index 3126989bad5..88501f61557 100644 --- a/packages/core/test-utils/src/utils.js +++ b/packages/core/test-utils/src/utils.js @@ -505,9 +505,9 @@ export function assertBundles( assets, }); }); - //console.log('Actual Bundles are', actualBundles); + console.log('Actual Bundles are', actualBundles); - //console.log('Expected Bundles are', expectedBundles); + console.log('Expected Bundles are', expectedBundles); for (let bundle of expectedBundles) { if (!Array.isArray(bundle.assets)) { throw new Error( From c1f82eb06620adef82604bd105186066a14b1f02 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Mon, 2 Aug 2021 18:18:14 -0400 Subject: [PATCH 28/73] WIP --- .../bundlers/default/src/DefaultBundler.js | 232 ++++++++++++------ packages/core/core/src/ContentGraph.js | 6 + 2 files changed, 160 insertions(+), 78 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 4af410834b3..c0b1ee2ce5c 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -6,6 +6,7 @@ import type { BundleBehavior, BundleGroup, Dependency, + DependencyPriority, Environment, Config, MutableBundleGraph, @@ -66,8 +67,19 @@ export type Bundle = {| type: string, |}; +type DependencyBundleGraph = ContentGraph< + | {| + value: Bundle, + type: 'bundle', + |} + | {| + value: Dependency, + type: 'dependency', + |}, + DependencyPriority, +>; type IdealGraph = {| - bundleLoadedByDependency: DefaultMap>, + dependencyBundleGraph: DependencyBundleGraph, bundleGraph: Graph, entryBundles: Array, assetReference: DefaultMap>, @@ -95,18 +107,27 @@ function decorateLegacyGraph( //TODO add in reference edges based on stored assets from create ideal graph let idealBundleToLegacyBundle: Map = new Map(); - let {bundleGraph: idealBundleGraph, bundleLoadedByDependency} = idealGraph; + let {bundleGraph: idealBundleGraph, dependencyBundleGraph} = idealGraph; let entryBundleToBundleGroup: Map = new Map(); for (let [bundleNodeId, idealBundle] of idealBundleGraph.nodes) { - let dependencies = bundleLoadedByDependency.get(bundleNodeId); - + let dependencies = dependencyBundleGraph + .getNodeIdsConnectedTo( + dependencyBundleGraph.getNodeIdByContentKey(String(bundleNodeId)), + ['lazy', 'sync'], + ) + .map(nodeId => { + let dependency = nullthrows(dependencyBundleGraph.getNode(nodeId)); + invariant(dependency.type === 'dependency'); + return dependency.value; + }); + console.log('deps are', dependencies); let entryAsset = bundleGraph.getAssetById(idealBundle.assetIds[0]); // This entry asset is the first asset of the bundle (not entry file asset) let bundleGroup; let bundle; - if (dependencies && dependencies.size > 0) { + if (dependencies && dependencies.length > 0) { //console.log('deps are', dependencies); for (let dependency of dependencies) { bundleGroup = bundleGraph.createBundleGroup( @@ -130,6 +151,7 @@ function decorateLegacyGraph( } else if (idealBundle.sourceBundles.length > 0) { //TODO this should be > 1 //this should only happen for shared bundles + bundle = nullthrows( bundleGraph.createBundle({ uniqueKey: @@ -229,10 +251,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // Asset to the bundle it's an entry of let bundleRoots: Map = new Map(); let bundles: Map = new Map(); - let bundleLoadedByDependency: DefaultMap< - NodeId, - Set, - > = new DefaultMap(() => new Set()); + let dependencyBundleGraph: DependencyBundleGraph = new ContentGraph(); let assetReference: DefaultMap< Asset, Array<[Dependency, Bundle]>, @@ -270,20 +289,30 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { asyncBundleRootGraph.setRootNodeId(rootNodeId); for (let [asset, dependency] of entries) { - let nodeId = bundleGraph.addNode( - createBundle({ - asset, - target: nullthrows(dependency.target), - needsStableName: dependency.isEntry, - }), - ); + let bundle = createBundle({ + asset, + target: nullthrows(dependency.target), + needsStableName: dependency.isEntry, + }); + let nodeId = bundleGraph.addNode(bundle); bundles.set(asset.id, nodeId); bundleRoots.set(asset, [nodeId, nodeId]); asyncBundleRootGraph.addEdge( rootNodeId, asyncBundleRootGraph.addNodeByContentKey(asset.id, asset), ); - bundleLoadedByDependency.get(nodeId).add(dependency); + + dependencyBundleGraph.addEdge( + dependencyBundleGraph.addNodeByContentKeyIfNeeded(dependency.id, { + value: dependency, + type: 'dependency', + }), + dependencyBundleGraph.addNodeByContentKeyIfNeeded(String(nodeId), { + value: bundle, + type: 'bundle', + }), + dependency.priority, + ); } let assets = []; @@ -326,17 +355,29 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // TODO: This bundle can be "created" by multiple dependencies? let bundleId = bundles.get(childAsset.id); if (bundleId == null) { - bundleId = bundleGraph.addNode( - createBundle({ - asset: childAsset, - target: nullthrows(bundleGraph.getNode(stack[0][1])).target, - }), - ); + let bundle = createBundle({ + asset: childAsset, + target: nullthrows(bundleGraph.getNode(stack[0][1])).target, + }); + bundleId = bundleGraph.addNode(bundle); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); - } - bundleLoadedByDependency.get(bundleId).add(dependency); + dependencyBundleGraph.addEdge( + dependencyBundleGraph.addNodeByContentKeyIfNeeded(dependency.id, { + value: dependency, + type: 'dependency', + }), + dependencyBundleGraph.addNodeByContentKeyIfNeeded( + String(bundleId), + { + value: bundle, + type: 'bundle', + }, + ), + dependency.priority, + ); + } // Walk up the stack until we hit a different asset type // and mark each bundle as reachable from every parent bundle for (let i = stack.length - 1; i >= 0; i--) { @@ -352,23 +393,15 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { if (i === stack.length - 1) { //Add child and connection from parent to child bundleRoot - let childNodeId = asyncBundleRootGraph.hasContentKey( + let childNodeId = asyncBundleRootGraph.addNodeByContentKeyIfNeeded( childAsset.id, - ) - ? asyncBundleRootGraph.getNodeIdByContentKey(childAsset.id) - : asyncBundleRootGraph.addNodeByContentKey( - childAsset.id, - childAsset, - ); - - let parentNodeId = asyncBundleRootGraph.hasContentKey( + childAsset, + ); + + let parentNodeId = asyncBundleRootGraph.addNodeByContentKeyIfNeeded( stackAsset.id, - ) - ? asyncBundleRootGraph.getNodeIdByContentKey(stackAsset.id) - : asyncBundleRootGraph.addNodeByContentKey( - stackAsset.id, - stackAsset, - ); + stackAsset, + ); asyncBundleRootGraph.addEdge(parentNodeId, childNodeId); } @@ -392,6 +425,20 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); + dependencyBundleGraph.addEdge( + dependencyBundleGraph.addNodeByContentKeyIfNeeded(dependency.id, { + value: dependency, + type: 'dependency', + }), + dependencyBundleGraph.addNodeByContentKeyIfNeeded( + String(bundleId), + { + value: bundle, + type: 'bundle', + }, + ), + 'parallel', + ); // Add an edge from the bundle group entry to the new bundle. // This indicates that the bundle is loaded together with the entry bundleGraph.addEdge(bundleGroupNodeId, bundleId); @@ -402,7 +449,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { return node; }, exit(node) { - if (stack[stack.length - 1] === node.value) { + if (stack[stack.length - 1]?.[0] === node.value) { stack.pop(); } }, @@ -417,40 +464,56 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { ); for (let [root] of bundleRoots) { - let rootNodeId = reachableRoots.hasContentKey(root.id) - ? reachableRoots.getNodeIdByContentKey(root.id) - : reachableRoots.addNodeByContentKey(root.id, root); + let rootNodeId = reachableRoots.addNodeByContentKeyIfNeeded(root.id, root); assetGraph.traverse((node, isAsync, actions) => { if (node.value === root) { return; } if (node.type === 'dependency') { - // Handle other situations in which bundles are created and maybe - // centralize that in a function - // if (isABundleRoot and dependency is not sync) { - if (node.value.priority === 'lazy') { - let assets = assetGraph.getDependencyAssets(node.value); - if (assets.length === 0) { - return node; - } + if (dependencyBundleGraph.hasContentKey(node.value.id)) { + if (node.value.priority === 'lazy') { + let assets = assetGraph.getDependencyAssets(node.value); + if (assets.length === 0) { + return node; + } - invariant(assets.length === 1); - let bundleRoot = assets[0]; - invariant(bundleRoots.has(bundleRoot)); + invariant(assets.length === 1); + let bundleRoot = assets[0]; + invariant(bundleRoots.has(bundleRoot)); - reachableAsyncRoots - .get(nullthrows(bundles.get(bundleRoot.id))) - .add(root); + reachableAsyncRoots + .get(nullthrows(bundles.get(bundleRoot.id))) + .add(root); + } actions.skipChildren(); + return; } - + // TODO Handle other situations in which bundles are created and maybe + // centralize that in a function + // if (isABundleRoot and dependency is not sync) { + // if (node.value.priority === 'lazy') { + // let assets = assetGraph.getDependencyAssets(node.value); + // if (assets.length === 0) { + // return node; + // } + + // invariant(assets.length === 1); + // let bundleRoot = assets[0]; + // invariant(bundleRoots.has(bundleRoot)); + + // reachableAsyncRoots + // .get(nullthrows(bundles.get(bundleRoot.id))) + // .add(root); + // actions.skipChildren(); + //} return; } - let nodeId = reachableRoots.hasContentKey(node.value.id) - ? reachableRoots.getNodeIdByContentKey(node.value.id) - : reachableRoots.addNodeByContentKey(node.value.id, node.value); + let nodeId = reachableRoots.addNodeByContentKeyIfNeeded( + node.value.id, + node.value, + ); reachableRoots.addEdge(rootNodeId, nodeId); }, root); } @@ -475,7 +538,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { let ancestorAssets: Map> = new Map(); //set difference //Map - console.log('toposort:', asyncBundleRootGraph.topoSort()); + //console.log('toposort:', asyncBundleRootGraph.topoSort()); for (let nodeId of asyncBundleRootGraph.topoSort()) { let bundleRoot = asyncBundleRootGraph.getNode(nodeId); if (bundleRoot === 'root') continue; @@ -487,7 +550,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { ) .map(id => nullthrows(reachableRoots.getNode(id))); //assets synchronously loaded when a is loaded - console.log({syncAssetsLoaded}); + //console.log({syncAssetsLoaded}); let ancestors = ancestorAssets.get(bundleRoot); @@ -495,7 +558,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { ? setUnion(ancestors, syncAssetsLoaded) : new Set(syncAssetsLoaded); let children = asyncBundleRootGraph.getNodeIdsConnectedFrom(nodeId); - console.log({children}); + //console.log({children}); for (let childId of children) { let child = asyncBundleRootGraph.getNode(childId); @@ -508,14 +571,14 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { asyncBundleRootGraph.getNodeIdsConnectedTo(childId).length > 1 ) { ancestorAssets.set(child, setIntersection(combined, availableAssets)); - console.log({combined}); + //console.log({combined}); } else { ancestorAssets.set(child, setUnion(combined, availableAssets)); } } } - console.log(ancestorAssets); + //console.log(ancestorAssets); // Step 3: Place all assets into bundles. Each asset is placed into a single // bundle based on the bundle entries it is reachable from. This creates a // maximally code split bundle graph with no duplication. @@ -586,7 +649,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { } } else if (reachable.length > 0) { // If the asset is reachable from more than one entry, find or create - // a bundle for that combination of bundles, and add the asset to it. + // a bundle for that combination of bundles (shared bundle), and add the asset to it. let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); let key = reachable.map(a => a.id).join(','); @@ -604,6 +667,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { bundle.sourceBundles = sourceBundles; bundleId = bundleGraph.addNode(bundle); bundles.set(key, bundleId); + console.log('creating shared bundle ', bundleId); } else { bundle = nullthrows(bundleGraph.getNode(bundleId)); } @@ -611,15 +675,13 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { bundle.size += asset.stats.size; // Add the bundle to each reachable bundle group. - for (let reachableAsset of reachable) { - let reachableRoot = nullthrows(bundleRoots.get(reachableAsset))[1]; - if (reachableRoot !== bundleId) { - bundleGraph.addEdge(reachableRoot, bundleId); - } + for (let sourceBundleId of sourceBundles) { + console.log('bundle id', bundleId, 'and source id is', sourceBundleId); + bundleGraph.addEdge(sourceBundleId, bundleId); } } } - + dumpGraphToGraphViz(bundleGraph, 'IdealBundleGraph'); // Step 4: Merge any sibling bundles required by entry bundles back into the entry bundle. // Entry bundles must be predictable, so cannot have unpredictable siblings. for (let entryAsset of entries.keys()) { @@ -627,6 +689,16 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); for (let siblingId of bundleGraph.getNodeIdsConnectedFrom(entryBundleId)) { let sibling = nullthrows(bundleGraph.getNode(siblingId)); + console.log( + 'Sibling type is ', + sibling.type, + 'sib id is', + siblingId, + 'and entry type is', + entryBundle.type, + 'id: ', + entryBundleId, + ); if (sibling.type !== entryBundle.type) { continue; } @@ -634,6 +706,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { for (let assetId of sibling.assetIds) { entryBundle.assetIds.push(assetId); } + console.log('Removing edge from', entryBundleId, 'to ', siblingId); bundleGraph.removeEdge(entryBundleId, siblingId); reachableAsyncRoots.get(siblingId).delete(entryAsset); if (sibling.sourceBundles.length > 1) { @@ -647,6 +720,7 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { for (let assetId of sibling.assetIds) { bundle.assetIds.push(assetId); } + console.log('Removing edge from', id, 'to ', siblingId); bundleGraph.removeEdge(id, siblingId); } } @@ -662,14 +736,16 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // $FlowFixMe dumpGraphToGraphViz(bundleGraph, 'IdealBundleGraph'); for (let [nodeId, node] of asyncBundleRootGraph.nodes) { - console.log('node id is', nodeId, 'and node is', node); - for (let edge of asyncBundleRootGraph.getNodeIdsConnectedFrom(nodeId)) { - console.log('Edge from ', edge, ' to ', nodeId); + //console.log('node id is', nodeId, 'and node is', node); + for (let otherNode of asyncBundleRootGraph.getNodeIdsConnectedFrom( + nodeId, + )) { + //console.log('Edge from ', nodeId, ' to ', otherNode); } } return { bundleGraph, - bundleLoadedByDependency, + dependencyBundleGraph, entryBundles: [...bundleRoots.values()].map(v => v[0]), assetReference, }; diff --git a/packages/core/core/src/ContentGraph.js b/packages/core/core/src/ContentGraph.js index 9cfdad0e651..43a4d77894f 100644 --- a/packages/core/core/src/ContentGraph.js +++ b/packages/core/core/src/ContentGraph.js @@ -57,6 +57,12 @@ export default class ContentGraph< return nodeId; } + addNodeByContentKeyIfNeeded(contentKey: ContentKey, node: TNode): NodeId { + return this.hasContentKey(contentKey) + ? this.getNodeIdByContentKey(contentKey) + : this.addNodeByContentKey(contentKey, node); + } + getNodeByContentKey(contentKey: ContentKey): ?TNode { let nodeId = this._contentKeyToNodeId.get(contentKey); if (nodeId != null) { From d4f25b276533f7d00b1f50601b0bada09de6562a Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Tue, 3 Aug 2021 11:45:41 -0700 Subject: [PATCH 29/73] do not add edge to itself when creating shared bundle --- packages/bundlers/default/src/DefaultBundler.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index c0b1ee2ce5c..7e7e8b3b042 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -677,8 +677,15 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { // Add the bundle to each reachable bundle group. for (let sourceBundleId of sourceBundles) { console.log('bundle id', bundleId, 'and source id is', sourceBundleId); - bundleGraph.addEdge(sourceBundleId, bundleId); + if (bundleId !== sourceBundleId) { + bundleGraph.addEdge(sourceBundleId, bundleId); + } } + + dependencyBundleGraph.addNodeByContentKeyIfNeeded(String(bundleId), { + value: bundle, + type: 'bundle', + }); } } dumpGraphToGraphViz(bundleGraph, 'IdealBundleGraph'); From 7cc52d59467c055e29536c04abbadfc2c6627e50 Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Tue, 3 Aug 2021 15:16:15 -0700 Subject: [PATCH 30/73] removeBundle --- .../bundlers/default/src/DefaultBundler.js | 129 ++++++++++-------- 1 file changed, 70 insertions(+), 59 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 7e7e8b3b042..d4290b6c535 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -39,6 +39,12 @@ type BundlerConfig = {| maxParallelRequests?: number, |}; +type ResolvedBundlerConfig = {| + minBundles: number, + minBundleSize: number, + maxParallelRequests: number, +|}; + // Default options by http version. const HTTP_OPTIONS = { '1': { @@ -56,7 +62,7 @@ const HTTP_OPTIONS = { type AssetId = string; type BundleRoot = Asset; export type Bundle = {| - assetIds: Array, + assets: Set, internalizedAssetIds: Array, bundleBehavior?: ?BundleBehavior, needsStableName: boolean, @@ -91,7 +97,7 @@ export default (new Bundler({ }, bundle({bundleGraph, config}) { - decorateLegacyGraph(createIdealGraph(bundleGraph), bundleGraph); + decorateLegacyGraph(createIdealGraph(bundleGraph, config), bundleGraph); }, optimize() {}, }): Bundler); @@ -121,14 +127,13 @@ function decorateLegacyGraph( invariant(dependency.type === 'dependency'); return dependency.value; }); - console.log('deps are', dependencies); - let entryAsset = bundleGraph.getAssetById(idealBundle.assetIds[0]); + + let [entryAsset] = [...idealBundle.assets]; // This entry asset is the first asset of the bundle (not entry file asset) let bundleGroup; let bundle; if (dependencies && dependencies.length > 0) { - //console.log('deps are', dependencies); for (let dependency of dependencies) { bundleGroup = bundleGraph.createBundleGroup( dependency, @@ -155,7 +160,7 @@ function decorateLegacyGraph( bundle = nullthrows( bundleGraph.createBundle({ uniqueKey: - idealBundle.assetIds.join(',') + + [...idealBundle.assets].map(asset => asset.id).join(',') + idealBundle.sourceBundles.join(','), needsStableName: idealBundle.needsStableName, bundleBehavior: idealBundle.bundleBehavior, @@ -177,13 +182,9 @@ function decorateLegacyGraph( idealBundleToLegacyBundle.set(idealBundle, bundle); - let assets = idealBundle.assetIds.map(a => bundleGraph.getAssetById(a)); - - for (let asset of assets) { + for (let asset of idealBundle.assets) { bundleGraph.addAssetToBundle(asset, bundle); } - - //console.log('INTERNALIZED', idealBundle.internalizedAssetIds); } for (let [, idealBundle] of idealBundleGraph.nodes) { @@ -197,7 +198,6 @@ function decorateLegacyGraph( incomingDep.priority === 'lazy' && bundle.hasDependency(incomingDep) ) { - //console.log('INTERNALIZING DEP', incomingDep); bundleGraph.internalizeAsyncDependency(bundle, incomingDep); } else { // console.log( @@ -247,7 +247,10 @@ function decorateLegacyGraph( } } -function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { +function createIdealGraph( + assetGraph: MutableBundleGraph, + config: ResolvedBundlerConfig, +): IdealGraph { // Asset to the bundle it's an entry of let bundleRoots: Map = new Map(); let bundles: Map = new Map(); @@ -354,30 +357,34 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { ) { // TODO: This bundle can be "created" by multiple dependencies? let bundleId = bundles.get(childAsset.id); + let bundle; if (bundleId == null) { - let bundle = createBundle({ + bundle = createBundle({ asset: childAsset, target: nullthrows(bundleGraph.getNode(stack[0][1])).target, }); bundleId = bundleGraph.addNode(bundle); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); - - dependencyBundleGraph.addEdge( - dependencyBundleGraph.addNodeByContentKeyIfNeeded(dependency.id, { - value: dependency, - type: 'dependency', - }), - dependencyBundleGraph.addNodeByContentKeyIfNeeded( - String(bundleId), - { - value: bundle, - type: 'bundle', - }, - ), - dependency.priority, - ); + } else { + bundle = nullthrows(bundleGraph.getNode(bundleId)); } + + dependencyBundleGraph.addEdge( + dependencyBundleGraph.addNodeByContentKeyIfNeeded(dependency.id, { + value: dependency, + type: 'dependency', + }), + dependencyBundleGraph.addNodeByContentKeyIfNeeded( + String(bundleId), + { + value: bundle, + type: 'bundle', + }, + ), + dependency.priority, + ); + // Walk up the stack until we hit a different asset type // and mark each bundle as reachable from every parent bundle for (let i = stack.length - 1; i >= 0; i--) { @@ -538,7 +545,6 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { let ancestorAssets: Map> = new Map(); //set difference //Map - //console.log('toposort:', asyncBundleRootGraph.topoSort()); for (let nodeId of asyncBundleRootGraph.topoSort()) { let bundleRoot = asyncBundleRootGraph.getNode(nodeId); if (bundleRoot === 'root') continue; @@ -550,15 +556,12 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { ) .map(id => nullthrows(reachableRoots.getNode(id))); //assets synchronously loaded when a is loaded - //console.log({syncAssetsLoaded}); - let ancestors = ancestorAssets.get(bundleRoot); let combined = ancestors ? setUnion(ancestors, syncAssetsLoaded) : new Set(syncAssetsLoaded); let children = asyncBundleRootGraph.getNodeIdsConnectedFrom(nodeId); - //console.log({children}); for (let childId of children) { let child = asyncBundleRootGraph.getNode(childId); @@ -571,14 +574,12 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { asyncBundleRootGraph.getNodeIdsConnectedTo(childId).length > 1 ) { ancestorAssets.set(child, setIntersection(combined, availableAssets)); - //console.log({combined}); } else { ancestorAssets.set(child, setUnion(combined, availableAssets)); } } } - //console.log(ancestorAssets); // Step 3: Place all assets into bundles. Each asset is placed into a single // bundle based on the bundle entries it is reachable from. This creates a // maximally code split bundle graph with no duplication. @@ -590,7 +591,6 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { reachableRoots, ); - //console.log('Reachable before for', asset.filePath, 'is ', reachable); // Filter out bundles when the asset is reachable in every parent bundle. // (Only keep a bundle if all of the others are not descendents of it) reachable = reachable.filter(b => !ancestorAssets.get(b)?.has(asset)); //don't want to filter out bundle if 'b' is not "reachable" from all of its (a) immediate parents @@ -667,16 +667,14 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { bundle.sourceBundles = sourceBundles; bundleId = bundleGraph.addNode(bundle); bundles.set(key, bundleId); - console.log('creating shared bundle ', bundleId); } else { bundle = nullthrows(bundleGraph.getNode(bundleId)); } - bundle.assetIds.push(asset.id); + bundle.assets.add(asset); bundle.size += asset.stats.size; // Add the bundle to each reachable bundle group. for (let sourceBundleId of sourceBundles) { - console.log('bundle id', bundleId, 'and source id is', sourceBundleId); if (bundleId !== sourceBundleId) { bundleGraph.addEdge(sourceBundleId, bundleId); } @@ -689,31 +687,27 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { } } dumpGraphToGraphViz(bundleGraph, 'IdealBundleGraph'); + // Step 4: Merge any sibling bundles required by entry bundles back into the entry bundle. // Entry bundles must be predictable, so cannot have unpredictable siblings. + for (let [bundleNodeId, bundle] of bundleGraph.nodes) { + if (bundle.sourceBundles.length > 0 && bundle.size < config.minBundleSize) { + removeBundle(bundleGraph, bundleNodeId); + } + } + for (let entryAsset of entries.keys()) { let entryBundleId = nullthrows(bundleRoots.get(entryAsset)?.[0]); let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); for (let siblingId of bundleGraph.getNodeIdsConnectedFrom(entryBundleId)) { let sibling = nullthrows(bundleGraph.getNode(siblingId)); - console.log( - 'Sibling type is ', - sibling.type, - 'sib id is', - siblingId, - 'and entry type is', - entryBundle.type, - 'id: ', - entryBundleId, - ); if (sibling.type !== entryBundle.type) { continue; } - - for (let assetId of sibling.assetIds) { - entryBundle.assetIds.push(assetId); + for (let asset of sibling.assets) { + entryBundle.assets.add(asset); + entryBundle.size += asset.stats.size; } - console.log('Removing edge from', entryBundleId, 'to ', siblingId); bundleGraph.removeEdge(entryBundleId, siblingId); reachableAsyncRoots.get(siblingId).delete(entryAsset); if (sibling.sourceBundles.length > 1) { @@ -724,10 +718,10 @@ function createIdealGraph(assetGraph: MutableBundleGraph): IdealGraph { if (sibling.sourceBundles.length === 1) { let id = sibling.sourceBundles.pop(); let bundle = nullthrows(bundleGraph.getNode(id)); - for (let assetId of sibling.assetIds) { - bundle.assetIds.push(assetId); + for (let asset of sibling.assets) { + bundle.assets.add(asset); + bundle.size += asset.stats.size; } - console.log('Removing edge from', id, 'to ', siblingId); bundleGraph.removeEdge(id, siblingId); } } @@ -796,7 +790,7 @@ function createBundle( ): Bundle { if (opts.asset == null) { return { - assetIds: [], + assets: new Set(), internalizedAssetIds: [], size: 0, sourceBundles: [], @@ -809,7 +803,7 @@ function createBundle( let asset = nullthrows(opts.asset); return { - assetIds: [asset.id], + assets: new Set([asset]), internalizedAssetIds: [], size: asset.stats.size, sourceBundles: [], @@ -821,7 +815,24 @@ function createBundle( }; } -async function loadBundlerConfig(config: Config, options: PluginOptions) { +function removeBundle(bundleGraph: Graph, bundleId: NodeId) { + let bundle = nullthrows(bundleGraph.getNode(bundleId)); + + for (let asset of bundle.assets) { + for (let sourceBundleId of bundle.sourceBundles) { + let sourceBundle = nullthrows(bundleGraph.getNode(sourceBundleId)); + sourceBundle.assets.add(asset); + sourceBundle.size += asset.stats.size; + } + } + + bundleGraph.removeNode(bundleId); +} + +async function loadBundlerConfig( + config: Config, + options: PluginOptions, +): Promise { let conf = await config.getConfig([], { packageKey: '@parcel/bundler-default', }); From 032590010db8437080444f75f3218fa4919b475c Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Mon, 9 Aug 2021 22:54:49 -0400 Subject: [PATCH 31/73] Dedup solution bundle group draft --- .../bundlers/default/src/DefaultBundler.js | 231 ++++++++++++++---- packages/core/core/src/dumpGraphToGraphViz.js | 11 +- packages/core/integration-tests/test/html.js | 2 +- packages/core/test-utils/src/utils.js | 3 + packages/core/types/index.js | 13 +- 5 files changed, 199 insertions(+), 61 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index d4290b6c535..c0360d787d5 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -12,6 +12,7 @@ import type { MutableBundleGraph, PluginOptions, Target, + FilePath, } from '@parcel/types'; import type {NodeId} from '@parcel/core/src/types'; import type {SchemaEntity} from '@parcel/utils'; @@ -87,6 +88,7 @@ type DependencyBundleGraph = ContentGraph< type IdealGraph = {| dependencyBundleGraph: DependencyBundleGraph, bundleGraph: Graph, + bundleGroupBundleIds: Array, entryBundles: Array, assetReference: DefaultMap>, |}; @@ -113,27 +115,30 @@ function decorateLegacyGraph( //TODO add in reference edges based on stored assets from create ideal graph let idealBundleToLegacyBundle: Map = new Map(); - let {bundleGraph: idealBundleGraph, dependencyBundleGraph} = idealGraph; + let { + bundleGraph: idealBundleGraph, + dependencyBundleGraph, + bundleGroupBundleIds, + } = idealGraph; let entryBundleToBundleGroup: Map = new Map(); for (let [bundleNodeId, idealBundle] of idealBundleGraph.nodes) { - let dependencies = dependencyBundleGraph - .getNodeIdsConnectedTo( - dependencyBundleGraph.getNodeIdByContentKey(String(bundleNodeId)), - ['lazy', 'sync'], - ) - .map(nodeId => { - let dependency = nullthrows(dependencyBundleGraph.getNode(nodeId)); - invariant(dependency.type === 'dependency'); - return dependency.value; - }); - let [entryAsset] = [...idealBundle.assets]; // This entry asset is the first asset of the bundle (not entry file asset) let bundleGroup; let bundle; - if (dependencies && dependencies.length > 0) { + if (bundleGroupBundleIds.includes(bundleNodeId)) { + let dependencies = dependencyBundleGraph + .getNodeIdsConnectedTo( + dependencyBundleGraph.getNodeIdByContentKey(String(bundleNodeId)), + ['lazy', 'sync', 'parallel'], + ) + .map(nodeId => { + let dependency = nullthrows(dependencyBundleGraph.getNode(nodeId)); + invariant(dependency.type === 'dependency'); + return dependency.value; + }); for (let dependency of dependencies) { bundleGroup = bundleGraph.createBundleGroup( dependency, @@ -270,6 +275,7 @@ function createIdealGraph( let asyncBundleRootGraph: ContentGraph< BundleRoot | 'root', > = new ContentGraph(); + let bundleGroupBundleIds: Array = []; //TODO of asyncBundleRootGraph: we should either add a root node or use bundleGraph which has a root automatically // Step 1: Create bundles for each entry. @@ -316,6 +322,7 @@ function createIdealGraph( }), dependency.priority, ); + bundleGroupBundleIds.push(nodeId); } let assets = []; @@ -355,17 +362,25 @@ function createIdealGraph( dependency.priority === 'lazy' || childAsset.bundleBehavior === 'isolated' ) { - // TODO: This bundle can be "created" by multiple dependencies? + // TODO: needsStableName if bundle exists here let bundleId = bundles.get(childAsset.id); let bundle; if (bundleId == null) { bundle = createBundle({ asset: childAsset, target: nullthrows(bundleGraph.getNode(stack[0][1])).target, + needsStableName: + dependency.bundleBehavior === 'inline' || + childAsset.bundleBehavior === 'inline' + ? false + : dependency.isEntry || dependency.needsStableName, + bundleBehavior: + dependency.bundleBehavior ?? childAsset.bundleBehavior, }); bundleId = bundleGraph.addNode(bundle); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); + bundleGroupBundleIds.push(bundleId); } else { bundle = nullthrows(bundleGraph.getNode(bundleId)); } @@ -496,24 +511,6 @@ function createIdealGraph( actions.skipChildren(); return; } - // TODO Handle other situations in which bundles are created and maybe - // centralize that in a function - // if (isABundleRoot and dependency is not sync) { - // if (node.value.priority === 'lazy') { - // let assets = assetGraph.getDependencyAssets(node.value); - // if (assets.length === 0) { - // return node; - // } - - // invariant(assets.length === 1); - // let bundleRoot = assets[0]; - // invariant(bundleRoots.has(bundleRoot)); - - // reachableAsyncRoots - // .get(nullthrows(bundles.get(bundleRoot.id))) - // .add(root); - // actions.skipChildren(); - //} return; } @@ -543,8 +540,20 @@ function createIdealGraph( // get all assets guarenteed to be loaded when bundle X is loaded // map.set(Z, {all assets gurenteed to be loaded at this point (by ancestors (X)) INTERSECTION WITH current map.get(z) }) + //TODO Consider BUNDLEGRROUPS let ancestorAssets: Map> = new Map(); - //set difference //Map + + // Using Nested Maps, we need to be able to query, for a bundleGroup, an asset within that bundleGroup, + // Any asset it "has", the ref when we first saw it + //This is a triply nest map :o + // AND we need a auxilary double map to keep current ref count for the next asset + // (this can be just 1 map num> because it will be reset when processing next bundleGroup ) + let assetRefsInBundleGroup: DefaultMap< + BundleRoot, + DefaultMap>, + > = new DefaultMap(() => new DefaultMap(() => new DefaultMap(() => 0))); + //FOR BUNDLEGROUPS, hold each BUNDLEGROUPROOT mapped to Roots within the bundle root, mapped to assets available, + // mapped to their number. We need duplicate entries for (let nodeId of asyncBundleRootGraph.topoSort()) { let bundleRoot = asyncBundleRootGraph.getNode(nodeId); if (bundleRoot === 'root') continue; @@ -558,6 +567,103 @@ function createIdealGraph( let ancestors = ancestorAssets.get(bundleRoot); + //Get all assets available in this bundleRoot's bundleGroup through ideal graph + //*****Bundle Group consideration start */ + let bundleGroupId = nullthrows(bundleRoots.get(bundleRoot))[1]; + let auxilaryRefCount: DefaultMap = new DefaultMap(() => 0); + + let availableAssetsfromBundleGroup = new Set(syncAssetsLoaded); // SET<[FILEPATH, NUMBER]> ? + let bundleRootInGroup; + let assetRefs = assetRefsInBundleGroup.get(bundleRoot); + //Process all nodes within bundleGroup that are NOT isolated + for (let bundleIdInGroup of bundleGraph.getNodeIdsConnectedFrom( + bundleGroupId, + )) { + let bundleInGroup = bundleGraph.getNode(bundleIdInGroup); //this is a bundle + + for (let asset of bundleInGroup.assets) { + //#1 getting first element of set -_- + if (bundleRoots.has(asset)) { + bundleRootInGroup = asset; + continue; + } + } + invariant(bundleRootInGroup); + + //Assets to consider = sync available assets plus itself + let assetsFromBundleRoot = reachableRoots + .getNodeIdsConnectedFrom( + reachableRoots.getNodeIdByContentKey(bundleRootInGroup.id), + ) + .map(id => nullthrows(reachableRoots.getNode(id))); + + assetsFromBundleRoot.push(...nullthrows(bundleInGroup).assets); + + for (let asset of assetsFromBundleRoot) { + //console.log('asset is ', asset.filePath); + if ( + bundleInGroup?.bundleBehavior != 'isolated' && + bundleInGroup?.bundleBehavior != 'inline' + ) { + //Add its assets + if (availableAssetsfromBundleGroup.has(asset)) { + //increment refs\ + auxilaryRefCount.set(asset, auxilaryRefCount.get(asset) + 1); + } else { + availableAssetsfromBundleGroup.add(asset); + auxilaryRefCount.set(asset, 1); + } + let countMap = assetRefs.get(bundleRootInGroup); + countMap.set(asset, auxilaryRefCount.get(asset)); + assetRefs.set(bundleRootInGroup, countMap); + } + } + } + // console.log('Ref counts are', assetRefsInBundleGroup); + // console.log( + // 'assets avaialble from bundle graph', + // availableAssetsfromBundleGroup, + // ); + for (let bundleIdInGroup of bundleGraph.getNodeIdsConnectedFrom( + bundleGroupId, + )) { + let bundleInGroup = bundleGraph.getNode(bundleIdInGroup); //this is a bundle + invariant(bundleInGroup != null && bundleInGroup.assets != null); + + for (let asset of bundleInGroup.assets) { + if (bundleRoots.has(asset)) { + bundleRootInGroup = asset; + continue; + } + } + invariant(bundleRootInGroup); + const availableAssets = ancestorAssets.get(bundleRootInGroup); + //if availabel assets is null, just set groupling bundleroots anc assets to the available assets in group? + if (availableAssets == null) { + ancestorAssets.set(bundleRootInGroup, availableAssetsfromBundleGroup); + } else if ( + bundleGraph.getNodeIdsConnectedTo(bundleIdInGroup).length > 1 + ) { + ancestorAssets.set( + bundleRootInGroup, + setIntersection(availableAssetsfromBundleGroup, availableAssets), + ); + } else { + ancestorAssets.set( + bundleRootInGroup, + setUnion(availableAssetsfromBundleGroup, availableAssets), + ); // Would this ever happen since if a child only has 1 parent, avail assets would be null? + } + // console.log( + // 'anc assets of groupling ', + // bundleIdInGroup, + // ' is ', + // ancestorAssets.get(bundleRootInGroup), + // ); + } + //**********End of bundle Group consideration */ + //should we edit Combined to include what available in it's bundlegroup? -we aren't editing parent AA + //THEN, process each ndoe in bundlegroup and do the same that we do below? let combined = ancestors ? setUnion(ancestors, syncAssetsLoaded) : new Set(syncAssetsLoaded); @@ -575,7 +681,7 @@ function createIdealGraph( ) { ancestorAssets.set(child, setIntersection(combined, availableAssets)); } else { - ancestorAssets.set(child, setUnion(combined, availableAssets)); + ancestorAssets.set(child, setUnion(combined, availableAssets)); // Would this ever happen since if a child only has 1 parent, avail assets would be null? } } } @@ -590,12 +696,49 @@ function createIdealGraph( asset, reachableRoots, ); - + //Don't have the notion of a bundlegroup here which is the problem // Filter out bundles when the asset is reachable in every parent bundle. // (Only keep a bundle if all of the others are not descendents of it) - reachable = reachable.filter(b => !ancestorAssets.get(b)?.has(asset)); //don't want to filter out bundle if 'b' is not "reachable" from all of its (a) immediate parents + let as = asset.filePath.split('/'); + reachable = reachable.filter(b => { + let one = !ancestorAssets.get(b)?.has(asset); + // console.log( + // 'for asset: ', + // as[as.length - 1], + // 'b is', + // b.filePath, + // 'one is', + // one, + // ); + let assetsBundleGroupRoot; + if (bundleRoots.get(b)) { + assetsBundleGroupRoot = bundleGraph.getNode(bundleRoots.get(b)[1]); + // console.log( + // 'bundleRoots', + // bundleRoots.get(b), + // 'and bundleroot bundlr is', + // assetsBundleGroupRoot, + // ); + assetsBundleGroupRoot = nullthrows( + [...assetsBundleGroupRoot.assets][0], + ); + } + if (assetsBundleGroupRoot) { + if ( + assetRefsInBundleGroup + .get(assetsBundleGroupRoot) + .get(b) + .get(asset) > 1 + ) { + return one || false; + } else { + return one || true; + } + } + return one; + }); //don't want to filter out bundle if 'b' is not "reachable" from all of its (a) immediate parents - //console.log('Reachable for', asset.filePath, 'is ', reachable); + //console.log('Reachable for', asset.filePath.split(), 'is ', reachable); //IDEA: reachableBundles as a graph so we can query an assets ancestors and/or decendants // BundleRoot = Root Asset of a bundle @@ -679,14 +822,12 @@ function createIdealGraph( bundleGraph.addEdge(sourceBundleId, bundleId); } } - dependencyBundleGraph.addNodeByContentKeyIfNeeded(String(bundleId), { value: bundle, type: 'bundle', }); } } - dumpGraphToGraphViz(bundleGraph, 'IdealBundleGraph'); // Step 4: Merge any sibling bundles required by entry bundles back into the entry bundle. // Entry bundles must be predictable, so cannot have unpredictable siblings. @@ -736,17 +877,11 @@ function createIdealGraph( // $FlowFixMe dumpGraphToGraphViz(bundleGraph, 'IdealBundleGraph'); - for (let [nodeId, node] of asyncBundleRootGraph.nodes) { - //console.log('node id is', nodeId, 'and node is', node); - for (let otherNode of asyncBundleRootGraph.getNodeIdsConnectedFrom( - nodeId, - )) { - //console.log('Edge from ', nodeId, ' to ', otherNode); - } - } + return { bundleGraph, dependencyBundleGraph, + bundleGroupBundleIds, entryBundles: [...bundleRoots.values()].map(v => v[0]), assetReference, }; @@ -779,6 +914,7 @@ function createBundle( env: Environment, type: string, needsStableName?: boolean, + bundleBehavior?: ?BundleBehavior, |} | {| target: Target, @@ -786,6 +922,7 @@ function createBundle( env?: Environment, type?: string, needsStableName?: boolean, + bundleBehavior?: ?BundleBehavior, |}, ): Bundle { if (opts.asset == null) { diff --git a/packages/core/core/src/dumpGraphToGraphViz.js b/packages/core/core/src/dumpGraphToGraphViz.js index 08cf979428f..ea0f66da3e0 100644 --- a/packages/core/core/src/dumpGraphToGraphViz.js +++ b/packages/core/core/src/dumpGraphToGraphViz.js @@ -54,10 +54,13 @@ export default async function dumpGraphToGraphViz( n.set('shape', 'box'); n.set('style', 'filled'); let label; - if (node.assetIds) { - label = `(${nodeId(id)}), (assetIds: ${node.assetIds.join( - ', ', - )}) (sourceBundles: ${node.sourceBundles.join( + if (node.assets) { + label = `(${nodeId(id)}), (assetIds: ${[...node.assets] + .map(a => { + let arr = a.filePath.split('/'); + return arr[arr.length - 1]; + }) + .join(', ')}) (sourceBundles: ${node.sourceBundles.join( ', ', )}) (bb ${node.bundleBehavior ?? 'none'})`; } else if (node.type) { diff --git a/packages/core/integration-tests/test/html.js b/packages/core/integration-tests/test/html.js index b2490f7cfb4..49fd24425f0 100644 --- a/packages/core/integration-tests/test/html.js +++ b/packages/core/integration-tests/test/html.js @@ -396,7 +396,7 @@ describe('html', function() { 2, ); }); - + //TODO it('should deduplicate shared code between script tags', async function() { let b = await bundle( path.join(__dirname, '/integration/html-js-dedup/index.html'), diff --git a/packages/core/test-utils/src/utils.js b/packages/core/test-utils/src/utils.js index 26a1e9946a1..84bd09b82f6 100644 --- a/packages/core/test-utils/src/utils.js +++ b/packages/core/test-utils/src/utils.js @@ -545,6 +545,9 @@ export function assertBundles( ); let i = 0; + // console.log('Actual Bundles are', actualBundles); + + // console.log('Expected Bundles are', expectedBundles); for (let bundle of expectedBundles) { let actualBundle = actualBundles[i++]; let name = bundle.name; diff --git a/packages/core/types/index.js b/packages/core/types/index.js index f4af00a618a..de80917edee 100644 --- a/packages/core/types/index.js +++ b/packages/core/types/index.js @@ -1326,7 +1326,10 @@ export interface BundleGraph { /** Returns a list of bundles in the bundle graph. By default, inline bundles are excluded. */ getBundles(opts?: {|includeInline: boolean|}): Array; /** Traverses the assets and dependencies in the bundle graph, in depth first order. */ - traverse(GraphVisitor): ?TContext; + traverse( + GraphVisitor, + ?Asset, + ): ?TContext; /** Traverses all bundles in the bundle graph, including inline bundles, in depth first order. */ traverseBundles( visit: GraphVisitor, @@ -1412,14 +1415,6 @@ export interface BundleGraph { asset: Asset, boundary: ?Bundle, ): Array; - traverse( - GraphVisitor, - ?Asset, - ): ?TContext; - traverseBundles( - visit: GraphVisitor, - startBundle: ?Bundle, - ): ?TContext; /** Returns a list of symbols from an asset or dependency that are referenced by a dependent asset. */ getUsedSymbols(Asset | Dependency): $ReadOnlySet; /** Returns the common root directory for the entry assets of a target. */ From 749da96f13793e9d396024001dded265653b4cf2 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Wed, 11 Aug 2021 11:26:19 -0400 Subject: [PATCH 32/73] WIP --- .../bundlers/default/src/DefaultBundler.js | 77 ++++++++++++------- .../core/integration-tests/test/javascript.js | 12 +-- 2 files changed, 54 insertions(+), 35 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index c0360d787d5..02d2db07f7f 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -571,7 +571,7 @@ function createIdealGraph( //*****Bundle Group consideration start */ let bundleGroupId = nullthrows(bundleRoots.get(bundleRoot))[1]; let auxilaryRefCount: DefaultMap = new DefaultMap(() => 0); - + //TODO should this include our bundle group root's assets ? let availableAssetsfromBundleGroup = new Set(syncAssetsLoaded); // SET<[FILEPATH, NUMBER]> ? let bundleRootInGroup; let assetRefs = assetRefsInBundleGroup.get(bundleRoot); @@ -619,11 +619,11 @@ function createIdealGraph( } } } - // console.log('Ref counts are', assetRefsInBundleGroup); - // console.log( - // 'assets avaialble from bundle graph', - // availableAssetsfromBundleGroup, - // ); + console.log('Ref counts are', assetRefsInBundleGroup); + console.log( + 'assets avaialble from bundle graph', + availableAssetsfromBundleGroup, + ); for (let bundleIdInGroup of bundleGraph.getNodeIdsConnectedFrom( bundleGroupId, )) { @@ -696,49 +696,65 @@ function createIdealGraph( asset, reachableRoots, ); + let small = asset.filePath.split('/'); + let toprint = small[small.length - 1] == 'lodash.js'; + if (toprint) { + console.log( + 'reachable before filtering for', + small[small.length - 1], + 'is ', + reachable, + ); + } + //Don't have the notion of a bundlegroup here which is the problem // Filter out bundles when the asset is reachable in every parent bundle. // (Only keep a bundle if all of the others are not descendents of it) - let as = asset.filePath.split('/'); reachable = reachable.filter(b => { + toprint && console.log('ancAssets are', ancestorAssets); let one = !ancestorAssets.get(b)?.has(asset); - // console.log( - // 'for asset: ', - // as[as.length - 1], - // 'b is', - // b.filePath, - // 'one is', - // one, - // ); let assetsBundleGroupRoot; if (bundleRoots.get(b)) { assetsBundleGroupRoot = bundleGraph.getNode(bundleRoots.get(b)[1]); - // console.log( - // 'bundleRoots', - // bundleRoots.get(b), - // 'and bundleroot bundlr is', - // assetsBundleGroupRoot, - // ); + assetsBundleGroupRoot = nullthrows( [...assetsBundleGroupRoot.assets][0], ); } + if (one === false) { + toprint && console.log('case 1asset b is ', b.filePath); + //TODO write this logic better + return one; // Bundle Root hierachy takes precedence over bundlegroup availability + } if (assetsBundleGroupRoot) { - if ( + //if this B is a bundleRoot, check + let two = !( assetRefsInBundleGroup .get(assetsBundleGroupRoot) .get(b) .get(asset) > 1 - ) { - return one || false; - } else { - return one || true; - } + ); + console.log('BUNDLE GROUP LOGIC: ', two, 'ROOT LOGIC', one); + return two; + + // return ( + // !( + // assetRefsInBundleGroup + // .get(assetsBundleGroupRoot) + // .get(b) + // .get(asset) > 1 + // ) && one + // ); } return one; }); //don't want to filter out bundle if 'b' is not "reachable" from all of its (a) immediate parents - - //console.log('Reachable for', asset.filePath.split(), 'is ', reachable); + toprint && + console.log( + 'reachable after filtering for', + small[small.length - 1], + 'is ', + reachable, + ); //IDEA: reachableBundles as a graph so we can query an assets ancestors and/or decendants // BundleRoot = Root Asset of a bundle @@ -798,7 +814,9 @@ function createIdealGraph( let bundleId = bundles.get(key); let bundle; + console.log('in shared bundle'); if (bundleId == null) { + console.log('new bundle shared'); let firstSourceBundle = nullthrows( bundleGraph.getNode(sourceBundles[0]), ); @@ -813,6 +831,7 @@ function createIdealGraph( } else { bundle = nullthrows(bundleGraph.getNode(bundleId)); } + console.log('asset is', asset.filePath); bundle.assets.add(asset); bundle.size += asset.stats.size; diff --git a/packages/core/integration-tests/test/javascript.js b/packages/core/integration-tests/test/javascript.js index e55e0432265..1a262e49bd2 100644 --- a/packages/core/integration-tests/test/javascript.js +++ b/packages/core/integration-tests/test/javascript.js @@ -1763,7 +1763,7 @@ describe('javascript', function() { ], }); }); - + //TODO Broke 1 it('should create a shared bundle to deduplicate assets in workers', async () => { let b = await bundle( path.join(__dirname, '/integration/worker-shared/index.js'), @@ -1847,7 +1847,7 @@ describe('javascript', function() { }, ]); }); - + // TODO Broke 2 it('should deduplicate and remove an unnecessary async bundle when it contains a cyclic reference to its entry', async () => { let b = await bundle( path.join( @@ -1876,7 +1876,7 @@ describe('javascript', function() { assert.deepEqual(await Promise.all((await run(b)).default), [5, 4]); }); - //TODO: Fixed + it('does not create bundles for dynamic imports when assets are available up the graph', async () => { let b = await bundle( path.join(__dirname, '/integration/internalize-no-bundle-split/index.js'), @@ -1918,8 +1918,8 @@ describe('javascript', function() { {assets: ['core.js', 'worker3.js']}, ]); }); - - it('should create a shared bundle between browser and worker contexts', async () => { + //TODO + it.only('should create a shared bundle between browser and worker contexts', async () => { let b = await bundle( path.join(__dirname, '/integration/html-shared-worker/index.html'), {mode: 'production', defaultTargetOptions: {shouldScopeHoist: false}}, @@ -2145,7 +2145,7 @@ describe('javascript', function() { assert.equal(typeof output, 'function'); assert.equal(await output(), 7); }); - + //TODO it('should not duplicate a module which is already in a parent bundle', async function() { let b = await bundle( path.join(__dirname, '/integration/dynamic-hoist-dup/index.js'), From 357463b6412cd0909dabd82d1334d7ae0b5a8133 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Wed, 11 Aug 2021 11:42:04 -0400 Subject: [PATCH 33/73] fix traversal type in bundlegraph --- packages/core/types/index.js | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/packages/core/types/index.js b/packages/core/types/index.js index f4af00a618a..de80917edee 100644 --- a/packages/core/types/index.js +++ b/packages/core/types/index.js @@ -1326,7 +1326,10 @@ export interface BundleGraph { /** Returns a list of bundles in the bundle graph. By default, inline bundles are excluded. */ getBundles(opts?: {|includeInline: boolean|}): Array; /** Traverses the assets and dependencies in the bundle graph, in depth first order. */ - traverse(GraphVisitor): ?TContext; + traverse( + GraphVisitor, + ?Asset, + ): ?TContext; /** Traverses all bundles in the bundle graph, including inline bundles, in depth first order. */ traverseBundles( visit: GraphVisitor, @@ -1412,14 +1415,6 @@ export interface BundleGraph { asset: Asset, boundary: ?Bundle, ): Array; - traverse( - GraphVisitor, - ?Asset, - ): ?TContext; - traverseBundles( - visit: GraphVisitor, - startBundle: ?Bundle, - ): ?TContext; /** Returns a list of symbols from an asset or dependency that are referenced by a dependent asset. */ getUsedSymbols(Asset | Dependency): $ReadOnlySet; /** Returns the common root directory for the entry assets of a target. */ From 324fc3c4981461af6d632227574487b9b504ce57 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Thu, 12 Aug 2021 13:10:39 -0400 Subject: [PATCH 34/73] reference edge removal and setIntersect alter --- .../bundlers/default/src/DefaultBundler.js | 18 +++--------------- .../core/core/src/public/MutableBundleGraph.js | 2 +- packages/core/types/index.js | 13 ++++--------- packages/core/utils/src/collection.js | 10 ++++------ packages/core/utils/src/index.js | 2 +- 5 files changed, 13 insertions(+), 32 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index d4290b6c535..937aabd4c38 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -25,7 +25,7 @@ import {Bundler} from '@parcel/plugin'; import { validateSchema, DefaultMap, - setIntersection, + setIntersect, setUnion, } from '@parcel/utils'; import {hashString} from '@parcel/hash'; @@ -570,12 +570,8 @@ function createIdealGraph( if (availableAssets == null) { ancestorAssets.set(child, combined); - } else if ( - asyncBundleRootGraph.getNodeIdsConnectedTo(childId).length > 1 - ) { - ancestorAssets.set(child, setIntersection(combined, availableAssets)); } else { - ancestorAssets.set(child, setUnion(combined, availableAssets)); + setIntersect(availableAssets, combined); } } } @@ -686,7 +682,6 @@ function createIdealGraph( }); } } - dumpGraphToGraphViz(bundleGraph, 'IdealBundleGraph'); // Step 4: Merge any sibling bundles required by entry bundles back into the entry bundle. // Entry bundles must be predictable, so cannot have unpredictable siblings. @@ -736,14 +731,7 @@ function createIdealGraph( // $FlowFixMe dumpGraphToGraphViz(bundleGraph, 'IdealBundleGraph'); - for (let [nodeId, node] of asyncBundleRootGraph.nodes) { - //console.log('node id is', nodeId, 'and node is', node); - for (let otherNode of asyncBundleRootGraph.getNodeIdsConnectedFrom( - nodeId, - )) { - //console.log('Edge from ', nodeId, ' to ', otherNode); - } - } + return { bundleGraph, dependencyBundleGraph, diff --git a/packages/core/core/src/public/MutableBundleGraph.js b/packages/core/core/src/public/MutableBundleGraph.js index 1aaeeffc728..ac9f5df430e 100644 --- a/packages/core/core/src/public/MutableBundleGraph.js +++ b/packages/core/core/src/public/MutableBundleGraph.js @@ -152,7 +152,7 @@ export default class MutableBundleGraph extends BundleGraph this.#graph._graph.addEdge(dependencyNodeId, bundleGroupNodeId); this.#graph._graph.replaceNodeIdsConnectedTo(bundleGroupNodeId, assetNodes); this.#graph._graph.addEdge(dependencyNodeId, resolvedNodeId, 'references'); - this.#graph.markDependencyReferenceable(dependencyNode.value); + //this.#graph.markDependencyReferenceable(dependencyNode.value); this.#graph._graph.removeEdge(dependencyNodeId, resolvedNodeId); //console.log('removed edge from', dependencyNodeId, 'to', resolvedNodeId); diff --git a/packages/core/types/index.js b/packages/core/types/index.js index f4af00a618a..de80917edee 100644 --- a/packages/core/types/index.js +++ b/packages/core/types/index.js @@ -1326,7 +1326,10 @@ export interface BundleGraph { /** Returns a list of bundles in the bundle graph. By default, inline bundles are excluded. */ getBundles(opts?: {|includeInline: boolean|}): Array; /** Traverses the assets and dependencies in the bundle graph, in depth first order. */ - traverse(GraphVisitor): ?TContext; + traverse( + GraphVisitor, + ?Asset, + ): ?TContext; /** Traverses all bundles in the bundle graph, including inline bundles, in depth first order. */ traverseBundles( visit: GraphVisitor, @@ -1412,14 +1415,6 @@ export interface BundleGraph { asset: Asset, boundary: ?Bundle, ): Array; - traverse( - GraphVisitor, - ?Asset, - ): ?TContext; - traverseBundles( - visit: GraphVisitor, - startBundle: ?Bundle, - ): ?TContext; /** Returns a list of symbols from an asset or dependency that are referenced by a dependent asset. */ getUsedSymbols(Asset | Dependency): $ReadOnlySet; /** Returns the common root directory for the entry assets of a target. */ diff --git a/packages/core/utils/src/collection.js b/packages/core/utils/src/collection.js index 08826aa3825..a7b656e8801 100644 --- a/packages/core/utils/src/collection.js +++ b/packages/core/utils/src/collection.js @@ -44,14 +44,12 @@ export function setDifference(a: Set, b: Set): Set { return difference; } -export function setIntersection(a: Set, b: Set): Set { - let intersection = new Set(); - for (let e of a) { - if (b.has(e)) { - intersection.add(e); +export function setIntersect(a: Set, b: Set): void { + for (let entry of a) { + if (!b.has(entry)) { + a.delete(entry); } } - return intersection; } export function setUnion(a: Iterable, b: Iterable): Set { diff --git a/packages/core/utils/src/index.js b/packages/core/utils/src/index.js index 6cedef6b6de..de17c8f6bcb 100644 --- a/packages/core/utils/src/index.js +++ b/packages/core/utils/src/index.js @@ -35,7 +35,7 @@ export { objectSortedEntries, objectSortedEntriesDeep, setDifference, - setIntersection, + setIntersect, setUnion, } from './collection'; export {resolveConfig, resolveConfigSync, loadConfig} from './config'; From ec341d06eb496cfc494f3501b2c69a77c1131155 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Thu, 12 Aug 2021 13:10:39 -0400 Subject: [PATCH 35/73] reference edge removal and setIntersect alter --- packages/bundlers/default/src/DefaultBundler.js | 8 ++------ packages/core/core/src/public/MutableBundleGraph.js | 2 +- packages/core/utils/src/collection.js | 10 ++++------ packages/core/utils/src/index.js | 2 +- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 02d2db07f7f..0f8584cb1db 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -26,7 +26,7 @@ import {Bundler} from '@parcel/plugin'; import { validateSchema, DefaultMap, - setIntersection, + setIntersect, setUnion, } from '@parcel/utils'; import {hashString} from '@parcel/hash'; @@ -676,12 +676,8 @@ function createIdealGraph( if (availableAssets == null) { ancestorAssets.set(child, combined); - } else if ( - asyncBundleRootGraph.getNodeIdsConnectedTo(childId).length > 1 - ) { - ancestorAssets.set(child, setIntersection(combined, availableAssets)); } else { - ancestorAssets.set(child, setUnion(combined, availableAssets)); // Would this ever happen since if a child only has 1 parent, avail assets would be null? + setIntersect(availableAssets, combined); } } } diff --git a/packages/core/core/src/public/MutableBundleGraph.js b/packages/core/core/src/public/MutableBundleGraph.js index 1aaeeffc728..ac9f5df430e 100644 --- a/packages/core/core/src/public/MutableBundleGraph.js +++ b/packages/core/core/src/public/MutableBundleGraph.js @@ -152,7 +152,7 @@ export default class MutableBundleGraph extends BundleGraph this.#graph._graph.addEdge(dependencyNodeId, bundleGroupNodeId); this.#graph._graph.replaceNodeIdsConnectedTo(bundleGroupNodeId, assetNodes); this.#graph._graph.addEdge(dependencyNodeId, resolvedNodeId, 'references'); - this.#graph.markDependencyReferenceable(dependencyNode.value); + //this.#graph.markDependencyReferenceable(dependencyNode.value); this.#graph._graph.removeEdge(dependencyNodeId, resolvedNodeId); //console.log('removed edge from', dependencyNodeId, 'to', resolvedNodeId); diff --git a/packages/core/utils/src/collection.js b/packages/core/utils/src/collection.js index 08826aa3825..a7b656e8801 100644 --- a/packages/core/utils/src/collection.js +++ b/packages/core/utils/src/collection.js @@ -44,14 +44,12 @@ export function setDifference(a: Set, b: Set): Set { return difference; } -export function setIntersection(a: Set, b: Set): Set { - let intersection = new Set(); - for (let e of a) { - if (b.has(e)) { - intersection.add(e); +export function setIntersect(a: Set, b: Set): void { + for (let entry of a) { + if (!b.has(entry)) { + a.delete(entry); } } - return intersection; } export function setUnion(a: Iterable, b: Iterable): Set { diff --git a/packages/core/utils/src/index.js b/packages/core/utils/src/index.js index 6cedef6b6de..de17c8f6bcb 100644 --- a/packages/core/utils/src/index.js +++ b/packages/core/utils/src/index.js @@ -35,7 +35,7 @@ export { objectSortedEntries, objectSortedEntriesDeep, setDifference, - setIntersection, + setIntersect, setUnion, } from './collection'; export {resolveConfig, resolveConfigSync, loadConfig} from './config'; From 368f12c08b92e5c5f88a69d167d07599ced1cc02 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Thu, 12 Aug 2021 14:43:00 -0400 Subject: [PATCH 36/73] add bundle group ids --- .../bundlers/default/src/DefaultBundler.js | 63 +++++++++---------- packages/core/core/src/dumpGraphToGraphViz.js | 11 ++-- 2 files changed, 38 insertions(+), 36 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 937aabd4c38..718ac79bb75 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -87,6 +87,7 @@ type DependencyBundleGraph = ContentGraph< type IdealGraph = {| dependencyBundleGraph: DependencyBundleGraph, bundleGraph: Graph, + bundleGroupBundleIds: Array, entryBundles: Array, assetReference: DefaultMap>, |}; @@ -113,27 +114,30 @@ function decorateLegacyGraph( //TODO add in reference edges based on stored assets from create ideal graph let idealBundleToLegacyBundle: Map = new Map(); - let {bundleGraph: idealBundleGraph, dependencyBundleGraph} = idealGraph; + let { + bundleGraph: idealBundleGraph, + dependencyBundleGraph, + bundleGroupBundleIds, + } = idealGraph; let entryBundleToBundleGroup: Map = new Map(); for (let [bundleNodeId, idealBundle] of idealBundleGraph.nodes) { - let dependencies = dependencyBundleGraph - .getNodeIdsConnectedTo( - dependencyBundleGraph.getNodeIdByContentKey(String(bundleNodeId)), - ['lazy', 'sync'], - ) - .map(nodeId => { - let dependency = nullthrows(dependencyBundleGraph.getNode(nodeId)); - invariant(dependency.type === 'dependency'); - return dependency.value; - }); - let [entryAsset] = [...idealBundle.assets]; // This entry asset is the first asset of the bundle (not entry file asset) let bundleGroup; let bundle; - if (dependencies && dependencies.length > 0) { + if (bundleGroupBundleIds.includes(bundleNodeId)) { + let dependencies = dependencyBundleGraph + .getNodeIdsConnectedTo( + dependencyBundleGraph.getNodeIdByContentKey(String(bundleNodeId)), + ['lazy', 'sync', 'parallel'], + ) + .map(nodeId => { + let dependency = nullthrows(dependencyBundleGraph.getNode(nodeId)); + invariant(dependency.type === 'dependency'); + return dependency.value; + }); for (let dependency of dependencies) { bundleGroup = bundleGraph.createBundleGroup( dependency, @@ -270,6 +274,7 @@ function createIdealGraph( let asyncBundleRootGraph: ContentGraph< BundleRoot | 'root', > = new ContentGraph(); + let bundleGroupBundleIds: Array = []; //TODO of asyncBundleRootGraph: we should either add a root node or use bundleGraph which has a root automatically // Step 1: Create bundles for each entry. @@ -316,6 +321,7 @@ function createIdealGraph( }), dependency.priority, ); + bundleGroupBundleIds.push(nodeId); } let assets = []; @@ -355,17 +361,25 @@ function createIdealGraph( dependency.priority === 'lazy' || childAsset.bundleBehavior === 'isolated' ) { - // TODO: This bundle can be "created" by multiple dependencies? + // TODO: needsStableName if bundle exists here let bundleId = bundles.get(childAsset.id); let bundle; if (bundleId == null) { bundle = createBundle({ asset: childAsset, target: nullthrows(bundleGraph.getNode(stack[0][1])).target, + needsStableName: + dependency.bundleBehavior === 'inline' || + childAsset.bundleBehavior === 'inline' + ? false + : dependency.isEntry || dependency.needsStableName, + bundleBehavior: + dependency.bundleBehavior ?? childAsset.bundleBehavior, }); bundleId = bundleGraph.addNode(bundle); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); + bundleGroupBundleIds.push(bundleId); } else { bundle = nullthrows(bundleGraph.getNode(bundleId)); } @@ -496,24 +510,6 @@ function createIdealGraph( actions.skipChildren(); return; } - // TODO Handle other situations in which bundles are created and maybe - // centralize that in a function - // if (isABundleRoot and dependency is not sync) { - // if (node.value.priority === 'lazy') { - // let assets = assetGraph.getDependencyAssets(node.value); - // if (assets.length === 0) { - // return node; - // } - - // invariant(assets.length === 1); - // let bundleRoot = assets[0]; - // invariant(bundleRoots.has(bundleRoot)); - - // reachableAsyncRoots - // .get(nullthrows(bundles.get(bundleRoot.id))) - // .add(root); - // actions.skipChildren(); - //} return; } @@ -735,6 +731,7 @@ function createIdealGraph( return { bundleGraph, dependencyBundleGraph, + bundleGroupBundleIds, entryBundles: [...bundleRoots.values()].map(v => v[0]), assetReference, }; @@ -767,6 +764,7 @@ function createBundle( env: Environment, type: string, needsStableName?: boolean, + bundleBehavior?: ?BundleBehavior, |} | {| target: Target, @@ -774,6 +772,7 @@ function createBundle( env?: Environment, type?: string, needsStableName?: boolean, + bundleBehavior?: ?BundleBehavior, |}, ): Bundle { if (opts.asset == null) { diff --git a/packages/core/core/src/dumpGraphToGraphViz.js b/packages/core/core/src/dumpGraphToGraphViz.js index 08cf979428f..ea0f66da3e0 100644 --- a/packages/core/core/src/dumpGraphToGraphViz.js +++ b/packages/core/core/src/dumpGraphToGraphViz.js @@ -54,10 +54,13 @@ export default async function dumpGraphToGraphViz( n.set('shape', 'box'); n.set('style', 'filled'); let label; - if (node.assetIds) { - label = `(${nodeId(id)}), (assetIds: ${node.assetIds.join( - ', ', - )}) (sourceBundles: ${node.sourceBundles.join( + if (node.assets) { + label = `(${nodeId(id)}), (assetIds: ${[...node.assets] + .map(a => { + let arr = a.filePath.split('/'); + return arr[arr.length - 1]; + }) + .join(', ')}) (sourceBundles: ${node.sourceBundles.join( ', ', )}) (bb ${node.bundleBehavior ?? 'none'})`; } else if (node.type) { From 638e5fd2bbad40b049d7429ac10a8f4c493ac85d Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Thu, 12 Aug 2021 17:40:42 -0400 Subject: [PATCH 37/73] WIP --- .../bundlers/default/src/DefaultBundler.js | 235 ++++++++---------- packages/core/integration-tests/test/html.js | 2 +- .../core/integration-tests/test/javascript.js | 2 +- 3 files changed, 106 insertions(+), 133 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 0f8584cb1db..8cb065f39b2 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -541,7 +541,13 @@ function createIdealGraph( // map.set(Z, {all assets gurenteed to be loaded at this point (by ancestors (X)) INTERSECTION WITH current map.get(z) }) //TODO Consider BUNDLEGRROUPS - let ancestorAssets: Map> = new Map(); + //Walk each bundlegroup entry + //get each asset in the group (instead of syncassetsloaded) + + let ancestorAssets: Map< + BundleRoot, + Map | null>, + > = new Map(); // Using Nested Maps, we need to be able to query, for a bundleGroup, an asset within that bundleGroup, // Any asset it "has", the ref when we first saw it @@ -550,123 +556,54 @@ function createIdealGraph( // (this can be just 1 map num> because it will be reset when processing next bundleGroup ) let assetRefsInBundleGroup: DefaultMap< BundleRoot, - DefaultMap>, - > = new DefaultMap(() => new DefaultMap(() => new DefaultMap(() => 0))); + DefaultMap, + > = new DefaultMap(() => new DefaultMap(() => 0)); //FOR BUNDLEGROUPS, hold each BUNDLEGROUPROOT mapped to Roots within the bundle root, mapped to assets available, // mapped to their number. We need duplicate entries for (let nodeId of asyncBundleRootGraph.topoSort()) { let bundleRoot = asyncBundleRootGraph.getNode(nodeId); if (bundleRoot === 'root') continue; invariant(bundleRoot != null); - - let syncAssetsLoaded = reachableRoots - .getNodeIdsConnectedFrom( - reachableRoots.getNodeIdByContentKey(bundleRoot.id), - ) - .map(id => nullthrows(reachableRoots.getNode(id))); //assets synchronously loaded when a is loaded - let ancestors = ancestorAssets.get(bundleRoot); //Get all assets available in this bundleRoot's bundleGroup through ideal graph //*****Bundle Group consideration start */ let bundleGroupId = nullthrows(bundleRoots.get(bundleRoot))[1]; - let auxilaryRefCount: DefaultMap = new DefaultMap(() => 0); - //TODO should this include our bundle group root's assets ? - let availableAssetsfromBundleGroup = new Set(syncAssetsLoaded); // SET<[FILEPATH, NUMBER]> ? - let bundleRootInGroup; + //TODO should this include our bundle group root's assets let assetRefs = assetRefsInBundleGroup.get(bundleRoot); //Process all nodes within bundleGroup that are NOT isolated for (let bundleIdInGroup of bundleGraph.getNodeIdsConnectedFrom( bundleGroupId, )) { - let bundleInGroup = bundleGraph.getNode(bundleIdInGroup); //this is a bundle - - for (let asset of bundleInGroup.assets) { - //#1 getting first element of set -_- - if (bundleRoots.has(asset)) { - bundleRootInGroup = asset; - continue; - } + let bundleInGroup = nullthrows(bundleGraph.getNode(bundleIdInGroup)); //this is a bundle + if ( + bundleInGroup.bundleBehavior === 'isolated' || + bundleInGroup.bundleBehavior === 'inline' + ) { + continue; } - invariant(bundleRootInGroup); + let [bundleRoot] = [...bundleInGroup.assets]; - //Assets to consider = sync available assets plus itself let assetsFromBundleRoot = reachableRoots .getNodeIdsConnectedFrom( - reachableRoots.getNodeIdByContentKey(bundleRootInGroup.id), + reachableRoots.getNodeIdByContentKey(bundleRoot.id), ) .map(id => nullthrows(reachableRoots.getNode(id))); - assetsFromBundleRoot.push(...nullthrows(bundleInGroup).assets); - for (let asset of assetsFromBundleRoot) { - //console.log('asset is ', asset.filePath); - if ( - bundleInGroup?.bundleBehavior != 'isolated' && - bundleInGroup?.bundleBehavior != 'inline' - ) { - //Add its assets - if (availableAssetsfromBundleGroup.has(asset)) { - //increment refs\ - auxilaryRefCount.set(asset, auxilaryRefCount.get(asset) + 1); - } else { - availableAssetsfromBundleGroup.add(asset); - auxilaryRefCount.set(asset, 1); - } - let countMap = assetRefs.get(bundleRootInGroup); - countMap.set(asset, auxilaryRefCount.get(asset)); - assetRefs.set(bundleRootInGroup, countMap); - } + assetRefs.set(asset, assetRefs.get(asset) + 1); } } - console.log('Ref counts are', assetRefsInBundleGroup); - console.log( - 'assets avaialble from bundle graph', - availableAssetsfromBundleGroup, - ); - for (let bundleIdInGroup of bundleGraph.getNodeIdsConnectedFrom( - bundleGroupId, - )) { - let bundleInGroup = bundleGraph.getNode(bundleIdInGroup); //this is a bundle - invariant(bundleInGroup != null && bundleInGroup.assets != null); - for (let asset of bundleInGroup.assets) { - if (bundleRoots.has(asset)) { - bundleRootInGroup = asset; - continue; - } - } - invariant(bundleRootInGroup); - const availableAssets = ancestorAssets.get(bundleRootInGroup); - //if availabel assets is null, just set groupling bundleroots anc assets to the available assets in group? - if (availableAssets == null) { - ancestorAssets.set(bundleRootInGroup, availableAssetsfromBundleGroup); - } else if ( - bundleGraph.getNodeIdsConnectedTo(bundleIdInGroup).length > 1 - ) { - ancestorAssets.set( - bundleRootInGroup, - setIntersection(availableAssetsfromBundleGroup, availableAssets), - ); - } else { - ancestorAssets.set( - bundleRootInGroup, - setUnion(availableAssetsfromBundleGroup, availableAssets), - ); // Would this ever happen since if a child only has 1 parent, avail assets would be null? - } - // console.log( - // 'anc assets of groupling ', - // bundleIdInGroup, - // ' is ', - // ancestorAssets.get(bundleRootInGroup), - // ); - } //**********End of bundle Group consideration */ //should we edit Combined to include what available in it's bundlegroup? -we aren't editing parent AA //THEN, process each ndoe in bundlegroup and do the same that we do below? + let bundleGroupAssets = new Set(assetRefs.keys()); + let combined = ancestors - ? setUnion(ancestors, syncAssetsLoaded) - : new Set(syncAssetsLoaded); + ? new Map([...bundleGroupAssets, ...ancestors.keys()].map(a => [a, null])) //sync assets loaded should be all available assets at that moment (from anc bundle groups) + : new Map([...bundleGroupAssets].map(a => [a, null])); + let children = asyncBundleRootGraph.getNodeIdsConnectedFrom(nodeId); for (let childId of children) { @@ -677,7 +614,28 @@ function createIdealGraph( if (availableAssets == null) { ancestorAssets.set(child, combined); } else { - setIntersect(availableAssets, combined); + ancestryIntersect(availableAssets, combined); + } + } + + let siblingAncestors = ancestors + ? ancestryUnion(new Set(ancestors.keys()), assetRefs, bundleGroupId) //sync assets loaded should be all available assets at that moment (from anc bundle groups) + : new Map([...bundleGroupAssets].map(a => [a, [bundleGroupId]])); + + for (let bundleIdInGroup of bundleGraph.getNodeIdsConnectedFrom( + bundleGroupId, + )) { + let bundleInGroup = bundleGraph.getNode(bundleIdInGroup); //this is a bundle + invariant(bundleInGroup != null && bundleInGroup.assets != null); + + let [bundleRoot] = [...bundleInGroup.assets]; + + const availableAssets = ancestorAssets.get(bundleRoot); + + if (availableAssets == null) { + ancestorAssets.set(bundleRoot, siblingAncestors); + } else { + ancestryIntersect(availableAssets, siblingAncestors); } } } @@ -707,51 +665,29 @@ function createIdealGraph( // Filter out bundles when the asset is reachable in every parent bundle. // (Only keep a bundle if all of the others are not descendents of it) reachable = reachable.filter(b => { - toprint && console.log('ancAssets are', ancestorAssets); - let one = !ancestorAssets.get(b)?.has(asset); - let assetsBundleGroupRoot; - if (bundleRoots.get(b)) { - assetsBundleGroupRoot = bundleGraph.getNode(bundleRoots.get(b)[1]); - - assetsBundleGroupRoot = nullthrows( - [...assetsBundleGroupRoot.assets][0], - ); - } - if (one === false) { - toprint && console.log('case 1asset b is ', b.filePath); - //TODO write this logic better - return one; // Bundle Root hierachy takes precedence over bundlegroup availability - } - if (assetsBundleGroupRoot) { - //if this B is a bundleRoot, check - let two = !( - assetRefsInBundleGroup - .get(assetsBundleGroupRoot) - .get(b) - .get(asset) > 1 - ); - console.log('BUNDLE GROUP LOGIC: ', two, 'ROOT LOGIC', one); - return two; - - // return ( - // !( - // assetRefsInBundleGroup - // .get(assetsBundleGroupRoot) - // .get(b) - // .get(asset) > 1 - // ) && one - // ); + let ancestry = ancestorAssets.get(b).get(asset); + if (ancestry === undefined) { + return true; + } else if (ancestry === null) { + return false; + } else { + if ( + ancestry.every( + bundleId => assetRefsInBundleGroup.get(bundleId).get(asset) > 1, + ) + ) { + for (let bundleRoot of ancestry) { + assetRefsInBundleGroup + .get(bundleRoot) + .set( + asset, + assetRefsInBundleGroup.get(bundleRoot).get(asset) - 1, + ); + } + return false; + } } - return one; }); //don't want to filter out bundle if 'b' is not "reachable" from all of its (a) immediate parents - toprint && - console.log( - 'reachable after filtering for', - small[small.length - 1], - 'is ', - reachable, - ); - //IDEA: reachableBundles as a graph so we can query an assets ancestors and/or decendants // BundleRoot = Root Asset of a bundle // reachableRoots = any asset => all BundleRoots that require it synchronously @@ -1017,6 +953,43 @@ async function loadBundlerConfig( }; } +function ancestryUnion( + ancestors: Set, + assetRefs: Map, + bundleId: BundleRoot, +): Map | null> { + let map = new Map(); + for (let a of ancestors) { + map.set(a, null); + } + for (let [asset, refCount] of assetRefs) { + if (!ancestors.has(asset)) { + map.set(asset, [bundleId]); + } + } + return map; +} + +function ancestryIntersect( + map: Map | null>, + previousMap: Map | null>, +): void { + for (let [asset, arrayOfBundleIds] of map) { + if (previousMap.has(asset)) { + let prevBundleIds = previousMap.get(asset); + if (prevBundleIds) { + if (arrayOfBundleIds) { + arrayOfBundleIds.push(...prevBundleIds); + } else { + map.set(asset, [...prevBundleIds]); + } + } + } else { + map.delete(asset); + } + } +} + function getReachableBundleRoots(asset, graph): Array { return graph .getNodeIdsConnectedTo(graph.getNodeIdByContentKey(asset.id)) diff --git a/packages/core/integration-tests/test/html.js b/packages/core/integration-tests/test/html.js index 49fd24425f0..192c88eecc7 100644 --- a/packages/core/integration-tests/test/html.js +++ b/packages/core/integration-tests/test/html.js @@ -397,7 +397,7 @@ describe('html', function() { ); }); //TODO - it('should deduplicate shared code between script tags', async function() { + it.only('should deduplicate shared code between script tags', async function() { let b = await bundle( path.join(__dirname, '/integration/html-js-dedup/index.html'), ); diff --git a/packages/core/integration-tests/test/javascript.js b/packages/core/integration-tests/test/javascript.js index 1a262e49bd2..454fe3306e9 100644 --- a/packages/core/integration-tests/test/javascript.js +++ b/packages/core/integration-tests/test/javascript.js @@ -1764,7 +1764,7 @@ describe('javascript', function() { }); }); //TODO Broke 1 - it('should create a shared bundle to deduplicate assets in workers', async () => { + it.only('should create a shared bundle to deduplicate assets in workers', async () => { let b = await bundle( path.join(__dirname, '/integration/worker-shared/index.js'), { From c681393e22fcb5ee1684d60eff9a9fa834620c2a Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Thu, 12 Aug 2021 15:58:47 -0700 Subject: [PATCH 38/73] Cleanup --- .../bundlers/default/src/DefaultBundler.js | 60 ++++--------------- packages/core/integration-tests/test/html.js | 8 +-- .../core/integration-tests/test/javascript.js | 4 +- 3 files changed, 17 insertions(+), 55 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 8cb065f39b2..43925ea15b2 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -12,7 +12,6 @@ import type { MutableBundleGraph, PluginOptions, Target, - FilePath, } from '@parcel/types'; import type {NodeId} from '@parcel/core/src/types'; import type {SchemaEntity} from '@parcel/utils'; @@ -23,13 +22,7 @@ import dumpGraphToGraphViz from '@parcel/core/src/dumpGraphToGraphViz'; import invariant from 'assert'; import {Bundler} from '@parcel/plugin'; -import { - validateSchema, - DefaultMap, - setIntersect, - setUnion, -} from '@parcel/utils'; -import {hashString} from '@parcel/hash'; +import {validateSchema, DefaultMap} from '@parcel/utils'; import nullthrows from 'nullthrows'; import {encodeJSONKeyComponent} from '@parcel/diagnostic'; @@ -204,13 +197,6 @@ function decorateLegacyGraph( bundle.hasDependency(incomingDep) ) { bundleGraph.internalizeAsyncDependency(bundle, incomingDep); - } else { - // console.log( - // 'NOT INTERNALIZING DEP', - // incomingDep, - // incomingDep.priority, - // bundle.hasDependency(incomingDep), - // ); } } } @@ -218,13 +204,6 @@ function decorateLegacyGraph( for (let [bundleId, bundleGroup] of entryBundleToBundleGroup) { let outboundNodeIds = idealBundleGraph.getNodeIdsConnectedFrom(bundleId); - let mainBundleOfBundleGroup = nullthrows( - idealBundleGraph.getNode(bundleId), - ); - let legacyMainBundleOfBundleGroup = nullthrows( - idealBundleToLegacyBundle.get(mainBundleOfBundleGroup), - ); - for (let id of outboundNodeIds) { let siblingBundle = nullthrows(idealBundleGraph.getNode(id)); let legacySiblingBundle = nullthrows( @@ -561,7 +540,7 @@ function createIdealGraph( //FOR BUNDLEGROUPS, hold each BUNDLEGROUPROOT mapped to Roots within the bundle root, mapped to assets available, // mapped to their number. We need duplicate entries for (let nodeId of asyncBundleRootGraph.topoSort()) { - let bundleRoot = asyncBundleRootGraph.getNode(nodeId); + const bundleRoot = asyncBundleRootGraph.getNode(nodeId); if (bundleRoot === 'root') continue; invariant(bundleRoot != null); let ancestors = ancestorAssets.get(bundleRoot); @@ -619,8 +598,8 @@ function createIdealGraph( } let siblingAncestors = ancestors - ? ancestryUnion(new Set(ancestors.keys()), assetRefs, bundleGroupId) //sync assets loaded should be all available assets at that moment (from anc bundle groups) - : new Map([...bundleGroupAssets].map(a => [a, [bundleGroupId]])); + ? ancestryUnion(new Set(ancestors.keys()), assetRefs, bundleRoot) //sync assets loaded should be all available assets at that moment (from anc bundle groups) + : new Map([...bundleGroupAssets].map(a => [a, [bundleRoot]])); for (let bundleIdInGroup of bundleGraph.getNodeIdsConnectedFrom( bundleGroupId, @@ -650,22 +629,12 @@ function createIdealGraph( asset, reachableRoots, ); - let small = asset.filePath.split('/'); - let toprint = small[small.length - 1] == 'lodash.js'; - if (toprint) { - console.log( - 'reachable before filtering for', - small[small.length - 1], - 'is ', - reachable, - ); - } //Don't have the notion of a bundlegroup here which is the problem // Filter out bundles when the asset is reachable in every parent bundle. // (Only keep a bundle if all of the others are not descendents of it) reachable = reachable.filter(b => { - let ancestry = ancestorAssets.get(b).get(asset); + let ancestry = ancestorAssets.get(b)?.get(asset); if (ancestry === undefined) { return true; } else if (ancestry === null) { @@ -686,6 +655,8 @@ function createIdealGraph( } return false; } + + return true; } }); //don't want to filter out bundle if 'b' is not "reachable" from all of its (a) immediate parents @@ -729,12 +700,6 @@ function createIdealGraph( let bundle = nullthrows( bundleGraph.getNode(nullthrows(bundles.get(bundleRoot.id))), ); - // console.log( - // 'PUSHING', - // asset.id, - // 'into bundle', - // nullthrows(bundles.get(bundleRoot.id)), - // ); bundle.internalizedAssetIds.push(asset.id); } } @@ -746,9 +711,7 @@ function createIdealGraph( let bundleId = bundles.get(key); let bundle; - console.log('in shared bundle'); if (bundleId == null) { - console.log('new bundle shared'); let firstSourceBundle = nullthrows( bundleGraph.getNode(sourceBundles[0]), ); @@ -763,7 +726,6 @@ function createIdealGraph( } else { bundle = nullthrows(bundleGraph.getNode(bundleId)); } - console.log('asset is', asset.filePath); bundle.assets.add(asset); bundle.size += asset.stats.size; @@ -956,15 +918,15 @@ async function loadBundlerConfig( function ancestryUnion( ancestors: Set, assetRefs: Map, - bundleId: BundleRoot, -): Map | null> { + bundleRoot: BundleRoot, +): Map | null> { let map = new Map(); for (let a of ancestors) { map.set(a, null); } for (let [asset, refCount] of assetRefs) { - if (!ancestors.has(asset)) { - map.set(asset, [bundleId]); + if (!ancestors.has(asset) && refCount > 1) { + map.set(asset, [bundleRoot]); } } return map; diff --git a/packages/core/integration-tests/test/html.js b/packages/core/integration-tests/test/html.js index 192c88eecc7..2cb4261c9f7 100644 --- a/packages/core/integration-tests/test/html.js +++ b/packages/core/integration-tests/test/html.js @@ -396,8 +396,8 @@ describe('html', function() { 2, ); }); - //TODO - it.only('should deduplicate shared code between script tags', async function() { + + it('should deduplicate shared code between script tags', async function() { let b = await bundle( path.join(__dirname, '/integration/html-js-dedup/index.html'), ); @@ -409,11 +409,11 @@ describe('html', function() { }, { type: 'js', - assets: ['component-1.js', 'obj.js', 'esmodule-helpers.js'], + assets: ['component-1.js'], }, { type: 'js', - assets: ['component-2.js'], + assets: ['component-2.js', 'obj.js', 'esmodule-helpers.js'], }, ]); diff --git a/packages/core/integration-tests/test/javascript.js b/packages/core/integration-tests/test/javascript.js index 454fe3306e9..075f37ff5ab 100644 --- a/packages/core/integration-tests/test/javascript.js +++ b/packages/core/integration-tests/test/javascript.js @@ -1764,7 +1764,7 @@ describe('javascript', function() { }); }); //TODO Broke 1 - it.only('should create a shared bundle to deduplicate assets in workers', async () => { + it('should create a shared bundle to deduplicate assets in workers', async () => { let b = await bundle( path.join(__dirname, '/integration/worker-shared/index.js'), { @@ -1919,7 +1919,7 @@ describe('javascript', function() { ]); }); //TODO - it.only('should create a shared bundle between browser and worker contexts', async () => { + it('should create a shared bundle between browser and worker contexts', async () => { let b = await bundle( path.join(__dirname, '/integration/html-shared-worker/index.html'), {mode: 'production', defaultTargetOptions: {shouldScopeHoist: false}}, From 0fcb6a7f11ac2866ce15b97e8b26b84eba56a51f Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Mon, 16 Aug 2021 18:07:22 -0400 Subject: [PATCH 39/73] for isolated bundles dont deduplicate --- .../bundlers/default/src/DefaultBundler.js | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 43925ea15b2..cd05719a733 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -472,9 +472,10 @@ function createIdealGraph( } if (node.type === 'dependency') { - if (dependencyBundleGraph.hasContentKey(node.value.id)) { - if (node.value.priority === 'lazy') { - let assets = assetGraph.getDependencyAssets(node.value); + let dependency = node.value; + if (dependencyBundleGraph.hasContentKey(dependency.id)) { + if (dependency.priority === 'lazy') { + let assets = assetGraph.getDependencyAssets(dependency); if (assets.length === 0) { return node; } @@ -482,10 +483,11 @@ function createIdealGraph( invariant(assets.length === 1); let bundleRoot = assets[0]; invariant(bundleRoots.has(bundleRoot)); - - reachableAsyncRoots - .get(nullthrows(bundles.get(bundleRoot.id))) - .add(root); + if (dependency.specifierType !== 'url') { + reachableAsyncRoots + .get(nullthrows(bundles.get(bundleRoot.id))) + .add(root); + } } actions.skipChildren(); return; @@ -629,8 +631,6 @@ function createIdealGraph( asset, reachableRoots, ); - - //Don't have the notion of a bundlegroup here which is the problem // Filter out bundles when the asset is reachable in every parent bundle. // (Only keep a bundle if all of the others are not descendents of it) reachable = reachable.filter(b => { @@ -659,18 +659,21 @@ function createIdealGraph( return true; } }); //don't want to filter out bundle if 'b' is not "reachable" from all of its (a) immediate parents - // BundleRoot = Root Asset of a bundle // reachableRoots = any asset => all BundleRoots that require it synchronously // reachableBundles = Some BundleRoot => all BundleRoot decendants // reachable = all bundle root assets that cant always have that asset reliably on page (so they need to be pulled in by shared bundle or other) let rootBundle = bundleRoots.get(asset); - if (rootBundle != null) { + if ( + rootBundle != null && + !nullthrows(bundleGraph.getNode(rootBundle[0])).env.isIsolated() + ) { // If the asset is a bundle root, add the bundle to every other reachable bundle group. if (!bundles.has(asset.id)) { bundles.set(asset.id, rootBundle[0]); } + for (let reachableAsset of reachable) { if (reachableAsset !== asset) { bundleGraph.addEdge( @@ -686,7 +689,6 @@ function createIdealGraph( ? reachableAsyncRoots.get(rootBundle[0]) : []), ]; - // TODO: is this correct? let willInternalizeRoots = reachableAsync.filter( b => From aaedcb39d8404654011ef249a26f730f0198d850 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Mon, 16 Aug 2021 18:43:18 -0400 Subject: [PATCH 40/73] dedup reverse order --- packages/bundlers/default/src/DefaultBundler.js | 15 +++++++++------ packages/runtimes/js/src/JSRuntime.js | 2 -- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index cd05719a733..cdb9e817b48 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -553,9 +553,10 @@ function createIdealGraph( //TODO should this include our bundle group root's assets let assetRefs = assetRefsInBundleGroup.get(bundleRoot); //Process all nodes within bundleGroup that are NOT isolated - for (let bundleIdInGroup of bundleGraph.getNodeIdsConnectedFrom( + for (let bundleIdInGroup of [ bundleGroupId, - )) { + ...bundleGraph.getNodeIdsConnectedFrom(bundleGroupId), + ]) { let bundleInGroup = nullthrows(bundleGraph.getNode(bundleIdInGroup)); //this is a bundle if ( bundleInGroup.bundleBehavior === 'isolated' || @@ -592,7 +593,7 @@ function createIdealGraph( invariant(child !== 'root' && child != null); const availableAssets = ancestorAssets.get(child); - if (availableAssets == null) { + if (availableAssets === undefined) { ancestorAssets.set(child, combined); } else { ancestryIntersect(availableAssets, combined); @@ -613,7 +614,7 @@ function createIdealGraph( const availableAssets = ancestorAssets.get(bundleRoot); - if (availableAssets == null) { + if (availableAssets === undefined) { ancestorAssets.set(bundleRoot, siblingAncestors); } else { ancestryIntersect(availableAssets, siblingAncestors); @@ -627,10 +628,12 @@ function createIdealGraph( for (let asset of assets) { // Find bundle entries reachable from the asset. + let reachable: Array = getReachableBundleRoots( asset, reachableRoots, - ); + ).reverse(); + // Filter out bundles when the asset is reachable in every parent bundle. // (Only keep a bundle if all of the others are not descendents of it) reachable = reachable.filter(b => { @@ -710,7 +713,7 @@ function createIdealGraph( // a bundle for that combination of bundles (shared bundle), and add the asset to it. let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); let key = reachable.map(a => a.id).join(','); - + console.log('Asset is', asset, 'and src bundles are', sourceBundles); let bundleId = bundles.get(key); let bundle; if (bundleId == null) { diff --git a/packages/runtimes/js/src/JSRuntime.js b/packages/runtimes/js/src/JSRuntime.js index 88c9cbabb99..93d3cf4243f 100644 --- a/packages/runtimes/js/src/JSRuntime.js +++ b/packages/runtimes/js/src/JSRuntime.js @@ -83,7 +83,6 @@ export default (new Runtime({ let assets = []; for (let dependency of asyncDependencies) { let resolved = bundleGraph.resolveAsyncDependency(dependency, bundle); - //console.log('resolved', dependency, dependency.priority, 'to', resolved); if (resolved == null) { continue; } @@ -512,7 +511,6 @@ function getURLRuntime( ): RuntimeAsset { let relativePathExpr = getRelativePathExpr(from, to, options); let code; - if (dependency.meta.webworker === true && !from.env.isLibrary) { code = `let workerURL = require('./helpers/get-worker-url');\n`; if ( From 138fff88ac4f14ad010804edfc6f11521114e347 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 17 Aug 2021 12:00:24 -0700 Subject: [PATCH 41/73] Cleanup --- packages/bundlers/default/src/DefaultBundler.js | 1 - packages/core/test-utils/src/utils.js | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index cdb9e817b48..3c20ccc0c2d 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -713,7 +713,6 @@ function createIdealGraph( // a bundle for that combination of bundles (shared bundle), and add the asset to it. let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); let key = reachable.map(a => a.id).join(','); - console.log('Asset is', asset, 'and src bundles are', sourceBundles); let bundleId = bundles.get(key); let bundle; if (bundleId == null) { diff --git a/packages/core/test-utils/src/utils.js b/packages/core/test-utils/src/utils.js index 84bd09b82f6..ab11e863ee5 100644 --- a/packages/core/test-utils/src/utils.js +++ b/packages/core/test-utils/src/utils.js @@ -514,9 +514,10 @@ export function assertBundles( assets, }); }); - console.log('Actual Bundles are', actualBundles); - console.log('Expected Bundles are', expectedBundles); + // console.log('Actual Bundles are', actualBundles); + // console.log('Expected Bundles are', expectedBundles); + for (let bundle of expectedBundles) { if (!Array.isArray(bundle.assets)) { throw new Error( From 757e2b81c7f1a57b3823b7a12737352fc43ba7ee Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 17 Aug 2021 14:45:50 -0700 Subject: [PATCH 42/73] Pass along provided bundleBehavior --- packages/bundlers/default/src/DefaultBundler.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 3c20ccc0c2d..85e49015e50 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -852,6 +852,7 @@ function createBundle( type: nullthrows(opts.type), env: nullthrows(opts.env), needsStableName: Boolean(opts.needsStableName), + bundleBehavior: opts.bundleBehavior, }; } @@ -865,7 +866,7 @@ function createBundle( type: opts.type ?? asset.type, env: opts.env ?? asset.env, needsStableName: Boolean(opts.needsStableName), - bundleBehavior: asset.bundleBehavior, + bundleBehavior: opts.bundleBehavior ?? asset.bundleBehavior, }; } From 232ff960c566ca8bc6d9c5ca028fa8fd0830ea61 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 17 Aug 2021 15:11:59 -0700 Subject: [PATCH 43/73] Extract Graph and ContentGraph into @parcel/graph package --- packages/core/core/package.json | 1 + packages/core/core/src/AssetGraph.js | 5 ++-- packages/core/core/src/BundleGraph.js | 9 +++---- packages/core/core/src/RequestTracker.js | 5 ++-- packages/core/core/src/applyRuntimes.js | 2 +- packages/core/core/src/dumpGraphToGraphViz.js | 4 +-- packages/core/core/src/public/Bundle.js | 2 +- packages/core/core/src/public/BundleGraph.js | 2 +- .../core/src/requests/AssetGraphRequest.js | 4 +-- .../core/core/src/requests/AssetRequest.js | 2 +- .../core/core/src/requests/PackageRequest.js | 4 ++- .../core/src/requests/ParcelBuildRequest.js | 6 +++-- .../core/src/requests/WriteBundleRequest.js | 6 +++-- .../core/src/requests/WriteBundlesRequest.js | 3 ++- packages/core/core/src/types.js | 18 +------------ packages/core/core/src/utils.js | 3 ++- packages/core/core/test/PublicBundle.test.js | 3 ++- packages/core/graph/package.json | 25 +++++++++++++++++++ .../core/{core => graph}/src/ContentGraph.js | 0 packages/core/{core => graph}/src/Graph.js | 0 packages/core/graph/src/index.js | 6 +++++ packages/core/graph/src/types.js | 18 +++++++++++++ .../{core => graph}/test/ContentGraph.test.js | 0 .../core/{core => graph}/test/Graph.test.js | 0 24 files changed, 84 insertions(+), 44 deletions(-) create mode 100644 packages/core/graph/package.json rename packages/core/{core => graph}/src/ContentGraph.js (100%) rename packages/core/{core => graph}/src/Graph.js (100%) create mode 100644 packages/core/graph/src/index.js create mode 100644 packages/core/graph/src/types.js rename packages/core/{core => graph}/test/ContentGraph.test.js (100%) rename packages/core/{core => graph}/test/Graph.test.js (100%) diff --git a/packages/core/core/package.json b/packages/core/core/package.json index 2dcc08ea3d8..b87f1ac3b0e 100644 --- a/packages/core/core/package.json +++ b/packages/core/core/package.json @@ -28,6 +28,7 @@ "@parcel/diagnostic": "2.0.0-rc.0", "@parcel/events": "2.0.0-rc.0", "@parcel/fs": "2.0.0-rc.0", + "@parcel/graph": "2.0.0-rc.0", "@parcel/hash": "2.0.0-rc.0", "@parcel/logger": "2.0.0-rc.0", "@parcel/package-manager": "2.0.0-rc.0", diff --git a/packages/core/core/src/AssetGraph.js b/packages/core/core/src/AssetGraph.js index 33bedd5bcd4..6c2f1e93523 100644 --- a/packages/core/core/src/AssetGraph.js +++ b/packages/core/core/src/AssetGraph.js @@ -1,20 +1,19 @@ // @flow strict-local import type {GraphVisitor} from '@parcel/types'; +import type {ContentKey, NodeId, SerializedContentGraph} from '@parcel/graph'; import type { Asset, AssetGraphNode, AssetGroup, AssetGroupNode, AssetNode, - ContentKey, Dependency, DependencyNode, Entry, EntryFileNode, EntrySpecifierNode, Environment, - NodeId, Target, } from './types'; @@ -22,7 +21,7 @@ import invariant from 'assert'; import {hashString, Hash} from '@parcel/hash'; import {hashObject, objectSortedEntries} from '@parcel/utils'; import nullthrows from 'nullthrows'; -import ContentGraph, {type SerializedContentGraph} from './ContentGraph'; +import {ContentGraph} from '@parcel/graph'; import {createDependency} from './Dependency'; import {type ProjectPath, fromProjectPathRelative} from './projectPath'; diff --git a/packages/core/core/src/BundleGraph.js b/packages/core/core/src/BundleGraph.js index 62d81976f73..94ba636d5c5 100644 --- a/packages/core/core/src/BundleGraph.js +++ b/packages/core/core/src/BundleGraph.js @@ -6,6 +6,7 @@ import type { Symbol, TraversalActions, } from '@parcel/types'; +import type {NodeId, SerializedContentGraph} from '@parcel/graph'; import querystring from 'querystring'; import type { @@ -16,7 +17,6 @@ import type { BundleGroup, Dependency, DependencyNode, - NodeId, InternalSourceLocation, Target, } from './types'; @@ -26,13 +26,12 @@ import type {ProjectPath} from './projectPath'; import assert from 'assert'; import invariant from 'assert'; import nullthrows from 'nullthrows'; -import {objectSortedEntriesDeep, getRootDir} from '@parcel/utils'; +import {ContentGraph, ALL_EDGE_TYPES, mapVisitor} from '@parcel/graph'; import {Hash, hashString} from '@parcel/hash'; -import {Priority, BundleBehavior} from './types'; +import {objectSortedEntriesDeep, getRootDir} from '@parcel/utils'; +import {Priority, BundleBehavior} from './types'; import {getBundleGroupId, getPublicId} from './utils'; -import {ALL_EDGE_TYPES, mapVisitor} from './Graph'; -import ContentGraph, {type SerializedContentGraph} from './ContentGraph'; import {ISOLATED_ENVS} from './public/Environment'; import {fromProjectPath} from './projectPath'; diff --git a/packages/core/core/src/RequestTracker.js b/packages/core/core/src/RequestTracker.js index 9cef0d5a9f2..1d03e49a6cf 100644 --- a/packages/core/core/src/RequestTracker.js +++ b/packages/core/core/src/RequestTracker.js @@ -4,9 +4,8 @@ import type {AbortSignal} from 'abortcontroller-polyfill/dist/cjs-ponyfill'; import type {Async, EnvMap} from '@parcel/types'; import type {EventType, Options as WatcherOptions} from '@parcel/watcher'; import type WorkerFarm from '@parcel/workers'; +import type {ContentKey, NodeId, SerializedContentGraph} from '@parcel/graph'; import type { - ContentKey, - NodeId, ParcelOptions, RequestInvalidation, InternalFile, @@ -24,7 +23,7 @@ import { makeDeferredWithPromise, } from '@parcel/utils'; import {hashString} from '@parcel/hash'; -import ContentGraph, {type SerializedContentGraph} from './ContentGraph'; +import {ContentGraph} from '@parcel/graph'; import {assertSignalNotAborted, hashFromOption} from './utils'; import { type ProjectPath, diff --git a/packages/core/core/src/applyRuntimes.js b/packages/core/core/src/applyRuntimes.js index 6a4b26392ee..b073ea28993 100644 --- a/packages/core/core/src/applyRuntimes.js +++ b/packages/core/core/src/applyRuntimes.js @@ -1,11 +1,11 @@ // @flow strict-local +import type {ContentKey} from '@parcel/graph'; import type {Dependency, NamedBundle as INamedBundle} from '@parcel/types'; import type {SharedReference} from '@parcel/workers'; import type { AssetGroup, Bundle as InternalBundle, - ContentKey, Config, DevDepRequest, ParcelOptions, diff --git a/packages/core/core/src/dumpGraphToGraphViz.js b/packages/core/core/src/dumpGraphToGraphViz.js index abc20b8d565..05aaaaabb0a 100644 --- a/packages/core/core/src/dumpGraphToGraphViz.js +++ b/packages/core/core/src/dumpGraphToGraphViz.js @@ -1,11 +1,11 @@ // @flow +import type {Graph} from '@parcel/graph'; import type {AssetGraphNode, BundleGraphNode, Environment} from './types'; -import type Graph from './Graph'; -import {SpecifierType, Priority} from './types'; import path from 'path'; import {fromProjectPathRelative} from './projectPath'; +import {SpecifierType, Priority} from './types'; const COLORS = { root: 'gray', diff --git a/packages/core/core/src/public/Bundle.js b/packages/core/core/src/public/Bundle.js index c7a5f539e5f..00793ad4c66 100644 --- a/packages/core/core/src/public/Bundle.js +++ b/packages/core/core/src/public/Bundle.js @@ -25,7 +25,7 @@ import nullthrows from 'nullthrows'; import {DefaultWeakMap} from '@parcel/utils'; import {assetToAssetValue, assetFromValue} from './Asset'; -import {mapVisitor} from '../Graph'; +import {mapVisitor} from '@parcel/graph'; import Environment from './Environment'; import Dependency, {dependencyToInternalDependency} from './Dependency'; import Target from './Target'; diff --git a/packages/core/core/src/public/BundleGraph.js b/packages/core/core/src/public/BundleGraph.js index 3567cc38e19..c538fe2b4de 100644 --- a/packages/core/core/src/public/BundleGraph.js +++ b/packages/core/core/src/public/BundleGraph.js @@ -20,11 +20,11 @@ import type InternalBundleGraph from '../BundleGraph'; import invariant from 'assert'; import nullthrows from 'nullthrows'; +import {mapVisitor} from '@parcel/graph'; import {assetFromValue, assetToAssetValue, Asset} from './Asset'; import {bundleToInternalBundle} from './Bundle'; import Dependency, {dependencyToInternalDependency} from './Dependency'; import {targetToInternalTarget} from './Target'; -import {mapVisitor} from '../Graph'; import {fromInternalSourceLocation} from '../utils'; import BundleGroup, {bundleGroupToInternalBundleGroup} from './BundleGroup'; diff --git a/packages/core/core/src/requests/AssetGraphRequest.js b/packages/core/core/src/requests/AssetGraphRequest.js index 82b17ce4a2f..25a05ebdf1e 100644 --- a/packages/core/core/src/requests/AssetGraphRequest.js +++ b/packages/core/core/src/requests/AssetGraphRequest.js @@ -1,8 +1,9 @@ // @flow strict-local +import type {Diagnostic} from '@parcel/diagnostic'; +import type {NodeId} from '@parcel/graph'; import type {Async, Symbol, Meta} from '@parcel/types'; import type {SharedReference} from '@parcel/workers'; -import type {Diagnostic} from '@parcel/diagnostic'; import type { Asset, AssetGroup, @@ -12,7 +13,6 @@ import type { DependencyNode, Entry, InternalSourceLocation, - NodeId, ParcelOptions, Target, } from '../types'; diff --git a/packages/core/core/src/requests/AssetRequest.js b/packages/core/core/src/requests/AssetRequest.js index 76d2323c6dc..65293c785f1 100644 --- a/packages/core/core/src/requests/AssetRequest.js +++ b/packages/core/core/src/requests/AssetRequest.js @@ -1,11 +1,11 @@ // @flow strict-local +import type {ContentKey} from '@parcel/graph'; import type {Async} from '@parcel/types'; import type {StaticRunOpts} from '../RequestTracker'; import type { AssetRequestInput, AssetRequestResult, - ContentKey, DevDepRequest, TransformationRequest, } from '../types'; diff --git a/packages/core/core/src/requests/PackageRequest.js b/packages/core/core/src/requests/PackageRequest.js index 1f5c34f485b..8356b1fed56 100644 --- a/packages/core/core/src/requests/PackageRequest.js +++ b/packages/core/core/src/requests/PackageRequest.js @@ -1,9 +1,11 @@ // @flow strict-local +import type {ContentKey} from '@parcel/graph'; import type {Async} from '@parcel/types'; import type {SharedReference} from '@parcel/workers'; + import type {StaticRunOpts} from '../RequestTracker'; -import type {Bundle, ContentKey} from '../types'; +import type {Bundle} from '../types'; import type BundleGraph from '../BundleGraph'; import type {BundleInfo} from '../PackagerRunner'; import type {ConfigAndCachePath} from './ParcelConfigRequest'; diff --git a/packages/core/core/src/requests/ParcelBuildRequest.js b/packages/core/core/src/requests/ParcelBuildRequest.js index eeb5e5c63ea..56b690afe78 100644 --- a/packages/core/core/src/requests/ParcelBuildRequest.js +++ b/packages/core/core/src/requests/ParcelBuildRequest.js @@ -1,11 +1,13 @@ // @flow strict-local +import type {ContentKey} from '@parcel/graph'; import type {Async} from '@parcel/types'; import type {SharedReference} from '@parcel/workers'; +import type {AbortSignal} from 'abortcontroller-polyfill/dist/cjs-ponyfill'; + import type {StaticRunOpts} from '../RequestTracker'; -import type {Asset, AssetGroup, ContentKey, PackagedBundleInfo} from '../types'; +import type {Asset, AssetGroup, PackagedBundleInfo} from '../types'; import type BundleGraph from '../BundleGraph'; -import type {AbortSignal} from 'abortcontroller-polyfill/dist/cjs-ponyfill'; import createAssetGraphRequest from './AssetGraphRequest'; import createBundleGraphRequest from './BundleGraphRequest'; diff --git a/packages/core/core/src/requests/WriteBundleRequest.js b/packages/core/core/src/requests/WriteBundleRequest.js index 634a81b7938..c39911aff35 100644 --- a/packages/core/core/src/requests/WriteBundleRequest.js +++ b/packages/core/core/src/requests/WriteBundleRequest.js @@ -1,9 +1,11 @@ // @flow strict-local +import type {FileSystem, FileOptions} from '@parcel/fs'; +import type {ContentKey} from '@parcel/graph'; import type {Async, FilePath} from '@parcel/types'; + import type {StaticRunOpts} from '../RequestTracker'; -import type {Bundle, ContentKey, PackagedBundleInfo} from '../types'; -import type {FileSystem, FileOptions} from '@parcel/fs'; +import type {Bundle, PackagedBundleInfo} from '../types'; import type BundleGraph from '../BundleGraph'; import type {BundleInfo} from '../PackagerRunner'; diff --git a/packages/core/core/src/requests/WriteBundlesRequest.js b/packages/core/core/src/requests/WriteBundlesRequest.js index 176be6408c2..e33974e8b93 100644 --- a/packages/core/core/src/requests/WriteBundlesRequest.js +++ b/packages/core/core/src/requests/WriteBundlesRequest.js @@ -1,9 +1,10 @@ // @flow strict-local +import type {ContentKey} from '@parcel/graph'; import type {Async} from '@parcel/types'; import type {SharedReference} from '@parcel/workers'; import type {StaticRunOpts} from '../RequestTracker'; -import type {ContentKey, PackagedBundleInfo} from '../types'; +import type {PackagedBundleInfo} from '../types'; import type BundleGraph from '../BundleGraph'; import type {BundleInfo} from '../PackagerRunner'; diff --git a/packages/core/core/src/types.js b/packages/core/core/src/types.js index ce1979955c4..7f6ab655c7b 100644 --- a/packages/core/core/src/types.js +++ b/packages/core/core/src/types.js @@ -1,5 +1,6 @@ // @flow strict-local +import type {ContentKey} from '@parcel/graph'; import type { ASTGenerator, BuildMode, @@ -284,23 +285,6 @@ export type ParcelOptions = {| |}, |}; -// forcing NodeId to be opaque as it should only be created once -export opaque type NodeId = number; -export function toNodeId(x: number): NodeId { - return x; -} -export function fromNodeId(x: NodeId): number { - return x; -} - -export type ContentKey = string; - -export type Edge = {| - from: NodeId, - to: NodeId, - type: TEdgeType, -|}; - export type AssetNode = {| id: ContentKey, +type: 'asset', diff --git a/packages/core/core/src/utils.js b/packages/core/core/src/utils.js index ca7de3b00ff..4f5f50150c9 100644 --- a/packages/core/core/src/utils.js +++ b/packages/core/core/src/utils.js @@ -15,11 +15,12 @@ import type { import invariant from 'assert'; import baseX from 'base-x'; +import {Graph} from '@parcel/graph'; import {hashObject} from '@parcel/utils'; + import {registerSerializableClass} from './serializer'; import AssetGraph from './AssetGraph'; import BundleGraph from './BundleGraph'; -import Graph from './Graph'; import ParcelConfig from './ParcelConfig'; import {RequestGraph} from './RequestTracker'; import Config from './public/Config'; diff --git a/packages/core/core/test/PublicBundle.test.js b/packages/core/core/test/PublicBundle.test.js index 231b537f687..39fe7baf6ba 100644 --- a/packages/core/core/test/PublicBundle.test.js +++ b/packages/core/core/test/PublicBundle.test.js @@ -2,11 +2,12 @@ import type {Bundle as InternalBundle} from '../src/types'; import assert from 'assert'; +import {ContentGraph} from '@parcel/graph'; + import {Bundle, NamedBundle, PackagedBundle} from '../src/public/Bundle'; import BundleGraph from '../src/BundleGraph'; import {createEnvironment} from '../src/Environment'; import {DEFAULT_OPTIONS} from './test-utils'; -import ContentGraph from '../src/ContentGraph'; import {toProjectPath} from '../src/projectPath'; describe('Public Bundle', () => { diff --git a/packages/core/graph/package.json b/packages/core/graph/package.json new file mode 100644 index 00000000000..48cf3915825 --- /dev/null +++ b/packages/core/graph/package.json @@ -0,0 +1,25 @@ +{ + "name": "@parcel/graph", + "version": "2.0.0-rc.0", + "description": "Blazing fast, zero configuration web application bundler", + "license": "MIT", + "publishConfig": { + "access": "public" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "repository": { + "type": "git", + "url": "https://github.com/parcel-bundler/parcel.git" + }, + "main": "lib/index.js", + "source": "src/index.js", + "engines": { + "node": ">= 12.0.0" + }, + "dependencies": { + "nullthrows": "^1.1.1" + } +} diff --git a/packages/core/core/src/ContentGraph.js b/packages/core/graph/src/ContentGraph.js similarity index 100% rename from packages/core/core/src/ContentGraph.js rename to packages/core/graph/src/ContentGraph.js diff --git a/packages/core/core/src/Graph.js b/packages/core/graph/src/Graph.js similarity index 100% rename from packages/core/core/src/Graph.js rename to packages/core/graph/src/Graph.js diff --git a/packages/core/graph/src/index.js b/packages/core/graph/src/index.js new file mode 100644 index 00000000000..a364b030efe --- /dev/null +++ b/packages/core/graph/src/index.js @@ -0,0 +1,6 @@ +// @flow strict-local + +export type {NodeId, ContentKey, Edge} from './types'; +export {toNodeId, fromNodeId} from './types'; +export {default as Graph, ALL_EDGE_TYPES, GraphOpts, mapVisitor} from './Graph'; +export {default as ContentGraph, SerializedContentGraph} from './ContentGraph'; diff --git a/packages/core/graph/src/types.js b/packages/core/graph/src/types.js new file mode 100644 index 00000000000..2e8e4d501e9 --- /dev/null +++ b/packages/core/graph/src/types.js @@ -0,0 +1,18 @@ +// @flow strict-local + +// forcing NodeId to be opaque as it should only be created once +export opaque type NodeId = number; +export function toNodeId(x: number): NodeId { + return x; +} +export function fromNodeId(x: NodeId): number { + return x; +} + +export type ContentKey = string; + +export type Edge = {| + from: NodeId, + to: NodeId, + type: TEdgeType, +|}; diff --git a/packages/core/core/test/ContentGraph.test.js b/packages/core/graph/test/ContentGraph.test.js similarity index 100% rename from packages/core/core/test/ContentGraph.test.js rename to packages/core/graph/test/ContentGraph.test.js diff --git a/packages/core/core/test/Graph.test.js b/packages/core/graph/test/Graph.test.js similarity index 100% rename from packages/core/core/test/Graph.test.js rename to packages/core/graph/test/Graph.test.js From fc68f39d7f5da4d88686766408471cb1a204d4e3 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Wed, 1 Sep 2021 17:09:44 -0400 Subject: [PATCH 44/73] do not remove entries after internalization and reduce reachable for shared bundle if parent child relationship exists --- .../bundlers/default/src/DefaultBundler.js | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index a48c6a6892d..8078dad0212 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -713,6 +713,17 @@ function createIdealGraph( // If the asset is reachable from more than one entry, find or create // a bundle for that combination of bundles (shared bundle), and add the asset to it. let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); + let sourceBundleSet = new Set([...sourceBundles]); + let toRemove: Set = new Set(); + for (let bundleId of sourceBundles) { + for (let id of bundleGraph.getNodeIdsConnectedTo(bundleId)) { + if (sourceBundleSet.has(id)) { + toRemove.add(id); + } + } + } + reachable = reachable.filter(a => !toRemove.has(bundles.get(a.id))); + let key = reachable.map(a => a.id).join(','); let bundleId = bundles.get(key); let bundle; @@ -789,6 +800,15 @@ function createIdealGraph( for (let [asyncBundleRoot, dependentRoots] of reachableAsyncRoots) { if (dependentRoots.size === 0) { + //TODO make get entry asset function + let bundleNode = bundleGraph.getNode(asyncBundleRoot); + + if (bundleNode?.assets) { + let [entryAsset] = [...bundleNode.assets]; + if (entries.has(entryAsset)) { + continue; + } + } bundleGraph.removeNode(asyncBundleRoot); } } From 8ae7088885ca9f6831cc15835036f0a760e940de Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 7 Sep 2021 15:31:07 -0700 Subject: [PATCH 45/73] Clean up orphaned subrequests manually in RequestGraph --- packages/core/core/src/RequestTracker.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/core/core/src/RequestTracker.js b/packages/core/core/src/RequestTracker.js index 1d03e49a6cf..a0931c9ef88 100644 --- a/packages/core/core/src/RequestTracker.js +++ b/packages/core/core/src/RequestTracker.js @@ -276,12 +276,25 @@ export class RequestGraph extends ContentGraph< } } + let previousSubrequestNodeIds = this.getNodeIdsConnectedFrom( + requestNodeId, + 'subrequest', + ); + this.replaceNodeIdsConnectedTo( requestNodeId, subrequestNodeIds, null, 'subrequest', ); + + for (let subrequestNodeId of previousSubrequestNodeIds) { + if ( + this.getNodeIdsConnectedTo(subrequestNodeId, 'subrequest').length === 0 + ) { + this.removeNode(subrequestNodeId); + } + } } invalidateNode(nodeId: NodeId, reason: InvalidateReason) { From 0352fd8806ada0c31b8669a0a1c7513263e6e274 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 7 Sep 2021 15:40:30 -0700 Subject: [PATCH 46/73] Adjust Graph orphan node tests to have roots --- packages/core/graph/src/Graph.js | 2 -- packages/core/graph/test/Graph.test.js | 10 ++++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/core/graph/src/Graph.js b/packages/core/graph/src/Graph.js index 24e97d6bdfb..8a396211793 100644 --- a/packages/core/graph/src/Graph.js +++ b/packages/core/graph/src/Graph.js @@ -235,8 +235,6 @@ export default class Graph { ); } - //console.trace('removeEdge: removing edge from', from, 'to', to, 'on', type); - this.outboundEdges.removeEdge(from, to, type); this.inboundEdges.removeEdge(to, from, type); diff --git a/packages/core/graph/test/Graph.test.js b/packages/core/graph/test/Graph.test.js index f311d01996b..68f1a9e1172 100644 --- a/packages/core/graph/test/Graph.test.js +++ b/packages/core/graph/test/Graph.test.js @@ -83,11 +83,14 @@ describe('Graph', () => { it('isOrphanedNode should return true or false if the node is orphaned or not', () => { let graph = new Graph(); + let rootNode = graph.addNode('root'); + graph.setRootNodeId(rootNode); + let nodeA = graph.addNode('a'); let nodeB = graph.addNode('b'); let nodeC = graph.addNode('c'); - graph.addEdge(nodeA, nodeB); - graph.addEdge(nodeA, nodeC, 'edgetype'); + graph.addEdge(rootNode, nodeB); + graph.addEdge(nodeB, nodeC, 'edgetype'); assert(graph.isOrphanedNode(nodeA)); assert(!graph.isOrphanedNode(nodeB)); assert(!graph.isOrphanedNode(nodeC)); @@ -101,6 +104,7 @@ describe('Graph', () => { // c let graph = new Graph(); let nodeA = graph.addNode('a'); + graph.setRootNodeId(nodeA); let nodeB = graph.addNode('b'); let nodeC = graph.addNode('c'); let nodeD = graph.addNode('d'); @@ -139,6 +143,7 @@ describe('Graph', () => { let graph = new Graph(); let nodeA = graph.addNode('a'); + graph.setRootNodeId(nodeA); let nodeB = graph.addNode('b'); let nodeC = graph.addNode('c'); let nodeD = graph.addNode('d'); @@ -267,6 +272,7 @@ describe('Graph', () => { it("replaceNodeIdsConnectedTo should update a node's downstream nodes", () => { let graph = new Graph(); let nodeA = graph.addNode('a'); + graph.setRootNodeId(nodeA); let nodeB = graph.addNode('b'); let nodeC = graph.addNode('c'); graph.addEdge(nodeA, nodeB); From 1b8d4b6012f6b35787c1b0ccdf32c41f7c559df8 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Wed, 8 Sep 2021 19:03:30 -0400 Subject: [PATCH 47/73] WIP change bundleRoot and Bundles structures, and change dynamic and url import test case --- .../bundlers/default/src/DefaultBundler.js | 87 +++++++++++++++---- .../core/integration-tests/test/javascript.js | 15 ++-- 2 files changed, 81 insertions(+), 21 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 8078dad0212..da70d360894 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -55,8 +55,12 @@ const HTTP_OPTIONS = { }; type AssetId = string; -type BundleRoot = Asset; +type BundleRoot = {| + asset: Asset, + bundleBehavior: BundleBehavior | null, +|}; export type Bundle = {| + entryAsset?: Asset, assets: Set, internalizedAssetIds: Array, bundleBehavior?: ?BundleBehavior, @@ -218,7 +222,6 @@ function decorateLegacyGraph( // ); } } - /** * TODO: Create all bundles, bundlegroups, without adding anything to them * Draw connections to bundles @@ -238,7 +241,10 @@ function createIdealGraph( ): IdealGraph { // Asset to the bundle it's an entry of let bundleRoots: Map = new Map(); - let bundles: Map = new Map(); + let bundles: DefaultMap< + string, + Map, + > = new DefaultMap(() => new Map()); let dependencyBundleGraph: DependencyBundleGraph = new ContentGraph(); let assetReference: DefaultMap< Asset, @@ -284,7 +290,15 @@ function createIdealGraph( needsStableName: dependency.isEntry, }); let nodeId = bundleGraph.addNode(bundle); - bundles.set(asset.id, nodeId); + console.log( + 'CREATING BUNDLE FOR ENTRY', + asset.filePath.split('/').pop(), + 'WITH NODE ID', + nodeId, + ); + + bundles.get(asset.id).set(dependency.bundleBehavior ?? null, bundle); + //bundles.set(getAssetKey(asset, dependency), nodeId); bundleRoots.set(asset, [nodeId, nodeId]); asyncBundleRootGraph.addEdge( rootNodeId, @@ -343,7 +357,9 @@ function createIdealGraph( childAsset.bundleBehavior === 'isolated' ) { // TODO: needsStableName if bundle exists here - let bundleId = bundles.get(childAsset.id); + let bundleId = bundles + .get(childAsset.id) + .get(dependency.bundleBehavior ?? null); let bundle; if (bundleId == null) { bundle = createBundle({ @@ -358,9 +374,18 @@ function createIdealGraph( dependency.bundleBehavior ?? childAsset.bundleBehavior, }); bundleId = bundleGraph.addNode(bundle); - bundles.set(childAsset.id, bundleId); + bundles + .get(childAsset.id) + .set(dependency.bundleBehavior ?? null, bundle); + //bundles.set(getAssetKey(childAsset, dependency), bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); bundleGroupBundleIds.push(bundleId); + console.log( + 'CREATING BUNDLE FOR ASYNC', + childAsset.filePath.split('/').pop(), + 'WITH NODE ID', + bundleId, + ); } else { bundle = nullthrows(bundleGraph.getNode(bundleId)); } @@ -423,8 +448,18 @@ function createIdealGraph( target: bundleGroup.target, needsStableName: dependency.bundleBehavior === 'inline', }); + let bundleId = bundleGraph.addNode(bundle); - bundles.set(childAsset.id, bundleId); + console.log( + 'CREATING BUNDLE FOR TYPE CHANGE', + childAsset.filePath.split('/').pop(), + 'WITH NODE ID', + bundleId, + ); + bundles + .get(childAsset.id) + .set(dependency.bundleBehavior ?? null, bundle); + //bundles.set(getAssetKey(childAsset, dependency), bundleId); bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); dependencyBundleGraph.addEdge( @@ -630,14 +665,18 @@ function createIdealGraph( for (let asset of assets) { // Find bundle entries reachable from the asset. - let reachable: Array = getReachableBundleRoots( + let reachable: Array = getReachableBundleRoots( asset, reachableRoots, - ).reverse(); + bundles, + ).reverse(); //must dededuplicate in order // Filter out bundles when the asset is reachable in every parent bundle. // (Only keep a bundle if all of the others are not descendents of it) reachable = reachable.filter(b => { + if (b.bundleBehavior === 'isolated') { + return true; + } let ancestry = ancestorAssets.get(b)?.get(asset); if (ancestry === undefined) { return true; @@ -675,7 +714,11 @@ function createIdealGraph( ) { // If the asset is a bundle root, add the bundle to every other reachable bundle group. if (!bundles.has(asset.id)) { - bundles.set(asset.id, rootBundle[0]); + bundles + .get(asset.id) + .set(null, nullthrows(bundleGraph.getNode(rootBundle[0]))); + //bundles.set(asset.id, rootBundle[0]); + //bundles.set(getAssetKey(asset, ""), rootBundle[0]); } for (let reachableAsset of reachable) { @@ -696,7 +739,7 @@ function createIdealGraph( // TODO: is this correct? let willInternalizeRoots = reachableAsync.filter( b => - !getReachableBundleRoots(asset, reachableRoots).every( + !getReachableBundleRoots(asset, reachableRoots, bundles).every( a => !(a === b || reachableBundles.get(a).has(b)), ), ); @@ -738,7 +781,14 @@ function createIdealGraph( }); bundle.sourceBundles = sourceBundles; bundleId = bundleGraph.addNode(bundle); - bundles.set(key, bundleId); + console.log( + 'CREATING BUNDLE FOR SHARED', + asset.filePath.split('/').pop(), + 'WITH NODE ID', + bundleId, + ); + bundles.get(key).set(null, bundle); + //bundles.set(key, bundleId); } else { bundle = nullthrows(bundleGraph.getNode(bundleId)); } @@ -771,13 +821,19 @@ function createIdealGraph( let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); for (let siblingId of bundleGraph.getNodeIdsConnectedFrom(entryBundleId)) { let sibling = nullthrows(bundleGraph.getNode(siblingId)); - if (sibling.type !== entryBundle.type) { + if ( + sibling.type !== entryBundle.type || + sibling.bundleBehavior === 'inline' || + sibling.bundleBehavior === 'isolated' || + sibling.env.isIsolated() + ) { continue; } for (let asset of sibling.assets) { entryBundle.assets.add(asset); entryBundle.size += asset.stats.size; } + console.log('1 REMOVING EDGE FROM', entryBundleId, 'TO', siblingId); bundleGraph.removeEdge(entryBundleId, siblingId); reachableAsyncRoots.get(siblingId).delete(entryAsset); if (sibling.sourceBundles.length > 1) { @@ -978,8 +1034,9 @@ function ancestryIntersect( } } -function getReachableBundleRoots(asset, graph): Array { +function getReachableBundleRoots(asset, graph, bundles): Array { return graph .getNodeIdsConnectedTo(graph.getNodeIdByContentKey(asset.id)) - .map(nodeId => nullthrows(graph.getNode(nodeId))); + .map(nodeId => nullthrows(graph.getNode(nodeId))) + .flatMap(asset => [...bundles.get(asset.id).values()]); } diff --git a/packages/core/integration-tests/test/javascript.js b/packages/core/integration-tests/test/javascript.js index ff24ec930b8..9bbbd7b4a7d 100644 --- a/packages/core/integration-tests/test/javascript.js +++ b/packages/core/integration-tests/test/javascript.js @@ -1015,24 +1015,27 @@ describe('javascript', function() { ); assertBundles(b, [ - { - assets: ['dedicated-worker.js', 'index.js'], - }, { name: 'index.js', assets: ['index.js', 'bundle-url.js', 'get-worker-url.js'], }, { - assets: ['shared-worker.js', 'index.js'], + assets: ['dedicated-worker.js'], + }, + { + assets: ['index.js'], + }, + { + assets: ['shared-worker.js'], }, ]); let dedicated, shared; b.traverseBundles((bundle, ctx, traversal) => { - if (bundle.getMainEntry().filePath.endsWith('shared-worker.js')) { + if (bundle.getMainEntry()?.filePath.endsWith('shared-worker.js')) { shared = bundle; } else if ( - bundle.getMainEntry().filePath.endsWith('dedicated-worker.js') + bundle.getMainEntry()?.filePath.endsWith('dedicated-worker.js') ) { dedicated = bundle; } From 75de592a29839cb5231982fd4e62cacddbea977a Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Mon, 13 Sep 2021 17:05:12 -0400 Subject: [PATCH 48/73] Revert default bundler back 1 --- .../bundlers/default/src/DefaultBundler.js | 87 ++++--------------- 1 file changed, 15 insertions(+), 72 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index da70d360894..8078dad0212 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -55,12 +55,8 @@ const HTTP_OPTIONS = { }; type AssetId = string; -type BundleRoot = {| - asset: Asset, - bundleBehavior: BundleBehavior | null, -|}; +type BundleRoot = Asset; export type Bundle = {| - entryAsset?: Asset, assets: Set, internalizedAssetIds: Array, bundleBehavior?: ?BundleBehavior, @@ -222,6 +218,7 @@ function decorateLegacyGraph( // ); } } + /** * TODO: Create all bundles, bundlegroups, without adding anything to them * Draw connections to bundles @@ -241,10 +238,7 @@ function createIdealGraph( ): IdealGraph { // Asset to the bundle it's an entry of let bundleRoots: Map = new Map(); - let bundles: DefaultMap< - string, - Map, - > = new DefaultMap(() => new Map()); + let bundles: Map = new Map(); let dependencyBundleGraph: DependencyBundleGraph = new ContentGraph(); let assetReference: DefaultMap< Asset, @@ -290,15 +284,7 @@ function createIdealGraph( needsStableName: dependency.isEntry, }); let nodeId = bundleGraph.addNode(bundle); - console.log( - 'CREATING BUNDLE FOR ENTRY', - asset.filePath.split('/').pop(), - 'WITH NODE ID', - nodeId, - ); - - bundles.get(asset.id).set(dependency.bundleBehavior ?? null, bundle); - //bundles.set(getAssetKey(asset, dependency), nodeId); + bundles.set(asset.id, nodeId); bundleRoots.set(asset, [nodeId, nodeId]); asyncBundleRootGraph.addEdge( rootNodeId, @@ -357,9 +343,7 @@ function createIdealGraph( childAsset.bundleBehavior === 'isolated' ) { // TODO: needsStableName if bundle exists here - let bundleId = bundles - .get(childAsset.id) - .get(dependency.bundleBehavior ?? null); + let bundleId = bundles.get(childAsset.id); let bundle; if (bundleId == null) { bundle = createBundle({ @@ -374,18 +358,9 @@ function createIdealGraph( dependency.bundleBehavior ?? childAsset.bundleBehavior, }); bundleId = bundleGraph.addNode(bundle); - bundles - .get(childAsset.id) - .set(dependency.bundleBehavior ?? null, bundle); - //bundles.set(getAssetKey(childAsset, dependency), bundleId); + bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); bundleGroupBundleIds.push(bundleId); - console.log( - 'CREATING BUNDLE FOR ASYNC', - childAsset.filePath.split('/').pop(), - 'WITH NODE ID', - bundleId, - ); } else { bundle = nullthrows(bundleGraph.getNode(bundleId)); } @@ -448,18 +423,8 @@ function createIdealGraph( target: bundleGroup.target, needsStableName: dependency.bundleBehavior === 'inline', }); - let bundleId = bundleGraph.addNode(bundle); - console.log( - 'CREATING BUNDLE FOR TYPE CHANGE', - childAsset.filePath.split('/').pop(), - 'WITH NODE ID', - bundleId, - ); - bundles - .get(childAsset.id) - .set(dependency.bundleBehavior ?? null, bundle); - //bundles.set(getAssetKey(childAsset, dependency), bundleId); + bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); dependencyBundleGraph.addEdge( @@ -665,18 +630,14 @@ function createIdealGraph( for (let asset of assets) { // Find bundle entries reachable from the asset. - let reachable: Array = getReachableBundleRoots( + let reachable: Array = getReachableBundleRoots( asset, reachableRoots, - bundles, - ).reverse(); //must dededuplicate in order + ).reverse(); // Filter out bundles when the asset is reachable in every parent bundle. // (Only keep a bundle if all of the others are not descendents of it) reachable = reachable.filter(b => { - if (b.bundleBehavior === 'isolated') { - return true; - } let ancestry = ancestorAssets.get(b)?.get(asset); if (ancestry === undefined) { return true; @@ -714,11 +675,7 @@ function createIdealGraph( ) { // If the asset is a bundle root, add the bundle to every other reachable bundle group. if (!bundles.has(asset.id)) { - bundles - .get(asset.id) - .set(null, nullthrows(bundleGraph.getNode(rootBundle[0]))); - //bundles.set(asset.id, rootBundle[0]); - //bundles.set(getAssetKey(asset, ""), rootBundle[0]); + bundles.set(asset.id, rootBundle[0]); } for (let reachableAsset of reachable) { @@ -739,7 +696,7 @@ function createIdealGraph( // TODO: is this correct? let willInternalizeRoots = reachableAsync.filter( b => - !getReachableBundleRoots(asset, reachableRoots, bundles).every( + !getReachableBundleRoots(asset, reachableRoots).every( a => !(a === b || reachableBundles.get(a).has(b)), ), ); @@ -781,14 +738,7 @@ function createIdealGraph( }); bundle.sourceBundles = sourceBundles; bundleId = bundleGraph.addNode(bundle); - console.log( - 'CREATING BUNDLE FOR SHARED', - asset.filePath.split('/').pop(), - 'WITH NODE ID', - bundleId, - ); - bundles.get(key).set(null, bundle); - //bundles.set(key, bundleId); + bundles.set(key, bundleId); } else { bundle = nullthrows(bundleGraph.getNode(bundleId)); } @@ -821,19 +771,13 @@ function createIdealGraph( let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); for (let siblingId of bundleGraph.getNodeIdsConnectedFrom(entryBundleId)) { let sibling = nullthrows(bundleGraph.getNode(siblingId)); - if ( - sibling.type !== entryBundle.type || - sibling.bundleBehavior === 'inline' || - sibling.bundleBehavior === 'isolated' || - sibling.env.isIsolated() - ) { + if (sibling.type !== entryBundle.type) { continue; } for (let asset of sibling.assets) { entryBundle.assets.add(asset); entryBundle.size += asset.stats.size; } - console.log('1 REMOVING EDGE FROM', entryBundleId, 'TO', siblingId); bundleGraph.removeEdge(entryBundleId, siblingId); reachableAsyncRoots.get(siblingId).delete(entryAsset); if (sibling.sourceBundles.length > 1) { @@ -1034,9 +978,8 @@ function ancestryIntersect( } } -function getReachableBundleRoots(asset, graph, bundles): Array { +function getReachableBundleRoots(asset, graph): Array { return graph .getNodeIdsConnectedTo(graph.getNodeIdByContentKey(asset.id)) - .map(nodeId => nullthrows(graph.getNode(nodeId))) - .flatMap(asset => [...bundles.get(asset.id).values()]); + .map(nodeId => nullthrows(graph.getNode(nodeId))); } From 617bd8f138bdf925e415f9215b1d465e099f72a9 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Mon, 20 Sep 2021 14:50:55 -0400 Subject: [PATCH 49/73] port over default bundler changes remove reachable async roots --- .../bundlers/default/src/DefaultBundler.js | 81 +++-- packages/core/test-utils/src/utils.js | 4 +- yarn.lock | 285 +++++++++++++++--- 3 files changed, 283 insertions(+), 87 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 8078dad0212..db07554e14e 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -336,7 +336,6 @@ function createIdealGraph( invariant(assets.length === 1); let childAsset = assets[0]; - // Create a new bundle as well as a new bundle group if the dependency is async. if ( dependency.priority === 'lazy' || @@ -392,21 +391,6 @@ function createIdealGraph( break; } reachableBundles.get(stackAsset).add(childAsset); - - if (i === stack.length - 1) { - //Add child and connection from parent to child bundleRoot - let childNodeId = asyncBundleRootGraph.addNodeByContentKeyIfNeeded( - childAsset.id, - childAsset, - ); - - let parentNodeId = asyncBundleRootGraph.addNodeByContentKeyIfNeeded( - stackAsset.id, - stackAsset, - ); - - asyncBundleRootGraph.addEdge(parentNodeId, childNodeId); - } } return node; } @@ -459,12 +443,13 @@ function createIdealGraph( // Step 2: Determine reachability for every asset from each bundle root. // This is later used to determine which bundles to place each asset in. - let reachableRoots: ContentGraph = new ContentGraph(); - - let reachableAsyncRoots: DefaultMap> = new DefaultMap( - () => new Set(), - ); + for (let [root] of bundleRoots) { + if (!entries.has(root)) { + asyncBundleRootGraph.addNodeByContentKey(root.id, root); + } + } + let reachableRoots: ContentGraph = new ContentGraph(); for (let [root] of bundleRoots) { let rootNodeId = reachableRoots.addNodeByContentKeyIfNeeded(root.id, root); assetGraph.traverse((node, isAsync, actions) => { @@ -483,11 +468,11 @@ function createIdealGraph( invariant(assets.length === 1); let bundleRoot = assets[0]; - invariant(bundleRoots.has(bundleRoot)); if (dependency.specifierType !== 'url') { - reachableAsyncRoots - .get(nullthrows(bundles.get(bundleRoot.id))) - .add(root); + asyncBundleRootGraph.addEdge( + asyncBundleRootGraph.getNodeIdByContentKey(root.id), + asyncBundleRootGraph.getNodeIdByContentKey(bundleRoot.id), + ); } } actions.skipChildren(); @@ -688,11 +673,17 @@ function createIdealGraph( } // reachableAsyncRoots = all bundleNodeId => all BundleRoots that require it asynchronously // reachableAsync = for one bundleRoot => all - let reachableAsync = [ - ...(reachableAsyncRoots.has(rootBundle[0]) - ? reachableAsyncRoots.get(rootBundle[0]) - : []), - ]; + let reachableAsync = asyncBundleRootGraph + .getNodeIdsConnectedTo( + asyncBundleRootGraph.getNodeIdByContentKey(asset.id), + ) + .map(id => nullthrows(asyncBundleRootGraph.getNode(id))) + .filter(node => node !== 'root') + .map(node => { + // TODO: Improve. Currently for flow + invariant(node !== 'root'); + return node; + }); // TODO: is this correct? let willInternalizeRoots = reachableAsync.filter( b => @@ -779,7 +770,7 @@ function createIdealGraph( entryBundle.size += asset.stats.size; } bundleGraph.removeEdge(entryBundleId, siblingId); - reachableAsyncRoots.get(siblingId).delete(entryAsset); + // reachableAsyncRoots.get(siblingId).delete(entryAsset); if (sibling.sourceBundles.length > 1) { let entryBundleIndex = sibling.sourceBundles.indexOf(entryBundleId); invariant(entryBundleIndex >= 0); @@ -798,20 +789,20 @@ function createIdealGraph( } } - for (let [asyncBundleRoot, dependentRoots] of reachableAsyncRoots) { - if (dependentRoots.size === 0) { - //TODO make get entry asset function - let bundleNode = bundleGraph.getNode(asyncBundleRoot); - - if (bundleNode?.assets) { - let [entryAsset] = [...bundleNode.assets]; - if (entries.has(entryAsset)) { - continue; - } - } - bundleGraph.removeNode(asyncBundleRoot); - } - } + // for (let [asyncBundleRoot, dependentRoots] of reachableAsyncRoots) { + // if (dependentRoots.size === 0) { + // //TODO make get entry asset function + // let bundleNode = bundleGraph.getNode(asyncBundleRoot); + + // if (bundleNode?.assets) { + // let [entryAsset] = [...bundleNode.assets]; + // if (entries.has(entryAsset)) { + // continue; + // } + // } + // bundleGraph.removeNode(asyncBundleRoot); + // } + // } // $FlowFixMe dumpGraphToGraphViz(bundleGraph, 'IdealBundleGraph'); diff --git a/packages/core/test-utils/src/utils.js b/packages/core/test-utils/src/utils.js index 9fb406f2c5b..75038229281 100644 --- a/packages/core/test-utils/src/utils.js +++ b/packages/core/test-utils/src/utils.js @@ -516,8 +516,8 @@ export function assertBundles( }); }); - // console.log('Actual Bundles are', actualBundles); - // console.log('Expected Bundles are', expectedBundles); + console.log('Actual Bundles are', actualBundles); + console.log('Expected Bundles are', expectedBundles); for (let bundle of expectedBundles) { if (!Array.isArray(bundle.assets)) { diff --git a/yarn.lock b/yarn.lock index 8b9b818743e..26101c9b39e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2004,14 +2004,14 @@ "@octokit/types" "^6.11.0" "@octokit/plugin-request-log@^1.0.0", "@octokit/plugin-request-log@^1.0.2": - version "1.0.3" - resolved "https://packages.atlassian.com/api/npm/npm-remote/@octokit/plugin-request-log/-/plugin-request-log-1.0.3.tgz#70a62be213e1edc04bb8897ee48c311482f9700d" - integrity sha1-cKYr4hPh7cBLuIl+5IwxFIL5cA0= + version "1.0.4" + resolved "https://registry.yarnpkg.com/@octokit/plugin-request-log/-/plugin-request-log-1.0.4.tgz#5e50ed7083a613816b1e4a28aeec5fb7f1462e85" + integrity sha512-mLUsMkgP7K/cnFEw07kWqXGF5LKrOkD+lhCrKvPHXWDywAwuDUeDwWBpc69XK3pNX0uKiVt8g5z96PJ6z9xCFA== "@octokit/plugin-rest-endpoint-methods@3.17.0": version "3.17.0" - resolved "https://packages.atlassian.com/api/npm/npm-remote/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-3.17.0.tgz#d8ba04eb883849dd98666c55bf49d8c9fe7be055" - integrity sha1-2LoE64g4Sd2YZmxVv0nYyf574FU= + resolved "https://registry.yarnpkg.com/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-3.17.0.tgz#d8ba04eb883849dd98666c55bf49d8c9fe7be055" + integrity sha512-NFV3vq7GgoO2TrkyBRUOwflkfTYkFKS0tLAPym7RNpkwLCttqShaEGjthOsPEEL+7LFcYv3mU24+F2yVd3npmg== dependencies: "@octokit/types" "^4.1.6" deprecation "^2.3.1" @@ -2075,8 +2075,8 @@ "@octokit/rest@^17.1.3": version "17.11.2" - resolved "https://packages.atlassian.com/api/npm/npm-remote/@octokit/rest/-/rest-17.11.2.tgz#f3dbd46f9f06361c646230fd0ef8598e59183ead" - integrity sha1-89vUb58GNhxkYjD9DvhZjlkYPq0= + resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-17.11.2.tgz#f3dbd46f9f06361c646230fd0ef8598e59183ead" + integrity sha512-4jTmn8WossTUaLfNDfXk4fVJgbz5JgZE8eCs4BvIb52lvIH8rpVMD1fgRCrHbSd6LRPE5JFZSfAEtszrOq3ZFQ== dependencies: "@octokit/core" "^2.4.3" "@octokit/plugin-paginate-rest" "^2.2.0" @@ -2102,15 +2102,15 @@ "@octokit/types@^4.1.6": version "4.1.10" - resolved "https://packages.atlassian.com/api/npm/npm-remote/@octokit/types/-/types-4.1.10.tgz#e4029c11e2cc1335051775bc1600e7e740e4aca4" - integrity sha1-5AKcEeLMEzUFF3W8FgDn50DkrKQ= + resolved "https://registry.yarnpkg.com/@octokit/types/-/types-4.1.10.tgz#e4029c11e2cc1335051775bc1600e7e740e4aca4" + integrity sha512-/wbFy1cUIE5eICcg0wTKGXMlKSbaAxEr00qaBXzscLXpqhcwgXeS6P8O0pkysBhRfyjkKjJaYrvR1ExMO5eOXQ== dependencies: "@types/node" ">= 8" "@octokit/types@^5.0.0": version "5.5.0" - resolved "https://packages.atlassian.com/api/npm/npm-remote/@octokit/types/-/types-5.5.0.tgz#e5f06e8db21246ca102aa28444cdb13ae17a139b" - integrity sha1-5fBujbISRsoQKqKERM2xOuF6E5s= + resolved "https://registry.yarnpkg.com/@octokit/types/-/types-5.5.0.tgz#e5f06e8db21246ca102aa28444cdb13ae17a139b" + integrity sha512-UZ1pErDue6bZNjYOotCNveTXArOMZQFG6hKJfOnGnulVCMcVVi7YIIuuR4WfBhjo7zgpmzn/BkPDnUXtNx+PcQ== dependencies: "@types/node" ">= 8" @@ -2291,6 +2291,11 @@ resolved "https://registry.yarnpkg.com/@trysound/sax/-/sax-0.1.1.tgz#3348564048e7a2d7398c935d466c0414ebb6a669" integrity sha512-Z6DoceYb/1xSg5+e+ZlPZ9v0N16ZvZ+wYMraFue4HYrE4ttONKtsvruIRf6t9TBR0YvSOfi1hUU0fJfBLCDYow== +"@trysound/sax@0.2.0": + version "0.2.0" + resolved "https://registry.yarnpkg.com/@trysound/sax/-/sax-0.2.0.tgz#cccaab758af56761eb7bf37af6f03f326dd798ad" + integrity sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA== + "@types/color-name@^1.1.1": version "1.1.1" resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0" @@ -3595,7 +3600,7 @@ charenc@~0.0.1: checkup@^1.3.0: version "1.3.0" - resolved "https://packages.atlassian.com/api/npm/npm-remote/checkup/-/checkup-1.3.0.tgz#d3800276fea5d0f247ffc951be78c8b02f8e0d76" + resolved "https://registry.yarnpkg.com/checkup/-/checkup-1.3.0.tgz#d3800276fea5d0f247ffc951be78c8b02f8e0d76" integrity sha1-04ACdv6l0PJH/8lRvnjIsC+ODXY= chokidar@3.5.1: @@ -3726,8 +3731,8 @@ cli-width@^3.0.0: clipanion@^2.6.2: version "2.6.2" - resolved "https://packages.atlassian.com/api/npm/npm-remote/clipanion/-/clipanion-2.6.2.tgz#820e7440812052442455b248f927b187ed732f71" - integrity sha1-gg50QIEgUkQkVbJI+Sexh+1zL3E= + resolved "https://registry.yarnpkg.com/clipanion/-/clipanion-2.6.2.tgz#820e7440812052442455b248f927b187ed732f71" + integrity sha512-0tOHJNMF9+4R3qcbBL+4IxLErpaYSYvzs10aXuECDbZdJOuJHdagJMAqvLdeaUQTI/o2uSCDRpet6ywDiKOAYw== cliui@^3.2.0: version "3.2.0" @@ -3897,11 +3902,21 @@ colord@^2.0.1: resolved "https://registry.yarnpkg.com/colord/-/colord-2.0.1.tgz#1e7fb1f9fa1cf74f42c58cb9c20320bab8435aa0" integrity sha512-vm5YpaWamD0Ov6TSG0GGmUIwstrWcfKQV/h2CmbR7PbNu41+qdB5PW9lpzhjedrpm08uuYvcXi0Oel1RLZIJuA== +colord@^2.6: + version "2.7.0" + resolved "https://registry.yarnpkg.com/colord/-/colord-2.7.0.tgz#706ea36fe0cd651b585eb142fe64b6480185270e" + integrity sha512-pZJBqsHz+pYyw3zpX6ZRXWoCHM1/cvFikY9TV8G3zcejCaKE0lhankoj8iScyrrePA8C7yJ5FStfA9zbcOnw7Q== + colorette@^1.2.2, colorette@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.3.0.tgz#ff45d2f0edb244069d3b772adeb04fed38d0a0af" integrity sha512-ecORCqbSFP7Wm8Y6lyqMJjexBQqXSF7SSeaTyGGphogUjBlFP9m9o08wy86HL2uB7fMTxtOUzLMk7ogKcxMg1w== +colorette@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.4.0.tgz#5190fbb87276259a86ad700bff2c6d6faa3fca40" + integrity sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g== + columnify@^1.5.4: version "1.5.4" resolved "https://registry.yarnpkg.com/columnify/-/columnify-1.5.4.tgz#4737ddf1c7b69a8a7c340570782e947eec8e78bb" @@ -4225,6 +4240,17 @@ cosmiconfig@^7.0.0: path-type "^4.0.0" yaml "^1.10.0" +cosmiconfig@^7.0.1: + version "7.0.1" + resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-7.0.1.tgz#714d756522cace867867ccb4474c5d01bbae5d6d" + integrity sha512-a1YWNUV2HwGimB7dU2s1wUMurNKjpx60HxBB6xUM8Re+2s1g1IIfJvFR0/iCF+XHdE0GMTKTuLR32UQff4TEyQ== + dependencies: + "@types/parse-json" "^4.0.0" + import-fresh "^3.2.1" + parse-json "^5.0.0" + path-type "^4.0.0" + yaml "^1.10.0" + create-ecdh@^4.0.0: version "4.0.3" resolved "https://registry.yarnpkg.com/create-ecdh/-/create-ecdh-4.0.3.tgz#c9111b6f33045c4697f144787f9254cdc77c45ff" @@ -4467,12 +4493,47 @@ cssnano-preset-default@^5.1.2: postcss-svgo "^5.0.2" postcss-unique-selectors "^5.0.1" +cssnano-preset-default@^5.1.4: + version "5.1.4" + resolved "https://registry.yarnpkg.com/cssnano-preset-default/-/cssnano-preset-default-5.1.4.tgz#359943bf00c5c8e05489f12dd25f3006f2c1cbd2" + integrity sha512-sPpQNDQBI3R/QsYxQvfB4mXeEcWuw0wGtKtmS5eg8wudyStYMgKOQT39G07EbW1LB56AOYrinRS9f0ig4Y3MhQ== + dependencies: + css-declaration-sorter "^6.0.3" + cssnano-utils "^2.0.1" + postcss-calc "^8.0.0" + postcss-colormin "^5.2.0" + postcss-convert-values "^5.0.1" + postcss-discard-comments "^5.0.1" + postcss-discard-duplicates "^5.0.1" + postcss-discard-empty "^5.0.1" + postcss-discard-overridden "^5.0.1" + postcss-merge-longhand "^5.0.2" + postcss-merge-rules "^5.0.2" + postcss-minify-font-values "^5.0.1" + postcss-minify-gradients "^5.0.2" + postcss-minify-params "^5.0.1" + postcss-minify-selectors "^5.1.0" + postcss-normalize-charset "^5.0.1" + postcss-normalize-display-values "^5.0.1" + postcss-normalize-positions "^5.0.1" + postcss-normalize-repeat-style "^5.0.1" + postcss-normalize-string "^5.0.1" + postcss-normalize-timing-functions "^5.0.1" + postcss-normalize-unicode "^5.0.1" + postcss-normalize-url "^5.0.2" + postcss-normalize-whitespace "^5.0.1" + postcss-ordered-values "^5.0.2" + postcss-reduce-initial "^5.0.1" + postcss-reduce-transforms "^5.0.1" + postcss-svgo "^5.0.2" + postcss-unique-selectors "^5.0.1" + cssnano-utils@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/cssnano-utils/-/cssnano-utils-2.0.1.tgz#8660aa2b37ed869d2e2f22918196a9a8b6498ce2" integrity sha512-i8vLRZTnEH9ubIyfdZCAdIdgnHAUeQeByEeQ2I7oTilvP9oHO6RScpeq3GsFUVqeB8uZgOQ9pw8utofNn32hhQ== -cssnano@^5.0.0, cssnano@^5.0.5: +cssnano@^5.0.5: version "5.0.5" resolved "https://registry.yarnpkg.com/cssnano/-/cssnano-5.0.5.tgz#6b8787123bf4cd5a220a2fa6cb5bc036b0854b48" integrity sha512-L2VtPXnq6rmcMC9vkBOP131sZu3ccRQI27ejKZdmQiPDpUlFkUbpXHgKN+cibeO1U4PItxVZp1zTIn5dHsXoyg== @@ -4481,6 +4542,16 @@ cssnano@^5.0.0, cssnano@^5.0.5: cssnano-preset-default "^5.1.2" is-resolvable "^1.1.0" +cssnano@^5.0.8: + version "5.0.8" + resolved "https://registry.yarnpkg.com/cssnano/-/cssnano-5.0.8.tgz#39ad166256980fcc64faa08c9bb18bb5789ecfa9" + integrity sha512-Lda7geZU0Yu+RZi2SGpjYuQz4HI4/1Y+BhdD0jL7NXAQ5larCzVn+PUGuZbDMYz904AXXCOgO5L1teSvgu7aFg== + dependencies: + cssnano-preset-default "^5.1.4" + is-resolvable "^1.1.0" + lilconfig "^2.0.3" + yaml "^1.10.2" + csso@^4.0.2, csso@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/csso/-/csso-4.2.0.tgz#ea3a561346e8dc9f546d6febedd50187cf389529" @@ -4968,6 +5039,13 @@ domhandler@^4.0.0, domhandler@^4.2.0: dependencies: domelementtype "^2.2.0" +domhandler@^4.2.2: + version "4.2.2" + resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-4.2.2.tgz#e825d721d19a86b8c201a35264e226c678ee755f" + integrity sha512-PzE9aBMsdZO8TK4BnuJwH0QT41wgMbRzuZrHUcpYncEjmQazq8QEaBWgLG7ZyC/DAZKEgglpIA6j4Qn/HmxS3w== + dependencies: + domelementtype "^2.2.0" + domutils@^1.5.1, domutils@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.7.0.tgz#56ea341e834e06e6748af7a1cb25da67ea9f8c2a" @@ -4985,6 +5063,15 @@ domutils@^2.4.2, domutils@^2.5.2, domutils@^2.6.0: domelementtype "^2.2.0" domhandler "^4.2.0" +domutils@^2.8.0: + version "2.8.0" + resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.8.0.tgz#4437def5db6e2d1f5d6ee859bd95ca7d02048135" + integrity sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A== + dependencies: + dom-serializer "^1.0.1" + domelementtype "^2.2.0" + domhandler "^4.2.0" + dot-prop@^4.2.0: version "4.2.1" resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-4.2.1.tgz#45884194a71fc2cda71cbb4bceb3a4dd2f433ba4" @@ -5147,6 +5234,11 @@ entities@^2.0.0: resolved "https://registry.yarnpkg.com/entities/-/entities-2.0.0.tgz#68d6084cab1b079767540d80e56a39b423e4abf4" integrity sha512-D9f7V0JSRwIxlRI2mjMqufDrRDnx8p+eEOz7aUM9SuvF8gsBzra0/6tbjl1m8eQHrZlYj6PxqE00hZ1SAIKPLw== +entities@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/entities/-/entities-3.0.1.tgz#2b887ca62585e96db3903482d336c1006c3001d4" + integrity sha512-WiyBqoomrwMdFG1e0kqvASYfnlb0lp8M5o5Fw2OFq1hNZxxcNk8Ik0Xm7LxzBhuidnZB/UtBqVCgUz3kBOP51Q== + env-paths@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-2.2.0.tgz#cdca557dc009152917d6166e2febe1f039685e43" @@ -6934,18 +7026,19 @@ html-void-elements@^1.0.0: integrity sha512-uE/TxKuyNIcx44cIWnjr/rfIATDH7ZaOMmstu0CwhFG1Dunhlp4OC6/NMbhiwoq5BpW0ubi303qnEk/PZj614w== htmlnano@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/htmlnano/-/htmlnano-1.0.0.tgz#b1119617d514051cefb3622a337baecd14edd08b" - integrity sha512-1lmF8MK6UZEEOtSusKn4/NJhJxw8vMb6lqie6Vmd80gxykPVstCqYGr4EYNlfkbLNQomQW++qHmnBDAe1XEytw== + version "1.1.1" + resolved "https://registry.yarnpkg.com/htmlnano/-/htmlnano-1.1.1.tgz#aea50e1d7ac156370ea766d4cd75f2d3d1a953cc" + integrity sha512-diMNyqTPx4uGwlxrTs0beZCy8L/GxGIFGHWv20OYhthLcdYkDOP/d4Ja5MbGgVJZMakZUM21KpMk5qWZrBGSdw== dependencies: - cssnano "^5.0.0" - postcss "^8.2.9" - posthtml "^0.15.2" + cosmiconfig "^7.0.1" + cssnano "^5.0.8" + postcss "^8.3.6" + posthtml "^0.16.5" purgecss "^4.0.0" relateurl "^0.2.7" - srcset "^3.0.0" - svgo "^2.3.0" - terser "^5.6.1" + srcset "^4.0.0" + svgo "^2.6.1" + terser "^5.8.0" timsort "^0.3.0" uncss "^0.17.3" @@ -6969,6 +7062,16 @@ htmlparser2@^6.0.0: domutils "^2.5.2" entities "^2.0.0" +htmlparser2@^7.1.1: + version "7.1.2" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-7.1.2.tgz#587923d38f03bc89e03076e00cba2c7473f37f7c" + integrity sha512-d6cqsbJba2nRdg8WW2okyD4ceonFHn9jLFxhwlNcLhQWcFPdxXeJulgOLjLKtAK9T6ahd+GQNZwG9fjmGW7lyg== + dependencies: + domelementtype "^2.0.1" + domhandler "^4.2.2" + domutils "^2.8.0" + entities "^3.0.1" + htmlparser2@~3.9.2: version "3.9.2" resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.9.2.tgz#1bdf87acca0f3f9e53fa4fcceb0f4b4cbb00b338" @@ -7812,7 +7915,7 @@ jake@^10.6.1: jju@^1.4.0: version "1.4.0" - resolved "https://packages.atlassian.com/api/npm/npm-remote/jju/-/jju-1.4.0.tgz#a3abe2718af241a2b2904f84a625970f389ae32a" + resolved "https://registry.yarnpkg.com/jju/-/jju-1.4.0.tgz#a3abe2718af241a2b2904f84a625970f389ae32a" integrity sha1-o6vicYryQaKykE+EpiWXDzia4yo= js-stringify@^1.0.2: @@ -8150,6 +8253,11 @@ liftoff@^3.1.0: rechoir "^0.6.2" resolve "^1.1.7" +lilconfig@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-2.0.3.tgz#68f3005e921dafbd2a2afb48379986aa6d2579fd" + integrity sha512-EHKqr/+ZvdKCifpNrJCKxBTgk5XupZA3y/aCPY9mxfgBzmgh93Mt/WqjjQ38oMxXuvDokaKiM3lAgvSH2sjtHg== + lines-and-columns@^1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00" @@ -9199,6 +9307,11 @@ normalize-url@^4.5.0: resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-4.5.0.tgz#453354087e6ca96957bd8f5baf753f5982142129" integrity sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ== +normalize-url@^6.0.1: + version "6.1.0" + resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a" + integrity sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A== + now-and-later@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/now-and-later/-/now-and-later-2.0.1.tgz#8e579c8685764a7cc02cb680380e94f43ccb1f7c" @@ -10119,6 +10232,15 @@ postcss-minify-gradients@^5.0.1: is-color-stop "^1.1.0" postcss-value-parser "^4.1.0" +postcss-minify-gradients@^5.0.2: + version "5.0.2" + resolved "https://registry.yarnpkg.com/postcss-minify-gradients/-/postcss-minify-gradients-5.0.2.tgz#7c175c108f06a5629925d698b3c4cf7bd3864ee5" + integrity sha512-7Do9JP+wqSD6Prittitt2zDLrfzP9pqKs2EcLX7HJYxsxCOwrrcLt4x/ctQTsiOw+/8HYotAoqNkrzItL19SdQ== + dependencies: + colord "^2.6" + cssnano-utils "^2.0.1" + postcss-value-parser "^4.1.0" + postcss-minify-params@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/postcss-minify-params/-/postcss-minify-params-5.0.1.tgz#371153ba164b9d8562842fdcd929c98abd9e5b6c" @@ -10277,6 +10399,15 @@ postcss-normalize-url@^5.0.1: normalize-url "^4.5.0" postcss-value-parser "^4.1.0" +postcss-normalize-url@^5.0.2: + version "5.0.2" + resolved "https://registry.yarnpkg.com/postcss-normalize-url/-/postcss-normalize-url-5.0.2.tgz#ddcdfb7cede1270740cf3e4dfc6008bd96abc763" + integrity sha512-k4jLTPUxREQ5bpajFQZpx8bCF2UrlqOTzP9kEqcEnOfwsRshWs2+oAFIHfDQB8GO2PaUaSE0NlTAYtbluZTlHQ== + dependencies: + is-absolute-url "^3.0.3" + normalize-url "^6.0.1" + postcss-value-parser "^4.1.0" + postcss-normalize-whitespace@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/postcss-normalize-whitespace/-/postcss-normalize-whitespace-5.0.1.tgz#b0b40b5bcac83585ff07ead2daf2dcfbeeef8e9a" @@ -10292,6 +10423,14 @@ postcss-ordered-values@^5.0.1: cssnano-utils "^2.0.1" postcss-value-parser "^4.1.0" +postcss-ordered-values@^5.0.2: + version "5.0.2" + resolved "https://registry.yarnpkg.com/postcss-ordered-values/-/postcss-ordered-values-5.0.2.tgz#1f351426977be00e0f765b3164ad753dac8ed044" + integrity sha512-8AFYDSOYWebJYLyJi3fyjl6CqMEG/UVworjiyK1r573I56kb3e879sCJLGvR3merj+fAdPpVplXKQZv+ey6CgQ== + dependencies: + cssnano-utils "^2.0.1" + postcss-value-parser "^4.1.0" + postcss-reduce-initial@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/postcss-reduce-initial/-/postcss-reduce-initial-5.0.1.tgz#9d6369865b0f6f6f6b165a0ef5dc1a4856c7e946" @@ -10383,7 +10522,7 @@ postcss@^7.0.14, postcss@^7.0.17, postcss@^7.0.32, postcss@^7.0.5, postcss@^7.0. source-map "^0.6.1" supports-color "^6.1.0" -postcss@^8.1.6, postcss@^8.2.1, postcss@^8.2.9, postcss@^8.3.0: +postcss@^8.1.6, postcss@^8.2.1, postcss@^8.3.0: version "8.3.0" resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.0.tgz#b1a713f6172ca427e3f05ef1303de8b65683325f" integrity sha512-+ogXpdAjWGa+fdYY5BQ96V/6tAo+TdSSIMP5huJBIygdWwKtVoB5JWZ7yUd4xZ8r+8Kvvx4nyg/PQ071H4UtcQ== @@ -10392,6 +10531,15 @@ postcss@^8.1.6, postcss@^8.2.1, postcss@^8.2.9, postcss@^8.3.0: nanoid "^3.1.23" source-map-js "^0.6.2" +postcss@^8.3.6: + version "8.3.6" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.6.tgz#2730dd76a97969f37f53b9a6096197be311cc4ea" + integrity sha512-wG1cc/JhRgdqB6WHEuyLTedf3KIRuD0hG6ldkFEZNCjRxiC+3i6kkWUUbiJQayP28iwG35cEmAbe98585BYV0A== + dependencies: + colorette "^1.2.2" + nanoid "^3.1.23" + source-map-js "^0.6.2" + posthtml-expressions@^1.7.1: version "1.7.1" resolved "https://registry.yarnpkg.com/posthtml-expressions/-/posthtml-expressions-1.7.1.tgz#b62b47a9c2cc8216cbbb07d2e9e7b293f3c1f5bc" @@ -10415,6 +10563,13 @@ posthtml-obfuscate@^0.1.5: resolved "https://registry.yarnpkg.com/posthtml-obfuscate/-/posthtml-obfuscate-0.1.5.tgz#d65f6642c96349002d33a2208845b2f7517d0fc8" integrity sha1-1l9mQsljSQAtM6IgiEWy91F9D8g= +posthtml-parser@^0.10.0: + version "0.10.1" + resolved "https://registry.yarnpkg.com/posthtml-parser/-/posthtml-parser-0.10.1.tgz#63c41931a9339cc2c32aba14f06286d98f107abf" + integrity sha512-i7w2QEHqiGtsvNNPty0Mt/+ERch7wkgnFh3+JnBI2VgDbGlBqKW9eDVd3ENUhE1ujGFe3e3E/odf7eKhvLUyDg== + dependencies: + htmlparser2 "^7.1.1" + posthtml-parser@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/posthtml-parser/-/posthtml-parser-0.6.0.tgz#52488cdb4fa591c3102de73197c471859ee0be63" @@ -10448,7 +10603,14 @@ posthtml-render@^2.0.6: dependencies: is-json "^2.0.1" -posthtml@^0.15.1, posthtml@^0.15.2: +posthtml-render@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/posthtml-render/-/posthtml-render-3.0.0.tgz#97be44931496f495b4f07b99e903cc70ad6a3205" + integrity sha512-z+16RoxK3fUPgwaIgH9NGnK1HKY9XIDpydky5eQGgAFVXTCSezalv9U2jQuNV+Z9qV1fDWNzldcw4eK0SSbqKA== + dependencies: + is-json "^2.0.1" + +posthtml@^0.15.1: version "0.15.2" resolved "https://registry.yarnpkg.com/posthtml/-/posthtml-0.15.2.tgz#739cf0d3ffec70868b87121dc7393478e1898c9c" integrity sha512-YugEJ5ze/0DLRIVBjCpDwANWL4pPj1kHJ/2llY8xuInr0nbkon3qTiMPe5LQa+cCwNjxS7nAZZTp+1M+6mT4Zg== @@ -10464,6 +10626,14 @@ posthtml@^0.16.4: posthtml-parser "^0.9.0" posthtml-render "^2.0.6" +posthtml@^0.16.5: + version "0.16.5" + resolved "https://registry.yarnpkg.com/posthtml/-/posthtml-0.16.5.tgz#d32f5cf32436516d49e0884b2367d0a1424136f6" + integrity sha512-1qOuPsywVlvymhTFIBniDXwUDwvlDri5KUQuBqjmCc8Jj4b/HDSVWU//P6rTWke5rzrk+vj7mms2w8e1vD0nnw== + dependencies: + posthtml-parser "^0.10.0" + posthtml-render "^3.0.0" + preact@^10.5.9: version "10.5.9" resolved "https://registry.yarnpkg.com/preact/-/preact-10.5.9.tgz#8caba9288b4db1d593be2317467f8735e43cda0b" @@ -11080,8 +11250,8 @@ readdirp@~3.6.0: readjson@^2.0.1: version "2.2.2" - resolved "https://packages.atlassian.com/api/npm/npm-remote/readjson/-/readjson-2.2.2.tgz#ed940ebdd72b88b383e02db7117402f980158959" - integrity sha1-7ZQOvdcriLOD4C23EXQC+YAViVk= + resolved "https://registry.yarnpkg.com/readjson/-/readjson-2.2.2.tgz#ed940ebdd72b88b383e02db7117402f980158959" + integrity sha512-PdeC9tsmLWBiL8vMhJvocq+OezQ3HhsH2HrN7YkhfYcTjQSa/iraB15A7Qvt7Xpr0Yd2rDNt6GbFwVQDg3HcAw== dependencies: jju "^1.4.0" try-catch "^3.0.0" @@ -11956,6 +12126,14 @@ source-map-support@^0.5.16, source-map-support@~0.5.19: buffer-from "^1.0.0" source-map "^0.6.0" +source-map-support@~0.5.20: + version "0.5.20" + resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.20.tgz#12166089f8f5e5e8c56926b377633392dd2cb6c9" + integrity sha512-n1lZZ8Ve4ksRqizaBQgxXDgKwttHDhyfQjA6YZZn8+AroHbsIz+JjwxQDxbp+7y5OYCI8t1Yk7etjD9CRd2hIw== + dependencies: + buffer-from "^1.0.0" + source-map "^0.6.0" + source-map-url@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.0.tgz#3e935d7ddd73631b97659956d55128e87b5084a3" @@ -12057,10 +12235,10 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= -srcset@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/srcset/-/srcset-3.0.1.tgz#3a09637782e71ded70126320e71b8eb92ce2ad6c" - integrity sha512-MM8wDGg5BQJEj94tDrZDrX9wrC439/Eoeg3sgmVLPMjHgrAFeXAKk3tmFlCbKw5k+yOEhPXRpPlRcisQmqWVSQ== +srcset@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/srcset/-/srcset-4.0.0.tgz#336816b665b14cd013ba545b6fe62357f86e65f4" + integrity sha512-wvLeHgcVHKO8Sc/H/5lkGreJQVeYMm9rlmt8PuR1xE31rIuXhuzznUUqAt8MqLhB3MqJdFzlNAfpcWnxiFUcPw== sshpk@^1.7.0: version "1.16.1" @@ -12496,6 +12674,19 @@ svgo@^2.3.0, svgo@^2.4.0: csso "^4.2.0" stable "^0.1.8" +svgo@^2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/svgo/-/svgo-2.6.1.tgz#60b613937e0081028cffc2369090e366b08f1f0e" + integrity sha512-SDo274ymyG1jJ3HtCr3hkfwS8NqWdF0fMr6xPlrJ5y2QMofsQxIEFWgR1epwb197teKGgnZbzozxvJyIeJpE2Q== + dependencies: + "@trysound/sax" "0.2.0" + colorette "^1.4.0" + commander "^7.2.0" + css-select "^4.1.3" + css-tree "^1.1.3" + csso "^4.2.0" + stable "^0.1.8" + symbol-tree@^3.2.2: version "3.2.4" resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" @@ -12623,7 +12814,7 @@ terminal-link@^2.1.1: ansi-escapes "^4.2.1" supports-hyperlinks "^2.0.0" -terser@^5.2.0, terser@^5.2.1, terser@^5.6.1: +terser@^5.2.0, terser@^5.2.1: version "5.7.0" resolved "https://registry.yarnpkg.com/terser/-/terser-5.7.0.tgz#a761eeec206bc87b605ab13029876ead938ae693" integrity sha512-HP5/9hp2UaZt5fYkuhNBR8YyRcT8juw8+uFbAme53iN9hblvKnLUTKkmwJG6ocWpIKf8UK4DoeWG4ty0J6S6/g== @@ -12632,6 +12823,15 @@ terser@^5.2.0, terser@^5.2.1, terser@^5.6.1: source-map "~0.7.2" source-map-support "~0.5.19" +terser@^5.8.0: + version "5.8.0" + resolved "https://registry.yarnpkg.com/terser/-/terser-5.8.0.tgz#c6d352f91aed85cc6171ccb5e84655b77521d947" + integrity sha512-f0JH+6yMpneYcRJN314lZrSwu9eKkUFEHLN/kNy8ceh8gaRiLgFPJqrB9HsXjhEGdv4e/ekjTOFxIlL6xlma8A== + dependencies: + commander "^2.20.0" + source-map "~0.7.2" + source-map-support "~0.5.20" + text-extensions@^1.0.0: version "1.9.0" resolved "https://registry.yarnpkg.com/text-extensions/-/text-extensions-1.9.0.tgz#1853e45fee39c945ce6f6c36b2d659b5aabc2a26" @@ -12869,13 +13069,13 @@ trough@^1.0.0: try-catch@^3.0.0: version "3.0.0" - resolved "https://packages.atlassian.com/api/npm/npm-remote/try-catch/-/try-catch-3.0.0.tgz#7996d8b89895e2e8ae62cbdbeb4fe17470f8131b" - integrity sha1-eZbYuJiV4uiuYsvb60/hdHD4Exs= + resolved "https://registry.yarnpkg.com/try-catch/-/try-catch-3.0.0.tgz#7996d8b89895e2e8ae62cbdbeb4fe17470f8131b" + integrity sha512-3uAqUnoemzca1ENvZ72EVimR+E8lqBbzwZ9v4CEbLjkaV3Q+FtdmPUt7jRtoSoTiYjyIMxEkf6YgUpe/voJ1ng== try-to-catch@^3.0.0: version "3.0.0" - resolved "https://packages.atlassian.com/api/npm/npm-remote/try-to-catch/-/try-to-catch-3.0.0.tgz#a1903b44d13d5124c54d14a461d22ec1f52ea14b" - integrity sha1-oZA7RNE9USTFTRSkYdIuwfUuoUs= + resolved "https://registry.yarnpkg.com/try-to-catch/-/try-to-catch-3.0.0.tgz#a1903b44d13d5124c54d14a461d22ec1f52ea14b" + integrity sha512-eIm6ZXwR35jVF8By/HdbbkcaCDTBI5PpCPkejRKrYp0jyf/DbCCcRhHD7/O9jtFI3ewsqo9WctFEiJTS6i+CQA== tsconfig-paths@^3.9.0: version "3.9.0" @@ -13258,8 +13458,8 @@ universal-user-agent@^4.0.0: universal-user-agent@^5.0.0: version "5.0.0" - resolved "https://packages.atlassian.com/api/npm/npm-remote/universal-user-agent/-/universal-user-agent-5.0.0.tgz#a3182aa758069bf0e79952570ca757de3579c1d9" - integrity sha1-oxgqp1gGm/DnmVJXDKdX3jV5wdk= + resolved "https://registry.yarnpkg.com/universal-user-agent/-/universal-user-agent-5.0.0.tgz#a3182aa758069bf0e79952570ca757de3579c1d9" + integrity sha512-B5TPtzZleXyPrUMKCpEHFmVhMN6EhmJYjG5PQna9s7mXeSqGTLap4OpqLl5FCEFUI3UBmllkETwKf/db66Y54Q== dependencies: os-name "^3.1.0" @@ -14141,6 +14341,11 @@ yaml@^1.10.0, yaml@^1.7.2: resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.0.tgz#3b593add944876077d4d683fee01081bd9fff31e" integrity sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg== +yaml@^1.10.2: + version "1.10.2" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" + integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== + yargs-parser@20.2.4: version "20.2.4" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.4.tgz#b42890f14566796f85ae8e3a25290d205f154a54" From e5e4a3fd01cd3d694aa54659309ba5e92b3ab9b7 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Mon, 20 Sep 2021 18:34:32 -0400 Subject: [PATCH 50/73] consolidate comments and annotate steps --- .../bundlers/default/src/DefaultBundler.js | 176 +++++------------- 1 file changed, 49 insertions(+), 127 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index db07554e14e..00d6b67740f 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -15,11 +15,7 @@ import type { } from '@parcel/types'; import type {NodeId} from '@parcel/graph'; import type {SchemaEntity} from '@parcel/utils'; - import {ContentGraph, Graph} from '@parcel/graph'; -// TODO: Remove before review! -// eslint-disable-next-line -import dumpGraphToGraphViz from '@parcel/core/src/dumpGraphToGraphViz'; import invariant from 'assert'; import {Bundler} from '@parcel/plugin'; @@ -98,15 +94,10 @@ export default (new Bundler({ optimize() {}, }): Bundler); -/** - Test: does not create bundles for dynamic imports when assets are available up the graph - Issue: Ideal bundlegraph creates dynamic import bundle & will not place asset in both bundle groups/bundles even if asset is present statically "up the tree" - */ function decorateLegacyGraph( idealGraph: IdealGraph, bundleGraph: MutableBundleGraph, ): void { - //TODO add in reference edges based on stored assets from create ideal graph let idealBundleToLegacyBundle: Map = new Map(); let { @@ -116,9 +107,9 @@ function decorateLegacyGraph( } = idealGraph; let entryBundleToBundleGroup: Map = new Map(); + // Step 1: Create bundle groups, bundles, and shared bundles and add assets to them for (let [bundleNodeId, idealBundle] of idealBundleGraph.nodes) { let [entryAsset] = [...idealBundle.assets]; - // This entry asset is the first asset of the bundle (not entry file asset) let bundleGroup; let bundle; @@ -153,9 +144,6 @@ function decorateLegacyGraph( bundleGraph.addBundleToBundleGroup(bundle, bundleGroup); } else if (idealBundle.sourceBundles.length > 0) { - //TODO this should be > 1 - //this should only happen for shared bundles - bundle = nullthrows( bundleGraph.createBundle({ uniqueKey: @@ -185,7 +173,7 @@ function decorateLegacyGraph( bundleGraph.addAssetToBundle(asset, bundle); } } - + // Step 2: Internalize dependencies for bundles for (let [, idealBundle] of idealBundleGraph.nodes) { let bundle = nullthrows(idealBundleToLegacyBundle.get(idealBundle)); for (let internalized of idealBundle.internalizedAssetIds) { @@ -202,7 +190,7 @@ function decorateLegacyGraph( } } } - + // Step 3: Add bundles to their bundle group for (let [bundleId, bundleGroup] of entryBundleToBundleGroup) { let outboundNodeIds = idealBundleGraph.getNodeIdsConnectedFrom(bundleId); for (let id of outboundNodeIds) { @@ -211,19 +199,10 @@ function decorateLegacyGraph( idealBundleToLegacyBundle.get(siblingBundle), ); bundleGraph.addBundleToBundleGroup(legacySiblingBundle, bundleGroup); - //TODO Put this back for shared bundles - // bundleGraph.createBundleReference( - // legacyMainBundleOfBundleGroup, - // legacySiblingBundle, - // ); } } - /** - * TODO: Create all bundles, bundlegroups, without adding anything to them - * Draw connections to bundles - * Add references to bundles - */ + // Step 4: Add references to all bundles for (let [asset, references] of idealGraph.assetReference) { for (let [dependency, bundle] of references) { let legacyBundle = nullthrows(idealBundleToLegacyBundle.get(bundle)); @@ -236,7 +215,7 @@ function createIdealGraph( assetGraph: MutableBundleGraph, config: ResolvedBundlerConfig, ): IdealGraph { - // Asset to the bundle it's an entry of + // assets to the bundle and group it's an entry of let bundleRoots: Map = new Map(); let bundles: Map = new Map(); let dependencyBundleGraph: DependencyBundleGraph = new ContentGraph(); @@ -244,23 +223,23 @@ function createIdealGraph( Asset, Array<[Dependency, Bundle]>, > = new DefaultMap(() => []); - // + + // bundleRoot to all bundleRoot decendants let reachableBundles: DefaultMap< BundleRoot, Set, > = new DefaultMap(() => new Set()); - // + let bundleGraph: Graph = new Graph(); let stack: Array<[BundleRoot, NodeId]> = []; + + // bundleGraph that models bundleRoots and async deps only let asyncBundleRootGraph: ContentGraph< BundleRoot | 'root', > = new ContentGraph(); let bundleGroupBundleIds: Array = []; - //TODO of asyncBundleRootGraph: we should either add a root node or use bundleGraph which has a root automatically - // Step 1: Create bundles for each entry. - // TODO: Try to not create bundles during this first path, only annotate - // BundleRoots + // Step 1: Find and create bundles for entries from assetGraph let entries: Map = new Map(); assetGraph.traverse((node, context, actions) => { if (node.type !== 'asset') { @@ -306,26 +285,25 @@ function createIdealGraph( } let assets = []; - // Traverse the asset graph and create bundles for asset type changes and async dependencies. - // This only adds the entry asset of each bundle, not the subgraph. + + // Step 2: Traverse the asset graph and create bundles for asset type changes and async dependencies, + // only adding the entry asset of each bundle, not the subgraph. assetGraph.traverse({ enter(node, context) { - //Discover if (node.type === 'asset') { assets.push(node.value); let bundleIdTuple = bundleRoots.get(node.value); if (bundleIdTuple) { - // Push to the stack when a new bundle is created. - stack.push([node.value, bundleIdTuple[1]]); // TODO: switch this to be push/pop instead of unshift + // Push to the stack when a new bundle is created + stack.push([node.value, bundleIdTuple[1]]); } } else if (node.type === 'dependency') { if (context == null) { return node; } - let dependency = node.value; - //TreeEdge Event + invariant(context?.type === 'asset'); let parentAsset = context.value; @@ -336,12 +314,10 @@ function createIdealGraph( invariant(assets.length === 1); let childAsset = assets[0]; - // Create a new bundle as well as a new bundle group if the dependency is async. if ( dependency.priority === 'lazy' || childAsset.bundleBehavior === 'isolated' ) { - // TODO: needsStableName if bundle exists here let bundleId = bundles.get(childAsset.id); let bundle; if (bundleId == null) { @@ -394,8 +370,6 @@ function createIdealGraph( } return node; } - - // Create a new bundle when the asset type changes. if ( parentAsset.type !== childAsset.type || childAsset.bundleBehavior === 'inline' @@ -441,7 +415,7 @@ function createIdealGraph( }, }); - // Step 2: Determine reachability for every asset from each bundle root. + // Step 3: Determine reachability for every asset from each bundleRoot. // This is later used to determine which bundles to place each asset in. for (let [root] of bundleRoots) { if (!entries.has(root)) { @@ -449,6 +423,7 @@ function createIdealGraph( } } + // graph that models bundleRoots and what require it synchronously let reachableRoots: ContentGraph = new ContentGraph(); for (let [root] of bundleRoots) { let rootNodeId = reachableRoots.addNodeByContentKeyIfNeeded(root.id, root); @@ -489,61 +464,36 @@ function createIdealGraph( }, root); } - // Step 2.5 - // IDEA 2: Somehow store all assets available (guarenteed to be loaded at this bundles load in time) at a certain point, for an asset/ bundleRoot, and do a lookup to - // determine what MUST be duplicated. - - // PART 1 (located in STEP 1) - // Make bundlegraph that models bundleRoots and async deps only [x] - // Turn reachableRoots into graph so that we have sync deps (Bidirectional) [x] - - // PART 2 - // traverse PART 2 BundleGraph (BFS) - // Maintain a MAP BundleRoot => Set of assets loaded thus far - - // At BundleRoot X - // Peek/Ask for children [Z..] - - // get all assets guarenteed to be loaded when bundle X is loaded - // map.set(Z, {all assets gurenteed to be loaded at this point (by ancestors (X)) INTERSECTION WITH current map.get(z) }) - - //TODO Consider BUNDLEGRROUPS - //Walk each bundlegroup entry - //get each asset in the group (instead of syncassetsloaded) - + // Tracks what assets have been loaded so far and, thus, available to a given bundleRoot let ancestorAssets: Map< BundleRoot, Map | null>, > = new Map(); - // Using Nested Maps, we need to be able to query, for a bundleGroup, an asset within that bundleGroup, - // Any asset it "has", the ref when we first saw it - //This is a triply nest map :o - // AND we need a auxilary double map to keep current ref count for the next asset - // (this can be just 1 map num> because it will be reset when processing next bundleGroup ) + // Counts references to assets available within a given bundle group for a bundleRoot let assetRefsInBundleGroup: DefaultMap< BundleRoot, DefaultMap, > = new DefaultMap(() => new DefaultMap(() => 0)); - //FOR BUNDLEGROUPS, hold each BUNDLEGROUPROOT mapped to Roots within the bundle root, mapped to assets available, - // mapped to their number. We need duplicate entries + + // Step 4: Determine what must be duplicated. For any child, visit all its parents first, peeking each time + // to inform what assets have been loaded thus far or are available to child for (let nodeId of asyncBundleRootGraph.topoSort()) { const bundleRoot = asyncBundleRootGraph.getNode(nodeId); if (bundleRoot === 'root') continue; invariant(bundleRoot != null); let ancestors = ancestorAssets.get(bundleRoot); - //Get all assets available in this bundleRoot's bundleGroup through ideal graph - //*****Bundle Group consideration start */ + // First Consider bundle group asset availability, processing only + // non-isolated bundles within that bundle group let bundleGroupId = nullthrows(bundleRoots.get(bundleRoot))[1]; - //TODO should this include our bundle group root's assets let assetRefs = assetRefsInBundleGroup.get(bundleRoot); - //Process all nodes within bundleGroup that are NOT isolated + for (let bundleIdInGroup of [ bundleGroupId, ...bundleGraph.getNodeIdsConnectedFrom(bundleGroupId), ]) { - let bundleInGroup = nullthrows(bundleGraph.getNode(bundleIdInGroup)); //this is a bundle + let bundleInGroup = nullthrows(bundleGraph.getNode(bundleIdInGroup)); if ( bundleInGroup.bundleBehavior === 'isolated' || bundleInGroup.bundleBehavior === 'inline' @@ -563,13 +513,13 @@ function createIdealGraph( } } - //**********End of bundle Group consideration */ - //should we edit Combined to include what available in it's bundlegroup? -we aren't editing parent AA - //THEN, process each ndoe in bundlegroup and do the same that we do below? + // Pre-process this node's children, modify it's available assets + // at that moment, using the total set of synchronous assets loaded thus + // far from bundleGroup consideration, and assets loaded by this parent. let bundleGroupAssets = new Set(assetRefs.keys()); let combined = ancestors - ? new Map([...bundleGroupAssets, ...ancestors.keys()].map(a => [a, null])) //sync assets loaded should be all available assets at that moment (from anc bundle groups) + ? new Map([...bundleGroupAssets, ...ancestors.keys()].map(a => [a, null])) : new Map([...bundleGroupAssets].map(a => [a, null])); let children = asyncBundleRootGraph.getNodeIdsConnectedFrom(nodeId); @@ -587,13 +537,13 @@ function createIdealGraph( } let siblingAncestors = ancestors - ? ancestryUnion(new Set(ancestors.keys()), assetRefs, bundleRoot) //sync assets loaded should be all available assets at that moment (from anc bundle groups) + ? ancestryUnion(new Set(ancestors.keys()), assetRefs, bundleRoot) : new Map([...bundleGroupAssets].map(a => [a, [bundleRoot]])); for (let bundleIdInGroup of bundleGraph.getNodeIdsConnectedFrom( bundleGroupId, )) { - let bundleInGroup = bundleGraph.getNode(bundleIdInGroup); //this is a bundle + let bundleInGroup = bundleGraph.getNode(bundleIdInGroup); invariant(bundleInGroup != null && bundleInGroup.assets != null); let [bundleRoot] = [...bundleInGroup.assets]; @@ -608,20 +558,21 @@ function createIdealGraph( } } - // Step 3: Place all assets into bundles. Each asset is placed into a single - // bundle based on the bundle entries it is reachable from. This creates a - // maximally code split bundle graph with no duplication. - + // Step 5: Place all assets into bundles or create shared bundles. Each asset + // is placed into a single bundle based on the bundle entries it is reachable from. + // This creates a maximally code split bundle graph with no duplication. for (let asset of assets) { - // Find bundle entries reachable from the asset. - + // all unreliable bundleRoot assets which need to pulled in by shared bundle or other means let reachable: Array = getReachableBundleRoots( asset, reachableRoots, ).reverse(); - // Filter out bundles when the asset is reachable in every parent bundle. - // (Only keep a bundle if all of the others are not descendents of it) + // Filter out bundles from this asset's reachable array if + // bundle does not contain the asset in its ancestry + // or if any of the bundles in the ancestry have a <= 1 reference to that asset, + // meaning it may not be deduplicated. Otherwise, decrement all references in + // the ancestry and keep it reachable = reachable.filter(b => { let ancestry = ancestorAssets.get(b)?.get(asset); if (ancestry === undefined) { @@ -644,21 +595,15 @@ function createIdealGraph( } return false; } - return true; } - }); //don't want to filter out bundle if 'b' is not "reachable" from all of its (a) immediate parents - // BundleRoot = Root Asset of a bundle - // reachableRoots = any asset => all BundleRoots that require it synchronously - // reachableBundles = Some BundleRoot => all BundleRoot decendants - // reachable = all bundle root assets that cant always have that asset reliably on page (so they need to be pulled in by shared bundle or other) + }); let rootBundle = bundleRoots.get(asset); if ( rootBundle != null && !nullthrows(bundleGraph.getNode(rootBundle[0])).env.isIsolated() ) { - // If the asset is a bundle root, add the bundle to every other reachable bundle group. if (!bundles.has(asset.id)) { bundles.set(asset.id, rootBundle[0]); } @@ -671,8 +616,7 @@ function createIdealGraph( ); } } - // reachableAsyncRoots = all bundleNodeId => all BundleRoots that require it asynchronously - // reachableAsync = for one bundleRoot => all + let reachableAsync = asyncBundleRootGraph .getNodeIdsConnectedTo( asyncBundleRootGraph.getNodeIdByContentKey(asset.id), @@ -680,11 +624,10 @@ function createIdealGraph( .map(id => nullthrows(asyncBundleRootGraph.getNode(id))) .filter(node => node !== 'root') .map(node => { - // TODO: Improve. Currently for flow invariant(node !== 'root'); return node; }); - // TODO: is this correct? + let willInternalizeRoots = reachableAsync.filter( b => !getReachableBundleRoots(asset, reachableRoots).every( @@ -701,12 +644,11 @@ function createIdealGraph( } } } else if (reachable.length > 0) { - // If the asset is reachable from more than one entry, find or create - // a bundle for that combination of bundles (shared bundle), and add the asset to it. let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); let sourceBundleSet = new Set([...sourceBundles]); let toRemove: Set = new Set(); for (let bundleId of sourceBundles) { + // entries should not be considered in reachablility for (let id of bundleGraph.getNodeIdsConnectedTo(bundleId)) { if (sourceBundleSet.has(id)) { toRemove.add(id); @@ -736,7 +678,6 @@ function createIdealGraph( bundle.assets.add(asset); bundle.size += asset.stats.size; - // Add the bundle to each reachable bundle group. for (let sourceBundleId of sourceBundles) { if (bundleId !== sourceBundleId) { bundleGraph.addEdge(sourceBundleId, bundleId); @@ -749,8 +690,8 @@ function createIdealGraph( } } - // Step 4: Merge any sibling bundles required by entry bundles back into the entry bundle. - // Entry bundles must be predictable, so cannot have unpredictable siblings. + // Step 7: Merge any sibling bundles required by entry bundles back into the entry bundle. + // Entry bundles must be predictable, so cannot have unpredictable siblings. for (let [bundleNodeId, bundle] of bundleGraph.nodes) { if (bundle.sourceBundles.length > 0 && bundle.size < config.minBundleSize) { removeBundle(bundleGraph, bundleNodeId); @@ -770,7 +711,6 @@ function createIdealGraph( entryBundle.size += asset.stats.size; } bundleGraph.removeEdge(entryBundleId, siblingId); - // reachableAsyncRoots.get(siblingId).delete(entryAsset); if (sibling.sourceBundles.length > 1) { let entryBundleIndex = sibling.sourceBundles.indexOf(entryBundleId); invariant(entryBundleIndex >= 0); @@ -789,24 +729,6 @@ function createIdealGraph( } } - // for (let [asyncBundleRoot, dependentRoots] of reachableAsyncRoots) { - // if (dependentRoots.size === 0) { - // //TODO make get entry asset function - // let bundleNode = bundleGraph.getNode(asyncBundleRoot); - - // if (bundleNode?.assets) { - // let [entryAsset] = [...bundleNode.assets]; - // if (entries.has(entryAsset)) { - // continue; - // } - // } - // bundleGraph.removeNode(asyncBundleRoot); - // } - // } - - // $FlowFixMe - dumpGraphToGraphViz(bundleGraph, 'IdealBundleGraph'); - return { bundleGraph, dependencyBundleGraph, From e1a47d5b249ca96973247efcbd5eae6a291560b9 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 21 Sep 2021 14:54:07 -0700 Subject: [PATCH 51/73] Use ancestorAssets for internalization --- .../bundlers/default/src/DefaultBundler.js | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 46bdada3aac..32ee9220e67 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -556,7 +556,6 @@ function createIdealGraph( let [bundleRoot] = [...bundleInGroup.assets]; const availableAssets = ancestorAssets.get(bundleRoot); - if (availableAssets === undefined) { ancestorAssets.set(bundleRoot, siblingAncestors); } else { @@ -624,23 +623,28 @@ function createIdealGraph( } } - let reachableAsync = asyncBundleRootGraph + let willInternalizeRoots = asyncBundleRootGraph .getNodeIdsConnectedTo( asyncBundleRootGraph.getNodeIdByContentKey(asset.id), ) .map(id => nullthrows(asyncBundleRootGraph.getNode(id))) - .filter(node => node !== 'root') - .map(node => { - invariant(node !== 'root'); - return node; - }); + .filter(bundleRoot => { + if (bundleRoot === 'root') { + return false; + } - let willInternalizeRoots = reachableAsync.filter( - b => - !getReachableBundleRoots(asset, reachableRoots).every( - a => !(a === b || reachableBundles.get(a).has(b)), - ), - ); + return ( + reachableRoots.hasEdge( + reachableRoots.getNodeIdByContentKey(bundleRoot.id), + reachableRoots.getNodeIdByContentKey(asset.id), + ) || ancestorAssets.get(bundleRoot)?.has(asset) + ); + }) + .map(bundleRoot => { + // For Flow + invariant(bundleRoot !== 'root'); + return bundleRoot; + }); for (let bundleRoot of willInternalizeRoots) { if (bundleRoot !== asset) { From 1a1013f405060602392dbde53454a63c2da0ef41 Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Tue, 21 Sep 2021 15:17:19 -0700 Subject: [PATCH 52/73] cleanup --- .../bundlers/default/src/DefaultBundler.js | 59 +++++++++++-------- .../core/src/public/MutableBundleGraph.js | 2 - .../test/integration/commonjs/index.js | 2 +- .../core/integration-tests/test/javascript.js | 10 ++-- packages/runtimes/js/src/JSRuntime.js | 1 - 5 files changed, 39 insertions(+), 35 deletions(-) diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 32ee9220e67..911da3afc30 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -85,7 +85,6 @@ type IdealGraph = {| dependencyBundleGraph: DependencyBundleGraph, bundleGraph: Graph, bundleGroupBundleIds: Array, - entryBundles: Array, assetReference: DefaultMap>, |}; @@ -231,7 +230,7 @@ function createIdealGraph( Array<[Dependency, Bundle]>, > = new DefaultMap(() => []); - // bundleRoot to all bundleRoot decendants + // bundleRoot to all bundleRoot descendants let reachableBundles: DefaultMap< BundleRoot, Set, @@ -430,7 +429,7 @@ function createIdealGraph( } } - // graph that models bundleRoots and what require it synchronously + // Models bundleRoots and the assets that require it synchronously let reachableRoots: ContentGraph = new ContentGraph(); for (let [root] of bundleRoots) { let rootNodeId = reachableRoots.addNodeByContentKeyIfNeeded(root.id, root); @@ -471,29 +470,32 @@ function createIdealGraph( }, root); } - // Tracks what assets have been loaded so far and, thus, available to a given bundleRoot + // Maps a given bundleRoot to the bundleRoots reachable from it, + // and the assets reachable from each of these bundleRoots let ancestorAssets: Map< BundleRoot, - Map | null>, + Map | null>, > = new Map(); - // Counts references to assets available within a given bundle group for a bundleRoot + // Reference count of each asset available within a given bundleRoot's bundle group let assetRefsInBundleGroup: DefaultMap< BundleRoot, DefaultMap, > = new DefaultMap(() => new DefaultMap(() => 0)); - // Step 4: Determine what must be duplicated. For any child, visit all its parents first, peeking each time - // to inform what assets have been loaded thus far or are available to child + // Step 4: Determine assets that should be duplicated by computing asset availability in each bundle group for (let nodeId of asyncBundleRootGraph.topoSort()) { const bundleRoot = asyncBundleRootGraph.getNode(nodeId); if (bundleRoot === 'root') continue; invariant(bundleRoot != null); + // BundleRoots reachable from current (bundleRoot) node mapped to assets that are available from it + // should we call these reachableBundleRoots/reachableBundleRootMap instead? let ancestors = ancestorAssets.get(bundleRoot); - // First Consider bundle group asset availability, processing only + // First consider bundle group asset availability, processing only // non-isolated bundles within that bundle group let bundleGroupId = nullthrows(bundleRoots.get(bundleRoot))[1]; + // Map of assets in the bundle group to their refcounts let assetRefs = assetRefsInBundleGroup.get(bundleRoot); for (let bundleIdInGroup of [ @@ -508,7 +510,7 @@ function createIdealGraph( continue; } let [bundleRoot] = [...bundleInGroup.assets]; - + // Assets directly connected to current bundleRoot let assetsFromBundleRoot = reachableRoots .getNodeIdsConnectedFrom( reachableRoots.getNodeIdByContentKey(bundleRoot.id), @@ -520,9 +522,8 @@ function createIdealGraph( } } - // Pre-process this node's children, modify it's available assets - // at that moment, using the total set of synchronous assets loaded thus - // far from bundleGroup consideration, and assets loaded by this parent. + // Enumerate bundleRoots connected to the node (parent), taking the intersection + // between the assets synchronously loaded by the parent, and those loaded by the child. let bundleGroupAssets = new Set(assetRefs.keys()); let combined = ancestors @@ -556,6 +557,7 @@ function createIdealGraph( let [bundleRoot] = [...bundleInGroup.assets]; const availableAssets = ancestorAssets.get(bundleRoot); + if (availableAssets === undefined) { ancestorAssets.set(bundleRoot, siblingAncestors); } else { @@ -576,14 +578,16 @@ function createIdealGraph( // Filter out bundles from this asset's reachable array if // bundle does not contain the asset in its ancestry - // or if any of the bundles in the ancestry have a <= 1 reference to that asset, + // or if any of the bundles in the ancestry have a refcount of <= 1 for that asset, // meaning it may not be deduplicated. Otherwise, decrement all references in // the ancestry and keep it reachable = reachable.filter(b => { let ancestry = ancestorAssets.get(b)?.get(asset); if (ancestry === undefined) { + // No reachable assets from this bundle return true; } else if (ancestry === null) { + // Asset is reachable via the bundle return false; } else { if ( @@ -670,6 +674,10 @@ function createIdealGraph( a => !toRemove.has(nullthrows(bundles.get(a.id))), ); + if (reachable.length < 1) { + continue; + } + let key = reachable.map(a => a.id).join(','); let bundleId = bundles.get(key); let bundle; @@ -746,7 +754,6 @@ function createIdealGraph( bundleGraph, dependencyBundleGraph, bundleGroupBundleIds, - entryBundles: [...bundleRoots.values()].map(v => v[0]), assetReference, }; } @@ -868,7 +875,7 @@ async function loadBundlerConfig( } function ancestryUnion( - ancestors: Set, + ancestors: Set, assetRefs: Map, bundleRoot: BundleRoot, ): Map | null> { @@ -885,21 +892,21 @@ function ancestryUnion( } function ancestryIntersect( - map: Map | null>, - previousMap: Map | null>, + currentMap: Map | null>, + map: Map | null>, ): void { - for (let [asset, arrayOfBundleIds] of map) { - if (previousMap.has(asset)) { - let prevBundleIds = previousMap.get(asset); - if (prevBundleIds) { - if (arrayOfBundleIds) { - arrayOfBundleIds.push(...prevBundleIds); + for (let [bundleRoot, currentAssets] of currentMap) { + if (map.has(bundleRoot)) { + let assets = map.get(bundleRoot); + if (assets) { + if (currentAssets) { + currentAssets.push(...assets); } else { - map.set(asset, [...prevBundleIds]); + currentMap.set(bundleRoot, [...assets]); } } } else { - map.delete(asset); + currentMap.delete(bundleRoot); } } } diff --git a/packages/core/core/src/public/MutableBundleGraph.js b/packages/core/core/src/public/MutableBundleGraph.js index 2fe8ec0d687..d2053db27bb 100644 --- a/packages/core/core/src/public/MutableBundleGraph.js +++ b/packages/core/core/src/public/MutableBundleGraph.js @@ -167,9 +167,7 @@ export default class MutableBundleGraph extends BundleGraph resolvedNodeId, bundleGraphEdgeTypes.references, ); - //this.#graph.markDependencyReferenceable(dependencyNode.value); this.#graph._graph.removeEdge(dependencyNodeId, resolvedNodeId); - //console.log('removed edge from', dependencyNodeId, 'to', resolvedNodeId); if (dependency.isEntry) { this.#graph._graph.addEdge( diff --git a/packages/core/integration-tests/test/integration/commonjs/index.js b/packages/core/integration-tests/test/integration/commonjs/index.js index 99a30d0574a..9a6fc97209e 100644 --- a/packages/core/integration-tests/test/integration/commonjs/index.js +++ b/packages/core/integration-tests/test/integration/commonjs/index.js @@ -1,6 +1,6 @@ var local = require('./local'); // eslint-disable-next-line no-unused-vars -//var url = require('url'); +var url = require('url'); module.exports = function () { return local.a + local.b; diff --git a/packages/core/integration-tests/test/javascript.js b/packages/core/integration-tests/test/javascript.js index d8b99a981e2..7acc9809076 100644 --- a/packages/core/integration-tests/test/javascript.js +++ b/packages/core/integration-tests/test/javascript.js @@ -1898,7 +1898,7 @@ describe('javascript', function() { ], }); }); - //TODO Broke 1 + it('should create a shared bundle to deduplicate assets in workers', async () => { let b = await bundle( path.join(__dirname, '/integration/worker-shared/index.js'), @@ -1982,7 +1982,7 @@ describe('javascript', function() { }, ]); }); - // TODO Broke 2 + it('should deduplicate and remove an unnecessary async bundle when it contains a cyclic reference to its entry', async () => { let b = await bundle( path.join( @@ -2053,7 +2053,7 @@ describe('javascript', function() { {assets: ['core.js', 'worker3.js']}, ]); }); - //TODO + it('should create a shared bundle between browser and worker contexts', async () => { let b = await bundle( path.join(__dirname, '/integration/html-shared-worker/index.html'), @@ -2169,7 +2169,7 @@ describe('javascript', function() { await run(b); }); - //TODO + it('should dynamic import files which import raw files', async function() { let b = await bundle( path.join(__dirname, '/integration/dynamic-references-raw/index.js'), @@ -2280,7 +2280,7 @@ describe('javascript', function() { assert.equal(typeof output, 'function'); assert.equal(await output(), 7); }); - //TODO + it('should not duplicate a module which is already in a parent bundle', async function() { let b = await bundle( path.join(__dirname, '/integration/dynamic-hoist-dup/index.js'), diff --git a/packages/runtimes/js/src/JSRuntime.js b/packages/runtimes/js/src/JSRuntime.js index 50bb1130cac..40164c723dc 100644 --- a/packages/runtimes/js/src/JSRuntime.js +++ b/packages/runtimes/js/src/JSRuntime.js @@ -92,7 +92,6 @@ export default (new Runtime({ // If this bundle already has the asset this dependency references, // return a simple runtime of `Promise.resolve(internalRequire(assetId))`. // The linker handles this for scope-hoisting. - console.log('creating promise.resolve runtime'); assets.push({ filePath: __filename, code: `module.exports = Promise.resolve(module.bundle.root(${JSON.stringify( From d6490eeaf51ec73eca97ac3714a11c1b304a9247 Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Thu, 23 Sep 2021 13:45:11 -0700 Subject: [PATCH 53/73] run tests with experimental bundler --- .../bundlers/default/src/DefaultBundler.js | 1107 +- packages/bundlers/experimental/package.json | 29 + .../experimental/src/ExperimentalBundler.js | 918 ++ .../experimental-bundler-config.json | 4 + packages/core/integration-tests/test/html.js | 4686 ++++---- .../core/integration-tests/test/javascript.js | 9882 +++++++++-------- .../integration-tests/test/scope-hoisting.js | 9430 ++++++++-------- 7 files changed, 13474 insertions(+), 12582 deletions(-) create mode 100644 packages/bundlers/experimental/package.json create mode 100644 packages/bundlers/experimental/src/ExperimentalBundler.js create mode 100644 packages/core/integration-tests/experimental-bundler-config.json diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 911da3afc30..c7c834b4e8e 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -2,24 +2,18 @@ import type { Asset, - Bundle as LegacyBundle, - BundleBehavior, + Bundle, BundleGroup, - Dependency, - Environment, Config, MutableBundleGraph, PluginOptions, - Target, } from '@parcel/types'; -import type {NodeId} from '@parcel/graph'; import type {SchemaEntity} from '@parcel/utils'; -import {ContentGraph, Graph} from '@parcel/graph'; import invariant from 'assert'; -import {ALL_EDGE_TYPES} from '@parcel/graph'; import {Bundler} from '@parcel/plugin'; -import {validateSchema, DefaultMap} from '@parcel/utils'; +import {validateSchema} from '@parcel/utils'; +import {hashString} from '@parcel/hash'; import nullthrows from 'nullthrows'; import {encodeJSONKeyComponent} from '@parcel/diagnostic'; @@ -30,12 +24,6 @@ type BundlerConfig = {| maxParallelRequests?: number, |}; -type ResolvedBundlerConfig = {| - minBundles: number, - minBundleSize: number, - maxParallelRequests: number, -|}; - // Default options by http version. const HTTP_OPTIONS = { '1': { @@ -50,712 +38,385 @@ const HTTP_OPTIONS = { }, }; -type AssetId = string; -type BundleRoot = Asset; -export type Bundle = {| - assets: Set, - internalizedAssetIds: Array, - bundleBehavior?: ?BundleBehavior, - needsStableName: boolean, - size: number, - sourceBundles: Array, - target: Target, - env: Environment, - type: string, -|}; - -const dependencyPriorityEdges = { - sync: 1, - parallel: 2, - lazy: 3, -}; - -type DependencyBundleGraph = ContentGraph< - | {| - value: Bundle, - type: 'bundle', - |} - | {| - value: Dependency, - type: 'dependency', - |}, - number, ->; -type IdealGraph = {| - dependencyBundleGraph: DependencyBundleGraph, - bundleGraph: Graph, - bundleGroupBundleIds: Array, - assetReference: DefaultMap>, -|}; +let skipOptimize = false; export default (new Bundler({ + // RULES: + // 1. If dep.isAsync or dep.isEntry, start a new bundle group. + // 2. If an asset is a different type than the current bundle, make a parallel bundle in the same bundle group. + // 3. If an asset is already in a parent bundle in the same entry point, exclude from child bundles. + // 4. If an asset is only in separate isolated entry points (e.g. workers, different HTML pages), duplicate it. + loadConfig({config, options}) { return loadBundlerConfig(config, options); }, bundle({bundleGraph, config}) { - decorateLegacyGraph(createIdealGraph(bundleGraph, config), bundleGraph); - }, - optimize() {}, -}): Bundler); - -function decorateLegacyGraph( - idealGraph: IdealGraph, - bundleGraph: MutableBundleGraph, -): void { - let idealBundleToLegacyBundle: Map = new Map(); - - let { - bundleGraph: idealBundleGraph, - dependencyBundleGraph, - bundleGroupBundleIds, - } = idealGraph; - let entryBundleToBundleGroup: Map = new Map(); - - // Step 1: Create bundle groups, bundles, and shared bundles and add assets to them - for (let [bundleNodeId, idealBundle] of idealBundleGraph.nodes) { - let [entryAsset] = [...idealBundle.assets]; - let bundleGroup; - let bundle; - - if (bundleGroupBundleIds.includes(bundleNodeId)) { - let dependencies = dependencyBundleGraph - .getNodeIdsConnectedTo( - dependencyBundleGraph.getNodeIdByContentKey(String(bundleNodeId)), - // $FlowFixMe[incompatible-call] - ALL_EDGE_TYPES, - ) - .map(nodeId => { - let dependency = nullthrows(dependencyBundleGraph.getNode(nodeId)); - invariant(dependency.type === 'dependency'); - return dependency.value; - }); - for (let dependency of dependencies) { - bundleGroup = bundleGraph.createBundleGroup( - dependency, - idealBundle.target, - ); - } - invariant(bundleGroup); - entryBundleToBundleGroup.set(bundleNodeId, bundleGroup); - - bundle = nullthrows( - bundleGraph.createBundle({ - entryAsset, - needsStableName: idealBundle.needsStableName, - bundleBehavior: idealBundle.bundleBehavior, - target: idealBundle.target, - }), - ); - - bundleGraph.addBundleToBundleGroup(bundle, bundleGroup); - } else if (idealBundle.sourceBundles.length > 0) { - bundle = nullthrows( - bundleGraph.createBundle({ - uniqueKey: - [...idealBundle.assets].map(asset => asset.id).join(',') + - idealBundle.sourceBundles.join(','), - needsStableName: idealBundle.needsStableName, - bundleBehavior: idealBundle.bundleBehavior, - type: idealBundle.type, - target: idealBundle.target, - env: idealBundle.env, - }), - ); - } else { - bundle = nullthrows( - bundleGraph.createBundle({ - entryAsset, - needsStableName: idealBundle.needsStableName, - bundleBehavior: idealBundle.bundleBehavior, - target: idealBundle.target, - }), - ); - } - - idealBundleToLegacyBundle.set(idealBundle, bundle); - - for (let asset of idealBundle.assets) { - bundleGraph.addAssetToBundle(asset, bundle); - } - } - // Step 2: Internalize dependencies for bundles - for (let [, idealBundle] of idealBundleGraph.nodes) { - let bundle = nullthrows(idealBundleToLegacyBundle.get(idealBundle)); - for (let internalized of idealBundle.internalizedAssetIds) { - let incomingDeps = bundleGraph.getIncomingDependencies( - bundleGraph.getAssetById(internalized), - ); - for (let incomingDep of incomingDeps) { - if ( - incomingDep.priority === 'lazy' && - bundle.hasDependency(incomingDep) - ) { - bundleGraph.internalizeAsyncDependency(bundle, incomingDep); + let bundleRoots: Map> = new Map(); + let bundlesByEntryAsset: Map = new Map(); + + // Step 1: create bundles for each of the explicit code split points. + bundleGraph.traverse({ + enter: (node, context, actions) => { + if (node.type !== 'dependency') { + return { + ...context, + bundleGroup: context?.bundleGroup, + bundleByType: context?.bundleByType, + parentNode: node, + parentBundle: + bundlesByEntryAsset.get(node.value) ?? context?.parentBundle, + }; } - } - } - } - // Step 3: Add bundles to their bundle group - for (let [bundleId, bundleGroup] of entryBundleToBundleGroup) { - let outboundNodeIds = idealBundleGraph.getNodeIdsConnectedFrom(bundleId); - for (let id of outboundNodeIds) { - let siblingBundle = nullthrows(idealBundleGraph.getNode(id)); - let legacySiblingBundle = nullthrows( - idealBundleToLegacyBundle.get(siblingBundle), - ); - bundleGraph.addBundleToBundleGroup(legacySiblingBundle, bundleGroup); - } - } - - // Step 4: Add references to all bundles - for (let [asset, references] of idealGraph.assetReference) { - for (let [dependency, bundle] of references) { - let legacyBundle = nullthrows(idealBundleToLegacyBundle.get(bundle)); - bundleGraph.createAssetReference(dependency, asset, legacyBundle); - } - } -} -function createIdealGraph( - assetGraph: MutableBundleGraph, - config: ResolvedBundlerConfig, -): IdealGraph { - // assets to the bundle and group it's an entry of - let bundleRoots: Map = new Map(); - let bundles: Map = new Map(); - let dependencyBundleGraph: DependencyBundleGraph = new ContentGraph(); - let assetReference: DefaultMap< - Asset, - Array<[Dependency, Bundle]>, - > = new DefaultMap(() => []); - - // bundleRoot to all bundleRoot descendants - let reachableBundles: DefaultMap< - BundleRoot, - Set, - > = new DefaultMap(() => new Set()); - - let bundleGraph: Graph = new Graph(); - let stack: Array<[BundleRoot, NodeId]> = []; - - // bundleGraph that models bundleRoots and async deps only - let asyncBundleRootGraph: ContentGraph< - BundleRoot | 'root', - > = new ContentGraph(); - let bundleGroupBundleIds: Array = []; - - // Step 1: Find and create bundles for entries from assetGraph - let entries: Map = new Map(); - assetGraph.traverse((node, context, actions) => { - if (node.type !== 'asset') { - return node; - } - - invariant( - context != null && context.type === 'dependency' && context.value.isEntry, - ); - entries.set(node.value, context.value); - actions.skipChildren(); - }); - - let rootNodeId = nullthrows(asyncBundleRootGraph.addNode('root')); - asyncBundleRootGraph.setRootNodeId(rootNodeId); - - for (let [asset, dependency] of entries) { - let bundle = createBundle({ - asset, - target: nullthrows(dependency.target), - needsStableName: dependency.isEntry, - }); - let nodeId = bundleGraph.addNode(bundle); - bundles.set(asset.id, nodeId); - bundleRoots.set(asset, [nodeId, nodeId]); - asyncBundleRootGraph.addEdge( - rootNodeId, - asyncBundleRootGraph.addNodeByContentKey(asset.id, asset), - ); - - dependencyBundleGraph.addEdge( - dependencyBundleGraph.addNodeByContentKeyIfNeeded(dependency.id, { - value: dependency, - type: 'dependency', - }), - dependencyBundleGraph.addNodeByContentKeyIfNeeded(String(nodeId), { - value: bundle, - type: 'bundle', - }), - dependencyPriorityEdges[dependency.priority], - ); - bundleGroupBundleIds.push(nodeId); - } - - let assets = []; - - // Step 2: Traverse the asset graph and create bundles for asset type changes and async dependencies, - // only adding the entry asset of each bundle, not the subgraph. - assetGraph.traverse({ - enter(node, context) { - if (node.type === 'asset') { - assets.push(node.value); - - let bundleIdTuple = bundleRoots.get(node.value); - if (bundleIdTuple) { - // Push to the stack when a new bundle is created - stack.push([node.value, bundleIdTuple[1]]); - } - } else if (node.type === 'dependency') { - if (context == null) { - return node; - } let dependency = node.value; - - invariant(context?.type === 'asset'); - let parentAsset = context.value; - - let assets = assetGraph.getDependencyAssets(dependency); - if (assets.length === 0) { - return node; + if (bundleGraph.isDependencySkipped(dependency)) { + actions.skipChildren(); + return; } - invariant(assets.length === 1); - let childAsset = assets[0]; + let assets = bundleGraph.getDependencyAssets(dependency); + let resolution = bundleGraph.getResolvedAsset(dependency); + let bundleGroup = context?.bundleGroup; + // Create a new bundle for entries, lazy/parallel dependencies, isolated/inline assets. if ( - dependency.priority === 'lazy' || - childAsset.bundleBehavior === 'isolated' + resolution && + (!bundleGroup || + dependency.priority === 'lazy' || + dependency.priority === 'parallel' || + resolution.bundleBehavior === 'isolated' || + resolution.bundleBehavior === 'inline') ) { - let bundleId = bundles.get(childAsset.id); - let bundle; - if (bundleId == null) { - bundle = createBundle({ - asset: childAsset, - target: nullthrows(bundleGraph.getNode(stack[0][1])).target, + let bundleByType: Map = + context?.bundleByType ?? new Map(); + + // Only create a new bundle group for entries, lazy dependencies, and isolated assets. + // Otherwise, the bundle is loaded together with the parent bundle. + if ( + !bundleGroup || + dependency.priority === 'lazy' || + resolution.bundleBehavior === 'isolated' + ) { + bundleGroup = bundleGraph.createBundleGroup( + dependency, + nullthrows(dependency.target ?? context?.bundleGroup?.target), + ); + + bundleByType = new Map(); + } + + for (let asset of assets) { + let bundle = bundleGraph.createBundle({ + entryAsset: asset, needsStableName: dependency.bundleBehavior === 'inline' || - childAsset.bundleBehavior === 'inline' + asset.bundleBehavior === 'inline' ? false : dependency.isEntry || dependency.needsStableName, - bundleBehavior: - dependency.bundleBehavior ?? childAsset.bundleBehavior, + bundleBehavior: dependency.bundleBehavior ?? asset.bundleBehavior, + target: bundleGroup.target, }); - bundleId = bundleGraph.addNode(bundle); - bundles.set(childAsset.id, bundleId); - bundleRoots.set(childAsset, [bundleId, bundleId]); - bundleGroupBundleIds.push(bundleId); - } else { - bundle = nullthrows(bundleGraph.getNode(bundleId)); - } + bundleByType.set(bundle.type, bundle); + bundlesByEntryAsset.set(asset, bundle); + bundleGraph.addBundleToBundleGroup(bundle, bundleGroup); - dependencyBundleGraph.addEdge( - dependencyBundleGraph.addNodeByContentKeyIfNeeded(dependency.id, { - value: dependency, - type: 'dependency', - }), - dependencyBundleGraph.addNodeByContentKeyIfNeeded( - String(bundleId), - { - value: bundle, - type: 'bundle', - }, - ), - dependencyPriorityEdges[dependency.priority], - ); + // The bundle may have already been created, and the graph gave us back the original one... + if (!bundleRoots.has(bundle)) { + bundleRoots.set(bundle, [asset]); + } - // Walk up the stack until we hit a different asset type - // and mark each bundle as reachable from every parent bundle - for (let i = stack.length - 1; i >= 0; i--) { - let [stackAsset] = stack[i]; - if ( - stackAsset.type !== childAsset.type || - stackAsset.env.context !== childAsset.env.context || - stackAsset.env.isIsolated() - ) { - break; + // If the bundle is in the same bundle group as the parent, create an asset reference + // between the dependency, the asset, and the target bundle. + if (bundleGroup === context?.bundleGroup) { + bundleGraph.createAssetReference(dependency, asset, bundle); } - reachableBundles.get(stackAsset).add(childAsset); } - return node; - } - if ( - parentAsset.type !== childAsset.type || - childAsset.bundleBehavior === 'inline' - ) { - let [, bundleGroupNodeId] = nullthrows(stack[stack.length - 1]); - let bundleGroup = nullthrows(bundleGraph.getNode(bundleGroupNodeId)); - let bundle = createBundle({ - asset: childAsset, - target: bundleGroup.target, - needsStableName: dependency.bundleBehavior === 'inline', - }); - let bundleId = bundleGraph.addNode(bundle); - bundles.set(childAsset.id, bundleId); - bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); - - dependencyBundleGraph.addEdge( - dependencyBundleGraph.addNodeByContentKeyIfNeeded(dependency.id, { - value: dependency, - type: 'dependency', - }), - dependencyBundleGraph.addNodeByContentKeyIfNeeded( - String(bundleId), - { - value: bundle, - type: 'bundle', - }, - ), - dependencyPriorityEdges.parallel, - ); - // Add an edge from the bundle group entry to the new bundle. - // This indicates that the bundle is loaded together with the entry - bundleGraph.addEdge(bundleGroupNodeId, bundleId); - assetReference.get(childAsset).push([dependency, bundle]); - return node; - } - } - return node; - }, - exit(node) { - if (stack[stack.length - 1]?.[0] === node.value) { - stack.pop(); - } - }, - }); - // Step 3: Determine reachability for every asset from each bundleRoot. - // This is later used to determine which bundles to place each asset in. - for (let [root] of bundleRoots) { - if (!entries.has(root)) { - asyncBundleRootGraph.addNodeByContentKey(root.id, root); - } - } + return { + bundleGroup, + bundleByType, + parentNode: node, + parentBundle: context?.parentBundle, + }; + } - // Models bundleRoots and the assets that require it synchronously - let reachableRoots: ContentGraph = new ContentGraph(); - for (let [root] of bundleRoots) { - let rootNodeId = reachableRoots.addNodeByContentKeyIfNeeded(root.id, root); - assetGraph.traverse((node, isAsync, actions) => { - if (node.value === root) { - return; - } + invariant(context != null); + invariant(context.parentNode.type === 'asset'); + invariant(context.parentBundle != null); + invariant(bundleGroup != null); + let parentAsset = context.parentNode.value; + let parentBundle = context.parentBundle; + let bundleByType = nullthrows(context.bundleByType); + + for (let asset of assets) { + if (parentAsset.type === asset.type) { + continue; + } - if (node.type === 'dependency') { - let dependency = node.value; - if (dependencyBundleGraph.hasContentKey(dependency.id)) { - if (dependency.priority === 'lazy') { - let assets = assetGraph.getDependencyAssets(dependency); - if (assets.length === 0) { - return node; - } + let existingBundle = bundleByType.get(asset.type); + if (existingBundle) { + // If a bundle of this type has already been created in this group, + // merge this subgraph into it. + nullthrows(bundleRoots.get(existingBundle)).push(asset); + bundlesByEntryAsset.set(asset, existingBundle); + bundleGraph.createAssetReference(dependency, asset, existingBundle); + } else { + let bundle = bundleGraph.createBundle({ + uniqueKey: asset.id, + env: asset.env, + type: asset.type, + target: bundleGroup.target, + needsStableName: + asset.bundleBehavior === 'inline' || + dependency.bundleBehavior === 'inline' || + (dependency.priority === 'parallel' && + !dependency.needsStableName) + ? false + : parentBundle.needsStableName, + bundleBehavior: dependency.bundleBehavior ?? asset.bundleBehavior, + isSplittable: asset.isBundleSplittable ?? true, + pipeline: asset.pipeline, + }); + bundleByType.set(bundle.type, bundle); + bundlesByEntryAsset.set(asset, bundle); + bundleGraph.createAssetReference(dependency, asset, bundle); - invariant(assets.length === 1); - let bundleRoot = assets[0]; - if (dependency.specifierType !== 'url') { - asyncBundleRootGraph.addEdge( - asyncBundleRootGraph.getNodeIdByContentKey(root.id), - asyncBundleRootGraph.getNodeIdByContentKey(bundleRoot.id), - ); + // The bundle may have already been created, and the graph gave us back the original one... + if (!bundleRoots.has(bundle)) { + bundleRoots.set(bundle, [asset]); } } - actions.skipChildren(); - return; } - return; - } - let nodeId = reachableRoots.addNodeByContentKeyIfNeeded( - node.value.id, - node.value, - ); - reachableRoots.addEdge(rootNodeId, nodeId); - }, root); - } + return { + ...context, + parentNode: node, + }; + }, + }); - // Maps a given bundleRoot to the bundleRoots reachable from it, - // and the assets reachable from each of these bundleRoots - let ancestorAssets: Map< - BundleRoot, - Map | null>, - > = new Map(); - - // Reference count of each asset available within a given bundleRoot's bundle group - let assetRefsInBundleGroup: DefaultMap< - BundleRoot, - DefaultMap, - > = new DefaultMap(() => new DefaultMap(() => 0)); - - // Step 4: Determine assets that should be duplicated by computing asset availability in each bundle group - for (let nodeId of asyncBundleRootGraph.topoSort()) { - const bundleRoot = asyncBundleRootGraph.getNode(nodeId); - if (bundleRoot === 'root') continue; - invariant(bundleRoot != null); - // BundleRoots reachable from current (bundleRoot) node mapped to assets that are available from it - // should we call these reachableBundleRoots/reachableBundleRootMap instead? - let ancestors = ancestorAssets.get(bundleRoot); - - // First consider bundle group asset availability, processing only - // non-isolated bundles within that bundle group - let bundleGroupId = nullthrows(bundleRoots.get(bundleRoot))[1]; - // Map of assets in the bundle group to their refcounts - let assetRefs = assetRefsInBundleGroup.get(bundleRoot); - - for (let bundleIdInGroup of [ - bundleGroupId, - ...bundleGraph.getNodeIdsConnectedFrom(bundleGroupId), - ]) { - let bundleInGroup = nullthrows(bundleGraph.getNode(bundleIdInGroup)); - if ( - bundleInGroup.bundleBehavior === 'isolated' || - bundleInGroup.bundleBehavior === 'inline' - ) { - continue; - } - let [bundleRoot] = [...bundleInGroup.assets]; - // Assets directly connected to current bundleRoot - let assetsFromBundleRoot = reachableRoots - .getNodeIdsConnectedFrom( - reachableRoots.getNodeIdByContentKey(bundleRoot.id), - ) - .map(id => nullthrows(reachableRoots.getNode(id))); - - for (let asset of assetsFromBundleRoot) { - assetRefs.set(asset, assetRefs.get(asset) + 1); + for (let [bundle, rootAssets] of bundleRoots) { + for (let asset of rootAssets) { + bundleGraph.addEntryToBundle(asset, bundle); } } - // Enumerate bundleRoots connected to the node (parent), taking the intersection - // between the assets synchronously loaded by the parent, and those loaded by the child. - let bundleGroupAssets = new Set(assetRefs.keys()); - - let combined = ancestors - ? new Map([...bundleGroupAssets, ...ancestors.keys()].map(a => [a, null])) - : new Map([...bundleGroupAssets].map(a => [a, null])); - - let children = asyncBundleRootGraph.getNodeIdsConnectedFrom(nodeId); - - for (let childId of children) { - let child = asyncBundleRootGraph.getNode(childId); - invariant(child !== 'root' && child != null); - const availableAssets = ancestorAssets.get(child); - - if (availableAssets === undefined) { - ancestorAssets.set(child, combined); - } else { - ancestryIntersect(availableAssets, combined); - } + // If there's only one bundle, we can skip the rest of the steps. + skipOptimize = bundleRoots.size === 1; + if (skipOptimize) { + return; } - let siblingAncestors = ancestors - ? ancestryUnion(new Set(ancestors.keys()), assetRefs, bundleRoot) - : new Map([...bundleGroupAssets].map(a => [a, [bundleRoot]])); + invariant(config != null); - for (let bundleIdInGroup of bundleGraph.getNodeIdsConnectedFrom( - bundleGroupId, - )) { - let bundleInGroup = bundleGraph.getNode(bundleIdInGroup); - invariant(bundleInGroup != null && bundleInGroup.assets != null); - - let [bundleRoot] = [...bundleInGroup.assets]; - - const availableAssets = ancestorAssets.get(bundleRoot); + // Step 2: Remove asset graphs that begin with entries to other bundles. + bundleGraph.traverseBundles(bundle => { + if ( + bundle.bundleBehavior === 'inline' || + bundle.bundleBehavior === 'isolated' || + !bundle.isSplittable || + bundle.env.isIsolated() + ) { + return; + } - if (availableAssets === undefined) { - ancestorAssets.set(bundleRoot, siblingAncestors); - } else { - ancestryIntersect(availableAssets, siblingAncestors); + // Skip bundles where the entry is reachable in a parent bundle. This can occur when both synchronously and + // asynchronously importing an asset from a bundle. This asset will later be internalized into the parent. + let entries = bundle.getEntryAssets(); + let mainEntry = entries[0]; + if ( + mainEntry == null || + entries.length !== 1 || + bundleGraph.isAssetReachableFromBundle(mainEntry, bundle) + ) { + return; } - } - } - // Step 5: Place all assets into bundles or create shared bundles. Each asset - // is placed into a single bundle based on the bundle entries it is reachable from. - // This creates a maximally code split bundle graph with no duplication. - for (let asset of assets) { - // all unreliable bundleRoot assets which need to pulled in by shared bundle or other means - let reachable: Array = getReachableBundleRoots( - asset, - reachableRoots, - ).reverse(); - - // Filter out bundles from this asset's reachable array if - // bundle does not contain the asset in its ancestry - // or if any of the bundles in the ancestry have a refcount of <= 1 for that asset, - // meaning it may not be deduplicated. Otherwise, decrement all references in - // the ancestry and keep it - reachable = reachable.filter(b => { - let ancestry = ancestorAssets.get(b)?.get(asset); - if (ancestry === undefined) { - // No reachable assets from this bundle - return true; - } else if (ancestry === null) { - // Asset is reachable via the bundle - return false; - } else { + let candidates = bundleGraph.getBundlesWithAsset(mainEntry).filter( + containingBundle => + containingBundle.id !== bundle.id && + // Don't add to BundleGroups for entry bundles, as that would require + // another entry bundle depending on these conditions, making it difficult + // to predict and reference. + // TODO: reconsider this. This is only true for the global output format. + !containingBundle.needsStableName && + containingBundle.bundleBehavior !== 'inline' && + containingBundle.bundleBehavior !== 'isolated' && + containingBundle.isSplittable, + ); + + for (let candidate of candidates) { + let bundleGroups = bundleGraph.getBundleGroupsContainingBundle( + candidate, + ); if ( - ancestry.every( - bundleId => assetRefsInBundleGroup.get(bundleId).get(asset) > 1, + Array.from(bundleGroups).every( + group => + bundleGraph.getBundlesInBundleGroup(group).length < + config.maxParallelRequests, ) ) { - for (let bundleRoot of ancestry) { - assetRefsInBundleGroup - .get(bundleRoot) - .set( - asset, - assetRefsInBundleGroup.get(bundleRoot).get(asset) - 1, - ); - } - return false; + bundleGraph.createBundleReference(candidate, bundle); + bundleGraph.removeAssetGraphFromBundle(mainEntry, candidate); } - return true; } }); - let rootBundle = bundleRoots.get(asset); - if ( - rootBundle != null && - !nullthrows(bundleGraph.getNode(rootBundle[0])).env.isIsolated() - ) { - if (!bundles.has(asset.id)) { - bundles.set(asset.id, rootBundle[0]); - } + // Step 3: Remove assets that are duplicated in a parent bundle. + deduplicate(bundleGraph); + internalizeReachableAsyncDependencies(bundleGraph); + }, + optimize({bundleGraph, config}) { + // if only one bundle, no need to optimize + if (skipOptimize) { + return; + } - for (let reachableAsset of reachable) { - if (reachableAsset !== asset) { - bundleGraph.addEdge( - nullthrows(bundleRoots.get(reachableAsset))[1], - rootBundle[0], - ); - } + invariant(config != null); + + // Step 5: Find duplicated assets in different bundle groups, and separate them into their own parallel bundles. + // If multiple assets are always seen together in the same bundles, combine them together. + // If the sub-graph from an asset is >= 30kb, and the number of parallel requests in the bundle group is < 5, create a new bundle containing the sub-graph. + let candidateBundles: Map< + string, + {| + assets: Array, + sourceBundles: Set, + size: number, + |}, + > = new Map(); + + bundleGraph.traverse((node, ctx, actions) => { + if (node.type !== 'asset') { + return; } - let willInternalizeRoots = asyncBundleRootGraph - .getNodeIdsConnectedTo( - asyncBundleRootGraph.getNodeIdByContentKey(asset.id), - ) - .map(id => nullthrows(asyncBundleRootGraph.getNode(id))) - .filter(bundleRoot => { - if (bundleRoot === 'root') { - return false; - } + let asset = node.value; + let containingBundles = bundleGraph + .getBundlesWithAsset(asset) + // Don't create shared bundles from entry bundles, as that would require + // another entry bundle depending on these conditions, making it difficult + // to predict and reference. + // TODO: reconsider this. This is only true for the global output format. + // This also currently affects other bundles with stable names, e.g. service workers. + .filter(b => { + let entries = b.getEntryAssets(); return ( - reachableRoots.hasEdge( - reachableRoots.getNodeIdByContentKey(bundleRoot.id), - reachableRoots.getNodeIdByContentKey(asset.id), - ) || ancestorAssets.get(bundleRoot)?.has(asset) + !b.needsStableName && + b.isSplittable && + entries.every(entry => entry.id !== asset.id) ); - }) - .map(bundleRoot => { - // For Flow - invariant(bundleRoot !== 'root'); - return bundleRoot; }); - for (let bundleRoot of willInternalizeRoots) { - if (bundleRoot !== asset) { - let bundle = nullthrows( - bundleGraph.getNode(nullthrows(bundles.get(bundleRoot.id))), - ); - bundle.internalizedAssetIds.push(asset.id); + if (containingBundles.length > config.minBundles) { + let id = containingBundles + .map(b => b.id) + .sort() + .join(':'); + + let candidate = candidateBundles.get(id); + if (candidate) { + candidate.assets.push(asset); + for (let bundle of containingBundles) { + candidate.sourceBundles.add(bundle); + } + candidate.size += bundleGraph.getTotalSize(asset); + } else { + candidateBundles.set(id, { + assets: [asset], + sourceBundles: new Set(containingBundles), + size: bundleGraph.getTotalSize(asset), + }); } + + // Skip children from consideration since we added a parent already. + actions.skipChildren(); } - } else if (reachable.length > 0) { - let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); - let sourceBundleSet = new Set([...sourceBundles]); - let toRemove: Set = new Set(); - for (let bundleId of sourceBundles) { - // entries should not be considered in reachablility - for (let id of bundleGraph.getNodeIdsConnectedTo(bundleId)) { - if (sourceBundleSet.has(id)) { - toRemove.add(id); - } + }); + + // Sort candidates by size (consider larger bundles first), and ensure they meet the size threshold + let sortedCandidates: Array<{| + assets: Array, + sourceBundles: Set, + size: number, + |}> = Array.from(candidateBundles.values()) + .filter(bundle => bundle.size >= config.minBundleSize) + .sort((a, b) => b.size - a.size); + + for (let {assets, sourceBundles} of sortedCandidates) { + let eligibleSourceBundles = new Set(); + + for (let bundle of sourceBundles) { + // Find all bundle groups connected to the original bundles + let bundleGroups = bundleGraph.getBundleGroupsContainingBundle(bundle); + // Check if all bundle groups are within the parallel request limit + if ( + bundleGroups.every( + group => + bundleGraph.getBundlesInBundleGroup(group).length < + config.maxParallelRequests, + ) + ) { + eligibleSourceBundles.add(bundle); } } - reachable = reachable.filter( - a => !toRemove.has(nullthrows(bundles.get(a.id))), - ); - if (reachable.length < 1) { + // Do not create a shared bundle unless there are at least 2 source bundles + if (eligibleSourceBundles.size < 2) { continue; } - let key = reachable.map(a => a.id).join(','); - let bundleId = bundles.get(key); - let bundle; - if (bundleId == null) { - let firstSourceBundle = nullthrows( - bundleGraph.getNode(sourceBundles[0]), - ); - bundle = createBundle({ - target: firstSourceBundle.target, - type: firstSourceBundle.type, - env: firstSourceBundle.env, - }); - bundle.sourceBundles = sourceBundles; - bundleId = bundleGraph.addNode(bundle); - bundles.set(key, bundleId); - } else { - bundle = nullthrows(bundleGraph.getNode(bundleId)); - } - bundle.assets.add(asset); - bundle.size += asset.stats.size; + let [firstBundle] = [...eligibleSourceBundles]; + let sharedBundle = bundleGraph.createBundle({ + uniqueKey: hashString( + [...eligibleSourceBundles].map(b => b.id).join(':'), + ), + // Allow this bundle to be deduplicated. It shouldn't be further split. + // TODO: Reconsider bundle/asset flags. + isSplittable: true, + env: firstBundle.env, + target: firstBundle.target, + type: firstBundle.type, + }); + + // Remove all of the root assets from each of the original bundles + // and reference the new shared bundle. + for (let asset of assets) { + bundleGraph.addAssetGraphToBundle(asset, sharedBundle); - for (let sourceBundleId of sourceBundles) { - if (bundleId !== sourceBundleId) { - bundleGraph.addEdge(sourceBundleId, bundleId); + for (let bundle of eligibleSourceBundles) { + { + bundleGraph.createBundleReference(bundle, sharedBundle); + bundleGraph.removeAssetGraphFromBundle(asset, bundle); + } } } - dependencyBundleGraph.addNodeByContentKeyIfNeeded(String(bundleId), { - value: bundle, - type: 'bundle', - }); } - } - // Step 7: Merge any sibling bundles required by entry bundles back into the entry bundle. - // Entry bundles must be predictable, so cannot have unpredictable siblings. - for (let [bundleNodeId, bundle] of bundleGraph.nodes) { - if (bundle.sourceBundles.length > 0 && bundle.size < config.minBundleSize) { - removeBundle(bundleGraph, bundleNodeId); - } - } + // Remove assets that are duplicated between shared bundles. + deduplicate(bundleGraph); + internalizeReachableAsyncDependencies(bundleGraph); + }, +}): Bundler); - for (let entryAsset of entries.keys()) { - let entryBundleId = nullthrows(bundleRoots.get(entryAsset)?.[0]); - let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); - for (let siblingId of bundleGraph.getNodeIdsConnectedFrom(entryBundleId)) { - let sibling = nullthrows(bundleGraph.getNode(siblingId)); - if (sibling.type !== entryBundle.type) { - continue; - } - for (let asset of sibling.assets) { - entryBundle.assets.add(asset); - entryBundle.size += asset.stats.size; - } - bundleGraph.removeEdge(entryBundleId, siblingId); - if (sibling.sourceBundles.length > 1) { - let entryBundleIndex = sibling.sourceBundles.indexOf(entryBundleId); - invariant(entryBundleIndex >= 0); - sibling.sourceBundles.splice(entryBundleIndex, 1); - - if (sibling.sourceBundles.length === 1) { - let id = sibling.sourceBundles.pop(); - let bundle = nullthrows(bundleGraph.getNode(id)); - for (let asset of sibling.assets) { - bundle.assets.add(asset); - bundle.size += asset.stats.size; - } - bundleGraph.removeEdge(id, siblingId); +function deduplicate(bundleGraph: MutableBundleGraph) { + bundleGraph.traverse(node => { + if (node.type === 'asset') { + let asset = node.value; + // Search in reverse order, so bundles that are loaded keep the duplicated asset, not later ones. + // This ensures that the earlier bundle is able to execute before the later one. + let bundles = bundleGraph.getBundlesWithAsset(asset).reverse(); + for (let bundle of bundles) { + if ( + bundle.hasAsset(asset) && + bundleGraph.isAssetReachableFromBundle(asset, bundle) + ) { + bundleGraph.removeAssetGraphFromBundle(asset, bundle); } } } - } - - return { - bundleGraph, - dependencyBundleGraph, - bundleGroupBundleIds, - assetReference, - }; + }); } const CONFIG_SCHEMA: SchemaEntity = { @@ -778,70 +439,7 @@ const CONFIG_SCHEMA: SchemaEntity = { additionalProperties: false, }; -function createBundle( - opts: - | {| - target: Target, - env: Environment, - type: string, - needsStableName?: boolean, - bundleBehavior?: ?BundleBehavior, - |} - | {| - target: Target, - asset: Asset, - env?: Environment, - type?: string, - needsStableName?: boolean, - bundleBehavior?: ?BundleBehavior, - |}, -): Bundle { - if (opts.asset == null) { - return { - assets: new Set(), - internalizedAssetIds: [], - size: 0, - sourceBundles: [], - target: opts.target, - type: nullthrows(opts.type), - env: nullthrows(opts.env), - needsStableName: Boolean(opts.needsStableName), - bundleBehavior: opts.bundleBehavior, - }; - } - - let asset = nullthrows(opts.asset); - return { - assets: new Set([asset]), - internalizedAssetIds: [], - size: asset.stats.size, - sourceBundles: [], - target: opts.target, - type: opts.type ?? asset.type, - env: opts.env ?? asset.env, - needsStableName: Boolean(opts.needsStableName), - bundleBehavior: opts.bundleBehavior ?? asset.bundleBehavior, - }; -} - -function removeBundle(bundleGraph: Graph, bundleId: NodeId) { - let bundle = nullthrows(bundleGraph.getNode(bundleId)); - - for (let asset of bundle.assets) { - for (let sourceBundleId of bundle.sourceBundles) { - let sourceBundle = nullthrows(bundleGraph.getNode(sourceBundleId)); - sourceBundle.assets.add(asset); - sourceBundle.size += asset.stats.size; - } - } - - bundleGraph.removeNode(bundleId); -} - -async function loadBundlerConfig( - config: Config, - options: PluginOptions, -): Promise { +async function loadBundlerConfig(config: Config, options: PluginOptions) { let conf = await config.getConfig([], { packageKey: '@parcel/bundler-default', }); @@ -874,45 +472,58 @@ async function loadBundlerConfig( }; } -function ancestryUnion( - ancestors: Set, - assetRefs: Map, - bundleRoot: BundleRoot, -): Map | null> { - let map = new Map(); - for (let a of ancestors) { - map.set(a, null); - } - for (let [asset, refCount] of assetRefs) { - if (!ancestors.has(asset) && refCount > 1) { - map.set(asset, [bundleRoot]); +function internalizeReachableAsyncDependencies( + bundleGraph: MutableBundleGraph, +): void { + // Mark async dependencies on assets that are already available in + // the bundle as internally resolvable. This removes the dependency between + // the bundle and the bundle group providing that asset. If all connections + // to that bundle group are removed, remove that bundle group. + let asyncBundleGroups: Set = new Set(); + bundleGraph.traverse((node, _, actions) => { + if ( + node.type !== 'dependency' || + node.value.isEntry || + node.value.priority !== 'lazy' + ) { + return; } - } - return map; -} -function ancestryIntersect( - currentMap: Map | null>, - map: Map | null>, -): void { - for (let [bundleRoot, currentAssets] of currentMap) { - if (map.has(bundleRoot)) { - let assets = map.get(bundleRoot); - if (assets) { - if (currentAssets) { - currentAssets.push(...assets); - } else { - currentMap.set(bundleRoot, [...assets]); - } + if (bundleGraph.isDependencySkipped(node.value)) { + actions.skipChildren(); + return; + } + + let dependency = node.value; + if (dependency.specifierType === 'url') { + // Don't internalize dependencies on URLs, e.g. `new Worker('foo.js')` + return; + } + + let resolution = bundleGraph.getResolvedAsset(dependency); + if (resolution == null) { + return; + } + + let externalResolution = bundleGraph.resolveAsyncDependency(dependency); + if (externalResolution?.type === 'bundle_group') { + asyncBundleGroups.add(externalResolution.value); + } + + for (let bundle of bundleGraph.getBundlesWithDependency(dependency)) { + if ( + bundle.hasAsset(resolution) || + bundleGraph.isAssetReachableFromBundle(resolution, bundle) + ) { + bundleGraph.internalizeAsyncDependency(bundle, dependency); } - } else { - currentMap.delete(bundleRoot); } - } -} + }); -function getReachableBundleRoots(asset, graph): Array { - return graph - .getNodeIdsConnectedTo(graph.getNodeIdByContentKey(asset.id)) - .map(nodeId => nullthrows(graph.getNode(nodeId))); + // Remove any bundle groups that no longer have any parent bundles. + for (let bundleGroup of asyncBundleGroups) { + if (bundleGraph.getParentBundlesOfBundleGroup(bundleGroup).length === 0) { + bundleGraph.removeBundleGroup(bundleGroup); + } + } } diff --git a/packages/bundlers/experimental/package.json b/packages/bundlers/experimental/package.json new file mode 100644 index 00000000000..a12e99ffa04 --- /dev/null +++ b/packages/bundlers/experimental/package.json @@ -0,0 +1,29 @@ +{ + "name": "@parcel/bundler-experimental", + "version": "2.0.0-rc.0", + "license": "MIT", + "publishConfig": { + "access": "public" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "repository": { + "type": "git", + "url": "https://github.com/parcel-bundler/parcel.git" + }, + "main": "lib/ExperimentalBundler.js", + "source": "src/ExperimentalBundler.js", + "engines": { + "node": ">= 12.0.0", + "parcel": "^2.0.0-beta.1" + }, + "dependencies": { + "@parcel/diagnostic": "2.0.0-rc.0", + "@parcel/hash": "2.0.0-rc.0", + "@parcel/plugin": "2.0.0-rc.0", + "@parcel/utils": "2.0.0-rc.0", + "nullthrows": "^1.1.1" + } +} diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js new file mode 100644 index 00000000000..6a595b289eb --- /dev/null +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -0,0 +1,918 @@ +// @flow strict-local + +import type { + Asset, + Bundle as LegacyBundle, + BundleBehavior, + BundleGroup, + Dependency, + Environment, + Config, + MutableBundleGraph, + PluginOptions, + Target, +} from '@parcel/types'; +import type {NodeId} from '@parcel/graph'; +import type {SchemaEntity} from '@parcel/utils'; +import {ContentGraph, Graph} from '@parcel/graph'; + +import invariant from 'assert'; +import {ALL_EDGE_TYPES} from '@parcel/graph'; +import {Bundler} from '@parcel/plugin'; +import {validateSchema, DefaultMap} from '@parcel/utils'; +import nullthrows from 'nullthrows'; +import {encodeJSONKeyComponent} from '@parcel/diagnostic'; + +type BundlerConfig = {| + http?: number, + minBundles?: number, + minBundleSize?: number, + maxParallelRequests?: number, +|}; + +type ResolvedBundlerConfig = {| + minBundles: number, + minBundleSize: number, + maxParallelRequests: number, +|}; + +// Default options by http version. +const HTTP_OPTIONS = { + '1': { + minBundles: 1, + minBundleSize: 30000, + maxParallelRequests: 6, + }, + '2': { + minBundles: 1, + minBundleSize: 20000, + maxParallelRequests: 25, + }, +}; + +type AssetId = string; +type BundleRoot = Asset; +export type Bundle = {| + assets: Set, + internalizedAssetIds: Array, + bundleBehavior?: ?BundleBehavior, + needsStableName: boolean, + size: number, + sourceBundles: Array, + target: Target, + env: Environment, + type: string, +|}; + +const dependencyPriorityEdges = { + sync: 1, + parallel: 2, + lazy: 3, +}; + +type DependencyBundleGraph = ContentGraph< + | {| + value: Bundle, + type: 'bundle', + |} + | {| + value: Dependency, + type: 'dependency', + |}, + number, +>; +type IdealGraph = {| + dependencyBundleGraph: DependencyBundleGraph, + bundleGraph: Graph, + bundleGroupBundleIds: Array, + assetReference: DefaultMap>, +|}; + +export default (new Bundler({ + loadConfig({config, options}) { + return loadBundlerConfig(config, options); + }, + + bundle({bundleGraph, config}) { + decorateLegacyGraph(createIdealGraph(bundleGraph, config), bundleGraph); + }, + optimize() {}, +}): Bundler); + +function decorateLegacyGraph( + idealGraph: IdealGraph, + bundleGraph: MutableBundleGraph, +): void { + let idealBundleToLegacyBundle: Map = new Map(); + + let { + bundleGraph: idealBundleGraph, + dependencyBundleGraph, + bundleGroupBundleIds, + } = idealGraph; + let entryBundleToBundleGroup: Map = new Map(); + + // Step 1: Create bundle groups, bundles, and shared bundles and add assets to them + for (let [bundleNodeId, idealBundle] of idealBundleGraph.nodes) { + let [entryAsset] = [...idealBundle.assets]; + let bundleGroup; + let bundle; + + if (bundleGroupBundleIds.includes(bundleNodeId)) { + let dependencies = dependencyBundleGraph + .getNodeIdsConnectedTo( + dependencyBundleGraph.getNodeIdByContentKey(String(bundleNodeId)), + // $FlowFixMe[incompatible-call] + ALL_EDGE_TYPES, + ) + .map(nodeId => { + let dependency = nullthrows(dependencyBundleGraph.getNode(nodeId)); + invariant(dependency.type === 'dependency'); + return dependency.value; + }); + for (let dependency of dependencies) { + bundleGroup = bundleGraph.createBundleGroup( + dependency, + idealBundle.target, + ); + } + invariant(bundleGroup); + entryBundleToBundleGroup.set(bundleNodeId, bundleGroup); + + bundle = nullthrows( + bundleGraph.createBundle({ + entryAsset, + needsStableName: idealBundle.needsStableName, + bundleBehavior: idealBundle.bundleBehavior, + target: idealBundle.target, + }), + ); + + bundleGraph.addBundleToBundleGroup(bundle, bundleGroup); + } else if (idealBundle.sourceBundles.length > 0) { + bundle = nullthrows( + bundleGraph.createBundle({ + uniqueKey: + [...idealBundle.assets].map(asset => asset.id).join(',') + + idealBundle.sourceBundles.join(','), + needsStableName: idealBundle.needsStableName, + bundleBehavior: idealBundle.bundleBehavior, + type: idealBundle.type, + target: idealBundle.target, + env: idealBundle.env, + }), + ); + } else { + bundle = nullthrows( + bundleGraph.createBundle({ + entryAsset, + needsStableName: idealBundle.needsStableName, + bundleBehavior: idealBundle.bundleBehavior, + target: idealBundle.target, + }), + ); + } + + idealBundleToLegacyBundle.set(idealBundle, bundle); + + for (let asset of idealBundle.assets) { + bundleGraph.addAssetToBundle(asset, bundle); + } + } + // Step 2: Internalize dependencies for bundles + for (let [, idealBundle] of idealBundleGraph.nodes) { + let bundle = nullthrows(idealBundleToLegacyBundle.get(idealBundle)); + for (let internalized of idealBundle.internalizedAssetIds) { + let incomingDeps = bundleGraph.getIncomingDependencies( + bundleGraph.getAssetById(internalized), + ); + for (let incomingDep of incomingDeps) { + if ( + incomingDep.priority === 'lazy' && + bundle.hasDependency(incomingDep) + ) { + bundleGraph.internalizeAsyncDependency(bundle, incomingDep); + } + } + } + } + // Step 3: Add bundles to their bundle groups + for (let [bundleId, bundleGroup] of entryBundleToBundleGroup) { + let outboundNodeIds = idealBundleGraph.getNodeIdsConnectedFrom(bundleId); + for (let id of outboundNodeIds) { + let siblingBundle = nullthrows(idealBundleGraph.getNode(id)); + let legacySiblingBundle = nullthrows( + idealBundleToLegacyBundle.get(siblingBundle), + ); + bundleGraph.addBundleToBundleGroup(legacySiblingBundle, bundleGroup); + } + } + + // Step 4: Add references to all bundles + for (let [asset, references] of idealGraph.assetReference) { + for (let [dependency, bundle] of references) { + let legacyBundle = nullthrows(idealBundleToLegacyBundle.get(bundle)); + bundleGraph.createAssetReference(dependency, asset, legacyBundle); + } + } +} + +function createIdealGraph( + assetGraph: MutableBundleGraph, + config: ResolvedBundlerConfig, +): IdealGraph { + // Asset to the bundle and group it's an entry of + let bundleRoots: Map = new Map(); + let bundles: Map = new Map(); + let dependencyBundleGraph: DependencyBundleGraph = new ContentGraph(); + let assetReference: DefaultMap< + Asset, + Array<[Dependency, Bundle]>, + > = new DefaultMap(() => []); + + // bundleRoot to all bundleRoot descendants + let reachableBundles: DefaultMap< + BundleRoot, + Set, + > = new DefaultMap(() => new Set()); + + let bundleGraph: Graph = new Graph(); + let stack: Array<[BundleRoot, NodeId]> = []; + + // bundleGraph that models bundleRoots and async deps only + let asyncBundleRootGraph: ContentGraph< + BundleRoot | 'root', + > = new ContentGraph(); + let bundleGroupBundleIds: Array = []; + + // Step 1: Find and create bundles for entries from assetGraph + let entries: Map = new Map(); + assetGraph.traverse((node, context, actions) => { + if (node.type !== 'asset') { + return node; + } + + invariant( + context != null && context.type === 'dependency' && context.value.isEntry, + ); + entries.set(node.value, context.value); + actions.skipChildren(); + }); + + let rootNodeId = nullthrows(asyncBundleRootGraph.addNode('root')); + asyncBundleRootGraph.setRootNodeId(rootNodeId); + + for (let [asset, dependency] of entries) { + let bundle = createBundle({ + asset, + target: nullthrows(dependency.target), + needsStableName: dependency.isEntry, + }); + let nodeId = bundleGraph.addNode(bundle); + bundles.set(asset.id, nodeId); + bundleRoots.set(asset, [nodeId, nodeId]); + asyncBundleRootGraph.addEdge( + rootNodeId, + asyncBundleRootGraph.addNodeByContentKey(asset.id, asset), + ); + + dependencyBundleGraph.addEdge( + dependencyBundleGraph.addNodeByContentKeyIfNeeded(dependency.id, { + value: dependency, + type: 'dependency', + }), + dependencyBundleGraph.addNodeByContentKeyIfNeeded(String(nodeId), { + value: bundle, + type: 'bundle', + }), + dependencyPriorityEdges[dependency.priority], + ); + bundleGroupBundleIds.push(nodeId); + } + + let assets = []; + + // Step 2: Traverse the asset graph and create bundles for asset type changes and async dependencies, + // only adding the entry asset of each bundle, not the subgraph. + assetGraph.traverse({ + enter(node, context) { + if (node.type === 'asset') { + assets.push(node.value); + + let bundleIdTuple = bundleRoots.get(node.value); + if (bundleIdTuple) { + // Push to the stack when a new bundle is created + stack.push([node.value, bundleIdTuple[1]]); + } + } else if (node.type === 'dependency') { + if (context == null) { + return node; + } + let dependency = node.value; + + invariant(context?.type === 'asset'); + let parentAsset = context.value; + + let assets = assetGraph.getDependencyAssets(dependency); + if (assets.length === 0) { + return node; + } + + invariant(assets.length === 1); + let childAsset = assets[0]; + if ( + dependency.priority === 'lazy' || + childAsset.bundleBehavior === 'isolated' + ) { + let bundleId = bundles.get(childAsset.id); + let bundle; + if (bundleId == null) { + bundle = createBundle({ + asset: childAsset, + target: nullthrows(bundleGraph.getNode(stack[0][1])).target, + needsStableName: + dependency.bundleBehavior === 'inline' || + childAsset.bundleBehavior === 'inline' + ? false + : dependency.isEntry || dependency.needsStableName, + bundleBehavior: + dependency.bundleBehavior ?? childAsset.bundleBehavior, + }); + bundleId = bundleGraph.addNode(bundle); + bundles.set(childAsset.id, bundleId); + bundleRoots.set(childAsset, [bundleId, bundleId]); + bundleGroupBundleIds.push(bundleId); + } else { + bundle = nullthrows(bundleGraph.getNode(bundleId)); + } + + dependencyBundleGraph.addEdge( + dependencyBundleGraph.addNodeByContentKeyIfNeeded(dependency.id, { + value: dependency, + type: 'dependency', + }), + dependencyBundleGraph.addNodeByContentKeyIfNeeded( + String(bundleId), + { + value: bundle, + type: 'bundle', + }, + ), + dependencyPriorityEdges[dependency.priority], + ); + + // Walk up the stack until we hit a different asset type + // and mark each bundle as reachable from every parent bundle + for (let i = stack.length - 1; i >= 0; i--) { + let [stackAsset] = stack[i]; + if ( + stackAsset.type !== childAsset.type || + stackAsset.env.context !== childAsset.env.context || + stackAsset.env.isIsolated() + ) { + break; + } + reachableBundles.get(stackAsset).add(childAsset); + } + return node; + } + if ( + parentAsset.type !== childAsset.type || + childAsset.bundleBehavior === 'inline' + ) { + let [, bundleGroupNodeId] = nullthrows(stack[stack.length - 1]); + let bundleGroup = nullthrows(bundleGraph.getNode(bundleGroupNodeId)); + let bundle = createBundle({ + asset: childAsset, + target: bundleGroup.target, + needsStableName: dependency.bundleBehavior === 'inline', + }); + let bundleId = bundleGraph.addNode(bundle); + bundles.set(childAsset.id, bundleId); + bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); + + dependencyBundleGraph.addEdge( + dependencyBundleGraph.addNodeByContentKeyIfNeeded(dependency.id, { + value: dependency, + type: 'dependency', + }), + dependencyBundleGraph.addNodeByContentKeyIfNeeded( + String(bundleId), + { + value: bundle, + type: 'bundle', + }, + ), + dependencyPriorityEdges.parallel, + ); + // Add an edge from the bundle group entry to the new bundle. + // This indicates that the bundle is loaded together with the entry + bundleGraph.addEdge(bundleGroupNodeId, bundleId); + assetReference.get(childAsset).push([dependency, bundle]); + return node; + } + } + return node; + }, + exit(node) { + if (stack[stack.length - 1]?.[0] === node.value) { + stack.pop(); + } + }, + }); + + // Step 3: Determine reachability for every asset from each bundleRoot. + // This is later used to determine which bundles to place each asset in. + for (let [root] of bundleRoots) { + if (!entries.has(root)) { + asyncBundleRootGraph.addNodeByContentKey(root.id, root); + } + } + + // Models bundleRoots and the assets that require it synchronously + let reachableRoots: ContentGraph = new ContentGraph(); + for (let [root] of bundleRoots) { + let rootNodeId = reachableRoots.addNodeByContentKeyIfNeeded(root.id, root); + assetGraph.traverse((node, isAsync, actions) => { + if (node.value === root) { + return; + } + + if (node.type === 'dependency') { + let dependency = node.value; + if (dependencyBundleGraph.hasContentKey(dependency.id)) { + if (dependency.priority === 'lazy') { + let assets = assetGraph.getDependencyAssets(dependency); + if (assets.length === 0) { + return node; + } + + invariant(assets.length === 1); + let bundleRoot = assets[0]; + if (dependency.specifierType !== 'url') { + asyncBundleRootGraph.addEdge( + asyncBundleRootGraph.getNodeIdByContentKey(root.id), + asyncBundleRootGraph.getNodeIdByContentKey(bundleRoot.id), + ); + } + } + actions.skipChildren(); + return; + } + return; + } + + let nodeId = reachableRoots.addNodeByContentKeyIfNeeded( + node.value.id, + node.value, + ); + reachableRoots.addEdge(rootNodeId, nodeId); + }, root); + } + + // Maps a given bundleRoot to the bundleRoots reachable from it, + // and the assets reachable from each of these bundleRoots + let ancestorAssets: Map< + BundleRoot, + Map | null>, + > = new Map(); + + // Reference count of each asset available within a given bundleRoot's bundle group + let assetRefsInBundleGroup: DefaultMap< + BundleRoot, + DefaultMap, + > = new DefaultMap(() => new DefaultMap(() => 0)); + + // Step 4: Determine assets that should be duplicated by computing asset availability in each bundle group + for (let nodeId of asyncBundleRootGraph.topoSort()) { + const bundleRoot = asyncBundleRootGraph.getNode(nodeId); + if (bundleRoot === 'root') continue; + invariant(bundleRoot != null); + // BundleRoots reachable from current (bundleRoot) node mapped to assets that are available from it + // should we call these reachableBundleRoots/reachableBundleRootMap instead? + let ancestors = ancestorAssets.get(bundleRoot); + + // First consider bundle group asset availability, processing only + // non-isolated bundles within that bundle group + let bundleGroupId = nullthrows(bundleRoots.get(bundleRoot))[1]; + // Map of assets in the bundle group to their refcounts + let assetRefs = assetRefsInBundleGroup.get(bundleRoot); + + for (let bundleIdInGroup of [ + bundleGroupId, + ...bundleGraph.getNodeIdsConnectedFrom(bundleGroupId), + ]) { + let bundleInGroup = nullthrows(bundleGraph.getNode(bundleIdInGroup)); + if ( + bundleInGroup.bundleBehavior === 'isolated' || + bundleInGroup.bundleBehavior === 'inline' + ) { + continue; + } + let [bundleRoot] = [...bundleInGroup.assets]; + // Assets directly connected to current bundleRoot + let assetsFromBundleRoot = reachableRoots + .getNodeIdsConnectedFrom( + reachableRoots.getNodeIdByContentKey(bundleRoot.id), + ) + .map(id => nullthrows(reachableRoots.getNode(id))); + + for (let asset of assetsFromBundleRoot) { + assetRefs.set(asset, assetRefs.get(asset) + 1); + } + } + + // Enumerate bundleRoots connected to the node (parent), taking the intersection + // between the assets synchronously loaded by the parent, and those loaded by the child. + let bundleGroupAssets = new Set(assetRefs.keys()); + + let combined = ancestors + ? new Map([...bundleGroupAssets, ...ancestors.keys()].map(a => [a, null])) + : new Map([...bundleGroupAssets].map(a => [a, null])); + + let children = asyncBundleRootGraph.getNodeIdsConnectedFrom(nodeId); + + for (let childId of children) { + let child = asyncBundleRootGraph.getNode(childId); + invariant(child !== 'root' && child != null); + const availableAssets = ancestorAssets.get(child); + + if (availableAssets === undefined) { + ancestorAssets.set(child, combined); + } else { + ancestryIntersect(availableAssets, combined); + } + } + + let siblingAncestors = ancestors + ? ancestryUnion(new Set(ancestors.keys()), assetRefs, bundleRoot) + : new Map([...bundleGroupAssets].map(a => [a, [bundleRoot]])); + + for (let bundleIdInGroup of bundleGraph.getNodeIdsConnectedFrom( + bundleGroupId, + )) { + let bundleInGroup = bundleGraph.getNode(bundleIdInGroup); + invariant(bundleInGroup != null && bundleInGroup.assets != null); + + let [bundleRoot] = [...bundleInGroup.assets]; + + const availableAssets = ancestorAssets.get(bundleRoot); + + if (availableAssets === undefined) { + ancestorAssets.set(bundleRoot, siblingAncestors); + } else { + ancestryIntersect(availableAssets, siblingAncestors); + } + } + } + + // Step 5: Place all assets into bundles or create shared bundles. Each asset + // is placed into a single bundle based on the bundle entries it is reachable from. + // This creates a maximally code split bundle graph with no duplication. + for (let asset of assets) { + // Unreliable bundleRoot assets which need to pulled in by shared bundles or other means + let reachable: Array = getReachableBundleRoots( + asset, + reachableRoots, + ).reverse(); + + // Filter out bundles from this asset's reachable array if + // bundle does not contain the asset in its ancestry + // or if any of the bundles in the ancestry have a refcount of <= 1 for that asset, + // meaning it may not be deduplicated. Otherwise, decrement all references in + // the ancestry and keep it + reachable = reachable.filter(b => { + let ancestry = ancestorAssets.get(b)?.get(asset); + if (ancestry === undefined) { + // No reachable assets from this bundle + return true; + } else if (ancestry === null) { + // Asset is reachable via the bundle + return false; + } else { + if ( + ancestry.every( + bundleId => assetRefsInBundleGroup.get(bundleId).get(asset) > 1, + ) + ) { + for (let bundleRoot of ancestry) { + assetRefsInBundleGroup + .get(bundleRoot) + .set( + asset, + assetRefsInBundleGroup.get(bundleRoot).get(asset) - 1, + ); + } + return false; + } + return true; + } + }); + + let rootBundle = bundleRoots.get(asset); + if ( + rootBundle != null && + !nullthrows(bundleGraph.getNode(rootBundle[0])).env.isIsolated() + ) { + if (!bundles.has(asset.id)) { + bundles.set(asset.id, rootBundle[0]); + } + + for (let reachableAsset of reachable) { + if (reachableAsset !== asset) { + bundleGraph.addEdge( + nullthrows(bundleRoots.get(reachableAsset))[1], + rootBundle[0], + ); + } + } + + let willInternalizeRoots = asyncBundleRootGraph + .getNodeIdsConnectedTo( + asyncBundleRootGraph.getNodeIdByContentKey(asset.id), + ) + .map(id => nullthrows(asyncBundleRootGraph.getNode(id))) + .filter(bundleRoot => { + if (bundleRoot === 'root') { + return false; + } + + return ( + reachableRoots.hasEdge( + reachableRoots.getNodeIdByContentKey(bundleRoot.id), + reachableRoots.getNodeIdByContentKey(asset.id), + ) || ancestorAssets.get(bundleRoot)?.has(asset) + ); + }) + .map(bundleRoot => { + // For Flow + invariant(bundleRoot !== 'root'); + return bundleRoot; + }); + + for (let bundleRoot of willInternalizeRoots) { + if (bundleRoot !== asset) { + let bundle = nullthrows( + bundleGraph.getNode(nullthrows(bundles.get(bundleRoot.id))), + ); + bundle.internalizedAssetIds.push(asset.id); + } + } + } else if (reachable.length > 0) { + let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); + let sourceBundleSet = new Set([...sourceBundles]); + let toRemove: Set = new Set(); + for (let bundleId of sourceBundles) { + // entries should not be considered in reachablility + for (let id of bundleGraph.getNodeIdsConnectedTo(bundleId)) { + if (sourceBundleSet.has(id)) { + toRemove.add(id); + } + } + } + reachable = reachable.filter( + a => !toRemove.has(nullthrows(bundles.get(a.id))), + ); + + if (reachable.length < 1) { + continue; + } + + let key = reachable.map(a => a.id).join(','); + let bundleId = bundles.get(key); + let bundle; + if (bundleId == null) { + let firstSourceBundle = nullthrows( + bundleGraph.getNode(sourceBundles[0]), + ); + bundle = createBundle({ + target: firstSourceBundle.target, + type: firstSourceBundle.type, + env: firstSourceBundle.env, + }); + bundle.sourceBundles = sourceBundles; + bundleId = bundleGraph.addNode(bundle); + bundles.set(key, bundleId); + } else { + bundle = nullthrows(bundleGraph.getNode(bundleId)); + } + bundle.assets.add(asset); + bundle.size += asset.stats.size; + + for (let sourceBundleId of sourceBundles) { + if (bundleId !== sourceBundleId) { + bundleGraph.addEdge(sourceBundleId, bundleId); + } + } + dependencyBundleGraph.addNodeByContentKeyIfNeeded(String(bundleId), { + value: bundle, + type: 'bundle', + }); + } + } + + // Step 7: Merge any sibling bundles required by entry bundles back into the entry bundle. + // Entry bundles must be predictable, so cannot have unpredictable siblings. + for (let [bundleNodeId, bundle] of bundleGraph.nodes) { + if (bundle.sourceBundles.length > 0 && bundle.size < config.minBundleSize) { + removeBundle(bundleGraph, bundleNodeId); + } + } + + for (let entryAsset of entries.keys()) { + let entryBundleId = nullthrows(bundleRoots.get(entryAsset)?.[0]); + let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); + for (let siblingId of bundleGraph.getNodeIdsConnectedFrom(entryBundleId)) { + let sibling = nullthrows(bundleGraph.getNode(siblingId)); + if (sibling.type !== entryBundle.type) { + continue; + } + for (let asset of sibling.assets) { + entryBundle.assets.add(asset); + entryBundle.size += asset.stats.size; + } + bundleGraph.removeEdge(entryBundleId, siblingId); + if (sibling.sourceBundles.length > 1) { + let entryBundleIndex = sibling.sourceBundles.indexOf(entryBundleId); + invariant(entryBundleIndex >= 0); + sibling.sourceBundles.splice(entryBundleIndex, 1); + + if (sibling.sourceBundles.length === 1) { + let id = sibling.sourceBundles.pop(); + let bundle = nullthrows(bundleGraph.getNode(id)); + for (let asset of sibling.assets) { + bundle.assets.add(asset); + bundle.size += asset.stats.size; + } + bundleGraph.removeEdge(id, siblingId); + } + } + } + } + + return { + bundleGraph, + dependencyBundleGraph, + bundleGroupBundleIds, + assetReference, + }; +} + +const CONFIG_SCHEMA: SchemaEntity = { + type: 'object', + properties: { + http: { + type: 'number', + enum: Object.keys(HTTP_OPTIONS).map(k => Number(k)), + }, + minBundles: { + type: 'number', + }, + minBundleSize: { + type: 'number', + }, + maxParallelRequests: { + type: 'number', + }, + }, + additionalProperties: false, +}; + +function createBundle( + opts: + | {| + target: Target, + env: Environment, + type: string, + needsStableName?: boolean, + bundleBehavior?: ?BundleBehavior, + |} + | {| + target: Target, + asset: Asset, + env?: Environment, + type?: string, + needsStableName?: boolean, + bundleBehavior?: ?BundleBehavior, + |}, +): Bundle { + if (opts.asset == null) { + return { + assets: new Set(), + internalizedAssetIds: [], + size: 0, + sourceBundles: [], + target: opts.target, + type: nullthrows(opts.type), + env: nullthrows(opts.env), + needsStableName: Boolean(opts.needsStableName), + bundleBehavior: opts.bundleBehavior, + }; + } + + let asset = nullthrows(opts.asset); + return { + assets: new Set([asset]), + internalizedAssetIds: [], + size: asset.stats.size, + sourceBundles: [], + target: opts.target, + type: opts.type ?? asset.type, + env: opts.env ?? asset.env, + needsStableName: Boolean(opts.needsStableName), + bundleBehavior: opts.bundleBehavior ?? asset.bundleBehavior, + }; +} + +function removeBundle(bundleGraph: Graph, bundleId: NodeId) { + let bundle = nullthrows(bundleGraph.getNode(bundleId)); + + for (let asset of bundle.assets) { + for (let sourceBundleId of bundle.sourceBundles) { + let sourceBundle = nullthrows(bundleGraph.getNode(sourceBundleId)); + sourceBundle.assets.add(asset); + sourceBundle.size += asset.stats.size; + } + } + + bundleGraph.removeNode(bundleId); +} + +async function loadBundlerConfig( + config: Config, + options: PluginOptions, +): Promise { + let conf = await config.getConfig([], { + packageKey: '@parcel/bundler-default', + }); + if (!conf) { + return HTTP_OPTIONS['2']; + } + + invariant(conf?.contents != null); + + validateSchema.diagnostic( + CONFIG_SCHEMA, + { + data: conf?.contents, + source: await options.inputFS.readFile(conf.filePath, 'utf8'), + filePath: conf.filePath, + prependKey: `/${encodeJSONKeyComponent('@parcel/bundler-default')}`, + }, + '@parcel/bundler-default', + 'Invalid config for @parcel/bundler-default', + ); + + let http = conf.contents.http ?? 2; + let defaults = HTTP_OPTIONS[http]; + + return { + minBundles: conf.contents.minBundles ?? defaults.minBundles, + minBundleSize: conf.contents.minBundleSize ?? defaults.minBundleSize, + maxParallelRequests: + conf.contents.maxParallelRequests ?? defaults.maxParallelRequests, + }; +} + +function ancestryUnion( + ancestors: Set, + assetRefs: Map, + bundleRoot: BundleRoot, +): Map | null> { + let map = new Map(); + for (let a of ancestors) { + map.set(a, null); + } + for (let [asset, refCount] of assetRefs) { + if (!ancestors.has(asset) && refCount > 1) { + map.set(asset, [bundleRoot]); + } + } + return map; +} + +function ancestryIntersect( + currentMap: Map | null>, + map: Map | null>, +): void { + for (let [bundleRoot, currentAssets] of currentMap) { + if (map.has(bundleRoot)) { + let assets = map.get(bundleRoot); + if (assets) { + if (currentAssets) { + currentAssets.push(...assets); + } else { + currentMap.set(bundleRoot, [...assets]); + } + } + } else { + currentMap.delete(bundleRoot); + } + } +} + +function getReachableBundleRoots(asset, graph): Array { + return graph + .getNodeIdsConnectedTo(graph.getNodeIdByContentKey(asset.id)) + .map(nodeId => nullthrows(graph.getNode(nodeId))); +} diff --git a/packages/core/integration-tests/experimental-bundler-config.json b/packages/core/integration-tests/experimental-bundler-config.json new file mode 100644 index 00000000000..709003dec50 --- /dev/null +++ b/packages/core/integration-tests/experimental-bundler-config.json @@ -0,0 +1,4 @@ +{ + "extends": "@parcel/config-default", + "bundler": "@parcel/bundler-experimental" +} diff --git a/packages/core/integration-tests/test/html.js b/packages/core/integration-tests/test/html.js index aee27a2f5ee..bf2161d089c 100644 --- a/packages/core/integration-tests/test/html.js +++ b/packages/core/integration-tests/test/html.js @@ -1,11 +1,12 @@ import assert from 'assert'; import { - bundle, - bundler, + bundle as _bundle, + bundler as _bundler, assertBundles, removeDistDirectory, distDir, getNextBuild, + mergeParcelOptions, run, inputFS, outputFS, @@ -14,2599 +15,2668 @@ import { } from '@parcel/test-utils'; import path from 'path'; -describe('html', function() { - beforeEach(async () => { - await removeDistDirectory(); - }); - - let subscription; - afterEach(async () => { - if (subscription) { - await subscription.unsubscribe(); - subscription = null; - } - }); - - it('should support bundling HTML', async () => { - let b = await bundle(path.join(__dirname, '/integration/html/index.html')); - - assertBundles(b, [ - { - type: 'css', - assets: ['index.html'], - }, - { - name: 'index.html', - assets: ['index.html'], - }, - { - name: 'index.html', - assets: ['index.html'], - }, - { - type: 'png', - assets: ['100x100.png'], - }, - { - type: 'svg', - assets: ['icons.svg'], - }, - { - type: 'css', - assets: ['index.css'], - }, - { - type: 'html', - assets: ['other.html'], - }, - { - type: 'js', - assets: ['index.js'], - }, - ]); - - let files = await outputFS.readdir(distDir); - let html = await outputFS.readFile( - path.join(distDir, 'index.html'), - 'utf8', +const configSpecifiers = [ + '@parcel/config-default', + path.resolve('experimental-bundler-config.json'), +]; + +for (let config of configSpecifiers) { + const bundle = (name, opts = {}) => { + return _bundle( + name, + // $FlowFixMe + mergeParcelOptions( + { + defaultConfig: config, + }, + opts, + ), ); - for (let file of files) { - if (file !== 'index.html' && path.extname(file) !== '.map') { - assert(html.includes(file)); - } - } + }; - assert(html.includes('#hash_link')); - assert(html.includes('mailto:someone@acme.com')); - assert(html.includes('tel:+33636757575')); - assert(html.includes('https://unpkg.com/parcel-bundler')); - - let iconsBundle = b.getBundles().find(b => b.name.startsWith('icons')); - assert( - html.includes('/' + path.basename(iconsBundle.filePath) + '#icon-code'), + const bundler = (name, opts = {}) => { + return _bundler( + name, + // $FlowFixMe + mergeParcelOptions( + { + defaultConfig: config, + }, + opts, + ), ); + }; - let value = null; - await run(b, { - alert: v => (value = v), + describe('html with ' + config, function() { + beforeEach(async () => { + await removeDistDirectory(); }); - assert.equal(value, 'Hi'); - }); - - it('should support pkg#source array as entrypoints', async () => { - let b = await bundle( - path.join(__dirname, '/integration/html-pkg-source-array'), - ); - assertBundles(b, [ - { - name: 'a.html', - assets: ['a.html'], - }, - { - name: 'b.html', - assets: ['b.html'], - }, - ]); - - assert(await outputFS.exists(path.join(distDir, 'a.html'), 'utf8')); - assert(await outputFS.exists(path.join(distDir, 'b.html'), 'utf8')); - }); + let subscription; + afterEach(async () => { + if (subscription) { + await subscription.unsubscribe(); + subscription = null; + } + }); - it('should find href attr when not first', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-attr-order/index.html'), - ); + it('should support bundling HTML', async () => { + let b = await bundle( + path.join(__dirname, '/integration/html/index.html'), + ); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - { - type: 'html', - assets: ['other.html'], - }, - ]); - }); + assertBundles(b, [ + { + type: 'css', + assets: ['index.html'], + }, + { + name: 'index.html', + assets: ['index.html'], + }, + { + name: 'index.html', + assets: ['index.html'], + }, + { + type: 'png', + assets: ['100x100.png'], + }, + { + type: 'svg', + assets: ['icons.svg'], + }, + { + type: 'css', + assets: ['index.css'], + }, + { + type: 'html', + assets: ['other.html'], + }, + { + type: 'js', + assets: ['index.js'], + }, + ]); - it('should insert empty script tag for HMR', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-no-js/index.html'), - { - hmrOptions: {}, - }, - ); + let files = await outputFS.readdir(distDir); + let html = await outputFS.readFile( + path.join(distDir, 'index.html'), + 'utf8', + ); + for (let file of files) { + if (file !== 'index.html' && path.extname(file) !== '.map') { + assert(html.includes(file)); + } + } - assertBundles(b, [ - { - type: 'js', - assets: ['index.html'], - }, - { - name: 'index.html', - assets: ['index.html'], - }, - ]); - }); + assert(html.includes('#hash_link')); + assert(html.includes('mailto:someone@acme.com')); + assert(html.includes('tel:+33636757575')); + assert(html.includes('https://unpkg.com/parcel-bundler')); - it('should support canonical links', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-canonical/index.html'), - ); + let iconsBundle = b.getBundles().find(b => b.name.startsWith('icons')); + assert( + html.includes('/' + path.basename(iconsBundle.filePath) + '#icon-code'), + ); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - ]); + let value = null; + await run(b, { + alert: v => (value = v), + }); + assert.equal(value, 'Hi'); + }); - let html = await outputFS.readFile( - path.join(distDir, 'index.html'), - 'utf8', - ); + it('should support pkg#source array as entrypoints', async () => { + let b = await bundle( + path.join(__dirname, '/integration/html-pkg-source-array'), + ); - assert(//.test(html)); - }); + assertBundles(b, [ + { + name: 'a.html', + assets: ['a.html'], + }, + { + name: 'b.html', + assets: ['b.html'], + }, + ]); - it('should support meta tags', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-meta/index.html'), - ); + assert(await outputFS.exists(path.join(distDir, 'a.html'), 'utf8')); + assert(await outputFS.exists(path.join(distDir, 'b.html'), 'utf8')); + }); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - { - name: 'logo.svg', - assets: ['logo.svg'], - }, - ]); - - let html = await outputFS.readFile( - path.join(distDir, 'index.html'), - 'utf8', - ); - assert(html.includes(``)); - assert(html.includes(``)); - assert( - html.includes( - ``, - ), - ); - }); + it('should find href attr when not first', async function() { + let b = await bundle( + path.join(__dirname, '/integration/html-attr-order/index.html'), + ); - it('should insert sibling CSS bundles for JS files in the HEAD', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-css/index.html'), - ); + assertBundles(b, [ + { + name: 'index.html', + assets: ['index.html'], + }, + { + type: 'html', + assets: ['other.html'], + }, + ]); + }); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - { - type: 'js', - assets: ['index.js'], - }, - { - type: 'css', - assets: ['index.css'], - }, - ]); - - let html = await outputFS.readFile( - path.join(distDir, 'index.html'), - 'utf8', - ); - assert( - //.test(html), - ); - }); + it('should insert empty script tag for HMR', async function() { + let b = await bundle( + path.join(__dirname, '/integration/html-no-js/index.html'), + { + hmrOptions: {}, + }, + ); - it('should insert sibling bundles before body element if no HEAD', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-css-head/index.html'), - ); + assertBundles(b, [ + { + type: 'js', + assets: ['index.html'], + }, + { + name: 'index.html', + assets: ['index.html'], + }, + ]); + }); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - { - type: 'js', - assets: ['index.js'], - }, - { - type: 'css', - assets: ['index.css'], - }, - ]); - - let html = await outputFS.readFile( - path.join(distDir, 'index.html'), - 'utf8', - ); - assert( - /\s*\s*/.test( - html, - ), - ); - }); + it('should support canonical links', async function() { + let b = await bundle( + path.join(__dirname, '/integration/html-canonical/index.html'), + ); - it('should insert sibling bundles after doctype if no html', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-css-doctype/index.html'), - ); + assertBundles(b, [ + { + name: 'index.html', + assets: ['index.html'], + }, + ]); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - { - type: 'js', - assets: ['index.js'], - }, - { - type: 'css', - assets: ['index.css'], - }, - ]); - - let html = await outputFS.readFile( - path.join(distDir, 'index.html'), - 'utf8', - ); - assert( - /^\s*\s*\s*'), - ); + assert.equal( + html.match( + //g, + ).length, + 1, + ); - // mergeStyles - assert(html.includes('')); + assert.equal( + html.match(/'), - ); - }); + let html = await outputFS.readFile(outputFile, 'utf8'); + assert(html.includes('Other page')); + }); - it('should not prepend the public path to hash links', async function() { - await bundle(path.join(__dirname, '/integration/html/index.html')); + it('should work with an empty html file', async function() { + let inputFile = path.join( + __dirname, + '/integration/html-empty/index.html', + ); + await bundle(inputFile, { + defaultTargetOptions: { + shouldOptimize: false, + }, + }); - let html = await outputFS.readFile( - path.join(distDir, 'index.html'), - 'utf8', - ); - assert(html.includes('')); - }); + let outputFile = path.join(distDir, 'index.html'); + let html = await outputFS.readFile(outputFile, 'utf8'); + assert.equal(html.length, 0); + }); - it('should detect virtual paths', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-virtualpath/index.html'), - ); + it('should work with an invalid html file', async function() { + let inputFile = path.join( + __dirname, + '/integration/html-invalid/index.html', + ); + await bundle(inputFile, { + defaultTargetOptions: { + shouldOptimize: false, + }, + }); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - { - type: 'html', - assets: ['other.html'], - }, - ]); - }); + let outputFile = path.join(distDir, 'index.html'); + let html = await outputFS.readFile(outputFile, 'utf8'); + assert(html.includes('This is a paragraph')); + }); - it('should not update root/main file in the bundles', async function() { - await bundle(path.join(__dirname, '/integration/html-root/index.html')); + it("should work with html that doesn't include optional closing tags", async function() { + let inputFile = path.join( + __dirname, + '/integration/html-optional-closing-tags/index.html', + ); + await bundle(inputFile, { + defaultTargetOptions: { + shouldOptimize: false, + }, + }); - let files = await outputFS.readdir(distDir); + let outputFile = path.join(distDir, 'index.html'); + let html = await outputFS.readFile(outputFile, 'utf8'); + assert(html.includes('Paragraph 1')); + }); - for (let file of files) { - if (file !== 'index.html' && file.endsWith('.html')) { - let html = await outputFS.readFile(path.join(distDir, file), 'utf8'); - assert(html.includes('index.html')); - } - } - }); + it('should read .htmlnanorc.json and minify HTML in production mode', async function() { + await bundle( + path.join(__dirname, '/integration/htmlnano-config/index.html'), + { + defaultTargetOptions: { + shouldOptimize: true, + }, + }, + ); - it('should preserve the spacing in the HTML tags', async function() { - await bundle(path.join(__dirname, '/integration/html/index.html')); + let html = await outputFS.readFile( + path.join(distDir, 'index.html'), + 'utf8', + ); - let html = await outputFS.readFile( - path.join(distDir, 'index.html'), - 'utf8', - ); - assert(/hello<\/i> world<\/i>/.test(html)); - }); + // minifyJson + assert( + html.includes(''), + ); - it('should support child bundles of different types', async function() { - let b = await bundle( - path.join( - __dirname, - '/integration/child-bundle-different-types/index.html', - ), - ); + // mergeStyles + assert(html.includes('')); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - { - type: 'js', - assets: ['main.js', 'util.js', 'other.js'], - }, - { - type: 'html', - assets: ['other.html'], - }, - { - type: 'js', - assets: ['index.js', 'util.js', 'other.js'], - }, - ]); - }); + assert(!html.includes('sourceMappingURL')); - it.skip('should support circular dependencies', async function() { - let b = await bundle( - path.join(__dirname, '/integration/circular/index.html'), - ); + // minifySvg is false + assert( + html.includes( + 'SVG', + ), + ); + }); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - { - type: 'html', - assets: ['about.html'], - }, - { - type: 'js', - assets: ['about.js', 'index.js'], - }, - { - type: 'html', - assets: ['test.html'], - }, - { - type: 'js', - assets: ['about.js', 'index.js'], - }, - ]); - }); + it('should not minify default values inside HTML in production mode', async function() { + let inputFile = path.join( + __dirname, + '/integration/htmlnano-defaults-form/index.html', + ); + await bundle(inputFile, { + defaultTargetOptions: { + shouldOptimize: true, + }, + }); - it('should support bundling HTM', async function() { - let b = await bundle( - path.join(__dirname, '/integration/htm-extension/index.htm'), - ); + let inputSize = (await inputFS.stat(inputFile)).size; - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.htm'], - type: 'html', - }, - { - type: 'js', - assets: ['index.js'], - }, - ]); - }); + let outputFile = path.join(distDir, '/index.html'); + let outputSize = (await outputFS.stat(outputFile)).size; - it('should detect srcset attribute', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-srcset/index.html'), - ); + assert(inputSize > outputSize); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - { - type: 'png', - assets: ['100x100.png'], - }, - { - type: 'png', - assets: ['200x200.png'], - }, - { - type: 'png', - assets: ['300x300.png'], - }, - ]); - }); + let html = await outputFS.readFile(outputFile, 'utf8'); + assert(html.includes('')); + }); - it('should detect srcset attribute of source element', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-source-srcset/index.html'), - ); + it('should not prepend the public path to assets with remote URLs', async function() { + await bundle(path.join(__dirname, '/integration/html/index.html')); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - { - type: 'png', - assets: ['100x100.png'], - }, - { - type: 'png', - assets: ['200x200.png'], - }, - { - type: 'png', - assets: ['300x300.png'], - }, - ]); - }); + let html = await outputFS.readFile( + path.join(distDir, 'index.html'), + 'utf8', + ); + assert( + html.includes( + '', + ), + ); + }); - it('should detect imagesrcset attribute', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-imagesrcset/index.html'), - ); + it('should not prepend the public path to hash links', async function() { + await bundle(path.join(__dirname, '/integration/html/index.html')); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - { - type: 'png', - assets: ['100x100.png'], - }, - { - type: 'png', - assets: ['200x200.png'], - }, - { - type: 'png', - assets: ['300x300.png'], - }, - ]); - }); + let html = await outputFS.readFile( + path.join(distDir, 'index.html'), + 'utf8', + ); + assert(html.includes('')); + }); - it.skip('should support webmanifest', async function() { - let b = await bundle( - path.join(__dirname, '/integration/webmanifest/index.html'), - ); + it('should detect virtual paths', async function() { + let b = await bundle( + path.join(__dirname, '/integration/html-virtualpath/index.html'), + ); - assertBundles(b, { - name: 'index.html', - assets: ['index.html'], - childBundles: [ + assertBundles(b, [ { - type: 'webmanifest', - assets: ['manifest.webmanifest'], - childBundles: [ - { - type: 'txt', - assets: ['some.txt'], - childBundles: [], - }, - ], + name: 'index.html', + assets: ['index.html'], }, - ], - }); - }); - - it.skip("should treat webmanifest as an entry module so it doesn't get content hashed", async function() { - const b = await bundle( - path.join(__dirname, '/integration/html-manifest/index.html'), - ); - - assertBundles(b, { - name: 'index.html', - assets: ['index.html'], - childBundles: [ { - type: 'webmanifest', - assets: ['manifest.webmanifest'], + type: 'html', + assets: ['other.html'], }, - ], + ]); }); - const html = await outputFS.readFile( - path.join(__dirname, '/dist/index.html'), - 'utf8', - ); - assert(html.includes('')); - }); + it('should not update root/main file in the bundles', async function() { + await bundle(path.join(__dirname, '/integration/html-root/index.html')); - it('should bundle svg files correctly', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-svg/index.html'), - ); + let files = await outputFS.readdir(distDir); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - { - type: 'svg', - assets: ['file.svg'], - }, - ]); - }); + for (let file of files) { + if (file !== 'index.html' && file.endsWith('.html')) { + let html = await outputFS.readFile(path.join(distDir, file), 'utf8'); + assert(html.includes('index.html')); + } + } + }); - it('should ignore svgs referencing local symbols via ', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-svg-local-symbol/index.html'), - { - mode: 'production', - }, - ); + it('should preserve the spacing in the HTML tags', async function() { + await bundle(path.join(__dirname, '/integration/html/index.html')); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - ]); + let html = await outputFS.readFile( + path.join(distDir, 'index.html'), + 'utf8', + ); + assert(/hello<\/i> world<\/i>/.test(html)); + }); - let contents = await outputFS.readFile(b.getBundles()[0].filePath, 'utf8'); - assert( - contents.includes( - '', - ), - ); - }); + it('should support child bundles of different types', async function() { + let b = await bundle( + path.join( + __dirname, + '/integration/child-bundle-different-types/index.html', + ), + ); - it('should bundle svg files using correctly', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-svg-image/index.html'), - ); + assertBundles(b, [ + { + name: 'index.html', + assets: ['index.html'], + }, + { + type: 'js', + assets: ['main.js', 'util.js', 'other.js'], + }, + { + type: 'html', + assets: ['other.html'], + }, + { + type: 'js', + assets: ['index.js', 'util.js', 'other.js'], + }, + ]); + }); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - { - type: 'svg', - assets: ['file.svg'], - }, - ]); - }); + it.skip('should support circular dependencies', async function() { + let b = await bundle( + path.join(__dirname, '/integration/circular/index.html'), + ); - // Based on https://developer.mozilla.org/en-US/docs/Web/SVG/Element/script - it('should bundle scripts inside svg', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-svg-script/index.html'), - ); + assertBundles(b, [ + { + name: 'index.html', + assets: ['index.html'], + }, + { + type: 'html', + assets: ['about.html'], + }, + { + type: 'js', + assets: ['about.js', 'index.js'], + }, + { + type: 'html', + assets: ['test.html'], + }, + { + type: 'js', + assets: ['about.js', 'index.js'], + }, + ]); + }); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - { - type: 'js', - assets: ['script-a.js'], - }, - { - type: 'js', - assets: ['script-b.js'], - }, - ]); - }); + it('should support bundling HTM', async function() { + let b = await bundle( + path.join(__dirname, '/integration/htm-extension/index.htm'), + ); - it('should support data attribute of object element', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-object/index.html'), - ); + assertBundles(b, [ + { + name: 'index.html', + assets: ['index.htm'], + type: 'html', + }, + { + type: 'js', + assets: ['index.js'], + }, + ]); + }); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - { - type: 'svg', - assets: ['file.svg'], - }, - ]); - }); + it('should detect srcset attribute', async function() { + let b = await bundle( + path.join(__dirname, '/integration/html-srcset/index.html'), + ); - it('should resolve assets containing spaces', async function() { - let b = await bundle( - path.join(__dirname, '/integration/resolve-spaces/index.html'), - ); + assertBundles(b, [ + { + name: 'index.html', + assets: ['index.html'], + }, + { + type: 'png', + assets: ['100x100.png'], + }, + { + type: 'png', + assets: ['200x200.png'], + }, + { + type: 'png', + assets: ['300x300.png'], + }, + ]); + }); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - { - type: 'html', - assets: ['other page.html'], - }, - ]); - }); + it('should detect srcset attribute of source element', async function() { + let b = await bundle( + path.join(__dirname, '/integration/html-source-srcset/index.html'), + ); - it('should process inline JS', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-inline-js/index.html'), - { - defaultTargetOptions: { - shouldOptimize: true, + assertBundles(b, [ + { + name: 'index.html', + assets: ['index.html'], }, - }, - ); - - // inline bundles are not output, but are apart of the bundleGraph - assertBundles(b, [ - {type: 'js', assets: ['index.html']}, - {type: 'js', assets: ['index.html']}, - {type: 'js', assets: ['index.html']}, - {type: 'js', assets: ['index.html']}, - {name: 'index.html', assets: ['index.html']}, - ]); - - let files = await outputFS.readdir(distDir); - // assert that the inline js files are not output - assert(!files.some(filename => filename.includes('js'))); - - let html = await outputFS.readFile( - path.join(distDir, 'index.html'), - 'utf-8', - ); + { + type: 'png', + assets: ['100x100.png'], + }, + { + type: 'png', + assets: ['200x200.png'], + }, + { + type: 'png', + assets: ['300x300.png'], + }, + ]); + }); - assert(!html.includes('someArgument')); - }); + it('should detect imagesrcset attribute', async function() { + let b = await bundle( + path.join(__dirname, '/integration/html-imagesrcset/index.html'), + ); - it('should process inline styles', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-inline-styles/index.html'), - { - defaultTargetOptions: { - shouldOptimize: true, + assertBundles(b, [ + { + name: 'index.html', + assets: ['index.html'], }, - }, - ); + { + type: 'png', + assets: ['100x100.png'], + }, + { + type: 'png', + assets: ['200x200.png'], + }, + { + type: 'png', + assets: ['300x300.png'], + }, + ]); + }); - assertBundles(b, [ - { - type: 'css', - assets: ['index.html'], - }, - { - type: 'css', - assets: ['index.html'], - }, - { - type: 'css', - assets: ['index.html'], - }, - { - type: 'css', - assets: ['index.html'], - }, - { - type: 'jpg', - assets: ['bg.jpg'], - }, - { - type: 'jpg', - assets: ['img.jpg'], - }, - { + it.skip('should support webmanifest', async function() { + let b = await bundle( + path.join(__dirname, '/integration/webmanifest/index.html'), + ); + + assertBundles(b, { name: 'index.html', assets: ['index.html'], - }, - ]); + childBundles: [ + { + type: 'webmanifest', + assets: ['manifest.webmanifest'], + childBundles: [ + { + type: 'txt', + assets: ['some.txt'], + childBundles: [], + }, + ], + }, + ], + }); + }); - let bundles = b.getBundles(); + it.skip("should treat webmanifest as an entry module so it doesn't get content hashed", async function() { + const b = await bundle( + path.join(__dirname, '/integration/html-manifest/index.html'), + ); - let html = await outputFS.readFile( - bundles.find(bundle => bundle.type === 'html').filePath, - 'utf8', - ); + assertBundles(b, { + name: 'index.html', + assets: ['index.html'], + childBundles: [ + { + type: 'webmanifest', + assets: ['manifest.webmanifest'], + }, + ], + }); - let urls = [...html.matchAll(/url\(([^)]*)\)/g)].map(m => m[1]); - assert.strictEqual(urls.length, 2); - for (let url of urls) { + const html = await outputFS.readFile( + path.join(__dirname, '/dist/index.html'), + 'utf8', + ); assert( - bundles.find( - bundle => - bundle.bundleBehavior !== 'inline' && - path.basename(bundle.filePath) === url, - ), + html.includes(''), ); - } - }); + }); - it('should process inline element styles', async function() { - let b = await bundle( - path.join( - __dirname, - '/integration/html-inline-styles-element/index.html', - ), - {shouldDisableCache: false}, - ); + it('should bundle svg files correctly', async function() { + let b = await bundle( + path.join(__dirname, '/integration/html-svg/index.html'), + ); - assertBundles(b, [ - { - type: 'css', - assets: ['index.html'], - }, - { - type: 'css', - assets: ['index.html'], - }, - { - type: 'css', - assets: ['index.html'], - }, - { - name: 'index.html', - assets: ['index.html'], - }, - ]); - }); - - it('should process inline styles using lang', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-inline-sass/index.html'), - { - defaultTargetOptions: { - shouldOptimize: true, + assertBundles(b, [ + { + name: 'index.html', + assets: ['index.html'], }, - }, - ); - - assertBundles(b, [ - { - type: 'css', - assets: ['index.html'], - }, - { - name: 'index.html', - assets: ['index.html'], - }, - ]); - - let html = await outputFS.readFile( - path.join(distDir, 'index.html'), - 'utf8', - ); - assert(html.includes('')); - assert(!html.includes('sourceMappingURL')); - }); + { + type: 'svg', + assets: ['file.svg'], + }, + ]); + }); - it('should process inline non-js scripts', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-inline-coffeescript/index.html'), - { - defaultTargetOptions: { - shouldOptimize: true, + it('should ignore svgs referencing local symbols via ', async function() { + let b = await bundle( + path.join(__dirname, '/integration/html-svg-local-symbol/index.html'), + { + mode: 'production', }, - }, - ); + ); - assertBundles(b, [ - { - type: 'js', - assets: ['index.html'], - }, - { - name: 'index.html', - assets: ['index.html'], - }, - ]); + assertBundles(b, [ + { + name: 'index.html', + assets: ['index.html'], + }, + ]); - let html = await outputFS.readFile( - path.join(distDir, 'index.html'), - 'utf8', - ); - assert(html.includes('alert("Hello, World!")')); - }); + let contents = await outputFS.readFile( + b.getBundles()[0].filePath, + 'utf8', + ); + assert( + contents.includes( + '', + ), + ); + }); - it('should handle inline css with @imports', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-inline-css-import/index.html'), - ); + it('should bundle svg files using correctly', async function() { + let b = await bundle( + path.join(__dirname, '/integration/html-svg-image/index.html'), + ); - assertBundles(b, [ - { - type: 'css', - assets: ['index.html', 'test.css'], - }, - { - name: 'index.html', - assets: ['index.html'], - }, - ]); + assertBundles(b, [ + { + name: 'index.html', + assets: ['index.html'], + }, + { + type: 'svg', + assets: ['file.svg'], + }, + ]); + }); - let html = await outputFS.readFile( - path.join(distDir, 'index.html'), - 'utf8', - ); - assert(!html.includes('@import')); - }); + // Based on https://developer.mozilla.org/en-US/docs/Web/SVG/Element/script + it('should bundle scripts inside svg', async function() { + let b = await bundle( + path.join(__dirname, '/integration/html-svg-script/index.html'), + ); - it('should not modify inline importmaps', async function() { - let b = await bundle( - path.join(__dirname, '/integration/html-inline-importmap/index.html'), - {}, - ); + assertBundles(b, [ + { + name: 'index.html', + assets: ['index.html'], + }, + { + type: 'js', + assets: ['script-a.js'], + }, + { + type: 'js', + assets: ['script-b.js'], + }, + ]); + }); - assertBundles(b, [ - { - name: 'index.html', - assets: ['index.html'], - }, - ]); + it('should support data attribute of object element', async function() { + let b = await bundle( + path.join(__dirname, '/integration/html-object/index.html'), + ); - let html = await outputFS.readFile(b.getBundles()[0].filePath, 'utf8'); - assert(html.includes('/node_modules/lit1.3.0/')); - }); + assertBundles(b, [ + { + name: 'index.html', + assets: ['index.html'], + }, + { + type: 'svg', + assets: ['file.svg'], + }, + ]); + }); - it('should expose top level declarations globally in inline ', + b: '