Skip to content

Commit

Permalink
Custom bundle loaders + WASM support -> master (#565)
Browse files Browse the repository at this point in the history 
* add support to rust/wasm

* use child-process-promise instead of async-child-process

* use parent extension instead of fromHtml flag

* create WasmAsset

* RustAsset refactor
 get rid of wargo
 use wasm32-unknown-unknown

* fix lint errors, remove toml package and delete jsconfig.json

* Inline wasm into JS, instantiate, and return exports

* Split bundle loaders into separate modules

Register custom loaders with `bundler.addBundleLoader`

* Split out build queue logic into its own class

Enables `bundler.getAsset` method to resolve and process an asset at any time, outside of the normal asset tree construction.

* Only add a single asset to bundle if raw packager is used

This gives us a separate bundle for each output file.

* Include used bundle loaders, and preload external modules

Allows synchronous import to preload modules in external files, e.g. .wasm file, prior to execution of the JS bundle.

Also processes the HMR runtime like other assets.

* Register wasm loader

* Replace dedicated WASMAsset with RawAsset

No longer returns a URL to the JS bundle if there is a bundle loader defined. This will cause the asset to be preloaded prior to JS bundle execution.

* Update tests

* Clean up bundling code

* Hopefully fix test in travis

* wasm tests

* Define correct wasm mime type

Until pillarjs/send#154 is merged.

* Use WebAssembly.instantiate instead of constructor

Should fix error “WebAssembly.Instance is disallowed on the main thread, if the buffer size is larger than 4KB”

* Fix test

* Fix PromiseQueue bug

* Remove rust for now. Will be added in a separate PR.
  • Loading branch information
@devongovett
devongovett committed Jan 16, 2018
1 parent af9102f commit b06d934
Show file tree
Hide file tree
Showing 33 changed files with 577 additions and 160 deletions.
2 changes: 2 additions & 0 deletions .eslintignore
View file
Expand Up @@ -10,6 +10,8 @@
/test/integration/dynamic-references-raw/index.js
/test/integration/dynamic-references-raw/local.js
/test/integration/hmr-dynamic/index.js
/test/integration/wasm-async/index.js
/test/integration/wasm-dynamic/index.js

# Generated by the build
lib
Expand Down
1 change: 1 addition & 0 deletions packages/core/parcel-bundler/package.json
View file
Expand Up @@ -15,6 +15,7 @@
"babylon-walk": "^1.0.2",
"browser-resolve": "^1.11.2",
"chalk": "^2.1.0",
"child-process-promise": "^2.2.1",
"chokidar": "^1.7.0",
"commander": "^2.11.0",
"cross-spawn": "^5.1.0",
Expand Down
39 changes: 31 additions & 8 deletions packages/core/parcel-bundler/src/Bundle.js
View file
Expand Up @@ -15,7 +15,20 @@ class Bundle {
this.entryAsset = null;
this.assets = new Set();
this.childBundles = new Set();
this.siblingBundles = new Map();
this.siblingBundles = new Set;
this.siblingBundlesMap = new Map();
}

static createWithAsset(asset, parentBundle) {
let bundle = new Bundle(
asset.type,
Path.join(asset.options.outDir, asset.generateBundleName()),
parentBundle
);

bundle.entryAsset = asset;
bundle.addAsset(asset);
return bundle;
}

addAsset(asset) {
Expand All @@ -33,26 +46,36 @@ class Bundle {
return this;
}

if (!this.siblingBundles.has(type)) {
let bundle = this.createChildBundle(
if (!this.siblingBundlesMap.has(type)) {
let bundle = new Bundle(
type,
Path.join(
Path.dirname(this.name),
Path.basename(this.name, Path.extname(this.name)) + '.' + type
)
),
this
);
this.siblingBundles.set(type, bundle);

this.childBundles.add(bundle);
this.siblingBundles.add(bundle);
this.siblingBundlesMap.set(type, bundle);
}

return this.siblingBundles.get(type);
return this.siblingBundlesMap.get(type);
}

createChildBundle(type, name) {
let bundle = new Bundle(type, name, this);
createChildBundle(entryAsset) {
let bundle = Bundle.createWithAsset(entryAsset, this);
this.childBundles.add(bundle);
return bundle;
}

createSiblingBundle(entryAsset) {
let bundle = this.createChildBundle(entryAsset);
this.siblingBundles.add(bundle);
return bundle;
}

get isEmpty() {
return this.assets.size === 0;
}
Expand Down
146 changes: 77 additions & 69 deletions packages/core/parcel-bundler/src/Bundler.js
View file
Expand Up @@ -15,6 +15,7 @@ const localRequire = require('./utils/localRequire');
const config = require('./utils/config');
const emoji = require('./utils/emoji');
const loadEnv = require('./utils/env');
const PromiseQueue = require('./utils/PromiseQueue');

/**
* The Bundler is the main entry point. It resolves and loads assets,
Expand All @@ -32,6 +33,17 @@ class Bundler extends EventEmitter {
this.cache = this.options.cache ? new FSCache(this.options) : null;
this.logger = new Logger(this.options);
this.delegate = options.delegate || {};
this.bundleLoaders = {};

this.addBundleLoader(
'wasm',
require.resolve('./builtins/loaders/wasm-loader')
);
this.addBundleLoader(
'css',
require.resolve('./builtins/loaders/css-loader')
);
this.addBundleLoader('js', require.resolve('./builtins/loaders/js-loader'));

this.pending = false;
this.loadedAssets = new Map();
Expand All @@ -41,7 +53,7 @@ class Bundler extends EventEmitter {
this.hmr = null;
this.bundleHashes = null;
this.errored = false;
this.buildQueue = new Set();
this.buildQueue = new PromiseQueue(this.processAsset.bind(this));
this.rebuildTimeout = null;
}

Expand Down Expand Up @@ -92,6 +104,18 @@ class Bundler extends EventEmitter {
this.packagers.add(type, packager);
}

addBundleLoader(type, path) {
if (typeof path !== 'string') {
throw new Error('Bundle loader should be a module path.');
}

if (this.farm) {
throw new Error('Bundle loaders must be added before bundling.');
}

this.bundleLoaders[type] = path;
}

async loadPlugins() {
let pkg = await config.load(this.mainFile, ['package.json']);
if (!pkg) {
Expand Down Expand Up @@ -141,8 +165,26 @@ class Bundler extends EventEmitter {
this.buildQueue.add(this.mainAsset);
}

// Build the queued assets, and produce a bundle tree.
let bundle = await this.buildQueuedAssets(isInitialBundle);
// Build the queued assets.
let loadedAssets = await this.buildQueue.run();

// Emit an HMR update for any new assets (that don't have a parent bundle yet)
// plus the asset that actually changed.
if (this.hmr && !isInitialBundle) {
this.hmr.emitUpdate([...this.findOrphanAssets(), ...loadedAssets]);
}

// Invalidate bundles
for (let asset of this.loadedAssets.values()) {
asset.invalidateBundle();
}

// Create a new bundle tree and package everything up.
let bundle = this.createBundleTree(this.mainAsset);
this.bundleHashes = await bundle.package(this, this.bundleHashes);

// Unload any orphaned assets
this.unloadOrphanedAssets();

let buildTime = Date.now() - startTime;
let time =
Expand All @@ -151,6 +193,7 @@ class Bundler extends EventEmitter {
: `${(buildTime / 1000).toFixed(2)}s`;
this.logger.status(emoji.success, `Built in ${time}.`, 'green');

this.emit('bundled', bundle);
return bundle;
} catch (err) {
this.errored = true;
Expand Down Expand Up @@ -182,8 +225,8 @@ class Bundler extends EventEmitter {
await loadEnv(this.mainFile);

this.options.extensions = Object.assign({}, this.parser.extensions);
this.options.bundleLoaders = this.bundleLoaders;
this.options.env = process.env;
this.farm = WorkerFarm.getShared(this.options);

if (this.options.watch) {
// FS events on macOS are flakey in the tests, which write lots of files very quickly
Expand All @@ -199,6 +242,8 @@ class Bundler extends EventEmitter {
this.hmr = new HMRServer();
this.options.hmrPort = await this.hmr.start(this.options.hmrPort);
}

this.farm = WorkerFarm.getShared(this.options);
}

stop() {
Expand All @@ -215,49 +260,11 @@ class Bundler extends EventEmitter {
}
}

async buildQueuedAssets(isInitialBundle = false) {
// Consume the rebuild queue until it is empty.
let loadedAssets = new Set();
while (this.buildQueue.size > 0) {
let promises = [];
for (let asset of this.buildQueue) {
// Invalidate the asset, unless this is the initial bundle
if (!isInitialBundle) {
asset.invalidate();
if (this.cache) {
this.cache.invalidate(asset.name);
}
}

promises.push(this.loadAsset(asset));
loadedAssets.add(asset);
}

// Wait for all assets to load. If there are more added while
// these are processing, they'll be loaded in the next batch.
await Promise.all(promises);
}

// Emit an HMR update for any new assets (that don't have a parent bundle yet)
// plus the asset that actually changed.
if (this.hmr && !isInitialBundle) {
this.hmr.emitUpdate([...this.findOrphanAssets(), ...loadedAssets]);
}

// Invalidate bundles
for (let asset of this.loadedAssets.values()) {
asset.invalidateBundle();
}

// Create a new bundle tree and package everything up.
let bundle = this.createBundleTree(this.mainAsset);
this.bundleHashes = await bundle.package(this, this.bundleHashes);

// Unload any orphaned assets
this.unloadOrphanedAssets();

this.emit('bundled', bundle);
return bundle;
async getAsset(name, parent) {
let asset = await this.resolveAsset(name, parent);
this.buildQueue.add(asset);
await this.buildQueue.run();
return asset;
}

async resolveAsset(name, parent) {
Expand Down Expand Up @@ -328,9 +335,19 @@ class Bundler extends EventEmitter {
}
}

async processAsset(asset, isRebuild) {
if (isRebuild) {
asset.invalidate();
if (this.cache) {
this.cache.invalidate(asset.name);
}
}

await this.loadAsset(asset);
}

async loadAsset(asset) {
if (asset.processed) {
this.buildQueue.delete(asset);
return;
}

Expand Down Expand Up @@ -386,8 +403,6 @@ class Bundler extends EventEmitter {
asset.depAssets.set(dep, assetDep);
}
});

this.buildQueue.delete(asset);
}

createBundleTree(asset, dep, bundle, parentBundles = new Set()) {
Expand Down Expand Up @@ -416,27 +431,20 @@ class Bundler extends EventEmitter {
}
}

// Create the root bundle if it doesn't exist
if (!bundle) {
bundle = new Bundle(
asset.type,
Path.join(this.options.outDir, asset.generateBundleName())
);
bundle.entryAsset = asset;
// Create the root bundle if it doesn't exist
bundle = Bundle.createWithAsset(asset);
} else if (dep && dep.dynamic) {
// Create a new bundle for dynamic imports
bundle = bundle.createChildBundle(asset);
} else if (asset.type && !this.packagers.has(asset.type)) {
// No packager is available for this asset type. Create a new bundle with only this asset.
bundle.createSiblingBundle(asset);
} else {
// Add the asset to the common bundle of the asset's type
bundle.getSiblingBundle(asset.type).addAsset(asset);
}

// Create a new bundle for dynamic imports
if (dep && dep.dynamic) {
bundle = bundle.createChildBundle(
asset.type,
Path.join(this.options.outDir, asset.generateBundleName())
);
bundle.entryAsset = asset;
}

// Add the asset to the bundle of the asset's type
bundle.getSiblingBundle(asset.type).addAsset(asset);

// If the asset generated a representation for the parent bundle type, also add it there
if (asset.generated[bundle.type] != null) {
bundle.addAsset(asset);
Expand Down Expand Up @@ -523,7 +531,7 @@ class Bundler extends EventEmitter {

// Add the asset to the rebuild queue, and reset the timeout.
for (let asset of assets) {
this.buildQueue.add(asset);
this.buildQueue.add(asset, true);
}

clearTimeout(this.rebuildTimeout);
Expand Down
4 changes: 4 additions & 0 deletions packages/core/parcel-bundler/src/Server.js
View file
Expand Up @@ -5,6 +5,10 @@ const getPort = require('get-port');
const serverErrors = require('./utils/customErrors').serverErrors;
const generateCertificate = require('./utils/generateCertificate');

serveStatic.mime.define({
'application/wasm': ['wasm']
});

function middleware(bundler) {
const serve = serveStatic(bundler.options.outDir, {index: false});

Expand Down
4 changes: 3 additions & 1 deletion packages/core/parcel-bundler/src/assets/HTMLAsset.js
View file
Expand Up @@ -47,7 +47,9 @@ class HTMLAsset extends Asset {
continue;
}
if (elements && elements.includes(node.tag)) {
let assetPath = this.addURLDependency(decodeURIComponent(node.attrs[attr]));
let assetPath = this.addURLDependency(
decodeURIComponent(node.attrs[attr])
);
if (!isURL(assetPath)) {
assetPath = urlJoin(this.options.publicURL, assetPath);
}
Expand Down
6 changes: 6 additions & 0 deletions packages/core/parcel-bundler/src/assets/RawAsset.js
View file
Expand Up @@ -6,6 +6,12 @@ class RawAsset extends Asset {
load() {}

generate() {
// Don't return a URL to the JS bundle if there is a bundle loader defined for this asset type.
// This will cause the actual asset to be automatically preloaded prior to the JS bundle running.
if (this.options.bundleLoaders[this.type]) {
return {};
}

const pathToAsset = urlJoin(
this.options.publicURL,
this.generateBundleName()
Expand Down

0 comments on commit b06d934

Please sign in to comment.