diff --git a/examples/assets/bundles/bundle.tar b/examples/assets/bundles/bundle.tar new file mode 100644 index 00000000000..99b1d1d53eb Binary files /dev/null and b/examples/assets/bundles/bundle.tar differ diff --git a/examples/bundle.tar b/examples/bundle.tar new file mode 100644 index 00000000000..99b1d1d53eb Binary files /dev/null and b/examples/bundle.tar differ diff --git a/examples/src/examples/loaders/bundle/config.mjs b/examples/src/examples/loaders/bundle/config.mjs new file mode 100644 index 00000000000..b7fc3987320 --- /dev/null +++ b/examples/src/examples/loaders/bundle/config.mjs @@ -0,0 +1,6 @@ +/** + * @type {import('../../../../types.mjs').ExampleConfig} + */ +export default { + WEBGPU_ENABLED: true +}; diff --git a/examples/src/examples/loaders/bundle/example.mjs b/examples/src/examples/loaders/bundle/example.mjs new file mode 100644 index 00000000000..9a5b364b1ca --- /dev/null +++ b/examples/src/examples/loaders/bundle/example.mjs @@ -0,0 +1,116 @@ +import * as pc from 'playcanvas'; +import { deviceType, rootPath } from '@examples/utils'; + +// The example demonstrates loading multiple assets from a single bundle file + +// This tar file has been created by a command line: +// : cd engine/examples/ +// : tar cvf bundle.tar assets/models/geometry-camera-light.glb assets/models/torus.png + +const canvas = document.getElementById('application-canvas'); +if (!(canvas instanceof HTMLCanvasElement)) { + throw new Error('No canvas found'); +} + +const assets = { + bundle: new pc.Asset('bundle', 'bundle', { url: '/static/assets/bundles/bundle.tar' }), + scene: new pc.Asset('scene', 'container', { url: 'assets/models/geometry-camera-light.glb' }), + torus: new pc.Asset('torus', 'container', { url: 'assets/models/torus.glb' }) +}; + +// Bundle should list asset IDs in its data +assets.bundle.data = { assets: [assets.scene.id, assets.torus.id] }; + +const gfxOptions = { + deviceTypes: [deviceType], + glslangUrl: rootPath + '/static/lib/glslang/glslang.js', + twgslUrl: rootPath + '/static/lib/twgsl/twgsl.js' +}; + +const device = await pc.createGraphicsDevice(canvas, gfxOptions); +const createOptions = new pc.AppOptions(); +createOptions.graphicsDevice = device; + +createOptions.componentSystems = [pc.RenderComponentSystem, pc.CameraComponentSystem, pc.LightComponentSystem]; +createOptions.resourceHandlers = [pc.TextureHandler, pc.ContainerHandler]; + +const app = new pc.AppBase(canvas); +app.init(createOptions); + +// Set the canvas to fill the window and automatically change resolution to be the same as the canvas size +app.setCanvasFillMode(pc.FILLMODE_FILL_WINDOW); +app.setCanvasResolution(pc.RESOLUTION_AUTO); + +// Ensure canvas is resized when window changes size +const resize = () => app.resizeCanvas(); +window.addEventListener('resize', resize); +app.on('destroy', () => { + window.removeEventListener('resize', resize); +}); + +// load assets +// notice that scene and torus are loaded as blob's and only tar file is downloaded +const assetListLoader = new pc.AssetListLoader(Object.values(assets), app.assets); +assetListLoader.load(() => { + + app.start(); + + /** + * the array will store loaded cameras + * @type {pc.CameraComponent[]} + */ + let camerasComponents = null; + + // glb lights use physical units + app.scene.physicalUnits = true; + + // create an instance using render component + const entity = assets.scene.resource.instantiateRenderEntity(); + app.root.addChild(entity); + + // create an instance using render component + const entityTorus = assets.torus.resource.instantiateRenderEntity(); + app.root.addChild(entityTorus); + entityTorus.setLocalPosition(0, 0, 2); + + // find all cameras - by default they are disabled + camerasComponents = entity.findComponents("camera"); + camerasComponents.forEach((component) => { + + // set the aspect ratio to automatic to work with any window size + component.aspectRatioMode = pc.ASPECT_AUTO; + + // set up exposure for physical units + component.aperture = 4; + component.shutter = 1 / 100; + component.sensitivity = 500; + }); + + /** @type {pc.LightComponent[]} */ + const lightComponents = entity.findComponents("light"); + lightComponents.forEach((component) => { + component.enabled = true; + }); + + let time = 0; + let activeCamera = 0; + app.on("update", function (dt) { + time -= dt; + + entityTorus.rotateLocal(360 * dt, 0, 0); + + // change the camera every few seconds + if (time <= 0) { + time = 2; + + // disable current camera + camerasComponents[activeCamera].enabled = false; + + // activate next camera + activeCamera = (activeCamera + 1) % camerasComponents.length; + camerasComponents[activeCamera].enabled = true; + } + }); +}); + +export { app }; diff --git a/src/core/event-handle.js b/src/core/event-handle.js index 1f5ce4068fa..a2a07cb6bdc 100644 --- a/src/core/event-handle.js +++ b/src/core/event-handle.js @@ -97,7 +97,7 @@ class EventHandle { /** * Mark if event has been removed. * @type {boolean} - * @internal + * @ignore */ set removed(value) { if (!value) return; diff --git a/src/framework/asset/asset-registry.js b/src/framework/asset/asset-registry.js index 0fcddca94e9..3383cbcb471 100644 --- a/src/framework/asset/asset-registry.js +++ b/src/framework/asset/asset-registry.js @@ -26,6 +26,14 @@ import { Asset } from './asset.js'; * @param {Asset} [asset] - The loaded asset if no errors were encountered. */ +/** + * Callback used by {@link ResourceLoader#load} and called when an asset is choosing a bundle + * to load from. Return a single bundle to ensure asset is loaded from it. + * + * @callback BundlesFilterCallback + * @param {import('../bundle/bundle.js').Bundle[]} bundles - List of bundles which contain the asset. + */ + /** * Container for all assets that are available to this application. Note that PlayCanvas scripts * are provided with an AssetRegistry instance as `app.assets`. @@ -189,6 +197,13 @@ class AssetRegistry extends EventHandler { */ prefix = null; + /** + * BundleRegistry + * + * @type {import('../bundle/bundle-registry.js').BundleRegistry|null} + */ + bundles = null; + /** * Create an instance of an AssetRegistry. * @@ -335,6 +350,15 @@ class AssetRegistry extends EventHandler { * out when it is loaded. * * @param {Asset} asset - The asset to load. + * @param {object} [options] - Options for asset loading. + * @param {boolean} [options.bundlesIgnore] - If set to true, then asset will not try to load + * from a bundle. Defaults to false. + * @param {boolean} [options.force] - If set to true, then the check of asset being loaded or + * is already loaded is bypassed, which forces loading of asset regardless. + * @param {BundlesFilterCallback} [options.bundlesFilter] - A callback that will be called + * when loading an asset that is contained in any of the bundles. It provides an array of + * bundles and will ensure asset is loaded from bundle returned from a callback. By default + * smallest filesize bundle is choosen. * @example * // load some assets * const assetsToLoad = [ @@ -352,16 +376,24 @@ class AssetRegistry extends EventHandler { * app.assets.load(assetToLoad); * }); */ - load(asset) { + load(asset, options) { // do nothing if asset is already loaded // note: lots of code calls assets.load() assuming this check is present // don't remove it without updating calls to assets.load() with checks for the asset.loaded state - if (asset.loading || asset.loaded) { + if ((asset.loading || asset.loaded) && !options?.force) { return; } const file = asset.file; + const _fireLoad = () => { + this.fire('load', asset); + this.fire('load:' + asset.id, asset); + if (file && file.url) + this.fire('load:url:' + file.url, asset); + asset.fire('load', asset); + }; + // open has completed on the resource const _opened = (resource) => { if (resource instanceof Array) { @@ -373,11 +405,28 @@ class AssetRegistry extends EventHandler { // let handler patch the resource this._loader.patch(asset, this); - this.fire('load', asset); - this.fire('load:' + asset.id, asset); - if (file && file.url) - this.fire('load:url:' + file.url, asset); - asset.fire('load', asset); + if (asset.type === 'bundle') { + const assetIds = asset.data.assets; + for (let i = 0; i < assetIds.length; i++) { + const assetInBundle = this._idToAsset.get(assetIds[i]); + if (assetInBundle && !assetInBundle.loaded) { + this.load(assetInBundle, { force: true }); + } + } + + if (asset.resource.loaded) { + _fireLoad(); + } else { + this.fire('load:start', asset); + this.fire('load:start:' + asset.id, asset); + if (file && file.url) + this.fire('load:start:url:' + file.url, asset); + asset.fire('load:start', asset); + asset.resource.on('load', _fireLoad); + } + } else { + _fireLoad(); + } }; // load has completed on the resource @@ -409,7 +458,26 @@ class AssetRegistry extends EventHandler { this.fire('load:' + asset.id + ':start', asset); asset.loading = true; - this._loader.load(asset.getFileUrl(), asset.type, _loaded, asset); + + const fileUrl = asset.getFileUrl(); + + // mark bundle assets as loading + if (asset.type === 'bundle') { + const assetIds = asset.data.assets; + for (let i = 0; i < assetIds.length; i++) { + const assetInBundle = this._idToAsset.get(assetIds[i]); + if (!assetInBundle) + continue; + + if (assetInBundle.loaded || assetInBundle.resource || assetInBundle.loading) + continue; + + assetInBundle.loading = true; + } + } + + + this._loader.load(fileUrl, asset.type, _loaded, asset, options); } else { // asset has no file to load, open it directly const resource = this._loader.open(asset.type, asset.data); diff --git a/src/framework/asset/asset.js b/src/framework/asset/asset.js index 37a360d7652..a68d7fe192e 100644 --- a/src/framework/asset/asset.js +++ b/src/framework/asset/asset.js @@ -133,7 +133,7 @@ class Asset extends EventHandler { * * @param {string} name - A non-unique but human-readable name which can be later used to * retrieve the asset. - * @param {string} type - Type of asset. One of ["animation", "audio", "binary", "container", + * @param {string} type - Type of asset. One of ["animation", "audio", "binary", "bundle", "container", * "cubemap", "css", "font", "json", "html", "material", "model", "script", "shader", "sprite", * "template", text", "texture", "textureatlas"] * @param {object} [file] - Details about the file the asset is made from. At the least must @@ -197,6 +197,8 @@ class Asset extends EventHandler { // This is where the loaded resource(s) will be this._resources = []; + this.urlObject = null; + // a string-assetId dictionary that maps // locale to asset id this._i18n = {}; @@ -533,6 +535,11 @@ class Asset extends EventHandler { const old = this._resources; + if (this.urlObject) { + URL.revokeObjectURL(this.urlObject); + this.urlObject = null; + } + // clear resources on the asset this.resources = []; this.loaded = false; diff --git a/src/framework/bundle/bundle-registry.js b/src/framework/bundle/bundle-registry.js index 6b7d5b7df0d..f215db2f5d9 100644 --- a/src/framework/bundle/bundle-registry.js +++ b/src/framework/bundle/bundle-registry.js @@ -4,6 +4,34 @@ * @ignore */ class BundleRegistry { + /** + * Index of bundle assets. + * @type {Map} + * @private + */ + _idToBundle = new Map(); + + /** + * Index of asset id to set of bundle assets. + * @type {Map>} + * @private + */ + _assetToBundles = new Map(); + + /** + * Index of file url to set of bundle assets. + * @type {Map>} + * @private + */ + _urlsToBundles = new Map(); + + /** + * Index of file request to load callbacks. + * @type {Map} + * @private + */ + _fileRequests = new Map(); + /** * Create a new BundleRegistry instance. * @@ -11,67 +39,58 @@ class BundleRegistry { */ constructor(assets) { this._assets = assets; - - // index of bundle assets - this._bundleAssets = {}; - // index asset id to one more bundle assets - this._assetsInBundles = {}; - // index file urls to one or more bundle assets - this._urlsInBundles = {}; - // contains requests to load file URLs indexed by URL - this._fileRequests = {}; - - this._assets.on('add', this._onAssetAdded, this); - this._assets.on('remove', this._onAssetRemoved, this); + this._assets.bundles = this; + this._assets.on('add', this._onAssetAdd, this); + this._assets.on('remove', this._onAssetRemove, this); } - // Add asset in internal indexes - _onAssetAdded(asset) { + /** + * Called when asset is added to AssetRegistry. + * + * @param {import('../asset/asset.js').Asset} asset - The asset that has been added. + * @private + */ + _onAssetAdd(asset) { // if this is a bundle asset then add it and // index its referenced assets if (asset.type === 'bundle') { - this._bundleAssets[asset.id] = asset; - - this._registerBundleEventListeners(asset.id); - - for (let i = 0, len = asset.data.assets.length; i < len; i++) { - this._indexAssetInBundle(asset.data.assets[i], asset); + this._idToBundle.set(asset.id, asset); + this._assets.on(`load:start:${asset.id}`, this._onBundleLoadStart, this); + this._assets.on(`load:${asset.id}`, this._onBundleLoad, this); + this._assets.on(`error:${asset.id}`, this._onBundleError, this); + + const assetIds = asset.data.assets; + for (let i = 0; i < assetIds.length; i++) { + this._indexAssetInBundle(assetIds[i], asset); } } else { // if this is not a bundle then index its URLs - if (this._assetsInBundles[asset.id]) { + if (this._assetToBundles.has(asset.id)) { this._indexAssetFileUrls(asset); } } } - _registerBundleEventListeners(bundleAssetId) { - this._assets.on('load:' + bundleAssetId, this._onBundleLoaded, this); - this._assets.on('error:' + bundleAssetId, this._onBundleError, this); - } - - _unregisterBundleEventListeners(bundleAssetId) { - this._assets.off('load:' + bundleAssetId, this._onBundleLoaded, this); - this._assets.off('error:' + bundleAssetId, this._onBundleError, this); + _unbindAssetEvents(id) { + this._assets.off('load:start:' + id, this._onBundleLoadStart, this); + this._assets.off('load:' + id, this._onBundleLoad, this); + this._assets.off('error:' + id, this._onBundleError, this); } // Index the specified asset id and its file URLs so that // the registry knows that the asset is in that bundle - _indexAssetInBundle(assetId, bundleAsset) { - if (!this._assetsInBundles[assetId]) { - this._assetsInBundles[assetId] = [bundleAsset]; - } else { - const bundles = this._assetsInBundles[assetId]; - const idx = bundles.indexOf(bundleAsset); - if (idx === -1) { - bundles.push(bundleAsset); - } - } + _indexAssetInBundle(id, bundle) { + let bundles = this._assetToBundles.get(id); - const asset = this._assets.get(assetId); - if (asset) { - this._indexAssetFileUrls(asset); + if (!bundles) { + bundles = new Set(); + this._assetToBundles.set(id, bundles); } + + bundles.add(bundle); + + const asset = this._assets.get(id); + if (asset) this._indexAssetFileUrls(asset); } // Index the file URLs of the specified asset @@ -79,14 +98,10 @@ class BundleRegistry { const urls = this._getAssetFileUrls(asset); if (!urls) return; - for (let i = 0, len = urls.length; i < len; i++) { - const url = urls[i]; - // Just set the URL to point to the same bundles as the asset does. - // This is a performance/memory optimization and it assumes that - // the URL will not exist in any other asset. If that does happen then - // this will not work as expected if the asset is removed, as the URL will - // be removed too. - this._urlsInBundles[url] = this._assetsInBundles[asset.id]; + for (let i = 0; i < urls.length; i++) { + const bundles = this._assetToBundles.get(asset.id); + if (!bundles) continue; + this._urlsToBundles.set(urls[i], bundles); } } @@ -95,7 +110,7 @@ class BundleRegistry { let url = asset.getFileUrl(); if (!url) return null; - url = this._normalizeUrl(url); + url = url.split('?')[0]; const urls = [url]; // a font might have additional files @@ -110,97 +125,96 @@ class BundleRegistry { return urls; } - // Removes query parameters from a URL - _normalizeUrl(url) { - return url && url.split('?')[0]; - } - // Remove asset from internal indexes - _onAssetRemoved(asset) { + _onAssetRemove(asset) { if (asset.type === 'bundle') { // remove bundle from index - delete this._bundleAssets[asset.id]; + this._idToBundle.delete(asset.id); // remove event listeners - this._unregisterBundleEventListeners(asset.id); - - // remove bundle from _assetsInBundles and _urlInBundles indexes - for (const id in this._assetsInBundles) { - const array = this._assetsInBundles[id]; - const idx = array.indexOf(asset); - if (idx !== -1) { - array.splice(idx, 1); - if (!array.length) { - delete this._assetsInBundles[id]; - - // make sure we do not leave that array in - // any _urlInBundles entries - for (const url in this._urlsInBundles) { - if (this._urlsInBundles[url] === array) { - delete this._urlsInBundles[url]; - } - } + this._unbindAssetEvents(asset.id); + + // remove bundle from _assetToBundles and _urlInBundles indexes + const assetIds = asset.data.assets; + for (let i = 0; i < assetIds.length; i++) { + const bundles = this._assetToBundles.get(assetIds[i]); + if (!bundles) continue; + bundles.delete(asset); + + if (bundles.size === 0) { + this._assetToBundles.delete(assetIds[i]); + for (const [url, otherBundles] of this._urlsToBundles) { + if (otherBundles !== bundles) + continue; + this._urlsToBundles.delete(url); } } } // fail any pending requests for this bundle - this._onBundleError(`Bundle ${asset.id} was removed`, asset); + this._onBundleError(`Bundle ${asset.id} was removed`); + } else { + const bundles = this._assetToBundles.get(asset.id); + if (!bundles) return; - } else if (this._assetsInBundles[asset.id]) { - // remove asset from _assetInBundles - delete this._assetsInBundles[asset.id]; + this._assetToBundles.delete(asset.id); - // remove asset urls from _urlsInBundles + // remove asset urls from _urlsToBundles const urls = this._getAssetFileUrls(asset); - for (let i = 0, len = urls.length; i < len; i++) { - delete this._urlsInBundles[urls[i]]; + if (!urls) return; + for (let i = 0; i < urls.length; i++) { + this._urlsToBundles.delete(urls[i]); } } } + _onBundleLoadStart(asset) { + asset.resource.on('add', (url, data) => { + const callbacks = this._fileRequests.get(url); + if (!callbacks) return; + for (let i = 0; i < callbacks.length; i++) { + callbacks[i](null, data); + } + this._fileRequests.delete(url); + }); + } + // If we have any pending file requests // that can be satisfied by the specified bundle // then resolve them - _onBundleLoaded(bundleAsset) { - // this can happen if the bundleAsset failed + _onBundleLoad(asset) { + // this can happen if the asset failed // to create its resource - if (!bundleAsset.resource) { - this._onBundleError(`Bundle ${bundleAsset.id} failed to load`, bundleAsset); + if (!asset.resource) { + this._onBundleError(`Bundle ${asset.id} failed to load`); return; } - // on next tick resolve the pending asset requests - // don't do it on the same tick because that ties the loading - // of the bundle to the loading of all the assets - requestAnimationFrame(() => { - // make sure the registry hasn't been destroyed already - if (!this._fileRequests) { - return; - } + // make sure the registry hasn't been destroyed already + if (!this._fileRequests) + return; - for (const url in this._fileRequests) { - const bundles = this._urlsInBundles[url]; - if (!bundles || bundles.indexOf(bundleAsset) === -1) continue; + for (const [url, requests] of this._fileRequests) { + const bundles = this._urlsToBundles.get(url); + if (!bundles || !bundles.has(asset)) continue; - const decodedUrl = decodeURIComponent(url); - let err = null; - if (!bundleAsset.resource.hasBlobUrl(decodedUrl)) { - err = `Bundle ${bundleAsset.id} does not contain URL ${url}`; - } + const decodedUrl = decodeURIComponent(url); - const requests = this._fileRequests[url]; - for (let i = 0, len = requests.length; i < len; i++) { - if (err) { - requests[i](err); - } else { - requests[i](null, bundleAsset.resource.getBlobUrl(decodedUrl)); - } - } + let err, data; - delete this._fileRequests[url]; + if (asset.resource.has(decodedUrl)) { + data = asset.resource.get(decodedUrl); + } else if (asset.resource.loaded) { + err = `Bundle ${asset.id} does not contain URL ${url}`; + } else { + continue; } - }); + + for (let i = 0; i < requests.length; i++) { + requests[i](err, err || data); + } + this._fileRequests.delete(url); + } } // If we have outstanding file requests for any @@ -208,17 +222,14 @@ class BundleRegistry { // other bundles that can satisfy these requests. // If we do not find any other bundles then fail // those pending file requests with the specified error. - _onBundleError(err, bundleAsset) { - for (const url in this._fileRequests) { + _onBundleError(err) { + for (const [url, requests] of this._fileRequests) { const bundle = this._findLoadedOrLoadingBundleForUrl(url); if (!bundle) { - const requests = this._fileRequests[url]; - for (let i = 0, len = requests.length; i < len; i++) { + for (let i = 0; i < requests.length; i++) requests[i](err); - } - - delete this._fileRequests[url]; + this._fileRequests.delete(url); } } } @@ -226,52 +237,42 @@ class BundleRegistry { // Finds a bundle that contains the specified URL but // only returns the bundle if it's either loaded or being loaded _findLoadedOrLoadingBundleForUrl(url) { - const bundles = this._urlsInBundles[url]; + const bundles = this._urlsToBundles.get(url); if (!bundles) return null; - // look for loaded bundle first... - const len = bundles.length; - for (let i = 0; i < len; i++) { - // 'loaded' can be true but if there was an error - // then 'resource' would be null - if (bundles[i].loaded && bundles[i].resource) { - return bundles[i]; - } - } + let candidate = null; - // ...then look for loading bundles - for (let i = 0; i < len; i++) { - if (bundles[i].loading) { - return bundles[i]; + for (const bundle of bundles) { + if (bundle.loaded && bundle.resource) { + return bundle; + } else if (bundle.loading) { + candidate = bundle; } } - return null; + return candidate; } /** - * Lists all of the available bundles that reference the specified asset id. + * Lists all of the available bundles that reference the specified asset. * - * @param {import('../asset/asset.js').Asset} asset - The asset. - * @returns {import('../asset/asset.js').Asset[]} An array of bundle assets or null if the + * @param {import('../asset/asset.js').Asset} asset - The asset to search by. + * @returns {import('../asset/asset.js').Asset[]|null} An array of bundle assets or null if the * asset is not in any bundle. */ listBundlesForAsset(asset) { - return this._assetsInBundles[asset.id] || null; + const bundles = this._assetToBundles.get(asset.id); + if (bundles) return Array.from(bundles); + return null; } /** - * Lists all of the available bundles. This includes bundles that are not loaded. + * Lists all bundle assets. * * @returns {import('../asset/asset.js').Asset[]} An array of bundle assets. */ list() { - const result = []; - for (const id in this._bundleAssets) { - result.push(this._bundleAssets[id]); - } - - return result; + return Array.from(this._idToBundle.values()); } /** @@ -281,7 +282,7 @@ class BundleRegistry { * @returns {boolean} True or false. */ hasUrl(url) { - return !!this._urlsInBundles[url]; + return this._urlsToBundles.has(url); } /** @@ -291,7 +292,7 @@ class BundleRegistry { * @param {string} url - The url. * @returns {boolean} True or false. */ - canLoadUrl(url) { + urlIsLoadedOrLoading(url) { return !!this._findLoadedOrLoadingBundleForUrl(url); } @@ -305,8 +306,8 @@ class BundleRegistry { * the second argument is the file blob URL. * @example * const url = asset.getFileUrl().split('?')[0]; // get normalized asset URL - * this.app.bundles.loadFile(url, function (err, blobUrl) { - * // do something with the blob URL + * this.app.bundles.loadFile(url, function (err, data) { + * // do something with the data * }); */ loadUrl(url, callback) { @@ -319,17 +320,22 @@ class BundleRegistry { // Only load files from bundles that're explicitly requested to be loaded. if (bundle.loaded) { const decodedUrl = decodeURIComponent(url); - if (!bundle.resource.hasBlobUrl(decodedUrl)) { + + if (bundle.resource.has(decodedUrl)) { + callback(null, bundle.resource.get(decodedUrl)); + return; + } else if (bundle.resource.loaded) { callback(`Bundle ${bundle.id} does not contain URL ${url}`); return; } + } - callback(null, bundle.resource.getBlobUrl(decodedUrl)); - } else if (this._fileRequests.hasOwnProperty(url)) { - this._fileRequests[url].push(callback); - } else { - this._fileRequests[url] = [callback]; + let callbacks = this._fileRequests.get(url); + if (!callbacks) { + callbacks = []; + this._fileRequests.set(url, callbacks); } + callbacks.push(callback); } /** @@ -337,17 +343,25 @@ class BundleRegistry { * should be unloaded by the {@link AssetRegistry}. */ destroy() { - this._assets.off('add', this._onAssetAdded, this); - this._assets.off('remove', this._onAssetRemoved, this); + this._assets.off('add', this._onAssetAdd, this); + this._assets.off('remove', this._onAssetRemove, this); - for (const id in this._bundleAssets) { - this._unregisterBundleEventListeners(id); + for (const id of this._idToBundle.keys()) { + this._unbindAssetEvents(id); } this._assets = null; - this._bundleAssets = null; - this._assetsInBundles = null; - this._urlsInBundles = null; + + this._idToBundle.clear(); + this._idToBundle = null; + + this._assetToBundles.clear(); + this._assetToBundles = null; + + this._urlsToBundles.clear(); + this._urlsToBundles = null; + + this._fileRequests.clear(); this._fileRequests = null; } } diff --git a/src/framework/bundle/bundle.js b/src/framework/bundle/bundle.js index 0ea05798213..475e3bdc72a 100644 --- a/src/framework/bundle/bundle.js +++ b/src/framework/bundle/bundle.js @@ -1,23 +1,59 @@ +import { EventHandler } from '../../core/event-handler.js'; + /** - * Represents the resource of a Bundle Asset, which contains an index that maps URLs to blob URLs. + * Represents the resource of a Bundle Asset, which contains an index that maps URLs to DataViews. * * @ignore */ -class Bundle { +class Bundle extends EventHandler { + /** + * Index of file url to to DataView. + * @type {Map} + * @private + */ + _index = new Map(); + + /** + * If Bundle has all files loaded. + * @type {boolean} + * @private + */ + _loaded = false; + + /** + * Fired when a file has been added to a Bundle. + * + * @event + * @example + * bundle.on("add", function (url, data) { + * console.log("file added: " + url); + * }); + */ + static EVENT_ADD = 'add'; + /** - * Create a new Bundle instance. + * Fired when all files of a Bundle has been loaded. * - * @param {object[]} files - An array of objects that have a name field and contain a - * getBlobUrl() function. + * @event + * @example + * bundle.on("load", function () { + * console.log("All Bundle files has been loaded"); + * }); */ - constructor(files) { - this._blobUrls = {}; + static EVENT_LOAD = 'load'; - for (let i = 0, len = files.length; i < len; i++) { - if (files[i].url) { - this._blobUrls[files[i].name] = files[i].url; - } - } + /** + * Add file to a Bundle. + * + * @param {string} url - A url of a file. + * @param {DataView} data - A DataView of a file. + * @ignore + */ + addFile(url, data) { + if (this._index.has(url)) + return; + this._index.set(url, data); + this.fire('add', url, data); } /** @@ -27,29 +63,42 @@ class Bundle { * the URL first. * @returns {boolean} True of false. */ - hasBlobUrl(url) { - return !!this._blobUrls[url]; + has(url) { + return this._index.has(url); } /** - * Returns a blob URL for the specified URL. + * Returns a DataView for the specified URL. * * @param {string} url - The original file URL. Make sure you have called decodeURIComponent on * the URL first. - * @returns {string} A blob URL. + * @returns {DataView|null} A DataView. */ - getBlobUrl(url) { - return this._blobUrls[url]; + get(url) { + return this._index.get(url) || null; } /** - * Destroys the bundle and frees up blob URLs. + * Destroys the bundle. */ destroy() { - for (const key in this._blobUrls) { - URL.revokeObjectURL(this._blobUrls[key]); - } - this._blobUrls = null; + this._index.clear(); + } + + /** + * True if all files of a Bundle are loaded. + * @type {boolean} + */ + set loaded(value) { + if (!value || this._loaded) + return; + + this._loaded = true; + this.fire('load'); + } + + get loaded() { + return this._loaded; } } diff --git a/src/framework/handlers/binary.js b/src/framework/handlers/binary.js index ce726608692..9031046e3c7 100644 --- a/src/framework/handlers/binary.js +++ b/src/framework/handlers/binary.js @@ -1,5 +1,4 @@ import { http, Http } from '../../platform/net/http.js'; - import { ResourceHandler } from './handler.js'; class BinaryHandler extends ResourceHandler { @@ -27,6 +26,16 @@ class BinaryHandler extends ResourceHandler { } }); } + + /** + * Parses raw DataView and returns ArrayBuffer. + * + * @param {DataView} data - The raw data as a DataView + * @returns {ArrayBuffer} The parsed resource data. + */ + openBinary(data) { + return data.buffer; + } } export { BinaryHandler }; diff --git a/src/framework/handlers/bundle.js b/src/framework/handlers/bundle.js index 752670360fd..8208f340ee0 100644 --- a/src/framework/handlers/bundle.js +++ b/src/framework/handlers/bundle.js @@ -1,10 +1,6 @@ -import { platform } from '../../core/platform.js'; - -import { http, Http } from '../../platform/net/http.js'; - import { Bundle } from '../bundle/bundle.js'; -import { Untar, UntarWorker } from './untar.js'; - +import { Debug } from '../../core/debug.js'; +import { Untar } from './untar.js'; import { ResourceHandler } from './handler.js'; /** @@ -22,7 +18,7 @@ class BundleHandler extends ResourceHandler { super(app, 'bundle'); this._assets = app.assets; - this._worker = null; + this.maxRetries = 0; } load(url, callback) { @@ -33,55 +29,42 @@ class BundleHandler extends ResourceHandler { }; } - const self = this; + fetch(url.load, { + mode: 'cors', + credentials: 'include' + }).then((res) => { + const bundle = new Bundle(); + callback(null, bundle); - http.get(url.load, { - responseType: Http.ResponseType.ARRAY_BUFFER, - retry: this.maxRetries > 0, - maxRetries: this.maxRetries - }, function (err, response) { - if (!err) { - try { - self._untar(response, callback); - } catch (ex) { - callback('Error loading bundle resource ' + url.original + ': ' + ex); - } - } else { - callback('Error loading bundle resource ' + url.original + ': ' + err); - } - }); - } - - _untar(response, callback) { - const self = this; + const untar = new Untar(res, this._assets.prefix); - // use web workers if available otherwise - // fallback to untar'ing in the main thread - if (platform.workers) { - // create web worker if necessary - if (!self._worker) { - self._worker = new UntarWorker(self._assets.prefix); - } + untar.on('file', (file) => { + bundle.addFile(file.name, file.data); + }); - self._worker.untar(response, function (err, files) { - callback(err, files); + untar.on('done', () => { + bundle.loaded = true; + }); - // if we have no more requests for this worker then - // destroy it - if (!self._worker.hasPendingRequests()) { - self._worker.destroy(); - self._worker = null; - } + untar.on('error', (err) => { + Debug.error(err); + callback(err); }); - } else { - const archive = new Untar(response); - const files = archive.untar(self._assets.prefix); - callback(null, files); - } + }).catch((err) => { + Debug.error(err); + callback(err); + }); } - open(url, data) { - return new Bundle(data); + /** + * Open the bundle. + * + * @param {string} url - The URL of the resource to open. + * @param {Bundle} bundle - Bundle to open. + * @returns {Bundle} The bundle. + */ + open(url, bundle) { + return bundle; } } diff --git a/src/framework/handlers/css.js b/src/framework/handlers/css.js index 4477ec53660..b555e3e7c47 100644 --- a/src/framework/handlers/css.js +++ b/src/framework/handlers/css.js @@ -1,8 +1,15 @@ import { http } from '../../platform/net/http.js'; - import { ResourceHandler } from './handler.js'; class CssHandler extends ResourceHandler { + /** + * TextDecoder for decoding binary data. + * + * @type {TextDecoder|null} + * @private + */ + decoder = null; + constructor(app) { super(app, 'css'); } @@ -26,6 +33,17 @@ class CssHandler extends ResourceHandler { } }); } + + /** + * Parses raw DataView and returns string. + * + * @param {DataView} data - The raw data as a DataView + * @returns {string} The parsed resource data. + */ + openBinary(data) { + this.decoder ??= new TextDecoder('utf-8'); + return this.decoder.decode(data); + } } export { CssHandler }; diff --git a/src/framework/handlers/html.js b/src/framework/handlers/html.js index 0a05fc709b8..e6b24653e53 100644 --- a/src/framework/handlers/html.js +++ b/src/framework/handlers/html.js @@ -1,8 +1,15 @@ import { http } from '../../platform/net/http.js'; - import { ResourceHandler } from './handler.js'; class HtmlHandler extends ResourceHandler { + /** + * TextDecoder for decoding binary data. + * + * @type {TextDecoder|null} + * @private + */ + decoder = null; + constructor(app) { super(app, 'html'); } @@ -26,6 +33,17 @@ class HtmlHandler extends ResourceHandler { } }); } + + /** + * Parses raw DataView and returns string. + * + * @param {DataView} data - The raw data as a DataView + * @returns {string} The parsed resource data. + */ + openBinary(data) { + this.decoder ??= new TextDecoder('utf-8'); + return this.decoder.decode(data); + } } export { HtmlHandler }; diff --git a/src/framework/handlers/json.js b/src/framework/handlers/json.js index b56d116fc26..7b16ed16228 100644 --- a/src/framework/handlers/json.js +++ b/src/framework/handlers/json.js @@ -1,8 +1,15 @@ import { http, Http } from '../../platform/net/http.js'; - import { ResourceHandler } from './handler.js'; class JsonHandler extends ResourceHandler { + /** + * TextDecoder for decoding binary data. + * + * @type {TextDecoder|null} + * @private + */ + decoder = null; + constructor(app) { super(app, 'json'); } @@ -33,6 +40,17 @@ class JsonHandler extends ResourceHandler { } }); } + + /** + * Parses raw DataView and returns string. + * + * @param {DataView} data - The raw data as a DataView + * @returns {object} The parsed resource data. + */ + openBinary(data) { + this.decoder ??= new TextDecoder('utf-8'); + return JSON.parse(this.decoder.decode(data)); + } } export { JsonHandler }; diff --git a/src/framework/handlers/loader.js b/src/framework/handlers/loader.js index b58b6fd03bd..6ca1929b878 100644 --- a/src/framework/handlers/loader.js +++ b/src/framework/handlers/loader.js @@ -94,12 +94,19 @@ class ResourceLoader { * an error occurs. Passed (err, resource) where err is null if there are no errors. * @param {import('../asset/asset.js').Asset} [asset] - Optional asset that is passed into * handler. + * @param {object} [options] - Additional options for loading. + * @param {boolean} [options.bundlesIgnore] - If set to true, then asset will not try to load + * from a bundle. Defaults to false. + * @param {import('../asset/asset-registry.js').BundlesFilterCallback} [options.bundlesFilter] - A callback that will be called + * when loading an asset that is contained in any of the bundles. It provides an array of + * bundles and will ensure asset is loaded from bundle returned from a callback. By default + * smallest filesize bundle is choosen. * @example * app.loader.load("../path/to/texture.png", "texture", function (err, texture) { * // use texture here * }); */ - load(url, type, callback, asset) { + load(url, type, callback, asset, options) { const handler = this._handlers[type]; if (!handler) { const err = `No resource handler for asset type: '${type}' when loading [${url}]`; @@ -134,6 +141,28 @@ class ResourceLoader { return; } + if (urlObj.load instanceof DataView) { + if (handler.openBinary) { + if (!self._requests[key]) + return; + + try { + const data = handler.openBinary(urlObj.load); + self._onSuccess(key, data); + } catch (err) { + self._onFailure(key, err); + } + return; + } + + urlObj.load = URL.createObjectURL(new Blob([urlObj.load])); + if (asset) { + if (asset.urlObject) + URL.revokeObjectURL(asset.urlObject); + asset.urlObject = urlObj.load; + } + } + handler.load(urlObj, function (err, data, extra) { // make sure key exists because loader // might have been destroyed by now @@ -155,10 +184,25 @@ class ResourceLoader { }; const normalizedUrl = url.split('?')[0]; - if (this._app.enableBundles && this._app.bundles.hasUrl(normalizedUrl)) { - if (!this._app.bundles.canLoadUrl(normalizedUrl)) { - handleLoad(`Bundle for ${url} not loaded yet`); - return; + if (this._app.enableBundles && this._app.bundles.hasUrl(normalizedUrl) && !(options && options.bundlesIgnore)) { + // if there is no loaded bundle with asset, then start loading a bundle + if (!this._app.bundles.urlIsLoadedOrLoading(normalizedUrl)) { + const bundles = this._app.bundles.listBundlesForAsset(asset); + let bundle; + + if (options && options.bundlesFilter) { + bundle = options.bundlesFilter(bundles); + } + + if (!bundle) { + // prioritize smallest bundle + bundles?.sort((a, b) => { + return a.file.size - b.file.size; + }); + bundle = bundles?.[0]; + } + + if (bundle) this._app.assets?.load(bundle); } this._app.bundles.loadUrl(normalizedUrl, function (err, fileUrlFromBundle) { diff --git a/src/framework/handlers/shader.js b/src/framework/handlers/shader.js index c3186152c56..c5a8840548b 100644 --- a/src/framework/handlers/shader.js +++ b/src/framework/handlers/shader.js @@ -1,8 +1,15 @@ import { http } from '../../platform/net/http.js'; - import { ResourceHandler } from './handler.js'; class ShaderHandler extends ResourceHandler { + /** + * TextDecoder for decoding binary data. + * + * @type {TextDecoder|null} + * @private + */ + decoder = null; + constructor(app) { super(app, 'shader'); } @@ -26,6 +33,17 @@ class ShaderHandler extends ResourceHandler { } }); } + + /** + * Parses raw DataView and returns string. + * + * @param {DataView} data - The raw data as a DataView + * @returns {string} The parsed resource data. + */ + openBinary(data) { + this.decoder ??= new TextDecoder('utf-8'); + return this.decoder.decode(data); + } } export { ShaderHandler }; diff --git a/src/framework/handlers/template.js b/src/framework/handlers/template.js index ad3a05d5ff7..4ec1fc005ee 100644 --- a/src/framework/handlers/template.js +++ b/src/framework/handlers/template.js @@ -1,10 +1,16 @@ import { http } from '../../platform/net/http.js'; - import { Template } from '../template.js'; - import { ResourceHandler } from './handler.js'; class TemplateHandler extends ResourceHandler { + /** + * TextDecoder for decoding binary data. + * + * @type {TextDecoder|null} + * @private + */ + decoder = null; + constructor(app) { super(app, 'template'); } @@ -35,6 +41,17 @@ class TemplateHandler extends ResourceHandler { open(url, data) { return new Template(this._app, data); } + + /** + * Parses raw DataView and returns string. + * + * @param {DataView} data - The raw data as a DataView + * @returns {Template} The parsed resource data. + */ + openBinary(data) { + this.decoder ??= new TextDecoder('utf-8'); + return new Template(this._app, JSON.parse(this.decoder.decode(data))); + } } export { TemplateHandler }; diff --git a/src/framework/handlers/text.js b/src/framework/handlers/text.js index 054fde79b67..0e2698a6c40 100644 --- a/src/framework/handlers/text.js +++ b/src/framework/handlers/text.js @@ -1,8 +1,15 @@ import { http } from '../../platform/net/http.js'; - import { ResourceHandler } from './handler.js'; class TextHandler extends ResourceHandler { + /** + * TextDecoder for decoding binary data. + * + * @type {TextDecoder|null} + * @private + */ + decoder = null; + constructor(app) { super(app, 'text'); } @@ -26,6 +33,17 @@ class TextHandler extends ResourceHandler { } }); } + + /** + * Parses raw DataView and returns string. + * + * @param {DataView} data - The raw data as a DataView + * @returns {string} The parsed resource data. + */ + openBinary(data) { + this.decoder ??= new TextDecoder('utf-8'); + return this.decoder.decode(data); + } } export { TextHandler }; diff --git a/src/framework/handlers/untar.js b/src/framework/handlers/untar.js index 8fcc479a76a..2e0c8f6088f 100644 --- a/src/framework/handlers/untar.js +++ b/src/framework/handlers/untar.js @@ -1,369 +1,201 @@ -let Untar; // see below why we declare this here - -// The UntarScope function is going to be used -// as the code that ends up in a Web Worker. -// The Untar variable is declared outside the scope so that -// we do not have to add a 'return' statement to the UntarScope function. -// We also have to make sure that we do not mangle 'Untar' variable otherwise -// the Worker will not work. -function UntarScope(isWorker) { - let utfDecoder; - let asciiDecoder; - - if (typeof TextDecoder !== 'undefined') { - try { - utfDecoder = new TextDecoder('utf-8'); - asciiDecoder = new TextDecoder('windows-1252'); - } catch (e) { - console.warn('TextDecoder not supported - pc.Untar module will not work'); - } - } else { - console.warn('TextDecoder not supported - pc.Untar module will not work'); - } - - function PaxHeader(fields) { - this._fields = fields; - } - - PaxHeader.parse = function (buffer, start, length) { - const paxArray = new Uint8Array(buffer, start, length); - let bytesRead = 0; - const fields = []; - - while (bytesRead < length) { - let spaceIndex; - for (spaceIndex = bytesRead; spaceIndex < length; spaceIndex++) { - if (paxArray[spaceIndex] === 0x20) - break; - } - - if (spaceIndex >= length) { - throw new Error('Invalid PAX header data format.'); - } - - const fieldLength = parseInt(utfDecoder.decode(new Uint8Array(buffer, start + bytesRead, spaceIndex - bytesRead)), 10); - const fieldText = utfDecoder.decode(new Uint8Array(buffer, start + spaceIndex + 1, fieldLength - (spaceIndex - bytesRead) - 2)); - const field = fieldText.split('='); - - if (field.length !== 2) { - throw new Error('Invalid PAX header data format.'); - } - - if (field[1].length === 0) { - field[1] = null; - } - - fields.push({ - name: field[0], - value: field[1] - }); - - bytesRead += fieldLength; - } - - return new PaxHeader(fields); - }; - - PaxHeader.prototype.applyHeader = function (file) { - for (let i = 0; i < this._fields.length; i++) { - let fieldName = this._fields[i].name; - const fieldValue = this._fields[i].value; - - if (fieldName === 'path') { - fieldName = 'name'; - } - - if (fieldValue === null) { - delete file[fieldName]; - } else { - file[fieldName] = fieldValue; - } - } - }; +import { EventHandler } from '../../core/event-handler.js'; +/** + * A utility class for untaring archives from a fetch request. It processes files from a tar file + * in a streamed manner, so asset parsing can happen in parallel instead of all at once at the end. + * + * @augments EventHandler + * @ignore + */ +class Untar extends EventHandler { /** + * @type {number} * @private - * @name Untar - * @classdesc Untars a tar archive in the form of an array buffer. - * @param {ArrayBuffer} arrayBuffer - The array buffer that holds the tar archive. - * @description Creates a new instance of Untar. */ - function UntarInternal(arrayBuffer) { - this._arrayBuffer = arrayBuffer || new ArrayBuffer(0); - this._bufferView = new DataView(this._arrayBuffer); - this._globalPaxHeader = null; - this._paxHeader = null; - this._bytesRead = 0; - } - - if (!isWorker) { - Untar = UntarInternal; - } + headerSize = 512; /** + * @type {number} * @private - * @function - * @name Untar#_hasNext - * @description Whether we have more files to untar. - * @returns {boolean} Returns true or false. */ - UntarInternal.prototype._hasNext = function () { - return this._bytesRead + 4 < this._arrayBuffer.byteLength && this._bufferView.getUint32(this._bytesRead) !== 0; - }; + paddingSize = 512; /** + * @type {number} * @private - * @function - * @name Untar#_readNextFile - * @description Untars the next file in the archive. - * @returns {object} Returns a file descriptor in the following format: - * {name, size, start, url}. */ - UntarInternal.prototype._readNextFile = function () { - const headersDataView = new DataView(this._arrayBuffer, this._bytesRead, 512); - const headers = asciiDecoder.decode(headersDataView); - this._bytesRead += 512; - - let name = headers.substring(0, 100).replace(/\0/g, ''); - const ustarFormat = headers.substring(257, 263); - const size = parseInt(headers.substring(124, 136), 8); - const type = headers.substring(156, 157); - const start = this._bytesRead; - let url = null; - - let normalFile = false; - switch (type) { - case '0': case '': // Normal file - // do not create blob URL if we are in a worker - // because if the worker is destroyed it will also destroy the blob URLs - normalFile = true; - if (!isWorker) { - const blob = new Blob([this._arrayBuffer.slice(this._bytesRead, this._bytesRead + size)]); - url = URL.createObjectURL(blob); - } - break; - case 'g': // Global PAX header - this._globalPaxHeader = PaxHeader.parse(this._arrayBuffer, this._bytesRead, size); - break; - case 'x': // PAX header - this._paxHeader = PaxHeader.parse(this._arrayBuffer, this._bytesRead, size); - break; - case '1': // Link to another file already archived - case '2': // Symbolic link - case '3': // Character special device - case '4': // Block special device - case '5': // Directory - case '6': // FIFO special file - case '7': // Reserved - default: // Unknown file type - } - - this._bytesRead += size; - - // File data is padded to reach a 512 byte boundary; skip the padded bytes too. - const remainder = size % 512; - if (remainder !== 0) { - this._bytesRead += (512 - remainder); - } - - if (!normalFile) { - return null; - } - - if (ustarFormat.indexOf('ustar') !== -1) { - const namePrefix = headers.substring(345, 500).replace(/\0/g, ''); - - if (namePrefix.length > 0) { - name = namePrefix.trim() + name.trim(); - } - } - - const file = { - name: name, - start: start, - size: size, - url: url - }; + bytesRead = 0; - if (this._globalPaxHeader) { - this._globalPaxHeader.applyHeader(file); - } + /** + * @type {number} + * @private + */ + bytesReceived = 0; - if (this._paxHeader) { - this._paxHeader.applyHeader(file); - this._paxHeader = null; - } + /** + * @type {boolean} + * @private + */ + headerRead = false; - return file; - }; + /** + * @type {ReadableStream|null} + * @private + */ + reader = null; /** + * @type {Uint8Array} * @private - * @function - * @name Untar#untar - * @description Untars the array buffer provided in the constructor. - * @param {string} [filenamePrefix] - The prefix for each filename in the tar archive. This is usually the {@link AssetRegistry} prefix. - * @returns {object[]} An array of files in this format {name, start, size, url}. */ - UntarInternal.prototype.untar = function (filenamePrefix) { - if (!utfDecoder) { - console.error('Cannot untar because TextDecoder interface is not available for this platform.'); - return []; - } + data = new Uint8Array(0); - const files = []; - while (this._hasNext()) { - const file = this._readNextFile(); - if (!file) continue; - if (filenamePrefix && file.name) { - file.name = filenamePrefix + file.name; - } - files.push(file); - } + /** + * @type {TextDecoder|null} + * @private + */ + decoder = null; - return files; - }; - - // if we are in a worker then create the onmessage handler using worker.self - if (isWorker) { - self.onmessage = function (e) { - const id = e.data.id; - - try { - const archive = new UntarInternal(e.data.arrayBuffer); - const files = archive.untar(e.data.prefix); - // The worker is done so send a message to the main thread. - // Notice we are sending the array buffer back as a Transferrable object - // so that the main thread can re-assume control of the array buffer. - postMessage({ - id: id, - files: files, - arrayBuffer: e.data.arrayBuffer - }, [e.data.arrayBuffer]); - } catch (err) { - postMessage({ - id: id, - error: err.toString() - }); - } - }; - } -} + /** + * @type {string} + * @private + */ + prefix = ''; -// this is the URL that is going to be used for workers -let workerUrl = null; + /** + * @type {string} + * @private + */ + fileName = ''; -// Convert the UntarScope function to a string and add -// the onmessage handler for the worker to untar archives -function getWorkerUrl() { - if (!workerUrl) { - // execute UntarScope function in the worker - const code = '(' + UntarScope.toString() + ')(true)\n\n'; + /** + * @type {number} + * @private + */ + fileSize = 0; - // create blob URL for the code above to be used for the worker - const blob = new Blob([code], { type: 'application/javascript' }); + /** + * @type {string} + * @private + */ + fileType = ''; - workerUrl = URL.createObjectURL(blob); - } - return workerUrl; -} + /** + * @type {string} + * @private + */ + ustarFormat = ''; -/** - * Wraps untar'ing a tar archive with a Web Worker. - * - * @ignore - */ -class UntarWorker { /** - * Creates new instance of an UntarWorker. + * Create an instance of an Untar. * - * @param {string} [filenamePrefix] - The prefix that should be added to each file name in the - * archive. This is usually the {@link AssetRegistry} prefix. + * @param {Promise} fetchPromise - A Promise object returned from a fetch request. + * @param {string} assetsPrefix - Assets registry files prefix. + * @ignore */ - constructor(filenamePrefix) { - this._requestId = 0; - this._pendingRequests = {}; - this._filenamePrefix = filenamePrefix; - this._worker = new Worker(getWorkerUrl()); - this._worker.addEventListener('message', this._onMessage.bind(this)); + constructor(fetchPromise, assetsPrefix = '') { + super(); + this.prefix = assetsPrefix || ''; + this.reader = fetchPromise.body.getReader(); + + this.reader.read().then((res) => { + this.pump(res.done, res.value); + }).catch((err) => { + this.fire('error', err); + }); } /** - * @param {MessageEvent} e - The message event from the worker. - * @private + * This method is called multiple times when the stream provides data. + * + * @param {boolean} done - True when reading data is complete. + * @param {Uint8Array} value - Chunk of data read from a stream. + * @returns {Promise|null} Return new pump Promise or null when no more data is available. + * @ignore */ - _onMessage(e) { - const id = e.data.id; - if (!this._pendingRequests[id]) return; - - const callback = this._pendingRequests[id]; + pump(done, value) { + if (done) { + this.fire('done'); + return null; + } - delete this._pendingRequests[id]; + this.bytesReceived += value.byteLength; - if (e.data.error) { - callback(e.data.error); - } else { - const arrayBuffer = e.data.arrayBuffer; + const data = new Uint8Array(this.data.length + value.length); + data.set(this.data); + data.set(value, this.data.length); + this.data = data; - // create blob URLs for each file. We are creating the URLs - // here - outside of the worker - so that the main thread owns them - for (let i = 0, len = e.data.files.length; i < len; i++) { - const file = e.data.files[i]; - const blob = new Blob([arrayBuffer.slice(file.start, file.start + file.size)]); - file.url = URL.createObjectURL(blob); - } + while (this.readFile()); - callback(null, e.data.files); - } + return this.reader.read().then((res) => { + this.pump(res.done, res.value); + }).catch((err) => { + this.fire('error', err); + }); } /** - * Untars the specified array buffer using a Web Worker and returns the result in the callback. + * Attempt to read file from an available data buffer * - * @param {ArrayBuffer} arrayBuffer - The array buffer that holds the tar archive. - * @param {Function} callback - The callback function called when the worker is finished or if - * there is an error. The callback has the following arguments: {error, files}, where error is - * a string if any, and files is an array of file descriptors. + * @returns {boolean} True if file was successfully read and more data is potentially available for + * processing. + * @ignore */ - untar(arrayBuffer, callback) { - const id = this._requestId++; - this._pendingRequests[id] = callback; - - // send data to the worker - notice the last argument - // converts the arrayBuffer to a Transferrable object - // to avoid copying the array buffer which would cause a stall. - // However this causes the worker to assume control of the array - // buffer so we cannot access this buffer until the worker is done with it. - this._worker.postMessage({ - id: id, - prefix: this._filenamePrefix, - arrayBuffer: arrayBuffer - }, [arrayBuffer]); - } + readFile() { + if (!this.headerRead && this.bytesReceived > (this.bytesRead + this.headerSize)) { + this.headerRead = true; + const view = new DataView(this.data.buffer, this.bytesRead, this.headerSize); + this.decoder ??= new TextDecoder('windows-1252'); + const headers = this.decoder.decode(view); + + this.fileName = headers.substring(0, 100).replace(/\0/g, ''); + this.fileSize = parseInt(headers.substring(124, 136), 8); + this.fileType = headers.substring(156, 157); + this.ustarFormat = headers.substring(257, 263); + + if (this.ustarFormat.indexOf('ustar') !== -1) { + const prefix = headers.substring(345, 500).replace(/\0/g, ''); + if (prefix.length > 0) { + this.fileName = prefix.trim() + this.fileName.trim(); + } + } - /** - * Returns whether the worker has pending requests to untar array buffers. - * - * @returns {boolean} Returns true if there are pending requests and false otherwise. - */ - hasPendingRequests() { - return Object.keys(this._pendingRequests).length > 0; - } + this.bytesRead += 512; + } - /** - * Destroys the internal Web Worker. - */ - destroy() { - if (this._worker) { - this._worker.terminate(); - this._worker = null; + if (this.headerRead) { + // buffer might be not long enough + if (this.bytesReceived < (this.bytesRead + this.fileSize)) { + return false; + } + + // normal file + if (this.fileType === '' || this.fileType === '0') { + const dataView = new DataView(this.data.buffer, this.bytesRead, this.fileSize); + + const file = { + name: this.prefix + this.fileName, + size: this.fileSize, + data: dataView + }; - this._pendingRequests = null; + this.fire('file', file); + } + + this.bytesRead += this.fileSize; + this.headerRead = false; + + // bytes padding + const bytesRemained = this.bytesRead % this.paddingSize; + if (bytesRemained !== 0) + this.bytesRead += this.paddingSize - bytesRemained; + + return true; } + + return false; } } -// execute the UntarScope function in order to declare the Untar constructor -UntarScope(); - -export { Untar, UntarWorker }; +export { Untar }; diff --git a/tests/framework/bundles/test_bundle_loader.js b/tests/framework/bundles/test_bundle_loader.js index 723c61e6515..f674cb45739 100644 --- a/tests/framework/bundles/test_bundle_loader.js +++ b/tests/framework/bundles/test_bundle_loader.js @@ -1,7 +1,5 @@ describe('Test Bundle Loader', function () { beforeEach(function () { - this._workers = pc.platform.workers; - // create app this.app = new pc.Application(document.createElement('canvas')); @@ -11,7 +9,7 @@ describe('Test Bundle Loader', function () { this.assets = [ new pc.Asset('css', 'css', { filename: 'css.css', - url: 'files/css/css.css' + url: 'base/tests/test-assets/bundles/css.css' }), new pc.Asset('html', 'html', { filename: 'html.html', @@ -33,22 +31,26 @@ describe('Test Bundle Loader', function () { // filename: 'cubemap.dds', // url: 'files/cubemap/cubemap.dds' // }), - new pc.Asset('model', 'model', { - filename: 'model.json', - url: 'files/model/model.json' + new pc.Asset('container', 'container', { + filename: 'container.glb', + url: 'files/container/container.glb' }), new pc.Asset('texture', 'texture', { filename: 'texture.jpg', url: 'files/texture/texture.jpg' }), new pc.Asset('atlas', 'textureatlas', { - filename: 'atlas.jpg', - url: 'files/textureatlas/atlas.jpg' + filename: 'texture.jpg', + url: 'files/textureatlas/texture.jpg' }), new pc.Asset('animation', 'animation', { filename: 'animation.json', url: 'files/animation/animation.json' }), + new pc.Asset('template', 'template', { + filename: 'template.json', + url: 'files/template/template.json' + }), new pc.Asset('font', 'font', { filename: 'font.png', url: 'files/font/font.png' @@ -89,6 +91,9 @@ describe('Test Bundle Loader', function () { cubemap: { instanceof: pc.Texture }, + template: { + instanceof: pc.Template + }, texture: { instanceof: pc.Texture }, @@ -98,6 +103,12 @@ describe('Test Bundle Loader', function () { model: { instanceof: pc.Model }, + render: { + instanceof: pc.Render + }, + container: { + instanceof: pc.Container + }, animation: { instanceof: pc.Animation }, @@ -111,7 +122,8 @@ describe('Test Bundle Loader', function () { // the bundle asset this.bundleAsset = new pc.Asset('bundle asset', 'bundle', { - url: 'base/tests/test-assets/bundles/bundle.tar.gz' + url: 'base/tests/test-assets/bundles/bundle.tar', + size: 133632 }, { assets: this.assets.map(function (asset) { return asset.id; @@ -120,143 +132,199 @@ describe('Test Bundle Loader', function () { }); afterEach(function () { - pc.platform.workers = this._workers; this.app.destroy(); }); - it('should load bundle asset', function (done) { - expect(pc.platform.workers).to.equal(true); - - var self = this; - self.app.assets.add(this.bundleAsset); - self.assets.forEach(function (asset) { - self.app.assets.add(asset); + it('should load bundle asset and its assets', function (done) { + this.app.assets.add(this.bundleAsset); + this.assets.forEach((asset) => { + this.app.assets.add(asset); }); - self.app.assets.load(this.bundleAsset); + this.app.assets.load(this.bundleAsset); - self.app.assets.on('load:' + self.bundleAsset.id, function () { - expect(self.bundleAsset.resource instanceof pc.Bundle).to.equal(true); - self.assets.forEach(function (asset) { - expect(self.bundleAsset.resource.hasBlobUrl(asset.file.url)).to.equal(true); + this.app.assets.on('load:' + this.bundleAsset.id, () => { + expect(this.bundleAsset.resource instanceof pc.Bundle).to.equal(true); + this.assets.forEach((asset) => { + const url = (this.app.assets.prefix || '') + asset.file.url; + expect(this.bundleAsset.resource.has(url)).to.equal(true); }); done(); }); }); - it('should load bundle asset without using web workers', function (done) { - pc.platform.workers = false; + it('should load assets from bundle', function (done) { + const self = this; + let loaded = 0; - var self = this; - self.app.assets.add(this.bundleAsset); - self.assets.forEach(function (asset) { - self.app.assets.add(asset); + this.app.assets.add(this.bundleAsset); + this.assets.forEach((asset) => { + this.app.assets.add(asset); }); - self.app.assets.load(this.bundleAsset); + this.app.assets.load(this.bundleAsset); - self.app.assets.on('load:' + self.bundleAsset.id, function () { - expect(self.bundleAsset.resource instanceof pc.Bundle).to.equal(true); - self.assets.forEach(function (asset) { - expect(self.bundleAsset.resource.hasBlobUrl(asset.file.url)).to.equal(true); - }); - done(); - }); - }); + const onLoad = function() { + loaded++; + var resource = this.type === 'cubemap' ? this.resources[1] : this.resource; + expect(resource).to.not.equal(null); + var expected = self.expectedTypes[this.type]; - it('should load assets from bundle', function (done) { - expect(pc.platform.workers).to.equal(true); - - var self = this; - var todo = 0; - - var onLoad = function () { - todo--; - if (todo === 0) { - self.assets.forEach(function (asset, index) { - var resource = asset.type === 'cubemap' ? asset.resources[1] : asset.resource; - expect(resource).to.not.equal(null); - var expected = self.expectedTypes[asset.type]; - - if (expected.typeof) { - expect(typeof resource).to.equal(expected.typeof); - } - - if (expected.instanceof) { - expect(resource instanceof expected.instanceof).to.equal(true); - } - - if (asset.type === 'font') { - expect(resource.textures.length).to.equal(2); - } - }); + if (expected.typeof) { + expect(typeof resource).to.equal(expected.typeof); + } + + if (expected.instanceof) { + expect(resource instanceof expected.instanceof).to.equal(true); + } + + if (this.type === 'font') { + expect(resource.textures.length).to.equal(2); + } + + if ((self.assets.length + 1) === loaded) { done(); } }; + this.assets.forEach((asset, index) => { + asset.on('load', onLoad); + }); this.bundleAsset.on('load', onLoad); - self.app.assets.add(this.bundleAsset); - self.app.assets.load(this.bundleAsset); - todo++; + }); - self.assets.forEach(function (asset) { - asset.on('load', onLoad); - self.app.assets.add(asset); - self.app.assets.load(asset); - todo++; + it('asset should load if bundle with that asset has loaded', function(done) { + this.app.assets.add(this.bundleAsset); + this.app.assets.add(this.assets[0]); + + expect(this.assets[0].loading).to.equal(false); + this.app.assets.load(this.bundleAsset); + expect(this.assets[0].loading).to.equal(true); + + this.assets[0].ready(() => { + done(); }); }); - it('should load assets from bundle without using web workers', function (done) { - pc.platform.workers = false; + it('bundle should load if asset from it has loaded', function(done) { + this.app.assets.add(this.bundleAsset); + this.app.assets.add(this.assets[0]); + + expect(this.bundleAsset.loading).to.equal(false); + this.app.assets.load(this.assets[0]); + expect(this.bundleAsset.loading).to.equal(true); - var self = this; - var todo = 0; + this.bundleAsset.ready(() => { + done(); + }); + }); - var onLoad = function () { - todo--; - if (todo === 0) { - self.assets.forEach(function (asset, index) { - var resource = asset.type === 'cubemap' ? asset.resources[1] : asset.resource; - expect(resource).to.not.equal(null); - var expected = self.expectedTypes[asset.type]; + it('bundle should load if asset from it has loaded', function(done) { + this.app.assets.add(this.bundleAsset); + this.app.assets.add(this.assets[0]); - if (expected.typeof) { - expect(typeof resource).to.equal(expected.typeof); - } + expect(this.bundleAsset.loading).to.equal(false); + this.app.assets.load(this.assets[0]); + expect(this.bundleAsset.loading).to.equal(true); - if (expected.instanceof) { - expect(resource instanceof expected.instanceof).to.equal(true); - } + this.bundleAsset.ready(() => { + done(); + }); + }); - if (asset.type === 'font') { - expect(resource.textures.length).to.equal(2); - } - }); - done(); + it('asset loading with bundlesIgnore option should not load bundle', function(done) { + this.app.assets.add(this.bundleAsset); + this.app.assets.add(this.assets[0]); + + let filterCalled = false; + + expect(this.bundleAsset.loading).to.equal(false); + this.app.assets.load(this.assets[0], { + bundlesIgnore: true, + bundlesFilter: (bundles) => { + filterCalled = true; } - }; + }); + expect(filterCalled).to.equal(false); + expect(this.bundleAsset.loading).to.equal(false); - this.bundleAsset.on('load', onLoad); - self.app.assets.add(this.bundleAsset); - self.app.assets.load(this.bundleAsset); - todo++; + this.assets[0].ready(() => { + done(); + }); + }); - self.assets.forEach(function (asset) { - asset.on('load', onLoad); - self.app.assets.add(asset); - self.app.assets.load(asset); - todo++; + it('asset loading should prefer smallest bundle', function(done) { + const bundleAsset2 = new pc.Asset('bundle asset 2', 'bundle', { + url: 'base/tests/test-assets/bundles/bundle.tar', + size: 133632 + 1 + }, { + assets: this.assets.map(function (asset) { + return asset.id; + }) + }); + + this.app.assets.add(bundleAsset2); + this.app.assets.add(this.bundleAsset); + this.app.assets.add(this.assets[0]); + + expect(this.bundleAsset.loading).to.equal(false); + this.app.assets.load(this.assets[0]); + expect(this.bundleAsset.loading).to.equal(true); + expect(bundleAsset2.loading).to.equal(false); + + this.assets[0].ready(() => { + done(); }); }); - it('should fail loading assets if the bundle has not started loading', function (done) { + it('asset loading with bundlesFilter', function(done) { + const bundleAsset2 = new pc.Asset('bundle asset 2', 'bundle', { + url: 'base/tests/test-assets/bundles/bundle.tar', + size: 133632 + 1 + }, { + assets: this.assets.map(function (asset) { + return asset.id; + }) + }); + + this.app.assets.add(bundleAsset2); this.app.assets.add(this.bundleAsset); + this.app.assets.add(this.assets[0]); + + let filterCalled = false; + + expect(bundleAsset2.loading).to.equal(false); + + this.app.assets.load(this.assets[0], { + bundlesFilter: (bundles) => { + filterCalled = true; + expect(bundles.length).to.equal(2); - this.app.assets.on('error:' + this.assets[0].id, function (err) { + if (bundles[0].name === 'bundle asset 2') { + return bundles[0]; + } else { + return bundles[1]; + } + } + }); + expect(filterCalled).to.equal(true); + expect(bundleAsset2.loading).to.equal(true); + expect(this.bundleAsset.loading).to.equal(false); + + this.assets[0].ready(() => { done(); }); + }); + + it('loadUrl() calls callback if bundle loaded', function (done) { + this.app.assets.add(this.bundleAsset); this.app.assets.add(this.assets[0]); - this.app.assets.load(this.assets[0]); + this.app.assets.load(this.bundleAsset); + + this.app.assets.bundles.loadUrl(this.assets[0].file.url, function (err, dataView) { + expect(err).to.equal(null); + expect(dataView instanceof DataView).to.equal(true); + done(); + }); }); }); diff --git a/tests/framework/bundles/test_bundle_registry.js b/tests/framework/bundles/test_bundle_registry.js index 434491109b1..98de7882a50 100644 --- a/tests/framework/bundles/test_bundle_registry.js +++ b/tests/framework/bundles/test_bundle_registry.js @@ -192,7 +192,7 @@ describe('pc.BundleRegistry', function () { expect(this.bundles.hasUrl('text.txt')).to.equal(false); }); - it('canLoadUrl() returns false if bundle not loaded', function () { + it('urlIsLoadedOrLoading() returns false if bundle not loaded', function () { var asset = new pc.Asset('asset', 'text', { url: 'text.txt' }); @@ -203,10 +203,10 @@ describe('pc.BundleRegistry', function () { }); this.assets.add(bundleAsset); - expect(this.bundles.canLoadUrl('text.txt')).to.equal(false); + expect(this.bundles.urlIsLoadedOrLoading('text.txt')).to.equal(false); }); - it('canLoadUrl() returns false if bundle loaded without a resource', function () { + it('urlIsLoadedOrLoading() returns false if bundle loaded without a resource', function () { var asset = new pc.Asset('asset', 'text', { url: 'text.txt' }); @@ -218,10 +218,10 @@ describe('pc.BundleRegistry', function () { this.assets.add(bundleAsset); bundleAsset.loaded = true; - expect(this.bundles.canLoadUrl('text.txt')).to.equal(false); + expect(this.bundles.urlIsLoadedOrLoading('text.txt')).to.equal(false); }); - it('canLoadUrl() returns true if bundle loaded', function () { + it('urlIsLoadedOrLoading() returns true if bundle loaded', function () { var asset = new pc.Asset('asset', 'text', { url: 'text.txt' }); @@ -235,10 +235,10 @@ describe('pc.BundleRegistry', function () { bundleAsset.loaded = true; bundleAsset.resource = sinon.fake(); - expect(this.bundles.canLoadUrl('text.txt')).to.equal(true); + expect(this.bundles.urlIsLoadedOrLoading('text.txt')).to.equal(true); }); - it('canLoadUrl() returns true if bundle being loaded', function () { + it('urlIsLoadedOrLoading() returns true if bundle being loaded', function () { var asset = new pc.Asset('asset', 'text', { url: 'text.txt' }); @@ -250,152 +250,7 @@ describe('pc.BundleRegistry', function () { this.assets.add(bundleAsset); bundleAsset.loading = true; - expect(this.bundles.canLoadUrl('text.txt')).to.equal(true); - }); - - it('loadUrl() calls callback if bundle loaded', function (done) { - var asset = new pc.Asset('asset', 'text', { - url: 'text.txt' - }); - this.assets.add(asset); - - var bundleAsset = new pc.Asset('bundle', 'bundle', null, { - assets: [asset.id] - }); - this.assets.add(bundleAsset); - bundleAsset.loaded = true; - bundleAsset.resource = sinon.fake(); - bundleAsset.resource.hasBlobUrl = sinon.fake.returns(true); - bundleAsset.resource.getBlobUrl = sinon.fake.returns('blob url'); - - this.bundles.loadUrl('text.txt', function (err, blobUrl) { - expect(err).to.equal(null); - expect(blobUrl).to.equal('blob url'); - done(); - }); - }); - - it('loadUrl() calls callback if bundle is loaded later', function (done) { - var asset = new pc.Asset('asset', 'text', { - url: 'text.txt' - }); - this.assets.add(asset); - - var bundleAsset = new pc.Asset('bundle', 'bundle', null, { - assets: [asset.id] - }); - this.assets.add(bundleAsset); - bundleAsset.loading = true; - - this.bundles.loadUrl('text.txt', function (err, blobUrl) { - expect(err).to.equal(null); - expect(blobUrl).to.equal('blob url'); - done(); - }); - - setTimeout(function () { - bundleAsset.loading = false; - bundleAsset.loaded = true; - bundleAsset.resource = sinon.fake(); - bundleAsset.resource.hasBlobUrl = sinon.fake.returns(true); - bundleAsset.resource.getBlobUrl = sinon.fake.returns('blob url'); - this.assets.fire('load:' + bundleAsset.id, bundleAsset); - }.bind(this)); - }); - - it('loadUrl() calls callback if other bundle that contains the asset is loaded later', function (done) { - var asset = new pc.Asset('asset', 'text', { - url: 'text.txt' - }); - this.assets.add(asset); - - var bundleAsset = new pc.Asset('bundle', 'bundle', null, { - assets: [asset.id] - }); - this.assets.add(bundleAsset); - bundleAsset.loading = true; - - var bundleAsset2 = new pc.Asset('bundle2', 'bundle', null, { - assets: [asset.id] - }); - this.assets.add(bundleAsset2); - bundleAsset2.loading = true; - - this.bundles.loadUrl('text.txt', function (err, blobUrl) { - expect(err).to.equal(null); - expect(blobUrl).to.equal('blob url'); - done(); - }); - - setTimeout(function () { - this.assets.remove(bundleAsset); - bundleAsset2.loading = false; - bundleAsset2.loaded = true; - bundleAsset2.resource = sinon.fake(); - bundleAsset2.resource.hasBlobUrl = sinon.fake.returns(true); - bundleAsset2.resource.getBlobUrl = sinon.fake.returns('blob url'); - this.assets.fire('load:' + bundleAsset2.id, bundleAsset2); - }.bind(this)); - }); - - it('loadUrl() calls callback with error if bundle that contains the asset is removed', function (done) { - var asset = new pc.Asset('asset', 'text', { - url: 'text.txt' - }); - this.assets.add(asset); - - var bundleAsset = new pc.Asset('bundle', 'bundle', null, { - assets: [asset.id] - }); - this.assets.add(bundleAsset); - bundleAsset.loading = true; - - this.bundles.loadUrl('text.txt', function (err, blobUrl) { - expect(err).to.be.a('string'); - done(); - }); - - setTimeout(function () { - this.assets.remove(bundleAsset); - }.bind(this)); - }); - - it('loadUrl() calls callback if bundle fails to load but another bundle that contains the asset is loaded later', function (done) { - var asset = new pc.Asset('asset', 'text', { - url: 'text.txt' - }); - this.assets.add(asset); - - var bundleAsset = new pc.Asset('bundle', 'bundle', null, { - assets: [asset.id] - }); - this.assets.add(bundleAsset); - bundleAsset.loading = true; - - var bundleAsset2 = new pc.Asset('bundle2', 'bundle', null, { - assets: [asset.id] - }); - this.assets.add(bundleAsset2); - bundleAsset2.loading = true; - - this.bundles.loadUrl('text.txt', function (err, blobUrl) { - expect(err).to.equal(null); - expect(blobUrl).to.equal('blob url'); - done(); - }); - - setTimeout(function () { - bundleAsset.loading = false; - bundleAsset.loaded = true; - this.assets.fire('error:' + bundleAsset.id, 'error'); - - bundleAsset2.loading = false; - bundleAsset2.loaded = true; - bundleAsset2.resource = sinon.fake(); - bundleAsset2.resource.hasBlobUrl = sinon.fake.returns(true); - bundleAsset2.resource.getBlobUrl = sinon.fake.returns('blob url'); - this.assets.fire('load:' + bundleAsset2.id, bundleAsset2); - }.bind(this)); + expect(this.bundles.urlIsLoadedOrLoading('text.txt')).to.equal(true); }); it('loadUrl() calls callback with error if bundle fails to load', function (done) { diff --git a/tests/test-assets/bundles/bundle.tar b/tests/test-assets/bundles/bundle.tar new file mode 100644 index 00000000000..f916062e191 Binary files /dev/null and b/tests/test-assets/bundles/bundle.tar differ diff --git a/tests/test-assets/bundles/bundle.tar.gz b/tests/test-assets/bundles/bundle.tar.gz deleted file mode 100644 index 851d84fc7d7..00000000000 Binary files a/tests/test-assets/bundles/bundle.tar.gz and /dev/null differ diff --git a/tests/test-assets/bundles/css.css b/tests/test-assets/bundles/css.css new file mode 100644 index 00000000000..b0b388b1487 --- /dev/null +++ b/tests/test-assets/bundles/css.css @@ -0,0 +1,3 @@ +body { + background: #f00; +} \ No newline at end of file