diff --git a/build/types/core b/build/types/core index 5f19623958..30d61997d6 100644 --- a/build/types/core +++ b/build/types/core @@ -82,6 +82,7 @@ +../../lib/util/mp4_box_parsers.js +../../lib/util/mp4_parser.js +../../lib/util/multi_map.js ++../../lib/util/mutex.js +../../lib/util/networking.js +../../lib/util/object_utils.js +../../lib/util/operation_manager.js diff --git a/docs/design/bg-fetch-after.gv b/docs/design/bg-fetch-after.gv new file mode 100644 index 0000000000..4a6b8f196b --- /dev/null +++ b/docs/design/bg-fetch-after.gv @@ -0,0 +1,29 @@ +# Generate png with: dot -Tpng -O after.gv +digraph storage_after { + subgraph cluster_0 { + label="Shaka Player"; + parse[label="Download and parse manifest (parseManifest)"]; + drm[label="Make DRM engine and load keys (createDrmEngine)"] + filter[label="Filter manifest (filterManifest_)"]; + segments[label="Download segments (downloadSegments_)"]; + store[label="Store manifest (cell.addManifests)"]; + parse -> drm; + drm -> filter; + filter -> store; + store -> segments[label="BG Fetch Not Available"]; + } + subgraph cluster_1 { + label="Service Worker"; + bgSegments[label="Download segments in background (backgroundFetch.fetch)"] + store -> bgSegments[label="BG Fetch Available"]; + } + subgraph cluster_2 { + label="Shaka Player Static Methods"; + storeSeg[label="Store segments one-by-one (assignStreamToManifest)"] + remove[label="Clean up (cleanStoredManifest)"]; + segments -> remove[label="On Fail"]; + segments -> storeSeg; + bgSegments -> storeSeg; + bgSegments -> remove[label="On Fail"]; + } +} \ No newline at end of file diff --git a/docs/design/bg-fetch-after.gv.png b/docs/design/bg-fetch-after.gv.png new file mode 100644 index 0000000000..1a0d5d5c13 Binary files /dev/null and b/docs/design/bg-fetch-after.gv.png differ diff --git a/docs/design/bg-fetch-before.gv b/docs/design/bg-fetch-before.gv new file mode 100644 index 0000000000..8da689b8cc --- /dev/null +++ b/docs/design/bg-fetch-before.gv @@ -0,0 +1,18 @@ +# Generate png with: dot -Tpng -O before.gv +digraph storage_before { + subgraph cluster_0 { + label="Shaka Player"; + parse[label="Download and parse manifest (parseManifest)"]; + drm[label="Make DRM engine and load keys (createDrmEngine)"] + filter[label="Filter manifest (filterManifest_)"]; + segments[label="Download and store segments (downloadManifest_)"]; + store[label="Store manifest (cell.addManifests)"]; + remove[label="Clean up (cell.removeSegments)"]; + parse -> drm; + drm -> filter; + filter -> segments; + segments -> store; + segments -> remove[label="On Fail"]; + store -> remove[label="On Fail"]; + } +} diff --git a/docs/design/bg-fetch-before.gv.png b/docs/design/bg-fetch-before.gv.png new file mode 100644 index 0000000000..832a340c80 Binary files /dev/null and b/docs/design/bg-fetch-before.gv.png differ diff --git a/docs/design/bg-fetch.md b/docs/design/bg-fetch.md new file mode 100644 index 0000000000..15dca62e1c --- /dev/null +++ b/docs/design/bg-fetch.md @@ -0,0 +1,134 @@ +# Shaka Player Background Fetch Support + +last update: 2021-7-12 + +by: [theodab@google.com](mailto:theodab@google.com) + + +## Overview + +The feature of background fetch has been in Shaka Player’s backlog [since 2017]. +At the time it was added to the backlog, the feature was not quite ready for +use. Since then, it has matured, and now is something we could feasibly use, but +it has still been a low-priority feature. + +[since 2017]: https://github.com/google/shaka-player/issues/879 + +## Design Concept + +This design attempts to reuse existing code whenever possible, in order to +minimize the amount of new code that has to be tested. The code will be made in +two main stages: +1. Refactor the offline download process to change the order that the asset is +downloaded. The manifest should be downloaded and stored first, and then every +segment should be downloaded. As a segment is downloaded, it should be stored. +The code for storing a segment, in particular, should be broken out into an +exported static (e.g. stateless) function. +1. Modify the Shaka Player wrapper to add the appropriate background fetch event +listeners if the environment is detected to be a service worker, so that a +compiled Shaka Player bundle can be used as a service worker. If background +fetch is used, the segment downloading step should be passed to this service +worker. When each segment is downloaded, it should be passed to the static +storage functions added in stage 1. + +By restructuring the offline storage code in this way, switching between +foreground and background fetch will just be a matter of calling a different +segment-downloading function. In addition, it is possible that, in the future, a +plugin interface could be made for this. That probably won’t be necessary unless +another browser makes a competing API for downloading in the background (which +is admittedly a possibility, as background fetch [is not yet a W3C standard]). + +[is not yet a W3C standard]: https://wicg.github.io/background-fetch/ + +### Storage System Process: Before + +![Shaka storage system flow before](bg-fetch-before.gv.png) + + +### Storage System Process: After + +![Shaka storage system flow after](bg-fetch-after.gv.png) + + +## Implementation + +### Changes to shaka.offline.Storage + +1. Change createOfflineManifest_ to leave the storage indexes on the segments +null at first. With this change, downloadManifest_ will now only be downloading +the encryption keys (which cannot be downloaded via background fetch, as they +require request bodies). +1. Create a new step within store_, after the manifest is stored, called +“downloadSegments_” that makes a Set of SegmentReference objects that need to be +downloaded. + 1. We use SegmentReference objects in order to contain the URI, startByte, + and endByte. + 1. This change also means we will no longer need an internal cache for + downloaded segments, as they will be deduplicated by the use of a Set. +1. If background fetch is not available, downloadSegments_ will simply download +the segments from this set as before, and then once they are all downloaded, +pass them all to assignStreamsToManifest. +1. If background fetch is available, this set will be turned into an array, +Request objects should be made for the individual uris (with appropriate headers +applied), and then that array will be passed to the service worker with a +background fetch call. The service worker will then, after everything is +downloaded and stored, call assignStreamsToManifest. An estimate of the total +download size will need to be computed here, and padded to avoid premature +cancellation for inaccurate manifests. +1. Create a new public static method, assignStreamToManifest. This is a static +method that requires no internal state, so that the service worker can call it. +It stores the data provided, loads the manifest from storage, applies the +storage id of the data to the appropriate segments (based on uri), and then +stores the modified manifest. It should have a mutex over the part that loads +and changes the manifest, to keep one invocation from overriding the manifest +changes of another. It should have the following parameters: + 1. manifestStorageId + 1. uri + 1. data + 1. throwIfAbortedFn +1. Create a second public static method, cleanStoredManifest. This method is +meant to be called by the service worker in the instance of the fetch operation +being aborted, and will simply clear the manifest away. It will also clear any +segments that have been stored already. This also means we will no longer need +the segmentsFromStore_ array, which we had previously been using to un-store +after canceled or failed downloads. It should have the following parameters: + 1. manifestStorageId +1. When filling out shaka.extern.StoredContent entries for the list() method, +the storage system should be sure to set the offlineUri field to null if the +manifest is still “isIncomplete”, to mark that the asset has not yet finished +downloading. This will help developers detect that an asset is mid-download on +page load, so that they can set up progress indicators if they so wish. + + +### Service Worker Design + +1. This code should go in, or at least be loaded in, the wrapper code. This will +let us access Shaka Player methods inside the service worker, without having to +coordinate how to load a compiled Shaka Player bundle from a service worker; +the user can simply load a Shaka Player bundle as a service worker. +1. When the background fetch message is called (see [the documentation]), the +“id” field should be set to the storage id of the manifest, with an added prefix +of “Shaka-”. The API does not provide any field for custom data, but this value +still needs to be provided to the service worker somehow. Luckily, this is the +only extra data the service worker needs, so it can just be the id of the fetch +operation. + 1. When handling background fetch-related events, we can simply ignore any + event that does not start with the prefix. This will help prevent any + contamination with other service worker code from the developer. +1. As each segment is downloaded, the assignStreamToManifest method should be +called to store that data in the manifest. +1. If the download is canceled, call the cleanStoredManifest method, so that the +player doesn’t pollute indexedDb with unused segment data. +1. As a service worker is essentially just a collection of event listeners, one +can theoretically listen to the same event multiple times. This is relevant +because [a given scope] can only have a single service worker, so our service +worker code will have to be something that other people can load into their +existing service workers, if they have any. +1. Our system should use the message event to pass a specific identifying +message to the service worker, and the service worker will be expected to +respond with a specific response message. This way, we won’t mistake an +unrelated service worker for our own. + 1. This message can also be used to make sure the versions are the same. + +[the documentation]: https://developers.google.com/web/updates/2018/12/background-fetch#starting_a_background_fetch +[a given scope]: https://developers.google.com/web/fundamentals/primers/service-workers#register_a_service_worker \ No newline at end of file diff --git a/lib/offline/indexeddb/storage_mechanism.js b/lib/offline/indexeddb/storage_mechanism.js index ccc6f0a6c4..88a2fb0afa 100644 --- a/lib/offline/indexeddb/storage_mechanism.js +++ b/lib/offline/indexeddb/storage_mechanism.js @@ -315,7 +315,7 @@ shaka.offline.indexeddb.StorageMechanism = class { const del = window.indexedDB.deleteDatabase(name); del.onblocked = (event) => { - shaka.log.warning('Deleting', name, 'is being blocked'); + shaka.log.warning('Deleting', name, 'is being blocked', event); }; del.onsuccess = (event) => { p.resolve(); diff --git a/lib/offline/storage.js b/lib/offline/storage.js index 5364bf30fb..93ec2fcd27 100644 --- a/lib/offline/storage.js +++ b/lib/offline/storage.js @@ -29,6 +29,7 @@ goog.require('shaka.util.Functional'); goog.require('shaka.util.IDestroyable'); goog.require('shaka.util.Iterables'); goog.require('shaka.util.MimeUtils'); +goog.require('shaka.util.Mutex'); goog.require('shaka.util.Platform'); goog.require('shaka.util.PlayerConfiguration'); goog.require('shaka.util.StreamUtils'); @@ -94,14 +95,6 @@ shaka.offline.Storage = class { this.networkingEngine_ = new shaka.net.NetworkingEngine(); } - /** - * A list of segment ids for all the segments that were added during the - * current store. If the store fails or is aborted, these need to be - * removed from storage. - * @private {!Array.} - */ - this.segmentsFromStore_ = []; - /** * A list of open operations that are being performed by this instance of * |shaka.offline.Storage|. @@ -448,28 +441,14 @@ shaka.offline.Storage = class { return shaka.offline.StoredContentUtils.fromManifestDB( offlineUri, manifestDB); } catch (e) { - // If we did start saving some data, we need to remove it all to avoid - // wasting storage. However if the muxer did not manage to initialize, - // then we won't have an active cell to remove the segments from. - if (activeHandle) { - await activeHandle.cell.removeSegments( - this.segmentsFromStore_, () => {}); - } - if (manifestId != null) { - const uri = shaka.offline.OfflineUri.manifest( - activeHandle.path.mechanism, - activeHandle.path.cell, - manifestId); - await this.remove_(uri.toString()); + await shaka.offline.Storage.cleanStoredManifest(manifestId); } // If we already had an error, ignore this error to avoid hiding // the original error. throw drmError || e; } finally { - this.segmentsFromStore_ = []; - await muxer.destroy(); if (parser) { @@ -500,8 +479,14 @@ shaka.offline.Storage = class { async downloadSegments_( toDownload, manifestId, manifestDB, downloader, config, storage, manifest, drmEngine) { - /** @param {!Array.} toDownload */ - const download = (toDownload) => { + /** + * @param {!Array.} toDownload + * @param {boolean} updateDRM + */ + const download = async (toDownload, updateDRM) => { + const throwIfAbortedFn = () => { + this.ensureNotDestroyed_(); + }; for (const download of toDownload) { /** @param {?BufferSource} data */ let data; @@ -516,67 +501,173 @@ shaka.offline.Storage = class { request, estimateId, isInitSegment, onDownloaded); downloader.queueWork(download.groupId, async () => { goog.asserts.assert(data, 'We should have loaded data by now'); + goog.asserts.assert(data instanceof ArrayBuffer, + 'The data should be an ArrayBuffer'); const ref = /** @type {!shaka.media.SegmentReference} */ ( download.ref); - const idForRef = shaka.offline.DownloadInfo.idForSegmentRef(ref); - - // Store the segment. - const ids = await storage.addSegments([{data: data}]); - this.ensureNotDestroyed_(); - this.segmentsFromStore_.push(ids[0]); - - // Attach the segment to the manifest. - let complete = true; - for (const stream of manifestDB.streams) { - for (const segment of stream.segments) { - if (segment.pendingSegmentRefId == idForRef) { - segment.dataKey = ids[0]; - // Now that the segment has been associated with the - // appropriate dataKey, the pendingSegmentRefId is no longer - // necessary. - segment.pendingSegmentRefId = undefined; - } - if (segment.pendingInitSegmentRefId == idForRef) { - segment.initSegmentKey = ids[0]; - // Now that the init segment has been associated with the - // appropriate initSegmentKey, the pendingInitSegmentRefId is - // no longer necessary. - segment.pendingInitSegmentRefId = undefined; - } - if (segment.pendingSegmentRefId) { - complete = false; - } - if (segment.pendingInitSegmentRefId) { - complete = false; - } - } - } - if (complete) { - manifestDB.isIncomplete = false; - } + manifestDB = (await shaka.offline.Storage.assignStreamToManifest( + manifestId, ref, {data}, throwIfAbortedFn)) || manifestDB; }); } + await downloader.waitToFinish(); + + if (updateDRM) { + // Re-store the manifest, to attach session IDs. + // These were (maybe) discovered inside the downloader; we can only add + // them now, at the end, since the manifestDB is in flux during the + // process of downloading and storing, and assignStreamToManifest does + // not know about the DRM engine. + this.ensureNotDestroyed_(); + this.setManifestDrmFields_(manifest, manifestDB, drmEngine, config); + await storage.updateManifest(manifestId, manifestDB); + } }; - if (this.getManifestIsEncrypted_(manifest) && + const usingBgFetch = false; // TODO: Get. + + if (this.getManifestIsEncrypted_(manifest) && usingBgFetch && !this.getManifestIncludesInitData_(manifest)) { // Background fetch can't make DRM sessions, so if we have to get the // init data from the init segments, download those first before anything // else. - download(toDownload.filter((info) => info.isInitSegment)); + await download(toDownload.filter((info) => info.isInitSegment), true); + this.ensureNotDestroyed_(); toDownload = toDownload.filter((info) => !info.isInitSegment); } - download(toDownload); + if (!usingBgFetch) { + await download(toDownload, false); + this.ensureNotDestroyed_(); - // Re-store the manifest, to update the size and attach session IDs. - manifestDB.size = await downloader.waitToFinish(); - this.setManifestDrmFields_(manifest, manifestDB, drmEngine, config); - goog.asserts.assert( - !manifestDB.isIncomplete, 'The manifest should be complete by now'); - await storage.updateManifest(manifestId, manifestDB); - this.ensureNotDestroyed_(); + goog.asserts.assert( + !manifestDB.isIncomplete, 'The manifest should be complete by now'); + } else { + // TODO: Send the request to the service worker. Don't await the result. + } + } + + /** + * Removes all of the contents for a given manifest, statelessly. + * + * @param {number} manifestId + * @return {!Promise} + */ + static async cleanStoredManifest(manifestId) { + const muxer = new shaka.offline.StorageMuxer(); + await muxer.init(); + const activeHandle = await muxer.getActive(); + const uri = shaka.offline.OfflineUri.manifest( + activeHandle.path.mechanism, + activeHandle.path.cell, + manifestId); + await muxer.destroy(); + const storage = new shaka.offline.Storage(); + await storage.remove(uri.toString()); + } + + /** + * Load the given manifest, modifies it by assigning the given data to the + * segments corresponding to "ref", then re-stores the manifest. + * The parts of this function that modify the manifest are protected by a + * mutex, to prevent race conditions; specifically, it prevents two parallel + * instances of this method from both loading the manifest into memory at the + * same time, which would result in the slower/later call overwriting the + * changes of the other. + * + * @param {number} manifestId + * @param {!shaka.media.SegmentReference} ref + * @param {shaka.extern.SegmentDataDB} data + * @param {function()} throwIfAbortedFn A function that should throw if the + * download has been aborted. + * @return {!Promise.} + */ + static async assignStreamToManifest(manifestId, ref, data, throwIfAbortedFn) { + /** @type {shaka.offline.StorageMuxer} */ + const muxer = new shaka.offline.StorageMuxer(); + + const idForRef = shaka.offline.DownloadInfo.idForSegmentRef(ref); + let manifestUpdated = false; + let dataKey; + let activeHandle; + /** @type {!shaka.extern.ManifestDB} */ + let manifestDB; + + let mutexId = 0; + + try { + await muxer.init(); + activeHandle = await muxer.getActive(); + + // Store the data. + const dataKeys = await activeHandle.cell.addSegments([data]); + dataKey = dataKeys[0]; + throwIfAbortedFn(); + + // Acquire the mutex before accessing the manifest, since there could be + // multiple instances of this method running at once. + mutexId = await shaka.offline.Storage.mutex_.acquire(); + throwIfAbortedFn(); + + // Load the manifest. + const manifests = await activeHandle.cell.getManifests([manifestId]); + throwIfAbortedFn(); + manifestDB = manifests[0]; + + // Assign the stored data to the manifest. + let complete = true; + for (const stream of manifestDB.streams) { + for (const segment of stream.segments) { + if (segment.pendingSegmentRefId == idForRef) { + segment.dataKey = dataKey; + // Now that the segment has been associated with the appropriate + // dataKey, the pendingSegmentRefId is no longer necessary. + segment.pendingSegmentRefId = undefined; + } + if (segment.pendingInitSegmentRefId == idForRef) { + segment.initSegmentKey = dataKey; + // Now that the init segment has been associated with the + // appropriate initSegmentKey, the pendingInitSegmentRefId is no + // longer necessary. + segment.pendingInitSegmentRefId = undefined; + } + if (segment.pendingSegmentRefId) { + complete = false; + } + if (segment.pendingInitSegmentRefId) { + complete = false; + } + } + } + + // Update the size of the manifest. + manifestDB.size += data.data.byteLength; + + // Mark the manifest as complete, if all segments are downloaded. + if (complete) { + manifestDB.isIncomplete = false; + } + + // Re-store the manifest. + await activeHandle.cell.updateManifest(manifestId, manifestDB); + manifestUpdated = true; + throwIfAbortedFn(); + } catch (e) { + await shaka.offline.Storage.cleanStoredManifest(manifestId); + + if (activeHandle && !manifestUpdated && dataKey) { + // The cleanStoredManifest method will not "see" any segments that have + // been downloaded but not assigned to the manifest yet. So un-store + // them separately. + await activeHandle.cell.removeSegments([dataKey], (key) => {}); + } + + throw e; + } finally { + await muxer.destroy(); + shaka.offline.Storage.mutex_.release(mutexId); + } + return manifestDB; } /** @@ -1568,6 +1659,9 @@ shaka.offline.Storage = class { } }; +/** @private {!shaka.util.Mutex} */ +shaka.offline.Storage.mutex_ = new shaka.util.Mutex(); + shaka.offline.Storage.defaultSystemIds_ = new Map() .set('org.w3.clearkey', '1077efecc0b24d02ace33c1e52e2fb4b') .set('com.widevine.alpha', 'edef8ba979d64acea3c827dcd51d21ed') diff --git a/lib/util/mutex.js b/lib/util/mutex.js new file mode 100644 index 0000000000..afb22f9b95 --- /dev/null +++ b/lib/util/mutex.js @@ -0,0 +1,52 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +goog.provide('shaka.util.Mutex'); + + +/** + * @summary A simple mutex. + */ +shaka.util.Mutex = class { + /** Creates the mutex. */ + constructor() { + /** @private {!Array.} */ + this.waiting_ = []; + + /** @private {number} */ + this.nextMutexId_ = 0; + + /** @private {number} */ + this.acquiredMutexId_ = 0; + } + + /** @return {!Promise.} mutexId */ + async acquire() { + const mutexId = ++this.nextMutexId_; + if (!this.acquiredMutexId_) { + this.acquiredMutexId_ = mutexId; + } else { + await (new Promise((resolve, reject) => { + this.waiting_.push(() => { + this.acquiredMutexId_ = mutexId; + resolve(); + }); + })); + } + return mutexId; + } + + /** @param {number} mutexId */ + release(mutexId) { + if (mutexId == this.acquiredMutexId_) { + this.acquiredMutexId_ = 0; + if (this.waiting_.length > 0) { + const callback = this.waiting_.shift(); + callback(); + } + } + } +};