diff --git a/build/types/offline b/build/types/offline index 4a6a930e55..aa7974f772 100644 --- a/build/types/offline +++ b/build/types/offline @@ -1,5 +1,6 @@ # The offline storage system and manifest parser plugin. ++../../lib/offline/download_info.js +../../lib/offline/download_manager.js +../../lib/offline/download_progress_estimator.js +../../lib/offline/indexeddb/base_storage_cell.js diff --git a/externs/shaka/offline.js b/externs/shaka/offline.js index ea7b6f06de..e55c7bfa1d 100644 --- a/externs/shaka/offline.js +++ b/externs/shaka/offline.js @@ -33,7 +33,8 @@ shaka.extern.OfflineSupport; * size: number, * expiration: number, * tracks: !Array., - * appMetadata: Object + * appMetadata: Object, + * isIncomplete: boolean * }} * * @property {?string} offlineUri @@ -53,6 +54,9 @@ shaka.extern.OfflineSupport; * The tracks that are stored. * @property {Object} appMetadata * The metadata passed to store(). + * @property {boolean} isIncomplete + * If true, the content is still downloading. Manifests with this set cannot + * be played yet. * @exportDoc */ shaka.extern.StoredContent; @@ -68,7 +72,8 @@ shaka.extern.StoredContent; * streams: !Array., * sessionIds: !Array., * drmInfo: ?shaka.extern.DrmInfo, - * appMetadata: Object + * appMetadata: Object, + * isIncomplete: (boolean|undefined) * }} * * @property {number} creationTime @@ -91,6 +96,8 @@ shaka.extern.StoredContent; * The DRM info used to initialize EME. * @property {Object} appMetadata * A metadata object passed from the application. + * @property {(boolean|undefined)} isIncomplete + * If true, the content is still downloading. */ shaka.extern.ManifestDB; @@ -195,6 +202,8 @@ shaka.extern.StreamDB; * appendWindowEnd: number, * timestampOffset: number, * tilesLayout: ?string, + * pendingSegmentRefId: (string|undefined), + * pendingInitSegmentRefId: (string|undefined), * dataKey: number * }} * @@ -215,6 +224,17 @@ shaka.extern.StreamDB; * The value is a grid-item-dimension consisting of two positive decimal * integers in the format: column-x-row ('4x3'). It describes the * arrangement of Images in a Grid. The minimum valid LAYOUT is '1x1'. + * @property {(string|undefined)} pendingSegmentRefId + * Contains an id that identifies what the segment was, originally. Used to + * coordinate where segments are stored, during the downloading process. + * If this field is non-null, it's assumed that the segment is not fully + * downloaded. + * @property {(string|undefined)} pendingInitSegmentRefId + * Contains an id that identifies what the init segment was, originally. + * Used to coordinate where init segments are stored, during the downloading + * process. + * If this field is non-null, it's assumed that the init segment is not fully + * downloaded. * @property {number} dataKey * The key to the data in storage. */ @@ -333,6 +353,15 @@ shaka.extern.StorageCell = class { */ addManifests(manifests) {} + /** + * Updates the given manifest, stored at the given key. + * + * @param {number} key + * @param {!shaka.extern.ManifestDB} manifest + * @return {!Promise} + */ + updateManifest(key, manifest) {} + /** * Replace the expiration time of the manifest stored under |key| with * |newExpiration|. If no manifest is found under |key| then this should diff --git a/lib/offline/download_info.js b/lib/offline/download_info.js new file mode 100644 index 0000000000..0c5ce747aa --- /dev/null +++ b/lib/offline/download_info.js @@ -0,0 +1,68 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +goog.provide('shaka.offline.DownloadInfo'); + +goog.require('shaka.util.Networking'); +goog.requireType('shaka.media.InitSegmentReference'); +goog.requireType('shaka.media.SegmentReference'); + + +/** + * An object that represents a single segment, that the storage system will soon + * download, but has not yet started downloading. + */ +shaka.offline.DownloadInfo = class { + /** + * @param {shaka.media.SegmentReference|shaka.media.InitSegmentReference} ref + * @param {number} estimateId + * @param {number} groupId + * @param {boolean} isInitSegment + */ + constructor(ref, estimateId, groupId, isInitSegment) { + /** @type {shaka.media.SegmentReference|shaka.media.InitSegmentReference} */ + this.ref = ref; + + /** @type {number} */ + this.estimateId = estimateId; + + /** @type {number} */ + this.groupId = groupId; + + /** @type {boolean} */ + this.isInitSegment = isInitSegment; + } + + /** + * Creates an ID that encapsulates all important information in the ref, which + * can then be used to check for equality. + * @param {shaka.media.SegmentReference|shaka.media.InitSegmentReference} ref + * @return {string} + */ + static idForSegmentRef(ref) { + // Escape the URIs using encodeURI, to make sure that a weirdly formed URI + // cannot cause two unrelated refs to be considered equivalent. + return ref.getUris().map((uri) => '{' + encodeURI(uri) + '}').join('') + + ':' + ref.startByte + ':' + ref.endByte; + } + + /** @return {string} */ + getRefId() { + return shaka.offline.DownloadInfo.idForSegmentRef(this.ref); + } + + /** + * @param {shaka.extern.PlayerConfiguration} config + * @return {!shaka.extern.Request} + */ + makeSegmentRequest(config) { + return shaka.util.Networking.createSegmentRequest( + this.ref.getUris(), + this.ref.startByte, + this.ref.endByte, + config.streaming.retryParameters); + } +}; diff --git a/lib/offline/download_manager.js b/lib/offline/download_manager.js index c5588d32dd..b5e3390c1b 100644 --- a/lib/offline/download_manager.js +++ b/lib/offline/download_manager.js @@ -102,6 +102,16 @@ shaka.offline.DownloadManager = class { return Promise.all(promises); } + /** + * Adds a byte length to the download estimate. + * + * @param {number} estimatedByteLength + * @return {number} estimateId + */ + addDownloadEstimate(estimatedByteLength) { + return this.estimator_.open(estimatedByteLength); + } + /** * Add a request to be downloaded as part of a group. * @@ -109,18 +119,16 @@ shaka.offline.DownloadManager = class { * The group to add this segment to. If the group does not exist, a new * group will be created. * @param {shaka.extern.Request} request - * @param {number} estimatedByteLength + * @param {number} estimateId * @param {boolean} isInitSegment * @param {function(BufferSource):!Promise} onDownloaded * The callback for when this request has been downloaded. Downloading for * |group| will pause until the promise returned by |onDownloaded| resolves. * @return {!Promise} Resolved when this request is complete. */ - queue(groupId, request, estimatedByteLength, isInitSegment, onDownloaded) { + queue(groupId, request, estimateId, isInitSegment, onDownloaded) { this.destroyer_.ensureNotDestroyed(); - const id = this.estimator_.open(estimatedByteLength); - const group = this.groups_.get(groupId) || Promise.resolve(); // Add another download to the group. @@ -148,7 +156,7 @@ shaka.offline.DownloadManager = class { } // Update all our internal stats. - this.estimator_.close(id, response.byteLength); + this.estimator_.close(estimateId, response.byteLength); this.onProgress_( this.estimator_.getEstimatedProgress(), this.estimator_.getTotalDownloaded()); diff --git a/lib/offline/indexeddb/base_storage_cell.js b/lib/offline/indexeddb/base_storage_cell.js index cfa3a15cd0..e97801af3b 100644 --- a/lib/offline/indexeddb/base_storage_cell.js +++ b/lib/offline/indexeddb/base_storage_cell.js @@ -68,6 +68,28 @@ shaka.offline.indexeddb.BaseStorageCell = class { return this.rejectAdd(this.manifestStore_); } + /** @override */ + updateManifest(key, manifest) { + // By default, reject all updates. + return this.rejectUpdate(this.manifestStore_); + } + + /** + * @param {number} key + * @param {!shaka.extern.ManifestDB} manifest + * @return {!Promise} + * @protected + */ + updateManifestImplementation(key, manifest) { + const op = this.connection_.startReadWriteOperation(this.manifestStore_); + const store = op.store(); + store.get(key).onsuccess = (e) => { + store.put(manifest, key); + }; + + return op.promise(); + } + /** @override */ updateManifestExpiration(key, newExpiration) { const op = this.connection_.startReadWriteOperation(this.manifestStore_); @@ -145,6 +167,19 @@ shaka.offline.indexeddb.BaseStorageCell = class { 'Cannot add new value to ' + storeName)); } + /** + * @param {string} storeName + * @return {!Promise} + * @protected + */ + rejectUpdate(storeName) { + return Promise.reject(new shaka.util.Error( + shaka.util.Error.Severity.CRITICAL, + shaka.util.Error.Category.STORAGE, + shaka.util.Error.Code.MODIFY_OPERATION_NOT_SUPPORTED, + 'Cannot modify values in ' + storeName)); + } + /** * @param {string} storeName * @param {!Array.} values diff --git a/lib/offline/indexeddb/v5_storage_cell.js b/lib/offline/indexeddb/v5_storage_cell.js index 277ad15709..b151b614ff 100644 --- a/lib/offline/indexeddb/v5_storage_cell.js +++ b/lib/offline/indexeddb/v5_storage_cell.js @@ -33,6 +33,11 @@ shaka.offline.indexeddb.V5StorageCell = class return this.add(this.manifestStore_, manifests); } + /** @override */ + updateManifest(key, manifest) { + return this.updateManifestImplementation(key, manifest); + } + /** @override */ convertManifest(old) { // JSON serialization turns Infinity into null, so turn it back now. diff --git a/lib/offline/storage.js b/lib/offline/storage.js index 1c8abb860d..a4eedbd443 100644 --- a/lib/offline/storage.js +++ b/lib/offline/storage.js @@ -13,6 +13,7 @@ goog.require('shaka.log'); goog.require('shaka.media.DrmEngine'); goog.require('shaka.media.ManifestParser'); goog.require('shaka.net.NetworkingEngine'); +goog.require('shaka.offline.DownloadInfo'); goog.require('shaka.offline.DownloadManager'); goog.require('shaka.offline.OfflineUri'); goog.require('shaka.offline.SessionDeleter'); @@ -28,11 +29,9 @@ goog.require('shaka.util.Functional'); goog.require('shaka.util.IDestroyable'); goog.require('shaka.util.Iterables'); goog.require('shaka.util.MimeUtils'); -goog.require('shaka.util.Networking'); goog.require('shaka.util.Platform'); goog.require('shaka.util.PlayerConfiguration'); goog.require('shaka.util.StreamUtils'); -goog.requireType('shaka.media.InitSegmentReference'); goog.requireType('shaka.media.SegmentReference'); goog.requireType('shaka.offline.StorageCellHandle'); @@ -285,7 +284,7 @@ shaka.offline.Storage = class { store(uri, appMetadata, mimeType) { goog.asserts.assert( this.networkingEngine_, - 'Cannot call |downloadManifest_| after calling |destroy|.'); + 'Cannot call |store| after calling |destroy|.'); // Get a copy of the current config. const config = this.getConfiguration(); @@ -371,6 +370,8 @@ shaka.offline.Storage = class { const muxer = new shaka.offline.StorageMuxer(); /** @type {?shaka.offline.StorageCellHandle} */ let activeHandle = null; + /** @type {?number} */ + let manifestId = null; // This will be used to store any errors from drm engine. Whenever drm // engine is passed to another function to do work, we should check if this @@ -398,6 +399,7 @@ shaka.offline.Storage = class { uri); } + // Create the DRM engine, and load the keys in the manifest. drmEngine = await this.createDrmEngine( manifest, (e) => { drmError = drmError || e; }, @@ -422,20 +424,32 @@ shaka.offline.Storage = class { this.ensureNotDestroyed_(); goog.asserts.assert(drmEngine, 'drmEngine should be non-null here.'); + const {manifestDB, toDownload} = this.makeManifestDB_( + drmEngine, manifest, uri, appMetadata, config, downloader); + + // Store the empty manifest, before downloading the segments. + const ids = await activeHandle.cell.addManifests([manifestDB]); + this.ensureNotDestroyed_(); + manifestId = ids[0]; - const manifestDB = await this.downloadManifest_( - activeHandle.cell, drmEngine, manifest, uri, appMetadata, config, - downloader); + goog.asserts.assert(drmEngine, 'drmEngine should be non-null here.'); this.ensureNotDestroyed_(); if (drmError) { throw drmError; } - const ids = await activeHandle.cell.addManifests([manifestDB]); + await this.downloadSegments_(toDownload, manifestId, manifestDB, + downloader, config, activeHandle.cell); + this.ensureNotDestroyed_(); + + // Now that we have the keys loaded into the DrmEngine, we can attach + // those fields to the manifest. + this.setManifestDrmFields_(manifest, manifestDB, drmEngine, config); + await activeHandle.cell.updateManifest(manifestId, manifestDB); this.ensureNotDestroyed_(); const offlineUri = shaka.offline.OfflineUri.manifest( - activeHandle.path.mechanism, activeHandle.path.cell, ids[0]); + activeHandle.path.mechanism, activeHandle.path.cell, manifestId); return shaka.offline.StoredContentUtils.fromManifestDB( offlineUri, manifestDB); @@ -448,6 +462,14 @@ shaka.offline.Storage = class { this.segmentsFromStore_, () => {}); } + if (manifestId != null) { + const uri = shaka.offline.OfflineUri.manifest( + activeHandle.path.mechanism, + activeHandle.path.cell, + manifestId); + await this.remove_(uri.toString()); + } + // If we already had an error, ignore this error to avoid hiding // the original error. throw drmError || e; @@ -466,6 +488,85 @@ shaka.offline.Storage = class { } } + /** + * Download and then store the contents of each segment. + * The promise this returns will wait for local downloads. + * + * @param {!Array.} toDownload + * @param {number} manifestId + * @param {shaka.extern.ManifestDB} manifestDB + * @param {!shaka.offline.DownloadManager} downloader + * @param {shaka.extern.PlayerConfiguration} config + * @param {shaka.extern.StorageCell} storage + * @return {!Promise} + * @private + */ + async downloadSegments_( + toDownload, manifestId, manifestDB, downloader, config, storage) { + for (const download of toDownload) { + /** @param {?BufferSource} data */ + let data; + const request = download.makeSegmentRequest(config); + const estimateId = download.estimateId; + const isInitSegment = download.isInitSegment; + const onDownloaded = (d) => { + data = d; + return Promise.resolve(); + }; + downloader.queue(download.groupId, + request, estimateId, isInitSegment, onDownloaded); + downloader.queueWork(download.groupId, async () => { + goog.asserts.assert(data, 'We should have loaded data by now'); + + const ref = /** @type {!shaka.media.SegmentReference} */ ( + download.ref); + const idForRef = shaka.offline.DownloadInfo.idForSegmentRef(ref); + + // Store the segment. + const ids = await storage.addSegments([{data: data}]); + this.ensureNotDestroyed_(); + this.segmentsFromStore_.push(ids[0]); + + // Attach the segment to the manifest. + let complete = true; + for (const stream of manifestDB.streams) { + for (const segment of stream.segments) { + if (segment.pendingSegmentRefId == idForRef) { + segment.dataKey = ids[0]; + // Now that the segment has been associated with the + // appropriate dataKey, the pendingSegmentRefId is no longer + // necessary. + segment.pendingSegmentRefId = undefined; + } + if (segment.pendingInitSegmentRefId == idForRef) { + segment.initSegmentKey = ids[0]; + // Now that the init segment has been associated with the + // appropriate initSegmentKey, the pendingInitSegmentRefId is + // no longer necessary. + segment.pendingInitSegmentRefId = undefined; + } + if (segment.pendingSegmentRefId) { + complete = false; + } + if (segment.pendingInitSegmentRefId) { + complete = false; + } + } + } + if (complete) { + manifestDB.isIncomplete = false; + } + }); + } + + // Re-store the manifest. + manifestDB.size = await downloader.waitToFinish(); + goog.asserts.assert( + !manifestDB.isIncomplete, 'The manifest should be complete by now'); + await storage.updateManifest(manifestId, manifestDB); + this.ensureNotDestroyed_(); + } + /** * Filter |manifest| such that it will only contain the variants and text * streams that we want to store and can actually play. @@ -584,19 +685,21 @@ shaka.offline.Storage = class { /** * Create a download manager and download the manifest. + * This also sets up download infos for each segment to be downloaded. * - * @param {shaka.extern.StorageCell} storage * @param {!shaka.media.DrmEngine} drmEngine * @param {shaka.extern.Manifest} manifest * @param {string} uri * @param {!Object} metadata * @param {shaka.extern.PlayerConfiguration} config * @param {!shaka.offline.DownloadManager} downloader - * @return {!Promise.} + * @return {{ + * manifestDB: shaka.extern.ManifestDB, + * toDownload: !Array. + * }} * @private */ - async downloadManifest_( - storage, drmEngine, manifest, uri, metadata, config, downloader) { + makeManifestDB_(drmEngine, manifest, uri, metadata, config, downloader) { const pendingContent = shaka.offline.StoredContentUtils.fromManifest( uri, manifest, /* size= */ 0, metadata); // In https://github.com/google/shaka-player/issues/2652, we found that this @@ -619,12 +722,73 @@ shaka.offline.Storage = class { }; downloader.setCallbacks(onProgress, onInitData); - const isEncrypted = manifest.variants.some((variant) => { + const needsInitData = this.getManifestIsEncrypted_(manifest) && + this.getManifestIncludesInitData_(manifest); + + let currentSystemId = null; + if (needsInitData) { + const drmInfo = drmEngine.getDrmInfo(); + currentSystemId = + shaka.offline.Storage.defaultSystemIds_.get(drmInfo.keySystem); + } + + // Make the estimator, which is used to make the download registries. + const estimator = new shaka.offline.StreamBandwidthEstimator(); + for (const stream of manifest.textStreams) { + estimator.addText(stream); + } + for (const stream of manifest.imageStreams) { + estimator.addImage(stream); + } + for (const variant of manifest.variants) { + estimator.addVariant(variant); + } + const {streams, toDownload} = this.createStreams_( + downloader, estimator, drmEngine, manifest, config); + + const drmInfo = drmEngine.getDrmInfo(); + const usePersistentLicense = config.offline.usePersistentLicense; + if (drmInfo && usePersistentLicense) { + // Don't store init data, since we have stored sessions. + drmInfo.initData = []; + } + + const manifestDB = { + creationTime: Date.now(), + originalManifestUri: uri, + duration: manifest.presentationTimeline.getDuration(), + size: 0, + expiration: drmEngine.getExpiration(), + streams, + sessionIds: usePersistentLicense ? drmEngine.getSessionIds() : [], + drmInfo, + appMetadata: metadata, + isIncomplete: true, + }; + + return {manifestDB, toDownload}; + } + + /** + * @param {shaka.extern.Manifest} manifest + * @return {boolean} + * @private + */ + getManifestIsEncrypted_(manifest) { + return manifest.variants.some((variant) => { const videoEncrypted = variant.video && variant.video.encrypted; const audioEncrypted = variant.audio && variant.audio.encrypted; return videoEncrypted || audioEncrypted; }); - const includesInitData = manifest.variants.some((variant) => { + } + + /** + * @param {shaka.extern.Manifest} manifest + * @return {boolean} + * @private + */ + getManifestIncludesInitData_(manifest) { + return manifest.variants.some((variant) => { const videoDrmInfos = variant.video ? variant.video.drmInfos : []; const audioDrmInfos = variant.audio ? variant.audio.drmInfos : []; const drmInfos = videoDrmInfos.concat(audioDrmInfos); @@ -632,36 +796,28 @@ shaka.offline.Storage = class { return drmInfos.initData && drmInfos.initData.length; }); }); - const needsInitData = isEncrypted && !includesInitData; - - let currentSystemId = null; - if (needsInitData) { - const drmInfo = drmEngine.getDrmInfo(); - currentSystemId = - shaka.offline.Storage.defaultSystemIds_.get(drmInfo.keySystem); - } - - try { - const manifestDB = this.createOfflineManifest_( - downloader, storage, drmEngine, manifest, uri, metadata, config); + } - manifestDB.size = await downloader.waitToFinish(); - manifestDB.expiration = drmEngine.getExpiration(); - const sessions = drmEngine.getSessionIds(); - manifestDB.sessionIds = config.offline.usePersistentLicense ? - sessions : []; + /** + * @param {shaka.extern.Manifest} manifest + * @param {shaka.extern.ManifestDB} manifestDB + * @param {!shaka.media.DrmEngine} drmEngine + * @param {shaka.extern.PlayerConfiguration} config + * @private + */ + setManifestDrmFields_(manifest, manifestDB, drmEngine, config) { + manifestDB.expiration = drmEngine.getExpiration(); - if (isEncrypted && config.offline.usePersistentLicense && - !sessions.length) { - throw new shaka.util.Error( - shaka.util.Error.Severity.CRITICAL, - shaka.util.Error.Category.STORAGE, - shaka.util.Error.Code.NO_INIT_DATA_FOR_OFFLINE); - } + const sessions = drmEngine.getSessionIds(); + manifestDB.sessionIds = config.offline.usePersistentLicense ? + sessions : []; - return manifestDB; - } finally { - await downloader.destroy(); + if (this.getManifestIsEncrypted_(manifest) && + config.offline.usePersistentLicense && !sessions.length) { + throw new shaka.util.Error( + shaka.util.Error.Severity.CRITICAL, + shaka.util.Error.Category.STORAGE, + shaka.util.Error.Code.NO_INIT_DATA_FOR_OFFLINE); } } @@ -764,7 +920,7 @@ shaka.offline.Storage = class { * @return {!Promise} * @private */ - removeFromStorage_( storage, uri, manifest) { + removeFromStorage_(storage, uri, manifest) { /** @type {!Array.} */ const segmentIds = shaka.offline.Storage.getAllSegmentIds_(manifest); @@ -1007,100 +1163,23 @@ shaka.offline.Storage = class { } /** - * Creates an offline 'manifest' for the real manifest. This does not store - * the segments yet, only adds them to the download manager through - * createStreams_. - * - * @param {!shaka.offline.DownloadManager} downloader - * @param {shaka.extern.StorageCell} storage - * @param {!shaka.media.DrmEngine} drmEngine - * @param {shaka.extern.Manifest} manifest - * @param {string} originalManifestUri - * @param {!Object} metadata - * @param {shaka.extern.PlayerConfiguration} config - * @return {shaka.extern.ManifestDB} - * @private - */ - createOfflineManifest_( - downloader, storage, drmEngine, manifest, originalManifestUri, metadata, - config) { - const estimator = new shaka.offline.StreamBandwidthEstimator(); - - const streams = this.createStreams_( - downloader, storage, estimator, drmEngine, manifest, config); - - const usePersistentLicense = config.offline.usePersistentLicense; - const drmInfo = drmEngine.getDrmInfo(); - - if (drmInfo && usePersistentLicense) { - // Don't store init data, since we have stored sessions. - drmInfo.initData = []; - } - - return { - creationTime: Date.now(), - originalManifestUri: originalManifestUri, - duration: manifest.presentationTimeline.getDuration(), - size: 0, - expiration: drmEngine.getExpiration(), - streams: streams, - sessionIds: usePersistentLicense ? drmEngine.getSessionIds() : [], - drmInfo: drmInfo, - appMetadata: metadata, - }; - } - - /** - * Converts manifest Streams to database Streams. This will use the current - * configuration to get the tracks to use, then it will search each segment - * index and add all the segments to the download manager through - * createStream_. + * Converts manifest Streams to database Streams. * * @param {!shaka.offline.DownloadManager} downloader - * @param {shaka.extern.StorageCell} storage * @param {shaka.offline.StreamBandwidthEstimator} estimator * @param {!shaka.media.DrmEngine} drmEngine * @param {shaka.extern.Manifest} manifest * @param {shaka.extern.PlayerConfiguration} config - * @return {!Array.} + * @return {{ + * streams: !Array., + * toDownload: !Array. + * }} * @private */ - createStreams_(downloader, storage, estimator, drmEngine, manifest, config) { - // Pass all variants and text streams to the estimator so that we can - // get the best estimate for each stream later. - for (const variant of manifest.variants) { - estimator.addVariant(variant); - } - for (const text of manifest.textStreams) { - estimator.addText(text); - } - for (const image of manifest.imageStreams) { - estimator.addImage(image); - } - - // TODO(joeyparrish): Break out stack-based state and method params into a - // separate class to clean up. See: - // https://github.com/google/shaka-player/issues/2781#issuecomment-678438039 - - /** - * A cache mapping init segment references to Promises to their DB key. - * - * @type {!Map.>} - */ - const initSegmentDbKeyCache = new Map(); - - // A null init segment reference always maps to a null DB key. - initSegmentDbKeyCache.set( - null, /** @type {!Promise.} */(Promise.resolve(null))); - - /** - * A cache mapping equivalent segment references to Promises to their DB - * key. The key in this map is a string of the form - * "--". - * - * @type {!Map.>} - */ - const segmentDbKeyCache = new Map(); + createStreams_(downloader, estimator, drmEngine, manifest, config) { + // Download infos are stored based on their refId, to dedup them. + /** @type {!Map.} */ + const toDownload = new Map(); // Find the streams we want to download and create a stream db instance // for each of them. @@ -1110,8 +1189,7 @@ shaka.offline.Storage = class { for (const stream of streamSet) { const streamDB = this.createStream_( - downloader, storage, estimator, manifest, stream, config, - initSegmentDbKeyCache, segmentDbKeyCache); + downloader, estimator, manifest, stream, config, toDownload); streamDBs.set(stream.id, streamDB); } @@ -1125,27 +1203,26 @@ shaka.offline.Storage = class { } } - return Array.from(streamDBs.values()); + return { + streams: Array.from(streamDBs.values()), + toDownload: Array.from(toDownload.values()), + }; } /** * Converts a manifest stream to a database stream. This will search the - * segment index and add all the segments to the download manager. + * segment index and add all the segments to the download infos. * * @param {!shaka.offline.DownloadManager} downloader - * @param {shaka.extern.StorageCell} storage * @param {shaka.offline.StreamBandwidthEstimator} estimator * @param {shaka.extern.Manifest} manifest * @param {shaka.extern.Stream} stream * @param {shaka.extern.PlayerConfiguration} config - * @param {!Map.>} - * initSegmentDbKeyCache - * @param {!Map.>} segmentDbKeyCache + * @param {!Map.} toDownload * @return {shaka.extern.StreamDB} * @private */ - createStream_(downloader, storage, estimator, manifest, stream, config, - initSegmentDbKeyCache, segmentDbKeyCache) { + createStream_(downloader, estimator, manifest, stream, config, toDownload) { /** @type {shaka.extern.StreamDB} */ const streamDb = { id: stream.id, @@ -1175,138 +1252,60 @@ shaka.offline.Storage = class { tilesLayout: stream.tilesLayout, }; - // Download each stream in parallel. - const downloadGroup = stream.id; - const startTime = manifest.presentationTimeline.getSegmentAvailabilityStart(); shaka.offline.Storage.forEachSegment_(stream, startTime, (segment) => { - const initSegmentKeyPromise = this.getInitSegmentDbKey_( - downloader, downloadGroup, stream.id, storage, estimator, - segment.initSegmentReference, config, initSegmentDbKeyCache); - - const segmentKeyPromise = this.getSegmentDbKey_( - downloader, downloadGroup, stream.id, storage, estimator, segment, - config, segmentDbKeyCache); - - downloader.queueWork(downloadGroup, async () => { - const initSegmentKey = await initSegmentKeyPromise; - const dataKey = await segmentKeyPromise; - - streamDb.segments.push({ - initSegmentKey, - startTime: segment.startTime, - endTime: segment.endTime, - appendWindowStart: segment.appendWindowStart, - appendWindowEnd: segment.appendWindowEnd, - timestampOffset: segment.timestampOffset, - tilesLayout: segment.tilesLayout, - dataKey, - }); - }); - }); - - return streamDb; - } - - /** - * Get a Promise to the DB key for a given init segment reference. - * - * The return values will be cached so that multiple calls with the same init - * segment reference will only trigger one request. - * - * @param {!shaka.offline.DownloadManager} downloader - * @param {number} downloadGroup - * @param {number} streamId - * @param {shaka.extern.StorageCell} storage - * @param {shaka.offline.StreamBandwidthEstimator} estimator - * @param {shaka.media.InitSegmentReference} initSegmentReference - * @param {shaka.extern.PlayerConfiguration} config - * @param {!Map.>} - * initSegmentDbKeyCache - * @return {!Promise.} - * @private - */ - getInitSegmentDbKey_( - downloader, downloadGroup, streamId, storage, estimator, - initSegmentReference, config, initSegmentDbKeyCache) { - if (initSegmentDbKeyCache.has(initSegmentReference)) { - return initSegmentDbKeyCache.get(initSegmentReference); - } - - const request = shaka.util.Networking.createSegmentRequest( - initSegmentReference.getUris(), - initSegmentReference.startByte, - initSegmentReference.endByte, - config.streaming.retryParameters); - - const promise = downloader.queue( - downloadGroup, - request, - estimator.getInitSegmentEstimate(streamId), - /* isInitSegment= */ true, - async (data) => { - /** @type {!Array.} */ - const ids = await storage.addSegments([{data: data}]); - this.segmentsFromStore_.push(ids[0]); - return ids[0]; - }); - - initSegmentDbKeyCache.set(initSegmentReference, promise); - return promise; - } + const pendingSegmentRefId = + shaka.offline.DownloadInfo.idForSegmentRef(segment); + let pendingInitSegmentRefId = undefined; + + // Set up the download for the segment, which will be downloaded later, + // perhaps in a service worker. + if (!toDownload.has(pendingSegmentRefId)) { + const estimateId = downloader.addDownloadEstimate( + estimator.getSegmentEstimate(stream.id, segment)); + const segmentDownload = new shaka.offline.DownloadInfo( + segment, + estimateId, + stream.id, + /* isInitSegment= */ false); + toDownload.set(pendingSegmentRefId, segmentDownload); + } - /** - * Get a Promise to the DB key for a given segment reference. - * - * The return values will be cached so that multiple calls with the same - * segment reference will only trigger one request. - * - * @param {!shaka.offline.DownloadManager} downloader - * @param {number} downloadGroup - * @param {number} streamId - * @param {shaka.extern.StorageCell} storage - * @param {shaka.offline.StreamBandwidthEstimator} estimator - * @param {shaka.media.SegmentReference} segmentReference - * @param {shaka.extern.PlayerConfiguration} config - * @param {!Map.>} segmentDbKeyCache - * @return {!Promise.} - * @private - */ - getSegmentDbKey_( - downloader, downloadGroup, streamId, storage, estimator, - segmentReference, config, segmentDbKeyCache) { - const mapKey = [ - segmentReference.getUris()[0], - segmentReference.startByte, - segmentReference.endByte, - ].join('-'); - - if (segmentDbKeyCache.has(mapKey)) { - return segmentDbKeyCache.get(mapKey); - } + // Set up the download for the init segment, similarly, if there is one. + if (segment.initSegmentReference) { + pendingInitSegmentRefId = shaka.offline.DownloadInfo.idForSegmentRef( + segment.initSegmentReference); + if (!toDownload.has(pendingInitSegmentRefId)) { + const estimateId = downloader.addDownloadEstimate( + estimator.getInitSegmentEstimate(stream.id)); + const initDownload = new shaka.offline.DownloadInfo( + segment.initSegmentReference, + estimateId, + stream.id, + /* isInitSegment= */ true); + toDownload.set(pendingInitSegmentRefId, initDownload); + } + } - const request = shaka.util.Networking.createSegmentRequest( - segmentReference.getUris(), - segmentReference.startByte, - segmentReference.endByte, - config.streaming.retryParameters); - - const promise = downloader.queue( - downloadGroup, - request, - estimator.getSegmentEstimate(streamId, segmentReference), - /* isInitSegment= */ false, - async (data) => { - /** @type {!Array.} */ - const ids = await storage.addSegments([{data: data}]); - this.segmentsFromStore_.push(ids[0]); - return ids[0]; - }); + /** @type {!shaka.extern.SegmentDB} */ + const segmentDB = { + pendingInitSegmentRefId, + initSegmentKey: pendingInitSegmentRefId ? 0 : null, + startTime: segment.startTime, + endTime: segment.endTime, + appendWindowStart: segment.appendWindowStart, + appendWindowEnd: segment.appendWindowEnd, + timestampOffset: segment.timestampOffset, + tilesLayout: segment.tilesLayout, + pendingSegmentRefId, + dataKey: 0, + }; + streamDb.segments.push(segmentDB); + }); - segmentDbKeyCache.set(mapKey, promise); - return promise; + return streamDb; } /** @@ -1408,21 +1407,21 @@ shaka.offline.Storage = class { * @private */ static getAllSegmentIds_(manifest) { - /** @type {!Array.} */ - const ids = []; + /** @type {!Set.} */ + const ids = new Set(); // Get every segment for every stream in the manifest. for (const stream of manifest.streams) { for (const segment of stream.segments) { if (segment.initSegmentKey != null) { - ids.push(segment.initSegmentKey); + ids.add(segment.initSegmentKey); } - ids.push(segment.dataKey); + ids.add(segment.dataKey); } } - return ids; + return Array.from(ids); } /** diff --git a/lib/offline/stored_content_utils.js b/lib/offline/stored_content_utils.js index 8c4cc22a2f..a85ed5dfae 100644 --- a/lib/offline/stored_content_utils.js +++ b/lib/offline/stored_content_utils.js @@ -47,6 +47,7 @@ shaka.offline.StoredContentUtils = class { expiration: Infinity, tracks: tracks, appMetadata: metadata, + isIncomplete: false, }; return content; @@ -87,6 +88,7 @@ shaka.offline.StoredContentUtils = class { expiration: manifestDB.expiration, tracks: tracks, appMetadata: metadata, + isIncomplete: (manifestDB.isIncomplete || false), }; return content; diff --git a/lib/util/error.js b/lib/util/error.js index d5a5d6d0bd..aaea12326e 100644 --- a/lib/util/error.js +++ b/lib/util/error.js @@ -986,6 +986,12 @@ shaka.util.Error.Code = { */ 'DOWNLOAD_SIZE_CALLBACK_ERROR': 9015, + /** + * The storage cell does not allow new operations that significantly change + * existing data. + */ + 'MODIFY_OPERATION_NOT_SUPPORTED': 9016, + /** * CS IMA SDK, required for ad insertion, has not been included on the page. */ diff --git a/test/offline/storage_integration.js b/test/offline/storage_integration.js index 8e6cffd521..d0a0ec893e 100644 --- a/test/offline/storage_integration.js +++ b/test/offline/storage_integration.js @@ -1234,11 +1234,11 @@ filterDescribe('Storage', storageSupport, () => { goog.asserts.assert( content.offlineUri != null, 'URI should not be null!'); - /** - * @type {!Array.} - */ + // We expect 5 progress events because there are 4 unique segments, plus + // the manifest. + /** @type {!Array.}*/ const progressSteps = [ - 0.111, 0.222, 0.333, 0.444, 0.555, 0.666, 0.777, 0.888, 1.0, + 0.2, 0.4, 0.6, 0.8, 1, ]; const progressCallback = (content, progress) => {