diff --git a/lib/cast/cast_utils.js b/lib/cast/cast_utils.js index 615f78111a..0ebcf89f1d 100644 --- a/lib/cast/cast_utils.js +++ b/lib/cast/cast_utils.js @@ -374,6 +374,7 @@ shaka.cast.CastUtils.PlayerInitAfterLoadState = [ shaka.cast.CastUtils.PlayerVoidMethods = [ 'addChaptersTrack', 'addTextTrackAsync', + 'addThumbnailsTrack', 'cancelTrickPlay', 'configure', 'getChapters', diff --git a/lib/player.js b/lib/player.js index 98282426a3..c9a8eeebfc 100644 --- a/lib/player.js +++ b/lib/player.js @@ -27,6 +27,7 @@ goog.require('shaka.media.QualityObserver'); goog.require('shaka.media.RegionObserver'); goog.require('shaka.media.RegionTimeline'); goog.require('shaka.media.SegmentIndex'); +goog.require('shaka.media.SegmentReference'); goog.require('shaka.media.SrcEqualsPlayhead'); goog.require('shaka.media.StreamingEngine'); goog.require('shaka.media.TimeRangesUtils'); @@ -4627,6 +4628,138 @@ shaka.Player = class extends shaka.util.FakeEventTarget { return shaka.util.StreamUtils.textStreamToTrack(stream); } + /** + * Adds the given thumbnails track to the loaded manifest. + * load() must resolve before calling. The presentation must + * have a duration. + * + * This returns the created track, which can immediately be used by the + * application. + * + * @param {string} uri + * @param {string=} mimeType + * @return {!Promise.} + * @export + */ + async addThumbnailsTrack(uri, mimeType) { + if (this.loadMode_ != shaka.Player.LoadMode.MEDIA_SOURCE && + this.loadMode_ != shaka.Player.LoadMode.SRC_EQUALS) { + shaka.log.error( + 'Must call load() and wait for it to resolve before adding image ' + + 'tracks.'); + throw new shaka.util.Error( + shaka.util.Error.Severity.RECOVERABLE, + shaka.util.Error.Category.PLAYER, + shaka.util.Error.Code.CONTENT_NOT_LOADED); + } + + if (this.loadMode_ == shaka.Player.LoadMode.SRC_EQUALS) { + shaka.log.error('Cannot add this thumbnail track when loaded with src='); + throw new shaka.util.Error( + shaka.util.Error.Severity.RECOVERABLE, + shaka.util.Error.Category.TEXT, + shaka.util.Error.Code.CANNOT_ADD_EXTERNAL_THUMBNAILS_TO_SRC_EQUALS); + } + + if (!mimeType) { + mimeType = await this.getTextMimetype_(uri); + } + + if (mimeType != 'text/vtt') { + throw new shaka.util.Error( + shaka.util.Error.Severity.RECOVERABLE, + shaka.util.Error.Category.TEXT, + shaka.util.Error.Code.UNSUPPORTED_EXTERNAL_THUMBNAILS_URI, + uri); + } + + const ContentType = shaka.util.ManifestParserUtils.ContentType; + + const duration = this.manifest_.presentationTimeline.getDuration(); + if (duration == Infinity) { + throw new shaka.util.Error( + shaka.util.Error.Severity.RECOVERABLE, + shaka.util.Error.Category.MANIFEST, + shaka.util.Error.Code.CANNOT_ADD_EXTERNAL_THUMBNAILS_TO_LIVE_STREAM); + } + + goog.asserts.assert( + this.networkingEngine_, 'Need networking engine.'); + const buffer = await this.getTextData_(uri, + this.networkingEngine_, + this.config_.streaming.retryParameters); + + const factory = shaka.text.TextEngine.findParser(mimeType); + if (!factory) { + throw new shaka.util.Error( + shaka.util.Error.Severity.CRITICAL, + shaka.util.Error.Category.TEXT, + shaka.util.Error.Code.MISSING_TEXT_PLUGIN, + mimeType); + } + const TextParser = factory(); + const time = { + periodStart: 0, + segmentStart: 0, + segmentEnd: duration, + vttOffset: 0, + }; + const data = shaka.util.BufferUtils.toUint8(buffer); + const cues = TextParser.parseMedia(data, time); + + const references = []; + for (const cue of cues) { + const imageUri = shaka.util.ManifestParserUtils.resolveUris( + [uri], [cue.payload])[0]; + if (imageUri.includes('#xywh')) { + shaka.log.alwaysWarn('Unsupported image uri', imageUri); + continue; + } + references.push(new shaka.media.SegmentReference( + cue.startTime, + cue.endTime, + () => [imageUri], + /* startByte= */ 0, + /* endByte= */ null, + /* initSegmentReference= */ null, + /* timestampOffset= */ 0, + /* appendWindowStart= */ 0, + /* appendWindowEnd= */ Infinity, + )); + } + + /** @type {shaka.extern.Stream} */ + const stream = { + id: this.nextExternalStreamId_++, + originalId: null, + createSegmentIndex: () => Promise.resolve(), + segmentIndex: new shaka.media.SegmentIndex(references), + mimeType: mimeType || '', + codecs: '', + kind: '', + encrypted: false, + drmInfos: [], + keyIds: new Set(), + language: 'und', + label: null, + type: ContentType.IMAGE, + primary: false, + trickModeVideo: null, + emsgSchemeIdUris: null, + roles: [], + forced: false, + channelsCount: null, + audioSamplingRate: null, + spatialAudio: false, + closedCaptions: null, + tilesLayout: '1x1', + }; + + this.manifest_.imageStreams.push(stream); + this.onTracksChanged_(); + return shaka.util.StreamUtils.imageStreamToTrack(stream); + } + /** * Adds the given chapters track to the loaded manifest. load() * must resolve before calling. The presentation must have a duration. diff --git a/lib/util/error.js b/lib/util/error.js index 45f7f4fd18..9cf5305cc7 100644 --- a/lib/util/error.js +++ b/lib/util/error.js @@ -358,6 +358,18 @@ shaka.util.Error.Code = { */ 'CHAPTERS_TRACK_FAILED': 2015, + /** + * External thumbnails tracks cannot be added in src= because native platform + * doesn't support it. + */ + 'CANNOT_ADD_EXTERNAL_THUMBNAILS_TO_SRC_EQUALS': 2016, + + /** + * Only external urls of WebVTT type are supported. + *
error.data[0] is the uri. + */ + 'UNSUPPORTED_EXTERNAL_THUMBNAILS_URI': 2017, + /** * Some component tried to read past the end of a buffer. The segment index, * init segment, or PSSH may be malformed. @@ -707,6 +719,11 @@ shaka.util.Error.Code = { */ 'HLS_AES_128_INVALID_KEY_LENGTH': 4044, + /** + * External thumbnails tracks cannot be added to live streams. + */ + 'CANNOT_ADD_EXTERNAL_THUMBNAILS_TO_LIVE_STREAM': 4045, + // RETIRED: 'INCONSISTENT_BUFFER_STATE': 5000, // RETIRED: 'INVALID_SEGMENT_INDEX': 5001, diff --git a/test/player_integration.js b/test/player_integration.js index 2f1479601c..7109fff5db 100644 --- a/test/player_integration.js +++ b/test/player_integration.js @@ -262,6 +262,49 @@ describe('Player', () => { expect(cues.length).toBeGreaterThan(0); }); + it('skip appended thumbnails for external thumbnails with sprites', + async () => { + await player.load('test:sintel_no_text_compiled'); + const locationUri = new goog.Uri(location.href); + const partialUri = + new goog.Uri('/base/test/test/assets/thumbnails-sprites.vtt'); + const absoluteUri = locationUri.resolve(partialUri); + const newTrack = + await player.addThumbnailsTrack(absoluteUri.toString()); + + expect(player.getImageTracks()).toEqual([newTrack]); + + const thumbnail1 = await player.getThumbnails(newTrack.id, 0); + expect(thumbnail1).toBe(null); + const thumbnail2 = await player.getThumbnails(newTrack.id, 10); + expect(thumbnail2).toBe(null); + const thumbnail3 = await player.getThumbnails(newTrack.id, 40); + expect(thumbnail3).toBe(null); + }); + + it('appends thumbnails for external thumbnails without sprites', + async () => { + await player.load('test:sintel_no_text_compiled'); + const locationUri = new goog.Uri(location.href); + const partialUri = + new goog.Uri('/base/test/test/assets/thumbnails.vtt'); + const absoluteUri = locationUri.resolve(partialUri); + const newTrack = + await player.addThumbnailsTrack(absoluteUri.toString()); + + expect(player.getImageTracks()).toEqual([newTrack]); + + const thumbnail1 = await player.getThumbnails(newTrack.id, 0); + expect(thumbnail1.startTime).toBe(0); + expect(thumbnail1.duration).toBe(5); + const thumbnail2 = await player.getThumbnails(newTrack.id, 10); + expect(thumbnail2.startTime).toBe(5); + expect(thumbnail2.duration).toBe(25); + const thumbnail3 = await player.getThumbnails(newTrack.id, 40); + expect(thumbnail3.startTime).toBe(30); + expect(thumbnail3.duration).toBe(30); + }); + // https://github.com/shaka-project/shaka-player/issues/2553 it('does not change the selected track', async () => { player.configure('streaming.alwaysStreamText', false); diff --git a/test/test/assets/thumbnails-sprites.vtt b/test/test/assets/thumbnails-sprites.vtt new file mode 100644 index 0000000000..42d2cf0fb5 --- /dev/null +++ b/test/test/assets/thumbnails-sprites.vtt @@ -0,0 +1,10 @@ +WEBVTT + +00:00.000 --> 00:05.000 +image1.jpg#xywh=0,0,160,90 + +00:05.000 --> 00:30.000 +image2.jpg#xywh=0,0,160,90 + +00:30.000 --> 01:00.000 +image3.jpg#xywh=0,0,160,90 \ No newline at end of file diff --git a/test/test/assets/thumbnails.vtt b/test/test/assets/thumbnails.vtt new file mode 100644 index 0000000000..b55896bf7c --- /dev/null +++ b/test/test/assets/thumbnails.vtt @@ -0,0 +1,10 @@ +WEBVTT + +00:00.000 --> 00:05.000 +image1.jpg + +00:05.000 --> 00:30.000 +image2.jpg + +00:30.000 --> 01:00.000 +image3.jpg \ No newline at end of file