diff --git a/lib/cast/cast_utils.js b/lib/cast/cast_utils.js
index 615f78111a..0ebcf89f1d 100644
--- a/lib/cast/cast_utils.js
+++ b/lib/cast/cast_utils.js
@@ -374,6 +374,7 @@ shaka.cast.CastUtils.PlayerInitAfterLoadState = [
shaka.cast.CastUtils.PlayerVoidMethods = [
'addChaptersTrack',
'addTextTrackAsync',
+ 'addThumbnailsTrack',
'cancelTrickPlay',
'configure',
'getChapters',
diff --git a/lib/player.js b/lib/player.js
index 98282426a3..c9a8eeebfc 100644
--- a/lib/player.js
+++ b/lib/player.js
@@ -27,6 +27,7 @@ goog.require('shaka.media.QualityObserver');
goog.require('shaka.media.RegionObserver');
goog.require('shaka.media.RegionTimeline');
goog.require('shaka.media.SegmentIndex');
+goog.require('shaka.media.SegmentReference');
goog.require('shaka.media.SrcEqualsPlayhead');
goog.require('shaka.media.StreamingEngine');
goog.require('shaka.media.TimeRangesUtils');
@@ -4627,6 +4628,138 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
return shaka.util.StreamUtils.textStreamToTrack(stream);
}
+ /**
+ * Adds the given thumbnails track to the loaded manifest.
+ * load()
must resolve before calling. The presentation must
+ * have a duration.
+ *
+ * This returns the created track, which can immediately be used by the
+ * application.
+ *
+ * @param {string} uri
+ * @param {string=} mimeType
+ * @return {!Promise.}
+ * @export
+ */
+ async addThumbnailsTrack(uri, mimeType) {
+ if (this.loadMode_ != shaka.Player.LoadMode.MEDIA_SOURCE &&
+ this.loadMode_ != shaka.Player.LoadMode.SRC_EQUALS) {
+ shaka.log.error(
+ 'Must call load() and wait for it to resolve before adding image ' +
+ 'tracks.');
+ throw new shaka.util.Error(
+ shaka.util.Error.Severity.RECOVERABLE,
+ shaka.util.Error.Category.PLAYER,
+ shaka.util.Error.Code.CONTENT_NOT_LOADED);
+ }
+
+ if (this.loadMode_ == shaka.Player.LoadMode.SRC_EQUALS) {
+ shaka.log.error('Cannot add this thumbnail track when loaded with src=');
+ throw new shaka.util.Error(
+ shaka.util.Error.Severity.RECOVERABLE,
+ shaka.util.Error.Category.TEXT,
+ shaka.util.Error.Code.CANNOT_ADD_EXTERNAL_THUMBNAILS_TO_SRC_EQUALS);
+ }
+
+ if (!mimeType) {
+ mimeType = await this.getTextMimetype_(uri);
+ }
+
+ if (mimeType != 'text/vtt') {
+ throw new shaka.util.Error(
+ shaka.util.Error.Severity.RECOVERABLE,
+ shaka.util.Error.Category.TEXT,
+ shaka.util.Error.Code.UNSUPPORTED_EXTERNAL_THUMBNAILS_URI,
+ uri);
+ }
+
+ const ContentType = shaka.util.ManifestParserUtils.ContentType;
+
+ const duration = this.manifest_.presentationTimeline.getDuration();
+ if (duration == Infinity) {
+ throw new shaka.util.Error(
+ shaka.util.Error.Severity.RECOVERABLE,
+ shaka.util.Error.Category.MANIFEST,
+ shaka.util.Error.Code.CANNOT_ADD_EXTERNAL_THUMBNAILS_TO_LIVE_STREAM);
+ }
+
+ goog.asserts.assert(
+ this.networkingEngine_, 'Need networking engine.');
+ const buffer = await this.getTextData_(uri,
+ this.networkingEngine_,
+ this.config_.streaming.retryParameters);
+
+ const factory = shaka.text.TextEngine.findParser(mimeType);
+ if (!factory) {
+ throw new shaka.util.Error(
+ shaka.util.Error.Severity.CRITICAL,
+ shaka.util.Error.Category.TEXT,
+ shaka.util.Error.Code.MISSING_TEXT_PLUGIN,
+ mimeType);
+ }
+ const TextParser = factory();
+ const time = {
+ periodStart: 0,
+ segmentStart: 0,
+ segmentEnd: duration,
+ vttOffset: 0,
+ };
+ const data = shaka.util.BufferUtils.toUint8(buffer);
+ const cues = TextParser.parseMedia(data, time);
+
+ const references = [];
+ for (const cue of cues) {
+ const imageUri = shaka.util.ManifestParserUtils.resolveUris(
+ [uri], [cue.payload])[0];
+ if (imageUri.includes('#xywh')) {
+ shaka.log.alwaysWarn('Unsupported image uri', imageUri);
+ continue;
+ }
+ references.push(new shaka.media.SegmentReference(
+ cue.startTime,
+ cue.endTime,
+ () => [imageUri],
+ /* startByte= */ 0,
+ /* endByte= */ null,
+ /* initSegmentReference= */ null,
+ /* timestampOffset= */ 0,
+ /* appendWindowStart= */ 0,
+ /* appendWindowEnd= */ Infinity,
+ ));
+ }
+
+ /** @type {shaka.extern.Stream} */
+ const stream = {
+ id: this.nextExternalStreamId_++,
+ originalId: null,
+ createSegmentIndex: () => Promise.resolve(),
+ segmentIndex: new shaka.media.SegmentIndex(references),
+ mimeType: mimeType || '',
+ codecs: '',
+ kind: '',
+ encrypted: false,
+ drmInfos: [],
+ keyIds: new Set(),
+ language: 'und',
+ label: null,
+ type: ContentType.IMAGE,
+ primary: false,
+ trickModeVideo: null,
+ emsgSchemeIdUris: null,
+ roles: [],
+ forced: false,
+ channelsCount: null,
+ audioSamplingRate: null,
+ spatialAudio: false,
+ closedCaptions: null,
+ tilesLayout: '1x1',
+ };
+
+ this.manifest_.imageStreams.push(stream);
+ this.onTracksChanged_();
+ return shaka.util.StreamUtils.imageStreamToTrack(stream);
+ }
+
/**
* Adds the given chapters track to the loaded manifest. load()
* must resolve before calling. The presentation must have a duration.
diff --git a/lib/util/error.js b/lib/util/error.js
index 45f7f4fd18..9cf5305cc7 100644
--- a/lib/util/error.js
+++ b/lib/util/error.js
@@ -358,6 +358,18 @@ shaka.util.Error.Code = {
*/
'CHAPTERS_TRACK_FAILED': 2015,
+ /**
+ * External thumbnails tracks cannot be added in src= because native platform
+ * doesn't support it.
+ */
+ 'CANNOT_ADD_EXTERNAL_THUMBNAILS_TO_SRC_EQUALS': 2016,
+
+ /**
+ * Only external urls of WebVTT type are supported.
+ *
error.data[0] is the uri.
+ */
+ 'UNSUPPORTED_EXTERNAL_THUMBNAILS_URI': 2017,
+
/**
* Some component tried to read past the end of a buffer. The segment index,
* init segment, or PSSH may be malformed.
@@ -707,6 +719,11 @@ shaka.util.Error.Code = {
*/
'HLS_AES_128_INVALID_KEY_LENGTH': 4044,
+ /**
+ * External thumbnails tracks cannot be added to live streams.
+ */
+ 'CANNOT_ADD_EXTERNAL_THUMBNAILS_TO_LIVE_STREAM': 4045,
+
// RETIRED: 'INCONSISTENT_BUFFER_STATE': 5000,
// RETIRED: 'INVALID_SEGMENT_INDEX': 5001,
diff --git a/test/player_integration.js b/test/player_integration.js
index 2f1479601c..7109fff5db 100644
--- a/test/player_integration.js
+++ b/test/player_integration.js
@@ -262,6 +262,49 @@ describe('Player', () => {
expect(cues.length).toBeGreaterThan(0);
});
+ it('skip appended thumbnails for external thumbnails with sprites',
+ async () => {
+ await player.load('test:sintel_no_text_compiled');
+ const locationUri = new goog.Uri(location.href);
+ const partialUri =
+ new goog.Uri('/base/test/test/assets/thumbnails-sprites.vtt');
+ const absoluteUri = locationUri.resolve(partialUri);
+ const newTrack =
+ await player.addThumbnailsTrack(absoluteUri.toString());
+
+ expect(player.getImageTracks()).toEqual([newTrack]);
+
+ const thumbnail1 = await player.getThumbnails(newTrack.id, 0);
+ expect(thumbnail1).toBe(null);
+ const thumbnail2 = await player.getThumbnails(newTrack.id, 10);
+ expect(thumbnail2).toBe(null);
+ const thumbnail3 = await player.getThumbnails(newTrack.id, 40);
+ expect(thumbnail3).toBe(null);
+ });
+
+ it('appends thumbnails for external thumbnails without sprites',
+ async () => {
+ await player.load('test:sintel_no_text_compiled');
+ const locationUri = new goog.Uri(location.href);
+ const partialUri =
+ new goog.Uri('/base/test/test/assets/thumbnails.vtt');
+ const absoluteUri = locationUri.resolve(partialUri);
+ const newTrack =
+ await player.addThumbnailsTrack(absoluteUri.toString());
+
+ expect(player.getImageTracks()).toEqual([newTrack]);
+
+ const thumbnail1 = await player.getThumbnails(newTrack.id, 0);
+ expect(thumbnail1.startTime).toBe(0);
+ expect(thumbnail1.duration).toBe(5);
+ const thumbnail2 = await player.getThumbnails(newTrack.id, 10);
+ expect(thumbnail2.startTime).toBe(5);
+ expect(thumbnail2.duration).toBe(25);
+ const thumbnail3 = await player.getThumbnails(newTrack.id, 40);
+ expect(thumbnail3.startTime).toBe(30);
+ expect(thumbnail3.duration).toBe(30);
+ });
+
// https://github.com/shaka-project/shaka-player/issues/2553
it('does not change the selected track', async () => {
player.configure('streaming.alwaysStreamText', false);
diff --git a/test/test/assets/thumbnails-sprites.vtt b/test/test/assets/thumbnails-sprites.vtt
new file mode 100644
index 0000000000..42d2cf0fb5
--- /dev/null
+++ b/test/test/assets/thumbnails-sprites.vtt
@@ -0,0 +1,10 @@
+WEBVTT
+
+00:00.000 --> 00:05.000
+image1.jpg#xywh=0,0,160,90
+
+00:05.000 --> 00:30.000
+image2.jpg#xywh=0,0,160,90
+
+00:30.000 --> 01:00.000
+image3.jpg#xywh=0,0,160,90
\ No newline at end of file
diff --git a/test/test/assets/thumbnails.vtt b/test/test/assets/thumbnails.vtt
new file mode 100644
index 0000000000..b55896bf7c
--- /dev/null
+++ b/test/test/assets/thumbnails.vtt
@@ -0,0 +1,10 @@
+WEBVTT
+
+00:00.000 --> 00:05.000
+image1.jpg
+
+00:05.000 --> 00:30.000
+image2.jpg
+
+00:30.000 --> 01:00.000
+image3.jpg
\ No newline at end of file