feat: Add external thumbnails support (shaka-project#4497)

JW spec:
https://docs.jwplayer.com/platform/docs/vdh-add-preview-thumbnails

Video.js implementation:
https://github.com/chrisboustead/videojs-vtt-thumbnails

Note: thumbnails with sprites are not supported yet.

lib/cast/cast_utils.js

@@ -374,6 +374,7 @@ shaka.cast.CastUtils.PlayerInitAfterLoadState = [
 shaka.cast.CastUtils.PlayerVoidMethods = [
   'addChaptersTrack',
   'addTextTrackAsync',
+  'addThumbnailsTrack',
   'cancelTrickPlay',
   'configure',
   'getChapters',

lib/player.js

@@ -27,6 +27,7 @@ goog.require('shaka.media.QualityObserver');
 goog.require('shaka.media.RegionObserver');
 goog.require('shaka.media.RegionTimeline');
 goog.require('shaka.media.SegmentIndex');
+goog.require('shaka.media.SegmentReference');
 goog.require('shaka.media.SrcEqualsPlayhead');
 goog.require('shaka.media.StreamingEngine');
 goog.require('shaka.media.TimeRangesUtils');
@@ -4627,6 +4628,138 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
     return shaka.util.StreamUtils.textStreamToTrack(stream);
   }
 
+  /**
+   * Adds the given thumbnails track to the loaded manifest.
+   * <code>load()</code> must resolve before calling.  The presentation must
+   * have a duration.
+   *
+   * This returns the created track, which can immediately be used by the
+   * application.
+   *
+   * @param {string} uri
+   * @param {string=} mimeType
+   * @return {!Promise.<shaka.extern.Track>}
+   * @export
+   */
+  async addThumbnailsTrack(uri, mimeType) {
+    if (this.loadMode_ != shaka.Player.LoadMode.MEDIA_SOURCE &&
+        this.loadMode_ != shaka.Player.LoadMode.SRC_EQUALS) {
+      shaka.log.error(
+          'Must call load() and wait for it to resolve before adding image ' +
+          'tracks.');
+      throw new shaka.util.Error(
+          shaka.util.Error.Severity.RECOVERABLE,
+          shaka.util.Error.Category.PLAYER,
+          shaka.util.Error.Code.CONTENT_NOT_LOADED);
+    }
+
+    if (this.loadMode_ == shaka.Player.LoadMode.SRC_EQUALS) {
+      shaka.log.error('Cannot add this thumbnail track when loaded with src=');
+      throw new shaka.util.Error(
+          shaka.util.Error.Severity.RECOVERABLE,
+          shaka.util.Error.Category.TEXT,
+          shaka.util.Error.Code.CANNOT_ADD_EXTERNAL_THUMBNAILS_TO_SRC_EQUALS);
+    }
+
+    if (!mimeType) {
+      mimeType = await this.getTextMimetype_(uri);
+    }
+
+    if (mimeType != 'text/vtt') {
+      throw new shaka.util.Error(
+          shaka.util.Error.Severity.RECOVERABLE,
+          shaka.util.Error.Category.TEXT,
+          shaka.util.Error.Code.UNSUPPORTED_EXTERNAL_THUMBNAILS_URI,
+          uri);
+    }
+
+    const ContentType = shaka.util.ManifestParserUtils.ContentType;
+
+    const duration = this.manifest_.presentationTimeline.getDuration();
+    if (duration == Infinity) {
+      throw new shaka.util.Error(
+          shaka.util.Error.Severity.RECOVERABLE,
+          shaka.util.Error.Category.MANIFEST,
+          shaka.util.Error.Code.CANNOT_ADD_EXTERNAL_THUMBNAILS_TO_LIVE_STREAM);
+    }
+
+    goog.asserts.assert(
+        this.networkingEngine_, 'Need networking engine.');
+    const buffer = await this.getTextData_(uri,
+        this.networkingEngine_,
+        this.config_.streaming.retryParameters);
+
+    const factory = shaka.text.TextEngine.findParser(mimeType);
+    if (!factory) {
+      throw new shaka.util.Error(
+          shaka.util.Error.Severity.CRITICAL,
+          shaka.util.Error.Category.TEXT,
+          shaka.util.Error.Code.MISSING_TEXT_PLUGIN,
+          mimeType);
+    }
+    const TextParser = factory();
+    const time = {
+      periodStart: 0,
+      segmentStart: 0,
+      segmentEnd: duration,
+      vttOffset: 0,
+    };
+    const data = shaka.util.BufferUtils.toUint8(buffer);
+    const cues = TextParser.parseMedia(data, time);
+
+    const references = [];
+    for (const cue of cues) {
+      const imageUri = shaka.util.ManifestParserUtils.resolveUris(
+          [uri], [cue.payload])[0];
+      if (imageUri.includes('#xywh')) {
+        shaka.log.alwaysWarn('Unsupported image uri', imageUri);
+        continue;
+      }
+      references.push(new shaka.media.SegmentReference(
+          cue.startTime,
+          cue.endTime,
+          () => [imageUri],
+          /* startByte= */ 0,
+          /* endByte= */ null,
+          /* initSegmentReference= */ null,
+          /* timestampOffset= */ 0,
+          /* appendWindowStart= */ 0,
+          /* appendWindowEnd= */ Infinity,
+      ));
+    }
+
+    /** @type {shaka.extern.Stream} */
+    const stream = {
+      id: this.nextExternalStreamId_++,
+      originalId: null,
+      createSegmentIndex: () => Promise.resolve(),
+      segmentIndex: new shaka.media.SegmentIndex(references),
+      mimeType: mimeType || '',
+      codecs: '',
+      kind: '',
+      encrypted: false,
+      drmInfos: [],
+      keyIds: new Set(),
+      language: 'und',
+      label: null,
+      type: ContentType.IMAGE,
+      primary: false,
+      trickModeVideo: null,
+      emsgSchemeIdUris: null,
+      roles: [],
+      forced: false,
+      channelsCount: null,
+      audioSamplingRate: null,
+      spatialAudio: false,
+      closedCaptions: null,
+      tilesLayout: '1x1',
+    };
+
+    this.manifest_.imageStreams.push(stream);
+    this.onTracksChanged_();
+    return shaka.util.StreamUtils.imageStreamToTrack(stream);
+  }
+
   /**
    * Adds the given chapters track to the loaded manifest.  <code>load()</code>
    * must resolve before calling.  The presentation must have a duration.

lib/util/error.js

@@ -358,6 +358,18 @@ shaka.util.Error.Code = {
    */
   'CHAPTERS_TRACK_FAILED': 2015,
 
+  /**
+   * External thumbnails tracks cannot be added in src= because native platform
+   * doesn't support it.
+   */
+  'CANNOT_ADD_EXTERNAL_THUMBNAILS_TO_SRC_EQUALS': 2016,
+
+  /**
+   * Only external urls of WebVTT type are supported.
+   * <br> error.data[0] is the uri.
+   */
+  'UNSUPPORTED_EXTERNAL_THUMBNAILS_URI': 2017,
+
   /**
    * Some component tried to read past the end of a buffer.  The segment index,
    * init segment, or PSSH may be malformed.
@@ -707,6 +719,11 @@ shaka.util.Error.Code = {
    */
   'HLS_AES_128_INVALID_KEY_LENGTH': 4044,
 
+  /**
+   * External thumbnails tracks cannot be added to live streams.
+   */
+  'CANNOT_ADD_EXTERNAL_THUMBNAILS_TO_LIVE_STREAM': 4045,
+
 
   // RETIRED: 'INCONSISTENT_BUFFER_STATE': 5000,
   // RETIRED: 'INVALID_SEGMENT_INDEX': 5001,

test/player_integration.js

@@ -262,6 +262,49 @@ describe('Player', () => {
       expect(cues.length).toBeGreaterThan(0);
     });
 
+    it('skip appended thumbnails for external thumbnails with sprites',
+        async () => {
+          await player.load('test:sintel_no_text_compiled');
+          const locationUri = new goog.Uri(location.href);
+          const partialUri =
+              new goog.Uri('/base/test/test/assets/thumbnails-sprites.vtt');
+          const absoluteUri = locationUri.resolve(partialUri);
+          const newTrack =
+              await player.addThumbnailsTrack(absoluteUri.toString());
+
+          expect(player.getImageTracks()).toEqual([newTrack]);
+
+          const thumbnail1 = await player.getThumbnails(newTrack.id, 0);
+          expect(thumbnail1).toBe(null);
+          const thumbnail2 = await player.getThumbnails(newTrack.id, 10);
+          expect(thumbnail2).toBe(null);
+          const thumbnail3 = await player.getThumbnails(newTrack.id, 40);
+          expect(thumbnail3).toBe(null);
+        });
+
+    it('appends thumbnails for external thumbnails without sprites',
+        async () => {
+          await player.load('test:sintel_no_text_compiled');
+          const locationUri = new goog.Uri(location.href);
+          const partialUri =
+              new goog.Uri('/base/test/test/assets/thumbnails.vtt');
+          const absoluteUri = locationUri.resolve(partialUri);
+          const newTrack =
+              await player.addThumbnailsTrack(absoluteUri.toString());
+
+          expect(player.getImageTracks()).toEqual([newTrack]);
+
+          const thumbnail1 = await player.getThumbnails(newTrack.id, 0);
+          expect(thumbnail1.startTime).toBe(0);
+          expect(thumbnail1.duration).toBe(5);
+          const thumbnail2 = await player.getThumbnails(newTrack.id, 10);
+          expect(thumbnail2.startTime).toBe(5);
+          expect(thumbnail2.duration).toBe(25);
+          const thumbnail3 = await player.getThumbnails(newTrack.id, 40);
+          expect(thumbnail3.startTime).toBe(30);
+          expect(thumbnail3.duration).toBe(30);
+        });
+
     // https://github.com/shaka-project/shaka-player/issues/2553
     it('does not change the selected track', async () => {
       player.configure('streaming.alwaysStreamText', false);

test/test/assets/thumbnails-sprites.vtt

@@ -0,0 +1,10 @@
+WEBVTT
+
+00:00.000 --> 00:05.000
+image1.jpg#xywh=0,0,160,90
+
+00:05.000 --> 00:30.000
+image2.jpg#xywh=0,0,160,90
+
+00:30.000 --> 01:00.000
+image3.jpg#xywh=0,0,160,90

test/test/assets/thumbnails.vtt

@@ -0,0 +1,10 @@
+WEBVTT
+
+00:00.000 --> 00:05.000
+image1.jpg
+
+00:05.000 --> 00:30.000
+image2.jpg
+
+00:30.000 --> 01:00.000
+image3.jpg