fix: Fix thumbnail duration, expose start time and duration

Thumbnail segments may be structured for a certain grid size and
duration, but the segment references may have their duration truncated
due to the end of a period or the end of the presentation.  This was
causing us to calculate the wrong duration for those individual
thumbnails, and therefore return the wrong
thumbnail.

We also did not have any way to indicate to an application how long a
thumbnail should be shown.

This fixes the duration calculation by retaining the original,
untruncated duration in SegmentReference.  This also exposes startTime
and duration information on the Thumbnail object, so that applications
know when/where and how long to show a given thumbnail.

Closes shaka-project#3517

Change-Id: I84aa7705a19691fc6ae68eee9944fecbd7067fe0

externs/shaka/player.js

@@ -1085,6 +1085,8 @@ shaka.extern.LanguageRole;
  *   height: number,
  *   positionX: number,
  *   positionY: number,
+ *   startTime: number,
+ *   duration: number,
  *   uris: !Array.<string>,
  *   width: number
  * }}
@@ -1095,6 +1097,10 @@ shaka.extern.LanguageRole;
  *    The thumbnail left position in px.
  * @property {number} positionY
  *    The thumbnail top position in px.
+ * @property {number} startTime
+ *    The start time of the thumbnail in the presentation timeline, in seconds.
+ * @property {number} duration
+ *    The duration of the thumbnail, in seconds.
  * @property {!Array.<string>} uris
  *   An array of URIs to attempt.  They will be tried in the order they are
  *   given.

lib/dash/segment_template.js

@@ -390,17 +390,17 @@ shaka.dash.SegmentTemplate = class {
 
       // Relative to the presentation.
       const segmentStart = segmentPeriodTime + periodStart;
+      const trueSegmentEnd = segmentStart + segmentDuration;
       // Cap the segment end at the period end so that references from the
       // next period will fit neatly after it.
-      const segmentEnd = Math.min(segmentStart + segmentDuration,
-          getPeriodEnd());
+      const segmentEnd = Math.min(trueSegmentEnd, getPeriodEnd());
 
       // This condition will be true unless the segmentStart was >= periodEnd.
       // If we've done the position calculations correctly, this won't happen.
       goog.asserts.assert(segmentStart < segmentEnd,
           'Generated a segment outside of the period!');
 
-      return new shaka.media.SegmentReference(
+      const ref = new shaka.media.SegmentReference(
           segmentStart,
           segmentEnd,
           getUris,
@@ -410,6 +410,9 @@ shaka.dash.SegmentTemplate = class {
           timestampOffset,
           /* appendWindowStart= */ periodStart,
           /* appendWindowEnd= */ getPeriodEnd());
+      // This is necessary information for thumbnail streams:
+      ref.trueEndTime = trueSegmentEnd;
+      return ref;
     };
 
     for (let position = minPosition; position <= maxPosition; ++position) {

lib/media/segment_reference.js

@@ -157,6 +157,14 @@ shaka.media.SegmentReference = class {
     /** @type {number} */
     this.endTime = endTime;
 
+    /**
+     * The "true" end time of the segment, without considering the period end
+     * time.  This is necessary for thumbnail segments, where timing requires us
+     * to know the original segment duration as described in the manifest.
+     * @type {number}
+     */
+    this.trueEndTime = endTime;
+
     /** @type {function():!Array.<string>} */
     this.getUrisInner = uris;
 

lib/player.js

@@ -3436,31 +3436,36 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
       const rows = parseInt(match[2], 10);
       const width = fullImageWidth / columns;
       const height = fullImageHeight / rows;
+      const totalImages = columns * rows;
+      const segmentDuration = reference.trueEndTime - reference.startTime;
+      const thumbnailDuration = segmentDuration / totalImages;
+      let thumbnailTime = reference.startTime;
       let positionX = 0;
       let positionY = 0;
-      const totalImages = columns * rows;
       // If the number of images in the segment is greater than 1, we have to
       // find the correct image. For that we will return to the app the
       // coordinates of the position of the correct image.
       // Image search is always from left to right and top to bottom.
       // Note: The time between images within the segment is always
       // equidistant.
       //
-      // Eg: Total images 5, tileLayout 5x1, segmentTime 5, thumbnailTime 2
+      // Eg: Total images 5, tileLayout 5x1, segmentDuration 5, thumbnailTime 2
       // positionX = 0.4 * fullImageWidth
       // positionY = 0
       if (totalImages > 1) {
-        const thumbnailTime = time - reference.startTime;
-        const segmentTime = reference.endTime - reference.startTime;
         const thumbnailPosition =
-            Math.floor(thumbnailTime * totalImages / segmentTime);
+            Math.floor((time - reference.startTime) / thumbnailDuration);
+        thumbnailTime = reference.startTime +
+            (thumbnailPosition * thumbnailDuration);
         positionX = (thumbnailPosition % columns) * width;
         positionY = Math.floor(thumbnailPosition / columns) * height;
       }
       return {
         height: height,
         positionX: positionX,
         positionY: positionY,
+        startTime: thumbnailTime,
+        duration: thumbnailDuration,
         uris: reference.getUris(),
         width: width,
       };

test/dash/dash_parser_live_unit.js

@@ -276,6 +276,7 @@ describe('DashParser Live', () => {
         ref.timestampOffset = pStart;
         ref.startTime += pStart;
         ref.endTime += pStart;
+        ref.trueEndTime += pStart;
       }
       /** @const {!Array.<!shaka.media.SegmentReference>} */
       const allRefs = period1Refs.concat(period2Refs);

test/dash/dash_parser_segment_template_unit.js

@@ -127,9 +127,9 @@ describe('DashParser SegmentTemplate', () => {
       // The first segment is number 1 and position 0.
       // Although the segment is 60 seconds long, it is clipped to the period
       // duration of 30 seconds.
-      const references = [
-        ManifestParser.makeReference('s1.mp4', 0, 30, baseUri),
-      ];
+      const ref = ManifestParser.makeReference('s1.mp4', 0, 30, baseUri);
+      ref.trueEndTime = 60;
+      const references = [ref];
       await Dash.testSegmentIndex(source, references);
     });
 

test/player_unit.js

@@ -3516,10 +3516,10 @@ describe('Player', () => {
     describe('getThumbnails', () => {
       it('returns correct thumbnail position for supplied time', async () => {
         const uris = () => ['thumbnail'];
-        const segment = new shaka.media.SegmentReference(
+        const ref = new shaka.media.SegmentReference(
             0, 60, uris, 0, null, null, 0, 0, Infinity, [],
         );
-        const index = new shaka.media.SegmentIndex([segment]);
+        const index = new shaka.media.SegmentIndex([ref]);
 
         manifest = shaka.test.ManifestGenerator.generate((manifest) => {
           manifest.addVariant(0, (variant) => {
@@ -3566,6 +3566,50 @@ describe('Player', () => {
           height: 50,
         }));
       });
+
+      it('returns correct duration for a partially-used segment', async () => {
+        const uris = () => ['thumbnail'];
+
+        const ref1 = new shaka.media.SegmentReference(
+            0, 60, uris, 0, null, null, 0, 0, Infinity);
+        const ref2 = new shaka.media.SegmentReference(
+            60, 90, uris, 0, null, null, 0, 0, Infinity);
+        ref2.trueEndTime = 120;
+
+        const index = new shaka.media.SegmentIndex([ref1, ref2]);
+
+        manifest = shaka.test.ManifestGenerator.generate((manifest) => {
+          manifest.addVariant(0, (variant) => {
+            variant.addVideo(1);
+          });
+          manifest.addImageStream(5, (stream) => {
+            stream.originalId = 'thumbnail';
+            stream.width = 200;
+            stream.height = 150;
+            stream.mimeType = 'image/jpeg';
+            stream.tilesLayout = '2x3';
+            stream.segmentIndex = index;
+          });
+        });
+
+        await player.load(fakeManifestUri, 0, fakeMimeType);
+
+        const thumbnail0 = await player.getThumbnails(5, 0);
+        expect(thumbnail0.startTime).toBe(0);
+        expect(thumbnail0.duration).toBe(10);
+
+        const thumbnail1 = await player.getThumbnails(5, 10);
+        expect(thumbnail1.startTime).toBe(10);
+        expect(thumbnail1.duration).toBe(10);
+
+        const thumbnail6 = await player.getThumbnails(5, 60);
+        expect(thumbnail6.startTime).toBe(60);
+        expect(thumbnail6.duration).toBe(10);
+
+        const thumbnail8 = await player.getThumbnails(5, 80);
+        expect(thumbnail8.startTime).toBe(80);
+        expect(thumbnail8.duration).toBe(10);
+      });
     });
   });