From d4a1255235e174d8a0ca6ca3a8edb5b10db59405 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 13 Jul 2021 16:28:56 -0700 Subject: [PATCH] fix: Fix thumbnail duration, expose start time and duration Thumbnail segments may be structured for a certain grid size and duration, but the segment references may have their duration truncated due to the end of a period or the end of the presentation. This was causing us to calculate the wrong duration for those individual thumbnails, and therefore return the wrong thumbnail. We also did not have any way to indicate to an application how long a thumbnail should be shown. This fixes the duration calculation by retaining the original, untruncated duration in SegmentReference. This also exposes startTime and duration information on the Thumbnail object, so that applications know when/where and how long to show a given thumbnail. Closes #3517 Change-Id: I84aa7705a19691fc6ae68eee9944fecbd7067fe0 --- externs/shaka/player.js | 6 +++ lib/dash/segment_template.js | 9 ++-- lib/media/segment_reference.js | 8 ++++ lib/player.js | 15 ++++-- test/dash/dash_parser_live_unit.js | 1 + .../dash/dash_parser_segment_template_unit.js | 6 +-- test/player_unit.js | 48 ++++++++++++++++++- 7 files changed, 80 insertions(+), 13 deletions(-) diff --git a/externs/shaka/player.js b/externs/shaka/player.js index 0ca8ffa8b6..6a8cefb962 100644 --- a/externs/shaka/player.js +++ b/externs/shaka/player.js @@ -1085,6 +1085,8 @@ shaka.extern.LanguageRole; * height: number, * positionX: number, * positionY: number, + * startTime: number, + * duration: number, * uris: !Array., * width: number * }} @@ -1095,6 +1097,10 @@ shaka.extern.LanguageRole; * The thumbnail left position in px. * @property {number} positionY * The thumbnail top position in px. + * @property {number} startTime + * The start time of the thumbnail in the presentation timeline, in seconds. + * @property {number} duration + * The duration of the thumbnail, in seconds. * @property {!Array.} uris * An array of URIs to attempt. They will be tried in the order they are * given. diff --git a/lib/dash/segment_template.js b/lib/dash/segment_template.js index 275993e564..d01eef23b1 100644 --- a/lib/dash/segment_template.js +++ b/lib/dash/segment_template.js @@ -390,17 +390,17 @@ shaka.dash.SegmentTemplate = class { // Relative to the presentation. const segmentStart = segmentPeriodTime + periodStart; + const trueSegmentEnd = segmentStart + segmentDuration; // Cap the segment end at the period end so that references from the // next period will fit neatly after it. - const segmentEnd = Math.min(segmentStart + segmentDuration, - getPeriodEnd()); + const segmentEnd = Math.min(trueSegmentEnd, getPeriodEnd()); // This condition will be true unless the segmentStart was >= periodEnd. // If we've done the position calculations correctly, this won't happen. goog.asserts.assert(segmentStart < segmentEnd, 'Generated a segment outside of the period!'); - return new shaka.media.SegmentReference( + const ref = new shaka.media.SegmentReference( segmentStart, segmentEnd, getUris, @@ -410,6 +410,9 @@ shaka.dash.SegmentTemplate = class { timestampOffset, /* appendWindowStart= */ periodStart, /* appendWindowEnd= */ getPeriodEnd()); + // This is necessary information for thumbnail streams: + ref.trueEndTime = trueSegmentEnd; + return ref; }; for (let position = minPosition; position <= maxPosition; ++position) { diff --git a/lib/media/segment_reference.js b/lib/media/segment_reference.js index 3ee1adb557..866e1d8d06 100644 --- a/lib/media/segment_reference.js +++ b/lib/media/segment_reference.js @@ -157,6 +157,14 @@ shaka.media.SegmentReference = class { /** @type {number} */ this.endTime = endTime; + /** + * The "true" end time of the segment, without considering the period end + * time. This is necessary for thumbnail segments, where timing requires us + * to know the original segment duration as described in the manifest. + * @type {number} + */ + this.trueEndTime = endTime; + /** @type {function():!Array.} */ this.getUrisInner = uris; diff --git a/lib/player.js b/lib/player.js index 977d7d3b40..7bed485f78 100644 --- a/lib/player.js +++ b/lib/player.js @@ -3436,9 +3436,12 @@ shaka.Player = class extends shaka.util.FakeEventTarget { const rows = parseInt(match[2], 10); const width = fullImageWidth / columns; const height = fullImageHeight / rows; + const totalImages = columns * rows; + const segmentDuration = reference.trueEndTime - reference.startTime; + const thumbnailDuration = segmentDuration / totalImages; + let thumbnailTime = reference.startTime; let positionX = 0; let positionY = 0; - const totalImages = columns * rows; // If the number of images in the segment is greater than 1, we have to // find the correct image. For that we will return to the app the // coordinates of the position of the correct image. @@ -3446,14 +3449,14 @@ shaka.Player = class extends shaka.util.FakeEventTarget { // Note: The time between images within the segment is always // equidistant. // - // Eg: Total images 5, tileLayout 5x1, segmentTime 5, thumbnailTime 2 + // Eg: Total images 5, tileLayout 5x1, segmentDuration 5, thumbnailTime 2 // positionX = 0.4 * fullImageWidth // positionY = 0 if (totalImages > 1) { - const thumbnailTime = time - reference.startTime; - const segmentTime = reference.endTime - reference.startTime; const thumbnailPosition = - Math.floor(thumbnailTime * totalImages / segmentTime); + Math.floor((time - reference.startTime) / thumbnailDuration); + thumbnailTime = reference.startTime + + (thumbnailPosition * thumbnailDuration); positionX = (thumbnailPosition % columns) * width; positionY = Math.floor(thumbnailPosition / columns) * height; } @@ -3461,6 +3464,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget { height: height, positionX: positionX, positionY: positionY, + startTime: thumbnailTime, + duration: thumbnailDuration, uris: reference.getUris(), width: width, }; diff --git a/test/dash/dash_parser_live_unit.js b/test/dash/dash_parser_live_unit.js index e28c9a5bbf..5657f177ec 100644 --- a/test/dash/dash_parser_live_unit.js +++ b/test/dash/dash_parser_live_unit.js @@ -276,6 +276,7 @@ describe('DashParser Live', () => { ref.timestampOffset = pStart; ref.startTime += pStart; ref.endTime += pStart; + ref.trueEndTime += pStart; } /** @const {!Array.} */ const allRefs = period1Refs.concat(period2Refs); diff --git a/test/dash/dash_parser_segment_template_unit.js b/test/dash/dash_parser_segment_template_unit.js index 39a599108b..8af8134b76 100644 --- a/test/dash/dash_parser_segment_template_unit.js +++ b/test/dash/dash_parser_segment_template_unit.js @@ -127,9 +127,9 @@ describe('DashParser SegmentTemplate', () => { // The first segment is number 1 and position 0. // Although the segment is 60 seconds long, it is clipped to the period // duration of 30 seconds. - const references = [ - ManifestParser.makeReference('s1.mp4', 0, 30, baseUri), - ]; + const ref = ManifestParser.makeReference('s1.mp4', 0, 30, baseUri); + ref.trueEndTime = 60; + const references = [ref]; await Dash.testSegmentIndex(source, references); }); diff --git a/test/player_unit.js b/test/player_unit.js index f8e956be64..0a9a3852e1 100644 --- a/test/player_unit.js +++ b/test/player_unit.js @@ -3516,10 +3516,10 @@ describe('Player', () => { describe('getThumbnails', () => { it('returns correct thumbnail position for supplied time', async () => { const uris = () => ['thumbnail']; - const segment = new shaka.media.SegmentReference( + const ref = new shaka.media.SegmentReference( 0, 60, uris, 0, null, null, 0, 0, Infinity, [], ); - const index = new shaka.media.SegmentIndex([segment]); + const index = new shaka.media.SegmentIndex([ref]); manifest = shaka.test.ManifestGenerator.generate((manifest) => { manifest.addVariant(0, (variant) => { @@ -3566,6 +3566,50 @@ describe('Player', () => { height: 50, })); }); + + it('returns correct duration for a partially-used segment', async () => { + const uris = () => ['thumbnail']; + + const ref1 = new shaka.media.SegmentReference( + 0, 60, uris, 0, null, null, 0, 0, Infinity); + const ref2 = new shaka.media.SegmentReference( + 60, 90, uris, 0, null, null, 0, 0, Infinity); + ref2.trueEndTime = 120; + + const index = new shaka.media.SegmentIndex([ref1, ref2]); + + manifest = shaka.test.ManifestGenerator.generate((manifest) => { + manifest.addVariant(0, (variant) => { + variant.addVideo(1); + }); + manifest.addImageStream(5, (stream) => { + stream.originalId = 'thumbnail'; + stream.width = 200; + stream.height = 150; + stream.mimeType = 'image/jpeg'; + stream.tilesLayout = '2x3'; + stream.segmentIndex = index; + }); + }); + + await player.load(fakeManifestUri, 0, fakeMimeType); + + const thumbnail0 = await player.getThumbnails(5, 0); + expect(thumbnail0.startTime).toBe(0); + expect(thumbnail0.duration).toBe(10); + + const thumbnail1 = await player.getThumbnails(5, 10); + expect(thumbnail1.startTime).toBe(10); + expect(thumbnail1.duration).toBe(10); + + const thumbnail6 = await player.getThumbnails(5, 60); + expect(thumbnail6.startTime).toBe(60); + expect(thumbnail6.duration).toBe(10); + + const thumbnail8 = await player.getThumbnails(5, 80); + expect(thumbnail8.startTime).toBe(80); + expect(thumbnail8.duration).toBe(10); + }); }); });