Skip to content

Commit

Permalink
fix: Account for server-side ad cue points in external text tracks. (s…
Browse files Browse the repository at this point in the history 
…haka-project#3617)

When an external subtitle track is added and you are using DAI, the external track does not take into account the ads that the video has, so this PR makes this internally take into account when generating the external track.
  • Loading branch information
Álvaro Velad Galván committed Sep 13, 2021
1 parent a17c988 commit a7f4db7
Show file tree
Hide file tree
Showing 9 changed files with 189 additions and 45 deletions.
23 changes: 23 additions & 0 deletions externs/shaka/ads.js
View file
Expand Up @@ -33,6 +33,24 @@
shaka.extern.AdsStats;


/**
* @typedef {{
* start: number,
* end: ?number
* }}
*
* @description
* Contains the times of a range of an Ad.
*
* @property {number} start
* The start time of the range, in milliseconds.
* @property {number} end
* The end time of the range, in milliseconds.
* @exportDoc
*/
shaka.extern.AdCuePoint;


/**
* An object that's responsible for all the ad-related logic
* in the player.
Expand Down Expand Up @@ -77,6 +95,11 @@ shaka.extern.IAdManager = class extends EventTarget {
*/
replaceServerSideAdTagParameters(adTagParameters) {}

/**
* @return {!Array.<!shaka.extern.AdCuePoint>}
*/
getServerSideCuePoints() {}

/**
* Get statistics for the current playback session. If the player is not
* playing content, this will return an empty stats object.
Expand Down
31 changes: 16 additions & 15 deletions lib/ads/ad_manager.js
View file
Expand Up @@ -6,7 +6,6 @@


goog.provide('shaka.ads.AdManager');
goog.provide('shaka.ads.CuePoint');

goog.require('shaka.Player');
goog.require('shaka.ads.AdsStats');
Expand Down Expand Up @@ -568,6 +567,22 @@ shaka.ads.AdManager = class extends shaka.util.FakeEventTarget {
}


/**
* @return {!Array.<!shaka.extern.AdCuePoint>}
* @override
* @export
*/
getServerSideCuePoints() {
if (!this.ssAdManager_) {
throw new shaka.util.Error(
shaka.util.Error.Severity.RECOVERABLE,
shaka.util.Error.Category.ADS,
shaka.util.Error.Code.SS_AD_MANAGER_NOT_INITIALIZED);
}
return this.ssAdManager_.getCuePoints();
}


/**
* @return {shaka.extern.AdsStats}
* @override
Expand Down Expand Up @@ -620,20 +635,6 @@ shaka.ads.AdManager = class extends shaka.util.FakeEventTarget {
}
};


shaka.ads.CuePoint = class {
/**
* @param {number} start
* @param {?number=} end
*/
constructor(start, end = null) {
/** @public {number} */
this.start = start;
/** @public {?number} */
this.end = end;
}
};

/**
* The event name for when a sequence of ads has been loaded.
*
Expand Down
8 changes: 6 additions & 2 deletions lib/ads/client_side_ad_manager.js
View file
Expand Up @@ -145,10 +145,14 @@ shaka.ads.ClientSideAdManager = class {

const cuePointStarts = this.imaAdsManager_.getCuePoints();
if (cuePointStarts.length) {
/** @type {!Array.<!shaka.ads.CuePoint>} */
/** @type {!Array.<!shaka.extern.AdCuePoint>} */
const cuePoints = [];
for (const start of cuePointStarts) {
const shakaCuePoint = new shaka.ads.CuePoint(start);
/** @type {shaka.extern.AdCuePoint} */
const shakaCuePoint = {
start: start,
end: null,
};
cuePoints.push(shakaCuePoint);
}

Expand Down
21 changes: 19 additions & 2 deletions lib/ads/server_side_ad_manager.js
View file
Expand Up @@ -62,6 +62,9 @@ shaka.ads.ServerSideAdManager = class {
/** @private {string} */
this.backupUrl_ = '';

/** @private {!Array.<!shaka.extern.AdCuePoint>} */
this.currentCuePoints_ = [];

/** @private {shaka.util.EventManager} */
this.eventManager_ = new shaka.util.EventManager();

Expand Down Expand Up @@ -212,6 +215,7 @@ shaka.ads.ServerSideAdManager = class {
// this.streamManager_.reset();
this.backupUrl_ = '';
this.snapForwardTime_ = null;
this.currentCuePoints_ = [];
}

/**
Expand Down Expand Up @@ -241,6 +245,13 @@ shaka.ads.ServerSideAdManager = class {
}
}

/**
* @return {!Array.<!shaka.extern.AdCuePoint>}
*/
getCuePoints() {
return this.currentCuePoints_;
}

/**
* If a seek jumped over the ad break, return to the start of the
* ad break, then complete the seek after the ad played through.
Expand Down Expand Up @@ -369,13 +380,19 @@ shaka.ads.ServerSideAdManager = class {
onCuePointsChanged_(e) {
const streamData = e.getStreamData();

/** @type {!Array.<!shaka.ads.CuePoint>} */
/** @type {!Array.<!shaka.extern.AdCuePoint>} */
const cuePoints = [];
for (const point of streamData.cuepoints) {
const shakaCuePoint = new shaka.ads.CuePoint(point.start, point.end);
/** @type {shaka.extern.AdCuePoint} */
const shakaCuePoint = {
start: point.start,
end: point.end,
};
cuePoints.push(shakaCuePoint);
}

this.currentCuePoints_ = cuePoints;

this.onEvent_(
new shaka.util.FakeEvent(shaka.ads.AdManager.CUEPOINTS_CHANGED,
{'cuepoints': cuePoints}));
Expand Down
47 changes: 38 additions & 9 deletions lib/player.js
View file
Expand Up @@ -4452,12 +4452,20 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
mimeType = await this.getTextMimetype_(uri);
}

let adCuePoints = [];
if (this.adManager_) {
try {
adCuePoints = this.adManager_.getServerSideCuePoints();
} catch (error) {}
}

if (this.loadMode_ == shaka.Player.LoadMode.SRC_EQUALS) {
if (forced) {
// See: https://github.com/whatwg/html/issues/4472
kind = 'forced';
}
await this.addSrcTrackElement_(uri, language, kind, mimeType, label);
await this.addSrcTrackElement_(uri, language, kind, mimeType, label || '',
adCuePoints);
const textTracks = this.getTextTracks();
const srcTrack = textTracks.find((t) => {
return t.language == language &&
Expand Down Expand Up @@ -4487,6 +4495,18 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
shaka.util.Error.Code.CANNOT_ADD_EXTERNAL_TEXT_TO_LIVE_STREAM);
}

if (adCuePoints.length) {
goog.asserts.assert(
this.networkingEngine_, 'Need networking engine.');
const data = await this.getTextData_(uri,
this.networkingEngine_,
this.config_.streaming.retryParameters);
const vvtText = this.convertToWebVTT_(data, mimeType, adCuePoints);
const blob = new Blob([vvtText], {type: 'text/vtt'});
uri = shaka.media.MediaSourceEngine.createObjectURL(blob);
mimeType = 'text/vtt';
}

/** @type {shaka.extern.Stream} */
const stream = {
id: this.nextExternalStreamId_++,
Expand Down Expand Up @@ -4559,8 +4579,14 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
if (!mimeType) {
mimeType = await this.getTextMimetype_(uri);
}
let adCuePoints = [];
if (this.adManager_) {
try {
adCuePoints = this.adManager_.getServerSideCuePoints();
} catch (error) {}
}
await this.addSrcTrackElement_(uri, language, /* kind= */ 'chapters',
mimeType);
mimeType, /* label= */ '', adCuePoints);
const chaptersTracks = this.getChaptersTracks();
const chaptersTrack = chaptersTracks.find((t) => {
return t.language == language;
Expand Down Expand Up @@ -4623,25 +4649,27 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
* @param {string} language
* @param {string} kind
* @param {string} mimeType
* @param {string=} label
* @param {string} label
* @param {!Array.<!shaka.extern.AdCuePoint>} adCuePoints
* @private
*/
async addSrcTrackElement_(uri, language, kind, mimeType, label) {
if (mimeType != 'text/vtt') {
async addSrcTrackElement_(uri, language, kind, mimeType, label,
adCuePoints) {
if (mimeType != 'text/vtt' || adCuePoints.length) {
goog.asserts.assert(
this.networkingEngine_, 'Need networking engine.');
const data = await this.getTextData_(uri,
this.networkingEngine_,
this.config_.streaming.retryParameters);
const vvtText = this.convertToWebVTT_(data, mimeType);
const vvtText = this.convertToWebVTT_(data, mimeType, adCuePoints);
const blob = new Blob([vvtText], {type: 'text/vtt'});
uri = shaka.media.MediaSourceEngine.createObjectURL(blob);
mimeType = 'text/vtt';
}
const trackElement =
/** @type {!HTMLTrackElement} */(document.createElement('track'));
trackElement.src = uri;
trackElement.label = label || '';
trackElement.label = label;
trackElement.kind = kind;
trackElement.srclang = language;
// Because we're pulling in the text track file via Javascript, the
Expand Down Expand Up @@ -4680,10 +4708,11 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
*
* @param {BufferSource} buffer
* @param {string} mimeType
* @param {!Array.<!shaka.extern.AdCuePoint>} adCuePoints
* @return {string}
* @private
*/
convertToWebVTT_(buffer, mimeType) {
convertToWebVTT_(buffer, mimeType, adCuePoints) {
const factory = shaka.text.TextEngine.findParser(mimeType);
if (factory) {
const obj = factory();
Expand All @@ -4694,7 +4723,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
};
const data = shaka.util.BufferUtils.toUint8(buffer);
const cues = obj.parseMedia(data, time);
return shaka.text.WebVttGenerator.convert(cues);
return shaka.text.WebVttGenerator.convert(cues, adCuePoints);
}
throw new shaka.util.Error(
shaka.util.Error.Severity.CRITICAL,
Expand Down
33 changes: 21 additions & 12 deletions lib/text/web_vtt_generator.js
View file
Expand Up @@ -17,9 +17,10 @@ goog.require('shaka.text.Cue');
shaka.text.WebVttGenerator = class {
/**
* @param {!Array.<!shaka.text.Cue>} cues
* @param {!Array.<!shaka.extern.AdCuePoint>} adCuePoints
* @return {string}
*/
static convert(cues) {
static convert(cues, adCuePoints) {
// Flatten nested cue payloads recursively. If a cue has nested cues,
// their contents should be combined and replace the payload of the parent.
const flattenPayload = (cue) => {
Expand Down Expand Up @@ -64,6 +65,25 @@ shaka.text.WebVttGenerator = class {
}
};

const webvttTimeString = (time) => {
let newTime = time;
for (const adCuePoint of adCuePoints) {
if (adCuePoint.end && adCuePoint.start < time) {
const offset = adCuePoint.end - adCuePoint.start;
newTime += offset;
}
}
const hours = Math.floor(newTime / 3600);
const minutes = Math.floor(newTime / 60 % 60);
const seconds = Math.floor(newTime % 60);
const milliseconds = Math.floor(newTime * 1000 % 1000);
return (hours < 10 ? '0' : '') + hours + ':' +
(minutes < 10 ? '0' : '') + minutes + ':' +
(seconds < 10 ? '0' : '') + seconds + '.' +
(milliseconds < 100 ? (milliseconds < 10 ? '00' : '0') : '') +
milliseconds;
};

// We don't want to modify the array or objects passed in, since we don't
// technically own them. So we build a new array and replace certain items
// in it if they need to be flattened.
Expand All @@ -80,17 +100,6 @@ shaka.text.WebVttGenerator = class {

let webvttString = 'WEBVTT\n\n';
for (const cue of flattenedCues) {
const webvttTimeString = (time) => {
const hours = Math.floor(time / 3600);
const minutes = Math.floor(time / 60 % 60);
const seconds = Math.floor(time % 60);
const milliseconds = Math.floor(time * 1000 % 1000);
return (hours < 10 ? '0' : '') + hours + ':' +
(minutes < 10 ? '0' : '') + minutes + ':' +
(seconds < 10 ? '0' : '') + seconds + '.' +
(milliseconds < 100 ? (milliseconds < 10 ? '00' : '0') : '') +
milliseconds;
};
const webvttSettings = (cue) => {
const settings = [];
const Cue = shaka.text.Cue;
Expand Down
7 changes: 7 additions & 0 deletions test/test/util/fake_ad_manager.js
View file
Expand Up @@ -50,6 +50,13 @@ shaka.test.FakeAdManager = class extends shaka.util.FakeEventTarget {
/** @override */
replaceServerSideAdTagParameters(adTagParameters) {}

/**
* @override
*/
getServerSideCuePoints() {
return [];
}

/** @override */
getStats() {
return this.stats_;
Expand Down

0 comments on commit a7f4db7

Please sign in to comment.