Skip to content

Commit

Permalink
fix(TTML): Fix support of urls in smpte:backgroundImage (#5851)
Browse files Browse the repository at this point in the history
Fixes #5049
  • Loading branch information
avelad committed Nov 6, 2023
1 parent 6a862d2 commit fa93d53
Show file tree
Hide file tree
Showing 12 changed files with 87 additions and 23 deletions.
4 changes: 3 additions & 1 deletion externs/shaka/text.js
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,13 @@ shaka.extern.TextParser = class {
* @param {shaka.extern.TextParser.TimeContext} timeContext
* The time information that should be used to adjust the times values
* for each cue.
* @param {?(string|undefined)} uri
* The media uri.
* @return {!Array.<!shaka.text.Cue>}
*
* @exportDoc
*/
parseMedia(data, timeContext) {}
parseMedia(data, timeContext, uri) {}

/**
* Notifies the stream if the manifest is in sequence mode or not.
Expand Down
3 changes: 2 additions & 1 deletion lib/media/media_source_engine.js
Original file line number Diff line number Diff line change
Expand Up @@ -830,7 +830,8 @@ shaka.media.MediaSourceEngine = class {
await this.textEngine_.appendBuffer(
data,
reference ? reference.startTime : null,
reference ? reference.endTime : null);
reference ? reference.endTime : null,
reference ? reference.getUris()[0] : null);
return;
}

Expand Down
4 changes: 2 additions & 2 deletions lib/player.js
Original file line number Diff line number Diff line change
Expand Up @@ -4706,7 +4706,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
vttOffset: 0,
};
const data = shaka.util.BufferUtils.toUint8(buffer);
const cues = TextParser.parseMedia(data, time);
const cues = TextParser.parseMedia(data, time, uri);

const references = [];
for (const cue of cues) {
Expand Down Expand Up @@ -4970,7 +4970,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
vttOffset: 0,
};
const data = shaka.util.BufferUtils.toUint8(buffer);
const cues = obj.parseMedia(data, time);
const cues = obj.parseMedia(data, time, /* uri= */ null);
return shaka.text.WebVttGenerator.convert(cues, adCuePoints);
}
throw new shaka.util.Error(
Expand Down
4 changes: 2 additions & 2 deletions lib/text/mp4_ttml_parser.js
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ shaka.text.Mp4TtmlParser = class {
* @override
* @export
*/
parseMedia(data, time) {
parseMedia(data, time, uri) {
const Mp4Parser = shaka.util.Mp4Parser;

let sawMDAT = false;
Expand All @@ -86,7 +86,7 @@ shaka.text.Mp4TtmlParser = class {
sawMDAT = true;
// Join this to any previous payload, in case the mp4 has multiple
// mdats.
payload = payload.concat(this.parser_.parseMedia(data, time));
payload = payload.concat(this.parser_.parseMedia(data, time, uri));
}));
parser.parse(data, /* partialOkay= */ false);

Expand Down
4 changes: 2 additions & 2 deletions lib/text/srt_text_parser.js
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ shaka.text.SrtTextParser = class {
* @override
* @export
*/
parseMedia(data, time) {
parseMedia(data, time, uri) {
const SrtTextParser = shaka.text.SrtTextParser;
const BufferUtils = shaka.util.BufferUtils;
const StringUtils = shaka.util.StringUtils;
Expand All @@ -67,7 +67,7 @@ shaka.text.SrtTextParser = class {

const newData = BufferUtils.toUint8(StringUtils.toUTF8(vvtText));

return this.parser_.parseMedia(newData, time);
return this.parser_.parseMedia(newData, time, uri);
}

/**
Expand Down
5 changes: 3 additions & 2 deletions lib/text/text_engine.js
Original file line number Diff line number Diff line change
Expand Up @@ -166,9 +166,10 @@ shaka.text.TextEngine = class {
* @param {BufferSource} buffer
* @param {?number} startTime relative to the start of the presentation
* @param {?number} endTime relative to the start of the presentation
* @param {?string=} uri
* @return {!Promise}
*/
async appendBuffer(buffer, startTime, endTime) {
async appendBuffer(buffer, startTime, endTime, uri) {
goog.asserts.assert(
this.parser_, 'The parser should already be initialized');

Expand Down Expand Up @@ -198,7 +199,7 @@ shaka.text.TextEngine = class {

// Parse the buffer and add the new cues.
const allCues = this.parser_.parseMedia(
shaka.util.BufferUtils.toUint8(buffer), time);
shaka.util.BufferUtils.toUint8(buffer), time, uri);
const cuesToAppend = allCues.filter((cue) => {
return cue.startTime >= this.appendWindowStart_ &&
cue.startTime < this.appendWindowEnd_;
Expand Down
33 changes: 28 additions & 5 deletions lib/text/ttml_text_parser.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
goog.provide('shaka.text.TtmlTextParser');

goog.require('goog.asserts');
goog.require('goog.Uri');
goog.require('shaka.log');
goog.require('shaka.text.Cue');
goog.require('shaka.text.CueRegion');
Expand Down Expand Up @@ -50,7 +51,7 @@ shaka.text.TtmlTextParser = class {
* @override
* @export
*/
parseMedia(data, time) {
parseMedia(data, time, uri) {
const TtmlTextParser = shaka.text.TtmlTextParser;
const XmlUtils = shaka.util.XmlUtils;
const ttpNs = TtmlTextParser.parameterNs_;
Expand Down Expand Up @@ -147,7 +148,7 @@ shaka.text.TtmlTextParser = class {
body, time, rateInfo, metadataElements, styles,
regionElements, cueRegions, whitespaceTrim,
cellResolutionInfo, /* parentCueElement= */ null,
/* isContent= */ false);
/* isContent= */ false, uri);
if (cue) {
// According to the TTML spec, backgrounds default to transparent.
// So default the background of the top-level element to transparent.
Expand Down Expand Up @@ -175,12 +176,14 @@ shaka.text.TtmlTextParser = class {
* @param {?{columns: number, rows: number}} cellResolution
* @param {?Element} parentCueElement
* @param {boolean} isContent
* @param {?(string|undefined)} uri
* @return {shaka.text.Cue}
* @private
*/
static parseCue_(
cueNode, timeContext, rateInfo, metadataElements, styles, regionElements,
cueRegions, whitespaceTrim, cellResolution, parentCueElement, isContent) {
cueRegions, whitespaceTrim, cellResolution, parentCueElement, isContent,
uri) {
/** @type {Element} */
let cueElement;
/** @type {Element} */
Expand Down Expand Up @@ -222,7 +225,21 @@ shaka.text.TtmlTextParser = class {
}
}

if (cueNode.nodeName == 'p' || imageElement) {
let imageUri = null;
const backgroundImage = shaka.util.XmlUtils.getAttributeNSList(
cueElement,
shaka.text.TtmlTextParser.smpteNsList_,
'backgroundImage');
if (uri && backgroundImage && !backgroundImage.startsWith('#')) {
const baseUri = new goog.Uri(uri);
const relativeUri = new goog.Uri(backgroundImage);
const newUri = baseUri.resolve(relativeUri).toString();
if (newUri) {
imageUri = newUri;
}
}

if (cueNode.nodeName == 'p' || imageElement || imageUri) {
isContent = true;
}

Expand Down Expand Up @@ -255,6 +272,7 @@ shaka.text.TtmlTextParser = class {
cellResolution,
cueElement,
isContent,
uri,
);

// This node may or may not generate a nested cue.
Expand Down Expand Up @@ -388,6 +406,7 @@ shaka.text.TtmlTextParser = class {
cueElement,
regionElementForStyle,
imageElement,
imageUri,
styles,
/** isNested= */ parentIsContent, // "nested in a <div>" doesn't count.
/** isLeaf= */ (nestedCues.length == 0));
Expand Down Expand Up @@ -490,13 +509,15 @@ shaka.text.TtmlTextParser = class {
* @param {!Element} cueElement
* @param {Element} region
* @param {Element} imageElement
* @param {?string} imageUri
* @param {!Array.<!Element>} styles
* @param {boolean} isNested
* @param {boolean} isLeaf
* @private
*/
static addStyle_(
cue, cueElement, region, imageElement, styles, isNested, isLeaf) {
cue, cueElement, region, imageElement, imageUri, styles,
isNested, isLeaf) {
const TtmlTextParser = shaka.text.TtmlTextParser;
const Cue = shaka.text.Cue;

Expand Down Expand Up @@ -664,6 +685,8 @@ shaka.text.TtmlTextParser = class {
backgroundImageData) {
cue.backgroundImage = 'data:image/png;base64,' + backgroundImageData;
}
} else if (imageUri) {
cue.backgroundImage = imageUri;
}

const textOutline = TtmlTextParser.getStyleAttribute_(
Expand Down
2 changes: 1 addition & 1 deletion test/media/media_source_engine_unit.js
Original file line number Diff line number Diff line change
Expand Up @@ -631,7 +631,7 @@ describe('MediaSourceEngine', () => {
ContentType.TEXT, data, reference, fakeStream,
/* hasClosedCaptions= */ false);
expect(mockTextEngine.appendBuffer).toHaveBeenCalledWith(
data, 0, 10);
data, 0, 10, 'foo://bar');
});

it('appends transmuxed data', async () => {
Expand Down
8 changes: 4 additions & 4 deletions test/text/mp4_ttml_parser_unit.js
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ describe('Mp4TtmlParser', () => {
parser.parseInit(ttmlInitSegment);
const time =
{periodStart: 0, segmentStart: 0, segmentEnd: 60, vttOffset: 0};
const ret = parser.parseMedia(ttmlSegmentMultipleMDAT, time);
const ret = parser.parseMedia(ttmlSegmentMultipleMDAT, time, null);
// Bodies.
expect(ret.length).toBe(2);
// Divs.
Expand All @@ -63,10 +63,10 @@ describe('Mp4TtmlParser', () => {
const parser = new shaka.text.Mp4TtmlParser();
parser.parseInit(ttmlInitSegment);

const ret1 = parser.parseMedia(ttmlSegment, time1);
const ret1 = parser.parseMedia(ttmlSegment, time1, null);
expect(ret1.length).toBeGreaterThan(0);

const ret2 = parser.parseMedia(ttmlSegment, time2);
const ret2 = parser.parseMedia(ttmlSegment, time2, null);
expect(ret2.length).toBeGreaterThan(0);

expect(ret2[0].startTime).toBe(ret1[0].startTime + 7);
Expand Down Expand Up @@ -164,7 +164,7 @@ describe('Mp4TtmlParser', () => {
parser.parseInit(ttmlInitSegment);
const time =
{periodStart: 0, segmentStart: 0, segmentEnd: 60, vttOffset: 0};
const result = parser.parseMedia(ttmlSegment, time);
const result = parser.parseMedia(ttmlSegment, time, null);
shaka.test.TtmlUtils.verifyHelper(
cues, result, {startTime: 23, endTime: 53.5});
});
Expand Down
2 changes: 1 addition & 1 deletion test/text/srt_text_parser_unit.js
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ describe('SrtTextParser', () => {
const data = BufferUtils.toUint8(StringUtils.toUTF8(text));

const parser = new shaka.text.SrtTextParser();
const result = parser.parseMedia(data, time);
const result = parser.parseMedia(data, time, null);

const expected = cues.map((cue) => {
if (cue.nestedCues) {
Expand Down
6 changes: 6 additions & 0 deletions test/text/text_engine_unit.js
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ describe('TextEngine', () => {
expect(mockParseMedia).toHaveBeenCalledOnceMoreWith([
dummyData,
{periodStart: 0, segmentStart: 0, segmentEnd: 3, vttOffset: 0},
undefined,
]);

expect(mockDisplayer.appendSpy).toHaveBeenCalledOnceMoreWith([
Expand All @@ -120,6 +121,7 @@ describe('TextEngine', () => {
expect(mockParseMedia).toHaveBeenCalledOnceMoreWith([
dummyData,
{periodStart: 0, segmentStart: 3, segmentEnd: 5, vttOffset: 0},
undefined,
]);

expect(mockDisplayer.appendSpy).toHaveBeenCalledOnceMoreWith([
Expand Down Expand Up @@ -272,6 +274,7 @@ describe('TextEngine', () => {
expect(mockParseMedia).toHaveBeenCalledOnceMoreWith([
dummyData,
{periodStart: 0, segmentStart: 0, segmentEnd: 3, vttOffset: 0},
undefined,
]);
expect(mockDisplayer.appendSpy).toHaveBeenCalledOnceMoreWith([
[
Expand All @@ -286,6 +289,7 @@ describe('TextEngine', () => {
expect(mockParseMedia).toHaveBeenCalledOnceMoreWith([
dummyData,
{periodStart: 4, segmentStart: 4, segmentEnd: 7, vttOffset: 4},
undefined,
]);
expect(mockDisplayer.appendSpy).toHaveBeenCalledOnceMoreWith([
[
Expand Down Expand Up @@ -316,6 +320,7 @@ describe('TextEngine', () => {
expect(mockParseMedia).toHaveBeenCalledOnceMoreWith([
dummyData,
{periodStart: 0, segmentStart: 0, segmentEnd: 3, vttOffset: 0},
undefined,
]);

textEngine.setTimestampOffset(8);
Expand All @@ -325,6 +330,7 @@ describe('TextEngine', () => {
expect(mockParseMedia).toHaveBeenCalledOnceMoreWith([
dummyData,
{periodStart: 8, segmentStart: 4, segmentEnd: 7, vttOffset: 4},
undefined,
]);
});
});
Expand Down
35 changes: 33 additions & 2 deletions test/text/ttml_text_parser_unit.js
Original file line number Diff line number Diff line change
Expand Up @@ -1263,6 +1263,35 @@ describe('TtmlTextParser', () => {
});
});

it('supports smpte:backgroundImage attribute with url', () => {
verifyHelper(
[
{
startTime: 62.05,
endTime: 3723.2,
payload: '',
},
],
'<tt ' +
'xmlns:ttm="http://www.w3.org/ns/ttml#metadata" ' +
'xmlns:smpte="http://www.smpte-ra.org/schemas/2052-1/2010/smpte-tt">' +
'<metadata>' +
'<smpte:image imageType="PNG" encoding="Base64" xml:id="img_0">' +
'base64EncodedImage</smpte:image>' +
'</metadata>' +
'<body><div smpte:backgroundImage="img_0.png">' +
'<p begin="01:02.05" end="01:02:03.200"></p>' +
'</div></body></tt>',
{periodStart: 0, segmentStart: 60, segmentEnd: 3730, vttOffset: 0},
{startTime: 62.05, endTime: 3723.2},
{
startTime: 62.05,
endTime: 3723.2,
backgroundImage: 'foo://bar/img_0.png',
isContainer: false,
});
});

it('supports smpte:backgroundImage attribute in div element', () => {
verifyHelper(
[],
Expand Down Expand Up @@ -2112,7 +2141,8 @@ describe('TtmlTextParser', () => {
function verifyHelper(cues, text, time, bodyProperties, divProperties) {
const data =
shaka.util.BufferUtils.toUint8(shaka.util.StringUtils.toUTF8(text));
const result = new shaka.text.TtmlTextParser().parseMedia(data, time);
const result = new shaka.text.TtmlTextParser()
.parseMedia(data, time, 'foo://bar');
shaka.test.TtmlUtils.verifyHelper(
cues, result, bodyProperties, divProperties);
}
Expand All @@ -2139,7 +2169,8 @@ describe('TtmlTextParser', () => {
expect(() => {
new shaka.text.TtmlTextParser().parseMedia(
shaka.util.BufferUtils.toUint8(data),
{periodStart: 0, segmentStart: 0, segmentEnd: 10, vttOffset: 0});
{periodStart: 0, segmentStart: 0, segmentEnd: 10, vttOffset: 0},
'foo://bar');
}).toThrow(error);
}
});

0 comments on commit fa93d53

Please sign in to comment.