From 37c68da45cc0c7ecb185dab632725eb764e5c570 Mon Sep 17 00:00:00 2001 From: Matt Juggins <32675386+mattjuggins@users.noreply.github.com> Date: Mon, 22 Jan 2024 14:23:36 +0000 Subject: [PATCH] DVB Font Downloads (#4338) * Initial dvb font download work * Continued dvb font download functionality * Seperate DVBFontUtils and TTML prefixing * Essential Property descriptors working and refactor DVB font handling * Ensure fonts are removed and handled with essential property descriptors, tidy up * Comment tidy up * Clean up and function commenting * Team feedback corrections * Non dvbFonts unit tests * Add DVBFonts unit tests * Update types * Update BSD-3 header in accordance with contributor guide * Remove font prefixing functionality * Remove references to prefixing fontFamilies in TTMLParser * Move events from CoreEvents to MediaPlayerEvents * Address PR comments on DashAdapter, DashManifestModel and TextSourceBuffer * Address PR comments for DVBFonts * Missing semicolon removal * Add DVB Font Download test streams to reference player * Use camelCase for DescriptorType dvb extensions * Handle disabled tracks correctly in the reference player controlbar * Fix controlbar text track and native track matching * Fix issue with disabled track cues * Add DVB font download test streams to functional tests * Update imscJS version and remove now unneeded fix --- contrib/akamai/controlbar/ControlBar.js | 81 +++++- index.d.ts | 70 ++++- package-lock.json | 47 ++-- package.json | 2 +- .../dash-if-reference-player/app/sources.json | 24 ++ src/dash/DashAdapter.js | 3 + src/dash/constants/DashConstants.js | 4 + src/dash/models/DashManifestModel.js | 34 ++- src/dash/vo/DescriptorType.js | 12 + src/dash/vo/MediaInfo.js | 2 + src/streaming/MediaPlayer.js | 1 + src/streaming/MediaPlayerEvents.js | 18 ++ src/streaming/constants/Constants.js | 4 + .../controllers/PlaybackController.js | 3 +- src/streaming/text/DVBFonts.js | 252 ++++++++++++++++++ src/streaming/text/TextController.js | 78 +++++- src/streaming/text/TextSourceBuffer.js | 7 +- src/streaming/text/TextTracks.js | 15 +- src/streaming/utils/Capabilities.js | 2 +- src/streaming/utils/TTMLParser.js | 5 +- test/functional-karma/config/subtitle.js | 32 +++ test/unit/dash.DashAdapter.js | 50 ++++ test/unit/dash.constants.DashConstants.js | 4 + test/unit/dash.models.DashManifestModel.js | 70 ++++- test/unit/dash.vo.DescriptorType.js | 44 +++ test/unit/data/subtitles/ttmlSample.ttml | 29 ++ test/unit/streaming.constants.Constants.js | 6 + test/unit/streaming.text.DVBFonts.js | 199 ++++++++++++++ test/unit/streaming.text.TextController.js | 6 +- .../streaming.utils.CapabilitiesFilter.js | 17 +- test/unit/streaming.utils.TTMLParser.js | 24 +- 31 files changed, 1064 insertions(+), 81 deletions(-) create mode 100644 src/streaming/text/DVBFonts.js create mode 100644 test/unit/dash.vo.DescriptorType.js create mode 100644 test/unit/data/subtitles/ttmlSample.ttml create mode 100644 test/unit/streaming.text.DVBFonts.js diff --git a/contrib/akamai/controlbar/ControlBar.js b/contrib/akamai/controlbar/ControlBar.js index b926b35e25..7abf429eff 100644 --- a/contrib/akamai/controlbar/ControlBar.js +++ b/contrib/akamai/controlbar/ControlBar.js @@ -65,6 +65,7 @@ var ControlBar = function (dashjsMediaPlayer, displayUTCTimeCodes) { seekbarPlay, seekbarBuffer, muteBtn, + nativeTextTracks, volumebar, fullscreenBtn, timeDisplay, @@ -595,6 +596,34 @@ var ControlBar = function (dashjsMediaPlayer, displayUTCTimeCodes) { } }; + // Match up the current dashjs text tracks against native video element tracks by ensuring they have matching properties + var _matchTrackWithNativeTrack = function(track, nativeTrack) { + let label = track.id !== undefined ? track.id.toString() : track.lang; + + return !!( + (track.kind === nativeTrack.kind) && + (track.lang === nativeTrack.language) && + (track.isTTML === nativeTrack.isTTML) && + (track.isEmbedded === nativeTrack.isEmbedded) && + (label === nativeTrack.label) + ); + } + + // Compare track information against native video element tracks to get the current track mode + var _getNativeVideoTrackMode = function (track) { + const nativeTracks = video.textTracks; + let trackMode; + for (let i = 0; i < nativeTracks.length; i++) { + const nativeTrack = nativeTracks[i]; + if (_matchTrackWithNativeTrack(track, nativeTrack)) { + trackMode = nativeTrack.mode; + break; + } + }; + + return (trackMode === undefined) ? 'showing' : trackMode; + }; + var createCaptionSwitchMenu = function (streamId) { // Subtitles/Captions Menu //XXX we need to add two layers for captions & subtitles if present. var activeStreamInfo = player.getActiveStream().getStreamInfo(); @@ -607,15 +636,24 @@ var ControlBar = function (dashjsMediaPlayer, displayUTCTimeCodes) { var tracks = textTrackList[streamId] || []; var contentFunc = function (element, index) { if (isNaN(index)) { - return 'OFF'; + return { + mode: 'showing', + text: 'OFF' + }; } var label = getLabelForLocale(element.labels); + var trackText; if (label) { - return label + ' : ' + element.type; + trackText = label + ' : ' + element.type; + } else { + trackText = element.lang + ' : ' + element.kind; } - return element.lang + ' : ' + element.kind; + return { + mode: _getNativeVideoTrackMode(element), + text: trackText + } }; captionMenu = createMenu({ menuType: 'caption', arr: tracks }, contentFunc); @@ -630,6 +668,11 @@ var ControlBar = function (dashjsMediaPlayer, displayUTCTimeCodes) { }; + var _onTracksChanged = function () { + var activeStreamInfo = player.getActiveStream().getStreamInfo(); + createCaptionSwitchMenu(activeStreamInfo.id); + } + var _onTracksAdded = function (e) { // Subtitles/Captions Menu //XXX we need to add two layers for captions & subtitles if present. if (!textTrackList[e.streamId]) { @@ -637,6 +680,10 @@ var ControlBar = function (dashjsMediaPlayer, displayUTCTimeCodes) { } textTrackList[e.streamId] = textTrackList[e.streamId].concat(e.tracks); + + nativeTextTracks = video.textTracks; + nativeTextTracks.addEventListener('change', _onTracksChanged); + createCaptionSwitchMenu(e.streamId); }; @@ -783,8 +830,15 @@ var ControlBar = function (dashjsMediaPlayer, displayUTCTimeCodes) { item.mediaType = mediaType; item.name = name; item.selected = false; - item.textContent = arr[i]; - + if (isObject(arr[i])) { + // text tracks need extra properties + item.mode = arr[i].mode; + item.textContent = arr[i].text; + } else { + // Other tracks will just have their text + item.textContent = arr[i]; + } + item.onmouseover = function (/*e*/) { if (this.selected !== true) { this.classList.add('menu-item-over'); @@ -802,7 +856,13 @@ var ControlBar = function (dashjsMediaPlayer, displayUTCTimeCodes) { el = menu.querySelector('.' + mediaType + '-menu-content'); } - el.appendChild(item); + if (mediaType === 'caption') { + if (item.mode !== 'disabled') { + el.appendChild(item); + } + } else { + el.appendChild(item); + } } return menu; @@ -952,6 +1012,14 @@ var ControlBar = function (dashjsMediaPlayer, displayUTCTimeCodes) { return !!navigator.userAgent.match(/Trident.*rv[ :]*11\./); }; + //************************************************************************************ + //Utilities + //************************************************************************************ + + var isObject = function (obj) { + return typeof obj === 'object' && !Array.isArray(obj) && obj !== null; + } + //************************************************************************************ // PUBLIC API //************************************************************************************ @@ -1035,6 +1103,7 @@ var ControlBar = function (dashjsMediaPlayer, displayUTCTimeCodes) { } if (menuHandlersList.caption) { captionBtn.removeEventListener('click', menuHandlersList.caption); + nativeTextTracks.removeEventListener('change', _onTracksChanged); } if (captionMenu) { this.removeMenu(captionMenu, captionBtn); diff --git a/index.d.ts b/index.d.ts index 0361673b11..3638092773 100644 --- a/index.d.ts +++ b/index.d.ts @@ -216,6 +216,10 @@ declare namespace dashjs { getSelectionPriority(realAdaptation: object): number; + getEssentialPropertiesForAdaptation(adaptation: object): object; + + getEssentialPropertiesAsArrayForAdaptation(adaptation: object): any[]; + getEssentialPropertiesForRepresentation(realRepresentation: object): {schemeIdUri: string, value: string} getRepresentationFor(index: number, adaptation: object): object; @@ -254,9 +258,19 @@ declare namespace dashjs { getServiceDescriptions(manifest: object): serviceDescriptions; - getSupplementalProperties(adaptation: object): object; getSegmentAlignment(adaptation: object): boolean; + getSubSegmentAlignment(adaptation: object): boolean; + + getSupplementalPropertiesForAdaptation(adaptation: object): object; + + getSupplementalPropertiesAsArrayForAdaptation(adaptation: object): any[]; + + getSupplementalPropertiesForRepresentation(representation: Representation): object; + + getSupplementalPropertiesAsArrayForRepresentation(representation: Representation): any[]; + + setConfig(config: object): void; } export interface PatchManifestModel { @@ -472,6 +486,9 @@ declare namespace dashjs { isEmbedded: any | null; selectionPriority: number; supplementalProperties: object; + supplementalPropertiesAsArray: any[]; + essentialProperties: object; + essentialPropertiesAsArray: any[]; segmentAlignment: boolean; subSegmentAlignment: boolean; } @@ -582,6 +599,9 @@ declare namespace dashjs { schemeIdUri: string; value: string; id: string; + dvbUrl?: string; + dvbMimeType?: string; + dvbFontFamily?: string; } export class ContentSteeringResponse { @@ -1512,7 +1532,10 @@ declare namespace dashjs { CAN_PLAY_THROUGH: 'canPlayThrough'; CAPTION_RENDERED: 'captionRendered'; CAPTION_CONTAINER_RESIZE: 'captionContainerResize'; - CONFORMANCE_VIOLATION: 'conformanceViolation' + CONFORMANCE_VIOLATION: 'conformanceViolation'; + DVB_FONT_DOWNLOAD_ADDED: 'dvbFontDownloadAdded'; + DVB_FONT_DOWNLOAD_COMPLETE: 'dvbFontDownloadComplete'; + DVB_FONT_DOWNLOAD_FAILED: 'dvbFontDownloadFailed'; DYNAMIC_TO_STATIC: 'dynamicToStatic'; ERROR: 'error'; EVENT_MODE_ON_RECEIVE: 'eventModeOnReceive'; @@ -1722,6 +1745,20 @@ declare namespace dashjs { type: MediaPlayerEvents['CAPTION_CONTAINER_RESIZE']; } + export interface dvbFontDownloadAdded extends Event { + type: MediaPlayerEvents['DVB_FONT_DOWNLOAD_ADDED']; + font: FontInfo; + } + + export interface dvbFontDownloadComplete extends Event { + type: MediaPlayerEvents['DVB_FONT_DOWNLOAD_COMPLETE']; + font: FontInfo; + } + + export interface dvbFontDownloadFailed extends Event { + type: MediaPlayerEvents['DVB_FONT_DOWNLOAD_FAILED']; + font: FontInfo; + } export interface DynamicToStaticEvent extends Event { type: MediaPlayerEvents['DYNAMIC_TO_STATIC']; } @@ -3504,7 +3541,32 @@ declare namespace dashjs { * Streaming - Text **/ - export type TextTrackType = 'subtitles' | 'caption' | 'descriptions' | 'chapters' | 'metadata'; + export type TextTrackType = 'subtitles' | 'caption' | 'descriptions' | 'chapters' | 'metadata'; + + export type FontDownloadStatus = 'unloaded' | 'loaded' | 'error'; + + export interface FontInfo { + fontFamily: string; + url: string; + mimeType: string; + trackId: number; + streamId: string; + isEssential: boolean; + status: FontDownloadStatus; + fontFace: FontFace; + } + + export interface DVBFonts { + addFontsFromTracks(tracks: TextTrackInfo, streamId: string): void; + + downloadFonts(): void; + + getFonts(): FontInfo[]; + + getFontsForTrackId(trackId: number): FontInfo[]; + + reset(): void; + } export interface EmbeddedTextHtmlRender { createHTMLCaptionsFromScreen(videoElement: HTMLVideoElement, startTime: number, endTime: number, captionScreen: any): any[]; @@ -3644,8 +3706,6 @@ declare namespace dashjs { deleteCuesFromTrackIdx(trackIdx: number, start: number, end: number): void; deleteAllTextTracks(): void; - - deleteTextTrack(idx: number): void; } /** diff --git a/package-lock.json b/package-lock.json index 672640a50c..ac47150c56 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,7 @@ "es6-promise": "^4.2.8", "fast-deep-equal": "2.0.1", "html-entities": "^1.2.1", - "imsc": "^1.1.4", + "imsc": "^1.1.5", "localforage": "^1.7.1", "path-browserify": "^1.0.1", "ua-parser-js": "^1.0.37" @@ -9047,9 +9047,9 @@ } }, "node_modules/imsc": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/imsc/-/imsc-1.1.4.tgz", - "integrity": "sha512-s/WbXG6IbeW6X/8sBJWcQD22mwRcnpI55b8Kr3sbcONUaeMLkpHle/PE1xcMN9HJrMc5idrCwNV7wtZ8EBsFnw==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/imsc/-/imsc-1.1.5.tgz", + "integrity": "sha512-V8je+CGkcvGhgl2C1GlhqFFiUOIEdwXbXLiu1Fcubvvbo+g9inauqT3l0pNYXGoLPBj3jxtZz9t+wCopMkwadQ==", "dependencies": { "sax": "1.2.1" } @@ -21114,22 +21114,19 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/@webpack-cli/configtest/-/configtest-2.0.1.tgz", "integrity": "sha512-njsdJXJSiS2iNbQVS0eT8A/KPnmyH4pv1APj2K0d1wrZcBLw+yppxOy4CGqa0OxDJkzfL/XELDhD8rocnIwB5A==", - "dev": true, - "requires": {} + "dev": true }, "@webpack-cli/info": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@webpack-cli/info/-/info-2.0.1.tgz", "integrity": "sha512-fE1UEWTwsAxRhrJNikE7v4EotYflkEhBL7EbajfkPlf6E37/2QshOy/D48Mw8G5XMFlQtS6YV42vtbG9zBpIQA==", - "dev": true, - "requires": {} + "dev": true }, "@webpack-cli/serve": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@webpack-cli/serve/-/serve-2.0.1.tgz", "integrity": "sha512-0G7tNyS+yW8TdgHwZKlDWYXFA6OJQnoLCQvYKkQP0Q2X205PSQ6RNUj0M+1OB/9gRQaUZ/ccYfaxd0nhaWKfjw==", - "dev": true, - "requires": {} + "dev": true }, "@xtuc/ieee754": { "version": "1.2.0", @@ -21163,15 +21160,13 @@ "version": "1.8.0", "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz", "integrity": "sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw==", - "dev": true, - "requires": {} + "dev": true }, "acorn-jsx": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.1.tgz", "integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==", - "dev": true, - "requires": {} + "dev": true }, "ajv": { "version": "6.12.6", @@ -21232,8 +21227,7 @@ "version": "3.5.2", "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", - "dev": true, - "requires": {} + "dev": true }, "align-text": { "version": "0.1.4", @@ -22273,8 +22267,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/chai-spies/-/chai-spies-1.0.0.tgz", "integrity": "sha512-elF2ZUczBsFoP07qCfMO/zeggs8pqCf3fZGyK5+2X4AndS8jycZYID91ztD9oQ7d/0tnS963dPkd0frQEThDsg==", - "dev": true, - "requires": {} + "dev": true }, "chalk": { "version": "1.1.3", @@ -23201,8 +23194,7 @@ "version": "8.2.3", "resolved": "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz", "integrity": "sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA==", - "dev": true, - "requires": {} + "dev": true } } }, @@ -24773,9 +24765,9 @@ } }, "imsc": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/imsc/-/imsc-1.1.4.tgz", - "integrity": "sha512-s/WbXG6IbeW6X/8sBJWcQD22mwRcnpI55b8Kr3sbcONUaeMLkpHle/PE1xcMN9HJrMc5idrCwNV7wtZ8EBsFnw==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/imsc/-/imsc-1.1.5.tgz", + "integrity": "sha512-V8je+CGkcvGhgl2C1GlhqFFiUOIEdwXbXLiu1Fcubvvbo+g9inauqT3l0pNYXGoLPBj3jxtZz9t+wCopMkwadQ==", "requires": { "sax": "1.2.1" } @@ -25835,8 +25827,7 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/karma-chai/-/karma-chai-0.1.0.tgz", "integrity": "sha1-vuWtQEAFF4Ea40u5RfdikJEIt5o=", - "dev": true, - "requires": {} + "dev": true }, "karma-chrome-launcher": { "version": "3.1.1", @@ -26468,8 +26459,7 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-5.3.0.tgz", "integrity": "sha512-/V1MnLL/rgJ3jkMWo84UR+K+jF1cxNG1a+KwqeXqTIJ+jtA8aWSHuigx8lTzauiIjBDbwF3NcWQMotd0Dm39jA==", - "dev": true, - "requires": {} + "dev": true }, "marked": { "version": "2.0.7", @@ -30934,8 +30924,7 @@ "version": "8.11.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz", "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==", - "dev": true, - "requires": {} + "dev": true }, "xmlbuilder": { "version": "10.1.1", diff --git a/package.json b/package.json index f3e155783b..f222259dab 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ "es6-promise": "^4.2.8", "fast-deep-equal": "2.0.1", "html-entities": "^1.2.1", - "imsc": "^1.1.4", + "imsc": "^1.1.5", "localforage": "^1.7.1", "path-browserify": "^1.0.1", "ua-parser-js": "^1.0.37" diff --git a/samples/dash-if-reference-player/app/sources.json b/samples/dash-if-reference-player/app/sources.json index 9b68c06e52..6e3a9f477e 100644 --- a/samples/dash-if-reference-player/app/sources.json +++ b/samples/dash-if-reference-player/app/sources.json @@ -424,6 +424,30 @@ "moreInfo": "https://rdmedia.bbc.co.uk/testcard/simulcast/", "provider": "bbc" }, + { + "name": "On-demand Testcard - WOFF Font Download signalled with supplemental property descriptor", + "url": "https://rdmedia.bbc.co.uk/testcard/vod/manifests/avc-ctv-stereo-en-sfdt-woff.mpd", + "moreInfo": "https://rdmedia.bbc.co.uk/testcard/vod/#feature-tests-font-downloads-for-subtitles", + "provider": "bbc" + }, + { + "name": "On-demand Testcard - WOFF Font Download signalled with essential property descriptor", + "url": "https://rdmedia.bbc.co.uk/testcard/vod/manifests/avc-ctv-stereo-en-efdt-woff.mpd", + "moreInfo": "https://rdmedia.bbc.co.uk/testcard/vod/#feature-tests-font-downloads-for-subtitles", + "provider": "bbc" + }, + { + "name": "Live Testcard - WOFF Font Download signalled with supplemental property descriptor", + "url": "https://rdmedia.bbc.co.uk/testcard/simulcast/manifests/avc-ctv-stereo-en-sfdt-woff.mpd", + "moreInfo": "https://rdmedia.bbc.co.uk/testcard/simulcast/#feature-tests-font-downloads-for-subtitles", + "provider": "bbc" + }, + { + "name": "Live Testcard - WOFF Font Download signalled with essential property descriptor", + "url": "https://rdmedia.bbc.co.uk/testcard/simulcast/manifests/avc-ctv-stereo-en-efdt-woff.mpd", + "moreInfo": "https://rdmedia.bbc.co.uk/testcard/simulcast/#feature-tests-font-downloads-for-subtitles", + "provider": "bbc" + }, { "url": "https://dash.akamaized.net/dash264/CTA/imsc1/IT1-20171027_dash.mpd", "name": "IMSC1 Text Subtitles via sidecar file", diff --git a/src/dash/DashAdapter.js b/src/dash/DashAdapter.js index 70ed7265ec..17821d5299 100644 --- a/src/dash/DashAdapter.js +++ b/src/dash/DashAdapter.js @@ -1121,6 +1121,9 @@ function DashAdapter() { } } + mediaInfo.essentialProperties = dashManifestModel.getEssentialPropertiesForAdaptation(realAdaptation); + mediaInfo.essentialPropertiesAsArray = dashManifestModel.getEssentialPropertiesAsArrayForAdaptation(realAdaptation); + mediaInfo.isFragmented = dashManifestModel.getIsFragmented(realAdaptation); mediaInfo.isEmbedded = false; diff --git a/src/dash/constants/DashConstants.js b/src/dash/constants/DashConstants.js index 00d97d3b0c..12479251a5 100644 --- a/src/dash/constants/DashConstants.js +++ b/src/dash/constants/DashConstants.js @@ -88,6 +88,7 @@ class DashConstants { this.AUDIO_CHANNEL_CONFIGURATION = 'AudioChannelConfiguration'; this.CONTENT_PROTECTION = 'ContentProtection'; this.ESSENTIAL_PROPERTY = 'EssentialProperty'; + this.ESSENTIAL_PROPERTY_ASARRAY = 'EssentialProperty_asArray'; this.SUPPLEMENTAL_PROPERTY = 'SupplementalProperty'; this.SUPPLEMENTAL_PROPERTY_ASARRAY = 'SupplementalProperty_asArray'; this.INBAND_EVENT_STREAM = 'InbandEventStream'; @@ -133,6 +134,9 @@ class DashConstants { this.CENC_DEFAULT_KID = 'cenc:default_KID'; this.DVB_PRIORITY = 'dvb:priority'; this.DVB_WEIGHT = 'dvb:weight'; + this.DVB_URL = 'dvb:url'; + this.DVB_MIMETYPE = 'dvb:mimeType'; + this.DVB_FONTFAMILY = 'dvb:fontFamily'; this.SUGGESTED_PRESENTATION_DELAY = 'suggestedPresentationDelay'; this.SERVICE_DESCRIPTION = 'ServiceDescription'; this.SERVICE_DESCRIPTION_SCOPE = 'Scope'; diff --git a/src/dash/models/DashManifestModel.js b/src/dash/models/DashManifestModel.js index 4c43373226..271c74e335 100644 --- a/src/dash/models/DashManifestModel.js +++ b/src/dash/models/DashManifestModel.js @@ -1295,8 +1295,9 @@ function DashManifestModel() { if (adaptation && adaptation.hasOwnProperty(DashConstants.SUPPLEMENTAL_PROPERTY_ASARRAY)) { for (const sp of adaptation.SupplementalProperty_asArray) { - if (sp.hasOwnProperty(Constants.SCHEME_ID_URI) && sp.hasOwnProperty(DashConstants.VALUE)) { - supplementalProperties[sp[Constants.SCHEME_ID_URI]] = sp[DashConstants.VALUE]; + if (sp.hasOwnProperty(Constants.SCHEME_ID_URI)) { + // N.B this will only work where there is a single SupplementalProperty descriptor with this SchemeIdUri + supplementalProperties[sp[Constants.SCHEME_ID_URI]] = {...sp}; } } } @@ -1316,8 +1317,9 @@ function DashManifestModel() { if (representation && representation.hasOwnProperty(DashConstants.SUPPLEMENTAL_PROPERTY_ASARRAY)) { for (const sp of representation.SupplementalProperty_asArray) { - if (sp.hasOwnProperty(Constants.SCHEME_ID_URI) && sp.hasOwnProperty(DashConstants.VALUE)) { - supplementalProperties[sp[Constants.SCHEME_ID_URI]] = sp[DashConstants.VALUE]; + if (sp.hasOwnProperty(Constants.SCHEME_ID_URI)) { + // N.B this will only work where there is a single SupplementalProperty descriptor with this SchemeIdUri + supplementalProperties[sp[Constants.SCHEME_ID_URI]] = {...sp}; } } } @@ -1332,6 +1334,28 @@ function DashManifestModel() { }); } + function getEssentialPropertiesForAdaptation(adaptation) { + const essentialProperties = {}; + + if (adaptation && adaptation.hasOwnProperty(DashConstants.ESSENTIAL_PROPERTY_ASARRAY)) { + for (const ep of adaptation.EssentialProperty_asArray) { + if (ep.hasOwnProperty(Constants.SCHEME_ID_URI)) { + // N.B this will only work where there is a single EssentialProperty descriptor with this SchemeIdUri + essentialProperties[ep[Constants.SCHEME_ID_URI]] = {...ep}; + } + } + } + return essentialProperties; + } + + function getEssentialPropertiesAsArrayForAdaptation(adaptation) { + if (!adaptation || !adaptation.hasOwnProperty(DashConstants.ESSENTIAL_PROPERTY_ASARRAY) || !adaptation.EssentialProperty_asArray.length) return []; + return adaptation.EssentialProperty_asArray.map( ep => { + const s = new DescriptorType(); + return s.init(ep); + }); + } + function setConfig(config) { if (!config) return; @@ -1382,6 +1406,8 @@ function DashManifestModel() { getRegularPeriods, getMpd, getEventsForPeriod, + getEssentialPropertiesForAdaptation, + getEssentialPropertiesAsArrayForAdaptation, getEssentialPropertiesForRepresentation, getEventStreamForAdaptationSet, getEventStreamForRepresentation, diff --git a/src/dash/vo/DescriptorType.js b/src/dash/vo/DescriptorType.js index be8a1460a7..987a5a82ad 100644 --- a/src/dash/vo/DescriptorType.js +++ b/src/dash/vo/DescriptorType.js @@ -32,6 +32,8 @@ * @class * @ignore */ +import DashConstants from '../constants/DashConstants' + class DescriptorType { constructor() { this.schemeIdUri = null; @@ -44,6 +46,16 @@ class DescriptorType { this.schemeIdUri = data.schemeIdUri ? data.schemeIdUri : null; this.value = data.value ? data.value : null; this.id = data.id ? data.id : null; + // Only add the DVB extensions if they exist + if (data[DashConstants.DVB_URL]) { + this.dvbUrl = data[DashConstants.DVB_URL] + } + if (data[DashConstants.DVB_MIMETYPE]) { + this.dvbMimeType = data[DashConstants.DVB_MIMETYPE] + } + if (data[DashConstants.DVB_FONTFAMILY]) { + this.dvbFontFamily = data[DashConstants.DVB_FONTFAMILY] + } } return this; } diff --git a/src/dash/vo/MediaInfo.js b/src/dash/vo/MediaInfo.js index eef44d33cb..0a99bbef59 100644 --- a/src/dash/vo/MediaInfo.js +++ b/src/dash/vo/MediaInfo.js @@ -60,6 +60,8 @@ class MediaInfo { this.selectionPriority = 1; this.supplementalProperties = {}; this.supplementalPropertiesAsArray = []; + this.essentialProperties = {}; + this.essentialPropertiesAsArray = []; this.segmentAlignment = false; this.subSegmentAlignment = false; } diff --git a/src/streaming/MediaPlayer.js b/src/streaming/MediaPlayer.js index ebe94fcddf..ad4b7454fd 100644 --- a/src/streaming/MediaPlayer.js +++ b/src/streaming/MediaPlayer.js @@ -2178,6 +2178,7 @@ function MediaPlayer() { manifestModel, adapter, mediaController, + baseURLController, videoModel, settings }); diff --git a/src/streaming/MediaPlayerEvents.js b/src/streaming/MediaPlayerEvents.js index e5c6f0be8b..0e9592ca15 100644 --- a/src/streaming/MediaPlayerEvents.js +++ b/src/streaming/MediaPlayerEvents.js @@ -81,6 +81,24 @@ class MediaPlayerEvents extends EventsBase { */ this.BUFFER_LEVEL_UPDATED = 'bufferLevelUpdated'; + /** + * Triggered when a font signalled by a DVB Font Download has been added to the document FontFaceSet interface. + * @event MediaPlayerEvents#DVB_FONT_DOWNLOAD_ADDED + */ + this.DVB_FONT_DOWNLOAD_ADDED = 'dvbFontDownloadAdded'; + + /** + * Triggered when a font signalled by a DVB Font Download has successfully downloaded and the FontFace can be used. + * @event MediaPlayerEvents#DVB_FONT_DOWNLOAD_COMPLETE + */ + this.DVB_FONT_DOWNLOAD_COMPLETE = 'dvbFontDownloadComplete'; + + /** + * Triggered when a font signalled by a DVB Font Download could not be successfully downloaded, so the FontFace will not be used. + * @event MediaPlayerEvents#DVB_FONT_DOWNLOAD_FAILED + */ + this.DVB_FONT_DOWNLOAD_FAILED = 'dvbFontDownloadFailed'; + /** * Triggered when a dynamic stream changed to static (transition phase between Live and On-Demand). * @event MediaPlayerEvents#DYNAMIC_TO_STATIC diff --git a/src/streaming/constants/Constants.js b/src/streaming/constants/Constants.js index 15d58e5528..5695a8bf47 100644 --- a/src/streaming/constants/Constants.js +++ b/src/streaming/constants/Constants.js @@ -278,6 +278,7 @@ class Constants { this.INITIALIZE = 'initialize'; this.TEXT_SHOWING = 'showing'; this.TEXT_HIDDEN = 'hidden'; + this.TEXT_DISABLED = 'disabled'; this.CC1 = 'CC1'; this.CC3 = 'CC3'; this.UTF8 = 'utf-8'; @@ -285,10 +286,13 @@ class Constants { this.START_TIME = 'starttime'; this.SERVICE_DESCRIPTION_DVB_LL_SCHEME = 'urn:dvb:dash:lowlatency:scope:2019'; this.SUPPLEMENTAL_PROPERTY_DVB_LL_SCHEME = 'urn:dvb:dash:lowlatency:critical:2019'; + this.FONT_DOWNLOAD_DVB_SCHEME = 'urn:dvb:dash:fontdownload:2014'; this.XML = 'XML'; this.ARRAY_BUFFER = 'ArrayBuffer'; this.DVB_REPORTING_URL = 'dvb:reportingUrl'; this.DVB_PROBABILITY = 'dvb:probability'; + this.OFF_MIMETYPE = 'application/font-sfnt'; + this.WOFF_MIMETYPE = 'application/font-woff'; this.VIDEO_ELEMENT_READY_STATES = { HAVE_NOTHING: 0, HAVE_METADATA: 1, diff --git a/src/streaming/controllers/PlaybackController.js b/src/streaming/controllers/PlaybackController.js index 68656ef6b0..d0ce36d9c7 100644 --- a/src/streaming/controllers/PlaybackController.js +++ b/src/streaming/controllers/PlaybackController.js @@ -836,7 +836,8 @@ function PlaybackController() { function _checkEnableLowLatency(mediaInfo) { if (mediaInfo && mediaInfo.supplementalProperties && - mediaInfo.supplementalProperties[Constants.SUPPLEMENTAL_PROPERTY_DVB_LL_SCHEME] === 'true') { + mediaInfo.supplementalProperties[Constants.SUPPLEMENTAL_PROPERTY_DVB_LL_SCHEME] && + mediaInfo.supplementalProperties[Constants.SUPPLEMENTAL_PROPERTY_DVB_LL_SCHEME].value === 'true') { logger.debug('Low Latency critical SupplementalProperty set: Enabling low Latency'); lowLatencyModeEnabled = true; } diff --git a/src/streaming/text/DVBFonts.js b/src/streaming/text/DVBFonts.js new file mode 100644 index 0000000000..ba12ce771d --- /dev/null +++ b/src/streaming/text/DVBFonts.js @@ -0,0 +1,252 @@ +/** + * The copyright in this software is being made available under the BSD License, + * included below. This software may be subject to other third party and contributor + * rights, including patent rights, and no such rights are granted under this license. + * + * Copyright (c) 2024, BBC. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. + * * Neither the name of BBC nor the names of its + * contributors may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS AS IS AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +import Constants from '../constants/Constants'; +import FactoryMaker from '../../core/FactoryMaker'; +import URLUtils from '../utils/URLUtils'; +import EventBus from '../../core/EventBus'; +import MediaPlayerEvents from '../MediaPlayerEvents'; +import Debug from '../../core/Debug'; + +function DVBFonts(config) { + + let context = this.context; + const eventBus = EventBus(context).getInstance(); + const urlUtils = URLUtils(context).getInstance(); + const adapter = config.adapter; + const baseURLController = config.baseURLController; + + const FONT_DOWNLOAD_STATUS = { + ERROR: 'error', + LOADED: 'loaded', + UNLOADED: 'unloaded' + }; + + let instance, + logger, + dvbFontList; + + function setup() { + logger = Debug(context).getInstance().getLogger(instance); + resetInitialSettings(); + } + + /** + * Add any dvb fonts from a single track to the dvbFontList + * @param {object} track - A text track + * @param {string} streamId - Id of current stream + * @private + */ + function _addFontFromTrack(track, streamId) { + let asBaseUrl; + let isEssential = false; + let dvbFontProps; + + // If there is a baseurl in the manifest resolve against a representation inside the current adaptation set + if (baseURLController.resolve()) { + const reps = adapter.getVoRepresentations(track); + asBaseUrl = baseURLController.resolve(reps[0].path).url + } + + const essentialTags = track.essentialPropertiesAsArray.filter(tag => + (tag.schemeIdUri && tag.schemeIdUri === Constants.FONT_DOWNLOAD_DVB_SCHEME) + ); + const supplementalTags = track.supplementalPropertiesAsArray.filter(tag => + (tag.schemeIdUri && tag.schemeIdUri === Constants.FONT_DOWNLOAD_DVB_SCHEME) + ); + + // When it comes to the property descriptors it's Essential OR Supplementary, with Essential taking preference + if (essentialTags.length > 0) { + isEssential = true; + dvbFontProps = essentialTags; + } else { + dvbFontProps = supplementalTags; + } + + dvbFontProps.forEach(attrs => { + if (_hasMandatoryDvbFontAttributes(attrs)) { + let resolvedUrl = _resolveFontUrl(attrs.dvbUrl, asBaseUrl); + dvbFontList.push({ + fontFamily: attrs.dvbFontFamily, + url: resolvedUrl, + mimeType: attrs.dvbMimeType, + trackId: track.id, + streamId, + isEssential, + status: FONT_DOWNLOAD_STATUS.UNLOADED, + fontFace: new FontFace( + attrs.dvbFontFamily, + `url(${resolvedUrl})`, + { display: 'swap' } + ) + }); + } + }); + } + + /** + * Clean up dvb font downloads + * @private + */ + function _cleanUpDvbCustomFonts() { + for (const font of dvbFontList) { + let deleted = document.fonts.delete(font.fontFace); + logger.debug(`Removal of fontFamily: ${font.fontFamily} was ${deleted ? 'successful' : 'unsuccessful'}`); + } + } + + /** + * Check the attributes of a supplemental or essential property descriptor to establish if + * it has the mandatory values for a dvb font download + * @param {object} attrs - property descriptor attributes + * @returns {boolean} true if mandatory attributes present + * @private + */ + function _hasMandatoryDvbFontAttributes(attrs) { + return !!((attrs.value && attrs.value === '1') && + (attrs.dvbUrl && attrs.dvbUrl.length > 0) && + (attrs.dvbFontFamily && attrs.dvbFontFamily.length > 0) && + (attrs.dvbMimeType && (attrs.dvbMimeType === Constants.OFF_MIMETYPE || attrs.dvbMimeType === Constants.WOFF_MIMETYPE))); + } + + /** + * Resolves a given font download URL. + * @param {string} fontUrl - URL as in the 'dvb:url' property + * @param {string} baseUrl - BaseURL for Adaptation Set + * @returns {string} resolved URL + * @private + */ + function _resolveFontUrl(fontUrl, baseUrl) { + if (urlUtils.isPathAbsolute(fontUrl)) { + return fontUrl; + } else if (urlUtils.isRelative(fontUrl)) { + if (baseUrl) { + return urlUtils.resolve(fontUrl, baseUrl); + } else { + return urlUtils.resolve(fontUrl); + } + } else { + return fontUrl; + } + } + + /** + * Updates the status of a given dvb font relative to whether it is loaded in the browser + * or if the download has failed + * @param {number} index - Index of font in dvbFontList + * @param {string} newStatus - Status value to update. Property of FONT_DOWNLOAD_STATUS + * @private + */ + function _updateFontStatus(index, newStatus) { + const font = dvbFontList[index]; + dvbFontList[index] = {...font, status: newStatus}; + } + + /** + * Adds all fonts to the dvb font list from all tracks + * @param {array} tracks - All text tracks + * @param {string} streamId - Id of the stream + */ + function addFontsFromTracks(tracks, streamId) { + if (tracks && Array.isArray(tracks) && streamId) { + for (let i = 0; i < tracks.length; i++) { + let track = tracks[i]; + _addFontFromTrack(track, streamId); + }; + } + } + + /** + * Initiate the download of a dvb custom font. + * The browser will neatly handle duplicate fonts + */ + function downloadFonts() { + for (let i = 0; i < dvbFontList.length; i++) { + let font = dvbFontList[i]; + + document.fonts.add(font.fontFace); + eventBus.trigger(MediaPlayerEvents.DVB_FONT_DOWNLOAD_ADDED, font); + + font.fontFace.load().then( + () => { + _updateFontStatus(i, FONT_DOWNLOAD_STATUS.LOADED); + eventBus.trigger(MediaPlayerEvents.DVB_FONT_DOWNLOAD_COMPLETE, font); + }, + (err) => { + _updateFontStatus(i, FONT_DOWNLOAD_STATUS.ERROR); + logger.debug('Font download error: ', err); + eventBus.trigger(MediaPlayerEvents.DVB_FONT_DOWNLOAD_FAILED, font); + } + ); + } + } + + /** + * Returns current list of all known DVB Fonts + * @returns {array} dvbFontList + */ + function getFonts() { + return dvbFontList; + } + + /** + * Returns dvbFonts relative to a track given a trackId + * @param {number} - TrackId + * @returns {array} filtered DVBFontList + */ + function getFontsForTrackId(trackId) { + return dvbFontList.filter(font => + (font.trackId && font.trackId === trackId) + ); + } + + function resetInitialSettings() { + dvbFontList = []; + } + + /** Reset DVBFonts instance */ + function reset() { + _cleanUpDvbCustomFonts(); + resetInitialSettings(); + } + + instance = { + addFontsFromTracks, + downloadFonts, + getFonts, + getFontsForTrackId, + reset + }; + setup(); + return instance; +} + +DVBFonts.__dashjs_factory_name = 'DVBFonts'; +export default FactoryMaker.getClassFactory(DVBFonts); diff --git a/src/streaming/text/TextController.js b/src/streaming/text/TextController.js index dff1fc8655..4c32ae61a6 100644 --- a/src/streaming/text/TextController.js +++ b/src/streaming/text/TextController.js @@ -35,10 +35,12 @@ import TextTracks from './TextTracks'; import VTTParser from '../utils/VTTParser'; import VttCustomRenderingParser from '../utils/VttCustomRenderingParser'; import TTMLParser from '../utils/TTMLParser'; +import Debug from '../../core/Debug'; import EventBus from '../../core/EventBus'; import Events from '../../core/events/Events'; import MediaPlayerEvents from '../../streaming/MediaPlayerEvents'; import {checkParameterType} from '../utils/SupervisorTools'; +import DVBFonts from './DVBFonts'; function TextController(config) { @@ -48,6 +50,7 @@ function TextController(config) { const errHandler = config.errHandler; const manifestModel = config.manifestModel; const mediaController = config.mediaController; + const baseURLController = config.baseURLController; const videoModel = config.videoModel; const settings = config.settings; @@ -62,7 +65,9 @@ function TextController(config) { allTracksAreDisabled, forceTextStreaming, textTracksAdded, - disableTextBeforeTextTracksAdded; + disableTextBeforeTextTracksAdded, + dvbFonts, + logger; function setup() { forceTextStreaming = false; @@ -73,12 +78,19 @@ function TextController(config) { vttCustomRenderingParser = VttCustomRenderingParser(context).getInstance(); ttmlParser = TTMLParser(context).getInstance(); eventBus = EventBus(context).getInstance(); + logger = Debug(context).getInstance().getLogger(instance); resetInitialSettings(); } function initialize() { + dvbFonts = DVBFonts(context).create({ + adapter, + baseURLController, + }); eventBus.on(Events.TEXT_TRACKS_QUEUE_INITIALIZED, _onTextTracksAdded, instance); + eventBus.on(Events.DVB_FONT_DOWNLOAD_FAILED, _onFontDownloadFailure, instance); + eventBus.on(Events.DVB_FONT_DOWNLOAD_COMPLETE, _onFontDownloadSuccess, instance); if (settings.get().streaming.text.webvtt.customRenderingEnabled) { eventBus.on(Events.PLAYBACK_TIME_UPDATED, _onPlaybackTimeUpdated, instance); eventBus.on(Events.PLAYBACK_SEEKING, _onPlaybackSeeking, instance); @@ -98,6 +110,7 @@ function TextController(config) { const textSourceBuffer = TextSourceBuffer(context).create({ errHandler, adapter, + dvbFonts, manifestModel, mediaController, videoModel, @@ -166,6 +179,36 @@ function TextController(config) { textSourceBuffers[streamId].addEmbeddedTrack(mediaInfo); } + /** + * Event that is triggered if a font download of a font described in an essential property descriptor + * tag fails. + * @param {FontInfo} font - font information + * @private + */ + function _onFontDownloadFailure(font) { + logger.error(`Could not download ${font.isEssential ? 'an essential' : 'a'} font - fontFamily: ${font.fontFamily}, url: ${font.url}`); + if (font.isEssential) { + let idx = textTracks[font.streamId].getTrackIdxForId(font.trackId); + textTracks[font.streamId].setModeForTrackIdx(idx, Constants.TEXT_DISABLED); + } + }; + + /** + * Set a font with an essential property + * @private + */ + function _onFontDownloadSuccess(font) { + logger.debug(`Successfully downloaded ${font.isEssential ? 'an essential' : 'a'} font - fontFamily: ${font.fontFamily}, url: ${font.url}`); + if (font.isEssential) { + let idx = textTracks[font.streamId].getTrackIdxForId(font.trackId); + if (idx === textTracks[font.streamId].getCurrentTrackIdx()) { + textTracks[font.streamId].setModeForTrackIdx(idx, Constants.TEXT_SHOWING); + } else { + textTracks[font.streamId].setModeForTrackIdx(idx, Constants.TEXT_HIDDEN); + } + } + } + function _onTextTracksAdded(e) { let tracks = e.tracks; let index = e.index; @@ -208,6 +251,18 @@ function TextController(config) { }); textTracksAdded = true; + + dvbFonts.addFontsFromTracks(tracks, streamId); + + // Initially disable any tracks with essential property font downloads + dvbFonts.getFonts().forEach(font => { + if (font.isEssential) { + let idx = textTracks[font.streamId].getTrackIdxForId(font.trackId); + textTracks[font.streamId].setModeForTrackIdx(idx, Constants.TEXT_DISABLED); + } + }); + + dvbFonts.downloadFonts(); } function _onPlaybackTimeUpdated(e) { @@ -289,14 +344,24 @@ function TextController(config) { return; } - textTracks[streamId].disableManualTracks(); - textTracks[streamId].setModeForTrackIdx(oldTrackIdx, Constants.TEXT_HIDDEN); + let currentTrackInfo = textTracks[streamId].getCurrentTrackInfo(); + let currentNativeTrackInfo = (currentTrackInfo) ? videoModel.getTextTrack(currentTrackInfo.kind, currentTrackInfo.id, currentTrackInfo.lang, currentTrackInfo.isTTML, currentTrackInfo.isEmbedded) : null; + + // Don't change disabled tracks - dvb font download for essential property failed or not complete + if (currentNativeTrackInfo && (currentNativeTrackInfo.mode !== Constants.TEXT_DISABLED)) { + textTracks[streamId].setModeForTrackIdx(oldTrackIdx, Constants.TEXT_HIDDEN); + } + textTracks[streamId].setCurrentTrackIdx(idx); - textTracks[streamId].setModeForTrackIdx(idx, Constants.TEXT_SHOWING); - let currentTrackInfo = textTracks[streamId].getCurrentTrackInfo(); + currentTrackInfo = textTracks[streamId].getCurrentTrackInfo(); + currentNativeTrackInfo = (currentTrackInfo) ? videoModel.getTextTrack(currentTrackInfo.kind, currentTrackInfo.id, currentTrackInfo.lang, currentTrackInfo.isTTML, currentTrackInfo.isEmbedded) : null; + + if (currentTrackInfo && (currentTrackInfo.mode !== Constants.TEXT_DISABLED)) { + textTracks[streamId].setModeForTrackIdx(idx, Constants.TEXT_SHOWING); + } if (currentTrackInfo && currentTrackInfo.isFragmented && !currentTrackInfo.isEmbedded) { _setFragmentedTextTrack(streamId, currentTrackInfo, oldTrackIdx); @@ -375,8 +440,11 @@ function TextController(config) { } function reset() { + dvbFonts.reset(); resetInitialSettings(); eventBus.off(Events.TEXT_TRACKS_QUEUE_INITIALIZED, _onTextTracksAdded, instance); + eventBus.off(Events.DVB_FONT_DOWNLOAD_FAILED, _onFontDownloadFailure, instance); + eventBus.off(Events.DVB_FONT_DOWNLOAD_COMPLETE, _onFontDownloadSuccess, instance); if (settings.get().streaming.text.webvtt.customRenderingEnabled) { eventBus.off(Events.PLAYBACK_TIME_UPDATED, _onPlaybackTimeUpdated, instance); eventBus.off(Events.PLAYBACK_SEEKING, _onPlaybackSeeking, instance) diff --git a/src/streaming/text/TextSourceBuffer.js b/src/streaming/text/TextSourceBuffer.js index 67492a3077..8402d74ebd 100644 --- a/src/streaming/text/TextSourceBuffer.js +++ b/src/streaming/text/TextSourceBuffer.js @@ -290,6 +290,7 @@ function TextSourceBuffer(config) { if (sampleList.length > 0) { firstFragmentedSubtitleStart = sampleList[0].cts - chunk.start * timescale; } + if (codecType.search(Constants.STPP) >= 0) { _appendFragmentedSttp(bytes, sampleList, codecType); } else { @@ -325,11 +326,11 @@ function TextSourceBuffer(config) { try { const manifest = manifestModel.getValue(); - // Only used for Miscrosoft Smooth Streaming support - caption time is relative to sample time. In this case, we apply an offset. + // Only used for Microsoft Smooth Streaming support - caption time is relative to sample time. In this case, we apply an offset. const offsetTime = manifest.ttmlTimeIsRelative ? sampleStart / timescale : 0; - - const result = parser.parse(ccContent, offsetTime, sampleStart / timescale, (sampleStart + sample.duration) / timescale, images); + const result = parser.parse(ccContent, offsetTime, (sampleStart / timescale), ((sampleStart + sample.duration) / timescale), images); textTracks.addCaptions(currFragmentedTrackIdx, timestampOffset, result); + } catch (e) { fragmentModel.removeExecutedRequestsBeforeTime(); this.remove(); diff --git a/src/streaming/text/TextTracks.js b/src/streaming/text/TextTracks.js index 3aa74ac23f..a87db21017 100644 --- a/src/streaming/text/TextTracks.js +++ b/src/streaming/text/TextTracks.js @@ -527,10 +527,15 @@ function TextTracks(config) { if (prevCue.isd) { prevCue.onexit = function () { }; } - track.addCue(cue); + // If cues are added when the track is disabled they can still persist in memory + if (track.mode !== Constants.TEXT_DISABLED) { + track.addCue(cue); + } } } else { - track.addCue(cue); + if (track.mode !== Constants.TEXT_DISABLED) { + track.addCue(cue); + } } } } @@ -894,11 +899,6 @@ function TextTracks(config) { clearCaptionContainer.call(this); } - function deleteTextTrack(idx) { - videoModel.removeChild(nativeTrackElementArr[idx]); - nativeTrackElementArr.splice(idx, 1); - } - /* Set native cue style to transparent background to avoid it being displayed. */ function setNativeCueStyle() { let styleElement = document.getElementById('native-cue-style'); @@ -969,7 +969,6 @@ function TextTracks(config) { setModeForTrackIdx, deleteCuesFromTrackIdx, deleteAllTextTracks, - deleteTextTrack, manualCueProcessing, disableManualTracks }; diff --git a/src/streaming/utils/Capabilities.js b/src/streaming/utils/Capabilities.js index 289041033f..c1c96cfb35 100644 --- a/src/streaming/utils/Capabilities.js +++ b/src/streaming/utils/Capabilities.js @@ -198,7 +198,7 @@ function Capabilities() { */ function supportsEssentialProperty(ep) { try { - return THUMBNAILS_SCHEME_ID_URIS.indexOf(ep.schemeIdUri) !== -1; + return (THUMBNAILS_SCHEME_ID_URIS.indexOf(ep.schemeIdUri) !== -1) || (Constants.FONT_DOWNLOAD_DVB_SCHEME === ep.schemeIdUri); } catch (e) { return true; } diff --git a/src/streaming/utils/TTMLParser.js b/src/streaming/utils/TTMLParser.js index 94ea3bd988..45b83ce952 100644 --- a/src/streaming/utils/TTMLParser.js +++ b/src/streaming/utils/TTMLParser.js @@ -66,7 +66,8 @@ function TTMLParser() { * @param {number} offsetTime - offset time to apply to cue time * @param {integer} startTimeSegment - startTime for the current segment * @param {integer} endTimeSegment - endTime for the current segment - * @param {Array} images - images array referenced by subs MP4 box + * @param {array} images - images array referenced by subs MP4 box + * @returns {array} captionArray */ function parse(data, offsetTime, startTimeSegment, endTimeSegment, images) { let errorMsg = ''; @@ -127,7 +128,7 @@ function TTMLParser() { eventBus.trigger(Events.TTML_TO_PARSE, content); - const imsc1doc = fromXML(content.data, function (msg) { + let imsc1doc = fromXML(content.data, function (msg) { errorMsg = msg; }, metadataHandler); diff --git a/test/functional-karma/config/subtitle.js b/test/functional-karma/config/subtitle.js index 23dad6860f..2d52095a49 100644 --- a/test/functional-karma/config/subtitle.js +++ b/test/functional-karma/config/subtitle.js @@ -89,6 +89,38 @@ export default [ type: 'vod', testcases: [Constants.TESTCASES.SIMPLE.SWITCH_TEXT], }, + { + 'name': 'On-demand Testcard - WOFF Font Download signalled with supplemental property descriptor', + 'url': 'https://rdmedia.bbc.co.uk/testcard/vod/manifests/avc-ctv-stereo-en-sfdt-woff.mpd', + 'moreInfo': 'https://rdmedia.bbc.co.uk/testcard/vod/#feature-tests-font-downloads-for-subtitles', + 'provider': 'bbc', + type: 'live', + testcases: [Constants.TESTCASES.SIMPLE.SWITCH_TEXT], + }, + { + 'name': 'On-demand Testcard - WOFF Font Download signalled with essential property descriptor', + 'url': 'https://rdmedia.bbc.co.uk/testcard/vod/manifests/avc-ctv-stereo-en-efdt-woff.mpd', + 'moreInfo': 'https://rdmedia.bbc.co.uk/testcard/vod/#feature-tests-font-downloads-for-subtitles', + 'provider': 'bbc', + type: 'live', + testcases: [Constants.TESTCASES.SIMPLE.SWITCH_TEXT], + }, + { + 'name': 'Live Testcard - WOFF Font Download signalled with supplemental property descriptor', + 'url': 'https://rdmedia.bbc.co.uk/testcard/simulcast/manifests/avc-ctv-stereo-en-sfdt-woff.mpd', + 'moreInfo': 'https://rdmedia.bbc.co.uk/testcard/simulcast/#feature-tests-font-downloads-for-subtitles', + 'provider': 'bbc', + type: 'live', + testcases: [Constants.TESTCASES.SIMPLE.SWITCH_TEXT], + }, + { + 'name': 'Live Testcard - WOFF Font Download signalled with essential property descriptor', + 'url': 'https://rdmedia.bbc.co.uk/testcard/simulcast/manifests/avc-ctv-stereo-en-efdt-woff.mpd', + 'moreInfo': 'https://rdmedia.bbc.co.uk/testcard/simulcast/#feature-tests-font-downloads-for-subtitles', + 'provider': 'bbc', + type: 'live', + testcases: [Constants.TESTCASES.SIMPLE.SWITCH_TEXT], + }, { 'url': 'https://dash.akamaized.net/dash264/CTA/imsc1/IT1-20171027_dash.mpd', 'name': 'IMSC1 Text Subtitles via sidecar file', diff --git a/test/unit/dash.DashAdapter.js b/test/unit/dash.DashAdapter.js index fa0a0305f0..fc354ff583 100644 --- a/test/unit/dash.DashAdapter.js +++ b/test/unit/dash.DashAdapter.js @@ -130,6 +130,20 @@ const manifest_with_supplemental_properties_on_only_one_repr = { }] }] }; +const manifest_with_essential_properties = { + loadedTime: new Date(), + mediaPresentationDuration: 10, + Period_asArray: [{ + AdaptationSet_asArray: [{ + id: 0, + mimeType: Constants.VIDEO, + EssentialProperty_asArray: [ + { schemeIdUri: 'test:scheme', value: 'value1'}, + { schemeIdUri: 'test:scheme', value: 'value2' } + ] + }] + }] +}; const manifest_with_audioChanCfg = { loadedTime: new Date(), mediaPresentationDuration: 10, @@ -603,6 +617,7 @@ describe('DashAdapter', function () { }); describe('mediainfo populated from manifest', function () { + it('supplemental properties should be empty if not defined', function () { const mediaInfoArray = dashAdapter.getAllMediaInfoForType({ id: 'defaultId_0', @@ -674,6 +689,41 @@ describe('DashAdapter', function () { expect(mediaInfoArray[0].supplementalPropertiesAsArray.length).equals(0); }); + it('essential properties should be empty if not defined', function () { + const mediaInfoArray = dashAdapter.getAllMediaInfoForType({ + id: 'defaultId_0', + index: 0 + }, Constants.VIDEO, manifest_without_supplemental_properties); + // works for no essential properties too + + expect(mediaInfoArray).to.be.instanceOf(Array); + expect(mediaInfoArray.length).equals(1); + + expect(mediaInfoArray[0].essentialProperties).not.to.be.null; + expect(Object.keys(mediaInfoArray[0].essentialProperties).length).equals(0); + + expect(mediaInfoArray[0].essentialPropertiesAsArray).to.be.instanceOf(Array); + expect(mediaInfoArray[0].essentialPropertiesAsArray.length).equals(0); + }); + + it('essential properties should be filled if correctly defined', function () { + const mediaInfoArray = dashAdapter.getAllMediaInfoForType({ + id: 'defaultId_0', + index: 0 + }, Constants.VIDEO, manifest_with_essential_properties); + + expect(mediaInfoArray).to.be.instanceOf(Array); + expect(mediaInfoArray.length).equals(1); + + expect(mediaInfoArray[0].codec).to.be.null; + + expect(mediaInfoArray[0].essentialProperties).not.to.be.null; + expect(Object.keys(mediaInfoArray[0].essentialProperties).length).equals(1); + + expect(mediaInfoArray[0].essentialPropertiesAsArray).to.be.instanceOf(Array); + expect(mediaInfoArray[0].essentialPropertiesAsArray.length).equals(2); + }); + it('audio channel config should be filled', function () { const mediaInfoArray = dashAdapter.getAllMediaInfoForType({ id: 'defaultId_0', diff --git a/test/unit/dash.constants.DashConstants.js b/test/unit/dash.constants.DashConstants.js index 7313c29558..a34e2169e5 100644 --- a/test/unit/dash.constants.DashConstants.js +++ b/test/unit/dash.constants.DashConstants.js @@ -56,6 +56,7 @@ describe('DashConstants', function () { expect(DashConstants.AUDIO_CHANNEL_CONFIGURATION).to.equal('AudioChannelConfiguration'); expect(DashConstants.CONTENT_PROTECTION).to.equal('ContentProtection'); expect(DashConstants.ESSENTIAL_PROPERTY).to.equal('EssentialProperty'); + expect(DashConstants.ESSENTIAL_PROPERTY_ASARRAY).to.equal('EssentialProperty_asArray'); expect(DashConstants.SUPPLEMENTAL_PROPERTY).to.equal('SupplementalProperty'); expect(DashConstants.SUPPLEMENTAL_PROPERTY_ASARRAY).to.equal('SupplementalProperty_asArray'); expect(DashConstants.INBAND_EVENT_STREAM).to.equal('InbandEventStream'); @@ -96,6 +97,9 @@ describe('DashConstants', function () { expect(DashConstants.CENC_DEFAULT_KID).to.equal('cenc:default_KID'); expect(DashConstants.DVB_PRIORITY).to.equal('dvb:priority'); expect(DashConstants.DVB_WEIGHT).to.equal('dvb:weight'); + expect(DashConstants.DVB_URL).to.equal('dvb:url'); + expect(DashConstants.DVB_MIMETYPE).to.equal('dvb:mimeType'); + expect(DashConstants.DVB_FONTFAMILY).to.equal('dvb:fontFamily'); expect(DashConstants.SUGGESTED_PRESENTATION_DELAY).to.equal('suggestedPresentationDelay'); expect(DashConstants.WALL_CLOCK_TIME).to.equal('wallClockTime'); expect(DashConstants.PRESENTATION_TIME).to.equal('presentationTime'); diff --git a/test/unit/dash.models.DashManifestModel.js b/test/unit/dash.models.DashManifestModel.js index 1e01e6c64c..564fbdb374 100644 --- a/test/unit/dash.models.DashManifestModel.js +++ b/test/unit/dash.models.DashManifestModel.js @@ -141,11 +141,12 @@ describe('DashManifestModel', function () { expect(rolesArray).to.be.empty; }); - it('should return an empty array when getSupplementalPropertiesForAdaptation', () => { - const suppPropArray = dashManifestModel.getSupplementalPropertiesForAdaptation(); + // Handling Supplemental Property Descriptors + it('should return an empty object when getSupplementalPropertiesForAdaptation', () => { + const suppProps = dashManifestModel.getSupplementalPropertiesForAdaptation(); - expect(suppPropArray).to.be.instanceOf(Object); - expect(suppPropArray).to.be.empty; + expect(suppProps).to.be.instanceOf(Object); + expect(suppProps).to.be.empty; }); it('should return an empty array when getSupplementalPropertiesAsArrayForAdaptation', () => { @@ -157,22 +158,29 @@ describe('DashManifestModel', function () { it('should return correct array of DescriptorType when getSupplementalPropertiesAsArrayForAdaptation is called', () => { const suppPropArray = dashManifestModel.getSupplementalPropertiesAsArrayForAdaptation({ - SupplementalProperty_asArray: [{schemeIdUri: 'test.scheme', value: 'testVal'},{schemeIdUri: 'test.scheme', value: 'test2Val'}] + SupplementalProperty_asArray: [ + {schemeIdUri: 'test.scheme0', value: 'testVal'}, + {schemeIdUri: 'test.scheme1', value: 'test2Val', 'dvb:mimeType': 'extVal'}, // e.g. dvb extensions + {schemeIdUri: 'test.scheme2'} // value not always required + ] }); expect(suppPropArray).to.be.instanceOf(Array); expect(suppPropArray[0]).to.be.instanceOf(DescriptorType); - expect(suppPropArray[0].schemeIdUri).equals('test.scheme'); + expect(suppPropArray[0].schemeIdUri).equals('test.scheme0'); expect(suppPropArray[0].value).equals('testVal'); - expect(suppPropArray[1].schemeIdUri).equals('test.scheme'); + expect(suppPropArray[1].schemeIdUri).equals('test.scheme1'); expect(suppPropArray[1].value).equals('test2Val'); + expect(suppPropArray[1].dvbMimeType).equals('extVal'); + expect(suppPropArray[2].schemeIdUri).equals('test.scheme2'); + expect(suppPropArray[2].value).to.be.null; }); - it('should return an empty array when getSupplementalPropertiesForRepresentation', () => { - const suppPropArray = dashManifestModel.getSupplementalPropertiesForRepresentation(); + it('should return an empty object when getSupplementalPropertiesForRepresentation', () => { + const suppProps = dashManifestModel.getSupplementalPropertiesForRepresentation(); - expect(suppPropArray).to.be.instanceOf(Object); - expect(suppPropArray).to.be.empty; + expect(suppProps).to.be.instanceOf(Object); + expect(suppProps).to.be.empty; }); it('should return an empty array when getSupplementalPropertiesAsArrayForRepresentation', () => { @@ -193,6 +201,46 @@ describe('DashManifestModel', function () { expect(suppPropArray[0].value).equals('testVal'); }); + // Handling Essential Property Descriptors + it('should return an empty object when getEssentialPropertiesForAdaptation', () => { + const essProps = dashManifestModel.getEssentialPropertiesForAdaptation(); + + expect(essProps).to.be.instanceOf(Object); + expect(essProps).to.be.empty; + }); + + it('should return an empty array when getEssentialPropertiesAsArrayForAdaptation', () => { + const essPropArray = dashManifestModel.getEssentialPropertiesAsArrayForAdaptation(); + + expect(essPropArray).to.be.instanceOf(Array); + expect(essPropArray).to.be.empty; + }); + + it('should return correct array of DescriptorType when getEssentialPropertiesAsArrayForAdaptation is called', () => { + const essPropArray = dashManifestModel.getEssentialPropertiesAsArrayForAdaptation({ + EssentialProperty_asArray: [ + {schemeIdUri: 'test.scheme0', value: 'testVal'}, + {schemeIdUri: 'test.scheme1', value: 'test2Val', 'dvb:mimeType': 'extVal'}, // e.g. dvb extensions + {schemeIdUri: 'test.scheme2'} // value not always required + ] + }); + + expect(essPropArray).to.be.instanceOf(Array); + expect(essPropArray[0]).to.be.instanceOf(DescriptorType); + expect(essPropArray[0].schemeIdUri).equals('test.scheme0'); + expect(essPropArray[0].value).equals('testVal'); + expect(essPropArray[1].schemeIdUri).equals('test.scheme1'); + expect(essPropArray[1].value).equals('test2Val'); + expect(essPropArray[1].dvbMimeType).equals('extVal'); + expect(essPropArray[2].schemeIdUri).equals('test.scheme2'); + expect(essPropArray[2].value).to.be.null; + }); + + // Works differently to the supplementalProperties counterpart + it('should return null when getEssentialPropertiesForRepresentation', () => { + expect(dashManifestModel.getEssentialPropertiesForRepresentation()).to.be.null; + }); + it('should return null when getAdaptationForId is called and id, manifest and periodIndex are undefined', () => { const adaptation = dashManifestModel.getAdaptationForId(undefined, undefined, undefined); diff --git a/test/unit/dash.vo.DescriptorType.js b/test/unit/dash.vo.DescriptorType.js new file mode 100644 index 0000000000..ed536b547d --- /dev/null +++ b/test/unit/dash.vo.DescriptorType.js @@ -0,0 +1,44 @@ +import DescriptorType from '../../src/dash/vo/DescriptorType'; + +const expect = require('chai').expect; + +describe('DescriptorType', () => { + + it('should be constructed with null values', () => { + const dt = new DescriptorType(); + expect(dt).to.deep.equal({ + schemeIdUri: null, + value: null, + id: null + }); + }); + + it('should initialise with correct base values', () => { + const dt = new DescriptorType(); + dt.init({ + schemeIdUri: 'testScheme', + value: '1', + }); + expect(dt).to.deep.equal({ + schemeIdUri: 'testScheme', + value: '1', + id: null + }); + }); + + it('should initialise with known dvb extensions if present', () => { + const dt = new DescriptorType(); + dt.init({ + schemeIdUri: 'testScheme', + value: '1', + 'dvb:url': 'testUrl' + }); + expect(dt).to.deep.equal({ + schemeIdUri: 'testScheme', + value: '1', + id: null, + dvbUrl: 'testUrl' + }); + }); + +}); \ No newline at end of file diff --git a/test/unit/data/subtitles/ttmlSample.ttml b/test/unit/data/subtitles/ttmlSample.ttml new file mode 100644 index 0000000000..b294d8d581 --- /dev/null +++ b/test/unit/data/subtitles/ttmlSample.ttml @@ -0,0 +1,29 @@ + + + + + +