Skip to content

Commit

Permalink
fix: Transmux containerless to the correct mimetype (#5205)
Browse files Browse the repository at this point in the history 
With this change, the manifest codecs are not filtered when using a raw
format, now it is filtered only when necessary.
  • Loading branch information
@avelad
avelad committed May 3, 2023
1 parent 18f5809 commit bb40d3b
Show file tree
Hide file tree
Showing 5 changed files with 73 additions and 53 deletions.
18 changes: 0 additions & 18 deletions lib/hls/hls_parser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ goog.require('shaka.log');
goog.require('shaka.media.DrmEngine');
goog.require('shaka.media.InitSegmentReference');
goog.require('shaka.media.ManifestParser');
goog.require('shaka.media.MediaSourceEngine');
goog.require('shaka.media.PresentationTimeline');
goog.require('shaka.media.SegmentIndex');
goog.require('shaka.media.SegmentReference');
Expand Down Expand Up @@ -1918,9 +1917,6 @@ shaka.hls.HlsParser = class {
// segment index, then download when createSegmentIndex is called.
const stream = this.makeStreamObject_(codecs, type, language, primary, name,
channelsCount, closedCaptions, characteristics, forced, spatialAudio);
if (shaka.media.MediaSourceEngine.RAW_FORMATS.includes(stream.mimeType)) {
stream.codecs = '';
}
const streamInfo = {
stream,
type,
Expand Down Expand Up @@ -1983,9 +1979,6 @@ shaka.hls.HlsParser = class {
stream.drmInfos = realStream.drmInfos;
stream.keyIds = realStream.keyIds;
stream.mimeType = realStream.mimeType;
if (shaka.media.MediaSourceEngine.RAW_FORMATS.includes(stream.mimeType)) {
stream.codecs = '';
}

// Since we lazy-loaded this content, the player may need to create new
// sessions for the DRM info in this stream.
Expand All @@ -2000,10 +1993,6 @@ shaka.hls.HlsParser = class {
// To aid manifest filtering, assume before loading that all video
// renditions have the same MIME type. (And likewise for audio.)
otherStreamInfo.stream.mimeType = realStream.mimeType;
if (shaka.media.MediaSourceEngine.RAW_FORMATS
.includes(otherStreamInfo.stream.mimeType)) {
otherStreamInfo.stream.codecs = '';
}
}
}
}
Expand Down Expand Up @@ -2134,13 +2123,6 @@ shaka.hls.HlsParser = class {
streamInfo.stream.segmentIndex.fit(/* periodStart= */ 0, minDuration);
}
}
// MediaSource expects no codec strings combined with raw formats.
for (const streamInfo of streamInfos) {
const stream = streamInfo.stream;
if (shaka.media.MediaSourceEngine.RAW_FORMATS.includes(stream.mimeType)) {
stream.codecs = '';
}
}
this.notifySegmentsForStreams_(streamInfos.map((s) => s.stream));
if (this.config_.hls.ignoreManifestProgramDateTime) {
this.syncStreamsWithSequenceNumber_(streamInfos);
Expand Down
38 changes: 16 additions & 22 deletions lib/media/media_source_engine.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,13 +181,15 @@ shaka.media.MediaSourceEngine = class {
* @return {boolean}
*/
static isStreamSupported(stream) {
const fullMimeType = shaka.util.MimeUtils.getFullType(
const MimeUtils = shaka.util.MimeUtils;
const fullMimeType = MimeUtils.getFullType(stream.mimeType, stream.codecs);
const extendedMimeType = MimeUtils.getExtendedType(stream);
const fullMimeTypeWithAllCodecs = MimeUtils.getFullTypeWithAllCodecs(
stream.mimeType, stream.codecs);
const extendedMimeType = shaka.util.MimeUtils.getExtendedType(stream);
const TransmuxerEngine = shaka.transmuxer.TransmuxerEngine;
return shaka.text.TextEngine.isTypeSupported(fullMimeType) ||
shaka.media.Capabilities.isTypeSupported(extendedMimeType) ||
TransmuxerEngine.isSupported(fullMimeType, stream.type);
TransmuxerEngine.isSupported(fullMimeTypeWithAllCodecs, stream.type);
}

/**
Expand Down Expand Up @@ -235,7 +237,7 @@ shaka.media.MediaSourceEngine = class {
'application/ttml+xml',
'application/mp4; codecs="stpp"',
// Containerless types
...shaka.media.MediaSourceEngine.RAW_FORMATS,
...shaka.util.MimeUtils.RAW_FORMATS,
];

const support = {};
Expand Down Expand Up @@ -391,17 +393,22 @@ shaka.media.MediaSourceEngine = class {
let needTransmux = this.config_.forceTransmux;
if (!shaka.media.Capabilities.isTypeSupported(mimeType) ||
(!sequenceMode &&
shaka.media.MediaSourceEngine.RAW_FORMATS.includes(mimeType))) {
shaka.util.MimeUtils.RAW_FORMATS.includes(mimeType))) {
needTransmux = true;
}
const mimeTypeWithAllCodecs =
shaka.util.MimeUtils.getFullTypeWithAllCodecs(
stream.mimeType, stream.codecs);
const TransmuxerEngine = shaka.transmuxer.TransmuxerEngine;
if (needTransmux &&
TransmuxerEngine.isSupported(mimeType, contentType)) {
const transmuxerPlugin = TransmuxerEngine.findTransmuxer(mimeType);
TransmuxerEngine.isSupported(mimeTypeWithAllCodecs, contentType)) {
const transmuxerPlugin =
TransmuxerEngine.findTransmuxer(mimeTypeWithAllCodecs);
if (transmuxerPlugin) {
const transmuxer = transmuxerPlugin();
this.transmuxers_[contentType] = transmuxer;
mimeType = transmuxer.convertCodecs(contentType, mimeType);
mimeType =
transmuxer.convertCodecs(contentType, mimeTypeWithAllCodecs);
}
}
const type = mimeType + this.config_.sourceBufferExtraFeatures;
Expand Down Expand Up @@ -639,7 +646,7 @@ shaka.media.MediaSourceEngine = class {
if (this.transmuxers_[contentType]) {
mimeType = this.transmuxers_[contentType].getOrginalMimeType();
}
if (shaka.media.MediaSourceEngine.RAW_FORMATS.includes(mimeType)) {
if (shaka.util.MimeUtils.RAW_FORMATS.includes(mimeType)) {
const frames = shaka.util.Id3Utils.getID3Frames(uint8ArrayData);
if (frames.length && reference) {
if (attemptTimestampOffsetCalculation) {
Expand Down Expand Up @@ -1547,16 +1554,3 @@ shaka.media.MediaSourceEngine.SourceBufferMode_ = {
SEQUENCE: 'sequence',
SEGMENTS: 'segments',
};


/**
* MIME types of raw formats.
*
* @const {!Array.<string>}
*/
shaka.media.MediaSourceEngine.RAW_FORMATS = [
'audio/aac',
'audio/ac3',
'audio/ec3',
'audio/mpeg',
];
12 changes: 9 additions & 3 deletions lib/transmuxer/muxjs_transmuxer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ goog.require('shaka.transmuxer.TransmuxerEngine');
goog.require('shaka.util.BufferUtils');
goog.require('shaka.util.Error');
goog.require('shaka.util.ManifestParserUtils');
goog.require('shaka.util.MimeUtils');
goog.require('shaka.util.PublicPromise');
goog.require('shaka.util.Uint8ArrayUtils');

Expand Down Expand Up @@ -85,7 +86,7 @@ shaka.transmuxer.MuxjsTransmuxer = class {
}

if (isAac) {
return Capabilities.isTypeSupported(this.convertAacCodecs_());
return Capabilities.isTypeSupported(this.convertAacCodecs_(mimeType));
}

if (contentType) {
Expand Down Expand Up @@ -130,7 +131,7 @@ shaka.transmuxer.MuxjsTransmuxer = class {
*/
convertCodecs(contentType, mimeType) {
if (this.isAacContainer_(mimeType)) {
return this.convertAacCodecs_();
return this.convertAacCodecs_(mimeType);
} else if (this.isTsContainer_(mimeType)) {
return this.convertTsCodecs_(contentType, mimeType);
}
Expand All @@ -140,10 +141,15 @@ shaka.transmuxer.MuxjsTransmuxer = class {

/**
* For aac stream, convert its codecs to MP4 codecs.
* @param {string} mimeType
* @return {string}
* @private
*/
convertAacCodecs_() {
convertAacCodecs_(mimeType) {
const codecs = shaka.util.MimeUtils.getCodecs(mimeType);
if (codecs != '') {
return `audio/mp4; codecs="${codecs}"`;
}
return 'audio/mp4; codecs="mp4a.40.2"';
}

Expand Down
46 changes: 41 additions & 5 deletions lib/util/mime_utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,31 @@ goog.require('shaka.util.ManifestParserUtils');
shaka.util.MimeUtils = class {
/**
* Takes a MIME type and optional codecs string and produces the full MIME
* type.
* type. Also remove the codecs for raw formats.
*
* @param {string} mimeType
* @param {string=} codecs
* @return {string}
* @export
*/
static getFullType(mimeType, codecs) {
let fullMimeType = mimeType;
if (codecs && !shaka.util.MimeUtils.RAW_FORMATS.includes(mimeType)) {
fullMimeType += '; codecs="' + codecs + '"';
}
return fullMimeType;
}

/**
* Takes a MIME type and optional codecs string and produces the full MIME
* type.
*
* @param {string} mimeType
* @param {string=} codecs
* @return {string}
* @export
*/
static getFullTypeWithAllCodecs(mimeType, codecs) {
let fullMimeType = mimeType;
if (codecs) {
fullMimeType += '; codecs="' + codecs + '"';
Expand All @@ -43,12 +60,16 @@ shaka.util.MimeUtils = class {
* @return {string}
*/
static getFullOrConvertedType(mimeType, codecs, contentType) {
const fullMimeType = shaka.util.MimeUtils.getFullType(mimeType, codecs);
const MimeUtils = shaka.util.MimeUtils;
const fullMimeType = MimeUtils.getFullType(mimeType, codecs);
const fullMimeTypeWithAllCodecs = MimeUtils.getFullTypeWithAllCodecs(
mimeType, codecs);
const ContentType = shaka.util.ManifestParserUtils.ContentType;
const TransmuxerEngine = shaka.transmuxer.TransmuxerEngine;

if (TransmuxerEngine.isSupported(fullMimeType, contentType)) {
return TransmuxerEngine.convertCodecs(contentType, fullMimeType);
if (TransmuxerEngine.isSupported(fullMimeTypeWithAllCodecs, contentType)) {
return TransmuxerEngine.convertCodecs(
contentType, fullMimeTypeWithAllCodecs);
} else if (mimeType != 'video/mp2t' && contentType == ContentType.AUDIO) {
// video/mp2t is the correct mime type for TS audio, so only replace the
// word "video" with "audio" for non-TS audio content.
Expand All @@ -71,7 +92,10 @@ shaka.util.MimeUtils = class {
const extendedMimeParams = shaka.util.MimeUtils.EXTENDED_MIME_PARAMETERS_;
extendedMimeParams.forEach((mimeKey, streamKey) => {
const value = stream[streamKey];
if (value) {
if (streamKey == 'codecs' &&
shaka.util.MimeUtils.RAW_FORMATS.includes(stream.mimeType)) {
// Skip codecs for raw formats
} else if (value) {
components.push(mimeKey + '="' + value + '"');
}
});
Expand Down Expand Up @@ -254,3 +278,15 @@ shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE = 'application/cea-608';
* @const {string}
*/
shaka.util.MimeUtils.CEA708_CLOSED_CAPTION_MIMETYPE = 'application/cea-708';

/**
* MIME types of raw formats.
*
* @const {!Array.<string>}
*/
shaka.util.MimeUtils.RAW_FORMATS = [
'audio/aac',
'audio/ac3',
'audio/ec3',
'audio/mpeg',
];
12 changes: 7 additions & 5 deletions test/hls/hls_parser_unit.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -560,7 +560,7 @@ describe('HlsParser', () => {
stream.mime('video/mp4', 'avc1');
});
variant.addPartialStream(ContentType.AUDIO, (stream) => {
stream.mime('audio/mp4', '');
stream.mime('audio/mp4', 'mp4a.40.34');
});
});
manifest.sequenceMode = sequenceMode;
Expand Down Expand Up @@ -589,7 +589,7 @@ describe('HlsParser', () => {
manifest.anyTimeline();
manifest.addPartialVariant((variant) => {
variant.addPartialStream(ContentType.AUDIO, (stream) => {
stream.mime('audio/aac', '');
stream.mime('audio/aac', 'mp4a.40.2');
});
});
manifest.sequenceMode = sequenceMode;
Expand Down Expand Up @@ -617,7 +617,7 @@ describe('HlsParser', () => {
manifest.anyTimeline();
manifest.addPartialVariant((variant) => {
variant.addPartialStream(ContentType.AUDIO, (stream) => {
stream.mime('audio/mpeg', '');
stream.mime('audio/mpeg', 'mp4a.40.34');
});
});
manifest.sequenceMode = sequenceMode;
Expand Down Expand Up @@ -4618,7 +4618,9 @@ describe('HlsParser', () => {

// Before loading, all MIME types agree, and are defaulted to audio/mp4.
expect(actualAudio0.mimeType).toBe('audio/mp4');
expect(actualAudio0.codecs).toBe('mp4a');
expect(actualAudio1.mimeType).toBe('audio/mp4');
expect(actualAudio1.codecs).toBe('mp4a');

await actualAudio0.createSegmentIndex();

Expand All @@ -4627,9 +4629,9 @@ describe('HlsParser', () => {
// This is how we avoid having the unloaded tracks filtered out during
// startup.
expect(actualAudio0.mimeType).toBe('audio/aac');
expect(actualAudio0.codecs).toBe('');
expect(actualAudio0.codecs).toBe('mp4a');
expect(actualAudio1.mimeType).toBe('audio/aac');
expect(actualAudio1.codecs).toBe('');
expect(actualAudio1.codecs).toBe('mp4a');
});

it('parses media playlists directly', async () => {
Expand Down

0 comments on commit bb40d3b

Please sign in to comment.