Skip to content

Commit

Permalink
feat(HLS): Optimization of LL-HLS with byterange (shaka-project#5319)
Browse files Browse the repository at this point in the history 
This solution is based on
https://www.akamai.com/blog/performance/-using-ll-hls-with-byte-range-addressing-to-achieve-interoperabi
  • Loading branch information
@avelad
avelad committed Jun 21, 2023
1 parent 4df8254 commit 9e6655a
Show file tree
Hide file tree
Showing 6 changed files with 177 additions and 36 deletions.
52 changes: 48 additions & 4 deletions lib/hls/hls_parser.js
View file
Expand Up @@ -2781,14 +2781,15 @@ shaka.hls.HlsParser = class {
* @param {!Map.<string, string>} variables
* @param {string} absoluteMediaPlaylistUri
* @param {string} type
* @param {string} mimeType
* @param {number} timestampOffset
* @param {shaka.extern.HlsAes128Key=} hlsAes128Key
* @return {shaka.media.SegmentReference}
* @private
*/
createSegmentReference_(
initSegmentReference, previousReference, hlsSegment, startTime,
variables, absoluteMediaPlaylistUri, type, timestampOffset,
variables, absoluteMediaPlaylistUri, type, mimeType, timestampOffset,
hlsAes128Key) {
const tags = hlsSegment.tags;
const extinfTag =
Expand Down Expand Up @@ -2836,8 +2837,18 @@ shaka.hls.HlsParser = class {
}

// Create SegmentReferences for the partial segments.
const partialSegmentRefs = [];
if (this.lowLatencyMode_) {
let partialSegmentRefs = [];

// Optimization for LL-HLS with byterange
// More info in https://tinyurl.com/hls-open-byte-range
let segmentWithByteRangeOptimization = false;
let getUrisOptimization = null;
let somePartialSegmentWithGap = false;

if (this.lowLatencyMode_ && hlsSegment.partialSegments.length) {
const byterangeOptimizationSupport = (mimeType == 'video/mp4' ||
mimeType == 'audio/mp4') && window.ReadableStream;

let partialSyncTime = syncTime;
for (let i = 0; i < hlsSegment.partialSegments.length; i++) {
const item = hlsSegment.partialSegments[i];
Expand Down Expand Up @@ -2870,6 +2881,11 @@ shaka.hls.HlsParser = class {
const pByterangeLength = item.getAttributeValue('BYTERANGE-LENGTH');
if (pByterangeLength) {
pEndByte = pStartByte + Number(pByterangeLength) - 1;
} else if (pStartByte) {
// If we have a non-zero start byte, but no end byte, follow the
// recommendation of https://tinyurl.com/hls-open-byte-range and
// set the end byte explicitly to a large integer.
pEndByte = Number.MAX_SAFE_INTEGER;
}
} else {
const pByterange = item.getAttributeValue('BYTERANGE');
Expand All @@ -2884,6 +2900,7 @@ shaka.hls.HlsParser = class {
let partialStatus = shaka.media.SegmentReference.Status.AVAILABLE;
if (item.getAttributeValue('GAP') == 'YES') {
partialStatus = shaka.media.SegmentReference.Status.MISSING;
somePartialSegmentWithGap = true;
}

let pAbsoluteUri = null;
Expand All @@ -2896,6 +2913,12 @@ shaka.hls.HlsParser = class {
return [pAbsoluteUri];
};

if (byterangeOptimizationSupport &&
pStartByte >= 0 && pEndByte != null) {
getUrisOptimization = getPartialUris;
segmentWithByteRangeOptimization = true;
}

const partial = new shaka.media.SegmentReference(
pStartTime,
pEndTime,
Expand Down Expand Up @@ -2947,6 +2970,17 @@ shaka.hls.HlsParser = class {
endTime = partialSegmentRefs[partialSegmentRefs.length - 1].endTime;
}

if (segmentWithByteRangeOptimization) {
// We cannot optimize segments with gaps, or with a start byte that is
// not 0.
if (somePartialSegmentWithGap || partialSegmentRefs[0].startByte != 0) {
segmentWithByteRangeOptimization = false;
getUrisOptimization = null;
} else {
partialSegmentRefs = [];
}
}

// If the segment has EXT-X-BYTERANGE tag, set the start byte and end byte
// base on the byterange information. If segment has no EXT-X-BYTERANGE tag
// and has partial segments, set the start byte and end byte base on the
Expand Down Expand Up @@ -2979,14 +3013,17 @@ shaka.hls.HlsParser = class {

let absoluteSegmentUri = null;
const getUris = () => {
if (getUrisOptimization) {
return getUrisOptimization();
}
if (absoluteSegmentUri == null) {
absoluteSegmentUri = this.variableSubstitution_(
hlsSegment.absoluteUri, variables);
}
return absoluteSegmentUri.length ? [absoluteSegmentUri] : [];
};

return new shaka.media.SegmentReference(
const reference = new shaka.media.SegmentReference(
startTime,
endTime,
getUris,
Expand All @@ -3003,6 +3040,12 @@ shaka.hls.HlsParser = class {
status,
hlsAes128Key,
);

if (segmentWithByteRangeOptimization) {
reference.markAsByterangeOptimization();
}

return reference;
}


Expand Down Expand Up @@ -3129,6 +3172,7 @@ shaka.hls.HlsParser = class {
variables,
playlist.absoluteUri,
type,
mimeType,
lastDiscontinuityStartTime,
hlsAes128Key);
previousReference = reference;
Expand Down
10 changes: 1 addition & 9 deletions lib/hls/manifest_text_parser.js
View file
Expand Up @@ -158,15 +158,7 @@ shaka.hls.ManifestTextParser = class {
partialSegmentTags.push(tag);
} else if (tag.name == 'EXT-X-PRELOAD-HINT') {
if (tag.getAttributeValue('TYPE') == 'PART') {
// Note: BYTERANGE-START without BYTERANGE-LENGTH is being
// ignored.
if (tag.getAttributeValue('BYTERANGE-START') != null) {
if (tag.getAttributeValue('BYTERANGE-LENGTH') != null) {
partialSegmentTags.push(tag);
}
} else {
partialSegmentTags.push(tag);
}
partialSegmentTags.push(tag);
} else if (tag.getAttributeValue('TYPE') == 'MAP') {
// Rename the Preload Hint tag to be a Map tag.
tag.setName('EXT-X-MAP');
Expand Down
8 changes: 7 additions & 1 deletion lib/media/segment_prefetch.js
View file
Expand Up @@ -67,7 +67,13 @@ shaka.media.SegmentPrefetch = class {
let reference = startReference;
while (this.segmentPrefetchMap_.size < this.prefetchLimit_ &&
reference != null) {
if (!this.segmentPrefetchMap_.has(reference)) {
// By default doesn't prefech preload partial segments when using
// byterange
let prefetchAllowed = true;
if (reference.isPreload() && reference.endByte != null) {
prefetchAllowed = false;
}
if (prefetchAllowed && !this.segmentPrefetchMap_.has(reference)) {
const segmentPrefetchOperation =
new shaka.media.SegmentPrefetchOperation(this.fetchDispatcher_);
segmentPrefetchOperation.dispatchFetch(reference, this.stream_);
Expand Down
25 changes: 25 additions & 0 deletions lib/media/segment_reference.js
View file
Expand Up @@ -268,6 +268,9 @@ shaka.media.SegmentReference = class {
/** @type {boolean} */
this.independent = true;

/** @type {boolean} */
this.byterangeOptimization = false;

/** @type {?shaka.extern.HlsAes128Key} */
this.hlsAes128Key = hlsAes128Key;

Expand Down Expand Up @@ -429,6 +432,28 @@ shaka.media.SegmentReference = class {
return this.independent;
}

/**
* Mark the reference as byterange optimization.
*
* The "byterange optimization" means that it is playable using MP4 low
* latency streaming with chunked data.
*
* @export
*/
markAsByterangeOptimization() {
this.byterangeOptimization = true;
}

/**
* Returns true if the segment has a byterange optimization.
*
* @return {boolean}
* @export
*/
hasByterangeOptimization() {
return this.byterangeOptimization;
}

/**
* Set the segment's thumbnail sprite.
*
Expand Down
5 changes: 3 additions & 2 deletions lib/media/streaming_engine.js
View file
Expand Up @@ -1330,9 +1330,10 @@ shaka.media.StreamingEngine = class {
stream.mimeType == 'audio/mp4';
const isReadableStreamSupported = window.ReadableStream;
// Enable MP4 low latency streaming with ReadableStream chunked data.
// And only for DASH.
// And only for DASH and HLS with byterange optimization.
if (this.config_.lowLatencyMode && isReadableStreamSupported && isMP4 &&
this.manifest_.type != shaka.media.ManifestParser.HLS) {
(this.manifest_.type != shaka.media.ManifestParser.HLS ||
reference.hasByterangeOptimization())) {
let remaining = new Uint8Array(0);
let processingResult = false;
let callbackCalled = false;
Expand Down
113 changes: 93 additions & 20 deletions test/hls/hls_live_unit.js
View file
Expand Up @@ -634,58 +634,132 @@ describe('HlsParser live', () => {
'#EXTM3U\n',
'#EXT-X-TARGETDURATION:5\n',
'#EXT-X-PART-INF:PART-TARGET=1.5\n',
'#EXT-X-MAP:URI="init.mp4",BYTERANGE="616@0"\n',
'#EXT-X-MAP:URI="init.mp4"\n',
'#EXT-X-MEDIA-SEQUENCE:0\n',
// ref includes partialRef, partialRef2
// partialRef
'#EXT-X-PART:DURATION=2,URI="partial.mp4",BYTERANGE=200@0,',
'INDEPENDENT=YES\n',
'#EXT-X-PART:DURATION=2,URI="partial.mp4",INDEPENDENT=YES\n',
// partialRef2
'#EXT-X-PART:DURATION=2,URI="partial2.mp4",BYTERANGE=230@200,',
'INDEPENDENT=YES\n',
'#EXT-X-PART:DURATION=2,URI="partial2.mp4",INDEPENDENT=YES\n',
'#EXTINF:4,\n',
'main.mp4\n',
// ref2 includes partialRef3, preloadRef
// partialRef3
'#EXT-X-PART:DURATION=2,URI="partial.mp4",BYTERANGE=210@0,',
'INDEPENDENT=YES\n',
'#EXT-X-PART:DURATION=2,URI="partial.mp4",INDEPENDENT=YES\n',
// preloadRef
'#EXT-X-PRELOAD-HINT:TYPE=PART,URI="partial.mp4",BYTERANGE-START=210,',
'BYTERANGE-LENGTH=210\n',
'#EXT-X-PRELOAD-HINT:TYPE=PART,URI="partial.mp4"\n',
].join('');

const partialRef = makeReference(
'test:/partial.mp4', 0, 2, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 199);
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null);

const partialRef2 = makeReference(
'test:/partial2.mp4', 2, 4, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 200, /* endByte= */ 429);
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null);

const ref = makeReference(
'test:/main.mp4', 0, 4, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 429,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null,
/* timestampOffset= */ 0, [partialRef, partialRef2]);

const partialRef3 = makeReference(
'test:/partial.mp4', 4, 6, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 209);
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null);

const preloadRef = makeReference(
'test:/partial.mp4', 6, 7.5, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 210, /* endByte= */ 419);
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null);
preloadRef.markAsPreload();
preloadRef.markAsNonIndependent();

// ref2 is not fully published yet, so it doesn't have a segment uri.
const ref2 = makeReference(
'', 4, 7.5, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 419,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null,
/* timestampOffset= */ 0, [partialRef3, preloadRef]);

await testInitialManifest(master, mediaWithPartialSegments, [ref, ref2]);
});

it('parses streams with partial and preload hinted segments and BYTERANGE', async () => { // eslint-disable-line max-len
playerInterface.isLowLatencyMode = () => true;
const mediaWithPartialSegments = [
'#EXTM3U\n',
'#EXT-X-TARGETDURATION:5\n',
'#EXT-X-PART-INF:PART-TARGET=1.5\n',
'#EXT-X-MAP:URI="init.mp4"\n',
'#EXT-X-MEDIA-SEQUENCE:0\n',
// ref includes partialRef, partialRef2
// partialRef
'#EXT-X-PART:DURATION=2,URI="ref1.mp4",BYTERANGE=200@0,',
'INDEPENDENT=YES\n',
// partialRef2
'#EXT-X-PART:DURATION=2,URI="ref1.mp4",BYTERANGE=230@200,',
'INDEPENDENT=YES\n',
'#EXTINF:4,\n',
'ref1.mp4\n',
// ref2 includes partialRef3, preloadRef
// partialRef3
'#EXT-X-PART:DURATION=2,URI="ref2.mp4",BYTERANGE=210@0,',
'INDEPENDENT=YES\n',
// preloadRef
'#EXT-X-PRELOAD-HINT:TYPE=PART,URI="ref2.mp4",BYTERANGE-START=210,',
'BYTERANGE-LENGTH=210\n',
].join('');

// If ReadableStream is defined we can apply some optimizations
if (window.ReadableStream) {
const ref = makeReference(
'test:/ref1.mp4', 0, 4, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null,
/* timestampOffset= */ 0);
ref.markAsByterangeOptimization();

// ref2 is not fully published yet, so it doesn't have a segment uri.
const ref2 = makeReference(
'test:/ref2.mp4', 4, 7.5, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null,
/* timestampOffset= */ 0);
ref2.markAsByterangeOptimization();

await testInitialManifest(master, mediaWithPartialSegments,
[ref, ref2]);
} else {
const partialRef = makeReference(
'test:/ref1.mp4', 0, 2, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 199);

const partialRef2 = makeReference(
'test:/ref1.mp4', 2, 4, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 200, /* endByte= */ 429);

const ref = makeReference(
'test:/ref1.mp4', 0, 4, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 429,
/* timestampOffset= */ 0, [partialRef, partialRef2]);

const partialRef3 = makeReference(
'test:/ref2.mp4', 4, 6, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 209);

const preloadRef = makeReference(
'test:/ref2.mp4', 6, 7.5, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 210, /* endByte= */ 419);
preloadRef.markAsPreload();
preloadRef.markAsNonIndependent();

// ref2 is not fully published yet, so it doesn't have a segment uri.
const ref2 = makeReference(
'', 4, 7.5, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 419,
/* timestampOffset= */ 0, [partialRef3, preloadRef]);

await testInitialManifest(master, mediaWithPartialSegments,
[ref, ref2]);
}
});

// Test for https://github.com/shaka-project/shaka-player/issues/4223
it('ignores preload hinted segments without target duration', async () => {
playerInterface.isLowLatencyMode = () => true;
Expand All @@ -701,10 +775,9 @@ describe('HlsParser live', () => {
'main.mp4\n',
// ref2 includes partialRef, but not preloadRef
// partialRef
'#EXT-X-PART:DURATION=2,URI="partial.mp4",BYTERANGE=210@0,',
'INDEPENDENT=YES\n',
'#EXT-X-PART:DURATION=2,URI="partial.mp4",INDEPENDENT=YES\n',
// preloadRef
'#EXT-X-PRELOAD-HINT:TYPE=PART,URI="partial.mp4",BYTERANGE-START=210\n',
'#EXT-X-PRELOAD-HINT:TYPE=PART,URI="partial.mp4"\n',
].join('');

const ref = makeReference(
Expand All @@ -714,12 +787,12 @@ describe('HlsParser live', () => {

const partialRef = makeReference(
'test:/partial.mp4', 4, 6, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 209);
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null);

// ref2 is not fully published yet, so it doesn't have a segment uri.
const ref2 = makeReference(
'', 4, 6, /* syncTime= */ null,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ 209,
/* baseUri= */ '', /* startByte= */ 0, /* endByte= */ null,
/* timestampOffset= */ 0, [partialRef]);

await testInitialManifest(master, mediaWithPartialSegments, [ref, ref2]);
Expand Down

0 comments on commit 9e6655a

Please sign in to comment.