From 94e15a2b03a9aefe0d87e922eebd618dc2148152 Mon Sep 17 00:00:00 2001 From: Rob Walch Date: Mon, 20 Feb 2023 11:19:35 -0800 Subject: [PATCH] Fix initPTS change between variants (timescale or basetime change) (#5235) Fixes #5195 --- src/remux/mp4-remuxer.ts | 57 +++++++++++++++++++++++--------- src/remux/passthrough-remuxer.ts | 25 +++++++++++--- 2 files changed, 63 insertions(+), 19 deletions(-) diff --git a/src/remux/mp4-remuxer.ts b/src/remux/mp4-remuxer.ts index a2a2d4b4947..54ade83328c 100644 --- a/src/remux/mp4-remuxer.ts +++ b/src/remux/mp4-remuxer.ts @@ -147,7 +147,12 @@ export default class MP4Remuxer implements Remuxer { if (canRemuxAvc) { if (!this.ISGenerated) { - initSegment = this.generateIS(audioTrack, videoTrack, timeOffset); + initSegment = this.generateIS( + audioTrack, + videoTrack, + timeOffset, + accurateTimeOffset + ); } const isVideoContiguous = this.isVideoContiguous; @@ -199,7 +204,12 @@ export default class MP4Remuxer implements Remuxer { logger.warn( '[mp4-remuxer]: regenerate InitSegment as audio detected' ); - initSegment = this.generateIS(audioTrack, videoTrack, timeOffset); + initSegment = this.generateIS( + audioTrack, + videoTrack, + timeOffset, + accurateTimeOffset + ); } audio = this.remuxAudio( audioTrack, @@ -219,7 +229,12 @@ export default class MP4Remuxer implements Remuxer { logger.warn( '[mp4-remuxer]: regenerate InitSegment as video detected' ); - initSegment = this.generateIS(audioTrack, videoTrack, timeOffset); + initSegment = this.generateIS( + audioTrack, + videoTrack, + timeOffset, + accurateTimeOffset + ); } video = this.remuxVideo( videoTrack, @@ -277,13 +292,15 @@ export default class MP4Remuxer implements Remuxer { generateIS( audioTrack: DemuxedAudioTrack, videoTrack: DemuxedAvcTrack, - timeOffset + timeOffset: number, + accurateTimeOffset: boolean ): InitSegmentData | undefined { const audioSamples = audioTrack.samples; const videoSamples = videoTrack.samples; const typeSupported = this.typeSupported; const tracks: TrackSet = {}; - const computePTSDTS = !this._initPTS; + const _initPTS = this._initPTS; + let computePTSDTS = !_initPTS || accurateTimeOffset; let container = 'audio/mp4'; let initPTS: number | undefined; let initDTS: number | undefined; @@ -325,9 +342,13 @@ export default class MP4Remuxer implements Remuxer { }; if (computePTSDTS) { timescale = audioTrack.inputTimeScale; - // remember first PTS of this demuxing context. for audio, PTS = DTS - initPTS = initDTS = - audioSamples[0].pts - Math.round(timescale * timeOffset); + if (!_initPTS || timescale !== _initPTS.timescale) { + // remember first PTS of this demuxing context. for audio, PTS = DTS + initPTS = initDTS = + audioSamples[0].pts - Math.round(timescale * timeOffset); + } else { + computePTSDTS = false; + } } } @@ -347,13 +368,17 @@ export default class MP4Remuxer implements Remuxer { }; if (computePTSDTS) { timescale = videoTrack.inputTimeScale; - const startPTS = this.getVideoStartPts(videoSamples); - const startOffset = Math.round(timescale * timeOffset); - initDTS = Math.min( - initDTS as number, - normalizePts(videoSamples[0].dts, startPTS) - startOffset - ); - initPTS = Math.min(initPTS as number, startPTS - startOffset); + if (!_initPTS || timescale !== _initPTS.timescale) { + const startPTS = this.getVideoStartPts(videoSamples); + const startOffset = Math.round(timescale * timeOffset); + initDTS = Math.min( + initDTS as number, + normalizePts(videoSamples[0].dts, startPTS) - startOffset + ); + initPTS = Math.min(initPTS as number, startPTS - startOffset); + } else { + computePTSDTS = false; + } } } @@ -368,6 +393,8 @@ export default class MP4Remuxer implements Remuxer { baseTime: initDTS as number, timescale: timescale as number, }; + } else { + initPTS = timescale = undefined; } return { diff --git a/src/remux/passthrough-remuxer.ts b/src/remux/passthrough-remuxer.ts index e0ed60f7e23..c822b942f82 100644 --- a/src/remux/passthrough-remuxer.ts +++ b/src/remux/passthrough-remuxer.ts @@ -122,7 +122,8 @@ class PassThroughRemuxer implements Remuxer { videoTrack: PassthroughTrack, id3Track: DemuxedMetadataTrack, textTrack: DemuxedUserdataTrack, - timeOffset: number + timeOffset: number, + accurateTimeOffset: boolean ): RemuxerResult { let { initPTS, lastEndTime } = this; const result: RemuxerResult = { @@ -167,7 +168,10 @@ class PassThroughRemuxer implements Remuxer { } const startDTS = getStartDTS(initData, data); - if (!initPTS) { + if ( + isInvalidInitPts(initPTS, startDTS, timeOffset) || + (initSegment.timescale !== initPTS.timescale && accurateTimeOffset) + ) { initSegment.initPTS = startDTS - timeOffset; this.initPTS = initPTS = { baseTime: initSegment.initPTS, @@ -177,10 +181,10 @@ class PassThroughRemuxer implements Remuxer { const duration = getDuration(data, initData); const startTime = audioTrack - ? startDTS - initPTS.baseTime + ? startDTS - initPTS.baseTime / initPTS.timescale : (lastEndTime as number); const endTime = startTime + duration; - offsetStartDTS(initData, data, initPTS.baseTime); + offsetStartDTS(initData, data, initPTS.baseTime / initPTS.timescale); if (duration > 0) { this.lastEndTime = endTime; @@ -236,6 +240,19 @@ class PassThroughRemuxer implements Remuxer { } } +function isInvalidInitPts( + initPTS: RationalTimestamp | null, + startDTS: number, + timeOffset: number +): initPTS is null { + if (initPTS === null) { + return true; + } + // InitPTS is invalid when it would cause start time to be negative, or distance from time offset to be more than 1 second + const startTime = startDTS - initPTS.baseTime / initPTS.timescale; + return startTime < 0 && Math.abs(startTime - timeOffset) > 1; +} + function getParsedTrackCodec( track: InitDataTrack | undefined, type: ElementaryStreamTypes.AUDIO | ElementaryStreamTypes.VIDEO