From 3d5d23778abc9cdc1bc94da6930be6feacad5815 Mon Sep 17 00:00:00 2001 From: olly Date: Tue, 2 Jul 2019 13:42:05 +0100 Subject: [PATCH] FLV extractor fixes 1. Only output video starting from a keyframe 2. When calculating the timestamp offset to adjust live streams to start at t=0, use the timestamp of the first tag from which a sample is actually output, rather than just the first audio/video tag. The test streams in the referenced GitHub issue start with a video tag whose packet type is AVC_PACKET_TYPE_SEQUENCE_HEADER (i.e. does not contain a sample) and whose timestamp is set to 0 (i.e. isn't set). The timestamp is set correctly on tags that from which a sample is actually output. Issue: #6111 PiperOrigin-RevId: 256147747 --- RELEASENOTES.md | 2 ++ .../extractor/flv/AudioTagPayloadReader.java | 8 ++++-- .../extractor/flv/FlvExtractor.java | 26 ++++++++++++------- .../extractor/flv/ScriptTagPayloadReader.java | 7 ++--- .../extractor/flv/TagPayloadReader.java | 16 ++++++------ .../extractor/flv/VideoTagPayloadReader.java | 18 ++++++++++--- 6 files changed, 51 insertions(+), 26 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 5dde993ae02..29f4d94946a 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -7,6 +7,8 @@ * SmoothStreaming: Parse text stream `Subtype` into `Format.roleFlags`. * Gracefully handle revoked `ACCESS_NETWORK_STATE` permission ([#6019](https://github.com/google/ExoPlayer/issues/6019)). +* FLV: Fix bug that caused playback of some live streams to not start + ([#6111](https://github.com/google/ExoPlayer/issues/6111)). ### 2.10.2 ### diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/AudioTagPayloadReader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/AudioTagPayloadReader.java index ec5ad88aebb..b10f2bf80bc 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/AudioTagPayloadReader.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/AudioTagPayloadReader.java @@ -86,11 +86,12 @@ protected boolean parseHeader(ParsableByteArray data) throws UnsupportedFormatEx } @Override - protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserException { + protected boolean parsePayload(ParsableByteArray data, long timeUs) throws ParserException { if (audioFormat == AUDIO_FORMAT_MP3) { int sampleSize = data.bytesLeft(); output.sampleData(data, sampleSize); output.sampleMetadata(timeUs, C.BUFFER_FLAG_KEY_FRAME, sampleSize, 0, null); + return true; } else { int packetType = data.readUnsignedByte(); if (packetType == AAC_PACKET_TYPE_SEQUENCE_HEADER && !hasOutputFormat) { @@ -104,12 +105,15 @@ protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserEx Collections.singletonList(audioSpecificConfig), null, 0, null); output.format(format); hasOutputFormat = true; + return false; } else if (audioFormat != AUDIO_FORMAT_AAC || packetType == AAC_PACKET_TYPE_AAC_RAW) { int sampleSize = data.bytesLeft(); output.sampleData(data, sampleSize); output.sampleMetadata(timeUs, C.BUFFER_FLAG_KEY_FRAME, sampleSize, 0, null); + return true; + } else { + return false; } } } - } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/FlvExtractor.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/FlvExtractor.java index 0a2c0c46f6b..de6ed2710ab 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/FlvExtractor.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/FlvExtractor.java @@ -74,6 +74,7 @@ public final class FlvExtractor implements Extractor { private ExtractorOutput extractorOutput; private @States int state; + private boolean outputFirstSample; private long mediaTagTimestampOffsetUs; private int bytesToNextTagHeader; private int tagType; @@ -90,7 +91,6 @@ public FlvExtractor() { tagData = new ParsableByteArray(); metadataReader = new ScriptTagPayloadReader(); state = STATE_READING_FLV_HEADER; - mediaTagTimestampOffsetUs = C.TIME_UNSET; } @Override @@ -132,7 +132,7 @@ public void init(ExtractorOutput output) { @Override public void seek(long position, long timeUs) { state = STATE_READING_FLV_HEADER; - mediaTagTimestampOffsetUs = C.TIME_UNSET; + outputFirstSample = false; bytesToNextTagHeader = 0; } @@ -253,14 +253,16 @@ private boolean readTagHeader(ExtractorInput input) throws IOException, Interrup */ private boolean readTagData(ExtractorInput input) throws IOException, InterruptedException { boolean wasConsumed = true; + boolean wasSampleOutput = false; + long timestampUs = getCurrentTimestampUs(); if (tagType == TAG_TYPE_AUDIO && audioReader != null) { ensureReadyForMediaOutput(); - audioReader.consume(prepareTagData(input), mediaTagTimestampOffsetUs + tagTimestampUs); + wasSampleOutput = audioReader.consume(prepareTagData(input), timestampUs); } else if (tagType == TAG_TYPE_VIDEO && videoReader != null) { ensureReadyForMediaOutput(); - videoReader.consume(prepareTagData(input), mediaTagTimestampOffsetUs + tagTimestampUs); + wasSampleOutput = videoReader.consume(prepareTagData(input), timestampUs); } else if (tagType == TAG_TYPE_SCRIPT_DATA && !outputSeekMap) { - metadataReader.consume(prepareTagData(input), tagTimestampUs); + wasSampleOutput = metadataReader.consume(prepareTagData(input), timestampUs); long durationUs = metadataReader.getDurationUs(); if (durationUs != C.TIME_UNSET) { extractorOutput.seekMap(new SeekMap.Unseekable(durationUs)); @@ -270,6 +272,11 @@ private boolean readTagData(ExtractorInput input) throws IOException, Interrupte input.skipFully(tagDataSize); wasConsumed = false; } + if (!outputFirstSample && wasSampleOutput) { + outputFirstSample = true; + mediaTagTimestampOffsetUs = + metadataReader.getDurationUs() == C.TIME_UNSET ? -tagTimestampUs : 0; + } bytesToNextTagHeader = 4; // There's a 4 byte previous tag size before the next header. state = STATE_SKIPPING_TO_TAG_HEADER; return wasConsumed; @@ -292,10 +299,11 @@ private void ensureReadyForMediaOutput() { extractorOutput.seekMap(new SeekMap.Unseekable(C.TIME_UNSET)); outputSeekMap = true; } - if (mediaTagTimestampOffsetUs == C.TIME_UNSET) { - mediaTagTimestampOffsetUs = - metadataReader.getDurationUs() == C.TIME_UNSET ? -tagTimestampUs : 0; - } } + private long getCurrentTimestampUs() { + return outputFirstSample + ? (mediaTagTimestampOffsetUs + tagTimestampUs) + : (metadataReader.getDurationUs() == C.TIME_UNSET ? 0 : tagTimestampUs); + } } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/ScriptTagPayloadReader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/ScriptTagPayloadReader.java index 2dec85ffcc9..eb1cc8f336d 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/ScriptTagPayloadReader.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/ScriptTagPayloadReader.java @@ -63,7 +63,7 @@ protected boolean parseHeader(ParsableByteArray data) { } @Override - protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserException { + protected boolean parsePayload(ParsableByteArray data, long timeUs) throws ParserException { int nameType = readAmfType(data); if (nameType != AMF_TYPE_STRING) { // Should never happen. @@ -72,12 +72,12 @@ protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserEx String name = readAmfString(data); if (!NAME_METADATA.equals(name)) { // We're only interested in metadata. - return; + return false; } int type = readAmfType(data); if (type != AMF_TYPE_ECMA_ARRAY) { // We're not interested in this metadata. - return; + return false; } // Set the duration to the value contained in the metadata, if present. Map metadata = readAmfEcmaArray(data); @@ -87,6 +87,7 @@ protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserEx durationUs = (long) (durationSeconds * C.MICROS_PER_SECOND); } } + return false; } private static int readAmfType(ParsableByteArray data) { diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/TagPayloadReader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/TagPayloadReader.java index e8652d653f9..48914b7c2c9 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/TagPayloadReader.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/TagPayloadReader.java @@ -58,12 +58,11 @@ protected TagPayloadReader(TrackOutput output) { * * @param data The payload data to consume. * @param timeUs The timestamp associated with the payload. + * @return Whether a sample was output. * @throws ParserException If an error occurs parsing the data. */ - public final void consume(ParsableByteArray data, long timeUs) throws ParserException { - if (parseHeader(data)) { - parsePayload(data, timeUs); - } + public final boolean consume(ParsableByteArray data, long timeUs) throws ParserException { + return parseHeader(data) && parsePayload(data, timeUs); } /** @@ -78,10 +77,11 @@ public final void consume(ParsableByteArray data, long timeUs) throws ParserExce /** * Parses tag payload. * - * @param data Buffer where tag payload is stored - * @param timeUs Time position of the frame + * @param data Buffer where tag payload is stored. + * @param timeUs Time position of the frame. + * @return Whether a sample was output. * @throws ParserException If an error occurs parsing the payload. */ - protected abstract void parsePayload(ParsableByteArray data, long timeUs) throws ParserException; - + protected abstract boolean parsePayload(ParsableByteArray data, long timeUs) + throws ParserException; } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/VideoTagPayloadReader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/VideoTagPayloadReader.java index 92db91e20b1..5ddaafb4a89 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/VideoTagPayloadReader.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/VideoTagPayloadReader.java @@ -47,6 +47,7 @@ // State variables. private boolean hasOutputFormat; + private boolean hasOutputKeyframe; private int frameType; /** @@ -60,7 +61,7 @@ public VideoTagPayloadReader(TrackOutput output) { @Override public void seek() { - // Do nothing. + hasOutputKeyframe = false; } @Override @@ -77,7 +78,7 @@ protected boolean parseHeader(ParsableByteArray data) throws UnsupportedFormatEx } @Override - protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserException { + protected boolean parsePayload(ParsableByteArray data, long timeUs) throws ParserException { int packetType = data.readUnsignedByte(); int compositionTimeMs = data.readInt24(); @@ -94,7 +95,12 @@ protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserEx avcConfig.initializationData, Format.NO_VALUE, avcConfig.pixelWidthAspectRatio, null); output.format(format); hasOutputFormat = true; + return false; } else if (packetType == AVC_PACKET_TYPE_AVC_NALU && hasOutputFormat) { + boolean isKeyframe = frameType == VIDEO_FRAME_KEYFRAME; + if (!hasOutputKeyframe && !isKeyframe) { + return false; + } // TODO: Deduplicate with Mp4Extractor. // Zero the top three bytes of the array that we'll use to decode nal unit lengths, in case // they're only 1 or 2 bytes long. @@ -123,8 +129,12 @@ protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserEx output.sampleData(data, bytesToWrite); bytesWritten += bytesToWrite; } - output.sampleMetadata(timeUs, frameType == VIDEO_FRAME_KEYFRAME ? C.BUFFER_FLAG_KEY_FRAME : 0, - bytesWritten, 0, null); + output.sampleMetadata( + timeUs, isKeyframe ? C.BUFFER_FLAG_KEY_FRAME : 0, bytesWritten, 0, null); + hasOutputKeyframe = true; + return true; + } else { + return false; } }