FLV extractor fixes

1. Only output video starting from a keyframe
2. When calculating the timestamp offset to adjust live streams to start
   at t=0, use the timestamp of the first tag from which a sample is actually
   output, rather than just the first audio/video tag. The test streams in
   the referenced GitHub issue start with a video tag whose packet type is
   AVC_PACKET_TYPE_SEQUENCE_HEADER (i.e. does not contain a sample) and whose
   timestamp is set to 0 (i.e. isn't set). The timestamp is set correctly on
   tags that from which a sample is actually output.

Issue: #6111
PiperOrigin-RevId: 256147747

RELEASENOTES.md

@@ -20,6 +20,8 @@
 * Wrap decoder exceptions in a new `DecoderException` class and report as
   renderer error.
 * SmoothStreaming: Parse text stream `Subtype` into `Format.roleFlags`.
+* FLV: Fix bug that caused playback of some live streams to not start
+  ([#6111](https://github.com/google/ExoPlayer/issues/6111)).
 
 ### 2.10.2 ###
 

library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/AudioTagPayloadReader.java

@@ -86,11 +86,12 @@ protected boolean parseHeader(ParsableByteArray data) throws UnsupportedFormatEx
   }
 
   @Override
-  protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserException {
+  protected boolean parsePayload(ParsableByteArray data, long timeUs) throws ParserException {
     if (audioFormat == AUDIO_FORMAT_MP3) {
       int sampleSize = data.bytesLeft();
       output.sampleData(data, sampleSize);
       output.sampleMetadata(timeUs, C.BUFFER_FLAG_KEY_FRAME, sampleSize, 0, null);
+      return true;
     } else {
       int packetType = data.readUnsignedByte();
       if (packetType == AAC_PACKET_TYPE_SEQUENCE_HEADER && !hasOutputFormat) {
@@ -104,12 +105,15 @@ protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserEx
             Collections.singletonList(audioSpecificConfig), null, 0, null);
         output.format(format);
         hasOutputFormat = true;
+        return false;
       } else if (audioFormat != AUDIO_FORMAT_AAC || packetType == AAC_PACKET_TYPE_AAC_RAW) {
         int sampleSize = data.bytesLeft();
         output.sampleData(data, sampleSize);
         output.sampleMetadata(timeUs, C.BUFFER_FLAG_KEY_FRAME, sampleSize, 0, null);
+        return true;
+      } else {
+        return false;
       }
     }
   }
-
 }

library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/FlvExtractor.java

@@ -73,6 +73,7 @@ public final class FlvExtractor implements Extractor {
 
   private ExtractorOutput extractorOutput;
   private @States int state;
+  private boolean outputFirstSample;
   private long mediaTagTimestampOffsetUs;
   private int bytesToNextTagHeader;
   private int tagType;
@@ -89,7 +90,6 @@ public FlvExtractor() {
     tagData = new ParsableByteArray();
     metadataReader = new ScriptTagPayloadReader();
     state = STATE_READING_FLV_HEADER;
-    mediaTagTimestampOffsetUs = C.TIME_UNSET;
   }
 
   @Override
@@ -131,7 +131,7 @@ public void init(ExtractorOutput output) {
   @Override
   public void seek(long position, long timeUs) {
     state = STATE_READING_FLV_HEADER;
-    mediaTagTimestampOffsetUs = C.TIME_UNSET;
+    outputFirstSample = false;
     bytesToNextTagHeader = 0;
   }
 
@@ -252,14 +252,16 @@ private boolean readTagHeader(ExtractorInput input) throws IOException, Interrup
    */
   private boolean readTagData(ExtractorInput input) throws IOException, InterruptedException {
     boolean wasConsumed = true;
+    boolean wasSampleOutput = false;
+    long timestampUs = getCurrentTimestampUs();
     if (tagType == TAG_TYPE_AUDIO && audioReader != null) {
       ensureReadyForMediaOutput();
-      audioReader.consume(prepareTagData(input), mediaTagTimestampOffsetUs + tagTimestampUs);
+      wasSampleOutput = audioReader.consume(prepareTagData(input), timestampUs);
     } else if (tagType == TAG_TYPE_VIDEO && videoReader != null) {
       ensureReadyForMediaOutput();
-      videoReader.consume(prepareTagData(input), mediaTagTimestampOffsetUs + tagTimestampUs);
+      wasSampleOutput = videoReader.consume(prepareTagData(input), timestampUs);
     } else if (tagType == TAG_TYPE_SCRIPT_DATA && !outputSeekMap) {
-      metadataReader.consume(prepareTagData(input), tagTimestampUs);
+      wasSampleOutput = metadataReader.consume(prepareTagData(input), timestampUs);
       long durationUs = metadataReader.getDurationUs();
       if (durationUs != C.TIME_UNSET) {
         extractorOutput.seekMap(new SeekMap.Unseekable(durationUs));
@@ -269,6 +271,11 @@ private boolean readTagData(ExtractorInput input) throws IOException, Interrupte
       input.skipFully(tagDataSize);
       wasConsumed = false;
     }
+    if (!outputFirstSample && wasSampleOutput) {
+      outputFirstSample = true;
+      mediaTagTimestampOffsetUs =
+          metadataReader.getDurationUs() == C.TIME_UNSET ? -tagTimestampUs : 0;
+    }
     bytesToNextTagHeader = 4; // There's a 4 byte previous tag size before the next header.
     state = STATE_SKIPPING_TO_TAG_HEADER;
     return wasConsumed;
@@ -291,10 +298,11 @@ private void ensureReadyForMediaOutput() {
       extractorOutput.seekMap(new SeekMap.Unseekable(C.TIME_UNSET));
       outputSeekMap = true;
     }
-    if (mediaTagTimestampOffsetUs == C.TIME_UNSET) {
-      mediaTagTimestampOffsetUs =
-          metadataReader.getDurationUs() == C.TIME_UNSET ? -tagTimestampUs : 0;
-    }
   }
 
+  private long getCurrentTimestampUs() {
+    return outputFirstSample
+        ? (mediaTagTimestampOffsetUs + tagTimestampUs)
+        : (metadataReader.getDurationUs() == C.TIME_UNSET ? 0 : tagTimestampUs);
+  }
 }

library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/ScriptTagPayloadReader.java

@@ -63,7 +63,7 @@ protected boolean parseHeader(ParsableByteArray data) {
   }
 
   @Override
-  protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserException {
+  protected boolean parsePayload(ParsableByteArray data, long timeUs) throws ParserException {
     int nameType = readAmfType(data);
     if (nameType != AMF_TYPE_STRING) {
       // Should never happen.
@@ -72,12 +72,12 @@ protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserEx
     String name = readAmfString(data);
     if (!NAME_METADATA.equals(name)) {
       // We're only interested in metadata.
-      return;
+      return false;
     }
     int type = readAmfType(data);
     if (type != AMF_TYPE_ECMA_ARRAY) {
       // We're not interested in this metadata.
-      return;
+      return false;
     }
     // Set the duration to the value contained in the metadata, if present.
     Map<String, Object> metadata = readAmfEcmaArray(data);
@@ -87,6 +87,7 @@ protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserEx
         durationUs = (long) (durationSeconds * C.MICROS_PER_SECOND);
       }
     }
+    return false;
   }
 
   private static int readAmfType(ParsableByteArray data) {

library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/TagPayloadReader.java

@@ -58,12 +58,11 @@ protected TagPayloadReader(TrackOutput output) {
    *
    * @param data The payload data to consume.
    * @param timeUs The timestamp associated with the payload.
+   * @return Whether a sample was output.
    * @throws ParserException If an error occurs parsing the data.
    */
-  public final void consume(ParsableByteArray data, long timeUs) throws ParserException {
-    if (parseHeader(data)) {
-      parsePayload(data, timeUs);
-    }
+  public final boolean consume(ParsableByteArray data, long timeUs) throws ParserException {
+    return parseHeader(data) && parsePayload(data, timeUs);
   }
 
   /**
@@ -78,10 +77,11 @@ public final void consume(ParsableByteArray data, long timeUs) throws ParserExce
   /**
    * Parses tag payload.
    *
-   * @param data Buffer where tag payload is stored
-   * @param timeUs Time position of the frame
+   * @param data Buffer where tag payload is stored.
+   * @param timeUs Time position of the frame.
+   * @return Whether a sample was output.
    * @throws ParserException If an error occurs parsing the payload.
    */
-  protected abstract void parsePayload(ParsableByteArray data, long timeUs) throws ParserException;
-
+  protected abstract boolean parsePayload(ParsableByteArray data, long timeUs)
+      throws ParserException;
 }

library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/VideoTagPayloadReader.java

@@ -47,6 +47,7 @@
 
   // State variables.
   private boolean hasOutputFormat;
+  private boolean hasOutputKeyframe;
   private int frameType;
 
   /**
@@ -60,7 +61,7 @@ public VideoTagPayloadReader(TrackOutput output) {
 
   @Override
   public void seek() {
-    // Do nothing.
+    hasOutputKeyframe = false;
   }
 
   @Override
@@ -77,7 +78,7 @@ protected boolean parseHeader(ParsableByteArray data) throws UnsupportedFormatEx
   }
 
   @Override
-  protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserException {
+  protected boolean parsePayload(ParsableByteArray data, long timeUs) throws ParserException {
     int packetType = data.readUnsignedByte();
     int compositionTimeMs = data.readInt24();
 
@@ -94,7 +95,12 @@ protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserEx
           avcConfig.initializationData, Format.NO_VALUE, avcConfig.pixelWidthAspectRatio, null);
       output.format(format);
       hasOutputFormat = true;
+      return false;
     } else if (packetType == AVC_PACKET_TYPE_AVC_NALU && hasOutputFormat) {
+      boolean isKeyframe = frameType == VIDEO_FRAME_KEYFRAME;
+      if (!hasOutputKeyframe && !isKeyframe) {
+        return false;
+      }
       // TODO: Deduplicate with Mp4Extractor.
       // Zero the top three bytes of the array that we'll use to decode nal unit lengths, in case
       // they're only 1 or 2 bytes long.
@@ -123,8 +129,12 @@ protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserEx
         output.sampleData(data, bytesToWrite);
         bytesWritten += bytesToWrite;
       }
-      output.sampleMetadata(timeUs, frameType == VIDEO_FRAME_KEYFRAME ? C.BUFFER_FLAG_KEY_FRAME : 0,
-          bytesWritten, 0, null);
+      output.sampleMetadata(
+          timeUs, isKeyframe ? C.BUFFER_FLAG_KEY_FRAME : 0, bytesWritten, 0, null);
+      hasOutputKeyframe = true;
+      return true;
+    } else {
+      return false;
     }
   }