Fix cue render order for ass/ssa.

RELEASENOTES.md

@@ -69,6 +69,10 @@
         SubStation Alpha subtitles (but ExoPlayer's parsing logic is the same
         for both variants)
         ([#2384](https://github.com/androidx/media/issues/2384)).
+    *   Add support for the `layer` property in SubStation Alpha (SSA) subtitle
+        files which is used to define the z-order of cues when more than one is
+        shown on screen at the same time
+        ([#2124](https://github.com/androidx/media/issues/2124)).
 *   Metadata:
 *   Image:
 *   DataSource:

libraries/common/src/main/java/androidx/media3/common/text/Cue.java

@@ -309,6 +309,10 @@ public final class Cue {
    */
   public final float shearDegrees;
 
+  /** The Z index for cue, the larger index will render above the smaller index. May be negative. */
+  @UnstableApi
+  public final int zIndex;
+
   private Cue(
       @Nullable CharSequence text,
       @Nullable Alignment textAlignment,
@@ -326,7 +330,8 @@ private Cue(
       boolean windowColorSet,
       int windowColor,
       @VerticalType int verticalType,
-      float shearDegrees) {
+      float shearDegrees,
+      int zIndex) {
     // Exactly one of text or bitmap should be set.
     if (text == null) {
       Assertions.checkNotNull(bitmap);
@@ -356,6 +361,7 @@ private Cue(
     this.textSize = textSize;
     this.verticalType = verticalType;
     this.shearDegrees = shearDegrees;
+    this.zIndex = zIndex;
   }
 
   /** Returns a new {@link Cue.Builder} initialized with the same values as this Cue. */
@@ -391,7 +397,8 @@ public boolean equals(@Nullable Object obj) {
         && textSizeType == that.textSizeType
         && textSize == that.textSize
         && verticalType == that.verticalType
-        && shearDegrees == that.shearDegrees;
+        && shearDegrees == that.shearDegrees
+        && zIndex == that.zIndex;
   }
 
   @Override
@@ -413,7 +420,8 @@ public int hashCode() {
         textSizeType,
         textSize,
         verticalType,
-        shearDegrees);
+        shearDegrees,
+        zIndex);
   }
 
   /** A builder for {@link Cue} objects. */
@@ -436,6 +444,7 @@ public static final class Builder {
     @ColorInt private int windowColor;
     private @VerticalType int verticalType;
     private float shearDegrees;
+    private int zIndex;
 
     public Builder() {
       text = null;
@@ -474,6 +483,7 @@ private Builder(Cue cue) {
       windowColor = cue.windowColor;
       verticalType = cue.verticalType;
       shearDegrees = cue.shearDegrees;
+      zIndex = cue.zIndex;
     }
 
     /**
@@ -806,6 +816,19 @@ public Builder setShearDegrees(float shearDegrees) {
       return verticalType;
     }
 
+    /** Sets the zIndex for this Cue. */
+    @CanIgnoreReturnValue
+    public Builder setZIndex(int zIndex) {
+      this.zIndex = zIndex;
+      return this;
+    }
+
+    /** Gets the zIndex for this Cue. */
+    @Pure
+    public int getZIndex() {
+      return zIndex;
+    }
+
     /** Build the cue. */
     public Cue build() {
       return new Cue(
@@ -825,7 +848,8 @@ public Cue build() {
           windowColorSet,
           windowColor,
           verticalType,
-          shearDegrees);
+          shearDegrees,
+          zIndex);
     }
   }
 
@@ -848,6 +872,7 @@ public Cue build() {
   private static final String FIELD_WINDOW_COLOR_SET = Util.intToStringMaxRadix(14);
   private static final String FIELD_VERTICAL_TYPE = Util.intToStringMaxRadix(15);
   private static final String FIELD_SHEAR_DEGREES = Util.intToStringMaxRadix(16);
+  private static final String FIELD_Z_INDEX = Util.intToStringMaxRadix(19);
 
   /**
    * Returns a {@link Bundle} that can be serialized to bytes.
@@ -923,6 +948,7 @@ private Bundle toBundleWithoutBitmap() {
     bundle.putInt(FIELD_WINDOW_COLOR, windowColor);
     bundle.putInt(FIELD_VERTICAL_TYPE, verticalType);
     bundle.putFloat(FIELD_SHEAR_DEGREES, shearDegrees);
+    bundle.putInt(FIELD_Z_INDEX, zIndex);
     return bundle;
   }
 
@@ -995,6 +1021,9 @@ public static Cue fromBundle(Bundle bundle) {
     if (bundle.containsKey(FIELD_SHEAR_DEGREES)) {
       builder.setShearDegrees(bundle.getFloat(FIELD_SHEAR_DEGREES));
     }
+    if (bundle.containsKey(FIELD_Z_INDEX)) {
+      builder.setZIndex(bundle.getInt(FIELD_Z_INDEX));
+    }
     return builder.build();
   }
 }

libraries/common/src/main/java/androidx/media3/common/text/CueGroup.java

@@ -23,12 +23,17 @@
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Ordering;
 import java.util.ArrayList;
 import java.util.List;
 
 /** Class to represent the state of active {@link Cue Cues} at a particular time. */
 public final class CueGroup {
 
+  /** An {@link Ordering} which sorts cues in ascending zIndex priority */
+  private static final Ordering<Cue> CUES_PRIORITY_COMPARATOR =
+      Ordering.<Integer>natural().onResultOf(c -> c.zIndex);
+
   /** An empty group with no {@link Cue Cues} and presentation time of zero. */
   @UnstableApi
   public static final CueGroup EMPTY_TIME_ZERO =
@@ -54,7 +59,7 @@ public final class CueGroup {
   /** Creates a CueGroup. */
   @UnstableApi
   public CueGroup(List<Cue> cues, long presentationTimeUs) {
-    this.cues = ImmutableList.copyOf(cues);
+    this.cues = ImmutableList.sortedCopyOf(CUES_PRIORITY_COMPARATOR, cues);
     this.presentationTimeUs = presentationTimeUs;
   }
 

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/SsaPlaybackTest.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2025 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.e2etest;
+
+import static androidx.media3.test.utils.robolectric.TestPlayerRunHelper.advance;
+
+import android.content.Context;
+import android.graphics.SurfaceTexture;
+import android.net.Uri;
+import android.view.Surface;
+import androidx.media3.common.C;
+import androidx.media3.common.MediaItem;
+import androidx.media3.common.MimeTypes;
+import androidx.media3.common.Player;
+import androidx.media3.exoplayer.ExoPlayer;
+import androidx.media3.test.utils.CapturingRenderersFactory;
+import androidx.media3.test.utils.DumpFileAsserts;
+import androidx.media3.test.utils.FakeClock;
+import androidx.media3.test.utils.robolectric.PlaybackOutput;
+import androidx.media3.test.utils.robolectric.ShadowMediaCodecConfig;
+import androidx.test.core.app.ApplicationProvider;
+import com.google.common.collect.ImmutableList;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.ParameterizedRobolectricTestRunner;
+
+/** End-to-end tests using side-loaded SSA subtitles. */
+@RunWith(ParameterizedRobolectricTestRunner.class)
+public class SsaPlaybackTest {
+  @ParameterizedRobolectricTestRunner.Parameters(name = "{0}")
+  public static ImmutableList<String> mediaSamples() {
+    return ImmutableList.of("overlapping_cues_different_layers");
+  }
+
+  @ParameterizedRobolectricTestRunner.Parameter public String inputFile;
+
+  @Rule
+  public ShadowMediaCodecConfig mediaCodecConfig =
+      ShadowMediaCodecConfig.withAllDefaultSupportedCodecs();
+
+  @Test
+  public void test() throws Exception {
+    Context applicationContext = ApplicationProvider.getApplicationContext();
+    CapturingRenderersFactory capturingRenderersFactory =
+        new CapturingRenderersFactory(applicationContext);
+    ExoPlayer player =
+        new ExoPlayer.Builder(applicationContext, capturingRenderersFactory)
+            .setClock(new FakeClock(/* isAutoAdvancing= */ true))
+            .build();
+    Surface surface = new Surface(new SurfaceTexture(/* texName= */ 1));
+    player.setVideoSurface(surface);
+    PlaybackOutput playbackOutput = PlaybackOutput.register(player, capturingRenderersFactory);
+    MediaItem mediaItem =
+        new MediaItem.Builder()
+            .setUri("asset:///media/mp4/preroll-5s.mp4")
+            .setSubtitleConfigurations(
+                ImmutableList.of(
+                    new MediaItem.SubtitleConfiguration.Builder(
+                            Uri.parse("asset:///media/ssa/" + inputFile))
+                        .setMimeType(MimeTypes.TEXT_SSA)
+                        .setLanguage("en")
+                        .setSelectionFlags(C.SELECTION_FLAG_DEFAULT)
+                        .build()))
+            .build();
+
+    player.setMediaItem(mediaItem);
+    player.prepare();
+    advance(player).untilState(Player.STATE_READY);
+    advance(player).untilFullyBuffered();
+    player.play();
+    advance(player).untilState(Player.STATE_ENDED);
+    player.release();
+    surface.release();
+
+    DumpFileAsserts.assertOutput(
+        applicationContext, playbackOutput, "playbackdumps/ssa/" + inputFile + ".dump");
+  }
+}

libraries/extractor/src/main/java/androidx/media3/extractor/text/ssa/SsaDialogueFormat.java

@@ -32,14 +32,21 @@
  */
 /* package */ final class SsaDialogueFormat {
 
+  public final int layerIndex;
   public final int startTimeIndex;
   public final int endTimeIndex;
   public final int styleIndex;
   public final int textIndex;
   public final int length;
 
   private SsaDialogueFormat(
-      int startTimeIndex, int endTimeIndex, int styleIndex, int textIndex, int length) {
+      int layerIndex,
+      int startTimeIndex,
+      int endTimeIndex,
+      int styleIndex,
+      int textIndex,
+      int length) {
+    this.layerIndex = layerIndex;
     this.startTimeIndex = startTimeIndex;
     this.endTimeIndex = endTimeIndex;
     this.styleIndex = styleIndex;
@@ -54,6 +61,7 @@ private SsaDialogueFormat(
    */
   @Nullable
   public static SsaDialogueFormat fromFormatLine(String formatLine) {
+    int layerIndex = C.INDEX_UNSET;
     int startTimeIndex = C.INDEX_UNSET;
     int endTimeIndex = C.INDEX_UNSET;
     int styleIndex = C.INDEX_UNSET;
@@ -62,6 +70,9 @@ public static SsaDialogueFormat fromFormatLine(String formatLine) {
     String[] keys = TextUtils.split(formatLine.substring(FORMAT_LINE_PREFIX.length()), ",");
     for (int i = 0; i < keys.length; i++) {
       switch (Ascii.toLowerCase(keys[i].trim())) {
+        case "layer":
+          layerIndex = i;
+          break;
         case "start":
           startTimeIndex = i;
           break;
@@ -79,7 +90,8 @@ public static SsaDialogueFormat fromFormatLine(String formatLine) {
     return (startTimeIndex != C.INDEX_UNSET
             && endTimeIndex != C.INDEX_UNSET
             && textIndex != C.INDEX_UNSET)
-        ? new SsaDialogueFormat(startTimeIndex, endTimeIndex, styleIndex, textIndex, keys.length)
+        ? new SsaDialogueFormat(
+            layerIndex, startTimeIndex, endTimeIndex, styleIndex, textIndex, keys.length)
         : null;
   }
 }

libraries/extractor/src/main/java/androidx/media3/extractor/text/ssa/SsaParser.java

@@ -328,6 +328,15 @@ private void parseDialogueLine(
       return;
     }
 
+    int layer = 0;
+    if (format.layerIndex != C.INDEX_UNSET) {
+      try {
+        layer = Integer.parseInt(lineValues[format.layerIndex].trim());
+      } catch (RuntimeException exception) {
+        Log.w(TAG, "Fail to parse layer: " + lineValues[format.layerIndex]);
+      }
+    }
+
     long startTimeUs = parseTimecodeUs(lineValues[format.startTimeIndex]);
     if (startTimeUs == C.TIME_UNSET) {
       Log.w(TAG, "Skipping invalid timing: " + dialogueLine);
@@ -352,7 +361,7 @@ private void parseDialogueLine(
             .replace("\\N", "\n")
             .replace("\\n", "\n")
             .replace("\\h", "\u00A0");
-    Cue cue = createCue(text, style, styleOverrides, screenWidth, screenHeight);
+    Cue cue = createCue(text, layer, style, styleOverrides, screenWidth, screenHeight);
 
     int startTimeIndex = addCuePlacerholderByTime(startTimeUs, cueTimesUs, cues);
     int endTimeIndex = addCuePlacerholderByTime(endTimeUs, cueTimesUs, cues);
@@ -383,12 +392,13 @@ private static long parseTimecodeUs(String timeString) {
 
   private static Cue createCue(
       String text,
+      int layer,
       @Nullable SsaStyle style,
       SsaStyle.Overrides styleOverrides,
       float screenWidth,
       float screenHeight) {
     SpannableString spannableText = new SpannableString(text);
-    Cue.Builder cue = new Cue.Builder().setText(spannableText);
+    Cue.Builder cue = new Cue.Builder().setText(spannableText).setZIndex(layer);
 
     if (style != null) {
       if (style.primaryColor != null) {

libraries/extractor/src/test/java/androidx/media3/extractor/text/ssa/SsaParserTest.java

@@ -56,6 +56,8 @@ public final class SsaParserTest {
   private static final String INVALID_TIMECODES = "media/ssa/invalid_timecodes";
   private static final String INVALID_POSITIONS = "media/ssa/invalid_positioning";
   private static final String POSITIONS_WITHOUT_PLAYRES = "media/ssa/positioning_without_playres";
+  private static final String LAYERS = "media/ssa/layer";
+  private static final String INVALID_LAYERS = "media/ssa/invalid_layer";
   private static final String STYLE_PRIMARY_COLOR = "media/ssa/style_primary_color";
   private static final String STYLE_OUTLINE_COLOR = "media/ssa/style_outline_color";
   private static final String STYLE_FONT_SIZE = "media/ssa/style_font_size";
@@ -411,6 +413,41 @@ public void parseInvalidPositions() throws IOException {
     assertThat(fourthCue.line).isEqualTo(0.5f);
   }
 
+  @Test
+  public void parseLayer() throws IOException {
+    SsaParser parser = new SsaParser();
+    byte[] bytes = TestUtil.getByteArray(ApplicationProvider.getApplicationContext(), LAYERS);
+    ImmutableList<CuesWithTiming> allCues = parseAllCues(parser, bytes);
+
+    // Check positive layer.
+    Cue firstCue = Iterables.getOnlyElement(allCues.get(0).cues);
+    assertThat(firstCue.zIndex).isEqualTo(1);
+
+    // Check negative layer.
+    Cue secondCue = Iterables.getOnlyElement(allCues.get(1).cues);
+    assertThat(secondCue.zIndex).isEqualTo(-1);
+  }
+
+  @Test
+  public void parseInvalidLayer() throws IOException {
+    SsaParser parser = new SsaParser();
+    byte[] bytes =
+        TestUtil.getByteArray(ApplicationProvider.getApplicationContext(), INVALID_LAYERS);
+    ImmutableList<CuesWithTiming> allCues = parseAllCues(parser, bytes);
+
+    // Check empty layer.
+    Cue firstCue = Iterables.getOnlyElement(allCues.get(0).cues);
+    assertThat(firstCue.zIndex).isEqualTo(0);
+
+    // Check non-numeric layer.
+    Cue secondCue = Iterables.getOnlyElement(allCues.get(1).cues);
+    assertThat(secondCue.zIndex).isEqualTo(0);
+
+    // Check non-integer layer.
+    Cue thirdCue = Iterables.getOnlyElement(allCues.get(2).cues);
+    assertThat(thirdCue.zIndex).isEqualTo(0);
+  }
+
   @Test
   public void parsePositionsWithMissingPlayResY() throws IOException {
     SsaParser parser = new SsaParser();