diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js
index 7a2acdc0e2..e98287254e 100644
--- a/lib/media/media_source_engine.js
+++ b/lib/media/media_source_engine.js
@@ -408,6 +408,13 @@ shaka.media.MediaSourceEngine = class {
               'expected \'open\'');
         }
 
+        if (sequenceMode && !streamsByType.has(ContentType.TEXT)) {
+          // There's no text stream, so we can set sequence mode early instead
+          // of setting it after the first segment is appended in appendBuffer_.
+          sourceBuffer.mode =
+              shaka.media.MediaSourceEngine.SourceBufferMode_.SEQUENCE;
+        }
+
         this.eventManager_.listen(
             sourceBuffer, 'error',
             () => this.onError_(contentType));
diff --git a/test/media/media_source_engine_unit.js b/test/media/media_source_engine_unit.js
index 4806977212..b3b9b62ed9 100644
--- a/test/media/media_source_engine_unit.js
+++ b/test/media/media_source_engine_unit.js
@@ -639,6 +639,7 @@ describe('MediaSourceEngine', () => {
       };
       const initObject = new Map();
       initObject.set(ContentType.VIDEO, fakeVideoStream);
+      initObject.set(ContentType.TEXT, fakeTextStream);
 
       await mediaSourceEngine.init(initObject, /* sequenceMode= */ true);