diff --git a/lib/player.js b/lib/player.js
index 67adda8146..f390694800 100644
--- a/lib/player.js
+++ b/lib/player.js
@@ -6161,6 +6161,9 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
    * @private
    */
   restoreDisabledVariants_(updateAbrManager=true) {
+    if (this.loadMode_ != shaka.Player.LoadMode.MEDIA_SOURCE) {
+      return;
+    }
     goog.asserts.assert(this.manifest_, 'Should have manifest!');
 
     shaka.log.v2('Restoring all disabled streams...');