diff --git a/lib/util/stream_utils.js b/lib/util/stream_utils.js
index c852706863..412fc7210a 100644
--- a/lib/util/stream_utils.js
+++ b/lib/util/stream_utils.js
@@ -768,26 +768,38 @@ shaka.util.StreamUtils = class {
         }
 
         if (audio) {
-          if (!keySystemConfig.audio) {
+          // See: https://github.com/shaka-project/shaka-player/issues/4659
+          if (info.audioRobustness != '') {
+            if (!keySystemConfig.audio) {
+              // KeySystemTrackConfiguration
+              keySystemConfig.audio = {
+                robustness: info.audioRobustness,
+              };
+            } else {
+              keySystemConfig.audio.robustness =
+                  keySystemConfig.audio.robustness || info.audioRobustness;
+            }
+          } else if (!keySystemConfig.audio) {
             // KeySystemTrackConfiguration
-            keySystemConfig.audio = {
-              robustness: info.audioRobustness,
-            };
-          } else {
-            keySystemConfig.audio.robustness =
-                keySystemConfig.audio.robustness || info.audioRobustness;
+            keySystemConfig.audio = {};
           }
         }
 
         if (video) {
-          if (!keySystemConfig.video) {
+          // See: https://github.com/shaka-project/shaka-player/issues/4659
+          if (info.videoRobustness != '') {
+            if (!keySystemConfig.video) {
+              // KeySystemTrackConfiguration
+              keySystemConfig.video = {
+                robustness: info.videoRobustness,
+              };
+            } else {
+              keySystemConfig.video.robustness =
+                  keySystemConfig.video.robustness || info.videoRobustness;
+            }
+          } else if (!keySystemConfig.video) {
             // KeySystemTrackConfiguration
-            keySystemConfig.video = {
-              robustness: info.videoRobustness,
-            };
-          } else {
-            keySystemConfig.video.robustness =
-                keySystemConfig.video.robustness || info.videoRobustness;
+            keySystemConfig.video = {};
           }
         }
       }