diff --git a/lib/transmuxer/aac_transmuxer.js b/lib/transmuxer/aac_transmuxer.js
index 9bca37a64d..5e11e77145 100644
--- a/lib/transmuxer/aac_transmuxer.js
+++ b/lib/transmuxer/aac_transmuxer.js
@@ -43,7 +43,7 @@ shaka.transmuxer.AacTransmuxer = class {
    * @export
    */
   destroy() {
-    // Nothing
+    this.initSegments.clear();
   }
 
 
diff --git a/lib/transmuxer/mp3_transmuxer.js b/lib/transmuxer/mp3_transmuxer.js
index 3323348e0f..4771876f8d 100644
--- a/lib/transmuxer/mp3_transmuxer.js
+++ b/lib/transmuxer/mp3_transmuxer.js
@@ -32,8 +32,8 @@ shaka.transmuxer.Mp3Transmuxer = class {
     /** @private {number} */
     this.frameIndex_ = 0;
 
-    /** @private {?Uint8Array} */
-    this.initSegment = null;
+    /** @private {!Map.<number, !Uint8Array>} */
+    this.initSegments = new Map();
   }
 
 
@@ -42,7 +42,7 @@ shaka.transmuxer.Mp3Transmuxer = class {
    * @export
    */
   destroy() {
-    // Nothing
+    this.initSegments.clear();
   }
 
 
@@ -179,13 +179,17 @@ shaka.transmuxer.Mp3Transmuxer = class {
       stream: stream,
     };
     const mp4Generator = new shaka.util.Mp4Generator(streamInfo);
-    if (!this.initSegment) {
-      this.initSegment = mp4Generator.initSegment();
+    let initSegment;
+    if (!this.initSegments.has(stream.id)) {
+      initSegment = mp4Generator.initSegment();
+      this.initSegments.set(stream.id, initSegment);
+    } else {
+      initSegment = this.initSegments.get(stream.id);
     }
     const segmentData = mp4Generator.segmentData();
 
     this.frameIndex_++;
-    const transmuxData = Uint8ArrayUtils.concat(this.initSegment, segmentData);
+    const transmuxData = Uint8ArrayUtils.concat(initSegment, segmentData);
     return Promise.resolve(transmuxData);
   }
 };