diff --git a/lib/util/periods.js b/lib/util/periods.js
index e814f67a49..0b9dfc9dee 100644
--- a/lib/util/periods.js
+++ b/lib/util/periods.js
@@ -460,9 +460,7 @@ shaka.util.PeriodCombiner = class {
       outputStreams, streamsPerPeriod, firstNewPeriodIndex, clone, concat,
       periodsMissing) {
     const unusedStreamsPerPeriod = [];
-    for (let i = 0; i < periodsMissing; i++) {
-      unusedStreamsPerPeriod.push(new Set());
-    }
+
     for (let i = 0; i < streamsPerPeriod.length; i++) {
       if (i >= firstNewPeriodIndex) {
         // This periods streams are all new.
@@ -478,7 +476,7 @@ shaka.util.PeriodCombiner = class {
       // eslint-disable-next-line no-await-in-loop
       const ok = await this.extendExistingOutputStream_(
           outputStream, streamsPerPeriod, firstNewPeriodIndex, concat,
-          unusedStreamsPerPeriod, periodsMissing > 0);
+          unusedStreamsPerPeriod, periodsMissing);
       if (!ok) {
         // This output Stream was not properly extended to include streams from
         // the new period.  This is likely a bug in our algorithm, so throw an
@@ -509,11 +507,6 @@ shaka.util.PeriodCombiner = class {
     }  // for (const unusedStreams of unusedStreamsPerPeriod)
 
     for (const unusedStreams of unusedStreamsPerPeriod) {
-      // eslint-disable-next-line no-restricted-syntax
-      if (unusedStreamsPerPeriod.indexOf(unusedStreams) <
-          periodsMissing && unusedStreams.size == 0) {
-        continue;
-      }
       for (const stream of unusedStreams) {
         if (shaka.util.PeriodCombiner.isDummy_(stream)) {
           // This is one of our dummy streams, so ignore it.  We may not use
@@ -554,7 +547,7 @@ shaka.util.PeriodCombiner = class {
    *   of the first.
    * @param {!Array.<!Set.<T>>} unusedStreamsPerPeriod An array of sets of
    *   unused streams from each period.
-   * @param {boolean} shouldAppend shall extend existing matching streams.
+   * @param {number} periodsMissing How many periods are missing in this update.
    *
    * @return {!Promise.<boolean>}
    *
@@ -566,9 +559,9 @@ shaka.util.PeriodCombiner = class {
    */
   async extendExistingOutputStream_(
       outputStream, streamsPerPeriod, firstNewPeriodIndex, concat,
-      unusedStreamsPerPeriod, shouldAppend) {
+      unusedStreamsPerPeriod, periodsMissing) {
     this.findMatchesInAllPeriods_(streamsPerPeriod,
-        outputStream, shouldAppend);
+        outputStream, periodsMissing > 0);
 
     // This only exists where T == Stream, and this should only ever be called
     // on Stream types.  StreamDB should not have pre-existing output streams.
@@ -589,7 +582,7 @@ shaka.util.PeriodCombiner = class {
     }
 
     shaka.util.PeriodCombiner.extendOutputStream_(outputStream,
-        firstNewPeriodIndex, concat, unusedStreamsPerPeriod);
+        firstNewPeriodIndex, concat, unusedStreamsPerPeriod, periodsMissing);
     return true;
   }
 
@@ -696,7 +689,8 @@ shaka.util.PeriodCombiner = class {
       return null;
     }
     shaka.util.PeriodCombiner.extendOutputStream_(outputStream,
-        /* firstNewPeriodIndex= */ 0, concat, unusedStreamsPerPeriod);
+        /* firstNewPeriodIndex= */ 0, concat, unusedStreamsPerPeriod,
+        /* periodsMissing= */ 0);
 
     return outputStream;
   }
@@ -710,6 +704,7 @@ shaka.util.PeriodCombiner = class {
    *   of the first.
    * @param {!Array.<!Set.<T>>} unusedStreamsPerPeriod An array of sets of
    *   unused streams from each period.
+   * @param {number} periodsMissing How many periods are missing in this update
    *
    * @template T
    * Accepts either a StreamDB or Stream type.
@@ -717,7 +712,8 @@ shaka.util.PeriodCombiner = class {
    * @private
    */
   static extendOutputStream_(
-      outputStream, firstNewPeriodIndex, concat, unusedStreamsPerPeriod) {
+      outputStream, firstNewPeriodIndex, concat, unusedStreamsPerPeriod,
+      periodsMissing) {
     const ContentType = shaka.util.ManifestParserUtils.ContentType;
     const LanguageUtils = shaka.util.LanguageUtils;
     const matches = outputStream.matchedStreams;
@@ -729,7 +725,8 @@ shaka.util.PeriodCombiner = class {
 
     // Concatenate the new matches onto the stream, starting at the first new
     // period.
-    for (let i = firstNewPeriodIndex; i < matches.length; i++) {
+    const start = firstNewPeriodIndex + periodsMissing;
+    for (let i = start; i < matches.length; i++) {
       const match = matches[i];
       concat(outputStream, match);
 
@@ -747,7 +744,7 @@ shaka.util.PeriodCombiner = class {
       }
 
       if (used) {
-        unusedStreamsPerPeriod[i].delete(match);
+        unusedStreamsPerPeriod[i - periodsMissing].delete(match);
         // Add the codec of this stream to the output stream's codecs.
         const codecs = new Set(outputStream.codecs.split(','));
         for (const codec of match.codecs.split(',')) {