diff --git a/lib/media/gap_jumping_controller.js b/lib/media/gap_jumping_controller.js
index cea9059c58..27d93ab6c1 100644
--- a/lib/media/gap_jumping_controller.js
+++ b/lib/media/gap_jumping_controller.js
@@ -58,6 +58,9 @@ shaka.media.GapJumpingController = class {
     /** @private {number} */
     this.prevReadyState_ = video.readyState;
 
+    /** @private {number} */
+    this.startTime_ = 0;
+
     /** @private {number} */
     this.gapsJumped_ = 0;
 
@@ -123,6 +126,18 @@ shaka.media.GapJumpingController = class {
     this.onPollGapJump_();
   }
 
+  /**
+   * Called when playback has started and the video element is
+   * listening for seeks.
+   *
+   * @param {number} startTime
+   */
+  onStarted(startTime) {
+    if (this.video_.seeking && !this.seekingEventReceived_) {
+      this.seekingEventReceived_ = true;
+      this.startTime_ = startTime;
+    }
+  }
 
   /** Called when a seek has started. */
   onSeeking() {
@@ -164,8 +179,8 @@ shaka.media.GapJumpingController = class {
     // while paused on a livestream.  We make an exception for time 0, since we
     // may be _required_ to seek on startup before play can begin, but only if
     // autoplay is enabled.
-    if (this.video_.paused && (this.video_.currentTime != 0 ||
-      (!this.video_.autoplay && this.video_.currentTime == 0))) {
+    if (this.video_.paused && (this.video_.currentTime != this.startTime_ ||
+      (!this.video_.autoplay && this.video_.currentTime == this.startTime_))) {
       return;
     }
 
diff --git a/lib/media/playhead.js b/lib/media/playhead.js
index 9cad395943..27a3a2e4aa 100644
--- a/lib/media/playhead.js
+++ b/lib/media/playhead.js
@@ -251,6 +251,7 @@ shaka.media.MediaSourcePlayhead = class {
     this.videoWrapper_ = new shaka.media.VideoWrapper(
         mediaElement,
         () => this.onSeeking_(),
+        (realStartTime) => this.onStarted_(realStartTime),
         this.getStartTime_(startTime));
 
     /** @type {shaka.util.Timer} */
@@ -390,6 +391,16 @@ shaka.media.MediaSourcePlayhead = class {
     }
   }
 
+  /**
+   * Called when the video element has started up and is listening for new seeks
+   *
+   * @param {number} startTime
+   * @private
+   */
+  onStarted_(startTime) {
+    this.gapController_.onStarted(startTime);
+  }
+
   /**
    * Handles when a seek happens on the video.
    *
diff --git a/lib/media/video_wrapper.js b/lib/media/video_wrapper.js
index 6ddc5e0346..a76539e804 100644
--- a/lib/media/video_wrapper.js
+++ b/lib/media/video_wrapper.js
@@ -27,15 +27,19 @@ shaka.media.VideoWrapper = class {
   /**
    * @param {!HTMLMediaElement} video
    * @param {function()} onSeek Called when the video seeks.
+   * @param {function(number)} onStarted Called when the video has started.
    * @param {number} startTime The time to start at.
    */
-  constructor(video, onSeek, startTime) {
+  constructor(video, onSeek, onStarted, startTime) {
     /** @private {HTMLMediaElement} */
     this.video_ = video;
 
     /** @private {function()} */
     this.onSeek_ = onSeek;
 
+    /** @private {function(number)} */
+    this.onStarted_ = onStarted;
+
     /** @private {number} */
     this.startTime_ = startTime;
 
@@ -165,6 +169,7 @@ shaka.media.VideoWrapper = class {
     this.started_ = true;
 
     this.eventManager_.listen(this.video_, 'seeking', () => this.onSeek_());
+    this.onStarted_(this.video_.currentTime);
   }
 };