diff --git a/build/types/core b/build/types/core
index fe2d4311bc..3a61689f34 100644
--- a/build/types/core
+++ b/build/types/core
@@ -9,6 +9,7 @@
 +../../lib/debug/log.js
 
 +../../lib/media/drm_engine.js
++../../lib/media/gap_jumping_controller.js
 +../../lib/media/manifest_parser.js
 +../../lib/media/media_source_engine.js
 +../../lib/media/mp4_segment_index_parser.js
@@ -20,6 +21,7 @@
 +../../lib/media/streaming_engine.js
 +../../lib/media/time_ranges_utils.js
 +../../lib/media/transmuxer.js
++../../lib/media/video_wrapper.js
 +../../lib/media/webm_segment_index_parser.js
 
 +../../lib/net/backoff.js
diff --git a/lib/media/gap_jumping_controller.js b/lib/media/gap_jumping_controller.js
new file mode 100644
index 0000000000..00d241f328
--- /dev/null
+++ b/lib/media/gap_jumping_controller.js
@@ -0,0 +1,267 @@
+/**
+ * @license
+ * Copyright 2016 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+goog.provide('shaka.media.GapJumpingController');
+
+goog.require('shaka.log');
+goog.require('shaka.media.TimeRangesUtils');
+goog.require('shaka.util.EventManager');
+goog.require('shaka.util.FakeEvent');
+goog.require('shaka.util.IDestroyable');
+goog.require('shaka.util.Timer');
+
+
+
+/**
+ * Creates a new GapJumpingController that handles jumping gaps that appear
+ * within the content.  This will only jump gaps between two buffered ranges,
+ * so we should not have to worry about the availability window.
+ *
+ * @param {!HTMLMediaElement} video
+ * @param {shakaExtern.Manifest} manifest
+ * @param {shakaExtern.StreamingConfiguration} config
+ * @param {function(!Event)} onEvent Called when an event is raised to be sent
+ *   to the application.
+ *
+ * @constructor
+ * @struct
+ * @implements {shaka.util.IDestroyable}
+ */
+shaka.media.GapJumpingController = function(video, manifest, config, onEvent) {
+  /** @private {HTMLMediaElement} */
+  this.video_ = video;
+
+  /** @private {?shakaExtern.Manifest} */
+  this.manifest_ = manifest;
+
+  /** @private {?shakaExtern.StreamingConfiguration} */
+  this.config_ = config;
+
+  /** @private {?function(!Event)} */
+  this.onEvent_ = onEvent;
+
+  /** @private {shaka.util.EventManager} */
+  this.eventManager_ = new shaka.util.EventManager();
+
+  /** @private {?shaka.util.Timer} */
+  this.gapJumpTimer_ = null;
+
+  /** @private {boolean} */
+  this.seekingEventReceived_ = false;
+
+  /** @private {number} */
+  this.prevReadyState_ = video.readyState;
+
+  /** @private {boolean} */
+  this.didFireLargeGap_ = false;
+
+  /**
+   * The wall-clock time (in milliseconds) that the stall occurred.  This is
+   * used to ensure we don't flush the pipeline too often.
+   * @private {number}
+   */
+  this.stallWallTime_ = -1;
+
+  /**
+   * The playhead time where we think a stall occurred.  When the ready state
+   * says we don't have enough data and the playhead stops too long, we assume
+   * we have stalled.
+   * @private {number}
+   */
+  this.stallPlayheadTime_ = -1;
+
+  /**
+   * True if we have already flushed the pipeline at stallPlayheadTime_.
+   * Allows us to avoid flushing multiple times for the same stall.
+   * @private {boolean}
+   */
+  this.stallCorrected_ = false;
+
+  /** @private {boolean} */
+  this.hadSegmentAppended_ = false;
+
+
+  let pollGap = this.onPollGapJump_.bind(this);
+  this.eventManager_.listen(video, 'waiting', pollGap);
+
+  // We can't trust readyState or 'waiting' events on all platforms.  So poll
+  // the current time and if we are in a gap, jump it.
+  // See: https://goo.gl/sbSHp9 and https://goo.gl/cuAcYd
+  this.gapJumpTimer_ = new shaka.util.Timer(pollGap);
+  this.gapJumpTimer_.scheduleRepeated(0.25);
+};
+
+
+/** @override */
+shaka.media.GapJumpingController.prototype.destroy = function() {
+  let p = this.eventManager_.destroy();
+  this.eventManager_ = null;
+  this.video_ = null;
+  this.manifest_ = null;
+  this.onEvent_ = null;
+
+  if (this.gapJumpTimer_ != null) {
+    this.gapJumpTimer_.cancel();
+    this.gapJumpTimer_ = null;
+  }
+
+  return p;
+};
+
+
+/**
+ * Called when a segment is appended by StreamingEngine, but not when a clear is
+ * pending.  This means StreamingEngine will continue buffering forward from
+ * what is buffered. So we know about any gaps before the start.
+ */
+shaka.media.GapJumpingController.prototype.onSegmentAppended = function() {
+  this.hadSegmentAppended_ = true;
+  this.onPollGapJump_();
+};
+
+
+/** Called when a seek has started. */
+shaka.media.GapJumpingController.prototype.onSeeking = function() {
+  this.seekingEventReceived_ = true;
+  this.hadSegmentAppended_ = false;
+  this.didFireLargeGap_ = false;
+};
+
+
+/**
+ * Called on a recurring timer to check for gaps in the media.  This is also
+ * called in a 'waiting' event.
+ *
+ * @private
+ */
+shaka.media.GapJumpingController.prototype.onPollGapJump_ = function() {
+  // Don't gap jump before the video is ready to play.
+  if (this.video_.readyState == 0) return;
+  // Do not gap jump if seeking has begun, but the seeking event has not
+  // yet fired for this particular seek.
+  if (this.video_.seeking) {
+    if (!this.seekingEventReceived_)
+      return;
+  } else {
+    this.seekingEventReceived_ = false;
+  }
+  // Don't gap jump while paused, so that you don't constantly jump ahead while
+  // paused on a livestream.
+  if (this.video_.paused) return;
+
+
+  // When the ready state changes, we have moved on, so we should fire the large
+  // gap event if we see one.
+  if (this.video_.readyState != this.prevReadyState_) {
+    this.didFireLargeGap_ = false;
+    this.prevReadyState_ = this.video_.readyState;
+  }
+
+  const smallGapLimit = this.config_.smallGapLimit;
+  let currentTime = this.video_.currentTime;
+  let buffered = this.video_.buffered;
+
+  let gapIndex = shaka.media.TimeRangesUtils.getGapIndex(buffered, currentTime);
+
+  // The current time is unbuffered or is too far from a gap.
+  if (gapIndex == null) {
+    this.handleStall_();
+    return;
+  }
+  // If we are before the first buffered range, this could be an unbuffered
+  // seek.  So wait until a segment is appended so we are sure it is a gap.
+  if (gapIndex == 0 && !this.hadSegmentAppended_)
+    return;
+
+  // StreamingEngine can buffer past the seek end, but still don't allow seeking
+  // past it.
+  let jumpTo = buffered.start(gapIndex);
+  let seekEnd = this.manifest_.presentationTimeline.getSeekRangeEnd();
+  if (jumpTo >= seekEnd)
+    return;
+
+  let jumpSize = jumpTo - currentTime;
+  let isGapSmall = jumpSize <= smallGapLimit;
+  let jumpLargeGap = false;
+
+  if (!isGapSmall && !this.didFireLargeGap_) {
+    this.didFireLargeGap_ = true;
+
+    // Event firing is synchronous.
+    let event = new shaka.util.FakeEvent(
+        'largegap', {'currentTime': currentTime, 'gapSize': jumpSize});
+    event.cancelable = true;
+    this.onEvent_(event);
+
+    if (this.config_.jumpLargeGaps && !event.defaultPrevented)
+      jumpLargeGap = true;
+    else
+      shaka.log.info('Ignoring large gap at', currentTime);
+  }
+
+  if (isGapSmall || jumpLargeGap) {
+    if (gapIndex == 0) {
+      shaka.log.info(
+          'Jumping forward', jumpSize,
+          'seconds because of gap before start time of', jumpTo);
+    } else {
+      shaka.log.info(
+          'Jumping forward', jumpSize, 'seconds because of gap starting at',
+          buffered.end(gapIndex - 1), 'and ending at', jumpTo);
+    }
+
+    this.video_.currentTime = jumpTo;
+  }
+};
+
+
+/**
+ * This determines if we are stalled inside a buffered range and corrects it if
+ * possible.
+ * @private
+ */
+shaka.media.GapJumpingController.prototype.handleStall_ = function() {
+  let currentTime = this.video_.currentTime;
+  let buffered = this.video_.buffered;
+
+  if (this.video_.readyState < 3 && this.video_.playbackRate > 0) {
+    // Some platforms/browsers can get stuck in the middle of a buffered range
+    // (e.g. when seeking in a background tab).  Flush the media pipeline to
+    // help. Flush once we have stopped for more than 1 second inside a buffered
+    // range.
+    if (this.stallPlayheadTime_ != currentTime) {
+      this.stallPlayheadTime_ = currentTime;
+      this.stallWallTime_ = Date.now();
+      this.stallCorrected_ = false;
+    } else if (!this.stallCorrected_ &&
+               this.stallWallTime_ < Date.now() - 1000) {
+      for (let i = 0; i < buffered.length; i++) {
+        // Ignore the end of the buffered range since it may not play any more
+        // on all platforms.
+        if (currentTime >= buffered.start(i) &&
+            currentTime < buffered.end(i) - 0.5) {
+          shaka.log.debug(
+              'Flushing media pipeline due to stall inside buffered range');
+          this.video_.currentTime += 0.1;
+          this.stallPlayheadTime_ = this.video_.currentTime;
+          this.stallCorrected_ = true;
+          break;
+        }
+      }
+    }
+  }
+};
diff --git a/lib/media/playhead.js b/lib/media/playhead.js
index 8fff136002..3110a57682 100644
--- a/lib/media/playhead.js
+++ b/lib/media/playhead.js
@@ -19,9 +19,9 @@ goog.provide('shaka.media.Playhead');
 
 goog.require('goog.asserts');
 goog.require('shaka.log');
+goog.require('shaka.media.GapJumpingController');
 goog.require('shaka.media.TimeRangesUtils');
-goog.require('shaka.util.EventManager');
-goog.require('shaka.util.FakeEvent');
+goog.require('shaka.media.VideoWrapper');
 goog.require('shaka.util.IDestroyable');
 goog.require('shaka.util.StreamUtils');
 goog.require('shaka.util.Timer');
@@ -35,7 +35,7 @@ goog.require('shaka.util.Timer');
  * restricting seeking to valid time ranges, and stopping playback for startup
  * and re- buffering.
  *
- * @param {HTMLMediaElement} video
+ * @param {!HTMLMediaElement} video
  * @param {shakaExtern.Manifest} manifest
  * @param {shakaExtern.StreamingConfiguration} config
  * @param {?number} startTime The playhead's initial position in seconds. If
@@ -61,130 +61,45 @@ shaka.media.Playhead = function(
   /** @private {?shakaExtern.StreamingConfiguration} */
   this.config_ = config;
 
-  /**
-   * The playhead's initial position in seconds, or null if it should
-   * automatically be calculated later.
-   * @private {?number}
-   */
-  this.startTime_ = startTime == null ? null :
-                    this.clampSeekToDuration_(startTime);
-
   /** @private {?function()} */
   this.onSeek_ = onSeek;
 
-  /** @private {?function(!Event)} */
-  this.onEvent_ = onEvent;
-
-  /** @private {shaka.util.EventManager} */
-  this.eventManager_ = new shaka.util.EventManager();
-
-  /** @private {boolean} */
-  this.buffering_ = false;
-
-  /** @private {number} */
-  this.playbackRate_ = 1;
-
   /** @private {?shaka.util.Timer} */
-  this.trickPlayTimer_ = null;
+  this.checkWindowTimer_ = null;
 
-  /** @private {?shaka.util.Timer} */
-  this.gapJumpTimer_ = null;
-
-  /** @private {boolean} */
-  this.seekingEventReceived_ = false;
-
-  /**
-   * Used to batch up early seeks and delay until video.currentTime is updated.
-   *
-   * @private {shaka.util.Timer}
-   */
-  this.earlySeekTimer_ = new shaka.util.Timer(this.onEarlySeek_.bind(this));
-
-  /** @private {number} */
-  this.prevReadyState_ = video.readyState;
-
-  /** @private {boolean} */
-  this.didFireLargeGap_ = false;
-
-  /**
-   * The wall-clock time (in milliseconds) that the stall occurred.  This is
-   * used to ensure we don't flush the pipeline too often.
-   * @private {number}
-   */
-  this.stallWallTime_ = -1;
-
-  /**
-   * The playhead time where we think a stall occurred.  When the ready state
-   * says we don't have enough data and the playhead stops too long, we assume
-   * we have stalled.
-   * @private {number}
-   */
-  this.stallPlayheadTime_ = -1;
-
-  /**
-   * True if we have already flushed the pipeline at stallPlayheadTime_.
-   * Allows us to avoid flushing multiple times for the same stall.
-   * @private {boolean}
-   */
-  this.stallCorrected_ = false;
-
-  /** @private {boolean} */
-  this.hadSegmentAppended_ = false;
-
-
-  // Check if the video has already loaded some metadata.
-  if (video.readyState > 0) {
-    this.onLoadedMetadata_();
-  } else {
-    this.eventManager_.listenOnce(
-        video, 'loadedmetadata', this.onLoadedMetadata_.bind(this));
-
-    // Check for early seeks before we have any media loaded.
-    // See onEarlySeek_() for details.
-    this.eventManager_.listen(video, 'timeupdate', function() {
-      // In practice, the value of video.currentTime doesn't change before the
-      // event is fired.  Delay 100ms and batch up changes.
-      this.earlySeekTimer_.schedule(0.1 /* seconds */);
-    }.bind(this));
-  }
+  /** @private {shaka.media.GapJumpingController} */
+  this.gapController_ =
+      new shaka.media.GapJumpingController(video, manifest, config, onEvent);
+
+  /** @private {shaka.media.VideoWrapper} */
+  this.videoWrapper_ = new shaka.media.VideoWrapper(
+      video, this.onSeeking_.bind(this), this.getStartTime_(startTime));
 
-  var pollGap = this.onPollGapJump_.bind(this);
-  this.eventManager_.listen(video, 'ratechange', this.onRateChange_.bind(this));
-  this.eventManager_.listen(video, 'waiting', pollGap);
 
-  // We can't trust readyState or 'waiting' events on all platforms.  So poll
-  // the current time and if we are in a gap, jump it.
-  // See: https://goo.gl/sbSHp9 and https://goo.gl/cuAcYd
-  this.gapJumpTimer_ = new shaka.util.Timer(pollGap);
-  this.gapJumpTimer_.scheduleRepeated(0.25);
+  let poll = this.onPollWindow_ .bind(this);
+  this.checkWindowTimer_ = new shaka.util.Timer(poll);
+  this.checkWindowTimer_.scheduleRepeated(0.25);
 };
 
 
 /** @override */
 shaka.media.Playhead.prototype.destroy = function() {
-  var p = this.eventManager_.destroy();
-  this.eventManager_ = null;
-
-  if (this.trickPlayTimer_ != null) {
-    this.trickPlayTimer_.cancel();
-    this.trickPlayTimer_ = null;
-  }
-
-  if (this.gapJumpTimer_ != null) {
-    this.gapJumpTimer_.cancel();
-    this.gapJumpTimer_ = null;
-  }
-
-  if (this.earlySeekTimer_ != null) {
-    this.earlySeekTimer_.cancel();
-    this.earlySeekTimer_ = null;
+  var p = Promise.all([
+    this.videoWrapper_.destroy(),
+    this.gapController_.destroy(),
+  ]);
+  this.videoWrapper_ = null;
+  this.gapController_ = null;
+
+  if (this.checkWindowTimer_ != null) {
+    this.checkWindowTimer_.cancel();
+    this.checkWindowTimer_ = null;
   }
 
   this.video_ = null;
   this.manifest_ = null;
   this.config_ = null;
   this.onSeek_ = null;
-  this.onEvent_ = null;
 
   return p;
 };
@@ -197,35 +112,7 @@ shaka.media.Playhead.prototype.destroy = function() {
  * @param {number} startTime
  */
 shaka.media.Playhead.prototype.setStartTime = function(startTime) {
-  if (this.video_.readyState > 0)
-    this.video_.currentTime = this.clampTime_(startTime);
-  else
-    this.startTime_ = startTime;
-};
-
-
-/**
- * Clamp seek times and playback start times so that we never seek to the
- * presentation duration.  Seeking to or starting at duration does not work
- * consistently across browsers.
- *
- * TODO: Clean up and simplify Playhead.  There are too many layers of, methods
- * for, and conditions on timestamp adjustment.
- *
- * @see https://github.com/google/shaka-player/issues/979
- * @param {number} time
- * @return {number} The adjusted seek time.
- * @private
- */
-shaka.media.Playhead.prototype.clampSeekToDuration_ = function(time) {
-  var timeline = this.manifest_.presentationTimeline;
-  var duration = timeline.getDuration();
-  if (time >= duration) {
-    goog.asserts.assert(this.config_.durationBackoff >= 0,
-                        'Duration backoff must be non-negative!');
-    return duration - this.config_.durationBackoff;
-  }
-  return time;
+  this.videoWrapper_.setTime(startTime);
 };
 
 
@@ -235,53 +122,46 @@ shaka.media.Playhead.prototype.clampSeekToDuration_ = function(time) {
  * @return {number}
  */
 shaka.media.Playhead.prototype.getTime = function() {
+  var time = this.videoWrapper_.getTime();
   if (this.video_.readyState > 0) {
     // We don't buffer when the livestream video is paused and the playhead time
     // is out of the seek range, thus we do not clamp the current time when the
     // video is paused.
     // https://github.com/google/shaka-player/issues/1121
-    if (this.video_.paused) return this.video_.currentTime;
-
+    //
     // Although we restrict the video's currentTime elsewhere, clamp it here to
     // ensure any timing issues (e.g., the user agent seeks and calls this
     // function before we receive the 'seeking' event) don't cause us to return
     // a time outside the segment availability window.
-    return this.clampTime_(this.video_.currentTime);
+    if (!this.video_.paused)
+      time = this.clampTime_(time);
   }
 
-  return this.getStartTime_();
+  return time;
 };
 
 
 /**
  * Gets the playhead's initial position in seconds.
  *
+ * @param {?number} startTime
  * @return {number}
  * @private
  */
-shaka.media.Playhead.prototype.getStartTime_ = function() {
-  if (this.startTime_ != null) {
-    return this.clampTime_(this.startTime_);
-  }
-
-  var startTime;
-  var timeline = this.manifest_.presentationTimeline;
-  if (timeline.getDuration() < Infinity) {
-    // If the presentation is VOD, or if the presentation is live but has
-    // finished broadcasting, then start from the beginning.
-    startTime = timeline.getSegmentAvailabilityStart();
-  } else {
-    // Otherwise, start near the live-edge.
-    startTime = timeline.getSeekRangeEnd();
+shaka.media.Playhead.prototype.getStartTime_ = function(startTime) {
+  if (startTime == null) {
+    var timeline = this.manifest_.presentationTimeline;
+    if (timeline.getDuration() < Infinity) {
+      // If the presentation is VOD, or if the presentation is live but has
+      // finished broadcasting, then start from the beginning.
+      startTime = timeline.getSegmentAvailabilityStart();
+    } else {
+      // Otherwise, start near the live-edge.
+      startTime = timeline.getSeekRangeEnd();
+    }
   }
 
-  // If we don't set this, getStartTime_ will continue to wander forward.  For
-  // a drifting stream, this is very bad, as we will never fall back into a
-  // playable range.
-  // TODO: re-evaluate after #999 (drift tolerance refactor) is resolved
-  this.startTime_ = this.clampSeekToDuration_(startTime);
-
-  return startTime;
+  return this.clampSeekToDuration_(this.clampTime_(startTime));
 };
 
 
@@ -292,10 +172,7 @@ shaka.media.Playhead.prototype.getStartTime_ = function() {
  *   continue.
  */
 shaka.media.Playhead.prototype.setBuffering = function(buffering) {
-  if (buffering != this.buffering_) {
-    this.buffering_ = buffering;
-    this.setPlaybackRate(this.playbackRate_);
-  }
+  this.videoWrapper_.setBuffering(buffering);
 };
 
 
@@ -305,7 +182,7 @@ shaka.media.Playhead.prototype.setBuffering = function(buffering) {
  * @return {number}
  */
 shaka.media.Playhead.prototype.getPlaybackRate = function() {
-  return this.playbackRate_;
+  return this.videoWrapper_.getPlaybackRate();
 };
 
 
@@ -314,25 +191,7 @@ shaka.media.Playhead.prototype.getPlaybackRate = function() {
  * @param {number} rate
  */
 shaka.media.Playhead.prototype.setPlaybackRate = function(rate) {
-  if (this.trickPlayTimer_ != null) {
-    this.trickPlayTimer_.cancel();
-    this.trickPlayTimer_ = null;
-  }
-
-  this.playbackRate_ = rate;
-  // All major browsers support playback rates above zero.  Only need fake
-  // trick play for negative rates.
-  this.video_.playbackRate = (this.buffering_ || rate < 0) ? 0 : rate;
-
-  if (!this.buffering_ && rate < 0) {
-    // Defer creating the timer until we stop buffering.  This function will be
-    // called again from setBuffering().
-    var trickPlay = (function() {
-      this.video_.currentTime += rate / 4;
-    }.bind(this));
-    this.trickPlayTimer_ = new shaka.util.Timer(trickPlay);
-    this.trickPlayTimer_.scheduleRepeated(0.25);
-  }
+  this.videoWrapper_.setPlaybackRate(rate);
 };
 
 
@@ -342,268 +201,78 @@ shaka.media.Playhead.prototype.setPlaybackRate = function(rate) {
  * what is buffered. So we know about any gaps before the start.
  */
 shaka.media.Playhead.prototype.onSegmentAppended = function() {
-  this.hadSegmentAppended_ = true;
-  this.onPollGapJump_();
-};
-
-
-/**
- * Handles a 'ratechange' event.
- *
- * @private
- */
-shaka.media.Playhead.prototype.onRateChange_ = function() {
-  // NOTE: This will not allow explicitly setting the playback rate to 0 while
-  // the playback rate is negative.  Pause will still work.
-  var expectedRate =
-      this.buffering_ || this.playbackRate_ < 0 ? 0 : this.playbackRate_;
-
-  // Native controls in Edge trigger a change to playbackRate and set it to 0
-  // when seeking.  If we don't exclude 0 from this check, we will force the
-  // rate to stay at 0 after a seek with Edge native controls.
-  // https://github.com/google/shaka-player/issues/951
-  if (this.video_.playbackRate && this.video_.playbackRate != expectedRate) {
-    shaka.log.debug('Video playback rate changed to', this.video_.playbackRate);
-    this.setPlaybackRate(this.video_.playbackRate);
-  }
-};
-
-
-/**
- * Handles a 'loadedmetadata' event.
- *
- * @private
- */
-shaka.media.Playhead.prototype.onLoadedMetadata_ = function() {
-  // Move the real playhead to the start time.
-  var targetTime = this.getStartTime_();
-
-  // We are out of the phase in which "early seek" may occur, so stop listening
-  // for the 'timeupdate' event.
-  this.eventManager_.unlisten(this.video_, 'timeupdate');
-
-  // Cancel any pending early seek timer.  Cancelation is important because it
-  // resolves a race between the first "timeupdate" and the "loadedmetadata"
-  // event.  On some browsers, "timeupdate" fires right before "loadedmetadata",
-  // which would cause earlySeekTimer_ to fire inappropriately and trigger a
-  // failed assertion.
-  this.earlySeekTimer_.cancel();
-
-  if (Math.abs(this.video_.currentTime - targetTime) < 0.001) {
-    this.eventManager_.listen(
-        this.video_, 'seeking', this.onSeeking_.bind(this));
-    this.eventManager_.listen(
-        this.video_, 'playing', this.onPlaying_.bind(this));
-  } else {
-    this.eventManager_.listenOnce(
-        this.video_, 'seeking', this.onSeekingToStartTime_.bind(this));
-    this.video_.currentTime = targetTime;
-  }
-};
-
-
-/**
- * Handles the 'seeking' event from the initial jump to the start time (if
- * there is one).
- *
- * @private
- */
-shaka.media.Playhead.prototype.onSeekingToStartTime_ = function() {
-  goog.asserts.assert(this.video_.readyState > 0,
-                      'readyState should be greater than 0');
-  this.eventManager_.listen(this.video_, 'seeking', this.onSeeking_.bind(this));
-  this.eventManager_.listen(this.video_, 'playing', this.onPlaying_.bind(this));
+  this.gapController_.onSegmentAppended();
 };
 
 
 /**
- * Called on a recurring timer to check for gaps in the media.  This is also
- * called in a 'waiting' event.
+ * Called on a recurring timer to keep the playhead from falling outside the
+ * availability window.
  *
  * @private
  */
-shaka.media.Playhead.prototype.onPollGapJump_ = function() {
-  // Don't gap jump before the video is ready to play.
-  if (this.video_.readyState == 0) return;
-  // Do not gap jump if seeking has begun, but the seeking event has not
-  // yet fired for this particular seek.
-  if (this.video_.seeking) {
-    if (!this.seekingEventReceived_)
-      return;
-  } else {
-    this.seekingEventReceived_ = false;
-  }
-  // Don't gap jump while paused, so that you don't constantly jump ahead while
-  // paused on a livestream.
-  if (this.video_.paused) return;
-
-  // When the ready state changes, we have moved on, so we should fire the large
-  // gap event if we see one.
-  if (this.video_.readyState != this.prevReadyState_) {
-    this.didFireLargeGap_ = false;
-    this.prevReadyState_ = this.video_.readyState;
-  }
-
-  var smallGapLimit = this.config_.smallGapLimit;
-  var currentTime = this.video_.currentTime;
-  var buffered = this.video_.buffered;
+shaka.media.Playhead.prototype.onPollWindow_ = function() {
+  // Don't keep in window when there is no content, we are seeking, or we are
+  // paused.
+  if (this.video_.readyState == 0 || this.video_.seeking || this.video_.paused)
+    return;
 
-  // If seeking is not possible, clamp the playhead manually here.
-  var timeline = this.manifest_.presentationTimeline;
-  var availabilityStart = timeline.getSegmentAvailabilityStart();
+  let currentTime = this.video_.currentTime;
+  let timeline = this.manifest_.presentationTimeline;
+  let availabilityStart = timeline.getSegmentAvailabilityStart();
   if (currentTime < availabilityStart) {
     // The availability window has moved past the playhead.
     // Move ahead to catch up.
-    var targetTime = this.reposition_(currentTime);
+    let targetTime = this.reposition_(currentTime);
     shaka.log.info('Jumping forward ' + (targetTime - currentTime) +
         ' seconds to catch up with the availability window.');
-    this.movePlayhead_(currentTime, targetTime);
-    return;
-  }
-
-  var gapIndex = shaka.media.TimeRangesUtils.getGapIndex(buffered, currentTime);
-
-  // The current time is unbuffered or is too far from a gap.
-  if (gapIndex == null) {
-    if (this.video_.readyState < 3 && this.video_.playbackRate > 0) {
-      // Some platforms/browsers can get stuck in the middle of a buffered range
-      // (e.g. when seeking in a background tab).  Flush the media pipeline to
-      // help.
-      //
-      // Flush once we have stopped for more than 1 second inside a buffered
-      // range.  Note that Chromecast takes a few seconds to start playing
-      // after any kind of seek, so wait 5 seconds between repeated flushes.
-      if (this.stallPlayheadTime_ != currentTime) {
-        this.stallPlayheadTime_ = currentTime;
-        this.stallWallTime_ = Date.now();
-        this.stallCorrected_ = false;
-      } else if (!this.stallCorrected_ &&
-                 this.stallWallTime_ < Date.now() - 1000) {
-        for (var i = 0; i < buffered.length; i++) {
-          // Ignore the end of the buffered range since it may not play any more
-          // on all platforms.
-          if (currentTime >= buffered.start(i) &&
-              currentTime < buffered.end(i) - 0.5) {
-            shaka.log.debug(
-                'Flushing media pipeline due to stall inside buffered range');
-            this.video_.currentTime += 0.1;
-            this.stallPlayheadTime_ = this.video_.currentTime;
-            this.stallCorrected_ = true;
-            break;
-          }
-        }
-      }
-    }
-    return;
-  }
-  // If we are before the first buffered range, this could be an unbuffered
-  // seek.  So wait until a segment is appended so we are sure it is a gap.
-  if (gapIndex == 0 && !this.hadSegmentAppended_)
-    return;
-
-  // StreamingEngine can buffer past the seek end, but still don't allow seeking
-  // past it.
-  var jumpTo = buffered.start(gapIndex);
-  var seekEnd = this.manifest_.presentationTimeline.getSeekRangeEnd();
-  if (jumpTo >= seekEnd)
-    return;
-
-  var jumpSize = jumpTo - currentTime;
-  var isGapSmall = jumpSize <= smallGapLimit;
-  var jumpLargeGap = false;
-
-  if (!isGapSmall && !this.didFireLargeGap_) {
-    this.didFireLargeGap_ = true;
-
-    // Event firing is synchronous.
-    var event = new shaka.util.FakeEvent(
-        'largegap', {'currentTime': currentTime, 'gapSize': jumpSize});
-    event.cancelable = true;
-    this.onEvent_(event);
-
-    if (this.config_.jumpLargeGaps && !event.defaultPrevented)
-      jumpLargeGap = true;
-    else
-      shaka.log.info('Ignoring large gap at', currentTime);
-  }
-
-  if (isGapSmall || jumpLargeGap) {
-    if (gapIndex == 0) {
-      shaka.log.info(
-          'Jumping forward', jumpSize,
-          'seconds because of gap before start time of', jumpTo);
-    } else {
-      shaka.log.info(
-          'Jumping forward', jumpSize, 'seconds because of gap starting at',
-          buffered.end(gapIndex - 1), 'and ending at', jumpTo);
-    }
-
-    this.movePlayhead_(currentTime, jumpTo);
+    this.video_.currentTime = targetTime;
   }
 };
 
 
 /**
- * Handles a 'timeupdate' event that occurs before metadata is loaded, which
- * would indicate that the user is seeking.
- *
- * Note that a 'seeking' event will not fire before content is loaded.  In this
- * state, the playhead can only move as a result of a seek action, so timeupdate
- * is a good choice.
- *
- * @private
- */
-shaka.media.Playhead.prototype.onEarlySeek_ = function() {
-  goog.asserts.assert(this.video_.readyState == 0,
-                      'readyState should be 0 for early seeking');
-
-  var currentTime = this.video_.currentTime;
-  var targetTime = this.reposition_(currentTime);
-
-  shaka.log.v1('Early seek to', currentTime, 'remapped to', targetTime);
-  this.startTime_ = targetTime;
-};
-
-
-/**
- * Handles a 'seeking' event.
+ * Handles when a seek happens on the video.
  *
  * @private
  */
 shaka.media.Playhead.prototype.onSeeking_ = function() {
-  goog.asserts.assert(this.video_.readyState > 0,
-                      'readyState should be greater than 0');
-
-  this.seekingEventReceived_ = true;
-  this.hadSegmentAppended_ = false;
-  var currentTime = this.video_.currentTime;
+  this.gapController_.onSeeking();
+  var currentTime = this.videoWrapper_.getTime();
   var targetTime = this.reposition_(currentTime);
 
   if (Math.abs(targetTime - currentTime) > 0.001) {
-    this.movePlayhead_(currentTime, targetTime);
+    this.videoWrapper_.setTime(targetTime);
     return;
   }
 
   shaka.log.v1('Seek to ' + currentTime);
-  this.didFireLargeGap_ = false;
   this.onSeek_();
 };
 
 
 /**
- * Handles a 'playing' event.
+ * Clamp seek times and playback start times so that we never seek to the
+ * presentation duration.  Seeking to or starting at duration does not work
+ * consistently across browsers.
  *
+ * TODO: Clean up and simplify Playhead.  There are too many layers of, methods
+ * for, and conditions on timestamp adjustment.
+ *
+ * @see https://github.com/google/shaka-player/issues/979
+ * @param {number} time
+ * @return {number} The adjusted seek time.
  * @private
  */
-shaka.media.Playhead.prototype.onPlaying_ = function() {
-  goog.asserts.assert(this.video_.readyState > 0,
-                      'readyState should be greater than 0');
-
-  var currentTime = this.video_.currentTime;
-  var targetTime = this.reposition_(currentTime);
-
-  if (Math.abs(targetTime - currentTime) > 0.001)
-    this.movePlayhead_(currentTime, targetTime);
+shaka.media.Playhead.prototype.clampSeekToDuration_ = function(time) {
+  var timeline = this.manifest_.presentationTimeline;
+  var duration = timeline.getDuration();
+  if (time >= duration) {
+    goog.asserts.assert(this.config_.durationBackoff >= 0,
+                        'Duration backoff must be non-negative!');
+    return duration - this.config_.durationBackoff;
+  }
+  return time;
 };
 
 
@@ -676,47 +345,6 @@ shaka.media.Playhead.prototype.reposition_ = function(currentTime) {
 };
 
 
-/**
- * Moves the playhead to the target time, triggering a call to onSeeking_().
- *
- * @param {number} currentTime
- * @param {number} targetTime
- * @private
- */
-shaka.media.Playhead.prototype.movePlayhead_ = function(
-    currentTime, targetTime) {
-  shaka.log.debug('Moving playhead...',
-                  'currentTime=' + currentTime,
-                  'targetTime=' + targetTime);
-  this.video_.currentTime = targetTime;
-
-  // Sometimes, IE and Edge ignore re-seeks.  Check every 100ms and try
-  // again if need be, up to 10 tries.
-  // Delay stats over 100 runs of a re-seeking integration test:
-  // IE     -   0ms -  47%
-  // IE     - 100ms -  63%
-  // Edge   -   0ms -   2%
-  // Edge   - 100ms -  40%
-  // Edge   - 200ms -  32%
-  // Edge   - 300ms -  24%
-  // Edge   - 400ms -   2%
-  // Chrome -   0ms - 100%
-  // TODO: File a bug on IE/Edge about this.
-  var tries = 0;
-  var recheck = (function() {
-    if (!this.video_) return;
-    if (tries++ >= 10) return;
-
-    if (this.video_.currentTime == currentTime) {
-      // Sigh.  Try again.
-      this.video_.currentTime = targetTime;
-      setTimeout(recheck, 100);
-    }
-  }).bind(this);
-  setTimeout(recheck, 100);
-};
-
-
 /**
  * Clamps the given time to the segment availability window.
  *
diff --git a/lib/media/video_wrapper.js b/lib/media/video_wrapper.js
new file mode 100644
index 0000000000..211a6ece1f
--- /dev/null
+++ b/lib/media/video_wrapper.js
@@ -0,0 +1,310 @@
+/**
+ * @license
+ * Copyright 2016 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+goog.provide('shaka.media.VideoWrapper');
+
+goog.require('goog.asserts');
+goog.require('shaka.log');
+goog.require('shaka.util.EventManager');
+goog.require('shaka.util.IDestroyable');
+goog.require('shaka.util.Timer');
+
+
+
+/**
+ * Creates a new VideoWrapper that manages setting current time and playback
+ * rate.  This handles seeks before content is loaded and ensuring the video
+ * time is set properly.  This doesn't handle repositioning within the
+ * presentation window.
+ *
+ * @param {!HTMLMediaElement} video
+ * @param {function()} onSeek Called when the video seeks.
+ * @param {number} startTime The time to start at.
+ *
+ * @constructor
+ * @struct
+ * @implements {shaka.util.IDestroyable}
+ */
+shaka.media.VideoWrapper = function(video, onSeek, startTime) {
+  /** @private {HTMLMediaElement} */
+  this.video_ = video;
+
+  /** @private {?function()} */
+  this.onSeek_ = onSeek;
+
+  /** @private {number} */
+  this.startTime_ = startTime;
+
+  /** @private {shaka.util.EventManager} */
+  this.eventManager_ = new shaka.util.EventManager();
+
+  /** @private {number} */
+  this.playbackRate_ = 1;
+
+  /** @private {boolean} */
+  this.buffering_ = false;
+
+  /** @private {shaka.util.Timer} */
+  this.trickPlayTimer_ = null;
+
+  /**
+   * Used to batch up early seeks and delay until video.currentTime is updated.
+   *
+   * @private {shaka.util.Timer}
+   */
+  this.earlySeekTimer_ = new shaka.util.Timer(this.onEarlySeek_.bind(this));
+
+
+  // Check if the video has already loaded some metadata.
+  if (video.readyState > 0) {
+    this.onLoadedMetadata_();
+  } else {
+    this.eventManager_.listenOnce(
+        video, 'loadedmetadata', this.onLoadedMetadata_.bind(this));
+
+    // Check for early seeks before we have any media loaded.
+    // See onEarlySeek_() for details.
+    this.eventManager_.listen(video, 'timeupdate', () => {
+      // In practice, the value of video.currentTime doesn't change before the
+      // event is fired.  Delay 100ms and batch up changes.
+      this.earlySeekTimer_.schedule(0.1 /* seconds */);
+    });
+  }
+
+  this.eventManager_.listen(video, 'ratechange', this.onRateChange_.bind(this));
+};
+
+
+/** @override */
+shaka.media.VideoWrapper.prototype.destroy = function() {
+  let p = this.eventManager_.destroy();
+  this.eventManager_ = null;
+
+  if (this.trickPlayTimer_ != null) {
+    this.trickPlayTimer_.cancel();
+    this.trickPlayTimer_ = null;
+  }
+
+  if (this.earlySeekTimer_ != null) {
+    this.earlySeekTimer_.cancel();
+    this.earlySeekTimer_ = null;
+  }
+
+  this.video_ = null;
+  this.onSeek_ = null;
+
+  return p;
+};
+
+
+/**
+ * Gets the video's current (logical) position.
+ *
+ * @return {number}
+ */
+shaka.media.VideoWrapper.prototype.getTime = function() {
+  if (this.video_.readyState > 0)
+    return this.video_.currentTime;
+  else
+    return this.startTime_;
+};
+
+
+/**
+ * Sets the current time of the video.
+ *
+ * @param {number} time
+ */
+shaka.media.VideoWrapper.prototype.setTime = function(time) {
+  if (this.video_.readyState > 0) {
+    this.movePlayhead_(this.video_.currentTime, time);
+  } else {
+    this.startTime_ = time;
+    setTimeout(this.onSeek_, 0);
+  }
+};
+
+
+/**
+ * Gets the current effective playback rate.  This may be negative even if the
+ * browser does not directly support rewinding.
+ * @return {number}
+ */
+shaka.media.VideoWrapper.prototype.getPlaybackRate = function() {
+  return this.playbackRate_;
+};
+
+
+/**
+ * Sets the playback rate.
+ * @param {number} rate
+ */
+shaka.media.VideoWrapper.prototype.setPlaybackRate = function(rate) {
+  if (this.trickPlayTimer_ != null) {
+    this.trickPlayTimer_.cancel();
+    this.trickPlayTimer_ = null;
+  }
+
+  this.playbackRate_ = rate;
+  // All major browsers support playback rates above zero.  Only need fake
+  // trick play for negative rates.
+  this.video_.playbackRate = (this.buffering_ || rate < 0) ? 0 : rate;
+
+  if (!this.buffering_ && rate < 0) {
+    // Defer creating the timer until we stop buffering.  This function will be
+    // called again from setBuffering().
+    let trickPlay = () => { this.video_.currentTime += rate / 4; };
+    this.trickPlayTimer_ = new shaka.util.Timer(trickPlay);
+    this.trickPlayTimer_.scheduleRepeated(0.25);
+  }
+};
+
+
+/**
+ * Stops the playhead for buffering, or resumes the playhead after buffering.
+ *
+ * @param {boolean} buffering True to stop the playhead; false to allow it to
+ *   continue.
+ */
+shaka.media.VideoWrapper.prototype.setBuffering = function(buffering) {
+  if (buffering != this.buffering_) {
+    this.buffering_ = buffering;
+    this.setPlaybackRate(this.playbackRate_);
+  }
+};
+
+
+/**
+ * Handles a 'ratechange' event.
+ *
+ * @private
+ */
+shaka.media.VideoWrapper.prototype.onRateChange_ = function() {
+  // NOTE: This will not allow explicitly setting the playback rate to 0 while
+  // the playback rate is negative.  Pause will still work.
+  let expectedRate =
+      this.buffering_ || this.playbackRate_ < 0 ? 0 : this.playbackRate_;
+
+  // Native controls in Edge trigger a change to playbackRate and set it to 0
+  // when seeking.  If we don't exclude 0 from this check, we will force the
+  // rate to stay at 0 after a seek with Edge native controls.
+  // https://github.com/google/shaka-player/issues/951
+  if (this.video_.playbackRate && this.video_.playbackRate != expectedRate) {
+    shaka.log.debug('Video playback rate changed to', this.video_.playbackRate);
+    this.setPlaybackRate(this.video_.playbackRate);
+  }
+};
+
+
+/**
+ * Handles a 'loadedmetadata' event.
+ *
+ * @private
+ */
+shaka.media.VideoWrapper.prototype.onLoadedMetadata_ = function() {
+  // We are out of the phase in which "early seek" may occur, so stop listening
+  // for the 'timeupdate' event.
+  this.eventManager_.unlisten(this.video_, 'timeupdate');
+
+  // Cancel any pending early seek timer.  Cancelation is important because it
+  // resolves a race between the first "timeupdate" and the "loadedmetadata"
+  // event.  On some browsers, "timeupdate" fires right before "loadedmetadata",
+  // which would cause earlySeekTimer_ to fire inappropriately and trigger a
+  // failed assertion.
+  this.earlySeekTimer_.cancel();
+
+  if (Math.abs(this.video_.currentTime - this.startTime_) < 0.001) {
+    this.onSeekingToStartTime_();
+  } else {
+    this.eventManager_.listenOnce(
+        this.video_, 'seeking', this.onSeekingToStartTime_.bind(this));
+    this.video_.currentTime = this.startTime_;
+  }
+};
+
+
+/**
+ * Handles the 'seeking' event from the initial jump to the start time (if
+ * there is one).
+ *
+ * @private
+ */
+shaka.media.VideoWrapper.prototype.onSeekingToStartTime_ = function() {
+  goog.asserts.assert(this.video_.readyState > 0,
+                      'readyState should be greater than 0');
+  this.eventManager_.listen(this.video_, 'seeking', () => this.onSeek_());
+};
+
+
+/**
+ * Handles a 'timeupdate' event that occurs before metadata is loaded, which
+ * would indicate that the user is seeking.
+ *
+ * Note that a 'seeking' event will not fire before content is loaded.  In this
+ * state, the playhead can only move as a result of a seek action, so timeupdate
+ * is a good choice.
+ *
+ * @private
+ */
+shaka.media.VideoWrapper.prototype.onEarlySeek_ = function() {
+  goog.asserts.assert(this.video_.readyState == 0,
+                      'readyState should be 0 for early seeking');
+
+  this.startTime_ = this.video_.currentTime;
+  this.onSeek_();
+};
+
+
+/**
+ * Moves the playhead to the target time, triggering a call to onSeeking_().
+ *
+ * @param {number} currentTime
+ * @param {number} targetTime
+ * @private
+ */
+shaka.media.VideoWrapper.prototype.movePlayhead_ = function(
+    currentTime, targetTime) {
+  shaka.log.debug('Moving playhead...',
+                  'currentTime=' + currentTime,
+                  'targetTime=' + targetTime);
+  this.video_.currentTime = targetTime;
+
+  // Sometimes, IE and Edge ignore re-seeks.  Check every 100ms and try
+  // again if need be, up to 10 tries.
+  // Delay stats over 100 runs of a re-seeking integration test:
+  // IE     -   0ms -  47%
+  // IE     - 100ms -  63%
+  // Edge   -   0ms -   2%
+  // Edge   - 100ms -  40%
+  // Edge   - 200ms -  32%
+  // Edge   - 300ms -  24%
+  // Edge   - 400ms -   2%
+  // Chrome -   0ms - 100%
+  // TODO: File a bug on IE/Edge about this.
+  let tries = 0;
+  let recheck = () => {
+    if (!this.video_) return;
+    if (tries++ >= 10) return;
+
+    if (this.video_.currentTime == currentTime) {
+      // Sigh.  Try again.
+      this.video_.currentTime = targetTime;
+      setTimeout(recheck, 100);
+    }
+  };
+  setTimeout(recheck, 100);
+};
diff --git a/lib/player.js b/lib/player.js
index ff9fb8d289..1664b3557f 100644
--- a/lib/player.js
+++ b/lib/player.js
@@ -816,6 +816,7 @@ shaka.Player.prototype.createNetworkingEngine = function() {
  */
 shaka.Player.prototype.createPlayhead = function(opt_startTime) {
   goog.asserts.assert(this.manifest_, 'Must have manifest');
+  goog.asserts.assert(this.video_, 'Must have video');
   var startTime = opt_startTime == undefined ? null : opt_startTime;
   return new shaka.media.Playhead(
       this.video_, this.manifest_, this.config_.streaming,
diff --git a/test/media/playhead_unit.js b/test/media/playhead_unit.js
index 64cd79e2e4..5a64ce1df9 100644
--- a/test/media/playhead_unit.js
+++ b/test/media/playhead_unit.js
@@ -573,6 +573,7 @@ describe('Playhead', function() {
     timeline.getSegmentAvailabilityStart.and.returnValue(1030);
     timeline.getSegmentAvailabilityEnd.and.returnValue(1030);
     video.on['waiting']();
+    jasmine.clock().tick(500);
     // We expect this to move to 15 seconds ahead of the start of the
     // availability window, due to the rebuffering goal (10s) and the 5s
     // for the Chromecast.
@@ -610,7 +611,7 @@ describe('Playhead', function() {
 
       // Because this is buffered, the playhead should move to (start + 5),
       // which will cause a 'seeking' event.
-      video.on['playing']();
+      jasmine.clock().tick(500);
       expect(video.currentTime).toBe(15);
       video.on['seeking']();
       expect(playhead.getTime()).toBe(15);
@@ -641,7 +642,7 @@ describe('Playhead', function() {
       timeline.getSafeAvailabilityStart.and.returnValue(10);
       timeline.getSegmentAvailabilityEnd.and.returnValue(70);
 
-      video.on['playing']();
+      jasmine.clock().tick(500);
       expect(video.currentTime).toBe(10);
       video.on['seeking']();
       expect(playhead.getTime()).toBe(10);