diff --git a/lib/abr/ewma_bandwidth_estimator.js b/lib/abr/ewma_bandwidth_estimator.js
index c7cffff070..4de4934c26 100644
--- a/lib/abr/ewma_bandwidth_estimator.js
+++ b/lib/abr/ewma_bandwidth_estimator.js
@@ -70,6 +70,14 @@ shaka.abr.EwmaBandwidthEstimator = function() {
    * @const
    */
   this.minBytes_ = 65536;
+
+
+  /**
+   * Minimum amount of time (in ms), under which samples are discarded.
+   * @private {number}
+   * @const
+   */
+  this.minTime_ = 200;
 };
 
 
@@ -90,7 +98,11 @@ shaka.abr.EwmaBandwidthEstimator.DEFAULT_ESTIMATE = 5e5;  // 500kbps
  */
 shaka.abr.EwmaBandwidthEstimator.prototype.sample = function(
     durationMs, numBytes) {
-  if (numBytes < this.minBytes_) {
+  // Ignore small requests.  This will protect against both caching and
+  // outliers.  Small requests in size will tend to vary a lot in the time they
+  // take, and overhead of JavaScript timings have a big effect.  Also, cached
+  // responses will always return in a small time.
+  if (numBytes < this.minBytes_ || durationMs < this.minTime_) {
     return;
   }