feat: Network stall detection (shaka-project#3227)

Closes: shaka-project#1582

demo/common/message_ids.js

@@ -152,6 +152,7 @@ shakaDemo.MessageIds = {
   BUFFER_BEHIND: 'DEMO_BUFFER_BEHIND',
   BUFFERING_GOAL: 'DEMO_BUFFERING_GOAL',
   CLOCK_SYNC_URI: 'DEMO_CLOCK_SYNC_URI',
+  CONNECTION_TIMEOUT: 'DEMO_CONNECTION_TIMEOUT',
   DEFAULT_PRESENTATION_DELAY: 'DEMO_DEFAULT_PRESENTATION_DELAY',
   DELAY_LICENSE: 'DEMO_DELAY_LICENSE',
   DISABLE_AUDIO: 'DEMO_DISABLE_AUDIO',
@@ -215,6 +216,7 @@ shakaDemo.MessageIds = {
   SHAKA_CONTROLS: 'DEMO_SHAKA_CONTROLS',
   STALL_DETECTOR_ENABLED: 'DEMO_STALL_DETECTOR_ENABLED',
   STALL_THRESHOLD: 'DEMO_STALL_THRESHOLD',
+  STALL_TIMEOUT: 'DEMO_STALL_TIMEOUT',
   START_AT_SEGMENT_BOUNDARY: 'DEMO_START_AT_SEGMENT_BOUNDARY',
   STREAMING_RETRY_SECTION_HEADER: 'DEMO_STREAMING_RETRY_SECTION_HEADER',
   STREAMING_SECTION_HEADER: 'DEMO_STREAMING_SECTION_HEADER',

demo/config.js

@@ -280,6 +280,11 @@ shakaDemo.Config = class {
         .addNumberInput_(MessageIds.FUZZ_FACTOR, prefix + 'fuzzFactor',
             /* canBeDecimal= */ true)
         .addNumberInput_(MessageIds.TIMEOUT, prefix + 'timeout',
+            /* canBeDecimal= */ true)
+        .addNumberInput_(MessageIds.STALL_TIMEOUT, prefix + 'stallTimeout',
+            /* canBeDecimal= */ true)
+        .addNumberInput_(MessageIds.CONNECTION_TIMEOUT,
+            prefix + 'connectionTimeout',
             /* canBeDecimal= */ true);
   }
 

demo/locales/en.json

@@ -35,6 +35,7 @@
   "DEMO_CLOCK_SYNC_URI": "Clock Sync URI",
   "DEMO_COMPILED_DEBUG": "Compiled (Debug)",
   "DEMO_COMPILED_RELEASE": "Compiled (Release)",
+  "DEMO_CONNECTION_TIMEOUT": "Connection timeout",
   "DEMO_CONFIG": "Shaka Player Demo Config",
   "DEMO_CONTAINER_SEARCH": "Container",
   "DEMO_CUSTOM": "Custom",
@@ -168,6 +169,7 @@
   "DEMO_SOURCE_SEARCH": "Source",
   "DEMO_STALL_DETECTOR_ENABLED": "Stall Detector Enabled",
   "DEMO_STALL_THRESHOLD": "Stall Threshold",
+  "DEMO_STALL_TIMEOUT": "Stall timeout",
   "DEMO_START_AT_SEGMENT_BOUNDARY": "Start At Segment Boundary",
   "DEMO_STORED": "Downloaded",
   "DEMO_STORED_SEARCH": "Filters for assets that have been stored offline.",

demo/locales/source.json

@@ -143,6 +143,10 @@
     "description": "A link in the footer, to the release build of the demo.",
     "message": "Compiled (Release)"
   },
+  "DEMO_CONNECTION_TIMEOUT": {
+    "description": "The name of a configuration value.",
+    "message": "Connection timeout"
+  },
   "DEMO_CONFIG": {
     "description": "A title on the configuration panel, labeling it as configuration.",
     "message": "[PROPER_NAME:Shaka Player] Demo Config"
@@ -675,6 +679,10 @@
     "description": "The name of a configuration value.",
     "message": "Stall Threshold"
   },
+  "DEMO_STALL_TIMEOUT": {
+    "description": "The name of a configuration value.",
+    "message": "Stall timeout"
+  },
   "DEMO_START_AT_SEGMENT_BOUNDARY": {
     "description": "The name of a configuration value.",
     "message": "Start At Segment Boundary"

docs/tutorials/network-and-buffering-config.md

@@ -14,7 +14,9 @@ identical:
 
 ```js
 retryParameters: {
-  timeout: 0,       // timeout in ms, after which we abort; 0 means never
+  timeout: 30000,       // timeout in ms, after which we abort
+  stallTimeout: 5000,  // stall timeout in ms, after which we abort
+  connectionTimeout: 10000, // connection timeout in ms, after which we abort
   maxAttempts: 2,   // the maximum number of requests before we fail
   baseDelay: 1000,  // the base delay in ms between retries
   backoffFactor: 2, // the multiplicative backoff factor between retries

externs/shaka/net.js

@@ -16,7 +16,9 @@
  *   baseDelay: number,
  *   backoffFactor: number,
  *   fuzzFactor: number,
- *   timeout: number
+ *   timeout: number,
+ *   stallTimeout: number,
+ *   connectionTimeout: number
  * }}
  *
  * @description
@@ -34,6 +36,12 @@
  * @property {number} timeout
  *   The request timeout, in milliseconds.  Zero means "unlimited".
  *   <i>Defaults to 30000 milliseconds.</i>
+ * @property {number} stallTimeout
+ *   The request stall timeout, in milliseconds.  Zero means "unlimited".
+ *   <i>Defaults to 5000 milliseconds.</i>
+ * @property {number} connectionTimeout
+ *   The request connection timeout, in milliseconds.  Zero means "unlimited".
+ *   <i>Defaults to 10000 milliseconds.</i>
  *
  * @tutorial network-and-buffering-config
  *

lib/media/streaming_engine.js

@@ -157,6 +157,8 @@ shaka.media.StreamingEngine = class {
       backoffFactor: config.retryParameters.backoffFactor,
       fuzzFactor: config.retryParameters.fuzzFactor,
       timeout: 0,  // irrelevant
+      stallTimeout: 0, // irrelevant
+      connectionTimeout: 0, // irrelevant
     };
 
     // We don't want to ever run out of attempts.  The application should be

lib/net/backoff.js

@@ -140,6 +140,8 @@ shaka.net.Backoff = class {
       backoffFactor: 2,
       fuzzFactor: 0.5,
       timeout: 30000,
+      stallTimeout: 5000,
+      connectionTimeout: 10000,
     };
   }
 

lib/net/http_fetch_plugin.js

@@ -284,8 +284,10 @@ shaka.net.HttpFetchPlugin.Headers_ = window.Headers;
 if (shaka.net.HttpFetchPlugin.isSupported()) {
   shaka.net.NetworkingEngine.registerScheme(
       'http', shaka.net.HttpFetchPlugin.parse,
-      shaka.net.NetworkingEngine.PluginPriority.PREFERRED);
+      shaka.net.NetworkingEngine.PluginPriority.PREFERRED,
+      /* progressSupport= */ true);
   shaka.net.NetworkingEngine.registerScheme(
       'https', shaka.net.HttpFetchPlugin.parse,
-      shaka.net.NetworkingEngine.PluginPriority.PREFERRED);
+      shaka.net.NetworkingEngine.PluginPriority.PREFERRED,
+      /* progressSupport= */ true);
 }

lib/net/http_xhr_plugin.js

@@ -130,8 +130,10 @@ shaka.net.HttpXHRPlugin.Xhr_ = window.XMLHttpRequest;
 
 shaka.net.NetworkingEngine.registerScheme(
     'http', shaka.net.HttpXHRPlugin.parse,
-    shaka.net.NetworkingEngine.PluginPriority.FALLBACK);
+    shaka.net.NetworkingEngine.PluginPriority.FALLBACK,
+    /* progressSupport= */ true);
 shaka.net.NetworkingEngine.registerScheme(
     'https', shaka.net.HttpXHRPlugin.parse,
-    shaka.net.NetworkingEngine.PluginPriority.FALLBACK);
+    shaka.net.NetworkingEngine.PluginPriority.FALLBACK,
+    /* progressSupport= */ true);
 

lib/net/networking_engine.js

@@ -19,6 +19,7 @@ goog.require('shaka.util.FakeEventTarget');
 goog.require('shaka.util.IDestroyable');
 goog.require('shaka.util.ObjectUtils');
 goog.require('shaka.util.OperationManager');
+goog.require('shaka.util.Timer');
 
 
 /**
@@ -88,9 +89,10 @@ shaka.net.NetworkingEngine = class extends shaka.util.FakeEventTarget {
    * @param {string} scheme
    * @param {shaka.extern.SchemePlugin} plugin
    * @param {number=} priority
+   * @param {boolean=} progressSupport
    * @export
    */
-  static registerScheme(scheme, plugin, priority) {
+  static registerScheme(scheme, plugin, priority, progressSupport = false) {
     goog.asserts.assert(
         priority == undefined || priority > 0, 'explicit priority must be > 0');
     priority =
@@ -100,6 +102,7 @@ shaka.net.NetworkingEngine = class extends shaka.util.FakeEventTarget {
       shaka.net.NetworkingEngine.schemes_[scheme] = {
         priority: priority,
         plugin: plugin,
+        progressSupport: progressSupport,
       };
     }
   }
@@ -428,13 +431,22 @@ shaka.net.NetworkingEngine = class extends shaka.util.FakeEventTarget {
               shaka.util.Error.Code.UNSUPPORTED_SCHEME,
               uri));
     }
+    const progressSupport = object.progressSupport;
 
 
     // Every attempt must have an associated backoff.attempt() call so that the
     // accounting is correct.
     const backoffOperation =
         shaka.util.AbortableOperation.notAbortable(backoff.attempt());
 
+    /** @type {?shaka.util.Timer} */
+    let connectionTimer = null;
+
+    /** @type {?shaka.util.Timer} */
+    let stallTimer = null;
+
+    let aborted = false;
+
     let startTimeMs;
     const sendOperation = backoffOperation.chain(() => {
       if (this.destroyed_) {
@@ -444,18 +456,54 @@ shaka.net.NetworkingEngine = class extends shaka.util.FakeEventTarget {
       startTimeMs = Date.now();
       const segment = shaka.net.NetworkingEngine.RequestType.SEGMENT;
 
-      return plugin(request.uris[index],
+      const requestPlugin = plugin(request.uris[index],
           request,
           type,
           // The following function is passed to plugin.
           (time, bytes, numBytesRemaining) => {
+            if (connectionTimer) {
+              connectionTimer.stop();
+            }
+            if (stallTimer) {
+              stallTimer.tickAfter(stallTimeoutMs / 1000);
+            }
             if (this.onProgressUpdated_ && type == segment) {
               this.onProgressUpdated_(time, bytes);
               gotProgress = true;
               numBytesRemainingObj.setBytes(numBytesRemaining);
             }
           });
+
+      if (!progressSupport) {
+        return requestPlugin;
+      }
+
+      const connectionTimeoutMs = request.retryParameters.connectionTimeout;
+      if (connectionTimeoutMs) {
+        connectionTimer = new shaka.util.Timer(() => {
+          aborted = true;
+          requestPlugin.abort();
+        });
+
+        connectionTimer.tickAfter(connectionTimeoutMs / 1000);
+      }
+
+      const stallTimeoutMs = request.retryParameters.stallTimeout;
+      if (stallTimeoutMs) {
+        stallTimer = new shaka.util.Timer(() => {
+          aborted = true;
+          requestPlugin.abort();
+        });
+      }
+
+      return requestPlugin;
     }).chain((response) => {
+      if (connectionTimer) {
+        connectionTimer.stop();
+      }
+      if (stallTimer) {
+        stallTimer.stop();
+      }
       if (response.timeMs == undefined) {
         response.timeMs = Date.now() - startTimeMs;
       }
@@ -466,10 +514,26 @@ shaka.net.NetworkingEngine = class extends shaka.util.FakeEventTarget {
 
       return responseAndGotProgress;
     }, (error) => {
+      if (connectionTimer) {
+        connectionTimer.stop();
+      }
+      if (stallTimer) {
+        stallTimer.stop();
+      }
       if (this.destroyed_) {
         return shaka.util.AbortableOperation.aborted();
       }
 
+      if (aborted) {
+        // It is necessary to change the error code to the correct one because
+        // otherwise the retry logic would not work.
+        error = new shaka.util.Error(
+            shaka.util.Error.Severity.RECOVERABLE,
+            shaka.util.Error.Category.NETWORK,
+            shaka.util.Error.Code.TIMEOUT,
+            request.uris[index], type);
+      }
+
       if (error instanceof shaka.util.Error) {
         if (error.code == shaka.util.Error.Code.OPERATION_ABORTED) {
           // Don't change anything if the operation was aborted.
@@ -663,12 +727,15 @@ shaka.net.NetworkingEngine.PluginPriority = {
 /**
  * @typedef {{
  *   plugin: shaka.extern.SchemePlugin,
- *   priority: number
+ *   priority: number,
+ *   progressSupport: boolean
  * }}
  * @property {shaka.extern.SchemePlugin} plugin
  *   The associated plugin.
  * @property {number} priority
  *   The plugin's priority.
+ * @property {boolean} progressSupport
+ *   The plugin's supports progress events
  */
 shaka.net.NetworkingEngine.SchemeObject;