chore(gzip) Retry without gzip when response seems broken

bda-research · Jul 27, 2018 · f3c86d8 · f3c86d8
1 parent a116947
commit f3c86d8
Showing 1 changed file with 19 additions and 7 deletions.
diff --git a/lib/crawler.js b/lib/crawler.js
@@ -333,13 +333,25 @@ Crawler.prototype._buildHttpRequest = function _buildHTTPRequest (options) {
 			   'followAllRedirects', 'maxRedirects','encoding','pool','timeout','proxy','auth','oauth','strictSSL',
 			   'jar','aws','gzip','time','tunnel','proxyHeaderWhiteList','proxyHeaderExclusiveList','localAddress','forever', 'agent'];
 
-	request(_.pick.apply(self,[ropts].concat(requestArgs)), function(error,response) {
-            if (error) {
-		return self._onContent(error, options);
-            }
-
-            self._onContent(error,options,response);
-	});
+  function tryWithOptions(requestOptions, onError) {
+      request(requestOptions, function(error, response) {
+          if (!error) {
+              return self._onContent(error, options, response);
+          }
+          onError(error);
+      });
+  }
+
+  var allRequestOptions = _.pick.apply(self,[ropts].concat(requestArgs));
+  tryWithOptions(allRequestOptions, function(error) {
+      if (error.code === 'Z_DATA_ERROR') {
+          tryWithOptions(_.assign(allRequestOptions, {gzip: false}), function(newError) {
+              return self._onContent(newError, options);
+          });
+      } else {
+          self._onContent(error, options);
+      }
+  });
     }
 
     if (options.preRequest && _.isFunction(options.preRequest)) {