0.11.3 Fixes (#192)

* recorder url filtering tweaks:
- add isSelfRedirect(), skip recording exact self-redirects
- when converting POST-to-GET, truncate long individual query args to 512, entire URL to 4096
to include remaining query args that may be shorter
- deps: bump to replaywebpage 1.8.13, wabac.js 2.16.12, browsertrix-behaviors 0.5.2
- bump to 0.11.3

package.json

@@ -1,18 +1,18 @@
 {
   "name": "@webrecorder/archivewebpage",
   "productName": "ArchiveWeb.page",
-  "version": "0.11.2",
+  "version": "0.11.3",
   "main": "index.js",
   "description": "Create Web Archives directly in your browser",
   "repository": "https://github.com/webrecorder/archiveweb.page",
   "author": "Webrecorder Software",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.13.0",
-    "@webrecorder/awp-sw": "^0.4.2",
-    "@webrecorder/wabac": "^2.16.9",
+    "@webrecorder/awp-sw": "^0.4.3",
+    "@webrecorder/wabac": "^2.16.12",
     "auto-js-ipfs": "^2.3.0",
-    "browsertrix-behaviors": "^0.5.1",
+    "browsertrix-behaviors": "^0.5.2",
     "btoa": "^1.2.1",
     "bulma": "^0.9.3",
     "client-zip": "^2.2.2",
@@ -22,7 +22,7 @@
     "multiformats": "^10.0.2",
     "node-fetch": "2.6.7",
     "pretty-bytes": "^5.6.0",
-    "replaywebpage": "^1.8.8",
+    "replaywebpage": "^1.8.13",
     "stream-browserify": "^3.0.0",
     "unused-filename": "^4.0.1",
     "uuid": "^8.3.2",

src/recorder.js

@@ -1170,6 +1170,13 @@ class Recorder {
     let data = null;
 
     if (params.redirectResponse) {
+
+      if (reqresp.isSelfRedirect()) {
+        console.warn(`Skip self redirect: ${reqresp.url}`);
+        this.removeReqResp(params.requestId);
+        return;
+      }
+
       reqresp.fillResponseRedirect(params);
       data = reqresp.toDBRecord(null, this.pageInfo);
     }

src/requestresponseinfo.js

@@ -7,6 +7,9 @@ import { postToGetUrl } from "warcio";
 // max URL length for post/put payload-converted URLs
 const MAX_URL_LENGTH = 4096;
 
+// max length for single query arg for post/put converted URLs
+const MAX_ARG_LEN = 512;
+
 const CONTENT_LENGTH = "content-length";
 const CONTENT_TYPE = "content-type";
 const EXCLUDE_HEADERS = ["content-encoding", "transfer-encoding"];
@@ -86,6 +89,18 @@ class RequestResponseInfo
     this._fillResponse(params.redirectResponse);
   }
 
+  isSelfRedirect() {
+    if (this.status < 300 || this.status >= 400 || this.status === 304) {
+      return false;
+    }
+    try {
+      const redirUrl = new URL(this.responseHeaders["location"], this.url).href;
+      return this.url === redirUrl;
+    } catch (e) {
+      return false;
+    }
+  }
+
   fillResponseReceived(params) {
     const response = params.response;
 
@@ -178,6 +193,17 @@ class RequestResponseInfo
       if (postToGetUrl(convData)) {
         //this.requestBody = convData.requestBody;
         // truncate to avoid extra long URLs
+        try {
+          const url = new URL(convData.url);
+          for (const [key, value] of url.searchParams.entries()) {
+            if (value && value.length > MAX_ARG_LEN) {
+              url.searchParams.set(key, value.slice(0, MAX_ARG_LEN));
+            }
+          }
+          convData.url = url.href;
+        } catch (e) {
+          //ignore
+        }
         this.url = convData.url.slice(0, MAX_URL_LENGTH);
       }
     }

yarn.lock

@@ -954,14 +954,14 @@
   resolved "https://registry.yarnpkg.com/@webpack-cli/serve/-/serve-1.6.0.tgz#2c275aa05c895eccebbfc34cfb223c6e8bd591a2"
   integrity sha512-ZkVeqEmRpBV2GHvjjUZqEai2PpUbuq8Bqd//vEYsp63J8WyexI8ppCqVS3Zs0QADf6aWuPdU+0XsPI647PVlQA==
 
-"@webrecorder/awp-sw@^0.4.2":
-  version "0.4.2"
-  resolved "https://registry.yarnpkg.com/@webrecorder/awp-sw/-/awp-sw-0.4.2.tgz#05eae20050c37e092090680559fe7f2f402a5adf"
-  integrity sha512-yn2idnbj71rVTozFn4cPlse0MYF/20SNQYUJ9AZSzXQLCTxQGMDUxP6JY8XrZVt0hrTNMCJNAfXjd/Zo5pb6vQ==
+"@webrecorder/awp-sw@^0.4.3":
+  version "0.4.3"
+  resolved "https://registry.yarnpkg.com/@webrecorder/awp-sw/-/awp-sw-0.4.3.tgz#d18503c7877182280b685657a34655a0878951be"
+  integrity sha512-erQWtLvrsEErfxsQb89DNwNVNxv/Y00TvpHzj06+pfNQqNMCg/lmTasMRhImDhUkg0WrAsLCOKK+c3NrEvmCiw==
   dependencies:
     "@ipld/car" "^5.1.1"
     "@ipld/unixfs" "^2.1.1"
-    "@webrecorder/wabac" "^2.16.9"
+    "@webrecorder/wabac" "^2.16.12"
     auto-js-ipfs "^2.3.0"
     client-zip "^2.3.0"
     hash-wasm "^4.9.0"
@@ -970,10 +970,10 @@
     uuid "^9.0.0"
     warcio "^2.2.0"
 
-"@webrecorder/wabac@^2.16.9":
-  version "2.16.9"
-  resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.16.9.tgz#49eab747d1a631e5aa46ee81530e0f2ed6fc0be7"
-  integrity sha512-uIL/Hyev44dFVJvGOmxQiqZ17UPEM/TRvIHBDuKZdTusKgWcffHJ0pjLR7e0vz04Y5ONnyOPSFRBM6VE7SBZBQ==
+"@webrecorder/wabac@^2.16.12":
+  version "2.16.12"
+  resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.16.12.tgz#cf9ce5490cffcc34f0c1c4a30245276a094d78b2"
+  integrity sha512-lqu9L4Ig2TWzt3t7cKs2CH9epkSt0k09NLx58xzytcHiH2sSEMpk5/ZvBQhNEjjd9Hb2gh3G7Clf7qdXB6b8lA==
   dependencies:
     "@peculiar/asn1-ecc" "^2.3.4"
     "@peculiar/asn1-schema" "^2.3.3"
@@ -1450,10 +1450,10 @@ browserslist@^4.14.5:
     node-releases "^2.0.1"
     picocolors "^1.0.0"
 
-browsertrix-behaviors@^0.5.1:
-  version "0.5.1"
-  resolved "https://registry.yarnpkg.com/browsertrix-behaviors/-/browsertrix-behaviors-0.5.1.tgz#c4756b349dcabd23e25f851cec804d92e94eb63b"
-  integrity sha512-cNSSpQyQT73Y5NcBn2PFDkZM2ptxHVVcqxstryvtzZNOW9gGqzJlLPo8tmCBY00JHrMyn5rm8qImbFglcG/DKg==
+browsertrix-behaviors@^0.5.2:
+  version "0.5.2"
+  resolved "https://registry.yarnpkg.com/browsertrix-behaviors/-/browsertrix-behaviors-0.5.2.tgz#d2fe1d6ff08815ff0dd68a05fe1a3cdc4bbec8ca"
+  integrity sha512-8nhpnzY8OM1mxQ+mZ+m10dpGgMuhCnKUV5YUlitDpMyEfKlEybUmTz5sroVQH8e//NcJox7W6QYjaU2Y/ygxww==
 
 btoa@^1.2.1:
   version "1.2.1"
@@ -5039,13 +5039,13 @@ registry-url@^5.0.0:
   dependencies:
     rc "^1.2.8"
 
-replaywebpage@^1.8.8:
-  version "1.8.8"
-  resolved "https://registry.yarnpkg.com/replaywebpage/-/replaywebpage-1.8.8.tgz#a1287ba9f11120cdb4c5e497c7b0b19271902d45"
-  integrity sha512-eoS1ZI5D/oBZILQbAMpYl5A8s2m47daVBgE1EjGswOATSGNLbQ3R/lfJNTvLd+Oentc3WqfQYGJUTNV37Eblqw==
+replaywebpage@^1.8.13:
+  version "1.8.13"
+  resolved "https://registry.yarnpkg.com/replaywebpage/-/replaywebpage-1.8.13.tgz#a846ea96f5444d09765a6eaddbe130f7dde2d3c1"
+  integrity sha512-haXIIwStg0STsgBadZ+wXPebIZ6zF/7CcqoJi0Xjvww73mxAiB7Kt9fl4yl3xgNaGQaXPn4HeTc/H0/2Gunayg==
   dependencies:
     "@fortawesome/fontawesome-free" "^5.15.4"
-    "@webrecorder/wabac" "^2.16.9"
+    "@webrecorder/wabac" "^2.16.12"
     bulma "^0.9.3"
     electron-log "^4.4.1"
     electron-updater "^5.3.0"