webrecorder · ikreymer · Jul 4, 2022 · Jun 24, 2022 · Jun 24, 2022 · Jun 24, 2022
diff --git a/.eslintignore b/.eslintignore
@@ -0,0 +1 @@
+src/static/*
diff --git a/.gitignore b/.gitignore
@@ -3,7 +3,6 @@
 *.warc.gz
 !examples/*.warc
 **/node_modules
-#wr-ext/replay/sw.js
-#wr-ext/replay/static/
 .DS_Store
+dist/
 
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@webrecorder/archivewebpage",
   "productName": "ArchiveWeb.page",
-  "version": "0.7.12",
+  "version": "0.8.0",
   "main": "index.js",
   "description": "Create Web Archives directly in your browser",
   "repository": "https://github.com/webrecorder/archiveweb.page",
@@ -22,7 +22,7 @@
     "node-fetch": "2.6.7",
     "pretty-bytes": "^5.6.0",
     "querystring-es3": "^0.2.1",
-    "replaywebpage": "^1.6.0",
+    "replaywebpage": "github:webrecorder/replayweb.page#rec-embed-custom",
     "stream-browserify": "^3.0.0",
     "url": "^0.11.0",
     "uuid": "^8.3.2",
@@ -32,7 +32,7 @@
     "copy-webpack-plugin": "^9.0.1",
     "css-loader": "^6.2.0",
     "electron": "^18.3.5",
-    "electron-builder": "^23.1.0",
+    "electron-builder": "^23.0.3",
     "electron-notarize": "^1.0.0",
     "generate-json-webpack-plugin": "^2.0.0",
     "mini-css-extract-plugin": "^2.3.0",
@@ -44,17 +44,18 @@
     "to-string-loader": "^1.1.6",
     "webpack": "^5.64.4",
     "webpack-cli": "^4.8.0",
+    "webpack-dev-server": "^4.9.3",
     "webpack-extension-reloader": "^1.1.4"
   },
   "files": [
-    "src/",
-    "wr-ext/"
+    "src/"
   ],
   "scripts": {
     "build": "webpack --mode production",
     "build-dev": "webpack --mode development",
-    "start-electron": "NODE_ENV=development electron ./dist/electron.js",
+    "start-electron": "NODE_ENV=development electron ./dist/electron/electron.js",
     "start-ext": "NODE_ENV=development webpack --mode=development --watch",
+    "start-embed": "webpack serve --mode development",
     "pack": "CSC_IDENTITY_AUTO_DISCOVERY=false electron-builder --publish never",
     "pack-signed": "electron-builder",
     "dist": "yarn run build && yarn run pack",
@@ -69,16 +70,13 @@
     "appId": "net.webrecorder.archivewebpage",
     "artifactName": "${productName}-${version}.${ext}",
     "extraMetadata": {
-      "main": "dist/electron.js"
+      "main": "dist/electron/electron.js"
     },
     "files": [
       "!**/node_modules/**/*",
-      "dist/*.js",
-      "dist/*.html",
-      "dist/prebuilds/${platform}-${arch}/*",
-      "dist/build/*",
-      "dist/replay/*",
-      "dist/ruffle/*"
+      "dist/electron/**/*",
+      "!dist/electron/prebuilds/**",
+      "dist/electron/prebuilds/${platform}-${arch}/*"
     ],
     "dmg": {
       "title": "ArchiveWeb.page"
@@ -105,7 +103,7 @@
       "extraResources": [
         "plugins-mac"
       ],
-      "singleArchFiles": "dist/prebuilds/**/*"
+      "singleArchFiles": "dist/electron/prebuilds/**/*"
     },
     "linux": {
       "category": "Archiving;Utility;",
@@ -124,7 +122,7 @@
     },
     "directories": {
       "buildResources": "build",
-      "output": "dist"
+      "output": "dist/bin/"
     }
   }
 }
diff --git a/src/embed.html b/src/embed.html
@@ -0,0 +1,14 @@
+<html>
+<head>
+<script src="ui.js"></script>
+<style>
+record-web-page {
+  border: 1px solid black;
+  display: flex;
+}
+</style>
+<head>
+<body>
+<record-web-page url="https://example.com/"></record-web-page>
+</body>
+</html>
diff --git a/wr-ext/icon-dev.png → src/static/icon-dev.png b/wr-ext/icon-dev.png → src/static/icon-dev.png
diff --git a/wr-ext/icon.png → src/static/icon.png b/wr-ext/icon.png → src/static/icon.png
diff --git a/wr-ext/pdf/pdf.min.js → src/static/pdf/pdf.min.js b/wr-ext/pdf/pdf.min.js → src/static/pdf/pdf.min.js
diff --git a/wr-ext/pdf/pdf.worker.min.js → src/static/pdf/pdf.worker.min.js b/wr-ext/pdf/pdf.worker.min.js → src/static/pdf/pdf.worker.min.js
diff --git a/wr-ext/popup.html → src/static/popup.html b/wr-ext/popup.html → src/static/popup.html
diff --git a/wr-ext/replay/index.html → src/static/replay/index.html b/wr-ext/replay/index.html → src/static/replay/index.html
diff --git a/wr-ext/ruffle/69052e08f0f291fb6d82.wasm → src/static/ruffle/69052e08f0f291fb6d82.wasm b/wr-ext/ruffle/69052e08f0f291fb6d82.wasm → src/static/ruffle/69052e08f0f291fb6d82.wasm
diff --git a/wr-ext/ruffle/824f57d600b8c30ee747.wasm → src/static/ruffle/824f57d600b8c30ee747.wasm b/wr-ext/ruffle/824f57d600b8c30ee747.wasm → src/static/ruffle/824f57d600b8c30ee747.wasm
diff --git a/wr-ext/ruffle/LICENSE_MIT → src/static/ruffle/LICENSE_MIT b/wr-ext/ruffle/LICENSE_MIT → src/static/ruffle/LICENSE_MIT
diff --git a/...uffle/core.ruffle.53a2873dad31763672e1.js → ...uffle/core.ruffle.53a2873dad31763672e1.js b/...uffle/core.ruffle.53a2873dad31763672e1.js → ...uffle/core.ruffle.53a2873dad31763672e1.js
diff --git a/...uffle/core.ruffle.be482e0399eaff7f47ec.js → ...uffle/core.ruffle.be482e0399eaff7f47ec.js b/...uffle/core.ruffle.be482e0399eaff7f47ec.js → ...uffle/core.ruffle.be482e0399eaff7f47ec.js
diff --git a/wr-ext/ruffle/ruffle.js → src/static/ruffle/ruffle.js b/wr-ext/ruffle/ruffle.js → src/static/ruffle/ruffle.js
diff --git a/src/sw/api.js b/src/sw/api.js
@@ -1,4 +1,5 @@
 import { API } from "@webrecorder/wabac/src/api";
+import { tsToDate } from "@webrecorder/wabac/src/utils";
 
 import { Downloader } from "../downloader";
 import { Signer } from "../keystore";
@@ -19,6 +20,8 @@ class ExtAPI extends API
     return {
       ...super.routes,
       "downloadPages": "c/:coll/dl",
+      "pageTitle": ["c/:coll/pageTitle", "POST"],
+      "publicKey": "publicKey",
     };
   }
 
@@ -44,10 +47,61 @@ class ExtAPI extends API
       return dl.download();
     }
 
+    case "pageTitle":
+      return await this.updatePageTitle(params.coll, request);
+
+    case "publicKey":
+      return await this.getPublicKey();
+
     default:
       return await super.handleApi(request, params);
     }
   }
+
+  async updatePageTitle(collId, request) {
+    const json = await request.json();
+    let {url, ts, title} = json;
+
+    ts = tsToDate(ts).getTime();
+
+    const coll = await this.collections.loadColl(collId);
+    if (!coll) {
+      return {error: "collection_not_found"};
+    }
+
+    //await coll.store.db.init();
+
+    const result = await coll.store.lookupUrl(url, ts);
+
+    if (!result) {
+      return {error: "page_not_found"};
+    }
+
+    // drop to second precision for comparison
+    const roundedTs = Math.floor(result.ts / 1000) * 1000;
+    if (url !== result.url || ts !== roundedTs) {
+      return {error: "no_exact_match"};
+    }
+
+    const page = await coll.store.db.getFromIndex("pages", "url", url);
+    if (!page) {
+      return {error: "page_not_found"};
+    }
+    page.title = title;
+    await coll.store.db.put("pages", page);
+
+    return {"added": true};
+  }
+
+  async getPublicKey() {
+    const signer = new Signer();
+    const keys = await signer.loadKeys();
+    if (!keys || !keys.public) {
+      return {};
+    } else {
+      return {publicKey: keys.public};
+    }
+  }
 }
 
 export { ExtAPI };
diff --git a/src/sw/main.js b/src/sw/main.js
@@ -2,10 +2,34 @@ import { SWReplay } from "@webrecorder/wabac/src/swmain";
 
 import { ExtAPI } from "./api";
 
-const defaultConfig = {
-  injectScripts: ["/ruffle/ruffle.js"],
-  baseUrlSourcePrefix: "/replay/index.html",
-  convertPostToGet: true
-};
+import { RecordingCollections } from "./recproxy";
 
-self.sw = new SWReplay({ApiClass: ExtAPI, useIPFS: false, defaultConfig});
+import REC_INDEX_HTML from "../static/replay/index.html";
+import RWP_INDEX_HTML from "replaywebpage/index.html";
+import { WorkerLoader } from "@webrecorder/wabac/src/loaders";
+
+if (self.registration) {
+  const defaultConfig = {
+    injectScripts: ["/ruffle/ruffle.js"],
+    baseUrlSourcePrefix: "/replay/index.html",
+    //convertPostToGet: true
+  };
+
+  const staticData = new Map();
+
+  const prefix = self.registration.scope;
+
+  // for backwards compatibility to support <replay-web-page> tag
+  staticData.set(prefix + "replay.html", {type: "text/html", content: RWP_INDEX_HTML});
+
+  // for use with <record-web-page> tag
+  staticData.set(prefix + "record.html", {type: "text/html", content: REC_INDEX_HTML});
+
+  const useIPFS = false;
+  const ApiClass = ExtAPI;
+  const CollectionsClass = RecordingCollections;
+
+  self.sw = new SWReplay({ApiClass, useIPFS, staticData, defaultConfig, CollectionsClass});
+} else {
+  new WorkerLoader(self);
+}
diff --git a/src/sw/recproxy.js b/src/sw/recproxy.js
@@ -0,0 +1,148 @@
+import { ArchiveDB } from "@webrecorder/wabac/src/archivedb";
+import { LiveProxy } from "@webrecorder/wabac/src/liveproxy";
+import { SWCollections } from "@webrecorder/wabac/src/swmain";
+import { randomId } from "@webrecorder/wabac/src/utils";
+import { postToGetUrl } from "warcio";
+
+
+// ===========================================================================
+class RecProxy extends ArchiveDB
+{
+  constructor(config, collLoader) {
+    super(config.dbname);
+
+    this.name = config.dbname.slice(3);
+
+    this.collLoader = collLoader;
+
+    this.recordProxied = config.extraConfig.recordProxied || false;
+
+    this.liveProxy = new LiveProxy(config.extraConfig, {cloneResponse: true, allowBody: true});
+
+    this.pageId = randomId();
+    this.isNew = true;
+
+    //this.cookie = "";
+  }
+
+  async getResource(request, prefix) {
+    let req;
+
+    if (request.method === "POST" || request.method === "PUT") {
+      req = request.request.clone();
+    } else {
+      req = request.request;
+    }
+
+    const response = await this.liveProxy.getResource(request, prefix);
+
+    //this.cookie = response.headers.get("x-wabac-preset-cookie");
+
+    // don't record content proxied from specified hosts
+    if (!this.recordProxied && this.liveProxy.hostProxy) {
+      const parsedUrl = new URL(request.url);
+      if (this.liveProxy.hostProxy[parsedUrl.host]) {
+        return response;
+      }
+    }
+
+    this.doRecord(response, req);
+
+    return response;
+  }
+
+  async doRecord(response, request) {
+    let url = response.url;
+    const ts = response.date.getTime();
+
+    const mime = (response.headers.get("content-type") || "").split(";")[0];
+
+    const range = response.headers.get("content-range");
+
+    if (range && !range.startsWith("bytes 0-")) {
+      console.log("skip range request: " + range);
+      return;
+    }
+
+    const status = response.status;
+    const statusText = response.statusText;
+
+    const respHeaders = Object.fromEntries(response.headers.entries());
+    const reqHeaders = Object.fromEntries(request.headers.entries());
+
+    const payload = new Uint8Array(await response.clonedResponse.arrayBuffer());
+
+    if (range) {
+      const expectedRange = `bytes 0-${payload.length - 1}/${payload.length}`;
+      if (range !== expectedRange) {
+        console.log("skip range request: " + range);
+        return;
+      }
+    }
+
+    if (request.mode === "navigate") {
+      this.pageId = randomId();
+      this.isNew = true;
+    }
+
+    const pageId = this.pageId;
+
+    const referrer = request.referrer;
+
+    if (request.method === "POST" || request.method === "PUT") {
+      const data = {
+        method: request.method,
+        postData: await request.text(),
+        headers: request.headers,
+        url,
+      };
+
+      if (postToGetUrl(data)) {
+        url = new URL(data.url).href;
+      }
+    }
+
+    const data = {
+      url,
+      ts,
+      status,
+      statusText,
+      pageId,
+      payload,
+      mime,
+      respHeaders,
+      reqHeaders,
+      referrer
+    };
+
+    await this.addResource(data);
+
+    await this.collLoader.updateSize(this.name, payload.length, payload.length);
+
+    // don't add page for redirects
+    if (this.isNew && (status < 301 || status >= 400) && request.mode === "navigate") {
+      //console.log("Page", url, "Referrer", referrer);
+      await this.addPages([{id: pageId, url, ts}]);
+      this.isNew = false;
+    }
+  }
+}
+
+// ===========================================================================
+export class RecordingCollections extends SWCollections
+{
+  async _initStore(type, config) {
+    let store;
+
+    switch (type) {
+    case "recordingproxy":
+      store = new RecProxy(config, this);
+      if (store.initing) {
+        await store.initing;
+      }
+      return store;
+    }
+
+    return await super._initStore(type, config);
+  }
+}