Add arg to write pages to Redis (#464)

Fixes #462 

Add --writePagesToRedis arg, for use conjunction with QA features in Browsertrix Cloud, to add
pages to the database for each crawl.
Ensure timestamp (as ISO date) is added to pages when they are serialized (both to pages.jsonl and redis)
Also include timestamp (as ISO date) in `pageinfo:` records

---------
Co-authored-by: Ilya Kreymer <ikreymer@gmail.com>

src/crawler.ts

@@ -90,6 +90,7 @@ type PageEntry = {
   seed?: boolean;
   text?: string;
   favIconUrl?: string;
+  ts?: string;
 };
 
 // ============================================================================
@@ -1928,13 +1929,22 @@ self.__bx_behaviors.selectMainBehavior();
     loadState,
     mime,
     favicon,
+    ts,
   }: PageState) {
     const row: PageEntry = { id: pageid!, url, title, loadState };
 
+    if (ts) {
+      row.ts = ts.toISOString();
+    }
+
     if (mime) {
       row.mime = mime;
     }
 
+    if (this.params.writePagesToRedis) {
+      await this.crawlState.writeToPagesQueue(JSON.stringify(row));
+    }
+
     if (depth === 0) {
       row.seed = true;
     }

src/util/argParser.ts

@@ -464,6 +464,12 @@ class ArgParser {
         default: false,
       },
 
+      writePagesToRedis: {
+        describe: "If set, write page objects to redis",
+        type: "boolean",
+        default: false,
+      },
+
       failOnFailedSeed: {
         describe:
           "If set, crawler will fail with exit code 1 if any seed fails",

src/util/recorder.ts

@@ -50,6 +50,8 @@ function logNetwork(msg: string, data: any) {
 export type PageInfoRecord = {
   pageid: string;
   urls: Record<string, number>;
+  url: string;
+  ts?: Date;
 };
 
 // =================================================================
@@ -463,6 +465,10 @@ export class Recorder {
       return false;
     }
 
+    if (url === this.pageUrl) {
+      this.pageInfo.ts = reqresp.ts;
+    }
+
     reqresp.fillFetchRequestPaused(params);
 
     if (this.noResponseForStatus(responseStatusCode)) {
@@ -622,7 +628,7 @@ export class Recorder {
     this.pendingRequests = new Map();
     this.skipIds = new Set();
     this.skipping = false;
-    this.pageInfo = { pageid, urls: {} };
+    this.pageInfo = { pageid, urls: {}, url };
   }
 
   addPageRecord(reqresp: RequestResponseInfo) {
@@ -685,6 +691,8 @@ export class Recorder {
     }
 
     await this.writePageInfoRecord();
+
+    return this.pageInfo.ts;
   }
 
   async onClosePage() {
@@ -1384,7 +1392,7 @@ function createResponse(
   const url = reqresp.url;
   const warcVersion = "WARC/1.1";
   const statusline = `HTTP/1.1 ${reqresp.status} ${reqresp.statusText}`;
-  const date = new Date().toISOString();
+  const date = new Date(reqresp.ts).toISOString();
 
   const httpHeaders = reqresp.getResponseHeadersDict(
     reqresp.payload ? reqresp.payload.length : 0,

src/util/reqresp.ts

@@ -16,12 +16,10 @@ const MAX_ARG_LEN = 512;
 
 // ===========================================================================
 export class RequestResponseInfo {
-  _created: Date = new Date();
+  ts: Date = new Date();
 
   requestId: string;
 
-  ts?: string;
-
   method?: string;
   url!: string;
   protocol?: string = "HTTP/1.1";

src/util/state.ts

@@ -51,6 +51,7 @@ export class PageState {
   pageid?: string;
   title?: string;
   mime?: string;
+  ts?: Date;
 
   callbacks: PageCallbacks = {};
 
@@ -135,6 +136,7 @@ export class RedisCrawlState {
   dkey: string;
   fkey: string;
   ekey: string;
+  pageskey: string;
 
   constructor(redis: Redis, key: string, maxPageTime: number, uid: string) {
     this.redis = redis;
@@ -152,6 +154,8 @@ export class RedisCrawlState {
     this.fkey = this.key + ":f";
     // crawler errors
     this.ekey = this.key + ":e";
+    // pages
+    this.pageskey = this.key + ":pages";
 
     this._initLuaCommands(this.redis);
   }
@@ -682,4 +686,8 @@ return 0;
   async logError(error: string) {
     return await this.redis.lpush(this.ekey, error);
   }
+
+  async writeToPagesQueue(value: string) {
+    return await this.redis.lpush(this.pageskey, value);
+  }
 }

src/util/worker.ts

@@ -298,7 +298,7 @@ export class PageWorker {
   async crawlPage(opts: WorkerState) {
     const res = await this.crawler.crawlPage(opts);
     if (this.recorder) {
-      await this.recorder.finishPage();
+      opts.data.ts = await this.recorder.finishPage();
     }
     return res;
   }