Add arg to write pages to Redis

src/crawler.ts

@@ -90,6 +90,7 @@ type PageEntry = {
   seed?: boolean;
   text?: string;
   favIconUrl?: string;
+  ts?: string;
 };
 
 // ============================================================================
@@ -1928,13 +1929,22 @@ self.__bx_behaviors.selectMainBehavior();
     loadState,
     mime,
     favicon,
+    ts,
   }: PageState) {
     const row: PageEntry = { id: pageid!, url, title, loadState };
 
+    if (ts) {
+      row.ts = ts.toISOString();
+    }
+
     if (mime) {
       row.mime = mime;
     }
 
+    if (this.params.writePagesToRedis) {
+      await this.crawlState.writeToPagesQueue(JSON.stringify(row));
+    }
+
     if (depth === 0) {
       row.seed = true;
     }

src/util/argParser.ts

@@ -464,6 +464,12 @@ class ArgParser {
         default: false,
       },
 
+      writePagesToRedis: {
+        describe: "If set, write page objects to redis",
+        type: "boolean",
+        default: false,
+      },
+
       failOnFailedSeed: {
         describe:
           "If set, crawler will fail with exit code 1 if any seed fails",

src/util/recorder.ts

@@ -50,6 +50,8 @@ function logNetwork(msg: string, data: any) {
 export type PageInfoRecord = {
   pageid: string;
   urls: Record<string, number>;
+  url: string;
+  ts?: Date;
 };
 
 // =================================================================
@@ -463,6 +465,10 @@ export class Recorder {
       return false;
     }
 
+    if (url === this.pageUrl) {
+      this.pageInfo.ts = reqresp.ts;
+    }
+
     reqresp.fillFetchRequestPaused(params);
 
     if (this.noResponseForStatus(responseStatusCode)) {
@@ -622,7 +628,7 @@ export class Recorder {
     this.pendingRequests = new Map();
     this.skipIds = new Set();
     this.skipping = false;
-    this.pageInfo = { pageid, urls: {} };
+    this.pageInfo = { pageid, urls: {}, url };
   }
 
   addPageRecord(reqresp: RequestResponseInfo) {
@@ -685,6 +691,8 @@ export class Recorder {
     }
 
     await this.writePageInfoRecord();
+
+    return this.pageInfo.ts;
   }
 
   async onClosePage() {
@@ -1384,7 +1392,7 @@ function createResponse(
   const url = reqresp.url;
   const warcVersion = "WARC/1.1";
   const statusline = `HTTP/1.1 ${reqresp.status} ${reqresp.statusText}`;
-  const date = new Date().toISOString();
+  const date = new Date(reqresp.ts).toISOString();
 
   const httpHeaders = reqresp.getResponseHeadersDict(
     reqresp.payload ? reqresp.payload.length : 0,

src/util/reqresp.ts

@@ -16,12 +16,10 @@ const MAX_ARG_LEN = 512;
 
 // ===========================================================================
 export class RequestResponseInfo {
-  _created: Date = new Date();
+  ts: Date = new Date();
 
   requestId: string;
 
-  ts?: string;
-
   method?: string;
   url!: string;
   protocol?: string = "HTTP/1.1";

src/util/state.ts

@@ -51,6 +51,7 @@ export class PageState {
   pageid?: string;
   title?: string;
   mime?: string;
+  ts?: Date;
 
   callbacks: PageCallbacks = {};
 
@@ -135,6 +136,7 @@ export class RedisCrawlState {
   dkey: string;
   fkey: string;
   ekey: string;
+  pageskey: string;
 
   constructor(redis: Redis, key: string, maxPageTime: number, uid: string) {
     this.redis = redis;
@@ -152,6 +154,8 @@ export class RedisCrawlState {
     this.fkey = this.key + ":f";
     // crawler errors
     this.ekey = this.key + ":e";
+    // pages
+    this.pageskey = this.key + ":pages";
 
     this._initLuaCommands(this.redis);
   }
@@ -682,4 +686,8 @@ return 0;
   async logError(error: string) {
     return await this.redis.lpush(this.ekey, error);
   }
+
+  async writeToPagesQueue(value: string) {
+    return await this.redis.lpush(this.pageskey, value);
+  }
 }

src/util/worker.ts

@@ -298,7 +298,7 @@ export class PageWorker {
   async crawlPage(opts: WorkerState) {
     const res = await this.crawler.crawlPage(opts);
     if (this.recorder) {
-      await this.recorder.finishPage();
+      opts.data.ts = await this.recorder.finishPage();
     }
     return res;
   }