Include WARC prefix for screenshots and text WARCs

src/crawler.ts

@@ -159,6 +159,8 @@ export class Crawler {
   maxHeapUsed = 0;
   maxHeapTotal = 0;
 
+  warcPrefix: string;
+
   driver!: (opts: {
     page: Page;
     data: PageState;
@@ -260,6 +262,12 @@ export class Crawler {
     this.customBehaviors = "";
 
     this.browser = new Browser();
+
+    this.warcPrefix = process.env.WARC_PREFIX || this.params.warcPrefix || "";
+
+    if (this.warcPrefix) {
+      this.warcPrefix += "-" + this.crawlId + "-";
+    }
   }
 
   configureUA() {
@@ -741,6 +749,7 @@ self.__bx_behaviors.selectMainBehavior();
         logger.debug("Skipping screenshots for non-HTML page", logDetails);
       }
       const screenshots = new Screenshots({
+        warcPrefix: this.warcPrefix,
         browser: this.browser,
         page,
         url,
@@ -761,6 +770,7 @@ self.__bx_behaviors.selectMainBehavior();
 
     if (data.isHTMLPage) {
       textextract = new TextExtractViaSnapshot(cdp, {
+        warcPrefix: this.warcPrefix,
         url,
         directory: archiveDir,
       });

src/util/argParser.ts

@@ -500,7 +500,6 @@ class ArgParser {
         describe:
           "prefix for WARC files generated, including WARCs added to WACZ",
         type: "string",
-        default: "rec",
       },
     };
   }

src/util/warcresourcewriter.ts

@@ -15,16 +15,18 @@ export class WARCResourceWriter {
     url,
     directory,
     date,
+    warcPrefix,
     warcName,
   }: {
     url: string;
     directory: string;
     date: Date;
+    warcPrefix: string;
     warcName: string;
   }) {
     this.url = url;
     this.directory = directory;
-    this.warcName = path.join(this.directory, warcName);
+    this.warcName = path.join(this.directory, warcPrefix + warcName);
     this.date = date ? date : new Date();
   }