more specific types additions (#434)

- add QueueEntry for type of json object stored in Redis
- and PageCallbacks for callback type
- use Crawler type

src/crawler.ts

@@ -10,6 +10,7 @@ import {
   QueueState,
   PageState,
   WorkerId,
+  PageCallbacks,
 } from "./util/state.js";
 
 import Sitemapper from "sitemapper";
@@ -554,8 +555,7 @@ export class Crawler {
     page: Page;
     cdp: CDPSession;
     workerid: WorkerId;
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    callbacks: any;
+    callbacks: PageCallbacks;
   }) {
     await this.browser.setupPage({ page, cdp });
 

src/util/state.ts

@@ -25,6 +25,20 @@ export enum QueueState {
 // ============================================================================
 export type WorkerId = number;
 
+// ============================================================================
+export type QueueEntry = {
+  added?: string;
+  url: string;
+  seedId: number;
+  depth: number;
+  extraHops: number;
+};
+
+// ============================================================================
+export type PageCallbacks = {
+  addLink?: (url: string) => void;
+};
+
 // ============================================================================
 export class PageState {
   url: string;
@@ -38,9 +52,7 @@ export class PageState {
   title?: string;
   mime?: string;
 
-  // TODO: Fix this the next time the file is edited.
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  callbacks: any;
+  callbacks: PageCallbacks = {};
 
   isHTMLPage?: boolean;
   text?: string;
@@ -52,16 +64,11 @@ export class PageState {
 
   logDetails = {};
 
-  constructor(redisData: {
-    url: string;
-    seedId: number;
-    depth: number;
-    extraHops: number;
-  }) {
+  constructor(redisData: QueueEntry) {
     this.url = redisData.url;
     this.seedId = redisData.seedId;
     this.depth = redisData.depth;
-    this.extraHops = redisData.extraHops;
+    this.extraHops = redisData.extraHops || 0;
   }
 }
 
@@ -290,10 +297,7 @@ return 0;
     await this.redis.srem(this.skey, url);
   }
 
-  recheckScope(
-    data: { url: string; depth: number; extraHops: number; seedId: number },
-    seeds: ScopedSeed[],
-  ) {
+  recheckScope(data: QueueEntry, seeds: ScopedSeed[]) {
     const seed = seeds[data.seedId];
 
     return seed.isIncluded(data.url, data.depth, data.extraHops);
@@ -434,21 +438,11 @@ return 0;
 
   //async addToQueue({url : string, seedId, depth = 0, extraHops = 0} = {}, limit = 0) {
   async addToQueue(
-    {
-      url,
-      seedId,
-      depth = 0,
-      extraHops = 0,
-    }: { url: string; seedId: number; depth?: number; extraHops?: number },
+    { url, seedId, depth = 0, extraHops = 0 }: QueueEntry,
     limit = 0,
   ) {
     const added = this._timestamp();
-    // TODO: Fix this the next time the file is edited.
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const data: any = { added, url, seedId, depth };
-    if (extraHops) {
-      data.extraHops = extraHops;
-    }
+    const data: QueueEntry = { added, url, seedId, depth, extraHops };
 
     // return codes
     // 0 - url queued successfully
@@ -500,7 +494,7 @@ return 0;
     return { done, queued, pending, failed, errors };
   }
 
-  _getScore(data: { depth: number; extraHops: number }) {
+  _getScore(data: QueueEntry) {
     return (data.depth || 0) + (data.extraHops || 0) * MAX_DEPTH;
   }
 

src/util/worker.ts

@@ -8,6 +8,7 @@ import { Recorder } from "./recorder.js";
 import { rxEscape } from "./seeds.js";
 import { CDPSession, Page } from "puppeteer-core";
 import { PageState, WorkerId } from "./state.js";
+import { Crawler } from "../crawler.js";
 
 const MAX_REUSE = 5;
 
@@ -17,9 +18,7 @@ const FINISHED_TIMEOUT = 60;
 
 // ===========================================================================
 export function runWorkers(
-  // TODO: Fix this the next time the file is edited.
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  crawler: any,
+  crawler: Crawler,
   numWorkers: number,
   maxPageTime: number,
   collDir: string,