feat: implement ErrorSnapshotter for error context capture

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -1363,7 +1363,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
         const shouldRetryRequest = this._canRequestBeRetried(request, error);
 
         if (shouldRetryRequest) {
-            this.stats.errorTrackerRetry.add(error);
+            await this.stats.errorTrackerRetry.addAsync(error, crawlingContext);
 
             if (error instanceof SessionError) {
                 await this._rotateSession(crawlingContext);
@@ -1389,7 +1389,15 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
             }
         }
 
-        this.stats.errorTracker.add(error);
+        // If the request is non-retryable, the error and snapshot aren't saved in the errorTrackerRetry object.
+        // Therefore, we pass the crawlingContext to the errorTracker.add method, enabling snapshot capture.
+        // This is to make sure the error snapshot is not duplicated in the errorTrackerRetry and errorTracker objects.
+        const { noRetry, maxRetries } = request;
+        if (noRetry || !maxRetries) {
+            await this.stats.errorTracker.addAsync(error, crawlingContext);
+        } else {
+            this.stats.errorTracker.add(error);
+        }
 
         // If we get here, the request is either not retryable
         // or failed more than retryCount times and will not be retried anymore.

packages/core/src/crawlers/error_snapshotter.ts

@@ -0,0 +1,135 @@
+import crypto from 'node:crypto';
+
+import type { ErrnoException } from './error_tracker';
+import type { CrawlingContext } from '../crawlers/crawler_commons';
+import type { KeyValueStore } from '../storages';
+import type { PlaywrightCrawlingContext, PuppeteerCrawlingContext, PlaywrightPage, PuppeteerPage } from '../typedefs';
+
+const { PWD, CRAWLEE_STORAGE_DIR, APIFY_IS_AT_HOME } = process.env;
+
+/**
+ * ErrorSnapshotter class is used to capture a screenshot of the page and a snapshot of the HTML when an error occur during web crawling.
+ */
+export class ErrorSnapshotter {
+    static readonly MAX_ERROR_CHARACTERS = 30;
+    static readonly MAX_HASH_LENGTH = 30;
+    static readonly MAX_FILENAME_LENGTH = 250;
+    static readonly BASE_MESSAGE = 'An error occurred';
+    static readonly SNAPSHOT_PREFIX = 'ERROR_SNAPSHOT';
+    static readonly KEY_VALUE_PLATFORM_PATH = 'https://api.apify.com/v2/key-value-stores';
+    static readonly KEY_VALUE_STORE_LOCAL_PATH = `file://${PWD}/${CRAWLEE_STORAGE_DIR || 'storage'}/key_value_stores`;
+
+    /**
+     * Capture a snapshot of the error context.
+     */
+    async captureSnapshot(error: ErrnoException, context: CrawlingContext): Promise<{ screenshotFileUrl?: string; htmlFileUrl?: string }> {
+        const { KEY_VALUE_PLATFORM_PATH, KEY_VALUE_STORE_LOCAL_PATH } = ErrorSnapshotter;
+        const page = context?.page as PuppeteerPage | PlaywrightPage | undefined;
+        const body = context?.body;
+
+        const keyValueStore = await context?.getKeyValueStore();
+        // If the key-value store is not available, or the body and page are not available, return empty filenames
+        if (!keyValueStore || (!body && !page)) {
+            return {};
+        }
+
+        const filename = this.generateFilename(error);
+
+        let screenshotFilename: string | undefined;
+        let htmlFileName: string | undefined;
+
+        if (page) {
+            const capturedFiles = await this.captureSnapShot(
+                context as unknown as
+                | PlaywrightCrawlingContext
+                | PuppeteerCrawlingContext,
+                filename,
+            );
+
+            if (capturedFiles) {
+                screenshotFilename = capturedFiles.screenshotFilename;
+                htmlFileName = capturedFiles.htmlFileName;
+            }
+
+            // If the snapshot for browsers failed to capture the HTML, try to capture it from the page content
+            if (!htmlFileName) {
+                const html = await page?.content() || undefined;
+                htmlFileName = html ? await this.saveHTMLSnapshot(html, keyValueStore, filename) : undefined;
+            }
+        } else if (typeof body === 'string') { // for non-browser contexts
+            htmlFileName = await this.saveHTMLSnapshot(body, keyValueStore, filename);
+        }
+
+        if (APIFY_IS_AT_HOME) {
+            const platformPath = `${KEY_VALUE_PLATFORM_PATH}/${keyValueStore.id}/records`;
+            return {
+                screenshotFileUrl: screenshotFilename ? `${platformPath}/${screenshotFilename}` : undefined,
+                htmlFileUrl: htmlFileName ? `${platformPath}/${htmlFileName}` : undefined,
+            };
+        }
+
+        const localPath = `${KEY_VALUE_STORE_LOCAL_PATH}/${keyValueStore.name || 'default'}`;
+        return {
+            screenshotFileUrl: screenshotFilename ? `${localPath}/${screenshotFilename}` : undefined,
+            htmlFileUrl: htmlFileName ? `${localPath}/${htmlFileName}` : undefined,
+        };
+    }
+
+    /**
+     * Capture a screenshot and HTML of the page (For Browser only), and return the filename with the extension.
+     */
+    async captureSnapShot(
+        context: PlaywrightCrawlingContext | PuppeteerCrawlingContext,
+        filename: string): Promise<{
+            screenshotFilename?: string;
+            htmlFileName?: string;
+        } | undefined> {
+        try {
+            await context.saveSnapshot({ key: filename });
+            return {
+                // The screenshot file extension is different for Apify and local environments
+                screenshotFilename: `${filename}${APIFY_IS_AT_HOME ? '.jpg' : '.jpeg'}`,
+                htmlFileName: `${filename}.html`,
+            };
+        } catch (e) {
+            return undefined;
+        }
+    }
+
+    /**
+     * Save the HTML snapshot of the page, and return the filename with the extension.
+     */
+    async saveHTMLSnapshot(html: string, keyValueStore: KeyValueStore, filename: string): Promise<string | undefined> {
+        try {
+            await keyValueStore.setValue(filename, html, { contentType: 'text/html' });
+            return `${filename}.html`;
+        } catch (e) {
+            return undefined;
+        }
+    }
+
+    /**
+     * Remove non-word characters from the start and end of a string.
+     */
+    sanitizeString(str: string): string {
+        return str.replace(/^\W+|\W+$/g, '');
+    }
+
+    /**
+     * Generate a unique filename for each error snapshot.
+     */
+    generateFilename(error: ErrnoException): string {
+        const { SNAPSHOT_PREFIX, BASE_MESSAGE, MAX_HASH_LENGTH, MAX_ERROR_CHARACTERS, MAX_FILENAME_LENGTH } = ErrorSnapshotter;
+        const { sanitizeString } = this;
+        // Create a hash of the error stack trace
+        const errorStackHash = crypto.createHash('sha1').update(error.stack || error.message || '').digest('hex').slice(0, MAX_HASH_LENGTH);
+        const errorMessagePrefix = (error.message || BASE_MESSAGE).slice(0, MAX_ERROR_CHARACTERS).trim();
+
+        // Generate filename and remove disallowed characters
+        const filename = `${SNAPSHOT_PREFIX}_${sanitizeString(errorStackHash)}_${sanitizeString(errorMessagePrefix)}`
+            .replace(/\W+/g, '-') // Replace non-word characters with a dash
+            .slice(0, MAX_FILENAME_LENGTH);
+
+        return filename;
+    }
+}

packages/utils/src/internals/error_tracker.ts → packages/core/src/crawlers/error_tracker.ts

@@ -1,9 +1,12 @@
 import { inspect } from 'node:util';
 
+import { ErrorSnapshotter } from './error_snapshotter';
+import type { CrawlingContext } from '../crawlers/crawler_commons';
+
 /**
  * Node.js Error interface
  */
- interface ErrnoException extends Error {
+export interface ErrnoException extends Error {
     errno?: number | undefined;
     code?: string | number | undefined;
     path?: string | undefined;
@@ -283,6 +286,8 @@ export class ErrorTracker {
 
     total: number;
 
+    errorSnapshotter: ErrorSnapshotter;
+
     constructor(options: Partial<ErrorTrackerOptions> = {}) {
         this.#options = {
             showErrorCode: true,
@@ -294,6 +299,8 @@ export class ErrorTracker {
             ...options,
         };
 
+        this.errorSnapshotter = new ErrorSnapshotter();
+
         this.result = Object.create(null);
         this.total = 0;
     }
@@ -326,6 +333,43 @@ export class ErrorTracker {
         }
     }
 
+    /**
+     * This method is async, because it captures a snapshot of the error context.
+     * We added this new method to avoid breaking changes.
+     */
+    async addAsync(error: ErrnoException, context?: CrawlingContext) {
+        this.total++;
+
+        let group = this.result;
+
+        if (this.#options.showStackTrace) {
+            group = getStackTraceGroup(error, group, this.#options.showFullStack);
+        }
+
+        if (this.#options.showErrorCode) {
+            group = getErrorCodeGroup(error, group);
+        }
+
+        if (this.#options.showErrorName) {
+            group = getErrorNameGroup(error, group);
+        }
+
+        if (this.#options.showErrorMessage) {
+            group = getErrorMessageGroup(error, group, this.#options.showFullMessage);
+        }
+
+        increaseCount(group as { count: number });
+
+        // Capture a snapshot (screenshot and HTML) on the first occurrence of an error
+        if (group.count === 1 && context) {
+            await this.captureSnapshot(group, error, context).catch(() => { });
+        }
+
+        if (typeof error.cause === 'object' && error.cause !== null) {
+            await this.addAsync(error.cause);
+        }
+    }
+
     getUniqueErrorCount() {
         let count = 0;
 
@@ -366,6 +410,13 @@ export class ErrorTracker {
         return result.sort((a, b) => b[0] - a[0]).slice(0, count);
     }
 
+    async captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext) {
+        const { screenshotFileUrl, htmlFileUrl } = await this.errorSnapshotter.captureSnapshot(error, context);
+
+        storage.firstErrorScreenshotUrl = screenshotFileUrl;
+        storage.firstErrorHtmlUrl = htmlFileUrl;
+    }
+
     reset() {
         // This actually safe, since we Object.create(null) so no prototype pollution can happen.
         // eslint-disable-next-line no-restricted-syntax, guard-for-in

packages/core/src/crawlers/index.ts

@@ -2,3 +2,5 @@ export * from './crawler_commons';
 export * from './crawler_extension';
 export * from './crawler_utils';
 export * from './statistics';
+export * from './error_tracker';
+export * from './error_snapshotter';

packages/core/src/crawlers/statistics.ts

@@ -1,6 +1,6 @@
-import { ErrorTracker } from '@crawlee/utils';
 import ow from 'ow';
 
+import { ErrorTracker } from './error_tracker';
 import { Configuration } from '../configuration';
 import type { EventManager } from '../events/event_manager';
 import { EventType } from '../events/event_manager';

packages/core/src/typedefs.ts

@@ -17,3 +17,17 @@ export function keys<T extends {}>(obj: T) {
 }
 
 export declare type AllowedHttpMethods = 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH';
+
+// Define the following types as we cannot import the complete types from the respective packages
+export interface PlaywrightCrawlingContext {
+    saveSnapshot: (options: { key: string }) => Promise<void>;
+}
+export interface PuppeteerCrawlingContext {
+    saveSnapshot: (options: { key: string }) => Promise<void>;
+}
+export interface PlaywrightPage {
+    content: () => Promise<string>;
+}
+export interface PuppeteerPage {
+    content: () => Promise<string>;
+}

packages/playwright-crawler/src/internals/utils/playwright-utils.ts

@@ -534,14 +534,16 @@ export async function saveSnapshot(page: Page, options: SaveSnapshotOptions = {}
     try {
         const store = await KeyValueStore.open(keyValueStoreName, { config: config ?? Configuration.getGlobalConfig() });
 
+        const { APIFY_IS_AT_HOME } = process.env;
+
         if (saveScreenshot) {
-            const screenshotName = `${key}.jpg`;
+            const screenshotName = APIFY_IS_AT_HOME ? `${key}.jpg` : key;
             const screenshotBuffer = await page.screenshot({ fullPage: true, quality: screenshotQuality, type: 'jpeg', animations: 'disabled' });
             await store.setValue(screenshotName, screenshotBuffer, { contentType: 'image/jpeg' });
         }
 
         if (saveHtml) {
-            const htmlName = `${key}.html`;
+            const htmlName = APIFY_IS_AT_HOME ? `${key}.html` : key;
             const html = await page.content();
             await store.setValue(htmlName, html, { contentType: 'text/html' });
         }

packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts

@@ -666,14 +666,16 @@ export async function saveSnapshot(page: Page, options: SaveSnapshotOptions = {}
     try {
         const store = await KeyValueStore.open(keyValueStoreName, { config: config ?? Configuration.getGlobalConfig() });
 
+        const { APIFY_IS_AT_HOME } = process.env;
+
         if (saveScreenshot) {
-            const screenshotName = `${key}.jpg`;
+            const screenshotName = APIFY_IS_AT_HOME ? `${key}.jpg` : key;
             const screenshotBuffer = await page.screenshot({ fullPage: true, quality: screenshotQuality, type: 'jpeg' });
             await store.setValue(screenshotName, screenshotBuffer, { contentType: 'image/jpeg' });
         }
 
         if (saveHtml) {
-            const htmlName = `${key}.html`;
+            const htmlName = APIFY_IS_AT_HOME ? `${key}.html` : key;
             const html = await page.content();
             await store.setValue(htmlName, html, { contentType: 'text/html' });
         }

packages/utils/src/index.ts

@@ -7,7 +7,6 @@ export * from './internals/memory-info';
 export * from './internals/debug';
 export * as social from './internals/social';
 export * from './internals/typedefs';
-export * from './internals/error_tracker';
 export * from './internals/open_graph_parser';
 export * from './internals/gotScraping';
 export * from './internals/robots';

packages/utils/test/non-error-objects-working.test.ts

@@ -1,4 +1,4 @@
-import { ErrorTracker } from '../src/internals/error_tracker';
+import { ErrorTracker } from '../../core/src/crawlers/error_tracker';
 
 describe('ErrorTracker', () => {
     test('processing a non-error error should not crash', () => {