Skip to content

Commit

Permalink
fix: ensure the status message cannot stuck the crawler (#2114)
Browse files Browse the repository at this point in the history
Recently we saw an actor being stuck after it finished. It might be
coming from the status message calls and the exponential backoff behind
the API calls. This makes sure the status message API call won't take
too long.
  • Loading branch information
B4nan committed Oct 4, 2023
1 parent 77600c7 commit 9034f08
Showing 1 changed file with 12 additions and 2 deletions.
14 changes: 12 additions & 2 deletions packages/basic-crawler/src/internals/basic-crawler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -741,12 +741,22 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
/**
* This method is periodically called by the crawler, every `statusMessageLoggingInterval` seconds.
*/
setStatusMessage(message: string, options: SetStatusMessageOptions = {}) {
async setStatusMessage(message: string, options: SetStatusMessageOptions = {}) {
const data = options.isStatusMessageTerminal != null ? { terminal: options.isStatusMessageTerminal } : undefined;
this.log.internal(LogLevel[options.level as 'DEBUG' ?? 'DEBUG'], message, data);

const client = this.config.getStorageClient();
return client.setStatusMessage?.(message, options);

if (!client.setStatusMessage) {
return;
}

// just to be sure, this should be fast
await addTimeoutToPromise(
() => client.setStatusMessage!(message, options),
1000,
`setting status message timed out`,
);
}

private getPeriodicLogger() {
Expand Down

0 comments on commit 9034f08

Please sign in to comment.