diff --git a/backend/src/build-system/handlers/ux/sitemap-structure/sms-page.ts b/backend/src/build-system/handlers/ux/sitemap-structure/sms-page.ts index 5969c004..4c05a88e 100644 --- a/backend/src/build-system/handlers/ux/sitemap-structure/sms-page.ts +++ b/backend/src/build-system/handlers/ux/sitemap-structure/sms-page.ts @@ -55,9 +55,6 @@ export class Level2UXSitemapStructureHandler implements BuildHandler { ); } - // Process all sections concurrently - const modelProvider = OpenAIModelProvider.getInstance(); - // Prepare all requests const requests = sections.map((section) => ({ model: 'gpt-4o-mini', diff --git a/backend/src/common/model-provider/openai-model-provider.ts b/backend/src/common/model-provider/openai-model-provider.ts index 67d27b6f..ac7b93b5 100644 --- a/backend/src/common/model-provider/openai-model-provider.ts +++ b/backend/src/common/model-provider/openai-model-provider.ts @@ -42,7 +42,7 @@ export class OpenAIModelProvider implements IModelProvider { const queue = new PQueue({ concurrency, - timeout: 30000, // 30 second timeout + timeout: 120000, // 120 second timeout }); // Log queue events for monitoring diff --git a/llm-server/src/model/remote-model-instance.ts b/llm-server/src/model/remote-model-instance.ts index 46458180..57248874 100644 --- a/llm-server/src/model/remote-model-instance.ts +++ b/llm-server/src/model/remote-model-instance.ts @@ -20,12 +20,17 @@ export class RemoteOpenAIModelEngine implements ModelInstance { baseURL: config.endpoint, }); + this.initializeQueue(); + } + + private initializeQueue() { // Initialize queue with 30 RPS limit this.queue = new PQueue({ - intervalCap: config.rps ?? 30, // 30 requests + intervalCap: this.config.rps ?? 30, // 30 requests interval: 1000, // per 1000ms (1 second) carryoverConcurrencyCount: true, // Carry over pending tasks - timeout: 30000, // 30 second timeout + // FIXME: hack way to set up timeout + timeout: 120000, // 120 second timeout to accommodate longer streams }); // Log queue events for monitoring @@ -73,11 +78,17 @@ export class RemoteOpenAIModelEngine implements ModelInstance { }); if (!result) { + this.logger.warn('Queue is closed, reinitializing queue'); + this.initializeQueue(); throw new Error('Queue is closed'); } return result; } catch (error) { + if (error.message === 'Queue is closed') { + this.logger.warn('Reinitializing queue due to closure'); + this.initializeQueue(); + } const modelError = this.createModelError(error); this.logger.error('Error in chat:', modelError); throw modelError; @@ -100,6 +111,8 @@ export class RemoteOpenAIModelEngine implements ModelInstance { ); if (!stream) { + this.logger.warn('Queue is closed, reinitializing queue'); + this.initializeQueue(); throw new Error('Queue is closed'); } @@ -108,6 +121,10 @@ export class RemoteOpenAIModelEngine implements ModelInstance { yield chunk; } } catch (error) { + if (error.message === 'Queue is closed') { + this.logger.warn('Reinitializing queue due to closure'); + this.initializeQueue(); + } const modelError = this.createModelError(error); this.logger.error('Error in chatStream:', modelError); throw modelError;