From a9c2fb0d8e2be879449dd585240844d6eae41d94 Mon Sep 17 00:00:00 2001 From: ZHallen122 Date: Sat, 8 Mar 2025 08:56:40 -0500 Subject: [PATCH 1/3] remove timeout --- llm-server/src/model/remote-model-instance.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm-server/src/model/remote-model-instance.ts b/llm-server/src/model/remote-model-instance.ts index f75876a6..27f4df09 100644 --- a/llm-server/src/model/remote-model-instance.ts +++ b/llm-server/src/model/remote-model-instance.ts @@ -30,7 +30,6 @@ export class RemoteOpenAIModelEngine implements ModelInstance { interval: 1000, // per 1000ms (1 second) carryoverConcurrencyCount: true, // Carry over pending tasks // FIXME: hack way to set up timeout - timeout: 120000, // 120 second timeout to accommodate longer streams }); // Log queue events for monitoring @@ -74,6 +73,7 @@ export class RemoteOpenAIModelEngine implements ModelInstance { return await this.client.chat.completions.create({ model: this.config.model, messages, + temperature: 1, // Default to 0.7 if not specified }); }); From 3fd7a200038ef17e9ae18c49976c0569ae1aa89c Mon Sep 17 00:00:00 2001 From: ZHallen122 Date: Sat, 8 Mar 2025 10:22:28 -0500 Subject: [PATCH 2/3] Add timesout for each chat --- llm-server/src/llm-provider.ts | 21 +++++++++++++++---- llm-server/src/main.ts | 3 ++- llm-server/src/model/remote-model-instance.ts | 1 - 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/llm-server/src/llm-provider.ts b/llm-server/src/llm-provider.ts index b3a033b3..c8a38b96 100644 --- a/llm-server/src/llm-provider.ts +++ b/llm-server/src/llm-provider.ts @@ -104,16 +104,29 @@ export class LLMProvider { return model; } - async chat(input: MessageInput): Promise { + async chat(input: MessageInput, timeoutMs: number): Promise { try { const model = this.getModelInstance(input.model); - const completion = await model.chat(input.messages); - return completion.choices[0].message.content || ''; + + // Set a timeout dynamically based on the provided value + const timeoutPromise = new Promise((_, reject) => + setTimeout(() => reject(new Error('Chat request timed out')), timeoutMs), + ); + + // Race between the actual model call and the timeout + const completion = await Promise.race([ + model.chat(input.messages), + timeoutPromise, + ]); + + return (completion as any).choices[0].message.content || ''; } catch (error) { - this.logger.error('Error in chat:', error); + this.logger.error(`Error in chat (Timeout: ${timeoutMs}ms):`, error); throw error; } } + + async *chatStream( input: MessageInput, diff --git a/llm-server/src/main.ts b/llm-server/src/main.ts index 289caf53..a16701c4 100644 --- a/llm-server/src/main.ts +++ b/llm-server/src/main.ts @@ -137,7 +137,8 @@ export class App { } } else { // Handle regular response - const response = await this.llmProvider.chat(input); + // TODO make it to dynamic Now is 200 second by defult. + const response = await this.llmProvider.chat(input, 200000); res.json({ model: input.model, choices: [ diff --git a/llm-server/src/model/remote-model-instance.ts b/llm-server/src/model/remote-model-instance.ts index 27f4df09..def1e589 100644 --- a/llm-server/src/model/remote-model-instance.ts +++ b/llm-server/src/model/remote-model-instance.ts @@ -73,7 +73,6 @@ export class RemoteOpenAIModelEngine implements ModelInstance { return await this.client.chat.completions.create({ model: this.config.model, messages, - temperature: 1, // Default to 0.7 if not specified }); }); From 9bac071cc637382cf7a11fc44340eeac762a8900 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Sat, 8 Mar 2025 15:23:59 +0000 Subject: [PATCH 3/3] [autofix.ci] apply automated fixes --- llm-server/src/llm-provider.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/llm-server/src/llm-provider.ts b/llm-server/src/llm-provider.ts index c8a38b96..bf2d2069 100644 --- a/llm-server/src/llm-provider.ts +++ b/llm-server/src/llm-provider.ts @@ -107,26 +107,27 @@ export class LLMProvider { async chat(input: MessageInput, timeoutMs: number): Promise { try { const model = this.getModelInstance(input.model); - + // Set a timeout dynamically based on the provided value const timeoutPromise = new Promise((_, reject) => - setTimeout(() => reject(new Error('Chat request timed out')), timeoutMs), + setTimeout( + () => reject(new Error('Chat request timed out')), + timeoutMs, + ), ); - + // Race between the actual model call and the timeout const completion = await Promise.race([ model.chat(input.messages), timeoutPromise, ]); - + return (completion as any).choices[0].message.content || ''; } catch (error) { this.logger.error(`Error in chat (Timeout: ${timeoutMs}ms):`, error); throw error; } } - - async *chatStream( input: MessageInput,