Exponential Backoff Mechanism for RateLimit Issues in /Chat

app/backend/approaches/chatreadretrieveread.py

@@ -8,6 +8,7 @@
 from core.messagebuilder import MessageBuilder
 from core.modelhelper import get_token_limit
 from text import nonewlines
+from tenacity import retry, stop_after_attempt, wait_random_exponential
 
 
 class ChatReadRetrieveReadApproach(Approach):
@@ -56,7 +57,8 @@ def __init__(self, search_client: SearchClient, chatgpt_deployment: str, chatgpt
         self.sourcepage_field = sourcepage_field
         self.content_field = content_field
         self.chatgpt_token_limit = get_token_limit(chatgpt_model)
-
+
+    @retry(wait=wait_random_exponential(multiplier=1, min=4, max=10), stop=stop_after_attempt(3))  
     def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any]) -> Any:
         has_text = overrides.get("retrieval_mode") in ["text", "hybrid", None]
         has_vector = overrides.get("retrieval_mode") in ["vectors", "hybrid", None]
@@ -154,7 +156,19 @@ def run(self, history: Sequence[dict[str, str]], overrides: dict[str, Any]) -> A
             temperature=overrides.get("temperature") or 0.7,
             max_tokens=1024,
             n=1)
-
+
+        # Retry on RateLimitError  
+        try:  
+         chat_completion = openai.ChatCompletion.create( 
+                deployment_id=self.chatgpt_deployment,  
+                model=self.chatgpt_model,  
+                messages=messages,  
+                temperature=overrides.get("temperature") or 0.7,
+                max_tokens=1024,  
+                n=1)  
+        except openai.error.RateLimitError as e:  
+            self.logger.error(f"Rate limit error: {e}")  
+            raise 
         chat_content = chat_completion.choices[0].message.content
 
         msg_to_display = '\n\n'.join([str(message) for message in messages])

app/backend/requirements.txt

@@ -5,3 +5,4 @@ openai[datalib]==0.27.8
 tiktoken==0.4.0
 azure-search-documents==11.4.0b6
 azure-storage-blob==12.14.1
+tenacity==8.2.2