add time interval options

README-EN.md

@@ -35,7 +35,7 @@ from random import randint
 
 msgs = [f"find the result of {randint(3, 100)} + {randint(4, 100)}" for _ in range(4)]
 # Annotate some data and get interrupted
-process_messages(msgs[:2], "test.jsonl")
+process_messages(msgs[:2], "test.jsonl", time_interval=5, max_tries=3)
 # Continue annotation
 process_messages(msgs, "test.jsonl")
 ```

README.md

@@ -36,7 +36,7 @@ from random import randint
 
 msgs = [f"find the result of {randint(3, 100)} + {randint(4, 100)}" for _ in range(4)]
 # 标注一部分后被中断
-process_messages(msgs[:2], "test.jsonl")
+process_messages(msgs[:2], "test.jsonl", time_interval=5, max_tries=3)
 # 继续标注
 process_messages(msgs, "test.jsonl")
 ```

setup.py

@@ -3,7 +3,7 @@
 """The setup script."""
 
 from setuptools import setup, find_packages
-version='0.1.0'
+VERSION = '0.1.1'
 
 with open('README.md') as readme_file:
     readme = readme_file.read()
@@ -29,14 +29,14 @@
     description="Wrapper of the web server of ChatGPT",
     install_requires=requirements,
     license="MIT license",
-    long_description=readme,
+    # long_description=readme,
     include_package_data=True,
     keywords='webchatter',
     name='webchatter',
     packages=find_packages(include=['webchatter', 'webchatter.*']),
     test_suite='tests',
     tests_require=test_requirements,
     url='https://github.com/RexWzh/webchatter',
-    version='0.1.0',
+    version=VERSION,
     zip_safe=False,
 )

webchatter/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = """Rex Wang"""
 __email__ = '1073853456@qq.com'
-__version__ = '0.1.0'
+__version__ = '0.1.1'
 
 import os, dotenv, requests
 from typing import Union

webchatter/checkpoint.py

@@ -1,15 +1,21 @@
 import os, json
 from webchatter import WebChat
 import tqdm, tqdm.notebook
+import time
 # from chattool import load_chats
 
-def process_messages(msgs, checkpoint:str, mode:str="delete", isjupyter:bool=False):
+def process_messages( msgs
+                    , checkpoint:str
+                    , time_interval:int=5
+                    , max_tries:int=-1
+                    , isjupyter:bool=False
+                    , interval_rate:float=1
+                    ):
     """Process the messages.
     
     Args:
         msgs (list): The messages.
         checkpoint (str): Store the checkpoint.
-        mode (str, optional): One of the three mode: delete, repeat, newchat. Defaults to "delete".
 
     Returns:
         list: The processed messages.
@@ -21,13 +27,22 @@ def process_messages(msgs, checkpoint:str, mode:str="delete", isjupyter:bool=Fal
         if len(processed) >= 1 and processed[0] != '':
             offset = len(processed)
     tq = tqdm.tqdm if not isjupyter else tqdm.notebook.tqdm
-    with open(checkpoint, 'a', encoding='utf-8') as f:    
+    chat = WebChat()
+    with open(checkpoint, 'a', encoding='utf-8') as f:
         for ind in tq(range(offset, len(msgs))):
-            msg = msgs[ind]
-            chat = WebChat()
-            ans = chat.ask(msg, keep=False)
-            data = {"index":ind + offset, "chat_log":{"user":msg, "assistant":ans}}
-            f.write(json.dumps(data) + '\n')
+            wait_time = time_interval
+            while max_tries:
+                try:
+                    msg = msgs[ind]
+                    ans = chat.ask(msg, keep=False)
+                    data = {"index":ind + offset, "chat_log":{"user":msg, "assistant":ans}}
+                    f.write(json.dumps(data) + '\n')
+                    break
+                except Exception as e:
+                    print(ind, e)
+                    max_tries -= 1
+                    time.sleep(wait_time)
+                    wait_time = wait_time * interval_rate
     return True
 
 def process_chats(chats, checkpoint:str):

webchatter/webchatter.py

@@ -6,7 +6,7 @@
 from .request import (
     get_account_status, get_models, get_beta_features,
     get_chat_list, get_chat_by_id, 
-    chat_completion, delete_chat,
+    chat_completion, delete_chat, get_conversation_limit
 )
 
 class Node():
@@ -152,6 +152,11 @@ def beta_features(self):
         url, token = self.backend_url, self.access_token
         return get_beta_features(url, token)
 
+    def conversation_limit(self):
+        """Get converation limitation"""
+        url, token = self.base_url, self.access_token
+        return get_conversation_limit(url, token)
+
     def chat_list(self, offset:int=0, limit:int=3, order:str="updated"):
         """Get the chat list."""
         url, token = self.backend_url, self.access_token