Possible workaround for Significant-Gravitas#796

requirements.txt

@@ -18,3 +18,4 @@ Pillow
 coverage
 flake8
 numpy
+tenacity

scripts/memory/base.py

@@ -2,16 +2,61 @@
 import abc
 from config import AbstractSingleton, Config
 import openai
+import numpy as np
+from itertools import islice
+import tiktoken
+from tenacity import retry, wait_random_exponential, stop_after_attempt, retry_if_not_exception_type
 
 cfg = Config()
 
+def batched(iterable, n):
+    """Batch data into tuples of length n. The last batch may be shorter."""
+    # batched('ABCDEFG', 3) --> ABC DEF G
+    if n < 1:
+        raise ValueError('n must be at least one')
+    it = iter(iterable)
+    while (batch := tuple(islice(it, n))):
+        yield batch
+
+def chunked_tokens(text, encoding_name, chunk_length):
+    encoding = tiktoken.get_encoding(encoding_name)
+    tokens = encoding.encode(text)
+    chunks_iterator = batched(tokens, chunk_length)
+    yield from chunks_iterator
+
+
+EMBEDDING_MODEL = 'text-embedding-ada-002'
+EMBEDDING_CTX_LENGTH = 8191
+EMBEDDING_ENCODING = 'cl100k_base'
+
+@retry(wait=wait_random_exponential(min=1, max=20), stop=stop_after_attempt(6), retry=retry_if_not_exception_type(openai.InvalidRequestError))
+def get_embedding(text_or_tokens, model=EMBEDDING_MODEL):
+    return openai.Embedding.create(input=text_or_tokens, model=model)["data"][0]["embedding"]
+
+def len_safe_get_embedding(text, model=EMBEDDING_MODEL, max_tokens=EMBEDDING_CTX_LENGTH, encoding_name=EMBEDDING_ENCODING, average=True):
+    chunk_embeddings = []
+    chunk_lens = []
+    for chunk in chunked_tokens(text, encoding_name=encoding_name, chunk_length=max_tokens):
+        chunk_embeddings.append(get_embedding(chunk, model=model))
+        chunk_lens.append(len(chunk))
+
+    if average:
+        chunk_embeddings = np.average(chunk_embeddings, axis=0, weights=chunk_lens)
+        chunk_embeddings = chunk_embeddings / np.linalg.norm(chunk_embeddings)  # normalizes length to 1
+        chunk_embeddings = chunk_embeddings.tolist()
+    return chunk_embeddings
+
+
 
 def get_ada_embedding(text):
     text = text.replace("\n", " ")
     if cfg.use_azure:
         return openai.Embedding.create(input=[text], engine=cfg.get_azure_deployment_id_for_model("text-embedding-ada-002"))["data"][0]["embedding"]
     else:
-        return openai.Embedding.create(input=[text], model="text-embedding-ada-002")["data"][0]["embedding"]
+        if (len(text) > EMBEDDING_CTX_LENGTH):
+            return len_safe_get_embedding(text)
+        else:
+            return openai.Embedding.create(input=[text], model="text-embedding-ada-002")["data"][0]["embedding"]
 
 
 class MemoryProviderSingleton(AbstractSingleton):