fix: avoid duplicate DDL and question-sql in chromadb

src/vanna/chromadb/chromadb_vector.py

@@ -8,6 +8,7 @@
 from chromadb.utils import embedding_functions
 
 from ..base import VannaBase
+from ..utils import deterministic_uuid
 
 default_ef = embedding_functions.DefaultEmbeddingFunction()
 
@@ -65,7 +66,7 @@ def add_question_sql(self, question: str, sql: str, **kwargs) -> str:
             },
             ensure_ascii=False,
         )
-        id = str(uuid.uuid4()) + "-sql"
+        id = deterministic_uuid(question_sql_json) + "-sql"
         self.sql_collection.add(
             documents=question_sql_json,
             embeddings=self.generate_embedding(question_sql_json),
@@ -75,7 +76,7 @@ def add_question_sql(self, question: str, sql: str, **kwargs) -> str:
         return id
 
     def add_ddl(self, ddl: str, **kwargs) -> str:
-        id = str(uuid.uuid4()) + "-ddl"
+        id = deterministic_uuid(ddl) + "-ddl"
         self.ddl_collection.add(
             documents=ddl,
             embeddings=self.generate_embedding(ddl),

src/vanna/utils.py

@@ -1,5 +1,8 @@
+import hashlib
 import os
 import re
+import uuid
+from typing import Union
 
 from .exceptions import ImproperlyConfigured, ValidationError
 
@@ -48,3 +51,27 @@ def sanitize_model_name(model_name):
         return model_name
     except Exception as e:
         raise ValidationError(e)
+
+
+def deterministic_uuid(content: Union[str, bytes]) -> str:
+    """Creates deterministic UUID on hash value of string or byte content.
+
+    Args:
+        content: String or byte representation of data.
+
+    Returns:
+        UUID of the content.
+    """
+    if isinstance(content, str):
+        content_bytes = content.encode("utf-8")
+    elif isinstance(content, bytes):
+        content_bytes = content
+    else:
+        raise ValueError(f"Content type {type(content)} not supported !")
+
+    hash_object = hashlib.sha256(content_bytes)
+    hash_hex = hash_object.hexdigest()
+    namespace = uuid.UUID("00000000-0000-0000-0000-000000000000")
+    content_uuid = str(uuid.uuid5(namespace, hash_hex))
+
+    return content_uuid