Update the benchmark result (#129)

Signed-off-by: SimFG <bang.fu@zilliz.com>
zilliztech · Apr 6, 2023 · b377fda · b377fda
1 parent d4e1502
commit b377fda
Show file tree

Hide file tree

Showing 6 changed files with 13 additions and 9 deletions.
diff --git a/examples/README.md b/examples/README.md
@@ -455,6 +455,6 @@ similarity evaluation func: pair_evaluation (search distance)
 
  | threshold | average time | positive | negative | fail count |
 |-----------|--------------|----------|----------|------------|
-| 20        | 0.04s        | 455      | 27       | 517        |
-| 50        | 0.09s        | 871      | 86       | 42         |
-| 100       | 0.12s        | 905      | 93       | 1          |
+| 0.95      | 0.12s        | 425      | 25       | 549        |
+| 0.9       | 0.23s        | 804      | 77       | 118        |
+| 0.8       | 0.26s        | 904      | 92       | 3          |
diff --git a/examples/benchmark/benchmark_sqlite_faiss_onnx.py b/examples/benchmark/benchmark_sqlite_faiss_onnx.py
@@ -31,6 +31,7 @@ def run():
     #         if rank1 <= 0.5:
     #             rank2 = evaluation_onnx.evaluation(src_dict, cache_dict, **kwargs)
     #             return rank2 if rank2 != 0 else 1
+    #         return 0
     # 
     #     def range(self):
     #         return 0.0, 1.0
@@ -40,16 +41,19 @@ def evaluation(self, src_dict, cache_dict, **kwargs):
             return super().evaluation(src_dict, cache_dict, **kwargs)
 
         def range(self):
-            return 0.0, 1.0
+            return super().range()
 
-    sqlite_file = 'gptcache.db'
+    sqlite_file = 'sqlite.db'
     faiss_file = 'faiss.index'
     has_data = os.path.isfile(sqlite_file) and os.path.isfile(faiss_file)
 
     data_manager = get_data_manager('sqlite', 'faiss', dimension=embedding_onnx.dimension, max_size=100000)
     cache.init(embedding_func=embedding_onnx.to_embeddings,
                data_manager=data_manager,
                similarity_evaluation=WrapEvaluation(),
+               config=Config(
+                   similarity_threshold=0.95
+               ),
                )
 
     i = 0

diff --git a/gptcache/manager/scalar_data/sqlalchemy.py b/gptcache/manager/scalar_data/sqlalchemy.py
@@ -56,7 +56,7 @@ class SQLDataBase(CacheStorage):
     """
     Using sqlalchemy to manage SQLite, PostgreSQL, MySQL, MariaDB, SQL Server and Oracle.
     """
-    def __init__(self, db_type: str = 'sqlite', url: str = 'sqlite:///./gptcache.db', table_name: str = 'gptcache'):
+    def __init__(self, db_type: str = 'sqlite', url: str = 'sqlite:///./sqlite.db', table_name: str = 'gptcache'):
         self._url = url
         self._model = get_model(table_name, db_type)
         self._engine = create_engine(self._url)

diff --git a/tests/integration_tests/common/common_type.py b/tests/integration_tests/common/common_type.py
@@ -1,3 +1,3 @@
 """ Initialized parameters """
-sqlite_file = 'gptcache.db'
+sqlite_file = 'sqlite.db'
 faiss_file = 'faiss.index'
diff --git a/tests/integration_tests/examples/sqlite_faiss_mock/sqlite_faiss_mock.py b/tests/integration_tests/examples/sqlite_faiss_mock/sqlite_faiss_mock.py
@@ -15,7 +15,7 @@ def mock_embeddings(data, **kwargs):
 
 
 def run():
-    sqlite_file = 'gptcache.db'
+    sqlite_file = 'sqlite.db'
     faiss_file = 'faiss.index'
     has_data = os.path.isfile(sqlite_file) and os.path.isfile(faiss_file)
     data_manager = get_data_manager('sqlite', 'faiss',

diff --git a/tests/integration_tests/examples/sqlite_faiss_onnx/sqlite_faiss_onnx.py b/tests/integration_tests/examples/sqlite_faiss_onnx/sqlite_faiss_onnx.py
@@ -11,7 +11,7 @@
 def run():
     onnx = Onnx()
 
-    sqlite_file = 'gptcache.db'
+    sqlite_file = 'sqlite.db'
     faiss_file = 'faiss.index'
     has_data = os.path.isfile(sqlite_file) and os.path.isfile(faiss_file)
     data_manager = get_data_manager('sqlite', 'faiss',