From 0d3ff45ddc6a0d0cdd32b6e78c548ee203ce89b9 Mon Sep 17 00:00:00 2001 From: SimFG Date: Fri, 7 Apr 2023 00:25:06 +0800 Subject: [PATCH] Update the benchmark result Signed-off-by: SimFG --- examples/README.md | 6 +++--- examples/benchmark/benchmark_sqlite_faiss_onnx.py | 8 ++++++-- gptcache/manager/scalar_data/sqlalchemy.py | 2 +- tests/integration_tests/common/common_type.py | 2 +- .../examples/sqlite_faiss_mock/sqlite_faiss_mock.py | 2 +- .../examples/sqlite_faiss_onnx/sqlite_faiss_onnx.py | 2 +- 6 files changed, 13 insertions(+), 9 deletions(-) diff --git a/examples/README.md b/examples/README.md index 83e11cb1..35fbb143 100644 --- a/examples/README.md +++ b/examples/README.md @@ -455,6 +455,6 @@ similarity evaluation func: pair_evaluation (search distance) | threshold | average time | positive | negative | fail count | |-----------|--------------|----------|----------|------------| -| 20 | 0.04s | 455 | 27 | 517 | -| 50 | 0.09s | 871 | 86 | 42 | -| 100 | 0.12s | 905 | 93 | 1 | +| 0.95 | 0.12s | 425 | 25 | 549 | +| 0.9 | 0.23s | 804 | 77 | 118 | +| 0.8 | 0.26s | 904 | 92 | 3 | diff --git a/examples/benchmark/benchmark_sqlite_faiss_onnx.py b/examples/benchmark/benchmark_sqlite_faiss_onnx.py index dde14d11..7011240f 100644 --- a/examples/benchmark/benchmark_sqlite_faiss_onnx.py +++ b/examples/benchmark/benchmark_sqlite_faiss_onnx.py @@ -31,6 +31,7 @@ def run(): # if rank1 <= 0.5: # rank2 = evaluation_onnx.evaluation(src_dict, cache_dict, **kwargs) # return rank2 if rank2 != 0 else 1 + # return 0 # # def range(self): # return 0.0, 1.0 @@ -40,9 +41,9 @@ def evaluation(self, src_dict, cache_dict, **kwargs): return super().evaluation(src_dict, cache_dict, **kwargs) def range(self): - return 0.0, 1.0 + return super().range() - sqlite_file = 'gptcache.db' + sqlite_file = 'sqlite.db' faiss_file = 'faiss.index' has_data = os.path.isfile(sqlite_file) and os.path.isfile(faiss_file) @@ -50,6 +51,9 @@ def range(self): cache.init(embedding_func=embedding_onnx.to_embeddings, data_manager=data_manager, similarity_evaluation=WrapEvaluation(), + config=Config( + similarity_threshold=0.95 + ), ) i = 0 diff --git a/gptcache/manager/scalar_data/sqlalchemy.py b/gptcache/manager/scalar_data/sqlalchemy.py index d6041369..5c27829d 100644 --- a/gptcache/manager/scalar_data/sqlalchemy.py +++ b/gptcache/manager/scalar_data/sqlalchemy.py @@ -56,7 +56,7 @@ class SQLDataBase(CacheStorage): """ Using sqlalchemy to manage SQLite, PostgreSQL, MySQL, MariaDB, SQL Server and Oracle. """ - def __init__(self, db_type: str = 'sqlite', url: str = 'sqlite:///./gptcache.db', table_name: str = 'gptcache'): + def __init__(self, db_type: str = 'sqlite', url: str = 'sqlite:///./sqlite.db', table_name: str = 'gptcache'): self._url = url self._model = get_model(table_name, db_type) self._engine = create_engine(self._url) diff --git a/tests/integration_tests/common/common_type.py b/tests/integration_tests/common/common_type.py index 0129392e..17806eb3 100644 --- a/tests/integration_tests/common/common_type.py +++ b/tests/integration_tests/common/common_type.py @@ -1,3 +1,3 @@ """ Initialized parameters """ -sqlite_file = 'gptcache.db' +sqlite_file = 'sqlite.db' faiss_file = 'faiss.index' \ No newline at end of file diff --git a/tests/integration_tests/examples/sqlite_faiss_mock/sqlite_faiss_mock.py b/tests/integration_tests/examples/sqlite_faiss_mock/sqlite_faiss_mock.py index a4d00bad..f3658040 100644 --- a/tests/integration_tests/examples/sqlite_faiss_mock/sqlite_faiss_mock.py +++ b/tests/integration_tests/examples/sqlite_faiss_mock/sqlite_faiss_mock.py @@ -15,7 +15,7 @@ def mock_embeddings(data, **kwargs): def run(): - sqlite_file = 'gptcache.db' + sqlite_file = 'sqlite.db' faiss_file = 'faiss.index' has_data = os.path.isfile(sqlite_file) and os.path.isfile(faiss_file) data_manager = get_data_manager('sqlite', 'faiss', diff --git a/tests/integration_tests/examples/sqlite_faiss_onnx/sqlite_faiss_onnx.py b/tests/integration_tests/examples/sqlite_faiss_onnx/sqlite_faiss_onnx.py index b1ccf685..411f5ef2 100644 --- a/tests/integration_tests/examples/sqlite_faiss_onnx/sqlite_faiss_onnx.py +++ b/tests/integration_tests/examples/sqlite_faiss_onnx/sqlite_faiss_onnx.py @@ -11,7 +11,7 @@ def run(): onnx = Onnx() - sqlite_file = 'gptcache.db' + sqlite_file = 'sqlite.db' faiss_file = 'faiss.index' has_data = os.path.isfile(sqlite_file) and os.path.isfile(faiss_file) data_manager = get_data_manager('sqlite', 'faiss',