Skip to content

Commit eb6e3de

Browse files
generallgenerall
committed
allow to skip incompatible combinations
1 parent 59bea06 commit eb6e3de

File tree

5 files changed

+51
-38
lines changed

5 files changed

+51
-38
lines changed

‎engine/base_client/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,7 @@
22
from engine.base_client.configure import BaseConfigurator
33
from engine.base_client.search import BaseSearcher
44
from engine.base_client.upload import BaseUploader
5+
6+
7+
class IncompatibilityError(Exception):
8+
pass

‎engine/clients/elasticsearch/configure.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
from elasticsearch import Elasticsearch, NotFoundError
22

3+
from engine.base_client import IncompatibilityError
34
from engine.base_client.configure import BaseConfigurator
45
from engine.base_client.distances import Distance
56
from engine.clients.elasticsearch import (
@@ -45,6 +46,9 @@ def recreate(
4546
vector_size,
4647
collection_params,
4748
):
49+
if distance == Distance.DOT:
50+
raise IncompatibilityError
51+
4852
self.client.indices.create(
4953
index=ELASTIC_INDEX,
5054
mappings={

‎experiments/configurations/elastic-single-node.json‎

Lines changed: 35 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@
77
},
88
"collection_params": { "index_options": { "m": 16, "ef_construction": 100 } },
99
"search_params": [
10-
{ "parallel": 1, "num_candidates": 64 }, { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
11-
{ "parallel": 2, "num_candidates": 64 }, { "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
12-
{ "parallel": 4, "num_candidates": 64 }, { "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
13-
{ "parallel": 8, "num_candidates": 64 }, { "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
14-
{ "parallel": 100, "num_candidates": 64 }, { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
10+
{ "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
11+
{ "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
12+
{ "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
13+
{ "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
14+
{ "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
1515
],
1616
"upload_params": { "parallel": 16 }
1717
},
@@ -23,11 +23,11 @@
2323
},
2424
"collection_params": { "index_options": { "m": 16, "ef_construction": 128 } },
2525
"search_params": [
26-
{ "parallel": 1, "num_candidates": 64 }, { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
27-
{ "parallel": 2, "num_candidates": 64 }, { "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
28-
{ "parallel": 4, "num_candidates": 64 }, { "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
29-
{ "parallel": 8, "num_candidates": 64 }, { "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
30-
{ "parallel": 100, "num_candidates": 64 }, { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
26+
{ "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
27+
{ "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
28+
{ "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
29+
{ "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
30+
{ "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
3131
],
3232
"upload_params": { "parallel": 16 }
3333
},
@@ -39,11 +39,11 @@
3939
},
4040
"collection_params": { "index_options": { "m": 32, "ef_construction": 128 } },
4141
"search_params": [
42-
{ "parallel": 1, "num_candidates": 64 }, { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
43-
{ "parallel": 2, "num_candidates": 64 }, { "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
44-
{ "parallel": 4, "num_candidates": 64 }, { "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
45-
{ "parallel": 8, "num_candidates": 64 }, { "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
46-
{ "parallel": 100, "num_candidates": 64 }, { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
42+
{ "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
43+
{ "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
44+
{ "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
45+
{ "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
46+
{ "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
4747
],
4848
"upload_params": { "parallel": 16 }
4949
},
@@ -55,11 +55,11 @@
5555
},
5656
"collection_params": { "index_options": { "m": 32, "ef_construction": 256 } },
5757
"search_params": [
58-
{ "parallel": 1, "num_candidates": 64 }, { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
59-
{ "parallel": 2, "num_candidates": 64 }, { "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
60-
{ "parallel": 4, "num_candidates": 64 }, { "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
61-
{ "parallel": 8, "num_candidates": 64 }, { "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
62-
{ "parallel": 100, "num_candidates": 64 }, { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
58+
{ "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
59+
{ "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
60+
{ "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
61+
{ "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
62+
{ "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
6363
],
6464
"upload_params": { "parallel": 16 }
6565
},
@@ -71,11 +71,11 @@
7171
},
7272
"collection_params": { "index_options": { "m": 32, "ef_construction": 512 } },
7373
"search_params": [
74-
{ "parallel": 1, "num_candidates": 64 }, { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
75-
{ "parallel": 2, "num_candidates": 64 }, { "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
76-
{ "parallel": 4, "num_candidates": 64 }, { "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
77-
{ "parallel": 8, "num_candidates": 64 }, { "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
78-
{ "parallel": 100, "num_candidates": 64 }, { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
74+
{ "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
75+
{ "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
76+
{ "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
77+
{ "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
78+
{ "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
7979
],
8080
"upload_params": { "parallel": 16 }
8181
},
@@ -87,11 +87,11 @@
8787
},
8888
"collection_params": { "index_options": { "m": 64, "ef_construction": 256 } },
8989
"search_params": [
90-
{ "parallel": 1, "num_candidates": 64 }, { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
91-
{ "parallel": 2, "num_candidates": 64 }, { "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
92-
{ "parallel": 4, "num_candidates": 64 }, { "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
93-
{ "parallel": 8, "num_candidates": 64 }, { "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
94-
{ "parallel": 100, "num_candidates": 64 }, { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
90+
{ "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
91+
{ "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
92+
{ "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
93+
{ "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
94+
{ "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
9595
],
9696
"upload_params": { "parallel": 16 }
9797
},
@@ -103,11 +103,11 @@
103103
},
104104
"collection_params": { "index_options": { "m": 64, "ef_construction": 512 } },
105105
"search_params": [
106-
{ "parallel": 1, "num_candidates": 64 }, { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
107-
{ "parallel": 2, "num_candidates": 64 }, { "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
108-
{ "parallel": 4, "num_candidates": 64 }, { "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
109-
{ "parallel": 8, "num_candidates": 64 }, { "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
110-
{ "parallel": 100, "num_candidates": 64 }, { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
106+
{ "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
107+
{ "parallel": 2, "num_candidates": 128 }, { "parallel": 2, "num_candidates": 256 }, { "parallel": 2, "num_candidates": 512 },
108+
{ "parallel": 4, "num_candidates": 128 }, { "parallel": 4, "num_candidates": 256 }, { "parallel": 4, "num_candidates": 512 },
109+
{ "parallel": 8, "num_candidates": 128 }, { "parallel": 8, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 512 },
110+
{ "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
111111
],
112112
"upload_params": { "parallel": 16 }
113113
}

‎run.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
from benchmark import DATASETS_DIR, ROOT_DIR
99
from benchmark.dataset import Dataset
10+
from engine.base_client import IncompatibilityError
1011
from engine.clients.client_factory import ClientFactory
1112

1213
app = typer.Typer()
@@ -61,7 +62,11 @@ def run(engines: str = "*", datasets: str = "*", host: str = "localhost"):
6162
client = ClientFactory(host).build_client(engine_config)
6263
dataset = Dataset(dataset_config)
6364
dataset.download()
64-
client.run_experiment(dataset)
65+
try:
66+
client.run_experiment(dataset)
67+
except IncompatibilityError as e:
68+
print(f"Skipping {engine_name} - {dataset_name}, incompatible params")
69+
continue
6570

6671

6772
if __name__ == "__main__":

‎run_all_engines.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ function run_exp() {
2020

2121
# run_exp "qdrant-single-node" 'qdrant-m-*'
2222
# run_exp "weaviate-single-node" 'weaviate-m-*'
23-
run_exp "milvus-single-node" 'milvus-m-*'
24-
run_exp "elastic-single-node" 'elastic-m-*'
23+
# run_exp "milvus-single-node" 'milvus-m-*'
24+
run_exp "elasticsearch-single-node" 'elastic-m-*'
2525

2626
# Extra: qdrant configured to tune RPS
2727

0 commit comments

Comments
 (0)