From 4de6b441bd4fa4b858cd7654459dd9e485512ee7 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Tue, 9 Apr 2024 11:42:10 +0700 Subject: [PATCH 01/33] Add support for manticoresearch engine --- engine/clients/client_factory.py | 9 ++ engine/clients/manticoresearch/__init__.py | 3 + engine/clients/manticoresearch/config.py | 2 + engine/clients/manticoresearch/configure.py | 62 ++++++++++++ engine/clients/manticoresearch/parser.py | 38 ++++++++ engine/clients/manticoresearch/search.py | 48 ++++++++++ engine/clients/manticoresearch/upload.py | 63 +++++++++++++ .../docker-compose.yaml | 18 ++++ .../manticoresearch-single-node.json | 94 +++++++++++++++++++ tools/run_benchmarks.sh | 2 +- 10 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 engine/clients/manticoresearch/__init__.py create mode 100644 engine/clients/manticoresearch/config.py create mode 100644 engine/clients/manticoresearch/configure.py create mode 100644 engine/clients/manticoresearch/parser.py create mode 100644 engine/clients/manticoresearch/search.py create mode 100644 engine/clients/manticoresearch/upload.py create mode 100644 engine/servers/manticoresearch-single-node/docker-compose.yaml create mode 100644 experiments/configurations/manticoresearch-single-node.json diff --git a/engine/clients/client_factory.py b/engine/clients/client_factory.py index a74df2ab4..53df7b2e2 100644 --- a/engine/clients/client_factory.py +++ b/engine/clients/client_factory.py @@ -31,6 +31,12 @@ WeaviateUploader, ) +from engine.clients.manticoresearch import ( + ManticoreSearchConfigurator, + ManticoreSearchSearcher, + ManticoreSearchUploader, +) + ENGINE_CONFIGURATORS = { "qdrant": QdrantConfigurator, "weaviate": WeaviateConfigurator, @@ -39,6 +45,7 @@ "opensearch": OpenSearchConfigurator, "redis": RedisConfigurator, "pgvector": PgVectorConfigurator, + "manticoresearch": ManticoreSearchConfigurator, } ENGINE_UPLOADERS = { @@ -49,6 +56,7 @@ "opensearch": OpenSearchUploader, "redis": RedisUploader, "pgvector": PgVectorUploader, + "manticoresearch": ManticoreSearchUploader, } ENGINE_SEARCHERS = { @@ -59,6 +67,7 @@ "opensearch": OpenSearchSearcher, "redis": RedisSearcher, "pgvector": PgVectorSearcher, + "manticoresearch": ManticoreSearchSearcher, } diff --git a/engine/clients/manticoresearch/__init__.py b/engine/clients/manticoresearch/__init__.py new file mode 100644 index 000000000..b4cad3e48 --- /dev/null +++ b/engine/clients/manticoresearch/__init__.py @@ -0,0 +1,3 @@ +from engine.clients.manticoresearch.configure import ManticoreSearchConfigurator +from engine.clients.manticoresearch.search import ManticoreSearchSearcher +from engine.clients.manticoresearch.upload import ManticoreSearchUploader diff --git a/engine/clients/manticoresearch/config.py b/engine/clients/manticoresearch/config.py new file mode 100644 index 000000000..9c92997c1 --- /dev/null +++ b/engine/clients/manticoresearch/config.py @@ -0,0 +1,2 @@ +MANTICORESEARCH_PORT = 9308 +MANTICORESEARCH_TABLE = "bench" diff --git a/engine/clients/manticoresearch/configure.py b/engine/clients/manticoresearch/configure.py new file mode 100644 index 000000000..2832574ec --- /dev/null +++ b/engine/clients/manticoresearch/configure.py @@ -0,0 +1,62 @@ +from benchmark.dataset import Dataset +from engine.base_client import IncompatibilityError +from engine.base_client.configure import BaseConfigurator +from engine.base_client.distances import Distance +from engine.clients.manticoresearch.config import ( + MANTICORESEARCH_PORT, + MANTICORESEARCH_TABLE, +) + +import requests +import json +class ManticoreSearchConfigurator(BaseConfigurator): + DISTANCE_MAPPING = { + Distance.L2: "L2", + Distance.COSINE: "COSINE", + } + + def __init__(self, host, collection_params, connection_params): + self.host = host + self.collection_params = collection_params + self.connection_params = connection_params + + def clean(self): + url = f'http://{self.host}:{MANTICORESEARCH_PORT}/sql?mode=raw' + query = f"DROP TABLE IF EXISTS `{MANTICORESEARCH_TABLE}`" + data = 'query=' + requests.utils.quote(query, safe='') + response = requests.post(url, data) + if response.status_code != 200: + print(f'Error cleaning table: {response.text}') + + def recreate(self, dataset, collection_params): + if dataset.config.distance == Distance.DOT: + raise IncompatibilityError + knn_options = collection_params.get('knn_options', {}) + hnsw_options = ' '.join([f"{key}='{value}'" for key, value in knn_options.items()]) + + vector_field = { + 'name': 'vector', + 'type': f"float_vector knn_type='hnsw' knn_dims='{dataset.config.vector_size}' hnsw_similarity='{self.DISTANCE_MAPPING[dataset.config.distance]}' {hnsw_options}", + } + + fields = [vector_field] + [ + { + 'name': field_name, + 'type': self.INDEX_TYPE_MAPPING.get(field_type, field_type), + } + for field_name, field_type in dataset.config.schema.items() + ] + + field_definitions = ', '.join([f"`{field['name']}` {field['type']}" for field in fields]) + + query = f""" + CREATE TABLE IF NOT EXISTS `{MANTICORESEARCH_TABLE}` ( + {field_definitions} + ) + """ + url = f'http://{self.host}:{MANTICORESEARCH_PORT}/sql?mode=raw' + data = 'query=' + requests.utils.quote(query, safe='') + response = requests.post(url, data) + + if response.status_code != 200: + print(f'Error creating table: {response.text}') diff --git a/engine/clients/manticoresearch/parser.py b/engine/clients/manticoresearch/parser.py new file mode 100644 index 000000000..c8bc957fc --- /dev/null +++ b/engine/clients/manticoresearch/parser.py @@ -0,0 +1,38 @@ +from typing import Any, List, Optional + +from engine.base_client.parser import BaseConditionParser, FieldValue + + +class ManticoreSearchConditionParser(BaseConditionParser): + def build_condition( + self, and_subfilters: Optional[List[Any]], or_subfilters: Optional[List[Any]] + ) -> Optional[Any]: + return { + "bool": { + "must": and_subfilters, + "should": or_subfilters, + } + } + + def build_exact_match_filter(self, field_name: str, value: FieldValue) -> Any: + return {"match": {field_name: value}} + + def build_range_filter( + self, + field_name: str, + lt: Optional[FieldValue], + gt: Optional[FieldValue], + lte: Optional[FieldValue], + gte: Optional[FieldValue], + ) -> Any: + return {"range": {field_name: {"lt": lt, "gt": gt, "lte": lte, "gte": gte}}} + + def build_geo_filter( + self, field_name: str, lat: float, lon: float, radius: float + ) -> Any: + return { + "geo_distance": { + "distance": f"{radius}m", + field_name: {"lat": lat, "lon": lon}, + } + } diff --git a/engine/clients/manticoresearch/search.py b/engine/clients/manticoresearch/search.py new file mode 100644 index 000000000..99bd8b917 --- /dev/null +++ b/engine/clients/manticoresearch/search.py @@ -0,0 +1,48 @@ +import multiprocessing as mp +import uuid +from typing import List, Tuple +from urllib.parse import urljoin + +from engine.base_client.search import BaseSearcher +from engine.clients.manticoresearch.config import ( + MANTICORESEARCH_PORT, + MANTICORESEARCH_TABLE, +) +from engine.clients.manticoresearch.parser import ManticoreSearchConditionParser +import requests + +class ManticoreSearchSearcher(BaseSearcher): + search_params = {} + parser = ManticoreSearchConditionParser() + + @classmethod + def get_mp_start_method(cls): + return "forkserver" if "forkserver" in mp.get_all_start_methods() else "spawn" + + @classmethod + def init_client(cls, host, distance, connection_params: dict, search_params: dict): + cls.session = requests.Session() + cls.session.headers.update({"Content-Type": "application/json"}) + cls.base_url = urljoin(f"http://{host}:{MANTICORESEARCH_PORT}", "/search") + cls.search_params = search_params + print(search_params) + + @classmethod + def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]: + knn = { + "index": MANTICORESEARCH_TABLE, + "knn": { + "field": "vector", + "query_vector": vector, + "k": top, + }, + **{"options": {"cutoff": 100}, **cls.search_params} + } + + meta_conditions = cls.parser.parse(meta_conditions) + if meta_conditions: + knn.update(meta_conditions) + + res = cls.session.post(cls.base_url, json=knn).json() + return [(int(hit["_id"]), hit["_score"]) for hit in res["hits"]["hits"]] + diff --git a/engine/clients/manticoresearch/upload.py b/engine/clients/manticoresearch/upload.py new file mode 100644 index 000000000..39e08ee41 --- /dev/null +++ b/engine/clients/manticoresearch/upload.py @@ -0,0 +1,63 @@ +import multiprocessing as mp +import uuid +from typing import List, Optional +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry +from engine.clients.manticoresearch.config import ( + MANTICORESEARCH_PORT, + MANTICORESEARCH_TABLE, +) + +from engine.base_client.upload import BaseUploader +import json + +class ClosableSession(requests.Session): + def __del__(self): + self.close() + +class ManticoreSearchUploader(BaseUploader): + api_url = None + session: requests.Session = None + upload_params = {} + + @classmethod + def get_mp_start_method(cls): + return "forkserver" if "forkserver" in mp.get_all_start_methods() else "spawn" + + @classmethod + def init_client(cls, host, distance, connection_params, upload_params): + cls.api_url = f"http://{host}:{MANTICORESEARCH_PORT}" + retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504]) + cls.host = host + cls.session = ClosableSession() + adapter = HTTPAdapter(max_retries=retries) + cls.session.mount("http://", adapter) + cls.session.headers.update({"Content-type": "application/x-ndjson"}) + cls.upload_params = upload_params + + @classmethod + def upload_batch(cls, ids: List[int], vectors: List[list], metadata: Optional[List[dict]]): + if metadata is None: + metadata = [{}] * len(vectors) + + docs = [] + for id, vector, payload in zip(ids, vectors, metadata): + data = { + "index": MANTICORESEARCH_TABLE, + "id": id, + "doc": { + "vector": vector + } + } + # data.update(payload) + docs.append({"insert": data}) + data = '\n'.join([json.dumps(item) for item in docs]) + response = cls.session.post(f"{cls.api_url}/bulk", data) + response.raise_for_status() + + @classmethod + def post_upload(cls, _distance): + response = cls.session.post(f"{cls.api_url}/sql?mode=raw", data=f"query=FLUSH%20RAMCHUNK%60{MANTICORESEARCH_TABLE}%60") + response.raise_for_status() + return {} diff --git a/engine/servers/manticoresearch-single-node/docker-compose.yaml b/engine/servers/manticoresearch-single-node/docker-compose.yaml new file mode 100644 index 000000000..323110ebc --- /dev/null +++ b/engine/servers/manticoresearch-single-node/docker-compose.yaml @@ -0,0 +1,18 @@ +version: '3.5' + +services: + searchd: + image: manticoresearch/manticore:dev-6.2.13-83733d1 + environment: + EXTRA: 1 + ports: + - "9308:9308" + logging: + driver: "json-file" + options: + max-file: 1 + max-size: 10m + deploy: + resources: + limits: + memory: 25Gb diff --git a/experiments/configurations/manticoresearch-single-node.json b/experiments/configurations/manticoresearch-single-node.json new file mode 100644 index 000000000..ae664c441 --- /dev/null +++ b/experiments/configurations/manticoresearch-single-node.json @@ -0,0 +1,94 @@ +[ + { + "name": "manticoresearch-default", + "engine": "manticoresearch", + "connection_params": { + "request_timeout": 10000 + }, + "collection_params": { "knn_options": { "hnsw_m": 16, "hnsw_ef_construction": 100 } }, + "search_params": [ + { "parallel": 1, "options": {"cutoff": 128} }, { "parallel": 1, "options": {"cutoff": 256} }, { "parallel": 1, "options": {"cutoff": 512} }, + { "parallel": 100, "options": {"cutoff": 128} }, { "parallel": 100, "options": {"cutoff": 256} }, { "parallel": 100, "options": {"cutoff": 512} } + ], + "upload_params": { "parallel": 16 } + }, + { + "name": "manticoresearch-m-16-ef-128", + "engine": "manticoresearch", + "connection_params": { + "request_timeout": 10000 + }, + "collection_params": { "knn_options": { "hnsw_m": 16, "hnsw_ef_construction": 128 } }, + "search_params": [ + { "parallel": 1, "options": {"cutoff": 128} }, { "parallel": 1, "options": {"cutoff": 256} }, { "parallel": 1, "options": {"cutoff": 512} }, + { "parallel": 100, "options": {"cutoff": 128} }, { "parallel": 100, "options": {"cutoff": 256} }, { "parallel": 100, "options": {"cutoff": 512} } + ], + "upload_params": { "parallel": 16 } + }, + { + "name": "manticoresearch-m-32-ef-128", + "engine": "manticoresearch", + "connection_params": { + "request_timeout": 10000 + }, + "collection_params": { "knn_options": { "hnsw_m": 32, "hnsw_ef_construction": 128 } }, + "search_params": [ + { "parallel": 1, "options": {"cutoff": 128} }, { "parallel": 1, "options": {"cutoff": 256} }, { "parallel": 1, "options": {"cutoff": 512} }, + { "parallel": 100, "options": {"cutoff": 128} }, { "parallel": 100, "options": {"cutoff": 256} }, { "parallel": 100, "options": {"cutoff": 512} } + ], + "upload_params": { "parallel": 16 } + }, + { + "name": "manticoresearch-m-32-ef-256", + "engine": "manticoresearch", + "connection_params": { + "request_timeout": 10000 + }, + "collection_params": { "knn_options": { "hnsw_m": 32, "hnsw_ef_construction": 256 } }, + "search_params": [ + { "parallel": 1, "options": {"cutoff": 128} }, { "parallel": 1, "options": {"cutoff": 256} }, { "parallel": 1, "options": {"cutoff": 512} }, + { "parallel": 100, "options": {"cutoff": 128} }, { "parallel": 100, "options": {"cutoff": 256} }, { "parallel": 100, "options": {"cutoff": 512} } + ], + "upload_params": { "parallel": 16 } + }, + { + "name": "manticoresearch-m-32-ef-512", + "engine": "manticoresearch", + "connection_params": { + "request_timeout": 10000 + }, + "collection_params": { "knn_options": { "hnsw_m": 32, "hnsw_ef_construction": 512 } }, + "search_params": [ + { "parallel": 1, "options": {"cutoff": 128} }, { "parallel": 1, "options": {"cutoff": 256} }, { "parallel": 1, "options": {"cutoff": 512} }, + { "parallel": 100, "options": {"cutoff": 128} }, { "parallel": 100, "options": {"cutoff": 256} }, { "parallel": 100, "options": {"cutoff": 512} } + ], + "upload_params": { "parallel": 16 } + }, + { + "name": "manticoresearch-m-64-ef-256", + "engine": "manticoresearch", + "connection_params": { + "request_timeout": 10000 + }, + "collection_params": { "knn_options": { "hnsw_m": 64, "hnsw_ef_construction": 256 } }, + "search_params": [ + { "parallel": 1, "options": {"cutoff": 128} }, { "parallel": 1, "options": {"cutoff": 256} }, { "parallel": 1, "options": {"cutoff": 512} }, + { "parallel": 100, "options": {"cutoff": 128} }, { "parallel": 100, "options": {"cutoff": 256} }, { "parallel": 100, "options": {"cutoff": 512} } + ], + "upload_params": { "parallel": 16 } + }, + { + "name": "manticoresearch-m-64-ef-512", + "engine": "manticoresearch", + "connection_params": { + "request_timeout": 10000 + }, + "collection_params": { "knn_options": { "hnsw_m": 64, "hnsw_ef_construction": 512 } }, + "search_params": [ + { "parallel": 1, "options": {"cutoff": 128} }, { "parallel": 1, "options": {"cutoff": 256} }, { "parallel": 1, "options": {"cutoff": 512} }, + { "parallel": 100, "options": {"cutoff": 128} }, { "parallel": 100, "options": {"cutoff": 256} }, { "parallel": 100, "options": {"cutoff": 512} } + ], + "upload_params": { "parallel": 16 } + } +] + diff --git a/tools/run_benchmarks.sh b/tools/run_benchmarks.sh index b07139142..fd3a187e6 100755 --- a/tools/run_benchmarks.sh +++ b/tools/run_benchmarks.sh @@ -9,7 +9,7 @@ SCRIPT_PATH="$( cd "$(dirname "$0")" &>/dev/null ; pwd -P )" export CLOUD_NAME=${CLOUD_NAME:-"custom"} DATASETS=("glove-100-angular" "deep-image-96" "gist-960-euclidean" "dbpedia-openai-1M-1536-angular") -VECTOR_DBS=("qdrant" "milvus" "elasticsearch" "weaviate" "redis") +VECTOR_DBS=("qdrant" "milvus" "elasticsearch" "weaviate" "redis" "manticoresearch") BRANCH="master" # Note: If you want to run with a different version of Qdrant. Set the QDRANT_VERSION env variable. From 553e67a7b4e8aaacc3b4ebd1e13d7c5995e8bf32 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Sun, 21 Apr 2024 19:38:01 +0700 Subject: [PATCH 02/33] Use ef in parameters for manticoresearch instead of cutoff that is wrong and update version --- engine/clients/manticoresearch/search.py | 3 +- .../docker-compose.yaml | 2 +- .../manticoresearch-single-node.json | 31 +++++++++---------- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/engine/clients/manticoresearch/search.py b/engine/clients/manticoresearch/search.py index 99bd8b917..413583b4b 100644 --- a/engine/clients/manticoresearch/search.py +++ b/engine/clients/manticoresearch/search.py @@ -35,8 +35,8 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]: "field": "vector", "query_vector": vector, "k": top, + **{**cls.search_params.get('options', {})}, }, - **{"options": {"cutoff": 100}, **cls.search_params} } meta_conditions = cls.parser.parse(meta_conditions) @@ -45,4 +45,3 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]: res = cls.session.post(cls.base_url, json=knn).json() return [(int(hit["_id"]), hit["_score"]) for hit in res["hits"]["hits"]] - diff --git a/engine/servers/manticoresearch-single-node/docker-compose.yaml b/engine/servers/manticoresearch-single-node/docker-compose.yaml index 323110ebc..16d170ef7 100644 --- a/engine/servers/manticoresearch-single-node/docker-compose.yaml +++ b/engine/servers/manticoresearch-single-node/docker-compose.yaml @@ -2,7 +2,7 @@ version: '3.5' services: searchd: - image: manticoresearch/manticore:dev-6.2.13-83733d1 + image: manticoresearch/manticore:dev-6.2.13-85fede4 environment: EXTRA: 1 ports: diff --git a/experiments/configurations/manticoresearch-single-node.json b/experiments/configurations/manticoresearch-single-node.json index ae664c441..e05e28485 100644 --- a/experiments/configurations/manticoresearch-single-node.json +++ b/experiments/configurations/manticoresearch-single-node.json @@ -7,8 +7,8 @@ }, "collection_params": { "knn_options": { "hnsw_m": 16, "hnsw_ef_construction": 100 } }, "search_params": [ - { "parallel": 1, "options": {"cutoff": 128} }, { "parallel": 1, "options": {"cutoff": 256} }, { "parallel": 1, "options": {"cutoff": 512} }, - { "parallel": 100, "options": {"cutoff": 128} }, { "parallel": 100, "options": {"cutoff": 256} }, { "parallel": 100, "options": {"cutoff": 512} } + { "parallel": 1, "options": {"ef": 128} }, { "parallel": 1, "options": {"ef": 256} }, { "parallel": 1, "options": {"ef": 512} }, + { "parallel": 100, "options": {"ef": 128} }, { "parallel": 100, "options": {"ef": 256} }, { "parallel": 100, "options": {"ef": 512} } ], "upload_params": { "parallel": 16 } }, @@ -20,8 +20,8 @@ }, "collection_params": { "knn_options": { "hnsw_m": 16, "hnsw_ef_construction": 128 } }, "search_params": [ - { "parallel": 1, "options": {"cutoff": 128} }, { "parallel": 1, "options": {"cutoff": 256} }, { "parallel": 1, "options": {"cutoff": 512} }, - { "parallel": 100, "options": {"cutoff": 128} }, { "parallel": 100, "options": {"cutoff": 256} }, { "parallel": 100, "options": {"cutoff": 512} } + { "parallel": 1, "options": {"ef": 128} }, { "parallel": 1, "options": {"ef": 256} }, { "parallel": 1, "options": {"ef": 512} }, + { "parallel": 100, "options": {"ef": 128} }, { "parallel": 100, "options": {"ef": 256} }, { "parallel": 100, "options": {"ef": 512} } ], "upload_params": { "parallel": 16 } }, @@ -33,8 +33,8 @@ }, "collection_params": { "knn_options": { "hnsw_m": 32, "hnsw_ef_construction": 128 } }, "search_params": [ - { "parallel": 1, "options": {"cutoff": 128} }, { "parallel": 1, "options": {"cutoff": 256} }, { "parallel": 1, "options": {"cutoff": 512} }, - { "parallel": 100, "options": {"cutoff": 128} }, { "parallel": 100, "options": {"cutoff": 256} }, { "parallel": 100, "options": {"cutoff": 512} } + { "parallel": 1, "options": {"ef": 128} }, { "parallel": 1, "options": {"ef": 256} }, { "parallel": 1, "options": {"ef": 512} }, + { "parallel": 100, "options": {"ef": 128} }, { "parallel": 100, "options": {"ef": 256} }, { "parallel": 100, "options": {"ef": 512} } ], "upload_params": { "parallel": 16 } }, @@ -46,8 +46,8 @@ }, "collection_params": { "knn_options": { "hnsw_m": 32, "hnsw_ef_construction": 256 } }, "search_params": [ - { "parallel": 1, "options": {"cutoff": 128} }, { "parallel": 1, "options": {"cutoff": 256} }, { "parallel": 1, "options": {"cutoff": 512} }, - { "parallel": 100, "options": {"cutoff": 128} }, { "parallel": 100, "options": {"cutoff": 256} }, { "parallel": 100, "options": {"cutoff": 512} } + { "parallel": 1, "options": {"ef": 128} }, { "parallel": 1, "options": {"ef": 256} }, { "parallel": 1, "options": {"ef": 512} }, + { "parallel": 100, "options": {"ef": 128} }, { "parallel": 100, "options": {"ef": 256} }, { "parallel": 100, "options": {"ef": 512} } ], "upload_params": { "parallel": 16 } }, @@ -59,8 +59,8 @@ }, "collection_params": { "knn_options": { "hnsw_m": 32, "hnsw_ef_construction": 512 } }, "search_params": [ - { "parallel": 1, "options": {"cutoff": 128} }, { "parallel": 1, "options": {"cutoff": 256} }, { "parallel": 1, "options": {"cutoff": 512} }, - { "parallel": 100, "options": {"cutoff": 128} }, { "parallel": 100, "options": {"cutoff": 256} }, { "parallel": 100, "options": {"cutoff": 512} } + { "parallel": 1, "options": {"ef": 128} }, { "parallel": 1, "options": {"ef": 256} }, { "parallel": 1, "options": {"ef": 512} }, + { "parallel": 100, "options": {"ef": 128} }, { "parallel": 100, "options": {"ef": 256} }, { "parallel": 100, "options": {"ef": 512} } ], "upload_params": { "parallel": 16 } }, @@ -72,8 +72,8 @@ }, "collection_params": { "knn_options": { "hnsw_m": 64, "hnsw_ef_construction": 256 } }, "search_params": [ - { "parallel": 1, "options": {"cutoff": 128} }, { "parallel": 1, "options": {"cutoff": 256} }, { "parallel": 1, "options": {"cutoff": 512} }, - { "parallel": 100, "options": {"cutoff": 128} }, { "parallel": 100, "options": {"cutoff": 256} }, { "parallel": 100, "options": {"cutoff": 512} } + { "parallel": 1, "options": {"ef": 128} }, { "parallel": 1, "options": {"ef": 256} }, { "parallel": 1, "options": {"ef": 512} }, + { "parallel": 100, "options": {"ef": 128} }, { "parallel": 100, "options": {"ef": 256} }, { "parallel": 100, "options": {"ef": 512} } ], "upload_params": { "parallel": 16 } }, @@ -85,10 +85,9 @@ }, "collection_params": { "knn_options": { "hnsw_m": 64, "hnsw_ef_construction": 512 } }, "search_params": [ - { "parallel": 1, "options": {"cutoff": 128} }, { "parallel": 1, "options": {"cutoff": 256} }, { "parallel": 1, "options": {"cutoff": 512} }, - { "parallel": 100, "options": {"cutoff": 128} }, { "parallel": 100, "options": {"cutoff": 256} }, { "parallel": 100, "options": {"cutoff": 512} } + { "parallel": 1, "options": {"ef": 128} }, { "parallel": 1, "options": {"ef": 256} }, { "parallel": 1, "options": {"ef": 512} }, + { "parallel": 100, "options": {"ef": 128} }, { "parallel": 100, "options": {"ef": 256} }, { "parallel": 100, "options": {"ef": 512} } ], "upload_params": { "parallel": 16 } } -] - +] \ No newline at end of file From 2ef6808274e9abbbc62c6c0ed6dc096f86c25be8 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Thu, 9 May 2024 14:25:09 +0700 Subject: [PATCH 03/33] Update manticore to the latest version --- engine/servers/manticoresearch-single-node/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/servers/manticoresearch-single-node/docker-compose.yaml b/engine/servers/manticoresearch-single-node/docker-compose.yaml index 16d170ef7..5276877b4 100644 --- a/engine/servers/manticoresearch-single-node/docker-compose.yaml +++ b/engine/servers/manticoresearch-single-node/docker-compose.yaml @@ -2,7 +2,7 @@ version: '3.5' services: searchd: - image: manticoresearch/manticore:dev-6.2.13-85fede4 + image: manticoresearch/manticore:dev-6.2.13-7047a53 environment: EXTRA: 1 ports: From d1dbce3041e31e4424854c1a352543209e813ed1 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Thu, 9 May 2024 14:50:19 +0700 Subject: [PATCH 04/33] Fix issues with limited output when K is high --- engine/base_client/search.py | 1 - engine/clients/manticoresearch/search.py | 4 ++-- experiments/configurations/manticoresearch-single-node.json | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/engine/base_client/search.py b/engine/base_client/search.py index 93368a3fe..df30e834e 100644 --- a/engine/base_client/search.py +++ b/engine/base_client/search.py @@ -52,7 +52,6 @@ def _search_one(cls, query, top: Optional[int] = None): if query.expected_result: ids = set(x[0] for x in search_res) precision = len(ids.intersection(query.expected_result[:top])) / top - return precision, end - start def search_all( diff --git a/engine/clients/manticoresearch/search.py b/engine/clients/manticoresearch/search.py index 413583b4b..da8ca56cc 100644 --- a/engine/clients/manticoresearch/search.py +++ b/engine/clients/manticoresearch/search.py @@ -37,11 +37,11 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]: "k": top, **{**cls.search_params.get('options', {})}, }, + "limit": top, } meta_conditions = cls.parser.parse(meta_conditions) if meta_conditions: knn.update(meta_conditions) - res = cls.session.post(cls.base_url, json=knn).json() - return [(int(hit["_id"]), hit["_score"]) for hit in res["hits"]["hits"]] + return [(int(hit["_id"]), hit["_knn_dist"]) for hit in res["hits"]["hits"]] diff --git a/experiments/configurations/manticoresearch-single-node.json b/experiments/configurations/manticoresearch-single-node.json index e05e28485..df7ce9988 100644 --- a/experiments/configurations/manticoresearch-single-node.json +++ b/experiments/configurations/manticoresearch-single-node.json @@ -90,4 +90,4 @@ ], "upload_params": { "parallel": 16 } } -] \ No newline at end of file +] From fd61682e9383c8ceb6a44a221b1bfda1ea5881d0 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Thu, 9 May 2024 16:54:28 +0700 Subject: [PATCH 05/33] Patch tqdm to start id from 1 instead of 0 cuz some engines uses 0 for auto increment --- engine/base_client/search.py | 7 ++++--- engine/base_client/tqdm_from_one.py | 6 ++++++ engine/base_client/upload.py | 6 +++--- 3 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 engine/base_client/tqdm_from_one.py diff --git a/engine/base_client/search.py b/engine/base_client/search.py index df30e834e..fb959311f 100644 --- a/engine/base_client/search.py +++ b/engine/base_client/search.py @@ -4,7 +4,7 @@ from typing import Iterable, List, Optional, Tuple import numpy as np -import tqdm +from engine.base_client.tqdm_from_one import TqdmFromOne from dataset_reader.base_reader import Query @@ -71,9 +71,10 @@ def search_all( search_one = functools.partial(self.__class__._search_one, top=top) if parallel == 1: + print(queries) start = time.perf_counter() precisions, latencies = list( - zip(*[search_one(query) for query in tqdm.tqdm(queries)]) + zip(*[search_one(query) for query in TqdmFromOne(queries)]) ) else: ctx = get_context(self.get_mp_start_method()) @@ -92,7 +93,7 @@ def search_all( time.sleep(15) # Wait for all processes to start start = time.perf_counter() precisions, latencies = list( - zip(*pool.imap_unordered(search_one, iterable=tqdm.tqdm(queries))) + zip(*pool.imap_unordered(search_one, iterable=TqdmFromOne(queries))) ) total_time = time.perf_counter() - start diff --git a/engine/base_client/tqdm_from_one.py b/engine/base_client/tqdm_from_one.py new file mode 100644 index 000000000..6454e1949 --- /dev/null +++ b/engine/base_client/tqdm_from_one.py @@ -0,0 +1,6 @@ +from tqdm import tqdm +class TqdmFromOne(tqdm): + def __init__(self, iterable, **kwargs): + super().__init__(iterable, **kwargs) + self.n = 1 # Set initial counter to 1 + diff --git a/engine/base_client/upload.py b/engine/base_client/upload.py index d9d53d94a..aeb372611 100644 --- a/engine/base_client/upload.py +++ b/engine/base_client/upload.py @@ -2,7 +2,7 @@ from multiprocessing import get_context from typing import Iterable, List, Optional, Tuple -import tqdm +from engine.base_client.tqdm_from_one import TqdmFromOne from dataset_reader.base_reader import Record from engine.base_client.utils import iter_batches @@ -39,7 +39,7 @@ def upload( ) if parallel == 1: - for batch in iter_batches(tqdm.tqdm(records), batch_size): + for batch in iter_batches(TqdmFromOne(records), batch_size): latencies.append(self._upload_batch(batch)) else: ctx = get_context(self.get_mp_start_method()) @@ -56,7 +56,7 @@ def upload( latencies = list( pool.imap( self.__class__._upload_batch, - iter_batches(tqdm.tqdm(records), batch_size), + iter_batches(TqdmFromOne(records), batch_size), ) ) From bd30f6802fd3fc643ba4f5d200b9f1ca6b5203a9 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Thu, 9 May 2024 17:06:07 +0700 Subject: [PATCH 06/33] Adjust json reader to start id from 1 instead of 0 to solve auto increment issue and precision lost --- dataset_reader/json_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataset_reader/json_reader.py b/dataset_reader/json_reader.py index 77a2eeb99..c48938af0 100644 --- a/dataset_reader/json_reader.py +++ b/dataset_reader/json_reader.py @@ -64,7 +64,7 @@ def read_data(self) -> Iterator[Record]: for idx, (vector, payload) in enumerate( zip(self.read_vectors(), self.read_payloads()) ): - yield Record(id=idx, vector=vector, metadata=payload) + yield Record(id=idx+1, vector=vector, metadata=payload) if __name__ == "__main__": From fda22c6a9970d346fe243bc2834ff7e68c50fd1e Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Thu, 9 May 2024 17:06:13 +0700 Subject: [PATCH 07/33] Revert "Patch tqdm to start id from 1 instead of 0 cuz some engines uses 0 for auto increment" This reverts commit fd61682e9383c8ceb6a44a221b1bfda1ea5881d0. --- engine/base_client/search.py | 7 +++---- engine/base_client/tqdm_from_one.py | 6 ------ engine/base_client/upload.py | 6 +++--- 3 files changed, 6 insertions(+), 13 deletions(-) delete mode 100644 engine/base_client/tqdm_from_one.py diff --git a/engine/base_client/search.py b/engine/base_client/search.py index fb959311f..df30e834e 100644 --- a/engine/base_client/search.py +++ b/engine/base_client/search.py @@ -4,7 +4,7 @@ from typing import Iterable, List, Optional, Tuple import numpy as np -from engine.base_client.tqdm_from_one import TqdmFromOne +import tqdm from dataset_reader.base_reader import Query @@ -71,10 +71,9 @@ def search_all( search_one = functools.partial(self.__class__._search_one, top=top) if parallel == 1: - print(queries) start = time.perf_counter() precisions, latencies = list( - zip(*[search_one(query) for query in TqdmFromOne(queries)]) + zip(*[search_one(query) for query in tqdm.tqdm(queries)]) ) else: ctx = get_context(self.get_mp_start_method()) @@ -93,7 +92,7 @@ def search_all( time.sleep(15) # Wait for all processes to start start = time.perf_counter() precisions, latencies = list( - zip(*pool.imap_unordered(search_one, iterable=TqdmFromOne(queries))) + zip(*pool.imap_unordered(search_one, iterable=tqdm.tqdm(queries))) ) total_time = time.perf_counter() - start diff --git a/engine/base_client/tqdm_from_one.py b/engine/base_client/tqdm_from_one.py deleted file mode 100644 index 6454e1949..000000000 --- a/engine/base_client/tqdm_from_one.py +++ /dev/null @@ -1,6 +0,0 @@ -from tqdm import tqdm -class TqdmFromOne(tqdm): - def __init__(self, iterable, **kwargs): - super().__init__(iterable, **kwargs) - self.n = 1 # Set initial counter to 1 - diff --git a/engine/base_client/upload.py b/engine/base_client/upload.py index aeb372611..d9d53d94a 100644 --- a/engine/base_client/upload.py +++ b/engine/base_client/upload.py @@ -2,7 +2,7 @@ from multiprocessing import get_context from typing import Iterable, List, Optional, Tuple -from engine.base_client.tqdm_from_one import TqdmFromOne +import tqdm from dataset_reader.base_reader import Record from engine.base_client.utils import iter_batches @@ -39,7 +39,7 @@ def upload( ) if parallel == 1: - for batch in iter_batches(TqdmFromOne(records), batch_size): + for batch in iter_batches(tqdm.tqdm(records), batch_size): latencies.append(self._upload_batch(batch)) else: ctx = get_context(self.get_mp_start_method()) @@ -56,7 +56,7 @@ def upload( latencies = list( pool.imap( self.__class__._upload_batch, - iter_batches(TqdmFromOne(records), batch_size), + iter_batches(tqdm.tqdm(records), batch_size), ) ) From 90b3e311efe9ae53f14b4239f65cdd7ed9ffc1d4 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Thu, 9 May 2024 17:25:28 +0700 Subject: [PATCH 08/33] Fix random-100 database neighbours --- datasets/random-100/neighbours.jsonl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasets/random-100/neighbours.jsonl b/datasets/random-100/neighbours.jsonl index 385fca207..884a11b77 100644 --- a/datasets/random-100/neighbours.jsonl +++ b/datasets/random-100/neighbours.jsonl @@ -1,4 +1,3 @@ -[0] [1] [2] [3] @@ -7,4 +6,5 @@ [6] [7] [8] -[9] \ No newline at end of file +[9] +[10] From 988b4e87403272cdd8d4ef86a51c474215ffb031 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Thu, 9 May 2024 17:43:35 +0700 Subject: [PATCH 09/33] Change approach how we altering ids --- .gitignore | 2 +- dataset_reader/json_reader.py | 2 +- engine/clients/manticoresearch/search.py | 2 +- engine/clients/manticoresearch/upload.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index e21c3643d..e11c19011 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ __pycache__ *.pyc NOTES.md - +.DS_Store results/* tools/custom/data.json diff --git a/dataset_reader/json_reader.py b/dataset_reader/json_reader.py index c48938af0..77a2eeb99 100644 --- a/dataset_reader/json_reader.py +++ b/dataset_reader/json_reader.py @@ -64,7 +64,7 @@ def read_data(self) -> Iterator[Record]: for idx, (vector, payload) in enumerate( zip(self.read_vectors(), self.read_payloads()) ): - yield Record(id=idx+1, vector=vector, metadata=payload) + yield Record(id=idx, vector=vector, metadata=payload) if __name__ == "__main__": diff --git a/engine/clients/manticoresearch/search.py b/engine/clients/manticoresearch/search.py index da8ca56cc..091bf8ee1 100644 --- a/engine/clients/manticoresearch/search.py +++ b/engine/clients/manticoresearch/search.py @@ -44,4 +44,4 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]: if meta_conditions: knn.update(meta_conditions) res = cls.session.post(cls.base_url, json=knn).json() - return [(int(hit["_id"]), hit["_knn_dist"]) for hit in res["hits"]["hits"]] + return [(int(hit["_id"]) - 1, hit["_knn_dist"]) for hit in res["hits"]["hits"]] diff --git a/engine/clients/manticoresearch/upload.py b/engine/clients/manticoresearch/upload.py index 39e08ee41..b2526ff16 100644 --- a/engine/clients/manticoresearch/upload.py +++ b/engine/clients/manticoresearch/upload.py @@ -45,7 +45,7 @@ def upload_batch(cls, ids: List[int], vectors: List[list], metadata: Optional[Li for id, vector, payload in zip(ids, vectors, metadata): data = { "index": MANTICORESEARCH_TABLE, - "id": id, + "id": id + 1, # we do not support id=0 "doc": { "vector": vector } From 67ede063b395ab91fcb622f7f05c0bd1ed8658b9 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Thu, 9 May 2024 17:53:20 +0700 Subject: [PATCH 10/33] Revert "Fix random-100 database neighbours" This reverts commit 90b3e311efe9ae53f14b4239f65cdd7ed9ffc1d4. --- datasets/random-100/neighbours.jsonl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasets/random-100/neighbours.jsonl b/datasets/random-100/neighbours.jsonl index 884a11b77..385fca207 100644 --- a/datasets/random-100/neighbours.jsonl +++ b/datasets/random-100/neighbours.jsonl @@ -1,3 +1,4 @@ +[0] [1] [2] [3] @@ -6,5 +7,4 @@ [6] [7] [8] -[9] -[10] +[9] \ No newline at end of file From 7cc72be5d1ac41b9dee1781da548b2cd906666d3 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Thu, 9 May 2024 19:43:26 +0700 Subject: [PATCH 11/33] Add timeouts to the connection --- engine/clients/manticoresearch/configure.py | 4 +- engine/clients/manticoresearch/search.py | 5 +- .../manticoresearch-single-node.json | 72 +++++++++++++------ 3 files changed, 55 insertions(+), 26 deletions(-) diff --git a/engine/clients/manticoresearch/configure.py b/engine/clients/manticoresearch/configure.py index 2832574ec..5fa1c8f6d 100644 --- a/engine/clients/manticoresearch/configure.py +++ b/engine/clients/manticoresearch/configure.py @@ -24,7 +24,7 @@ def clean(self): url = f'http://{self.host}:{MANTICORESEARCH_PORT}/sql?mode=raw' query = f"DROP TABLE IF EXISTS `{MANTICORESEARCH_TABLE}`" data = 'query=' + requests.utils.quote(query, safe='') - response = requests.post(url, data) + response = requests.post(url, data, **self.connection_params) if response.status_code != 200: print(f'Error cleaning table: {response.text}') @@ -56,7 +56,7 @@ def recreate(self, dataset, collection_params): """ url = f'http://{self.host}:{MANTICORESEARCH_PORT}/sql?mode=raw' data = 'query=' + requests.utils.quote(query, safe='') - response = requests.post(url, data) + response = requests.post(url, data, **self.connection_params) if response.status_code != 200: print(f'Error creating table: {response.text}') diff --git a/engine/clients/manticoresearch/search.py b/engine/clients/manticoresearch/search.py index 091bf8ee1..22782ec8a 100644 --- a/engine/clients/manticoresearch/search.py +++ b/engine/clients/manticoresearch/search.py @@ -12,6 +12,7 @@ import requests class ManticoreSearchSearcher(BaseSearcher): + connection_params = {} search_params = {} parser = ManticoreSearchConditionParser() @@ -25,7 +26,7 @@ def init_client(cls, host, distance, connection_params: dict, search_params: dic cls.session.headers.update({"Content-Type": "application/json"}) cls.base_url = urljoin(f"http://{host}:{MANTICORESEARCH_PORT}", "/search") cls.search_params = search_params - print(search_params) + cls.connection_params = connection_params @classmethod def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]: @@ -43,5 +44,5 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]: meta_conditions = cls.parser.parse(meta_conditions) if meta_conditions: knn.update(meta_conditions) - res = cls.session.post(cls.base_url, json=knn).json() + res = cls.session.post(cls.base_url, json=knn, **cls.connection_params).json() return [(int(hit["_id"]) - 1, hit["_knn_dist"]) for hit in res["hits"]["hits"]] diff --git a/experiments/configurations/manticoresearch-single-node.json b/experiments/configurations/manticoresearch-single-node.json index df7ce9988..fbfefbe52 100644 --- a/experiments/configurations/manticoresearch-single-node.json +++ b/experiments/configurations/manticoresearch-single-node.json @@ -1,14 +1,18 @@ -[ +[ { "name": "manticoresearch-default", "engine": "manticoresearch", "connection_params": { - "request_timeout": 10000 + "timeout": 30 }, "collection_params": { "knn_options": { "hnsw_m": 16, "hnsw_ef_construction": 100 } }, "search_params": [ - { "parallel": 1, "options": {"ef": 128} }, { "parallel": 1, "options": {"ef": 256} }, { "parallel": 1, "options": {"ef": 512} }, - { "parallel": 100, "options": {"ef": 128} }, { "parallel": 100, "options": {"ef": 256} }, { "parallel": 100, "options": {"ef": 512} } + { "parallel": 1, "options": { "ef": 128 } }, + { "parallel": 1, "options": { "ef": 256 } }, + { "parallel": 1, "options": { "ef": 512 } }, + { "parallel": 100, "options": { "ef": 128 } }, + { "parallel": 100, "options": { "ef": 256 } }, + { "parallel": 100, "options": { "ef": 512 } } ], "upload_params": { "parallel": 16 } }, @@ -16,12 +20,16 @@ "name": "manticoresearch-m-16-ef-128", "engine": "manticoresearch", "connection_params": { - "request_timeout": 10000 + "timeout": 30 }, "collection_params": { "knn_options": { "hnsw_m": 16, "hnsw_ef_construction": 128 } }, "search_params": [ - { "parallel": 1, "options": {"ef": 128} }, { "parallel": 1, "options": {"ef": 256} }, { "parallel": 1, "options": {"ef": 512} }, - { "parallel": 100, "options": {"ef": 128} }, { "parallel": 100, "options": {"ef": 256} }, { "parallel": 100, "options": {"ef": 512} } + { "parallel": 1, "options": { "ef": 128 } }, + { "parallel": 1, "options": { "ef": 256 } }, + { "parallel": 1, "options": { "ef": 512 } }, + { "parallel": 100, "options": { "ef": 128 } }, + { "parallel": 100, "options": { "ef": 256 } }, + { "parallel": 100, "options": { "ef": 512 } } ], "upload_params": { "parallel": 16 } }, @@ -29,12 +37,16 @@ "name": "manticoresearch-m-32-ef-128", "engine": "manticoresearch", "connection_params": { - "request_timeout": 10000 + "timeout": 30 }, "collection_params": { "knn_options": { "hnsw_m": 32, "hnsw_ef_construction": 128 } }, "search_params": [ - { "parallel": 1, "options": {"ef": 128} }, { "parallel": 1, "options": {"ef": 256} }, { "parallel": 1, "options": {"ef": 512} }, - { "parallel": 100, "options": {"ef": 128} }, { "parallel": 100, "options": {"ef": 256} }, { "parallel": 100, "options": {"ef": 512} } + { "parallel": 1, "options": { "ef": 128 } }, + { "parallel": 1, "options": { "ef": 256 } }, + { "parallel": 1, "options": { "ef": 512 } }, + { "parallel": 100, "options": { "ef": 128 } }, + { "parallel": 100, "options": { "ef": 256 } }, + { "parallel": 100, "options": { "ef": 512 } } ], "upload_params": { "parallel": 16 } }, @@ -42,12 +54,16 @@ "name": "manticoresearch-m-32-ef-256", "engine": "manticoresearch", "connection_params": { - "request_timeout": 10000 + "timeout": 30 }, "collection_params": { "knn_options": { "hnsw_m": 32, "hnsw_ef_construction": 256 } }, "search_params": [ - { "parallel": 1, "options": {"ef": 128} }, { "parallel": 1, "options": {"ef": 256} }, { "parallel": 1, "options": {"ef": 512} }, - { "parallel": 100, "options": {"ef": 128} }, { "parallel": 100, "options": {"ef": 256} }, { "parallel": 100, "options": {"ef": 512} } + { "parallel": 1, "options": { "ef": 128 } }, + { "parallel": 1, "options": { "ef": 256 } }, + { "parallel": 1, "options": { "ef": 512 } }, + { "parallel": 100, "options": { "ef": 128 } }, + { "parallel": 100, "options": { "ef": 256 } }, + { "parallel": 100, "options": { "ef": 512 } } ], "upload_params": { "parallel": 16 } }, @@ -55,12 +71,16 @@ "name": "manticoresearch-m-32-ef-512", "engine": "manticoresearch", "connection_params": { - "request_timeout": 10000 + "timeout": 30 }, "collection_params": { "knn_options": { "hnsw_m": 32, "hnsw_ef_construction": 512 } }, "search_params": [ - { "parallel": 1, "options": {"ef": 128} }, { "parallel": 1, "options": {"ef": 256} }, { "parallel": 1, "options": {"ef": 512} }, - { "parallel": 100, "options": {"ef": 128} }, { "parallel": 100, "options": {"ef": 256} }, { "parallel": 100, "options": {"ef": 512} } + { "parallel": 1, "options": { "ef": 128 } }, + { "parallel": 1, "options": { "ef": 256 } }, + { "parallel": 1, "options": { "ef": 512 } }, + { "parallel": 100, "options": { "ef": 128 } }, + { "parallel": 100, "options": { "ef": 256 } }, + { "parallel": 100, "options": { "ef": 512 } } ], "upload_params": { "parallel": 16 } }, @@ -68,12 +88,16 @@ "name": "manticoresearch-m-64-ef-256", "engine": "manticoresearch", "connection_params": { - "request_timeout": 10000 + "timeout": 30 }, "collection_params": { "knn_options": { "hnsw_m": 64, "hnsw_ef_construction": 256 } }, "search_params": [ - { "parallel": 1, "options": {"ef": 128} }, { "parallel": 1, "options": {"ef": 256} }, { "parallel": 1, "options": {"ef": 512} }, - { "parallel": 100, "options": {"ef": 128} }, { "parallel": 100, "options": {"ef": 256} }, { "parallel": 100, "options": {"ef": 512} } + { "parallel": 1, "options": { "ef": 128 } }, + { "parallel": 1, "options": { "ef": 256 } }, + { "parallel": 1, "options": { "ef": 512 } }, + { "parallel": 100, "options": { "ef": 128 } }, + { "parallel": 100, "options": { "ef": 256 } }, + { "parallel": 100, "options": { "ef": 512 } } ], "upload_params": { "parallel": 16 } }, @@ -81,12 +105,16 @@ "name": "manticoresearch-m-64-ef-512", "engine": "manticoresearch", "connection_params": { - "request_timeout": 10000 + "timeout": 30 }, "collection_params": { "knn_options": { "hnsw_m": 64, "hnsw_ef_construction": 512 } }, "search_params": [ - { "parallel": 1, "options": {"ef": 128} }, { "parallel": 1, "options": {"ef": 256} }, { "parallel": 1, "options": {"ef": 512} }, - { "parallel": 100, "options": {"ef": 128} }, { "parallel": 100, "options": {"ef": 256} }, { "parallel": 100, "options": {"ef": 512} } + { "parallel": 1, "options": { "ef": 128 } }, + { "parallel": 1, "options": { "ef": 256 } }, + { "parallel": 1, "options": { "ef": 512 } }, + { "parallel": 100, "options": { "ef": 128 } }, + { "parallel": 100, "options": { "ef": 256 } }, + { "parallel": 100, "options": { "ef": 512 } } ], "upload_params": { "parallel": 16 } } From 41dc94375c07bbf0c65516e40e6eb981b7c7a210 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Thu, 9 May 2024 20:08:25 +0700 Subject: [PATCH 12/33] Try to use keep alive connection for session when we benchmarking search queries --- engine/clients/manticoresearch/search.py | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/clients/manticoresearch/search.py b/engine/clients/manticoresearch/search.py index 22782ec8a..a77fb6f67 100644 --- a/engine/clients/manticoresearch/search.py +++ b/engine/clients/manticoresearch/search.py @@ -23,6 +23,7 @@ def get_mp_start_method(cls): @classmethod def init_client(cls, host, distance, connection_params: dict, search_params: dict): cls.session = requests.Session() + cls.session.headers.update({"Connection": "keep-alive"}) cls.session.headers.update({"Content-Type": "application/json"}) cls.base_url = urljoin(f"http://{host}:{MANTICORESEARCH_PORT}", "/search") cls.search_params = search_params From 2b49697d53d6787d977f1d3d6482a5b03d71b46d Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Fri, 10 May 2024 15:23:26 +0700 Subject: [PATCH 13/33] Fetch only id for manticoresearch client the same way as other clients do --- engine/clients/manticoresearch/search.py | 1 + .../servers/manticoresearch-single-node/docker-compose.yaml | 4 +++- engine/servers/qdrant-single-node/docker-compose.yaml | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/engine/clients/manticoresearch/search.py b/engine/clients/manticoresearch/search.py index a77fb6f67..e252946b0 100644 --- a/engine/clients/manticoresearch/search.py +++ b/engine/clients/manticoresearch/search.py @@ -33,6 +33,7 @@ def init_client(cls, host, distance, connection_params: dict, search_params: dic def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]: knn = { "index": MANTICORESEARCH_TABLE, + "_source": "id", "knn": { "field": "vector", "query_vector": vector, diff --git a/engine/servers/manticoresearch-single-node/docker-compose.yaml b/engine/servers/manticoresearch-single-node/docker-compose.yaml index 5276877b4..cb1743c17 100644 --- a/engine/servers/manticoresearch-single-node/docker-compose.yaml +++ b/engine/servers/manticoresearch-single-node/docker-compose.yaml @@ -1,10 +1,11 @@ -version: '3.5' +version: '3.7' services: searchd: image: manticoresearch/manticore:dev-6.2.13-7047a53 environment: EXTRA: 1 + searchd_threads: 8 ports: - "9308:9308" logging: @@ -16,3 +17,4 @@ services: resources: limits: memory: 25Gb + cpus: "1.0" diff --git a/engine/servers/qdrant-single-node/docker-compose.yaml b/engine/servers/qdrant-single-node/docker-compose.yaml index 8285e9d3f..628f1b578 100644 --- a/engine/servers/qdrant-single-node/docker-compose.yaml +++ b/engine/servers/qdrant-single-node/docker-compose.yaml @@ -16,3 +16,4 @@ services: resources: limits: memory: 25Gb + cpus: "1.0" From b62d2170698072178283e94cbb22be0fbec7a400 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Mon, 10 Mar 2025 14:40:25 +0700 Subject: [PATCH 14/33] Update docker image and add quantization configs both for qdrant and manticore to compare it --- .../docker-compose.yaml | 2 +- .../manticoresearch-single-node.json | 36 ++++++ .../configurations/qdrant-single-node.json | 104 ++++++++++++++++++ 3 files changed, 141 insertions(+), 1 deletion(-) diff --git a/engine/servers/manticoresearch-single-node/docker-compose.yaml b/engine/servers/manticoresearch-single-node/docker-compose.yaml index cb1743c17..2478e3e95 100644 --- a/engine/servers/manticoresearch-single-node/docker-compose.yaml +++ b/engine/servers/manticoresearch-single-node/docker-compose.yaml @@ -2,7 +2,7 @@ version: '3.7' services: searchd: - image: manticoresearch/manticore:dev-6.2.13-7047a53 + image: manticoresearch/manticore:dev-7.4.7-39a6f50 environment: EXTRA: 1 searchd_threads: 8 diff --git a/experiments/configurations/manticoresearch-single-node.json b/experiments/configurations/manticoresearch-single-node.json index fbfefbe52..973fdb482 100644 --- a/experiments/configurations/manticoresearch-single-node.json +++ b/experiments/configurations/manticoresearch-single-node.json @@ -16,6 +16,42 @@ ], "upload_params": { "parallel": 16 } }, + { + "name": "manticoresearch-quantization-1bit", + "engine": "manticoresearch", + "connection_params": { "timeout": 30 }, + "collection_params": { "knn_options": { "hnsw_m": 32, "hnsw_ef_construction": 256, "quantization": "1bit" } }, + "search_params": [ + { "parallel": 1, "options": { "ef": 256 } }, + { "parallel": 8, "options": { "ef": 256 } }, + { "parallel": 16, "options": { "ef": 256 } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "manticoresearch-quantization-4bit", + "engine": "manticoresearch", + "connection_params": { "timeout": 30 }, + "collection_params": { "knn_options": { "hnsw_m": 32, "hnsw_ef_construction": 256, "quantization": "4bit" } }, + "search_params": [ + { "parallel": 1, "options": { "ef": 256 } }, + { "parallel": 8, "options": { "ef": 256 } }, + { "parallel": 16, "options": { "ef": 256 } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "manticoresearch-quantization-8bit", + "engine": "manticoresearch", + "connection_params": { "timeout": 30 }, + "collection_params": { "knn_options": { "hnsw_m": 32, "hnsw_ef_construction": 256, "quantization": "8bit" } }, + "search_params": [ + { "parallel": 1, "options": { "ef": 256 } }, + { "parallel": 8, "options": { "ef": 256 } }, + { "parallel": 16, "options": { "ef": 256 } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, { "name": "manticoresearch-m-16-ef-128", "engine": "manticoresearch", diff --git a/experiments/configurations/qdrant-single-node.json b/experiments/configurations/qdrant-single-node.json index 3b46f9f36..2cfaad327 100644 --- a/experiments/configurations/qdrant-single-node.json +++ b/experiments/configurations/qdrant-single-node.json @@ -11,6 +11,110 @@ ], "upload_params": { "parallel": 16, "batch_size": 1024 } }, + { + "name": "qdrant-quantization-4bit", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "hnsw_config": { + "m": 32, + "ef_construct": 256 + }, + "quantization_config": { + "scalar": { + "type": "int4", + "quantile": 0.99 + } + }, + "optimizers_config": { + "max_segment_size": 1000000, + "default_segment_number": 3, + "memmap_threshold": 10000000 + } + }, + "search_params": [ + { + "parallel": 1, + "search_params": { + "hnsw_ef": 256, + "quantization": { + "oversampling": 2.0 + } + } + }, + { + "parallel": 8, + "search_params": { + "hnsw_ef": 256, + "quantization": { + "oversampling": 2.0 + } + } + }, + { + "parallel": 16, + "search_params": { + "hnsw_ef": 256, + "quantization": { + "oversampling": 2.0 + } + } + } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-quantization-8bit", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "hnsw_config": { + "m": 32, + "ef_construct": 256 + }, + "quantization_config": { + "scalar": { + "type": "int8", + "quantile": 0.99 + } + }, + "optimizers_config": { + "max_segment_size": 1000000, + "default_segment_number": 3, + "memmap_threshold": 10000000 + } + }, + "search_params": [ + { + "parallel": 1, + "search_params": { + "hnsw_ef": 256, + "quantization": { + "oversampling": 2.0 + } + } + }, + { + "parallel": 8, + "search_params": { + "hnsw_ef": 256, + "quantization": { + "oversampling": 2.0 + } + } + }, + { + "parallel": 16, + "search_params": { + "hnsw_ef": 256, + "quantization": { + "oversampling": 2.0 + } + } + } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, { "name": "qdrant-continuous-benchmark", "engine": "qdrant", From 74599c5ac267d50e06721fc2b0075534776b7d9d Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Mon, 10 Mar 2025 17:00:23 +0700 Subject: [PATCH 15/33] Run Optimize for manticore and add config of binary quantization for qdrant --- engine/clients/manticoresearch/upload.py | 3 + .../configurations/qdrant-single-node.json | 63 +++++++++++++++++-- 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/engine/clients/manticoresearch/upload.py b/engine/clients/manticoresearch/upload.py index b2526ff16..5b21d78c6 100644 --- a/engine/clients/manticoresearch/upload.py +++ b/engine/clients/manticoresearch/upload.py @@ -60,4 +60,7 @@ def upload_batch(cls, ids: List[int], vectors: List[list], metadata: Optional[Li def post_upload(cls, _distance): response = cls.session.post(f"{cls.api_url}/sql?mode=raw", data=f"query=FLUSH%20RAMCHUNK%60{MANTICORESEARCH_TABLE}%60") response.raise_for_status() + + response = cls.session.post(f"{cls.api_url}/sql?mode=raw", data=f"query=OPTIMIZE%20TABLE%20{MANTICORESEARCH_TABLE}%20sync%3D1%60") + response.raise_for_status() return {} diff --git a/experiments/configurations/qdrant-single-node.json b/experiments/configurations/qdrant-single-node.json index 2cfaad327..6d2166b1c 100644 --- a/experiments/configurations/qdrant-single-node.json +++ b/experiments/configurations/qdrant-single-node.json @@ -11,6 +11,57 @@ ], "upload_params": { "parallel": 16, "batch_size": 1024 } }, + { + "name": "qdrant-quantization-1bit", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "hnsw_config": { + "m": 32, + "ef_construct": 256 + }, + "quantization_config": { + "binary": { + "always_ram": true + } + }, + "optimizers_config": { + "max_segment_size": 1000000, + "default_segment_number": 3, + "memmap_threshold": 10000000 + } + }, + "search_params": [ + { + "parallel": 1, + "search_params": { + "hnsw_ef": 256, + "quantization": { + "oversampling": 1.0 + } + } + }, + { + "parallel": 8, + "search_params": { + "hnsw_ef": 256, + "quantization": { + "oversampling": 1.0 + } + } + }, + { + "parallel": 16, + "search_params": { + "hnsw_ef": 256, + "quantization": { + "oversampling": 1.0 + } + } + } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, { "name": "qdrant-quantization-4bit", "engine": "qdrant", @@ -38,7 +89,7 @@ "search_params": { "hnsw_ef": 256, "quantization": { - "oversampling": 2.0 + "oversampling": 1.0 } } }, @@ -47,7 +98,7 @@ "search_params": { "hnsw_ef": 256, "quantization": { - "oversampling": 2.0 + "oversampling": 1.0 } } }, @@ -56,7 +107,7 @@ "search_params": { "hnsw_ef": 256, "quantization": { - "oversampling": 2.0 + "oversampling": 1.0 } } } @@ -90,7 +141,7 @@ "search_params": { "hnsw_ef": 256, "quantization": { - "oversampling": 2.0 + "oversampling": 1.0 } } }, @@ -99,7 +150,7 @@ "search_params": { "hnsw_ef": 256, "quantization": { - "oversampling": 2.0 + "oversampling": 1.0 } } }, @@ -108,7 +159,7 @@ "search_params": { "hnsw_ef": 256, "quantization": { - "oversampling": 2.0 + "oversampling": 1.0 } } } From 996d83005aa73d6fcaf347ad753f8dce26b40129 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Mon, 10 Mar 2025 17:39:38 +0700 Subject: [PATCH 16/33] Disable timeout for optimize --- engine/clients/manticoresearch/upload.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/engine/clients/manticoresearch/upload.py b/engine/clients/manticoresearch/upload.py index 5b21d78c6..056f178d3 100644 --- a/engine/clients/manticoresearch/upload.py +++ b/engine/clients/manticoresearch/upload.py @@ -61,6 +61,10 @@ def post_upload(cls, _distance): response = cls.session.post(f"{cls.api_url}/sql?mode=raw", data=f"query=FLUSH%20RAMCHUNK%60{MANTICORESEARCH_TABLE}%60") response.raise_for_status() + # Unset timeout for session cuz optimize is quite long query + old_timeout = cls.session.timeout + cls.session.timeout = None response = cls.session.post(f"{cls.api_url}/sql?mode=raw", data=f"query=OPTIMIZE%20TABLE%20{MANTICORESEARCH_TABLE}%20sync%3D1%60") response.raise_for_status() + cls.session.timeout = old_timeout return {} From e84f304271595fc959cca5333251d4cefe5f1ecd Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Mon, 10 Mar 2025 17:58:00 +0700 Subject: [PATCH 17/33] Set timeout in proper way --- engine/clients/manticoresearch/upload.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/engine/clients/manticoresearch/upload.py b/engine/clients/manticoresearch/upload.py index 056f178d3..db3d26626 100644 --- a/engine/clients/manticoresearch/upload.py +++ b/engine/clients/manticoresearch/upload.py @@ -61,10 +61,10 @@ def post_upload(cls, _distance): response = cls.session.post(f"{cls.api_url}/sql?mode=raw", data=f"query=FLUSH%20RAMCHUNK%60{MANTICORESEARCH_TABLE}%60") response.raise_for_status() - # Unset timeout for session cuz optimize is quite long query - old_timeout = cls.session.timeout - cls.session.timeout = None - response = cls.session.post(f"{cls.api_url}/sql?mode=raw", data=f"query=OPTIMIZE%20TABLE%20{MANTICORESEARCH_TABLE}%20sync%3D1%60") + response = cls.session.post( + f"{cls.api_url}/sql?mode=raw", + data=f"query=OPTIMIZE%20TABLE%20%60{MANTICORESEARCH_TABLE}%60%20sync%3D1", + timeout=None + ) response.raise_for_status() - cls.session.timeout = old_timeout return {} From c2057dc6d417d8835487f59729b61391c0294273 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Mon, 10 Mar 2025 18:13:52 +0700 Subject: [PATCH 18/33] Add missing option word in optimize --- engine/clients/manticoresearch/upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/clients/manticoresearch/upload.py b/engine/clients/manticoresearch/upload.py index db3d26626..3cc6675ff 100644 --- a/engine/clients/manticoresearch/upload.py +++ b/engine/clients/manticoresearch/upload.py @@ -63,7 +63,7 @@ def post_upload(cls, _distance): response = cls.session.post( f"{cls.api_url}/sql?mode=raw", - data=f"query=OPTIMIZE%20TABLE%20%60{MANTICORESEARCH_TABLE}%60%20sync%3D1", + data=f"query=OPTIMIZE%20TABLE%20%60{MANTICORESEARCH_TABLE}%60%20OPTION%20sync%3D1", timeout=None ) response.raise_for_status() From face9f6a22ceebe309f4cfb3aac4c476bbc1d4cb Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Tue, 11 Mar 2025 11:08:35 +0700 Subject: [PATCH 19/33] Update qdrant config --- .../configurations/qdrant-single-node.json | 58 ++----------------- 1 file changed, 6 insertions(+), 52 deletions(-) diff --git a/experiments/configurations/qdrant-single-node.json b/experiments/configurations/qdrant-single-node.json index 6d2166b1c..a7fe853da 100644 --- a/experiments/configurations/qdrant-single-node.json +++ b/experiments/configurations/qdrant-single-node.json @@ -37,6 +37,7 @@ "search_params": { "hnsw_ef": 256, "quantization": { + "rescore": false, "oversampling": 1.0 } } @@ -46,6 +47,7 @@ "search_params": { "hnsw_ef": 256, "quantization": { + "rescore": false, "oversampling": 1.0 } } @@ -55,58 +57,7 @@ "search_params": { "hnsw_ef": 256, "quantization": { - "oversampling": 1.0 - } - } - } - ], - "upload_params": { "parallel": 16, "batch_size": 1024 } - }, - { - "name": "qdrant-quantization-4bit", - "engine": "qdrant", - "connection_params": { "timeout": 30 }, - "collection_params": { - "hnsw_config": { - "m": 32, - "ef_construct": 256 - }, - "quantization_config": { - "scalar": { - "type": "int4", - "quantile": 0.99 - } - }, - "optimizers_config": { - "max_segment_size": 1000000, - "default_segment_number": 3, - "memmap_threshold": 10000000 - } - }, - "search_params": [ - { - "parallel": 1, - "search_params": { - "hnsw_ef": 256, - "quantization": { - "oversampling": 1.0 - } - } - }, - { - "parallel": 8, - "search_params": { - "hnsw_ef": 256, - "quantization": { - "oversampling": 1.0 - } - } - }, - { - "parallel": 16, - "search_params": { - "hnsw_ef": 256, - "quantization": { + "rescore": false, "oversampling": 1.0 } } @@ -141,6 +92,7 @@ "search_params": { "hnsw_ef": 256, "quantization": { + "rescore": false, "oversampling": 1.0 } } @@ -150,6 +102,7 @@ "search_params": { "hnsw_ef": 256, "quantization": { + "rescore": false, "oversampling": 1.0 } } @@ -159,6 +112,7 @@ "search_params": { "hnsw_ef": 256, "quantization": { + "rescore": false, "oversampling": 1.0 } } From 847972472a57a402317cfb115dcf100993dea904 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Thu, 13 Mar 2025 10:24:26 +0700 Subject: [PATCH 20/33] Add cutoff=1 to manticore and tune configs, update qdrant version --- engine/clients/manticoresearch/upload.py | 2 +- .../docker-compose.yaml | 1 - .../qdrant-single-node/docker-compose.yaml | 2 +- .../configurations/qdrant-single-node.json | 109 ++++++++++++++++++ 4 files changed, 111 insertions(+), 3 deletions(-) diff --git a/engine/clients/manticoresearch/upload.py b/engine/clients/manticoresearch/upload.py index 3cc6675ff..26dc1318d 100644 --- a/engine/clients/manticoresearch/upload.py +++ b/engine/clients/manticoresearch/upload.py @@ -63,7 +63,7 @@ def post_upload(cls, _distance): response = cls.session.post( f"{cls.api_url}/sql?mode=raw", - data=f"query=OPTIMIZE%20TABLE%20%60{MANTICORESEARCH_TABLE}%60%20OPTION%20sync%3D1", + data=f"query=OPTIMIZE%20TABLE%20%60{MANTICORESEARCH_TABLE}%60%20OPTION%20sync%3D1%20%2Ccutoff%3D1", timeout=None ) response.raise_for_status() diff --git a/engine/servers/manticoresearch-single-node/docker-compose.yaml b/engine/servers/manticoresearch-single-node/docker-compose.yaml index 2478e3e95..c9ccc1eeb 100644 --- a/engine/servers/manticoresearch-single-node/docker-compose.yaml +++ b/engine/servers/manticoresearch-single-node/docker-compose.yaml @@ -5,7 +5,6 @@ services: image: manticoresearch/manticore:dev-7.4.7-39a6f50 environment: EXTRA: 1 - searchd_threads: 8 ports: - "9308:9308" logging: diff --git a/engine/servers/qdrant-single-node/docker-compose.yaml b/engine/servers/qdrant-single-node/docker-compose.yaml index 628f1b578..6e7ab0d03 100644 --- a/engine/servers/qdrant-single-node/docker-compose.yaml +++ b/engine/servers/qdrant-single-node/docker-compose.yaml @@ -2,7 +2,7 @@ version: '3.7' services: qdrant_bench: - image: ${CONTAINER_REGISTRY:-docker.io}/qdrant/qdrant:v1.8.2 + image: ${CONTAINER_REGISTRY:-docker.io}/qdrant/qdrant:v1.13.4 network_mode: host logging: driver: "json-file" diff --git a/experiments/configurations/qdrant-single-node.json b/experiments/configurations/qdrant-single-node.json index a7fe853da..7462f5649 100644 --- a/experiments/configurations/qdrant-single-node.json +++ b/experiments/configurations/qdrant-single-node.json @@ -120,6 +120,115 @@ ], "upload_params": { "parallel": 16, "batch_size": 1024 } }, + { + "name": "qdrant-quantization-1bit-rescore", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "hnsw_config": { + "m": 32, + "ef_construct": 256 + }, + "quantization_config": { + "binary": { + "always_ram": true + } + }, + "optimizers_config": { + "max_segment_size": 1000000, + "default_segment_number": 3, + "memmap_threshold": 10000000 + } + }, + "search_params": [ + { + "parallel": 1, + "search_params": { + "hnsw_ef": 256, + "quantization": { + "rescore": true, + "oversampling": 1.0 + } + } + }, + { + "parallel": 8, + "search_params": { + "hnsw_ef": 256, + "quantization": { + "rescore": true, + "oversampling": 1.0 + } + } + }, + { + "parallel": 16, + "search_params": { + "hnsw_ef": 256, + "quantization": { + "rescore": true, + "oversampling": 1.0 + } + } + } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-quantization-8bit-rescore", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "hnsw_config": { + "m": 32, + "ef_construct": 256 + }, + "quantization_config": { + "scalar": { + "type": "int8", + "quantile": 0.99 + } + }, + "optimizers_config": { + "max_segment_size": 1000000, + "default_segment_number": 3, + "memmap_threshold": 10000000 + } + }, + "search_params": [ + { + "parallel": 1, + "search_params": { + "hnsw_ef": 256, + "quantization": { + "rescore": true, + "oversampling": 1.0 + } + } + }, + { + "parallel": 8, + "search_params": { + "hnsw_ef": 256, + "quantization": { + "rescore": true, + "oversampling": 1.0 + } + } + }, + { + "parallel": 16, + "search_params": { + "hnsw_ef": 256, + "quantization": { + "rescore": true, + "oversampling": 1.0 + } + } + } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, { "name": "qdrant-continuous-benchmark", "engine": "qdrant", From b8c8f77b7b8fb42cb666ea8d5e0b061634571a60 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Sun, 16 Mar 2025 14:42:07 +0700 Subject: [PATCH 21/33] Remove thread limitation for qdrant --- engine/servers/qdrant-single-node/docker-compose.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/engine/servers/qdrant-single-node/docker-compose.yaml b/engine/servers/qdrant-single-node/docker-compose.yaml index 6e7ab0d03..34fdb5186 100644 --- a/engine/servers/qdrant-single-node/docker-compose.yaml +++ b/engine/servers/qdrant-single-node/docker-compose.yaml @@ -9,9 +9,6 @@ services: options: max-file: 1 max-size: 10m - environment: - - QDRANT__STORAGE__PERFORMANCE__OPTIMIZER_CPU_BUDGET=8 - - QDRANT__STORAGE__PERFORMANCE__MAX_SEARCH_THREADS=8 deploy: resources: limits: From 27683b81edc84fbf28b5874b5174d52e2175afd1 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Fri, 18 Apr 2025 10:52:21 +0700 Subject: [PATCH 22/33] Add config for elasticsearch quantization --- .../docker-compose.yaml | 2 +- .../elasticsearch-single-node.json | 60 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/engine/servers/elasticsearch-single-node/docker-compose.yaml b/engine/servers/elasticsearch-single-node/docker-compose.yaml index fa767bb94..4cccee248 100644 --- a/engine/servers/elasticsearch-single-node/docker-compose.yaml +++ b/engine/servers/elasticsearch-single-node/docker-compose.yaml @@ -2,7 +2,7 @@ version: '3.5' services: es: - image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2 + image: docker.elastic.co/elasticsearch/elasticsearch:8.18.0 environment: ELASTIC_PASSWORD: "passwd" KIBANA_PASSWORD: "passwd" diff --git a/experiments/configurations/elasticsearch-single-node.json b/experiments/configurations/elasticsearch-single-node.json index b3f0f609c..120ea2fca 100644 --- a/experiments/configurations/elasticsearch-single-node.json +++ b/experiments/configurations/elasticsearch-single-node.json @@ -12,6 +12,66 @@ ], "upload_params": { "parallel": 16 } }, + { + "name": "elasticsearch-quantization-1bit", + "engine": "elasticsearch", + "connection_params": { + "request_timeout": 10000 + }, + "collection_params": { "index_options": { "m": 32, "ef_construction": 256 }, "binary_quantization": { "type": "random_projection" } }, + "search_params": [ + { "parallel": 1, "num_candidates": 256 }, + { "parallel": 8, "num_candidates": 256 }, + { "parallel": 16, "num_candidates": 256 }, + ], + "upload_params": { "parallel": 16 } + }, + { + "name": "elasticsearch-quantization-4bit", + "engine": "elasticsearch", + "connection_params": { + "request_timeout": 10000 + }, + "collection_params": { + "index_options": { + "m": 32, + "ef_construction": 256 + }, + "scalar_quantization": { + "type": "float4" + } + }, + "search_params": [ + { "parallel": 1, "num_candidates": 256 }, + { "parallel": 8, "num_candidates": 256 }, + { "parallel": 16, "num_candidates": 256 }, + ], + "upload_params": { "parallel": 16 } + }, + { + "name": "elasticsearch-quantization-8bit", + "engine": "elasticsearch", + "connection_params": { + "request_timeout": 10000 + }, + "collection_params": { + "index_options": { + "m": 32, + "ef_construction": 256 + }, + "scalar_quantization": { + "type": "float8" + } + }, + "search_params": [ + { "parallel": 1, "num_candidates": 256 }, + { "parallel": 8, "num_candidates": 256 }, + { "parallel": 16, "num_candidates": 256 }, + ], + "upload_params": { "parallel": 16 } + }, + + { "name": "elasticsearch-m-16-ef-128", "engine": "elasticsearch", From 9b7ed6171386ea2bef65d14a206ee52e04d26f6f Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Fri, 18 Apr 2025 10:59:03 +0700 Subject: [PATCH 23/33] Fix elasticserch config --- .../elasticsearch-single-node.json | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/experiments/configurations/elasticsearch-single-node.json b/experiments/configurations/elasticsearch-single-node.json index 120ea2fca..de1fbe74f 100644 --- a/experiments/configurations/elasticsearch-single-node.json +++ b/experiments/configurations/elasticsearch-single-node.json @@ -18,11 +18,19 @@ "connection_params": { "request_timeout": 10000 }, - "collection_params": { "index_options": { "m": 32, "ef_construction": 256 }, "binary_quantization": { "type": "random_projection" } }, + "collection_params": { + "index_options": { + "m": 32, + "ef_construction": 256 + }, + "binary_quantization": { + "type": "random_projection" + } + }, "search_params": [ { "parallel": 1, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 256 }, - { "parallel": 16, "num_candidates": 256 }, + { "parallel": 16, "num_candidates": 256 } ], "upload_params": { "parallel": 16 } }, @@ -44,7 +52,7 @@ "search_params": [ { "parallel": 1, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 256 }, - { "parallel": 16, "num_candidates": 256 }, + { "parallel": 16, "num_candidates": 256 } ], "upload_params": { "parallel": 16 } }, @@ -66,12 +74,10 @@ "search_params": [ { "parallel": 1, "num_candidates": 256 }, { "parallel": 8, "num_candidates": 256 }, - { "parallel": 16, "num_candidates": 256 }, + { "parallel": 16, "num_candidates": 256 } ], "upload_params": { "parallel": 16 } }, - - { "name": "elasticsearch-m-16-ef-128", "engine": "elasticsearch", From f9c39bd3529d5543058a85ad6c5e3e174bb03f2c Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Fri, 18 Apr 2025 11:15:58 +0700 Subject: [PATCH 24/33] Set exact version of elasticsearch cuz of incompatibility issue --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 129a58285..42eea5578 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ typer = "^0.6.1" jsons = "^1.6.3" h5py = "^3.7.0" weaviate-client = "^4.5.0" -elasticsearch = "^8.10.0" +elasticsearch = "8.18.0" pymilvus = "^2.3.1" redis = "^5.0.1" ipdb = "^0.13.9" From 4d30aa92bf5d82a97ef1f116940937157c00c9c3 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Fri, 18 Apr 2025 14:45:58 +0700 Subject: [PATCH 25/33] Fix elasticsearch config --- .../elasticsearch-single-node.json | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/experiments/configurations/elasticsearch-single-node.json b/experiments/configurations/elasticsearch-single-node.json index de1fbe74f..f04b177df 100644 --- a/experiments/configurations/elasticsearch-single-node.json +++ b/experiments/configurations/elasticsearch-single-node.json @@ -7,8 +7,8 @@ }, "collection_params": { "index_options": { "m": 16, "ef_construction": 100 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -28,9 +28,9 @@ } }, "search_params": [ - { "parallel": 1, "num_candidates": 256 }, - { "parallel": 8, "num_candidates": 256 }, - { "parallel": 16, "num_candidates": 256 } + { "parallel": 1, "search_params": {"num_candidates": 256} }, + { "parallel": 8, "search_params": {"num_candidates": 256} }, + { "parallel": 16, "search_params": {"num_candidates": 256} } ], "upload_params": { "parallel": 16 } }, @@ -50,9 +50,9 @@ } }, "search_params": [ - { "parallel": 1, "num_candidates": 256 }, - { "parallel": 8, "num_candidates": 256 }, - { "parallel": 16, "num_candidates": 256 } + { "parallel": 1, "search_params": {"num_candidates": 256} }, + { "parallel": 8, "search_params": {"num_candidates": 256} }, + { "parallel": 16, "search_params": {"num_candidates": 256} } ], "upload_params": { "parallel": 16 } }, @@ -72,9 +72,9 @@ } }, "search_params": [ - { "parallel": 1, "num_candidates": 256 }, - { "parallel": 8, "num_candidates": 256 }, - { "parallel": 16, "num_candidates": 256 } + { "parallel": 1, "search_params": {"num_candidates": 256} }, + { "parallel": 8, "search_params": {"num_candidates": 256} }, + { "parallel": 16, "search_params": {"num_candidates": 256} } ], "upload_params": { "parallel": 16 } }, @@ -86,8 +86,8 @@ }, "collection_params": { "index_options": { "m": 16, "ef_construction": 128 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -99,8 +99,8 @@ }, "collection_params": { "index_options": { "m": 32, "ef_construction": 128 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -112,8 +112,8 @@ }, "collection_params": { "index_options": { "m": 32, "ef_construction": 256 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -125,8 +125,8 @@ }, "collection_params": { "index_options": { "m": 32, "ef_construction": 512 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -138,8 +138,8 @@ }, "collection_params": { "index_options": { "m": 64, "ef_construction": 256 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -151,8 +151,8 @@ }, "collection_params": { "index_options": { "m": 64, "ef_construction": 512 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } } From c81e1105d64ea7caeafce1245895fea931474894 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Fri, 18 Apr 2025 21:58:48 +0700 Subject: [PATCH 26/33] Exclude parallel key for elastic --- engine/clients/elasticsearch/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/clients/elasticsearch/search.py b/engine/clients/elasticsearch/search.py index 29d20ec5f..22933854e 100644 --- a/engine/clients/elasticsearch/search.py +++ b/engine/clients/elasticsearch/search.py @@ -51,7 +51,7 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]: "field": "vector", "query_vector": vector, "k": top, - **{"num_candidates": 100, **cls.search_params}, + {"num_candidates": 100, **{k: v for k, v in cls.search_params.items() if k != 'parallel'}} } meta_conditions = cls.parser.parse(meta_conditions) From aa026b2da0783f596954e526c87c45f1c53a49d7 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Fri, 18 Apr 2025 21:58:57 +0700 Subject: [PATCH 27/33] Revert "Fix elasticsearch config" This reverts commit 4d30aa92bf5d82a97ef1f116940937157c00c9c3. --- .../elasticsearch-single-node.json | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/experiments/configurations/elasticsearch-single-node.json b/experiments/configurations/elasticsearch-single-node.json index f04b177df..de1fbe74f 100644 --- a/experiments/configurations/elasticsearch-single-node.json +++ b/experiments/configurations/elasticsearch-single-node.json @@ -7,8 +7,8 @@ }, "collection_params": { "index_options": { "m": 16, "ef_construction": 100 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -28,9 +28,9 @@ } }, "search_params": [ - { "parallel": 1, "search_params": {"num_candidates": 256} }, - { "parallel": 8, "search_params": {"num_candidates": 256} }, - { "parallel": 16, "search_params": {"num_candidates": 256} } + { "parallel": 1, "num_candidates": 256 }, + { "parallel": 8, "num_candidates": 256 }, + { "parallel": 16, "num_candidates": 256 } ], "upload_params": { "parallel": 16 } }, @@ -50,9 +50,9 @@ } }, "search_params": [ - { "parallel": 1, "search_params": {"num_candidates": 256} }, - { "parallel": 8, "search_params": {"num_candidates": 256} }, - { "parallel": 16, "search_params": {"num_candidates": 256} } + { "parallel": 1, "num_candidates": 256 }, + { "parallel": 8, "num_candidates": 256 }, + { "parallel": 16, "num_candidates": 256 } ], "upload_params": { "parallel": 16 } }, @@ -72,9 +72,9 @@ } }, "search_params": [ - { "parallel": 1, "search_params": {"num_candidates": 256} }, - { "parallel": 8, "search_params": {"num_candidates": 256} }, - { "parallel": 16, "search_params": {"num_candidates": 256} } + { "parallel": 1, "num_candidates": 256 }, + { "parallel": 8, "num_candidates": 256 }, + { "parallel": 16, "num_candidates": 256 } ], "upload_params": { "parallel": 16 } }, @@ -86,8 +86,8 @@ }, "collection_params": { "index_options": { "m": 16, "ef_construction": 128 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -99,8 +99,8 @@ }, "collection_params": { "index_options": { "m": 32, "ef_construction": 128 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -112,8 +112,8 @@ }, "collection_params": { "index_options": { "m": 32, "ef_construction": 256 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -125,8 +125,8 @@ }, "collection_params": { "index_options": { "m": 32, "ef_construction": 512 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -138,8 +138,8 @@ }, "collection_params": { "index_options": { "m": 64, "ef_construction": 256 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -151,8 +151,8 @@ }, "collection_params": { "index_options": { "m": 64, "ef_construction": 512 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } } From ed35127dac1bf9f8642c45b543e86339e0c607e2 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Fri, 18 Apr 2025 22:00:10 +0700 Subject: [PATCH 28/33] Reapply "Fix elasticsearch config" This reverts commit aa026b2da0783f596954e526c87c45f1c53a49d7. --- .../elasticsearch-single-node.json | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/experiments/configurations/elasticsearch-single-node.json b/experiments/configurations/elasticsearch-single-node.json index de1fbe74f..f04b177df 100644 --- a/experiments/configurations/elasticsearch-single-node.json +++ b/experiments/configurations/elasticsearch-single-node.json @@ -7,8 +7,8 @@ }, "collection_params": { "index_options": { "m": 16, "ef_construction": 100 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -28,9 +28,9 @@ } }, "search_params": [ - { "parallel": 1, "num_candidates": 256 }, - { "parallel": 8, "num_candidates": 256 }, - { "parallel": 16, "num_candidates": 256 } + { "parallel": 1, "search_params": {"num_candidates": 256} }, + { "parallel": 8, "search_params": {"num_candidates": 256} }, + { "parallel": 16, "search_params": {"num_candidates": 256} } ], "upload_params": { "parallel": 16 } }, @@ -50,9 +50,9 @@ } }, "search_params": [ - { "parallel": 1, "num_candidates": 256 }, - { "parallel": 8, "num_candidates": 256 }, - { "parallel": 16, "num_candidates": 256 } + { "parallel": 1, "search_params": {"num_candidates": 256} }, + { "parallel": 8, "search_params": {"num_candidates": 256} }, + { "parallel": 16, "search_params": {"num_candidates": 256} } ], "upload_params": { "parallel": 16 } }, @@ -72,9 +72,9 @@ } }, "search_params": [ - { "parallel": 1, "num_candidates": 256 }, - { "parallel": 8, "num_candidates": 256 }, - { "parallel": 16, "num_candidates": 256 } + { "parallel": 1, "search_params": {"num_candidates": 256} }, + { "parallel": 8, "search_params": {"num_candidates": 256} }, + { "parallel": 16, "search_params": {"num_candidates": 256} } ], "upload_params": { "parallel": 16 } }, @@ -86,8 +86,8 @@ }, "collection_params": { "index_options": { "m": 16, "ef_construction": 128 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -99,8 +99,8 @@ }, "collection_params": { "index_options": { "m": 32, "ef_construction": 128 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -112,8 +112,8 @@ }, "collection_params": { "index_options": { "m": 32, "ef_construction": 256 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -125,8 +125,8 @@ }, "collection_params": { "index_options": { "m": 32, "ef_construction": 512 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -138,8 +138,8 @@ }, "collection_params": { "index_options": { "m": 64, "ef_construction": 256 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -151,8 +151,8 @@ }, "collection_params": { "index_options": { "m": 64, "ef_construction": 512 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } } From c259610cad98821a48e3aaf7fb4b03cc4c6ed601 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Fri, 18 Apr 2025 22:00:48 +0700 Subject: [PATCH 29/33] Update config of elastic again --- .../elasticsearch-single-node.json | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/experiments/configurations/elasticsearch-single-node.json b/experiments/configurations/elasticsearch-single-node.json index f04b177df..6e7382703 100644 --- a/experiments/configurations/elasticsearch-single-node.json +++ b/experiments/configurations/elasticsearch-single-node.json @@ -7,8 +7,8 @@ }, "collection_params": { "index_options": { "m": 16, "ef_construction": 100 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates":256 }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates":256 }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -28,9 +28,9 @@ } }, "search_params": [ - { "parallel": 1, "search_params": {"num_candidates": 256} }, - { "parallel": 8, "search_params": {"num_candidates": 256} }, - { "parallel": 16, "search_params": {"num_candidates": 256} } + { "parallel": 1, "num_candidates":256 }, + { "parallel": 8, "num_candidates":256 }, + { "parallel": 16, "num_candidates":256 } ], "upload_params": { "parallel": 16 } }, @@ -50,9 +50,9 @@ } }, "search_params": [ - { "parallel": 1, "search_params": {"num_candidates": 256} }, - { "parallel": 8, "search_params": {"num_candidates": 256} }, - { "parallel": 16, "search_params": {"num_candidates": 256} } + { "parallel": 1, "num_candidates":256 }, + { "parallel": 8, "num_candidates":256 }, + { "parallel": 16, "num_candidates":256 } ], "upload_params": { "parallel": 16 } }, @@ -72,9 +72,9 @@ } }, "search_params": [ - { "parallel": 1, "search_params": {"num_candidates": 256} }, - { "parallel": 8, "search_params": {"num_candidates": 256} }, - { "parallel": 16, "search_params": {"num_candidates": 256} } + { "parallel": 1, "num_candidates":256 }, + { "parallel": 8, "num_candidates":256 }, + { "parallel": 16, "num_candidates":256 } ], "upload_params": { "parallel": 16 } }, @@ -86,8 +86,8 @@ }, "collection_params": { "index_options": { "m": 16, "ef_construction": 128 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates":256 }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates":256 }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -99,8 +99,8 @@ }, "collection_params": { "index_options": { "m": 32, "ef_construction": 128 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates":256 }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates":256 }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -112,8 +112,8 @@ }, "collection_params": { "index_options": { "m": 32, "ef_construction": 256 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates":256 }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates":256 }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -125,8 +125,8 @@ }, "collection_params": { "index_options": { "m": 32, "ef_construction": 512 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates":256 }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates":256 }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -138,8 +138,8 @@ }, "collection_params": { "index_options": { "m": 64, "ef_construction": 256 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates":256 }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates":256 }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } }, @@ -151,8 +151,8 @@ }, "collection_params": { "index_options": { "m": 64, "ef_construction": 512 } }, "search_params": [ - { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "search_params": {"num_candidates": 256} }, { "parallel": 1, "num_candidates": 512 }, - { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "search_params": {"num_candidates": 256} }, { "parallel": 100, "num_candidates": 512 } + { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates":256 }, { "parallel": 1, "num_candidates": 512 }, + { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates":256 }, { "parallel": 100, "num_candidates": 512 } ], "upload_params": { "parallel": 16 } } From 2b53b015a0b6ec45424afec13f0aab28481f21d7 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Fri, 18 Apr 2025 22:01:32 +0700 Subject: [PATCH 30/33] Revert "Exclude parallel key for elastic" This reverts commit c81e1105d64ea7caeafce1245895fea931474894. --- engine/clients/elasticsearch/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/clients/elasticsearch/search.py b/engine/clients/elasticsearch/search.py index 22933854e..29d20ec5f 100644 --- a/engine/clients/elasticsearch/search.py +++ b/engine/clients/elasticsearch/search.py @@ -51,7 +51,7 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]: "field": "vector", "query_vector": vector, "k": top, - {"num_candidates": 100, **{k: v for k, v in cls.search_params.items() if k != 'parallel'}} + **{"num_candidates": 100, **cls.search_params}, } meta_conditions = cls.parser.parse(meta_conditions) From 74a8ac91db3e36a8b11557ca2bcecb244771aab1 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Fri, 18 Apr 2025 22:02:27 +0700 Subject: [PATCH 31/33] fix --- engine/clients/elasticsearch/search.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/engine/clients/elasticsearch/search.py b/engine/clients/elasticsearch/search.py index 29d20ec5f..c8560cf62 100644 --- a/engine/clients/elasticsearch/search.py +++ b/engine/clients/elasticsearch/search.py @@ -51,7 +51,10 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]: "field": "vector", "query_vector": vector, "k": top, - **{"num_candidates": 100, **cls.search_params}, + **{ + "num_candidates": 100, + **{k: v for k, v in cls.search_params.items() if k != 'parallel'} + } } meta_conditions = cls.parser.parse(meta_conditions) From 16adbd55bf6c7a89eee66ace7a0afe5b9bbb58a7 Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Mon, 21 Apr 2025 14:09:29 +0700 Subject: [PATCH 32/33] Fix elasticsearch config --- .../elasticsearch-single-node.json | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/experiments/configurations/elasticsearch-single-node.json b/experiments/configurations/elasticsearch-single-node.json index 6e7382703..3a7d44cbe 100644 --- a/experiments/configurations/elasticsearch-single-node.json +++ b/experiments/configurations/elasticsearch-single-node.json @@ -21,10 +21,8 @@ "collection_params": { "index_options": { "m": 32, - "ef_construction": 256 - }, - "binary_quantization": { - "type": "random_projection" + "ef_construction": 256, + "type": "bbq_hnsw" } }, "search_params": [ @@ -43,10 +41,8 @@ "collection_params": { "index_options": { "m": 32, - "ef_construction": 256 - }, - "scalar_quantization": { - "type": "float4" + "ef_construction": 256, + "type": "int4_hnsw" } }, "search_params": [ @@ -65,10 +61,8 @@ "collection_params": { "index_options": { "m": 32, - "ef_construction": 256 - }, - "scalar_quantization": { - "type": "float8" + "ef_construction": 256, + "type": "int8_hnsw" } }, "search_params": [ From 66d0839e6f83833f7370b22d31d66f39b1e609bb Mon Sep 17 00:00:00 2001 From: Don Hardman Date: Wed, 23 Apr 2025 15:49:26 +0700 Subject: [PATCH 33/33] Try to fix manticore search index mappings --- engine/clients/manticoresearch/configure.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/engine/clients/manticoresearch/configure.py b/engine/clients/manticoresearch/configure.py index 5fa1c8f6d..8b480190d 100644 --- a/engine/clients/manticoresearch/configure.py +++ b/engine/clients/manticoresearch/configure.py @@ -14,6 +14,15 @@ class ManticoreSearchConfigurator(BaseConfigurator): Distance.L2: "L2", Distance.COSINE: "COSINE", } + + + INDEX_TYPE_MAPPING = { + "int": "uint", + "keyword": "string", + "text": "text", + "float": "float", + "geo": "json", # Manticore typically handles geo as JSON + } def __init__(self, host, collection_params, connection_params): self.host = host