From 3209e3c14eb41f1e80c5ebb55e04bbdaf1259f70 Mon Sep 17 00:00:00 2001 From: Marcin Pozniak Date: Thu, 26 Jun 2025 08:54:31 +0000 Subject: [PATCH 1/4] Add dbpedia calibration json --- engine/clients/redis/search.py | 4 +- experiments/configurations/create-svs.py | 26 +-- .../configurations/dbpedia-calibration.json | 192 ++++++++++++++++++ 3 files changed, 207 insertions(+), 15 deletions(-) create mode 100644 experiments/configurations/dbpedia-calibration.json diff --git a/engine/clients/redis/search.py b/engine/clients/redis/search.py index 88fef8a2..31cf27bd 100644 --- a/engine/clients/redis/search.py +++ b/engine/clients/redis/search.py @@ -38,7 +38,7 @@ def init_client(cls, host, distance, connection_params: dict, search_params: dic if cls.hybrid_policy != "ADHOC_BF": cls.knn_conditions = "EF_RUNTIME $EF" elif cls.algorithm == "SVS-VAMANA": - cls.knn_conditions = "WS_SEARCH $SEARCH_WINDOW_SIZE" + cls.knn_conditions = "WS_SEARCH $WS_SEARCH" cls.data_type = "FLOAT32" if "search_params" in cls.search_params: cls.data_type = ( @@ -97,7 +97,7 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]: if cls.hybrid_policy != "ADHOC_BF": params_dict["EF"] = cls.search_params["search_params"]["ef"] if cls.algorithm == "SVS-VAMANA": - params_dict["SEARCH_WINDOW_SIZE"] = cls.search_params["search_params"]["SEARCH_WINDOW_SIZE"] + params_dict["WS_SEARCH"] = cls.search_params["search_params"]["WS_SEARCH"] results = cls._ft.search(q, query_params=params_dict) return [(int(result.id), float(result.vector_score)) for result in results.docs] diff --git a/experiments/configurations/create-svs.py b/experiments/configurations/create-svs.py index 105bc97b..86774557 100644 --- a/experiments/configurations/create-svs.py +++ b/experiments/configurations/create-svs.py @@ -1,14 +1,14 @@ import json threads = [16] -ws_constructs = [100] -ws_search = [32, 40, 48, 64] +ws_constructs = [200] +ws_search = [177] #ws_search = [48] graph_degree = [32] -#quantization = ["NO_COMPRESSION", "LVQ4x4", "LVQ4x8", "LVQ4", "LVQ8", "LeanVec4x8", "LeanVec8x8"] -quantization = ["LVQ4x8"] -topKs = [10] -data_types = ["FLOAT32"] +#quantization = ["0", "4x4", "4x8", "8", "4"] +quantization = ["NO"] +topKs = [100] +data_types = ["FLOAT16", "FLOAT32"] for algo in ["svs-vamana"]: for data_type in data_types: @@ -18,34 +18,34 @@ configs = [] for thread in threads: config = { - "name": f"svs-test-algo-{algo}-graph-{graph_d}-ws-con-{ws_construct}-quant-{quant}-threads-{thread}-dt-{data_type}", + "name": f"cohere-{algo}-quant-{quant}-dt-{data_type}", "engine": "redis", "connection_params": {}, "collection_params": { "algorithm": algo, "data_type": data_type, - f"{algo}_config": {"NUM_THREADS": thread, "GRAPH_MAX_DEGREE": graph_d, "CONSTRUCTION_WINDOW_SIZE": ws_construct, "COMPRESSION": quant}, + f"{algo}_config": {"NUM_THREADS": thread, "GRAPH_MAX_DEGREE": graph_d, "CONSTRUCTION_WINDOW_SIZE": ws_construct, "compression": quant}, }, "search_params": [], "upload_params": { - "parallel": 128, + "parallel": 100, "data_type": data_type, "algorithm": algo, }, } - for client in [1, 8, 16, 32, 64, 128]: + for client in [100]: for ws_s in ws_search: for top in topKs: test_config = { "algorithm": algo, "parallel": client, "top": top, - "search_params": {"SEARCH_WINDOW_SIZE": ws_s, "data_type": data_type}, + "search_params": {"WS_SEARCH": ws_s, "data_type": data_type}, } config["search_params"].append(test_config) configs.append(config) - fname = f"svs-test-algo-{algo}-graph-{graph_d}-ws-con-{ws_construct}-quant-{quant}-threads-{thread}-dt-{data_type}.json" + fname = f"cohere-{algo}-quant-{quant}-dt-{data_type}.json" with open(fname, "w") as json_fd: json.dump(configs, json_fd, indent=2) - print(f"Created {len(configs)} configs for {fname}.") \ No newline at end of file + print(f"Created {len(configs)} configs for {fname}.") diff --git a/experiments/configurations/dbpedia-calibration.json b/experiments/configurations/dbpedia-calibration.json new file mode 100644 index 00000000..d6ae2ae5 --- /dev/null +++ b/experiments/configurations/dbpedia-calibration.json @@ -0,0 +1,192 @@ +[ + { + "name": "dbpedia-cal-hnsw-float16", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "hnsw", + "data_type": "FLOAT16", + "hnsw_config": { + "M": 16, + "DISTANCE_METRIC": "L2", + "EF_CONSTRUCTION": 200 + } + }, + "search_params": [ + { + "parallel": 100, + "top": 100, + "calibration_param": "ef", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT16" + } + } + ], + "upload_params": { + "parallel": 100, + "algorithm": "hnsw", + "data_type": "FLOAT16" + } + }, + { + "name": "dbpedia-cal-hnsw-float32", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "hnsw", + "data_type": "FLOAT32", + "hnsw_config": { + "M": 16, + "DISTANCE_METRIC": "L2", + "EF_CONSTRUCTION": 200 + } + }, + "search_params": [ + { + "parallel": 100, + "top": 100, + "calibration_param": "ef", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT32" + } + } + ], + "upload_params": { + "parallel": 100, + "algorithm": "hnsw", + "data_type": "FLOAT32" + } + }, +{ + "name": "dbpedia-cal-svs-noquant-float16", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT16", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "L2", + "GRAPH_MAX_DEGREE": 32, + "CONSTRUCTION_WINDOW_SIZE": 200 + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT16" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT16", + "algorithm": "svs-vamana" + } + }, + { + "name": "dbpedia-cal-svs-noquant-float32", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT32", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "L2", + "GRAPH_MAX_DEGREE": 32, + "CONSTRUCTION_WINDOW_SIZE": 200 + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT32" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT32", + "algorithm": "svs-vamana" + } + }, + { + "name": "dbpedia-cal-svs-SQ8-float16", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT16", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "L2", + "GRAPH_MAX_DEGREE": 32, + "CONSTRUCTION_WINDOW_SIZE": 200, + "compression": "LVQ4X8" + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT16" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT16", + "algorithm": "svs-vamana" + } + }, + { + "name": "dbpedia-cal-svs-SQ8-float32", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT32", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "L2", + "GRAPH_MAX_DEGREE": 32, + "CONSTRUCTION_WINDOW_SIZE": 200, + "compression": "LVQ4X4" + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT32" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT32", + "algorithm": "svs-vamana" + } + } +] \ No newline at end of file From d573c7f4746188117ab6e8d37accb509c4656046 Mon Sep 17 00:00:00 2001 From: Marcin Pozniak Date: Thu, 26 Jun 2025 09:00:08 +0000 Subject: [PATCH 2/4] Add LVQ's to dbpedia-calibration configuration --- .../configurations/dbpedia-calibration.json | 70 ++++++++++++++++++- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/experiments/configurations/dbpedia-calibration.json b/experiments/configurations/dbpedia-calibration.json index d6ae2ae5..6524bef9 100644 --- a/experiments/configurations/dbpedia-calibration.json +++ b/experiments/configurations/dbpedia-calibration.json @@ -124,7 +124,7 @@ } }, { - "name": "dbpedia-cal-svs-SQ8-float16", + "name": "dbpedia-cal-svs-LVQ4X8-float16", "engine": "redis", "connection_params": {}, "collection_params": { @@ -157,7 +157,40 @@ } }, { - "name": "dbpedia-cal-svs-SQ8-float32", + "name": "dbpedia-cal-svs-LVQ4X8-float32", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT32", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "L2", + "GRAPH_MAX_DEGREE": 32, + "CONSTRUCTION_WINDOW_SIZE": 200, + "compression": "LVQ4X8" + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT32" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT32", + "algorithm": "svs-vamana" + } + }, + { + "name": "dbpedia-cal-svs-LVQ4X4-float32", "engine": "redis", "connection_params": {}, "collection_params": { @@ -188,5 +221,38 @@ "data_type": "FLOAT32", "algorithm": "svs-vamana" } + }, + { + "name": "dbpedia-cal-svs-LVQ4X4-float16", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT16", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "L2", + "GRAPH_MAX_DEGREE": 32, + "CONSTRUCTION_WINDOW_SIZE": 200, + "compression": "LVQ4X4" + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT16" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT16", + "algorithm": "svs-vamana" + } } ] \ No newline at end of file From fd08e9ccdbac71fc8907e5b02e27964720a3623f Mon Sep 17 00:00:00 2001 From: Marcin Pozniak Date: Thu, 26 Jun 2025 09:02:37 +0000 Subject: [PATCH 3/4] Fix create-svs --- experiments/configurations/create-svs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/experiments/configurations/create-svs.py b/experiments/configurations/create-svs.py index 86774557..b904de3b 100644 --- a/experiments/configurations/create-svs.py +++ b/experiments/configurations/create-svs.py @@ -5,8 +5,8 @@ ws_search = [177] #ws_search = [48] graph_degree = [32] -#quantization = ["0", "4x4", "4x8", "8", "4"] -quantization = ["NO"] +#quantization = ["LVQ4X4", "LVQ4x8", "LVQ8", "LVQ4"] +quantization = ["LVQ4X4"] topKs = [100] data_types = ["FLOAT16", "FLOAT32"] @@ -18,7 +18,7 @@ configs = [] for thread in threads: config = { - "name": f"cohere-{algo}-quant-{quant}-dt-{data_type}", + "name": f"svs-{algo}-quant-{quant}-dt-{data_type}", "engine": "redis", "connection_params": {}, "collection_params": { @@ -45,7 +45,7 @@ config["search_params"].append(test_config) configs.append(config) - fname = f"cohere-{algo}-quant-{quant}-dt-{data_type}.json" + fname = f"svs-{algo}-quant-{quant}-dt-{data_type}.json" with open(fname, "w") as json_fd: json.dump(configs, json_fd, indent=2) print(f"Created {len(configs)} configs for {fname}.") From 7602ebb7a2cc27136489fe678f9140c81b1dc648 Mon Sep 17 00:00:00 2001 From: Marcin Pozniak Date: Thu, 26 Jun 2025 09:25:28 +0000 Subject: [PATCH 4/4] Add LeanVec4X8 --- .../configurations/dbpedia-calibration.json | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/experiments/configurations/dbpedia-calibration.json b/experiments/configurations/dbpedia-calibration.json index 6524bef9..a315de2e 100644 --- a/experiments/configurations/dbpedia-calibration.json +++ b/experiments/configurations/dbpedia-calibration.json @@ -254,5 +254,71 @@ "data_type": "FLOAT16", "algorithm": "svs-vamana" } + }, + { + "name": "dbpedia-cal-svs-LeanVec4x8-float16", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT16", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "L2", + "GRAPH_MAX_DEGREE": 32, + "CONSTRUCTION_WINDOW_SIZE": 200, + "compression": "LeanVec4x8" + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT16" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT16", + "algorithm": "svs-vamana" + } + }, + { + "name": "dbpedia-cal-svs-LeanVec4x8-float32", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT32", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "L2", + "GRAPH_MAX_DEGREE": 32, + "CONSTRUCTION_WINDOW_SIZE": 200, + "compression": "LeanVec4x8" + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT32" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT32", + "algorithm": "svs-vamana" + } } ] \ No newline at end of file