From e2534120ec0d92296cf8fe9f4e86bba86de24536 Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Fri, 10 May 2024 12:43:44 -0400 Subject: [PATCH] cleanup dead code --- .../QdrantCreateCollectionOpDispenser.java | 267 ------------------ .../QdrantSearchPointsOpDispenser.java | 25 -- .../qdrant/ops/QdrantCountPointsOp.java | 2 +- .../activities/qdrant_vectors_live.yaml | 10 +- 4 files changed, 6 insertions(+), 298 deletions(-) diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreateCollectionOpDispenser.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreateCollectionOpDispenser.java index ef5b18d12..e9d162ed9 100644 --- a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreateCollectionOpDispenser.java +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantCreateCollectionOpDispenser.java @@ -26,7 +26,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -58,20 +57,10 @@ public LongFunction getParamFunc( LongFunction ebF = l -> CreateCollection.newBuilder().setCollectionName(targetF.apply(l)); - // new code - incomplete LongFunction> namedVectorParamsMap = buildNamedVectorsStruct(op); final LongFunction namedVectorsF = ebF; ebF = l -> namedVectorsF.apply(l).setVectorsConfig(VectorsConfig.newBuilder().setParamsMap( VectorParamsMap.newBuilder().putAllMap(namedVectorParamsMap.apply(l)).build())); - // new code - incomplete - // old code -// Map namedVectorParamsMap1 = buildNamedVectorsStruct( -// op.getAsSubOps("vectors", ParsedOp.SubOpNaming.SubKey) -// ); -// final LongFunction namedVectorsF1 = ebF; -// ebF = l -> namedVectorsF1.apply(l).setVectorsConfig(VectorsConfig.newBuilder().setParamsMap( -// VectorParamsMap.newBuilder().putAllMap(namedVectorParamsMap1).build())); - // old code ebF = op.enhanceFuncOptionally(ebF, "on_disk_payload", Boolean.class, CreateCollection.Builder::setOnDiskPayload); @@ -92,9 +81,6 @@ public LongFunction getParamFunc( LongFunction wcdF = buildWalConfigDiff(walF.get()); ebF = l -> wallFunc.apply(l).setWalConfig(wcdF.apply(l)); } -// WalConfigDiff walConfig = buildWalConfigDiff(op); -// final LongFunction walConfigF = ebF; -// ebF = l -> walConfigF.apply(l).setWalConfig(walConfig); Optional> optConDifF = op.getAsOptionalFunction("optimizers_config", Map.class); if (optConDifF.isPresent()) { @@ -103,10 +89,6 @@ public LongFunction getParamFunc( ebF = l -> wallFunc.apply(l).setOptimizersConfig(ocdF.apply(l)); } -// OptimizersConfigDiff ocDiff = buildOptimizerConfigDiff(op); -// final LongFunction ocF = ebF; -// ebF = l -> ocF.apply(l).setOptimizersConfig(ocDiff); - Optional> hnswConfigDiffF = op.getAsOptionalFunction("hnsw_config", Map.class); if (hnswConfigDiffF.isPresent()) { final LongFunction hnswConfigF = ebF; @@ -114,21 +96,12 @@ public LongFunction getParamFunc( ebF = l -> hnswConfigF.apply(l).setHnswConfig(hcdF.apply(l)); } -// HnswConfigDiff hnswConfigDiff = buildHnswConfigDiff(op); -// final LongFunction hnswConfigF = ebF; -// ebF = l -> hnswConfigF.apply(l).setHnswConfig(hnswConfigDiff); - Optional> quantConfigF = op.getAsOptionalFunction("quantization_config", Map.class); if (quantConfigF.isPresent()) { final LongFunction qConF = ebF; LongFunction qcDiffF = buildQuantizationConfig(quantConfigF.get()); ebF = l -> qConF.apply(l).setQuantizationConfig(qcDiffF.apply(l)); } -// QuantizationConfig qcDiff = buildQuantizationConfig(op); -// if (qcDiff != null) { -// final LongFunction qcConfigF = ebF; -// ebF = l -> qcConfigF.apply(l).setQuantizationConfig(qcDiff); -// } Optional> sparseVectorsF = op.getAsOptionalFunction("sparse_vectors", Map.class); if (sparseVectorsF.isPresent()) { @@ -136,59 +109,11 @@ public LongFunction getParamFunc( LongFunction sparseVectorsMap = buildSparseVectorsStruct(sparseVectorsF.get()); ebF = l -> sparseVecF.apply(l).setSparseVectorsConfig(sparseVectorsMap.apply(l)); } -// if (op.isDefined("sparse_vectors")) { -// SparseVectorConfig sparseVectorsMap = buildSparseVectorsStruct( -// op.getAsSubOps("sparse_vectors", ParsedOp.SubOpNaming.SubKey) -// ); -// final LongFunction sparseVectorsF = ebF; -// ebF = l -> sparseVectorsF.apply(l).setSparseVectorsConfig(sparseVectorsMap); -// } final LongFunction lastF = ebF; return l -> lastF.apply(l).build(); } - /** - * Build the {@link OptimizersConfigDiff} from the provided {@link ParsedOp}. - * - * @param op {@link ParsedOp} containing the optimizer config data. - * @return {@link OptimizersConfigDiff} containing the optimizer config data - */ - private OptimizersConfigDiff buildOptimizerConfigDiff(ParsedOp op) { - OptimizersConfigDiff.Builder ocDiffBuilder = OptimizersConfigDiff.newBuilder(); - op.getOptionalStaticValue("optimizers_config", Map.class).ifPresent(ocData -> { - if (ocData.isEmpty()) { - return; - } else { - if (ocData.containsKey("deleted_threshold")) { - ocDiffBuilder.setDeletedThreshold(((Number) ocData.get("deleted_threshold")).doubleValue()); - } - if (ocData.containsKey("vacuum_min_vector_number")) { - ocDiffBuilder.setVacuumMinVectorNumber(((Number) ocData.get("vacuum_min_vector_number")).longValue()); - } - if (ocData.containsKey("default_segment_number")) { - ocDiffBuilder.setDefaultSegmentNumber(((Number) ocData.get("default_segment_number")).longValue()); - } - if (ocData.containsKey("max_segment_size")) { - ocDiffBuilder.setMaxSegmentSize(((Number) ocData.get("max_segment_size")).longValue()); - } - if (ocData.containsKey("memmap_threshold")) { - ocDiffBuilder.setMemmapThreshold(((Number) ocData.get("memmap_threshold")).longValue()); - } - if (ocData.containsKey("indexing_threshold")) { - ocDiffBuilder.setIndexingThreshold(((Number) ocData.get("indexing_threshold")).longValue()); - } - if (ocData.containsKey(("flush_interval_sec"))) { - ocDiffBuilder.setFlushIntervalSec(((Number) ocData.get("flush_interval_sec")).longValue()); - } - if (ocData.containsKey("max_optimization_threads")) { - ocDiffBuilder.setMaxOptimizationThreads(((Number) ocData.get("max_optimization_threads")).intValue()); - } - } - }); - return ocDiffBuilder.build(); - } - /** * Build the {@link OptimizersConfigDiff} from the provided {@link ParsedOp}. * @@ -229,30 +154,6 @@ private LongFunction buildOptimizerConfigDiff(LongFunction }; } - /** - * Build the {@link WalConfigDiff} from the provided {@link ParsedOp}. - * - * @param op {@link ParsedOp} containing the WAL config data. - * @return {@link WalConfigDiff} containing the WAL config data - */ - @Deprecated - private WalConfigDiff buildWalConfigDiff(ParsedOp op) { - WalConfigDiff.Builder walConfigDiffBuilder = WalConfigDiff.newBuilder(); - op.getOptionalStaticValue("wal_config", Map.class).ifPresent(walConfigData -> { - if (walConfigData.isEmpty()) { - return; - } else { - if (walConfigData.containsKey("wal_capacity_mb")) { - walConfigDiffBuilder.setWalCapacityMb(((Number) walConfigData.get("wal_capacity_mb")).longValue()); - } - if (walConfigData.containsKey("wal_segments_ahead")) { - walConfigDiffBuilder.setWalSegmentsAhead(((Number) walConfigData.get("wal_segments_ahead")).longValue()); - } - } - }); - return walConfigDiffBuilder.build(); - } - /** * Build the {@link WalConfigDiff} from the provided {@link ParsedOp}. * @@ -275,32 +176,6 @@ private LongFunction buildWalConfigDiff(LongFunction mapLong }; } - /** - * Only named vectors are supported at this time in this driver. - * - * @param {@link Map} namedVectorsData - * @return {@link VectorParams} containing the named vectors - */ - @Deprecated - private Map buildNamedVectorsStruct(Map namedVectorsData) { - Map namedVectors = new HashMap<>(); - VectorParams.Builder builder = VectorParams.newBuilder(); - namedVectorsData.forEach((name, fieldSpec) -> { - builder.setDistanceValue(fieldSpec.getStaticValue("distance_value", Number.class).intValue()); - builder.setSize(fieldSpec.getStaticValue("size", Number.class).longValue()); - fieldSpec.getOptionalStaticValue("on_disk", Boolean.class) - .ifPresent(builder::setOnDisk); - fieldSpec.getOptionalStaticValue("datatype_value", Number.class) - .ifPresent((Number value) -> builder.setDatatypeValue(value.intValue())); - - builder.setHnswConfig(buildHnswConfigDiff(fieldSpec)); - builder.setQuantizationConfig(buildQuantizationConfig(fieldSpec)); - - namedVectors.put(name, builder.build()); - }); - return namedVectors; - } - /** * Only named vectors are supported at this time in this driver. * @@ -348,124 +223,6 @@ private LongFunction> buildNamedVectorsStruct(ParsedOp }).orElse(null); } - /** - * Build the {@link QuantizationConfig} from the provided {@link ParsedOp}. - * - * @param fieldSpec The {@link ParsedOp} containing the quantization config data - * @return The {@link QuantizationConfig} built from the provided {@link ParsedOp} - * @see Quantization Config - */ - @Deprecated - private QuantizationConfig buildQuantizationConfig(ParsedOp fieldSpec) { - QuantizationConfig.Builder qcBuilder = QuantizationConfig.newBuilder(); - fieldSpec.getOptionalStaticValue("quantization_config", Map.class).ifPresent(qcData -> { - if (qcData.isEmpty()) { - return; - } else { - // TODO - Approach #1 - feels ugly - Arrays.asList("binary", "product", "scalar") - .forEach(key -> { - if (qcData.containsKey(key)) { - switch (key) { - case "binary": - BinaryQuantization.Builder binaryBuilder = BinaryQuantization.newBuilder(); - Map binaryQCData = (Map) qcData.get("binary"); - if (null != binaryQCData && !binaryQCData.isEmpty()) { - if (binaryQCData.containsKey("always_ram")) { - binaryBuilder.setAlwaysRam((Boolean) binaryQCData.get("always_ram")); - } - } - qcBuilder.setBinary(binaryBuilder); - break; - case "product": - ProductQuantization.Builder productBuilder = ProductQuantization.newBuilder(); - Map productQCData = (Map) qcData.get("product"); - if (null != productQCData && !productQCData.isEmpty()) { - // Mandatory field - productBuilder.setAlwaysRam((Boolean) productQCData.get("always_ram")); - // Optional field(s) below - if (productQCData.containsKey("compression")) { - productBuilder.setCompression(CompressionRatio.valueOf((String) productQCData.get("compression"))); - } - } - qcBuilder.setProduct(productBuilder); - break; - case "scalar": - ScalarQuantization.Builder scalarBuilder = ScalarQuantization.newBuilder(); - Map scalarQCData = (Map) qcData.get("scalar"); - if (null != scalarQCData && !scalarQCData.isEmpty()) { - // Mandatory field - scalarBuilder.setType(QuantizationType.forNumber(((Number) scalarQCData.get("type")).intValue())); - // Optional field(s) below - if (scalarQCData.containsKey("always_ram")) { - scalarBuilder.setAlwaysRam((Boolean) scalarQCData.get("always_ram")); - } - if (scalarQCData.containsKey("quantile")) { - scalarBuilder.setQuantile(((Number) scalarQCData.get("quantile")).floatValue()); - } - } - qcBuilder.setScalar(scalarBuilder); - break; - } - } - }); - // TODO - Approach #2 - equally feels ugly too. -// if (qcData.containsKey("binary")) { -// if (qcData.containsKey("scalar") || qcData.containsKey("product")) { -// throw new UnsupportedOperationException("Only one of binary, scalar, or product can be specified for quantization config"); -// } -// BinaryQuantization.Builder binaryBuilder = BinaryQuantization.newBuilder(); -// Map binaryQCData = (Map) qcData.get("binary"); -// if (null != binaryQCData && !binaryQCData.isEmpty()) { -// if (binaryQCData.containsKey("always_ram")) { -// binaryBuilder.setAlwaysRam((Boolean) binaryQCData.get("always_ram")); -// } -// } -// qcBuilder.setBinary(binaryBuilder); -// } else if (qcData.containsKey("product")) { -// if (qcData.containsKey("binary") || qcData.containsKey("scalar")) { -// throw new UnsupportedOperationException("Only one of binary, scalar, or product can be specified for quantization config"); -// } -// ProductQuantization.Builder productBuilder = ProductQuantization.newBuilder(); -// Map productQCData = (Map) qcData.get("product"); -// if (null != productQCData && !productQCData.isEmpty()) { -// // Mandatory field -// productBuilder.setAlwaysRam((Boolean) productQCData.get("always_ram")); -// // Optional field(s) below -// if (productQCData.containsKey("compression")) { -// productBuilder.setCompression(CompressionRatio.valueOf((String) productQCData.get("compression"))); -// } -// } -// qcBuilder.setProduct(productBuilder); -// } else if (qcData.containsKey("scalar")) { -// if (qcData.containsKey("binary") || qcData.containsKey("product")) { -// throw new UnsupportedOperationException("Only one of binary, scalar, or product can be specified for quantization config"); -// } -// ScalarQuantization.Builder scalarBuilder = ScalarQuantization.newBuilder(); -// Map scalarQCData = (Map) qcData.get("scalar"); -// if (null != scalarQCData && !scalarQCData.isEmpty()) { -// // Mandatory field -// scalarBuilder.setType(QuantizationType.valueOf((String) scalarQCData.get("type"))); -// // Optional field(s) below -// if (scalarQCData.containsKey("always_ram")) { -// scalarBuilder.setAlwaysRam((Boolean) scalarQCData.get("always_ram")); -// } -// if (scalarQCData.containsKey("quantile")) { -// scalarBuilder.setQuantile((Float) scalarQCData.get("quantile")); -// } -// } -// qcBuilder.setScalar(scalarBuilder); -// } - } - }); - - // The below check is required to avoid INVALID_ARGUMENT: Unable to convert quantization config - if (qcBuilder.hasBinary() || qcBuilder.hasProduct() || qcBuilder.hasScalar()) { - return qcBuilder.build(); - } - return null; - } - private LongFunction buildQuantizationConfig(LongFunction quantConfMapLongFunc) { return l -> this.buildQuantizationConfig(quantConfMapLongFunc.apply(l)); } @@ -592,30 +349,6 @@ private HnswConfigDiff buildHnswConfigDiff(Map hnswConfigDiffMap return hnswConfigBuilder.build(); } - /** - * Build the {@link SparseVectorConfig} from the provided {@link ParsedOp}. - * - * @param sparseVectorsData The {@link ParsedOp} containing the sparse vectors data - * @return The {@link SparseVectorConfig} built from the provided {@link ParsedOp} - */ - @Deprecated - private SparseVectorConfig buildSparseVectorsStruct(Map sparseVectorsData) { - SparseVectorConfig.Builder builder = SparseVectorConfig.newBuilder(); - sparseVectorsData.forEach((name, fieldSpec) -> { - SparseVectorParams.Builder svpBuilder = SparseVectorParams.newBuilder(); - SparseIndexConfig.Builder sicBuilder = SparseIndexConfig.newBuilder(); - - fieldSpec.getOptionalStaticValue("full_scan_threshold", Number.class) - .ifPresent((Number value) -> sicBuilder.setFullScanThreshold(value.intValue())); - fieldSpec.getOptionalStaticValue("on_disk", Boolean.class) - .ifPresent(sicBuilder::setOnDisk); - - svpBuilder.setIndex(sicBuilder); - builder.putMap(name, svpBuilder.build()); - }); - return builder.build(); - } - /** * Build the {@link SparseVectorConfig} from the provided {@link ParsedOp}. * diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantSearchPointsOpDispenser.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantSearchPointsOpDispenser.java index 715fde0cd..61a8419d8 100644 --- a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantSearchPointsOpDispenser.java +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/opdispensers/QdrantSearchPointsOpDispenser.java @@ -216,29 +216,4 @@ private LongFunction buildReadConsistency(LongFunction } }; } - - @Deprecated - private Optional>> getVectorFieldsFunction(ParsedOp op, String namedVectors) { -// return l -> { -// if (!op.isDefined(namedVectors)) { -// return Optional.empty(); -// } -// List fields = op.get(namedVectors, 0L); -// if (fields == null) { -// fields = op.get(namedVectors, 0L); -// } -// return fields; -// }; - LongFunction rowF = op.getAsRequiredFunction(namedVectors, Object.class); - Object testObject = rowF.apply(0L); - LongFunction> rowsF = null; - if(testObject instanceof List list) { - if (list.isEmpty()) { - throw new OpConfigError("Unable to detect type of list object for empty list for op named '" + op.getName() + "'"); - } else if (list.get(0) instanceof Float) { - rowsF = l -> (List) rowF.apply(l); - } - } - return Optional.ofNullable(rowsF); - } } diff --git a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCountPointsOp.java b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCountPointsOp.java index 1cd82aca8..48f8e0ba5 100644 --- a/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCountPointsOp.java +++ b/nb-adapters/adapter-qdrant/src/main/java/io/nosqlbench/adapter/qdrant/ops/QdrantCountPointsOp.java @@ -37,7 +37,7 @@ public Object applyOp(long value) { request.getExact(), Duration.ofMinutes(5) // opinionated default of 5 minutes for timeout ).get(); - logger.info("Total vector points counted: {}", result); + logger.info("[QdrantCountPointsOp] Total vector points counted: {}", result); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } diff --git a/nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml b/nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml index 53fa40ef0..547c15373 100644 --- a/nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml +++ b/nb-adapters/adapter-qdrant/src/main/resources/activities/qdrant_vectors_live.yaml @@ -17,27 +17,27 @@ scenarios: run tags==block:delete_collection errors===stop cycles===UNDEF threads===UNDEF - uri=TEMPLATE(qdranthost) token_file=TEMPLATE(token_file) + uri=TEMPLATE(qdranthost) grpc_port=TEMPLATE(grpc_port,6334) token_file=TEMPLATE(token_file) schema_collection: >- run tags==block:schema_collection errors===stop cycles===UNDEF threads===UNDEF - uri=TEMPLATE(qdranthost) token_file=TEMPLATE(token_file) + uri=TEMPLATE(qdranthost) grpc_port=TEMPLATE(grpc_port,6334) token_file=TEMPLATE(token_file) rampup: >- run tags==block:rampup errors===warn,counter,retry cycles===TEMPLATE(train_cycles,TEMPLATE(trainsize,1000)) threads===TEMPLATE(train_threads,AUTO) - uri=TEMPLATE(qdranthost) token_file=TEMPLATE(token_file) + uri=TEMPLATE(qdranthost) grpc_port=TEMPLATE(grpc_port,6334) token_file=TEMPLATE(token_file) count_vectors: >- run tags==block:count_vectors errors===stop cycles===UNDEF threads===UNDEF - uri=TEMPLATE(qdranthost) token_file=TEMPLATE(token_file) + uri=TEMPLATE(qdranthost) grpc_port=TEMPLATE(grpc_port,6334) token_file=TEMPLATE(token_file) search_points: >- run tags==block:search_points errors===warn,counter cycles===TEMPLATE(testann_cycles,TEMPLATE(testsize,1000)) threads===TEMPLATE(testann_threads,AUTO) - uri=TEMPLATE(qdranthost) token_file=TEMPLATE(token_file) + uri=TEMPLATE(qdranthost) grpc_port=TEMPLATE(grpc_port,6334) token_file=TEMPLATE(token_file) params: driver: qdrant