From 1d3331eb37a9a7428a2631cb7291b867424f9d84 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Thu, 16 Oct 2025 15:37:39 +0200 Subject: [PATCH 1/5] enhance import script to reduce memory usage --- cmd/import/main.go | 71 ++++++++++++++++++++++++++++++++++++--- docker-compose.qdrant.yml | 11 ++++++ 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/cmd/import/main.go b/cmd/import/main.go index edb74c0..c953973 100644 --- a/cmd/import/main.go +++ b/cmd/import/main.go @@ -238,6 +238,18 @@ func main() { for _, collectionName := range collections { showCollectionStats(ctx, client, collectionName) } + + // Re-enable production HNSW indexing for all collections + log.Println("\n=== Enabling Production HNSW Indexing ===") + log.Println("Re-enabling HNSW indexing (M=48) for production queries...") + for _, collectionName := range collections { + if err := enableProductionIndexing(ctx, client, collectionName); err != nil { + log.Printf("WARNING: Failed to enable production indexing for %s: %v", collectionName, err) + } + } + log.Println("\nāœ“ Production indexing enabled for all collections.") + log.Println("The Qdrant optimizer will build HNSW indexes in the background.") + log.Println("Monitor collection stats to track indexing progress.") } // Create a language-based collection with named vectors (dirs, names, contents). @@ -245,32 +257,39 @@ func createCollection(ctx context.Context, client *qdrant.Client, collectionName log.Printf("Creating language-based collection with named vectors: %s", collectionName) // Create named vectors configuration for dirs, names, and contents + // Optimized for bulk import: vectors on disk, HNSW disabled (M=0) namedVectors := map[string]*qdrant.VectorParams{ "dirs": { Size: VectorDim, Distance: qdrant.Distance_Manhattan, + OnDisk: qdrant.PtrOf(true), // Store vectors on disk to reduce RAM during import HnswConfig: &qdrant.HnswConfigDiff{ - M: qdrant.PtrOf(uint64(48)), + M: qdrant.PtrOf(uint64(0)), // Disable HNSW during import, re-enabled after EfConstruct: qdrant.PtrOf(uint64(500)), FullScanThreshold: qdrant.PtrOf(uint64(100000)), + OnDisk: qdrant.PtrOf(true), // Store HNSW index on disk }, }, "names": { Size: VectorDim, Distance: qdrant.Distance_Manhattan, + OnDisk: qdrant.PtrOf(true), // Store vectors on disk to reduce RAM during import HnswConfig: &qdrant.HnswConfigDiff{ - M: qdrant.PtrOf(uint64(48)), + M: qdrant.PtrOf(uint64(0)), // Disable HNSW during import, re-enabled after EfConstruct: qdrant.PtrOf(uint64(500)), FullScanThreshold: qdrant.PtrOf(uint64(100000)), + OnDisk: qdrant.PtrOf(true), // Store HNSW index on disk }, }, "contents": { Size: VectorDim, Distance: qdrant.Distance_Manhattan, + OnDisk: qdrant.PtrOf(true), // Store vectors on disk to reduce RAM during import HnswConfig: &qdrant.HnswConfigDiff{ - M: qdrant.PtrOf(uint64(48)), + M: qdrant.PtrOf(uint64(0)), // Disable HNSW during import, re-enabled after EfConstruct: qdrant.PtrOf(uint64(500)), FullScanThreshold: qdrant.PtrOf(uint64(100000)), + OnDisk: qdrant.PtrOf(true), // Store HNSW index on disk }, }, } @@ -288,7 +307,7 @@ func createCollection(ctx context.Context, client *qdrant.Client, collectionName QuantizationConfig: &qdrant.QuantizationConfig{ Quantization: &qdrant.QuantizationConfig_Binary{ Binary: &qdrant.BinaryQuantization{ - AlwaysRam: qdrant.PtrOf(true), // Keep quantized vectors in RAM + AlwaysRam: qdrant.PtrOf(false), // Allow quantized vectors on disk to reduce RAM }, }, }, @@ -329,6 +348,50 @@ func createCollection(ctx context.Context, client *qdrant.Client, collectionName } } +// enableProductionIndexing re-enables HNSW indexing after bulk import is complete. +// This should be called after all data has been imported to optimize for production queries. +func enableProductionIndexing(ctx context.Context, client *qdrant.Client, collectionName string) error { + log.Printf("Enabling production HNSW indexing for collection: %s", collectionName) + + // Update HNSW config for each named vector (dirs, names, contents) + for _, vectorName := range []string{"dirs", "names", "contents"} { + log.Printf(" Updating HNSW config for vector: %s", vectorName) + + err := client.UpdateCollection(ctx, &qdrant.UpdateCollection{ + CollectionName: collectionName, + HnswConfig: &qdrant.HnswConfigDiff{ + M: qdrant.PtrOf(uint64(48)), + }, + VectorsConfig: &qdrant.VectorsConfigDiff{ + Config: &qdrant.VectorsConfigDiff_Params{ + Params: &qdrant.VectorParamsDiff{ + HnswConfig: &qdrant.HnswConfigDiff{ + M: qdrant.PtrOf(uint64(48)), + OnDisk: qdrant.PtrOf(false), + }, + }, + }, + }, + OptimizersConfig: &qdrant.OptimizersConfigDiff{ + IndexingThreshold: qdrant.PtrOf(uint64(0)), + }, + QuantizationConfig: &qdrant.QuantizationConfigDiff{ + Quantization: &qdrant.QuantizationConfigDiff_Binary{ + Binary: &qdrant.BinaryQuantization{ + AlwaysRam: qdrant.PtrOf(true), + }, + }, + }, + }) + if err != nil { + return fmt.Errorf("failed to update HNSW for vector %s: %w", vectorName, err) + } + } + + log.Printf("āœ“ HNSW indexing enabled for %s. Optimizer will build indexes in background.", collectionName) + return nil +} + // Import data from a CSV file to separate collections. func importCSVFileWithProgress(ctx context.Context, client *qdrant.Client, filePath string, batchSize int, progress *progresstracker.ProgressTracker) (int, error) { file, err := os.Open(filePath) diff --git a/docker-compose.qdrant.yml b/docker-compose.qdrant.yml index 34f72df..0308fe0 100644 --- a/docker-compose.qdrant.yml +++ b/docker-compose.qdrant.yml @@ -7,6 +7,17 @@ services: - 6334:6334 # gRPC API port (used by our Go client) volumes: - ./qdrant_data:/qdrant/storage + environment: + # Optimize storage performance for large-scale imports + - QDRANT__STORAGE__OPTIMIZERS__MAX_SEGMENT_SIZE=500000 + - QDRANT__STORAGE__PERFORMANCE__MAX_OPTIMIZATION_THREADS=4 + # Enable WAL for durability during bulk imports + - QDRANT__STORAGE__WAL__WAL_CAPACITY_MB=32 + deploy: + resources: + limits: + cpus: "16" # Adjust based on your server (recommended: 8-16 for 100M vectors) + memory: 32G # Adjust based on your server (recommended: 16-32 GB for 100M vectors) expose: - 6333 - 6334 From 23cbabc6bef3d1d3edc0d0d407d5f30476f47c7e Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 22 Oct 2025 09:23:08 +0200 Subject: [PATCH 2/5] update docker compose --- docker-compose.qdrant.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docker-compose.qdrant.yml b/docker-compose.qdrant.yml index 0308fe0..7596230 100644 --- a/docker-compose.qdrant.yml +++ b/docker-compose.qdrant.yml @@ -9,15 +9,10 @@ services: - ./qdrant_data:/qdrant/storage environment: # Optimize storage performance for large-scale imports - - QDRANT__STORAGE__OPTIMIZERS__MAX_SEGMENT_SIZE=500000 + - QDRANT__STORAGE__OPTIMIZERS__OVERWRITE__MAX_SEGMENT_SIZE=500000 - QDRANT__STORAGE__PERFORMANCE__MAX_OPTIMIZATION_THREADS=4 # Enable WAL for durability during bulk imports - QDRANT__STORAGE__WAL__WAL_CAPACITY_MB=32 - deploy: - resources: - limits: - cpus: "16" # Adjust based on your server (recommended: 8-16 for 100M vectors) - memory: 32G # Adjust based on your server (recommended: 16-32 GB for 100M vectors) expose: - 6333 - 6334 From 83d69bf15d9e777bb25bf4efa60119cfb9a7c445 Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 22 Oct 2025 09:57:34 +0200 Subject: [PATCH 3/5] chore: add qdrant's docker compose to release artifacts --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6385da6..814d1e4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,3 +45,4 @@ jobs: target/scanoss-folder-hashing-import-linux-arm64 scanoss-folder-hashing-api_linux-amd64_${{ github.ref_name }}-1.tgz scanoss-folder-hashing-api_linux-arm64_${{ github.ref_name }}-1.tgz + docker-compose.qdrant.yml From 4110185e679544ff9fa2838aa7153a0a356305ae Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 22 Oct 2025 10:56:01 +0200 Subject: [PATCH 4/5] fix: qdrant import script named vectors config --- cmd/import/main.go | 55 ++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/cmd/import/main.go b/cmd/import/main.go index c953973..8bd8ed7 100644 --- a/cmd/import/main.go +++ b/cmd/import/main.go @@ -353,39 +353,42 @@ func createCollection(ctx context.Context, client *qdrant.Client, collectionName func enableProductionIndexing(ctx context.Context, client *qdrant.Client, collectionName string) error { log.Printf("Enabling production HNSW indexing for collection: %s", collectionName) - // Update HNSW config for each named vector (dirs, names, contents) + // Build named vectors config map + namedVectorsConfig := make(map[string]*qdrant.VectorParamsDiff) for _, vectorName := range []string{"dirs", "names", "contents"} { - log.Printf(" Updating HNSW config for vector: %s", vectorName) - - err := client.UpdateCollection(ctx, &qdrant.UpdateCollection{ - CollectionName: collectionName, + namedVectorsConfig[vectorName] = &qdrant.VectorParamsDiff{ HnswConfig: &qdrant.HnswConfigDiff{ - M: qdrant.PtrOf(uint64(48)), + M: qdrant.PtrOf(uint64(48)), + OnDisk: qdrant.PtrOf(false), }, - VectorsConfig: &qdrant.VectorsConfigDiff{ - Config: &qdrant.VectorsConfigDiff_Params{ - Params: &qdrant.VectorParamsDiff{ - HnswConfig: &qdrant.HnswConfigDiff{ - M: qdrant.PtrOf(uint64(48)), - OnDisk: qdrant.PtrOf(false), - }, - }, + OnDisk: qdrant.PtrOf(true), // Keep vectors on disk, only HNSW in RAM + } + } + + // Update all named vectors and collection settings in a single call + err := client.UpdateCollection(ctx, &qdrant.UpdateCollection{ + CollectionName: collectionName, + VectorsConfig: &qdrant.VectorsConfigDiff{ + Config: &qdrant.VectorsConfigDiff_ParamsMap{ + ParamsMap: &qdrant.VectorParamsDiffMap{ + Map: namedVectorsConfig, }, }, - OptimizersConfig: &qdrant.OptimizersConfigDiff{ - IndexingThreshold: qdrant.PtrOf(uint64(0)), - }, - QuantizationConfig: &qdrant.QuantizationConfigDiff{ - Quantization: &qdrant.QuantizationConfigDiff_Binary{ - Binary: &qdrant.BinaryQuantization{ - AlwaysRam: qdrant.PtrOf(true), - }, + }, + OptimizersConfig: &qdrant.OptimizersConfigDiff{ + IndexingThreshold: qdrant.PtrOf(uint64(0)), + }, + QuantizationConfig: &qdrant.QuantizationConfigDiff{ + Quantization: &qdrant.QuantizationConfigDiff_Binary{ + Binary: &qdrant.BinaryQuantization{ + AlwaysRam: qdrant.PtrOf(true), }, }, - }) - if err != nil { - return fmt.Errorf("failed to update HNSW for vector %s: %w", vectorName, err) - } + }, + }) + + if err != nil { + return fmt.Errorf("failed to update HNSW config: %w", err) } log.Printf("āœ“ HNSW indexing enabled for %s. Optimizer will build indexes in background.", collectionName) From 3e6d9d0e036af2ca08567c4abaa04c9aee79110d Mon Sep 17 00:00:00 2001 From: Matias Daloia Date: Wed, 22 Oct 2025 12:29:32 +0200 Subject: [PATCH 5/5] fix: lint error --- cmd/import/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/import/main.go b/cmd/import/main.go index 8bd8ed7..c474fa8 100644 --- a/cmd/import/main.go +++ b/cmd/import/main.go @@ -386,7 +386,6 @@ func enableProductionIndexing(ctx context.Context, client *qdrant.Client, collec }, }, }) - if err != nil { return fmt.Errorf("failed to update HNSW config: %w", err) }