Merge pull request #517 from cococo2000/update-milvus

update milvus 2.4.x, add index hnsw, flat, ivfflat, ivfsq8, ivfpq, scann
erikbern · May 13, 2024 · cc3a748 · cc3a748
2 parents 1c3f702 + 09d63b8
commit cc3a748
Show file tree

Hide file tree

Showing 4 changed files with 349 additions and 41 deletions.
diff --git a/ann_benchmarks/algorithms/milvus/Dockerfile b/ann_benchmarks/algorithms/milvus/Dockerfile
@@ -1,11 +1,24 @@
-FROM milvusdb/knowhere-env:pyknowhere1.3-20230303
+FROM ann-benchmarks
 
-WORKDIR /home/app
-COPY run_algorithm.py requirements.txt ./
+# Add Docker's official GPG key:
+RUN apt-get update
+RUN apt-get install ca-certificates curl
+RUN install -m 0755 -d /etc/apt/keyrings
+RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+RUN chmod a+r /etc/apt/keyrings/docker.asc
 
-# update python packages
-RUN pip3 install -r requirements.txt
-RUN pip3 install scipy==1.10.1
-RUN python3 -c 'import pyknowhere'
+# Add the repository to Apt sources:
+RUN echo \
+    "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+    $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+    tee /etc/apt/sources.list.d/docker.list > /dev/null
 
-ENTRYPOINT ["python3", "-u", "run_algorithm.py"]
+RUN apt-get update 
+RUN apt-get install -y wget docker-ce-cli docker-compose-plugin
+
+RUN pip3 install pymilvus==2.4.1
+RUN python3 -c "from pymilvus import Collection"
+
+# Install milvus standalone by docker-compose.yml
+RUN wget https://github.com/milvus-io/milvus/releases/download/v2.4.1/milvus-standalone-docker-compose.yml -O docker-compose.yml
+RUN sed -i 's|${DOCKER_VOLUME_DIRECTORY:-.}|/tmp|g' docker-compose.yml
diff --git a/ann_benchmarks/algorithms/milvus/config.yml b/ann_benchmarks/algorithms/milvus/config.yml
@@ -1,15 +1,69 @@
 float:
   any:
-  - base_args: ['@metric', '@dimension']
-    constructor: Milvus
-    disabled: false
-    docker_tag: ann-benchmarks-milvus
-    module: ann_benchmarks.algorithms.milvus
-    name: Milvus(Knowhere)
-    run_groups:
-      HNSW:
-        args:
-          M: [4, 8, 12, 16, 24, 36, 48, 64, 96]
-          efConstruction: 500
-        query_args: [[10, 15, 20, 25, 30, 35, 40, 50, 60, 80, 120, 200, 400, 600,
-            800]]
+    - base_args: ["@metric", "@dimension"]
+      constructor: MilvusFLAT
+      disabled: false
+      docker_tag: ann-benchmarks-milvus
+      module: ann_benchmarks.algorithms.milvus
+      name: milvus-flat
+      run_groups:
+        FLAT:
+          args:
+            placeholder: [0]
+    - base_args: ["@metric", "@dimension"]
+      constructor: MilvusIVFFLAT
+      disabled: false
+      docker_tag: ann-benchmarks-milvus
+      module: ann_benchmarks.algorithms.milvus
+      name: milvus-ivfflat
+      run_groups:
+        IVFFLAT:
+          args:
+            nlist: [128, 256, 512, 1024, 2048, 4096]
+          query_args: [[1, 10, 20, 50, 100]]
+    - base_args: ["@metric", "@dimension"]
+      constructor: MilvusIVFSQ8
+      disabled: false
+      docker_tag: ann-benchmarks-milvus
+      module: ann_benchmarks.algorithms.milvus
+      name: milvus-ivfsq8
+      run_groups:
+        IVFSQ8:
+          args:
+            nlist: [128, 256, 512, 1024, 2048, 4096]
+          query_args: [[1, 10, 20, 50, 100]]
+    - base_args: ["@metric", "@dimension"]
+      constructor: MilvusIVFPQ
+      disabled: false
+      docker_tag: ann-benchmarks-milvus
+      module: ann_benchmarks.algorithms.milvus
+      name: milvus-ivfpq
+      run_groups:
+        IVFPQ:
+          args:
+            nlist: [128, 256, 512, 1024, 2048, 4096]
+            m: [2, 4]
+          query_args: [[1, 10, 20, 50, 100]]
+    - base_args: ["@metric", "@dimension"]
+      constructor: MilvusHNSW
+      disabled: false
+      docker_tag: ann-benchmarks-milvus
+      module: ann_benchmarks.algorithms.milvus
+      name: milvus-hnsw
+      run_groups:
+        HNSW:
+          args:
+            M: [4, 8, 12, 16, 24, 36, 48, 64, 96]
+            efConstruction: [200, 500]
+          query_args: [[10, 20, 40, 80, 120, 200, 400, 600, 800]]
+    - base_args: ["@metric", "@dimension"]
+      constructor: MilvusSCANN
+      disabled: false
+      docker_tag: ann-benchmarks-milvus
+      module: ann_benchmarks.algorithms.milvus
+      name: milvus-scann
+      run_groups:
+        SCANN:
+          args:
+            nlist: [64, 128, 256, 512, 1024, 2048, 4096, 8192]
+          query_args: [[1, 10, 20, 30, 50]]