fix: Reduce duplicate PKs in segcore (milvus-io#34267) (milvus-io#34302)

issue: issue: milvus-io#34247 pr: milvus-io#34267 Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
bigsheeper · Jul 12, 2024 · 9f7f3ac · 9f7f3ac
1 parent 6c64c0d
commit 9f7f3ac
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 5 deletions.
diff --git a/internal/core/src/segcore/InsertRecord.h b/internal/core/src/segcore/InsertRecord.h
@@ -146,7 +146,10 @@ class OffsetOrderedMap : public OffsetMap {
         seg_offsets.reserve(limit);
         auto it = map_.begin();
         for (; hit_num < limit && it != map_.end(); it++) {
-            for (auto seg_offset : it->second) {
+            // Offsets in the growing segment are ordered by timestamp,
+            // so traverse from back to front to obtain the latest offset.
+            for (int i = it->second.size() - 1; i >= 0; --i) {
+                auto seg_offset = it->second[i];
                 if (seg_offset >= size) {
                     // Frequently concurrent insert/query will cause this case.
                     continue;
@@ -155,9 +158,8 @@ class OffsetOrderedMap : public OffsetMap {
                 if (!(bitset[seg_offset] ^ false_filtered_out)) {
                     seg_offsets.push_back(seg_offset);
                     hit_num++;
-                    if (hit_num >= limit) {
-                        break;
-                    }
+                    // PK hit, no need to continue traversing offsets with the same PK.
+                    break;
                 }
             }
         }

diff --git a/internal/core/unittest/test_c_api.cpp b/internal/core/unittest/test_c_api.cpp
@@ -847,7 +847,7 @@ TEST(CApiTest, DeleteRepeatedPksFromGrowingSegment) {
     auto suc = query_result->ParseFromArray(retrieve_result.proto_blob,
                                             retrieve_result.proto_size);
     ASSERT_TRUE(suc);
-    ASSERT_EQ(query_result->ids().int_id().data().size(), 6);
+    ASSERT_EQ(query_result->ids().int_id().data().size(), 3);
     DeleteRetrieveResult(&retrieve_result);
 
     // delete data pks = {1, 2, 3}