enhance: Update knowhere commit (#34223)

Signed-off-by: Cai Yudong <yudong.cai@zilliz.com>
milvus-io · Jun 27, 2024 · ad90360 · ad90360
1 parent f2107ad
commit ad90360
Show file tree

Hide file tree

Showing 12 changed files with 32 additions and 40 deletions.
diff --git a/internal/core/src/common/QueryResult.h b/internal/core/src/common/QueryResult.h
@@ -93,7 +93,7 @@ struct VectorIterator {
         return !heap_.empty();
     }
     bool
-    AddIterator(std::shared_ptr<knowhere::IndexNode::iterator> iter) {
+    AddIterator(knowhere::IndexNode::IteratorPtr iter) {
         if (!sealed && iter != nullptr) {
             iterators_.emplace_back(iter);
             return true;
@@ -130,7 +130,7 @@ struct VectorIterator {
     }
 
  private:
-    std::vector<std::shared_ptr<knowhere::IndexNode::iterator>> iterators_;
+    std::vector<knowhere::IndexNode::IteratorPtr> iterators_;
     std::priority_queue<std::shared_ptr<OffsetDisPair>,
                         std::vector<std::shared_ptr<OffsetDisPair>>,
                         OffsetDisPairComparator>
@@ -161,8 +161,7 @@ struct SearchResult {
         int64_t nq,
         int chunk_count,
         int64_t rows_per_chunk,
-        const std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>&
-            kw_iterators) {
+        const std::vector<knowhere::IndexNode::IteratorPtr>& kw_iterators) {
         AssertInfo(kw_iterators.size() == nq * chunk_count,
                    "kw_iterators count:{} is not equal to nq*chunk_count:{}, "
                    "wrong state",

diff --git a/internal/core/src/index/VectorDiskIndex.cpp b/internal/core/src/index/VectorDiskIndex.cpp
@@ -375,7 +375,7 @@ VectorDiskAnnIndex<T>::Query(const DatasetPtr dataset,
                                       search_config[RANGE_FILTER],
                                       GetMetricType());
             }
-            auto res = index_.RangeSearch(*dataset, search_config, bitset);
+            auto res = index_.RangeSearch(dataset, search_config, bitset);
 
             if (!res.has_value()) {
                 PanicInfo(ErrorCode::UnexpectedError,
@@ -386,7 +386,7 @@ VectorDiskAnnIndex<T>::Query(const DatasetPtr dataset,
             return ReGenRangeSearchResult(
                 res.value(), topk, num_queries, GetMetricType());
         } else {
-            auto res = index_.Search(*dataset, search_config, bitset);
+            auto res = index_.Search(dataset, search_config, bitset);
             if (!res.has_value()) {
                 PanicInfo(ErrorCode::UnexpectedError,
                           fmt::format("failed to search: {}: {}",
@@ -419,11 +419,11 @@ VectorDiskAnnIndex<T>::Query(const DatasetPtr dataset,
 }
 
 template <typename T>
-knowhere::expected<std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>>
+knowhere::expected<std::vector<knowhere::IndexNode::IteratorPtr>>
 VectorDiskAnnIndex<T>::VectorIterators(const DatasetPtr dataset,
                                        const knowhere::Json& conf,
                                        const BitsetView& bitset) const {
-    return this->index_.AnnIterator(*dataset, conf, bitset);
+    return this->index_.AnnIterator(dataset, conf, bitset);
 }
 
 template <typename T>
@@ -440,7 +440,7 @@ VectorDiskAnnIndex<T>::GetVector(const DatasetPtr dataset) const {
         PanicInfo(ErrorCode::UnexpectedError,
                   "failed to get vector, index is sparse");
     }
-    auto res = index_.GetVectorByIds(*dataset);
+    auto res = index_.GetVectorByIds(dataset);
     if (!res.has_value()) {
         PanicInfo(ErrorCode::UnexpectedError,
                   fmt::format("failed to get vector: {}: {}",

diff --git a/internal/core/src/index/VectorDiskIndex.h b/internal/core/src/index/VectorDiskIndex.h
@@ -106,8 +106,7 @@ class VectorDiskAnnIndex : public VectorIndex {
 
     void CleanLocalData() override;
 
-    knowhere::expected<
-        std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>>
+    knowhere::expected<std::vector<knowhere::IndexNode::IteratorPtr>>
     VectorIterators(const DatasetPtr dataset,
                     const knowhere::Json& json,
                     const BitsetView& bitset) const override;

diff --git a/internal/core/src/index/VectorIndex.h b/internal/core/src/index/VectorIndex.h
@@ -60,8 +60,7 @@ class VectorIndex : public IndexBase {
           const BitsetView& bitset,
           SearchResult& search_result) const = 0;
 
-    virtual knowhere::expected<
-        std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>>
+    virtual knowhere::expected<std::vector<knowhere::IndexNode::IteratorPtr>>
     VectorIterators(const DatasetPtr dataset,
                     const knowhere::Json& json,
                     const BitsetView& bitset) const {

diff --git a/internal/core/src/index/VectorMemIndex.cpp b/internal/core/src/index/VectorMemIndex.cpp
@@ -147,11 +147,11 @@ VectorMemIndex<T>::UploadV2(const Config& config) {
 }
 
 template <typename T>
-knowhere::expected<std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>>
+knowhere::expected<std::vector<knowhere::IndexNode::IteratorPtr>>
 VectorMemIndex<T>::VectorIterators(const milvus::DatasetPtr dataset,
                                    const knowhere::Json& conf,
                                    const milvus::BitsetView& bitset) const {
-    return this->index_.AnnIterator(*dataset, conf, bitset);
+    return this->index_.AnnIterator(dataset, conf, bitset);
 }
 
 template <typename T>
@@ -434,7 +434,7 @@ VectorMemIndex<T>::BuildWithDataset(const DatasetPtr& dataset,
     SetDim(dataset->GetDim());
 
     knowhere::TimeRecorder rc("BuildWithoutIds", 1);
-    auto stat = index_.Build(*dataset, index_config);
+    auto stat = index_.Build(dataset, index_config);
     if (stat != knowhere::Status::success)
         PanicInfo(ErrorCode::IndexBuildError,
                   "failed to build index, " + KnowhereStatusString(stat));
@@ -572,7 +572,7 @@ VectorMemIndex<T>::AddWithDataset(const DatasetPtr& dataset,
     index_config.update(config);
 
     knowhere::TimeRecorder rc("AddWithDataset", 1);
-    auto stat = index_.Add(*dataset, index_config);
+    auto stat = index_.Add(dataset, index_config);
     if (stat != knowhere::Status::success)
         PanicInfo(ErrorCode::IndexBuildError,
                   "failed to append index, " + KnowhereStatusString(stat));
@@ -601,7 +601,7 @@ VectorMemIndex<T>::Query(const DatasetPtr dataset,
                                       GetMetricType());
             }
             milvus::tracer::AddEvent("start_knowhere_index_range_search");
-            auto res = index_.RangeSearch(*dataset, search_conf, bitset);
+            auto res = index_.RangeSearch(dataset, search_conf, bitset);
             milvus::tracer::AddEvent("finish_knowhere_index_range_search");
             if (!res.has_value()) {
                 PanicInfo(ErrorCode::UnexpectedError,
@@ -615,7 +615,7 @@ VectorMemIndex<T>::Query(const DatasetPtr dataset,
             return result;
         } else {
             milvus::tracer::AddEvent("start_knowhere_index_search");
-            auto res = index_.Search(*dataset, search_conf, bitset);
+            auto res = index_.Search(dataset, search_conf, bitset);
             milvus::tracer::AddEvent("finish_knowhere_index_search");
             if (!res.has_value()) {
                 PanicInfo(ErrorCode::UnexpectedError,
@@ -662,7 +662,7 @@ VectorMemIndex<T>::GetVector(const DatasetPtr dataset) const {
                   "failed to get vector, index is sparse");
     }
 
-    auto res = index_.GetVectorByIds(*dataset);
+    auto res = index_.GetVectorByIds(dataset);
     if (!res.has_value()) {
         PanicInfo(ErrorCode::UnexpectedError,
                   "failed to get vector, " + KnowhereStatusString(res.error()));
@@ -680,7 +680,7 @@ VectorMemIndex<T>::GetVector(const DatasetPtr dataset) const {
 template <typename T>
 std::unique_ptr<const knowhere::sparse::SparseRow<float>[]>
 VectorMemIndex<T>::GetSparseVector(const DatasetPtr dataset) const {
-    auto res = index_.GetVectorByIds(*dataset);
+    auto res = index_.GetVectorByIds(dataset);
     if (!res.has_value()) {
         PanicInfo(ErrorCode::UnexpectedError,
                   "failed to get vector, " + KnowhereStatusString(res.error()));

diff --git a/internal/core/src/index/VectorMemIndex.h b/internal/core/src/index/VectorMemIndex.h
@@ -94,8 +94,7 @@ class VectorMemIndex : public VectorIndex {
     BinarySet
     UploadV2(const Config& config = {}) override;
 
-    knowhere::expected<
-        std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>>
+    knowhere::expected<std::vector<knowhere::IndexNode::IteratorPtr>>
     VectorIterators(const DatasetPtr dataset,
                     const knowhere::Json& json,
                     const BitsetView& bitset) const override;

diff --git a/internal/core/src/query/GroupByOperator.h b/internal/core/src/query/GroupByOperator.h
@@ -133,8 +133,7 @@ PrepareVectorIteratorsFromIndex(const SearchInfo& search_info,
     if (search_info.group_by_field_id_.has_value()) {
         try {
             auto search_conf = search_info.search_params_;
-            knowhere::expected<
-                std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>>
+            knowhere::expected<std::vector<knowhere::IndexNode::IteratorPtr>>
                 iterators_val =
                     index.VectorIterators(dataset, search_conf, bitset);
             if (iterators_val.has_value()) {

diff --git a/internal/core/src/query/SearchBruteForce.cpp b/internal/core/src/query/SearchBruteForce.cpp
@@ -205,8 +205,7 @@ BruteForceSearchIterators(const dataset::SearchDataset& dataset,
     }
     auto search_cfg = PrepareBFSearchParams(search_info);
 
-    knowhere::expected<
-        std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>>
+    knowhere::expected<std::vector<knowhere::IndexNode::IteratorPtr>>
         iterators_val;
     switch (data_type) {
         case DataType::VECTOR_FLOAT:

diff --git a/internal/core/src/query/SubSearchResult.h b/internal/core/src/query/SubSearchResult.h
@@ -27,8 +27,7 @@ class SubSearchResult {
         int64_t topk,
         const MetricType& metric_type,
         int64_t round_decimal,
-        const std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>&
-            iters)
+        const std::vector<knowhere::IndexNode::IteratorPtr>& iters)
         : num_queries_(num_queries),
           topk_(topk),
           round_decimal_(round_decimal),
@@ -47,7 +46,7 @@ class SubSearchResult {
               topk,
               metric_type,
               round_decimal,
-              std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>{}) {
+              std::vector<knowhere::IndexNode::IteratorPtr>{}) {
     }
 
     SubSearchResult(SubSearchResult&& other) noexcept
@@ -114,7 +113,7 @@ class SubSearchResult {
     void
     merge(const SubSearchResult& other);
 
-    const std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>&
+    const std::vector<knowhere::IndexNode::IteratorPtr>&
     chunk_iterators() {
         return this->chunk_iterators_;
     }
@@ -131,7 +130,7 @@ class SubSearchResult {
     knowhere::MetricType metric_type_;
     std::vector<int64_t> seg_offsets_;
     std::vector<float> distances_;
-    std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>
+    std::vector<knowhere::IndexNode::IteratorPtr>
         chunk_iterators_;
 };
 

diff --git a/internal/core/thirdparty/knowhere/CMakeLists.txt b/internal/core/thirdparty/knowhere/CMakeLists.txt
@@ -13,7 +13,7 @@
 
 # Update KNOWHERE_VERSION for the first occurrence
 set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES "")
-set( KNOWHERE_VERSION d69e0f33 )
+set( KNOWHERE_VERSION 6bd3ba4c )
 set( GIT_REPOSITORY  "https://github.com/zilliztech/knowhere.git")
 message(STATUS "Knowhere repo: ${GIT_REPOSITORY}")
 message(STATUS "Knowhere version: ${KNOWHERE_VERSION}")

diff --git a/internal/core/unittest/test_c_api.cpp b/internal/core/unittest/test_c_api.cpp
@@ -1958,8 +1958,8 @@ TEST(CApiTest, LoadIndexInfo) {
                        {knowhere::indexparam::NPROBE, 4}};
 
     auto database = knowhere::GenDataSet(N, DIM, raw_data.data());
-    indexing.Train(*database, conf);
-    indexing.Add(*database, conf);
+    indexing.Train(database, conf);
+    indexing.Add(database, conf);
     EXPECT_EQ(indexing.Count(), N);
     EXPECT_EQ(indexing.Dim(), DIM);
     knowhere::BinarySet binary_set;
@@ -2009,8 +2009,8 @@ TEST(CApiTest, LoadIndexSearch) {
                        {knowhere::indexparam::NPROBE, 4}};
 
     auto database = knowhere::GenDataSet(N, DIM, raw_data.data());
-    indexing.Train(*database, conf);
-    indexing.Add(*database, conf);
+    indexing.Train(database, conf);
+    indexing.Add(database, conf);
 
     EXPECT_EQ(indexing.Count(), N);
     EXPECT_EQ(indexing.Dim(), DIM);
@@ -2033,7 +2033,7 @@ TEST(CApiTest, LoadIndexSearch) {
     auto query_dataset =
         knowhere::GenDataSet(num_query, DIM, raw_data.data() + BIAS * DIM);
 
-    auto result = indexing.Search(*query_dataset, conf, nullptr);
+    auto result = indexing.Search(query_dataset, conf, nullptr);
 }
 
 TEST(CApiTest, Indexing_Without_Predicate) {

diff --git a/internal/core/unittest/test_indexing.cpp b/internal/core/unittest/test_indexing.cpp
@@ -449,8 +449,7 @@ TEST(Indexing, Iterator) {
     searchInfo.search_params_ = search_conf;
     auto vec_index = dynamic_cast<index::VectorIndex*>(index.get());
 
-    knowhere::expected<
-        std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>>
+    knowhere::expected<std::vector<knowhere::IndexNode::IteratorPtr>>
         kw_iterators = vec_index->VectorIterators(
             query_ds, searchInfo.search_params_, view);
     ASSERT_TRUE(kw_iterators.has_value());