Cherry-pick milvus-io#17891, upgrade to knowhere-v1.1.13

Signed-off-by: yudong.cai <yudong.cai@zilliz.com>
cydrain · Aug 20, 2022 · 64042ff · 64042ff
1 parent f5aef8f
commit 64042ff
Show file tree

Hide file tree

Showing 40 changed files with 165 additions and 286 deletions.
diff --git a/internal/core/src/common/CMakeLists.txt b/internal/core/src/common/CMakeLists.txt
@@ -11,7 +11,6 @@
 
 set(COMMON_SRC
         Schema.cpp
-        Types.cpp
         SystemProperty.cpp
         vector_index_c.cpp
         memory_c.cpp

diff --git a/internal/core/src/common/FieldMeta.h b/internal/core/src/common/FieldMeta.h
@@ -146,7 +146,8 @@ class FieldMeta {
         Assert(is_string());
     }
 
-    FieldMeta(const FieldName& name, FieldId id, DataType type, int64_t dim, std::optional<MetricType> metric_type)
+    FieldMeta(
+        const FieldName& name, FieldId id, DataType type, int64_t dim, std::optional<knowhere::MetricType> metric_type)
         : name_(name), id_(id), type_(type), vector_info_(VectorInfo{dim, metric_type}) {
         Assert(is_vector());
     }
@@ -177,7 +178,7 @@ class FieldMeta {
         return string_info_->max_length;
     }
 
-    std::optional<MetricType>
+    std::optional<knowhere::MetricType>
     get_metric_type() const {
         Assert(is_vector());
         Assert(vector_info_.has_value());
@@ -213,7 +214,7 @@ class FieldMeta {
  private:
     struct VectorInfo {
         int64_t dim_;
-        std::optional<MetricType> metric_type_;
+        std::optional<knowhere::MetricType> metric_type_;
     };
     struct StringInfo {
         int64_t max_length;

diff --git a/internal/core/src/common/Schema.cpp b/internal/core/src/common/Schema.cpp
@@ -65,7 +65,7 @@ Schema::ParseFrom(const milvus::proto::schema::CollectionSchema& schema_proto) {
             if (!index_map.count("metric_type")) {
                 schema->AddField(name, field_id, data_type, dim, std::nullopt);
             } else {
-                auto metric_type = GetMetricType(index_map.at("metric_type"));
+                auto metric_type = index_map.at("metric_type");
                 schema->AddField(name, field_id, data_type, dim, metric_type);
             }
         } else if (datatype_is_string(data_type)) {

diff --git a/internal/core/src/common/Schema.h b/internal/core/src/common/Schema.h
@@ -43,7 +43,10 @@ class Schema {
 
     // auto gen field_id for convenience
     FieldId
-    AddDebugField(const std::string& name, DataType data_type, int64_t dim, std::optional<MetricType> metric_type) {
+    AddDebugField(const std::string& name,
+                  DataType data_type,
+                  int64_t dim,
+                  std::optional<knowhere::MetricType> metric_type) {
         auto field_id = FieldId(debug_id);
         debug_id++;
         auto field_meta = FieldMeta(FieldName(name), field_id, data_type, dim, metric_type);
@@ -71,7 +74,7 @@ class Schema {
              const FieldId id,
              DataType data_type,
              int64_t dim,
-             std::optional<MetricType> metric_type) {
+             std::optional<knowhere::MetricType> metric_type) {
         auto field_meta = FieldMeta(name, id, data_type, dim, metric_type);
         this->AddField(std::move(field_meta));
     }

diff --git a/internal/core/src/common/Types.cpp b/internal/core/src/common/Types.cpp
diff --git a/internal/core/src/common/Types.h b/internal/core/src/common/Types.h
@@ -30,7 +30,7 @@
 #include <NamedType/named_type.hpp>
 #include <variant>
 
-#include "knowhere/common/MetricType.h"
+#include "knowhere/index/vector_index/helpers/IndexParameter.h"
 #include "pb/schema.pb.h"
 #include "pb/segcore.pb.h"
 #include "pb/plan.pb.h"
@@ -70,21 +70,16 @@ using ScalarArray = proto::schema::ScalarField;
 using DataArray = proto::schema::FieldData;
 using VectorArray = proto::schema::VectorField;
 using IdArray = proto::schema::IDs;
-using MetricType = faiss::MetricType;
 using InsertData = proto::segcore::InsertRecord;
 using PkType = std::variant<std::monostate, int64_t, std::string>;
 // tbb::concurrent_unordered_multimap equal_range too slow when multi repeated key
 // using Pk2OffsetType = tbb::concurrent_unordered_multimap<PkType, int64_t, std::hash<PkType>>;
 using Pk2OffsetType = std::unordered_map<PkType, std::vector<int64_t>, std::hash<PkType>>;
 
-MetricType
-GetMetricType(const std::string& type);
-
-std::string
-MetricTypeToName(MetricType metric_type);
-
-bool
-IsPrimaryKeyDataType(DataType data_type);
+inline bool
+IsPrimaryKeyDataType(DataType data_type) {
+    return data_type == DataType::INT64 || data_type == DataType::VARCHAR;
+}
 
 // NOTE: dependent type
 // used at meta-template programming

diff --git a/internal/core/src/indexbuilder/VecIndexCreator.cpp b/internal/core/src/indexbuilder/VecIndexCreator.cpp
@@ -139,7 +139,7 @@ VecIndexCreator::check_parameter(knowhere::Config& conf,
 
 template <typename T>
 std::optional<T>
-VecIndexCreator::get_config_by_name(std::string_view name) {
+VecIndexCreator::get_config_by_name(const std::string& name) {
     if (config_.contains(name)) {
         return knowhere::GetValueFromConfig<T>(config_, name);
     }

diff --git a/internal/core/src/indexbuilder/VecIndexCreator.h b/internal/core/src/indexbuilder/VecIndexCreator.h
@@ -74,7 +74,7 @@ class VecIndexCreator : public IndexCreatorBase {
 
     template <typename T>
     std::optional<T>
-    get_config_by_name(std::string_view name);
+    get_config_by_name(const std::string& name);
 
     void
     StoreRawData(const knowhere::DatasetPtr& dataset);

diff --git a/internal/core/src/query/Parser.cpp b/internal/core/src/query/Parser.cpp
@@ -200,7 +200,7 @@ Parser::ParseVecNode(const Json& out_body) {
         }
     }();
     vec_node->search_info_.topk_ = topk;
-    vec_node->search_info_.metric_type_ = GetMetricType(vec_info.at("metric_type"));
+    vec_node->search_info_.metric_type_ = vec_info.at("metric_type");
     vec_node->search_info_.search_params_ = vec_info.at("params");
     vec_node->search_info_.field_id_ = field_id;
     vec_node->search_info_.round_decimal_ = vec_info.at("round_decimal");

diff --git a/internal/core/src/query/PlanNode.h b/internal/core/src/query/PlanNode.h
@@ -38,7 +38,7 @@ struct SearchInfo {
     int64_t topk_;
     int64_t round_decimal_;
     FieldId field_id_;
-    MetricType metric_type_;
+    knowhere::MetricType metric_type_;
     knowhere::Config search_params_;
 };
 

diff --git a/internal/core/src/query/PlanProto.cpp b/internal/core/src/query/PlanProto.cpp
@@ -145,7 +145,7 @@ ProtoParser::PlanNodeFromProto(const planpb::PlanNode& plan_node_proto) {
     auto field_id = FieldId(anns_proto.field_id());
     search_info.field_id_ = field_id;
 
-    search_info.metric_type_ = GetMetricType(query_info_proto.metric_type());
+    search_info.metric_type_ = query_info_proto.metric_type();
     search_info.topk_ = query_info_proto.topk();
     search_info.round_decimal_ = query_info_proto.round_decimal();
     search_info.search_params_ = json::parse(query_info_proto.search_params());

diff --git a/internal/core/src/query/SearchBruteForce.cpp b/internal/core/src/query/SearchBruteForce.cpp
@@ -25,7 +25,7 @@ namespace milvus::query {
 // copy from faiss/IndexBinaryFlat.cpp::IndexBinaryFlat::search()
 // disable lint to make further migration easier
 static void
-binary_search(MetricType metric_type,
+binary_search(const knowhere::MetricType& metric_type,
               const uint8_t* xb,
               int64_t ntotal,
               int code_size,
@@ -36,28 +36,28 @@ binary_search(MetricType metric_type,
               idx_t* labels,
               const BitsetView bitset) {
     using namespace faiss;  // NOLINT
-    if (metric_type == METRIC_Jaccard || metric_type == METRIC_Tanimoto) {
+    if (metric_type == knowhere::metric::JACCARD || metric_type == knowhere::metric::TANIMOTO) {
         float_maxheap_array_t res = {size_t(n), size_t(k), labels, D};
         binary_distance_knn_hc(METRIC_Jaccard, &res, x, xb, ntotal, code_size, bitset);
 
-        if (metric_type == METRIC_Tanimoto) {
+        if (metric_type == knowhere::metric::TANIMOTO) {
             for (int i = 0; i < k * n; i++) {
                 D[i] = Jaccard_2_Tanimoto(D[i]);
             }
         }
-    } else if (metric_type == METRIC_Hamming) {
+    } else if (metric_type == knowhere::metric::HAMMING) {
         std::vector<int32_t> int_distances(n * k);
         int_maxheap_array_t res = {size_t(n), size_t(k), labels, int_distances.data()};
         binary_distance_knn_hc(METRIC_Hamming, &res, x, xb, ntotal, code_size, bitset);
         for (int i = 0; i < n * k; ++i) {
             D[i] = int_distances[i];
         }
-    } else if (metric_type == METRIC_Substructure || metric_type == METRIC_Superstructure) {
+    } else if (metric_type == knowhere::metric::SUBSTRUCTURE || metric_type == knowhere::metric::SUPERSTRUCTURE) {
         // only matched ids will be chosen, not to use heap
-        binary_distance_knn_mc(metric_type, x, xb, n, ntotal, k, code_size, D, labels, bitset);
+        auto faiss_metric_type = knowhere::GetFaissMetricType(metric_type);
+        binary_distance_knn_mc(faiss_metric_type, x, xb, n, ntotal, k, code_size, D, labels, bitset);
     } else {
-        std::string msg =
-            std::string("binary search not support metric type: ") + segcore::MetricTypeToString(metric_type);
+        std::string msg = "binary search not support metric type: " + metric_type;
         PanicInfo(msg);
     }
 }
@@ -97,16 +97,16 @@ FloatSearchBruteForce(const dataset::SearchDataset& dataset,
     SubSearchResult sub_qr(num_queries, topk, metric_type, round_decimal);
     auto query_data = reinterpret_cast<const float*>(dataset.query_data);
     auto chunk_data = reinterpret_cast<const float*>(chunk_data_raw);
-    if (metric_type == MetricType::METRIC_L2) {
+    if (metric_type == knowhere::metric::L2) {
         faiss::float_maxheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_seg_offsets(),
                                          sub_qr.get_distances()};
         faiss::knn_L2sqr(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, nullptr, bitset);
-    } else if (metric_type == MetricType::METRIC_INNER_PRODUCT) {
+    } else if (metric_type == knowhere::metric::IP) {
         faiss::float_minheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_seg_offsets(),
                                          sub_qr.get_distances()};
         faiss::knn_inner_product(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, bitset);
     } else {
-        std::string msg = "search not support metric type: " + MetricTypeToName(metric_type);
+        std::string msg = "search not support metric type: " + metric_type;
         PanicInfo(msg);
     }
     sub_qr.round_values();

diff --git a/internal/core/src/query/SearchOnSealed.cpp b/internal/core/src/query/SearchOnSealed.cpp
@@ -47,7 +47,7 @@ SearchOnSealed(const Schema& schema,
 
         auto conf = search_info.search_params_;
         knowhere::SetMetaTopk(conf, search_info.topk_);
-        knowhere::SetMetaMetricType(conf, MetricTypeToName(field_indexing->metric_type_));
+        knowhere::SetMetaMetricType(conf, field_indexing->metric_type_);
         auto index_type = field_indexing->indexing_->index_type();
         auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_type);
         try {

diff --git a/internal/core/src/query/SubSearchResult.h b/internal/core/src/query/SubSearchResult.h
@@ -19,7 +19,7 @@ namespace milvus::query {
 
 class SubSearchResult {
  public:
-    SubSearchResult(int64_t num_queries, int64_t topk, MetricType metric_type, int64_t round_decimal)
+    SubSearchResult(int64_t num_queries, int64_t topk, const knowhere::MetricType& metric_type, int64_t round_decimal)
         : metric_type_(metric_type),
           num_queries_(num_queries),
           topk_(topk),
@@ -29,15 +29,15 @@ class SubSearchResult {
     }
 
  public:
-    static constexpr float
-    init_value(MetricType metric_type) {
+    static float
+    init_value(const knowhere::MetricType& metric_type) {
         return (is_descending(metric_type) ? -1 : 1) * std::numeric_limits<float>::max();
     }
 
-    static constexpr bool
-    is_descending(MetricType metric_type) {
+    static bool
+    is_descending(const knowhere::MetricType& metric_type) {
         // TODO(dog): more types
-        if (metric_type == MetricType::METRIC_INNER_PRODUCT) {
+        if (metric_type == knowhere::metric::IP) {
             return true;
         } else {
             return false;
@@ -103,7 +103,7 @@ class SubSearchResult {
     int64_t num_queries_;
     int64_t topk_;
     int64_t round_decimal_;
-    MetricType metric_type_;
+    knowhere::MetricType metric_type_;
     std::vector<int64_t> seg_offsets_;
     std::vector<float> distances_;
 };

diff --git a/internal/core/src/query/helper.h b/internal/core/src/query/helper.h
@@ -21,7 +21,7 @@ namespace milvus::query {
 namespace dataset {
 
 struct SearchDataset {
-    MetricType metric_type;
+    knowhere::MetricType metric_type;
     int64_t num_queries;
     int64_t topk;
     int64_t round_decimal;

diff --git a/internal/core/src/query/visitors/ShowPlanNodeVisitor.cpp b/internal/core/src/query/visitors/ShowPlanNodeVisitor.cpp
@@ -53,12 +53,12 @@ ShowPlanNodeVisitor::visit(FloatVectorANNS& node) {
     assert(!ret_);
     auto& info = node.search_info_;
     Json json_body{
-        {"node_type", "FloatVectorANNS"},                      //
-        {"metric_type", MetricTypeToName(info.metric_type_)},  //
-        {"field_id_", info.field_id_.get()},                   //
-        {"topk", info.topk_},                                  //
-        {"search_params", info.search_params_},                //
-        {"placeholder_tag", node.placeholder_tag_},            //
+        {"node_type", "FloatVectorANNS"},            //
+        {"metric_type", info.metric_type_},          //
+        {"field_id_", info.field_id_.get()},         //
+        {"topk", info.topk_},                        //
+        {"search_params", info.search_params_},      //
+        {"placeholder_tag", node.placeholder_tag_},  //
     };
     if (node.predicate_.has_value()) {
         ShowExprVisitor expr_show;
@@ -75,12 +75,12 @@ ShowPlanNodeVisitor::visit(BinaryVectorANNS& node) {
     assert(!ret_);
     auto& info = node.search_info_;
     Json json_body{
-        {"node_type", "BinaryVectorANNS"},                     //
-        {"metric_type", MetricTypeToName(info.metric_type_)},  //
-        {"field_id_", info.field_id_.get()},                   //
-        {"topk", info.topk_},                                  //
-        {"search_params", info.search_params_},                //
-        {"placeholder_tag", node.placeholder_tag_},            //
+        {"node_type", "BinaryVectorANNS"},           //
+        {"metric_type", info.metric_type_},          //
+        {"field_id_", info.field_id_.get()},         //
+        {"topk", info.topk_},                        //
+        {"search_params", info.search_params_},      //
+        {"placeholder_tag", node.placeholder_tag_},  //
     };
     if (node.predicate_.has_value()) {
         ShowExprVisitor expr_show;