diff --git a/core/src/db/SSDBImpl.cpp b/core/src/db/SSDBImpl.cpp index 94db95de783a5..69086fd405a2c 100644 --- a/core/src/db/SSDBImpl.cpp +++ b/core/src/db/SSDBImpl.cpp @@ -234,8 +234,7 @@ SSDBImpl::DropIndex(const std::string& collection_name, const std::string& field } Status -SSDBImpl::PreloadCollection(const std::shared_ptr& context, const std::string& collection_name, - bool force) { +SSDBImpl::PreloadCollection(const server::ContextPtr& context, const std::string& collection_name, bool force) { CHECK_INITIALIZED; snapshot::ScopedSnapshotT ss; @@ -247,6 +246,26 @@ SSDBImpl::PreloadCollection(const std::shared_ptr& context, con return handler->GetStatus(); } +Status +SSDBImpl::GetEntityByID(const std::string& collection_name, const IDNumbers& id_array, + const std::vector& field_names, std::vector& vector_data, + std::vector& attr_type, std::vector& attr_data) { + CHECK_INITIALIZED; + + snapshot::ScopedSnapshotT ss; + STATUS_CHECK(snapshot::Snapshots::GetInstance().GetSnapshot(ss, collection_name)); + + auto handler = std::make_shared(nullptr, ss, id_array, field_names); + handler->Iterate(); + STATUS_CHECK(handler->GetStatus()); + + vector_data = std::move(handler->vector_data_); + attr_type = std::move(handler->attr_type_); + attr_data = std::move(handler->attr_data_); + + return Status::OK(); +} + //////////////////////////////////////////////////////////////////////////////// // Internal APIs //////////////////////////////////////////////////////////////////////////////// diff --git a/core/src/db/SSDBImpl.h b/core/src/db/SSDBImpl.h index f85dde7c7fcca..0883153f2acc8 100644 --- a/core/src/db/SSDBImpl.h +++ b/core/src/db/SSDBImpl.h @@ -61,8 +61,7 @@ class SSDBImpl { GetCollectionRowCount(const std::string& collection_name, uint64_t& row_count); Status - PreloadCollection(const std::shared_ptr& context, const std::string& collection_name, - bool force = false); + PreloadCollection(const server::ContextPtr& context, const std::string& collection_name, bool force = false); Status CreatePartition(const std::string& collection_name, const std::string& partition_name); @@ -76,6 +75,11 @@ class SSDBImpl { Status DropIndex(const std::string& collection_name, const std::string& field_name, const std::string& field_element_name); + Status + GetEntityByID(const std::string& collection_name, const IDNumbers& id_array, + const std::vector& field_names, std::vector& vector_data, + std::vector& attr_type, std::vector& attr_data); + private: void InternalFlush(const std::string& collection_id = ""); diff --git a/core/src/db/SnapshotHandlers.cpp b/core/src/db/SnapshotHandlers.cpp index f16476a4d0bd8..396ba9e37f24f 100644 --- a/core/src/db/SnapshotHandlers.cpp +++ b/core/src/db/SnapshotHandlers.cpp @@ -11,6 +11,13 @@ #include "db/SnapshotHandlers.h" #include "db/meta/MetaTypes.h" +#include "db/snapshot/ResourceHelper.h" +#include "db/snapshot/Snapshot.h" +#include "knowhere/index/vector_index/helpers/IndexParameter.h" +#include "segment/SegmentReader.h" + +#include +#include namespace milvus { namespace engine { @@ -62,6 +69,7 @@ LoadVectorFieldHandler::Handle(const snapshot::FieldPtr& field) { return status; } +/////////////////////////////////////////////////////////////////////////////// SegmentsToSearchCollector::SegmentsToSearchCollector(snapshot::ScopedSnapshotT ss, meta::FilesHolder& holder) : BaseT(ss), holder_(holder) { } @@ -93,5 +101,122 @@ SegmentsToSearchCollector::Handle(const snapshot::SegmentCommitPtr& segment_comm holder_.MarkFile(schema); } +/////////////////////////////////////////////////////////////////////////////// +GetEntityByIdSegmentHandler::GetEntityByIdSegmentHandler(const std::shared_ptr& context, + engine::snapshot::ScopedSnapshotT ss, const IDNumbers& ids, + const std::vector& field_names) + : BaseT(ss), context_(context), ids_(ids), field_names_(field_names), vector_data_(), attr_type_(), attr_data_() { +} + +Status +GetEntityByIdSegmentHandler::Handle(const snapshot::SegmentPtr& segment) { + LOG_ENGINE_DEBUG_ << "Get entity by id in segment " << segment->GetID(); + + // Load bloom filter + std::string segment_dir = snapshot::GetResPath(segment); + segment::SegmentReader segment_reader(segment_dir); + + segment::IdBloomFilterPtr id_bloom_filter_ptr; + STATUS_CHECK(segment_reader.LoadBloomFilter(id_bloom_filter_ptr)); + + // Load uids and check if the id is indeed present. If yes, find its offset. + std::vector uids; + STATUS_CHECK(segment_reader.LoadUids(uids)); + + // Check whether the id has been deleted + segment::DeletedDocsPtr deleted_docs_ptr; + STATUS_CHECK(segment_reader.LoadDeletedDocs(deleted_docs_ptr)); + auto& deleted_docs = deleted_docs_ptr->GetDeletedDocs(); + + for (auto id : ids_) { + AttrsData& attr_ref = attr_data_[id]; + VectorsData& vector_ref = vector_data_[id]; + + /* fast check using bloom filter */ + if (!id_bloom_filter_ptr->Check(id)) { + continue; + } + + /* check if id really exists in uids */ + auto found = std::find(uids.begin(), uids.end(), id); + if (found == uids.end()) { + continue; + } + + /* check if this id is deleted */ + auto offset = std::distance(uids.begin(), found); + auto deleted = std::find(deleted_docs.begin(), deleted_docs.end(), offset); + if (deleted != deleted_docs.end()) { + continue; + } + + std::unordered_map> raw_attrs; + for (auto& field_name : field_names_) { + auto field_ptr = ss_->GetField(field_name); + auto field_params = field_ptr->GetParams(); + auto dim = field_params[knowhere::meta::DIM].get(); + auto field_type = field_ptr->GetFtype(); + + if (field_ptr->GetFtype() == (int64_t)meta::hybrid::DataType::VECTOR_BINARY) { + size_t vector_size = dim / 8; + std::vector raw_vector; + STATUS_CHECK(segment_reader.LoadVectors(offset * vector_size, vector_size, raw_vector)); + + vector_ref.vector_count_ = 1; + vector_ref.binary_data_.swap(raw_vector); + } else if (field_ptr->GetFtype() == (int64_t)meta::hybrid::DataType::VECTOR_FLOAT) { + size_t vector_size = dim * sizeof(float); + std::vector raw_vector; + STATUS_CHECK(segment_reader.LoadVectors(offset * vector_size, vector_size, raw_vector)); + + vector_ref.vector_count_ = 1; + std::vector float_vector; + float_vector.resize(dim); + memcpy(float_vector.data(), raw_vector.data(), vector_size); + vector_ref.float_data_.swap(float_vector); + } else { + size_t num_bytes; + switch (field_type) { + case (int64_t)meta::hybrid::DataType::INT8: { + num_bytes = sizeof(int8_t); + break; + } + case (int64_t)meta::hybrid::DataType::INT16: { + num_bytes = sizeof(int16_t); + break; + } + case (int64_t)meta::hybrid::DataType::INT32: { + num_bytes = sizeof(int32_t); + break; + } + case (int64_t)meta::hybrid::DataType::INT64: { + num_bytes = sizeof(int64_t); + break; + } + case (int64_t)meta::hybrid::DataType::FLOAT: { + num_bytes = sizeof(float); + break; + } + case (int64_t)meta::hybrid::DataType::DOUBLE: { + num_bytes = sizeof(double); + break; + } + default: { + std::string msg = "Field type of " + field_name + " not supported"; + return Status(DB_ERROR, msg); + } + } + std::vector raw_attr; + STATUS_CHECK(segment_reader.LoadAttrs(field_name, offset * num_bytes, num_bytes, raw_attr)); + raw_attrs.insert(std::make_pair(field_name, raw_attr)); + } + } + + attr_ref.attr_count_ = 1; + attr_ref.attr_data_ = raw_attrs; + } + + return Status::OK(); +} } // namespace engine } // namespace milvus diff --git a/core/src/db/SnapshotHandlers.h b/core/src/db/SnapshotHandlers.h index c695954e5a200..13c4ac458e89d 100644 --- a/core/src/db/SnapshotHandlers.h +++ b/core/src/db/SnapshotHandlers.h @@ -11,12 +11,15 @@ #pragma once +#include "db/Types.h" #include "db/meta/FilesHolder.h" #include "db/snapshot/Snapshot.h" #include "server/context/Context.h" #include "utils/Log.h" #include +#include +#include namespace milvus { namespace engine { @@ -24,25 +27,25 @@ namespace engine { struct LoadVectorFieldElementHandler : public snapshot::IterateHandler { using ResourceT = snapshot::FieldElement; using BaseT = snapshot::IterateHandler; - LoadVectorFieldElementHandler(const std::shared_ptr& context, snapshot::ScopedSnapshotT ss, + LoadVectorFieldElementHandler(const server::ContextPtr& context, snapshot::ScopedSnapshotT ss, const snapshot::FieldPtr& field); Status Handle(const typename ResourceT::Ptr&) override; - const std::shared_ptr& context_; - const snapshot::FieldPtr& field_; + const server::ContextPtr context_; + const snapshot::FieldPtr field_; }; struct LoadVectorFieldHandler : public snapshot::IterateHandler { using ResourceT = snapshot::Field; using BaseT = snapshot::IterateHandler; - LoadVectorFieldHandler(const std::shared_ptr& context, snapshot::ScopedSnapshotT ss); + LoadVectorFieldHandler(const server::ContextPtr& context, snapshot::ScopedSnapshotT ss); Status Handle(const typename ResourceT::Ptr&) override; - const std::shared_ptr& context_; + const server::ContextPtr context_; }; struct SegmentsToSearchCollector : public snapshot::IterateHandler { @@ -56,5 +59,23 @@ struct SegmentsToSearchCollector : public snapshot::IterateHandler { + using ResourceT = snapshot::Segment; + using BaseT = snapshot::IterateHandler; + GetEntityByIdSegmentHandler(const server::ContextPtr& context, snapshot::ScopedSnapshotT ss, const IDNumbers& ids, + const std::vector& field_names); + + Status + Handle(const typename ResourceT::Ptr&) override; + + const server::ContextPtr context_; + const engine::IDNumbers ids_; + const std::vector field_names_; + std::vector vector_data_; + std::vector attr_type_; + std::vector attr_data_; +}; + } // namespace engine } // namespace milvus diff --git a/core/src/db/Types.h b/core/src/db/Types.h index 8ad2c4df7eb33..d2f66b6cf8bbf 100644 --- a/core/src/db/Types.h +++ b/core/src/db/Types.h @@ -33,6 +33,9 @@ typedef segment::doc_id_t IDNumber; typedef IDNumber* IDNumberPtr; typedef std::vector IDNumbers; +typedef faiss::Index::distance_t VectorDistance; +typedef std::vector VectorDistances; + typedef std::vector ResultIds; typedef std::vector ResultDistances; diff --git a/core/src/db/meta/MetaTypes.h b/core/src/db/meta/MetaTypes.h index e988769030722..b926c2d789fcb 100644 --- a/core/src/db/meta/MetaTypes.h +++ b/core/src/db/meta/MetaTypes.h @@ -102,20 +102,21 @@ using Table2FileRef = std::map; namespace hybrid { enum class DataType { - INT8 = 1, - INT16 = 2, - INT32 = 3, - INT64 = 4, + NONE = 0, + BOOL = 1, + INT8 = 2, + INT16 = 3, + INT32 = 4, + INT64 = 5, - STRING = 20, - - BOOL = 30, + FLOAT = 10, + DOUBLE = 11, - FLOAT = 40, - DOUBLE = 41, + STRING = 20, - VECTOR = 100, - UNKNOWN = 9999, + VECTOR_BINARY = 100, + VECTOR_FLOAT = 101, + VECTOR = 200, }; struct VectorFieldSchema { @@ -133,27 +134,10 @@ struct VectorFieldsSchema { using VectorFieldSchemaPtr = std::shared_ptr; struct FieldSchema { - typedef enum { - INT8 = 1, - INT16 = 2, - INT32 = 3, - INT64 = 4, - - STRING = 20, - - BOOL = 30, - - FLOAT = 40, - DOUBLE = 41, - - VECTOR = 100, - UNKNOWN = 9999, - } FIELD_TYPE; - // TODO(yukun): need field_id? std::string collection_id_; std::string field_name_; - int32_t field_type_ = (int)INT8; + int32_t field_type_; std::string field_params_; }; diff --git a/core/src/db/snapshot/Event.h b/core/src/db/snapshot/Event.h index b7396f75d651a..bf91faa486c5f 100644 --- a/core/src/db/snapshot/Event.h +++ b/core/src/db/snapshot/Event.h @@ -49,17 +49,14 @@ class ResourceGCEvent : public Event { STATUS_CHECK((*sd_op)(store)); /* TODO: physically clean resource */ - std::vector res_file_list; - STATUS_CHECK(GetResFiles(res_file_list, res_)); - for (auto& res_file : res_file_list) { - if (!boost::filesystem::exists(res_file)) { - continue; - } - if (boost::filesystem::is_directory(res_file)) { - boost::filesystem::remove_all(res_file); - } else { - boost::filesystem::remove(res_file); - } + std::string res_path = GetResPath(res_); + if (!boost::filesystem::exists(res_path)) { + return Status::OK(); + } + if (boost::filesystem::is_directory(res_path)) { + boost::filesystem::remove_all(res_path); + } else { + boost::filesystem::remove(res_path); } /* remove resource from meta */ diff --git a/core/src/db/snapshot/ResourceHelper.h b/core/src/db/snapshot/ResourceHelper.h index 995d48a222f6f..b515ee961821a 100644 --- a/core/src/db/snapshot/ResourceHelper.h +++ b/core/src/db/snapshot/ResourceHelper.h @@ -20,55 +20,51 @@ namespace milvus::engine::snapshot { template -inline Status -GetResFiles(std::vector& file_list, typename ResourceT::Ptr& res_ptr) { - return Status::OK(); +inline std::string +GetResPath(const typename ResourceT::Ptr& res_ptr) { + return std::string(); } template <> -inline Status -GetResFiles(std::vector& file_list, Collection::Ptr& res_ptr) { +inline std::string +GetResPath(const Collection::Ptr& res_ptr) { std::stringstream ss; ss << res_ptr->GetID(); - file_list.push_back(ss.str()); - return Status::OK(); + return ss.str(); } template <> -inline Status -GetResFiles(std::vector& file_list, Partition::Ptr& res_ptr) { +inline std::string +GetResPath(const Partition::Ptr& res_ptr) { std::stringstream ss; ss << res_ptr->GetCollectionId() << "/"; ss << res_ptr->GetID(); - file_list.push_back(ss.str()); - return Status::OK(); + return ss.str(); } template <> -inline Status -GetResFiles(std::vector& file_list, Segment::Ptr& res_ptr) { +inline std::string +GetResPath(const Segment::Ptr& res_ptr) { std::stringstream ss; ss << res_ptr->GetCollectionId() << "/"; ss << res_ptr->GetPartitionId() << "/"; ss << res_ptr->GetID(); - file_list.push_back(ss.str()); - return Status::OK(); + return ss.str(); } template <> -inline Status -GetResFiles(std::vector& file_list, SegmentFile::Ptr& res_ptr) { +inline std::string +GetResPath(const SegmentFile::Ptr& res_ptr) { std::stringstream ss; ss << res_ptr->GetCollectionId() << "/"; ss << res_ptr->GetPartitionId() << "/"; ss << res_ptr->GetSegmentId() << "/"; ss << res_ptr->GetID(); - file_list.push_back(ss.str()); - return Status::OK(); + return ss.str(); } } // namespace milvus::engine::snapshot diff --git a/core/src/db/snapshot/Resources.cpp b/core/src/db/snapshot/Resources.cpp index 0a24822d421ed..5867b63b0502c 100644 --- a/core/src/db/snapshot/Resources.cpp +++ b/core/src/db/snapshot/Resources.cpp @@ -16,7 +16,7 @@ namespace milvus::engine::snapshot { -Collection::Collection(const std::string& name, const std::string& params, ID_TYPE id, LSN_TYPE lsn, State state, +Collection::Collection(const std::string& name, const json& params, ID_TYPE id, LSN_TYPE lsn, State state, TS_TYPE created_on, TS_TYPE updated_on) : NameField(name), ParamsField(params), @@ -159,8 +159,8 @@ SchemaCommit::SchemaCommit(ID_TYPE collection_id, const MappingT& mappings, ID_T UpdatedOnField(updated_on) { } -Field::Field(const std::string& name, NUM_TYPE num, FTYPE_TYPE ftype, const std::string& params, ID_TYPE id, - LSN_TYPE lsn, State state, TS_TYPE created_on, TS_TYPE updated_on) +Field::Field(const std::string& name, NUM_TYPE num, FTYPE_TYPE ftype, const json& params, ID_TYPE id, LSN_TYPE lsn, + State state, TS_TYPE created_on, TS_TYPE updated_on) : NameField(name), NumField(num), FtypeField(ftype), @@ -185,7 +185,7 @@ FieldCommit::FieldCommit(ID_TYPE collection_id, ID_TYPE field_id, const MappingT } FieldElement::FieldElement(ID_TYPE collection_id, ID_TYPE field_id, const std::string& name, FTYPE_TYPE ftype, - const std::string& params, ID_TYPE id, LSN_TYPE lsn, State state, TS_TYPE created_on, + const json& params, ID_TYPE id, LSN_TYPE lsn, State state, TS_TYPE created_on, TS_TYPE updated_on) : CollectionIdField(collection_id), FieldIdField(field_id), diff --git a/core/src/db/snapshot/Resources.h b/core/src/db/snapshot/Resources.h index 9da7fc59cc1ba..9ef67a42e0ff6 100644 --- a/core/src/db/snapshot/Resources.h +++ b/core/src/db/snapshot/Resources.h @@ -29,7 +29,7 @@ using milvus::engine::utils::GetMicroSecTimeStamp; namespace milvus::engine::snapshot { -static constexpr const char* JEmpty = "{}"; +static const json JEmpty = {}; class MappingsField { public: @@ -296,22 +296,16 @@ class NameField { class ParamsField { public: - explicit ParamsField(std::string params) : params_(std::move(params)), json_params_(json::parse(params_)) { + explicit ParamsField(const json& params) : params_(params) { } - const std::string& + const json& GetParams() const { return params_; } - const json& - GetParamsJson() const { - return json_params_; - } - protected: - std::string params_; - json json_params_; + json params_; }; class SizeField { @@ -364,7 +358,7 @@ class Collection : public BaseResource, using VecT = std::vector; static constexpr const char* Name = "Collection"; - explicit Collection(const std::string& name, const std::string& params = JEmpty, ID_TYPE id = 0, LSN_TYPE lsn = 0, + explicit Collection(const std::string& name, const json& params = JEmpty, ID_TYPE id = 0, LSN_TYPE lsn = 0, State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(), TS_TYPE UpdatedOnField = GetMicroSecTimeStamp()); }; @@ -574,7 +568,7 @@ class Field : public BaseResource, using VecT = std::vector; static constexpr const char* Name = "Field"; - Field(const std::string& name, NUM_TYPE num, FTYPE_TYPE ftype, const std::string& params = JEmpty, ID_TYPE id = 0, + Field(const std::string& name, NUM_TYPE num, FTYPE_TYPE ftype, const json& params = JEmpty, ID_TYPE id = 0, LSN_TYPE lsn = 0, State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(), TS_TYPE UpdatedOnField = GetMicroSecTimeStamp()); }; @@ -624,7 +618,7 @@ class FieldElement : public BaseResource, using VecT = std::vector; static constexpr const char* Name = "FieldElement"; FieldElement(ID_TYPE collection_id, ID_TYPE field_id, const std::string& name, FTYPE_TYPE ftype, - const std::string& params = JEmpty, ID_TYPE id = 0, LSN_TYPE lsn = 0, State status = PENDING, + const json& params = JEmpty, ID_TYPE id = 0, LSN_TYPE lsn = 0, State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(), TS_TYPE UpdatedOnField = GetMicroSecTimeStamp()); }; diff --git a/core/src/db/snapshot/Snapshot.cpp b/core/src/db/snapshot/Snapshot.cpp index f7571a6ba71c3..c33c5907b2c77 100644 --- a/core/src/db/snapshot/Snapshot.cpp +++ b/core/src/db/snapshot/Snapshot.cpp @@ -29,91 +29,99 @@ Snapshot::UnRefAll() { std::apply([this](auto&... resource) { ((DoUnRef(resource)), ...); }, resources_); } -Snapshot::Snapshot(ID_TYPE id) { - auto collection_commit = CollectionCommitsHolder::GetInstance().GetResource(id, false); - AddResource(collection_commit); - max_lsn_ = collection_commit->GetLsn(); - auto& schema_holder = SchemaCommitsHolder::GetInstance(); - auto current_schema = schema_holder.GetResource(collection_commit->GetSchemaId(), false); - AddResource(current_schema); - current_schema_id_ = current_schema->GetID(); +Snapshot::Snapshot(ID_TYPE ss_id) { + auto& collection_commits_holder = CollectionCommitsHolder::GetInstance(); + auto& collections_holder = CollectionsHolder::GetInstance(); + auto& schema_commits_holder = SchemaCommitsHolder::GetInstance(); auto& field_commits_holder = FieldCommitsHolder::GetInstance(); auto& fields_holder = FieldsHolder::GetInstance(); auto& field_elements_holder = FieldElementsHolder::GetInstance(); - - auto collection = CollectionsHolder::GetInstance().GetResource(collection_commit->GetCollectionId(), false); - AddResource(collection); - auto& mappings = collection_commit->GetMappings(); auto& partition_commits_holder = PartitionCommitsHolder::GetInstance(); auto& partitions_holder = PartitionsHolder::GetInstance(); - auto& segments_holder = SegmentsHolder::GetInstance(); auto& segment_commits_holder = SegmentCommitsHolder::GetInstance(); + auto& segments_holder = SegmentsHolder::GetInstance(); auto& segment_files_holder = SegmentFilesHolder::GetInstance(); - auto ssid = id; - for (auto& id : mappings) { - auto partition_commit = partition_commits_holder.GetResource(id, false); - auto partition = partitions_holder.GetResource(partition_commit->GetPartitionId(), false); + auto collection_commit = collection_commits_holder.GetResource(ss_id, false); + AddResource(collection_commit); + + max_lsn_ = collection_commit->GetLsn(); + auto schema_commit = schema_commits_holder.GetResource(collection_commit->GetSchemaId(), false); + AddResource(schema_commit); + + current_schema_id_ = schema_commit->GetID(); + auto collection = collections_holder.GetResource(collection_commit->GetCollectionId(), false); + AddResource(collection); + + auto& collection_commit_mappings = collection_commit->GetMappings(); + for (auto p_c_id : collection_commit_mappings) { + auto partition_commit = partition_commits_holder.GetResource(p_c_id, false); + auto partition_id = partition_commit->GetPartitionId(); + auto partition = partitions_holder.GetResource(partition_id, false); + auto partition_name = partition->GetName(); AddResource(partition_commit); - p_pc_map_[partition_commit->GetPartitionId()] = partition_commit->GetID(); + + p_pc_map_[partition_id] = partition_commit->GetID(); AddResource(partition); - partition_names_map_[partition->GetName()] = partition->GetID(); - p_max_seg_num_[partition->GetID()] = 0; - auto& s_c_mappings = partition_commit->GetMappings(); - /* std::cout << "SS-" << ssid << "PC_MAP=("; */ + partition_names_map_[partition_name] = partition_id; + p_max_seg_num_[partition_id] = 0; + /* std::cout << "SS-" << ss_id << "PC_MAP=("; */ /* for (auto id : s_c_mappings) { */ /* std::cout << id << ","; */ /* } */ /* std::cout << ")" << std::endl; */ - for (auto& s_c_id : s_c_mappings) { + auto& partition_commit_mappings = partition_commit->GetMappings(); + for (auto s_c_id : partition_commit_mappings) { auto segment_commit = segment_commits_holder.GetResource(s_c_id, false); - auto segment = segments_holder.GetResource(segment_commit->GetSegmentId(), false); - auto schema = schema_holder.GetResource(segment_commit->GetSchemaId(), false); - AddResource(schema); + auto segment_id = segment_commit->GetSegmentId(); + auto segment = segments_holder.GetResource(segment_id, false); + auto segment_schema_id = segment_commit->GetSchemaId(); + auto segment_schema = schema_commits_holder.GetResource(segment_schema_id, false); + auto segment_partition_id = segment->GetPartitionId(); + AddResource(segment_schema); AddResource(segment_commit); - if (segment->GetNum() > p_max_seg_num_[segment->GetPartitionId()]) { - p_max_seg_num_[segment->GetPartitionId()] = segment->GetNum(); + if (segment->GetNum() > p_max_seg_num_[segment_partition_id]) { + p_max_seg_num_[segment_partition_id] = segment->GetNum(); } AddResource(segment); - seg_segc_map_[segment->GetID()] = segment_commit->GetID(); - auto& s_f_mappings = segment_commit->GetMappings(); - for (auto& s_f_id : s_f_mappings) { + + seg_segc_map_[segment_id] = segment_commit->GetID(); + auto& segment_commit_mappings = segment_commit->GetMappings(); + for (auto s_f_id : segment_commit_mappings) { auto segment_file = segment_files_holder.GetResource(s_f_id, false); - auto field_element = field_elements_holder.GetResource(segment_file->GetFieldElementId(), false); + auto segment_file_id = segment_file->GetID(); + auto field_element_id = segment_file->GetFieldElementId(); + auto field_element = field_elements_holder.GetResource(field_element_id, false); AddResource(field_element); AddResource(segment_file); - auto entry = element_segfiles_map_.find(segment_file->GetFieldElementId()); - if (entry == element_segfiles_map_.end()) { - element_segfiles_map_[segment_file->GetFieldElementId()] = { - {segment_file->GetSegmentId(), segment_file->GetID()}}; - } else { - entry->second[segment_file->GetSegmentId()] = segment_file->GetID(); - } + element_segfiles_map_[field_element_id][segment_id] = segment_file_id; } } } - for (auto& kv : GetResources()) { - if (kv.first > latest_schema_commit_id_) + auto& schema_commit_mappings = schema_commit->GetMappings(); + auto& schema_commits = GetResources(); + for (auto& kv : schema_commits) { + if (kv.first > latest_schema_commit_id_) { latest_schema_commit_id_ = kv.first; + } auto& schema_commit = kv.second; - auto& s_c_m = current_schema->GetMappings(); - for (auto field_commit_id : s_c_m) { + for (auto field_commit_id : schema_commit_mappings) { auto field_commit = field_commits_holder.GetResource(field_commit_id, false); AddResource(field_commit); - auto field = fields_holder.GetResource(field_commit->GetFieldId(), false); + + auto field_id = field_commit->GetFieldId(); + auto field = fields_holder.GetResource(field_id, false); + auto field_name = field->GetName(); AddResource(field); - field_names_map_[field->GetName()] = field->GetID(); - auto& f_c_m = field_commit->GetMappings(); - for (auto field_element_id : f_c_m) { + + field_names_map_[field_name] = field_id; + auto& field_commit_mappings = field_commit->GetMappings(); + for (auto field_element_id : field_commit_mappings) { auto field_element = field_elements_holder.GetResource(field_element_id, false); AddResource(field_element); - auto entry = field_element_names_map_.find(field->GetName()); - if (entry == field_element_names_map_.end()) { - field_element_names_map_[field->GetName()] = {{field_element->GetName(), field_element->GetID()}}; - } else { - entry->second[field_element->GetName()] = field_element->GetID(); - } + auto field_element_name = field_element->GetName(); + field_element_names_map_[field_name][field_element_name] = field_element_id; } } } diff --git a/core/src/db/snapshot/Snapshot.h b/core/src/db/snapshot/Snapshot.h index 7adcdbe0ce834..0866f51e16801 100644 --- a/core/src/db/snapshot/Snapshot.h +++ b/core/src/db/snapshot/Snapshot.h @@ -50,7 +50,7 @@ class Snapshot : public ReferenceProxy { return GetCollectionCommit()->GetID(); } - [[nodiscard]] ID_TYPE + ID_TYPE GetCollectionId() const { auto it = GetResources().cbegin(); return it->first; @@ -67,17 +67,17 @@ class Snapshot : public ReferenceProxy { return GetResource(id); } - [[nodiscard]] const std::string& + const std::string& GetName() const { return GetResources().cbegin()->second->GetName(); } - [[nodiscard]] size_t + size_t NumberOfPartitions() const { return GetResources().size(); } - [[nodiscard]] const LSN_TYPE& + const LSN_TYPE& GetMaxLsn() const { return max_lsn_; } @@ -107,7 +107,7 @@ class Snapshot : public ReferenceProxy { return GetResources().cbegin()->second.Get(); } - [[nodiscard]] ID_TYPE + ID_TYPE GetLatestSchemaCommitId() const { return latest_schema_commit_id_; } @@ -167,7 +167,7 @@ class Snapshot : public ReferenceProxy { handler->SetStatus(status); } - [[nodiscard]] std::vector + std::vector GetFieldNames() const { std::vector names; for (auto& kv : field_names_map_) { @@ -176,7 +176,7 @@ class Snapshot : public ReferenceProxy { return std::move(names); } - [[nodiscard]] bool + bool HasField(const std::string& name) const { auto it = field_names_map_.find(name); return it != field_names_map_.end(); @@ -185,13 +185,13 @@ class Snapshot : public ReferenceProxy { FieldPtr GetField(const std::string& name) const; - [[nodiscard]] bool + bool HasFieldElement(const std::string& field_name, const std::string& field_element_name) const { auto id = GetFieldElementId(field_name, field_element_name); return id > 0; } - [[nodiscard]] ID_TYPE + ID_TYPE GetSegmentFileId(const std::string& field_name, const std::string& field_element_name, ID_TYPE segment_id) const { auto field_element_id = GetFieldElementId(field_name, field_element_name); auto it = element_segfiles_map_.find(field_element_id); @@ -205,13 +205,13 @@ class Snapshot : public ReferenceProxy { return its->second; } - [[nodiscard]] bool + bool HasSegmentFile(const std::string& field_name, const std::string& field_element_name, ID_TYPE segment_id) const { auto id = GetSegmentFileId(field_name, field_element_name, segment_id); return id > 0; } - [[nodiscard]] ID_TYPE + ID_TYPE GetFieldElementId(const std::string& field_name, const std::string& field_element_name) const { auto itf = field_element_names_map_.find(field_name); if (itf == field_element_names_map_.end()) @@ -287,7 +287,7 @@ class Snapshot : public ReferenceProxy { } template - [[nodiscard]] const typename ResourceT::ScopedMapT& + const typename ResourceT::ScopedMapT& GetResources() const { return std::get::value>(resources_); } @@ -300,7 +300,6 @@ class Snapshot : public ReferenceProxy { if (it == resources.end()) { return nullptr; } - return it->second.Get(); } diff --git a/core/src/grpc/gen-milvus/milvus.pb.cc b/core/src/grpc/gen-milvus/milvus.pb.cc index 9503eedfe588a..ad2262c1d907d 100644 --- a/core/src/grpc/gen-milvus/milvus.pb.cc +++ b/core/src/grpc/gen-milvus/milvus.pb.cc @@ -1673,92 +1673,93 @@ const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE( "y\030\002 \003(\003\"\221\001\n\013HIndexParam\022#\n\006status\030\001 \001(\0132" "\023.milvus.grpc.Status\022\027\n\017collection_name\030" "\002 \001(\t\022\023\n\013field_names\030\003 \003(\t\022/\n\014extra_para" - "ms\030\004 \003(\0132\031.milvus.grpc.KeyValuePair*\206\001\n\010" - "DataType\022\010\n\004NULL\020\000\022\010\n\004INT8\020\001\022\t\n\005INT16\020\002\022" - "\t\n\005INT32\020\003\022\t\n\005INT64\020\004\022\n\n\006STRING\020\024\022\010\n\004BOO" - "L\020\036\022\t\n\005FLOAT\020(\022\n\n\006DOUBLE\020)\022\n\n\006VECTOR\020d\022\014" - "\n\007UNKNOWN\020\217N*C\n\017CompareOperator\022\006\n\002LT\020\000\022" - "\007\n\003LTE\020\001\022\006\n\002EQ\020\002\022\006\n\002GT\020\003\022\007\n\003GTE\020\004\022\006\n\002NE\020" - "\005*8\n\005Occur\022\013\n\007INVALID\020\000\022\010\n\004MUST\020\001\022\n\n\006SHO" - "ULD\020\002\022\014\n\010MUST_NOT\020\0032\256\030\n\rMilvusService\022H\n" - "\020CreateCollection\022\035.milvus.grpc.Collecti" - "onSchema\032\023.milvus.grpc.Status\"\000\022F\n\rHasCo" - "llection\022\033.milvus.grpc.CollectionName\032\026." - "milvus.grpc.BoolReply\"\000\022R\n\022DescribeColle" - "ction\022\033.milvus.grpc.CollectionName\032\035.mil" - "vus.grpc.CollectionSchema\"\000\022Q\n\017CountColl" - "ection\022\033.milvus.grpc.CollectionName\032\037.mi" - "lvus.grpc.CollectionRowCount\"\000\022J\n\017ShowCo" - "llections\022\024.milvus.grpc.Command\032\037.milvus" - ".grpc.CollectionNameList\"\000\022P\n\022ShowCollec" - "tionInfo\022\033.milvus.grpc.CollectionName\032\033." - "milvus.grpc.CollectionInfo\"\000\022D\n\016DropColl" - "ection\022\033.milvus.grpc.CollectionName\032\023.mi" - "lvus.grpc.Status\"\000\022=\n\013CreateIndex\022\027.milv" - "us.grpc.IndexParam\032\023.milvus.grpc.Status\"" - "\000\022G\n\rDescribeIndex\022\033.milvus.grpc.Collect" - "ionName\032\027.milvus.grpc.IndexParam\"\000\022\?\n\tDr" - "opIndex\022\033.milvus.grpc.CollectionName\032\023.m" - "ilvus.grpc.Status\"\000\022E\n\017CreatePartition\022\033" - ".milvus.grpc.PartitionParam\032\023.milvus.grp" - "c.Status\"\000\022E\n\014HasPartition\022\033.milvus.grpc" - ".PartitionParam\032\026.milvus.grpc.BoolReply\"" - "\000\022K\n\016ShowPartitions\022\033.milvus.grpc.Collec" - "tionName\032\032.milvus.grpc.PartitionList\"\000\022C" - "\n\rDropPartition\022\033.milvus.grpc.PartitionP" - "aram\032\023.milvus.grpc.Status\"\000\022<\n\006Insert\022\030." - "milvus.grpc.InsertParam\032\026.milvus.grpc.Ve" - "ctorIds\"\000\022J\n\016GetVectorsByID\022\034.milvus.grp" - "c.VectorsIdentity\032\030.milvus.grpc.VectorsD" - "ata\"\000\022H\n\014GetVectorIDs\022\036.milvus.grpc.GetV" - "ectorIDsParam\032\026.milvus.grpc.VectorIds\"\000\022" - "B\n\006Search\022\030.milvus.grpc.SearchParam\032\034.mi" - "lvus.grpc.TopKQueryResult\"\000\022J\n\nSearchByI" - "D\022\034.milvus.grpc.SearchByIDParam\032\034.milvus" - ".grpc.TopKQueryResult\"\000\022P\n\rSearchInFiles" - "\022\037.milvus.grpc.SearchInFilesParam\032\034.milv" - "us.grpc.TopKQueryResult\"\000\0227\n\003Cmd\022\024.milvu" - "s.grpc.Command\032\030.milvus.grpc.StringReply" - "\"\000\022A\n\nDeleteByID\022\034.milvus.grpc.DeleteByI" - "DParam\032\023.milvus.grpc.Status\"\000\022G\n\021Preload" - "Collection\022\033.milvus.grpc.CollectionName\032" - "\023.milvus.grpc.Status\"\000\022I\n\016ReloadSegments" - "\022 .milvus.grpc.ReLoadSegmentsParam\032\023.mil" - "vus.grpc.Status\"\000\0227\n\005Flush\022\027.milvus.grpc" - ".FlushParam\032\023.milvus.grpc.Status\"\000\022=\n\007Co" - "mpact\022\033.milvus.grpc.CollectionName\032\023.mil" - "vus.grpc.Status\"\000\022E\n\026CreateHybridCollect" - "ion\022\024.milvus.grpc.Mapping\032\023.milvus.grpc." - "Status\"\000\022L\n\023HasHybridCollection\022\033.milvus" - ".grpc.CollectionName\032\026.milvus.grpc.BoolR" - "eply\"\000\022J\n\024DropHybridCollection\022\033.milvus." + "ms\030\004 \003(\0132\031.milvus.grpc.KeyValuePair*\236\001\n\010" + "DataType\022\010\n\004NONE\020\000\022\010\n\004BOOL\020\001\022\010\n\004INT8\020\002\022\t" + "\n\005INT16\020\003\022\t\n\005INT32\020\004\022\t\n\005INT64\020\005\022\t\n\005FLOAT" + "\020\n\022\n\n\006DOUBLE\020\013\022\n\n\006STRING\020\024\022\021\n\rVECTOR_BIN" + "ARY\020d\022\020\n\014VECTOR_FLOAT\020e\022\013\n\006VECTOR\020\310\001*C\n\017" + "CompareOperator\022\006\n\002LT\020\000\022\007\n\003LTE\020\001\022\006\n\002EQ\020\002" + "\022\006\n\002GT\020\003\022\007\n\003GTE\020\004\022\006\n\002NE\020\005*8\n\005Occur\022\013\n\007IN" + "VALID\020\000\022\010\n\004MUST\020\001\022\n\n\006SHOULD\020\002\022\014\n\010MUST_NO" + "T\020\0032\256\030\n\rMilvusService\022H\n\020CreateCollectio" + "n\022\035.milvus.grpc.CollectionSchema\032\023.milvu" + "s.grpc.Status\"\000\022F\n\rHasCollection\022\033.milvu" + "s.grpc.CollectionName\032\026.milvus.grpc.Bool" + "Reply\"\000\022R\n\022DescribeCollection\022\033.milvus.g" + "rpc.CollectionName\032\035.milvus.grpc.Collect" + "ionSchema\"\000\022Q\n\017CountCollection\022\033.milvus." + "grpc.CollectionName\032\037.milvus.grpc.Collec" + "tionRowCount\"\000\022J\n\017ShowCollections\022\024.milv" + "us.grpc.Command\032\037.milvus.grpc.Collection" + "NameList\"\000\022P\n\022ShowCollectionInfo\022\033.milvu" + "s.grpc.CollectionName\032\033.milvus.grpc.Coll" + "ectionInfo\"\000\022D\n\016DropCollection\022\033.milvus." "grpc.CollectionName\032\023.milvus.grpc.Status" - "\"\000\022O\n\030DescribeHybridCollection\022\033.milvus." - "grpc.CollectionName\032\024.milvus.grpc.Mappin" - "g\"\000\022W\n\025CountHybridCollection\022\033.milvus.gr" - "pc.CollectionName\032\037.milvus.grpc.Collecti" - "onRowCount\"\000\022I\n\025ShowHybridCollections\022\024." - "milvus.grpc.Command\032\030.milvus.grpc.Mappin" - "gList\"\000\022V\n\030ShowHybridCollectionInfo\022\033.mi" - "lvus.grpc.CollectionName\032\033.milvus.grpc.C" - "ollectionInfo\"\000\022M\n\027PreloadHybridCollecti" - "on\022\033.milvus.grpc.CollectionName\032\023.milvus" - ".grpc.Status\"\000\022D\n\021CreateHybridIndex\022\030.mi" - "lvus.grpc.HIndexParam\032\023.milvus.grpc.Stat" - "us\"\000\022D\n\014InsertEntity\022\031.milvus.grpc.HInse" - "rtParam\032\027.milvus.grpc.HEntityIDs\"\000\022J\n\016Hy" - "bridSearchPB\022\033.milvus.grpc.HSearchParamP" - "B\032\031.milvus.grpc.HQueryResult\"\000\022F\n\014Hybrid" - "Search\022\031.milvus.grpc.HSearchParam\032\031.milv" - "us.grpc.HQueryResult\"\000\022]\n\026HybridSearchIn" - "Segments\022#.milvus.grpc.HSearchInSegments" - "Param\032\034.milvus.grpc.TopKQueryResult\"\000\022E\n" - "\rGetEntityByID\022\034.milvus.grpc.VectorsIden" - "tity\032\024.milvus.grpc.HEntity\"\000\022J\n\014GetEntit" - "yIDs\022\037.milvus.grpc.HGetEntityIDsParam\032\027." - "milvus.grpc.HEntityIDs\"\000\022J\n\022DeleteEntiti" - "esByID\022\035.milvus.grpc.HDeleteByIDParam\032\023." - "milvus.grpc.Status\"\000b\006proto3" + "\"\000\022=\n\013CreateIndex\022\027.milvus.grpc.IndexPar" + "am\032\023.milvus.grpc.Status\"\000\022G\n\rDescribeInd" + "ex\022\033.milvus.grpc.CollectionName\032\027.milvus" + ".grpc.IndexParam\"\000\022\?\n\tDropIndex\022\033.milvus" + ".grpc.CollectionName\032\023.milvus.grpc.Statu" + "s\"\000\022E\n\017CreatePartition\022\033.milvus.grpc.Par" + "titionParam\032\023.milvus.grpc.Status\"\000\022E\n\014Ha" + "sPartition\022\033.milvus.grpc.PartitionParam\032" + "\026.milvus.grpc.BoolReply\"\000\022K\n\016ShowPartiti" + "ons\022\033.milvus.grpc.CollectionName\032\032.milvu" + "s.grpc.PartitionList\"\000\022C\n\rDropPartition\022" + "\033.milvus.grpc.PartitionParam\032\023.milvus.gr" + "pc.Status\"\000\022<\n\006Insert\022\030.milvus.grpc.Inse" + "rtParam\032\026.milvus.grpc.VectorIds\"\000\022J\n\016Get" + "VectorsByID\022\034.milvus.grpc.VectorsIdentit" + "y\032\030.milvus.grpc.VectorsData\"\000\022H\n\014GetVect" + "orIDs\022\036.milvus.grpc.GetVectorIDsParam\032\026." + "milvus.grpc.VectorIds\"\000\022B\n\006Search\022\030.milv" + "us.grpc.SearchParam\032\034.milvus.grpc.TopKQu" + "eryResult\"\000\022J\n\nSearchByID\022\034.milvus.grpc." + "SearchByIDParam\032\034.milvus.grpc.TopKQueryR" + "esult\"\000\022P\n\rSearchInFiles\022\037.milvus.grpc.S" + "earchInFilesParam\032\034.milvus.grpc.TopKQuer" + "yResult\"\000\0227\n\003Cmd\022\024.milvus.grpc.Command\032\030" + ".milvus.grpc.StringReply\"\000\022A\n\nDeleteByID" + "\022\034.milvus.grpc.DeleteByIDParam\032\023.milvus." + "grpc.Status\"\000\022G\n\021PreloadCollection\022\033.mil" + "vus.grpc.CollectionName\032\023.milvus.grpc.St" + "atus\"\000\022I\n\016ReloadSegments\022 .milvus.grpc.R" + "eLoadSegmentsParam\032\023.milvus.grpc.Status\"" + "\000\0227\n\005Flush\022\027.milvus.grpc.FlushParam\032\023.mi" + "lvus.grpc.Status\"\000\022=\n\007Compact\022\033.milvus.g" + "rpc.CollectionName\032\023.milvus.grpc.Status\"" + "\000\022E\n\026CreateHybridCollection\022\024.milvus.grp" + "c.Mapping\032\023.milvus.grpc.Status\"\000\022L\n\023HasH" + "ybridCollection\022\033.milvus.grpc.Collection" + "Name\032\026.milvus.grpc.BoolReply\"\000\022J\n\024DropHy" + "bridCollection\022\033.milvus.grpc.CollectionN" + "ame\032\023.milvus.grpc.Status\"\000\022O\n\030DescribeHy" + "bridCollection\022\033.milvus.grpc.CollectionN" + "ame\032\024.milvus.grpc.Mapping\"\000\022W\n\025CountHybr" + "idCollection\022\033.milvus.grpc.CollectionNam" + "e\032\037.milvus.grpc.CollectionRowCount\"\000\022I\n\025" + "ShowHybridCollections\022\024.milvus.grpc.Comm" + "and\032\030.milvus.grpc.MappingList\"\000\022V\n\030ShowH" + "ybridCollectionInfo\022\033.milvus.grpc.Collec" + "tionName\032\033.milvus.grpc.CollectionInfo\"\000\022" + "M\n\027PreloadHybridCollection\022\033.milvus.grpc" + ".CollectionName\032\023.milvus.grpc.Status\"\000\022D" + "\n\021CreateHybridIndex\022\030.milvus.grpc.HIndex" + "Param\032\023.milvus.grpc.Status\"\000\022D\n\014InsertEn" + "tity\022\031.milvus.grpc.HInsertParam\032\027.milvus" + ".grpc.HEntityIDs\"\000\022J\n\016HybridSearchPB\022\033.m" + "ilvus.grpc.HSearchParamPB\032\031.milvus.grpc." + "HQueryResult\"\000\022F\n\014HybridSearch\022\031.milvus." + "grpc.HSearchParam\032\031.milvus.grpc.HQueryRe" + "sult\"\000\022]\n\026HybridSearchInSegments\022#.milvu" + "s.grpc.HSearchInSegmentsParam\032\034.milvus.g" + "rpc.TopKQueryResult\"\000\022E\n\rGetEntityByID\022\034" + ".milvus.grpc.VectorsIdentity\032\024.milvus.gr" + "pc.HEntity\"\000\022J\n\014GetEntityIDs\022\037.milvus.gr" + "pc.HGetEntityIDsParam\032\027.milvus.grpc.HEnt" + "ityIDs\"\000\022J\n\022DeleteEntitiesByID\022\035.milvus." + "grpc.HDeleteByIDParam\032\023.milvus.grpc.Stat" + "us\"\000b\006proto3" ; static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_milvus_2eproto_deps[1] = { &::descriptor_table_status_2eproto, @@ -1818,7 +1819,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_mil static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_milvus_2eproto_once; static bool descriptor_table_milvus_2eproto_initialized = false; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_milvus_2eproto = { - &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 8908, + &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 8932, &descriptor_table_milvus_2eproto_once, descriptor_table_milvus_2eproto_sccs, descriptor_table_milvus_2eproto_deps, 50, 1, schemas, file_default_instances, TableStruct_milvus_2eproto::offsets, file_level_metadata_milvus_2eproto, 51, file_level_enum_descriptors_milvus_2eproto, file_level_service_descriptors_milvus_2eproto, @@ -1839,12 +1840,13 @@ bool DataType_IsValid(int value) { case 2: case 3: case 4: + case 5: + case 10: + case 11: case 20: - case 30: - case 40: - case 41: case 100: - case 9999: + case 101: + case 200: return true; default: return false; diff --git a/core/src/grpc/gen-milvus/milvus.pb.h b/core/src/grpc/gen-milvus/milvus.pb.h index 4ae825adb2995..6d072f0ba142c 100644 --- a/core/src/grpc/gen-milvus/milvus.pb.h +++ b/core/src/grpc/gen-milvus/milvus.pb.h @@ -270,23 +270,24 @@ namespace milvus { namespace grpc { enum DataType : int { - NULL_ = 0, - INT8 = 1, - INT16 = 2, - INT32 = 3, - INT64 = 4, + NONE = 0, + BOOL = 1, + INT8 = 2, + INT16 = 3, + INT32 = 4, + INT64 = 5, + FLOAT = 10, + DOUBLE = 11, STRING = 20, - BOOL = 30, - FLOAT = 40, - DOUBLE = 41, - VECTOR = 100, - UNKNOWN = 9999, + VECTOR_BINARY = 100, + VECTOR_FLOAT = 101, + VECTOR = 200, DataType_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits<::PROTOBUF_NAMESPACE_ID::int32>::min(), DataType_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits<::PROTOBUF_NAMESPACE_ID::int32>::max() }; bool DataType_IsValid(int value); -constexpr DataType DataType_MIN = NULL_; -constexpr DataType DataType_MAX = UNKNOWN; +constexpr DataType DataType_MIN = NONE; +constexpr DataType DataType_MAX = VECTOR; constexpr int DataType_ARRAYSIZE = DataType_MAX + 1; const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* DataType_descriptor(); diff --git a/core/src/grpc/milvus.proto b/core/src/grpc/milvus.proto index 330ab454f0a1a..f8c6e894c08a4 100644 --- a/core/src/grpc/milvus.proto +++ b/core/src/grpc/milvus.proto @@ -224,21 +224,21 @@ message GetVectorIDsParam { /********************************************************************************************************************/ enum DataType { - NULL = 0; - INT8 = 1; - INT16 = 2; - INT32 = 3; - INT64 = 4; + NONE = 0; + BOOL = 1; + INT8 = 2; + INT16 = 3; + INT32 = 4; + INT64 = 5; - STRING = 20; - - BOOL = 30; + FLOAT = 10; + DOUBLE = 11; - FLOAT = 40; - DOUBLE = 41; + STRING = 20; - VECTOR = 100; - UNKNOWN = 9999; + VECTOR_BINARY = 100; + VECTOR_FLOAT = 101; + VECTOR = 200; } /////////////////////////////////////////////////////////////////// diff --git a/core/unittest/ssdb/test_snapshot.cpp b/core/unittest/ssdb/test_snapshot.cpp index 82f16230561ea..f549033978d56 100644 --- a/core/unittest/ssdb/test_snapshot.cpp +++ b/core/unittest/ssdb/test_snapshot.cpp @@ -22,11 +22,10 @@ TEST_F(SnapshotTest, ResourcesTest) { int nprobe = 16; milvus::json params = {{"nprobe", nprobe}}; - ParamsField p_field(params.dump()); - ASSERT_EQ(params.dump(), p_field.GetParams()); - ASSERT_EQ(params, p_field.GetParamsJson()); + ParamsField p_field(params); + ASSERT_EQ(params, p_field.GetParams()); - auto nprobe_real = p_field.GetParamsJson().at("nprobe").get(); + auto nprobe_real = p_field.GetParams().at("nprobe").get(); ASSERT_EQ(nprobe, nprobe_real); }