From 067234777581dd1a95fb2ff87e8da1d7a34e71fb Mon Sep 17 00:00:00 2001 From: "shengjun.li" Date: Sat, 4 Jul 2020 09:25:48 +0800 Subject: [PATCH] Create index failed caused by server crashed (#2736) Signed-off-by: shengjun.li Co-authored-by: groot --- CHANGELOG.md | 1 + .../index/vector_index/IndexBinaryIDMAP.cpp | 16 ++++++++++++++++ .../index/vector_index/IndexBinaryIDMAP.h | 8 ++------ .../index/vector_index/IndexBinaryIVF.cpp | 16 ++++++++++++++++ .../knowhere/index/vector_index/IndexBinaryIVF.h | 8 ++------ .../knowhere/index/vector_index/IndexIDMAP.cpp | 16 ++++++++++++++++ .../knowhere/index/vector_index/IndexIDMAP.h | 8 ++------ .../knowhere/index/vector_index/IndexIVF.cpp | 16 ++++++++++++++++ .../knowhere/index/vector_index/IndexIVF.h | 8 ++------ .../knowhere/index/vector_index/IndexNSG.cpp | 6 ++++++ .../knowhere/index/vector_index/IndexSPTAG.cpp | 6 ++++++ .../index/vector_offset_index/IndexIVF_NM.cpp | 16 ++++++++++++++++ .../index/vector_offset_index/IndexIVF_NM.h | 8 ++------ .../index/vector_offset_index/IndexNSG_NM.cpp | 6 ++++++ core/src/index/thirdparty/hnswlib/hnswalg_nm.h | 11 ----------- 15 files changed, 109 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afe7c631a718..dc26aaabd191 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Please mark all changes in change log and use the issue from GitHub - \#2598 Fix Milvus docker image report illegal instruction - \#2617 Fix HNSW and RNSG index files size - \#2637 Suit the range of HNSW parameters +- \#2642 Create index failed and server crashed - \#2649 Search parameter of annoy has conflict with document - \#2690 Remove body parser in show-partitions endpoints diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp index a38fba01b461..056a955abea5 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp @@ -112,6 +112,22 @@ BinaryIDMAP::QueryById(const DatasetPtr& dataset_ptr, const Config& config) { } #endif +int64_t +BinaryIDMAP::Count() { + if (!index_) { + KNOWHERE_THROW_MSG("index not initialize"); + } + return index_->ntotal; +} + +int64_t +BinaryIDMAP::Dim() { + if (!index_) { + KNOWHERE_THROW_MSG("index not initialize"); + } + return index_->d; +} + void BinaryIDMAP::Add(const DatasetPtr& dataset_ptr, const Config& config) { if (!index_) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.h index bf3b57808363..ce7da9bf0404 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.h @@ -56,14 +56,10 @@ class BinaryIDMAP : public VecIndex, public FaissBaseBinaryIndex { #endif int64_t - Count() override { - return index_->ntotal; - } + Count() override; int64_t - Dim() override { - return index_->d; - } + Dim() override; int64_t IndexSize() override { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp index 616f4c8d23fb..8036a42859a5 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp @@ -129,6 +129,22 @@ BinaryIVF::QueryById(const DatasetPtr& dataset_ptr, const Config& config) { } #endif +int64_t +BinaryIVF::Count() { + if (!index_) { + KNOWHERE_THROW_MSG("index not initialize"); + } + return index_->ntotal; +} + +int64_t +BinaryIVF::Dim() { + if (!index_) { + KNOWHERE_THROW_MSG("index not initialize"); + } + return index_->d; +} + void BinaryIVF::Train(const DatasetPtr& dataset_ptr, const Config& config) { GETTENSORWITHIDS(dataset_ptr) diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.h index e46a64e8d6c1..ae0991428c41 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.h @@ -68,14 +68,10 @@ class BinaryIVF : public VecIndex, public FaissBaseBinaryIndex { #endif int64_t - Count() override { - return index_->ntotal; - } + Count() override; int64_t - Dim() override { - return index_->d; - } + Dim() override; #if 0 DatasetPtr diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp index 8f1babc840e4..612de45547af 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp @@ -142,6 +142,22 @@ IDMAP::QueryById(const DatasetPtr& dataset_ptr, const Config& config) { } #endif +int64_t +IDMAP::Count() { + if (!index_) { + KNOWHERE_THROW_MSG("index not initialize"); + } + return index_->ntotal; +} + +int64_t +IDMAP::Dim() { + if (!index_) { + KNOWHERE_THROW_MSG("index not initialize"); + } + return index_->d; +} + VecIndexPtr IDMAP::CopyCpuToGpu(const int64_t device_id, const Config& config) { #ifdef MILVUS_GPU_VERSION diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h index f15c665d40b7..128d9f99b468 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h @@ -54,14 +54,10 @@ class IDMAP : public VecIndex, public FaissBaseIndex { #endif int64_t - Count() override { - return index_->ntotal; - } + Count() override; int64_t - Dim() override { - return index_->d; - } + Dim() override; int64_t IndexSize() override { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 94133acc2fb6..67c797740d00 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -217,6 +217,22 @@ IVF::GetVectorById(const DatasetPtr& dataset_ptr, const Config& config) { } #endif +int64_t +IVF::Count() { + if (!index_) { + KNOWHERE_THROW_MSG("index not initialize"); + } + return index_->ntotal; +} + +int64_t +IVF::Dim() { + if (!index_) { + KNOWHERE_THROW_MSG("index not initialize"); + } + return index_->d; +} + void IVF::Seal() { if (!index_ || !index_->is_trained) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h index 612abc6bd11b..291efe7436f8 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h @@ -59,14 +59,10 @@ class IVF : public VecIndex, public FaissBaseIndex { #endif int64_t - Count() override { - return index_->ntotal; - } + Count() override; int64_t - Dim() override { - return index_->d; - } + Dim() override; #if 0 DatasetPtr diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp index 1a55701307b4..c6f97513cae4 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp @@ -149,11 +149,17 @@ NSG::Train(const DatasetPtr& dataset_ptr, const Config& config) { int64_t NSG::Count() { + if (!index_) { + KNOWHERE_THROW_MSG("index not initialize"); + } return index_->ntotal; } int64_t NSG::Dim() { + if (!index_) { + KNOWHERE_THROW_MSG("index not initialize"); + } return index_->dimension; } diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp index ddbc37ec80b5..951a5fc31470 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp @@ -200,11 +200,17 @@ CPUSPTAGRNG::Query(const DatasetPtr& dataset_ptr, const Config& config) { int64_t CPUSPTAGRNG::Count() { + if (!index_ptr_) { + KNOWHERE_THROW_MSG("index not initialize"); + } return index_ptr_->GetNumSamples(); } int64_t CPUSPTAGRNG::Dim() { + if (!index_ptr_) { + KNOWHERE_THROW_MSG("index not initialize"); + } return index_ptr_->GetFeatureDim(); } diff --git a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp index 370c25b0225e..0bc22ccc68e8 100644 --- a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp @@ -335,5 +335,21 @@ IVF_NM::SealImpl() { #endif } +int64_t +IVF_NM::Count() { + if (!index_) { + KNOWHERE_THROW_MSG("index not initialize"); + } + return index_->ntotal; +} + +int64_t +IVF_NM::Dim() { + if (!index_) { + KNOWHERE_THROW_MSG("index not initialize"); + } + return index_->d; +} + } // namespace knowhere } // namespace milvus diff --git a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.h b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.h index 4371388afc6d..eed2bf76b97a 100644 --- a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.h +++ b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.h @@ -59,14 +59,10 @@ class IVF_NM : public VecIndex, public OffsetBaseIndex { #endif int64_t - Count() override { - return index_->ntotal; - } + Count() override; int64_t - Dim() override { - return index_->d; - } + Dim() override; #if 0 DatasetPtr diff --git a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexNSG_NM.cpp b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexNSG_NM.cpp index c71ef81867e8..905c3868c221 100644 --- a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexNSG_NM.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexNSG_NM.cpp @@ -151,11 +151,17 @@ NSG_NM::Train(const DatasetPtr& dataset_ptr, const Config& config) { int64_t NSG_NM::Count() { + if (!index_) { + KNOWHERE_THROW_MSG("index not initialize"); + } return index_->ntotal; } int64_t NSG_NM::Dim() { + if (!index_) { + KNOWHERE_THROW_MSG("index not initialize"); + } return index_->dimension; } diff --git a/core/src/index/thirdparty/hnswlib/hnswalg_nm.h b/core/src/index/thirdparty/hnswlib/hnswalg_nm.h index 81fbe2a2a4c6..0bf533582328 100644 --- a/core/src/index/thirdparty/hnswlib/hnswalg_nm.h +++ b/core/src/index/thirdparty/hnswlib/hnswalg_nm.h @@ -46,9 +46,6 @@ namespace hnswlib { dist_func_param_ = s->get_dist_func_param(); M_ = M; maxM_ = M_; - cmli_cnt_ = 0; - cmli_cnt2_ = 0; - cmli_time_ = 0.0; maxM0_ = M_ * 2; ef_construction_ = std::max(ef_construction,M_); ef_ = 10; @@ -124,9 +121,6 @@ namespace hnswlib { size_t M_; size_t maxM_; size_t maxM0_; - size_t cmli_cnt_; - size_t cmli_cnt2_; - double cmli_time_; size_t ef_construction_; double mult_, revSize_; @@ -497,13 +491,8 @@ namespace hnswlib { dist_func_param_), data[j]); } - if (candidates.size() >= Mcurmax) cmli_cnt2_ ++; size_t indx = 0; - auto t0 = std::chrono::high_resolution_clock::now(); getNeighborsByHeuristic2(candidates, Mcurmax, data, indx, pdata); - auto t1 = std::chrono::high_resolution_clock::now(); - cmli_time_ += (double)std::chrono::duration_cast( t1 - t0 ).count(); - cmli_cnt_ ++; // while (candidates.size() > 0) { // data[indx] = candidates.top().second;