Skip to content

Commit

Permalink
return status in CreateVectorTable
Browse files Browse the repository at this point in the history
  • Loading branch information
@wxingda
wxingda committed Apr 19, 2024
1 parent e2b18e2 commit 1a457df
Show file tree
Hide file tree
Showing 13 changed files with 151 additions and 165 deletions.
6 changes: 2 additions & 4 deletions internal/engine/c_api/api_data/request.h
View file
Expand Up @@ -26,10 +26,8 @@ class Request : public RawData {
}

virtual ~Request() {
if (ranker_) {
delete ranker_;
ranker_ = nullptr;
}
delete ranker_;
ranker_ = nullptr;
}

virtual int Serialize(char **out, int *out_len);
Expand Down
24 changes: 8 additions & 16 deletions internal/engine/index/impl/gamma_index_binary_ivf.cc
View file
Expand Up @@ -67,18 +67,12 @@ GammaIndexBinaryIVF::GammaIndexBinaryIVF() {
}

GammaIndexBinaryIVF::~GammaIndexBinaryIVF() {
if (rt_invert_index_ptr_) {
delete rt_invert_index_ptr_;
rt_invert_index_ptr_ = nullptr;
}
if (invlists) {
delete invlists;
invlists = nullptr;
}
if (quantizer) {
delete quantizer; // it will not be delete in parent class
quantizer = nullptr;
}
delete rt_invert_index_ptr_;
rt_invert_index_ptr_ = nullptr;
delete invlists;
invlists = nullptr;
delete quantizer; // it will not be delete in parent class
quantizer = nullptr;
}

Status GammaIndexBinaryIVF::Init(const std::string &model_parameters,
Expand Down Expand Up @@ -117,10 +111,8 @@ Status GammaIndexBinaryIVF::Init(const std::string &model_parameters,

this->nprobe = 20;

if (this->invlists) {
delete this->invlists;
this->invlists = nullptr;
}
delete this->invlists;
this->invlists = nullptr;

bool ret = rt_invert_index_ptr_->Init();

Expand Down
15 changes: 8 additions & 7 deletions internal/engine/index/impl/gamma_index_ivfflat.cc
View file
Expand Up @@ -199,10 +199,8 @@ Status GammaIndexIVFFlat::Init(const std::string &model_parameters,
this->nlist, this->code_size, raw_vec->VidMgr(), raw_vec->Bitmap(),
params.bucket_init_size, params.bucket_max_size);

if (this->invlists) {
delete this->invlists;
this->invlists = nullptr;
}
delete this->invlists;
this->invlists = nullptr;

bool ret = rt_invert_index_ptr_->Init();
if (!ret) {
Expand Down Expand Up @@ -442,7 +440,9 @@ int GammaIndexIVFFlat::Search(RetrievalContext *retrieval_context, int n,
nprobe = retrieval_params->Nprobe();
} else {
retrieval_params->SetNprobe(this->nprobe);
LOG(WARNING) << "nlist = " << this->nlist << "nprobe = " << retrieval_params->Nprobe() << "invalid, now use:" << this->nprobe;
LOG(WARNING) << "nlist = " << this->nlist
<< "nprobe = " << retrieval_params->Nprobe()
<< "invalid, now use:" << this->nprobe;
}
#else
int nprobe = this->nprobe;
Expand All @@ -455,7 +455,7 @@ int GammaIndexIVFFlat::Search(RetrievalContext *retrieval_context, int n,
quantizer->search(n, x, nprobe, coarse_dis.get(), idx.get());

search_preassigned(retrieval_context, n, x, k, idx.get(), coarse_dis.get(),
distances, labels, nprobe, false);
distances, labels, nprobe, false);

return 0;
}
Expand All @@ -482,7 +482,8 @@ void GammaIndexIVFFlat::search_preassigned(RetrievalContext *retrieval_context,
retrieval_context->retrieval_params_ =
new FlatRetrievalParameters(retrieval_params->ParallelOnQueries(),
retrieval_params->GetDistanceComputeType());
GammaFLATIndex::Search(retrieval_context, n, (const uint8_t*)x, k, distances, labels);
GammaFLATIndex::Search(retrieval_context, n, (const uint8_t *)x, k,
distances, labels);
return;
}
faiss::MetricType metric_type;
Expand Down
38 changes: 17 additions & 21 deletions internal/engine/index/impl/gamma_index_ivfpq.cc
View file
Expand Up @@ -52,22 +52,14 @@ GammaIVFPQIndex::GammaIVFPQIndex() : indexed_vec_count_(0) {
}

GammaIVFPQIndex::~GammaIVFPQIndex() {
if (rt_invert_index_ptr_) {
delete rt_invert_index_ptr_;
rt_invert_index_ptr_ = nullptr;
}
if (invlists) {
delete invlists;
invlists = nullptr;
}
if (quantizer) {
delete quantizer; // it will not be delete in parent class
quantizer = nullptr;
}
if (opq_) {
delete opq_;
opq_ = nullptr;
}
delete rt_invert_index_ptr_;
rt_invert_index_ptr_ = nullptr;
delete invlists;
invlists = nullptr;
delete quantizer; // it will not be delete in parent class
quantizer = nullptr;
delete opq_;
opq_ = nullptr;

CHECK_DELETE(model_param_);
}
Expand Down Expand Up @@ -132,8 +124,10 @@ Status GammaIVFPQIndex::Init(const std::string &model_parameters,
if (training_threshold) {
training_threshold_ = training_threshold;
} else {
// shouldn't less than max_points_per_centroid because of pq.train() when nbit = 8 and ksub = 2^8
training_threshold_ = std::max((int)nlist * min_points_per_centroid, max_points_per_centroid);
// shouldn't less than max_points_per_centroid because of pq.train() when
// nbit = 8 and ksub = 2^8
training_threshold_ =
std::max((int)nlist * min_points_per_centroid, max_points_per_centroid);
}
ivfpq_param.training_threshold = training_threshold_;
LOG(INFO) << ivfpq_param.ToString();
Expand Down Expand Up @@ -516,7 +510,9 @@ int GammaIVFPQIndex::Search(RetrievalContext *retrieval_context, int n,
nprobe = retrieval_params->Nprobe();
} else {
retrieval_params->SetNprobe(this->nprobe);
LOG(WARNING) << "nlist = " << this->nlist << "nprobe = " << retrieval_params->Nprobe() << "invalid, now use:" << this->nprobe;
LOG(WARNING) << "nlist = " << this->nlist
<< "nprobe = " << retrieval_params->Nprobe()
<< "invalid, now use:" << this->nprobe;
}

const float *xq = reinterpret_cast<const float *>(x);
Expand Down Expand Up @@ -611,8 +607,8 @@ size_t scan_one_list(GammaInvertedListScanner *scanner, idx_t key,

void compute_dis(int k, const float *xi, float *simi, idx_t *idxi,
float *recall_simi, idx_t *recall_idxi, int recall_num,
bool rerank, faiss::MetricType metric_type,
VectorReader *vec, RetrievalContext *retrieval_context) {
bool rerank, faiss::MetricType metric_type, VectorReader *vec,
RetrievalContext *retrieval_context) {
if (rerank == true) {
ScopeVectors scope_vecs;
std::vector<idx_t> vids(recall_idxi, recall_idxi + recall_num);
Expand Down
24 changes: 8 additions & 16 deletions internal/engine/index/impl/gamma_index_ivfpqfs.cc
View file
Expand Up @@ -48,22 +48,14 @@ GammaIVFPQFastScanIndex::GammaIVFPQFastScanIndex() : indexed_vec_count_(0) {
}

GammaIVFPQFastScanIndex::~GammaIVFPQFastScanIndex() {
if (rt_invert_index_ptr_) {
delete rt_invert_index_ptr_;
rt_invert_index_ptr_ = nullptr;
}
if (invlists) {
delete invlists;
invlists = nullptr;
}
if (quantizer) {
delete quantizer; // it will not be delete in parent class
quantizer = nullptr;
}
if (opq_) {
delete opq_;
opq_ = nullptr;
}
delete rt_invert_index_ptr_;
rt_invert_index_ptr_ = nullptr;
delete invlists;
invlists = nullptr;
delete quantizer; // it will not be delete in parent class
quantizer = nullptr;
delete opq_;
opq_ = nullptr;

CHECK_DELETE(model_param_);
int ret = pthread_rwlock_destroy(&shared_mutex_);
Expand Down
13 changes: 4 additions & 9 deletions internal/engine/index/impl/gpu/gamma_index_ivfpq_gpu.cc
View file
Expand Up @@ -246,15 +246,10 @@ GammaIVFPQGPUIndex::~GammaIVFPQGPUIndex() {
std::lock_guard<std::mutex> lock(indexing_mutex_);
b_exited_ = true;
std::this_thread::sleep_for(std::chrono::seconds(2));
if (cpu_index_) {
delete cpu_index_;
cpu_index_ = nullptr;
}

if (gpu_index_) {
delete gpu_index_;
gpu_index_ = nullptr;
}
delete cpu_index_;
cpu_index_ = nullptr;
delete gpu_index_;
gpu_index_ = nullptr;

for (auto &resource : resources_) {
delete resource;
Expand Down
19 changes: 11 additions & 8 deletions internal/engine/search/engine.cc
View file
Expand Up @@ -490,8 +490,10 @@ Status Engine::CreateTable(TableInfo &table) {
}
}

if (vec_manager_->CreateVectorTable(table, meta_jp) != 0) {
std::string msg = space_name_ + " cannot create VectorTable!";
Status status = vec_manager_->CreateVectorTable(table, meta_jp);
if (status != Status::OK()) {
std::string msg =
space_name_ + " cannot create VectorTable: " + status.ToString();
LOG(ERROR) << msg;
vec_manager_->Close();
return Status::ParamError(msg);
Expand All @@ -502,7 +504,7 @@ Status Engine::CreateTable(TableInfo &table) {
meta_jp->GetObject("table", table_jp);
disk_table_params.Parse(table_jp);
}
Status status = table_->CreateTable(table, disk_table_params, docids_bitmap_);
status = table_->CreateTable(table, disk_table_params, docids_bitmap_);
training_threshold_ = table.TrainingThreshold();
LOG(INFO) << space_name_
<< " init training_threshold=" << training_threshold_;
Expand Down Expand Up @@ -775,7 +777,7 @@ int Engine::RebuildIndex(int drop_before_rebuild, int limit_cpu, int describe) {
std::map<std::string, IndexModel *> vector_indexes;

if (!drop_before_rebuild) {
ret =
Status status =
vec_manager_->CreateVectorIndexes(training_threshold_, vector_indexes);
if (vec_manager_->TrainIndex(vector_indexes) != 0) {
LOG(ERROR) << "RebuildIndex TrainIndex failed!";
Expand All @@ -795,10 +797,11 @@ int Engine::RebuildIndex(int drop_before_rebuild, int limit_cpu, int describe) {
vec_manager_->DestroyVectorIndexes();

if (drop_before_rebuild) {
ret = vec_manager_->CreateVectorIndexes(training_threshold_,
vec_manager_->VectorIndexes());
if (ret) {
LOG(ERROR) << "RebuildIndex CreateVectorIndexes failed, ret: " << ret;
Status status = vec_manager_->CreateVectorIndexes(
training_threshold_, vec_manager_->VectorIndexes());
if (status != Status::OK()) {
LOG(ERROR) << "RebuildIndex CreateVectorIndexes failed: "
<< status.ToString();
vec_manager_->DestroyVectorIndexes();
return ret;
}
Expand Down
9 changes: 4 additions & 5 deletions internal/engine/table/table.cc
View file
Expand Up @@ -108,10 +108,8 @@ Table::Table(const string &root_path, const string &space_name)
Table::~Table() {
bitmap_mgr_ = nullptr;
CHECK_DELETE(table_params_);
if (storage_mgr_) {
delete storage_mgr_;
storage_mgr_ = nullptr;
}
delete storage_mgr_;
storage_mgr_ = nullptr;

delete item_to_docid_;
item_to_docid_ = nullptr;
Expand Down Expand Up @@ -351,7 +349,8 @@ int Table::Update(const std::unordered_map<std::string, struct Field> &fields,
int field_id = it->second;
int offset = idx_attr_offset_[field_id];

if (field.datatype == DataType::STRING || field.datatype == DataType::STRINGARRAY) {
if (field.datatype == DataType::STRING ||
field.datatype == DataType::STRINGARRAY) {
int len = field.value.size();
storage_mgr_->UpdateString(docid, name, field.value.c_str(), len);
} else {
Expand Down
6 changes: 2 additions & 4 deletions internal/engine/vector/raw_vector_common.h
View file
Expand Up @@ -46,10 +46,8 @@ class VIDMgr {
~VIDMgr() {
if (multi_vids_) {
for (size_t i = 0; i < docid2vid_.size(); i++) {
if (docid2vid_[i] != nullptr) {
delete[] docid2vid_[i];
docid2vid_[i] = nullptr;
}
delete[] docid2vid_[i];
docid2vid_[i] = nullptr;
}
}
}
Expand Down
6 changes: 2 additions & 4 deletions internal/engine/vector/rocksdb_raw_vector.cc
View file
Expand Up @@ -25,10 +25,8 @@ RocksDBRawVector::RocksDBRawVector(VectorMetaInfo *meta_info,
}

RocksDBRawVector::~RocksDBRawVector() {
if (db_) {
delete db_;
db_ = nullptr;
}
delete db_;
db_ = nullptr;
}

int RocksDBRawVector::InitStore(std::string &vec_name) {
Expand Down
6 changes: 2 additions & 4 deletions internal/engine/vector/rocksdb_wrapper.cc
View file
Expand Up @@ -16,10 +16,8 @@ namespace vearch {
RocksDBWrapper::RocksDBWrapper() : db_(nullptr) {}

RocksDBWrapper::~RocksDBWrapper() {
if (db_) {
delete db_;
db_ = nullptr;
}
delete db_;
db_ = nullptr;
}

Status RocksDBWrapper::Open(string db_path, size_t block_cache_size) {
Expand Down

0 comments on commit 1a457df

Please sign in to comment.