Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix warning master #2894

Merged
merged 5 commits into from Jul 18, 2020
Merged

Fix warning master #2894

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
dc37a40
fix clean unused variable warning in knowhere
cqy123456 Jul 17, 2020
d0dedfc
clean unused variable warning in knowhere
cqy123456 Jul 17, 2020
7c1cbad
clean unused variable warning in knowhere
cqy123456 Jul 17, 2020
bafc30d
clean unused variable warning in knowhere
cqy123456 Jul 17, 2020
4649303
fix konwhere warning
cqy123456 Jul 18, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion CHANGELOG.md
View file
Open in desktop
Expand Up @@ -42,7 +42,7 @@ Please mark all changes in change log and use the issue from GitHub
- \#2561 Clean util dependencies with other modules
- \#2612 Move all APIs in utils into namespace milvus
- \#2675 Print out system memory size when report invalid cpu cache size
- \#2828 Let Faiss not to compile half float by default
- \#2828 Let Faiss not compile half float by default
- \#2841 Replace IndexType/EngineType/MetricType
- \#2858 Unify index name in db
- \#2884 Using BlockingQueue in JobMgr
Expand Down
4 changes: 2 additions & 2 deletions core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.cpp
View file
Open in desktop
Expand Up @@ -86,7 +86,7 @@ IndexAnnoy::BuildAll(const DatasetPtr& dataset_ptr, const Config& config) {
return;
}

GETTENSORWITHIDS(dataset_ptr)
GET_TENSOR(dataset_ptr)

metric_type_ = config[Metric::TYPE];
if (metric_type_ == Metric::L2) {
Expand All @@ -110,7 +110,7 @@ IndexAnnoy::Query(const DatasetPtr& dataset_ptr, const Config& config) {
KNOWHERE_THROW_MSG("index not initialize or trained");
}

GETTENSOR(dataset_ptr)
GET_TENSOR_DATA_DIM(dataset_ptr)
auto k = config[meta::TOPK].get<int64_t>();
auto search_k = config[IndexParams::search_k].get<int64_t>();
auto all_num = rows * k;
Expand Down
6 changes: 3 additions & 3 deletions core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp
View file
Open in desktop
Expand Up @@ -44,7 +44,7 @@ BinaryIDMAP::Query(const DatasetPtr& dataset_ptr, const Config& config) {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA(dataset_ptr)

int64_t k = config[meta::TOPK].get<int64_t>();
auto elems = rows * k;
Expand Down Expand Up @@ -135,7 +135,7 @@ BinaryIDMAP::Add(const DatasetPtr& dataset_ptr, const Config& config) {
}

std::lock_guard<std::mutex> lk(mutex_);
GETTENSORWITHIDS(dataset_ptr)
GET_TENSOR_DATA_ID(dataset_ptr)

index_->add_with_ids(rows, (uint8_t*)p_data, p_ids);
}
Expand Down Expand Up @@ -177,7 +177,7 @@ BinaryIDMAP::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config)
}

std::lock_guard<std::mutex> lk(mutex_);
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA(dataset_ptr)

std::vector<int64_t> new_ids(rows);
for (int i = 0; i < rows; ++i) {
Expand Down
4 changes: 2 additions & 2 deletions core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp
View file
Open in desktop
Expand Up @@ -48,7 +48,7 @@ BinaryIVF::Query(const DatasetPtr& dataset_ptr, const Config& config) {
KNOWHERE_THROW_MSG("index not initialize or trained");
}

GETTENSOR(dataset_ptr)
GET_TENSOR_DATA(dataset_ptr)

try {
int64_t k = config[meta::TOPK].get<int64_t>();
Expand Down Expand Up @@ -147,7 +147,7 @@ BinaryIVF::Dim() {

void
BinaryIVF::Train(const DatasetPtr& dataset_ptr, const Config& config) {
GETTENSORWITHIDS(dataset_ptr)
GET_TENSOR(dataset_ptr)

int64_t nlist = config[IndexParams::nlist];
faiss::MetricType metric_type = GetMetricType(config[Metric::TYPE].get<std::string>());
Expand Down
7 changes: 4 additions & 3 deletions core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp
View file
Open in desktop
Expand Up @@ -78,7 +78,8 @@ IndexHNSW::Load(const BinarySet& index_binary) {
void
IndexHNSW::Train(const DatasetPtr& dataset_ptr, const Config& config) {
try {
GETTENSOR(dataset_ptr)
int64_t dim = dataset_ptr->Get<int64_t>(meta::DIM);
int64_t rows = dataset_ptr->Get<int64_t>(meta::ROWS);

hnswlib::SpaceInterface<float>* space;
if (config[Metric::TYPE] == Metric::L2) {
Expand All @@ -102,7 +103,7 @@ IndexHNSW::Add(const DatasetPtr& dataset_ptr, const Config& config) {

std::lock_guard<std::mutex> lk(mutex_);

GETTENSORWITHIDS(dataset_ptr)
GET_TENSOR_DATA_ID(dataset_ptr)

// if (normalize) {
// std::vector<float> ep_norm_vector(Dim());
Expand Down Expand Up @@ -135,7 +136,7 @@ IndexHNSW::Query(const DatasetPtr& dataset_ptr, const Config& config) {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize or trained");
}
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA(dataset_ptr)

size_t k = config[meta::TOPK].get<int64_t>();
size_t id_size = sizeof(int64_t) * k;
Expand Down
4 changes: 2 additions & 2 deletions core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp
View file
Open in desktop
Expand Up @@ -68,7 +68,7 @@ IDMAP::Add(const DatasetPtr& dataset_ptr, const Config& config) {
}

std::lock_guard<std::mutex> lk(mutex_);
GETTENSORWITHIDS(dataset_ptr)
GET_TENSOR_DATA_ID(dataset_ptr)
index_->add_with_ids(rows, (float*)p_data, p_ids);
}

Expand Down Expand Up @@ -96,7 +96,7 @@ IDMAP::Query(const DatasetPtr& dataset_ptr, const Config& config) {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize");
}
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA(dataset_ptr)

int64_t k = config[meta::TOPK].get<int64_t>();
auto elems = rows * k;
Expand Down
8 changes: 4 additions & 4 deletions core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
View file
Open in desktop
Expand Up @@ -64,7 +64,7 @@ IVF::Load(const BinarySet& binary_set) {

void
IVF::Train(const DatasetPtr& dataset_ptr, const Config& config) {
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA_DIM(dataset_ptr)

faiss::MetricType metric_type = GetMetricType(config[Metric::TYPE].get<std::string>());
faiss::Index* coarse_quantizer = new faiss::IndexFlat(dim, metric_type);
Expand All @@ -80,7 +80,7 @@ IVF::Add(const DatasetPtr& dataset_ptr, const Config& config) {
}

std::lock_guard<std::mutex> lk(mutex_);
GETTENSORWITHIDS(dataset_ptr)
GET_TENSOR_DATA_ID(dataset_ptr)
index_->add_with_ids(rows, (float*)p_data, p_ids);
}

Expand All @@ -91,7 +91,7 @@ IVF::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) {
}

std::lock_guard<std::mutex> lk(mutex_);
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA(dataset_ptr)
index_->add(rows, (float*)p_data);
}

Expand All @@ -101,7 +101,7 @@ IVF::Query(const DatasetPtr& dataset_ptr, const Config& config) {
KNOWHERE_THROW_MSG("index not initialize or trained");
}

GETTENSOR(dataset_ptr)
GET_TENSOR_DATA(dataset_ptr)

try {
fiu_do_on("IVF.Search.throw_std_exception", throw std::exception());
Expand Down
2 changes: 1 addition & 1 deletion core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp
View file
Open in desktop
Expand Up @@ -33,7 +33,7 @@ namespace knowhere {

void
IVFPQ::Train(const DatasetPtr& dataset_ptr, const Config& config) {
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA_DIM(dataset_ptr)

faiss::MetricType metric_type = GetMetricType(config[Metric::TYPE].get<std::string>());
faiss::Index* coarse_quantizer = new faiss::IndexFlat(dim, metric_type);
Expand Down
2 changes: 1 addition & 1 deletion core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp
View file
Open in desktop
Expand Up @@ -35,7 +35,7 @@ namespace knowhere {

void
IVFSQ::Train(const DatasetPtr& dataset_ptr, const Config& config) {
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA_DIM(dataset_ptr)

// std::stringstream index_type;
// index_type << "IVF" << config[IndexParams::nlist] << ","
Expand Down
4 changes: 2 additions & 2 deletions core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp
View file
Open in desktop
Expand Up @@ -78,7 +78,7 @@ NSG::Query(const DatasetPtr& dataset_ptr, const Config& config) {
KNOWHERE_THROW_MSG("index not initialize or trained");
}

GETTENSOR(dataset_ptr)
GET_TENSOR_DATA_DIM(dataset_ptr)

try {
auto elems = rows * config[meta::TOPK].get<int64_t>();
Expand Down Expand Up @@ -139,7 +139,7 @@ NSG::Train(const DatasetPtr& dataset_ptr, const Config& config) {
b_params.out_degree = config[IndexParams::out_degree];
b_params.search_length = config[IndexParams::search_length];

GETTENSORWITHIDS(dataset_ptr)
GET_TENSOR(dataset_ptr)

impl::NsgIndex::Metric_Type metric;
auto metric_str = config[Metric::TYPE].get<std::string>();
Expand Down
4 changes: 2 additions & 2 deletions core/src/index/knowhere/knowhere/index/vector_index/adapter/SptagAdapter.cpp
View file
Open in desktop
Expand Up @@ -32,7 +32,7 @@ ConvertToMetadataSet(const DatasetPtr& dataset_ptr) {

std::shared_ptr<SPTAG::VectorSet>
ConvertToVectorSet(const DatasetPtr& dataset_ptr) {
GETTENSOR(dataset_ptr);
GET_TENSOR_DATA_DIM(dataset_ptr)
size_t num_bytes = rows * dim * sizeof(float);
SPTAG::ByteArray byte_array((uint8_t*)p_data, num_bytes, false);

Expand All @@ -42,7 +42,7 @@ ConvertToVectorSet(const DatasetPtr& dataset_ptr) {

std::vector<SPTAG::QueryResult>
ConvertToQueryResult(const DatasetPtr& dataset_ptr, const Config& config) {
GETTENSOR(dataset_ptr);
GET_TENSOR_DATA_DIM(dataset_ptr);

int64_t k = config[meta::TOPK].get<int64_t>();
std::vector<SPTAG::QueryResult> query_results(rows, SPTAG::QueryResult(nullptr, k, true));
Expand Down
18 changes: 12 additions & 6 deletions core/src/index/knowhere/knowhere/index/vector_index/adapter/VectorAdapter.h
View file
Open in desktop
Expand Up @@ -18,15 +18,21 @@
namespace milvus {
namespace knowhere {

#define GETTENSOR(dataset_ptr) \
int64_t dim = dataset_ptr->Get<int64_t>(meta::DIM); \
#define GET_TENSOR_DATA(dataset_ptr) \
int64_t rows = dataset_ptr->Get<int64_t>(meta::ROWS); \
const void* p_data = dataset_ptr->Get<const void*>(meta::TENSOR);

#define GETTENSORWITHIDS(dataset_ptr) \
int64_t dim = dataset_ptr->Get<int64_t>(meta::DIM); \
int64_t rows = dataset_ptr->Get<int64_t>(meta::ROWS); \
const void* p_data = dataset_ptr->Get<const void*>(meta::TENSOR); \
#define GET_TENSOR_DATA_DIM(dataset_ptr) \
GET_TENSOR_DATA(dataset_ptr) \
int64_t dim = dataset_ptr->Get<int64_t>(meta::DIM);

#define GET_TENSOR_DATA_ID(dataset_ptr) \
GET_TENSOR_DATA(dataset_ptr) \
const int64_t* p_ids = dataset_ptr->Get<const int64_t*>(meta::IDS);

#define GET_TENSOR(dataset_ptr) \
GET_TENSOR_DATA(dataset_ptr) \
int64_t dim = dataset_ptr->Get<int64_t>(meta::DIM); \
const int64_t* p_ids = dataset_ptr->Get<const int64_t*>(meta::IDS);

extern DatasetPtr
Expand Down
2 changes: 1 addition & 1 deletion core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp
View file
Open in desktop
Expand Up @@ -30,7 +30,7 @@ namespace knowhere {

void
GPUIVF::Train(const DatasetPtr& dataset_ptr, const Config& config) {
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA_DIM(dataset_ptr)
gpu_id_ = config[knowhere::meta::DEVICEID];

auto gpu_res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_);
Expand Down
2 changes: 1 addition & 1 deletion core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFPQ.cpp
View file
Open in desktop
Expand Up @@ -27,7 +27,7 @@ namespace knowhere {

void
GPUIVFPQ::Train(const DatasetPtr& dataset_ptr, const Config& config) {
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA_DIM(dataset_ptr)
gpu_id_ = config[knowhere::meta::DEVICEID];

auto gpu_res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_);
Expand Down
2 changes: 1 addition & 1 deletion core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQ.cpp
View file
Open in desktop
Expand Up @@ -28,7 +28,7 @@ namespace knowhere {

void
GPUIVFSQ::Train(const DatasetPtr& dataset_ptr, const Config& config) {
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA_DIM(dataset_ptr)
gpu_id_ = config[knowhere::meta::DEVICEID];

// std::stringstream index_type;
Expand Down
2 changes: 1 addition & 1 deletion core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQNR.cpp
View file
Open in desktop
Expand Up @@ -28,7 +28,7 @@ namespace knowhere {

void
GPUIVFSQNR::Train(const DatasetPtr& dataset_ptr, const Config& config) {
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA_DIM(dataset_ptr)
gpu_id_ = config[knowhere::meta::DEVICEID];

// std::stringstream index_type;
Expand Down
2 changes: 1 addition & 1 deletion core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp
View file
Open in desktop
Expand Up @@ -30,7 +30,7 @@ namespace knowhere {

void
IVFSQHybrid::Train(const DatasetPtr& dataset_ptr, const Config& config) {
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA_DIM(dataset_ptr)
gpu_id_ = config[knowhere::meta::DEVICEID];

std::stringstream index_type;
Expand Down
7 changes: 4 additions & 3 deletions core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_NM.cpp
View file
Open in desktop
Expand Up @@ -79,7 +79,8 @@ IndexHNSW_NM::Load(const BinarySet& index_binary) {
void
IndexHNSW_NM::Train(const DatasetPtr& dataset_ptr, const Config& config) {
try {
GETTENSOR(dataset_ptr)
int64_t dim = dataset_ptr->Get<int64_t>(meta::DIM);
int64_t rows = dataset_ptr->Get<int64_t>(meta::ROWS);

hnswlib_nm::SpaceInterface<float>* space;
if (config[Metric::TYPE] == Metric::L2) {
Expand All @@ -106,7 +107,7 @@ IndexHNSW_NM::Add(const DatasetPtr& dataset_ptr, const Config& config) {

std::lock_guard<std::mutex> lk(mutex_);

GETTENSORWITHIDS(dataset_ptr)
GET_TENSOR_DATA_ID(dataset_ptr)

auto base = index_->getCurrentElementCount();
auto pp_data = const_cast<void*>(p_data);
Expand All @@ -123,7 +124,7 @@ IndexHNSW_NM::Query(const DatasetPtr& dataset_ptr, const Config& config) {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize or trained");
}
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA_DIM(dataset_ptr)

size_t k = config[meta::TOPK].get<int64_t>();
size_t id_size = sizeof(int64_t) * k;
Expand Down
6 changes: 3 additions & 3 deletions core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_SQ8NR.cpp
View file
Open in desktop
Expand Up @@ -73,7 +73,7 @@ IndexHNSW_SQ8NR::Load(const BinarySet& index_binary) {
void
IndexHNSW_SQ8NR::Train(const DatasetPtr& dataset_ptr, const Config& config) {
try {
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA_DIM(dataset_ptr)

hnswlib_nm::SpaceInterface<float>* space;
if (config[Metric::TYPE] == Metric::L2) {
Expand Down Expand Up @@ -103,7 +103,7 @@ IndexHNSW_SQ8NR::Add(const DatasetPtr& dataset_ptr, const Config& config) {

std::lock_guard<std::mutex> lk(mutex_);

GETTENSORWITHIDS(dataset_ptr)
GET_TENSOR_DATA_ID(dataset_ptr)

auto base = index_->getCurrentElementCount();
auto pp_data = const_cast<void*>(p_data);
Expand All @@ -120,7 +120,7 @@ IndexHNSW_SQ8NR::Query(const DatasetPtr& dataset_ptr, const Config& config) {
if (!index_) {
KNOWHERE_THROW_MSG("index not initialize or trained");
}
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA_DIM(dataset_ptr)

size_t k = config[meta::TOPK].get<int64_t>();
size_t id_size = sizeof(int64_t) * k;
Expand Down
8 changes: 4 additions & 4 deletions core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVFSQNR_NM.cpp
View file
Open in desktop
Expand Up @@ -116,7 +116,7 @@ IVFSQNR_NM::Load(const BinarySet& binary_set) {

void
IVFSQNR_NM::Train(const DatasetPtr& dataset_ptr, const Config& config) {
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA_DIM(dataset_ptr)

faiss::MetricType metric_type = GetMetricType(config[Metric::TYPE].get<std::string>());
faiss::Index* coarse_quantizer = new faiss::IndexFlat(dim, metric_type);
Expand All @@ -134,7 +134,7 @@ IVFSQNR_NM::Add(const DatasetPtr& dataset_ptr, const Config& config) {
}

std::lock_guard<std::mutex> lk(mutex_);
GETTENSORWITHIDS(dataset_ptr)
GET_TENSOR_DATA_ID(dataset_ptr)
index_->add_with_ids_without_codes(rows, (float*)p_data, p_ids);

ArrangeCodes(dataset_ptr, config);
Expand All @@ -147,7 +147,7 @@ IVFSQNR_NM::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) {
}

std::lock_guard<std::mutex> lk(mutex_);
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA(dataset_ptr)
index_->add_without_codes(rows, (float*)p_data);

ArrangeCodes(dataset_ptr, config);
Expand Down Expand Up @@ -175,7 +175,7 @@ IVFSQNR_NM::CopyCpuToGpu(const int64_t device_id, const Config& config) {

void
IVFSQNR_NM::ArrangeCodes(const DatasetPtr& dataset_ptr, const Config& config) {
GETTENSOR(dataset_ptr)
GET_TENSOR_DATA_DIM(dataset_ptr)

// Construct arranged sq8 data from original data
const float* original_data = (const float*)p_data;
Expand Down