From ef52a5ba19e4c840777d6cbfab017eb61c533327 Mon Sep 17 00:00:00 2001 From: Buqian Zheng Date: Wed, 15 Nov 2023 17:45:43 +0800 Subject: [PATCH] Move read/writeBinaryPOD from hnswlib to knowhere utils. Signed-off-by: Buqian Zheng --- include/knowhere/utils.h | 12 ++++++++++++ thirdparty/hnswlib/hnswlib/hnswalg.h | 4 ++++ thirdparty/hnswlib/hnswlib/hnswlib.h | 24 ------------------------ 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/include/knowhere/utils.h b/include/knowhere/utils.h index efc80d5c..a6ff6bed 100644 --- a/include/knowhere/utils.h +++ b/include/knowhere/utils.h @@ -80,4 +80,16 @@ ConvertIVFFlat(const BinarySet& binset, const MetricType metric_type, const uint bool UseDiskLoad(const std::string& index_type, const int32_t& /*version*/); +template +static void +writeBinaryPOD(W& out, const T& podRef) { + out.write((char*)&podRef, sizeof(T)); +} + +template +static void +readBinaryPOD(R& in, T& podRef) { + in.read((char*)&podRef, sizeof(T)); +} + } // namespace knowhere diff --git a/thirdparty/hnswlib/hnswlib/hnswalg.h b/thirdparty/hnswlib/hnswlib/hnswalg.h index 1dfbc1ac..c71ed51b 100644 --- a/thirdparty/hnswlib/hnswlib/hnswalg.h +++ b/thirdparty/hnswlib/hnswlib/hnswalg.h @@ -27,6 +27,7 @@ #include "hnswlib.h" #include "io/memory_io.h" #include "knowhere/config.h" +#include "knowhere/utils.h" #include "knowhere/heap.h" #include "neighbor.h" #include "visited_list_pool.h" @@ -661,6 +662,7 @@ class HierarchicalNSW : public AlgorithmInterface { void loadIndex(const std::string& location, const knowhere::Config& config, size_t max_elements_i = 0) { + using knowhere::readBinaryPOD; auto cfg = static_cast(config); auto input = knowhere::FileReader(location); @@ -770,6 +772,7 @@ class HierarchicalNSW : public AlgorithmInterface { void saveIndex(knowhere::MemoryIOWriter& output) { + using knowhere::writeBinaryPOD; // write l2/ip calculator writeBinaryPOD(output, metric_type_); writeBinaryPOD(output, data_size_); @@ -807,6 +810,7 @@ class HierarchicalNSW : public AlgorithmInterface { void loadIndex(knowhere::MemoryIOReader& input, size_t max_elements_i = 0) { + using knowhere::readBinaryPOD; // linxj: init with metrictype size_t dim; readBinaryPOD(input, metric_type_); diff --git a/thirdparty/hnswlib/hnswlib/hnswlib.h b/thirdparty/hnswlib/hnswlib/hnswlib.h index 482f1545..74104381 100644 --- a/thirdparty/hnswlib/hnswlib/hnswlib.h +++ b/thirdparty/hnswlib/hnswlib/hnswlib.h @@ -148,30 +148,6 @@ class pairGreater { } }; -template -static void -writeBinaryPOD(std::ostream& out, const T& podRef) { - out.write((char*)&podRef, sizeof(T)); -} - -template -static void -readBinaryPOD(std::istream& in, T& podRef) { - in.read((char*)&podRef, sizeof(T)); -} - -template -static void -writeBinaryPOD(W& out, const T& podRef) { - out.write((char*)&podRef, sizeof(T)); -} - -template -static void -readBinaryPOD(R& in, T& podRef) { - in.read((char*)&podRef, sizeof(T)); -} - template using DISTFUNC = MTYPE (*)(const void*, const void*, const void*);