[C API] Implement memory estimation functions for index builders and tests

bindings/c/CMakeLists.txt

@@ -35,6 +35,7 @@ set(SVS_C_API_SOURCES
     src/svs_c.cpp
     src/dispatcher_vamana.cpp
     src/dispatcher_dynamic_vamana.cpp
+    src/data_builder.cpp
 )
 
 add_library(${TARGET_NAME} SHARED

bindings/c/include/svs/c_api/svs_c.h

@@ -355,6 +355,37 @@ SVS_API bool svs_index_builder_set_threadpool_custom(
     svs_index_builder_h builder, svs_threadpool_i pool, svs_error_h out_err /*=NULL*/
 );
 
+/// @brief Estimate the memory usage of an index based on the builder configuration and
+/// number of vectors
+/// @param builder The index builder handle
+/// @param num_vectors The number of vectors to be indexed
+/// @param out_breakdown Pointer to a structure to hold the memory breakdown
+/// @param out_err An optional error handle to capture errors
+/// @return true on success, false on failure
+SVS_API bool svs_index_builder_estimate_memory(
+    svs_index_builder_h builder,
+    size_t num_vectors,
+    svs_memory_breakdown_t* out_breakdown,
+    svs_error_h out_err /*=NULL*/
+);
+
+/// @brief Estimate the memory usage of a dynamic index based on the builder configuration,
+/// number of vectors, and block size
+/// @param builder The index builder handle
+/// @param num_vectors The number of vectors to be indexed
+/// @param blocksize_bytes The block size in bytes for dynamic index building (0 for
+/// default)
+/// @param out_breakdown Pointer to a structure to hold the memory breakdown
+/// @param out_err An optional error handle to capture errors
+/// @return true on success, false on failure
+SVS_API bool svs_index_builder_estimate_memory_dynamic(
+    svs_index_builder_h builder,
+    size_t num_vectors,
+    size_t blocksize_bytes,
+    svs_memory_breakdown_t* out_breakdown,
+    svs_error_h out_err /*=NULL*/
+);
+
 /// @brief Build an index from the provided data
 /// @param builder The index builder handle
 /// @param data Pointer to the vector data (float array)

bindings/c/src/data_builder.cpp

@@ -0,0 +1,133 @@
+/*
+ * Copyright 2026 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "data_builder.hpp"
+
+#include "storage.hpp"
+
+#include <svs/core/data/simple.h>
+#include <svs/lib/dispatcher.h>
+#include <svs/lib/misc.h>
+
+#include <variant>
+
+namespace svs::c_runtime {
+// namespace {
+template <typename DataBuilder>
+size_t
+estimate_size(DataBuilder builder, size_t num_vectors, size_t dimension, svs::lib::Empty) {
+    using allocator_type = typename DataBuilder::allocator_type;
+    static_assert(
+        !svs::data::is_blocked_v<allocator_type>,
+        "estimate_size requires a non-blocked allocator type."
+    );
+    return builder.estimate_size(num_vectors, dimension, allocator_type{});
+}
+
+template <typename DataBuilder>
+size_t estimate_blocked_size(
+    DataBuilder builder, size_t num_vectors, size_t dimension, size_t blocksize_bytes
+) {
+    using allocator_type = typename DataBuilder::allocator_type;
+    static_assert(
+        svs::data::is_blocked_v<allocator_type>,
+        "estimate_blocked_size requires a blocked allocator type."
+    );
+    svs::data::BlockingParameters block_params;
+    if (blocksize_bytes != 0) {
+        block_params.blocksize_bytes = svs::lib::prevpow2(blocksize_bytes);
+    }
+    auto allocator = allocator_type{block_params};
+    return builder.estimate_size(num_vectors, dimension, allocator);
+}
+
+template <typename Dispatcher>
+void register_data_size_specializations(Dispatcher& dispatcher) {
+    auto size_closure = [&dispatcher]<typename DataBuilder, typename = void>() {
+        dispatcher.register_target(&estimate_size<DataBuilder>);
+    };
+
+    for_simple_specializations<false>(size_closure);
+    for_leanvec_specializations<false>(size_closure);
+    for_lvq_specializations<false>(size_closure);
+    for_sq_specializations<false>(size_closure);
+
+    auto blocked_size_closure = [&dispatcher]<typename DataBuilder, typename = void>() {
+        dispatcher.register_target(&estimate_blocked_size<DataBuilder>);
+    };
+
+    for_simple_specializations<true>(blocked_size_closure);
+    for_leanvec_specializations<true>(blocked_size_closure);
+    for_lvq_specializations<true>(blocked_size_closure);
+    for_sq_specializations<true>(blocked_size_closure);
+}
+
+using BlocksizeArg = std::variant<svs::lib::Empty, size_t>;
+
+using EstimateSizeDispatcher =
+    svs::lib::Dispatcher<size_t, const Storage*, size_t, size_t, BlocksizeArg>;
+
+const EstimateSizeDispatcher& build_data_size_dispatcher() {
+    static EstimateSizeDispatcher dispatcher = [] {
+        EstimateSizeDispatcher d{};
+        register_data_size_specializations(d);
+        return d;
+    }();
+    return dispatcher;
+}
+
+size_t dispatch_data_size_estimation(
+    const Storage* storage,
+    size_t num_vectors,
+    size_t dimension,
+    BlocksizeArg blocksize_bytes
+) {
+    return build_data_size_dispatcher().invoke(
+        storage, num_vectors, dimension, blocksize_bytes
+    );
+}
+//} // namespace
+
+size_t estimate_data_size(const Storage* storage, size_t num_vectors, size_t dimension) {
+    if (storage == nullptr) {
+        throw std::invalid_argument("Storage pointer cannot be null.");
+    }
+    if (num_vectors == 0) {
+        throw std::invalid_argument("Number of vectors must be greater than zero.");
+    }
+    if (dimension == 0) {
+        throw std::invalid_argument("Dimension must be greater than zero.");
+    }
+    return dispatch_data_size_estimation(
+        storage, num_vectors, dimension, svs::lib::Empty{}
+    );
+}
+
+size_t estimate_data_size_blocked(
+    const Storage* storage, size_t num_vectors, size_t dimension, size_t blocksize_bytes
+) {
+    if (storage == nullptr) {
+        throw std::invalid_argument("Storage pointer cannot be null.");
+    }
+    if (num_vectors == 0) {
+        throw std::invalid_argument("Number of vectors must be greater than zero.");
+    }
+    if (dimension == 0) {
+        throw std::invalid_argument("Dimension must be greater than zero.");
+    }
+    return dispatch_data_size_estimation(storage, num_vectors, dimension, blocksize_bytes);
+}
+} // namespace svs::c_runtime

bindings/c/src/data_builder.hpp

@@ -19,3 +19,11 @@
 #include "data_builder/lvq.hpp"
 #include "data_builder/simple.hpp"
 #include "data_builder/sq.hpp"
+#include "storage.hpp"
+
+namespace svs::c_runtime {
+size_t estimate_data_size(const Storage* storage, size_t num_vectors, size_t dimension);
+size_t estimate_data_size_blocked(
+    const Storage* storage, size_t num_vectors, size_t dimension, size_t blocksize_bytes
+);
+} // namespace svs::c_runtime

bindings/c/src/data_builder/leanvec.hpp

@@ -19,6 +19,7 @@
 
 #include "svs/c_api/svs_c.h"
 
+#include "data_builder/lvq.hpp"
 #include "storage.hpp"
 #include "types_support.hpp"
 
@@ -75,6 +76,42 @@ class LeanVecDataBuilder {
     load(const std::filesystem::path& path, const allocator_type& allocator = {}) {
         return svs::lib::load_from_disk<data_type>(path, allocator);
     }
+
+    size_t estimate_size(
+        size_t num_vectors, size_t dimension, const allocator_type& allocator = {}
+    ) const {
+        // Current version of LeanVecDataBuilder supports LVQ-only datasets, so we can
+        // directly reuse LVQDataBuilder::estimate_size()
+        //
+        // LeanDataset uses primary-only LVQ (ResidualBits == 0), so we can use
+        // LVQDataBuilder<I1, 0> and LVQDataBuilder<I2, 0> to estimate sizes for primary and
+        // secondary datasets.
+
+        // Estimate primary size
+        using primary_data_builder = LVQDataBuilder<I1, 0, allocator_type>;
+        const auto primary_size =
+            primary_data_builder{}.estimate_size(num_vectors, leanvec_dims_, allocator);
+
+        // Estimate secondary size
+        using secondary_data_builder = LVQDataBuilder<I2, 0, allocator_type>;
+        const auto secondary_size =
+            secondary_data_builder{}.estimate_size(num_vectors, dimension, allocator);
+
+        // TODO: Fix the actual memory breakdown reported by index by implementing
+        // dataset_allocated_bytes() specialization for LeanDataset. LeanVec matrices are 2
+        // SimpleData matrices of float, each of size (dimension x leanvec_dims)
+        const size_t matrices_size = 0; // 2 * dimension * leanvec_dims_ * sizeof(float);
+
+        // LeanVec means is the vector of double of size (dimension)
+        const size_t means_size = 0; // dimension * sizeof(double);
+
+        // is_pca_ flag is a boolean, so it takes 1 byte
+        const size_t is_pca_size = 0; // sizeof(bool);
+
+        const auto total_size =
+            primary_size + secondary_size + matrices_size + means_size + is_pca_size;
+        return total_size;
+    }
 };
 
 template <size_t I1, size_t I2, typename Alloc>

bindings/c/src/data_builder/lvq.hpp

@@ -52,11 +52,18 @@ class LVQDataBuilder {
   public:
     LVQDataBuilder() {}
 
+    // Follow the logic of svs::leanvec::detail::PickContainer which looks like:
+    // "Use Turbo-encoding for 4-bit LVQ."
+    using Sequential = svs::quantization::lvq::Sequential;
+    using Turbo16x8 = svs::quantization::lvq::Turbo<16, 8>;
+    template <size_t Primary, size_t Residual>
+    using AutoStrategy = std::conditional_t<(Primary == 4), Turbo16x8, Sequential>;
+
     using data_type = svs::quantization::lvq::LVQDataset<
         PrimaryBits,
         ResidualBits,
         svs::Dynamic,
-        svs::quantization::lvq::Sequential,
+        AutoStrategy<PrimaryBits, ResidualBits>,
         Allocator>;
     using allocator_type = Allocator;
 
@@ -73,6 +80,52 @@ class LVQDataBuilder {
     load(const std::filesystem::path& path, const allocator_type& allocator = {}) {
         return svs::lib::load_from_disk<data_type>(path, allocator);
     }
+
+    static constexpr size_t primary_element_size(size_t dimension, size_t alignment = 0) {
+        using primary_type = typename data_type::primary_type;
+        using layout_type = typename primary_type::helper_type;
+        using layout_dims_type = svs::lib::MaybeStatic<data_type::extent>;
+        const auto layout_dims = layout_dims_type{dimension};
+        return primary_type::compute_data_dimensions(layout_type{layout_dims}, alignment);
+    }
+
+    static constexpr size_t residual_element_size(size_t dims) {
+        if constexpr (ResidualBits == 0) {
+            return 0;
+        } else {
+            using residual_type = typename data_type::residual_type;
+            using dims_type = svs::lib::MaybeStatic<data_type::extent>;
+            auto residual_dims = dims_type{dims};
+            return residual_type::total_bytes(residual_dims);
+        }
+    }
+
+    size_t estimate_size(
+        size_t num_vectors, size_t dimension, const allocator_type& allocator = {}
+    ) const {
+        const size_t alignment = 0; // Assuming no specific alignment for estimation
+
+        const auto primary_element_sz = primary_element_size(dimension, alignment);
+        const auto primary_size =
+            svs::c_runtime::adjust_blocked_size(num_vectors, primary_element_sz, allocator);
+
+        const auto residual_element_sz = residual_element_size(dimension);
+        const auto residual_size = svs::c_runtime::adjust_blocked_size(
+            num_vectors, residual_element_sz, allocator
+        );
+
+        // Assuming a single centroid for estimation purposes
+        const size_t num_centroids = 1; // Assuming 1 centroid for estimation
+        // TODO: Fix the actual memory breakdown reported by index by implementing
+        // dataset_allocated_bytes() specialization for LVQDataset.
+        const size_t centroid_size = 0; // Skipping centroids for estimation
+        // const auto centroid_size =
+        //     sizeof(typename data_type::centroid_type::element_type) * dimension;
+
+        const auto total_size =
+            primary_size + residual_size + num_centroids * centroid_size;
+        return total_size;
+    }
 };
 
 template <size_t PrimaryBits, size_t ResidualBits, typename Alloc>

bindings/c/src/data_builder/simple.hpp

@@ -59,6 +59,15 @@ class SimpleDataBuilder {
     load(const std::filesystem::path& path, const allocator_type& allocator = {}) {
         return svs::lib::load_from_disk<data_type>(path, allocator);
     }
+
+    size_t estimate_size(
+        size_t num_vectors, size_t dimension, const allocator_type& allocator = {}
+    ) const {
+        const auto element_size = sizeof(typename data_type::element_type) * dimension;
+        const auto total_size =
+            svs::c_runtime::adjust_blocked_size(num_vectors, element_size, allocator);
+        return total_size;
+    }
 };
 
 template <Arithmetic T, typename Alloc>

bindings/c/src/data_builder/sq.hpp

@@ -57,6 +57,18 @@ template <Arithmetic T, typename Allocator = svs::lib::Allocator<T>> class SQDat
     load(const std::filesystem::path& path, const allocator_type& allocator = {}) {
         return svs::lib::load_from_disk<data_type>(path, allocator);
     }
+
+    size_t estimate_size(
+        size_t num_vectors, size_t dimension, const allocator_type& allocator = {}
+    ) const {
+        const auto element_size = sizeof(typename data_type::element_type) * dimension;
+        const auto data_size =
+            svs::c_runtime::adjust_blocked_size(num_vectors, element_size, allocator);
+        // TODO: Fix the actual memory breakdown reported by index by implementing
+        // dataset_allocated_bytes() specialization for SQDataset.
+        const size_t scale_bias_size = 0; // sizeof(float) * 2;
+        return data_size + scale_bias_size;
+    }
 };
 
 template <Arithmetic T, typename Alloc>