fix: fix bug that set incorrect info to columnbase (#34621)

pr: #34428 Signed-off-by: luzhang <luzhang@zilliz.com> Co-authored-by: luzhang <luzhang@zilliz.com>
milvus-io · Jul 12, 2024 · 91952e8 · 91952e8
1 parent 9bcab2e
commit 91952e8
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 19 deletions.
diff --git a/internal/core/src/common/FieldMeta.h b/internal/core/src/common/FieldMeta.h
@@ -136,6 +136,7 @@ class FieldMeta {
         if (is_vector()) {
             return GetDataTypeSize(type_, get_dim());
         } else if (is_string()) {
+            Assert(string_info_.has_value());
             return string_info_->max_length;
         } else if (IsVariableDataType(type_)) {
             return type_ == DataType::ARRAY ? ARRAY_SIZE : JSON_SIZE;

diff --git a/internal/core/src/mmap/Column.h b/internal/core/src/mmap/Column.h
@@ -64,16 +64,16 @@ class ColumnBase {
     };
     // memory mode ctor
     ColumnBase(size_t reserve, const FieldMeta& field_meta)
-        : type_size_(IsSparseFloatVectorDataType(field_meta.get_data_type())
-                         ? 1
-                         : field_meta.get_sizeof()),
-          mapping_type_(MappingType::MAP_WITH_ANONYMOUS) {
-        SetPaddingSize(field_meta.get_data_type());
+        : mapping_type_(MappingType::MAP_WITH_ANONYMOUS) {
+        auto data_type = field_meta.get_data_type();
+        SetPaddingSize(data_type);
 
-        if (IsVariableDataType(field_meta.get_data_type())) {
+        if (IsVariableDataType(data_type)) {
             return;
         }
 
+        type_size_ = field_meta.get_sizeof();
+
         cap_size_ = type_size_ * reserve;
 
         // use anon mapping so we are able to free these memory with munmap only
@@ -120,12 +120,13 @@ class ColumnBase {
     // mmap mode ctor
     // User must call Seal to build the view for variable length column.
     ColumnBase(const File& file, size_t size, const FieldMeta& field_meta)
-        : type_size_(IsSparseFloatVectorDataType(field_meta.get_data_type())
-                         ? 1
-                         : field_meta.get_sizeof()),
-          mapping_type_(MappingType::MAP_WITH_FILE),
-          num_rows_(size / type_size_) {
-        SetPaddingSize(field_meta.get_data_type());
+        : mapping_type_(MappingType::MAP_WITH_FILE) {
+        auto data_type = field_meta.get_data_type();
+        SetPaddingSize(data_type);
+        if (!IsVariableDataType(data_type)) {
+            type_size_ = field_meta.get_sizeof();
+            num_rows_ = size / type_size_;
+        }
 
         size_ = size;
         cap_size_ = size;
@@ -148,19 +149,18 @@ class ColumnBase {
                size_t size,
                int dim,
                const DataType& data_type)
-        : type_size_(IsSparseFloatVectorDataType(data_type)
-                         ? 1
-                         : GetDataTypeSize(data_type, dim)),
-          num_rows_(
-              IsSparseFloatVectorDataType(data_type) ? 1 : (size / type_size_)),
-          size_(size),
+        : size_(size),
           cap_size_(size),
           mapping_type_(MappingType::MAP_WITH_FILE) {
         SetPaddingSize(data_type);
 
         // use exact same size of file, padding shall be written in file already
         // see also https://github.com/milvus-io/milvus/issues/34442
         size_t mapped_size = cap_size_;
+        if (!IsVariableDataType(data_type)) {
+            type_size_ = GetDataTypeSize(data_type, dim);
+            num_rows_ = size / type_size_;
+        }
         data_ = static_cast<char*>(mmap(
             nullptr, mapped_size, PROT_READ, MAP_SHARED, file.Descriptor(), 0));
         AssertInfo(data_ != MAP_FAILED,
@@ -355,7 +355,7 @@ class ColumnBase {
     size_t cap_size_{0};
     size_t padding_{0};
     // type_size_ is not used for sparse float vector column.
-    const size_t type_size_{1};
+    size_t type_size_{1};
     size_t num_rows_{0};
 
     // length in bytes