segment writer (#2873)

* redefine SegmentWriter Signed-off-by: yhmo <yihua.mo@zilliz.com> * redefine SegmentWriter Signed-off-by: yhmo <yihua.mo@zilliz.com> * refine code Signed-off-by: yhmo <yihua.mo@zilliz.com> * mem insert Signed-off-by: yhmo <yihua.mo@zilliz.com> * unittest Signed-off-by: yhmo <yihua.mo@zilliz.com> * typo Signed-off-by: yhmo <yihua.mo@zilliz.com> * typo Signed-off-by: yhmo <yihua.mo@zilliz.com> * fix Signed-off-by: yhmo <yihua.mo@zilliz.com> * fix some issues Signed-off-by: yhmo <yihua.mo@zilliz.com> * typo Signed-off-by: yhmo <yihua.mo@zilliz.com> * segment reader Signed-off-by: yhmo <yihua.mo@zilliz.com> * segment writer Signed-off-by: yhmo <yihua.mo@zilliz.com> * remove files Signed-off-by: yhmo <yihua.mo@zilliz.com> * remove code Signed-off-by: yhmo <yihua.mo@zilliz.com> * typo Signed-off-by: yhmo <yihua.mo@zilliz.com> * typo Signed-off-by: yhmo <yihua.mo@zilliz.com> * typo Signed-off-by: yhmo <yihua.mo@zilliz.com> * merge Signed-off-by: yhmo <yihua.mo@zilliz.com> * fix merge Signed-off-by: yhmo <yihua.mo@zilliz.com> * get entities Signed-off-by: yhmo <yihua.mo@zilliz.com>
milvus-io · Jul 17, 2020 · 708bad3 · 708bad3
1 parent 771585b
commit 708bad3
Show file tree

Hide file tree

Showing 41 changed files with 1,403 additions and 1,656 deletions.
diff --git a/core/src/codecs/snapshot/SSAttrsFormat.cpp b/core/src/codecs/snapshot/SSAttrsFormat.cpp
diff --git a/core/src/codecs/snapshot/SSAttrsFormat.h b/core/src/codecs/snapshot/SSAttrsFormat.h
diff --git a/core/src/codecs/snapshot/SSBlockFormat.cpp b/core/src/codecs/snapshot/SSBlockFormat.cpp
@@ -48,6 +48,78 @@ SSBlockFormat::read(const storage::FSHandlerPtr& fs_ptr, const std::string& file
     fs_ptr->reader_ptr_->close();
 }
 
+void
+SSBlockFormat::read(const storage::FSHandlerPtr& fs_ptr, const std::string& file_path, int64_t offset,
+                    int64_t num_bytes, std::vector<uint8_t>& raw) {
+    if (offset < 0 || num_bytes <= 0) {
+        std::string err_msg = "Invalid input to read: " + file_path;
+        LOG_ENGINE_ERROR_ << err_msg;
+        throw Exception(SERVER_INVALID_ARGUMENT, err_msg);
+    }
+
+    if (!fs_ptr->reader_ptr_->open(file_path.c_str())) {
+        std::string err_msg = "Failed to open file: " + file_path + ", error: " + std::strerror(errno);
+        LOG_ENGINE_ERROR_ << err_msg;
+        throw Exception(SERVER_CANNOT_OPEN_FILE, err_msg);
+    }
+
+    size_t total_num_bytes;
+    fs_ptr->reader_ptr_->read(&total_num_bytes, sizeof(size_t));
+
+    offset += sizeof(size_t);  // Beginning of file is num_bytes
+    if (offset + num_bytes > total_num_bytes) {
+        std::string err_msg = "Invalid input to read: " + file_path;
+        LOG_ENGINE_ERROR_ << err_msg;
+        throw Exception(SERVER_INVALID_ARGUMENT, err_msg);
+    }
+
+    raw.resize(num_bytes);
+    fs_ptr->reader_ptr_->seekg(offset);
+    fs_ptr->reader_ptr_->read(raw.data(), num_bytes);
+    fs_ptr->reader_ptr_->close();
+}
+
+void
+SSBlockFormat::read(const storage::FSHandlerPtr& fs_ptr, const std::string& file_path, const ReadRanges& read_ranges,
+                    std::vector<uint8_t>& raw) {
+    if (read_ranges.empty()) {
+        return;
+    }
+
+    if (!fs_ptr->reader_ptr_->open(file_path.c_str())) {
+        std::string err_msg = "Failed to open file: " + file_path + ", error: " + std::strerror(errno);
+        LOG_ENGINE_ERROR_ << err_msg;
+        throw Exception(SERVER_CANNOT_OPEN_FILE, err_msg);
+    }
+
+    size_t total_num_bytes;
+    fs_ptr->reader_ptr_->read(&total_num_bytes, sizeof(size_t));
+
+    int64_t total_bytes = 0;
+    for (auto& range : read_ranges) {
+        int64_t offset = range.offset_ + sizeof(size_t);
+        if (offset + range.num_bytes_ > total_num_bytes) {
+            std::string err_msg = "Invalid input to read: " + file_path;
+            LOG_ENGINE_ERROR_ << err_msg;
+            throw Exception(SERVER_INVALID_ARGUMENT, err_msg);
+        }
+
+        total_bytes += range.num_bytes_;
+    }
+
+    raw.clear();
+    raw.resize(total_bytes);
+    int64_t poz = 0;
+    for (auto& range : read_ranges) {
+        int64_t offset = range.offset_ + sizeof(size_t);
+        fs_ptr->reader_ptr_->seekg(offset);
+        fs_ptr->reader_ptr_->read(raw.data() + poz, range.num_bytes_);
+        poz += range.num_bytes_;
+    }
+
+    fs_ptr->reader_ptr_->close();
+}
+
 void
 SSBlockFormat::write(const storage::FSHandlerPtr& fs_ptr, const std::string& file_path,
                      const std::vector<uint8_t>& raw) {