SDSTOR-22799 feat: chunk ownership tracking, remove GC shard header/footer I/O, blob commit validation

conanfile.py

@@ -10,7 +10,7 @@
 
 class HomeObjectConan(ConanFile):
     name = "homeobject"
-    version = "4.2.2"
+    version = "4.2.3"
 
     homepage = "https://github.com/eBay/HomeObject"
     description = "Blob Store built on HomeStore"

src/lib/homestore_backend/CMakeLists.txt

@@ -177,3 +177,17 @@ add_test(NAME HomestoreTestGC_ShardRaceTests
         --override_config hs_backend_config.gc_garbage_rate_threshold=0
         --override_config hs_backend_config.gc_scan_interval_sec=5
         --gtest_filter=HomeObjectFixture.StalePChunkRouteAfterGC:HomeObjectFixture.StaleBlobRouteAfterSealAndGC)
+
+# All tests share disk, port, and shared-memory resources, so run them serially to avoid parallel conflicts.
+set_tests_properties(
+    HomestoreTestBlob
+    HomestoreTestMisc
+    HomestoreTestReplaceMember
+    HomestoreTestReplaceMemberWithBaselineResync
+    HomestoreTestBaselineResync
+    HomestoreResyncTestWithFollowerRestart
+    HomestoreResyncTestWithLeaderRestart
+    FetchDataWithOriginatorGC
+    HomestoreTestGC
+    HomestoreTestGC_ShardRaceTests
+    PROPERTIES RUN_SERIAL TRUE)

src/lib/homestore_backend/gc_manager.hpp

@@ -291,10 +291,6 @@ class GCManager {
 
         pdev_gc_metrics& metrics() { return metrics_; }
 
-    private:
-        // utils
-        sisl::sg_list generate_shard_super_blk_sg_list(shard_id_t shard_id);
-
     private:
         friend class gc_task_guard;
         uint32_t m_pdev_id;

src/lib/homestore_backend/heap_chunk_selector.cpp

@@ -44,34 +44,10 @@ void HeapChunkSelector::add_chunk_internal(const chunk_num_t p_chunk_id, bool ad
     }
 }
 
-// select_chunk will only be called in homestore when creating a shard.
+// select_chunk will never be called in homestore since create shard is log only.
 csharedChunk HeapChunkSelector::select_chunk(homestore::blk_count_t count, const homestore::blk_alloc_hints& hint) {
-    auto& chunkIdHint = hint.chunk_id_hint;
-    if (chunkIdHint.has_value()) {
-        LOGWARNMOD(homeobject, "should not allocated a chunk with exiting chunkIdHint={} in hint!",
-                   chunkIdHint.value());
-        return nullptr;
-    }
-
-    if (!hint.application_hint.has_value()) {
-        LOGWARNMOD(homeobject, "should not allocated a chunk without exiting application_hint in hint!");
-        return nullptr;
-    } else {
-        // Both chunk_num_t and pg_id_t are of type uint16_t.
-        static_assert(std::is_same< pg_id_t, uint16_t >::value, "pg_id_t is not uint16_t");
-        static_assert(std::is_same< homestore::chunk_num_t, uint16_t >::value, "chunk_num_t is not uint16_t");
-        auto application_hint = hint.application_hint.value();
-        pg_id_t pg_id = (uint16_t)(application_hint >> 16 & 0xFFFF);
-        homestore::chunk_num_t v_chunk_id = (uint16_t)(application_hint & 0xFFFF);
-        auto exVChunk = select_specific_chunk(pg_id, v_chunk_id);
-        if (exVChunk == nullptr) {
-            LOGWARNMOD(homeobject, "failed to select chunk with pg_id={} and v_chunk_id={} from application_hint={}",
-                       pg_id, v_chunk_id, application_hint);
-            return nullptr;
-        } else {
-            return exVChunk->get_internal_chunk();
-        }
-    }
+    RELEASE_ASSERT(false, "create shard is log only, this function should never be called");
+    return nullptr;
 }
 
 bool HeapChunkSelector::try_mark_chunk_to_gc_state(const chunk_num_t chunk_id, bool force) {
@@ -114,9 +90,9 @@ void HeapChunkSelector::mark_chunk_out_of_gc_state(const chunk_num_t chunk_id, c
 }
 
 homestore::cshared< HeapChunkSelector::ExtendedVChunk >
-HeapChunkSelector::select_specific_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id) {
+HeapChunkSelector::select_specific_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id,
+                                         const shard_id_t owner_shard_id) {
     homestore::shared< ExtendedVChunk > chunk;
-
     while (true) {
         {
             std::unique_lock lock_guard(m_chunk_selector_mtx);
@@ -125,36 +101,34 @@ HeapChunkSelector::select_specific_chunk(const pg_id_t pg_id, const chunk_num_t
                 LOGWARNMOD(homeobject, "No pg found for pg={}", pg_id);
                 return nullptr;
             }
-
             auto pg_chunk_collection = pg_it->second;
             auto& pg_chunks = pg_chunk_collection->m_pg_chunks;
             std::scoped_lock lock(pg_chunk_collection->mtx);
             if (v_chunk_id >= pg_chunks.size()) {
                 LOGWARNMOD(homeobject, "No chunk found for v_chunk_id={}", v_chunk_id);
                 return nullptr;
             }
-
             chunk = pg_chunks[v_chunk_id];
-
             if (chunk->m_state == ChunkState::GC) {
                 LOGDEBUGMOD(homeobject, "v_chunk_id={} for pg={} is pchunk_id={}, in GC state, wait and retry!",
                             v_chunk_id, pg_id, chunk->get_chunk_id());
-            } else {
-                if (chunk->m_state == ChunkState::AVAILABLE) {
-                    chunk->m_state = ChunkState::INUSE;
-                    --pg_chunk_collection->available_num_chunks;
-                    pg_chunk_collection->available_blk_count -= chunk->available_blks();
-                }
+            } else if (chunk->m_state == ChunkState::AVAILABLE) {
+                chunk->m_state = ChunkState::INUSE;
+                chunk->m_owner_shard_id = owner_shard_id;
+                --pg_chunk_collection->available_num_chunks;
+                pg_chunk_collection->available_blk_count -= chunk->available_blks();
                 break;
+            } else {
+                // INUSE: same owner → idempotent; different owner → error.
+                if (chunk->m_owner_shard_id == owner_shard_id) { break; }
+                LOGWARNMOD(homeobject,
+                           "v_chunk_id={} for pg={} is INUSE by shard=0x{:x}, cannot select for shard=0x{:x}",
+                           v_chunk_id, pg_id, chunk->m_owner_shard_id.value_or(0), owner_shard_id);
+                return nullptr;
             }
         }
-
-        // if the chunk is not available, probably being gc. we wait for a while and retry
         std::this_thread::sleep_for(std::chrono::seconds(1));
     }
-
-    LOGDEBUGMOD(homeobject, "chunk={} is selected for v_chunk_id={}, pg={}", chunk->get_chunk_id(), v_chunk_id, pg_id);
-
     return chunk;
 }
 
@@ -164,14 +138,14 @@ void HeapChunkSelector::foreach_chunks(std::function< void(csharedChunk&) >&& cb
                   [cb = std::move(cb)](auto& p) { cb(p.second->get_internal_chunk()); });
 }
 
-bool HeapChunkSelector::release_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id) {
+bool HeapChunkSelector::release_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id,
+                                      const shard_id_t owner_shard_id) {
     std::unique_lock lock_guard(m_chunk_selector_mtx);
     auto pg_it = m_per_pg_chunks.find(pg_id);
     if (pg_it == m_per_pg_chunks.end()) {
         LOGWARNMOD(homeobject, "No pg found for pg={}", pg_id);
         return false;
     }
-
     auto pg_chunk_collection = pg_it->second;
     auto& pg_chunks = pg_chunk_collection->m_pg_chunks;
     if (v_chunk_id >= pg_chunks.size()) {
@@ -180,8 +154,17 @@ bool HeapChunkSelector::release_chunk(const pg_id_t pg_id, const chunk_num_t v_c
     }
     std::scoped_lock lock(pg_chunk_collection->mtx);
     auto chunk = pg_chunks[v_chunk_id];
+    if (chunk->m_state == ChunkState::AVAILABLE) {
+        return true; // idempotent for replay
+    }
     if (chunk->m_state == ChunkState::INUSE) {
+        if (chunk->m_owner_shard_id != owner_shard_id) {
+            LOGWARNMOD(homeobject, "v_chunk_id={} for pg={} is owned by shard=0x{:x}, release by shard=0x{:x} rejected",
+                       v_chunk_id, pg_id, chunk->m_owner_shard_id.value_or(0), owner_shard_id);
+            return false;
+        }
         chunk->m_state = ChunkState::AVAILABLE;
+        chunk->m_owner_shard_id.reset();
         ++pg_chunk_collection->available_num_chunks;
         pg_chunk_collection->available_blk_count += chunk->available_blks();
     }
@@ -228,6 +211,7 @@ bool HeapChunkSelector::return_pg_chunks_to_dev_heap(const pg_id_t pg_id) {
             } // with shard which should be first
             chunk->m_pg_id = std::nullopt;
             chunk->m_v_chunk_id = std::nullopt;
+            chunk->m_owner_shard_id = std::nullopt;
 
             pdev_heap->m_heap.emplace(chunk);
             pdev_heap->available_blk_count += chunk->available_blks();
@@ -347,10 +331,14 @@ void HeapChunkSelector::update_vchunk_info_after_gc(const chunk_num_t move_from_
     // 1 change the pg_id and vchunk_id of the move_to_chunk according to metablk
     move_to_vchunk->m_pg_id = pg_id;
     move_to_vchunk->m_v_chunk_id = vchunk_id;
+    // GC transfers data but not ownership: move_to inherits move_from's owner_shard_id unchanged.
+    // (nullopt for normal GC since move_from was AVAILABLE; open shard id for EGC.)
+    move_to_vchunk->m_owner_shard_id = move_from_vchunk->m_owner_shard_id;
 
     // 2 update the chunk state of move_from_chunk, now it is a reserved chunk
     move_from_vchunk->m_pg_id = std::nullopt;
     move_from_vchunk->m_v_chunk_id = std::nullopt;
+    move_from_vchunk->m_owner_shard_id = std::nullopt;
 
     // 3 update the state of move_to_chunk, so that it can be used for creating shard or putting blob. we need to do
     // this after reserved_chunk meta blk is updated, so that if crash happens, we recovery the move_to_chunk is the
@@ -471,8 +459,7 @@ void HeapChunkSelector::build_pdev_available_chunk_heap() {
     }
 }
 
-bool HeapChunkSelector::recover_pg_chunks_states(pg_id_t pg_id,
-                                                 const std::unordered_set< chunk_num_t >& excluding_v_chunk_ids) {
+bool HeapChunkSelector::recover_pg_chunks_states(pg_id_t pg_id) {
     std::unique_lock lock_guard(m_chunk_selector_mtx);
     auto pg_it = m_per_pg_chunks.find(pg_id);
     if (pg_it == m_per_pg_chunks.end()) {
@@ -487,14 +474,8 @@ bool HeapChunkSelector::recover_pg_chunks_states(pg_id_t pg_id,
     for (size_t v_chunk_id = 0; v_chunk_id < pg_chunks.size(); ++v_chunk_id) {
         auto chunk = pg_chunks[v_chunk_id];
         pg_chunk_collection->m_total_blks += chunk->get_total_blks();
-        if (excluding_v_chunk_ids.find(v_chunk_id) == excluding_v_chunk_ids.end()) {
-            chunk->m_state = ChunkState::AVAILABLE;
-            ++pg_chunk_collection->available_num_chunks;
-            pg_chunk_collection->available_blk_count += chunk->available_blks();
-
-        } else {
-            chunk->m_state = ChunkState::INUSE;
-        }
+        ++pg_chunk_collection->available_num_chunks;
+        pg_chunk_collection->available_blk_count += chunk->available_blks();
     }
     return true;
 }
@@ -519,7 +500,7 @@ std::shared_ptr< const std::vector< homestore::chunk_num_t > > HeapChunkSelector
 }
 
 homestore::cshared< HeapChunkSelector::ExtendedVChunk >
-HeapChunkSelector::pick_most_available_blk_chunk(uint64_t ctx, pg_id_t pg_id) {
+HeapChunkSelector::pick_most_available_blk_chunk(shard_id_t shard_id, pg_id_t pg_id) {
     std::shared_lock lock_guard(m_chunk_selector_mtx);
     auto pg_it = m_per_pg_chunks.find(pg_id);
     if (pg_it == m_per_pg_chunks.end()) {
@@ -535,16 +516,18 @@ HeapChunkSelector::pick_most_available_blk_chunk(uint64_t ctx, pg_id_t pg_id) {
                              return !a->available() || (b->available() && a->available_blks() < b->available_blks());
                          });
     if (!(*max_it)->available()) {
-        LOGWARNMOD(homeobject, "No available chunk for pg={}, ctx=0x{:x}", pg_id, ctx);
+        LOGWARNMOD(homeobject, "No available chunk for pg={}, shard_id=0x{:x}", pg_id, shard_id);
         return nullptr;
     }
     auto v_chunk_id = std::distance(pg_chunks.begin(), max_it);
-    LOGDEBUGMOD(homeobject, "Picked v_chunk_id={} : [p_chunk_id={}, avail={}], ctx=0x{:x}", v_chunk_id,
-                pg_chunks[v_chunk_id]->get_chunk_id(), pg_chunks[v_chunk_id]->available_blks(), ctx);
-    pg_chunks[v_chunk_id]->m_state = ChunkState::INUSE;
+    auto& picked = pg_chunks[v_chunk_id];
+    LOGDEBUGMOD(homeobject, "Picked v_chunk_id={} : [p_chunk_id={}, avail={}], shard_id=0x{:x}", v_chunk_id,
+                picked->get_chunk_id(), picked->available_blks(), shard_id);
+    picked->m_state = ChunkState::INUSE;
+    picked->m_owner_shard_id = shard_id;
     --pg_chunk_collection->available_num_chunks;
-    pg_chunk_collection->available_blk_count -= pg_chunks[v_chunk_id]->available_blks();
-    return pg_chunks[v_chunk_id];
+    pg_chunk_collection->available_blk_count -= picked->available_blks();
+    return picked;
 }
 
 // return the maximum number of chunks that can be allocated on pdev

src/lib/homestore_backend/heap_chunk_selector.h

@@ -32,11 +32,12 @@ class HeapChunkSelector : public homestore::ChunkSelector {
     class ExtendedVChunk : public VChunk {
     public:
         ExtendedVChunk(csharedChunk const& chunk) :
-                VChunk(chunk), m_state(ChunkState::AVAILABLE), m_pg_id(), m_v_chunk_id() {}
+                VChunk(chunk), m_state(ChunkState::AVAILABLE), m_pg_id(), m_v_chunk_id(), m_owner_shard_id() {}
         ~ExtendedVChunk() = default;
         ChunkState m_state;
         std::optional< pg_id_t > m_pg_id;
         std::optional< chunk_num_t > m_v_chunk_id;
+        std::optional< shard_id_t > m_owner_shard_id; // runtime only, not persisted
         bool available() const { return m_state == ChunkState::AVAILABLE; }
     };
 
@@ -73,9 +74,12 @@ class HeapChunkSelector : public homestore::ChunkSelector {
 
     csharedChunk select_chunk([[maybe_unused]] homestore::blk_count_t nblks, const homestore::blk_alloc_hints& hints);
 
-    // this function will be used by create shard or recovery flow to mark one specific chunk to be busy, caller should
-    // be responsible to use release_chunk() interface to release it when no longer to use the chunk anymore.
-    homestore::cshared< ExtendedVChunk > select_specific_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id);
+    // State machine: GC->wait+retry, AVAILABLE->INUSE(owner=shard), INUSE+same_owner->success,
+    // INUSE+other_owner->fail.
+    // During shard creation, leader touches the target chunk in two steps:
+    // pick_most_available_blk_chunk() then select_specific_chunk(). Follower only runs select_specific_chunk().
+    homestore::cshared< ExtendedVChunk > select_specific_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id,
+                                                               const shard_id_t owner_shard_id);
 
     /**
      * try to mark a chunk as gc state, so that it will not be selected by any creating shard.
@@ -89,9 +93,9 @@ class HeapChunkSelector : public homestore::ChunkSelector {
 
     void mark_chunk_out_of_gc_state(const chunk_num_t chunk_id, const ChunkState final_state, const uint64_t task_id);
 
-    // This function returns a chunk back to ChunkSelector.
-    // It is used in two scenarios: 1. seal shard  2. create shard rollback
-    bool release_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id);
+    // Release a vchunk back to AVAILABLE. Idempotent if already AVAILABLE (safe for replay).
+    // INUSE + wrong owner → logs warning, returns false.
+    bool release_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id, const shard_id_t owner_shard_id);
 
     bool reset_pg_chunks(pg_id_t pg_id);
 
@@ -122,20 +126,20 @@ class HeapChunkSelector : public homestore::ChunkSelector {
     /**
      * pop pg top chunk
      *
-     * @param ctx  only for logging.
+     * @param shard_id  select chunk for this shard
      * @param pg_id The ID of the pg.
      * @return The most available ExtendedVChunk for the pg, or nullptr if no chunk is available.
      */
-    homestore::cshared< ExtendedVChunk > pick_most_available_blk_chunk(uint64_t ctx, pg_id_t pg_id);
+    homestore::cshared< ExtendedVChunk > pick_most_available_blk_chunk(shard_id_t shard_id, pg_id_t pg_id);
 
     // this should be called on each pg meta blk found
     bool recover_pg_chunks(pg_id_t pg_id, std::vector< chunk_num_t >&& p_chunk_ids);
 
     // this should be called after all pg meta blk and gc reserved chunk recovered
     void build_pdev_available_chunk_heap();
 
-    // this should be called after ShardManager is initialized and get all the open shards
-    bool recover_pg_chunks_states(pg_id_t pg_id, const std::unordered_set< chunk_num_t >& excluding_v_chunk_ids);
+    // this should be called after ShardManager is initialized
+    bool recover_pg_chunks_states(pg_id_t pg_id);
 
     /**
      * Returns the number of available blocks of the given device id.

src/lib/homestore_backend/hs_blob_manager.cpp

@@ -229,6 +229,22 @@ bool HSHomeObject::local_add_blob_info(pg_id_t const pg_id, BlobInfo const& blob
         return false;
     }
     if (!exist_already) {
+        // Validate the blob was written to the pchunk the shard records.
+        // Catches GC-related inconsistencies (EGC moved chunk while blob was in-flight).
+        homestore::chunk_num_t sb_pchunk_id;
+        {
+            std::shared_lock lock_guard(_shard_lock);
+            const auto iter = _shard_map.find(blob_info.shard_id);
+            RELEASE_ASSERT(iter != _shard_map.end(), "shardID=0x{:x}, pg={}, shard=0x{:x}, shard does not exist",
+                           blob_info.shard_id, (blob_info.shard_id >> homeobject::shard_width),
+                           (blob_info.shard_id & homeobject::shard_mask));
+            sb_pchunk_id = d_cast< HS_Shard* >((*iter->second).get())->p_chunk_id();
+        }
+        RELEASE_ASSERT(blob_info.pbas.chunk_num() == sb_pchunk_id,
+                       "traceID={}, commit-time pchunk mismatch: blob pchunk={} shard pchunk={} "
+                       "shard=0x{:x} blob={} pg={}",
+                       tid, blob_info.pbas.chunk_num(), sb_pchunk_id, blob_info.shard_id, blob_info.blob_id, pg_id);
+
         // The PG superblock (durable entities) will be persisted as part of HS_CLIENT Checkpoint, which is always
         // done ahead of the Index Checkpoint. Hence, if the index already has this entity, whatever durable
         // counters updated as part of the update would have been persisted already in PG superblock. So if we were
@@ -280,8 +296,8 @@ void HSHomeObject::on_blob_put_commit(int64_t lsn, sisl::blob const& header, sis
 
     int64_t shard_sealed_lsn;
     {
-        std::scoped_lock lock_guard(_shard_lock);
-        auto iter = _shard_map.find(shard_id);
+        std::shared_lock lock_guard(_shard_lock);
+        const auto iter = _shard_map.find(shard_id);
         RELEASE_ASSERT(iter != _shard_map.end(), "shardID=0x{:x}, pg={}, shard=0x{:x}, shard does not exist", shard_id,
                        (shard_id >> homeobject::shard_width), (shard_id & homeobject::shard_mask));
         shard_sealed_lsn = (*iter->second)->info.sealed_lsn;