Add RetiredCommitted state to consensus

doc/schemas/node_openapi.json

@@ -690,7 +690,8 @@
         "enum": [
           "Ordered",
           "Signed",
-          "Completed"
+          "Completed",
+          "RetiredCommitted"
         ],
         "type": "string"
       },

src/consensus/aft/impl/state.h

@@ -185,6 +185,9 @@ namespace aft
     std::optional<ccf::SeqNo> retirement_idx = std::nullopt;
     // Earliest index at which this node's retirement can be committed
     std::optional<ccf::SeqNo> retirement_committable_idx = std::nullopt;
+    // Index at which this node observes its retired_committed, only set when
+    // that index itself is committed
+    std::optional<ccf::SeqNo> retired_committed_idx = std::nullopt;
   };
   DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(State);
   DECLARE_JSON_REQUIRED_FIELDS(
@@ -198,5 +201,9 @@ namespace aft
     membership_state,
     committable_indices);
   DECLARE_JSON_OPTIONAL_FIELDS(
-    State, retirement_phase, retirement_idx, retirement_committable_idx);
+    State,
+    retirement_phase,
+    retirement_idx,
+    retirement_committable_idx,
+    retired_committed_idx);
 }

src/consensus/aft/raft.h

@@ -291,6 +291,18 @@ namespace aft
       return state->membership_state == kv::MembershipState::Retired;
     }
 
+    void set_retired_committed(ccf::SeqNo seqno) override
+    {
+      state->retirement_phase = kv::RetirementPhase::RetiredCommitted;
+      CCF_ASSERT_FMT(
+        state->retired_committed_idx == state->commit_idx,
+        "Retired "
+        "committed index {} does not match current commit index {}",
+        state->retired_committed_idx.value_or(0),
+        state->commit_idx);
+      state->retired_committed_idx = seqno;
+    }
+
     Index last_committable_index() const
     {
       return state->committable_indices.empty() ?

src/kv/kv_types.h

@@ -163,14 +163,16 @@ namespace kv
   {
     Ordered = 1,
     Signed = 2,
-    Completed = 3
+    Completed = 3,
+    RetiredCommitted = 4,
   };
 
   DECLARE_JSON_ENUM(
     RetirementPhase,
     {{RetirementPhase::Ordered, "Ordered"},
      {RetirementPhase::Signed, "Signed"},
-     {RetirementPhase::Completed, "Completed"}});
+     {RetirementPhase::Completed, "Completed"},
+     {RetirementPhase::RetiredCommitted, "RetiredCommitted"}});
 
   DECLARE_JSON_TYPE(Configuration);
   DECLARE_JSON_REQUIRED_FIELDS(Configuration, idx, nodes, rid);
@@ -473,6 +475,8 @@ namespace kv
     virtual void periodic_end() {}
 
     virtual void enable_all_domains() {}
+
+    virtual void set_retired_committed(ccf::SeqNo){};
   };
 
   struct PendingTxInfo

src/node/node_state.h

@@ -2144,6 +2144,29 @@ namespace ccf
             return;
           }));
 
+      network.tables->set_global_hook(
+        network.nodes.get_name(),
+        network.nodes.wrap_commit_hook(
+          [this](kv::Version hook_version, const Nodes::Write& w) {
+            for (const auto& [node_id, node_info] : w)
+            {
+              if (node_id != self)
+              {
+                // Only update our own state
+                continue;
+              }
+
+              if (node_info.has_value())
+              {
+                if (node_info->retired_committed)
+                {
+                  consensus->set_retired_committed(hook_version);
+                }
+                return;
+              }
+            }
+          }));
+
       // Service-endorsed certificate is passed to history as early as _local_
       // commit since a new node may become primary (and thus, e.g. generate
       // signatures) before the transaction that added it is _globally_

tests/infra/network.py

@@ -1046,7 +1046,7 @@ def replace_stopped_node(
                     )
         except (ValueError, TimeoutError):
             LOG.error(
-                f"NFailed to replace {node_to_retire.node_id} with {node_to_add.node_id}"
+                f"Failed to replace {node_to_retire.node_id} with {node_to_add.node_id}"
             )
             node_to_add.stop()
             raise