Add RetiredCommitted state to consensus

[achamayou]

Exploratory change connected to #5951

• Removed kv::RetirementPhase::Committed. Introduced in Raft state refactoring #3218, to enable experimentl support for multiple reconfiguration implementations (1tx and 2tx). The corresponding membership state was removed in Raft tracing #5201, after experimental 2tx support was taken out.
• Add a RetiredCommitted state (name open to change), matching the KV flag introduced in Safe node removal #4008 to expose a safe node removal interface to operators. The matching retired_committed_index is also added to the state. This is expected to be useful to enable gating signing, acking and running for election at a point where liveness is guaranteed.

[ccf-bot]

add_finalised_state_to_raft@81163 aka 20240201.29 vs main ewma over 20 builds from 80913 to 81155

Click to see table

main

build_id	build_number	Commit latency factor	tpcc_sgx_cft^	tpcc_sgx_cft_mem	pi_basic_mt_sgx_cft^	pi_basic_mt_sgx_cft_mem	ls_sgx_cft^	ls_sgx_cft_mem	pi_ls_sgx_cft^	pi_ls_sgx_cft_mem	pi_basic_sgx_cft^	pi_basic_sgx_cft_mem	pi_basic_mt_virtual_cft^	pi_basic_js_sgx_cft^	pi_basic_js_sgx_cft_mem	tpcc_virtual_cft^	ls_jwt_sgx_cft^	ls_jwt_sgx_cft_mem	pi_ls_jwt_sgx_cft^	pi_ls_jwt_sgx_cft_mem	ls_virtual_cft^	pi_ls_virtual_cft^	pi_basic_virtual_cft^	ls_js_sgx_cft^	ls_js_sgx_cft_mem	pi_basic_js_virtual_cft^	ls_full_js_sgx_cft^	ls_full_js_sgx_cft_mem	ls_jwt_virtual_cft^	pi_ls_jwt_virtual_cft^	ls_js_virtual_cft^	ls_js_jwt_sgx_cft^	ls_js_jwt_sgx_cft_mem	ls_full_js_virtual_cft^	ls_js_jwt_virtual_cft^	hist_sgx_cft^	RB put (/s)^	CHAMP put (/s)^	RB get (/s)^	CHAMP get (/s)^	tlc_sim_traces	tlc_sim_levelmean	tlc_3node_fixed_duration_s	tlc_3node_fixed_states	tlc_atomic_reconfig_duration_s	tlc_atomic_reconfig_states	tlc_reconfig_duration_s	tlc_reconfig_states
80913	20240126.22	0.798196	5544.65	8.59996e+07	27633.1	2.30851e+07	14072.8	1.88908e+07	14171.3	1.05021e+07	15604.7	1.46964e+07	78695.8	1426.7	1.25993e+07	17357.8	6826.65	1.67936e+07	6972.1	6.30784e+06	56357.1	57377.9	62184.5	5772.85	1.67936e+07	4613	5499.35	1.67936e+07	20610.7	22195.6	17639.8	4007.51	1.67936e+07	17657.8	11838	44960.5	820210	1.18087e+06	8.1734e+06	3.06706e+07	45182	386	6	77678	157	4.37276e+06	533	1.33071e+07
80929	20240129.1	0.79025	5560.94	8.59996e+07	27801.3	2.51822e+07	14067.5	1.88908e+07	14124.3	1.05021e+07	15649.1	1.25993e+07	69311.6	1433.8	1.25993e+07	17321.1	7237.53	1.67936e+07	7150.5	6.30784e+06	55744.7	57291.2	62629	5802.21	1.67936e+07	4624.2	5482.25	1.67936e+07	21008	22043.4	20881.8	4005.69	1.67936e+07	17589.2	11830.8	43810	823514	1.17755e+06	8.13596e+06	3.06697e+07	39518	381	7	77678	157	4.37276e+06	552	1.33071e+07
80937	20240129.4	0.78287	5580.23	8.59996e+07	27193.3	2.51822e+07	13990.8	1.67936e+07	14165.4	1.05021e+07	15426.1	1.46964e+07	74944.2	1432.6	1.25993e+07	17299.5	6765.84	1.67936e+07	6957.4	6.30784e+06	56071.6	57785.4	62215.5	5805.44	1.67936e+07	4592.5	5462.25	1.88908e+07	20671.2	22288.6	20939.7	3990.7	1.67936e+07	17352.4	11726.1	47710.8	832294	1.1809e+06	8.1479e+06	3.08095e+07	41838	387	7	77678	155	4.37276e+06	551	1.33071e+07
80946	20240129.7	0.786277	5621.41	8.59996e+07	27708.8	2.51822e+07	14068.7	1.88908e+07	14089.3	1.05021e+07	15631.3	1.46964e+07	86602.4	1435.3	1.25993e+07	17437.1	6818.8	1.67936e+07	6970.9	6.30784e+06	56117.1	58060.9	62257.9	5808.9	1.67936e+07	4608.7	5498.45	1.88908e+07	21140.4	22156.2	20830.9	3982.5	1.67936e+07	17956.4	11817.4	45043.1	837351	1.1781e+06	8.153e+06	3.06468e+07	43306	383	6	77678	159	4.37276e+06	553	1.33071e+07
80954	20240129.9	0.795995	5528.42	8.59996e+07	28029.8	2.30851e+07	13964.6	1.88908e+07	14050.9	1.05021e+07	15396	1.46964e+07	74524.5	1425.8	1.25993e+07	17283.9	7202.09	1.67936e+07	6869.9	6.30784e+06	56124.4	57241.6	53736.8	5775.4	1.67936e+07	4641.7	5491.38	1.67936e+07	20831.6	22114.1	20603.8	3992.87	1.67936e+07	17453.8	11626.6	43231.7	839186	1.18597e+06	8.13525e+06	3.08132e+07	45595	389	7	77678	167	4.37276e+06	564	1.33071e+07
80964	20240130.1	0.775029	5654.41	8.59996e+07	27939.8	2.51822e+07	14041.4	1.88908e+07	14190	1.05021e+07	15622.9	1.25993e+07	81759.2	1434.3	1.25993e+07	17349.2	6856.94	1.67936e+07	6978	6.30784e+06	56180	57864.7	62566.6	5808.29	1.67936e+07	4625	5426.51	1.67936e+07	20370.9	22148.9	20438.2	3999.38	1.67936e+07	17668.8	11990.8	45266.7	804386	1.18303e+06	8.15209e+06	3.19048e+07	44358	392	6	77678	159	4.37276e+06	581	1.33071e+07
80975	20240130.6	0.82787	5579	8.59996e+07	28076.8	2.30851e+07	14118.3	1.88908e+07	14266.4	1.05021e+07	15691.5	1.46964e+07	79318.4	1436.3	1.25993e+07	17378.1	7278.09	1.67936e+07	6980.8	6.30784e+06	53240.6	57493.4	62191.5	5818.7	1.67936e+07	4592.2	5768.96	1.67936e+07	20854.3	20002.8	17228.5	4009.96	1.67936e+07	17752.1	11706.2	46062.9	833293	1.17779e+06	8.12715e+06	3.09651e+07	45137	385	6	77678	164	4.37276e+06	554	1.33071e+07
80989	20240130.12	0.784843	5504.8	8.59996e+07	28284.3	2.51822e+07	14010.7	1.88908e+07	14134.7	1.05021e+07	15620.1	1.25993e+07	85648.2	1424.9	1.25993e+07	17580.7	7202.09	1.88908e+07	6884.2	6.30784e+06	56075.2	57522.2	62184.5	5800.16	1.67936e+07	4632	5488.67	1.67936e+07	20992	21272.5	20794.6	3999.09	1.67936e+07	17603.7	11667.3	45389.1	835228	1.18493e+06	8.15076e+06	3.07489e+07	45551	386	6	77678	156	4.37276e+06	557	1.33071e+07
80995	20240130.14	0.787181	5659.57	8.59996e+07	27844.6	2.51822e+07	14098.8	1.67936e+07	14192.3	1.05021e+07	15626.6	1.46964e+07	78242.1	1428.6	1.25993e+07	17352	7239.63	1.67936e+07	6897.6	6.30784e+06	55926.3	57441.2	62110.1	5821.54	1.67936e+07	4624	5477.94	1.67936e+07	20672.4	21807.9	20799.7	3995.74	1.67936e+07	17593.9	11720.2	47653.6	835866	1.18124e+06	8.15647e+06	3.08494e+07	42258	388	7	77678	164	4.37276e+06	545	1.33071e+07
81010	20240130.20	0.797706	5665.74	8.59996e+07	28317.5	2.51822e+07	14099.1	1.67936e+07	14188.3	1.05021e+07	15685.2	1.46964e+07	85150.2	1439.3	1.25993e+07	17472.6	7275.97	1.67936e+07	6970	6.30784e+06	55750.4	58130.4	61859.1	5813.75	1.67936e+07	4586.7	5733.26	1.67936e+07	20685.9	22327.4	20877	4003.24	1.67936e+07	17610.4	11745.3	39948.5	840538	1.18233e+06	8.1491e+06	3.07125e+07	45109	394	7	77678	158	4.37276e+06	548	1.33071e+07
81020	20240130.24	0.803394	5590.79	8.59996e+07	28178.8	2.30851e+07	13982.1	1.88908e+07	14101.6	1.05021e+07	15540.2	1.25993e+07	82804.4	1426.6	1.25993e+07	17185.5	6844.82	1.67936e+07	6927.4	6.30784e+06	53316.6	58131.8	62663.5	5803.64	1.67936e+07	4579.1	5452.53	1.67936e+07	21261.2	22214.3	20865.6	3986.21	1.67936e+07	17313	11711.5	44792.7	835477	1.17224e+06	8.15342e+06	3.09398e+07	44286	379	7	77678	160	4.37276e+06	560	1.33071e+07
81037	20240131.2	0.782807	5595.55	8.59996e+07	28026.2	2.51822e+07	13996.1	1.88908e+07	14122.8	1.05021e+07	15579.2	1.46964e+07	78766.9	1435	1.25993e+07	17366.6	7217	1.67936e+07	7087	6.30784e+06	56476.7	58274.4	62357	5775.32	1.67936e+07	4636.4	5468.8	1.67936e+07	20649.9	22196.6	17589.5	4013.81	1.67936e+07	17601.5	11740.7	41791.6	830278	1.18138e+06	8.17235e+06	3.12805e+07	44114	391	6	77678	158	4.37276e+06	546	1.33071e+07
81052	20240131.8	0.797422	5640.19	8.59996e+07	27954.4	2.30851e+07	14011.7	1.88908e+07	14104.9	1.05021e+07	15545.5	1.25993e+07	82825	1431.6	1.25993e+07	17252.4	6810.67	1.67936e+07	7144.5	6.30784e+06	52840.3	57793.4	62956.6	5776.62	1.67936e+07	4677.1	5492.64	1.67936e+07	21098.2	22290.6	20895.3	4005.12	1.67936e+07	17638.4	11786.7	45970.9	834427	1.18542e+06	8.16489e+06	3.07831e+07	42274	388	7	77678	165	4.37276e+06	561	1.33071e+07
81082	20240131.21	0.775369	5624.37	8.59996e+07	28031.6	2.30851e+07	14004.3	1.88908e+07	14153.2	1.05021e+07	15519.6	1.46964e+07	85782.3	1434.6	1.25993e+07	17242.7	6814.83	1.88908e+07	6921.8	6.30784e+06	53185.6	58230.6	61698.8	5810.37	1.67936e+07	4599.1	5737.02	1.67936e+07	20956.9	22145.5	20760	3991.71	1.67936e+07	17866.7	11791.7	44870.4	836799	1.17473e+06	8.13431e+06	3.15786e+07	41737	391	7	77678	163	4.37276e+06	554	1.33071e+07
81093	20240201.3	0.777591	5608	8.59996e+07	27841.7	2.51822e+07	14013.2	1.88908e+07	14185	1.05021e+07	15594.1	1.46964e+07	83402.1	1435.3	1.25993e+07	17295.4	6857.42	1.67936e+07	7086	6.30784e+06	53364.3	57728.1	62992.6	5805.16	1.67936e+07	4659.8	5453.77	1.67936e+07	20680.4	21886.6	20679.6	3974.74	1.67936e+07	17573.8	11777.3	47043.4	832580	1.1843e+06	8.15014e+06	3.0882e+07	40666	391	6	77678	158	4.37276e+06	552	1.33071e+07
81114	20240201.11	0.822915	5513.38	8.59996e+07	28072.4	2.51822e+07	14011.9	1.88908e+07	14121.1	1.05021e+07	15543.7	1.25993e+07	79807	1422	1.25993e+07	17304.8	7183.82	1.88908e+07	7058.8	6.30784e+06	56139.4	57508.6	62598.8	5813.99	1.67936e+07	4639.2	5731.83	1.88908e+07	20556.5	21777	20887.8	3967.96	1.67936e+07	17575.8	11723.6	45387.7	836976	1.17864e+06	8.15459e+06	3.10124e+07	44839	392	7	77678	160	4.37276e+06	551	1.33071e+07
81124	20240201.15	0.764222	5616.65	8.59996e+07	28137.9	2.51822e+07	14093.5	1.88908e+07	14176.1	1.05021e+07	15633.1	1.25993e+07	96228.1	1424.1	1.25993e+07	17327.1	6803.97	1.88908e+07	6943.7	6.30784e+06	53350.9	58149.3	61837.4	5765.82	1.67936e+07	4612.9	5738.28	1.67936e+07	21209.2	21195.9	17421.1	3989.03	1.67936e+07	17629.9	11831	44049.2	834565	1.18157e+06	8.14907e+06	3.13303e+07	45276	378	5	58806	165	4.51944e+06	575	1.35909e+07
81141	20240201.21	0.830179	5603.72	8.59996e+07	28194	2.51822e+07	14032.8	1.88908e+07	14138	1.05021e+07	15624.9	1.46964e+07	69246.9	1422.7	1.25993e+07	17379.3	7222.9	1.67936e+07	6881.9	6.30784e+06	56204.6	57632.2	61922.3	5765.47	1.67936e+07	4617.9	5714.02	1.88908e+07	21011.5	22151.3	20824.9	3981.21	1.67936e+07	17266.5	11777.6	45071.3	829086	1.18319e+06	8.13505e+06	3.09207e+07	45002	374	5	58806	167	4.51944e+06	564	1.35909e+07
81143	20240201.22	0.788425	5550.38	8.59996e+07	28003.9	2.30851e+07	13997	1.88908e+07	14131.3	1.05021e+07	15659.6	1.25993e+07	76974.1	1438.8	1.25993e+07	17353.5	6788.68	1.88908e+07	6851.8	6.30784e+06	55835.3	57480.8	61038.5	5805.87	1.67936e+07	4609.9	5460.09	1.67936e+07	21136.2	21619.8	20837.9	3990.5	1.67936e+07	17714.7	11811.6	43160	827345	1.17974e+06	8.13328e+06	3.07859e+07	44907	383	5	58806	173	4.51944e+06	586	1.35909e+07
81155	20240201.27	0.754562	5512.97	8.59996e+07	27721.2	2.30851e+07	13974.9	1.88908e+07	14047.6	1.05021e+07	15473.6	1.25993e+07	79007.3	1428.8	1.25993e+07	17327.2	6809.82	1.67936e+07	6972.8	6.30784e+06	52936.4	55935.3	61111.5	5757.94	1.67936e+07	4588.6	5454.24	1.67936e+07	21159.9	21663.3	20596.9	3965.43	1.67936e+07	17353.6	11679.6	44675.3	820629	1.16845e+06	8.14927e+06	3.04445e+07	45810	375	5	58806	164	4.51944e+06	606	1.35909e+07

add_finalised_state_to_raft

build_id	build_number	Commit latency factor	tpcc_sgx_cft^	tpcc_sgx_cft_mem	ls_sgx_cft^	ls_sgx_cft_mem	pi_ls_sgx_cft^	pi_ls_sgx_cft_mem	pi_basic_sgx_cft^	pi_basic_sgx_cft_mem	pi_basic_js_sgx_cft^	pi_basic_js_sgx_cft_mem	ls_jwt_sgx_cft^	ls_jwt_sgx_cft_mem	pi_ls_jwt_sgx_cft^	pi_ls_jwt_sgx_cft_mem	ls_js_sgx_cft^	ls_js_sgx_cft_mem	ls_full_js_sgx_cft^	ls_full_js_sgx_cft_mem	ls_js_jwt_sgx_cft^	ls_js_jwt_sgx_cft_mem	hist_sgx_cft^	pi_basic_mt_sgx_cft^	pi_basic_mt_sgx_cft_mem	tlc_3node_fixed_duration_s	tlc_3node_fixed_states	tlc_atomic_reconfig_duration_s	tlc_atomic_reconfig_states	tlc_reconfig_duration_s	tlc_reconfig_states	RB put (/s)^	CHAMP put (/s)^	RB get (/s)^	CHAMP get (/s)^	tlc_sim_traces	tlc_sim_levelmean	pi_basic_mt_virtual_cft^	tpcc_virtual_cft^	ls_virtual_cft^	pi_ls_virtual_cft^	pi_basic_virtual_cft^	pi_basic_js_virtual_cft^	ls_jwt_virtual_cft^	pi_ls_jwt_virtual_cft^	ls_js_virtual_cft^	ls_full_js_virtual_cft^	ls_js_jwt_virtual_cft^
81057	20240131.9	0.770284	5588.36	8.59996e+07	14011.6	1.67936e+07	14155	1.05021e+07	15503.6	1.46964e+07	1431.9	1.25993e+07	7206.44	1.67936e+07	7067.3	6.30784e+06	5766.77	1.67936e+07	5506.67	1.67936e+07	4014.21	1.67936e+07	45375.5	27855	2.51822e+07	6	77678	152	4.37276e+06	547	1.33071e+07	837391	1.17859e+06	8.14175e+06	3.1973e+07	44069	385	77786.1	17105.3	52898.4	56111.4	61512.9	4587.6	21177.4	21493.8	18157.7	17476.1	11800.2
81111	20240201.9	0.790758	5626.21	8.59996e+07	14060.3	1.88908e+07	14115.6	1.05021e+07	15605	1.46964e+07	1434.6	1.25993e+07	6778.12	1.67936e+07	6929.8	6.30784e+06	5811.06	1.67936e+07	5473.42	1.67936e+07	4003.36	1.67936e+07	42676.1	28072.3	2.51822e+07	7	77678	165	4.37276e+06	578	1.33071e+07	828032	1.18384e+06	8.15352e+06	3.11858e+07	46132	386	81131.8	17359.9	52783.4	56879.6	61487.3	4560	21121.4	22204	17730.7	17450.3	11719.6
81148	20240201.23	0.812637	5606.47	8.59996e+07	14106.8	1.88908e+07	14196.6	1.05021e+07	15624.3	1.25993e+07	1434.7	1.05021e+07	6924.91	1.88908e+07	6921.1	6.30784e+06	5797.64	1.67936e+07	5488.87	1.67936e+07	4001.99	1.67936e+07	45466.9	28005.8	2.51822e+07	5	58806	164	4.51944e+06	593	1.35909e+07	837226	1.18355e+06	8.12692e+06	3.23334e+07	45072	381	85030.5	17310.8	53259.5	56188.6	61617.2	4577.3	20587.3	22064.3	17685.4	17555.3	11624.9
81163	20240201.29	0.812256	5567.35	8.59996e+07	14064.5	1.88908e+07	14210.9	1.05021e+07	15565.7	1.46964e+07	1424.2	1.25993e+07	6866.82	1.88908e+07	7036.5	6.30784e+06	5799.1	1.67936e+07	5478.6	1.67936e+07	3987.62	1.67936e+07	45276.6	27752.4	2.30851e+07	5	58806	165	4.51944e+06	579	1.35909e+07	839761	1.17119e+06	8.15628e+06	3.06316e+07	43662	378	83057.8	17221.6	53193.2	56497.5	62908.4	4606.1	20664.3	22274.2	20675.4	17672.8	11757.4