Improved the cluster raft and tests covarage and performance now the code is more stable when running in a cluster

.config/nextest.toml

@@ -3,38 +3,119 @@
 retries = 3
 test-threads = 15
 
+[test-groups]
+# Keep these known noisy shared-cluster/perf tests from overlapping with each
+# other, while still allowing unrelated tests to use the remaining default
+# concurrency budget.
+stateful-heavy = { max-threads = 1 }
+
 [[profile.default.overrides]]
 filter = 'test(test_scenario_08_subscription_reconnect)'
 slow-timeout = { period = "60s", terminate-after = 2 }
 
 [[profile.default.overrides]]
 filter = 'test(test_setup_complete_environment)'
-threads-required = 15
+test-group = "stateful-heavy"
 
 [[profile.default.overrides]]
 filter = 'test(idle_autocommit_transaction_checks_add_no_extra_allocations)'
 # This perf/allocation regression is stable in isolation but noisy under full workspace contention.
-threads-required = 15
+test-group = "stateful-heavy"
 
 [[profile.default.overrides]]
 filter = 'test(autocommit_read_write_latency_regression_stays_within_five_percent)'
 # This perf regression compares two nearly identical code paths and should run in isolation.
-threads-required = 15
+test-group = "stateful-heavy"
 
 [[profile.default.overrides]]
 filter = 'test(e2e_perf_sequential_insert_100)'
 # This PG perf check is stable in isolation but noisy under full-suite contention.
-threads-required = 15
+test-group = "stateful-heavy"
 
 [[profile.default.overrides]]
 filter = 'test(e2e_perf_cross_verify_latency)'
 # This latency check is sensitive to concurrent heavy tests and should run in isolation.
-threads-required = 15
+test-group = "stateful-heavy"
 
 [[profile.default.overrides]]
 filter = 'test(e2e_perf_local_memory_stays_bounded_under_batch_insert_and_scan)'
 # This memory-bound perf check is meaningful only without competing suite load.
-threads-required = 15
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_topic_http_consume_direct_multi_user_publishers_no_missing_changes)'
+# These topic smoke tests are stable in isolation but can contend under full workspace load.
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_topic_http_consume_preserves_impersonated_user_and_payloads)'
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_topic_consume_option_matrix_start_batch_auto_ack_modes)'
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_topic_high_load_two_consumers_same_group_single_delivery)'
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_topic_four_consumers_same_group_no_duplicates)'
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_topic_ack_failure_recovery_no_message_loss_with_latency)'
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_topic_fan_out_different_groups_receive_all)'
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_event_counter_integrity_through_multiple_outages)'
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_gradual_latency_ramp_forces_reconnect_then_recovers)'
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_heavy_write_burst_during_outage_all_delivered)'
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_large_initial_snapshot_survives_repeated_outages)'
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_latency_spike_during_initial_snapshot_recovers)'
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_proxy_server_down_during_live_updates_resumes)'
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_loading_snapshot_with_live_writes_resumes_without_duplicate_rows)'
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_proxy_three_subscriptions_resume_after_server_bounce)'
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_shared_connection_recovers_subscriptions_in_different_stages)'
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_cli_syntax_error_handling)'
+test-group = "stateful-heavy"
+
+[[profile.default.overrides]]
+filter = 'test(test_connection_timeout_option)'
+# Pure options test, but full-suite CLI subprocess load can make nextest assign leaked output
+# handles here; keep it isolated from subprocess-heavy tests.
+test-group = "stateful-heavy"
 
 [profile.ci]
 # Store test results in JUnit format