Introduce rabbit_chaos, a module for chaos testing

MODULE.bazel

@@ -349,8 +349,8 @@ erlang_package.hex_package(
     name = "ra",
     build_file = "@rabbitmq-server//bazel:BUILD.ra",
     pkg = "ra",
-    sha256 = "ee7f63461e9b1734e99a7d38cb601803dc0868f015672d795857fa329287288c",
-    version = "2.9.1",
+    sha256 = "e08dae61a7e0cb036dbadad4c9fc498bbdd991bbb8470c69eb678f9e3ffda1c3",
+    version = "2.10.0-pre.3",
 )
 
 erlang_package.git_package(

deps/rabbit/app.bzl

@@ -72,6 +72,7 @@ def all_beam_files(name = "all_beam_files"):
             "src/rabbit_channel_sup_sup.erl",
             "src/rabbit_channel_tracking.erl",
             "src/rabbit_channel_tracking_handler.erl",
+            "src/rabbit_chaos.erl",
             "src/rabbit_classic_queue.erl",
             "src/rabbit_classic_queue_index_v2.erl",
             "src/rabbit_classic_queue_store_v2.erl",
@@ -341,6 +342,7 @@ def all_test_beam_files(name = "all_test_beam_files"):
             "src/rabbit_channel_sup_sup.erl",
             "src/rabbit_channel_tracking.erl",
             "src/rabbit_channel_tracking_handler.erl",
+            "src/rabbit_chaos.erl",
             "src/rabbit_classic_queue.erl",
             "src/rabbit_classic_queue_index_v2.erl",
             "src/rabbit_classic_queue_store_v2.erl",
@@ -626,6 +628,7 @@ def all_srcs(name = "all_srcs"):
             "src/rabbit_channel_sup_sup.erl",
             "src/rabbit_channel_tracking.erl",
             "src/rabbit_channel_tracking_handler.erl",
+            "src/rabbit_chaos.erl",
             "src/rabbit_classic_queue.erl",
             "src/rabbit_classic_queue_index_v2.erl",
             "src/rabbit_classic_queue_store_v2.erl",

deps/rabbit/src/rabbit.erl

@@ -168,6 +168,13 @@
                     {requires,    kernel_ready},
                     {enables,     core_initialized}]}).
 
+-rabbit_boot_step({rabbit_chaos,
+                   [{description, "rabbit node chaos server"},
+                    {mfa,         {rabbit_sup, start_restartable_child,
+                                   [rabbit_chaos]}},
+                    {requires,    [database]},
+                    {enables,     core_initialized}]}).
+
 -rabbit_boot_step({rabbit_node_monitor,
                    [{description, "node monitor"},
                     {mfa,         {rabbit_sup, start_restartable_child,

deps/rabbit/src/rabbit_chaos.erl

@@ -0,0 +1,142 @@
+%% This Source Code Form is subject to the terms of the Mozilla Public
+%% License, v. 2.0. If a copy of the MPL was not distributed with this
+%% file, You can obtain one at https://mozilla.org/MPL/2.0/.
+%%
+%% Copyright (c) 2007-2023 Broadcom. All Rights Reserved. The term “Broadcom”
+%% refers to Broadcom Inc. and/or its subsidiaries.  All rights reserved.
+%%
+
+-module(rabbit_chaos).
+
+-behaviour(gen_server).
+
+-export([start_link/0]).
+-export([
+         begin_default/0,
+         begin_default/1,
+         begin_chaos/1
+        ]).
+
+
+-export([init/1,
+         handle_call/3,
+         handle_cast/2,
+         handle_info/2,
+         terminate/2,
+         code_change/3]).
+
+-type chaos_event() :: {Name :: atom(),
+                        {kill_named_proc, Process :: atom()} |
+                        kill_quorum_queue_member}.
+
+-type chaos_cfg() :: #{interval := non_neg_integer(),
+                       events := [chaos_event()]}.
+-define(SERVER, ?MODULE).
+
+-record(?MODULE, {cfg :: chaos_cfg()}).
+
+-export_type([chaos_cfg/0,
+              chaos_event/0]).
+
+%%----------------------------------------------------------------------------
+%% A chaos server that can be enabled to create periodic configurable chaos
+%% inside the broker.
+%%----------------------------------------------------------------------------
+
+begin_default() ->
+    begin_default(20000).
+
+begin_default(Interval) ->
+    Events = [
+              {kill_qq_wal, 5, {kill_named_proc, ra_log_wal}},
+              {kill_qq_seq_writer, 1, {kill_named_proc, ra_log_segment_writer}},
+              {kill_qq_member, 1, kill_ra_member},
+              {kill_qq_member, 1, restart_ra_member},
+              {multi_kill_qq_wal, 1,
+               {multi, 3, 100, {kill_named_proc, ra_log_wal}}}
+             ],
+    begin_chaos(#{interval => Interval,
+                  events => Events}).
+
+begin_chaos(Cfg) ->
+    gen_server:call(?SERVER, {begin_chaos, Cfg}).
+
+-spec start_link() -> rabbit_types:ok_pid_or_error().
+start_link() ->
+    gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
+
+init([]) ->
+    process_flag(trap_exit, true),
+    Cfg = #{interval => 20000,
+            events => []},
+    {ok, #?MODULE{cfg = Cfg}}.
+
+handle_call({begin_chaos, #{interval := Interval} = Cfg}, _From, State) ->
+    _ = erlang:send_after(Interval, self(), do_chaos),
+    {reply, ok, State#?MODULE{cfg = Cfg}}.
+
+handle_cast(_Request, State) ->
+    {noreply, State}.
+
+handle_info(do_chaos, #?MODULE{cfg = #{interval := Interval} = Cfg} = State) ->
+    Events = maps:get(events, Cfg),
+    {Name, _, Event} = pick_event(Events),
+    do_event(Name, Event),
+    _ = erlang:send_after(Interval, self(), do_chaos),
+    {noreply, State};
+handle_info(_, #?MODULE{} = State) ->
+    {noreply, State}.
+
+terminate(_Reason, #?MODULE{}) ->
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+%% internal
+
+do_event(Name, {kill_named_proc, ProcName}) ->
+    rabbit_log:info("~s: doing event ~s...", [?MODULE, Name]),
+    catch exit(whereis(ProcName), chaos),
+    ok;
+do_event(Name, kill_ra_member) ->
+    rabbit_log:info("~s: doing event ~s...", [?MODULE, Name]),
+    Procs = ets:tab2list(ra_leaderboard),
+    At = rand:uniform(length(Procs)),
+    Selected = lists:nth(At, Procs),
+    catch exit(whereis(element(1, Selected)), kill),
+    ok;
+do_event(Name, restart_ra_member = Type) ->
+    rabbit_log:info("~s: doing event ~s of type ~s", [?MODULE, Name, Type]),
+    Queues = rabbit_amqqueue:list_local_quorum_queues(),
+    At = rand:uniform(length(Queues)),
+    Selected = lists:nth(At, Queues),
+    ServerId = amqqueue:get_pid(Selected),
+    ra:stop_server(quorum_queues, ServerId),
+    ra:restart_server(quorum_queues, ServerId),
+    ok;
+do_event(Name, {multi, Num, Interval, Event}) ->
+    rabbit_log:info("~s: doing multi event ~s...",
+                    [?MODULE, Name]),
+    catch [begin
+               do_event(Name, Event),
+               timer:sleep(Interval)
+           end || _ <- lists:seq(1, Num)],
+    ok.
+
+pick_event(Events) ->
+    TotalWeight = lists:sum([element(2, E) || E <- Events]),
+    Pick = rand:uniform(TotalWeight),
+    event_at_weight_point(Pick, 0, Events).
+
+
+event_at_weight_point(_Pick, _Cur, []) ->
+    undefined;
+event_at_weight_point(Pick, Cur0, [{_, W, _} = E | Events]) ->
+    Cur = Cur0 + W,
+    case Pick =< Cur of
+        true ->
+           E;
+        false ->
+            event_at_weight_point(Pick, Cur, Events)
+    end.

deps/rabbit/src/rabbit_ra_systems.erl

@@ -119,7 +119,8 @@ get_config(quorum_queues = RaSystem) ->
                    default_max_append_entries_rpc_batch_size => AERBatchSize,
                    wal_compute_checksums => WalChecksums,
                    segment_compute_checksums => SegmentChecksums,
-                   compress_mem_tables => CompressMemTables};
+                   compress_mem_tables => CompressMemTables,
+                   server_recovery_strategy => registered};
 get_config(coordination = RaSystem) ->
     DefaultConfig = get_default_config(),
     CoordDataDir = filename:join(

moduleindex.yaml

@@ -572,6 +572,7 @@ rabbit:
 - rabbit_channel_sup_sup
 - rabbit_channel_tracking
 - rabbit_channel_tracking_handler
+- rabbit_chaos
 - rabbit_classic_queue
 - rabbit_classic_queue_index_v2
 - rabbit_classic_queue_store_v2

rabbitmq-components.mk

@@ -122,7 +122,7 @@ dep_khepri = hex 0.13.0
 dep_khepri_mnesia_migration = hex 0.4.0
 dep_looking_glass = git https://github.com/rabbitmq/looking_glass.git main
 dep_prometheus = hex 4.11.0
-dep_ra = hex 2.9.1
+dep_ra = hex 2.10.0-pre.3
 dep_ranch = hex 2.1.0
 dep_recon = hex 2.5.3
 dep_redbug = hex 2.0.7