diff --git a/deps/rabbit/src/rabbit_amqqueue.erl b/deps/rabbit/src/rabbit_amqqueue.erl index 91f4a3f1306d..435d9560147b 100644 --- a/deps/rabbit/src/rabbit_amqqueue.erl +++ b/deps/rabbit/src/rabbit_amqqueue.erl @@ -778,6 +778,7 @@ declare_args() -> {<<"x-message-ttl">>, fun check_message_ttl_arg/2}, {<<"x-dead-letter-exchange">>, fun check_dlxname_arg/2}, {<<"x-dead-letter-routing-key">>, fun check_dlxrk_arg/2}, + {<<"x-dead-letter-strategy">>, fun check_dlxstrategy_arg/2}, {<<"x-max-length">>, fun check_non_neg_int_arg/2}, {<<"x-max-length-bytes">>, fun check_non_neg_int_arg/2}, {<<"x-max-in-memory-length">>, fun check_non_neg_int_arg/2}, @@ -945,6 +946,22 @@ check_dlxrk_arg(Val, Args) when is_binary(Val) -> check_dlxrk_arg(_Val, _Args) -> {error, {unacceptable_type, "expected a string"}}. +-define(KNOWN_DLX_STRATEGIES, [<<"at-most-once">>, <<"at-least-once">>]). +check_dlxstrategy_arg({longstr, Val}, _Args) -> + case lists:member(Val, ?KNOWN_DLX_STRATEGIES) of + true -> ok; + false -> {error, invalid_dlx_strategy} + end; +check_dlxstrategy_arg({Type, _}, _Args) -> + {error, {unacceptable_type, Type}}; +check_dlxstrategy_arg(Val, _Args) when is_binary(Val) -> + case lists:member(Val, ?KNOWN_DLX_STRATEGIES) of + true -> ok; + false -> {error, invalid_dlx_strategy} + end; +check_dlxstrategy_arg(_Val, _Args) -> + {error, invalid_dlx_strategy}. + -define(KNOWN_OVERFLOW_MODES, [<<"drop-head">>, <<"reject-publish">>, <<"reject-publish-dlx">>]). check_overflow({longstr, Val}, _Args) -> case lists:member(Val, ?KNOWN_OVERFLOW_MODES) of @@ -1641,8 +1658,8 @@ credit(Q, CTag, Credit, Drain, QStates) -> {'ok', non_neg_integer(), qmsg(), rabbit_queue_type:state()} | {'empty', rabbit_queue_type:state()} | {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. -basic_get(Q, NoAck, LimiterPid, CTag, QStates0) -> - rabbit_queue_type:dequeue(Q, NoAck, LimiterPid, CTag, QStates0). +basic_get(Q, NoAck, LimiterPid, CTag, QStates) -> + rabbit_queue_type:dequeue(Q, NoAck, LimiterPid, CTag, QStates). -spec basic_consume(amqqueue:amqqueue(), boolean(), pid(), pid(), boolean(), @@ -1654,7 +1671,7 @@ basic_get(Q, NoAck, LimiterPid, CTag, QStates0) -> {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. basic_consume(Q, NoAck, ChPid, LimiterPid, LimiterActive, ConsumerPrefetchCount, ConsumerTag, - ExclusiveConsume, Args, OkMsg, ActingUser, Contexts) -> + ExclusiveConsume, Args, OkMsg, ActingUser, QStates) -> QName = amqqueue:get_name(Q), %% first phase argument validation @@ -1670,7 +1687,7 @@ basic_consume(Q, NoAck, ChPid, LimiterPid, args => Args, ok_msg => OkMsg, acting_user => ActingUser}, - rabbit_queue_type:consume(Q, Spec, Contexts). + rabbit_queue_type:consume(Q, Spec, QStates). -spec basic_cancel(amqqueue:amqqueue(), rabbit_types:ctag(), any(), rabbit_types:username(), diff --git a/deps/rabbit/src/rabbit_basic.erl b/deps/rabbit/src/rabbit_basic.erl index cc7c00047e63..b42e832f71eb 100644 --- a/deps/rabbit/src/rabbit_basic.erl +++ b/deps/rabbit/src/rabbit_basic.erl @@ -12,7 +12,8 @@ -export([publish/4, publish/5, publish/1, message/3, message/4, properties/1, prepend_table_header/3, extract_headers/1, extract_timestamp/1, map_headers/2, delivery/4, - header_routes/1, parse_expiration/1, header/2, header/3]). + header_routes/1, parse_expiration/1, header/2, header/3, + is_message_persistent/1]). -export([build_content/2, from_content/1, msg_size/1, maybe_gc_large_msg/1, maybe_gc_large_msg/2]). -export([add_header/4, diff --git a/deps/rabbit/src/rabbit_classic_queue.erl b/deps/rabbit/src/rabbit_classic_queue.erl index b720cfc96ea7..f15632b7b980 100644 --- a/deps/rabbit/src/rabbit_classic_queue.erl +++ b/deps/rabbit/src/rabbit_classic_queue.erl @@ -445,8 +445,10 @@ recover_durable_queues(QueuesAndRecoveryTerms) -> capabilities() -> #{unsupported_policies => [ %% Stream policies - <<"max-age">>, <<"stream-max-segment-size-bytes">>, - <<"queue-leader-locator">>, <<"initial-cluster-size">>], + <<"max-age">>, <<"stream-max-segment-size-bytes">>, + <<"queue-leader-locator">>, <<"initial-cluster-size">>, + %% Quorum policies + <<"dead-letter-strategy">>], queue_arguments => [<<"x-expires">>, <<"x-message-ttl">>, <<"x-dead-letter-exchange">>, <<"x-dead-letter-routing-key">>, <<"x-max-length">>, <<"x-max-length-bytes">>, <<"x-max-in-memory-length">>, diff --git a/deps/rabbit/src/rabbit_dead_letter.erl b/deps/rabbit/src/rabbit_dead_letter.erl index f13b409dce85..c3865d31b696 100644 --- a/deps/rabbit/src/rabbit_dead_letter.erl +++ b/deps/rabbit/src/rabbit_dead_letter.erl @@ -7,7 +7,9 @@ -module(rabbit_dead_letter). --export([publish/5]). +-export([publish/5, + make_msg/5, + detect_cycles/3]). -include_lib("rabbit_common/include/rabbit.hrl"). -include_lib("rabbit_common/include/rabbit_framing.hrl"). @@ -39,7 +41,7 @@ make_msg(Msg = #basic_message{content = Content, undefined -> {RoutingKeys, fun (H) -> H end}; _ -> {[RK], fun (H) -> lists:keydelete(<<"CC">>, 1, H) end} end, - ReasonBin = list_to_binary(atom_to_list(Reason)), + ReasonBin = atom_to_binary(Reason), TimeSec = os:system_time(seconds), PerMsgTTL = per_msg_ttl_header(Content#content.properties), HeadersFun2 = diff --git a/deps/rabbit/src/rabbit_fifo.erl b/deps/rabbit/src/rabbit_fifo.erl index cb2fe7fd7819..c98a231b6ea6 100644 --- a/deps/rabbit/src/rabbit_fifo.erl +++ b/deps/rabbit/src/rabbit_fifo.erl @@ -20,11 +20,13 @@ -include_lib("rabbit_common/include/rabbit.hrl"). -export([ + %% ra_machine callbacks init/1, apply/3, state_enter/2, tick/2, overview/1, + get_checked_out/4, %% versioning version/0, @@ -51,7 +53,10 @@ %% misc dehydrate_state/1, + dehydrate_message/1, normalize/1, + get_msg_header/1, + get_header/2, %% protocol helpers make_enqueue/3, @@ -103,7 +108,7 @@ #update_config{} | #garbage_collection{}. --type command() :: protocol() | ra_machine:builtin_command(). +-type command() :: protocol() | rabbit_fifo_dlx:protocol() | ra_machine:builtin_command(). %% all the command types supported by ra fifo -type client_msg() :: delivery(). @@ -126,6 +131,8 @@ state/0, config/0]). +%% This function is never called since only rabbit_fifo_v0:init/1 is called. +%% See https://github.com/rabbitmq/ra/blob/e0d1e6315a45f5d3c19875d66f9d7bfaf83a46e3/src/ra_machine.erl#L258-L265 -spec init(config()) -> state(). init(#{name := Name, queue_resource := Resource} = Conf) -> @@ -143,6 +150,7 @@ update_config(Conf, State) -> MaxMemoryBytes = maps:get(max_in_memory_bytes, Conf, undefined), DeliveryLimit = maps:get(delivery_limit, Conf, undefined), Expires = maps:get(expires, Conf, undefined), + MsgTTL = maps:get(msg_ttl, Conf, undefined), ConsumerStrategy = case maps:get(single_active_consumer_on, Conf, false) of true -> single_active; @@ -153,6 +161,7 @@ update_config(Conf, State) -> RCISpec = {RCI, RCI}, LastActive = maps:get(created, Conf, undefined), + MaxMemoryBytes = maps:get(max_in_memory_bytes, Conf, undefined), State#?MODULE{cfg = Cfg#cfg{release_cursor_interval = RCISpec, dead_letter_handler = DLH, become_leader_handler = BLH, @@ -163,8 +172,9 @@ update_config(Conf, State) -> max_in_memory_bytes = MaxMemoryBytes, consumer_strategy = ConsumerStrategy, delivery_limit = DeliveryLimit, - expires = Expires}, - last_active = LastActive}. + expires = Expires, + msg_ttl = MsgTTL}, + last_active = LastActive}. zero(_) -> 0. @@ -201,30 +211,46 @@ apply(Meta, case Cons0 of #{ConsumerId := Con0} -> complete_and_checkout(Meta, MsgIds, ConsumerId, - Con0, [], State); + Con0, [], State, true); _ -> {State, ok} end; apply(Meta, #discard{msg_ids = MsgIds, consumer_id = ConsumerId}, - #?MODULE{consumers = Cons0} = State0) -> - case Cons0 of - #{ConsumerId := #consumer{checked_out = Checked} = Con0} -> - % Discarded maintains same order as MsgIds (so that publishing to - % dead-letter exchange will be in same order as messages got rejected) - Discarded = lists:filtermap(fun(Id) -> - case maps:find(Id, Checked) of - {ok, Msg} -> - {true, Msg}; - error -> - false - end - end, MsgIds), - Effects = dead_letter_effects(rejected, Discarded, State0, []), - complete_and_checkout(Meta, MsgIds, ConsumerId, Con0, - Effects, State0); + #?MODULE{consumers = Cons, + dlx = DlxState0, + cfg = #cfg{dead_letter_handler = DLH}} = State) -> + case Cons of + #{ConsumerId := #consumer{checked_out = Checked} = Con} -> + case DLH of + at_least_once -> + DlxState = lists:foldl(fun(MsgId, S) -> + case maps:find(MsgId, Checked) of + {ok, Msg} -> + rabbit_fifo_dlx:discard(Msg, rejected, S); + error -> + S + end + end, DlxState0, MsgIds), + complete_and_checkout(Meta, MsgIds, ConsumerId, Con, + [], State#?MODULE{dlx = DlxState}, false); + _ -> + % Discarded maintains same order as MsgIds (so that publishing to + % dead-letter exchange will be in same order as messages got rejected) + Discarded = lists:filtermap(fun(Id) -> + case maps:find(Id, Checked) of + {ok, Msg} -> + {true, Msg}; + error -> + false + end + end, MsgIds), + Effects = dead_letter_effects(rejected, Discarded, State, []), + complete_and_checkout(Meta, MsgIds, ConsumerId, Con, + Effects, State, true) + end; _ -> - {State0, ok} + {State, ok} end; apply(Meta, #return{msg_ids = MsgIds, consumer_id = ConsumerId}, #?MODULE{consumers = Cons0} = State) -> @@ -319,17 +345,17 @@ apply(#{index := Index, State1 = update_consumer(ConsumerId, ConsumerMeta, {once, 1, simple_prefetch}, 0, State0), - {success, _, MsgId, Msg, State2} = checkout_one(Meta, State1), + {success, _, MsgId, Msg, State2, Effects0} = checkout_one(Meta, State1, []), {State4, Effects1} = case Settlement of unsettled -> {_, Pid} = ConsumerId, - {State2, [{monitor, process, Pid}]}; + {State2, [{monitor, process, Pid} | Effects0]}; settled -> %% immediately settle the checkout - {State3, _, Effects0} = + {State3, _, SettleEffects} = apply(Meta, make_settle(ConsumerId, [MsgId]), State2), - {State3, Effects0} + {State3, SettleEffects ++ Effects0} end, {Reply, Effects2} = case Msg of @@ -366,34 +392,45 @@ apply(#{index := Index}, #purge{}, #?MODULE{messages_total = Tot, returns = Returns, messages = Messages, - ra_indexes = Indexes0} = State0) -> - Total = messages_ready(State0), - Indexes1 = lists:foldl(fun (?INDEX_MSG(I, _), Acc0) when is_integer(I) -> + ra_indexes = Indexes0, + dlx = DlxState0} = State0) -> + NumReady = messages_ready(State0), + Indexes1 = lists:foldl(fun (?INDEX_MSG(I, ?MSG(_, _)), Acc0) when is_integer(I) -> rabbit_fifo_index:delete(I, Acc0); (_, Acc) -> Acc end, Indexes0, lqueue:to_list(Returns)), - Indexes = lists:foldl(fun (?INDEX_MSG(I, _), Acc0) when is_integer(I) -> + Indexes2 = lists:foldl(fun (?INDEX_MSG(I, ?MSG(_, _)), Acc0) when is_integer(I) -> rabbit_fifo_index:delete(I, Acc0); (_, Acc) -> Acc end, Indexes1, lqueue:to_list(Messages)), + {DlxState, DiscardMsgs} = rabbit_fifo_dlx:purge(DlxState0), + Indexes = lists:foldl(fun (?INDEX_MSG(I, ?MSG(_, _)), Acc0) when is_integer(I) -> + rabbit_fifo_index:delete(I, Acc0); + (_, Acc) -> + Acc + end, Indexes2, DiscardMsgs), + NumPurged = NumReady + length(DiscardMsgs), State1 = State0#?MODULE{ra_indexes = Indexes, messages = lqueue:new(), - messages_total = Tot - Total, + messages_total = Tot - NumPurged, returns = lqueue:new(), + dlx = DlxState, msg_bytes_enqueue = 0, prefix_msgs = {0, [], 0, []}, msg_bytes_in_memory = 0, msgs_ready_in_memory = 0}, Effects0 = [garbage_collection], - Reply = {purge, Total}, + Reply = {purge, NumPurged}, {State, _, Effects} = evaluate_limit(Index, false, State0, State1, Effects0), update_smallest_raft_index(Index, Reply, State, Effects); apply(#{index := Idx}, #garbage_collection{}, State) -> update_smallest_raft_index(Idx, ok, State, [{aux, garbage_collection}]); +apply(Meta, {timeout, expire_msgs}, State) -> + checkout(Meta, State, State, [], false); apply(#{system_time := Ts} = Meta, {down, Pid, noconnection}, #?MODULE{consumers = Cons0, cfg = #cfg{consumer_strategy = single_active}, @@ -531,12 +568,71 @@ apply(#{index := Idx} = Meta, #purge_nodes{nodes = Nodes}, State0) -> purge_node(Meta, Node, S, E) end, {State0, []}, Nodes), update_smallest_raft_index(Idx, ok, State, Effects); -apply(#{index := Idx} = Meta, #update_config{config = Conf}, State0) -> - {State, Reply, Effects} = checkout(Meta, State0, update_config(Conf, State0), []), +apply(#{index := Idx} = Meta, #update_config{config = Conf}, + #?MODULE{cfg = #cfg{dead_letter_handler = Old_DLH}} = State0) -> + #?MODULE{cfg = #cfg{dead_letter_handler = DLH}, + dlx = DlxState, + ra_indexes = Indexes0, + messages_total = Tot} = State1 = update_config(Conf, State0), + %%TODO return aux effect here and move logic over to handle_aux/6 which can return effects as last arguments. + {State3, Effects1} = case DLH of + at_least_once -> + case rabbit_fifo_dlx:consumer_pid(DlxState) of + undefined -> + %% Policy changed from at-most-once to at-least-once. + %% Therefore, start rabbit_fifo_dlx_worker on leader. + {State1, [{aux, start_dlx_worker}]}; + DlxWorkerPid -> + %% Leader already exists. + %% Notify leader of new policy. + Effect = {send_msg, DlxWorkerPid, lookup_topology, ra_event}, + {State1, [Effect]} + end; + _ when Old_DLH =:= at_least_once -> + %% Cleanup any remaining messages stored by rabbit_fifo_dlx + %% by either dropping or at-most-once dead-lettering. + ReasonMsgs = rabbit_fifo_dlx:cleanup(DlxState), + Len = length(ReasonMsgs), + rabbit_log:debug("Cleaning up ~b dead-lettered messages " + "since dead_letter_handler changed from ~s to ~p", + [Len, Old_DLH, DLH]), + Effects0 = dead_letter_effects(undefined, ReasonMsgs, State1, []), + {_, Msgs} = lists:unzip(ReasonMsgs), + Indexes = delete_indexes(Msgs, Indexes0), + State2 = State1#?MODULE{dlx = rabbit_fifo_dlx:init(), + ra_indexes = Indexes, + messages_total = Tot - Len}, + {State2, Effects0}; + _ -> + {State1, []} + end, + {State, Reply, Effects} = checkout(Meta, State0, State3, Effects1), update_smallest_raft_index(Idx, Reply, State, Effects); apply(_Meta, {machine_version, FromVersion, ToVersion}, V0State) -> State = convert(FromVersion, ToVersion, V0State), - {State, ok, []}; + {State, ok, [{aux, start_dlx_worker}]}; +%%TODO are there better approach to +%% 1. matching against opaque rabbit_fifo_dlx:protocol / record (without exposing all the protocol details), and +%% 2. Separate the logic running in rabbit_fifo and rabbit_fifo_dlx when dead-letter messages is acked? +apply(#{index := IncomingRaftIdx} = Meta, {dlx, Cmd}, + #?MODULE{dlx = DlxState0, + messages_total = Total0, + ra_indexes = Indexes0} = State0) when element(1, Cmd) =:= settle -> + {DlxState, AckedMsgs} = rabbit_fifo_dlx:apply(Cmd, DlxState0), + Indexes = delete_indexes(AckedMsgs, Indexes0), + Total = Total0 - length(AckedMsgs), + State1 = State0#?MODULE{dlx = DlxState, + messages_total = Total, + ra_indexes = Indexes}, + {State, ok, Effects} = checkout(Meta, State0, State1, [], false), + update_smallest_raft_index(IncomingRaftIdx, State, Effects); +apply(Meta, {dlx, Cmd}, + #?MODULE{dlx = DlxState0} = State0) -> + {DlxState, ok} = rabbit_fifo_dlx:apply(Cmd, DlxState0), + State1 = State0#?MODULE{dlx = DlxState}, + %% Run a checkout so that a new DLX consumer will be delivered discarded messages + %% directly after it subscribes. + checkout(Meta, State0, State1, [], false); apply(_Meta, Cmd, State) -> %% handle unhandled commands gracefully rabbit_log:debug("rabbit_fifo: unhandled command ~W", [Cmd, 10]), @@ -627,11 +723,31 @@ convert_v1_to_v2(V1State) -> end, Ch)} end, ConsumersV1), + %% The (old) format of dead_letter_handler in RMQ < v3.10 is: + %% {Module, Function, Args} + %% The (new) format of dead_letter_handler in RMQ >= v3.10 is: + %% undefined | {at_most_once, {Module, Function, Args}} | at_least_once + %% + %% Note that the conversion must convert both from old format to new format + %% as well as from new format to new format. The latter is because quorum queues + %% created in RMQ >= v3.10 are still initialised with rabbit_fifo_v0 as described in + %% https://github.com/rabbitmq/ra/blob/e0d1e6315a45f5d3c19875d66f9d7bfaf83a46e3/src/ra_machine.erl#L258-L265 + DLH = case rabbit_fifo_v1:get_cfg_field(dead_letter_handler, V1State) of + {_M, _F, _A = [_DLX = undefined|_]} -> + %% queue was declared in RMQ < v3.10 and no DLX configured + undefined; + {_M, _F, _A} = MFA -> + %% queue was declared in RMQ < v3.10 and DLX configured + {at_most_once, MFA}; + Other -> + Other + end, + %% Then add all pending messages back into the index Cfg = #cfg{name = rabbit_fifo_v1:get_cfg_field(name, V1State), resource = rabbit_fifo_v1:get_cfg_field(resource, V1State), release_cursor_interval = rabbit_fifo_v1:get_cfg_field(release_cursor_interval, V1State), - dead_letter_handler = rabbit_fifo_v1:get_cfg_field(dead_letter_handler, V1State), + dead_letter_handler = DLH, become_leader_handler = rabbit_fifo_v1:get_cfg_field(become_leader_handler, V1State), %% TODO: what if policy enabling reject_publish was applied before conversion? overflow_strategy = rabbit_fifo_v1:get_cfg_field(overflow_strategy, V1State), @@ -670,14 +786,20 @@ purge_node(Meta, Node, State, Effects) -> end, {State, Effects}, all_pids_for(Node, State)). %% any downs that re not noconnection -handle_down(Meta, Pid, #?MODULE{consumers = Cons0, - enqueuers = Enqs0} = State0) -> +handle_down(#{system_time := DownTs} = Meta, Pid, #?MODULE{consumers = Cons0, + enqueuers = Enqs0} = State0) -> % Remove any enqueuer for the same pid and enqueue any pending messages % This should be ok as we won't see any more enqueues from this pid State1 = case maps:take(Pid, Enqs0) of {#enqueuer{pending = Pend}, Enqs} -> - lists:foldl(fun ({_, RIdx, RawMsg}, S) -> - enqueue(RIdx, RawMsg, S) + lists:foldl(fun ({_, RIdx, Ts, RawMsg}, S) -> + enqueue(RIdx, Ts, RawMsg, S); + ({_, RIdx, RawMsg}, S) -> + %% This is an edge case: It is an out-of-order delivery + %% from machine version 1. + %% If message TTL is configured, expiration will be delayed + %% for the time the message has been pending. + enqueue(RIdx, DownTs, RawMsg, S) end, State0#?MODULE{enqueuers = Enqs}, Pend); error -> State0 @@ -738,7 +860,16 @@ update_waiting_consumer_status(Node, Consumer#consumer.status =/= cancelled]. -spec state_enter(ra_server:ra_state(), state()) -> ra_machine:effects(). -state_enter(leader, #?MODULE{consumers = Cons, +state_enter(RaState, #?MODULE{cfg = #cfg{dead_letter_handler = at_least_once, + resource = QRef, + name = QName}, + dlx = DlxState} = State) -> + rabbit_fifo_dlx:state_enter(RaState, QRef, QName, DlxState), + state_enter0(RaState, State); +state_enter(RaState, State) -> + state_enter0(RaState, State). + +state_enter0(leader, #?MODULE{consumers = Cons, enqueuers = Enqs, waiting_consumers = WaitingConsumers, cfg = #cfg{name = Name, @@ -761,7 +892,7 @@ state_enter(leader, #?MODULE{consumers = Cons, {Mod, Fun, Args} -> [{mod_call, Mod, Fun, Args ++ [Name]} | Effects] end; -state_enter(eol, #?MODULE{enqueuers = Enqs, +state_enter0(eol, #?MODULE{enqueuers = Enqs, consumers = Custs0, waiting_consumers = WaitingConsumers0}) -> Custs = maps:fold(fun({_, P}, V, S) -> S#{P => V} end, #{}, Custs0), @@ -772,14 +903,13 @@ state_enter(eol, #?MODULE{enqueuers = Enqs, || P <- maps:keys(maps:merge(Enqs, AllConsumers))] ++ [{aux, eol}, {mod_call, rabbit_quorum_queue, file_handle_release_reservation, []}]; -state_enter(State, #?MODULE{cfg = #cfg{resource = _Resource}}) when State =/= leader -> +state_enter0(State, #?MODULE{cfg = #cfg{resource = _Resource}}) when State =/= leader -> FHReservation = {mod_call, rabbit_quorum_queue, file_handle_other_reservation, []}, [FHReservation]; - state_enter(_, _) -> +state_enter0(_, _) -> %% catch all as not handling all states []. - -spec tick(non_neg_integer(), state()) -> ra_machine:effects(). tick(Ts, #?MODULE{cfg = #cfg{name = Name, resource = QName}, @@ -805,6 +935,7 @@ overview(#?MODULE{consumers = Cons, enqueuers = Enqs, release_cursors = Cursors, enqueue_count = EnqCount, + dlx = DlxState, msg_bytes_enqueue = EnqueueBytes, msg_bytes_checkout = CheckoutBytes, cfg = Cfg} = State) -> @@ -818,23 +949,26 @@ overview(#?MODULE{consumers = Cons, max_in_memory_length => Cfg#cfg.max_in_memory_length, max_in_memory_bytes => Cfg#cfg.max_in_memory_bytes, expires => Cfg#cfg.expires, + msg_ttl => Cfg#cfg.msg_ttl, delivery_limit => Cfg#cfg.delivery_limit - }, + }, {Smallest, _} = smallest_raft_index(State), - #{type => ?MODULE, - config => Conf, - num_consumers => maps:size(Cons), - num_checked_out => num_checked_out(State), - num_enqueuers => maps:size(Enqs), - num_ready_messages => messages_ready(State), - num_pending_messages => messages_pending(State), - num_messages => messages_total(State), - num_release_cursors => lqueue:len(Cursors), - release_cursors => [{I, messages_total(S)} || {_, I, S} <- lqueue:to_list(Cursors)], - release_cursor_enqueue_counter => EnqCount, - enqueue_message_bytes => EnqueueBytes, - checkout_message_bytes => CheckoutBytes, - smallest_raft_index => Smallest}. + Overview = #{type => ?MODULE, + config => Conf, + num_consumers => maps:size(Cons), + num_checked_out => num_checked_out(State), + num_enqueuers => maps:size(Enqs), + num_ready_messages => messages_ready(State), + num_pending_messages => messages_pending(State), + num_messages => messages_total(State), + num_release_cursors => lqueue:len(Cursors), + release_cursors => [{I, messages_total(S)} || {_, I, S} <- lqueue:to_list(Cursors)], + release_cursor_enqueue_counter => EnqCount, + enqueue_message_bytes => EnqueueBytes, + checkout_message_bytes => CheckoutBytes, + smallest_raft_index => Smallest}, + DlxOverview = rabbit_fifo_dlx:overview(DlxState), + maps:merge(Overview, DlxOverview). -spec get_checked_out(consumer_id(), msg_id(), msg_id(), state()) -> [delivery_msg()]. @@ -917,8 +1051,15 @@ handle_aux(_RaState, {call, _From}, {peek, Pos}, Aux0, {reply, {ok, {Header, Msg}}, Aux0, Log0}; Err -> {reply, Err, Aux0, Log0} - end. - + end; +handle_aux(leader, _, start_dlx_worker, Aux, Log, + #?MODULE{cfg = #cfg{resource = QRef, + name = QName, + dead_letter_handler = at_least_once}}) -> + rabbit_fifo_dlx:start_worker(QRef, QName), + {no_reply, Aux, Log}; +handle_aux(_, _, start_dlx_worker, Aux, Log, _) -> + {no_reply, Aux, Log}. eval_gc(Log, #?MODULE{cfg = #cfg{resource = QR}} = MacState, #aux{gc = #aux_gc{last_raft_idx = LastGcIdx} = Gc} = AuxState) -> @@ -1113,8 +1254,15 @@ messages_total(#?MODULE{messages = _M, messages_total = Total, ra_indexes = _Indexes, prefix_msgs = {_RCnt, _R, _PCnt, _P}}) -> - Total. % lqueue:len(M) + rabbit_fifo_index:size(Indexes) + RCnt + PCnt. + Total; +%% release cursors might be old state (e.g. after recent upgrade) +messages_total(State) + when element(1, State) =:= rabbit_fifo_v1 -> + rabbit_fifo_v1:query_messages_total(State); +messages_total(State) + when element(1, State) =:= rabbit_fifo_v0 -> + rabbit_fifo_v0:query_messages_total(State). update_use({inactive, _, _, _} = CUInfo, inactive) -> CUInfo; @@ -1265,8 +1413,9 @@ maybe_return_all(#{system_time := Ts} = Meta, ConsumerId, Consumer, S0, Effects0 Effects1} end. -apply_enqueue(#{index := RaftIdx} = Meta, From, Seq, RawMsg, State0) -> - case maybe_enqueue(RaftIdx, From, Seq, RawMsg, [], State0) of +apply_enqueue(#{index := RaftIdx, + system_time := Ts} = Meta, From, Seq, RawMsg, State0) -> + case maybe_enqueue(RaftIdx, Ts, From, Seq, RawMsg, [], State0) of {ok, State1, Effects1} -> State2 = incr_enqueue_count(incr_total(State1)), {State, ok, Effects} = checkout(Meta, State0, State2, Effects1, false), @@ -1305,10 +1454,11 @@ drop_head(#?MODULE{ra_indexes = Indexes0} = State0, Effects0) -> {State0, Effects0} end. -enqueue(RaftIdx, RawMsg, #?MODULE{messages = Messages} = State0) -> +enqueue(RaftIdx, Ts, RawMsg, #?MODULE{messages = Messages} = State0) -> %% the initial header is an integer only - it will get expanded to a map %% when the next required key is added - Header = message_size(RawMsg), + Header0 = message_size(RawMsg), + Header = maybe_set_msg_ttl(RawMsg, Ts, Header0, State0), {State1, Msg} = case evaluate_memory_limit(Header, State0) of true -> @@ -1322,6 +1472,39 @@ enqueue(RaftIdx, RawMsg, #?MODULE{messages = Messages} = State0) -> State = add_bytes_enqueue(Header, State1), State#?MODULE{messages = lqueue:in(Msg, Messages)}. +maybe_set_msg_ttl(#basic_message{content = #content{properties = none}}, + _, Header, + #?MODULE{cfg = #cfg{msg_ttl = undefined}}) -> + Header; +maybe_set_msg_ttl(#basic_message{content = #content{properties = none}}, + RaCmdTs, Header, + #?MODULE{cfg = #cfg{msg_ttl = PerQueueMsgTTL}}) -> + update_expiry_header(RaCmdTs, PerQueueMsgTTL, Header); +maybe_set_msg_ttl(#basic_message{content = #content{properties = Props}}, + RaCmdTs, Header, + #?MODULE{cfg = #cfg{msg_ttl = PerQueueMsgTTL}}) -> + %% rabbit_quorum_queue will leave the properties decoded if and only if + %% per message message TTL is set. + %% We already check in the channel that expiration must be valid. + {ok, PerMsgMsgTTL} = rabbit_basic:parse_expiration(Props), + TTL = min(PerMsgMsgTTL, PerQueueMsgTTL), + update_expiry_header(RaCmdTs, TTL, Header). + +update_expiry_header(_, undefined, Header) -> + Header; +update_expiry_header(RaCmdTs, 0, Header) -> + %% We do not comply exactly with the "TTL=0 models AMQP immediate flag" semantics + %% as done for classic queues where the message is discarded if it cannot be + %% consumed immediately. + %% Instead, we discard the message if it cannot be consumed within the same millisecond + %% when it got enqueued. This behaviour should be good enough. + update_expiry_header(RaCmdTs + 1, Header); +update_expiry_header(RaCmdTs, TTL, Header) -> + update_expiry_header(RaCmdTs + TTL, Header). + +update_expiry_header(ExpiryTs, Header) -> + update_header(expiry, fun(Ts) -> Ts end, ExpiryTs, Header). + incr_enqueue_count(#?MODULE{enqueue_count = EC, cfg = #cfg{release_cursor_interval = {_Base, C}} } = State0) when EC >= C -> @@ -1363,39 +1546,39 @@ maybe_store_dehydrated_state(_RaftIdx, State) -> enqueue_pending(From, #enqueuer{next_seqno = Next, - pending = [{Next, RaftIdx, RawMsg} | Pending]} = Enq0, + pending = [{Next, RaftIdx, Ts, RawMsg} | Pending]} = Enq0, State0) -> - State = enqueue(RaftIdx, RawMsg, State0), + State = enqueue(RaftIdx, Ts, RawMsg, State0), Enq = Enq0#enqueuer{next_seqno = Next + 1, pending = Pending}, enqueue_pending(From, Enq, State); enqueue_pending(From, Enq, #?MODULE{enqueuers = Enqueuers0} = State) -> State#?MODULE{enqueuers = Enqueuers0#{From => Enq}}. -maybe_enqueue(RaftIdx, undefined, undefined, RawMsg, Effects, State0) -> +maybe_enqueue(RaftIdx, Ts, undefined, undefined, RawMsg, Effects, State0) -> % direct enqueue without tracking - State = enqueue(RaftIdx, RawMsg, State0), + State = enqueue(RaftIdx, Ts, RawMsg, State0), {ok, State, Effects}; -maybe_enqueue(RaftIdx, From, MsgSeqNo, RawMsg, Effects0, +maybe_enqueue(RaftIdx, Ts, From, MsgSeqNo, RawMsg, Effects0, #?MODULE{enqueuers = Enqueuers0, ra_indexes = Indexes0} = State0) -> case maps:get(From, Enqueuers0, undefined) of undefined -> State1 = State0#?MODULE{enqueuers = Enqueuers0#{From => #enqueuer{}}}, - {ok, State, Effects} = maybe_enqueue(RaftIdx, From, MsgSeqNo, + {ok, State, Effects} = maybe_enqueue(RaftIdx, Ts, From, MsgSeqNo, RawMsg, Effects0, State1), {ok, State, [{monitor, process, From} | Effects]}; #enqueuer{next_seqno = MsgSeqNo} = Enq0 -> % it is the next expected seqno - State1 = enqueue(RaftIdx, RawMsg, State0), + State1 = enqueue(RaftIdx, Ts, RawMsg, State0), Enq = Enq0#enqueuer{next_seqno = MsgSeqNo + 1}, State = enqueue_pending(From, Enq, State1), {ok, State, Effects0}; #enqueuer{next_seqno = Next, pending = Pending0} = Enq0 when MsgSeqNo > Next -> - % out of order enqueue - Pending = [{MsgSeqNo, RaftIdx, RawMsg} | Pending0], + % out of order delivery + Pending = [{MsgSeqNo, RaftIdx, Ts, RawMsg} | Pending0], Enq = Enq0#enqueuer{pending = lists:sort(Pending)}, %% if the enqueue it out of order we need to mark it in the %% index @@ -1426,29 +1609,39 @@ return(#{index := IncomingRaftIdx} = Meta, ConsumerId, Returned, {State, ok, Effects} = checkout(Meta, State0, State2, Effects1, false), update_smallest_raft_index(IncomingRaftIdx, State, Effects). -% used to processes messages that are finished +% used to process messages that are finished complete(Meta, ConsumerId, DiscardedMsgIds, - #consumer{checked_out = Checked} = Con0, Effects, + #consumer{checked_out = Checked} = Con0, #?MODULE{messages_total = Tot, - ra_indexes = Indexes0} = State0) -> + ra_indexes = Indexes0} = State0, Delete) -> %% credit_mode = simple_prefetch should automatically top-up credit %% as messages are simple_prefetch or otherwise returned Discarded = maps:with(DiscardedMsgIds, Checked), + DiscardedMsgs = maps:values(Discarded), + Len = length(DiscardedMsgs), Con = Con0#consumer{checked_out = maps:without(DiscardedMsgIds, Checked), - credit = increase_credit(Con0, map_size(Discarded))}, + credit = increase_credit(Con0, Len)}, State1 = update_or_remove_sub(Meta, ConsumerId, Con, State0), + State = lists:foldl(fun(Msg, Acc) -> + add_bytes_settle( + get_msg_header(Msg), Acc) + end, State1, DiscardedMsgs), + case Delete of + true -> + Indexes = delete_indexes(DiscardedMsgs, Indexes0), + State#?MODULE{messages_total = Tot - Len, + ra_indexes = Indexes}; + false -> + State + end. + +delete_indexes(Msgs, Indexes) -> %% TODO: optimise by passing a list to rabbit_fifo_index - Indexes = maps:fold(fun (_, ?INDEX_MSG(I, _), Acc0) when is_integer(I) -> - rabbit_fifo_index:delete(I, Acc0); - (_, _, Acc) -> - Acc - end, Indexes0, Discarded), - State = maps:fold(fun(_, Msg, Acc) -> - add_bytes_settle( - get_msg_header(Msg), Acc) - end, State1, Discarded), - {State#?MODULE{messages_total = Tot - length(DiscardedMsgIds), - ra_indexes = Indexes}, Effects}. + lists:foldl(fun (?INDEX_MSG(I, ?MSG(_,_)), Acc) when is_integer(I) -> + rabbit_fifo_index:delete(I, Acc); + (_, Acc) -> + Acc + end, Indexes, Msgs). increase_credit(#consumer{lifetime = once, credit = Credit}, _) -> @@ -1464,10 +1657,9 @@ increase_credit(#consumer{credit = Current}, Credit) -> complete_and_checkout(#{index := IncomingRaftIdx} = Meta, MsgIds, ConsumerId, #consumer{} = Con0, - Effects0, State0) -> - {State1, Effects1} = complete(Meta, ConsumerId, MsgIds, Con0, - Effects0, State0), - {State, ok, Effects} = checkout(Meta, State0, State1, Effects1, false), + Effects0, State0, Delete) -> + State1 = complete(Meta, ConsumerId, MsgIds, Con0, State0, Delete), + {State, ok, Effects} = checkout(Meta, State0, State1, Effects0, false), update_smallest_raft_index(IncomingRaftIdx, State, Effects). dead_letter_effects(_Reason, _Discarded, @@ -1475,12 +1667,14 @@ dead_letter_effects(_Reason, _Discarded, Effects) -> Effects; dead_letter_effects(Reason, Discarded, - #?MODULE{cfg = #cfg{dead_letter_handler = {Mod, Fun, Args}}}, + #?MODULE{cfg = #cfg{dead_letter_handler = {at_most_once, {Mod, Fun, Args}}}}, Effects) -> RaftIdxs = lists:filtermap( fun (?INDEX_MSG(RaftIdx, ?DISK_MSG(_Header))) -> {true, RaftIdx}; - (_) -> + ({_PerMsgReason, ?INDEX_MSG(RaftIdx, ?DISK_MSG(_Header))}) when Reason =:= undefined -> + {true, RaftIdx}; + (_IgnorePrefixMessage) -> false end, Discarded), [{log, RaftIdxs, @@ -1492,7 +1686,12 @@ dead_letter_effects(Reason, Discarded, {true, {Reason, Msg}}; (?INDEX_MSG(_, ?MSG(_Header, Msg))) -> {true, {Reason, Msg}}; - (_) -> + ({PerMsgReason, ?INDEX_MSG(RaftIdx, ?DISK_MSG(_Header))}) when Reason =:= undefined -> + {enqueue, _, _, Msg} = maps:get(RaftIdx, Lookup), + {true, {PerMsgReason, Msg}}; + ({PerMsgReason, ?INDEX_MSG(_, ?MSG(_Header, Msg))}) when Reason =:= undefined -> + {true, {PerMsgReason, Msg}}; + (_IgnorePrefixMessage) -> false end, Discarded), [{mod_call, Mod, Fun, Args ++ [DeadLetters]}] @@ -1592,7 +1791,9 @@ get_header(Key, Header) when is_map(Header) -> return_one(Meta, MsgId, Msg0, #?MODULE{returns = Returns, consumers = Consumers, - cfg = #cfg{delivery_limit = DeliveryLimit}} = State0, + dlx = DlxState0, + cfg = #cfg{delivery_limit = DeliveryLimit, + dead_letter_handler = DLH}} = State0, Effects0, ConsumerId) -> #consumer{checked_out = Checked} = Con0 = maps:get(ConsumerId, Consumers), Msg = update_msg_header(delivery_count, fun (C) -> C + 1 end, 1, Msg0), @@ -1600,9 +1801,17 @@ return_one(Meta, MsgId, Msg0, case get_header(delivery_count, Header) of DeliveryCount when DeliveryCount > DeliveryLimit -> %% TODO: don't do for prefix msgs - Effects = dead_letter_effects(delivery_limit, [Msg], - State0, Effects0), - complete(Meta, ConsumerId, [MsgId], Con0, Effects, State0); + case DLH of + at_least_once -> + DlxState = rabbit_fifo_dlx:discard(Msg, delivery_limit, DlxState0), + State = complete(Meta, ConsumerId, [MsgId], Con0, State0#?MODULE{dlx = DlxState}, false), + {State, Effects0}; + _ -> + Effects = dead_letter_effects(delivery_limit, [Msg], + State0, Effects0), + State = complete(Meta, ConsumerId, [MsgId], Con0, State0, true), + {State, Effects} + end; _ -> Con = Con0#consumer{checked_out = maps:remove(MsgId, Checked)}, @@ -1649,11 +1858,15 @@ return_all(Meta, #?MODULE{consumers = Cons} = State0, Effects0, ConsumerId, checkout(Meta, OldState, State, Effects) -> checkout(Meta, OldState, State, Effects, true). -checkout(#{index := Index} = Meta, #?MODULE{cfg = #cfg{resource = QName}} = OldState, +checkout(#{index := Index} = Meta, + #?MODULE{cfg = #cfg{resource = QName}} = OldState, State0, Effects0, HandleConsumerChanges) -> - {State1, _Result, Effects1} = checkout0(Meta, checkout_one(Meta, State0), - Effects0, #{}), - case evaluate_limit(Index, false, OldState, State1, Effects1) of + {#?MODULE{dlx = DlxState0} = State1, _Result, Effects1} = checkout0(Meta, checkout_one(Meta, State0, Effects0), #{}), + %%TODO For now we checkout the discards queue here. Move it to a better place + {DlxState1, DlxDeliveryEffects} = rabbit_fifo_dlx:checkout(DlxState0), + State2 = State1#?MODULE{dlx = DlxState1}, + Effects2 = DlxDeliveryEffects ++ Effects1, + case evaluate_limit(Index, false, OldState, State2, Effects2) of {State, true, Effects} -> case maybe_notify_decorators(State, HandleConsumerChanges) of {true, {MaxActivePriority, IsEmpty}} -> @@ -1673,27 +1886,31 @@ checkout(#{index := Index} = Meta, #?MODULE{cfg = #cfg{resource = QName}} = OldS end. checkout0(Meta, {success, ConsumerId, MsgId, - ?INDEX_MSG(RaftIdx, ?DISK_MSG(Header)), State}, - Effects, SendAcc0) when is_integer(RaftIdx) -> + ?INDEX_MSG(RaftIdx, ?DISK_MSG(Header)), State, Effects}, + SendAcc0) when is_integer(RaftIdx) -> DelMsg = {RaftIdx, {MsgId, Header}}, SendAcc = maps:update_with(ConsumerId, - fun ({InMem, LogMsgs}) -> - {InMem, [DelMsg | LogMsgs]} - end, {[], [DelMsg]}, SendAcc0), - checkout0(Meta, checkout_one(Meta, State), Effects, SendAcc); + fun ({InMem, LogMsgs}) -> + {InMem, [DelMsg | LogMsgs]} + end, {[], [DelMsg]}, SendAcc0), + checkout0(Meta, checkout_one(Meta, State, Effects), SendAcc); checkout0(Meta, {success, ConsumerId, MsgId, - ?INDEX_MSG(Idx, ?MSG(Header, Msg)), State}, Effects, + ?INDEX_MSG(Idx, ?MSG(Header, Msg)), State, Effects}, SendAcc0) when is_integer(Idx) -> DelMsg = {MsgId, {Header, Msg}}, SendAcc = maps:update_with(ConsumerId, - fun ({InMem, LogMsgs}) -> - {[DelMsg | InMem], LogMsgs} - end, {[DelMsg], []}, SendAcc0), - checkout0(Meta, checkout_one(Meta, State), Effects, SendAcc); -checkout0(Meta, {success, _ConsumerId, _MsgId, ?TUPLE(_, _), State}, Effects, + fun ({InMem, LogMsgs}) -> + {[DelMsg | InMem], LogMsgs} + end, {[DelMsg], []}, SendAcc0), + checkout0(Meta, checkout_one(Meta, State, Effects), SendAcc); +checkout0(Meta, {success, _ConsumerId, _MsgId, ?TUPLE(_, _), State, Effects}, SendAcc) -> - checkout0(Meta, checkout_one(Meta, State), Effects, SendAcc); -checkout0(_Meta, {Activity, State0}, Effects0, SendAcc) -> + %% Do not append delivery effect for prefix messages. + %% Prefix messages do not exist anymore, but they still go through the + %% normal checkout flow to derive correct consumer states + %% after recovery and will still be settled or discarded later on. + checkout0(Meta, checkout_one(Meta, State, Effects), SendAcc); +checkout0(_Meta, {Activity, State0, Effects0}, SendAcc) -> Effects1 = case Activity of nochange -> append_delivery_effects(Effects0, SendAcc); @@ -1844,9 +2061,12 @@ reply_log_effect(RaftIdx, MsgId, Header, Ready, From) -> {dequeue, {MsgId, {Header, Msg}}, Ready}}}] end}. -checkout_one(Meta, #?MODULE{service_queue = SQ0, - messages = Messages0, - consumers = Cons0} = InitState) -> +checkout_one(#{system_time := Ts} = Meta, InitState0, Effects0) -> + %% Before checking out any messsage to any consumer, + %% first remove all expired messages from the head of the queue. + {#?MODULE{service_queue = SQ0, + messages = Messages0, + consumers = Cons0} = InitState, Effects1} = expire_msgs(Ts, InitState0, Effects0), case priority_queue:out(SQ0) of {{value, ConsumerId}, SQ1} when is_map_key(ConsumerId, Cons0) -> @@ -1859,11 +2079,11 @@ checkout_one(Meta, #?MODULE{service_queue = SQ0, %% no credit but was still on queue %% can happen when draining %% recurse without consumer on queue - checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}); + checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}, Effects1); #consumer{status = cancelled} -> - checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}); + checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}, Effects1); #consumer{status = suspected_down} -> - checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}); + checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}, Effects1); #consumer{checked_out = Checked0, next_msg_id = Next, credit = Credit, @@ -1881,27 +2101,102 @@ checkout_one(Meta, #?MODULE{service_queue = SQ0, true -> add_bytes_checkout(Header, State1); false -> + %% TODO do not subtract from memory here since + %% messages are still in memory when checked out subtract_in_memory_counts( Header, add_bytes_checkout(Header, State1)) end, - {success, ConsumerId, Next, ConsumerMsg, State}; + {success, ConsumerId, Next, ConsumerMsg, State, Effects1}; error -> %% consumer did not exist but was queued, recurse - checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}) + checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}, Effects1) end; empty -> - {nochange, InitState} + {nochange, InitState, Effects1} end; {{value, _ConsumerId}, SQ1} -> %% consumer did not exist but was queued, recurse - checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}); + checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}, Effects1); {empty, _} -> + Effects = timer_effect(Ts, InitState, Effects1), case lqueue:len(Messages0) of - 0 -> {nochange, InitState}; - _ -> {inactive, InitState} + 0 -> + {nochange, InitState, Effects}; + _ -> + {inactive, InitState, Effects} end end. +%% dequeue all expired messages +expire_msgs(RaCmdTs, State0, Effects0) -> + case take_next_msg(State0) of + {?INDEX_MSG(Idx, ?MSG(#{expiry := Expiry} = Header, _) = Msg) = FullMsg, State1} + when RaCmdTs >= Expiry -> + #?MODULE{dlx = DlxState0, + cfg = #cfg{dead_letter_handler = DLH}, + ra_indexes = Indexes0} = State2 = add_bytes_drop(Header, State1), + case DLH of + at_least_once -> + DlxState = rabbit_fifo_dlx:discard(FullMsg, expired, DlxState0), + State = State2#?MODULE{dlx = DlxState}, + expire_msgs(RaCmdTs, State, Effects0); + _ -> + Indexes = rabbit_fifo_index:delete(Idx, Indexes0), + State3 = decr_total(State2), + State4 = case Msg of + ?DISK_MSG(_) -> + State3; + _ -> + subtract_in_memory_counts(Header, State3) + end, + Effects = dead_letter_effects(expired, [FullMsg], + State4, Effects0), + State = State4#?MODULE{ra_indexes = Indexes}, + expire_msgs(RaCmdTs, State, Effects) + end; + {?PREFIX_MEM_MSG(#{expiry := Expiry} = Header) = Msg, State1} + when RaCmdTs >= Expiry -> + State2 = expire_prefix_msg(Msg, Header, State1), + expire_msgs(RaCmdTs, State2, Effects0); + {?DISK_MSG(#{expiry := Expiry} = Header) = Msg, State1} + when RaCmdTs >= Expiry -> + State2 = expire_prefix_msg(Msg, Header, State1), + expire_msgs(RaCmdTs, State2, Effects0); + _ -> + {State0, Effects0} + end. + +expire_prefix_msg(Msg, Header, State0) -> + #?MODULE{dlx = DlxState0, + cfg = #cfg{dead_letter_handler = DLH}} = State1 = add_bytes_drop(Header, State0), + case DLH of + at_least_once -> + DlxState = rabbit_fifo_dlx:discard(Msg, expired, DlxState0), + State1#?MODULE{dlx = DlxState}; + _ -> + State2 = case Msg of + ?DISK_MSG(_) -> + State1; + _ -> + subtract_in_memory_counts(Header, State1) + end, + decr_total(State2) + end. + +%%TODO make sure effect is re-issued when becoming leader +timer_effect(RaCmdTs, State, Effects) -> + T = case take_next_msg(State) of + {?INDEX_MSG(_, ?MSG(#{expiry := Expiry}, _)), _} when is_number(Expiry) -> + %% Next message contains 'expiry' header. + %% (Re)set timer so that mesage will be dropped or dead-lettered on time. + Expiry - RaCmdTs; + _ -> + %% Next message does not contain 'expiry' header. + %% Therefore, do not set timer or cancel timer if it was set. + infinity + end, + [{timer, expire_msgs, T} | Effects]. + update_or_remove_sub(_Meta, ConsumerId, #consumer{lifetime = auto, credit = 0} = Con, #?MODULE{consumers = Cons} = State) -> @@ -1996,21 +2291,22 @@ maybe_queue_consumer(ConsumerId, #consumer{credit = Credit} = Con, %% creates a dehydrated version of the current state to be cached and %% potentially used to for a snaphot at a later point dehydrate_state(#?MODULE{msg_bytes_in_memory = 0, - cfg = #cfg{max_length = 0}, + cfg = #cfg{max_in_memory_length = 0}, consumers = Consumers} = State) -> - %% no messages are kept in memory, no need to - %% overly mutate the current state apart from removing indexes and cursors + % no messages are kept in memory, no need to + % overly mutate the current state apart from removing indexes and cursors State#?MODULE{ - ra_indexes = rabbit_fifo_index:empty(), - consumers = maps:map(fun (_, C) -> - dehydrate_consumer(C) - end, Consumers), - release_cursors = lqueue:new()}; + ra_indexes = rabbit_fifo_index:empty(), + consumers = maps:map(fun (_, C) -> + dehydrate_consumer(C) + end, Consumers), + release_cursors = lqueue:new()}; dehydrate_state(#?MODULE{messages = Messages, consumers = Consumers, returns = Returns, prefix_msgs = {PRCnt, PrefRet0, PPCnt, PrefMsg0}, - waiting_consumers = Waiting0} = State) -> + waiting_consumers = Waiting0, + dlx = DlxState} = State) -> RCnt = lqueue:len(Returns), %% TODO: optimise this function as far as possible PrefRet1 = lists:foldr(fun (M, Acc) -> @@ -2031,7 +2327,8 @@ dehydrate_state(#?MODULE{messages = Messages, returns = lqueue:new(), prefix_msgs = {PRCnt + RCnt, PrefRet, PPCnt + lqueue:len(Messages), PrefMsgs}, - waiting_consumers = Waiting}. + waiting_consumers = Waiting, + dlx = rabbit_fifo_dlx:dehydrate(DlxState)}. dehydrate_messages(Msgs0) -> {OutRes, Msgs} = lqueue:out(Msgs0), @@ -2053,7 +2350,8 @@ dehydrate_message(?PREFIX_MEM_MSG(_) = M) -> dehydrate_message(?DISK_MSG(_) = M) -> M; dehydrate_message(?INDEX_MSG(_Idx, ?DISK_MSG(_Header) = Msg)) -> - %% use disk msgs directly as prefix messages + %% Use disk msgs directly as prefix messages. + %% This avoids memory allocation since we do not convert. Msg; dehydrate_message(?INDEX_MSG(Idx, ?MSG(Header, _))) when is_integer(Idx) -> ?PREFIX_MEM_MSG(Header). @@ -2062,11 +2360,13 @@ dehydrate_message(?INDEX_MSG(Idx, ?MSG(Header, _))) when is_integer(Idx) -> normalize(#?MODULE{ra_indexes = _Indexes, returns = Returns, messages = Messages, - release_cursors = Cursors} = State) -> + release_cursors = Cursors, + dlx = DlxState} = State) -> State#?MODULE{ returns = lqueue:from_list(lqueue:to_list(Returns)), messages = lqueue:from_list(lqueue:to_list(Messages)), - release_cursors = lqueue:from_list(lqueue:to_list(Cursors))}. + release_cursors = lqueue:from_list(lqueue:to_list(Cursors)), + dlx = rabbit_fifo_dlx:normalize(DlxState)}. is_over_limit(#?MODULE{cfg = #cfg{max_length = undefined, max_bytes = undefined}}) -> diff --git a/deps/rabbit/src/rabbit_fifo.hrl b/deps/rabbit/src/rabbit_fifo.hrl index c797c9d9bd07..ca37fbca7981 100644 --- a/deps/rabbit/src/rabbit_fifo.hrl +++ b/deps/rabbit/src/rabbit_fifo.hrl @@ -2,16 +2,21 @@ %% macros for memory optimised tuple structures -define(TUPLE(A, B), [A | B]). --define(DISK_MSG_TAG, '$disk'). -% -define(PREFIX_DISK_MSG_TAG, '$prefix_disk'). --define(PREFIX_MEM_MSG_TAG, '$prefix_inmem'). +%% We want short atoms since their binary representations will get +%% persisted in a snapshot for every message. +%% '$d' stand for 'disk'. +-define(DISK_MSG_TAG, '$d'). +%% '$m' stand for 'memory'. +-define(PREFIX_MEM_MSG_TAG, '$m'). -define(DISK_MSG(Header), [Header | ?DISK_MSG_TAG]). -define(MSG(Header, RawMsg), [Header | RawMsg]). -define(INDEX_MSG(Index, Msg), [Index | Msg]). +-define(PREFIX_MEM_MSG(Header), [Header | ?PREFIX_MEM_MSG_TAG]). + +% -define(PREFIX_DISK_MSG_TAG, '$prefix_disk'). % -define(PREFIX_DISK_MSG(Header), [?PREFIX_DISK_MSG_TAG | Header]). % -define(PREFIX_DISK_MSG(Header), ?DISK_MSG(Header)). --define(PREFIX_MEM_MSG(Header), [?PREFIX_MEM_MSG_TAG | Header]). -type option(T) :: undefined | T. @@ -32,11 +37,14 @@ %% same process -type msg_header() :: msg_size() | - #{size := msg_size(), - delivery_count => non_neg_integer()}. +#{size := msg_size(), + delivery_count => non_neg_integer(), + expiry => milliseconds()}. %% The message header: %% delivery_count: the number of unsuccessful delivery attempts. %% A non-zero value indicates a previous attempt. +%% expiry: Epoch time in ms when a message expires. Set during enqueue. +%% Value is determined by per-queue or per-message message TTL. %% If it only contains the size it can be condensed to an integer only -type msg() :: ?MSG(msg_header(), raw_msg()) | @@ -122,7 +130,7 @@ -record(enqueuer, {next_seqno = 1 :: msg_seqno(), % out of order enqueues - sorted list - pending = [] :: [{msg_seqno(), ra:index(), raw_msg()}], + pending = [] :: [{msg_seqno(), ra:index(), milliseconds(), raw_msg()}], status = up :: up | suspected_down, %% it is useful to have a record of when this was blocked @@ -137,7 +145,7 @@ {name :: atom(), resource :: rabbit_types:r('queue'), release_cursor_interval :: option({non_neg_integer(), non_neg_integer()}), - dead_letter_handler :: option(applied_mfa()), + dead_letter_handler :: option({at_most_once, applied_mfa()} | at_least_once), become_leader_handler :: option(applied_mfa()), overflow_strategy = drop_head :: drop_head | reject_publish, max_length :: option(non_neg_integer()), @@ -149,6 +157,7 @@ max_in_memory_length :: option(non_neg_integer()), max_in_memory_bytes :: option(non_neg_integer()), expires :: undefined | milliseconds(), + msg_ttl :: undefined | milliseconds(), unused_1, unused_2 }). @@ -166,6 +175,7 @@ % queue of returned msg_in_ids - when checking out it picks from returns = lqueue:new() :: lqueue:lqueue(term()), % a counter of enqueues - used to trigger shadow copy points + % reset to 0 when release_cursor gets stored enqueue_count = 0 :: non_neg_integer(), % a map containing all the live processes that have ever enqueued % a message to this queue as well as a cached value of the smallest @@ -177,11 +187,19 @@ % index when there are large gaps but should be faster than gb_trees % for normal appending operations as it's backed by a map ra_indexes = rabbit_fifo_index:empty() :: rabbit_fifo_index:state(), + %% A release cursor is essentially a snapshot without message bodies + %% (aka. "dehydrated state") taken at time T in order to truncate + %% the log at some point in the future when all messages that were enqueued + %% up to time T have been removed (e.g. consumed, dead-lettered, or dropped). + %% This concept enables snapshots to not contain any message bodies. + %% Advantage: Smaller snapshots are sent between Ra nodes. + %% Working assumption: Messages are consumed in a FIFO-ish order because + %% the log is truncated only until the oldest message. release_cursors = lqueue:new() :: lqueue:lqueue({release_cursor, ra:index(), #rabbit_fifo{}}), % consumers need to reflect consumer state at time of snapshot % needs to be part of snapshot - consumers = #{} :: #{consumer_id() => #consumer{}}, + consumers = #{} :: #{consumer_id() => consumer()}, % consumers that require further service are queued here % needs to be part of snapshot service_queue = priority_queue:new() :: priority_queue:q(), @@ -194,7 +212,10 @@ %% overflow calculations). %% This is done so that consumers are still served in a deterministic %% order on recovery. + %% TODO Remove this field and store prefix messages in-place. This will + %% simplify the checkout logic. prefix_msgs = {0, [], 0, []} :: prefix_msgs(), + dlx = rabbit_fifo_dlx:init() :: rabbit_fifo_dlx:state(), msg_bytes_enqueue = 0 :: non_neg_integer(), msg_bytes_checkout = 0 :: non_neg_integer(), %% waiting consumers, one is picked active consumer is cancelled or dies @@ -209,7 +230,7 @@ -type config() :: #{name := atom(), queue_resource := rabbit_types:r('queue'), - dead_letter_handler => applied_mfa(), + dead_letter_handler => option({at_most_once, applied_mfa()} | at_least_once), become_leader_handler => applied_mfa(), release_cursor_interval => non_neg_integer(), max_length => non_neg_integer(), @@ -220,5 +241,6 @@ single_active_consumer_on => boolean(), delivery_limit => non_neg_integer(), expires => non_neg_integer(), + msg_ttl => non_neg_integer(), created => non_neg_integer() }. diff --git a/deps/rabbit/src/rabbit_fifo_client.erl b/deps/rabbit/src/rabbit_fifo_client.erl index 3f5315de08b2..0faf32fd300d 100644 --- a/deps/rabbit/src/rabbit_fifo_client.erl +++ b/deps/rabbit/src/rabbit_fifo_client.erl @@ -531,7 +531,7 @@ update_machine_state(Server, Conf) -> %% `{internal, AppliedCorrelations, State}' if the event contained an internally %% handled event such as a notification and a correlation was included with %% the command (e.g. in a call to `enqueue/3' the correlation terms are returned -%% here. +%% here). %% %% `{RaFifoEvent, State}' if the event contained a client message generated by %% the `rabbit_fifo' state machine such as a delivery. diff --git a/deps/rabbit/src/rabbit_fifo_dlx.erl b/deps/rabbit/src/rabbit_fifo_dlx.erl new file mode 100644 index 000000000000..1e0bc5fcd4a6 --- /dev/null +++ b/deps/rabbit/src/rabbit_fifo_dlx.erl @@ -0,0 +1,314 @@ +-module(rabbit_fifo_dlx). + +-include("rabbit_fifo_dlx.hrl"). +-include("rabbit_fifo.hrl"). + +% client API, e.g. for rabbit_fifo_dlx_client +-export([make_checkout/2, + make_settle/1]). + +% called by rabbit_fifo delegating DLX handling to this module +-export([init/0, apply/2, discard/3, overview/1, + checkout/1, state_enter/4, + start_worker/2, terminate_worker/1, cleanup/1, purge/1, + consumer_pid/1, dehydrate/1, normalize/1]). + +%% This module handles the dead letter (DLX) part of the rabbit_fifo state machine. +%% This is a separate module to better unit test and provide separation of concerns. +%% This module maintains its own state: +%% a queue of DLX messages, a single node local DLX consumer, and some stats. +%% The state of this module is included into rabbit_fifo state because there can only by one Ra state machine. +%% The rabbit_fifo module forwards all DLX commands to this module where we then update the DLX specific state only: +%% e.g. DLX consumer subscribed, adding / removing discarded messages, stats +%% +%% It also runs its own checkout logic sending DLX messages to the DLX consumer. +%% +%% TODO: Does it hook into the tick as well? +%% + +-record(checkout,{ + consumer :: atom(), + prefetch :: non_neg_integer() + }). +-record(settle, {msg_ids :: [msg_id()]}). +-opaque protocol() :: {dlx, #checkout{} | #settle{}}. +-opaque state() :: #?MODULE{}. +-export_type([state/0, protocol/0, reason/0]). + +init() -> + #?MODULE{}. + +make_checkout(RegName, NumUnsettled) -> + {dlx, #checkout{consumer = RegName, + prefetch = NumUnsettled + }}. + +make_settle(MessageIds) when is_list(MessageIds) -> + {dlx, #settle{msg_ids = MessageIds}}. + +overview(#?MODULE{consumer = undefined, + msg_bytes = MsgBytes, + msg_bytes_checkout = 0, + discards = Discards}) -> + overview0(Discards, #{}, MsgBytes, 0); +overview(#?MODULE{consumer = #dlx_consumer{checked_out = Checked}, + msg_bytes = MsgBytes, + msg_bytes_checkout = MsgBytesCheckout, + discards = Discards}) -> + overview0(Discards, Checked, MsgBytes, MsgBytesCheckout). + +overview0(Discards, Checked, MsgBytes, MsgBytesCheckout) -> + #{num_discarded => lqueue:len(Discards), + num_discard_checked_out => map_size(Checked), + discard_message_bytes => MsgBytes, + discard_checkout_message_bytes => MsgBytesCheckout}. + +apply(#checkout{consumer = RegName, + prefetch = Prefetch}, + #?MODULE{consumer = undefined} = State0) -> + State = State0#?MODULE{consumer = #dlx_consumer{registered_name = RegName, + prefetch = Prefetch}}, + {State, ok}; +apply(#checkout{consumer = RegName, + prefetch = Prefetch}, + #?MODULE{consumer = #dlx_consumer{checked_out = CheckedOutOldConsumer}, + discards = Discards0, + msg_bytes = Bytes, + msg_bytes_checkout = BytesCheckout} = State0) -> + %% Since we allow only a single consumer, the new consumer replaces the old consumer. + %% All checked out messages to the old consumer need to be returned to the discards queue + %% such that these messages can be (eventually) re-delivered to the new consumer. + %% When inserting back into the discards queue, we respect the original order in which messages + %% were discarded. + Checked0 = maps:to_list(CheckedOutOldConsumer), + Checked1 = lists:keysort(1, Checked0), + {Discards, BytesMoved} = lists:foldr(fun({_Id, {_Reason, IdxMsg} = Msg}, {D, B}) -> + {lqueue:in_r(Msg, D), B + size_in_bytes(IdxMsg)} + end, {Discards0, 0}, Checked1), + State = State0#?MODULE{consumer = #dlx_consumer{registered_name = RegName, + prefetch = Prefetch}, + discards = Discards, + msg_bytes = Bytes + BytesMoved, + msg_bytes_checkout = BytesCheckout - BytesMoved}, + {State, ok}; +apply(#settle{msg_ids = MsgIds}, + #?MODULE{consumer = #dlx_consumer{checked_out = Checked} = C, + msg_bytes_checkout = BytesCheckout} = State0) -> + Acked = maps:with(MsgIds, Checked), + AckedRsnMsgs = maps:values(Acked), + AckedMsgs = lists:map(fun({_Reason, Msg}) -> Msg end, AckedRsnMsgs), + AckedBytes = lists:foldl(fun(Msg, Bytes) -> + Bytes + size_in_bytes(Msg) + end, 0, AckedMsgs), + Unacked = maps:without(MsgIds, Checked), + State = State0#?MODULE{consumer = C#dlx_consumer{checked_out = Unacked}, + msg_bytes_checkout = BytesCheckout - AckedBytes}, + {State, AckedMsgs}. + +%%TODO delete delivery_count header to save space? +%% It's not needed anymore. +discard(Msg, Reason, #?MODULE{discards = Discards0, + msg_bytes = MsgBytes0} = State) -> + Discards = lqueue:in({Reason, Msg}, Discards0), + MsgBytes = MsgBytes0 + size_in_bytes(Msg), + State#?MODULE{discards = Discards, + msg_bytes = MsgBytes}. + +checkout(#?MODULE{consumer = undefined, + discards = Discards} = State) -> + case lqueue:is_empty(Discards) of + true -> + ok; + false -> + rabbit_log:warning("there are dead-letter messages but no dead-letter consumer") + end, + {State, []}; +checkout(State) -> + checkout0(checkout_one(State), {[],[]}). + +checkout0({success, MsgId, {Reason, ?INDEX_MSG(RaftIdx, ?DISK_MSG(Header))}, State}, {InMemMsgs, LogMsgs}) when is_integer(RaftIdx) -> + DelMsg = {RaftIdx, {Reason, MsgId, Header}}, + SendAcc = {InMemMsgs, [DelMsg|LogMsgs]}, + checkout0(checkout_one(State ), SendAcc); +checkout0({success, MsgId, {Reason, ?INDEX_MSG(Idx, ?MSG(Header, Msg))}, State}, {InMemMsgs, LogMsgs}) when is_integer(Idx) -> + DelMsg = {MsgId, {Reason, Header, Msg}}, + SendAcc = {[DelMsg|InMemMsgs], LogMsgs}, + checkout0(checkout_one(State), SendAcc); +checkout0({success, _MsgId, {_Reason, ?TUPLE(_, _)}, State}, SendAcc) -> + %% This is a prefix message which means we are recovering from a snapshot. + %% We know: + %% 1. This message was already delivered in the past, and + %% 2. The recovery Raft log ahead of this Raft command will defintely settle this message. + %% Therefore, here, we just check this message out to the consumer but do not re-deliver this message + %% so that we will end up with the correct and deterministic state once the whole recovery log replay is completed. + checkout0(checkout_one(State), SendAcc); +checkout0(#?MODULE{consumer = #dlx_consumer{registered_name = RegName}} = State, SendAcc) -> + Effects = delivery_effects(whereis(RegName), SendAcc), + {State, Effects}. + +checkout_one(#?MODULE{consumer = #dlx_consumer{checked_out = Checked, + prefetch = Prefetch}} = State) when map_size(Checked) >= Prefetch -> + State; +checkout_one(#?MODULE{consumer = #dlx_consumer{checked_out = Checked0, + next_msg_id = Next} = Con0} = State0) -> + case take_next_msg(State0) of + {{_, Msg} = ReasonMsg, State1} -> + Checked = maps:put(Next, ReasonMsg, Checked0), + State2 = State1#?MODULE{consumer = Con0#dlx_consumer{checked_out = Checked, + next_msg_id = Next + 1}}, + Bytes = size_in_bytes(Msg), + State = add_bytes_checkout(Bytes, State2), + {success, Next, ReasonMsg, State}; + empty -> + State0 + end. + +take_next_msg(#?MODULE{discards = Discards0} = State) -> + case lqueue:out(Discards0) of + {empty, _} -> + empty; + {{value, ReasonMsg}, Discards} -> + {ReasonMsg, State#?MODULE{discards = Discards}} + end. + +add_bytes_checkout(Size, #?MODULE{msg_bytes = Bytes, + msg_bytes_checkout = BytesCheckout} = State) -> + State#?MODULE{msg_bytes = Bytes - Size, + msg_bytes_checkout = BytesCheckout + Size}. + +size_in_bytes(Msg) -> + Header = rabbit_fifo:get_msg_header(Msg), + rabbit_fifo:get_header(size, Header). + +%% returns at most one delivery effect because there is only one consumer +delivery_effects(_CPid, {[], []}) -> + []; +delivery_effects(CPid, {InMemMsgs, []}) -> + [{send_msg, CPid, {dlx_delivery, lists:reverse(InMemMsgs)}, [ra_event]}]; +delivery_effects(CPid, {InMemMsgs, IdxMsgs0}) -> + IdxMsgs = lists:reverse(IdxMsgs0), + {RaftIdxs, Data} = lists:unzip(IdxMsgs), + [{log, RaftIdxs, + fun(Log) -> + Msgs0 = lists:zipwith(fun ({enqueue, _, _, Msg}, {Reason, MsgId, Header}) -> + {MsgId, {Reason, Header, Msg}} + end, Log, Data), + Msgs = case InMemMsgs of + [] -> + Msgs0; + _ -> + lists:sort(InMemMsgs ++ Msgs0) + end, + [{send_msg, CPid, {dlx_delivery, Msgs}, [ra_event]}] + end}]. + %%TODO rabbit_fifo_prop_SUITE:dlx_04 fails with below line + % {local, node(CPid)}}]. + +state_enter(leader, QRef, QName, _State) -> + start_worker(QRef, QName); +state_enter(_, _, _, State) -> + terminate_worker(State). + +start_worker(QRef, QName) -> + RegName = registered_name(QName), + %% We must ensure that starting the rabbit_fifo_dlx_worker succeeds. + %% Therefore, we don't use an effect. + %% Also therefore, if starting the rabbit_fifo_dlx_worker fails, let the whole Ra server process crash + %% in which case another Ra node will become leader. + %% supervisor:start_child/2 blocks until rabbit_fifo_dlx_worker:init/1 returns (TODO check if this is correct). + %% That's okay since rabbit_fifo_dlx_worker:init/1 returns immediately by delegating + %% initial setup to handle_continue/2. + case whereis(RegName) of + undefined -> + {ok, Pid} = supervisor:start_child(rabbit_fifo_dlx_sup, [QRef, RegName]), + rabbit_log:debug("started rabbit_fifo_dlx_worker (~s ~p)", [RegName, Pid]); + Pid -> + rabbit_log:debug("rabbit_fifo_dlx_worker (~s ~p) already started", [RegName, Pid]) + end. + +terminate_worker(#?MODULE{consumer = #dlx_consumer{registered_name = RegName}}) -> + case whereis(RegName) of + undefined -> + ok; + Pid -> + %% Note that we can't return a mod_call effect here because mod_call is executed on the leader only. + ok = supervisor:terminate_child(rabbit_fifo_dlx_sup, Pid), + rabbit_log:debug("terminated rabbit_fifo_dlx_worker (~s ~p)", [RegName, Pid]) + end; +terminate_worker(_) -> + ok. + +%% TODO consider not registering the worker name at all +%% because if there is a new worker process, it will always subscribe and tell us its new pid +registered_name(QName) when is_atom(QName) -> + list_to_atom(atom_to_list(QName) ++ "_dlx"). + +consumer_pid(#?MODULE{consumer = #dlx_consumer{registered_name = Name}}) -> + whereis(Name); +consumer_pid(_) -> + undefined. + +%% called when switching from at-least-once to at-most-once +cleanup(#?MODULE{consumer = Consumer, + discards = Discards} = State) -> + terminate_worker(State), + %% Return messages in the order they got discarded originally + %% for the final at-most-once dead-lettering. + CheckedReasonMsgs = case Consumer of + #dlx_consumer{checked_out = Checked} when is_map(Checked) -> + L0 = maps:to_list(Checked), + L1 = lists:keysort(1, L0), + {_, L2} = lists:unzip(L1), + L2; + _ -> + [] + end, + DiscardReasonMsgs = lqueue:to_list(Discards), + CheckedReasonMsgs ++ DiscardReasonMsgs. + +purge(#?MODULE{consumer = Con0, + discards = Discards} = State0) -> + {Con, CheckedMsgs} = case Con0 of + #dlx_consumer{checked_out = Checked} when is_map(Checked) -> + L = maps:to_list(Checked), + {_, CheckedReasonMsgs} = lists:unzip(L), + {_, Msgs} = lists:unzip(CheckedReasonMsgs), + C = Con0#dlx_consumer{checked_out = #{}}, + {C, Msgs}; + _ -> + {Con0, []} + end, + DiscardReasonMsgs = lqueue:to_list(Discards), + {_, DiscardMsgs} = lists:unzip(DiscardReasonMsgs), + PurgedMsgs = CheckedMsgs ++ DiscardMsgs, + State = State0#?MODULE{consumer = Con, + discards = lqueue:new(), + msg_bytes = 0, + msg_bytes_checkout = 0 + }, + {State, PurgedMsgs}. + +%% TODO Consider alternative to not dehydrate at all +%% by putting messages to disk before enqueueing them in discards queue. +dehydrate(#?MODULE{discards = Discards, + consumer = Con} = State) -> + State#?MODULE{discards = dehydrate_messages(Discards), + consumer = dehydrate_consumer(Con)}. + +dehydrate_messages(Discards) -> + L0 = lqueue:to_list(Discards), + L1 = lists:map(fun({_Reason, Msg}) -> + {?NIL, rabbit_fifo:dehydrate_message(Msg)} + end, L0), + lqueue:from_list(L1). + +dehydrate_consumer(#dlx_consumer{checked_out = Checked0} = Con) -> + Checked = maps:map(fun (_, {_, Msg}) -> + {?NIL, rabbit_fifo:dehydrate_message(Msg)} + end, Checked0), + Con#dlx_consumer{checked_out = Checked}; +dehydrate_consumer(undefined) -> + undefined. + +normalize(#?MODULE{discards = Discards} = State) -> + State#?MODULE{discards = lqueue:from_list(lqueue:to_list(Discards))}. diff --git a/deps/rabbit/src/rabbit_fifo_dlx.hrl b/deps/rabbit/src/rabbit_fifo_dlx.hrl new file mode 100644 index 000000000000..1b545ab4027e --- /dev/null +++ b/deps/rabbit/src/rabbit_fifo_dlx.hrl @@ -0,0 +1,31 @@ +-define(NIL, []). + +%% At-least-once dead-lettering does not support reason 'maxlen'. +%% Reason of prefix messages is [] because the message will not be +%% actually delivered and storing 2 bytes in the persisted snapshot +%% is less than the reason atom. +-type reason() :: 'expired' | 'rejected' | delivery_limit | ?NIL. + +% See snapshot scenarios in rabbit_fifo_prop_SUITE. Add dlx dehydrate tests. +-record(dlx_consumer,{ + %% We don't require a consumer tag because a consumer tag is a means to distinguish + %% multiple consumers in the same channel. The rabbit_fifo_dlx_worker channel like process however + %% creates only a single consumer to this quorum queue's discards queue. + registered_name :: atom(), + prefetch :: non_neg_integer(), + %%TODO use ?TUPLE for memory optimisation? + checked_out = #{} :: #{msg_id() => {reason(), indexed_msg()}}, + next_msg_id = 0 :: msg_id() % part of snapshot data + % total number of checked out messages - ever + % incremented for each delivery + % delivery_count = 0 :: non_neg_integer(), + % status = up :: up | suspected_down | cancelled + }). + +-record(rabbit_fifo_dlx,{ + consumer = undefined :: #dlx_consumer{} | undefined, + %% Queue of dead-lettered messages. + discards = lqueue:new() :: lqueue:lqueue({reason(), indexed_msg()}), + msg_bytes = 0 :: non_neg_integer(), + msg_bytes_checkout = 0 :: non_neg_integer() + }). diff --git a/deps/rabbit/src/rabbit_fifo_dlx_client.erl b/deps/rabbit/src/rabbit_fifo_dlx_client.erl new file mode 100644 index 000000000000..466a2523194a --- /dev/null +++ b/deps/rabbit/src/rabbit_fifo_dlx_client.erl @@ -0,0 +1,90 @@ +-module(rabbit_fifo_dlx_client). + +-export([checkout/4, settle/2, handle_ra_event/3, + overview/1]). + +-record(state,{ + queue_resource :: rabbit_tyes:r(queue), + leader :: ra:server_id(), + last_msg_id :: non_neg_integer | -1 + }). +-opaque state() :: #state{}. +-export_type([state/0]). + +checkout(RegName, QResource, Leader, NumUnsettled) -> + Cmd = rabbit_fifo_dlx:make_checkout(RegName, NumUnsettled), + State = #state{queue_resource = QResource, + leader = Leader, + last_msg_id = -1}, + process_command(Cmd, State, 5). + +settle(MsgIds, State) when is_list(MsgIds) -> + Cmd = rabbit_fifo_dlx:make_settle(MsgIds), + process_command(Cmd, State, 2). + +process_command(_Cmd, _State, 0) -> + {error, ra_command_failed}; +process_command(Cmd, #state{leader = Leader} = State, Tries) -> + case ra:process_command(Leader, Cmd, 60_000) of + {ok, ok, Leader} -> + {ok, State#state{leader = Leader}}; + {ok, ok, L} -> + rabbit_log:warning("Failed to process command ~p on quorum queue leader ~p because actual leader is ~p.", + [Cmd, Leader, L]), + {error, ra_command_failed}; + Err -> + rabbit_log:warning("Failed to process command ~p on quorum queue leader ~p: ~p~n" + "Trying ~b more time(s)...", + [Cmd, Leader, Err, Tries]), + process_command(Cmd, State, Tries - 1) + end. + +handle_ra_event(Leader, {machine, {dlx_delivery, _} = Del}, #state{leader = Leader} = State) -> + handle_delivery(Del, State); +handle_ra_event(_From, Evt, State) -> + rabbit_log:warning("~s received unknown ra event: ~p", [?MODULE, Evt]), + {ok, State, []}. + +handle_delivery({dlx_delivery, [{FstId, _} | _] = IdMsgs}, + #state{queue_resource = QRes, + last_msg_id = Prev} = State0) -> + %% format as a deliver action + {LastId, _} = lists:last(IdMsgs), + Del = {deliver, transform_msgs(QRes, IdMsgs)}, + case Prev of + Prev when FstId =:= Prev+1 -> + %% expected message ID(s) got delivered + State = State0#state{last_msg_id = LastId}, + {ok, State, [Del]}; + Prev when FstId > Prev+1 -> + %% messages ID(s) are missing, therefore fetch all checked-out discarded messages + %% TODO implement as done in + %% https://github.com/rabbitmq/rabbitmq-server/blob/b4eb5e2cfd7f85a1681617dc489dd347fa9aac72/deps/rabbit/src/rabbit_fifo_client.erl#L732-L744 + exit(not_implemented); + Prev when FstId =< Prev -> + rabbit_log:debug("dropping messages with duplicate IDs (~b to ~b) consumed from ~s", + [FstId, Prev, rabbit_misc:rs(QRes)]), + case lists:dropwhile(fun({Id, _}) -> Id =< Prev end, IdMsgs) of + [] -> + {ok, State0, []}; + IdMsgs2 -> + handle_delivery({dlx_delivery, IdMsgs2}, State0) + end; + _ when FstId =:= 0 -> + % the very first delivery + % TODO We init last_msg_id with -1. So, why would we ever run into this branch? + rabbit_log:debug("very first delivery consumed from ~s", [rabbit_misc:rs(QRes)]), + State = State0#state{last_msg_id = 0}, + {ok, State, [Del]} + end. + +transform_msgs(QRes, Msgs) -> + lists:map( + fun({MsgId, {Reason, _MsgHeader, Msg}}) -> + {QRes, MsgId, Msg, Reason} + end, Msgs). + +overview(#state{leader = Leader, + last_msg_id = LastMsgId}) -> + #{leader => Leader, + last_msg_id => LastMsgId}. diff --git a/deps/rabbit/src/rabbit_fifo_dlx_sup.erl b/deps/rabbit/src/rabbit_fifo_dlx_sup.erl new file mode 100644 index 000000000000..29043eec3f06 --- /dev/null +++ b/deps/rabbit/src/rabbit_fifo_dlx_sup.erl @@ -0,0 +1,37 @@ +-module(rabbit_fifo_dlx_sup). + +-behaviour(supervisor). + +-rabbit_boot_step({?MODULE, + [{description, "supervisor of quorum queue dead-letter workers"}, + {mfa, {rabbit_sup, start_supervisor_child, [?MODULE]}}, + {requires, kernel_ready}, + {enables, core_initialized}]}). + +%% supervisor callback +-export([init/1]). +%% client API +-export([start_link/0]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + FeatureFlag = quorum_queue, + %%TODO rabbit_feature_flags:is_enabled(FeatureFlag) ? + case rabbit_ff_registry:is_enabled(FeatureFlag) of + true -> + SupFlags = #{strategy => simple_one_for_one, + intensity => 1, + period => 5}, + Worker = rabbit_fifo_dlx_worker, + ChildSpec = #{id => Worker, + start => {Worker, start_link, []}, + type => worker, + modules => [Worker]}, + {ok, {SupFlags, [ChildSpec]}}; + false -> + rabbit_log:info("not starting supervisor ~s because feature flag ~s is disabled", + [?MODULE, FeatureFlag]), + ignore + end. diff --git a/deps/rabbit/src/rabbit_fifo_dlx_worker.erl b/deps/rabbit/src/rabbit_fifo_dlx_worker.erl new file mode 100644 index 000000000000..af05402c27cb --- /dev/null +++ b/deps/rabbit/src/rabbit_fifo_dlx_worker.erl @@ -0,0 +1,566 @@ +%% This module consumes from a single quroum queue's discards queue (containing dead-letttered messages) +%% and forwards the DLX messages at least once to every target queue. +%% +%% Some parts of this module resemble the channel process in the sense that it needs to keep track what messages +%% are consumed but not acked yet and what messages are published but not confirmed yet. +%% Compared to the channel process, this module is protocol independent since it doesn't deal with AMQP clients. +%% +%% This module consumes directly from the rabbit_fifo_dlx_client bypassing the rabbit_queue_type interface, +%% but publishes via the rabbit_queue_type interface. +%% While consuming via rabbit_queue_type interface would have worked in practice (by using a special consumer argument, +%% e.g. {<<"x-internal-queue">>, longstr, <<"discards">>} ) using the rabbit_fifo_dlx_client directly provides +%% separation of concerns making things much easier to test, to debug, and to understand. + +-module(rabbit_fifo_dlx_worker). + +-include_lib("rabbit_common/include/rabbit.hrl"). +-include_lib("rabbit_common/include/rabbit_framing.hrl"). + +-behaviour(gen_server2). + +-export([start_link/2]). +%% gen_server2 callbacks +-export([init/1, terminate/2, handle_continue/2, + handle_cast/2, handle_call/3, handle_info/2, + code_change/3, format_status/2]). + +%%TODO make configurable or leave at 0 which means 2000 as in +%% https://github.com/rabbitmq/rabbitmq-server/blob/1e7df8c436174735b1d167673afd3f1642da5cdc/deps/rabbit/src/rabbit_quorum_queue.erl#L726-L729 +-define(CONSUMER_PREFETCH_COUNT, 10). +-define(HIBERNATE_AFTER, 180_000). +%% If no publisher confirm was received for at least SETTLE_TIMEOUT, message will be redelivered. +%% To prevent duplicates in the target queue and to ensure message will eventually be acked to the source queue, +%% set this value higher than the maximum time it takes for a queue to settle a message. +-define(SETTLE_TIMEOUT, 120_000). + +-record(pending, { + %% consumed_msg_id is not to be confused with consumer delivery tag. + %% The latter represents a means for AMQP clients to (multi-)ack to a channel process. + %% However, queues are not aware of delivery tags. + %% This rabbit_fifo_dlx_worker does not have the concept of delivery tags because it settles (acks) + %% message IDs directly back to the queue (and there is no AMQP consumer). + consumed_msg_id :: non_neg_integer(), + content :: rabbit_types:decoded_content(), + %% TODO Reason is already stored in first x-death header of #content.properties.#'P_basic'.headers + %% So, we could remove this convenience field and lookup the 1st header when redelivering. + reason :: rabbit_fifo_dlx:reason(), + %% target queues for which publisher confirm has not been received yet + unsettled = [] :: [rabbit_amqqueue:name()], + %% target queues for which publisher confirm was received + settled = [] :: [rabbit_amqqueue:name()], + %% Number of times the message was published (i.e. rabbit_queue_type:deliver/3 invoked). + %% Can be 0 if the message was never published (for example no route exists). + publish_count = 0 :: non_neg_integer(), + %% Epoch time in milliseconds when the message was last published (i.e. rabbit_queue_type:deliver/3 invoked). + %% It can be 'undefined' if the message was never published (for example no route exists). + last_published_at :: undefined | integer(), + %% Epoch time in milliseconds when the message was consumed from the source quorum queue. + %% This value never changes. + %% It's mainly informational and meant for debugging to understand for how long the message + %% is sitting around without having received all publisher confirms. + consumed_at :: integer() + }). + +-record(state, { + registered_name :: atom(), + %% There is one rabbit_fifo_dlx_worker per source quorum queue + %% (if dead-letter-strategy at-least-once is used). + queue_ref :: rabbit_amqqueue:name(), + %% configured (x-)dead-letter-exchange of source queue + exchange_ref, + %% configured (x-)dead-letter-routing-key of source queue + routing_key, + dlx_client_state :: rabbit_fifo_dlx_client:state(), + queue_type_state :: rabbit_queue_type:state(), + %% Consumed messages for which we have not received all publisher confirms yet. + %% Therefore, they have not been ACKed yet to the consumer queue. + %% This buffer contains at most CONSUMER_PREFETCH_COUNT pending messages at any given point in time. + pendings = #{} :: #{OutSeq :: non_neg_integer() => #pending{}}, + %% next publisher confirm delivery tag sequence number + next_out_seq = 1, + %% Timer firing every SETTLE_TIMEOUT milliseconds + %% redelivering messages for which not all publisher confirms were received. + %% If there are no pending messages, this timer will eventually be cancelled to allow + %% this worker to hibernate. + timer :: undefined | reference() + }). + +% -type state() :: #state{}. + +%%TODO add metrics like global counters for messages routed, delivered, etc. + +start_link(QRef, RegName) -> + gen_server:start_link({local, RegName}, + ?MODULE, {QRef, RegName}, + [{hibernate_after, ?HIBERNATE_AFTER}]). + +-spec init({rabbit_amqqueue:name(), atom()}) -> {ok, undefined, {continue, {rabbit_amqqueue:name(), atom()}}}. +init(Arg) -> + {ok, undefined, {continue, Arg}}. + +handle_continue({QRef, RegName}, undefined) -> + State = lookup_topology(#state{queue_ref = QRef}), + {ok, Q} = rabbit_amqqueue:lookup(QRef), + {ClusterName, _MaybeOldLeaderNode} = amqqueue:get_pid(Q), + {ok, ConsumerState} = rabbit_fifo_dlx_client:checkout(RegName, + QRef, + {ClusterName, node()}, + ?CONSUMER_PREFETCH_COUNT), + {noreply, State#state{registered_name = RegName, + dlx_client_state = ConsumerState, + queue_type_state = rabbit_queue_type:init()}}. + +terminate(_Reason, _State) -> + %%TODO cancel subscription? + %%TODO cancel timer? + ok. + +handle_call(Request, From, State) -> + rabbit_log:warning("~s received unhandled call from ~p: ~p", [?MODULE, From, Request]), + {noreply, State}. + +handle_cast({queue_event, QRef, {_From, {machine, lookup_topology}}}, + #state{queue_ref = QRef} = State0) -> + State = lookup_topology(State0), + redeliver_and_ack(State); +handle_cast({queue_event, QRef, {From, Evt}}, + #state{queue_ref = QRef, + dlx_client_state = DlxState0} = State0) -> + %% received dead-letter messsage from source queue + % rabbit_log:debug("~s received queue event: ~p", [rabbit_misc:rs(QRef), E]), + {ok, DlxState, Actions} = rabbit_fifo_dlx_client:handle_ra_event(From, Evt, DlxState0), + State1 = State0#state{dlx_client_state = DlxState}, + State = handle_queue_actions(Actions, State1), + {noreply, State}; +handle_cast({queue_event, QRef, Evt}, + #state{queue_type_state = QTypeState0} = State0) -> + %% received e.g. confirm from target queue + case rabbit_queue_type:handle_event(QRef, Evt, QTypeState0) of + {ok, QTypeState1, Actions} -> + State1 = State0#state{queue_type_state = QTypeState1}, + State = handle_queue_actions(Actions, State1), + {noreply, State}; + %% TODO handle as done in + %% https://github.com/rabbitmq/rabbitmq-server/blob/9cf18e83f279408e20430b55428a2b19156c90d7/deps/rabbit/src/rabbit_channel.erl#L771-L783 + eol -> + {noreply, State0}; + {protocol_error, _Type, _Reason, _ReasonArgs} -> + {noreply, State0} + end; +handle_cast(settle_timeout, State0) -> + State = State0#state{timer = undefined}, + redeliver_and_ack(State); +handle_cast(Request, State) -> + rabbit_log:warning("~s received unhandled cast ~p", [?MODULE, Request]), + {noreply, State}. + +redeliver_and_ack(State0) -> + State1 = redeliver_messsages(State0), + %% Routes could have been changed dynamically. + %% If a publisher confirm timed out for a target queue to which we now don't route anymore, ack the message. + State2 = maybe_ack(State1), + State = maybe_set_timer(State2), + {noreply, State}. + +handle_info({'DOWN', _MRef, process, QPid, Reason}, + #state{queue_type_state = QTypeState0} = State0) -> + %% received from target classic queue + State = case rabbit_queue_type:handle_down(QPid, Reason, QTypeState0) of + {ok, QTypeState, Actions} -> + State1 = State0#state{queue_type_state = QTypeState}, + handle_queue_actions(Actions, State1); + {eol, QTypeState1, QRef} -> + QTypeState = rabbit_queue_type:remove(QRef, QTypeState1), + State0#state{queue_type_state = QTypeState} + end, + {noreply, State}; +handle_info(Info, State) -> + rabbit_log:warning("~s received unhandled info ~p", [?MODULE, Info]), + {noreply, State}. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +lookup_topology(#state{queue_ref = {resource, Vhost, queue, _} = QRef} = State) -> + {ok, Q} = rabbit_amqqueue:lookup(QRef), + DLRKey = rabbit_queue_type_util:args_policy_lookup(<<"dead-letter-routing-key">>, fun(_Pol, QArg) -> QArg end, Q), + DLX = rabbit_queue_type_util:args_policy_lookup(<<"dead-letter-exchange">>, fun(_Pol, QArg) -> QArg end, Q), + DLXRef = rabbit_misc:r(Vhost, exchange, DLX), + State#state{exchange_ref = DLXRef, + routing_key = DLRKey}. + +%% https://github.com/rabbitmq/rabbitmq-server/blob/9cf18e83f279408e20430b55428a2b19156c90d7/deps/rabbit/src/rabbit_channel.erl#L2855-L2888 +handle_queue_actions(Actions, State0) -> + lists:foldl( + fun ({deliver, Msgs}, S0) -> + S1 = handle_deliver(Msgs, S0), + maybe_set_timer(S1); + ({settled, QRef, MsgSeqs}, S0) -> + S1 = handle_settled(QRef, MsgSeqs, S0), + S2 = maybe_ack(S1), + maybe_cancel_timer(S2); + ({rejected, QRef, MsgSeqNos}, S0) -> + rabbit_log:debug("Ignoring rejected messages ~p from ~s", [MsgSeqNos, rabbit_misc:rs(QRef)]), + S0; + ({queue_down, QRef}, S0) -> + %% target classic queue is down, but not deleted + rabbit_log:debug("Ignoring DOWN from ~s", [rabbit_misc:rs(QRef)]), + S0 + end, State0, Actions). + +handle_deliver(Msgs, #state{queue_ref = QRef} = State) when is_list(Msgs) -> + DLX = lookup_dlx(State), + lists:foldl(fun({_QRef, MsgId, Msg, Reason}, S) -> + forward(Msg, MsgId, QRef, DLX, Reason, S) + end, State, Msgs). + +lookup_dlx(#state{exchange_ref = DLXRef, + queue_ref = QRef}) -> + case rabbit_exchange:lookup(DLXRef) of + {error, not_found} -> + rabbit_log:warning("Cannot forward any dead-letter messages from source quorum ~s because its configured " + "dead-letter-exchange ~s does not exist. " + "Either create the configured dead-letter-exchange or re-configure " + "the dead-letter-exchange policy for the source quorum queue to prevent " + "dead-lettered messages from piling up in the source quorum queue.", + [rabbit_misc:rs(QRef), rabbit_misc:rs(DLXRef)]), + not_found; + {ok, X} -> + X + end. + +forward(ConsumedMsg, ConsumedMsgId, ConsumedQRef, DLX, Reason, + #state{next_out_seq = OutSeq, + pendings = Pendings, + exchange_ref = DLXRef, + routing_key = RKey} = State0) -> + #basic_message{content = Content, routing_keys = RKeys} = Msg = + rabbit_dead_letter:make_msg(ConsumedMsg, Reason, DLXRef, RKey, ConsumedQRef), + %% Field 'mandatory' is set to false because our module checks on its own whether the message is routable. + Delivery = rabbit_basic:delivery(_Mandatory = false, _Confirm = true, Msg, OutSeq), + TargetQs = case DLX of + not_found -> + []; + _ -> + RouteToQs = rabbit_exchange:route(DLX, Delivery), + case rabbit_dead_letter:detect_cycles(Reason, Msg, RouteToQs) of + {[], []} -> + rabbit_log:warning("Cannot deliver message with sequence number ~b " + "(for consumed message sequence number ~b) " + "because no queue is bound to dead-letter ~s with routing keys ~p.", + [OutSeq, ConsumedMsgId, rabbit_misc:rs(DLXRef), RKeys]), + []; + {Qs, []} -> + %% the "normal" case, i.e. no dead-letter-topology misconfiguration + Qs; + {[], Cycles} -> + %%TODO introduce structured logging in rabbit_log by using type logger:report + rabbit_log:warning("Cannot route to any queues. Detected dead-letter queue cycles. " + "Fix the dead-letter routing topology to prevent dead-letter messages from " + "piling up in source quorum queue. " + "outgoing_sequene_number=~b " + "consumed_message_sequence_number=~b " + "consumed_queue=~s " + "dead_letter_exchange=~s " + "effective_dead_letter_routing_keys=~p " + "routed_to_queues=~s " + "dead_letter_queue_cycles=~p", + [OutSeq, ConsumedMsgId, rabbit_misc:rs(ConsumedQRef), + rabbit_misc:rs(DLXRef), RKeys, strings(RouteToQs), Cycles]), + []; + {Qs, Cycles} -> + rabbit_log:warning("Detected dead-letter queue cycles. " + "Fix the dead-letter routing topology. " + "outgoing_sequene_number=~b " + "consumed_message_sequence_number=~b " + "consumed_queue=~s " + "dead_letter_exchange=~s " + "effective_dead_letter_routing_keys=~p " + "routed_to_queues_desired=~s " + "routed_to_queues_effective=~s " + "dead_letter_queue_cycles=~p", + [OutSeq, ConsumedMsgId, rabbit_misc:rs(ConsumedQRef), + rabbit_misc:rs(DLXRef), RKeys, strings(RouteToQs), strings(Qs), Cycles]), + %% Ignore the target queues resulting in cycles. + %% We decide it's good enough to deliver to only routable target queues. + Qs + end + end, + Now = os:system_time(millisecond), + State1 = State0#state{next_out_seq = OutSeq + 1}, + Pend0 = #pending{ + consumed_msg_id = ConsumedMsgId, + consumed_at = Now, + content = Content, + reason = Reason + }, + case TargetQs of + [] -> + %% We can't deliver this message since there is no target queue we can route to. + %% Under no circumstances should we drop a message with dead-letter-strategy at-least-once. + %% We buffer this message and retry to send every SETTLE_TIMEOUT milliseonds + %% (until the user has fixed the dead-letter routing topology). + State1#state{pendings = maps:put(OutSeq, Pend0, Pendings)}; + _ -> + Pend = Pend0#pending{publish_count = 1, + last_published_at = Now, + unsettled = TargetQs}, + State = State1#state{pendings = maps:put(OutSeq, Pend, Pendings)}, + deliver_to_queues(Delivery, TargetQs, State) + end. + +deliver_to_queues(Delivery, RouteToQNames, #state{queue_type_state = QTypeState0} = State0) -> + Qs = rabbit_amqqueue:lookup(RouteToQNames), + {ok, QTypeState1, Actions} = rabbit_queue_type:deliver(Qs, Delivery, QTypeState0), + State = State0#state{queue_type_state = QTypeState1}, + handle_queue_actions(Actions, State). + +handle_settled(QRef, MsgSeqs, #state{pendings = Pendings0} = State) -> + Pendings = lists:foldl(fun (MsgSeq, P0) -> + handle_settled0(QRef, MsgSeq, P0) + end, Pendings0, MsgSeqs), + State#state{pendings = Pendings}. + +handle_settled0(QRef, MsgSeq, Pendings) -> + case maps:find(MsgSeq, Pendings) of + {ok, #pending{unsettled = Unset0, settled = Set0} = Pend0} -> + Unset = lists:delete(QRef, Unset0), + Set = [QRef | Set0], + Pend = Pend0#pending{unsettled = Unset, settled = Set}, + maps:update(MsgSeq, Pend, Pendings); + error -> + rabbit_log:warning("Ignoring publisher confirm for sequence number ~b " + "from target dead letter ~s after settle timeout of ~bms. " + "Troubleshoot why that queue confirms so slowly.", + [MsgSeq, rabbit_misc:rs(QRef), ?SETTLE_TIMEOUT]), + Pendings + end. + +maybe_ack(#state{pendings = Pendings0, + dlx_client_state = DlxState0} = State) -> + Settled = maps:filter(fun(_OutSeq, #pending{unsettled = [], settled = [_|_]}) -> + %% Ack because there is at least one target queue and all + %% target queues settled (i.e. combining publisher confirm + %% and mandatory flag semantics). + true; + (_, _) -> + false + end, Pendings0), + case maps:size(Settled) of + 0 -> + %% nothing to ack + State; + _ -> + Ids = lists:map(fun(#pending{consumed_msg_id = Id}) -> Id end, maps:values(Settled)), + case rabbit_fifo_dlx_client:settle(Ids, DlxState0) of + {ok, DlxState} -> + SettledOutSeqs = maps:keys(Settled), + Pendings = maps:without(SettledOutSeqs, Pendings0), + State#state{pendings = Pendings, + dlx_client_state = DlxState}; + {error, _Reason} -> + %% Failed to ack. Ack will be retried in the next maybe_ack/1 + State + end + end. + +%% Re-deliver messages that timed out waiting on publisher confirm and +%% messages that got never sent due to routing topology misconfiguration. +redeliver_messsages(#state{pendings = Pendings} = State) -> + case lookup_dlx(State) of + not_found -> + %% Configured dead-letter-exchange does (still) not exist. + %% Warning got already logged. + %% Keep the same Pendings in our state until user creates or re-configures the dead-letter-exchange. + State; + DLX -> + Now = os:system_time(millisecond), + maps:fold(fun(OutSeq, #pending{last_published_at = LastPub} = Pend, S0) + when LastPub + ?SETTLE_TIMEOUT =< Now -> + %% Publisher confirm timed out. + redeliver(Pend, DLX, OutSeq, S0); + (OutSeq, #pending{last_published_at = undefined} = Pend, S0) -> + %% Message was never published due to dead-letter routing topology misconfiguration. + redeliver(Pend, DLX, OutSeq, S0); + (_OutSeq, _Pending, S) -> + %% Publisher confirm did not time out. + S + end, State, Pendings) + end. + +redeliver(#pending{content = Content} = Pend, DLX, OldOutSeq, + #state{routing_key = undefined} = State) -> + %% No dead-letter-routing-key defined for source quorum queue. + %% Therefore use all of messages's original routing keys (which can include CC and BCC recipients). + %% This complies with the behaviour of the rabbit_dead_letter module. + %% We stored these original routing keys in the 1st (i.e. most recent) x-death entry. + #content{properties = #'P_basic'{headers = Headers}} = + rabbit_binary_parser:ensure_content_decoded(Content), + {array, [{table, MostRecentDeath}|_]} = rabbit_misc:table_lookup(Headers, <<"x-death">>), + {<<"routing-keys">>, array, Routes0} = lists:keyfind(<<"routing-keys">>, 1, MostRecentDeath), + Routes = [Route || {longstr, Route} <- Routes0], + redeliver0(Pend, DLX, Routes, OldOutSeq, State); +redeliver(Pend, DLX, OldOutSeq, #state{routing_key = DLRKey} = State) -> + redeliver0(Pend, DLX, [DLRKey], OldOutSeq, State). + +%% Quorum queues maintain their own Raft sequene number mapping to the message sequence number (= Raft correlation ID). +%% So, they would just send us a 'settled' queue action containing the correct message sequence number. +%% +%% Classic queues however maintain their state by mapping the message sequence number to pending and confirmed queues. +%% While re-using the same message sequence number could work there as well, it just gets unnecssary complicated when +%% different target queues settle two separate deliveries referring to the same message sequence number (and same basic message). +%% +%% Therefore, to keep things simple, create a brand new delivery, store it in our state and forget about the old delivery and +%% sequence number. +%% +%% If a sequene number gets settled after SETTLE_TIMEOUT, we can't map it anymore to the #pending{}. Hence, we ignore it. +%% +%% This can lead to issues when SETTLE_TIMEOUT is too low and time to settle takes too long. +%% For example, if SETTLE_TIMEOUT is set to only 10 seconds, but settling a message takes always longer than 10 seconds +%% (e.g. due to extremly slow hypervisor disks that ran out of credit), we will re-deliver the same message all over again +%% leading to many duplicates in the target queue without ever acking the message back to the source discards queue. +%% +%% Therefore, set SETTLE_TIMEOUT reasonably high (e.g. 2 minutes). +%% +%% TODO do not log per message? +redeliver0(#pending{consumed_msg_id = ConsumedMsgId, + content = Content, + unsettled = Unsettled, + settled = Settled, + publish_count = PublishCount, + reason = Reason} = Pend0, + DLX, DLRKeys, OldOutSeq, + #state{next_out_seq = OutSeq, + queue_ref = QRef, + pendings = Pendings0, + exchange_ref = DLXRef} = State0) when is_list(DLRKeys) -> + BasicMsg = #basic_message{exchange_name = DLXRef, + routing_keys = DLRKeys, + %% BCC Header was already stripped previously + content = Content, + id = rabbit_guid:gen(), + is_persistent = rabbit_basic:is_message_persistent(Content) + }, + %% Field 'mandatory' is set to false because our module checks on its own whether the message is routable. + Delivery = rabbit_basic:delivery(_Mandatory = false, _Confirm = true, BasicMsg, OutSeq), + RouteToQs0 = rabbit_exchange:route(DLX, Delivery), + %% Do not re-deliver to queues for which we already received a publisher confirm. + RouteToQs1 = RouteToQs0 -- Settled, + {RouteToQs, Cycles} = rabbit_dead_letter:detect_cycles(Reason, BasicMsg, RouteToQs1), + Prefix = io_lib:format("Message has not received required publisher confirm(s). " + "Received confirm from: [~s]. " + "Did not receive confirm from: [~s]. " + "timeout=~bms " + "message_sequence_number=~b " + "consumed_message_sequence_number=~b " + "publish_count=~b.", + [strings(Settled), strings(Unsettled), ?SETTLE_TIMEOUT, + OldOutSeq, ConsumedMsgId, PublishCount]), + case {RouteToQs, Cycles, Settled} of + {[], [], []} -> + rabbit_log:warning("~s Failed to re-deliver this message because no queue is bound " + "to dead-letter ~s with routing keys ~p.", + [Prefix, rabbit_misc:rs(DLXRef), DLRKeys]), + State0; + {[], [], [_|_]} -> + rabbit_log:debug("~s Routes changed dynamically so that this message does not need to be routed " + "to any queue anymore. This message will be acknowledged to the source ~s.", + [Prefix, rabbit_misc:rs(QRef)]), + State0; + {[], [_|_], []} -> + rabbit_log:warning("~s Failed to re-deliver this message because dead-letter queue cycles " + "got detected: ~p", + [Prefix, Cycles]), + State0; + {[], [_|_], [_|_]} -> + rabbit_log:warning("~s Dead-letter queue cycles detected: ~p. " + "This message will nevertheless be acknowledged to the source ~s " + "because it received at least one publisher confirm.", + [Prefix, Cycles, rabbit_misc:rs(QRef)]), + State0; + _ -> + case Cycles of + [] -> + rabbit_log:debug("~s Re-delivering this message to ~s", + [Prefix, strings(RouteToQs)]); + [_|_] -> + rabbit_log:warning("~s Dead-letter queue cycles detected: ~p. " + "Re-delivering this message only to ~s", + [Prefix, Cycles, strings(RouteToQs)]) + end, + Pend = Pend0#pending{publish_count = PublishCount + 1, + last_published_at = os:system_time(millisecond), + %% override 'unsettled' because topology could have changed + unsettled = RouteToQs}, + Pendings1 = maps:remove(OldOutSeq, Pendings0), + Pendings = maps:put(OutSeq, Pend, Pendings1), + State = State0#state{next_out_seq = OutSeq + 1, + pendings = Pendings}, + deliver_to_queues(Delivery, RouteToQs, State) + end. + +strings(QRefs) when is_list(QRefs) -> + L0 = lists:map(fun rabbit_misc:rs/1, QRefs), + L1 = lists:join(", ", L0), + lists:flatten(L1). + +maybe_set_timer(#state{timer = TRef} = State) when is_reference(TRef) -> + State; +maybe_set_timer(#state{timer = undefined, + pendings = Pendings} = State) when map_size(Pendings) =:= 0 -> + State; +maybe_set_timer(#state{timer = undefined} = State) -> + TRef = erlang:send_after(?SETTLE_TIMEOUT, self(), {'$gen_cast', settle_timeout}), + % rabbit_log:debug("set timer"), + State#state{timer = TRef}. + +maybe_cancel_timer(#state{timer = undefined} = State) -> + State; +maybe_cancel_timer(#state{timer = TRef, + pendings = Pendings} = State) -> + case maps:size(Pendings) of + 0 -> + erlang:cancel_timer(TRef, [{async, true}, {info, false}]), + % rabbit_log:debug("cancelled timer"), + State#state{timer = undefined}; + _ -> + State + end. + +%% Avoids large message contents being logged. +format_status(_Opt, [_PDict, #state{ + registered_name = RegisteredName, + queue_ref = QueueRef, + exchange_ref = ExchangeRef, + routing_key = RoutingKey, + dlx_client_state = DlxClientState, + queue_type_state = QueueTypeState, + pendings = Pendings, + next_out_seq = NextOutSeq, + timer = Timer + }]) -> + S = #{registered_name => RegisteredName, + queue_ref => QueueRef, + exchange_ref => ExchangeRef, + routing_key => RoutingKey, + dlx_client_state => rabbit_fifo_dlx_client:overview(DlxClientState), + queue_type_state => QueueTypeState, + pendings => maps:map(fun(_, P) -> format_pending(P) end, Pendings), + next_out_seq => NextOutSeq, + timer_is_active => Timer =/= undefined}, + [{data, [{"State", S}]}]. + +format_pending(#pending{consumed_msg_id = ConsumedMsgId, + reason = Reason, + unsettled = Unsettled, + settled = Settled, + publish_count = PublishCount, + last_published_at = LastPublishedAt, + consumed_at = ConsumedAt}) -> + #{consumed_msg_id => ConsumedMsgId, + reason => Reason, + unsettled => Unsettled, + settled => Settled, + publish_count => PublishCount, + last_published_at => LastPublishedAt, + consumed_at => ConsumedAt}. diff --git a/deps/rabbit/src/rabbit_fifo_v1.erl b/deps/rabbit/src/rabbit_fifo_v1.erl index a59a5c9250ae..51150b0f7089 100644 --- a/deps/rabbit/src/rabbit_fifo_v1.erl +++ b/deps/rabbit/src/rabbit_fifo_v1.erl @@ -130,6 +130,8 @@ state/0, config/0]). +%% This function is never called since only rabbit_fifo_v0:init/1 is called. +%% See https://github.com/rabbitmq/ra/blob/e0d1e6315a45f5d3c19875d66f9d7bfaf83a46e3/src/ra_machine.erl#L258-L265 -spec init(config()) -> state(). init(#{name := Name, queue_resource := Resource} = Conf) -> diff --git a/deps/rabbit/src/rabbit_policies.erl b/deps/rabbit/src/rabbit_policies.erl index 062635c5b431..89d2a3eadea0 100644 --- a/deps/rabbit/src/rabbit_policies.erl +++ b/deps/rabbit/src/rabbit_policies.erl @@ -30,6 +30,7 @@ register() -> {Class, Name} <- [{policy_validator, <<"alternate-exchange">>}, {policy_validator, <<"dead-letter-exchange">>}, {policy_validator, <<"dead-letter-routing-key">>}, + {policy_validator, <<"dead-letter-strategy">>}, {policy_validator, <<"message-ttl">>}, {policy_validator, <<"expires">>}, {policy_validator, <<"max-length">>}, @@ -84,6 +85,13 @@ validate_policy0(<<"dead-letter-routing-key">>, Value) validate_policy0(<<"dead-letter-routing-key">>, Value) -> {error, "~p is not a valid dead letter routing key", [Value]}; +validate_policy0(<<"dead-letter-strategy">>, <<"at-most-once">>) -> + ok; +validate_policy0(<<"dead-letter-strategy">>, <<"at-least-once">>) -> + ok; +validate_policy0(<<"dead-letter-strategy">>, Value) -> + {error, "~p is not a valid dead letter strategy", [Value]}; + validate_policy0(<<"message-ttl">>, Value) when is_integer(Value), Value >= 0 -> ok; diff --git a/deps/rabbit/src/rabbit_quorum_queue.erl b/deps/rabbit/src/rabbit_quorum_queue.erl index a4c6d5dd5f46..4ffed0052795 100644 --- a/deps/rabbit/src/rabbit_quorum_queue.erl +++ b/deps/rabbit/src/rabbit_quorum_queue.erl @@ -71,6 +71,7 @@ -include_lib("stdlib/include/qlc.hrl"). -include_lib("rabbit_common/include/rabbit.hrl"). +-include_lib("rabbit_common/include/rabbit_framing.hrl"). -include("amqqueue.hrl"). -type msg_id() :: non_neg_integer(). @@ -227,18 +228,17 @@ ra_machine_config(Q) when ?is_amqqueue(Q) -> {Name, _} = amqqueue:get_pid(Q), %% take the minimum value of the policy and the queue arg if present MaxLength = args_policy_lookup(<<"max-length">>, fun min/2, Q), - %% prefer the policy defined strategy if available - Overflow = args_policy_lookup(<<"overflow">>, fun (A, _B) -> A end , Q), + OverflowBin = args_policy_lookup(<<"overflow">>, fun policyHasPrecedence/2, Q), + Overflow = overflow(OverflowBin, drop_head, QName), MaxBytes = args_policy_lookup(<<"max-length-bytes">>, fun min/2, Q), MaxMemoryLength = args_policy_lookup(<<"max-in-memory-length">>, fun min/2, Q), MaxMemoryBytes = args_policy_lookup(<<"max-in-memory-bytes">>, fun min/2, Q), DeliveryLimit = args_policy_lookup(<<"delivery-limit">>, fun min/2, Q), - Expires = args_policy_lookup(<<"expires">>, - fun (A, _B) -> A end, - Q), + Expires = args_policy_lookup(<<"expires">>, fun policyHasPrecedence/2, Q), + MsgTTL = args_policy_lookup(<<"message-ttl">>, fun min/2, Q), #{name => Name, queue_resource => QName, - dead_letter_handler => dlx_mfa(Q), + dead_letter_handler => dead_letter_handler(Q, Overflow), become_leader_handler => {?MODULE, become_leader, [QName]}, max_length => MaxLength, max_bytes => MaxBytes, @@ -246,11 +246,17 @@ ra_machine_config(Q) when ?is_amqqueue(Q) -> max_in_memory_bytes => MaxMemoryBytes, single_active_consumer_on => single_active_consumer_on(Q), delivery_limit => DeliveryLimit, - overflow_strategy => overflow(Overflow, drop_head, QName), + overflow_strategy => Overflow, created => erlang:system_time(millisecond), - expires => Expires + expires => Expires, + msg_ttl => MsgTTL }. +policyHasPrecedence(Policy, _QueueArg) -> + Policy. +queueArgHasPrecedence(_Policy, QueueArg) -> + QueueArg. + single_active_consumer_on(Q) -> QArguments = amqqueue:get_arguments(Q), case rabbit_misc:table_lookup(QArguments, <<"x-single-active-consumer">>) of @@ -293,7 +299,7 @@ become_leader(QName, Name) -> end, %% as this function is called synchronously when a ra node becomes leader %% we need to ensure there is no chance of blocking as else the ra node - %% may not be able to establish it's leadership + %% may not be able to establish its leadership spawn(fun() -> rabbit_misc:execute_mnesia_transaction( fun() -> @@ -377,19 +383,20 @@ filter_quorum_critical(Queues, ReplicaStates) -> capabilities() -> #{unsupported_policies => [ %% Classic policies - <<"message-ttl">>, <<"max-priority">>, <<"queue-mode">>, + <<"max-priority">>, <<"queue-mode">>, <<"single-active-consumer">>, <<"ha-mode">>, <<"ha-params">>, <<"ha-sync-mode">>, <<"ha-promote-on-shutdown">>, <<"ha-promote-on-failure">>, <<"queue-master-locator">>, %% Stream policies <<"max-age">>, <<"stream-max-segment-size-bytes">>, <<"queue-leader-locator">>, <<"initial-cluster-size">>], - queue_arguments => [<<"x-expires">>, <<"x-dead-letter-exchange">>, - <<"x-dead-letter-routing-key">>, <<"x-max-length">>, - <<"x-max-length-bytes">>, <<"x-max-in-memory-length">>, - <<"x-max-in-memory-bytes">>, <<"x-overflow">>, - <<"x-single-active-consumer">>, <<"x-queue-type">>, - <<"x-quorum-initial-group-size">>, <<"x-delivery-limit">>], + queue_arguments => [<<"x-dead-letter-exchange">>, <<"x-dead-letter-routing-key">>, + <<"x-dead-letter-strategy">>, <<"x-expires">>, <<"x-max-length">>, + <<"x-max-length-bytes">>, <<"x-max-in-memory-length">>, + <<"x-max-in-memory-bytes">>, <<"x-overflow">>, + <<"x-single-active-consumer">>, <<"x-queue-type">>, + <<"x-quorum-initial-group-size">>, <<"x-delivery-limit">>, + <<"x-message-ttl">>], consumer_arguments => [<<"x-priority">>, <<"x-credit">>], server_named => false}. @@ -839,8 +846,11 @@ deliver(true, Delivery, QState0) -> rabbit_fifo_client:enqueue(Delivery#delivery.msg_seq_no, Delivery#delivery.message, QState0). -deliver(QSs, #delivery{confirm = Confirm} = Delivery0) -> - Delivery = clean_delivery(Delivery0), +deliver(QSs, #delivery{message = #basic_message{content = Content0} = Msg, + confirm = Confirm} = Delivery0) -> + %% TODO: we could also consider clearing out the message id here + Content = prepare_content(Content0), + Delivery = Delivery0#delivery{message = Msg#basic_message{content = Content}}, lists:foldl( fun({Q, stateless}, {Qs, Actions}) -> QRef = amqqueue:get_pid(Q), @@ -1253,20 +1263,45 @@ reclaim_memory(Vhost, QueueName) -> ra_log_wal:force_roll_over({?RA_WAL_NAME, Node}). %%---------------------------------------------------------------------------- -dlx_mfa(Q) -> - DLX = init_dlx(args_policy_lookup(<<"dead-letter-exchange">>, - fun res_arg/2, Q), Q), - DLXRKey = args_policy_lookup(<<"dead-letter-routing-key">>, - fun res_arg/2, Q), - {?MODULE, dead_letter_publish, [DLX, DLXRKey, amqqueue:get_name(Q)]}. - -init_dlx(undefined, _Q) -> - undefined; -init_dlx(DLX, Q) when ?is_amqqueue(Q) -> +dead_letter_handler(Q, Overflow) -> + %% Queue arg continues to take precedence to not break existing configurations + %% for queues upgraded from =v3.10 + Exchange = args_policy_lookup(<<"dead-letter-exchange">>, fun queueArgHasPrecedence/2, Q), + RoutingKey = args_policy_lookup(<<"dead-letter-routing-key">>, fun queueArgHasPrecedence/2, Q), + %% Policy takes precedence because it's a new key introduced in v3.10 and we want + %% users to use policies instead of queue args allowing dynamic reconfiguration. + Strategy = args_policy_lookup(<<"dead-letter-strategy">>, fun policyHasPrecedence/2, Q), QName = amqqueue:get_name(Q), - rabbit_misc:r(QName, exchange, DLX). + dlh(Exchange, RoutingKey, Strategy, Overflow, QName). -res_arg(_PolVal, ArgVal) -> ArgVal. +dlh(undefined, undefined, undefined, _, _) -> + undefined; +dlh(undefined, RoutingKey, undefined, _, QName) -> + rabbit_log:warning("Disabling dead-lettering for ~s despite configured dead-letter-routing-key '~s' " + "because dead-letter-exchange is not configured.", + [rabbit_misc:rs(QName), RoutingKey]), + undefined; +dlh(undefined, _, Strategy, _, QName) -> + rabbit_log:warning("Disabling dead-lettering for ~s despite configured dead-letter-strategy '~s' " + "because dead-letter-exchange is not configured.", + [rabbit_misc:rs(QName), Strategy]), + undefined; +dlh(_, _, <<"at-least-once">>, reject_publish, _) -> + at_least_once; +dlh(Exchange, RoutingKey, <<"at-least-once">>, drop_head, QName) -> + rabbit_log:warning("Falling back to dead-letter-strategy at-most-once for ~s " + "because configured dead-letter-strategy at-least-once is incompatible with " + "effective overflow strategy drop-head. To enable dead-letter-strategy " + "at-least-once, set overflow strategy to reject-publish.", + [rabbit_misc:rs(QName)]), + dlh_at_most_once(Exchange, RoutingKey, QName); +dlh(Exchange, RoutingKey, _, _, QName) -> + dlh_at_most_once(Exchange, RoutingKey, QName). + +dlh_at_most_once(Exchange, RoutingKey, QName) -> + DLX = rabbit_misc:r(QName, exchange, Exchange), + MFA = {?MODULE, dead_letter_publish, [DLX, RoutingKey, QName]}, + {at_most_once, MFA}. dead_letter_publish(undefined, _, _, _) -> ok; @@ -1582,7 +1617,7 @@ overflow(undefined, Def, _QName) -> Def; overflow(<<"reject-publish">>, _Def, _QName) -> reject_publish; overflow(<<"drop-head">>, _Def, _QName) -> drop_head; overflow(<<"reject-publish-dlx">> = V, Def, QName) -> - rabbit_log:warning("Invalid overflow strategy ~p for quorum queue: ~p", + rabbit_log:warning("Invalid overflow strategy ~p for quorum queue: ~s", [V, rabbit_misc:rs(QName)]), Def. @@ -1626,19 +1661,15 @@ notify_decorators(QName, F, A) -> end. %% remove any data that a quorum queue doesn't need -clean_delivery(#delivery{message = - #basic_message{content = Content0} = Msg} = Delivery) -> - Content = case Content0 of - #content{properties = none} -> - Content0; - #content{protocol = none} -> - Content0; - #content{properties = Props, - protocol = Proto} -> - Content0#content{properties = none, - properties_bin = Proto:encode_properties(Props)} - end, - - %% TODO: we could also consider clearing out the message id here - Delivery#delivery{message = Msg#basic_message{content = Content}}. - +prepare_content(#content{properties = none} = Content) -> + Content; +prepare_content(#content{protocol = none} = Content) -> + Content; +prepare_content(#content{properties = #'P_basic'{expiration = undefined} = Props, + protocol = Proto} = Content) -> + Content#content{properties = none, + properties_bin = Proto:encode_properties(Props)}; +prepare_content(Content) -> + %% expiration is set. Therefore, leave properties decoded so that + %% rabbit_fifo can directly parse it without having to decode again. + Content. diff --git a/deps/rabbit/src/rabbit_stream_queue.erl b/deps/rabbit/src/rabbit_stream_queue.erl index dee8740a2907..33d32ece4c5f 100644 --- a/deps/rabbit/src/rabbit_stream_queue.erl +++ b/deps/rabbit/src/rabbit_stream_queue.erl @@ -955,7 +955,9 @@ capabilities() -> <<"single-active-consumer">>, <<"delivery-limit">>, <<"ha-mode">>, <<"ha-params">>, <<"ha-sync-mode">>, <<"ha-promote-on-shutdown">>, <<"ha-promote-on-failure">>, - <<"queue-master-locator">>], + <<"queue-master-locator">>, + %% Quorum policies + <<"dead-letter-strategy">>], queue_arguments => [<<"x-dead-letter-exchange">>, <<"x-dead-letter-routing-key">>, <<"x-max-length">>, <<"x-max-length-bytes">>, <<"x-single-active-consumer">>, <<"x-queue-type">>, diff --git a/deps/rabbit/test/dead_lettering_SUITE.erl b/deps/rabbit/test/dead_lettering_SUITE.erl index 132ef92fae48..007cc9f1e350 100644 --- a/deps/rabbit/test/dead_lettering_SUITE.erl +++ b/deps/rabbit/test/dead_lettering_SUITE.erl @@ -91,7 +91,9 @@ init_per_group(quorum_queue, Config) -> ok -> rabbit_ct_helpers:set_config( Config, - [{queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}]}, + [{queue_args, [{<<"x-queue-type">>, longstr, <<"quorum">>}, + %%TODO add at-least-once tests + {<<"x-dead-letter-strategy">>, longstr, <<"at-most-once">>}]}, {queue_durable, true}]); Skip -> Skip @@ -703,7 +705,9 @@ dead_letter_policy(Config) -> {_Conn, Ch} = rabbit_ct_client_helpers:open_connection_and_channel(Config, 0), QName = ?config(queue_name, Config), DLXQName = ?config(queue_name_dlx, Config), - Args = ?config(queue_args, Config), + Args0 = ?config(queue_args, Config), + %% declaring a quorum queue with x-dead-letter-strategy without defining a DLX will fail + Args = proplists:delete(<<"x-dead-letter-strategy">>, Args0), Durable = ?config(queue_durable, Config), DLXExchange = ?config(dlx_exchange, Config), diff --git a/deps/rabbit/test/rabbit_fifo_prop_SUITE.erl b/deps/rabbit/test/rabbit_fifo_prop_SUITE.erl index a22b0a286eb4..d4a061c1d591 100644 --- a/deps/rabbit/test/rabbit_fifo_prop_SUITE.erl +++ b/deps/rabbit/test/rabbit_fifo_prop_SUITE.erl @@ -10,6 +10,11 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("rabbit/src/rabbit_fifo.hrl"). +-include_lib("rabbit/src/rabbit_fifo_dlx.hrl"). +-include_lib("rabbit_common/include/rabbit.hrl"). +-include_lib("rabbit_common/include/rabbit_framing.hrl"). + +-define(record_info(T,R),lists:zip(record_info(fields,T),tl(tuple_to_list(R)))). %%%=================================================================== %%% Common Test callbacks @@ -25,7 +30,6 @@ all_tests() -> [ test_run_log, snapshots, - scenario1, scenario2, scenario3, scenario4, @@ -69,7 +73,16 @@ all_tests() -> single_active_ordering_01, single_active_ordering_03, in_memory_limit, - max_length + max_length, + snapshots_dlx, + dlx_01, + dlx_02, + dlx_03, + dlx_04, + dlx_05, + dlx_06, + dlx_07, + dlx_08 % single_active_ordering_02 ]. @@ -103,32 +116,14 @@ end_per_testcase(_TestCase, _Config) -> % -type log_op() :: % {enqueue, pid(), maybe(msg_seqno()), Msg :: raw_msg()}. -scenario1(_Config) -> - C1 = {<<>>, c:pid(0,6723,1)}, - C2 = {<<0>>,c:pid(0,6723,1)}, - E = c:pid(0,6720,1), - - Commands = [ - make_checkout(C1, {auto,2,simple_prefetch}), - make_enqueue(E,1,msg1), - make_enqueue(E,2,msg2), - make_checkout(C1, cancel), %% both on returns queue - make_checkout(C2, {auto,1,simple_prefetch}), - make_return(C2, [0]), %% E1 in returns, E2 with C2 - make_return(C2, [1]), %% E2 in returns E1 with C2 - make_settle(C2, [2]) %% E2 with C2 - ], - run_snapshot_test(#{name => ?FUNCTION_NAME}, Commands), - ok. - scenario2(_Config) -> C1 = {<<>>, c:pid(0,346,1)}, C2 = {<<>>,c:pid(0,379,1)}, E = c:pid(0,327,1), Commands = [make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E,1,msg1), + make_enqueue(E,1,msg(<<"msg1">>)), make_checkout(C1, cancel), - make_enqueue(E,2,msg2), + make_enqueue(E,2,msg(<<"msg2">>)), make_checkout(C2, {auto,1,simple_prefetch}), make_settle(C1, [0]), make_settle(C2, [0]) @@ -140,10 +135,10 @@ scenario3(_Config) -> C1 = {<<>>, c:pid(0,179,1)}, E = c:pid(0,176,1), Commands = [make_checkout(C1, {auto,2,simple_prefetch}), - make_enqueue(E,1,msg1), + make_enqueue(E,1,msg(<<"msg1">>)), make_return(C1, [0]), - make_enqueue(E,2,msg2), - make_enqueue(E,3,msg3), + make_enqueue(E,2,msg(<<"msg2">>)), + make_enqueue(E,3,msg(<<"msg3">>)), make_settle(C1, [1]), make_settle(C1, [2]) ], @@ -154,7 +149,7 @@ scenario4(_Config) -> C1 = {<<>>, c:pid(0,179,1)}, E = c:pid(0,176,1), Commands = [make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E,1,msg), + make_enqueue(E,1,msg(<<"msg">>)), make_settle(C1, [0]) ], run_snapshot_test(#{name => ?FUNCTION_NAME}, Commands), @@ -163,17 +158,17 @@ scenario4(_Config) -> scenario5(_Config) -> C1 = {<<>>, c:pid(0,505,0)}, E = c:pid(0,465,9), - Commands = [make_enqueue(E,1,<<0>>), + Commands = [make_enqueue(E,1,msg(<<0>>)), make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E,2,<<>>), + make_enqueue(E,2,msg(<<>>)), make_settle(C1,[0])], run_snapshot_test(#{name => ?FUNCTION_NAME}, Commands), ok. scenario6(_Config) -> E = c:pid(0,465,9), - Commands = [make_enqueue(E,1,<<>>), %% 1 msg on queue - snap: prefix 1 - make_enqueue(E,2,<<>>) %% 1. msg on queue - snap: prefix 1 + Commands = [make_enqueue(E,1,msg(<<>>)), %% 1 msg on queue - snap: prefix 1 + make_enqueue(E,2,msg(<<>>)) %% 1. msg on queue - snap: prefix 1 ], run_snapshot_test(#{name => ?FUNCTION_NAME, max_length => 1}, Commands), @@ -183,10 +178,10 @@ scenario7(_Config) -> C1 = {<<>>, c:pid(0,208,0)}, E = c:pid(0,188,0), Commands = [ - make_enqueue(E,1,<<>>), + make_enqueue(E,1,msg(<<>>)), make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E,2,<<>>), - make_enqueue(E,3,<<>>), + make_enqueue(E,2,msg(<<>>)), + make_enqueue(E,3,msg(<<>>)), make_settle(C1,[0])], run_snapshot_test(#{name => ?FUNCTION_NAME, max_length => 1}, Commands), @@ -196,8 +191,8 @@ scenario8(_Config) -> C1 = {<<>>, c:pid(0,208,0)}, E = c:pid(0,188,0), Commands = [ - make_enqueue(E,1,<<>>), - make_enqueue(E,2,<<>>), + make_enqueue(E,1,msg(<<>>)), + make_enqueue(E,2,msg(<<>>)), make_checkout(C1, {auto,1,simple_prefetch}), % make_checkout(C1, cancel), {down, E, noconnection}, @@ -209,9 +204,9 @@ scenario8(_Config) -> scenario9(_Config) -> E = c:pid(0,188,0), Commands = [ - make_enqueue(E,1,<<>>), - make_enqueue(E,2,<<>>), - make_enqueue(E,3,<<>>)], + make_enqueue(E,1,msg(<<>>)), + make_enqueue(E,2,msg(<<>>)), + make_enqueue(E,3,msg(<<>>))], run_snapshot_test(#{name => ?FUNCTION_NAME, max_length => 1}, Commands), ok. @@ -221,7 +216,7 @@ scenario10(_Config) -> E = c:pid(0,188,0), Commands = [ make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E,1,<<>>), + make_enqueue(E,1,msg(<<>>)), make_settle(C1, [0]) ], run_snapshot_test(#{name => ?FUNCTION_NAME, @@ -232,10 +227,10 @@ scenario11(_Config) -> C1 = {<<>>, c:pid(0,215,0)}, E = c:pid(0,217,0), Commands = [ - make_enqueue(E,1,<<"1">>), % 1 + make_enqueue(E,1,msg(<<"1">>)), % 1 make_checkout(C1, {auto,1,simple_prefetch}), % 2 make_checkout(C1, cancel), % 3 - make_enqueue(E,2,<<"22">>), % 4 + make_enqueue(E,2,msg(<<"22">>)), % 4 make_checkout(C1, {auto,1,simple_prefetch}), % 5 make_settle(C1, [0]), % 6 make_checkout(C1, cancel) % 7 @@ -246,19 +241,19 @@ scenario11(_Config) -> scenario12(_Config) -> E = c:pid(0,217,0), - Commands = [make_enqueue(E,1,<<0>>), - make_enqueue(E,2,<<0>>), - make_enqueue(E,3,<<0>>)], + Commands = [make_enqueue(E,1,msg(<<0>>)), + make_enqueue(E,2,msg(<<0>>)), + make_enqueue(E,3,msg(<<0>>))], run_snapshot_test(#{name => ?FUNCTION_NAME, max_bytes => 2}, Commands), ok. scenario13(_Config) -> E = c:pid(0,217,0), - Commands = [make_enqueue(E,1,<<0>>), - make_enqueue(E,2,<<>>), - make_enqueue(E,3,<<>>), - make_enqueue(E,4,<<>>) + Commands = [make_enqueue(E,1,msg(<<0>>)), + make_enqueue(E,2,msg(<<>>)), + make_enqueue(E,3,msg(<<>>)), + make_enqueue(E,4,msg(<<>>)) ], run_snapshot_test(#{name => ?FUNCTION_NAME, max_length => 2}, Commands), @@ -266,7 +261,7 @@ scenario13(_Config) -> scenario14(_Config) -> E = c:pid(0,217,0), - Commands = [make_enqueue(E,1,<<0,0>>)], + Commands = [make_enqueue(E,1,msg(<<0,0>>))], run_snapshot_test(#{name => ?FUNCTION_NAME, max_bytes => 1}, Commands), ok. @@ -274,8 +269,8 @@ scenario14(_Config) -> scenario14b(_Config) -> E = c:pid(0,217,0), Commands = [ - make_enqueue(E,1,<<0>>), - make_enqueue(E,2,<<0>>) + make_enqueue(E,1,msg(<<0>>)), + make_enqueue(E,2,msg(<<0>>)) ], run_snapshot_test(#{name => ?FUNCTION_NAME, max_bytes => 1}, Commands), @@ -285,8 +280,8 @@ scenario15(_Config) -> C1 = {<<>>, c:pid(0,179,1)}, E = c:pid(0,176,1), Commands = [make_checkout(C1, {auto,2,simple_prefetch}), - make_enqueue(E, 1, msg1), - make_enqueue(E, 2, msg2), + make_enqueue(E, 1, msg(<<"msg1">>)), + make_enqueue(E, 2, msg(<<"msg2">>)), make_return(C1, [0]), make_return(C1, [2]), make_settle(C1, [1]) @@ -302,11 +297,11 @@ scenario16(_Config) -> E = c:pid(0,176,1), Commands = [ make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E, 1, msg1), + make_enqueue(E, 1, msg(<<"msg1">>)), make_checkout(C2, {auto,1,simple_prefetch}), {down, C1Pid, noproc}, %% msg1 allocated to C2 make_return(C2, [0]), %% msg1 returned - make_enqueue(E, 2, <<>>), + make_enqueue(E, 2, msg(<<>>)), make_settle(C2, [0]) ], run_snapshot_test(#{name => ?FUNCTION_NAME, @@ -321,11 +316,11 @@ scenario17(_Config) -> E = test_util:fake_pid(rabbit@fake_node2), Commands = [ make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E,1,<<"one">>), + make_enqueue(E,1,msg(<<"one">>)), make_checkout(C2, {auto,1,simple_prefetch}), {down, C1Pid, noconnection}, make_checkout(C2, cancel), - make_enqueue(E,2,<<"two">>), + make_enqueue(E,2,msg(<<"two">>)), {nodeup,rabbit@fake_node1}, %% this has no effect as was returned make_settle(C1, [0]), @@ -339,11 +334,11 @@ scenario17(_Config) -> scenario18(_Config) -> E = c:pid(0,176,1), - Commands = [make_enqueue(E,1,<<"1">>), - make_enqueue(E,2,<<"2">>), - make_enqueue(E,3,<<"3">>), - make_enqueue(E,4,<<"4">>), - make_enqueue(E,5,<<"5">>) + Commands = [make_enqueue(E,1,msg(<<"1">>)), + make_enqueue(E,2,msg(<<"2">>)), + make_enqueue(E,3,msg(<<"3">>)), + make_enqueue(E,4,msg(<<"4">>)), + make_enqueue(E,5,msg(<<"5">>)) ], run_snapshot_test(#{name => ?FUNCTION_NAME, %% max_length => 3, @@ -354,10 +349,10 @@ scenario19(_Config) -> C1Pid = c:pid(0,883,1), C1 = {<<>>, C1Pid}, E = c:pid(0,176,1), - Commands = [make_enqueue(E,1,<<"1">>), - make_enqueue(E,2,<<"2">>), + Commands = [make_enqueue(E,1,msg(<<"1">>)), + make_enqueue(E,2,msg(<<"2">>)), make_checkout(C1, {auto,2,simple_prefetch}), - make_enqueue(E,3,<<"3">>), + make_enqueue(E,3,msg(<<"3">>)), make_settle(C1, [0, 1]) ], run_snapshot_test(#{name => ?FUNCTION_NAME, @@ -369,15 +364,15 @@ scenario20(_Config) -> C1Pid = c:pid(0,883,1), C1 = {<<>>, C1Pid}, E = c:pid(0,176,1), - Commands = [make_enqueue(E,1,<<>>), - make_enqueue(E,2,<<1>>), + Commands = [make_enqueue(E,1,msg(<<>>)), + make_enqueue(E,2,msg(<<1>>)), make_checkout(C1, {auto,2,simple_prefetch}), {down, C1Pid, noconnection}, - make_enqueue(E,3,<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>), - make_enqueue(E,4,<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>), - make_enqueue(E,5,<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>), - make_enqueue(E,6,<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>), - make_enqueue(E,7,<<0,0,0,0,0,0,0,0,0,0,0,0,0,0>>) + make_enqueue(E,3,msg(<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>)), + make_enqueue(E,4,msg(<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>)), + make_enqueue(E,5,msg(<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>)), + make_enqueue(E,6,msg(<<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0>>)), + make_enqueue(E,7,msg(<<0,0,0,0,0,0,0,0,0,0,0,0,0,0>>)) ], run_snapshot_test(#{name => ?FUNCTION_NAME, max_length => 4, @@ -391,15 +386,15 @@ scenario21(_Config) -> E = c:pid(0,176,1), Commands = [ make_checkout(C1, {auto,2,simple_prefetch}), - make_enqueue(E,1,<<"1">>), - make_enqueue(E,2,<<"2">>), - make_enqueue(E,3,<<"3">>), + make_enqueue(E,1,msg(<<"1">>)), + make_enqueue(E,2,msg(<<"2">>)), + make_enqueue(E,3,msg(<<"3">>)), rabbit_fifo:make_discard(C1, [0]), rabbit_fifo:make_settle(C1, [1]) ], run_snapshot_test(#{name => ?FUNCTION_NAME, release_cursor_interval => 1, - dead_letter_handler => {?MODULE, banana, []}}, + dead_letter_handler => {at_most_once, {?MODULE, banana, []}}}, Commands), ok. @@ -408,16 +403,16 @@ scenario22(_Config) -> % C1 = {<<>>, C1Pid}, E = c:pid(0,176,1), Commands = [ - make_enqueue(E,1,<<"1">>), - make_enqueue(E,2,<<"2">>), - make_enqueue(E,3,<<"3">>), - make_enqueue(E,4,<<"4">>), - make_enqueue(E,5,<<"5">>) + make_enqueue(E,1,msg(<<"1">>)), + make_enqueue(E,2,msg(<<"2">>)), + make_enqueue(E,3,msg(<<"3">>)), + make_enqueue(E,4,msg(<<"4">>)), + make_enqueue(E,5,msg(<<"5">>)) ], run_snapshot_test(#{name => ?FUNCTION_NAME, release_cursor_interval => 1, max_length => 3, - dead_letter_handler => {?MODULE, banana, []}}, + dead_letter_handler => {at_most_once, {?MODULE, banana, []}}}, Commands), ok. @@ -429,10 +424,10 @@ scenario24(_Config) -> Commands = [ make_checkout(C1, {auto,2,simple_prefetch}), %% 1 make_checkout(C2, {auto,1,simple_prefetch}), %% 2 - make_enqueue(E,1,<<"1">>), %% 3 - make_enqueue(E,2,<<"2b">>), %% 4 - make_enqueue(E,3,<<"3">>), %% 5 - make_enqueue(E,4,<<"4">>), %% 6 + make_enqueue(E,1,msg(<<"1">>)), %% 3 + make_enqueue(E,2,msg(<<"2b">>)), %% 4 + make_enqueue(E,3,msg(<<"3">>)), %% 5 + make_enqueue(E,4,msg(<<"4">>)), %% 6 {down, E, noconnection} %% 7 ], run_snapshot_test(#{name => ?FUNCTION_NAME, @@ -440,7 +435,7 @@ scenario24(_Config) -> deliver_limit => undefined, max_length => 3, overflow_strategy => drop_head, - dead_letter_handler => {?MODULE, banana, []} + dead_letter_handler => {at_most_once, {?MODULE, banana, []}} }, Commands), ok. @@ -453,12 +448,12 @@ scenario25(_Config) -> E = c:pid(0,280,0), Commands = [ make_checkout(C1, {auto,2,simple_prefetch}), %% 1 - make_enqueue(E,1,<<0>>), %% 2 + make_enqueue(E,1,msg(<<0>>)), %% 2 make_checkout(C2, {auto,1,simple_prefetch}), %% 3 - make_enqueue(E,2,<<>>), %% 4 - make_enqueue(E,3,<<>>), %% 5 + make_enqueue(E,2,msg(<<>>)), %% 4 + make_enqueue(E,3,msg(<<>>)), %% 5 {down, C1Pid, noproc}, %% 6 - make_enqueue(E,4,<<>>), %% 7 + make_enqueue(E,4,msg(<<>>)), %% 7 rabbit_fifo:make_purge() %% 8 ], run_snapshot_test(#{name => ?FUNCTION_NAME, @@ -466,7 +461,7 @@ scenario25(_Config) -> release_cursor_interval => 0, deliver_limit => undefined, overflow_strategy => drop_head, - dead_letter_handler => {?MODULE, banana, []} + dead_letter_handler => {at_most_once, {?MODULE, banana, []}} }, Commands), ok. @@ -477,19 +472,19 @@ scenario26(_Config) -> E1 = c:pid(0,436,0), E2 = c:pid(0,435,0), Commands = [ - make_enqueue(E1,2,<<>>), %% 1 - make_enqueue(E1,3,<<>>), %% 2 - make_enqueue(E2,1,<<>>), %% 3 - make_enqueue(E2,2,<<>>), %% 4 - make_enqueue(E1,4,<<>>), %% 5 - make_enqueue(E1,5,<<>>), %% 6 - make_enqueue(E1,6,<<>>), %% 7 - make_enqueue(E1,7,<<>>), %% 8 - make_enqueue(E1,1,<<>>), %% 9 + make_enqueue(E1,2,msg(<<>>)), %% 1 + make_enqueue(E1,3,msg(<<>>)), %% 2 + make_enqueue(E2,1,msg(<<>>)), %% 3 + make_enqueue(E2,2,msg(<<>>)), %% 4 + make_enqueue(E1,4,msg(<<>>)), %% 5 + make_enqueue(E1,5,msg(<<>>)), %% 6 + make_enqueue(E1,6,msg(<<>>)), %% 7 + make_enqueue(E1,7,msg(<<>>)), %% 8 + make_enqueue(E1,1,msg(<<>>)), %% 9 make_checkout(C1, {auto,5,simple_prefetch}), %% 1 - make_enqueue(E1,8,<<>>), %% 2 - make_enqueue(E1,9,<<>>), %% 2 - make_enqueue(E1,10,<<>>), %% 2 + make_enqueue(E1,8,msg(<<>>)), %% 2 + make_enqueue(E1,9,msg(<<>>)), %% 2 + make_enqueue(E1,10,msg(<<>>)), %% 2 {down, C1Pid, noconnection} ], run_snapshot_test(#{name => ?FUNCTION_NAME, @@ -497,22 +492,22 @@ scenario26(_Config) -> deliver_limit => undefined, max_length => 8, overflow_strategy => drop_head, - dead_letter_handler => {?MODULE, banana, []} + dead_letter_handler => {at_most_once, {?MODULE, banana, []}} }, Commands), ok. scenario28(_Config) -> E = c:pid(0,151,0), - Conf = #{dead_letter_handler => {rabbit_fifo_prop_SUITE,banana,[]}, + Conf = #{dead_letter_handler => {at_most_once, {rabbit_fifo_prop_SUITE,banana,[]}}, delivery_limit => undefined, max_in_memory_bytes => undefined, max_length => 1,name => ?FUNCTION_NAME,overflow_strategy => drop_head, release_cursor_interval => 100,single_active_consumer_on => false}, Commands = [ - make_enqueue(E,2, <<>>), - make_enqueue(E,3, <<>>), - make_enqueue(E,1, <<>>) + make_enqueue(E,2,msg( <<>>)), + make_enqueue(E,3,msg( <<>>)), + make_enqueue(E,1,msg( <<>>)) ], ?assert(single_active_prop(Conf, Commands, false)), ok. @@ -525,39 +520,39 @@ scenario27(_Config) -> E = c:pid(0,151,0), E2 = c:pid(0,152,0), Commands = [ - make_enqueue(E,1,<<>>), - make_enqueue(E2,1,<<28,202>>), - make_enqueue(E,2,<<"Î2">>), + make_enqueue(E,1,msg(<<>>)), + make_enqueue(E2,1,msg(<<28,202>>)), + make_enqueue(E,2,msg(<<"Î2">>)), {down, E, noproc}, - make_enqueue(E2,2,<<"ê">>), + make_enqueue(E2,2,msg(<<"ê">>)), {nodeup,fakenode@fake}, - make_enqueue(E2,3,<<>>), - make_enqueue(E2,4,<<>>), - make_enqueue(E2,5,<<>>), - make_enqueue(E2,6,<<>>), - make_enqueue(E2,7,<<>>), - make_enqueue(E2,8,<<>>), - make_enqueue(E2,9,<<>>), + make_enqueue(E2,3,msg(<<>>)), + make_enqueue(E2,4,msg(<<>>)), + make_enqueue(E2,5,msg(<<>>)), + make_enqueue(E2,6,msg(<<>>)), + make_enqueue(E2,7,msg(<<>>)), + make_enqueue(E2,8,msg(<<>>)), + make_enqueue(E2,9,msg(<<>>)), {purge}, - make_enqueue(E2,10,<<>>), - make_enqueue(E2,11,<<>>), - make_enqueue(E2,12,<<>>), - make_enqueue(E2,13,<<>>), - make_enqueue(E2,14,<<>>), - make_enqueue(E2,15,<<>>), - make_enqueue(E2,16,<<>>), - make_enqueue(E2,17,<<>>), - make_enqueue(E2,18,<<>>), + make_enqueue(E2,10,msg(<<>>)), + make_enqueue(E2,11,msg(<<>>)), + make_enqueue(E2,12,msg(<<>>)), + make_enqueue(E2,13,msg(<<>>)), + make_enqueue(E2,14,msg(<<>>)), + make_enqueue(E2,15,msg(<<>>)), + make_enqueue(E2,16,msg(<<>>)), + make_enqueue(E2,17,msg(<<>>)), + make_enqueue(E2,18,msg(<<>>)), {nodeup,fakenode@fake}, - make_enqueue(E2,19,<<>>), + make_enqueue(E2,19,msg(<<>>)), make_checkout(C1, {auto,77,simple_prefetch}), - make_enqueue(E2,20,<<>>), - make_enqueue(E2,21,<<>>), - make_enqueue(E2,22,<<>>), - make_enqueue(E2,23,<<"Ýý">>), + make_enqueue(E2,20,msg(<<>>)), + make_enqueue(E2,21,msg(<<>>)), + make_enqueue(E2,22,msg(<<>>)), + make_enqueue(E2,23,msg(<<"Ýý">>)), make_checkout(C2, {auto,66,simple_prefetch}), {purge}, - make_enqueue(E2,24,<<>>) + make_enqueue(E2,24,msg(<<>>)) ], ?assert( single_active_prop(#{name => ?FUNCTION_NAME, @@ -569,7 +564,7 @@ scenario27(_Config) -> max_in_memory_bytes => 691, overflow_strategy => drop_head, single_active_consumer_on => true, - dead_letter_handler => {?MODULE, banana, []} + dead_letter_handler => {at_most_once, {?MODULE, banana, []}} }, Commands, false)), ok. @@ -578,11 +573,11 @@ scenario30(_Config) -> C1 = {<<>>, C1Pid}, E = c:pid(0,240,0), Commands = [ - make_enqueue(E,1,<<>>), %% 1 - make_enqueue(E,2,<<1>>), %% 2 + make_enqueue(E,1,msg(<<>>)), %% 1 + make_enqueue(E,2,msg(<<1>>)), %% 2 make_checkout(C1, {auto,1,simple_prefetch}), %% 3 {down, C1Pid, noconnection}, %% 4 - make_enqueue(E,3,<<>>) %% 5 + make_enqueue(E,3,msg(<<>>)) %% 5 ], run_snapshot_test(#{name => ?FUNCTION_NAME, release_cursor_interval => 0, @@ -590,7 +585,7 @@ scenario30(_Config) -> max_length => 1, max_in_memory_length => 1, overflow_strategy => drop_head, - dead_letter_handler => {?MODULE, banana, []}, + dead_letter_handler => {at_most_once, {?MODULE, banana, []}}, single_active_consumer_on => true }, Commands), @@ -609,8 +604,8 @@ scenario31(_Config) -> % {auto,1,simple_prefetch}, % #{ack => true,args => [],prefetch => 1,username => <<"user">>}}}, % {4,{purge}}] - make_enqueue(E1,1,<<>>), %% 1 - make_enqueue(E2,2,<<1>>), %% 2 + make_enqueue(E1,1,msg(<<>>)), %% 1 + make_enqueue(E2,2,msg(<<1>>)), %% 2 make_checkout(C1, {auto,1,simple_prefetch}), %% 3 {purge} %% 4 ], @@ -618,7 +613,7 @@ scenario31(_Config) -> release_cursor_interval => 0, deliver_limit => undefined, overflow_strategy => drop_head, - dead_letter_handler => {?MODULE, banana, []} + dead_letter_handler => {at_most_once, {?MODULE, banana, []}} }, Commands), ok. @@ -626,17 +621,17 @@ scenario31(_Config) -> scenario32(_Config) -> E1 = c:pid(0,314,0), Commands = [ - make_enqueue(E1,1,<<0>>), %% 1 - make_enqueue(E1,2,<<0,0>>), %% 2 - make_enqueue(E1,4,<<0,0,0,0>>), %% 3 - make_enqueue(E1,3,<<0,0,0>>) %% 4 + make_enqueue(E1,1,msg(<<0>>)), %% 1 + make_enqueue(E1,2,msg(<<0,0>>)), %% 2 + make_enqueue(E1,4,msg(<<0,0,0,0>>)), %% 3 + make_enqueue(E1,3,msg(<<0,0,0>>)) %% 4 ], run_snapshot_test(#{name => ?FUNCTION_NAME, release_cursor_interval => 0, max_length => 3, deliver_limit => undefined, overflow_strategy => drop_head, - dead_letter_handler => {?MODULE, banana, []} + dead_letter_handler => {at_most_once, {?MODULE, banana, []}} }, Commands), ok. @@ -646,14 +641,14 @@ scenario29(_Config) -> C1 = {<<>>, C1Pid}, E = c:pid(0,240,0), Commands = [ - make_enqueue(E,1,<<>>), %% 1 - make_enqueue(E,2,<<>>), %% 2 + make_enqueue(E,1,msg(<<>>)), %% 1 + make_enqueue(E,2,msg(<<>>)), %% 2 make_checkout(C1, {auto,2,simple_prefetch}), %% 2 - make_enqueue(E,3,<<>>), %% 3 - make_enqueue(E,4,<<>>), %% 4 - make_enqueue(E,5,<<>>), %% 5 - make_enqueue(E,6,<<>>), %% 6 - make_enqueue(E,7,<<>>), %% 7 + make_enqueue(E,3,msg(<<>>)), %% 3 + make_enqueue(E,4,msg(<<>>)), %% 4 + make_enqueue(E,5,msg(<<>>)), %% 5 + make_enqueue(E,6,msg(<<>>)), %% 6 + make_enqueue(E,7,msg(<<>>)), %% 7 {down, E, noconnection} %% 8 ], run_snapshot_test(#{name => ?FUNCTION_NAME, @@ -662,7 +657,7 @@ scenario29(_Config) -> max_length => 5, max_in_memory_length => 1, overflow_strategy => drop_head, - dead_letter_handler => {?MODULE, banana, []}, + dead_letter_handler => {at_most_once, {?MODULE, banana, []}}, single_active_consumer_on => true }, Commands), @@ -672,19 +667,19 @@ scenario23(_Config) -> C1 = {<<>>, C1Pid}, E = c:pid(0,240,0), Commands = [ - make_enqueue(E,1,<<>>), %% 1 + make_enqueue(E,1,msg(<<>>)), %% 1 make_checkout(C1, {auto,2,simple_prefetch}), %% 2 - make_enqueue(E,2,<<>>), %% 3 - make_enqueue(E,3,<<>>), %% 4 + make_enqueue(E,2,msg(<<>>)), %% 3 + make_enqueue(E,3,msg(<<>>)), %% 4 {down, E, noconnection}, %% 5 - make_enqueue(E,4,<<>>) %% 6 + make_enqueue(E,4,msg(<<>>)) %% 6 ], run_snapshot_test(#{name => ?FUNCTION_NAME, release_cursor_interval => 0, deliver_limit => undefined, max_length => 2, overflow_strategy => drop_head, - dead_letter_handler => {?MODULE, banana, []} + dead_letter_handler => {at_most_once, {?MODULE, banana, []}} }, Commands), ok. @@ -697,7 +692,7 @@ single_active_01(_Config) -> E = test_util:fake_pid(rabbit@fake_node2), Commands = [ make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E,1,<<"one">>), + make_enqueue(E,1,msg(<<"one">>)), make_checkout(C2, {auto,1,simple_prefetch}), make_checkout(C1, cancel), {nodeup,rabbit@fake_node1} @@ -716,7 +711,7 @@ single_active_02(_Config) -> E = test_util:fake_pid(node()), Commands = [ make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E,1,<<"one">>), + make_enqueue(E,1,msg(<<"one">>)), {down,E,noconnection}, make_checkout(C2, {auto,1,simple_prefetch}), make_checkout(C2, cancel), @@ -735,8 +730,8 @@ single_active_03(_Config) -> E = test_util:fake_pid(rabbit@fake_node2), Commands = [ make_checkout(C1, {auto,2,simple_prefetch}), - make_enqueue(E, 1, 0), - make_enqueue(E, 2, 1), + make_enqueue(E, 1, msg(<<0>>)), + make_enqueue(E, 2, msg(<<1>>)), {down, Pid, noconnection}, {nodeup, node()} ], @@ -754,10 +749,10 @@ single_active_04(_Config) -> Commands = [ % make_checkout(C1, {auto,2,simple_prefetch}), - make_enqueue(E, 1, <<>>), - make_enqueue(E, 2, <<>>), - make_enqueue(E, 3, <<>>), - make_enqueue(E, 4, <<>>) + make_enqueue(E, 1, msg(<<>>)), + make_enqueue(E, 2, msg(<<>>)), + make_enqueue(E, 3, msg(<<>>)), + make_enqueue(E, 4, msg(<<>>)) % {down, Pid, noconnection}, % {nodeup, node()} ], @@ -796,15 +791,16 @@ snapshots(_Config) -> fun () -> ?FORALL({Length, Bytes, SingleActiveConsumer, DeliveryLimit, InMemoryLength, InMemoryBytes, - Overflow}, - frequency([{10, {0, 0, false, 0, 0, 0, drop_head}}, + Overflow, DeadLetterHandler}, + frequency([{10, {0, 0, false, 0, 0, 0, drop_head, undefined}}, {5, {oneof([range(1, 10), undefined]), oneof([range(1, 1000), undefined]), boolean(), oneof([range(1, 3), undefined]), oneof([range(1, 10), undefined]), oneof([range(1, 1000), undefined]), - oneof([drop_head, reject_publish]) + oneof([drop_head, reject_publish]), + oneof([undefined, {at_most_once, {?MODULE, banana, []}}]) }}]), begin Config = config(?FUNCTION_NAME, @@ -814,13 +810,43 @@ snapshots(_Config) -> DeliveryLimit, InMemoryLength, InMemoryBytes, - Overflow), + Overflow, + DeadLetterHandler), ?FORALL(O, ?LET(Ops, log_gen(256), expand(Ops, Config)), collect({log_size, length(O)}, snapshots_prop(Config, O))) end) end, [], 1000). +snapshots_dlx(_Config) -> + run_proper( + fun () -> + ?FORALL({Length, Bytes, SingleActiveConsumer, + DeliveryLimit, InMemoryLength, InMemoryBytes}, + frequency([{10, {0, 0, false, 0, 0, 0}}, + {5, {oneof([range(1, 10), undefined]), + oneof([range(1, 1000), undefined]), + boolean(), + oneof([range(1, 3), undefined]), + oneof([range(1, 10), undefined]), + oneof([range(1, 1000), undefined]) + }}]), + begin + Config = config(?FUNCTION_NAME, + Length, + Bytes, + SingleActiveConsumer, + DeliveryLimit, + InMemoryLength, + InMemoryBytes, + reject_publish, + at_least_once), + ?FORALL(O, ?LET(Ops, log_gen_dlx(256), expand(Ops, Config)), + collect({log_size, length(O)}, + snapshots_prop(Config, O))) + end) + end, [], 1000). + single_active(_Config) -> Size = 300, run_proper( @@ -867,7 +893,10 @@ upgrade(_Config) -> SingleActive, DeliveryLimit, InMemoryLength, - undefined), + undefined, + drop_head, + {?MODULE, banana, []} + ), ?FORALL(O, ?LET(Ops, log_gen(Size), expand(Ops, Config)), collect({log_size, length(O)}, upgrade_prop(Config, O))) @@ -923,10 +952,10 @@ single_active_ordering_01(_Config) -> E = test_util:fake_pid(rabbit@fake_node2), E2 = test_util:fake_pid(rabbit@fake_node2), Commands = [ - make_enqueue(E, 1, 0), - make_enqueue(E, 2, 1), + make_enqueue(E, 1, msg(<<"0">>)), + make_enqueue(E, 2, msg(<<"1">>)), make_checkout(C1, {auto,2,simple_prefetch}), - make_enqueue(E2, 1, 2), + make_enqueue(E2, 1, msg(<<"2">>)), make_settle(C1, [0]) ], Conf = config(?FUNCTION_NAME, 0, 0, true, 0, 0, 0), @@ -945,7 +974,7 @@ single_active_ordering_02(_Config) -> E = test_util:fake_pid(node()), Commands = [ make_checkout(C1, {auto,1,simple_prefetch}), - make_enqueue(E, 2, 1), + make_enqueue(E, 2, msg(<<"1">>)), %% CANNOT HAPPEN {down,E,noproc}, make_settle(C1, [0]) @@ -961,9 +990,9 @@ single_active_ordering_03(_Config) -> C2 = {<<2>>, C2Pid}, E = test_util:fake_pid(rabbit@fake_node2), Commands = [ - make_enqueue(E, 1, 0), - make_enqueue(E, 2, 1), - make_enqueue(E, 3, 2), + make_enqueue(E, 1, msg(<<"0">>)), + make_enqueue(E, 2, msg(<<"1">>)), + make_enqueue(E, 3, msg(<<"2">>)), make_checkout(C1, {auto,1,simple_prefetch}), make_checkout(C2, {auto,1,simple_prefetch}), make_settle(C1, [0]), @@ -1045,17 +1074,232 @@ max_length(_Config) -> end) end, [], Size). -config(Name, Length, Bytes, SingleActive, DeliveryLimit, - InMemoryLength, InMemoryBytes) -> -config(Name, Length, Bytes, SingleActive, DeliveryLimit, - InMemoryLength, InMemoryBytes, drop_head). +%% Test that rabbit_fifo_dlx can check out a prefix message. +dlx_01(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 1), + make_checkout(C1, {auto,1,simple_prefetch}), + make_enqueue(E,1,msg(<<"1">>)), + make_enqueue(E,2,msg(<<"2">>)), + rabbit_fifo:make_discard(C1, [0]), + rabbit_fifo_dlx:make_settle([0]), + rabbit_fifo:make_discard(C1, [1]), + rabbit_fifo_dlx:make_settle([1]) + ], + Config = config(?FUNCTION_NAME, 8, undefined, false, 2, 5, 100, reject_publish, at_least_once), + ?assert(snapshots_prop(Config, Commands)), + ok. + +%% Test that dehydrating dlx_consumer works. +dlx_02(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 1), + make_checkout(C1, {auto,1,simple_prefetch}), + make_enqueue(E,1,msg(<<"1">>)), + %% State contains release cursor A. + rabbit_fifo:make_discard(C1, [0]), + make_enqueue(E,2,msg(<<"2">>)), + %% State contains release cursor B + %% with the 1st msg being checked out to dlx_consumer and + %% being dehydrated. + rabbit_fifo_dlx:make_settle([0]) + %% Release cursor A got emitted. + ], + Config = config(?FUNCTION_NAME, 10, undefined, false, 5, 5, 100, reject_publish, at_least_once), + ?assert(snapshots_prop(Config, Commands)), + ok. + +%% Test that dehydrating discards queue works. +dlx_03(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + make_enqueue(E,1,msg(<<"1">>)), + %% State contains release cursor A. + make_checkout(C1, {auto,1,simple_prefetch}), + rabbit_fifo:make_discard(C1, [0]), + make_enqueue(E,2,msg(<<"2">>)), + %% State contains release cursor B. + %% 1st message sitting in discards queue got dehydrated. + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 1), + rabbit_fifo_dlx:make_settle([0]) + %% Release cursor A got emitted. + ], + Config = config(?FUNCTION_NAME, 10, undefined, false, 5, 5, 100, reject_publish, at_least_once), + ?assert(snapshots_prop(Config, Commands)), + ok. + +dlx_04(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 3), + make_enqueue(E,1,msg(<<>>)), + make_enqueue(E,2,msg(<<>>)), + make_enqueue(E,3,msg(<<>>)), + make_enqueue(E,4,msg(<<>>)), + make_enqueue(E,5,msg(<<>>)), + make_enqueue(E,6,msg(<<>>)), + make_checkout(C1, {auto,6,simple_prefetch}), + rabbit_fifo:make_discard(C1, [0,1,2,3,4,5]), + rabbit_fifo_dlx:make_settle([0,1,2]) + ], + Config = config(?FUNCTION_NAME, undefined, undefined, true, 1, 5, 136, reject_publish, at_least_once), + ?assert(snapshots_prop(Config, Commands)), + ok. + +%% Test that discards queue gets dehydrated with 1 message that has empty message body. +dlx_05(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + make_enqueue(E,1,msg(<<>>)), + make_enqueue(E,2,msg(<<"msg2">>)), + %% 0,1 in messages + make_checkout(C1, {auto,1,simple_prefetch}), + rabbit_fifo:make_discard(C1, [0]), + %% 0 in discards, 1 in checkout + make_enqueue(E,3,msg(<<"msg3">>)), + %% 0 in discards (rabbit_fifo_dlx msg_bytes is still 0 because body of msg 0 is empty), + %% 1 in checkout, 2 in messages + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 1), + %% 0 in dlx_checkout, 1 in checkout, 2 in messages + make_settle(C1, [1]), + %% 0 in dlx_checkout, 2 in checkout + rabbit_fifo_dlx:make_settle([0]) + %% 2 in checkout + ], + Config = config(?FUNCTION_NAME, 0, 0, false, 0, 0, 0, reject_publish, at_least_once), + ?assert(snapshots_prop(Config, Commands)), + ok. + +% Test that after recovery we can differentiate between index messge and (prefix) disk message +dlx_06(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + make_enqueue(E,1,msg(<<>>)), + %% The following message has 3 bytes. + %% If we cannot differentiate between disk message and prefix disk message, + %% rabbit_fifo:delete_indexes/2 will not know whether it's a disk message or + %% prefix disk message and it will therefore falsely think that 3 is an index + %% instead of a size header resulting in message 3 being deleted from the index + %% after recovery. + make_enqueue(E,2,msg(<<"111">>)), + make_enqueue(E,3,msg(<<>>)), + %% 0,1,2 in messages + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 2), + make_checkout(C1, {auto,3,simple_prefetch}), + %% 0,1,2 in checkout + rabbit_fifo:make_discard(C1, [0,1,2]), + %% 0,1 in dlx_checkout, 3 in discards + rabbit_fifo_dlx:make_settle([0,1]) + %% 3 in dlx_checkout + ], + Config = config(?FUNCTION_NAME, undefined, 749, false, 1, 1, 131, reject_publish, at_least_once), + ?assert(snapshots_prop(Config, Commands)), + ok. + +dlx_07(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + make_checkout(C1, {auto,1,simple_prefetch}), + make_enqueue(E,1,msg(<<"12">>)), + %% 0 in checkout + rabbit_fifo:make_discard(C1, [0]), + %% 0 in discard + make_enqueue(E,2,msg(<<"1234567">>)), + %% 0 in discard, 1 in checkout + rabbit_fifo:make_discard(C1, [1]), + %% 0, 1 in discard + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 1), + %% 0 in dlx_checkout, 1 in discard + make_enqueue(E,3,msg(<<"123">>)), + %% 0 in dlx_checkout, 1 in discard, 2 in checkout + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 2), + %% 0,1 in dlx_checkout, 2 in checkout + rabbit_fifo_dlx:make_settle([0]), + %% 1 in dlx_checkout, 2 in checkout + make_settle(C1, [2]), + %% 1 in dlx_checkout + make_enqueue(E,4,msg(<<>>)), + %% 1 in dlx_checkout, 3 in checkout + rabbit_fifo_dlx:make_settle([0,1]) + %% 3 in checkout + ], + Config = config(?FUNCTION_NAME, undefined, undefined, false, undefined, undefined, undefined, + reject_publish, at_least_once), + ?assert(snapshots_prop(Config, Commands)), + ok. + +%% This test fails if discards queue is not normalized for comparison. +dlx_08(_Config) -> + C1Pid = c:pid(0,883,1), + C1 = {<<>>, C1Pid}, + E = c:pid(0,176,1), + Commands = [ + make_enqueue(E,1,msg(<<>>)), + %% 0 in messages + make_checkout(C1, {auto,1,simple_prefetch}), + %% 0 in checkout + make_enqueue(E,2,msg(<<>>)), + %% 1 in messages, 0 in checkout + rabbit_fifo:make_discard(C1, [0]), + %% 1 in checkout, 0 in discards + make_enqueue(E,3,msg(<<>>)), + %% 2 in messages, 1 in checkout, 0 in discards + rabbit_fifo:make_discard(C1, [1]), + %% 2 in checkout, 0,1 in discards + rabbit_fifo:make_discard(C1, [2]), + %% 0,1,2 in discards + make_enqueue(E,4,msg(<<>>)), + %% 3 in checkout, 0,1,2 in discards + %% last command emitted this release cursor + make_settle(C1, [3]), + make_enqueue(E,5,msg(<<>>)), + make_enqueue(E,6,msg(<<>>)), + rabbit_fifo:make_discard(C1, [4]), + rabbit_fifo:make_discard(C1, [5]), + make_enqueue(E,7,msg(<<>>)), + make_enqueue(E,8,msg(<<>>)), + make_enqueue(E,9,msg(<<>>)), + rabbit_fifo:make_discard(C1, [6]), + rabbit_fifo:make_discard(C1, [7]), + rabbit_fifo_dlx:make_checkout(my_dlx_worker, 1), + make_enqueue(E,10,msg(<<>>)), + rabbit_fifo:make_discard(C1, [8]), + rabbit_fifo_dlx:make_settle([0]), + rabbit_fifo:make_discard(C1, [9]), + rabbit_fifo_dlx:make_settle([1]), + rabbit_fifo_dlx:make_settle([2]) + ], + Config = config(?FUNCTION_NAME, undefined, undefined, false, undefined, undefined, undefined, + reject_publish, at_least_once), + ?assert(snapshots_prop(Config, Commands)), + ok. + +config(Name, Length, Bytes, SingleActive, DeliveryLimit, InMemoryLength, InMemoryBytes) -> +config(Name, Length, Bytes, SingleActive, DeliveryLimit, InMemoryLength, InMemoryBytes, + drop_head, {at_most_once, {?MODULE, banana, []}}). config(Name, Length, Bytes, SingleActive, DeliveryLimit, - InMemoryLength, InMemoryBytes, Overflow) -> + InMemoryLength, InMemoryBytes, Overflow, DeadLetterHandler) -> #{name => Name, max_length => map_max(Length), max_bytes => map_max(Bytes), - dead_letter_handler => {?MODULE, banana, []}, + dead_letter_handler => DeadLetterHandler, single_active_consumer_on => SingleActive, delivery_limit => map_max(DeliveryLimit), max_in_memory_length => map_max(InMemoryLength), @@ -1121,6 +1365,16 @@ validate_idx_order(Idxs, ReleaseCursorIdx) -> ok end. +%%TODO write separate generator for dlx using single_active_prop() or +%% messages_total_prop() as base template. +%% +%% E.g. enqueue few messages and have a consumer rejecting those. +%% The invariant could be: Delivery effects to dlx_worker must match the number of dead-lettered messages. +%% +%% Other invariants could be: +%% * if new consumer subscribes, messages are checked out to new consumer +%% * if dlx_worker fails receiving DOWN, messages are still in state. + single_active_prop(Conf0, Commands, ValidateOrder) -> Conf = Conf0#{release_cursor_interval => 100}, Indexes = lists:seq(1, length(Commands)), @@ -1166,14 +1420,23 @@ messages_total_invariant() -> consumers = C, enqueuers = E, prefix_msgs = {PTot, _, RTot, _}, - returns = R} = S) -> + returns = R, + dlx = #rabbit_fifo_dlx{discards = D, + consumer = DlxCon}} = S) -> Base = lqueue:len(M) + lqueue:len(R) + PTot + RTot, CTot = maps:fold(fun (_, #consumer{checked_out = Ch}, Acc) -> Acc + map_size(Ch) end, Base, C), - Tot = maps:fold(fun (_, #enqueuer{pending = P}, Acc) -> + Tot0 = maps:fold(fun (_, #enqueuer{pending = P}, Acc) -> Acc + length(P) end, CTot, E), + Tot1 = Tot0 + lqueue:len(D), + Tot = case DlxCon of + undefined -> + Tot1; + #dlx_consumer{checked_out = DlxChecked} -> + Tot1 + map_size(DlxChecked) + end, QTot = rabbit_fifo:query_messages_total(S), case Tot == QTot of true -> true; @@ -1262,9 +1525,6 @@ snapshots_prop(Conf, Commands) -> end. log_gen(Size) -> - log_gen(Size, binary()). - -log_gen(Size, _Body) -> Nodes = [node(), fakenode@fake, fakenode@fake2 @@ -1287,6 +1547,35 @@ log_gen(Size, _Body) -> {1, purge} ]))))). +log_gen_dlx(Size) -> + Nodes = [node(), + fakenode@fake, + fakenode@fake2 + ], + ?LET(EPids, vector(2, pid_gen(Nodes)), + ?LET(CPids, vector(2, pid_gen(Nodes)), + resize(Size, + list( + frequency( + [{20, enqueue_gen(oneof(EPids))}, + {40, {input_event, + frequency([{1, settle}, + {1, return}, + %% dead-letter many messages + {5, discard}, + {1, requeue}])}}, + {2, checkout_gen(oneof(CPids))}, + {1, checkout_cancel_gen(oneof(CPids))}, + {1, down_gen(oneof(EPids ++ CPids))}, + {1, nodeup_gen(Nodes)}, + {1, purge}, + %% same dlx_worker can subscribe multiple times, + %% e.g. after it dlx_worker crashed + %% "last subscriber wins" + {2, {checkout_dlx, choose(1,10)}} + ]))))). + + log_gen_config(Size) -> Nodes = [node(), fakenode@fake, @@ -1359,7 +1648,18 @@ enqueue_gen(Pid, Enq, Del) -> ?LET(E, {enqueue, Pid, frequency([{Enq, enqueue}, {Del, delay}]), - binary()}, E). + msg_gen()}, E). + +%% It's fair to assume that every message enqueued is a #basic_message. +%% That's what the channel expects and what rabbit_quorum_queue invokes rabbit_fifo_client with. +msg_gen() -> + ?LET(Bin, binary(), + #basic_message{content = #content{payload_fragments_rev = [Bin], + properties = none}}). + +msg(Bin) when is_binary(Bin) -> + #basic_message{content = #content{payload_fragments_rev = [Bin], + properties = none}}. checkout_cancel_gen(Pid) -> {checkout, Pid, cancel}. @@ -1368,11 +1668,7 @@ checkout_gen(Pid) -> %% pid, tag, prefetch ?LET(C, {checkout, {binary(), Pid}, choose(1, 100)}, C). - --record(t, {state = rabbit_fifo:init(#{name => proper, - queue_resource => blah, - release_cursor_interval => 1}) - :: rabbit_fifo:state(), +-record(t, {state :: rabbit_fifo:state(), index = 1 :: non_neg_integer(), %% raft index enqueuers = #{} :: #{pid() => term()}, consumers = #{} :: #{{binary(), pid()} => term()}, @@ -1387,20 +1683,34 @@ checkout_gen(Pid) -> expand(Ops, Config) -> expand(Ops, Config, {undefined, fun ra_lib:id/1}). +%% generates a sequence of Raft commands expand(Ops, Config, EnqFun) -> %% execute each command against a rabbit_fifo state and capture all relevant %% effects - T = #t{enq_body_fun = EnqFun, + InitConfig0 = #{name => proper, + queue_resource => blah, + release_cursor_interval => 1}, + InitConfig = case Config of + #{dead_letter_handler := at_least_once} -> + %% Configure rabbit_fifo config with at_least_once so that + %% rabbit_fifo_dlx outputs dlx_delivery effects + %% which we are going to settle immediately in enq_effs/2. + %% Therefore the final generated Raft commands will include + %% {dlx, {checkout, ...}} and {dlx, {settle, ...}} Raft commands. + maps:put(dead_letter_handler, at_least_once, InitConfig0); + _ -> + InitConfig0 + end, + T = #t{state = rabbit_fifo:init(InitConfig), + enq_body_fun = EnqFun, config = Config}, #t{effects = Effs} = T1 = lists:foldl(fun handle_op/2, T, Ops), %% process the remaining effect #t{log = Log} = lists:foldl(fun do_apply/2, T1#t{effects = queue:new()}, queue:to_list(Effs)), - lists:reverse(Log). - handle_op({enqueue, Pid, When, Data}, #t{enqueuers = Enqs0, enq_body_fun = {EnqSt0, Fun}, @@ -1493,6 +1803,9 @@ handle_op({input_event, Settlement}, #t{effects = Effs, false -> do_apply(Cmd, T#t{effects = Q}) end; + {{value, {dlx, {settle, MsgIds}}}, Q} -> + Cmd = rabbit_fifo_dlx:make_settle(MsgIds), + do_apply(Cmd, T#t{effects = Q}); _ -> T end; @@ -1500,7 +1813,10 @@ handle_op(purge, T) -> do_apply(rabbit_fifo:make_purge(), T); handle_op({update_config, Changes}, #t{config = Conf} = T) -> Config = maps:merge(Conf, Changes), - do_apply(rabbit_fifo:make_update_config(Config), T). + do_apply(rabbit_fifo:make_update_config(Config), T); +handle_op({checkout_dlx, Prefetch}, #t{config = #{dead_letter_handler := at_least_once}} = T) -> + Cmd = rabbit_fifo_dlx:make_checkout(proper_dlx_worker, Prefetch), + do_apply(Cmd, T). do_apply(Cmd, #t{effects = Effs, @@ -1534,14 +1850,17 @@ enq_effs([{send_msg, P, {delivery, CTag, Msgs}, _Opts} | Rem], Q) -> %% they can be changed depending on the input event later Cmd = rabbit_fifo:make_settle({CTag, P}, MsgIds), enq_effs(Rem, queue:in(Cmd, Q)); +enq_effs([{send_msg, _, {dlx_delivery, Msgs}, _Opts} | Rem], Q) -> + MsgIds = [I || {I, _} <- Msgs], + Cmd = rabbit_fifo_dlx:make_settle(MsgIds), + enq_effs(Rem, queue:in(Cmd, Q)); enq_effs([_ | Rem], Q) -> enq_effs(Rem, Q). %% Utility run_proper(Fun, Args, NumTests) -> - ?assertEqual( - true, + ?assert( proper:counterexample( erlang:apply(Fun, Args), [{numtests, NumTests}, @@ -1585,10 +1904,12 @@ run_snapshot_test0(Conf, Commands, Invariant) -> State -> ok; _ -> ct:pal("Snapshot tests failed run log:~n" - "~p~n from ~n~p~n Entries~n~p~n" + "~p~n from snapshot index ~b " + "with snapshot state~n~p~n Entries~n~p~n" "Config: ~p~n", - [Filtered, SnapState, Entries, Conf]), - ct:pal("Expected~n~p~nGot:~n~p", [State, S]), + [Filtered, SnapIdx, SnapState, Entries, Conf]), + ct:pal("Expected~n~p~nGot:~n~p~n", [?record_info(rabbit_fifo, State), + ?record_info(rabbit_fifo, S)]), ?assertEqual(State, S) end end || {release_cursor, SnapIdx, SnapState} <- Cursors], diff --git a/deps/rabbit_common/include/rabbit.hrl b/deps/rabbit_common/include/rabbit.hrl index 1fb3d4e6ea44..86779c0042ea 100644 --- a/deps/rabbit_common/include/rabbit.hrl +++ b/deps/rabbit_common/include/rabbit.hrl @@ -112,7 +112,7 @@ -record(basic_message, {exchange_name, %% The exchange where the message was received routing_keys = [], %% Routing keys used during publish - content, %% The message content + content, %% The message #content record id, %% A `rabbit_guid:gen()` generated id is_persistent}). %% Whether the message was published as persistent diff --git a/deps/rabbitmq_management/priv/www/js/global.js b/deps/rabbitmq_management/priv/www/js/global.js index 335c3607cefd..8660d5556667 100644 --- a/deps/rabbitmq_management/priv/www/js/global.js +++ b/deps/rabbitmq_management/priv/www/js/global.js @@ -174,7 +174,7 @@ const QUEUE_EXTRA_CONTENT_REQUESTS = []; // All help ? popups var HELP = { 'delivery-limit': - 'The number of allowed unsuccessful delivery attempts. Once a message has been delivered unsuccessfully this many times it will be dropped or dead-lettered, depending on the queue configuration.', + 'The number of allowed unsuccessful delivery attempts. Once a message has been delivered unsuccessfully more than this many times it will be dropped or dead-lettered, depending on the queue configuration.', 'exchange-auto-delete': 'If yes, the exchange will delete itself after at least one queue or exchange has been bound to this one, and then all queues or exchanges have been unbound.', @@ -218,6 +218,9 @@ var HELP = { 'queue-dead-letter-routing-key': 'Optional replacement routing key to use when a message is dead-lettered. If this is not set, the message\'s original routing key will be used.
(Sets the "x-dead-letter-routing-key" argument.)', + 'queue-dead-letter-strategy': + 'Valid values are at-most-once or at-least-once. It defaults to at-most-once. This setting is understood only by quorum queues. If at-least-once is set, Overflow behaviour must be set to reject-publish. Otherwise, dead letter strategy will fall back to at-most-once.', + 'queue-single-active-consumer': 'If set, makes sure only one consumer at a time consumes from the queue and fails over to another registered consumer in case the active one is cancelled or dies.
(Sets the "x-single-active-consumer" argument.)', @@ -243,7 +246,7 @@ var HELP = { 'Set the queue initial cluster size.', 'queue-type': - 'Set the queue type, determining the type of queue to use: raft-based high availability or classic queue. Valid values are quorum or classic. It defaults to classic.
', + 'Set the queue type, determining the type of queue to use: raft-based high availability or classic queue. Valid values are quorum or classic. It defaults to classic.
', 'queue-messages': '

Message counts.

Note that "in memory" and "persistent" are not mutually exclusive; persistent messages can be in memory as well as on disc, and transient messages can be paged out if memory is tight. Non-durable queues will consider all messages to be transient.

', diff --git a/deps/rabbitmq_management/priv/www/js/tmpl/policies.ejs b/deps/rabbitmq_management/priv/www/js/tmpl/policies.ejs index 91aaefb80b0b..f87b32f7ce02 100644 --- a/deps/rabbitmq_management/priv/www/js/tmpl/policies.ejs +++ b/deps/rabbitmq_management/priv/www/js/tmpl/policies.ejs @@ -103,7 +103,8 @@ Overflow behaviour | Auto expire
Dead letter exchange | - Dead letter routing key
+ Dead letter routing key
+ Message TTL
Queues [Classic] @@ -114,7 +115,6 @@ HA mirror promotion on shutdown | HA mirror promotion on failure
- Message TTL | Lazy mode | Master Locator
@@ -127,7 +127,9 @@ Max in memory bytes | Delivery limit - +
+ Dead letter strategy + @@ -270,13 +272,14 @@ Max length | Max length bytes | Overflow behaviour - +
+ Message TTL + Queues [Classic] - Message TTL | Auto expire diff --git a/deps/rabbitmq_management/priv/www/js/tmpl/queues.ejs b/deps/rabbitmq_management/priv/www/js/tmpl/queues.ejs index cc489eece766..e387b6753376 100644 --- a/deps/rabbitmq_management/priv/www/js/tmpl/queues.ejs +++ b/deps/rabbitmq_management/priv/www/js/tmpl/queues.ejs @@ -312,13 +312,11 @@ Add - <% if (queue_type == "classic") { %> - Message TTL | - <% } %> <% if (queue_type != "stream") { %> Auto expire | - Overflow behaviour | - Single active consumer
+ Message TTL | + Overflow behaviour
+ Single active consumer | Dead letter exchange | Dead letter routing key
Max length | @@ -333,7 +331,8 @@ Delivery limit | Max in memory length | Max in memory bytes - | Initial cluster size
+ | Initial cluster size
+ Dead letter strategy
<% } %> <% if (queue_type == "stream") { %> Max time retention