Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(retainer): fix topic search by index #12303

Merged
merged 1 commit into from Jan 11, 2024
Merged

fix(retainer): fix topic search by index #12303

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
27 changes: 16 additions & 11 deletions apps/emqx_retainer/src/emqx_retainer_index.erl
View file
Open in desktop
Expand Up @@ -88,13 +88,16 @@ select_index(Tokens, Indices) ->
select_index(Tokens, Indices, 0, undefined).

%% @doc For an index and a wildcard topic
%% returns a matchspec pattern for the corresponding index key.
%% returns a tuple of:
%% * matchspec pattern for the corresponding index key
%% * boolean flag indicating whether the pattern is exact
%%
%% E.g. for `[2, 3]' index and <code>['+', <<"b">>, '+', <<"d">>]</code> wildcard topic
%% returns <code>{[2, 3], {[<<"b">>, '_'], ['_', <<"d">>]}}</code> pattern.
-spec condition(index(), emqx_types:words()) -> match_pattern_part().
-spec condition(index(), emqx_types:words()) -> {match_pattern_part(), boolean()}.
condition(Index, Tokens) ->
{Index, condition(Index, Tokens, 1, [], [])}.
{Condition, IsExact} = condition(Index, Tokens, 1, [], []),
{{Index, Condition}, IsExact}.

%% @doc Returns a matchspec pattern for a wildcard topic.
%%
Expand All @@ -103,15 +106,17 @@ condition(Index, Tokens) ->
-spec condition(emqx_types:words()) -> match_pattern_part().
condition(Tokens) ->
Tokens1 = [
case W =:= '+' of
true -> '_';
case W of
'+' -> '_';
_ -> W
end
|| W <- Tokens
],
case length(Tokens1) > 0 andalso lists:last(Tokens1) =:= '#' of
false -> Tokens1;
_ -> (Tokens1 -- ['#']) ++ '_'
false ->
Tokens1;
_ ->
(Tokens1 -- ['#']) ++ '_'
end.

%% @doc Restores concrete topic from its index key representation.
Expand Down Expand Up @@ -162,13 +167,13 @@ select_index(Tokens, [Index | Indices], MaxScore, SelectedIndex) ->
end.

condition([_NIndex | _OtherIndex], ['#' | _OtherTokens], _N, IndexMatch, OtherMatch) ->
{lists:reverse(IndexMatch) ++ '_', lists:reverse(OtherMatch) ++ '_'};
{{lists:reverse(IndexMatch) ++ '_', lists:reverse(OtherMatch) ++ '_'}, false};
condition([], ['#' | _OtherTokens], _N, IndexMatch, OtherMatch) ->
{lists:reverse(IndexMatch), lists:reverse(OtherMatch) ++ '_'};
{{lists:reverse(IndexMatch), lists:reverse(OtherMatch) ++ '_'}, true};
condition([], Tokens, _N, IndexMatch, OtherMatch) ->
{lists:reverse(IndexMatch), lists:reverse(OtherMatch) ++ condition(Tokens)};
{{lists:reverse(IndexMatch), lists:reverse(OtherMatch) ++ condition(Tokens)}, true};
condition([_NIndex | _OtherIndex], [], _N, IndexMatch, OtherMatch) ->
{lists:reverse(IndexMatch) ++ '_', lists:reverse(OtherMatch)};
{{lists:reverse(IndexMatch), lists:reverse(OtherMatch)}, true};
condition([NIndex | OtherIndex], ['+' | OtherTokens], N, IndexMatch, OtherMatch) when
NIndex =:= N
->
Expand Down
13 changes: 8 additions & 5 deletions apps/emqx_retainer/src/emqx_retainer_mnesia.erl
View file
Open in desktop
Expand Up @@ -330,15 +330,15 @@ search_table(Tokens, Now) ->
search_table(undefined, Tokens, Now) ->
Ms = make_message_match_spec(Tokens, Now),
ets:table(?TAB_MESSAGE, [{traverse, {select, Ms}}]);
search_table(Index, Tokens, Now) ->
Ms = make_index_match_spec(Index, Tokens, Now),
search_table(Index, FilterTokens, Now) ->
{Ms, IsExactMs} = make_index_match_spec(Index, FilterTokens, Now),
Topics = [
emqx_retainer_index:restore_topic(Key)
|| #retained_index{key = Key} <- ets:select(?TAB_INDEX, Ms)
],
RetainedMsgQH = qlc:q([
ets:lookup(?TAB_MESSAGE, TopicTokens)
|| TopicTokens <- Topics
|| TopicTokens <- Topics, match(IsExactMs, TopicTokens, FilterTokens)
]),
qlc:q([
RetainedMsg
Expand All @@ -350,6 +350,9 @@ search_table(Index, Tokens, Now) ->
(ExpiryTime == 0) or (ExpiryTime > Now)
]).

match(_IsExactMs = true, _TopicTokens, _FilterTokens) -> true;
match(_IsExactMs = false, TopicTokens, FilterTokens) -> emqx_topic:match(TopicTokens, FilterTokens).

clear_batch(Indices, QC) ->
{Result, Rows} = qlc_next_answers(QC, ?CLEAR_BATCH_SIZE),
lists:foreach(
Expand Down Expand Up @@ -423,9 +426,9 @@ make_message_match_spec(Tokens, NowMs) ->
[{MsHd, [{'orelse', {'=:=', '$3', 0}, {'>', '$3', NowMs}}], ['$_']}].

make_index_match_spec(Index, Tokens, NowMs) ->
Cond = emqx_retainer_index:condition(Index, Tokens),
{Cond, IsExact} = emqx_retainer_index:condition(Index, Tokens),
MsHd = #retained_index{key = Cond, expiry_time = '$3'},
[{MsHd, [{'orelse', {'=:=', '$3', 0}, {'>', '$3', NowMs}}], ['$_']}].
{[{MsHd, [{'orelse', {'=:=', '$3', 0}, {'>', '$3', NowMs}}], ['$_']}], IsExact}.

is_table_full() ->
Limit = emqx:get_config([retainer, backend, max_retained_messages]),
Expand Down