Merge pull request #9440 from zmstone/1128-chore-sync-release-branch-…

…back-to-master

1128 chore sync release branch back to master

.github/workflows/release.yaml

@@ -90,12 +90,3 @@ jobs:
                 -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ github.ref_name }}\"}}" \
                 "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
           fi
-      - uses: emqx/push-helm-action@v1
-        if: github.event_name == 'release' || inputs.publish_release_artefacts
-        with:
-          charts_dir: "${{ github.workspace }}/deploy/charts/${{ steps.profile.outputs.profile }}"
-          version: ${{ steps.profile.outputs.version }}
-          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws_region: "us-west-2"
-          aws_bucket_name: "repos-emqx-io"

.github/workflows/upload-helm-charts.yaml

@@ -0,0 +1,52 @@
+name: Upload helm charts
+on:
+  release:
+    types:
+      - published
+  workflow_dispatch:
+    inputs:
+      tag:
+        type: string
+        required: true
+
+jobs:
+  upload:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v1-node16
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.inputs.tag }}
+      - name: Detect profile
+        id: profile
+        run: |
+          if git describe --tags --match '[v|e]*' --exact; then
+            REF=$(git describe --tags --match '[v|e]*' --exact)
+          else
+            echo "Only release tags matching '[v|e]*' are supported"
+            exit 1
+          fi
+          case "$REF" in
+            v*)
+              echo "profile=emqx" >> $GITHUB_OUTPUT
+              echo "version=$(./pkg-vsn.sh emqx)" >> $GITHUB_OUTPUT
+              ;;
+            e*)
+              echo "profile=emqx-enterprise" >> $GITHUB_OUTPUT
+              echo "version=$(./pkg-vsn.sh emqx-enterprise)" >> $GITHUB_OUTPUT
+              ;;
+          esac
+      - uses: emqx/push-helm-action@v1
+        with:
+          charts_dir: "${{ github.workspace }}/deploy/charts/${{ steps.profile.outputs.profile }}"
+          version: ${{ steps.profile.outputs.version }}
+          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_region: "us-west-2"
+          aws_bucket_name: "repos-emqx-io"

Makefile

@@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d
 export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
-export EMQX_DASHBOARD_VERSION ?= v1.1.1
+export EMQX_DASHBOARD_VERSION ?= v1.1.2
 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.0
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1

apps/emqx/i18n/emqx_schema_i18n.conf

@@ -2045,12 +2045,18 @@ Type of the rate limit.
 base_listener_enable_authn {
     desc {
         en: """
-Set <code>true</code> (default) to enable client authentication on this listener.
-When set to <code>false</code> clients will be allowed to connect without authentication.
+Set <code>true</code> (default) to enable client authentication on this listener, the authentication
+process goes through the configured authentication chain.
+When set to <code>false</code> to allow any clients with or without authentication information such as username or password to log in.
+When set to <code>quick_deny_anonymous<code>, it behaves like when set to <code>true</code> but clients will be
+denied immediately without going through any authenticators if <code>username</code> is not provided. This is useful to fence off
+anonymous clients early.
 """
         zh: """
-配置 <code>true</code> （默认值）启用客户端进行身份认证。
-配置 <code>false</code> 时，将不对客户端做任何认证。
+配置 <code>true</code> （默认值）启用客户端进行身份认证，通过检查认配置的认认证器链来决定是否允许接入。
+配置 <code>false</code> 时，将不对客户端做任何认证，任何客户端，不论是不是携带用户名等认证信息，都可以接入。
+配置 <code>quick_deny_anonymous</code> 时，行为跟 <code>true</code> 类似，但是会对匿名
+客户直接拒绝，不做使用任何认证器对客户端进行身份检查。
 """
     }
     label: {

apps/emqx/include/emqx_release.hrl

@@ -32,7 +32,7 @@
 %% `apps/emqx/src/bpapi/README.md'
 
 %% Community edition
--define(EMQX_RELEASE_CE, "5.0.10").
+-define(EMQX_RELEASE_CE, "5.0.11").
 
 %% Enterprise edition
 -define(EMQX_RELEASE_EE, "5.0.0-alpha.1").

apps/emqx/src/emqx.app.src

@@ -3,7 +3,7 @@
     {id, "emqx"},
     {description, "EMQX Core"},
     % strict semver, bump manually!
-    {vsn, "5.0.11"},
+    {vsn, "5.0.12"},
     {modules, []},
     {registered, []},
     {applications, [

apps/emqx/src/emqx_access_control.erl

@@ -38,11 +38,22 @@
     | {ok, map(), binary()}
     | {continue, map()}
     | {continue, binary(), map()}
-    | {error, term()}.
+    | {error, not_authorized}.
 authenticate(Credential) ->
-    case run_hooks('client.authenticate', [Credential], {ok, #{is_superuser => false}}) of
+    %% pre-hook quick authentication or
+    %% if auth backend returning nothing but just 'ok'
+    %% it means it's not a superuser, or there is no way to tell.
+    NotSuperUser = #{is_superuser => false},
+    case emqx_authentication:pre_hook_authenticate(Credential) of
         ok ->
-            {ok, #{is_superuser => false}};
+            {ok, NotSuperUser};
+        continue ->
+            case run_hooks('client.authenticate', [Credential], {ok, #{is_superuser => false}}) of
+                ok ->
+                    {ok, NotSuperUser};
+                Other ->
+                    Other
+            end;
         Other ->
             Other
     end.

apps/emqx/src/emqx_alarm.erl

@@ -41,7 +41,8 @@
     delete_all_deactivated_alarms/0,
     get_alarms/0,
     get_alarms/1,
-    format/1
+    format/1,
+    format/2
 ]).
 
 %% gen_server callbacks
@@ -169,28 +170,31 @@ get_alarms(activated) ->
 get_alarms(deactivated) ->
     gen_server:call(?MODULE, {get_alarms, deactivated}).
 
-format(#activated_alarm{name = Name, message = Message, activate_at = At, details = Details}) ->
+format(Alarm) ->
+    format(node(), Alarm).
+
+format(Node, #activated_alarm{name = Name, message = Message, activate_at = At, details = Details}) ->
     Now = erlang:system_time(microsecond),
     %% mnesia db stored microsecond for high frequency alarm
     %% format for dashboard using millisecond
     #{
-        node => node(),
+        node => Node,
         name => Name,
         message => Message,
         %% to millisecond
         duration => (Now - At) div 1000,
         activate_at => to_rfc3339(At),
         details => Details
     };
-format(#deactivated_alarm{
+format(Node, #deactivated_alarm{
     name = Name,
     message = Message,
     activate_at = At,
     details = Details,
     deactivate_at = DAt
 }) ->
     #{
-        node => node(),
+        node => Node,
         name => Name,
         message => Message,
         %% to millisecond

apps/emqx/src/emqx_authentication.erl

@@ -29,9 +29,13 @@
 -include_lib("stdlib/include/ms_transform.hrl").
 
 -define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
+-define(IS_UNDEFINED(X), (X =:= undefined orelse X =:= <<>>)).
 
 %% The authentication entrypoint.
--export([authenticate/2]).
+-export([
+    pre_hook_authenticate/1,
+    authenticate/2
+]).
 
 %% Authenticator manager process start/stop
 -export([
@@ -221,10 +225,23 @@ when
 %%------------------------------------------------------------------------------
 %% Authenticate
 %%------------------------------------------------------------------------------
-
-authenticate(#{enable_authn := false}, _AuthResult) ->
+-spec pre_hook_authenticate(emqx_types:clientinfo()) ->
+    ok | continue | {error, not_authorized}.
+pre_hook_authenticate(#{enable_authn := false}) ->
     inc_authenticate_metric('authentication.success.anonymous'),
-    ?TRACE_RESULT("authentication_result", ignore, enable_authn_false);
+    ?TRACE_RESULT("authentication_result", ok, enable_authn_false);
+pre_hook_authenticate(#{enable_authn := quick_deny_anonymous} = Credential) ->
+    case maps:get(username, Credential, undefined) of
+        U when ?IS_UNDEFINED(U) ->
+            ?TRACE_RESULT(
+                "authentication_result", {error, not_authorized}, enable_authn_false
+            );
+        _ ->
+            continue
+    end;
+pre_hook_authenticate(_) ->
+    continue.
+
 authenticate(#{listener := Listener, protocol := Protocol} = Credential, _AuthResult) ->
     case get_authenticators(Listener, global_chain(Protocol)) of
         {ok, ChainName, Authenticators} ->

apps/emqx/src/emqx_cm.erl

@@ -650,8 +650,8 @@ init([]) ->
     TabOpts = [public, {write_concurrency, true}],
     ok = emqx_tables:new(?CHAN_TAB, [bag, {read_concurrency, true} | TabOpts]),
     ok = emqx_tables:new(?CHAN_CONN_TAB, [bag | TabOpts]),
-    ok = emqx_tables:new(?CHAN_INFO_TAB, [set, compressed | TabOpts]),
-    ok = emqx_tables:new(?CHAN_LIVE_TAB, [set, {write_concurrency, true} | TabOpts]),
+    ok = emqx_tables:new(?CHAN_INFO_TAB, [ordered_set, compressed | TabOpts]),
+    ok = emqx_tables:new(?CHAN_LIVE_TAB, [ordered_set, {write_concurrency, true} | TabOpts]),
     ok = emqx_stats:update_interval(chan_stats, fun ?MODULE:stats_fun/0),
     State = #{chan_pmon => emqx_pmon:new()},
     {ok, State}.

apps/emqx/src/emqx_map_lib.erl

@@ -23,6 +23,7 @@
     deep_force_put/3,
     deep_remove/2,
     deep_merge/2,
+    binary_key_map/1,
     safe_atom_key_map/1,
     unsafe_atom_key_map/1,
     jsonable_map/1,
@@ -153,6 +154,17 @@ deep_convert(Val, _, _Args) ->
 unsafe_atom_key_map(Map) ->
     covert_keys_to_atom(Map, fun(K) -> binary_to_atom(K, utf8) end).
 
+-spec binary_key_map(map()) -> map().
+binary_key_map(Map) ->
+    deep_convert(
+        Map,
+        fun
+            (K, V) when is_atom(K) -> {atom_to_binary(K, utf8), V};
+            (K, V) when is_binary(K) -> {K, V}
+        end,
+        []
+    ).
+
 -spec safe_atom_key_map(#{binary() | atom() => any()}) -> #{atom() => any()}.
 safe_atom_key_map(Map) ->
     covert_keys_to_atom(Map, fun(K) -> binary_to_existing_atom(K, utf8) end).

apps/emqx/src/emqx_schema.erl

@@ -1668,7 +1668,7 @@ base_listener(Bind) ->
             )},
         {"enable_authn",
             sc(
-                boolean(),
+                hoconsc:enum([true, false, quick_deny_anonymous]),
                 #{
                     desc => ?DESC(base_listener_enable_authn),
                     default => true

apps/emqx/test/emqx_access_control_SUITE.erl

@@ -37,7 +37,8 @@ init_per_testcase(_, Config) ->
     Config.
 
 end_per_testcase(_, _Config) ->
-    ok = emqx_hooks:del('client.authorize', {?MODULE, authz_stub}).
+    ok = emqx_hooks:del('client.authorize', {?MODULE, authz_stub}),
+    ok = emqx_hooks:del('client.authenticate', {?MODULE, quick_deny_anonymous_authn}).
 
 t_authenticate(_) ->
     ?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())).
@@ -60,13 +61,49 @@ t_delayed_authorize(_) ->
     ?assertEqual(deny, emqx_access_control:authorize(clientinfo(), Publish2, InvalidTopic)),
     ok.
 
+t_quick_deny_anonymous(_) ->
+    ok = emqx_hooks:put(
+        'client.authenticate',
+        {?MODULE, quick_deny_anonymous_authn, []},
+        ?HP_AUTHN
+    ),
+
+    RawClient0 = clientinfo(),
+    RawClient = RawClient0#{username => undefined},
+
+    %% No name, No authn
+    Client1 = RawClient#{enable_authn => false},
+    ?assertMatch({ok, _}, emqx_access_control:authenticate(Client1)),
+
+    %% No name, With quick_deny_anonymous
+    Client2 = RawClient#{enable_authn => quick_deny_anonymous},
+    ?assertMatch({error, _}, emqx_access_control:authenticate(Client2)),
+
+    %% Bad name, With quick_deny_anonymous
+    Client3 = RawClient#{enable_authn => quick_deny_anonymous, username => <<"badname">>},
+    ?assertMatch({error, _}, emqx_access_control:authenticate(Client3)),
+
+    %% Good name, With quick_deny_anonymous
+    Client4 = RawClient#{enable_authn => quick_deny_anonymous, username => <<"goodname">>},
+    ?assertMatch({ok, _}, emqx_access_control:authenticate(Client4)),
+
+    %% Name, With authn
+    Client5 = RawClient#{enable_authn => true, username => <<"badname">>},
+    ?assertMatch({error, _}, emqx_access_control:authenticate(Client5)),
+    ok.
+
 %%--------------------------------------------------------------------
 %% Helper functions
 %%--------------------------------------------------------------------
 
 authz_stub(_Client, _PubSub, ValidTopic, _DefaultResult, ValidTopic) -> {stop, #{result => allow}};
 authz_stub(_Client, _PubSub, _Topic, _DefaultResult, _ValidTopic) -> {stop, #{result => deny}}.
 
+quick_deny_anonymous_authn(#{username := <<"badname">>}, _AuthResult) ->
+    {stop, {error, not_authorized}};
+quick_deny_anonymous_authn(_ClientInfo, _AuthResult) ->
+    {stop, {ok, #{is_superuser => false}}}.
+
 clientinfo() -> clientinfo(#{}).
 clientinfo(InitProps) ->
     maps:merge(

apps/emqx/test/emqx_bpapi_static_checks.erl

@@ -62,9 +62,10 @@
 %% List of business-layer functions that are exempt from the checks:
 %% erlfmt-ignore
 -define(EXEMPTIONS,
-    "emqx_mgmt_api:do_query/6"  % Reason: legacy code. A fun and a QC query are
-                                % passed in the args, it's futile to try to statically
-                                % check it
+    % Reason: legacy code. A fun and a QC query are
+    % passed in the args, it's futile to try to statically
+    % check it
+    "emqx_mgmt_api:do_query/2, emqx_mgmt_api:collect_total_from_tail_nodes/3"
 ).
 
 -define(XREF, myxref).