Merge branch 'release-57' into sync-r57-m-20240430

.ci/docker-compose-file/docker-compose-kafka.yaml

@@ -18,7 +18,7 @@ services:
       - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret
   kdc:
     hostname: kdc.emqx.net
-    image:  ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04
+    image:  ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04
     container_name: kdc.emqx.net
     expose:
       - 88 # kdc

.ci/docker-compose-file/docker-compose.yaml

@@ -3,7 +3,7 @@ version: '3.9'
 services:
   erlang:
     container_name: erlang
-    image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04}
+    image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04}
     env_file:
       - credentials.env
       - conf.env

.github/workflows/_pr_entrypoint.yaml

@@ -20,15 +20,15 @@ permissions:
 jobs:
   sanity-checks:
     runs-on: ubuntu-22.04
-    container: "ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04"
+    container: "ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04"
     outputs:
       ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
       ct-host: ${{ steps.matrix.outputs.ct-host }}
       ct-docker: ${{ steps.matrix.outputs.ct-docker }}
       version-emqx: ${{ steps.matrix.outputs.version-emqx }}
       version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }}
-      builder: "ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04"
-      builder_vsn: "5.3-2"
+      builder: "ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04"
+      builder_vsn: "5.3-4"
       otp_vsn: "26.2.1-2"
       elixir_vsn: "1.15.7"
 
@@ -95,12 +95,12 @@ jobs:
           MATRIX="$(echo "${APPS}" | jq -c '
             [
               (.[] | select(.profile == "emqx") | . + {
-                builder: "5.3-2",
+                builder: "5.3-4",
                 otp: "26.2.1-2",
                 elixir: "1.15.7"
               }),
               (.[] | select(.profile == "emqx-enterprise") | . + {
-                builder: "5.3-2",
+                builder: "5.3-4",
                 otp: ["26.2.1-2"][],
                 elixir: "1.15.7"
               })

.github/workflows/_push-entrypoint.yaml

@@ -23,16 +23,16 @@ env:
 jobs:
   prepare:
     runs-on: ubuntu-22.04
-    container: 'ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04'
+    container: 'ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04'
     outputs:
       profile: ${{ steps.parse-git-ref.outputs.profile }}
       release: ${{ steps.parse-git-ref.outputs.release }}
       latest: ${{ steps.parse-git-ref.outputs.latest }}
       ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
       ct-host: ${{ steps.matrix.outputs.ct-host }}
       ct-docker: ${{ steps.matrix.outputs.ct-docker }}
-      builder: 'ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04'
-      builder_vsn: '5.3-2'
+      builder: 'ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04'
+      builder_vsn: '5.3-4'
       otp_vsn: '26.2.1-2'
       elixir_vsn: '1.15.7'
 
@@ -62,12 +62,12 @@ jobs:
           MATRIX="$(echo "${APPS}" | jq -c '
             [
               (.[] | select(.profile == "emqx") | . + {
-                builder: "5.3-2",
+                builder: "5.3-4",
                 otp: "26.2.1-2",
                 elixir: "1.15.7"
               }),
               (.[] | select(.profile == "emqx-enterprise") | . + {
-                builder: "5.3-2",
+                builder: "5.3-4",
                 otp: ["26.2.1-2"][],
                 elixir: "1.15.7"
               })

.github/workflows/build_and_push_docker_images.yaml

@@ -61,7 +61,7 @@ on:
       builder_vsn:
         required: false
         type: string
-        default: '5.3-2'
+        default: '5.3-4'
 
 permissions:
   contents: read

.github/workflows/build_packages.yaml

@@ -63,7 +63,7 @@ on:
       builder_vsn:
         required: false
         type: string
-        default: '5.3-2'
+        default: '5.3-4'
 
 permissions:
   contents: read

.github/workflows/build_packages_cron.yaml

@@ -23,8 +23,8 @@ jobs:
       fail-fast: false
       matrix:
         profile:
-          - ['emqx', 'master', '5.3-2:1.15.7-26.2.1-2']
-          - ['emqx-enterprise', 'release-56', '5.3-2:1.15.7-26.2.1-2']
+          - ['emqx', 'master', '5.3-4:1.15.7-26.2.1-2']
+          - ['emqx-enterprise', 'release-56', '5.3-4:1.15.7-26.2.1-2']
         os:
           - debian10
           - ubuntu22.04

.github/workflows/build_slim_packages.yaml

@@ -27,11 +27,11 @@ on:
       builder:
         required: false
         type: string
-        default: 'ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04'
+        default: 'ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04'
       builder_vsn:
         required: false
         type: string
-        default: '5.3-2'
+        default: '5.3-4'
       otp_vsn:
         required: false
         type: string

.github/workflows/codeql.yaml

@@ -17,7 +17,7 @@ jobs:
       actions: read
       security-events: write
     container:
-      image: ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04
+      image: ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04
 
     strategy:
       fail-fast: false

.github/workflows/performance_test.yaml

@@ -26,7 +26,7 @@ jobs:
   prepare:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'emqx'
-    container: ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu20.04
     outputs:
       BENCH_ID: ${{ steps.prepare.outputs.BENCH_ID }}
       PACKAGE_FILE: ${{ steps.package_file.outputs.PACKAGE_FILE }}

Makefile

@@ -7,7 +7,7 @@ REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
 export EMQX_RELUP ?= true
-export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-debian12
+export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12
 export EMQX_DEFAULT_RUNNER = public.ecr.aws/debian/debian:12-slim
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1
@@ -20,8 +20,8 @@ endif
 
 # Dashboard version
 # from https://github.com/emqx/emqx-dashboard5
-export EMQX_DASHBOARD_VERSION ?= v1.8.1
-export EMQX_EE_DASHBOARD_VERSION ?= e1.6.1
+export EMQX_DASHBOARD_VERSION ?= v1.9.0-beta.1
+export EMQX_EE_DASHBOARD_VERSION ?= e1.7.0-beta.1
 
 PROFILE ?= emqx
 REL_PROFILES := emqx emqx-enterprise

README-CN.md

@@ -86,7 +86,7 @@ EMQX Cloud 文档：[docs.emqx.com/zh/cloud/latest/](https://docs.emqx.com/zh/cl
 
 `master` 分支是最新的 5 版本，`main-v4.4` 是 4.4 版本。
 
-EMQX 4.4 版本需要 OTP 24；5 版本则可以使用 OTP 24 和 25 构建。
+EMQX 4.4 版本需要 OTP 24；5 版本则可以使用 OTP 25 和 26 构建。
 
 ```bash
 git clone https://github.com/emqx/emqx.git

README.md

@@ -98,7 +98,7 @@ The `master` branch tracks the latest version 5. For version 4.4 checkout the `m
 
 EMQX 4.4 requires OTP 24.
 EMQX 5.0 ~ 5.3 can be built with OTP 24 or 25.
-EMQX 5.4 and newer can be built with OTP 24 or 25.
+EMQX 5.4 and newer can be built with OTP 25 or 26.
 
 ```bash
 git clone https://github.com/emqx/emqx.git

apps/emqx/include/emqx_release.hrl

@@ -32,7 +32,7 @@
 %% `apps/emqx/src/bpapi/README.md'
 
 %% Opensource edition
--define(EMQX_RELEASE_CE, "5.6.1").
+-define(EMQX_RELEASE_CE, "5.7.0-alpha.1").
 
 %% Enterprise edition
--define(EMQX_RELEASE_EE, "5.6.1").
+-define(EMQX_RELEASE_EE, "5.7.0-alpha.1").

apps/emqx/include/emqx_trace.hrl

@@ -38,4 +38,10 @@
 -define(SHARD, ?COMMON_SHARD).
 -define(MAX_SIZE, 30).
 
+-define(EMQX_TRACE_STOP_ACTION(REASON),
+    {unrecoverable_error, {action_stopped_after_template_rendering, REASON}}
+).
+
+-define(EMQX_TRACE_STOP_ACTION_MATCH, ?EMQX_TRACE_STOP_ACTION(_)).
+
 -endif.

apps/emqx/priv/bpapi.versions

@@ -46,6 +46,7 @@
 {emqx_metrics,2}.
 {emqx_mgmt_api_plugins,1}.
 {emqx_mgmt_api_plugins,2}.
+{emqx_mgmt_api_plugins,3}.
 {emqx_mgmt_cluster,1}.
 {emqx_mgmt_cluster,2}.
 {emqx_mgmt_cluster,3}.

apps/emqx/rebar.config

@@ -30,7 +30,7 @@
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.11.2"}}},
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.19.3"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}},
-    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.42.1"}}},
+    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.42.2"}}},
     {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
     {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},

apps/emqx/src/emqx_channel.erl

@@ -1636,8 +1636,17 @@ maybe_set_client_initial_attrs(ConnPkt, #{zone := Zone} = ClientInfo) ->
 initialize_client_attrs(Inits, ClientInfo) ->
     lists:foldl(
         fun(#{expression := Variform, set_as_attr := Name}, Acc) ->
-            Attrs = maps:get(client_attrs, ClientInfo, #{}),
+            Attrs = maps:get(client_attrs, Acc, #{}),
             case emqx_variform:render(Variform, ClientInfo) of
+                {ok, <<>>} ->
+                    ?SLOG(
+                        debug,
+                        #{
+                            msg => "client_attr_rednered_to_empty_string",
+                            set_as_attr => Name
+                        }
+                    ),
+                    Acc;
                 {ok, Value} ->
                     ?SLOG(
                         debug,

apps/emqx/src/emqx_logger_jsonfmt.erl

@@ -32,7 +32,7 @@
 -export([format/2]).
 
 %% For CLI HTTP API outputs
--export([best_effort_json/1, best_effort_json/2]).
+-export([best_effort_json/1, best_effort_json/2, best_effort_json_obj/1]).
 
 -ifdef(TEST).
 -include_lib("proper/include/proper.hrl").
@@ -65,10 +65,13 @@
 best_effort_json(Input) ->
     best_effort_json(Input, [pretty, force_utf8]).
 best_effort_json(Input, Opts) ->
-    Config = #{depth => unlimited, single_line => true, chars_limit => unlimited},
-    JsonReady = best_effort_json_obj(Input, Config),
+    JsonReady = best_effort_json_obj(Input),
     emqx_utils_json:encode(JsonReady, Opts).
 
+best_effort_json_obj(Input) ->
+    Config = #{depth => unlimited, single_line => true, chars_limit => unlimited},
+    best_effort_json_obj(Input, Config).
+
 -spec format(logger:log_event(), config()) -> iodata().
 format(#{level := Level, msg := Msg, meta := Meta}, Config0) when is_map(Config0) ->
     Config = add_default_config(Config0),

apps/emqx/src/emqx_schema.erl

@@ -217,14 +217,7 @@ roots(high) ->
                     importance => ?IMPORTANCE_MEDIUM
                 }
             )},
-        {zones,
-            sc(
-                map(name, ref("zone")),
-                #{
-                    desc => ?DESC(zones),
-                    importance => ?IMPORTANCE_HIDDEN
-                }
-            )}
+        {zones, zones_field_schema()}
     ] ++
         emqx_schema_hooks:injection_point(
             'roots.high',
@@ -1859,7 +1852,7 @@ base_listener(Bind) ->
                 #{
                     desc => ?DESC(base_listener_zone),
                     default => 'default',
-                    importance => ?IMPORTANCE_HIDDEN
+                    importance => ?IMPORTANCE_LOW
                 }
             )},
         {"limiter",
@@ -1883,6 +1876,22 @@ base_listener(Bind) ->
             )}
     ] ++ emqx_limiter_schema:short_paths_fields().
 
+%% @hidden Starting from 5.7, listeners.{TYPE}.{NAME}.zone is no longer hidden
+%% However, the root key 'zones' is still hidden because the fields' schema
+%% just repeat other root field's schema, which makes the dumped schema doc
+%% unnecessarily bloated.
+%%
+%% zone schema is documented here since 5.7:
+%% https://docs.emqx.com/en/enterprise/latest/configuration/configuration.html
+zones_field_schema() ->
+    sc(
+        map(name, ref("zone")),
+        #{
+            desc => ?DESC(zones),
+            importance => ?IMPORTANCE_HIDDEN
+        }
+    ).
+
 desc("persistent_session_store") ->
     "Settings for message persistence.";
 desc("persistent_session_builtin") ->
@@ -3712,7 +3721,7 @@ default_mem_check_interval() ->
 
 description_schema() ->
     sc(
-        string(),
+        binary(),
         #{
             default => <<"">>,
             desc => ?DESC(description),