From 7de9111203c4cfc0e34038914e88e12ae283d720 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:56:41 +0100 Subject: [PATCH] fix(ds): Fix schema checker warnings for DS schema --- apps/emqx/src/emqx_ds_schema.erl | 56 ++++++++++++++++++------------- apps/emqx/src/emqx_schema.erl | 2 ++ changes/ce/fix-12562.en.md | 3 ++ rel/i18n/emqx_ds_schema.hocon | 40 ++++++++++++++-------- scripts/spellcheck/dicts/emqx.txt | 2 ++ 5 files changed, 66 insertions(+), 37 deletions(-) create mode 100644 changes/ce/fix-12562.en.md diff --git a/apps/emqx/src/emqx_ds_schema.erl b/apps/emqx/src/emqx_ds_schema.erl index b1aff4a901..80170ba508 100644 --- a/apps/emqx/src/emqx_ds_schema.erl +++ b/apps/emqx/src/emqx_ds_schema.erl @@ -76,12 +76,12 @@ schema() -> [ {messages, ds_schema(#{ - desc => ?DESC(messages), - importance => ?IMPORTANCE_MEDIUM, default => #{ <<"backend">> => builtin - } + }, + importance => ?IMPORTANCE_MEDIUM, + desc => ?DESC(messages) })} ]. @@ -92,38 +92,38 @@ fields(builtin) -> sc( builtin, #{ - importance => ?IMPORTANCE_MEDIUM, 'readOnly' => true, default => builtin, - desc => ?DESC(builtin) + importance => ?IMPORTANCE_MEDIUM, + desc => ?DESC(builtin_backend) } )}, {'_config_handler', sc( {module(), atom()}, #{ - importance => ?IMPORTANCE_HIDDEN, 'readOnly' => true, - default => {?MODULE, translate_builtin} + default => {?MODULE, translate_builtin}, + importance => ?IMPORTANCE_HIDDEN } )}, {data_dir, sc( string(), #{ - desc => ?DESC(builtin_data_dir), mapping => "emqx_durable_storage.db_data_dir", required => false, - importance => ?IMPORTANCE_MEDIUM + importance => ?IMPORTANCE_MEDIUM, + desc => ?DESC(builtin_data_dir) } )}, {n_shards, sc( pos_integer(), #{ + default => 16, importance => ?IMPORTANCE_MEDIUM, - desc => ?DESC(builtin_n_shards), - default => 16 + desc => ?DESC(builtin_n_shards) } )}, {replication_factor, @@ -134,22 +134,23 @@ fields(builtin) -> importance => ?IMPORTANCE_HIDDEN } )}, - {egress, + {local_write_buffer, sc( - ref(builtin_egress), + ref(builtin_local_write_buffer), #{ - desc => ?DESC(builtin_egress), - importance => ?IMPORTANCE_MEDIUM + importance => ?IMPORTANCE_MEDIUM, + desc => ?DESC(builtin_local_write_buffer) } )}, {layout, sc( hoconsc:union([ - ref(layout_builtin_wildcard_optimized), ref(layout_builtin_reference) + ref(layout_builtin_wildcard_optimized) + % , ref(layout_builtin_reference) ]), #{ desc => ?DESC(builtin_layout), - importance => ?IMPORTANCE_HIDDEN, + importance => ?IMPORTANCE_MEDIUM, default => #{ <<"type">> => wildcard_optimized @@ -157,7 +158,7 @@ fields(builtin) -> } )} ]; -fields(builtin_egress) -> +fields(builtin_local_write_buffer) -> [ {max_items, sc( @@ -166,7 +167,7 @@ fields(builtin_egress) -> default => 1000, mapping => "emqx_durable_storage.egress_batch_size", importance => ?IMPORTANCE_MEDIUM, - desc => ?DESC(egress_max_items) + desc => ?DESC(builtin_local_write_buffer_max_items) } )}, {flush_interval, @@ -176,7 +177,7 @@ fields(builtin_egress) -> default => 100, mapping => "emqx_durable_storage.egress_flush_interval", importance => ?IMPORTANCE_MEDIUM, - desc => ?DESC(egress_flush_interval) + desc => ?DESC(builtin_local_write_buffer_flush_interval) } )} ]; @@ -186,9 +187,9 @@ fields(layout_builtin_wildcard_optimized) -> sc( wildcard_optimized, #{ - desc => ?DESC(layout_wildcard_optimized), 'readOnly' => true, - default => wildcard_optimized + default => wildcard_optimized, + desc => ?DESC(layout_builtin_wildcard_optimized_type) } )}, {bits_per_topic_level, @@ -222,10 +223,19 @@ fields(layout_builtin_reference) -> {type, sc( reference, - #{'readOnly' => true} + #{ + 'readOnly' => true, + importance => ?IMPORTANCE_HIDDEN + } )} ]. +desc(builtin) -> + ?DESC(builtin); +desc(builtin_local_write_buffer) -> + ?DESC(builtin_local_write_buffer); +desc(layout_builtin_wildcard_optimized) -> + ?DESC(layout_builtin_wildcard_optimized); desc(_) -> undefined. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 5d2459a81b..ccf91943de 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1985,6 +1985,8 @@ desc("crl_cache") -> "Global CRL cache options."; desc("session_persistence") -> "Settings governing durable sessions persistence."; +desc("durable_storage") -> + ?DESC(durable_storage); desc(_) -> undefined. diff --git a/changes/ce/fix-12562.en.md b/changes/ce/fix-12562.en.md new file mode 100644 index 0000000000..af16d1cc38 --- /dev/null +++ b/changes/ce/fix-12562.en.md @@ -0,0 +1,3 @@ +Add a new configuration root: `durable_storage`. + +This configuration tree contains the settings related to the new persistent session feature. diff --git a/rel/i18n/emqx_ds_schema.hocon b/rel/i18n/emqx_ds_schema.hocon index 7468746206..b283b6ab56 100644 --- a/rel/i18n/emqx_ds_schema.hocon +++ b/rel/i18n/emqx_ds_schema.hocon @@ -10,23 +10,28 @@ builtin.desc: """~ Builtin session storage backend utilizing embedded RocksDB key-value store.~""" +builtin_backend.label: "Backend type" +builtin_backend.desc: + """~ + Built-in backend.~""" + builtin_data_dir.label: "Database location" builtin_data_dir.desc: """~ File system directory where the database is located. - By default it is equal to `node.data_dir`.~""" + By default, it is equal to `node.data_dir`.~""" builtin_n_shards.label: "Number of shards" builtin_n_shards.desc: """~ - The builtin durable storage partitions data into shards. + The built-in durable storage partitions data into shards. This configuration parameter defines the number of shards. Please note that it takes effect only during the initialization of the durable storage database. Changing this configuration parameter after the database has been already created won't take any effect.~""" -builtin_egress.label: "Egress configuration" -builtin_egress.desc: +builtin_local_write_buffer.label: "Builtin local write buffer" +builtin_local_write_buffer.desc: """~ Configuration related to the buffering of messages from the local node to the shard leader.~""" @@ -37,8 +42,8 @@ builtin_layout.desc: Depending on the type of workload and the topic structure, different types of strategies for storing the data can be employed to maximize efficiency of reading messages from the durable storage.~""" -layout_wildcard_optimized.label: "Wildcard-optimized storage layout" -layout_wildcard_optimized.desc: +layout_builtin_wildcard_optimized.label: "Wildcard-optimized storage layout" +layout_builtin_wildcard_optimized.desc: """~ _Wildcard-optimized_ layout is designed to maximize the throughput of wildcard subscriptions covering large numbers of topics. @@ -47,11 +52,16 @@ layout_wildcard_optimized.desc: This layout is efficient for non-wildcard subscriptions as well.~""" +layout_builtin_wildcard_optimized_type.label: "Layout type" +layout_builtin_wildcard_optimized_type.desc: + """~ + Wildcard-optimized layout type.~""" + wildcard_optimized_epoch_bits.label: "Epoch bits" wildcard_optimized_epoch_bits.desc: """~ Wildcard-optimized layout partitions messages recorded at different times into "epochs". - Reading messages from a single epoch can be done very efficiently, so larger epochs improve the throughput of subscribers, but may increase end-to-end latency + Reading messages from a single epoch can be done very efficiently, so larger epochs improve the throughput of subscribers, but may increase end-to-end latency. Time span covered by each epoch grows exponentially with the value of `epoch_bits`: @@ -60,17 +70,19 @@ wildcard_optimized_epoch_bits.desc: ... - `epoch_bits = 10`: 1024 milliseconds - `epoch_bits = 13`: ~8 seconds - ....~""" + ...~""" -egress_max_items.label: "Max items" -egress_max_items.desc: +builtin_local_write_buffer_max_items.label: "Max items" +builtin_local_write_buffer_max_items.desc: """~ - This configuration parameter defines maximum number of buffered messages stored in the egress buffer.~""" + This configuration parameter defines maximum number of messages stored in the local write buffer.~""" -egress_flush_interval.label: "Flush interval" -egress_flush_interval.desc: +builtin_local_write_buffer_flush_interval.label: "Flush interval" +builtin_local_write_buffer_flush_interval.desc: """~ Maximum linger time for the buffered messages. - Egress buffer will be flushed _at least_ as often as `flush_interval`.~""" + Local write buffer will be flushed _at least_ as often as `flush_interval`. + + Larger values of `flush_interval` may lead to higher throughput and better overall performance, but may increase end-to-end latency.~""" } diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index c2f5f54ef7..bb8bb397aa 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -284,9 +284,11 @@ TDengine clickhouse FormatType RocketMQ +RocksDB Keyspace OpenTSDB saml +storages idp ocpp OCPP