Handle node-specific options correctly in reload_cluster

doc/developers-guide/flat_options.md

@@ -0,0 +1,215 @@
+# Flat options format
+
+`'FLAT'` means that the value was separated into more options. For example,
+module options are separated into a list of `module_opt`-s.
+
+Each flat option key starts with an option type:
+
+- `l` - local options
+- `h` - local host options
+- `g` - global options
+
+# Flat options example
+
+```erlang
+{[l,listen],'FLAT'}.
+{[l,listener,{5280,{0,0,0,0},tcp},ejabberd_cowboy],'FLAT'}.
+{[l,listener_opt,{5280,{0,0,0,0},tcp},ejabberd_cowboy,num_acceptors],10}.
+{[l,listener_opt,{5280,{0,0,0,0},tcp},ejabberd_cowboy,transport_options],
+ [{max_connections,1024}]}.
+{[l,listener_opt,{5280,{0,0,0,0},tcp},ejabberd_cowboy,modules],
+ [{"_","/http-bind",mod_bosh},
+  {"_","/ws-xmpp",mod_websockets,
+   [{ejabberd_service,[{access,all},
+                       {shaper_rule,fast},
+                       {ip,{127,0,0,1}},
+                       {password,"secret"}]}]}]}.
+{[l,listener,{5285,{0,0,0,0},tcp},ejabberd_cowboy],'FLAT'}.
+{[l,listener_opt,{5285,{0,0,0,0},tcp},ejabberd_cowboy,num_acceptors],10}.
+{[l,listener_opt,{5285,{0,0,0,0},tcp},ejabberd_cowboy,transport_options],
+ [{max_connections,1024}]}.
+{[l,listener_opt,{5285,{0,0,0,0},tcp},ejabberd_cowboy,ssl],
+ [{certfile,"priv/ssl/fake_cert.pem"},
+  {keyfile,"priv/ssl/fake_key.pem"},
+  {password,[]}]}.
+{[l,listener_opt,{5285,{0,0,0,0},tcp},ejabberd_cowboy,modules],
+ [{"_","/http-bind",mod_bosh},{"_","/ws-xmpp",mod_websockets,[]}]}.
+{[l,listener,{8088,{127,0,0,1},tcp},ejabberd_cowboy],'FLAT'}.
+{[l,listener_opt,{8088,{127,0,0,1},tcp},ejabberd_cowboy,num_acceptors],10}.
+{[l,listener_opt,{8088,{127,0,0,1},tcp},ejabberd_cowboy,transport_options],
+ [{max_connections,1024}]}.
+{[l,listener_opt,{8088,{127,0,0,1},tcp},ejabberd_cowboy,modules],
+ [{"localhost","/api",mongoose_api_admin,[]}]}.
+{[l,listener,{8089,{0,0,0,0},tcp},ejabberd_cowboy],'FLAT'}.
+{[l,listener_opt,{8089,{0,0,0,0},tcp},ejabberd_cowboy,num_acceptors],10}.
+{[l,listener_opt,{8089,{0,0,0,0},tcp},ejabberd_cowboy,transport_options],
+ [{max_connections,1024}]}.
+{[l,listener_opt,{8089,{0,0,0,0},tcp},ejabberd_cowboy,protocol_options],
+ [{compress,true}]}.
+{[l,listener_opt,{8089,{0,0,0,0},tcp},ejabberd_cowboy,ssl],
+ [{certfile,"priv/ssl/fake_cert.pem"},
+  {keyfile,"priv/ssl/fake_key.pem"},
+  {password,[]}]}.
+{[l,listener_opt,{8089,{0,0,0,0},tcp},ejabberd_cowboy,modules],
+ [{"_","/api/sse",lasse_handler,[mongoose_client_api_sse]},
+  {"_","/api/messages/[:with]",mongoose_client_api_messages,[]},
+  {"_","/api/contacts/[:jid]",mongoose_client_api_contacts,[]},
+  {"_","/api/rooms/[:id]",mongoose_client_api_rooms,[]},
+  {"_","/api/rooms/:id/users/[:user]",mongoose_client_api_rooms_users,[]},
+  {"_","/api/rooms/[:id]/messages",mongoose_client_api_rooms_messages,[]}]}.
+{[l,listener,{5288,{127,0,0,1},tcp},ejabberd_cowboy],'FLAT'}.
+{[l,listener_opt,{5288,{127,0,0,1},tcp},ejabberd_cowboy,num_acceptors],10}.
+{[l,listener_opt,{5288,{127,0,0,1},tcp},ejabberd_cowboy,transport_options],
+ [{max_connections,1024}]}.
+{[l,listener_opt,{5288,{127,0,0,1},tcp},ejabberd_cowboy,modules],
+ [{"localhost","/api",mongoose_api,
+   [{handlers,[mongoose_api_metrics,mongoose_api_users]}]}]}.
+{[l,listener,{5222,{0,0,0,0},tcp},ejabberd_c2s],'FLAT'}.
+{[l,listener_opt,{5222,{0,0,0,0},tcp},ejabberd_c2s,certfile],
+ "priv/ssl/fake_server.pem"}.
+{[l,listener_simple_opt,{5222,{0,0,0,0},tcp},ejabberd_c2s,starttls],simple}.
+{[l,listener_opt,{5222,{0,0,0,0},tcp},ejabberd_c2s,zlib],10000}.
+{[l,listener_opt,{5222,{0,0,0,0},tcp},ejabberd_c2s,access],c2s}.
+{[l,listener_opt,{5222,{0,0,0,0},tcp},ejabberd_c2s,shaper],c2s_shaper}.
+{[l,listener_opt,{5222,{0,0,0,0},tcp},ejabberd_c2s,max_stanza_size],65536}.
+{[l,listener_opt,{5222,{0,0,0,0},tcp},ejabberd_c2s,protocol_options],
+ ["no_sslv3"]}.
+{[l,listener_opt,{5222,{0,0,0,0},tcp},ejabberd_c2s,dhfile],
+ "priv/ssl/fake_dh_server.pem"}.
+{[l,listener,{5223,{0,0,0,0},tcp},ejabberd_c2s],'FLAT'}.
+{[l,listener_opt,{5223,{0,0,0,0},tcp},ejabberd_c2s,zlib],4096}.
+{[l,listener_opt,{5223,{0,0,0,0},tcp},ejabberd_c2s,access],c2s}.
+{[l,listener_opt,{5223,{0,0,0,0},tcp},ejabberd_c2s,shaper],c2s_shaper}.
+{[l,listener_opt,{5223,{0,0,0,0},tcp},ejabberd_c2s,max_stanza_size],65536}.
+{[l,listener,{5269,{0,0,0,0},tcp},ejabberd_s2s_in],'FLAT'}.
+{[l,listener_opt,{5269,{0,0,0,0},tcp},ejabberd_s2s_in,shaper],s2s_shaper}.
+{[l,listener_opt,{5269,{0,0,0,0},tcp},ejabberd_s2s_in,max_stanza_size],131072}.
+{[l,listener_opt,{5269,{0,0,0,0},tcp},ejabberd_s2s_in,protocol_options],
+ ["no_sslv3"]}.
+{[l,listener_opt,{5269,{0,0,0,0},tcp},ejabberd_s2s_in,dhfile],
+ "priv/ssl/fake_dh_server.pem"}.
+{[l,listener,{8888,{127,0,0,1},tcp},ejabberd_service],'FLAT'}.
+{[l,listener_opt,{8888,{127,0,0,1},tcp},ejabberd_service,access],all}.
+{[l,listener_opt,{8888,{127,0,0,1},tcp},ejabberd_service,shaper_rule],fast}.
+{[l,listener_opt,{8888,{127,0,0,1},tcp},ejabberd_service,password],"secret"}.
+{[l,listener,{8189,{127,0,0,1},tcp},ejabberd_service],'FLAT'}.
+{[l,listener_opt,{8189,{127,0,0,1},tcp},ejabberd_service,access],all}.
+{[l,listener_opt,{8189,{127,0,0,1},tcp},ejabberd_service,hidden_components],
+ true}.
+{[l,listener_opt,{8189,{127,0,0,1},tcp},ejabberd_service,shaper_rule],fast}.
+{[l,listener_opt,{8189,{127,0,0,1},tcp},ejabberd_service,password],"secret"}.
+{[l,all_metrics_are_global],false}.
+{[l,s2s_certfile],"priv/ssl/fake_server.pem"}.
+{[l,node_start],{1530,15976,143119}}.
+{[l,odbc_pools],[]}.
+{[l,registration_timeout],infinity}.
+{[l,outgoing_s2s_port],5299}.
+{[l,services],
+ [{service_admin_extra,[{submods,[node,accounts,sessions,vcard,roster,last,
+                                  private,stanza,stats]}]}]}.
+{[l,max_fsm_queue],1000}.
+{[l,s2s_use_starttls],optional}.
+{[h,<<"anonymous.localhost">>,auth_method],anonymous}.
+{[h,<<"anonymous.localhost">>,s2s_default_policy],allow}.
+{[h,<<"localhost.bis">>,modules],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_carboncopy],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_stream_management],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_muc_commands],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_amp],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_offline],'FLAT'}.
+{[h,<<"localhost.bis">>,module_opt,mod_offline,access_max_user_messages],
+ max_user_offline_messages}.
+{[h,<<"localhost.bis">>,module,mod_last],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_roster],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_bosh],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_blocking],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_vcard],'FLAT'}.
+{[h,<<"localhost.bis">>,module_opt,mod_vcard,host],"vjud.@HOST@"}.
+{[h,<<"localhost.bis">>,module,mod_muc_light_commands],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_commands],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_disco],'FLAT'}.
+{[h,<<"localhost.bis">>,module_opt,mod_disco,users_can_see_hidden_services],
+ false}.
+{[h,<<"localhost.bis">>,module,mod_privacy],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_private],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_sic],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_adhoc],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_register],'FLAT'}.
+{[h,<<"localhost.bis">>,module_opt,mod_register,welcome_message],{[]}}.
+{[h,<<"localhost.bis">>,module_opt,mod_register,ip_access],'FLAT'}.
+{[h,<<"localhost.bis">>,module_subopt,mod_register,ip_access,allow],
+ "127.0.0.0/8"}.
+{[h,<<"localhost.bis">>,module_subopt,mod_register,ip_access,deny],
+ "0.0.0.0/0"}.
+{[h,<<"localhost.bis">>,module_opt,mod_register,access],register}.
+{[h,<<"localhost">>,modules],'FLAT'}.
+{[h,<<"localhost">>,module,mod_carboncopy],'FLAT'}.
+{[h,<<"localhost">>,module,mod_stream_management],'FLAT'}.
+{[h,<<"localhost">>,module,mod_muc_commands],'FLAT'}.
+{[h,<<"localhost">>,module,mod_amp],'FLAT'}.
+{[h,<<"localhost">>,module,mod_offline],'FLAT'}.
+{[h,<<"localhost">>,module_opt,mod_offline,access_max_user_messages],
+ max_user_offline_messages}.
+{[h,<<"localhost">>,module,mod_last],'FLAT'}.
+{[h,<<"localhost">>,module,mod_roster],'FLAT'}.
+{[h,<<"localhost">>,module,mod_bosh],'FLAT'}.
+{[h,<<"localhost">>,module,mod_blocking],'FLAT'}.
+{[h,<<"localhost">>,module,mod_vcard],'FLAT'}.
+{[h,<<"localhost">>,module_opt,mod_vcard,host],"vjud.@HOST@"}.
+{[h,<<"localhost">>,module,mod_muc_light_commands],'FLAT'}.
+{[h,<<"localhost">>,module,mod_commands],'FLAT'}.
+{[h,<<"localhost">>,module,mod_disco],'FLAT'}.
+{[h,<<"localhost">>,module_opt,mod_disco,users_can_see_hidden_services],false}.
+{[h,<<"localhost">>,module,mod_privacy],'FLAT'}.
+{[h,<<"localhost">>,module,mod_private],'FLAT'}.
+{[h,<<"localhost">>,module,mod_sic],'FLAT'}.
+{[h,<<"localhost">>,module,mod_adhoc],'FLAT'}.
+{[h,<<"localhost">>,module,mod_register],'FLAT'}.
+{[h,<<"localhost">>,module_opt,mod_register,welcome_message],{[]}}.
+{[h,<<"localhost">>,module_opt,mod_register,ip_access],'FLAT'}.
+{[h,<<"localhost">>,module_subopt,mod_register,ip_access,allow],"127.0.0.0/8"}.
+{[h,<<"localhost">>,module_subopt,mod_register,ip_access,deny],"0.0.0.0/0"}.
+{[h,<<"localhost">>,module_opt,mod_register,access],register}.
+{[h,<<"localhost">>,auth_method],internal}.
+{[h,<<"anonymous.localhost">>,auth_opts],[]}.
+{[h,<<"fed1">>,s2s_addr],{127,0,0,1}}.
+{[h,<<"localhost.bis">>,auth_opts],[]}.
+{[h,<<"localhost">>,auth_opts],[]}.
+{[h,<<"anonymous.localhost">>,modules],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module,mod_carboncopy],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module,mod_stream_management],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module,mod_muc_commands],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module,mod_amp],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module,mod_offline],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module_opt,mod_offline,access_max_user_messages],
+ max_user_offline_messages}.
+{[h,<<"anonymous.localhost">>,module,mod_last],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module,mod_roster],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module,mod_bosh],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module,mod_blocking],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module,mod_vcard],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module_opt,mod_vcard,host],"vjud.@HOST@"}.
+{[h,<<"anonymous.localhost">>,module,mod_muc_light_commands],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module,mod_commands],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module,mod_disco],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module_opt,mod_disco,
+  users_can_see_hidden_services],
+ false}.
+{[h,<<"anonymous.localhost">>,module,mod_privacy],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module,mod_private],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module,mod_sic],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module,mod_adhoc],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module,mod_register],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module_opt,mod_register,welcome_message],{[]}}.
+{[h,<<"anonymous.localhost">>,module_opt,mod_register,ip_access],'FLAT'}.
+{[h,<<"anonymous.localhost">>,module_subopt,mod_register,ip_access,allow],
+ "127.0.0.0/8"}.
+{[h,<<"anonymous.localhost">>,module_subopt,mod_register,ip_access,deny],
+ "0.0.0.0/0"}.
+{[h,<<"anonymous.localhost">>,module_opt,mod_register,access],register}.
+{[h,<<"anonymous.localhost">>,allow_multiple_connections],true}.
+{[h,<<"localhost.bis">>,auth_method],internal}.
+{[h,<<"localhost.bis">>,s2s_default_policy],allow}.
+{[h,<<"localhost">>,s2s_default_policy],allow}.
+{[h,<<"anonymous.localhost">>,anonymous_protocol],both}.
+```

doc/operation-and-maintenance/Reloading-configuration-on-a-running-system.md

@@ -1,19 +1,24 @@
 ### Reloading configuration on a running system 
 
-`mongooseimctl` subcommands `reload_cluster` and `reload_local` are now available. The syntax is:
+`mongooseimctl` subcommands for configuration reloading are:
 
 `mongooseimctl reload_local`
 
 `mongooseimctl reload_cluster`
 
+`mongooseimctl reload_cluster_dryrun`
+
 `reload_local` is unsafe as it reloads the configuration only on the local node.
 This might introduce inconsistencies between different nodes of the cluster.
 It's available as a safety mechanism for the rare case of a cluster-global reload failing.
 
-`reload_cluster` is generally safe. It will try to apply the configuration
-on all nodes of the cluster.
-The prerequisite is that the modified config file must be available on
-all nodes at the same location (the location where MongooseIM expects its config file).
+`reload_cluster` applies the configuration on all nodes of the cluster.
+The prerequisite is that the same version of config file must be available on
+all nodes. All nodes in a cluster must have the same config loaded into memory
+as well. There is a small exception from this rule, see Node specific options.
+
+`reload_cluster_dryrun` calculates and prints config changes,
+but does not applies them. Useful for debugging.
 
 ### Non-reloadable options
 Some options require restarting the server in order to be reloaded.
@@ -22,3 +27,147 @@ The following options' changes will be ignored when using `mongooseimctl` tool:
 * s2s_*
 * all_metrics_are_global
 * odbc_*
+
+
+### Node-specific options
+
+Very rarely we want different configs for each node in cluster.
+Than `reload_cluster` would detect configuration inconsistency and would not
+allow configuration updates.
+
+To tell `reload_cluster` to ignore such options, extra information should be
+provided in `ejabberd.cfg`.
+
+It's called `node_specific_options`.
+
+They are defined on top level of the configuration file using
+`node_specific_options` tuple. This tuple should be the same for all configs
+in cluster.
+
+`node_specific_options` contains a list of match patterns. If you are familiar
+with ETS tables or Mnesia tuple matching - it's the same thing.
+Match patterns are documented in
+[Erlang/OTP docs](http://erlang.org/doc/apps/erts/match_spec.html).
+
+The pattern mechanism is also documented in
+[Learn you some Erlang book](http://learnyousomeerlang.com/ets).
+
+Basically, it allows you to put `'_'` to match every possible host.
+
+`[h,'_',module_opt,mod_muc_log, outdir]` would match
+`[h,<<"localhost">>,module_opt,mod_muc_log, outdir]` and
+`[h,<<"any.other.host">>,module_opt,mod_muc_log, outdir]`.
+
+Example showing where to put `node_specific_options`.
+
+```erlang
+{hosts, ["localhost"]}.
+{node_specific_options, [
+    [h,'_',module_opt,mod_muc_log, outdir]
+ ]}.
+{modules, [....]}.
+```
+
+The `node_specific_options` patterns are matched against flat configuration
+options. To print your config in flat form, use the command with running
+node `mongooseimctl print_flat_config`.
+
+Example:
+
+```erlang
+_build/mim1/rel/mongooseim/bin/mongooseimctl print_flat_config
+Flat options:
+{[l,listen],'FLAT'}.
+{[l,listener,{5280,{0,0,0,0},tcp},ejabberd_cowboy],'FLAT'}.
+{[l,listener_opt,{5280,{0,0,0,0},tcp},ejabberd_cowboy,num_acceptors],10}.
+{[l,listener_opt,{5280,{0,0,0,0},tcp},ejabberd_cowboy,transport_options],
+ [{max_connections,1024}]}.
+...
+{[h,<<"anonymous.localhost">>,auth_method],anonymous}.
+{[h,<<"localhost.bis">>,modules],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_carboncopy],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_stream_management],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_muc_commands],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_amp],'FLAT'}.
+{[h,<<"localhost.bis">>,module,mod_offline],'FLAT'}.
+{[h,<<"localhost.bis">>,module_opt,mod_offline,access_max_user_messages],
+ max_user_offline_messages}.
+...
+```
+
+[More information about flat options format](../developers-guide/flat_options.md)
+
+#### Node-specific options for Global Distribution
+
+Node-specific options mechanism was designed to allow `mod_global_distrib`
+to be configured with different parameters for each node.
+
+Real life configuration example:
+
+```erlang
+{node_specific_options, [
+    [h,'_',module_opt,mod_global_distrib,endpoints],
+    [h,'_',module_opt,mod_global_distrib,advertised_endpoints],
+    [h,'_',module_subopt,mod_global_distrib,connections,endpoints],
+    [h,'_',module_subopt,mod_global_distrib,connections,advertised_endpoints],
+    [h,'_',module_subopt,mod_global_distrib,redis,server],
+    [h,'_',module_subopt,mod_global_distrib_bounce,connections,advertised_endpoints],
+    [h,'_',module_subopt,mod_global_distrib_bounce,connections,advertised_endpoints,'_'],
+    [h,'_',module_subopt,mod_global_distrib_bounce,connections,endpoints],
+    [h,'_',module_subopt,mod_global_distrib_bounce,connections,endpoints,'_'],
+    [h,'_',module_subopt,mod_global_distrib_disco,connections,advertised_endpoints],
+    [h,'_',module_subopt,mod_global_distrib_disco,connections,advertised_endpoints,'_'],
+    [h,'_',module_subopt,mod_global_distrib_hosts_refresher,connections,advertised_endpoints],
+    [h,'_',module_subopt,mod_global_distrib_hosts_refresher,connections,advertised_endpoints,'_'],
+    [h,'_',module_subopt,mod_global_distrib_hosts_refresher,connections,endpoints],
+    [h,'_',module_subopt,mod_global_distrib_hosts_refresher,connections,endpoints,'_'],
+    [h,'_',module_subopt,mod_global_distrib_hosts_refresher,redis,server],
+    [h,'_',module_subopt,mod_global_distrib_mapping,connections,advertised_endpoints],
+    [h,'_',module_subopt,mod_global_distrib_mapping,connections,advertised_endpoints,'_'],
+    [h,'_',module_subopt,mod_global_distrib_mapping,connections,endpoints],
+    [h,'_',module_subopt,mod_global_distrib_mapping,connections,endpoints,'_'],
+    [h,'_',module_subopt,mod_global_distrib_mapping,redis,server],
+    [h,'_',module_subopt,mod_global_distrib_receiver,advertised_endpoints],
+    [h,'_',module_subopt,mod_global_distrib_receiver,advertised_endpoints,'_'],
+    [h,'_',module_subopt,mod_global_distrib_receiver,connections,advertised_endpoints],
+    [h,'_',module_subopt,mod_global_distrib_receiver,connections,advertised_endpoints,'_'],
+    [h,'_',module_subopt,mod_global_distrib_receiver,connections,endpoints],
+    [h,'_',module_subopt,mod_global_distrib_receiver,connections,endpoints,'_'],
+    [h,'_',module_subopt,mod_global_distrib_receiver,endpoints],
+    [h,'_',module_subopt,mod_global_distrib_receiver,endpoints,'_'],
+    [h,'_',module_subopt,mod_global_distrib_receiver,redis,server],
+    [h,'_',module_subopt,mod_global_distrib_sender,advertised_endpoints],
+    [h,'_',module_subopt,mod_global_distrib_sender,advertised_endpoints,'_'],
+    [h,'_',module_subopt,mod_global_distrib_sender,connections,advertised_endpoints],
+    [h,'_',module_subopt,mod_global_distrib_sender,connections,advertised_endpoints,'_'],
+    [h,'_',module_subopt,mod_global_distrib_sender,connections,endpoints],
+    [h,'_',module_subopt,mod_global_distrib_sender,connections,endpoints,'_'],
+    [h,'_',module_subopt,mod_global_distrib_sender,endpoints],
+    [h,'_',module_subopt,mod_global_distrib_sender,endpoints,'_'],
+    [h,'_',module_subopt,mod_global_distrib_sender,redis,server]
+]}.
+```
+
+`'_'` means that any value can be there.
+
+Usually, all modules are configured using just one level of option nesting.
+`module_subopt` means that we are interested in a nested option.
+
+`node_specific_options` can be used with any module, not just
+`mod_global_distrib` (but usually you want all options to be the same on all
+nodes!).
+
+Any flat option can be used in `node_specific_options`.
+
+
+#### Node-specific modules
+
+We don't compare options of node-specific modules for configuration consistency
+check. We also don't check, if all nodes run these modules (it's fine to run
+them only on some nodes in a cluster).
+
+```erlang
+{node_specific_options, [
+    [h,'_',module,mod_global_distrib]
+]}.
+```