From df5f9e0803bbfef1ef55e6acac57e5cca3ea328b Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Mon, 1 Nov 2021 12:38:16 -0700 Subject: [PATCH 01/33] MB-48047:[BP] Introduce inaddr_any/2 Introduced here: http://review.couchbase.org/c/ns_server/+/137791 Change-Id: I558a031a89f3186ff038411d4d5dbf33a3a65f74 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164903 Well-Formed: Restriction Checker Tested-by: Abhijeeth Nuthan Reviewed-by: Steve Watanabe --- src/misc.erl | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/misc.erl b/src/misc.erl index ec805d930c..df8372cf15 100644 --- a/src/misc.erl +++ b/src/misc.erl @@ -1537,16 +1537,15 @@ inaddr_any() -> -spec inaddr_any([] | [url]) -> string(). inaddr_any(Options) -> - case is_ipv6() of - true -> - case Options of - [] -> - "::"; - [url] -> - "[::]" - end; - false -> - "0.0.0.0" + inaddr_any(get_net_family(), Options). + +inaddr_any(inet, _Options) -> "0.0.0.0"; +inaddr_any(inet6, Options) -> + case Options of + [] -> + "::"; + [url] -> + "[::]" end. -spec local_url(integer(), From 13bb6224e109ce9e3d6de1bc44af0671447a7af5 Mon Sep 17 00:00:00 2001 From: Sam Cramer Date: Wed, 17 Mar 2021 13:48:04 -0700 Subject: [PATCH 02/33] MB-48047:[BP] Add functions which determine the specific os type Reviewed-on: http://review.couchbase.org/c/ns_server/+/148877 Change-Id: Ib151bb1472dd37e7ddc3783acedfc58d4efb2721 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164904 Well-Formed: Restriction Checker Tested-by: Abhijeeth Nuthan Reviewed-by: Steve Watanabe --- src/misc.erl | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/misc.erl b/src/misc.erl index df8372cf15..1b94ad11b6 100644 --- a/src/misc.erl +++ b/src/misc.erl @@ -1702,6 +1702,32 @@ parse_cpuset_cpus_test() -> ?assertException(error, badarg, parse_cpuset_cpus("1,aa")). -endif. +%% What platform are we running on? + +is_linux() -> + case os:type() of + {unix, linux} -> + true; + {_, _} -> + false + end. + +is_macos() -> + case os:type() of + {unix, darwin} -> + true; + {_, _} -> + false + end. + +is_windows() -> + case os:type() of + {win32, _} -> + true; + {_, _} -> + false + end. + ensure_writable_dir(Path) -> filelib:ensure_dir(Path), case filelib:is_dir(Path) of From 95dde81b93dee8053c10cc7a18a77b129a1cc924 Mon Sep 17 00:00:00 2001 From: Timofey Barmin Date: Wed, 4 Mar 2020 12:35:29 -0800 Subject: [PATCH 03/33] MB-48047:[BP] Introduce misc:localhost/2 Reviewed-on: http://review.couchbase.org/123261 Change-Id: Icd7f5bc174890f9ead79a8b956238d317da30b06 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164905 Well-Formed: Restriction Checker Tested-by: Abhijeeth Nuthan Reviewed-by: Steve Watanabe --- src/misc.erl | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/misc.erl b/src/misc.erl index 1b94ad11b6..c1259c9eb1 100644 --- a/src/misc.erl +++ b/src/misc.erl @@ -1517,17 +1517,11 @@ localhost() -> -spec localhost([] | [url]) -> string(). localhost(Options) -> - case is_ipv6() of - true -> - case Options of - [] -> - "::1"; - [url] -> - "[::1]" - end; - false -> - "127.0.0.1" - end. + localhost(get_net_family(), Options). + +localhost(inet, _Options) -> "127.0.0.1"; +localhost(inet6, [url]) -> "[::1]"; +localhost(inet6, []) -> "::1". localhost_alias() -> "cb.local". From c973d0518fa385039f51785788ef87a8f486f94e Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Wed, 11 Nov 2020 16:37:52 -0800 Subject: [PATCH 04/33] MB-48047:[BP] Some refactor of webconfig function Reviewed-on: http://review.couchbase.org/c/ns_server/+/140071 Change-Id: I845a48d46dea7c33ec3498e14a0185fe6c73d357 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164906 Well-Formed: Restriction Checker Tested-by: Abhijeeth Nuthan Reviewed-by: Steve Watanabe --- src/menelaus_sup.erl | 3 +-- src/menelaus_web.erl | 17 ++++++++--------- src/menelaus_web_settings.erl | 4 ++-- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/menelaus_sup.erl b/src/menelaus_sup.erl index 87a72432b4..0a69dd8e57 100644 --- a/src/menelaus_sup.erl +++ b/src/menelaus_sup.erl @@ -41,8 +41,7 @@ %% @doc API for starting the supervisor. start_link() -> Result = supervisor:start_link({local, ?MODULE}, ?MODULE, []), - WConfig = menelaus_web:webconfig(), - Port = proplists:get_value(port, WConfig), + Port = menelaus_web:webconfig(port), case Result of {ok, _Pid} -> ?user_log(?START_OK, diff --git a/src/menelaus_web.erl b/src/menelaus_web.erl index fa588966f7..773ed5e846 100644 --- a/src/menelaus_web.erl +++ b/src/menelaus_web.erl @@ -103,7 +103,8 @@ http_server(Options) -> end end. -webconfig(Config) -> +webconfig() -> + Config = ns_config:get(), Ip = case os:getenv("MOCHIWEB_IP") of false -> misc:inaddr_any(); Any -> Any @@ -114,15 +115,13 @@ webconfig(Config) -> P -> list_to_integer(P) end, - WebConfig = [{ip, Ip}, - {port, Port}, - {nodelay, true}, - {approot, menelaus_deps:local_path(["priv","public"], - ?MODULE)}], - WebConfig. + [{ip, Ip}, + {port, Port}, + {nodelay, true}, + {approot, menelaus_deps:local_path(["priv","public"], ?MODULE)}]. -webconfig() -> - webconfig(ns_config:get()). +webconfig(Prop) -> + proplists:get_value(Prop, webconfig()). loop(Req, Config) -> ok = menelaus_sup:barrier_wait(), diff --git a/src/menelaus_web_settings.erl b/src/menelaus_web_settings.erl index 1e1990f566..a883865cb0 100644 --- a/src/menelaus_web_settings.erl +++ b/src/menelaus_web_settings.erl @@ -534,7 +534,7 @@ handle_settings_web(Req) -> reply_json(Req, build_settings_web()). build_settings_web() -> - Port = proplists:get_value(port, menelaus_web:webconfig()), + Port = menelaus_web:webconfig(port), User = case ns_config_auth:get_user(admin) of undefined -> ""; @@ -734,7 +734,7 @@ handle_settings_web_post(Req) -> do_handle_settings_web_post(Port, U, P, Req) -> PortInt = case Port of - "SAME" -> proplists:get_value(port, menelaus_web:webconfig()); + "SAME" -> menelaus_web:webconfig(port); _ -> list_to_integer(Port) end, From af83339c6a9a79373fc8484c43230ab6ba5d5626 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Wed, 11 Nov 2020 16:52:40 -0800 Subject: [PATCH 05/33] MB-48047:[BP] Remove support for MOCHIWEB_IP wasn't working anyway Reviewed-on: http://review.couchbase.org/c/ns_server/+/140072 Change-Id: Ide4eea81dc32ad35704f1ff9a4d229fa69ea13ef Reviewed-on: http://review.couchbase.org/c/ns_server/+/164907 Well-Formed: Restriction Checker Tested-by: Abhijeeth Nuthan Reviewed-by: Steve Watanabe --- src/menelaus_web.erl | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/menelaus_web.erl b/src/menelaus_web.erl index 773ed5e846..0dcfc2bed0 100644 --- a/src/menelaus_web.erl +++ b/src/menelaus_web.erl @@ -105,18 +105,13 @@ http_server(Options) -> webconfig() -> Config = ns_config:get(), - Ip = case os:getenv("MOCHIWEB_IP") of - false -> misc:inaddr_any(); - Any -> Any - end, Port = case os:getenv("MOCHIWEB_PORT") of false -> service_ports:get_port(rest_port, Config); P -> list_to_integer(P) end, - [{ip, Ip}, - {port, Port}, + [{port, Port}, {nodelay, true}, {approot, menelaus_deps:local_path(["priv","public"], ?MODULE)}]. From 38dd8042698ad846d06197b3d392a84c0cb8f02f Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Mon, 16 Nov 2020 14:50:53 -0800 Subject: [PATCH 06/33] MB-48047:[BP] Remove support for MOCHIWEB_PORT Noone seems to use it. Reviewed-on: http://review.couchbase.org/c/ns_server/+/140459 Change-Id: Iceeeacbf00237278d5dda2fa5f6748412208bde4 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164928 Well-Formed: Restriction Checker Tested-by: Abhijeeth Nuthan Reviewed-by: Steve Watanabe --- src/menelaus_web.erl | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/menelaus_web.erl b/src/menelaus_web.erl index 0dcfc2bed0..ed68e77108 100644 --- a/src/menelaus_web.erl +++ b/src/menelaus_web.erl @@ -104,14 +104,7 @@ http_server(Options) -> end. webconfig() -> - Config = ns_config:get(), - Port = case os:getenv("MOCHIWEB_PORT") of - false -> - service_ports:get_port(rest_port, Config); - P -> - list_to_integer(P) - end, - [{port, Port}, + [{port, service_ports:get_port(rest_port)}, {nodelay, true}, {approot, menelaus_deps:local_path(["priv","public"], ?MODULE)}]. From 9fd8353844460b20b26c365958fde5f2b7de2800 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Wed, 4 Nov 2020 10:43:20 -0800 Subject: [PATCH 07/33] MB-48047:[BP] Introduce new 'strict' cluster encryption level Builds on top of level 'all' in that it also enforces only TLS ports for intra-cluster and external communication. Backports change related to MB-42373. Reviewed-on: http://review.couchbase.org/c/ns_server/+/139625 Change-Id: I6a49243c69f0e3bcd3e9aaf4af870bc60d370a0c Reviewed-on: http://review.couchbase.org/c/ns_server/+/164929 Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker Tested-by: Abhijeeth Nuthan --- src/menelaus_web_node.erl | 10 ++++++---- src/menelaus_web_settings.erl | 18 +++++++++++++----- src/misc.erl | 22 +++++++++++++--------- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/menelaus_web_node.erl b/src/menelaus_web_node.erl index b6128b5a3d..7f9c6a8aba 100644 --- a/src/menelaus_web_node.erl +++ b/src/menelaus_web_node.erl @@ -823,7 +823,7 @@ verify_net_config_allowed(State) -> AFamily = validator:get_value(afamily, State), AutoFailover = ns_config_auth:is_system_provisioned() andalso auto_failover:is_enabled(), - EncryptLevelAll = (misc:get_cluster_encryption_level() =:= all), + Encrypted = misc:should_cluster_data_be_encrypted(), IsCommunity = not cluster_compat_mode:is_enterprise(), if @@ -837,10 +837,12 @@ verify_net_config_allowed(State) -> M = "Can't change network configuration when auto-failover " "is enabled.", validator:return_error('_', M, State); - EncryptLevelAll andalso NodeEncryption =:= false -> + Encrypted andalso NodeEncryption =:= false -> M = <<"Can't disable nodeEncryption when the cluster " - "encryption level has been set to 'all'">>, - validator:return_error(nodeEncryption, M, State); + "encryption level has been set to ">>, + EL = atom_to_binary(misc:get_cluster_encryption_level(), latin1), + validator:return_error(nodeEncryption, <>, + State); true -> State end. diff --git a/src/menelaus_web_settings.erl b/src/menelaus_web_settings.erl index a883865cb0..7930073f89 100644 --- a/src/menelaus_web_settings.erl +++ b/src/menelaus_web_settings.erl @@ -140,11 +140,12 @@ get_cipher_suites(Str) -> end. get_cluster_encryption(Level) -> - SupportedLevels = ["control", "all"], + SupportedLevels = ["control", "all", "strict"], IsCEncryptEnabled = misc:is_cluster_encryption_fully_enabled(), ValidLevel = lists:member(Level, SupportedLevels), IsMandatory = (ns_ssl_services_setup:client_cert_auth_state() =:= "mandatory"), + IsStrictPossible = misc:is_strict_possible(), if not IsCEncryptEnabled -> @@ -152,11 +153,16 @@ get_cluster_encryption(Level) -> "is disabled.", {error, M}; not ValidLevel -> - M = "Cluster encryption level must be one of ['control', 'all'].", + M = io_lib:format("Cluster encryption level must be one of ~p", + [SupportedLevels]), + {error, lists:flatten(M)}; + IsMandatory andalso (Level =:= "all" orelse Level =:= "strict") -> + M = "Can't set cluster encryption level to '" ++ Level ++ + "' when client certificate authentication state is set " + "to 'mandatory'.", {error, M}; - IsMandatory andalso Level =:= "all" -> - M = "Can't set cluster encryption level to 'all' when client " - "certificate authentication state is set to 'mandatory'.", + Level =:= "strict" andalso not IsStrictPossible -> + M = "Can't set cluster encryption level to 'strict'.", {error, M}; true -> {ok, list_to_atom(Level)} @@ -229,6 +235,8 @@ conf(internal) -> {drop_request_memory_threshold_mib, dropRequestMemoryThresholdMiB, undefined, get_number(0, 99999, undefined)}, {gotraceback, gotraceback, <<"single">>, fun get_string/1}, + {can_enable_strict_encryption, canEnableStrictEncryption, false, + fun get_bool/1}, {{auto_failover_disabled, index}, indexAutoFailoverDisabled, true, fun get_bool/1}, {{cert, use_sha1}, certUseSha1, false, fun get_bool/1}]; diff --git a/src/misc.erl b/src/misc.erl index c1259c9eb1..fec62c2744 100644 --- a/src/misc.erl +++ b/src/misc.erl @@ -1434,6 +1434,10 @@ try_with_maybe_ignorant_after(TryBody, AfterBody) -> letrec(Args, F) -> erlang:apply(F, [F | Args]). +-spec is_strict_possible() -> true | false. +is_strict_possible() -> + ns_config:read_key_fast(can_enable_strict_encryption, false). + -spec is_ipv6() -> true | false. is_ipv6() -> get_net_family() == inet6. @@ -1464,7 +1468,7 @@ is_cluster_encryption_fully_disabled() -> [] =:= [N || N <- ns_node_disco:nodes_wanted(), misc:is_node_encryption_enabled(Cfg, N)]. --spec get_cluster_encryption_level() -> none | control | all. +-spec get_cluster_encryption_level() -> none | control | all | strict. get_cluster_encryption_level() -> Default = case is_cluster_encryption_fully_enabled() of true -> @@ -1474,7 +1478,8 @@ get_cluster_encryption_level() -> end, ns_config:search(ns_config:latest(), cluster_encryption_level, Default). --spec get_effective_cluster_encryption_level(term()) -> none | control | all. +-spec get_effective_cluster_encryption_level(term()) -> none | control | + all | strict. get_effective_cluster_encryption_level(Config) -> case is_cluster_encryption_fully_enabled() of true -> @@ -1485,16 +1490,15 @@ get_effective_cluster_encryption_level(Config) -> -spec should_cluster_data_be_encrypted() -> true | false. should_cluster_data_be_encrypted() -> - get_cluster_encryption_level() =:= all. + case get_cluster_encryption_level() of + all -> true; + strict -> true; + _ -> false + end. -spec disable_non_ssl_ports() -> true | false. disable_non_ssl_ports() -> - %% TODO: Disabling the non-SSL ports in not targeted for the 6.5 - %% release. Hence returning 'false' unconditionally. The thinking is to - %% implement this API correctly in the next release when this feature - %% will be supported. This will avoid making changes to the plumbing to - %% get this info to other services via cbauth. - false. + get_cluster_encryption_level() =:= strict. -spec get_net_family() -> inet:address_family(). get_net_family() -> From 7eef6efd1432641dfff777b91f2510178a300a2a Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Wed, 11 Nov 2020 17:17:18 -0800 Subject: [PATCH 08/33] MB-48047:[BP] Enforce 'strict' encryption level in ns_server Backports change related to MB-42373. Reviewed-on: http://review.couchbase.org/c/ns_server/+/140073 Change-Id: I2cee40e459fd03eefdf8d9d13c9e721b9bf132af Reviewed-on: http://review.couchbase.org/c/ns_server/+/164930 Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker Tested-by: Abhijeeth Nuthan --- src/menelaus_event.erl | 18 ++++++++++++++---- src/menelaus_web.erl | 36 ++++++++++++++++++++++++----------- src/ns_ssl_services_setup.erl | 1 - 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/src/menelaus_event.erl b/src/menelaus_event.erl index 76ca29acd4..0b8b9e9d9c 100644 --- a/src/menelaus_event.erl +++ b/src/menelaus_event.erl @@ -32,7 +32,9 @@ -export([init/1, handle_event/2, handle_call/2, handle_info/2, terminate/2, code_change/3]). --record(state, {webconfig, watchers = []}). +-record(state, {webconfig, + disable_non_ssl_ports, + watchers = []}). -include("ns_common.hrl"). @@ -95,6 +97,7 @@ unregister_watcher(Pid) -> init(ns_config_events) -> {ok, #state{watchers = [], + disable_non_ssl_ports = misc:disable_non_ssl_ports(), webconfig = menelaus_web:webconfig()}}; init(_) -> @@ -137,6 +140,10 @@ handle_event({rest, _}, State) -> NewState = maybe_restart(State), {ok, NewState}; +handle_event({cluster_encryption_level, _}, State) -> + NewState = maybe_restart(State), + {ok, NewState}; + handle_event(Event, State) -> case is_interesting_to_watchers(Event) of true -> @@ -192,12 +199,15 @@ notify_watchers(#state{watchers = Watchers}) -> Watchers), ok. -maybe_restart(#state{webconfig=WebConfigOld} = State) -> +maybe_restart(#state{webconfig = WebConfigOld, + disable_non_ssl_ports = DisableOld} = State) -> WebConfigNew = menelaus_web:webconfig(), - case WebConfigNew =:= WebConfigOld of + DisableNew = misc:disable_non_ssl_ports(), + case WebConfigNew =:= WebConfigOld andalso DisableOld =:= DisableNew of true -> State; false -> spawn(fun menelaus_sup:restart_web_servers/0), - State#state{webconfig=WebConfigNew} + State#state{webconfig = WebConfigNew, + disable_non_ssl_ports = DisableNew} end. flush_watcher_notifications(PrevID) -> diff --git a/src/menelaus_web.erl b/src/menelaus_web.erl index ed68e77108..ac09820366 100644 --- a/src/menelaus_web.erl +++ b/src/menelaus_web.erl @@ -64,13 +64,22 @@ init([Options0]) -> list_to_atom(lists:flatten(io_lib:format("~p_~s", [N, S]))) end, IsEnterprise = cluster_compat_mode:is_enterprise(), + {IPv4Addr, IPv6Addr} = case misc:disable_non_ssl_ports() andalso + not proplists:get_bool(ssl, Options) of + true -> + {misc:localhost(inet, []), + misc:localhost(inet6, [])}; + false -> + {misc:inaddr_any(inet, []), + misc:inaddr_any(inet6, [])} + end, Specs = [{menelaus_web_ipv4, {?MODULE, http_server, - [[{ip, "0.0.0.0"}, {name, CreateName(Name, "ipv4")} | Options]]}, + [[{ip, IPv4Addr}, {name, CreateName(Name, "ipv4")} | Options]]}, permanent, 5000, worker, dynamic}] ++ [{menelaus_web_ipv6, {?MODULE, http_server, - [[{ip, "::"}, {name, CreateName(Name, "ipv6")} | Options]]}, + [[{ip, IPv6Addr}, {name, CreateName(Name, "ipv6")} | Options]]}, permanent, 5000, worker, dynamic} || IsEnterprise], {ok, {{one_for_all, 10, 10}, Specs}}. @@ -84,21 +93,26 @@ http_server(Options) -> Loop = fun (Req) -> ?MODULE:loop(Req, {AppRoot, IsSSL, Plugins}) end, + LogOptions = [{K, V} || {K, V} <- Options1, + lists:member(K, [ssl, ssl_opts, ip, port])], case mochiweb_http:start_link([{loop, Loop} | Options1]) of - {ok, Pid} -> {ok, Pid}; + {ok, Pid} -> + ?log_info("Started web service with ~p", [LogOptions]), + {ok, Pid}; Other -> AFamily = misc:get_net_family(), - case {proplists:get_value(ip, Options1, "0.0.0.0"), AFamily} of - {"0.0.0.0", inet6} -> - ?log_warning("Failed to start IPv4 web service, ignoring"), + {ok, Addr} = inet:parse_address(proplists:get_value(ip, Options1)), + {Msg, Values} = {"Failed to start web service with ~p, Reason : ~p", + [LogOptions, Other]}, + case {Addr, AFamily} of + {{_, _, _, _}, inet6} -> + ?log_warning("Ignoring error: " ++ Msg, Values), ignore; - {"::", inet} -> - ?log_warning("Failed to start IPv6 web service, ignoring"), + {{_, _, _, _, _, _, _, _}, inet} -> + ?log_warning("Ignoring error: " ++ Msg, Values), ignore; _ -> - ?MENELAUS_WEB_LOG(?START_FAIL, - "Failed to start web service: ~p", - [Other]), + ?MENELAUS_WEB_LOG(?START_FAIL, Msg, Values), Other end end. diff --git a/src/ns_ssl_services_setup.erl b/src/ns_ssl_services_setup.erl index dce0913705..6bb2b8688f 100644 --- a/src/ns_ssl_services_setup.erl +++ b/src/ns_ssl_services_setup.erl @@ -461,7 +461,6 @@ init([]) -> Data = get_node_cert_data(), apply_node_cert_data(Data), - ?log_info("Used ssl options:~n~p", [ssl_server_opts()]), RetrySvc = case misc:marker_exists(marker_path()) of true -> Self ! notify_services, From 5bdc43a15308c77a2d4ebcb3fd9d73684a5f5ad6 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Thu, 19 Nov 2020 15:20:17 -0800 Subject: [PATCH 09/33] MB-48047:[BP] Make memcached interfaces address family specific Before interfaces, "interfaces": [ { "host": "*", "port": 12000, "ipv4": "required", "ipv6": "optional" }, { "host": "*", "port": 11999, "system": true, "ipv4": "required", "ipv6": "optional" }, { "host": "*", "port": 11998, "ssl": { "key": "/config/memcached-key.pem", "cert": "/config/memcached-cert.pem" }, "ipv4": "required", "ipv6": "optional" }, { "host": "*", "port": 11997, "system": true, "ssl": { "key": "/config/memcached-key.pem", "cert": "/config/memcached-cert.pem" }, "ipv4": "required", "ipv6": "optional" } ] After interfaces, "interfaces": [ { "port": 12000, "host": "*", "ipv4": "required", "ipv6": "off" }, { "port": 11999, "system": true, "host": "*", "ipv4": "required", "ipv6": "off" }, { "port": 11998, "ssl": { "key": "/config/memcached-key.pem", "cert": "/config/memcached-cert.pem" }, "host": "*", "ipv4": "required", "ipv6": "off" }, { "port": 11997, "system": true, "ssl": { "key": "/config/memcached-key.pem", "cert": "/config/memcached-cert.pem" }, "host": "*", "ipv4": "required", "ipv6": "off" }, { "port": 12000, "host": "*", "ipv4": "off", "ipv6": "optional" }, { "port": 11999, "system": true, "host": "*", "ipv4": "off", "ipv6": "optional" }, { "port": 11998, "ssl": { "key": "/config/memcached-key.pem", "cert": "/config/memcached-cert.pem" }, "host": "*", "ipv4": "off", "ipv6": "optional" }, { "port": 11997, "system": true, "ssl": { "key": "/config/memcached-key.pem", "cert": "/config/memcached-cert.pem" }, "host": "*", "ipv4": "off", "ipv6": "optional" } ] Reviewed-on: http://review.couchbase.org/c/ns_server/+/140669 Change-Id: I56b987d96f2edf3d232c82ddae4026a062baf8b8 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164931 Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker Tested-by: Abhijeeth Nuthan --- src/memcached_config_mgr.erl | 54 ++++++++++++++++++++++++++++++------ src/ns_config_default.erl | 30 +------------------- 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/src/memcached_config_mgr.erl b/src/memcached_config_mgr.erl index 783bc6737a..b1738c0ba5 100644 --- a/src/memcached_config_mgr.erl +++ b/src/memcached_config_mgr.erl @@ -24,12 +24,12 @@ -export([start_link/0]). %% referenced from ns_config_default --export([get_minidump_dir/2, omit_missing_mcd_ports/2, ssl_minimum_protocol/2, +-export([get_minidump_dir/2, get_interfaces/2, ssl_minimum_protocol/2, client_cert_auth/2, is_snappy_enabled/2, is_snappy_enabled/0, collections_enabled/2, get_fallback_salt/2, get_external_users_push_interval/2, get_ssl_cipher_list/2, get_ssl_cipher_order/2, get_external_auth_service/2, - is_external_auth_service_enabled/0, get_afamily_type/2]). + is_external_auth_service_enabled/0]). %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, @@ -345,13 +345,10 @@ get_minidump_dir([], Params) -> list_to_binary(proplists:get_value(breakpad_minidump_dir_path, Params, proplists:get_value(log_path, Params))). -omit_missing_mcd_ports(Interfaces, MCDParams) -> - Expanded = expand_memcached_config(Interfaces, MCDParams), - [Obj || Obj <- Expanded, - case Obj of - {Props} -> +get_interfaces([], MCDParams) -> + lists:filter(fun ({Props}) -> proplists:get_value(port, Props) =/= undefined - end]. + end, generate_interfaces(MCDParams)). ssl_minimum_protocol([], _Params) -> ns_ssl_services_setup:ssl_minimum_protocol(kv). @@ -431,9 +428,48 @@ format_ciphers(RFCCipherNames) -> get_ssl_cipher_order([], _Params) -> ns_ssl_services_setup:honor_cipher_order(kv). -get_afamily_type([AFamily], _Params) -> +get_afamily_type(AFamily) -> Required = ns_config:read_key_fast({node, node(), address_family}, inet), case AFamily of Required -> <<"required">>; _ -> <<"optional">> end. + +generate_interfaces(MCDParams) -> + SSL = {[{key, list_to_binary(ns_ssl_services_setup:memcached_key_path())}, + {cert, + list_to_binary(ns_ssl_services_setup:memcached_cert_path())}]}, + GetPort = fun (Port) -> + {Port, Value} = lists:keyfind(Port, 1, MCDParams), + Value + end, + InterProps = [{[{port, GetPort(port)}]}, + + {[{port, GetPort(dedicated_port)}, + {system, true}]}, + + {[{port, GetPort(ssl_port)}, + {ssl, SSL}]}, + + {[{port, GetPort(dedicated_ssl_port)}, + {system, true}, + {ssl, SSL}]}], + + IPv4Interfaces = lists:map( + fun ({Props}) -> + Additonal = [{host, get_host()}, + {ipv4, get_afamily_type(inet)}, + {ipv6, <<"off">>}], + {Props ++ Additonal} + end, InterProps), + IPv6Interfaces = lists:map( + fun ({Props}) -> + Additonal = [{host, get_host()}, + {ipv4, <<"off">>}, + {ipv6, get_afamily_type(inet6)}], + {Props ++ Additonal} + end, InterProps), + IPv4Interfaces ++ IPv6Interfaces. + +get_host() -> + <<"*">>. diff --git a/src/ns_config_default.erl b/src/ns_config_default.erl index 3d8c71f3da..f499bf3e8d 100644 --- a/src/ns_config_default.erl +++ b/src/ns_config_default.erl @@ -208,35 +208,7 @@ default() -> {{node, node(), memcached_config}, {[ - {interfaces, - {memcached_config_mgr, omit_missing_mcd_ports, - [ - {[{host, <<"*">>}, - {port, port}, - {ipv4, {memcached_config_mgr, get_afamily_type, [inet]}}, - {ipv6, {memcached_config_mgr, get_afamily_type, [inet6]}}]}, - - {[{host, <<"*">>}, - {port, dedicated_port}, - {system, true}, - {ipv4, {memcached_config_mgr, get_afamily_type, [inet]}}, - {ipv6, {memcached_config_mgr, get_afamily_type, [inet6]}}]}, - {[{host, <<"*">>}, - {port, ssl_port}, - {ssl, {[{key, list_to_binary(ns_ssl_services_setup:memcached_key_path())}, - {cert, list_to_binary(ns_ssl_services_setup:memcached_cert_path())}]}}, - {ipv4, {memcached_config_mgr, get_afamily_type, [inet]}}, - {ipv6, {memcached_config_mgr, get_afamily_type, [inet6]}}]}, - - {[{host, <<"*">>}, - {port, dedicated_ssl_port}, - {system, true}, - {ssl, {[{key, list_to_binary(ns_ssl_services_setup:memcached_key_path())}, - {cert, list_to_binary(ns_ssl_services_setup:memcached_cert_path())}]}}, - {ipv4, {memcached_config_mgr, get_afamily_type, [inet]}}, - {ipv6, {memcached_config_mgr, get_afamily_type, [inet6]}}]} - ]}}, - + {interfaces, {memcached_config_mgr, get_interfaces, []}}, {ssl_cipher_list, {memcached_config_mgr, get_ssl_cipher_list, []}}, {ssl_cipher_order, {memcached_config_mgr, get_ssl_cipher_order, []}}, {client_cert_auth, {memcached_config_mgr, client_cert_auth, []}}, From 6a05ef2f6ef384f70976c3038949a5a402ffc2b6 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Thu, 19 Nov 2020 15:39:35 -0800 Subject: [PATCH 10/33] MB-48047:[BP] Enforce 'strict' encryption level for memcached Backports change related to MB-42373. Reviewed-on: http://review.couchbase.org/c/ns_server/+/140512 Change-Id: Idaf474956ac8eb9e2722c699c6bc4689393826ed Reviewed-on: http://review.couchbase.org/c/ns_server/+/164932 Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker Tested-by: Abhijeeth Nuthan --- src/memcached_config_mgr.erl | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/memcached_config_mgr.erl b/src/memcached_config_mgr.erl index b1738c0ba5..73a6d8cfbb 100644 --- a/src/memcached_config_mgr.erl +++ b/src/memcached_config_mgr.erl @@ -123,6 +123,7 @@ is_notable_config_key(scramsha_fallback_salt) -> true; is_notable_config_key(external_auth_polling_interval) -> true; is_notable_config_key(cipher_suites) -> true; is_notable_config_key(honor_cipher_order) -> true; +is_notable_config_key(cluster_encryption_level) -> true; is_notable_config_key({security_settings, kv}) -> true; is_notable_config_key(ldap_settings) -> true; is_notable_config_key(saslauthd_auth_settings) -> true; @@ -457,19 +458,26 @@ generate_interfaces(MCDParams) -> IPv4Interfaces = lists:map( fun ({Props}) -> - Additonal = [{host, get_host()}, + IsSSL = proplists:is_defined(ssl, Props), + Additonal = [{host, get_host(inet, IsSSL)}, {ipv4, get_afamily_type(inet)}, {ipv6, <<"off">>}], {Props ++ Additonal} end, InterProps), IPv6Interfaces = lists:map( fun ({Props}) -> - Additonal = [{host, get_host()}, + IsSSL = proplists:is_defined(ssl, Props), + Additonal = [{host, get_host(inet6, IsSSL)}, {ipv4, <<"off">>}, {ipv6, get_afamily_type(inet6)}], {Props ++ Additonal} end, InterProps), IPv4Interfaces ++ IPv6Interfaces. -get_host() -> - <<"*">>. +get_host(Proto, IsSSL) -> + case (not IsSSL) andalso misc:disable_non_ssl_ports() of + true -> + list_to_binary(misc:localhost(Proto, [])); + false -> + <<"*">> + end. From d91253c03266f29f2f9da3ae2e90523c742f8257 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Tue, 24 Nov 2020 10:26:28 -0800 Subject: [PATCH 11/33] MB-48047:[BP] Refactor menelaus_web http server start code Reviewed-on: http://review.couchbase.org/c/ns_server/+/141025 Change-Id: Icc6a0cd36d7bb1df0038b606dcbbead4cece2751 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164933 Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker Tested-by: Abhijeeth Nuthan --- src/menelaus_web.erl | 68 ++++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/src/menelaus_web.erl b/src/menelaus_web.erl index ac09820366..31bed1a833 100644 --- a/src/menelaus_web.erl +++ b/src/menelaus_web.erl @@ -55,60 +55,66 @@ start_link() -> start_link([]). start_link(Options) -> - supervisor:start_link(?MODULE, [Options]). + Defaults = [{name, ?MODULE} | webconfig()], + MergedOptions = misc:update_proplist(Defaults, Options), + supervisor:start_link(?MODULE, [MergedOptions]). -init([Options0]) -> - Name = proplists:get_value(name, Options0, ?MODULE), - Options = proplists:delete(name, Options0), - CreateName = fun (N, S) -> - list_to_atom(lists:flatten(io_lib:format("~p_~s", [N, S]))) - end, +init([Options]) -> IsEnterprise = cluster_compat_mode:is_enterprise(), - {IPv4Addr, IPv6Addr} = case misc:disable_non_ssl_ports() andalso - not proplists:get_bool(ssl, Options) of - true -> - {misc:localhost(inet, []), - misc:localhost(inet6, [])}; - false -> - {misc:inaddr_any(inet, []), - misc:inaddr_any(inet6, [])} - end, Specs = [{menelaus_web_ipv4, - {?MODULE, http_server, - [[{ip, IPv4Addr}, {name, CreateName(Name, "ipv4")} | Options]]}, + {?MODULE, http_server, [[{afamily, inet} | Options]]}, permanent, 5000, worker, dynamic}] ++ [{menelaus_web_ipv6, - {?MODULE, http_server, - [[{ip, IPv6Addr}, {name, CreateName(Name, "ipv6")} | Options]]}, + {?MODULE, http_server, [[{afamily, inet6} | Options]]}, permanent, 5000, worker, dynamic} || IsEnterprise], {ok, {{one_for_all, 10, 10}, Specs}}. -http_server(Options) -> - Defaults = webconfig(), - Options0 = misc:update_proplist(Defaults, Options), - {AppRoot, Options1} = get_option(approot, Options0), +get_addr(AFamily, IsSSL) -> + case misc:disable_non_ssl_ports() andalso not IsSSL of + true -> + misc:localhost(AFamily, []); + false -> + misc:inaddr_any(AFamily, []) + end. +get_name(Name, inet) -> + get_name(Name, "ipv4"); +get_name(Name, inet6) -> + get_name(Name, "ipv6"); +get_name(Name, String) when is_list(String) -> + list_to_atom(lists:flatten(io_lib:format("~p_~s", [Name, String]))). + +generate_http_server_options(Options) -> + {AppRoot, Options1} = get_option(approot, Options), + {AFamily, Options2} = get_option(afamily, Options1), + {Name, Options3} = get_option(name, Options2), Plugins = menelaus_pluggable_ui:find_plugins(), - IsSSL = proplists:get_value(ssl, Options1, false), + IsSSL = proplists:get_bool(ssl, Options3), Loop = fun (Req) -> ?MODULE:loop(Req, {AppRoot, IsSSL, Plugins}) end, - LogOptions = [{K, V} || {K, V} <- Options1, + [{ip, get_addr(AFamily, IsSSL)}, + {name, get_name(Name, AFamily)}, + {loop, Loop}] ++ Options3. + +http_server(Options) -> + ServerAFamily = proplists:get_value(afamily, Options), + ServerOptions = generate_http_server_options(Options), + LogOptions = [{K, V} || {K, V} <- ServerOptions, lists:member(K, [ssl, ssl_opts, ip, port])], - case mochiweb_http:start_link([{loop, Loop} | Options1]) of + case mochiweb_http:start_link(ServerOptions) of {ok, Pid} -> ?log_info("Started web service with ~p", [LogOptions]), {ok, Pid}; Other -> AFamily = misc:get_net_family(), - {ok, Addr} = inet:parse_address(proplists:get_value(ip, Options1)), {Msg, Values} = {"Failed to start web service with ~p, Reason : ~p", [LogOptions, Other]}, - case {Addr, AFamily} of - {{_, _, _, _}, inet6} -> + case {ServerAFamily, AFamily} of + {inet, inet6} -> ?log_warning("Ignoring error: " ++ Msg, Values), ignore; - {{_, _, _, _, _, _, _, _}, inet} -> + {inet6, inet} -> ?log_warning("Ignoring error: " ++ Msg, Values), ignore; _ -> From 66af0495a398ccea99d28926adc7776b839b0869 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Tue, 1 Dec 2020 16:36:38 -0800 Subject: [PATCH 12/33] MB-48047:[BP] Slight refactor of menelaus_event code Reviewed-on: http://review.couchbase.org/c/ns_server/+/141383 Change-Id: I8142cf113ecababf3fcef16be93b6d698d9563b8 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164934 Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker Tested-by: Abhijeeth Nuthan --- src/menelaus_event.erl | 43 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/menelaus_event.erl b/src/menelaus_event.erl index 0b8b9e9d9c..6686b25632 100644 --- a/src/menelaus_event.erl +++ b/src/menelaus_event.erl @@ -132,26 +132,10 @@ is_interesting_to_watchers({client_cert_auth, _}) -> true; is_interesting_to_watchers({audit_uid_change, _}) -> true; is_interesting_to_watchers(_) -> false. -handle_event({{node, Node, rest}, _}, State) when Node =:= node() -> - NewState = maybe_restart(State), - {ok, NewState}; - -handle_event({rest, _}, State) -> - NewState = maybe_restart(State), - {ok, NewState}; - -handle_event({cluster_encryption_level, _}, State) -> - NewState = maybe_restart(State), - {ok, NewState}; - handle_event(Event, State) -> - case is_interesting_to_watchers(Event) of - true -> - ok = notify_watchers(State); - _ -> - ok - end, - {ok, State}. + NewState = maybe_restart(Event, State), + maybe_notify_watchers(Event, State), + {ok, NewState}. handle_call({register_watcher, Pid}, #state{watchers = Watchers} = State) -> @@ -191,13 +175,28 @@ handle_info(_Info, State) -> % ------------------------------------------------------------ +maybe_notify_watchers(Event, State) -> + case is_interesting_to_watchers(Event) of + true -> notify_watchers(State); + false -> ok + end. + notify_watchers(#state{watchers = Watchers}) -> UpdateID = erlang:unique_integer(), lists:foreach(fun({Pid, _}) -> Pid ! {notify_watcher, UpdateID} - end, - Watchers), - ok. + end, Watchers). + +restart_event({{node, N, rest}, _}) when N =:= node() -> true; +restart_event({rest, _}) -> true; +restart_event({cluster_encryption_level, _}) -> true; +restart_event(_) -> false. + +maybe_restart(Event, State) -> + case restart_event(Event) of + true -> maybe_restart(State); + false -> State + end. maybe_restart(#state{webconfig = WebConfigOld, disable_non_ssl_ports = DisableOld} = State) -> From 3b92f7b3a73fe7280e2ca40c9e7da26d7ae21b69 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Tue, 1 Dec 2020 16:42:22 -0800 Subject: [PATCH 13/33] MB-48047:[BP] Separate gen_event and internal functions Reviewed-on: http://review.couchbase.org/c/ns_server/+/141384 Change-Id: I8f47226346b6f0815852c7b10b5019d842a52247 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164935 Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker Tested-by: Abhijeeth Nuthan --- src/menelaus_event.erl | 51 +++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/src/menelaus_event.erl b/src/menelaus_event.erl index 6686b25632..d883e4ed97 100644 --- a/src/menelaus_event.erl +++ b/src/menelaus_event.erl @@ -106,32 +106,6 @@ init(_) -> terminate(_Reason, _State) -> ok. code_change(_OldVsn, State, _) -> {ok, State}. -is_interesting_to_watchers({significant_buckets_change, _}) -> true; -is_interesting_to_watchers({memcached, _}) -> true; -is_interesting_to_watchers({{node, _, memcached}, _}) -> true; -is_interesting_to_watchers({{node, _, membership}, _}) -> true; -is_interesting_to_watchers({rebalance_status, _}) -> true; -is_interesting_to_watchers({recovery_status, _}) -> true; -is_interesting_to_watchers({buckets, _}) -> true; -is_interesting_to_watchers({nodes_wanted, _}) -> true; -is_interesting_to_watchers({server_groups, _}) -> true; -is_interesting_to_watchers({ns_node_disco_events, _NodesBefore, _NodesAfter}) -> true; -is_interesting_to_watchers({autocompaction, _}) -> true; -is_interesting_to_watchers({cluster_compat_version, _}) -> true; -is_interesting_to_watchers({developer_preview_enabled, _}) -> true; -is_interesting_to_watchers({cluster_name, _}) -> true; -is_interesting_to_watchers({memory_quota, _}) -> true; -is_interesting_to_watchers({index_settings_change, memoryQuota, _}) -> true; -is_interesting_to_watchers({indexes_change, index, _}) -> true; -is_interesting_to_watchers({goxdcr_enabled, _}) -> true; -is_interesting_to_watchers({{node, _, stop_xdcr}, _}) -> true; -is_interesting_to_watchers({{node, _, services}, _}) -> true; -is_interesting_to_watchers({{service_map, _}, _}) -> true; -is_interesting_to_watchers({user_roles, _}) -> true; -is_interesting_to_watchers({client_cert_auth, _}) -> true; -is_interesting_to_watchers({audit_uid_change, _}) -> true; -is_interesting_to_watchers(_) -> false. - handle_event(Event, State) -> NewState = maybe_restart(Event, State), maybe_notify_watchers(Event, State), @@ -174,6 +148,31 @@ handle_info(_Info, State) -> {ok, State}. % ------------------------------------------------------------ +is_interesting_to_watchers({significant_buckets_change, _}) -> true; +is_interesting_to_watchers({memcached, _}) -> true; +is_interesting_to_watchers({{node, _, memcached}, _}) -> true; +is_interesting_to_watchers({{node, _, membership}, _}) -> true; +is_interesting_to_watchers({rebalance_status, _}) -> true; +is_interesting_to_watchers({recovery_status, _}) -> true; +is_interesting_to_watchers({buckets, _}) -> true; +is_interesting_to_watchers({nodes_wanted, _}) -> true; +is_interesting_to_watchers({server_groups, _}) -> true; +is_interesting_to_watchers({ns_node_disco_events, _NodesBefore, _NodesAfter}) -> true; +is_interesting_to_watchers({autocompaction, _}) -> true; +is_interesting_to_watchers({cluster_compat_version, _}) -> true; +is_interesting_to_watchers({developer_preview_enabled, _}) -> true; +is_interesting_to_watchers({cluster_name, _}) -> true; +is_interesting_to_watchers({memory_quota, _}) -> true; +is_interesting_to_watchers({index_settings_change, memoryQuota, _}) -> true; +is_interesting_to_watchers({indexes_change, index, _}) -> true; +is_interesting_to_watchers({goxdcr_enabled, _}) -> true; +is_interesting_to_watchers({{node, _, stop_xdcr}, _}) -> true; +is_interesting_to_watchers({{node, _, services}, _}) -> true; +is_interesting_to_watchers({{service_map, _}, _}) -> true; +is_interesting_to_watchers({user_roles, _}) -> true; +is_interesting_to_watchers({client_cert_auth, _}) -> true; +is_interesting_to_watchers({audit_uid_change, _}) -> true; +is_interesting_to_watchers(_) -> false. maybe_notify_watchers(Event, State) -> case is_interesting_to_watchers(Event) of From a8c7ea48eec0d037d4dcea8a712ecdca1ad0597d Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Thu, 14 Oct 2021 11:30:29 -0700 Subject: [PATCH 14/33] MB-48047:[BP] Start menelaus_event before menelaus_web in new sup Changes in webconfig and/or clusterEncryptionLevel can be missed if the changes occur 1. between the time when menelaus_web has started to when menelaus_event was started. 2. when menelaus_event had crashed but not up and running again. Also, make sure we handle error on restart of web servers. Reviewed-on: http://review.couchbase.org/c/ns_server/+/140663 Change-Id: I990362197830f2f7a5e27353ad90ec9be527debd Reviewed-on: http://review.couchbase.org/c/ns_server/+/164936 Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker Tested-by: Abhijeeth Nuthan --- src/menelaus_event.erl | 2 +- src/menelaus_sup.erl | 53 +++----------------------- src/menelaus_web_sup.erl | 81 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 48 deletions(-) create mode 100644 src/menelaus_web_sup.erl diff --git a/src/menelaus_event.erl b/src/menelaus_event.erl index d883e4ed97..db4ed293cd 100644 --- a/src/menelaus_event.erl +++ b/src/menelaus_event.erl @@ -203,7 +203,7 @@ maybe_restart(#state{webconfig = WebConfigOld, DisableNew = misc:disable_non_ssl_ports(), case WebConfigNew =:= WebConfigOld andalso DisableOld =:= DisableNew of true -> State; - false -> spawn(fun menelaus_sup:restart_web_servers/0), + false -> {ok, _} = menelaus_web_sup:restart_web_servers(), State#state{webconfig = WebConfigNew, disable_non_ssl_ports = DisableNew} end. diff --git a/src/menelaus_sup.erl b/src/menelaus_sup.erl index 0a69dd8e57..0d77ecaac2 100644 --- a/src/menelaus_sup.erl +++ b/src/menelaus_sup.erl @@ -19,45 +19,21 @@ -author('Northscale '). -behaviour(supervisor). --behaviour(ns_log_categorizing). - --define(START_OK, 1). --define(START_FAIL, 2). %% External exports -export([start_link/0, barrier_spec/1, barrier_notify_spec/1, - barrier_start_link/0, barrier_notify/0, barrier_wait/0, - restart_web_servers/0]). + barrier_start_link/0, barrier_notify/0, barrier_wait/0]). %% supervisor callbacks -export([init/1]). --export([ns_log_cat/1, ns_log_code_string/1]). - -include("ns_common.hrl"). %% @spec start_link() -> ServerRet %% @doc API for starting the supervisor. start_link() -> - Result = supervisor:start_link({local, ?MODULE}, ?MODULE, []), - Port = menelaus_web:webconfig(port), - case Result of - {ok, _Pid} -> - ?user_log(?START_OK, - "Couchbase Server has started on web port ~p on node ~p. Version: ~p.", - [Port, node(), ns_info:version(ns_server)]); - _Err -> - %% The exact error message is not logged here since this - %% is a supervisor start, but a more helpful message - %% should've been logged before. - ?user_log(?START_FAIL, - "Couchbase Server has failed to start on web port ~p on node ~p. " ++ - "Perhaps another process has taken port ~p already? " ++ - "If so, please stop that process first before trying again.", - [Port, node(), Port]) - end, - Result. + supervisor:start_link({local, ?MODULE}, ?MODULE, []). barrier_spec(Id) -> {Id, {menelaus_sup, barrier_start_link, []}, @@ -118,18 +94,14 @@ init([]) -> {gen_event, start_link, [{local, json_rpc_events}]}, permanent, 1000, worker, []}, - Web = restartable:spec({menelaus_web, - {menelaus_web, start_link, []}, - permanent, infinity, supervisor, dynamic}), + WebSup = {menelaus_web_sup, + {menelaus_web_sup, start_link, []}, + permanent, infinity, supervisor, [menelaus_web_sup]}, Alerts = {menelaus_web_alerts_srv, {menelaus_web_alerts_srv, start_link, []}, permanent, 5000, worker, dynamic}, - WebEvent = {menelaus_event, - {menelaus_event, start_link, []}, - permanent, 5000, worker, dynamic}, - HotKeysKeeper = {hot_keys_keeper, {hot_keys_keeper, start_link, []}, permanent, 5000, worker, dynamic}, @@ -139,18 +111,5 @@ init([]) -> permanent, 1000, worker, dynamic}, Processes = [UIAuth, ScramSha, LocalAuth, Cache, StatsGatherer, RpcEvents, - Web, WebEvent, HotKeysKeeper, Alerts, CBAuth], + WebSup, HotKeysKeeper, Alerts, CBAuth], {ok, {{one_for_one, 10, 10}, Processes}}. - -ns_log_cat(?START_OK) -> - info; -ns_log_cat(?START_FAIL) -> - crit. - -ns_log_code_string(?START_OK) -> - "web start ok"; -ns_log_code_string(?START_FAIL) -> - "web start fail". - -restart_web_servers() -> - restartable:restart(?MODULE, menelaus_web). diff --git a/src/menelaus_web_sup.erl b/src/menelaus_web_sup.erl new file mode 100644 index 0000000000..ccdb4d7173 --- /dev/null +++ b/src/menelaus_web_sup.erl @@ -0,0 +1,81 @@ +%% @author Couchbase +%% @copyright 2020 Couchbase, Inc. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(menelaus_web_sup). + +-behaviour(supervisor). +-behaviour(ns_log_categorizing). + +-define(START_OK, 1). +-define(START_FAIL, 2). + +%% External exports +-export([start_link/0, + restart_web_servers/0]). + +%% supervisor callbacks +-export([init/1]). + +%% ns_log_categorizing callbacks. +-export([ns_log_cat/1, ns_log_code_string/1]). + +-include("ns_common.hrl"). + +restart_web_servers() -> + restartable:restart(?MODULE, menelaus_web). + +start_link() -> + Result = supervisor:start_link({local, ?MODULE}, ?MODULE, []), + Port = menelaus_web:webconfig(port), + case Result of + {ok, _Pid} -> + ?user_log(?START_OK, + "Couchbase Server has started on web port ~p on node ~p. Version: ~p.", + [Port, node(), ns_info:version(ns_server)]); + _Err -> + %% The exact error message is not logged here since this + %% is a supervisor start, but a more helpful message + %% should've been logged before. + ?user_log(?START_FAIL, + "Couchbase Server has failed to start on web port ~p on node ~p. " ++ + "Perhaps another process has taken port ~p already? " ++ + "If so, please stop that process first before trying again.", + [Port, node(), Port]) + end, + Result. + +init([]) -> + %% We restart menelaus_web from menelaus_event, make sure we start + %% menelaus_event first so that we don't miss any events where + %% menelaus_web needs to be restarted. Also, we restart menelaus_web if + %% menelaus_event crashes. + Processes = [{menelaus_event, + {menelaus_event, start_link, []}, + permanent, 5000, worker, dynamic}, + + restartable:spec({menelaus_web, + {menelaus_web, start_link, []}, + permanent, infinity, supervisor, dynamic})], + {ok, {{rest_for_one, 10, 10}, Processes}}. + +ns_log_cat(?START_OK) -> + info; +ns_log_cat(?START_FAIL) -> + crit. + +ns_log_code_string(?START_OK) -> + "web start ok"; +ns_log_code_string(?START_FAIL) -> + "web start fail". From 86ee18320ceed6a6561c1c8332c7b364cdce42c0 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Thu, 19 Nov 2020 12:06:34 -0800 Subject: [PATCH 15/33] MB-48047:[BP] Fix theoretical race in misc:start_event_link In misc:start_event_link there is also a theoretical race from when spawn_link returns and SubscriptionBody completes. Reviewed-on: http://review.couchbase.org/c/ns_server/+/140664 Change-Id: If546f93fb75d4986c0d0f7d200070236c1629101 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164937 Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker Tested-by: Build Bot Tested-by: Abhijeeth Nuthan --- src/misc.erl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/misc.erl b/src/misc.erl index fec62c2744..0158be32d8 100644 --- a/src/misc.erl +++ b/src/misc.erl @@ -1095,13 +1095,15 @@ absname(Name) -> end. start_event_link(SubscriptionBody) -> - {ok, - spawn_link(fun () -> - SubscriptionBody(), - receive - _ -> ok - end - end)}. + proc_lib:start_link( + erlang, apply, + [fun () -> + SubscriptionBody(), + proc_lib:init_ack({ok, self()}), + receive + _ -> ok + end + end, []]). %% Writes to file atomically using write_file + atomic_rename atomic_write_file(Path, BodyOrBytes) From fbfd0e4fc309d7319d47af19073f1b34e27d1c3d Mon Sep 17 00:00:00 2001 From: Timofey Barmin Date: Mon, 7 Dec 2020 13:29:12 -0800 Subject: [PATCH 16/33] MB-48047:[BP] cb_dist: Disable ipv6 listener when ... ... node is ipv4 and vise versa Backports change related to MB-41189. Reviewed-on: http://review.couchbase.org/c/ns_server/+/141647 Change-Id: I59fba6cb3d44b1efd360479fd1d1a20389025de3 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164938 Tested-by: Abhijeeth Nuthan Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker --- src/cb_dist.erl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/cb_dist.erl b/src/cb_dist.erl index c5994be2c2..0aaeea9040 100644 --- a/src/cb_dist.erl +++ b/src/cb_dist.erl @@ -650,8 +650,8 @@ conf(Prop, Conf) -> defaults() -> [{preferred_external_proto, inet_tcp_dist}, {preferred_local_proto, inet_tcp_dist}, - {local_listeners, [inet_tcp_dist, inet6_tcp_dist]}, - {external_listeners, [inet_tcp_dist, inet6_tcp_dist]}]. + {local_listeners, [inet_tcp_dist]}, + {external_listeners, [inet_tcp_dist]}]. transform_old_to_new_config(Dist) -> DistType = list_to_atom((atom_to_list(Dist) ++ "_dist")), @@ -819,9 +819,10 @@ import_props_to_config(Props, Cfg) -> undefined -> CurListeners; L -> [netsettings2proto(E) || E <- L] end, - [{external_listeners, Listeners} || Listeners =/= undefined] ++ - [{preferred_external_proto, PrefExt} || PrefExt =/= undefined] ++ - [{preferred_local_proto, PrefLocal} || PrefLocal =/= undefined]. + [{external_listeners, Listeners} || Listeners =/= undefined] ++ + [{local_listeners, [PrefLocal]}, + {preferred_external_proto, PrefExt}, + {preferred_local_proto, PrefLocal}]. store_config(DCfgFile, DCfg) -> DirName = filename:dirname(DCfgFile), From 4721ae35857309f14bab6655242a4040033f77b8 Mon Sep 17 00:00:00 2001 From: Timofey Barmin Date: Wed, 9 Dec 2020 12:58:50 -0800 Subject: [PATCH 17/33] MB-48047:[BP] Make engage cluster wait for web server restart finish Backports change related to MB-43166. Reviewed-on: http://review.couchbase.org/c/ns_server/+/141820 Change-Id: I846c07684ccc8e8236fc73e730f93af4505c69d7 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164939 Tested-by: Abhijeeth Nuthan Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker --- src/dist_manager.erl | 3 +-- src/menelaus_event.erl | 9 ++++++++- src/menelaus_web_cert.erl | 4 ++-- src/menelaus_web_node.erl | 11 +++++++++-- src/ns_cluster.erl | 14 +++++++++++--- src/ns_ssl_services_setup.erl | 9 +++++++-- 6 files changed, 38 insertions(+), 12 deletions(-) diff --git a/src/dist_manager.erl b/src/dist_manager.erl index 8973b46c09..b0fb3a05ef 100644 --- a/src/dist_manager.erl +++ b/src/dist_manager.erl @@ -377,8 +377,7 @@ complete_rename(OldNode) -> %% rename command to be synchronous. They can perform calls to web server %% immediately after rename is done. In order to avoid the race we need %% to make sure web server is restarted by the time the rename is finished. - cluster_compat_mode:is_enterprise() andalso - ns_ssl_services_setup:sync_local_cert_and_pkey_change(), + cluster_compat_mode:is_enterprise() andalso ns_ssl_services_setup:sync(), misc:remove_marker(ns_cluster:rename_marker_path()). rename_node_in_config(Old, New) -> diff --git a/src/menelaus_event.erl b/src/menelaus_event.erl index db4ed293cd..8bf9a616d0 100644 --- a/src/menelaus_event.erl +++ b/src/menelaus_event.erl @@ -25,7 +25,8 @@ -export([register_watcher/1, unregister_watcher/1, - flush_watcher_notifications/1]). + flush_watcher_notifications/1, + sync/1]). %% gen_event callbacks @@ -93,6 +94,9 @@ unregister_watcher(Pid) -> {?MODULE, audit_events}, {unregister_watcher, Pid}). +sync(Module) -> + gen_event:call(Module, {?MODULE, Module}, sync). + %% Implementation init(ns_config_events) -> @@ -130,6 +134,9 @@ handle_call({unregister_watcher, Pid}, end, {ok, ok, State#state{watchers = Watchers2}}; +handle_call(sync, State) -> + {ok, ok, State}; + handle_call(Request, State) -> ?log_warning("Unexpected handle_call(~p, ~p)", [Request, State]), {ok, ok, State}. diff --git a/src/menelaus_web_cert.erl b/src/menelaus_web_cert.erl index de5c515506..3de12cc5e4 100644 --- a/src/menelaus_web_cert.erl +++ b/src/menelaus_web_cert.erl @@ -92,7 +92,7 @@ handle_regenerate_certificate(Req) -> assert_n2n_encryption_is_disabled(), ns_server_cert:generate_and_set_cert_and_pkey(), - ns_ssl_services_setup:sync_local_cert_and_pkey_change(), + ns_ssl_services_setup:sync(), ?log_info("Completed certificate regeneration"), ns_audit:regenerate_certificate(Req), handle_cluster_certificate_simple(Req). @@ -136,7 +136,7 @@ handle_reload_node_certificate(Req) -> ns_audit:reload_node_certificate(Req, proplists:get_value(subject, Props), proplists:get_value(expires, Props)), - ns_ssl_services_setup:sync_local_cert_and_pkey_change(), + ns_ssl_services_setup:sync(), case netconfig_updater:ensure_tls_dist_started(Nodes) of ok -> menelaus_util:reply(Req, 200); diff --git a/src/menelaus_web_node.erl b/src/menelaus_web_node.erl index 7f9c6a8aba..7af98e84e9 100644 --- a/src/menelaus_web_node.erl +++ b/src/menelaus_web_node.erl @@ -871,8 +871,15 @@ handle_setup_net_config(Req) -> fun (Values) -> erlang:process_flag(trap_exit, true), case netconfig_updater:apply_config(Values) of - ok -> menelaus_util:reply(Req, 200); - {error, Msg} -> menelaus_util:reply_global_error(Req, Msg) + ok -> + %% Wait for web servers to restart + ns_config:sync_announcements(), + menelaus_event:sync(ns_config_events), + cluster_compat_mode:is_enterprise() andalso + ns_ssl_services_setup:sync(), + menelaus_util:reply(Req, 200); + {error, Msg} -> + menelaus_util:reply_global_error(Req, Msg) end, erlang:exit(normal) end, Req, form, net_config_validators(false)). diff --git a/src/ns_cluster.erl b/src/ns_cluster.erl index c378e7bd43..bc62bc20ea 100644 --- a/src/ns_cluster.erl +++ b/src/ns_cluster.erl @@ -121,7 +121,7 @@ engage_cluster_apply_certs(NodeKVList) -> NeedEncryption} of {Cert, true} when Cert =/= undefined -> ns_server_cert:set_generated_public_ca(Cert), - ns_ssl_services_setup:sync_local_cert_and_pkey_change(), + ns_ssl_services_setup:sync(), ?log_info("Generated certificate was loaded on the node " "before joining. Cert: ~p", [Cert]); _ -> @@ -140,7 +140,7 @@ apply_certs(ClusterCA) -> {ok, _} -> case ns_server_cert:apply_certificate_chain_from_inbox(ClusterCA) of {ok, Props} -> - ns_ssl_services_setup:sync_local_cert_and_pkey_change(), + ns_ssl_services_setup:sync(), ?log_info("Custom certificate was loaded on the node " "before joining. Props: ~p", [Props]), ok; @@ -172,7 +172,15 @@ apply_net_config(NodeKVList) -> Props = [{externalListeners, Protos}, {afamily, AFamily}, {nodeEncryption, NEncryption}], - netconfig_updater:apply_config(Props); + case netconfig_updater:apply_config(Props) of + ok -> + ns_config:sync_announcements(), + menelaus_event:sync(ns_config_events), + cluster_compat_mode:is_enterprise() andalso + ns_ssl_services_setup:sync(), + ok; + {error, Msg} -> {error, Msg} + end; {error, Msg} -> {error, Msg} end; {error, Msg} -> {error, Msg} diff --git a/src/ns_ssl_services_setup.erl b/src/ns_ssl_services_setup.erl index 6bb2b8688f..3756b504ae 100644 --- a/src/ns_ssl_services_setup.erl +++ b/src/ns_ssl_services_setup.erl @@ -29,7 +29,7 @@ ssl_cacert_key_path/0, memcached_cert_path/0, memcached_key_path/0, - sync_local_cert_and_pkey_change/0, + sync/0, ssl_minimum_protocol/1, ssl_minimum_protocol/2, client_cert_auth/0, @@ -428,8 +428,13 @@ do_check_local_cert_and_pkey(ClusterCertPEM, Node) -> Digest = erlang:md5(term_to_binary({Node, ClusterCertPEM, LocalCert, LocalPKey})), {proplists:get_value(digest, Meta) =:= Digest, LocalCert, LocalPKey}. -sync_local_cert_and_pkey_change() -> +sync() -> ns_config:sync_announcements(), + %% First ping guarantees that trigger_ssl_reload has sent + %% the notify_services message + ok = gen_server:call(?MODULE, ping, infinity), + %% Second ping guarantees that the notify_services message message + %% has been handled ok = gen_server:call(?MODULE, ping, infinity). set_node_certificate_chain(Props, CAChain, Cert, PKey) -> From 177c882ecf2aea4bf919e8cd9672e0d3f10e0717 Mon Sep 17 00:00:00 2001 From: Timofey Barmin Date: Tue, 8 Dec 2020 11:20:09 -0800 Subject: [PATCH 18/33] MB-48047:[BP] cb_dist: Listen on loopback when possible Backports change related to MB-42373. Reviewed-on: http://review.couchbase.org/c/ns_server/+/141722 Change-Id: I507b8b21fdfe8f121d78b9ca60c745496f280141 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164940 Well-Formed: Restriction Checker Reviewed-by: Timofey Barmin Tested-by: Build Bot Tested-by: Abhijeeth Nuthan --- src/cb_dist.erl | 134 +++++++++++++++++++++++++++++------------------- 1 file changed, 82 insertions(+), 52 deletions(-) diff --git a/src/cb_dist.erl b/src/cb_dist.erl index 0aaeea9040..8361ef4f50 100644 --- a/src/cb_dist.erl +++ b/src/cb_dist.erl @@ -255,8 +255,10 @@ handle_call({listen, Name}, _From, #s{creation = Creation} = State) -> %% to start later. To work around that start the ipv4 listener before %% the ipv6 one. OrderFun = - fun (inet_tcp_dist, _) -> true; - (_, inet_tcp_dist) -> false; + fun ({_, inet_tcp_dist}, _) -> true; + (_, {_, inet_tcp_dist}) -> false; + ({_, inet_tls_dist}, _) -> true; + (_, {_, inet_tls_dist}) -> false; (A, B) -> A =< B end, Protos = lists:sort(OrderFun, Protos0), @@ -287,8 +289,8 @@ handle_call({listen, Name}, _From, #s{creation = Creation} = State) -> handle_call({accept, KernelPid}, _From, #s{listeners = Listeners} = State) -> Acceptors = lists:map( - fun ({Module, {LSocket, _Addr, _Creation}}) -> - {Module:accept(LSocket), Module} + fun ({{_AType, Module} = Listener, {LSocket, _Addr, _Creation}}) -> + {Module:accept(LSocket), Listener} end, Listeners), {reply, self(), ensure_config(State#s{acceptors = Acceptors, @@ -296,7 +298,7 @@ handle_call({accept, KernelPid}, _From, #s{listeners = Listeners} = State) -> handle_call({get_module_by_acceptor, AcceptorPid}, _From, #s{acceptors = Acceptors} = State) -> - Module = proplists:get_value(AcceptorPid, Acceptors), + {_, Module} = proplists:get_value(AcceptorPid, Acceptors), {reply, Module, State}; handle_call({get_preferred, Target}, _From, #s{name = Name, @@ -326,7 +328,7 @@ handle_call(ensure_config, _From, State) -> case not_started_required_listeners(NewState) of [] -> {reply, ok, NewState}; List -> - ProtoList = [proto2netsettings(L) || L <- List], + ProtoList = [proto2netsettings(L) || {_, L} <- List], {reply, {error, {not_started, ProtoList}}, NewState} end; @@ -358,10 +360,11 @@ handle_call({update_connection_pid, Ref, Pid}, _From, State) -> {reply, ok, update_connection_pid(Ref, Pid, State)}; handle_call(restart_tls, _From, #s{connections = Connections, - kernel_pid = KernelPid} = State) -> + kernel_pid = KernelPid, + listeners = Listeners} = State) -> info_msg("Restarting tls distribution protocols (if any)", []), - NewState = functools:chain(State, [remove_proto(inet6_tls_dist, _), - remove_proto(inet_tls_dist, _)]), + TLSListeners = [L || {{_, P} = L, _} <- Listeners, proto_to_encryption(P)], + NewState = lists:foldl(fun remove_proto/2, State, TLSListeners), NewConnections = lists:filtermap( @@ -403,8 +406,10 @@ handle_info({accept, AcceptorPid, ConSocket, _Family, _Protocol}, connections = Connections, acceptors = Acceptors} = State) -> Ref = make_ref(), - Con = #con{ref = Ref, mod = proplists:get_value(AcceptorPid, Acceptors)}, - info_msg("Accepted new connection from ~p: ~p", [AcceptorPid, Con]), + {_, Module} = proplists:get_value(AcceptorPid, Acceptors), + Con = #con{ref = Ref, mod = Module}, + info_msg("Accepted new connection from ~p DistCtrl ~p: ~p", + [AcceptorPid, ConSocket, Con]), KernelPid ! {accept, self(), {Ref, AcceptorPid, ConSocket}, ?family, ?proto}, {noreply, State#s{connections = [Con | Connections]}}; @@ -460,13 +465,15 @@ close_listeners(#s{listeners = Listeners} = State) -> %% inet_*_dist modules use inet_dist_listen_min and inet_dist_listen_max to %% choose port for listening. Since we need them to choose Port we set those %% variables to Port, call listen function, and then change variables back. -with_dist_port(noport, _Fun) -> ignore; -with_dist_port(Port, Fun) -> +with_dist_ip_and_port(_, noport, _Fun) -> ignore; +with_dist_ip_and_port(IP, Port, Fun) -> OldMin = application:get_env(kernel,inet_dist_listen_min), OldMax = application:get_env(kernel,inet_dist_listen_max), + OldIP = application:get_env(kernel,inet_dist_use_interface), try application:set_env(kernel, inet_dist_listen_min, Port), application:set_env(kernel, inet_dist_listen_max, Port), + application:set_env(kernel, inet_dist_use_interface, IP), Fun() after case OldMin of @@ -476,27 +483,34 @@ with_dist_port(Port, Fun) -> case OldMax of undefined -> application:unset_env(kernel, inet_dist_listen_max); {ok, V2} -> application:set_env(kernel, inet_dist_listen_max, V2) + end, + case OldIP of + undefined -> + application:unset_env(kernel, inet_dist_use_interface); + {ok, V3} -> + application:set_env(kernel, inet_dist_use_interface, V3) end end. -add_proto(Mod, #s{name = NodeName, listeners = Listeners, - acceptors = Acceptors} = State) -> - case can_add_proto(Mod, State) of +add_proto({_AddrType, Mod} = Listener, + #s{name = NodeName, listeners = Listeners, + acceptors = Acceptors} = State) -> + case can_add_proto(Listener, State) of ok -> - case listen_proto(Mod, NodeName) of + case listen_proto(Listener, NodeName) of {ok, L = {LSocket, _, _}} -> try APid = Mod:accept(LSocket), true = is_pid(APid), - State#s{listeners = [{Mod, L}|Listeners], - acceptors = [{APid, Mod}|Acceptors]} + State#s{listeners = [{Listener, L}|Listeners], + acceptors = [{APid, Listener}|Acceptors]} catch _:E -> ST = erlang:get_stacktrace(), catch Mod:close(LSocket), error_msg( "Accept failed for protocol ~p with reason: ~p~n" - "Stacktrace: ~p", [Mod, E, ST]), + "Stacktrace: ~p", [Listener, E, ST]), start_ensure_config_timer(State) end; {error, eafnosupport} -> State; @@ -505,7 +519,7 @@ add_proto(Mod, #s{name = NodeName, listeners = Listeners, _Error -> start_ensure_config_timer(State) end; {error, Reason} -> - error_msg("Ignoring ~p listener, reason: ~p", [Mod, Reason]), + error_msg("Ignoring ~p listener, reason: ~p", [Listener, Reason]), State end. @@ -516,12 +530,12 @@ start_ensure_config_timer(#s{ensure_config_timer = undefined} = State) -> start_ensure_config_timer(#s{} = State) -> State. -remove_proto(Mod, #s{listeners = Listeners, acceptors = Acceptors} = State) -> - case proplists:get_value(Mod, Listeners) of +remove_proto({_AddrType, Mod} = Listener, + #s{listeners = Listeners, acceptors = Acceptors} = State) -> + case proplists:get_value(Listener, Listeners) of {LSocket, _, _} -> - info_msg("Closing listener ~p", [Mod]), - [erlang:unlink(P) || {P, M} <- Acceptors, M =:= Mod], - + info_msg("Closing listener ~p", [Listener]), + [erlang:unlink(P) || {P, M} <- Acceptors, M =:= Listener], ToWait = case lists:member(Mod, [inet_tls_dist, inet6_tls_dist]) of true -> @@ -551,14 +565,14 @@ remove_proto(Mod, #s{listeners = Listeners, acceptors = Acceptors} = State) -> "reason: ~p", [Proc, Reason]), exit(Proc, kill) end - end, lists:usort([P || {P, M} <- Acceptors, M =:= Mod] ++ - ToWait)), + end, lists:usort([P || {P, M} <- Acceptors, M =:= Listener] ++ + ToWait)), - State#s{listeners = proplists:delete(Mod, Listeners), - acceptors = [{P, M} || {P, M} <- Acceptors, M =/= Mod]}; + State#s{listeners = proplists:delete(Listener, Listeners), + acceptors = [{P, M} || {P, M} <- Acceptors, M =/= Listener]}; undefined -> info_msg("ignoring closing of ~p because listener is not started", - [Mod]), + [Listener]), State end. @@ -572,7 +586,7 @@ extract_tls_acceptors() -> {[LocalSocket, WorldSocket], [LocalCon, WorldCon]} end. -listen_proto(Module, NodeName) -> +listen_proto({AddrType, Module}, NodeName) -> NameStr = atom_to_list(NodeName), Port = cb_epmd:port_for_node(Module, NameStr), info_msg("Starting ~p listener on ~p...", [Module, Port]), @@ -587,7 +601,8 @@ listen_proto(Module, NodeName) -> Error -> Error end end, - case with_dist_port(Port, ListenFun) of + ListenAddr = get_listen_addr(AddrType, Module), + case with_dist_ip_and_port(ListenAddr, Port, ListenFun) of {ok, Res} -> {ok, Res}; ignore -> info_msg("Ignoring starting dist ~p on port ~p", [Module, Port]), @@ -598,6 +613,15 @@ listen_proto(Module, NodeName) -> Error end. +get_listen_addr(AddrType, Module) -> + AFamily = proto_to_family(Module), + IPStr = case AddrType of + local -> misc:localhost(AFamily, []); + external -> misc:inaddr_any(AFamily, []) + end, + {ok, IPParsed} = inet:parse_address(IPStr), + IPParsed. + %% Backward compat: we need to register ns_server non tls port on epmd to allow %% old nodes to find this node %% This code can be dropped if upgrade from Alice is not supported @@ -618,14 +642,15 @@ maybe_register_on_epmd(Module, NodeName, PortNo) maybe_register_on_epmd(_Module, _NodeName, _PortNo) -> ok. -can_add_proto(P, #s{listeners = L}) -> - case is_valid_protocol(P) of +can_add_proto({_AddrType, Proto}, #s{listeners = Listeners}) -> + case is_valid_protocol(Proto) of true -> - case proplists:is_defined(P, L) of + Protos = [P || {{_, P}, _} <- Listeners], + case lists:member(Proto, Protos) of false -> - HasInet6Tls = proplists:is_defined(inet6_tls_dist, L), - HasInetTls = proplists:is_defined(inet_tls_dist, L), - case P of + HasInet6Tls = lists:member(inet6_tls_dist, Protos), + HasInetTls = lists:member(inet_tls_dist, Protos), + case Proto of inet6_tls_dist when HasInetTls -> {error, {already_has, inet_tls_dist}}; inet_tls_dist when HasInet6Tls -> @@ -696,7 +721,7 @@ handle_reload_config(State) -> case not_started_required_listeners(NewState) of [] -> #s{listeners = Listeners} = NewState, - L = [proto2netsettings(M) || {M, _} <- Listeners], + L = [proto2netsettings(M) || {{_, M}, _} <- Listeners], {reply, {ok, L}, NewState}; NotStartedRequired -> error_msg("Failed to start required dist listeners ~p", @@ -726,23 +751,28 @@ ensure_config(#s{listeners = Listeners} = State) -> State3. get_protos(#s{name = Name, config = Config}) -> - Protos = - case cb_epmd:is_local_node(Name) of - true -> - conf(local_listeners, Config); - false -> - conf(external_listeners, Config) ++ - conf(local_listeners, Config) - end, - lists:usort(Protos). + case cb_epmd:is_local_node(Name) of + true -> + [{local, P} || P <- lists:usort(conf(local_listeners, Config))]; + false -> + External = lists:usort(conf(external_listeners, Config)), + Local = lists:usort(conf(local_listeners, Config)), + OnlyLocal = Local -- External, + [{local, P} || P <- OnlyLocal] ++ + [{external, P} || P <- External] + end. get_required_protos(#s{name = Name, config = Config}) -> Local = conf(preferred_local_proto, Config), Ext = conf(preferred_external_proto, Config), case {cb_epmd:node_type(atom_to_list(Name)), cb_epmd:is_local_node(Name)} of {executioner, _} -> []; - {_, true} -> [Local]; - {_, false} -> lists:usort([Local, Ext]) + {_, true} -> [{local, Local}]; + {_, false} -> + case Ext == Local of + true -> [{external, Ext}]; + false -> [{local, Local}, {external, Ext}] + end end. info_msg(F, A) -> @@ -856,7 +886,7 @@ store_config(Cfg) -> end. format_error({not_started, Protocols}) -> - PS = string:join([proto2str(P) || P <- Protocols], ", "), + PS = string:join([proto2str(P) || {_, P} <- Protocols], ", "), io_lib:format("Failed to start the following required listeners: ~p", [PS]); format_error({invalid_cb_dist_config, File, invalid_format}) -> io_lib:format("Invalid format of cb_dist config file (~s)", [File]); From c60f1c5c50ab96233628543102268451761862cb Mon Sep 17 00:00:00 2001 From: Timofey Barmin Date: Tue, 8 Dec 2020 11:34:11 -0800 Subject: [PATCH 19/33] MB-48047:[BP] cb_dist: Use IPv{4|6} instead of ipv{4|6} Reviewed-on: http://review.couchbase.org/c/ns_server/+/141723 Change-Id: Ib73c70f517ae6d5eafae9bb1625332387590c654 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164941 Reviewed-by: Timofey Barmin Tested-by: Abhijeeth Nuthan Well-Formed: Restriction Checker --- src/cb_dist.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cb_dist.erl b/src/cb_dist.erl index 8361ef4f50..af9e6d6467 100644 --- a/src/cb_dist.erl +++ b/src/cb_dist.erl @@ -907,10 +907,10 @@ format_error(Unknown) -> netsettings2str(S) -> proto2str(netsettings2proto(S)). -proto2str(inet_tcp_dist) -> "TCP-ipv4"; -proto2str(inet_tls_dist) -> "TLS-ipv4"; -proto2str(inet6_tcp_dist) -> "TCP-ipv6"; -proto2str(inet6_tls_dist) -> "TLS-ipv6". +proto2str(inet_tcp_dist) -> "TCP-IPv4"; +proto2str(inet_tls_dist) -> "TLS-IPv4"; +proto2str(inet6_tcp_dist) -> "TCP-IPv6"; +proto2str(inet6_tls_dist) -> "TLS-IPv6". netsettings2proto({inet, false}) -> inet_tcp_dist; netsettings2proto({inet, true}) -> inet_tls_dist; From a495a44a85b57a82c843fb32068200f342eb30e6 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Fri, 11 Dec 2020 14:16:51 -0800 Subject: [PATCH 20/33] MB-48047:[BP] Enforce 'strict' clusterEncryptionLevel in CAPI The SSL endpoint should always open ports on INADDR_ANY, and the non-ssl endpoint should listen on loopback whenever clusterEncryptionLevel is 'strict'. Backports change related to MB-42373. Reviewed-on: http://review.couchbase.org/c/ns_server/+/142000 Change-Id: I8a1476b470c7d6a28c2c8daa56b3374e9bb44749 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164942 Reviewed-by: Timofey Barmin Tested-by: Abhijeeth Nuthan Well-Formed: Restriction Checker --- deps/ns_couchdb/src/cb_config_couch_sync.erl | 10 ++++++++++ src/menelaus_web.erl | 1 + src/ns_ssl_services_setup.erl | 11 +++-------- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/deps/ns_couchdb/src/cb_config_couch_sync.erl b/deps/ns_couchdb/src/cb_config_couch_sync.erl index b1689d72e7..c07f114240 100644 --- a/deps/ns_couchdb/src/cb_config_couch_sync.erl +++ b/deps/ns_couchdb/src/cb_config_couch_sync.erl @@ -64,6 +64,13 @@ handle_call(_Request, _From, State) -> handle_cast(_Request, State) -> {noreply, State}. +handle_info({notable_change, cluster_encryption_level = Key}, State) -> + flush_for_key(Key), + apply_to_couch_config(httpd, ip4_bind_address, + menelaus_web:get_addr(inet, false), true), + apply_to_couch_config(httpd, ip6_bind_address, + menelaus_web:get_addr(inet6, false), true), + {noreply, State}; handle_info({notable_change, secure_headers = Key}, State) -> flush_for_key(Key), @@ -122,6 +129,7 @@ is_notable_key({couchdb, _}) -> true; is_notable_key({{node, Node, {couchdb, _}}}) when Node =:= node() -> true; is_notable_key(secure_headers) -> true; is_notable_key(audit) -> true; +is_notable_key(cluster_encryption_level) -> true; is_notable_key(_) -> false. handle_config_event({Key, _Value}) -> @@ -136,6 +144,8 @@ handle_config_event(_) -> to_global_key({couchdb, _} = Key) -> Key; +to_global_key(cluster_encryption_level = Key) -> + Key; to_global_key(secure_headers = Key) -> Key; to_global_key(audit = Key) -> diff --git a/src/menelaus_web.erl b/src/menelaus_web.erl index 31bed1a833..eab594244b 100644 --- a/src/menelaus_web.erl +++ b/src/menelaus_web.erl @@ -34,6 +34,7 @@ webconfig/0, webconfig/1, get_uuid/0, + get_addr/2, init/1]). -export([ns_log_cat/1, ns_log_code_string/1, ns_log_prepare_message/2]). diff --git a/src/ns_ssl_services_setup.erl b/src/ns_ssl_services_setup.erl index 3756b504ae..13ddec9c1c 100644 --- a/src/ns_ssl_services_setup.erl +++ b/src/ns_ssl_services_setup.erl @@ -84,7 +84,8 @@ do_start_link_capi_service(SSLPort) -> Options = [{port, SSLPort}, {ssl, true}, - {ssl_opts, ssl_server_opts()}], + {ssl_opts, ssl_server_opts()}, + {ip, misc:inaddr_any()}], %% the following is copied almost verbatim from couch_httpd. The %% difference is that we don't touch "ssl" key of couch config and @@ -96,11 +97,6 @@ do_start_link_capi_service(SSLPort) -> %% fossilized couchdb code. %% %% Also couchdb code is reformatted for ns_server formatting - Field = case misc:is_ipv6() of - true -> "ip6_bind_address"; - false -> "ip4_bind_address" - end, - BindAddress = couch_config:get("httpd", Field, any), DefaultSpec = "{couch_httpd_db, handle_request}", DefaultFun = make_arity_1_fun( couch_config:get("httpd", "default_handler", DefaultSpec)), @@ -151,8 +147,7 @@ do_start_link_capi_service(SSLPort) -> %% set mochiweb options FinalOptions = lists:append([Options, ServerOptions, [{loop, Loop}, - {name, https}, - {ip, BindAddress}]]), + {name, https}]]), %% launch mochiweb {ok, _Pid} = case mochiweb_http:start(FinalOptions) of From 4085808c7cf0b760dbb086a64f8e98c1f595c8dd Mon Sep 17 00:00:00 2001 From: Timofey Barmin Date: Thu, 17 Dec 2020 17:28:02 -0800 Subject: [PATCH 21/33] MB-48047:[BP] Make switching to afamily for which we don't ... ... have external listener forbidden. As it might lead to incorrect state where we have only loopback listener for preferred afamily Backports change related to MB-42373. Reviewed-on: http://review.couchbase.org/c/ns_server/+/142448 Change-Id: I7b437379505054e5fd80dd459e7f9acb4574464e Reviewed-on: http://review.couchbase.org/c/ns_server/+/164943 Reviewed-by: Timofey Barmin Tested-by: Abhijeeth Nuthan Well-Formed: Restriction Checker --- src/cb_dist.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cb_dist.erl b/src/cb_dist.erl index af9e6d6467..e40fa4dc83 100644 --- a/src/cb_dist.erl +++ b/src/cb_dist.erl @@ -823,7 +823,7 @@ validate_config_file(CfgFile) -> orelse throw(not_unique_listeners), length(lists:usort(ExternalListeners)) == length(ExternalListeners) orelse throw(not_unique_listeners), - lists:member(ExtPreferred, ExternalListeners ++ LocalListeners) + lists:member(ExtPreferred, ExternalListeners) orelse throw({missing_preferred_external_listener, ExtPreferred}), lists:member(LocalPreferred, LocalListeners) orelse throw({missing_preferred_local_listener, LocalPreferred}), From 76c04baafe80f0a6f1bfa8b177e67b15ec4eb260 Mon Sep 17 00:00:00 2001 From: Timofey Barmin Date: Fri, 18 Dec 2020 13:08:25 -0800 Subject: [PATCH 22/33] MB-48047:[BP] Add endpoint to disable unused cb_dist listeners In some situations, when changing net settings we don't know the previous state of the system. It makes disabling of unused interfaces problematic. The new api just closes all the listeners that are in use. It will be used by UI for correct node initialization and by CLI for changing ip-family and n2n encryption. Backports change related to MB-42373. Reviewed-on: http://review.couchbase.org/c/ns_server/+/142449 Change-Id: Ic6a66d72df4a15a923497035eab462191a83b82a Reviewed-on: http://review.couchbase.org/c/ns_server/+/164944 Reviewed-by: Timofey Barmin Tested-by: Abhijeeth Nuthan Well-Formed: Restriction Checker --- src/menelaus_web.erl | 3 +++ src/menelaus_web_node.erl | 5 +++++ src/netconfig_updater.erl | 9 +++++++++ 3 files changed, 17 insertions(+) diff --git a/src/menelaus_web.erl b/src/menelaus_web.erl index eab594244b..76c0953b80 100644 --- a/src/menelaus_web.erl +++ b/src/menelaus_web.erl @@ -538,6 +538,9 @@ get_action(Req, {AppRoot, IsSSL, Plugins}, Path, PathTokens) -> ["node", "controller", "disableExternalListener"] -> {{[admin, setup], write}, fun menelaus_web_node:handle_change_external_listeners/2, [disable]}; + ["node", "controller", "disableUnusedExternalListeners"] -> + {{[admin, setup], write}, + fun menelaus_web_node:handle_change_external_listeners/2, [disable_unused]}; ["settings", "web"] -> {{[admin, setup], write}, fun menelaus_web_settings:handle_settings_web_post/1}; ["settings", "alerts"] -> diff --git a/src/menelaus_web_node.erl b/src/menelaus_web_node.erl index 7af98e84e9..19bf5c6d99 100644 --- a/src/menelaus_web_node.erl +++ b/src/menelaus_web_node.erl @@ -884,6 +884,11 @@ handle_setup_net_config(Req) -> erlang:exit(normal) end, Req, form, net_config_validators(false)). +handle_change_external_listeners(disable_unused, Req) -> + case netconfig_updater:change_external_listeners(disable_unused, []) of + ok -> menelaus_util:reply(Req, 200); + {error, Msg} -> menelaus_util:reply_global_error(Req, Msg) + end; handle_change_external_listeners(Action, Req) -> menelaus_util:assert_is_65(), validator:handle( diff --git a/src/netconfig_updater.erl b/src/netconfig_updater.erl index 0197f4cba1..c7ced8163d 100644 --- a/src/netconfig_updater.erl +++ b/src/netconfig_updater.erl @@ -70,6 +70,15 @@ handle_call({apply_config, Config}, _From, State) -> {error, _} = Error -> {reply, Error, State, hibernate} end; +handle_call({change_listeners, disable_unused, _Config}, _From, State) -> + CurListeners = cb_dist:external_listeners(), + CurAFamily = cb_dist:address_family(), + CurNEncrypt = cb_dist:external_encryption(), + NewListeners = [{CurAFamily, CurNEncrypt}], + NewConfig = [{externalListeners, NewListeners}], + CurConfig = [{externalListeners, CurListeners}], + handle_with_marker(apply_config, CurConfig, NewConfig, State); + handle_call({change_listeners, Action, Config}, _From, State) -> CurProtos = cb_dist:external_listeners(), AFamily = proplists:get_value(afamily, Config, cb_dist:address_family()), From b9d71745bc0061227f9519744921a19df639a3b1 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Tue, 2 Mar 2021 13:43:34 -0800 Subject: [PATCH 23/33] MB-48047:[BP] flush accept messages after we kill the acceptor We receive accept messages from the acceptor as a consequence of cb_dist being the middle man for handling the erlang distribution. When we remove a particular proto we make sure to kill the acceptor before we do so we can potentially have message(s) in our mailbox from the killed acceptor, make sure we clean up the messages before proceeding. Backports change related to MB-44626. Reviewed-on: http://review.couchbase.org/c/ns_server/+/147393 Change-Id: I961a177df79a7e5657776a9e2dd4e11cd91d65e7 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164945 Tested-by: Abhijeeth Nuthan Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker --- src/cb_dist.erl | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/cb_dist.erl b/src/cb_dist.erl index e40fa4dc83..0a9be2455c 100644 --- a/src/cb_dist.erl +++ b/src/cb_dist.erl @@ -564,7 +564,10 @@ remove_proto({_AddrType, Mod} = Listener, error_msg("Wait for acceptor: ~p failed with " "reason: ~p", [Proc, Reason]), exit(Proc, kill) - end + end, + %% Since we killed the acceptor flush the accept messages + %% from it. + flush_accept_messages(Proc) end, lists:usort([P || {P, M} <- Acceptors, M =:= Listener] ++ ToWait)), @@ -586,6 +589,16 @@ extract_tls_acceptors() -> {[LocalSocket, WorldSocket], [LocalCon, WorldCon]} end. +flush_accept_messages(AcceptorPid) -> + receive + {accept, AcceptorPid, _, _, _} = Msg -> + info_msg("Ignoring message from acceptor ~p", [Msg]), + flush_accept_messages(AcceptorPid) + after + 0 -> + ok + end. + listen_proto({AddrType, Module}, NodeName) -> NameStr = atom_to_list(NodeName), Port = cb_epmd:port_for_node(Module, NameStr), From 59a650d0b49d0526e4d8402d9175e56db066beb6 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Tue, 2 Mar 2021 11:31:22 -0800 Subject: [PATCH 24/33] MB-48047:[BP] Graceful exit of connector won't work till ... ... connector pid receives controller message from acceptor. Reviewed-on: http://review.couchbase.org/c/ns_server/+/147516 Change-Id: I64affbc6976fe9505df56d0e1c9c38b5b88bde57 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164946 Tested-by: Abhijeeth Nuthan Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker --- src/cb_dist.erl | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/cb_dist.erl b/src/cb_dist.erl index 0a9be2455c..f7d04fb4d3 100644 --- a/src/cb_dist.erl +++ b/src/cb_dist.erl @@ -379,8 +379,7 @@ handle_call(restart_tls, _From, #s{connections = Connections, _ -> info_msg("Closing connection ~p because of " "tls restart", [Pid]), - catch erlang:demonitor(Mon, [flush]), - close_dist_connection(Pid, KernelPid), + close_dist_connection(Mon, Pid, KernelPid), false end; {_, _} -> @@ -950,15 +949,19 @@ with_registered_connection(Fun, Module) -> erlang:raise(C, E, ST) end. -update_connection_pid(Ref, Pid, #s{connections = Connections, - kernel_pid = KernelPid} = State) -> +update_connection_pid(Ref, Pid, #s{connections = Connections} = State) -> case lists:keytake(Ref, #con.ref, Connections) of {value, Con, Rest} when Pid =:= undefined -> info_msg("Removed connection: ~p", [Con]), State#s{connections = Rest}; {value, #con{pid = shutdown}, Rest} -> info_msg("Closing connection ~p because of tls restart", [Pid]), - close_dist_connection(Pid, KernelPid), + %% No point in using close_dist_connection here as {KernelPid, + %% disconnect} message is only useful after we forward the + %% controller message to AcceptorPid(which no longer exists), and it + %% in turn sends another controller message to ConPid to proceed + %% accepting connection. + force_close_dist_connection(Pid), State#s{connections = Rest}; {value, Con, Rest} -> MonRef = erlang:monitor(process, Pid), @@ -970,11 +973,15 @@ update_connection_pid(Ref, Pid, #s{connections = Connections, State end. -close_dist_connection(Pid, KernelPid) -> +close_dist_connection(MonRef, Pid, KernelPid) -> + catch erlang:demonitor(MonRef, [flush]), Pid ! {KernelPid, disconnect}, case misc:wait_for_process(Pid, ?TERMINATE_TIMEOUT) of ok -> ok; {error, Reason} -> error_msg("Close connection ~p error: ~p", [Pid, Reason]), - exit(Pid, kill) + force_close_dist_connection(Pid) end. + +force_close_dist_connection(ConPid) -> + exit(ConPid, kill). From 8af09c1171b46f104d4c73e310abafeecbb27ac5 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Tue, 3 Aug 2021 14:22:23 -0700 Subject: [PATCH 25/33] MB-48047:[BP] Optionally register node with epmd. Do not crash if fail to register with epmd, we might have killed it. Backports change related to MB-47702. Reviewed-on: http://review.couchbase.org/c/ns_server/+/158664 Change-Id: Ia6abb52fd34c361e3168a2907c5f049abb4e890d Reviewed-on: http://review.couchbase.org/c/ns_server/+/164947 Tested-by: Abhijeeth Nuthan Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker --- src/cb_dist.erl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/cb_dist.erl b/src/cb_dist.erl index f7d04fb4d3..cc8c4251c8 100644 --- a/src/cb_dist.erl +++ b/src/cb_dist.erl @@ -606,10 +606,8 @@ listen_proto({AddrType, Module}, NodeName) -> fun () -> case Module:listen(NodeName) of {ok, _} = Res -> - case maybe_register_on_epmd(Module, NodeName, Port) of - ok -> Res; - {error, _} = Error -> Error - end; + maybe_register_on_epmd(Module, NodeName, Port), + Res; Error -> Error end end, @@ -646,7 +644,10 @@ maybe_register_on_epmd(Module, NodeName, PortNo) case erl_epmd:register_node(NodeName, PortNo, Family) of {ok, _} -> ok; {error, already_registered} -> ok; - Error -> Error + Error -> + info_msg("Failed to register ~p with epmd. Reason ~p", + [{NodeName, PortNo, Family}, Error]), + Error end; _ -> ok From bc4a5be05409a2735f4af44d07781a41e44ff1a6 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Tue, 3 Aug 2021 10:04:54 -0700 Subject: [PATCH 26/33] MB-48047:[BP] Conditionally kill epmd on settings change and restart In order to support EnforceTLS(don't listen on unencrypted ports) and address family only feature, it was decided to kill epmd. Both the above features are only supported above 7.0. The reason for killing epmd is that it no longer required from 6.5 and it is hard to make it listen only on a particular address family. The ports opened by epmd are also unencrypted ports. In order, to kill epmd we need to start it with relaxed_command_check option. Using "epmd -kill" since we can only kill epmd if, 1. no node names are registered 2. started with relaxed_command_check option Backports change related to MB-47702. Reviewed-on: http://review.couchbase.org/c/ns_server/+/158666 Change-Id: I537e68a14b0455de5e4dd15bf40fcc91e76ad742 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164948 Tested-by: Abhijeeth Nuthan Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker --- cluster_run | 14 +++++++++ couchbase-server.sh.in | 16 +++++++--- src/menelaus_event.erl | 1 + src/netconfig_updater.erl | 61 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 4 deletions(-) diff --git a/cluster_run b/cluster_run index be40375cc1..a052b292c9 100755 --- a/cluster_run +++ b/cluster_run @@ -252,6 +252,7 @@ def erlang_args_for_node(i, ebin_path, extra_args, args_prefix): "-proto_dist", "cb", "-ssl_dist_optfile", ssloptfile, "-epmd_module", "cb_epmd", + "-start_epmd", "false", "-hidden", "-kernel", "dist_config_file", quote_string_for_erl(cb_dist_config), "-kernel", "inetrc", "\"etc/hosts.cfg\"", @@ -318,6 +319,8 @@ def erlang_args_for_node(i, ebin_path, extra_args, args_prefix): return args +def abs_path_join(*args): + return os.path.abspath(os.path.join(*args)) def start_cluster(num_nodes, start_index, extra_args, args_prefix, force_community): @@ -352,6 +355,7 @@ def start_cluster(num_nodes, start_index, extra_args, args_prefix, params['env']['ERL_CRASH_DUMP'] = crash_dump_base + '.babysitter' params['env']['COUCHBASE_SMALLER_PKEYS'] = '1' + params['env']['ERL_EPMD_RELAXED_COMMAND_CHECK'] = '1' params['close_fds'] = True if platform.system() == "Windows": @@ -371,6 +375,16 @@ def start_cluster(num_nodes, start_index, extra_args, args_prefix, # ability to it shutdown carefully or otherwise params['preexec_fn'] = os.setpgrp + node_name = 'n_%d' % node_num + script_dir = os.path.dirname(os.path.realpath(__file__)) + if not os.path.isfile(abs_path_join(script_dir, 'data', node_name, + 'no_epmd')): + pr_epmd = subprocess.Popen(["erl", "-noshell", "-setcookie", + "nocookie", "-sname", "init", "-run", + "init", "stop"], + env=params['env']) + pr_epmd.wait() + pr = subprocess.Popen(args, **params) if w is not None: os.close(r) diff --git a/couchbase-server.sh.in b/couchbase-server.sh.in index 9833098e9d..9873d7af32 100644 --- a/couchbase-server.sh.in +++ b/couchbase-server.sh.in @@ -65,6 +65,9 @@ export ERL_CRASH_DUMP ERL_FULLSWEEP_AFTER=512 export ERL_FULLSWEEP_AFTER +ERL_EPMD_RELAXED_COMMAND_CHECK=1 +export ERL_EPMD_RELAXED_COMMAND_CHECK + # For some obscure reason erl requires HOME environment variable to be set. if [ -z "$HOME" ] then @@ -106,11 +109,14 @@ _prepare_datadir () { } _maybe_start_epmd () { - # Initialize distributed erlang on the system (i.e. epmd) - erl -noshell -setcookie nocookie -sname init -run init stop 2>&1 > /dev/null - if [ $? -ne 0 ] + if [ ! -f $CB_DATA_DIR/no_epmd ] then - exit 1 + # Initialize distributed erlang on the system (i.e. epmd) + erl -noshell -setcookie nocookie -sname init -run init stop 2>&1 > /dev/null + if [ $? -ne 0 ] + then + exit 1 + fi fi } @@ -173,6 +179,7 @@ _start() { -name 'babysitter_of_ns_1@cb.local' \ -proto_dist cb \ -epmd_module cb_epmd \ + -start_epmd false \ -ssl_dist_optfile $SSL_DIST_OPTFILE \ -setcookie nocookie \ $* \ @@ -196,6 +203,7 @@ _stop() { inetrc "\"$HOSTS_CFG_FILE\"" \ -proto_dist cb \ -epmd_module cb_epmd \ + -start_epmd false \ -ssl_dist_optfile $SSL_DIST_OPTFILE \ -noshell \ -hidden \ diff --git a/src/menelaus_event.erl b/src/menelaus_event.erl index 8bf9a616d0..e9c3477069 100644 --- a/src/menelaus_event.erl +++ b/src/menelaus_event.erl @@ -211,6 +211,7 @@ maybe_restart(#state{webconfig = WebConfigOld, case WebConfigNew =:= WebConfigOld andalso DisableOld =:= DisableNew of true -> State; false -> {ok, _} = menelaus_web_sup:restart_web_servers(), + netconfig_updater:maybe_kill_epmd(), State#state{webconfig = WebConfigNew, disable_non_ssl_ports = DisableNew} end. diff --git a/src/netconfig_updater.erl b/src/netconfig_updater.erl index c7ced8163d..77f34ee09b 100644 --- a/src/netconfig_updater.erl +++ b/src/netconfig_updater.erl @@ -4,6 +4,7 @@ %% API -export([start_link/0, + maybe_kill_epmd/0, apply_config/1, change_external_listeners/2, ensure_tls_dist_started/1]). @@ -17,6 +18,8 @@ -include_lib("kernel/include/net_address.hrl"). -include("ns_common.hrl"). +-define(CAN_KILL_EPMD, ?get_param(can_kill_epmd, true)). + %%%=================================================================== %%% API %%%=================================================================== @@ -41,6 +44,10 @@ init([]) -> ServerName = ?MODULE, register(ServerName, self()), ensure_ns_config_settings_in_order(), + %% We choose to kill epmd at startup if required. This is mainly required + %% for windows as for unix systems epmd will not be started because of + %% no_epmd file. + misc:is_windows() andalso maybe_kill_epmd(), proc_lib:init_ack({ok, self()}), case misc:consult_marker(update_marker_path()) of {ok, [Cmd]} -> @@ -348,6 +355,60 @@ check_nodename_resolvable(Node, AFamily) -> {error, iolist_to_binary(M)} end. +epmd_executable() -> + case misc:is_windows() of + true -> + %% Epmd doesn't exist in the bin path for windows so we pass the + %% erts_bin_path env to point us to it. + {ok, ERTSPath} = application:get_env(ns_server, + erts_bin_path), + filename:join(ERTSPath, "epmd.exe"); + false -> + path_config:component_path(bin, "epmd") + end. + +kill_epmd() -> + Path = epmd_executable(), + Port = erlang:open_port({spawn_executable, Path}, + [stderr_to_stdout, binary, + stream, exit_status, hide, + {args, ["-kill"]}]), + {ExitStatus, Output} = wait_for_exit(Port, []), + case ExitStatus of + 0 -> + ok; + _ -> + ?log_error("Failed to kill epmd: ~p", [{ExitStatus, Output}]), + error + end. + +wait_for_exit(Port, Output) -> + receive + {Port, {data, Data}} -> + wait_for_exit(Port, Output ++ binary_to_list(Data)); + {Port, {exit_status, Status}} -> + {Status, Output} + end. + +maybe_kill_epmd() -> + NoEpmdFile = path_config:component_path(data, "no_epmd"), + case ?CAN_KILL_EPMD andalso misc:disable_non_ssl_ports() andalso + misc:is_strict_possible() of + true -> + try + misc:create_marker(NoEpmdFile), + ?log_info("Killing epmd ..."), + kill_epmd() + catch + T:E -> + S = erlang:get_stacktrace(), + ?log_error("Exception while killing epmd ~p", [{T, E, S}]) + end; + false -> + file:delete(NoEpmdFile), + ok + end. + %% This function is needed in two cases: %% - migration for address family settings to 6.5 %% - allow manual changes in dist_cfg file From cb4c747fe6c15a6a11090aa5c105288b29913e76 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Thu, 6 May 2021 17:28:06 -0700 Subject: [PATCH 27/33] MB-48047:[BP] Set appropriate defaults for listeners On upgrade, we won't find local_listerners in the dist_cfg file as we never recorded them. Since we no longer start both inet and inet6 listeners for both local and external, always have the preferred proto as the default list of listeners. This avoids the situation where we start the wrong listener on upgrade. Backports change related to MB-46105. Reviewed-on: http://review.couchbase.org/c/ns_server/+/153079 Change-Id: Ieef1c28ac579cbede8d0a8d3d5dd572b71acf843 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164949 Tested-by: Abhijeeth Nuthan Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker --- src/cb_dist.erl | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/cb_dist.erl b/src/cb_dist.erl index cc8c4251c8..31b35644dc 100644 --- a/src/cb_dist.erl +++ b/src/cb_dist.erl @@ -683,13 +683,25 @@ is_valid_protocol(P) -> inet6_tls_dist]). conf(Prop, Conf) -> - proplists:get_value(Prop, Conf, proplists:get_value(Prop, defaults())). - -defaults() -> + %% See comments for how we determine defaults. + proplists:get_value(Prop, Conf, proplists:get_value(Prop, defaults(Conf))). + +%% From 6.6.4, +%% 1. we do not start both [inet_tcp_dist, inet6_tcp_dist] protos for +%% local_listeners and external_listeners by default as we introduced the +%% address family only feature. +%% 2. we write the local_listeners to the dist_cfg file. +%% +%% In cases, like on upgrade from pre 7.0 we don't find local_listeners in the +%% dist_cfg and we always need to start the preferred proto for local and +%% external. Therefore, the defaults should reflect that for listeners. +defaults(Conf) -> [{preferred_external_proto, inet_tcp_dist}, {preferred_local_proto, inet_tcp_dist}, - {local_listeners, [inet_tcp_dist]}, - {external_listeners, [inet_tcp_dist]}]. + {local_listeners, [proplists:get_value(preferred_local_proto, Conf, + inet_tcp_dist)]}, + {external_listeners, [proplists:get_value(preferred_external_proto, Conf, + inet_tcp_dist)]}]. transform_old_to_new_config(Dist) -> DistType = list_to_atom((atom_to_list(Dist) ++ "_dist")), @@ -819,7 +831,7 @@ validate_config_file(CfgFile) -> Cfg = read_config(CfgFile, false), is_list(Cfg) orelse throw(not_list), ([E || {_, _} = E <- Cfg] == Cfg) orelse throw(not_proplist), - Unknown = proplists:get_keys(Cfg) -- proplists:get_keys(defaults()), + Unknown = proplists:get_keys(Cfg) -- proplists:get_keys(defaults(Cfg)), (Unknown == []) orelse throw({unknown_props, Unknown}), ExtPreferred = conf(preferred_external_proto, Cfg), is_valid_protocol(ExtPreferred) From 7ddc15bfe4b027fbcef445537ba871e60dcdb778 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Thu, 19 Aug 2021 12:44:40 -0700 Subject: [PATCH 28/33] MB-48047:[BP] restart cb_config_couch_sync on crash Backports change related to MB-48031. Reviewed-on: http://review.couchbase.org/c/ns_server/+/159728 Change-Id: Ifbcffed7eff7808e7b7e7d694488b1f32fa73006 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164950 Tested-by: Abhijeeth Nuthan Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker --- deps/ns_couchdb/src/cb_config_couch_sync.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/ns_couchdb/src/cb_config_couch_sync.erl b/deps/ns_couchdb/src/cb_config_couch_sync.erl index c07f114240..7d87b51816 100644 --- a/deps/ns_couchdb/src/cb_config_couch_sync.erl +++ b/deps/ns_couchdb/src/cb_config_couch_sync.erl @@ -31,7 +31,7 @@ -record(state, {}). start_link() -> - gen_server:start({local, ?MODULE}, ?MODULE, [], []). + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). init([]) -> ns_pubsub:subscribe_link(ns_config_events, fun handle_config_event/1), From 390acaff218eaf5dbe3fa58495583ef91cf86fe4 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Mon, 23 Aug 2021 09:26:51 -0700 Subject: [PATCH 29/33] MB-48047:[BP] Always annouce cluster_encryption_level on start. When a node is removed from cluster, the cluster_encryption_level key is deleted from the ns_config. Before this patch, we would only generate a change event when the key is present in the config. We would miss any changes to cluster_encryption_level in this case. Therefore, always generate a change message for cluster_encryption_level on startup. Backports change related to MB-48084. Reviewed-on: http://review.couchbase.org/c/ns_server/+/159921 Change-Id: Iad43c8b8f621bdd383fcb52251a4195850d9e075 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164951 Tested-by: Abhijeeth Nuthan Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker --- deps/ns_couchdb/src/cb_config_couch_sync.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deps/ns_couchdb/src/cb_config_couch_sync.erl b/deps/ns_couchdb/src/cb_config_couch_sync.erl index 7d87b51816..213b2e1b9d 100644 --- a/deps/ns_couchdb/src/cb_config_couch_sync.erl +++ b/deps/ns_couchdb/src/cb_config_couch_sync.erl @@ -172,6 +172,8 @@ do_flush_for_key(Key) -> announce_notable_keys() -> KVList = ns_config:get_kv_list(), + %% Always annouce cluster_encryption_level even if not present in ns_config. + ?MODULE ! {notable_change, cluster_encryption_level}, lists:foreach( fun ({Key, _Value}) -> case is_notable_key(Key) of From 65d77eaa6430b76f799d6bf712ef50c57ea92a0c Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Thu, 19 Aug 2021 13:36:35 -0700 Subject: [PATCH 30/33] MB-48047:[BP] Prevent socket close on clusterEncryptionLevel change When clusterEncryptionLevel changes from strict to other setting or vice versa we restart web server. On occasion the web server is restarted prior to settings API response. Backports change related to MB-47663. Reviewed-on: http://review.couchbase.org/c/ns_server/+/159808 Change-Id: I283d8419b923640a5479f92bc8196704c064db81 Reviewed-on: http://review.couchbase.org/c/ns_server/+/164952 Tested-by: Abhijeeth Nuthan Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker --- src/menelaus_web_settings.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/menelaus_web_settings.erl b/src/menelaus_web_settings.erl index 7930073f89..ce6ac1c5f6 100644 --- a/src/menelaus_web_settings.erl +++ b/src/menelaus_web_settings.erl @@ -319,6 +319,9 @@ handle_post(Type, Req) -> handle_post(Type, [], Req). handle_post(Type, Keys, Req) -> + %% NOTE: due to a potential restart we need to protect + %% ourselves from 'death signal' of parent + erlang:process_flag(trap_exit, true), case parse_post_data(conf(Type), Keys, mochiweb_request:recv_body(Req), fun is_allowed_setting/1) of {ok, ToSet} -> @@ -332,7 +335,8 @@ handle_post(Type, Keys, Req) -> end; {error, Errors} -> reply_json(Req, {struct, [{errors, Errors}]}, 400) - end. + end, + erlang:exit(normal). set_keys_in_txn(Cfg, SetFn, ToSet) -> {NewCfg, AuditProps} = From 921a62514492de92f9f2cf336100e76522e9fca2 Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Fri, 11 Dec 2020 23:04:50 -0800 Subject: [PATCH 31/33] MB-48047:[BP] During config reload try connecting to kv on all afamily and use the connection which works. When we attempt to reload memcached config, we may have a situation where the address family in ns_server has changed but not propogated to memcached yet. We might fail to communicate with memcached over new address family as memcached may not be listening on the new address family. This bug is especially seen when we go from ipv4-only to ipv6-only and vice versa. Since we cannot know for sure which address family memcached is listening on, due to various error paths that can lead to mismatch in what ns_server perceive the memcached config is and what in fact is applied in memcached, it is simplest to attempt connection using both address family. Backports change related to MB-43196. Reviewed-on: http://review.couchbase.org/c/ns_server/+/142182 Change-Id: Id8fb97470bbd2bdb4573c8a01b16650cc652601b Reviewed-on: http://review.couchbase.org/c/ns_server/+/164953 Tested-by: Abhijeeth Nuthan Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker --- src/memcached_config_mgr.erl | 23 +++++++++++++----- src/ns_memcached.erl | 46 ++++++++++++++++++++++++------------ 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/src/memcached_config_mgr.erl b/src/memcached_config_mgr.erl index 73a6d8cfbb..b6ef10b6ca 100644 --- a/src/memcached_config_mgr.erl +++ b/src/memcached_config_mgr.erl @@ -175,10 +175,20 @@ code_change(_OldVsn, State, _Extra) -> {ok, State}. apply_changed_memcached_config(DifferentConfig, State) -> - case ns_memcached:config_validate(DifferentConfig) of + %% We might have changed address family in ns_server and we don't want to + %% fail after trying to connect to the address family which might not exist + %% in memcached yet. + %% Since we cannot know for sure which address family memcached is + %% listening on, due to various error paths that can lead to mismatch in + %% what this module perceives the memcached config is and what in fact is + %% applied in memcached, it is simplest to attempt connection using both + %% address family. + AFamiliesToTry = [inet, inet6], + + case ns_memcached:config_validate(DifferentConfig, AFamiliesToTry) of ok -> ?log_debug("New memcached config is hot-reloadable."), - hot_reload_config(DifferentConfig, State, 10, []), + hot_reload_config(DifferentConfig, AFamiliesToTry, State, 10, []), {noreply, State#state{memcached_config = DifferentConfig}}; {memcached_error, einval, _} -> ?log_debug("Memcached config is not hot-reloadable"), @@ -221,14 +231,14 @@ changed_keys(BlobBefore, BlobAfter) -> unknown end. -hot_reload_config(NewMcdConfig, State, Tries, LastErr) when Tries < 1 -> +hot_reload_config(NewMcdConfig, _, State, Tries, LastErr) when Tries < 1 -> ale:error(?USER_LOGGER, "Unable to apply memcached config update that was supposed to " "succeed. Error: ~p. Giving up. Restart memcached to apply that " "config change. Updated keys: ~p", [LastErr, changed_keys(State#state.memcached_config, NewMcdConfig)]); -hot_reload_config(NewMcdConfig, State, Tries, _LastErr) -> +hot_reload_config(NewMcdConfig, AFamiliesToTry, State, Tries, _LastErr) -> FilePath = get_memcached_config_path(), PrevFilePath = FilePath ++ ".prev", @@ -243,7 +253,7 @@ hot_reload_config(NewMcdConfig, State, Tries, _LastErr) -> %% we'll be able to retry hot or cold config update ok = misc:atomic_write_file(FilePath, NewMcdConfig), - case ns_memcached:config_reload() of + case ns_memcached:config_reload(AFamiliesToTry) of ok -> delete_prev_config_file(), ale:info(?USER_LOGGER, @@ -255,7 +265,8 @@ hot_reload_config(NewMcdConfig, State, Tries, _LastErr) -> ?log_error("Failed to reload memcached config. " "Will retry. Error: ~p", [Err]), timer:sleep(1000), - hot_reload_config(NewMcdConfig, State, Tries - 1, Err) + hot_reload_config(NewMcdConfig, AFamiliesToTry, State, + Tries - 1, Err) end. get_memcached_config_path() -> diff --git a/src/ns_memcached.erl b/src/ns_memcached.erl index 296f2bc13b..6a09bc4e0d 100644 --- a/src/ns_memcached.erl +++ b/src/ns_memcached.erl @@ -116,8 +116,8 @@ get_vbucket_high_seqno/2, wait_for_seqno_persistence/3, get_keys/3, - config_validate/1, - config_reload/0, + config_validate/2, + config_reload/1, get_failover_log/2, get_failover_logs/2, get_collections_uid/1 @@ -1184,13 +1184,27 @@ do_connect(Options) -> Port = service_ports:get_port(memcached_dedicated_port, Config), User = ns_config:search_node_prop(Config, memcached, admin_user), Pass = ns_config:search_node_prop(Config, memcached, admin_pass), - {ok, Sock} = gen_tcp:connect(misc:localhost(), Port, - [misc:get_net_family(), - binary, - {packet, 0}, - {active, false}, - {recbuf, ?RECBUF}, - {sndbuf, ?SNDBUF}]), + AFamilies = proplists:get_value(try_afamily, Options, + [misc:get_net_family()]), + HelloFeatures = proplists:delete(try_afamily, Options), + {ok, Sock} = lists:foldl( + fun (_AFamily, {ok, Socket}) -> + {ok, Socket}; + (AFamily, Acc) -> + RV = gen_tcp:connect(misc:localhost(AFamily, []), + Port, + [AFamily, + binary, + {packet, 0}, + {active, false}, + {recbuf, ?RECBUF}, + {sndbuf, ?SNDBUF}]), + case RV of + {ok, S} -> {ok, S}; + _ -> [{AFamily, RV} | Acc] + end + end, [], AFamilies), + try case mc_client_binary:auth(Sock, {<<"PLAIN">>, {list_to_binary(User), @@ -1201,7 +1215,7 @@ do_connect(Options) -> "provided password ~s", [User, Pass]), error({auth_failure, Err}) end, - Features = mc_client_binary:hello_features(Options), + Features = mc_client_binary:hello_features(HelloFeatures), {ok, Negotiated} = mc_client_binary:hello(Sock, "regular", Features), Failed = Features -- Negotiated, Failed == [] orelse error({feature_negotiation_failed, Failed}), @@ -1447,18 +1461,20 @@ do_get_keys(Bucket, NodeVBuckets, Params) -> end end, NodeVBuckets, ?GET_KEYS_OUTER_TIMEOUT). --spec config_validate(binary()) -> ok | mc_error(). -config_validate(NewConfig) -> +-spec config_validate(binary(), [inet | inet6]) -> ok | mc_error(). +config_validate(NewConfig, AFamilies) -> misc:executing_on_new_process( fun () -> - {ok, Sock} = connect([{retries, 1}]), + {ok, Sock} = connect([{retries, 1}, + {try_afamily, AFamilies}]), mc_client_binary:config_validate(Sock, NewConfig) end). -config_reload() -> +config_reload(AFamilies) -> misc:executing_on_new_process( fun () -> - {ok, Sock} = connect([{retries, 1}]), + {ok, Sock} = connect([{retries, 1}, + {try_afamily, AFamilies}]), mc_client_binary:config_reload(Sock) end). From 11613933d283f822029d508b4a54430b0fc9284c Mon Sep 17 00:00:00 2001 From: Abhijeeth Nuthan Date: Mon, 8 Nov 2021 09:49:45 -0800 Subject: [PATCH 32/33] MB-48047: Kill epmd when joining cluster if required. Change-Id: Ifab29901b3b9a5a4fde1f4a12f8c180144b34a2d Reviewed-on: http://review.couchbase.org/c/ns_server/+/165274 Tested-by: Build Bot Tested-by: Abhijeeth Nuthan Reviewed-by: Timofey Barmin Well-Formed: Restriction Checker --- src/ns_cluster.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ns_cluster.erl b/src/ns_cluster.erl index bc62bc20ea..4a7fe4dbc3 100644 --- a/src/ns_cluster.erl +++ b/src/ns_cluster.erl @@ -1317,6 +1317,7 @@ perform_actual_join(RemoteNode, NewCookie) -> ok = ns_config_rep:pull_from_one_node_directly(RemoteNode), ?cluster_debug("pre-join merged config is:~n~p", [ns_config_log:sanitize(ns_config:get())]), + netconfig_updater:maybe_kill_epmd(), {ok, ok} catch From fc3b6e3e9759d6d118d9e211d7700b56dc093c46 Mon Sep 17 00:00:00 2001 From: Timofey Barmin Date: Tue, 26 Oct 2021 20:32:39 -0700 Subject: [PATCH 33/33] MB-48047:[BP] Make terminate timeout smaller 5s connection termination timeout leads to timeouts for external cb_dist calls, which leads to crashes and can be avoided by decreasing the termination timeout. It should be ok to brutally kill connections. Backports change related to MB-49138. Reviewed-on: http://review.couchbase.org/c/ns_server/+/164388 Change-Id: Ia89e09c601a017779dd3648b9f5a07086211e14b Reviewed-on: http://review.couchbase.org/c/ns_server/+/165292 Well-Formed: Restriction Checker Tested-by: Abhijeeth Nuthan Reviewed-by: Timofey Barmin --- src/cb_dist.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cb_dist.erl b/src/cb_dist.erl index 31b35644dc..c27c272ddc 100644 --- a/src/cb_dist.erl +++ b/src/cb_dist.erl @@ -64,7 +64,7 @@ -define(family, ?MODULE). -define(proto, ?MODULE). --define(TERMINATE_TIMEOUT, 5000). +-define(TERMINATE_TIMEOUT, 1000). -define(ENSURE_CONFIG_TIMEOUT, 60000). -type socket() :: any().