MB-44361: Multiple CA support core changes and refactoring

- save ca's in a separate key in chronicle
- reduce the number of cert related files
  (there should be only three files: chain, pkey, ca's)
- use certs.tmp when loading certs in order to avoid situations where
  only one file of two is updated (either chain or pkey); we should make
  sure we update them atomicaly
- don't keep node cert hash in state of ns_ssl_services_setup, in order
  to avoid restart of cb_dist ssl connection every time the process
  starts
- load ca's from disk in order to avoid situation when a person with
  password can add their node to a cluster without having correct certs

Change-Id: I41c44f5566919944f9f219b40505f257b191edbb
Reviewed-on: http://review.couchbase.org/c/ns_server/+/158779
Tested-by: Timofey Barmin <timofey.barmin@couchbase.com>
Reviewed-by: Artem Stemkovski <artem@couchbase.com>

src/memcached_config_mgr.erl

@@ -436,9 +436,8 @@ prometheus_cfg([], _Params) ->
       {family, ns_config:read_key_fast({node, node(), address_family}, inet)}]}.
 
 generate_interfaces(MCDParams) ->
-    SSL = {[{key, list_to_binary(ns_ssl_services_setup:memcached_key_path())},
-            {cert,
-             list_to_binary(ns_ssl_services_setup:memcached_cert_path())}]},
+    SSL = {[{key, list_to_binary(ns_ssl_services_setup:pkey_file_path())},
+            {cert, list_to_binary(ns_ssl_services_setup:chain_file_path())}]},
     GetPort = fun (Port) ->
                       {Port, Value} = lists:keyfind(Port, 1, MCDParams),
                       Value

src/menelaus_web.erl

@@ -585,6 +585,9 @@ get_action(Req, {AppRoot, IsSSL, Plugins}, Path, PathTokens) ->
                 ["node", "controller", "reloadCertificate"] ->
                     {{[admin, security], write},
                      fun menelaus_web_cert:handle_reload_node_certificate/1};
+                ["node", "controller", "loadCAcertificates"] ->
+                    {{[admin, security], write},
+                     fun menelaus_web_cert:handle_load_ca_certs/1};
                 ["node", "controller", "changeMasterPassword"] ->
                     {{[admin, security], write},
                      fun menelaus_web_secrets:handle_change_master_password/1};

src/menelaus_web_cert.erl

@@ -15,7 +15,8 @@
 
 -export([handle_cluster_certificate/1,
          handle_regenerate_certificate/1,
-         handle_upload_cluster_ca/1,
+         handle_load_ca_certs/1,
+         handle_upload_cluster_ca/1, %% deprecated
          handle_reload_node_certificate/1,
          handle_get_node_certificate/2,
          handle_client_cert_auth_settings/1,
@@ -59,8 +60,11 @@ translate_warning(Warning) ->
     warning_props(Warning).
 
 jsonify_cert_props(Props) ->
-    lists:map(fun ({expires, UTCSeconds}) ->
-                      {expires, format_time(UTCSeconds)};
+    lists:map(fun ({K, UTCSeconds}) when K =:= expires;
+                                         K =:= not_before;
+                                         K =:= not_after;
+                                         K =:= load_timestamp ->
+                      {K, format_time(UTCSeconds)};
                   ({K, V}) when is_list(V) ->
                       {K, list_to_binary(V)};
                   (Pair) ->
@@ -95,6 +99,26 @@ reply_error(Req, Error) ->
     menelaus_util:reply_json(
       Req, {[{error, ns_error_messages:cert_validation_error_message(Error)}]}, 400).
 
+handle_load_ca_certs(Req) ->
+    menelaus_util:assert_is_enterprise(),
+    menelaus_util:assert_is_NEO(),
+    Nodes = nodes(),
+    case ns_server_cert:load_CAs_from_inbox() of
+        {ok, NewCertsProps} ->
+            ns_ssl_services_setup:sync(),
+            case netconfig_updater:ensure_tls_dist_started(Nodes) of
+                ok ->
+                    CertsJson = [{jsonify_cert_props(C)} || C <- NewCertsProps],
+                    menelaus_util:reply_json(Req, CertsJson, 200);
+                {error, ErrorMsg} ->
+                    menelaus_util:reply_json(Req, ErrorMsg, 400)
+            end;
+        {error, Error} ->
+            ?log_error("Error loading CA certificates: ~p", [Error]),
+            menelaus_util:reply_json(
+              Req, ns_error_messages:load_CAs_from_inbox_error(Error), 400)
+    end.
+
 handle_upload_cluster_ca(Req) ->
     menelaus_util:assert_is_enterprise(),
     assert_n2n_encryption_is_disabled(),
@@ -125,11 +149,11 @@ assert_n2n_encryption_is_disabled() ->
 handle_reload_node_certificate(Req) ->
     menelaus_util:assert_is_enterprise(),
     Nodes = nodes(),
-    case ns_server_cert:apply_certificate_chain_from_inbox() of
+    case ns_server_cert:load_node_certs_from_inbox() of
         {ok, Props} ->
             ns_audit:reload_node_certificate(Req,
                                              proplists:get_value(subject, Props),
-                                             proplists:get_value(expires, Props)),
+                                             proplists:get_value(not_after, Props)),
             ns_ssl_services_setup:sync(),
             case netconfig_updater:ensure_tls_dist_started(Nodes) of
                 ok ->

src/ns_cluster.erl

@@ -151,7 +151,7 @@ engage_cluster_apply_certs(NodeKVList) ->
 apply_certs(ClusterCA) ->
     case ns_server_cert:set_cluster_ca(ClusterCA) of
         {ok, _} ->
-            case ns_server_cert:apply_certificate_chain_from_inbox(ClusterCA) of
+            case ns_server_cert:load_node_certs_from_inbox() of
                 {ok, Props} ->
                     ns_ssl_services_setup:sync(),
                     ?log_info("Custom certificate was loaded on the node "
@@ -846,29 +846,36 @@ do_add_node_with_connectivity(Scheme, RemoteAddr, RestPort, Auth, GroupUUID,
 
     Encryption = proplists:get_bool(nodeEncryption, Props),
 
+    Config = ns_config:get(),
+
     Props1 =
-        case ns_server_cert:cluster_ca() of
-             {CAProps, _, _} ->
-                 [{<<"clusterCA">>, proplists:get_value(pem, CAProps)}];
-             {CA, PKey} when Encryption ->
-                 {NewNodeCert, NewNodeKey} =
-                     ns_ssl_services_setup:generate_node_certs(CA, PKey,
-                                                               RemoteAddr),
-                 %% Sending PKey only in case if encryption is enabled (so
-                 %% remote node really needs it) and only if cluster uses
-                 %% autogenerated certs (otherwise user is supposed to provision
-                 %% new node with certs)
-                 [{<<"autogeneratedCA">>, CA},
-                  {<<"autogeneratedCert">>, NewNodeCert}] ++
-                 case cluster_compat_mode:is_cluster_70() of
-                     true ->
-                         [{<<"autogeneratedKey">>, NewNodeKey}];
-                     false ->
-                         [{<<"autogeneratedKey">>, {[{otpCookie, NewNodeKey}]}}]
-                 end;
-             {CA, _PKey} ->
-                 [{<<"autogeneratedCA">>, CA}]
-         end ++ Props,
+        case ns_server_cert:this_node_uses_self_generated_certs(Config) of
+            false ->
+                CA = ns_server_cert:this_node_ca(Config),
+                [{<<"clusterCA">>, CA}];
+            true when Encryption ->
+                {CA, NewNodeCert, NewNodeKey} =
+                    case ns_server_cert:generate_node_certs(RemoteAddr) of
+                        no_private_key ->
+                            _ = ns_server_cert:generate_and_set_cert_and_pkey(),
+                            ns_server_cert:generate_node_certs(RemoteAddr);
+                        Certs -> Certs
+                    end,
+                %% Sending PKey only in case if encryption is enabled (so
+                %% remote node really needs it) and only if cluster uses
+                %% autogenerated certs (otherwise user is supposed to provision
+                %% new node with certs)
+                [{<<"autogeneratedCA">>, CA},
+                 {<<"autogeneratedCert">>, NewNodeCert}] ++
+                case cluster_compat_mode:is_cluster_70() of
+                    true ->
+                        [{<<"autogeneratedKey">>, NewNodeKey}];
+                    false ->
+                        [{<<"autogeneratedKey">>, {[{otpCookie, NewNodeKey}]}}]
+                end;
+            true ->
+                [{<<"autogeneratedCA">>, ns_server_cert:self_generated_ca()}]
+        end ++ Props,
 
     Options = [{connect_options, [misc:get_net_family()]}],
 
@@ -1002,7 +1009,7 @@ check_otp_tls_connectivity(Host, Port, AFamily) ->
 do_add_node_engaged(NodeKVList, Auth, GroupUUID, Services, Scheme) ->
     OtpNode = expect_json_property_atom(<<"otpNode">>, NodeKVList),
 
-    VerifyOpts = case ns_server_cert:cluster_ca() of
+    VerifyOpts = case ns_server_cert:this_node_uses_self_generated_certs() of
                      %% Do not test TLS connection in case of self-generated
                      %% certs.
                      %% Reason: the remote node can't have correct certs
@@ -1011,8 +1018,8 @@ do_add_node_engaged(NodeKVList, Auth, GroupUUID, Services, Scheme) ->
                      %% to us (because it knows our CA), then it will sync
                      %% config, and only then it will be able to generate
                      %% correct certs
-                     {_, _} -> [tcp_only];
-                     {_, _, _} -> []
+                     true -> [tcp_only];
+                     false -> []
                  end,
     RV = verify_otp_connectivity(OtpNode, VerifyOpts),
     case RV of

src/ns_error_messages.erl

@@ -27,7 +27,8 @@
          not_absolute_path/1,
          empty_param/1,
          preview_cluster_join_error/0,
-         address_check_error/2]).
+         address_check_error/2,
+         load_CAs_from_inbox_error/1]).
 
 -spec connection_error_message(term(), string(), string() | integer()) -> binary() | undefined.
 connection_error_message({tls_alert, "bad record mac"}, Host, Port) ->
@@ -228,8 +229,20 @@ file_read_error(enomem) ->
 file_read_error(Reason) ->
     atom_to_list(Reason).
 
+load_CAs_from_inbox_error({Path, empty}) ->
+    list_to_binary(io_lib:format("Directory ~s is empty", [Path]));
+load_CAs_from_inbox_error({File, {read, Error}}) ->
+    list_to_binary(io_lib:format("Couldn't load CA certificate from ~s. ~s",
+                                 [File, file_read_error(Error)]));
+load_CAs_from_inbox_error({File, {bad_cert, Error}}) ->
+    list_to_binary(io_lib:format("Couldn't load CA certificate from ~s. ~s",
+                                 [File, cert_validation_error_message(Error)])).
+
 reload_node_certificate_error(no_cluster_ca) ->
     <<"Cluster CA needs to be set before setting node certificate.">>;
+reload_node_certificate_error(no_ca) ->
+    <<"CA certificate for this chain is not found "
+      "in the list of trusted CA's">>;
 reload_node_certificate_error({bad_cert, {invalid_root_issuer, Subject, RootSubject}}) ->
     list_to_binary(io_lib:format("Last certificate of the chain ~p is not issued by the "
                                  "cluster root certificate ~p",

src/ns_ports_setup.erl

@@ -245,8 +245,8 @@ build_https_args(PortName, PortArg, PortPrefix, CertArg, KeyArg, Config) ->
             [];
         Port ->
             [PortArg ++ "=" ++ PortPrefix ++ integer_to_list(Port),
-             CertArg ++ "=" ++ ns_ssl_services_setup:memcached_cert_path(),
-             KeyArg ++ "=" ++ ns_ssl_services_setup:memcached_key_path()]
+             CertArg ++ "=" ++ ns_ssl_services_setup:chain_file_path(),
+             KeyArg ++ "=" ++ ns_ssl_services_setup:pkey_file_path()]
     end.
 
 build_port_arg(ArgName, PortName, Config) ->
@@ -380,7 +380,7 @@ goport_args(goxdcr, Config, _Cmd, _NodeUUID) ->
     build_port_args([{"-sourceKVAdminPort", rest_port},
                      {"-xdcrRestPort", xdcr_rest_port}], Config) ++
         [IsEnterprise | build_afamily_requirement("-")] ++
-        ["-caFile=" ++ ns_ssl_services_setup:memcached_cert_path()];
+        ["-caFile=" ++ ns_ssl_services_setup:ca_file_path()];
 
 goport_args(indexer, Config, _Cmd, NodeUUID) ->
     {ok, LogDir} = application:get_env(ns_server, error_logger_mf_dir),