diff --git a/db/config.cc b/db/config.cc index 7fae92855743..c317c1cda5e8 100644 --- a/db/config.cc +++ b/db/config.cc @@ -764,6 +764,8 @@ db::config::config(std::shared_ptr exts) "\tworkdir the node will open the maintenance socket on the path /cql.m,\n" "\t where is a path defined by the workdir configuration option\n" "\t the node will open the maintenance socket on the path ") + , maintenance_socket_group(this, "maintenance_socket_group", value_status::Used, "", + "The group that the maintenance socket will be owned by. If not set, the group will be the same as the user running the scylla node.") , maintenance_mode(this, "maintenance_mode", value_status::Used, false, "If set to true, the node will not connect to other nodes. It will only serve requests to its local data.") , native_transport_port_ssl(this, "native_transport_port_ssl", value_status::Used, 9142, "Port on which the CQL TLS native transport listens for clients." diff --git a/db/config.hh b/db/config.hh index dbdfd861b86f..bcfbbb00bf4f 100644 --- a/db/config.hh +++ b/db/config.hh @@ -281,6 +281,7 @@ public: named_value start_native_transport; named_value native_transport_port; named_value maintenance_socket; + named_value maintenance_socket_group; named_value maintenance_mode; named_value native_transport_port_ssl; named_value native_shard_aware_transport_port; diff --git a/docs/operating-scylla/admin-tools/maintenance-socket.rst b/docs/operating-scylla/admin-tools/maintenance-socket.rst index f35d7e67598e..b95a67a95f27 100644 --- a/docs/operating-scylla/admin-tools/maintenance-socket.rst +++ b/docs/operating-scylla/admin-tools/maintenance-socket.rst @@ -11,11 +11,15 @@ To set up the maintenance socket, use the `maintenance-socket` option when start * If set to `workdir` maintenance socket will be created in `/cql.m`. * Otherwise maintenance socket will be created in the specified path. + The maintenance socket path has to satisfy following restrictions: * the path has to be shorter than `108` chars (due to linux limits), * a file or a directory cannot exists in this path. +Option `maintenance-socket-group` sets the owning group of the maintenance socket. If not set, the group will be the same as the user running the scylla node. +The user running the scylla node has to be in the group specified by `maintenance-socket-group` option or have root privileges. + Connect to maintenance socket ----------------------------- diff --git a/generic_server.cc b/generic_server.cc index daec5d47e84b..28a6c8096f16 100644 --- a/generic_server.cc +++ b/generic_server.cc @@ -145,7 +145,7 @@ future<> server::shutdown() { } future<> -server::listen(socket_address addr, std::shared_ptr creds, bool is_shard_aware, bool keepalive) { +server::listen(socket_address addr, std::shared_ptr creds, bool is_shard_aware, bool keepalive, std::optional unix_domain_socket_permissions) { auto f = make_ready_future>(nullptr); if (creds) { f = creds->build_reloadable_server_credentials([this](const std::unordered_set& files, std::exception_ptr ep) { @@ -156,9 +156,10 @@ server::listen(socket_address addr, std::shared_ptr creds) { + return f.then([this, addr, is_shard_aware, keepalive, unix_domain_socket_permissions](shared_ptr creds) { listen_options lo; lo.reuse_address = true; + lo.unix_domain_socket_permissions = unix_domain_socket_permissions; if (is_shard_aware) { lo.lba = server_socket::load_balancing_algorithm::port; } diff --git a/generic_server.hh b/generic_server.hh index 5593d42fb67d..2371b9122775 100644 --- a/generic_server.hh +++ b/generic_server.hh @@ -14,6 +14,7 @@ #include +#include #include #include #include @@ -107,7 +108,7 @@ public: future<> shutdown(); future<> stop(); - future<> listen(socket_address addr, std::shared_ptr creds, bool is_shard_aware, bool keepalive); + future<> listen(socket_address addr, std::shared_ptr creds, bool is_shard_aware, bool keepalive, std::optional unix_domain_socket_permissions); future<> do_accepts(int which, bool keepalive, socket_address server_addr); diff --git a/redis/controller.cc b/redis/controller.cc index ca711eb8758c..e55ea0d1284d 100644 --- a/redis/controller.cc +++ b/redis/controller.cc @@ -87,7 +87,7 @@ future<> controller::listen(seastar::sharded& auth_service, db::c return f.then([server, configs = std::move(configs), keepalive] { return parallel_for_each(configs, [server, keepalive](const listen_cfg & cfg) { - return server->invoke_on_all(&redis_transport::redis_server::listen, cfg.addr, cfg.cred, false, keepalive).then([cfg] { + return server->invoke_on_all(&redis_transport::redis_server::listen, cfg.addr, cfg.cred, false, keepalive, std::nullopt).then([cfg] { slogger.info("Starting listening for REDIS clients on {} ({})", cfg.addr, cfg.cred ? "encrypted" : "unencrypted"); }); }); diff --git a/transport/controller.cc b/transport/controller.cc index 9882dcaa0053..ae814db5ddcc 100644 --- a/transport/controller.cc +++ b/transport/controller.cc @@ -6,10 +6,12 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ +#include #include "transport/controller.hh" #include #include #include +#include #include "transport/server.hh" #include "service/memory_limiter.hh" #include "db/config.hh" @@ -65,6 +67,142 @@ future<> controller::start_server() { return do_start_server().finally([this] { _ops_sem.signal(); }); } +static future<> listen_on_all_shards(sharded& cserver, socket_address addr, std::shared_ptr creds, bool is_shard_aware, bool keepalive, std::optional unix_domain_socket_permissions) { + co_await cserver.invoke_on_all([addr, creds, is_shard_aware, keepalive, unix_domain_socket_permissions] (cql_server& server) { + return server.listen(addr, creds, is_shard_aware, keepalive, unix_domain_socket_permissions); + }); + + logger.info("Starting listening for CQL clients on {} ({}, {})" + , addr, creds ? "encrypted" : "unencrypted", is_shard_aware ? "shard-aware" : "non-shard-aware" + ); +} + +future<> controller::start_listening_on_tcp_sockets(sharded& cserver) { + auto& cfg = _config; + auto preferred = cfg.rpc_interface_prefer_ipv6() ? std::make_optional(net::inet_address::family::INET6) : std::nullopt; + auto family = cfg.enable_ipv6_dns_lookup() || preferred ? std::nullopt : std::make_optional(net::inet_address::family::INET); + auto ceo = cfg.client_encryption_options(); + auto keepalive = cfg.rpc_keepalive(); + + struct listen_cfg { + socket_address addr; + bool is_shard_aware; + std::shared_ptr cred; + }; + + _listen_addresses.clear(); + std::vector configs; + + const seastar::net::inet_address ip = utils::resolve(cfg.rpc_address, family, preferred).get0(); + int native_port_idx = -1, native_shard_aware_port_idx = -1; + + if (cfg.native_transport_port.is_set() || + (!cfg.native_transport_port_ssl.is_set() && !cfg.native_transport_port.is_set())) { + // Non-SSL port is specified || neither SSL nor non-SSL ports are specified + configs.emplace_back(listen_cfg{ socket_address{ip, cfg.native_transport_port()}, false }); + _listen_addresses.push_back(configs.back().addr); + native_port_idx = 0; + } + if (cfg.native_shard_aware_transport_port.is_set() || + (!cfg.native_shard_aware_transport_port_ssl.is_set() && !cfg.native_shard_aware_transport_port.is_set())) { + configs.emplace_back(listen_cfg{ socket_address{ip, cfg.native_shard_aware_transport_port()}, true }); + _listen_addresses.push_back(configs.back().addr); + native_shard_aware_port_idx = native_port_idx + 1; + } + + // main should have made sure values are clean and neatish + if (utils::is_true(utils::get_or_default(ceo, "enabled", "false"))) { + auto cred = std::make_shared(); + utils::configure_tls_creds_builder(*cred, std::move(ceo)).get(); + + logger.info("Enabling encrypted CQL connections between client and server"); + + if (cfg.native_transport_port_ssl.is_set() && + (!cfg.native_transport_port.is_set() || + cfg.native_transport_port_ssl() != cfg.native_transport_port())) { + // SSL port is specified && non-SSL port is either left out or set to a different value + configs.emplace_back(listen_cfg{{ip, cfg.native_transport_port_ssl()}, false, cred}); + _listen_addresses.push_back(configs.back().addr); + } else if (native_port_idx >= 0) { + configs[native_port_idx].cred = cred; + } + if (cfg.native_shard_aware_transport_port_ssl.is_set() && + (!cfg.native_shard_aware_transport_port.is_set() || + cfg.native_shard_aware_transport_port_ssl() != cfg.native_shard_aware_transport_port())) { + configs.emplace_back(listen_cfg{{ip, cfg.native_shard_aware_transport_port_ssl()}, true, std::move(cred)}); + _listen_addresses.push_back(configs.back().addr); + } else if (native_shard_aware_port_idx >= 0) { + configs[native_shard_aware_port_idx].cred = std::move(cred); + } + } + + return parallel_for_each(configs, [&cserver, keepalive](const listen_cfg & cfg) { + return listen_on_all_shards(cserver, cfg.addr, cfg.cred, cfg.is_shard_aware, keepalive, std::nullopt); + }); +} + +future<> controller::start_listening_on_maintenance_socket(sharded& cserver) { + auto socket = _config.maintenance_socket(); + + if (socket == "workdir") { + socket = _config.work_directory() + "/cql.m"; + } + + auto max_socket_length = sizeof(sockaddr_un::sun_path); + if (socket.length() > max_socket_length - 1) { + throw std::runtime_error(format("Maintenance socket path is too long: {}. Change it to string shorter than {} chars.", socket, max_socket_length)); + } + + struct stat statbuf; + auto stat_result = ::stat(socket.c_str(), &statbuf); + if (stat_result == 0) { + // Check if it is a unix domain socket, not a regular file or directory + if (!S_ISSOCK(statbuf.st_mode)) { + throw std::runtime_error(format("Under maintenance socket path ({}) there is something else.", socket)); + } + } else if (errno != ENOENT) { + // Other error than "file does not exist" + throw std::runtime_error(format("Failed to stat {}: {}", socket, strerror(errno))); + } + + // Remove the socket if it already exists, otherwise when the server + // tries to listen on it, it will hang on bind(). + auto unlink_result = ::unlink(socket.c_str()); + if (unlink_result < 0 && errno != ENOENT) { + // Other error than "file does not exist" + throw std::runtime_error(format("Failed to unlink {}: {}", socket, strerror(errno))); + } + + auto addr = socket_address { unix_domain_addr { socket } }; + _listen_addresses.push_back(addr); + + logger.info("Setting up maintenance socket on {}", socket); + + auto unix_domain_socket_permissions = + file_permissions::user_read | file_permissions::user_write | + file_permissions::group_read | file_permissions::group_write; + + co_await listen_on_all_shards(cserver, addr, nullptr, false, _config.rpc_keepalive(), unix_domain_socket_permissions); + + if (_config.maintenance_socket_group.is_set()) { + auto group_name = _config.maintenance_socket_group(); + struct group *grp; + grp = ::getgrnam(group_name.c_str()); + if (!grp) { + throw std::runtime_error(format("Group id of {} not found. Make sure the group exists.", group_name)); + } + + auto chown_result = ::chown(socket.c_str(), ::geteuid(), grp->gr_gid); + if (chown_result < 0) { + if (errno == EPERM) { + throw std::runtime_error(format("Failed to change group of {}: Permission denied. Make sure the user has the root privilege or is a member of the group {}.", socket, group_name)); + } else { + throw std::runtime_error(format("Failed to chown {}: {} ()", socket, strerror(errno))); + } + } + } +} + future<> controller::do_start_server() { if (_server) { return make_ready_future<>(); @@ -74,10 +212,6 @@ future<> controller::do_start_server() { auto cserver = std::make_unique>(); auto& cfg = _config; - auto preferred = cfg.rpc_interface_prefer_ipv6() ? std::make_optional(net::inet_address::family::INET6) : std::nullopt; - auto family = cfg.enable_ipv6_dns_lookup() || preferred ? std::nullopt : std::make_optional(net::inet_address::family::INET); - auto ceo = cfg.client_encryption_options(); - auto keepalive = cfg.rpc_keepalive(); smp_service_group_config cql_server_smp_service_group_config; cql_server_smp_service_group_config.max_nonlocal_requests = 5000; auto bounce_request_smp_service_group = create_smp_service_group(cql_server_smp_service_group_config).get(); @@ -110,92 +244,6 @@ future<> controller::do_start_server() { std::shared_ptr cred; }; - _listen_addresses.clear(); - std::vector configs; - - if (!_used_by_maintenance_socket) { - const seastar::net::inet_address ip = utils::resolve(cfg.rpc_address, family, preferred).get(); - int native_port_idx = -1, native_shard_aware_port_idx = -1; - - if (cfg.native_transport_port.is_set() || - (!cfg.native_transport_port_ssl.is_set() && !cfg.native_transport_port.is_set())) { - // Non-SSL port is specified || neither SSL nor non-SSL ports are specified - configs.emplace_back(listen_cfg{ socket_address{ip, cfg.native_transport_port()}, false }); - _listen_addresses.push_back(configs.back().addr); - native_port_idx = 0; - } - if (cfg.native_shard_aware_transport_port.is_set() || - (!cfg.native_shard_aware_transport_port_ssl.is_set() && !cfg.native_shard_aware_transport_port.is_set())) { - configs.emplace_back(listen_cfg{ socket_address{ip, cfg.native_shard_aware_transport_port()}, true }); - _listen_addresses.push_back(configs.back().addr); - native_shard_aware_port_idx = native_port_idx + 1; - } - - // main should have made sure values are clean and neatish - if (utils::is_true(utils::get_or_default(ceo, "enabled", "false"))) { - auto cred = std::make_shared(); - utils::configure_tls_creds_builder(*cred, std::move(ceo)).get(); - - logger.info("Enabling encrypted CQL connections between client and server"); - - if (cfg.native_transport_port_ssl.is_set() && - (!cfg.native_transport_port.is_set() || - cfg.native_transport_port_ssl() != cfg.native_transport_port())) { - // SSL port is specified && non-SSL port is either left out or set to a different value - configs.emplace_back(listen_cfg{{ip, cfg.native_transport_port_ssl()}, false, cred}); - _listen_addresses.push_back(configs.back().addr); - } else if (native_port_idx >= 0) { - configs[native_port_idx].cred = cred; - } - if (cfg.native_shard_aware_transport_port_ssl.is_set() && - (!cfg.native_shard_aware_transport_port.is_set() || - cfg.native_shard_aware_transport_port_ssl() != cfg.native_shard_aware_transport_port())) { - configs.emplace_back(listen_cfg{{ip, cfg.native_shard_aware_transport_port_ssl()}, true, std::move(cred)}); - _listen_addresses.push_back(configs.back().addr); - } else if (native_shard_aware_port_idx >= 0) { - configs[native_shard_aware_port_idx].cred = std::move(cred); - } - } - } else { - auto socket = cfg.maintenance_socket(); - - if (socket == "workdir") { - socket = cfg.work_directory() + "/cql.m"; - } - - if (socket.length() > 107) { - throw std::runtime_error(format("Maintenance socket path is too long: {}. Change it to string shorter than 108 chars.", socket)); - } - - struct stat statbuf; - auto stat_result = ::stat(socket.c_str(), &statbuf); - if (stat_result == 0) { - // Check if it is a unix domain socket, not a regular file or directory - if (!S_ISSOCK(statbuf.st_mode)) { - throw std::runtime_error(format("Under maintenance socket path ({}) there is something else.", socket)); - } - } else if (errno != ENOENT) { - // Other error than "file does not exist" - throw std::runtime_error(format("Failed to stat {}: {}", socket, strerror(errno))); - } - - // Remove the socket if it already exists, otherwise when the server - // tries to listen on it, it will hang on bind(). - auto unlink_result = ::unlink(socket.c_str()); - if (unlink_result < 0 && errno != ENOENT) { - // Other error than "file does not exist" - throw std::runtime_error(format("Failed to unlink {}: {}", socket, strerror(errno))); - } - - configs.emplace_back(listen_cfg { - .addr = socket_address { unix_domain_addr { socket } }, - .is_shard_aware = false - }); - _listen_addresses.push_back(configs.back().addr); - - logger.info("Setting up maintenance socket on {}", socket); - } - cserver->start(std::ref(_qp), std::ref(_auth_service), std::ref(_mem_limiter), std::move(get_cql_server_config), std::ref(cfg), std::ref(_sl_controller), std::ref(_gossiper), _cql_opcode_stats_key, _used_by_maintenance_socket).get(); auto on_error = defer([&cserver] { cserver->stop().get(); }); @@ -204,13 +252,12 @@ future<> controller::do_start_server() { unsubscribe_server(*cserver).get(); }); - parallel_for_each(configs, [&cserver, keepalive](const listen_cfg & cfg) { - return cserver->invoke_on_all(&cql_server::listen, cfg.addr, cfg.cred, cfg.is_shard_aware, keepalive).then([cfg] { - logger.info("Starting listening for CQL clients on {} ({}, {})" - , cfg.addr, cfg.cred ? "encrypted" : "unencrypted", cfg.is_shard_aware ? "shard-aware" : "non-shard-aware" - ); - }); - }).get(); + _listen_addresses.clear(); + if (!_used_by_maintenance_socket) { + start_listening_on_tcp_sockets(*cserver).get(); + } else { + start_listening_on_maintenance_socket(*cserver).get(); + } if (!_used_by_maintenance_socket) { set_cql_ready(true).get(); diff --git a/transport/controller.hh b/transport/controller.hh index edd627592c45..411556a0384d 100644 --- a/transport/controller.hh +++ b/transport/controller.hh @@ -59,6 +59,9 @@ class controller : public protocol_server { future<> subscribe_server(sharded& server); future<> unsubscribe_server(sharded& server); + future<> start_listening_on_tcp_sockets(sharded& cserver); + future<> start_listening_on_maintenance_socket(sharded& cserver); + maintenance_socket_enabled _used_by_maintenance_socket; public: