Skip to content

Commit

Permalink
Merge 'Maintenance socket: set filesystem permissions to 660' from Mi…
Browse files Browse the repository at this point in the history 
…kołaj Grzebieluch

Set filesystem permissions for the maintenance socket to 660 (previously it was 755) to allow a scyllaadm's group to connect.
Split the logic of creating sockets into two separate functions, one for each case: when it is a regular cql controller or used by maintenance_socket.

Fixes #16487.

Closes #17113

* github.com:scylladb/scylladb:
  maintenance_socket: add option to set owning group
  transport/controller: get rid of magic number for socket path's maximal length
  transport/controller: set unix_domain_socket_permissions for maintenance_socket
  transport/controller: pass unix_domain_socket_permissions to generic_server::listen
  transport/controller: split configuring sockets into separate functions
  • Loading branch information
@avikivity
avikivity committed Feb 20, 2024
2 parents 73a3a3f + 182cfeb commit 93af3dd
Show file tree
Hide file tree
Showing 8 changed files with 160 additions and 101 deletions.
2 changes: 2 additions & 0 deletions db/config.cc
View file
Expand Up @@ -764,6 +764,8 @@ db::config::config(std::shared_ptr<db::extensions> exts)
"\tworkdir the node will open the maintenance socket on the path <scylla's workdir>/cql.m,\n"
"\t where <scylla's workdir> is a path defined by the workdir configuration option\n"
"\t<socket path> the node will open the maintenance socket on the path <socket path>")
, maintenance_socket_group(this, "maintenance_socket_group", value_status::Used, "",
"The group that the maintenance socket will be owned by. If not set, the group will be the same as the user running the scylla node.")
, maintenance_mode(this, "maintenance_mode", value_status::Used, false, "If set to true, the node will not connect to other nodes. It will only serve requests to its local data.")
, native_transport_port_ssl(this, "native_transport_port_ssl", value_status::Used, 9142,
"Port on which the CQL TLS native transport listens for clients."
Expand Down
1 change: 1 addition & 0 deletions db/config.hh
View file
Expand Up @@ -281,6 +281,7 @@ public:
named_value<bool> start_native_transport;
named_value<uint16_t> native_transport_port;
named_value<sstring> maintenance_socket;
named_value<sstring> maintenance_socket_group;
named_value<bool> maintenance_mode;
named_value<uint16_t> native_transport_port_ssl;
named_value<uint16_t> native_shard_aware_transport_port;
Expand Down
4 changes: 4 additions & 0 deletions docs/operating-scylla/admin-tools/maintenance-socket.rst
View file
Expand Up @@ -11,11 +11,15 @@ To set up the maintenance socket, use the `maintenance-socket` option when start
* If set to `workdir` maintenance socket will be created in `<node's workdir>/cql.m`.
* Otherwise maintenance socket will be created in the specified path.


The maintenance socket path has to satisfy following restrictions:

* the path has to be shorter than `108` chars (due to linux limits),
* a file or a directory cannot exists in this path.

Option `maintenance-socket-group` sets the owning group of the maintenance socket. If not set, the group will be the same as the user running the scylla node.
The user running the scylla node has to be in the group specified by `maintenance-socket-group` option or have root privileges.

Connect to maintenance socket
-----------------------------

Expand Down
5 changes: 3 additions & 2 deletions generic_server.cc
View file
Expand Up @@ -145,7 +145,7 @@ future<> server::shutdown() {
}

future<>
server::listen(socket_address addr, std::shared_ptr<seastar::tls::credentials_builder> creds, bool is_shard_aware, bool keepalive) {
server::listen(socket_address addr, std::shared_ptr<seastar::tls::credentials_builder> creds, bool is_shard_aware, bool keepalive, std::optional<file_permissions> unix_domain_socket_permissions) {
auto f = make_ready_future<shared_ptr<seastar::tls::server_credentials>>(nullptr);
if (creds) {
f = creds->build_reloadable_server_credentials([this](const std::unordered_set<sstring>& files, std::exception_ptr ep) {
Expand All @@ -156,9 +156,10 @@ server::listen(socket_address addr, std::shared_ptr<seastar::tls::credentials_bu
}
});
}
return f.then([this, addr, is_shard_aware, keepalive](shared_ptr<seastar::tls::server_credentials> creds) {
return f.then([this, addr, is_shard_aware, keepalive, unix_domain_socket_permissions](shared_ptr<seastar::tls::server_credentials> creds) {
listen_options lo;
lo.reuse_address = true;
lo.unix_domain_socket_permissions = unix_domain_socket_permissions;
if (is_shard_aware) {
lo.lba = server_socket::load_balancing_algorithm::port;
}
Expand Down
3 changes: 2 additions & 1 deletion generic_server.hh
View file
Expand Up @@ -14,6 +14,7 @@

#include <list>

#include <seastar/core/file-types.hh>
#include <seastar/core/future.hh>
#include <seastar/core/gate.hh>
#include <seastar/net/api.hh>
Expand Down Expand Up @@ -107,7 +108,7 @@ public:
future<> shutdown();
future<> stop();

future<> listen(socket_address addr, std::shared_ptr<seastar::tls::credentials_builder> creds, bool is_shard_aware, bool keepalive);
future<> listen(socket_address addr, std::shared_ptr<seastar::tls::credentials_builder> creds, bool is_shard_aware, bool keepalive, std::optional<file_permissions> unix_domain_socket_permissions);

future<> do_accepts(int which, bool keepalive, socket_address server_addr);

Expand Down
2 changes: 1 addition & 1 deletion redis/controller.cc
View file
Expand Up @@ -87,7 +87,7 @@ future<> controller::listen(seastar::sharded<auth::service>& auth_service, db::c

return f.then([server, configs = std::move(configs), keepalive] {
return parallel_for_each(configs, [server, keepalive](const listen_cfg & cfg) {
return server->invoke_on_all(&redis_transport::redis_server::listen, cfg.addr, cfg.cred, false, keepalive).then([cfg] {
return server->invoke_on_all(&redis_transport::redis_server::listen, cfg.addr, cfg.cred, false, keepalive, std::nullopt).then([cfg] {
slogger.info("Starting listening for REDIS clients on {} ({})", cfg.addr, cfg.cred ? "encrypted" : "unencrypted");
});
});
Expand Down
241 changes: 144 additions & 97 deletions transport/controller.cc
View file
Expand Up @@ -6,10 +6,12 @@
* SPDX-License-Identifier: AGPL-3.0-or-later
*/

#include <grp.h>
#include "transport/controller.hh"
#include <seastar/core/sharded.hh>
#include <seastar/net/socket_defs.hh>
#include <seastar/net/unix_address.hh>
#include <seastar/core/file-types.hh>
#include "transport/server.hh"
#include "service/memory_limiter.hh"
#include "db/config.hh"
Expand Down Expand Up @@ -65,6 +67,142 @@ future<> controller::start_server() {
return do_start_server().finally([this] { _ops_sem.signal(); });
}

static future<> listen_on_all_shards(sharded<cql_server>& cserver, socket_address addr, std::shared_ptr<seastar::tls::credentials_builder> creds, bool is_shard_aware, bool keepalive, std::optional<file_permissions> unix_domain_socket_permissions) {
co_await cserver.invoke_on_all([addr, creds, is_shard_aware, keepalive, unix_domain_socket_permissions] (cql_server& server) {
return server.listen(addr, creds, is_shard_aware, keepalive, unix_domain_socket_permissions);
});

logger.info("Starting listening for CQL clients on {} ({}, {})"
, addr, creds ? "encrypted" : "unencrypted", is_shard_aware ? "shard-aware" : "non-shard-aware"
);
}

future<> controller::start_listening_on_tcp_sockets(sharded<cql_server>& cserver) {
auto& cfg = _config;
auto preferred = cfg.rpc_interface_prefer_ipv6() ? std::make_optional(net::inet_address::family::INET6) : std::nullopt;
auto family = cfg.enable_ipv6_dns_lookup() || preferred ? std::nullopt : std::make_optional(net::inet_address::family::INET);
auto ceo = cfg.client_encryption_options();
auto keepalive = cfg.rpc_keepalive();

struct listen_cfg {
socket_address addr;
bool is_shard_aware;
std::shared_ptr<seastar::tls::credentials_builder> cred;
};

_listen_addresses.clear();
std::vector<listen_cfg> configs;

const seastar::net::inet_address ip = utils::resolve(cfg.rpc_address, family, preferred).get0();
int native_port_idx = -1, native_shard_aware_port_idx = -1;

if (cfg.native_transport_port.is_set() ||
(!cfg.native_transport_port_ssl.is_set() && !cfg.native_transport_port.is_set())) {
// Non-SSL port is specified || neither SSL nor non-SSL ports are specified
configs.emplace_back(listen_cfg{ socket_address{ip, cfg.native_transport_port()}, false });
_listen_addresses.push_back(configs.back().addr);
native_port_idx = 0;
}
if (cfg.native_shard_aware_transport_port.is_set() ||
(!cfg.native_shard_aware_transport_port_ssl.is_set() && !cfg.native_shard_aware_transport_port.is_set())) {
configs.emplace_back(listen_cfg{ socket_address{ip, cfg.native_shard_aware_transport_port()}, true });
_listen_addresses.push_back(configs.back().addr);
native_shard_aware_port_idx = native_port_idx + 1;
}

// main should have made sure values are clean and neatish
if (utils::is_true(utils::get_or_default(ceo, "enabled", "false"))) {
auto cred = std::make_shared<seastar::tls::credentials_builder>();
utils::configure_tls_creds_builder(*cred, std::move(ceo)).get();

logger.info("Enabling encrypted CQL connections between client and server");

if (cfg.native_transport_port_ssl.is_set() &&
(!cfg.native_transport_port.is_set() ||
cfg.native_transport_port_ssl() != cfg.native_transport_port())) {
// SSL port is specified && non-SSL port is either left out or set to a different value
configs.emplace_back(listen_cfg{{ip, cfg.native_transport_port_ssl()}, false, cred});
_listen_addresses.push_back(configs.back().addr);
} else if (native_port_idx >= 0) {
configs[native_port_idx].cred = cred;
}
if (cfg.native_shard_aware_transport_port_ssl.is_set() &&
(!cfg.native_shard_aware_transport_port.is_set() ||
cfg.native_shard_aware_transport_port_ssl() != cfg.native_shard_aware_transport_port())) {
configs.emplace_back(listen_cfg{{ip, cfg.native_shard_aware_transport_port_ssl()}, true, std::move(cred)});
_listen_addresses.push_back(configs.back().addr);
} else if (native_shard_aware_port_idx >= 0) {
configs[native_shard_aware_port_idx].cred = std::move(cred);
}
}

return parallel_for_each(configs, [&cserver, keepalive](const listen_cfg & cfg) {
return listen_on_all_shards(cserver, cfg.addr, cfg.cred, cfg.is_shard_aware, keepalive, std::nullopt);
});
}

future<> controller::start_listening_on_maintenance_socket(sharded<cql_server>& cserver) {
auto socket = _config.maintenance_socket();

if (socket == "workdir") {
socket = _config.work_directory() + "/cql.m";
}

auto max_socket_length = sizeof(sockaddr_un::sun_path);
if (socket.length() > max_socket_length - 1) {
throw std::runtime_error(format("Maintenance socket path is too long: {}. Change it to string shorter than {} chars.", socket, max_socket_length));
}

struct stat statbuf;
auto stat_result = ::stat(socket.c_str(), &statbuf);
if (stat_result == 0) {
// Check if it is a unix domain socket, not a regular file or directory
if (!S_ISSOCK(statbuf.st_mode)) {
throw std::runtime_error(format("Under maintenance socket path ({}) there is something else.", socket));
}
} else if (errno != ENOENT) {
// Other error than "file does not exist"
throw std::runtime_error(format("Failed to stat {}: {}", socket, strerror(errno)));
}

// Remove the socket if it already exists, otherwise when the server
// tries to listen on it, it will hang on bind().
auto unlink_result = ::unlink(socket.c_str());
if (unlink_result < 0 && errno != ENOENT) {
// Other error than "file does not exist"
throw std::runtime_error(format("Failed to unlink {}: {}", socket, strerror(errno)));
}

auto addr = socket_address { unix_domain_addr { socket } };
_listen_addresses.push_back(addr);

logger.info("Setting up maintenance socket on {}", socket);

auto unix_domain_socket_permissions =
file_permissions::user_read | file_permissions::user_write |
file_permissions::group_read | file_permissions::group_write;

co_await listen_on_all_shards(cserver, addr, nullptr, false, _config.rpc_keepalive(), unix_domain_socket_permissions);

if (_config.maintenance_socket_group.is_set()) {
auto group_name = _config.maintenance_socket_group();
struct group *grp;
grp = ::getgrnam(group_name.c_str());
if (!grp) {
throw std::runtime_error(format("Group id of {} not found. Make sure the group exists.", group_name));
}

auto chown_result = ::chown(socket.c_str(), ::geteuid(), grp->gr_gid);
if (chown_result < 0) {
if (errno == EPERM) {
throw std::runtime_error(format("Failed to change group of {}: Permission denied. Make sure the user has the root privilege or is a member of the group {}.", socket, group_name));
} else {
throw std::runtime_error(format("Failed to chown {}: {} ()", socket, strerror(errno)));
}
}
}
}

future<> controller::do_start_server() {
if (_server) {
return make_ready_future<>();
Expand All @@ -74,10 +212,6 @@ future<> controller::do_start_server() {
auto cserver = std::make_unique<sharded<cql_server>>();

auto& cfg = _config;
auto preferred = cfg.rpc_interface_prefer_ipv6() ? std::make_optional(net::inet_address::family::INET6) : std::nullopt;
auto family = cfg.enable_ipv6_dns_lookup() || preferred ? std::nullopt : std::make_optional(net::inet_address::family::INET);
auto ceo = cfg.client_encryption_options();
auto keepalive = cfg.rpc_keepalive();
smp_service_group_config cql_server_smp_service_group_config;
cql_server_smp_service_group_config.max_nonlocal_requests = 5000;
auto bounce_request_smp_service_group = create_smp_service_group(cql_server_smp_service_group_config).get();
Expand Down Expand Up @@ -110,92 +244,6 @@ future<> controller::do_start_server() {
std::shared_ptr<seastar::tls::credentials_builder> cred;
};

_listen_addresses.clear();
std::vector<listen_cfg> configs;

if (!_used_by_maintenance_socket) {
const seastar::net::inet_address ip = utils::resolve(cfg.rpc_address, family, preferred).get();
int native_port_idx = -1, native_shard_aware_port_idx = -1;

if (cfg.native_transport_port.is_set() ||
(!cfg.native_transport_port_ssl.is_set() && !cfg.native_transport_port.is_set())) {
// Non-SSL port is specified || neither SSL nor non-SSL ports are specified
configs.emplace_back(listen_cfg{ socket_address{ip, cfg.native_transport_port()}, false });
_listen_addresses.push_back(configs.back().addr);
native_port_idx = 0;
}
if (cfg.native_shard_aware_transport_port.is_set() ||
(!cfg.native_shard_aware_transport_port_ssl.is_set() && !cfg.native_shard_aware_transport_port.is_set())) {
configs.emplace_back(listen_cfg{ socket_address{ip, cfg.native_shard_aware_transport_port()}, true });
_listen_addresses.push_back(configs.back().addr);
native_shard_aware_port_idx = native_port_idx + 1;
}

// main should have made sure values are clean and neatish
if (utils::is_true(utils::get_or_default(ceo, "enabled", "false"))) {
auto cred = std::make_shared<seastar::tls::credentials_builder>();
utils::configure_tls_creds_builder(*cred, std::move(ceo)).get();

logger.info("Enabling encrypted CQL connections between client and server");

if (cfg.native_transport_port_ssl.is_set() &&
(!cfg.native_transport_port.is_set() ||
cfg.native_transport_port_ssl() != cfg.native_transport_port())) {
// SSL port is specified && non-SSL port is either left out or set to a different value
configs.emplace_back(listen_cfg{{ip, cfg.native_transport_port_ssl()}, false, cred});
_listen_addresses.push_back(configs.back().addr);
} else if (native_port_idx >= 0) {
configs[native_port_idx].cred = cred;
}
if (cfg.native_shard_aware_transport_port_ssl.is_set() &&
(!cfg.native_shard_aware_transport_port.is_set() ||
cfg.native_shard_aware_transport_port_ssl() != cfg.native_shard_aware_transport_port())) {
configs.emplace_back(listen_cfg{{ip, cfg.native_shard_aware_transport_port_ssl()}, true, std::move(cred)});
_listen_addresses.push_back(configs.back().addr);
} else if (native_shard_aware_port_idx >= 0) {
configs[native_shard_aware_port_idx].cred = std::move(cred);
}
}
} else {
auto socket = cfg.maintenance_socket();

if (socket == "workdir") {
socket = cfg.work_directory() + "/cql.m";
}

if (socket.length() > 107) {
throw std::runtime_error(format("Maintenance socket path is too long: {}. Change it to string shorter than 108 chars.", socket));
}

struct stat statbuf;
auto stat_result = ::stat(socket.c_str(), &statbuf);
if (stat_result == 0) {
// Check if it is a unix domain socket, not a regular file or directory
if (!S_ISSOCK(statbuf.st_mode)) {
throw std::runtime_error(format("Under maintenance socket path ({}) there is something else.", socket));
}
} else if (errno != ENOENT) {
// Other error than "file does not exist"
throw std::runtime_error(format("Failed to stat {}: {}", socket, strerror(errno)));
}

// Remove the socket if it already exists, otherwise when the server
// tries to listen on it, it will hang on bind().
auto unlink_result = ::unlink(socket.c_str());
if (unlink_result < 0 && errno != ENOENT) {
// Other error than "file does not exist"
throw std::runtime_error(format("Failed to unlink {}: {}", socket, strerror(errno)));
}

configs.emplace_back(listen_cfg {
.addr = socket_address { unix_domain_addr { socket } },
.is_shard_aware = false
});
_listen_addresses.push_back(configs.back().addr);

logger.info("Setting up maintenance socket on {}", socket);
}

cserver->start(std::ref(_qp), std::ref(_auth_service), std::ref(_mem_limiter), std::move(get_cql_server_config), std::ref(cfg), std::ref(_sl_controller), std::ref(_gossiper), _cql_opcode_stats_key, _used_by_maintenance_socket).get();
auto on_error = defer([&cserver] { cserver->stop().get(); });

Expand All @@ -204,13 +252,12 @@ future<> controller::do_start_server() {
unsubscribe_server(*cserver).get();
});

parallel_for_each(configs, [&cserver, keepalive](const listen_cfg & cfg) {
return cserver->invoke_on_all(&cql_server::listen, cfg.addr, cfg.cred, cfg.is_shard_aware, keepalive).then([cfg] {
logger.info("Starting listening for CQL clients on {} ({}, {})"
, cfg.addr, cfg.cred ? "encrypted" : "unencrypted", cfg.is_shard_aware ? "shard-aware" : "non-shard-aware"
);
});
}).get();
_listen_addresses.clear();
if (!_used_by_maintenance_socket) {
start_listening_on_tcp_sockets(*cserver).get();
} else {
start_listening_on_maintenance_socket(*cserver).get();
}

if (!_used_by_maintenance_socket) {
set_cql_ready(true).get();
Expand Down
3 changes: 3 additions & 0 deletions transport/controller.hh
View file
Expand Up @@ -59,6 +59,9 @@ class controller : public protocol_server {
future<> subscribe_server(sharded<cql_server>& server);
future<> unsubscribe_server(sharded<cql_server>& server);

future<> start_listening_on_tcp_sockets(sharded<cql_server>& cserver);
future<> start_listening_on_maintenance_socket(sharded<cql_server>& cserver);

maintenance_socket_enabled _used_by_maintenance_socket;

public:
Expand Down

0 comments on commit 93af3dd

Please sign in to comment.