From 6b178f9a4a90176ed4534a70468b63efd2e9dfc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Grzebieluch?= Date: Fri, 26 Jan 2024 16:17:55 +0100 Subject: [PATCH 1/5] transport/controller: split configuring sockets into separate functions TCP sockets and unix domain sockets don't share common listen options excluding `socket_address`. For unix domain sockets, available options will be expanded to cover also filesystem permissions and owner for the socket. Storing listen options for both types of sockets in one structure would become messy. For now, both use `listen_cfg`. In a singular cql controller, only sockets of one type are created, thus it can be easily split into two cases. Isolate maintenance socket from `listen_cfg`. --- transport/controller.cc | 216 ++++++++++++++++++++++------------------ transport/controller.hh | 3 + 2 files changed, 122 insertions(+), 97 deletions(-) diff --git a/transport/controller.cc b/transport/controller.cc index 8b9ac261da89..5829e6306885 100644 --- a/transport/controller.cc +++ b/transport/controller.cc @@ -65,6 +65,119 @@ future<> controller::start_server() { return do_start_server().finally([this] { _ops_sem.signal(); }); } +static future<> listen_on_all_shards(sharded& cserver, socket_address addr, std::shared_ptr creds, bool is_shard_aware, bool keepalive) { + co_await cserver.invoke_on_all([addr, creds, is_shard_aware, keepalive] (cql_server& server) { + return server.listen(addr, creds, is_shard_aware, keepalive); + }); + + logger.info("Starting listening for CQL clients on {} ({}, {})" + , addr, creds ? "encrypted" : "unencrypted", is_shard_aware ? "shard-aware" : "non-shard-aware" + ); +} + +future<> controller::start_listening_on_tcp_sockets(sharded& cserver) { + auto& cfg = _config; + auto preferred = cfg.rpc_interface_prefer_ipv6() ? std::make_optional(net::inet_address::family::INET6) : std::nullopt; + auto family = cfg.enable_ipv6_dns_lookup() || preferred ? std::nullopt : std::make_optional(net::inet_address::family::INET); + auto ceo = cfg.client_encryption_options(); + auto keepalive = cfg.rpc_keepalive(); + + struct listen_cfg { + socket_address addr; + bool is_shard_aware; + std::shared_ptr cred; + }; + + _listen_addresses.clear(); + std::vector configs; + + const seastar::net::inet_address ip = utils::resolve(cfg.rpc_address, family, preferred).get0(); + int native_port_idx = -1, native_shard_aware_port_idx = -1; + + if (cfg.native_transport_port.is_set() || + (!cfg.native_transport_port_ssl.is_set() && !cfg.native_transport_port.is_set())) { + // Non-SSL port is specified || neither SSL nor non-SSL ports are specified + configs.emplace_back(listen_cfg{ socket_address{ip, cfg.native_transport_port()}, false }); + _listen_addresses.push_back(configs.back().addr); + native_port_idx = 0; + } + if (cfg.native_shard_aware_transport_port.is_set() || + (!cfg.native_shard_aware_transport_port_ssl.is_set() && !cfg.native_shard_aware_transport_port.is_set())) { + configs.emplace_back(listen_cfg{ socket_address{ip, cfg.native_shard_aware_transport_port()}, true }); + _listen_addresses.push_back(configs.back().addr); + native_shard_aware_port_idx = native_port_idx + 1; + } + + // main should have made sure values are clean and neatish + if (utils::is_true(utils::get_or_default(ceo, "enabled", "false"))) { + auto cred = std::make_shared(); + utils::configure_tls_creds_builder(*cred, std::move(ceo)).get(); + + logger.info("Enabling encrypted CQL connections between client and server"); + + if (cfg.native_transport_port_ssl.is_set() && + (!cfg.native_transport_port.is_set() || + cfg.native_transport_port_ssl() != cfg.native_transport_port())) { + // SSL port is specified && non-SSL port is either left out or set to a different value + configs.emplace_back(listen_cfg{{ip, cfg.native_transport_port_ssl()}, false, cred}); + _listen_addresses.push_back(configs.back().addr); + } else if (native_port_idx >= 0) { + configs[native_port_idx].cred = cred; + } + if (cfg.native_shard_aware_transport_port_ssl.is_set() && + (!cfg.native_shard_aware_transport_port.is_set() || + cfg.native_shard_aware_transport_port_ssl() != cfg.native_shard_aware_transport_port())) { + configs.emplace_back(listen_cfg{{ip, cfg.native_shard_aware_transport_port_ssl()}, true, std::move(cred)}); + _listen_addresses.push_back(configs.back().addr); + } else if (native_shard_aware_port_idx >= 0) { + configs[native_shard_aware_port_idx].cred = std::move(cred); + } + } + + return parallel_for_each(configs, [&cserver, keepalive](const listen_cfg & cfg) { + return listen_on_all_shards(cserver, cfg.addr, cfg.cred, cfg.is_shard_aware, keepalive); + }); +} + +future<> controller::start_listening_on_maintenance_socket(sharded& cserver) { + auto socket = _config.maintenance_socket(); + + if (socket == "workdir") { + socket = _config.work_directory() + "/cql.m"; + } + + if (socket.length() > 107) { + throw std::runtime_error(format("Maintenance socket path is too long: {}. Change it to string shorter than 108 chars.", socket)); + } + + struct stat statbuf; + auto stat_result = ::stat(socket.c_str(), &statbuf); + if (stat_result == 0) { + // Check if it is a unix domain socket, not a regular file or directory + if (!S_ISSOCK(statbuf.st_mode)) { + throw std::runtime_error(format("Under maintenance socket path ({}) there is something else.", socket)); + } + } else if (errno != ENOENT) { + // Other error than "file does not exist" + throw std::runtime_error(format("Failed to stat {}: {}", socket, strerror(errno))); + } + + // Remove the socket if it already exists, otherwise when the server + // tries to listen on it, it will hang on bind(). + auto unlink_result = ::unlink(socket.c_str()); + if (unlink_result < 0 && errno != ENOENT) { + // Other error than "file does not exist" + throw std::runtime_error(format("Failed to unlink {}: {}", socket, strerror(errno))); + } + + auto addr = socket_address { unix_domain_addr { socket } }; + _listen_addresses.push_back(addr); + + logger.info("Setting up maintenance socket on {}", socket); + + return listen_on_all_shards(cserver, addr, nullptr, false, _config.rpc_keepalive()); +} + future<> controller::do_start_server() { if (_server) { return make_ready_future<>(); @@ -74,10 +187,6 @@ future<> controller::do_start_server() { auto cserver = std::make_unique>(); auto& cfg = _config; - auto preferred = cfg.rpc_interface_prefer_ipv6() ? std::make_optional(net::inet_address::family::INET6) : std::nullopt; - auto family = cfg.enable_ipv6_dns_lookup() || preferred ? std::nullopt : std::make_optional(net::inet_address::family::INET); - auto ceo = cfg.client_encryption_options(); - auto keepalive = cfg.rpc_keepalive(); smp_service_group_config cql_server_smp_service_group_config; cql_server_smp_service_group_config.max_nonlocal_requests = 5000; auto bounce_request_smp_service_group = create_smp_service_group(cql_server_smp_service_group_config).get(); @@ -110,92 +219,6 @@ future<> controller::do_start_server() { std::shared_ptr cred; }; - _listen_addresses.clear(); - std::vector configs; - - if (!_used_by_maintenance_socket) { - const seastar::net::inet_address ip = utils::resolve(cfg.rpc_address, family, preferred).get(); - int native_port_idx = -1, native_shard_aware_port_idx = -1; - - if (cfg.native_transport_port.is_set() || - (!cfg.native_transport_port_ssl.is_set() && !cfg.native_transport_port.is_set())) { - // Non-SSL port is specified || neither SSL nor non-SSL ports are specified - configs.emplace_back(listen_cfg{ socket_address{ip, cfg.native_transport_port()}, false }); - _listen_addresses.push_back(configs.back().addr); - native_port_idx = 0; - } - if (cfg.native_shard_aware_transport_port.is_set() || - (!cfg.native_shard_aware_transport_port_ssl.is_set() && !cfg.native_shard_aware_transport_port.is_set())) { - configs.emplace_back(listen_cfg{ socket_address{ip, cfg.native_shard_aware_transport_port()}, true }); - _listen_addresses.push_back(configs.back().addr); - native_shard_aware_port_idx = native_port_idx + 1; - } - - // main should have made sure values are clean and neatish - if (utils::is_true(utils::get_or_default(ceo, "enabled", "false"))) { - auto cred = std::make_shared(); - utils::configure_tls_creds_builder(*cred, std::move(ceo)).get(); - - logger.info("Enabling encrypted CQL connections between client and server"); - - if (cfg.native_transport_port_ssl.is_set() && - (!cfg.native_transport_port.is_set() || - cfg.native_transport_port_ssl() != cfg.native_transport_port())) { - // SSL port is specified && non-SSL port is either left out or set to a different value - configs.emplace_back(listen_cfg{{ip, cfg.native_transport_port_ssl()}, false, cred}); - _listen_addresses.push_back(configs.back().addr); - } else if (native_port_idx >= 0) { - configs[native_port_idx].cred = cred; - } - if (cfg.native_shard_aware_transport_port_ssl.is_set() && - (!cfg.native_shard_aware_transport_port.is_set() || - cfg.native_shard_aware_transport_port_ssl() != cfg.native_shard_aware_transport_port())) { - configs.emplace_back(listen_cfg{{ip, cfg.native_shard_aware_transport_port_ssl()}, true, std::move(cred)}); - _listen_addresses.push_back(configs.back().addr); - } else if (native_shard_aware_port_idx >= 0) { - configs[native_shard_aware_port_idx].cred = std::move(cred); - } - } - } else { - auto socket = cfg.maintenance_socket(); - - if (socket == "workdir") { - socket = cfg.work_directory() + "/cql.m"; - } - - if (socket.length() > 107) { - throw std::runtime_error(format("Maintenance socket path is too long: {}. Change it to string shorter than 108 chars.", socket)); - } - - struct stat statbuf; - auto stat_result = ::stat(socket.c_str(), &statbuf); - if (stat_result == 0) { - // Check if it is a unix domain socket, not a regular file or directory - if (!S_ISSOCK(statbuf.st_mode)) { - throw std::runtime_error(format("Under maintenance socket path ({}) there is something else.", socket)); - } - } else if (errno != ENOENT) { - // Other error than "file does not exist" - throw std::runtime_error(format("Failed to stat {}: {}", socket, strerror(errno))); - } - - // Remove the socket if it already exists, otherwise when the server - // tries to listen on it, it will hang on bind(). - auto unlink_result = ::unlink(socket.c_str()); - if (unlink_result < 0 && errno != ENOENT) { - // Other error than "file does not exist" - throw std::runtime_error(format("Failed to unlink {}: {}", socket, strerror(errno))); - } - - configs.emplace_back(listen_cfg { - .addr = socket_address { unix_domain_addr { socket } }, - .is_shard_aware = false - }); - _listen_addresses.push_back(configs.back().addr); - - logger.info("Setting up maintenance socket on {}", socket); - } - cserver->start(std::ref(_qp), std::ref(_auth_service), std::ref(_mem_limiter), std::move(get_cql_server_config), std::ref(cfg), std::ref(_sl_controller), std::ref(_gossiper), _cql_opcode_stats_key, _used_by_maintenance_socket).get(); auto on_error = defer([&cserver] { cserver->stop().get(); }); @@ -204,13 +227,12 @@ future<> controller::do_start_server() { unsubscribe_server(*cserver).get(); }); - parallel_for_each(configs, [&cserver, keepalive](const listen_cfg & cfg) { - return cserver->invoke_on_all(&cql_server::listen, cfg.addr, cfg.cred, cfg.is_shard_aware, keepalive).then([cfg] { - logger.info("Starting listening for CQL clients on {} ({}, {})" - , cfg.addr, cfg.cred ? "encrypted" : "unencrypted", cfg.is_shard_aware ? "shard-aware" : "non-shard-aware" - ); - }); - }).get(); + _listen_addresses.clear(); + if (!_used_by_maintenance_socket) { + start_listening_on_tcp_sockets(*cserver).get(); + } else { + start_listening_on_maintenance_socket(*cserver).get(); + } set_cql_ready(true).get(); diff --git a/transport/controller.hh b/transport/controller.hh index edd627592c45..411556a0384d 100644 --- a/transport/controller.hh +++ b/transport/controller.hh @@ -59,6 +59,9 @@ class controller : public protocol_server { future<> subscribe_server(sharded& server); future<> unsubscribe_server(sharded& server); + future<> start_listening_on_tcp_sockets(sharded& cserver); + future<> start_listening_on_maintenance_socket(sharded& cserver); + maintenance_socket_enabled _used_by_maintenance_socket; public: From 4cecda7ead1479a253e6ff0c2e631d94976369c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Grzebieluch?= Date: Thu, 1 Feb 2024 15:46:33 +0100 Subject: [PATCH 2/5] transport/controller: pass unix_domain_socket_permissions to generic_server::listen --- generic_server.cc | 5 +++-- generic_server.hh | 3 ++- redis/controller.cc | 2 +- transport/controller.cc | 11 ++++++----- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/generic_server.cc b/generic_server.cc index daec5d47e84b..28a6c8096f16 100644 --- a/generic_server.cc +++ b/generic_server.cc @@ -145,7 +145,7 @@ future<> server::shutdown() { } future<> -server::listen(socket_address addr, std::shared_ptr creds, bool is_shard_aware, bool keepalive) { +server::listen(socket_address addr, std::shared_ptr creds, bool is_shard_aware, bool keepalive, std::optional unix_domain_socket_permissions) { auto f = make_ready_future>(nullptr); if (creds) { f = creds->build_reloadable_server_credentials([this](const std::unordered_set& files, std::exception_ptr ep) { @@ -156,9 +156,10 @@ server::listen(socket_address addr, std::shared_ptr creds) { + return f.then([this, addr, is_shard_aware, keepalive, unix_domain_socket_permissions](shared_ptr creds) { listen_options lo; lo.reuse_address = true; + lo.unix_domain_socket_permissions = unix_domain_socket_permissions; if (is_shard_aware) { lo.lba = server_socket::load_balancing_algorithm::port; } diff --git a/generic_server.hh b/generic_server.hh index 5593d42fb67d..2371b9122775 100644 --- a/generic_server.hh +++ b/generic_server.hh @@ -14,6 +14,7 @@ #include +#include #include #include #include @@ -107,7 +108,7 @@ public: future<> shutdown(); future<> stop(); - future<> listen(socket_address addr, std::shared_ptr creds, bool is_shard_aware, bool keepalive); + future<> listen(socket_address addr, std::shared_ptr creds, bool is_shard_aware, bool keepalive, std::optional unix_domain_socket_permissions); future<> do_accepts(int which, bool keepalive, socket_address server_addr); diff --git a/redis/controller.cc b/redis/controller.cc index ca711eb8758c..e55ea0d1284d 100644 --- a/redis/controller.cc +++ b/redis/controller.cc @@ -87,7 +87,7 @@ future<> controller::listen(seastar::sharded& auth_service, db::c return f.then([server, configs = std::move(configs), keepalive] { return parallel_for_each(configs, [server, keepalive](const listen_cfg & cfg) { - return server->invoke_on_all(&redis_transport::redis_server::listen, cfg.addr, cfg.cred, false, keepalive).then([cfg] { + return server->invoke_on_all(&redis_transport::redis_server::listen, cfg.addr, cfg.cred, false, keepalive, std::nullopt).then([cfg] { slogger.info("Starting listening for REDIS clients on {} ({})", cfg.addr, cfg.cred ? "encrypted" : "unencrypted"); }); }); diff --git a/transport/controller.cc b/transport/controller.cc index 5829e6306885..b873da3668bf 100644 --- a/transport/controller.cc +++ b/transport/controller.cc @@ -10,6 +10,7 @@ #include #include #include +#include #include "transport/server.hh" #include "service/memory_limiter.hh" #include "db/config.hh" @@ -65,9 +66,9 @@ future<> controller::start_server() { return do_start_server().finally([this] { _ops_sem.signal(); }); } -static future<> listen_on_all_shards(sharded& cserver, socket_address addr, std::shared_ptr creds, bool is_shard_aware, bool keepalive) { - co_await cserver.invoke_on_all([addr, creds, is_shard_aware, keepalive] (cql_server& server) { - return server.listen(addr, creds, is_shard_aware, keepalive); +static future<> listen_on_all_shards(sharded& cserver, socket_address addr, std::shared_ptr creds, bool is_shard_aware, bool keepalive, std::optional unix_domain_socket_permissions) { + co_await cserver.invoke_on_all([addr, creds, is_shard_aware, keepalive, unix_domain_socket_permissions] (cql_server& server) { + return server.listen(addr, creds, is_shard_aware, keepalive, unix_domain_socket_permissions); }); logger.info("Starting listening for CQL clients on {} ({}, {})" @@ -135,7 +136,7 @@ future<> controller::start_listening_on_tcp_sockets(sharded& cserver } return parallel_for_each(configs, [&cserver, keepalive](const listen_cfg & cfg) { - return listen_on_all_shards(cserver, cfg.addr, cfg.cred, cfg.is_shard_aware, keepalive); + return listen_on_all_shards(cserver, cfg.addr, cfg.cred, cfg.is_shard_aware, keepalive, std::nullopt); }); } @@ -175,7 +176,7 @@ future<> controller::start_listening_on_maintenance_socket(sharded& logger.info("Setting up maintenance socket on {}", socket); - return listen_on_all_shards(cserver, addr, nullptr, false, _config.rpc_keepalive()); + return listen_on_all_shards(cserver, addr, nullptr, false, _config.rpc_keepalive(), std::nullopt); } future<> controller::do_start_server() { From fffb732704c52d600bb7301929c1918596adf927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Grzebieluch?= Date: Thu, 1 Feb 2024 15:47:34 +0100 Subject: [PATCH 3/5] transport/controller: set unix_domain_socket_permissions for maintenance_socket Set filesystem permissions for the maintenance socket to 660. Fixes #16487 --- transport/controller.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/transport/controller.cc b/transport/controller.cc index b873da3668bf..2a452f9955c0 100644 --- a/transport/controller.cc +++ b/transport/controller.cc @@ -176,7 +176,11 @@ future<> controller::start_listening_on_maintenance_socket(sharded& logger.info("Setting up maintenance socket on {}", socket); - return listen_on_all_shards(cserver, addr, nullptr, false, _config.rpc_keepalive(), std::nullopt); + auto unix_domain_socket_permissions = + file_permissions::user_read | file_permissions::user_write | + file_permissions::group_read | file_permissions::group_write; + + return listen_on_all_shards(cserver, addr, nullptr, false, _config.rpc_keepalive(), unix_domain_socket_permissions); } future<> controller::do_start_server() { From 38191144ac361757ebb3d756d03fd5a18c3bee60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Grzebieluch?= Date: Mon, 5 Feb 2024 14:19:20 +0100 Subject: [PATCH 4/5] transport/controller: get rid of magic number for socket path's maximal length Calculate `max_socket_length` from the size of the structure representing the Unix domain socket address. --- transport/controller.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transport/controller.cc b/transport/controller.cc index 2a452f9955c0..e23904944da6 100644 --- a/transport/controller.cc +++ b/transport/controller.cc @@ -147,8 +147,9 @@ future<> controller::start_listening_on_maintenance_socket(sharded& socket = _config.work_directory() + "/cql.m"; } - if (socket.length() > 107) { - throw std::runtime_error(format("Maintenance socket path is too long: {}. Change it to string shorter than 108 chars.", socket)); + auto max_socket_length = sizeof(sockaddr_un::sun_path); + if (socket.length() > max_socket_length - 1) { + throw std::runtime_error(format("Maintenance socket path is too long: {}. Change it to string shorter than {} chars.", socket, max_socket_length)); } struct stat statbuf; From 182cfebe40bd49ca23070ebca9b18e71e78bfbc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Grzebieluch?= Date: Mon, 5 Feb 2024 17:50:50 +0100 Subject: [PATCH 5/5] maintenance_socket: add option to set owning group Option `maintenance-socket-group` sets the owning group of the maintenance socket. If not set, the group will be the same as the user running the scylla node. --- db/config.cc | 2 ++ db/config.hh | 1 + .../admin-tools/maintenance-socket.rst | 4 ++++ transport/controller.cc | 21 ++++++++++++++++++- 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/db/config.cc b/db/config.cc index d6475729e159..790d42ec3b53 100644 --- a/db/config.cc +++ b/db/config.cc @@ -764,6 +764,8 @@ db::config::config(std::shared_ptr exts) "\tworkdir the node will open the maintenance socket on the path /cql.m,\n" "\t where is a path defined by the workdir configuration option\n" "\t the node will open the maintenance socket on the path ") + , maintenance_socket_group(this, "maintenance_socket_group", value_status::Used, "", + "The group that the maintenance socket will be owned by. If not set, the group will be the same as the user running the scylla node.") , maintenance_mode(this, "maintenance_mode", value_status::Used, false, "If set to true, the node will not connect to other nodes. It will only serve requests to its local data.") , native_transport_port_ssl(this, "native_transport_port_ssl", value_status::Used, 9142, "Port on which the CQL TLS native transport listens for clients." diff --git a/db/config.hh b/db/config.hh index 5a6e440ec040..a6de585e4698 100644 --- a/db/config.hh +++ b/db/config.hh @@ -276,6 +276,7 @@ public: named_value start_native_transport; named_value native_transport_port; named_value maintenance_socket; + named_value maintenance_socket_group; named_value maintenance_mode; named_value native_transport_port_ssl; named_value native_shard_aware_transport_port; diff --git a/docs/operating-scylla/admin-tools/maintenance-socket.rst b/docs/operating-scylla/admin-tools/maintenance-socket.rst index f35d7e67598e..b95a67a95f27 100644 --- a/docs/operating-scylla/admin-tools/maintenance-socket.rst +++ b/docs/operating-scylla/admin-tools/maintenance-socket.rst @@ -11,11 +11,15 @@ To set up the maintenance socket, use the `maintenance-socket` option when start * If set to `workdir` maintenance socket will be created in `/cql.m`. * Otherwise maintenance socket will be created in the specified path. + The maintenance socket path has to satisfy following restrictions: * the path has to be shorter than `108` chars (due to linux limits), * a file or a directory cannot exists in this path. +Option `maintenance-socket-group` sets the owning group of the maintenance socket. If not set, the group will be the same as the user running the scylla node. +The user running the scylla node has to be in the group specified by `maintenance-socket-group` option or have root privileges. + Connect to maintenance socket ----------------------------- diff --git a/transport/controller.cc b/transport/controller.cc index e23904944da6..00a87ebe06a8 100644 --- a/transport/controller.cc +++ b/transport/controller.cc @@ -6,6 +6,7 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ +#include #include "transport/controller.hh" #include #include @@ -181,7 +182,25 @@ future<> controller::start_listening_on_maintenance_socket(sharded& file_permissions::user_read | file_permissions::user_write | file_permissions::group_read | file_permissions::group_write; - return listen_on_all_shards(cserver, addr, nullptr, false, _config.rpc_keepalive(), unix_domain_socket_permissions); + co_await listen_on_all_shards(cserver, addr, nullptr, false, _config.rpc_keepalive(), unix_domain_socket_permissions); + + if (_config.maintenance_socket_group.is_set()) { + auto group_name = _config.maintenance_socket_group(); + struct group *grp; + grp = ::getgrnam(group_name.c_str()); + if (!grp) { + throw std::runtime_error(format("Group id of {} not found. Make sure the group exists.", group_name)); + } + + auto chown_result = ::chown(socket.c_str(), ::geteuid(), grp->gr_gid); + if (chown_result < 0) { + if (errno == EPERM) { + throw std::runtime_error(format("Failed to change group of {}: Permission denied. Make sure the user has the root privilege or is a member of the group {}.", socket, group_name)); + } else { + throw std::runtime_error(format("Failed to chown {}: {} ()", socket, strerror(errno))); + } + } + } } future<> controller::do_start_server() {