From c52efc17b40b74749eace00b9756cc359dc88730 Mon Sep 17 00:00:00 2001 From: Jens Elkner Date: Sat, 18 Nov 2023 21:07:10 +0100 Subject: [PATCH 01/12] use enums instead of raw values --- src/apps/relay/ns_ioalib_engine_impl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/apps/relay/ns_ioalib_engine_impl.c b/src/apps/relay/ns_ioalib_engine_impl.c index c62d92e22..5d3448b28 100644 --- a/src/apps/relay/ns_ioalib_engine_impl.c +++ b/src/apps/relay/ns_ioalib_engine_impl.c @@ -3507,13 +3507,13 @@ void stun_report_binding(void *a, STUN_PROMETHEUS_METRIC_TYPE type) { #if !defined(TURN_NO_PROMETHEUS) UNUSED_ARG(a); switch (type) { - case 0: + case STUN_PROMETHEUS_METRIC_TYPE_REQUEST: prom_inc_stun_binding_request(); break; - case 1: + case STUN_PROMETHEUS_METRIC_TYPE_RESPONSE: prom_inc_stun_binding_response(); break; - case 2: + case STUN_PROMETHEUS_METRIC_TYPE_ERROR: prom_inc_stun_binding_error(); break; default: From fe5477f3f6ea4134dc86a0711710e3a9ad0a6cc8 Mon Sep 17 00:00:00 2001 From: Jens Elkner Date: Tue, 21 Nov 2023 03:33:38 +0100 Subject: [PATCH 02/12] libprom support use https://github.com/jelmd/libprom instead of the bugyy digitalocean/prometheus-client-c including its compact feature. --- README.turnserver | 7 ++ man/man1/turnserver.1 | 9 +++ src/apps/relay/mainrelay.c | 7 ++ src/apps/relay/mainrelay.h | 1 + src/apps/relay/prom_server.c | 140 +++++++++++++++-------------------- src/apps/relay/prom_server.h | 5 +- 6 files changed, 84 insertions(+), 85 deletions(-) diff --git a/README.turnserver b/README.turnserver index de4728c0b..d10dc65db 100644 --- a/README.turnserver +++ b/README.turnserver @@ -297,6 +297,13 @@ Flags: --prometheus-port Prometheus listener port (Default: 9641). +--prometheus-compact Omit the HELP and TYPE comments when generating + the prometheus metrics report. Usually it is sufficient to get and + record them once. There is no need to send them again and again on + each report - would just waste bandwith and demands more resources + to parse and process the report. On production systems it is + recommended to use this option. + -h Help. Options with values: diff --git a/man/man1/turnserver.1 b/man/man1/turnserver.1 index 993206896..e56ddc1ad 100644 --- a/man/man1/turnserver.1 +++ b/man/man1/turnserver.1 @@ -444,6 +444,15 @@ authentication with ephemeral usernames (e.g. TURN REST API). .B \fB\-\-prometheus\-port\fP Prometheus listener port (Default: 9641). + +.TP +.B --prometheus-compact +Omit the HELP and TYPE comments when generating the prometheus metrics +report. Usually it is sufficient to get and record them once. There is +no need to send them again and again on each report - would just waste +bandwith and demands more resources to parse and process the report. +On production systems it is recommended to use this option. + .TP .B \fB\-h\fP diff --git a/src/apps/relay/mainrelay.c b/src/apps/relay/mainrelay.c index 88ed1e07f..a1b972274 100644 --- a/src/apps/relay/mainrelay.c +++ b/src/apps/relay/mainrelay.c @@ -214,6 +214,7 @@ turn_params_t turn_params = { 0, /* prometheus disabled by default */ DEFAULT_PROM_SERVER_PORT, /* prometheus port */ 0, /* prometheus username labelling disabled by default when prometheus is enabled */ + 0, /* prometheus compact output */ ///////////// Users DB ////////////// {(TURN_USERDB_TYPE)0, {"\0", "\0"}, {0, NULL, {NULL, 0}}}, @@ -1070,6 +1071,7 @@ static char Usage[] = " also the path / on this port can be used as a health check\n" " --prometheus-port Prometheus metrics port (Default: 9641).\n" " --prometheus-username-labels When metrics are enabled, add labels with client usernames.\n" + " --prometheus-compact Omit the optional # HELP and # TYPE comments for each metric output.\n" #endif " --use-auth-secret TURN REST API flag.\n" " Flag that sets a special authorization option that is based upon " @@ -1367,6 +1369,7 @@ enum EXTRA_OPTS { PROMETHEUS_OPT, PROMETHEUS_PORT_OPT, PROMETHEUS_ENABLE_USERNAMES_OPT, + PROMETHEUS_COMPACT_OPT, AUTH_SECRET_OPT, NO_AUTH_PINGS_OPT, NO_DYNAMIC_IP_LIST_OPT, @@ -1487,6 +1490,7 @@ static const struct myoption long_options[] = { {"prometheus", optional_argument, NULL, PROMETHEUS_OPT}, {"prometheus-port", optional_argument, NULL, PROMETHEUS_PORT_OPT}, {"prometheus-username-labels", optional_argument, NULL, PROMETHEUS_ENABLE_USERNAMES_OPT}, + {"prometheus-compact", optional_argument, NULL, PROMETHEUS_COMPACT_OPT}, #endif {"use-auth-secret", optional_argument, NULL, AUTH_SECRET_OPT}, {"static-auth-secret", required_argument, NULL, STATIC_AUTH_SECRET_VAL_OPT}, @@ -2130,6 +2134,9 @@ static void set_option(int c, char *value) { case PROMETHEUS_ENABLE_USERNAMES_OPT: turn_params.prometheus_username_labels = 1; break; + case PROMETHEUS_COMPACT_OPT: + turn_params.prometheus_compact = 1; + break; case AUTH_SECRET_OPT: turn_params.use_auth_secret_with_timestamp = 1; use_tltc = 1; diff --git a/src/apps/relay/mainrelay.h b/src/apps/relay/mainrelay.h index 98b901358..c2aa1e058 100644 --- a/src/apps/relay/mainrelay.h +++ b/src/apps/relay/mainrelay.h @@ -310,6 +310,7 @@ typedef struct _turn_params_ { int prometheus; int prometheus_port; int prometheus_username_labels; + int prometheus_compact; /////// Users DB /////////// diff --git a/src/apps/relay/prom_server.c b/src/apps/relay/prom_server.c index ea6104ed4..293ea5887 100644 --- a/src/apps/relay/prom_server.c +++ b/src/apps/relay/prom_server.c @@ -1,3 +1,5 @@ +#include + #include "prom_server.h" #include "mainrelay.h" #include "ns_turn_utils.h" @@ -18,24 +20,21 @@ prom_counter_t *turn_traffic_peer_rcvb; prom_counter_t *turn_traffic_peer_sentp; prom_counter_t *turn_traffic_peer_sentb; -prom_counter_t *turn_total_traffic_rcvp; -prom_counter_t *turn_total_traffic_rcvb; -prom_counter_t *turn_total_traffic_sentp; -prom_counter_t *turn_total_traffic_sentb; - -prom_counter_t *turn_total_traffic_peer_rcvp; -prom_counter_t *turn_total_traffic_peer_rcvb; -prom_counter_t *turn_total_traffic_peer_sentp; -prom_counter_t *turn_total_traffic_peer_sentb; - prom_gauge_t *turn_total_allocations; void start_prometheus_server(void) { + PROM_INIT_FLAGS features = PROM_PROCESS|PROM_SCRAPETIME_ALL; if (turn_params.prometheus == 0) { TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "prometheus collector disabled, not started\n"); return; } - prom_collector_registry_default_init(); + if (turn_params.prometheus_compact) + features |= PROM_COMPACT; + if (pcr_init(features, "coturn_")) { + TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, "prometheus collector disabled - init failed.\n"); + turn_params.prometheus = 0; + return; + } const char *label[] = {"realm", NULL}; size_t nlabels = 1; @@ -46,80 +45,65 @@ void start_prometheus_server(void) { } // Create STUN counters - stun_binding_request = prom_collector_registry_must_register_metric( - prom_counter_new("stun_binding_request", "Incoming STUN Binding requests", 0, NULL)); - stun_binding_response = prom_collector_registry_must_register_metric( - prom_counter_new("stun_binding_response", "Outgoing STUN Binding responses", 0, NULL)); - stun_binding_error = prom_collector_registry_must_register_metric( - prom_counter_new("stun_binding_error", "STUN Binding errors", 0, NULL)); + stun_binding_request = pcr_must_register_metric(prom_counter_new( + "bind_requests","Incoming STUN Binding requests", 0, NULL)); + stun_binding_response = pcr_must_register_metric(prom_counter_new( + "bind_responses","Outgoing STUN Binding responses", 0, NULL)); + stun_binding_error = pcr_must_register_metric(prom_counter_new( + "bind_errors","STUN Binding errors", 0, NULL)); // Create TURN traffic counter metrics - turn_traffic_rcvp = prom_collector_registry_must_register_metric( - prom_counter_new("turn_traffic_rcvp", "Represents finished sessions received packets", nlabels, label)); - turn_traffic_rcvb = prom_collector_registry_must_register_metric( - prom_counter_new("turn_traffic_rcvb", "Represents finished sessions received bytes", nlabels, label)); - turn_traffic_sentp = prom_collector_registry_must_register_metric( - prom_counter_new("turn_traffic_sentp", "Represents finished sessions sent packets", nlabels, label)); - turn_traffic_sentb = prom_collector_registry_must_register_metric( - prom_counter_new("turn_traffic_sentb", "Represents finished sessions sent bytes", nlabels, label)); + // see tcp_client_input_handler_rfc6062data() + turn_traffic_rcvp = pcr_must_register_metric(prom_counter_new( + "rx_msgs","Messages received in a session from the turn client.", nlabels, label)); + turn_traffic_peer_sentp = pcr_must_register_metric(prom_counter_new( + "peer_tx_msgs","Messages sent in a session to the turn client.", nlabels, label)); + turn_traffic_rcvb = pcr_must_register_metric(prom_counter_new( + "rx_bytes","Bytes received in a session from the turn client.", nlabels, label)); + turn_traffic_peer_sentb = pcr_must_register_metric(prom_counter_new( + "peer_tx_bytes","Bytes sent in a session to the turn client.", nlabels, label)); // Create finished sessions traffic for peers counter metrics - turn_traffic_peer_rcvp = prom_collector_registry_must_register_metric( - prom_counter_new("turn_traffic_peer_rcvp", "Represents finished sessions peer received packets", nlabels, label)); - turn_traffic_peer_rcvb = prom_collector_registry_must_register_metric( - prom_counter_new("turn_traffic_peer_rcvb", "Represents finished sessions peer received bytes", nlabels, label)); - turn_traffic_peer_sentp = prom_collector_registry_must_register_metric( - prom_counter_new("turn_traffic_peer_sentp", "Represents finished sessions peer sent packets", nlabels, label)); - turn_traffic_peer_sentb = prom_collector_registry_must_register_metric( - prom_counter_new("turn_traffic_peer_sentb", "Represents finished sessions peer sent bytes", nlabels, label)); - - // Create total finished traffic counter metrics - turn_total_traffic_rcvp = prom_collector_registry_must_register_metric( - prom_counter_new("turn_total_traffic_rcvp", "Represents total finished sessions received packets", 0, NULL)); - turn_total_traffic_rcvb = prom_collector_registry_must_register_metric( - prom_counter_new("turn_total_traffic_rcvb", "Represents total finished sessions received bytes", 0, NULL)); - turn_total_traffic_sentp = prom_collector_registry_must_register_metric( - prom_counter_new("turn_total_traffic_sentp", "Represents total finished sessions sent packets", 0, NULL)); - turn_total_traffic_sentb = prom_collector_registry_must_register_metric( - prom_counter_new("turn_total_traffic_sentb", "Represents total finished sessions sent bytes", 0, NULL)); - - // Create total finished sessions traffic for peers counter metrics - turn_total_traffic_peer_rcvp = prom_collector_registry_must_register_metric(prom_counter_new( - "turn_total_traffic_peer_rcvp", "Represents total finished sessions peer received packets", 0, NULL)); - turn_total_traffic_peer_rcvb = prom_collector_registry_must_register_metric(prom_counter_new( - "turn_total_traffic_peer_rcvb", "Represents total finished sessions peer received bytes", 0, NULL)); - turn_total_traffic_peer_sentp = prom_collector_registry_must_register_metric(prom_counter_new( - "turn_total_traffic_peer_sentp", "Represents total finished sessions peer sent packets", 0, NULL)); - turn_total_traffic_peer_sentb = prom_collector_registry_must_register_metric( - prom_counter_new("turn_total_traffic_peer_sentb", "Represents total finished sessions peer sent bytes", 0, NULL)); + // see tcp_peer_input_handler() + turn_traffic_peer_rcvp = pcr_must_register_metric(prom_counter_new( + "peer_rx_pkts","Messages received in a session from the peer.", nlabels, label)); + turn_traffic_sentp = pcr_must_register_metric(prom_counter_new( + "tx_msgs","Messages sent in a session to the peer.", nlabels, label)); + turn_traffic_peer_rcvb = pcr_must_register_metric(prom_counter_new( + "peer_rx_bytes","Bytes received in a session from peer.", nlabels, label)); + turn_traffic_sentb = pcr_must_register_metric(prom_counter_new( + "tx_bytes","Bytes sent in a session to the peer.", nlabels, label)); // Create total allocations number gauge metric const char *typeLabel[] = {"type"}; - turn_total_allocations = prom_collector_registry_must_register_metric( - prom_gauge_new("turn_total_allocations", "Represents current allocations number", 1, typeLabel)); + turn_total_allocations = pcr_must_register_metric(prom_gauge_new( + "allocations", "Current allocations", 1, typeLabel)); promhttp_set_active_collector_registry(NULL); // some flags appeared first in microhttpd v0.9.53 - unsigned int flags = MHD_USE_DUAL_STACK -#if MHD_VERSION >= 0x00095300 - | MHD_USE_ERROR_LOG -#endif - ; - if (MHD_is_feature_supported(MHD_FEATURE_EPOLL)) { -#if MHD_VERSION >= 0x00095300 - flags |= MHD_USE_EPOLL_INTERNAL_THREAD; + unsigned int flags; +#ifdef MHD_USE_AUTO + flags = MHD_USE_AUTO; + // EITHER +# ifdef MHD_USE_INTERNAL_POLLING_THREAD + flags |= MHD_USE_INTERNAL_POLLING_THREAD; // EPOLL if avail or POLL +# endif + /* OR +# ifdef MHD_USE_THREAD_PER_CONNECTION + flags |= MHD_USE_THREAD_PER_CONNECTION; // implies POLL +# endif + */ #else - flags |= MHD_USE_EPOLL_INTERNALLY_LINUX_ONLY; // old versions of microhttpd + flags = MHD_USE_POLL_INTERNALLY; // internal polling thread + /* OR + flags = MHD_USE_THREAD_PER_CONNECTION; // implies POLL + */ #endif - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "prometheus exporter server will start using EPOLL\n"); - } else { - flags |= MHD_USE_SELECT_INTERNALLY; - // Select() will not work if all 1024 first file-descriptors are used. - // In this case the prometheus server will be unreachable - TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, "prometheus exporter server will start using SELECT. " - "The exporter might be unreachable on highly used servers\n"); - } +#ifdef MHD_USE_DEBUG + flags |= MHD_USE_DEBUG; // same as MHD_USE_ERROR_LOG +#endif + struct MHD_Daemon *daemon = promhttp_start_daemon(flags, turn_params.prometheus_port, NULL, NULL); if (daemon == NULL) { TURN_LOG_FUNC(TURN_LOG_LEVEL_ERROR, "could not start prometheus collector\n"); @@ -131,6 +115,8 @@ void start_prometheus_server(void) { return; } +// This is total non-sense right now, because called at the end of a session, +// only. So would only appear as a tiny spike in a timeseries visualization. void prom_set_finished_traffic(const char *realm, const char *user, unsigned long rsvp, unsigned long rsvb, unsigned long sentp, unsigned long sentb, bool peer) { if (turn_params.prometheus == 1) { @@ -145,21 +131,11 @@ void prom_set_finished_traffic(const char *realm, const char *user, unsigned lon prom_counter_add(turn_traffic_peer_rcvb, rsvb, label); prom_counter_add(turn_traffic_peer_sentp, sentp, label); prom_counter_add(turn_traffic_peer_sentb, sentb, label); - - prom_counter_add(turn_total_traffic_peer_rcvp, rsvp, NULL); - prom_counter_add(turn_total_traffic_peer_rcvb, rsvb, NULL); - prom_counter_add(turn_total_traffic_peer_sentp, sentp, NULL); - prom_counter_add(turn_total_traffic_peer_sentb, sentb, NULL); } else { prom_counter_add(turn_traffic_rcvp, rsvp, label); prom_counter_add(turn_traffic_rcvb, rsvb, label); prom_counter_add(turn_traffic_sentp, sentp, label); prom_counter_add(turn_traffic_sentb, sentb, label); - - prom_counter_add(turn_total_traffic_rcvp, rsvp, NULL); - prom_counter_add(turn_total_traffic_rcvb, rsvb, NULL); - prom_counter_add(turn_total_traffic_sentp, sentp, NULL); - prom_counter_add(turn_total_traffic_sentb, sentb, NULL); } } } diff --git a/src/apps/relay/prom_server.h b/src/apps/relay/prom_server.h index 4c05e5c2e..c766ff8b8 100644 --- a/src/apps/relay/prom_server.h +++ b/src/apps/relay/prom_server.h @@ -16,9 +16,8 @@ #ifdef __cplusplus extern "C" { #endif -#include -#include -#include +#include +#include #ifdef __cplusplus } #endif /* __clplusplus */ From 4cd10e18c3426781872923ff9dbd355761045993 Mon Sep 17 00:00:00 2001 From: Jens Elkner Date: Thu, 30 Nov 2023 21:10:24 +0100 Subject: [PATCH 03/12] life metrics with libprom --- README.turnserver | 56 ++-- docs/Metrics.md | 276 +++++++++++++++++ src/apps/relay/http_server.c | 4 +- src/apps/relay/http_server.h | 2 +- src/apps/relay/mainrelay.c | 167 +++++++---- src/apps/relay/mainrelay.h | 15 +- src/apps/relay/netengine.c | 12 +- src/apps/relay/ns_ioalib_engine_impl.c | 386 +++++++++++++++++++----- src/apps/relay/ns_ioalib_impl.h | 4 + src/apps/relay/prom_server.c | 394 +++++++++++++++++-------- src/apps/relay/prom_server.h | 66 ++--- src/apps/relay/turn_admin_server.c | 45 ++- src/apps/relay/turn_admin_server.h | 8 +- src/apps/relay/turn_ports.h | 4 +- src/client/ns_turn_msg_defs.h | 10 +- src/ns_turn_defs.h | 4 +- src/server/ns_turn_allocation.c | 2 +- src/server/ns_turn_allocation.h | 1 + src/server/ns_turn_ioalib.h | 16 +- src/server/ns_turn_server.c | 149 ++++++++-- src/server/ns_turn_server.h | 27 +- src/server/ns_turn_session.h | 30 ++ 22 files changed, 1302 insertions(+), 376 deletions(-) create mode 100644 docs/Metrics.md diff --git a/README.turnserver b/README.turnserver index d10dc65db..5a80f06ef 100644 --- a/README.turnserver +++ b/README.turnserver @@ -162,6 +162,8 @@ User database settings: Flags: +-h Help. + -v, --verbose Moderate verbose mode. -V, --Verbose Extra verbose mode, very annoying and not recommended. @@ -285,28 +287,46 @@ Flags: main ORIGIN attribute value (if the ORIGIN was initially used by the session). ---prometheus Enable prometheus metrics. By default it is - disabled. Would listen on port 9641 under the path /metrics - also the path / on this port can be used as a health check. - Unavailable on apt installations. - ---prometheus-username-labels Enable labeling prometheus traffic - metrics with client usernames. Labeling with client usernames is - disabled by default, because this may cause memory leaks when using - authentication with ephemeral usernames (e.g. TURN REST API). ---prometheus-port Prometheus listener port (Default: 9641). +Prometheus/OpenMetrics options: + +--prom Enable metric collections to monitor this application and make them + available via HTTP in reports using the Prometheus/OpenMetrics + exposition format via http://host:port/metrics . All options + starting with "--prom-" are only effective if this one is enabled. + For more information and details you should read `docs/Metrics.md`. + +--prom-port= Listen on the given port for /metric requests. Default: 9641 + +--prom-compact Omit the HELP and TYPE comments when generating the metric + reports. Recommended for all production systems. + +--prom-realm Label the session state metric with the session realm. Usually just overhead. + +--prom-usernames Label the session state metric with client usernames. One + should not use, unless really needed! + +--prom-uniq-sid Label session related metrics with the unique ID of the session. + One should not use, unless really needed! + +--prom-sid Reuse session IDs of already closed sessions to reduce the number + of generate metrics. However, still a vulnerable point for DoS attacks + so should not be used or in an environment, where one has pretty good + control over the clients accessing this service. + NOTE: --prom-sid supersedes the --prom-sid option. + +--prom-sid-retain= Set the time in seconds a session must have been closed + before its ID can be re-used by a new session. + +--log-ip Log the IP address of each session endpoint, when a session gets created. + Right now IP addresses are logged in verbose mode at the session end, only. + This is not sufficient or too late if one needs to correlate metrics. So + this option allows one to turn it on on demand. Default: 0 ---prometheus-compact Omit the HELP and TYPE comments when generating - the prometheus metrics report. Usually it is sufficient to get and - record them once. There is no need to send them again and again on - each report - would just waste bandwith and demands more resources - to parse and process the report. On production systems it is - recommended to use this option. +For production the recommended options are --prom and --prom-compact. --h Help. -Options with values: +Other Options with values: --stale-nonce[=] Use extra security with nonce value having limited lifetime, in seconds (default 600 secs). diff --git a/docs/Metrics.md b/docs/Metrics.md new file mode 100644 index 000000000..5760e5283 --- /dev/null +++ b/docs/Metrics.md @@ -0,0 +1,276 @@ +# Introduction + +The option **--prom** enables more or less coarse grained data collection for +life monitoring the the running turnserver instances - per default one per +CPU thread or strand or whatever is configured via the option **relay-threads**. +The label pair **tid="inctanceNumber"** allows one to determine, to which +turnserver instance the related metric is associated. + +Just monitoring at turnserver instance level is pretty save wrt. to the number +of generated metrics and the thread, which generates and delivers the related +report formatted as plain text in the +[Prometheus/OpenMetrics exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/) +via **http://your-turnserver.do.main/metrics** . Collecting the data itself +is pretty light weight (just atomic add, sub, or set). The related metrics are described in the next section. + +# Metrics + +## version +The TURN server software name and version. + +## bind\_requests +The number of valid STUN binding requests (RFC 5780) a turnserver instance +received so far. If a binding request contains one or more unrecognized +attributes, or other errors (see below), it gets counted via bind\_errors, +not here. If `no-stun-backward-compatibility` is set, old STUN binding requests +are ignored and thus not counted. If `no-stun` is set, all STUN binding +requests are ignored and thus not counted, excluded from reports. + +## bind\_responses +The number of STUN binding responses to valid STUN binding requests (RFC 5780) +the turnserver instance sent so far. It obviously gets sent only if the related +request had no errors and a proper answer could be constructed. The response +gets counted as soon as it has been put into the server's "sent-queue", i.e. +takes not into account, whether the delivery was successful. Excluded from +reports if `no-stun` is set. + + +## bind\_errors +The number of errors related to STUN binding occurred so far. The label pair +**code="number"** can be used to distinguish the errors. The error codes used +so far are: + - 420 + - unknown attribute + - Unknown attribute: TURN server was configured without RFC 5780 support + - 400 + - wrong response format + - Wrong request: applicable only to UDP protocol + - Wrong request format: you cannot use PADDING and RESPONSE\_PORT together + - Wrong request: padding applicable only to UDP and DTLS protocols + +NOTE: If no binding errors occured so far, excluded from reports. + +## rx\_msgs, rx\_bytes +Messages and bytes received from turn clients or peers. The label pair +**peer="value"** is used to specify, whether the metric refers to +the turn client (`0`) or the the peer (`1`). + +## tx\_msgs, tx\_bytes +Messages and bytes sent to turn clients or peers. The label pair +**peer="value"** is used to specify, whether the metric refers to +to the turn client (`0`) or the the peer (`1`). + +## allocations +The number of created allocations (label `created="1"`) and the number of +active, i.e. not yet deleted allocations (label `created="0"`). Wrt. to the +turnserver this is usually what matters and since there is a 1:1 relation to +the owning session, sessions are not yet tracked in a separate metric. But +note, that a session may exists without a [not yet created] allocation. + +## lifetime +The lifetime of current allocations in seconds. This metric gets automatically +enabled, if monitoring at the session level is enabled (at turnserver level it +does not make any sense). Because per contract there is max. one allocation +between a client and the turnserver, and the allocation is always related to +the session between both, the related allocation can be identified by the +**tid="inctanceNumber"** and **"sid=sessionID"**. When an allocation gets +deleted (usually when the session gets closed), the lifetime is set to `-1` +to get a marker for the allocation/session end. + +## session\_state +The current state of a session. This metric gets automatically enabled, if +monitoring at the session level is enabled (at turnserver level it does not +make any sense). Sessions can be +distinguished by the labels **tid="inctanceNumber"** and **"sid=sessionID"**. +The metric value denotes the following states: + - 0 .. session is closed. + - 1 .. allocation got deleted. Because this usually happen on a session + shutdown, and the time between an allocation got deleted and the session + closed is very short, you probably will not see it very often. + - 2 .. session shutdown initiated. Because the time between the shutdown init + and the session end is very short, you probably will not see it very often. + - 3 .. session is open, however no allocation request has been seen so far. + - 4 . an allocation got created, however, no refresh for it has been seen + so far. The **lifetime** metric can be used to check the allocation's + lifetime in seconds. + - 5 .. an allocation exist and has been refreshed by the client at least once. + On each refresh the related **lifetime** metric gets an update. + +## process\_\* +For now 17 metrics about the coturn process itself. Most relevant are probably +the total number of threads (threads\_total), the physical RAM in use +(resident\_memory\_bytes), and the number of open file descriptors (open\_fds) +vs. the maximum number of open file descriptors (max\_fds). If they are too +close, you may adjust this limit by adding e.g. `LimitNOFILE=64535` to the +`[Service]` section of the systemd service or using `ulimit -S -n 64535` in +your coturn service startup script. Last but not least: if there are too many +major page faults (major\_pagefaults) there might be a memory related problem +on the box running the app. + +## scrape\_duration\_seconds +The metric labeled with `collector="default"` reports how long it took to collect +and format the turnserver related metrics (i.e. all except `process` and +`scrape_duration_seconds`). If this takes longer than 1 second, you probably +have either a "too many metrics" or "not enough CPU resources" problem. + +The metric labeled `collector="process"` reports how long it took to collect +and format the coturn process related data exposed via the `process_*` metrics. + +Last but not least the `collector="libprom"` tells you, how long it took to +collect the reports from all other collectors (default and process) and to +create the final report exposed via HTTP. + +## Summary +So monitoring at turnserver level (the default) produces max. +`1 + (1 + 1 + 2 + 2*2 + 2*2 + 2) * relay-threads + 17 + 3 = 14 * relay-threads + 21` +metrics. +This means for a box with 2x Xeon(R) CPU E5-2690 v4 per default max. +`56 * 14 + 21 = 805` metrics. +This is not a big deal and easy to handle by the coturn application as +well as metrics agents like VictoriaMetric's vmagent or Prometheus, +time series databases like VictoriaMetric's vmdb or Prometheus Mimir, and +visualizers like Grafana or Netdata. Should be also sufficient to monitor +the health of your coturn applications. + + +# Metrics at session level +Logging at session level may help: + - in development to debug traffic issues + - to understand your traffic patterns + - to discover bottlenecks + - support better troubleshooting + +Using the option **--prom-uniq-sid** or **--prom-sid** enables monitoring +at the session label. The difference between both options is, that `--prom-sid` +recycles used session IDs by putting the session IDs of closed session back +into a pool and tries to reuse them for new sessions after a certain time has +passed (see option **--prom-sid-retain**). + +If any of the sid options is enabled, the **session\_state** and **lifetime** +metrics get enabled automatically and all metrics get a **"sid=sessionID"** +label pair. This means the max. number of metrics is now for `--prom-sid`: +`21 + (14 + 2) * relay-threads * sum(countPerPool(unique(sessionIDs)))/relay-threads` +or assuming that coturn distributes requests/sessions evenly over the running +relay-threads as rule of thumb: +`21 + 16 * relay-threads * averageSessionIdPoolSize` or +`21 + 16 * relay-threads * max(concurrentSessionsAtATime)` +and for `--prom-uniq-sid`: +`21 + 16 * relay-threads * all_sessions_handled_so_far_by_the_app`. + +So it is pretty obvious, that these options possibly open the door for +**DoS attacks**! Therefore one should think at least twice and understand +the impact on produced metrics as described below, before using it. Having a +pretty good control over the clients (authentication, session frequency, +kindness of your clients) may help as well ;-). + +If enabled, it is certainly a good idea to setup an alert manager, which +monitors the scrapetime or size of the metrics reports and alerts you, if it +takes too long or gets too big, which indicates that there are probably too +many metrics and may get you into trouble. In case you monitor the coturn log, +watch out for entries containing `grew rsid pool`: `maxId` says, how many +unique session IDs the relay-thread with id `tid` has seen so far. So the +current number of metrics for this thread is about `16 * maxSid`. + +BTW: Usually a separate single thread per request generates and delivers the +metric report (using libmicrohttp), not the turnserver threads themselves. +However, it needs some CPU cycles to get the work done and thus may have an +impact on other tasks running on the box, if CPU resources are not sufficient. + +Anyway, as you deduce from the calculations above, one should definitely avoid +using the `--prom-uniq-sid` option unless you really know, what you are doing. +If you really need it, enable it only for the time needed and make sure, that +scraping agents use a separate time series DB or storage place, where the +metrics data can be easily discarded, when the work is done and have +not really an impact on your other services in production. + +## Tuning +As already said, to mitigate a possible "too many metrics" problem, the option +**--prom-sid** got introduced, which basically uses a dynamic pool of session +IDs per turnserver instance, where the IDs of closed sessions get put back and +reused by new sessions. So if you have per average only 100 sessions or +allocations concurrently running, it would result at least into +`56 * 16 * ceil(100/40)) + 21 = 2709` metrics. +Still easy to handle by coturn and not really a problem for report collectors, +time series DBs and visualizers. +**BUT** if one decides to challenge your server, +the numbers may change dramatically. So there might be a desire to adjust the +time a session ID has to stay in the ID pool before it can be reused. The +option **--prom-sid-retain=num** allows you to adjust this parameter measured +in seconds. For now it defaults to 60 - a more or less conservative value, +depending on the environment. To get a feeling, how it works out for you, +the following test may reveal data for more or less proper settings: + +## Testing +- setup a coturn test instance +- ensure that `lt-cred-mech` is enabled and no other mechanism +- ensure that a realm is configured, e.g. `realm = test.do.main` +- set `relay-threads = 5` +- add a user to the coturn db you wanna use for your tests, e.g. if you have + `userdb=/data/coturn/db` set, use: + `turnadmin -a -b /data/coturn/db -u foo -p bar -r test.do.main` +- make sure, your firewall allows the related traffic on the server as well + as the client zone/container. +- run the test on the client machine, e.g.: + `time turnutils_uclient -p 3478 -DgXv -y -c -u foo -w bar -m 40 server_hostname`. + If you have changed the `listening-port`, adjust the `-p` option accordingly. + Only simulate as many clients as CPU threads/strands are available on the + client machine (option `-m`). +- when finished: + - have a look at the log file (defaults to /var/log/coturn.log) + - get the metrics report, e.g. `curl -o /tmp/report.om http://localhost:9641/metrics` + - to find out the number of metrics and size of the report, one may run: + `grep ^coturn_ /tmp/report.om |wc -lc`. The first number in the output + is the number of lines and thus number of metrics in the report, the + 2nd number is the size of the report without any `# TYPE` or `# HELP` + comments. + - to find out, how many bytes you can drop from transmission of each report + using the option **--prom-compact**, one may run: + `grep -v ^coturn_ /tmp/report.om |wc -lc`. + +- you may also try the same test using an invalid password or running a test + peer e.g. on machineC (`turnutils_peer -v -p 16384`) and add the options + `-e machineC -r 16384`to the `turnutils_uclient` command. + + +# Misc options + +## **--prom-compact** +Should be used on all production systems to avoid the inclusion of the `HELP` +and `TYPE` comments for each metric in each report. +Usually it is sufficient to get and record them once, when a demo is required +or Grafana charts get setup. There is no need to send them again and again in +each report - would just waste bandwidth and demands more resources to parse +and process the report. At least on production systems it is recommended to +enable this option. + +## **--prom-realm** +Label the `session_state` metric with the session realm. If one uses different +realms for user authentication it could make sense to add a realm label, so one +may select the interested time series by the realm. If you only use one and the +same realm it is usually just overhead and should be skipped. Because rarely +used, it is not taken into account in the calculations of number of metrics +above. + +## **--prom-usernames** +Label the `session_state` metric with client usernames. One should not do this, +unless really needed: Modern apps often generate usernames on the fly (e.g. +hash(currentTimeAsString + `:` + username)) and thus all the names are +different or even unique for a long time. This in turn means, that a new state +metric gets created for each new session. This stresses the server resources +the same way as `--prom-uniq-sid` does, because a sample for each metric gets +kept in memory and needs to be formatted and transferred on each /metric +request. When the report gets collected and stored in a time series DB, this +would in turn create a new time series for each new session +(metricName + label:value pairs form the key for a time series) and would let +your DB "explode", get it performance and resource wise probably into trouble, +because it has to dig into such a huge set of time series (high cardinality) +and replace "active/cached" ones (high churn rate). So do NOT use it, unless +you known what you are doing! + + +# More Information +More information related to labels and cardinality can be found e.g. via: +- https://docs.victoriametrics.com/keyConcepts.html +- https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality +- https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate + diff --git a/src/apps/relay/http_server.c b/src/apps/relay/http_server.c index 2ba9f214d..ec6a8d739 100644 --- a/src/apps/relay/http_server.c +++ b/src/apps/relay/http_server.c @@ -350,9 +350,9 @@ void str_buffer_append_sz(struct str_buffer *sb, size_t sz) { str_buffer_append(sb, ssz); } -void str_buffer_append_sid(struct str_buffer *sb, turnsession_id sid) { +void str_buffer_append_sid(struct str_buffer *sb, turnsession_id sid, uint32_t rsid) { char ssz[129]; - snprintf(ssz, sizeof(ssz) - 1, "%018llu", (unsigned long long)sid); + snprintf(ssz, sizeof(ssz) - 1, "%012llu.%d", (unsigned long long)sid, rsid); str_buffer_append(sb, ssz); } diff --git a/src/apps/relay/http_server.h b/src/apps/relay/http_server.h index a4a11c1cf..dc84c34a0 100644 --- a/src/apps/relay/http_server.h +++ b/src/apps/relay/http_server.h @@ -69,7 +69,7 @@ struct str_buffer; struct str_buffer *str_buffer_new(void); void str_buffer_append(struct str_buffer *sb, const char *str); void str_buffer_append_sz(struct str_buffer *sb, size_t sz); -void str_buffer_append_sid(struct str_buffer *sb, turnsession_id sid); +void str_buffer_append_sid(struct str_buffer *sb, turnsession_id sid, uint32_t rsid); const char *str_buffer_get_str(const struct str_buffer *sb); size_t str_buffer_get_str_len(const struct str_buffer *sb); void str_buffer_free(struct str_buffer *sb); diff --git a/src/apps/relay/mainrelay.c b/src/apps/relay/mainrelay.c index a1b972274..6cf8ba3e1 100644 --- a/src/apps/relay/mainrelay.c +++ b/src/apps/relay/mainrelay.c @@ -211,11 +211,17 @@ turn_params_t turn_params = { 0, /* bps_capacity_allocated */ 0, /* total_quota */ 0, /* user_quota */ - 0, /* prometheus disabled by default */ - DEFAULT_PROM_SERVER_PORT, /* prometheus port */ - 0, /* prometheus username labelling disabled by default when prometheus is enabled */ - 0, /* prometheus compact output */ - +#if !defined(TURN_NO_PROMETHEUS) + 0, /* prom disabled by default */ + DEFAULT_PROM_SERVER_PORT, /* prom port */ + 0, /* prom compact output */ + 0, /* prom realm labels */ + 0, /* prom username labels */ + 0, /* prom unique sessionID labels */ + 0, /* prom sessionID labels, reusable */ + DEFAULT_PROM_SID_RETAIN, /* prom retain sessionID secs before reuse */ + 0, /* log ip on session create */ +#endif ///////////// Users DB ////////////// {(TURN_USERDB_TYPE)0, {"\0", "\0"}, {0, NULL, {NULL, 0}}}, @@ -861,18 +867,19 @@ int get_a_local_relay(int family, ioa_addr *relay_addr) { } ////////////////////////////////////////////////// - +#define _STRVAL(s) _STR(s) +#define _STR(s) #s static char Usage[] = "Usage: turnserver [options]\n" "Options:\n" " -d, --listening-device Listener interface device (NOT RECOMMENDED. Optional, Linux " "only).\n" - " -p, --listening-port TURN listener port (Default: 3478).\n" + " -p, --listening-port TURN listener port. Default: " _STRVAL(DEFAULT_STUN_PORT) ".\n" " Note: actually, TLS & DTLS sessions can connect to the \"plain\" TCP & " "UDP port(s), too,\n" " if allowed by configuration.\n" - " --tls-listening-port TURN listener port for TLS & DTLS listeners\n" - " (Default: 5349).\n" + " --tls-listening-port TURN listener port for TLS & DTLS listeners.\n" + " Default: " _STRVAL(DEFAULT_STUN_TLS_PORT) ".\n" " Note: actually, \"plain\" TCP & UDP sessions can connect to the TLS & " "DTLS port(s), too,\n" " if allowed by configuration. The TURN server\n" @@ -953,10 +960,10 @@ static char Usage[] = " the number of listening endpoints (unless -m 0 is set).\n" " --min-port Lower bound of the UDP port range for relay endpoints " "allocation.\n" - " Default value is 49152, according to RFC 5766.\n" + " Default according to RFC 5766: " _STRVAL(LOW_DEFAULT_PORTS_BOUNDARY) ".\n" " --max-port Upper bound of the UDP port range for relay endpoints " "allocation.\n" - " Default value is 65535, according to RFC 5766.\n" + " Default according to RFC 5766: " _STRVAL(HIGH_DEFAULT_PORTS_BOUNDARY) ".\n" " -v, --verbose 'Moderate' verbose mode.\n" " -V, --Verbose Extra verbose mode, very annoying (for debug purposes only).\n" " -o, --daemon Start process as daemon (detach from current shell).\n" @@ -995,14 +1002,14 @@ static char Usage[] = "allocate\n" " for the sessions, combined (input and output network streams are " "treated separately).\n" - " -c Configuration file name (default - turnserver.conf).\n" + " -c Configuration file name. Default: " DEFAULT_CONFIG_FILE ".\n" #if !defined(TURN_NO_SQLITE) " -b, , --db, --userdb SQLite database file name; default - /var/db/turndb or\n" " /usr/local/var/db/turndb or /var/lib/turn/turndb.\n" #endif #if !defined(TURN_NO_PQ) - " -e, --psql-userdb, --sql-userdb PostgreSQL database connection string, if used (default - " - "empty, no PostgreSQL DB used).\n" + " -e, --psql-userdb, --sql-userdb PostgreSQL database connection string, if used. Default: " + "empty, no PostgreSQL DB used.\n" " This database can be used for long-term credentials mechanism users,\n" " and it can store the secret value(s) for secret-based timed " "authentication in TURN REST API.\n" @@ -1014,7 +1021,7 @@ static char Usage[] = " for 9.x and newer connection string formats.\n" #endif #if !defined(TURN_NO_MYSQL) - " -M, --mysql-userdb MySQL database connection string, if used (default - empty, no " + " -M, --mysql-userdb MySQL database connection string, if used. Default: empty, no " "MySQL DB used).\n" " This database can be used for long-term credentials mechanism users,\n" " and it can store the secret value(s) for secret-based timed " @@ -1039,15 +1046,15 @@ static char Usage[] = " If you want to use cleartext password then do not set this option!\n" #endif #if !defined(TURN_NO_MONGO) - " -J, --mongo-userdb MongoDB connection string, if used (default - empty, no " - "MongoDB used).\n" + " -J, --mongo-userdb MongoDB connection string, if used. Default: empty, no " + "MongoDB used.\n" " This database can be used for long-term credentials mechanism users,\n" " and it can store the secret value(s) for secret-based timed " "authentication in TURN REST API.\n" #endif #if !defined(TURN_NO_HIREDIS) - " -N, --redis-userdb Redis user database connection string, if used (default - " - "empty, no Redis DB used).\n" + " -N, --redis-userdb Redis user database connection string, if used. Default: " + "empty, no Redis DB used.\n" " This database can be used for long-term credentials mechanism users,\n" " and it can store the secret value(s) for secret-based timed " "authentication in TURN REST API.\n" @@ -1057,8 +1064,7 @@ static char Usage[] = "connect_timeout=\".\n\n" " All connection-string parameters are optional.\n\n" " -O, --redis-statsdb Redis status and statistics database connection string, if " - "used \n" - " (default - empty, no Redis stats DB used).\n" + "used. Default: empty, no Redis stats DB used.\n" " This database keeps allocations status information, and it can be also " "used for publishing\n" " and delivering traffic and allocation event notifications.\n" @@ -1066,12 +1072,20 @@ static char Usage[] = "connection string.\n" #endif #if !defined(TURN_NO_PROMETHEUS) - " --prometheus Enable prometheus metrics. It is disabled by default. If it is " - "enabled it will listen on port 9641 under the path /metrics\n" - " also the path / on this port can be used as a health check\n" - " --prometheus-port Prometheus metrics port (Default: 9641).\n" - " --prometheus-username-labels When metrics are enabled, add labels with client usernames.\n" - " --prometheus-compact Omit the optional # HELP and # TYPE comments for each metric output.\n" + " --prom Enable metric collections and provide reports in\n" + " Prometheus/OpenMetrics format via http://localhost:" _STRVAL(DEFAULT_PROM_SERVER_PORT) "/metrics .\n" + " All --prom- options are only effective, if this option is enabled.\n" + " --prom-port= Provide /metric reports on the given port. Default: " _STRVAL(DEFAULT_PROM_SERVER_PORT) ".\n" + " --prom-compact Omit the optional # HELP and # TYPE comments for each metric output.\n" + " --prom-realm Add a realm label to the session state metric.\n" + " --prom-usernames Add a username label to the session state metric.\n" + " --prom-uniq-sid Add a unique session ID label to all session metrics.\n" + " --prom-sid Add a session ID label to all session metrics. IDs of closed session\n" + " will be reused. Also enables session_state metrics.\n" + " Supersedes --prom-uniq-sid.\n" + " --prom-sid-retain= Wait the given number of seconds, before an ID of a closed session gets\n" + " reused. Default: " _STRVAL(DEFAULT_PROM_SID_RETAIN) "\n" + " --log-ip Log the endpoint IP addresses for each new session: Default: 0.\n" #endif " --use-auth-secret TURN REST API flag.\n" " Flag that sets a special authorization option that is based upon " @@ -1114,10 +1128,7 @@ static char Usage[] = " By default, no CA is set and no client certificate check is " "performed.\n" " --ec-curve-name Curve name for EC ciphers, if supported by OpenSSL\n" - " library (TLS and DTLS). The default value is prime256v1,\n" - " if pre-OpenSSL 1.0.2 is used. With OpenSSL 1.0.2+,\n" - " an optimal curve will be automatically calculated, if not defined\n" - " by this option.\n" + " library (TLS and DTLS). Default: " DEFAULT_EC_CURVE_NAME ".\n" " --dh566 Use 566 bits predefined DH TLS key. Default size of the predefined key " "is 2066.\n" " --dh1066 Use 1066 bits predefined DH TLS key. Default size of the predefined " @@ -1157,7 +1168,7 @@ static char Usage[] = " (unless the log file itself is stdout).\n" " --syslog Output all log information into the system log (syslog), do not use " "the file output.\n" - " --syslog-facility Set syslog facility for syslog messages. Default is ''.\n" + " --syslog-facility Set syslog facility for syslog messages. Default: ''.\n" " --simple-log This flag means that no log file rollover will be used, and " "the log file\n" " name will be constructed as-is, without PID and date appendage.\n" @@ -1168,14 +1179,11 @@ static char Usage[] = "--new-log-timestamp to be enabled.\n" " --log-binding Log STUN binding request. It is now disabled by default to " "avoid DoS attacks.\n" - " --stale-nonce[=] Use extra security with nonce value having limited lifetime (default " - "600 secs).\n" - " --max-allocate-lifetime Set the maximum value for the allocation lifetime. Default to 3600 " - "secs.\n" - " --channel-lifetime Set the lifetime for channel binding, default to 600 secs.\n" + " --stale-nonce[=] Use extra security with nonce value having limited lifetime in seconds. Default: " _STRVAL(STUN_DEFAULT_ALLOCATE_LIFETIME) ".\n" + " --max-allocate-lifetime Set the maximum value for the allocation lifetime in seconds. Default: " _STRVAL(STUN_DEFAULT_MAX_ALLOCATE_LIFETIME) ".\n" + " --channel-lifetime Set the lifetime for channel binding in seconds. Default: " _STRVAL(STUN_DEFAULT_CHANNEL_LIFETIME) ".\n" " This value MUST not be changed for production purposes.\n" - " --permission-lifetime Set the value for the lifetime of the permission. Default to " - "300 secs.\n" + " --permission-lifetime Set the value for the lifetime of the permission in seconds. Default: " _STRVAL(STUN_DEFAULT_PERMISSION_LIFETIME) ".\n" " This MUST not be changed for production purposes.\n" " -S, --stun-only Option to set standalone STUN operation only, all TURN requests will " "be ignored.\n" @@ -1193,9 +1201,9 @@ static char Usage[] = " See the docs for more information.\n" " -C, --rest-api-separator This is the timestamp/username separator symbol (character) in TURN " "REST API.\n" - " The default value is ':'.\n" + " Default: ':'.\n" " --max-allocate-timeout= Max time, in seconds, allowed for full allocation establishment. " - "Default is 60.\n" + "Default: " _STRVAL(TURN_MAX_ALLOCATE_TIMEOUT) ".\n" " --allowed-peer-ip= Specifies an ip or range of ips that are explicitly allowed to " "connect to the \n" " turn server. Multiple allowed-peer-ip can be set.\n" @@ -1232,18 +1240,16 @@ static char Usage[] = " so use other option values with care!\n" " --no-cli Turn OFF the CLI support. By default it is always ON.\n" " --cli-ip= Local system IP address to be used for CLI server endpoint. " - "Default value\n" - " is 127.0.0.1.\n" - " --cli-port= CLI server port. Default is 5766.\n" + "Default: " CLI_DEFAULT_IP ".\n" + " --cli-port= CLI server port. Default: " _STRVAL(CLI_DEFAULT_PORT) ".\n" " --cli-password= CLI access password. Default is empty (no password).\n" " For the security reasons, it is recommended to use the encrypted\n" " for of the password (see the -P command in the turnadmin utility).\n" " The dollar signs in the encrypted form must be escaped.\n" " --web-admin Enable Turn Web-admin support. By default it is disabled.\n" " --web-admin-ip= Local system IP address to be used for Web-admin server " - "endpoint. Default value\n" - " is 127.0.0.1.\n" - " --web-admin-port= Web-admin server port. Default is 8080.\n" + "endpoint. Default: " WEB_ADMIN_DEFAULT_IP ".\n" + " --web-admin-port= Web-admin server port. Default: " _STRVAL(WEB_ADMIN_DEFAULT_PORT) ".\n" " --web-admin-listen-on-workers Enable for web-admin server to listens on STUN/TURN workers " "STUN/TURN ports.\n" " By default it is disabled for security reasons!\n" @@ -1366,10 +1372,17 @@ enum EXTRA_OPTS { MAX_ALLOCATE_LIFETIME_OPT, CHANNEL_LIFETIME_OPT, PERMISSION_LIFETIME_OPT, - PROMETHEUS_OPT, - PROMETHEUS_PORT_OPT, - PROMETHEUS_ENABLE_USERNAMES_OPT, - PROMETHEUS_COMPACT_OPT, +#if !defined(TURN_NO_PROMETHEUS) + PROM_OPT, + PROM_PORT_OPT, + PROM_COMPACT_OPT, + PROM_REALM_OPT, + PROM_USERNAMES_OPT, + PROM_USID_OPT, + PROM_RSID_OPT, + PROM_SID_RETAIN_OPT, + LOG_IP_OPT, +#endif AUTH_SECRET_OPT, NO_AUTH_PINGS_OPT, NO_DYNAMIC_IP_LIST_OPT, @@ -1487,10 +1500,15 @@ static const struct myoption long_options[] = { {"redis-statsdb", required_argument, NULL, 'O'}, #endif #if !defined(TURN_NO_PROMETHEUS) - {"prometheus", optional_argument, NULL, PROMETHEUS_OPT}, - {"prometheus-port", optional_argument, NULL, PROMETHEUS_PORT_OPT}, - {"prometheus-username-labels", optional_argument, NULL, PROMETHEUS_ENABLE_USERNAMES_OPT}, - {"prometheus-compact", optional_argument, NULL, PROMETHEUS_COMPACT_OPT}, + {"prom", optional_argument, NULL, PROM_OPT}, + {"prom-port", required_argument, NULL, PROM_PORT_OPT}, + {"prom-compact", optional_argument, NULL, PROM_COMPACT_OPT}, + {"prom-realm", optional_argument, NULL, PROM_REALM_OPT}, + {"prom-usernames", optional_argument, NULL, PROM_USERNAMES_OPT}, + {"prom-uniq-sid", optional_argument, NULL, PROM_USID_OPT}, + {"prom-sid", optional_argument, NULL, PROM_RSID_OPT}, + {"prom-sid-retain", required_argument, NULL, PROM_SID_RETAIN_OPT}, + {"log-ip", optional_argument, NULL, LOG_IP_OPT}, #endif {"use-auth-secret", optional_argument, NULL, AUTH_SECRET_OPT}, {"static-auth-secret", required_argument, NULL, STATIC_AUTH_SECRET_VAL_OPT}, @@ -1791,6 +1809,8 @@ static int get_bool_value(const char *s) { } static void set_option(int c, char *value) { + int n; + if (value && value[0] == '=') { TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, "WARNING: option -%c is possibly used incorrectly. The short form of the option must be used as " @@ -2125,18 +2145,39 @@ static void set_option(int c, char *value) { turn_params.use_redis_statsdb = 1; break; #endif - case PROMETHEUS_OPT: - turn_params.prometheus = 1; +#if !defined(TURN_NO_PROMETHEUS) + case PROM_OPT: + turn_params.prom = get_bool_value(value); + break; + case PROM_PORT_OPT: + n = atoi(value); + if (n > 0) + turn_params.prom_port = n; + break; + case PROM_COMPACT_OPT: + turn_params.prom_compact = get_bool_value(value); + break; + case PROM_REALM_OPT: + turn_params.prom_realm = get_bool_value(value); break; - case PROMETHEUS_PORT_OPT: - turn_params.prometheus_port = atoi(value); + case PROM_USERNAMES_OPT: + turn_params.prom_usernames = get_bool_value(value); break; - case PROMETHEUS_ENABLE_USERNAMES_OPT: - turn_params.prometheus_username_labels = 1; + case PROM_USID_OPT: + turn_params.prom_usid = get_bool_value(value); break; - case PROMETHEUS_COMPACT_OPT: - turn_params.prometheus_compact = 1; + case PROM_RSID_OPT: + turn_params.prom_rsid = get_bool_value(value); break; + case PROM_SID_RETAIN_OPT: + n = atoi(value); + if (n >= 0) + turn_params.prom_sid_retain = n; + break; + case LOG_IP_OPT: + turn_params.log_ip = get_bool_value(value); + break; +#endif case AUTH_SECRET_OPT: turn_params.use_auth_secret_with_timestamp = 1; use_tltc = 1; diff --git a/src/apps/relay/mainrelay.h b/src/apps/relay/mainrelay.h index c2aa1e058..5274761f3 100644 --- a/src/apps/relay/mainrelay.h +++ b/src/apps/relay/mainrelay.h @@ -307,10 +307,17 @@ typedef struct _turn_params_ { band_limit_t bps_capacity_allocated; vint total_quota; vint user_quota; - int prometheus; - int prometheus_port; - int prometheus_username_labels; - int prometheus_compact; +#if !defined(TURN_NO_PROMETHEUS) + int prom; + int prom_port; + int prom_compact; + int prom_realm; + int prom_usernames; + int prom_usid; + int prom_rsid; + vint prom_sid_retain; + vint log_ip; +#endif /////// Users DB /////////// diff --git a/src/apps/relay/netengine.c b/src/apps/relay/netengine.c index 5df4cfcd5..2f0ce2d54 100644 --- a/src/apps/relay/netengine.c +++ b/src/apps/relay/netengine.c @@ -45,6 +45,7 @@ static pthread_barrier_t barrier; ////////////// Auth Server //////////////// +uint32_t relay_servers_in_use = 0; typedef unsigned char authserver_id; struct auth_server { @@ -1626,8 +1627,12 @@ static void setup_relay_server(struct relay_server *rs, ioa_engine_handle e, int DONT_FRAGMENT_SUPPORTED, start_user_check, check_new_allocation_quota, release_allocation_quota, turn_params.external_ip, &turn_params.check_origin, &turn_params.no_tcp_relay, &turn_params.no_udp_relay, &turn_params.stale_nonce, &turn_params.max_allocate_lifetime, &turn_params.channel_lifetime, - &turn_params.permission_lifetime, &turn_params.stun_only, &turn_params.no_stun, - &turn_params.no_software_attribute, &turn_params.web_admin_listen_on_workers, &turn_params.alternate_servers_list, + &turn_params.permission_lifetime, &turn_params.stun_only, &turn_params.no_stun, &turn_params.no_software_attribute, +#if !defined(TURN_NO_PROMETHEUS) + &turn_params.prom_sid_retain, + &turn_params.log_ip, +#endif + &turn_params.web_admin_listen_on_workers, &turn_params.alternate_servers_list, &turn_params.tls_alternate_servers_list, &turn_params.aux_servers_list, turn_params.udp_self_balance, &turn_params.no_multicast_peers, &turn_params.allow_loopback_peers, &turn_params.ip_whitelist, &turn_params.ip_blacklist, send_socket_to_relay, &turn_params.secure_stun, &turn_params.mobility, @@ -1671,7 +1676,8 @@ static void *run_general_relay_thread(void *arg) { static void setup_general_relay_servers(void) { size_t i = 0; - for (i = 0; i < get_real_general_relay_servers_number(); i++) { + relay_servers_in_use = get_real_general_relay_servers_number(); + for (i = 0; i < relay_servers_in_use; i++) { if (turn_params.general_relay_servers_number == 0) { general_relay_servers[i] = (struct relay_server *)allocate_super_memory_engine(turn_params.listener.ioa_eng, diff --git a/src/apps/relay/ns_ioalib_engine_impl.c b/src/apps/relay/ns_ioalib_engine_impl.c index 5d3448b28..dd403950b 100644 --- a/src/apps/relay/ns_ioalib_engine_impl.c +++ b/src/apps/relay/ns_ioalib_engine_impl.c @@ -183,10 +183,12 @@ static void log_socket_event(ioa_socket_handle s, const char *msg, int error) { if (!msg) msg = "General socket event"; turnsession_id id = 0; + int32_t rsid = 0; { ts_ur_super_session *ss = s->session; if (ss) { id = ss->id; + rsid = ss->rsid; } else { return; } @@ -205,10 +207,10 @@ static void log_socket_event(ioa_socket_handle s, const char *msg, int error) { addr_to_string(&(s->local_addr), (uint8_t *)sladdr); if (EVUTIL_SOCKET_ERROR()) { - TURN_LOG_FUNC(ll, "session %018llu: %s: %s (local %s, remote %s)\n", (unsigned long long)id, msg, + TURN_LOG_FUNC(ll, "session %012llu.%d: %s: %s (local %s, remote %s)\n", (unsigned long long)id, rsid, msg, evutil_socket_error_to_string(EVUTIL_SOCKET_ERROR()), sladdr, sraddr); } else { - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %018llu: %s (local %s, remote %s)\n", (unsigned long long)id, msg, + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %012llu.%d: %s (local %s, remote %s)\n", (unsigned long long)id, rsid, msg, sladdr, sraddr); } } @@ -2842,8 +2844,8 @@ static void eventcb_bev(struct bufferevent *bev, short events, void *arg) { addr_to_string(&(s->remote_addr), (uint8_t *)sraddr); if (events & BEV_EVENT_EOF) { if (server->verbose) - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %018llu: %s socket closed remotely %s\n", - (unsigned long long)(ss->id), socket_type_name(s->st), sraddr); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %012llu.%d: %s socket closed remotely %s\n", + (unsigned long long)(ss->id), ss->rsid, socket_type_name(s->st), sraddr); if (s == ss->client_socket) { char msg[256]; snprintf(msg, sizeof(msg) - 1, "%s connection closed by client (callback)", socket_type_name(s->st)); @@ -2866,12 +2868,12 @@ static void eventcb_bev(struct bufferevent *bev, short events, void *arg) { } } else if (events & BEV_EVENT_ERROR) { if (EVUTIL_SOCKET_ERROR()) { - TURN_LOG_FUNC(TURN_LOG_LEVEL_ERROR, "session %018llu: %s socket error: %s %s\n", - (unsigned long long)(ss->id), socket_type_name(s->st), + TURN_LOG_FUNC(TURN_LOG_LEVEL_ERROR, "session %012llu.%d: %s socket error: %s %s\n", + (unsigned long long)(ss->id), ss->rsid, socket_type_name(s->st), evutil_socket_error_to_string(EVUTIL_SOCKET_ERROR()), sraddr); } else if (server->verbose) { - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %018llu: %s socket disconnected: %s\n", - (unsigned long long)(ss->id), socket_type_name(s->st), sraddr); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %012llu.%d: %s socket disconnected: %s\n", + (unsigned long long)(ss->id), ss->rsid, socket_type_name(s->st), sraddr); } char msg[256]; snprintf(msg, sizeof(msg) - 1, "%s socket buffer operation error (callback)", socket_type_name(s->st)); @@ -3503,25 +3505,270 @@ const char *get_ioa_socket_ssl_method(ioa_socket_handle s) { return "no SSL"; } -void stun_report_binding(void *a, STUN_PROMETHEUS_METRIC_TYPE type) { +/** + * @brief turnserver: Get a uniqe session ID wrt. all sessions currently + * managed by the associated turnserver. + * + * It is different wrt. the sessionID, because sessionID is a composit of the + * turnserverID and its session count and thus always in any scope (global or + * local) unique for the whole lifetime of the turnserver. + * + * However if sessionID would be used as parameter for timeseries metrics, + * this would let explode the number of timeseries (every sessionID creates a + * new one) and extremly stress the TDB. + * + * Using a recyclable ID instead reduces such timeseries to a more or less + * predictable level and improves reporting as well as data management time and + * server resources quiet a bit. + * + * The simple idea is: If a session gets closed, its ID gets returned into the + * related pool. Only if the pool has no ID, which is in there for the given + * duration, a new ID gets allocated/returned. + * + * @param ss The sessions, which needs to get a recyclable ID. + */ +#ifdef POOL_TRACE + #define _TRACE TURN_LOG_FUNC +#else + // dirty talk - is for developers, only + #define _TRACE(level, fmt, ...) +#endif +void +acquire_recyclable_session_id(ts_ur_super_session *ss) { +#if defined(TURN_NO_PROMETHEUS) + UNUSED_ARG(ss); +#else + if (!prom_rsids()) + return; + + // gets called by initialized servers, only. So no ts->var NULL checks. + turn_turnserver *server = (turn_turnserver *)(ss->server); + id_pool_t *pool = server->rsid_pool; + _TRACE(TURN_LOG_LEVEL_DEBUG, "tid: %d usid: %d wants a sid (has %d) start: %d\n", + server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, pool->start); + + if (pool->id == NULL || pool->len == 0) { + (pool->max_id)++; + _TRACE(TURN_LOG_LEVEL_DEBUG,"tid: %d usid: %d <= newId: %d " + "capacity: %d len: %d\n", + server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), pool->max_id, + pool->capacity, pool->len); + ss->rsid = pool->max_id; + return; + } + if (pool->release[pool->start] < server->ctime) { + ss->rsid = pool->id[pool->start]; + // not really needed but + pool->release[pool->start] = INT_MAX; + pool->id[pool->start] = -1; + + (pool->start)++; + if (pool->start == pool->capacity) + pool->start = 0; + (pool->len)--; + (pool->recycled)++; + _TRACE(TURN_LOG_LEVEL_DEBUG, "tid: %d usid: %d <= newId: %d " + "capacity: %d len: %d recycledSids: %d startNow: %d\n", + server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, + pool->capacity, pool->len, pool->recycled, pool->start); + return; + } + // all IDs in use + (pool->max_id)++; + _TRACE(TURN_LOG_LEVEL_DEBUG, "tid: %d usid: %d <= newId: %d " + "capacity: %d len: %d\n", + server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), pool->max_id, + pool->capacity,pool->len); + ss->rsid = pool->max_id; +#endif +} + +#define INITIAL_ID_POOL_SZ 4 +#define ID_POOL_NOMEM TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, \ + "Turnserver %d: Not enough memory to maintain the pool for " \ + "recyclable session IDs.\n", server->id); + +/** + * Delete the recyclable ID of the given session and and return it to the + * related server's rsid_pool. + * + * @param ss The sessions, which needs to give its recyclable ID back to the + * server's rsid_pool. + */ +void +release_recyclable_session_id(ts_ur_super_session *ss) { +#if defined(TURN_NO_PROMETHEUS) + UNUSED_ARG(ss); +#else + int n, m; + + if (!prom_rsids()) + return; + + turn_turnserver *server = (turn_turnserver *)(ss->server); + id_pool_t *pool = server->rsid_pool; + _TRACE(TURN_LOG_LEVEL_DEBUG, "tid: %d usid: %d returns sid %d%s\n", + server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, + ss->rsid < 1 ? " INVALID" : ""); + + if (ss->rsid < 1) + return; + + _TRACE(TURN_LOG_LEVEL_DEBUG,"tid: %d usid: %d rsid: %d recycle before: " + "capacity: %d start: %d len: %d maxId: %d\n", + server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, + pool->capacity, pool->start, pool->len, pool->max_id); + + if (pool->id == NULL) { + int32_t *ia = (int32_t *) calloc(sizeof(int32_t), INITIAL_ID_POOL_SZ); + if (ia == NULL) { + ID_POOL_NOMEM + return; + } + time_t *t = (time_t *) calloc(sizeof(time_t), INITIAL_ID_POOL_SZ); + if (t == NULL) { + free(ia); + ID_POOL_NOMEM + return; + } + pool->id = ia; + pool->release = t; + pool->capacity = INITIAL_ID_POOL_SZ; + } else if (pool->len == pool->capacity) { + // grow slowly + n = pool->capacity + (pool->capacity >> 1); + // scrapetime for 1 session report takes probably ~ 160K CPU cycles + // to avoid complexity we assume a 2.6 GHz machine + double max = n * relay_servers_in_use * 16.0/260000; + // 0.8 s/report should be ok =~ 16K sessions =~ 256K metrics + TURN_LOG_LEVEL l = max < 0.8 ? TURN_LOG_LEVEL_INFO : TURN_LOG_LEVEL_WARNING; + int32_t *ia = (int32_t *) calloc(sizeof(int32_t), n); + if (ia == NULL) { + ID_POOL_NOMEM + return; + } + time_t *ta = (time_t *) calloc(sizeof(time_t), n); + if (ta == NULL) { + free(ia); + ID_POOL_NOMEM + return; + } + if (pool->start == 0) { + TURN_LOG_FUNC(l, "tid: %d usid: %d rsid: %d grew rsid pool from " + "%d to %d start: %d len: %d maxSid: %d\n", + server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, + pool->capacity, n, pool->start, pool->len, pool->max_id); + memcpy(ia, pool->id, pool->len * sizeof(int32_t)); + memcpy(ta, pool->release, pool->len * sizeof(time_t)); + } else { + m = pool->capacity - pool->start; + TURN_LOG_FUNC(l, "tid: %d usid: %d rsid: %d grew sid pool from " + "%d to %d start: %d len: %d/end: %d len: %d maxSid: %d\n", + server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, + pool->capacity,n, pool->start,m, 0, pool->len - m, pool->max_id); + memcpy(ia, &(pool->id[pool->start]), m * sizeof(int32_t)); + memcpy(ta, &(pool->release[pool->start]), m * sizeof(time_t)); + memcpy(&(ia[m]), pool->id, (pool->len - m) * sizeof(int32_t)); + memcpy(&(ta[m]), pool->release, (pool->len - m) * sizeof(time_t)); + pool->start = 0; + } + free(pool->id); + free(pool->release); + pool->id = ia; + pool->release = ta; + pool->capacity = n; + } + n = pool->start + pool->len; + if (n >= pool->capacity) + n -= pool->capacity; + pool->id[n] = ss->rsid; + pool->release[n] = server->ctime + *(server->sid_retain); + _TRACE(TURN_LOG_LEVEL_DEBUG, "tid: %d usid: %d rsid: %d idx: %d %d %ld\n", + server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, + n, pool->id[n], pool->release[n]); + (pool->len)++; + ss->rsid *= -1; + _TRACE(TURN_LOG_LEVEL_DEBUG, "tid: %d usid: %d rsid: %d recycle after: " + "capacity: %d start: %d len: %d maxId: %d\n", + server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, + pool->capacity, pool->start, pool->len, pool->max_id); +#endif +} + +void attach_samples(ts_ur_super_session *ss) { +#if defined(TURN_NO_PROMETHEUS) + UNUSED_ARG(ss); +#else + if (ss == NULL || prom_disabled() || ss->server == NULL) + return; + + if (ss->rsid < 0) + ss->rsid = -1; + uint32_t tid = ((turn_turnserver *) ss->server)->id; + + // no reflection in C, so ... + ss->sample_session_state = get_state_sample(tid, ss->rsid, ss->id, + ss->realm_options.name, (char *)ss->username); + if (ss->sample_session_state) + pms_set(ss->sample_session_state, SESSION_STATE_OPEN); + + ss->sample_rx_msgs = get_session_sample(METRIC_RX_MSGS, false, + tid, ss->rsid, ss->id); + ss->sample_tx_msgs = get_session_sample(METRIC_TX_MSGS, false, + tid, ss->rsid, ss->id); + ss->sample_rx_bytes = get_session_sample(METRIC_RX_BYTES, false, + tid, ss->rsid, ss->id); + ss->sample_tx_bytes = get_session_sample(METRIC_TX_BYTES, false, + tid, ss->rsid, ss->id); + + ss->sample_peer_rx_msgs = get_session_sample(METRIC_RX_MSGS, true, + tid, ss->rsid, ss->id); + ss->sample_peer_tx_msgs = get_session_sample(METRIC_TX_MSGS, true, + tid, ss->rsid, ss->id); + ss->sample_peer_rx_bytes = get_session_sample(METRIC_RX_BYTES, true, + tid, ss->rsid, ss->id); + ss->sample_peer_tx_bytes = get_session_sample(METRIC_TX_BYTES, true, + tid, ss->rsid, ss->id); + + ss->sample_lifetime = get_session_sample(METRIC_LIFETIME, false, + tid, ss->rsid, ss->id); + if (ss->sample_lifetime) + pms_set(ss->sample_lifetime, 0); + ss->sample_allocations_running = + get_session_sample(METRIC_ALLOCATIONS_RUNNING, false, + tid, ss->rsid, ss->id); + ss->sample_allocations_created = + get_session_sample(METRIC_ALLOCATIONS_CREATED, false, + tid, ss->rsid, ss->id); + ss->sample_stun_req = get_session_sample(METRIC_STUN_REQUEST, false, + tid, ss->rsid, ss->id); + ss->sample_stun_resp = get_session_sample(METRIC_STUN_RESPONSE, false, + tid, ss->rsid, ss->id); +#endif +} + +void stun_report_binding(ts_ur_super_session *ss, STUN_PROMETHEUS_METRIC_TYPE type, int err_code) { #if !defined(TURN_NO_PROMETHEUS) - UNUSED_ARG(a); + if (! ss->sample_stun_req) + return; + switch (type) { case STUN_PROMETHEUS_METRIC_TYPE_REQUEST: - prom_inc_stun_binding_request(); + pms_add(ss->sample_stun_req, 1); break; case STUN_PROMETHEUS_METRIC_TYPE_RESPONSE: - prom_inc_stun_binding_response(); + pms_add(ss->sample_stun_resp, 1); break; case STUN_PROMETHEUS_METRIC_TYPE_ERROR: - prom_inc_stun_binding_error(); + prom_binding_error(((turn_turnserver *) ss->server)->id, ss->rsid, ss->id, err_code); break; default: break; } #else - UNUSED_ARG(a); + UNUSED_ARG(ss); UNUSED_ARG(type); + UNUSED_ARG(err_code); #endif } @@ -3536,26 +3783,45 @@ void turn_report_allocation_set(void *a, turn_time_t lifetime, int refresh) { if (server) { ioa_engine_handle e = turn_server_get_engine(server); if (e && e->verbose && ss->client_socket) { + SOCKET_TYPE socket_type = get_ioa_socket_type(ss->client_socket); + const char *socket_type_str = socket_type_name(socket_type); if (ss->client_socket->ssl) { TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, - "session %018llu: %s, realm=<%s>, username=<%s>, lifetime=%lu, cipher=%s, method=%s\n", - (unsigned long long)ss->id, status, (char *)ss->realm_options.name, (char *)ss->username, + "session %012llu.%d: %s, realm=<%s>, username=<%s>, lifetime=%lu, cipher=%s, method=%s, socket=%s\n", + (unsigned long long)ss->id, ss->rsid, status, (char *)ss->realm_options.name, (char *)ss->username, (unsigned long)lifetime, SSL_get_cipher(ss->client_socket->ssl), - turn_get_ssl_method(ss->client_socket->ssl, "UNKNOWN")); + turn_get_ssl_method(ss->client_socket->ssl, "UNKNOWN"), + socket_type_str); } else { - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %018llu: %s, realm=<%s>, username=<%s>, lifetime=%lu\n", - (unsigned long long)ss->id, status, (char *)ss->realm_options.name, (char *)ss->username, - (unsigned long)lifetime); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %012llu.%d: %s, realm=<%s>, username=<%s>, lifetime=%lu, socket=%s\n", + (unsigned long long)ss->id, ss->rsid, status, (char *)ss->realm_options.name, (char *)ss->username, + (unsigned long)lifetime, socket_type_str); + } + } + +#if !defined(TURN_NO_PROMETHEUS) + if (ss->sample_allocations_running) { + if (!refresh) { + pms_add(ss->sample_allocations_running, 1); + pms_add(ss->sample_allocations_created, 1); + } + if (ss->sample_session_state) { + pms_set(ss->sample_session_state, refresh + ? SESSION_STATE_REFRESH + : SESSION_STATE_ALLOCATED); + pms_set(ss->sample_lifetime, lifetime); } } +#endif + #if !defined(TURN_NO_HIREDIS) - { + if (e->rch) { char key[1024]; if (ss->realm_options.name[0]) { - snprintf(key, sizeof(key), "turn/realm/%s/user/%s/allocation/%018llu/status", ss->realm_options.name, + snprintf(key, sizeof(key), "turn/realm/%s/user/%s/allocation/%012llu/status", ss->realm_options.name, (char *)ss->username, (unsigned long long)ss->id); } else { - snprintf(key, sizeof(key), "turn/user/%s/allocation/%018llu/status", (char *)ss->username, + snprintf(key, sizeof(key), "turn/user/%s/allocation/%012llu/status", (char *)ss->username, (unsigned long long)ss->id); } uint8_t saddr[129]; @@ -3572,10 +3838,6 @@ void turn_report_allocation_set(void *a, turn_time_t lifetime, int refresh) { (unsigned long)lifetime, type, saddr, rsaddr, ssl, cipher); } #endif - { - if (!refresh) - prom_inc_allocation(get_ioa_socket_type(ss->client_socket)); - } } } } @@ -3589,17 +3851,28 @@ void turn_report_allocation_delete(void *a, SOCKET_TYPE socket_type) { if (server) { ioa_engine_handle e = turn_server_get_engine(server); if (e && e->verbose) { - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %018llu: delete: realm=<%s>, username=<%s>\n", - (unsigned long long)ss->id, (char *)ss->realm_options.name, (char *)ss->username); + const char *socket_type_str = socket_type_name(socket_type); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %012llu.%d: delete: realm=<%s>, username=<%s>, socket=%s\n", + (unsigned long long)ss->id, ss->rsid, (char *)ss->realm_options.name, (char *)ss->username, socket_type_str); } + +#if !defined(TURN_NO_PROMETHEUS) + if (ss->sample_allocations_running) + pms_sub(ss->sample_allocations_running, 1); + if (ss->sample_session_state) { + pms_set(ss->sample_session_state, SESSION_STATE_DEALLOCATED); + pms_set(ss->sample_lifetime, -1); + } +#endif + #if !defined(TURN_NO_HIREDIS) - { + if (e->rch) { char key[1024]; if (ss->realm_options.name[0]) { - snprintf(key, sizeof(key), "turn/realm/%s/user/%s/allocation/%018llu/status", ss->realm_options.name, + snprintf(key, sizeof(key), "turn/realm/%s/user/%s/allocation/%012llu/status", ss->realm_options.name, (char *)ss->username, (unsigned long long)ss->id); } else { - snprintf(key, sizeof(key), "turn/user/%s/allocation/%018llu/status", (char *)ss->username, + snprintf(key, sizeof(key), "turn/user/%s/allocation/%012llu/status", (char *)ss->username, (unsigned long long)ss->id); } send_message_to_redis(e->rch, "del", key, ""); @@ -3607,20 +3880,20 @@ void turn_report_allocation_delete(void *a, SOCKET_TYPE socket_type) { // report total traffic usage for this allocation if (ss->realm_options.name[0]) { - snprintf(key, sizeof(key), "turn/realm/%s/user/%s/allocation/%018llu/total_traffic", ss->realm_options.name, + snprintf(key, sizeof(key), "turn/realm/%s/user/%s/allocation/%012llu/total_traffic", ss->realm_options.name, (char *)ss->username, (unsigned long long)ss->id); } else { - snprintf(key, sizeof(key), "turn/user/%s/allocation/%018llu/total_traffic", (char *)ss->username, + snprintf(key, sizeof(key), "turn/user/%s/allocation/%012llu/total_traffic", (char *)ss->username, (unsigned long long)ss->id); } send_message_to_redis(e->rch, "publish", key, "rcvp=%lu, rcvb=%lu, sentp=%lu, sentb=%lu", (unsigned long)(ss->t_received_packets), (unsigned long)(ss->t_received_bytes), (unsigned long)(ss->t_sent_packets), (unsigned long)(ss->t_sent_bytes)); if (ss->realm_options.name[0]) { - snprintf(key, sizeof(key), "turn/realm/%s/user/%s/allocation/%018llu/total_traffic/peer", + snprintf(key, sizeof(key), "turn/realm/%s/user/%s/allocation/%012llu/total_traffic/peer", ss->realm_options.name, (char *)ss->username, (unsigned long long)(ss->id)); } else { - snprintf(key, sizeof(key), "turn/user/%s/allocation/%018llu/total_traffic/peer", (char *)ss->username, + snprintf(key, sizeof(key), "turn/user/%s/allocation/%012llu/total_traffic/peer", (char *)ss->username, (unsigned long long)(ss->id)); } send_message_to_redis(e->rch, "publish", key, "rcvp=%lu, rcvb=%lu, sentp=%lu, sentb=%lu", @@ -3629,29 +3902,6 @@ void turn_report_allocation_delete(void *a, SOCKET_TYPE socket_type) { (unsigned long)(ss->t_peer_sent_bytes)); } #endif - { - if (ss->realm_options.name[0]) { - - // Set prometheus traffic metrics - prom_set_finished_traffic(ss->realm_options.name, (const char *)ss->username, - (unsigned long)(ss->t_received_packets), (unsigned long)(ss->t_received_bytes), - (unsigned long)(ss->t_sent_packets), (unsigned long)(ss->t_sent_bytes), false); - prom_set_finished_traffic( - ss->realm_options.name, (const char *)ss->username, (unsigned long)(ss->t_peer_received_packets), - (unsigned long)(ss->t_peer_received_bytes), (unsigned long)(ss->t_peer_sent_packets), - (unsigned long)(ss->t_peer_sent_bytes), true); - } else { - // Set prometheus traffic metrics - prom_set_finished_traffic(NULL, (const char *)ss->username, (unsigned long)(ss->t_received_packets), - (unsigned long)(ss->t_received_bytes), (unsigned long)(ss->t_sent_packets), - (unsigned long)(ss->t_sent_bytes), false); - prom_set_finished_traffic(NULL, (const char *)ss->username, (unsigned long)(ss->t_peer_received_packets), - (unsigned long)(ss->t_peer_received_bytes), - (unsigned long)(ss->t_peer_sent_packets), (unsigned long)(ss->t_peer_sent_bytes), - true); - } - prom_dec_allocation(socket_type); - } } } } @@ -3667,34 +3917,34 @@ void turn_report_session_usage(void *session, int force_invalid) { force_invalid) { if (e && e->verbose) { TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, - "session %018llu: usage: realm=<%s>, username=<%s>, rp=%lu, rb=%lu, sp=%lu, sb=%lu\n", - (unsigned long long)(ss->id), (char *)ss->realm_options.name, (char *)ss->username, + "session %012llu.%d: usage: realm=<%s>, username=<%s>, rp=%lu, rb=%lu, sp=%lu, sb=%lu\n", + (unsigned long long)(ss->id), ss->rsid, (char *)ss->realm_options.name, (char *)ss->username, (unsigned long)(ss->received_packets), (unsigned long)(ss->received_bytes), (unsigned long)(ss->sent_packets), (unsigned long)(ss->sent_bytes)); TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, - "session %018llu: peer usage: realm=<%s>, username=<%s>, rp=%lu, rb=%lu, sp=%lu, sb=%lu\n", - (unsigned long long)(ss->id), (char *)ss->realm_options.name, (char *)ss->username, + "session %012llu.%d: peer usage: realm=<%s>, username=<%s>, rp=%lu, rb=%lu, sp=%lu, sb=%lu\n", + (unsigned long long)(ss->id), ss->rsid, (char *)ss->realm_options.name, (char *)ss->username, (unsigned long)(ss->peer_received_packets), (unsigned long)(ss->peer_received_bytes), (unsigned long)(ss->peer_sent_packets), (unsigned long)(ss->peer_sent_bytes)); } #if !defined(TURN_NO_HIREDIS) - { + if (e->chr) { char key[1024]; if (ss->realm_options.name[0]) { - snprintf(key, sizeof(key), "turn/realm/%s/user/%s/allocation/%018llu/traffic", ss->realm_options.name, + snprintf(key, sizeof(key), "turn/realm/%s/user/%s/allocation/%012llu/traffic", ss->realm_options.name, (char *)ss->username, (unsigned long long)(ss->id)); } else { - snprintf(key, sizeof(key), "turn/user/%s/allocation/%018llu/traffic", (char *)ss->username, + snprintf(key, sizeof(key), "turn/user/%s/allocation/%012llu/traffic", (char *)ss->username, (unsigned long long)(ss->id)); } send_message_to_redis(e->rch, "publish", key, "rcvp=%lu, rcvb=%lu, sentp=%lu, sentb=%lu", (unsigned long)(ss->received_packets), (unsigned long)(ss->received_bytes), (unsigned long)(ss->sent_packets), (unsigned long)(ss->sent_bytes)); if (ss->realm_options.name[0]) { - snprintf(key, sizeof(key), "turn/realm/%s/user/%s/allocation/%018llu/traffic/peer", ss->realm_options.name, + snprintf(key, sizeof(key), "turn/realm/%s/user/%s/allocation/%012llu/traffic/peer", ss->realm_options.name, (char *)ss->username, (unsigned long long)(ss->id)); } else { - snprintf(key, sizeof(key), "turn/user/%s/allocation/%018llu/traffic/peer", (char *)ss->username, + snprintf(key, sizeof(key), "turn/user/%s/allocation/%012llu/traffic/peer", (char *)ss->username, (unsigned long long)(ss->id)); } send_message_to_redis(e->rch, "publish", key, "rcvp=%lu, rcvb=%lu, sentp=%lu, sentb=%lu", diff --git a/src/apps/relay/ns_ioalib_impl.h b/src/apps/relay/ns_ioalib_impl.h index 39b984f80..71b523491 100644 --- a/src/apps/relay/ns_ioalib_impl.h +++ b/src/apps/relay/ns_ioalib_impl.h @@ -125,6 +125,10 @@ struct message_to_relay { struct relay_server; typedef struct relay_server *relay_server_handle; +// should be read-only and rw for src/apps/relay/netengine.c to let consumers +// know, how many relay-threads are actually in use. +extern uint32_t relay_servers_in_use; + typedef int (*ioa_engine_new_connection_event_handler)(ioa_engine_handle e, struct message_to_relay *sm); typedef int (*ioa_engine_udp_event_handler)(relay_server_handle rs, struct message_to_relay *sm); diff --git a/src/apps/relay/prom_server.c b/src/apps/relay/prom_server.c index 293ea5887..0802ea474 100644 --- a/src/apps/relay/prom_server.c +++ b/src/apps/relay/prom_server.c @@ -6,78 +6,175 @@ #if !defined(TURN_NO_PROMETHEUS) -prom_counter_t *stun_binding_request; -prom_counter_t *stun_binding_response; -prom_counter_t *stun_binding_error; - -prom_counter_t *turn_traffic_rcvp; -prom_counter_t *turn_traffic_rcvb; -prom_counter_t *turn_traffic_sentp; -prom_counter_t *turn_traffic_sentb; +prom_counter_t *stun_binding_request = NULL; +prom_counter_t *stun_binding_response = NULL; +prom_counter_t *stun_binding_error = NULL; + +prom_counter_t *turn_rx_msgs = NULL; +prom_counter_t *turn_rx_bytes = NULL; +prom_counter_t *turn_tx_msgs = NULL; +prom_counter_t *turn_tx_bytes = NULL; + +prom_gauge_t *turn_lifetime = NULL; // -1 .. closed, otherwise lifetime [s] +prom_gauge_t *turn_allocations = NULL; +prom_gauge_t *turn_state = NULL; + +static bool is_prom_enabled = false; +bool prom_disabled(void) { return !is_prom_enabled; } + +static bool use_rsids = false; +bool prom_rsids(void) { return use_rsids; } + +// metrics need to have the same labels over their whole lifetime, which means +// here, over the lifetime of the app. Therefore labels get calculated once +// on server startup an d reused as needed. +#define LABEL_PEER "peer" +#define LABEL_REALM "realm" +#define LABEL_USER "user" +#define LABEL_TURNSERVER_ID "tid" +#define LABEL_SESSION_ID "sid" +#define LABEL_STUN_ERR "code" +#define LABEL_ALLOCATIONS "created" + +static const char **state_labels; +static size_t state_label_count; +static bool use_sid_labels = true; + +static const char *session_labels[] = + { LABEL_TURNSERVER_ID, LABEL_SESSION_ID, LABEL_PEER }; +static size_t session_label_count = 3; + +static bool +init_state_labels(void) { + size_t n; + + if (turn_params.prom_usid && turn_params.prom_rsid) { + TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, "prom-sid takes precedence " + "over prom-usid. Metrics get tagged with dynamic session IDs to save" + "resources and hopefully keep your timeseries DB happy.\n"); + turn_params.prom_usid = 0; + } + if (!turn_params.prom_rsid && !turn_params.prom_usid) { + use_sid_labels = false; + if (!turn_params.prom_usernames) { + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "Neither prom-usid nor " + "prom-sid nor prom-usernames is given (makes your server safe " + "wrt. too many metrics attacks): session_state and lifetime " + "metrics get not enabled.\n"); + } + state_label_count = 0; // use as indicator for state n/a + return false; + } + + state_label_count = turn_params.prom_realm + turn_params.prom_usernames + + (use_sid_labels ? 2 : 1); + + state_labels = (const char **) + malloc(state_label_count * sizeof(const char *)); + if (state_labels == NULL) { + TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, + "Memory problem - metric collector disabled - init failed.\n"); + state_label_count = 0; + turn_params.prom = 0; + return false; + } + n = 0; + state_labels[n++] = LABEL_TURNSERVER_ID; + if (use_sid_labels) + state_labels[n++] = LABEL_SESSION_ID; + if (turn_params.prom_realm) + state_labels[n++] = LABEL_REALM; + if (turn_params.prom_usernames) + state_labels[n++] = LABEL_USER; + return true; +} -prom_counter_t *turn_traffic_peer_rcvp; -prom_counter_t *turn_traffic_peer_rcvb; -prom_counter_t *turn_traffic_peer_sentp; -prom_counter_t *turn_traffic_peer_sentb; +static void +init_metrics(void) { + const char *str; -prom_gauge_t *turn_total_allocations; + prom_metric_t *version = pcr_must_register_metric(prom_gauge_new( + "version","TURN server version.", 2,(const char *[]) {"name", "release"})); + prom_gauge_set(version, 1, (const char *[]) {"Coturn",TURN_SERVER_VERSION}); -void start_prometheus_server(void) { - PROM_INIT_FLAGS features = PROM_PROCESS|PROM_SCRAPETIME_ALL; - if (turn_params.prometheus == 0) { - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "prometheus collector disabled, not started\n"); - return; + if (init_state_labels()) { + turn_state = pcr_must_register_metric(prom_gauge_new( + "session_state", "The state of a client or peer session. " + "0 .. closed, 1 .. allocation deleted, 2 .. closing, 3 .. open, " + "4 .. allocation created, 5 .. allocation refresh seen.", + state_label_count, state_labels)); } - if (turn_params.prometheus_compact) - features |= PROM_COMPACT; - if (pcr_init(features, "coturn_")) { - TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, "prometheus collector disabled - init failed.\n"); - turn_params.prometheus = 0; - return; + + if (!use_sid_labels) { + session_labels[--session_label_count] = NULL; + session_labels[session_label_count - 1] = LABEL_PEER; + } else if (turn_params.prom_rsid) { + use_rsids = true; } - const char *label[] = {"realm", NULL}; - size_t nlabels = 1; + // Create TURN traffic counter metrics + turn_rx_msgs = pcr_must_register_metric(prom_counter_new( + "rx_msgs","Messages received from the turn client or peer.", + session_label_count, session_labels)); + turn_tx_msgs = pcr_must_register_metric(prom_counter_new( + "tx_msgs","Messages sent to the turn client or peer.", + session_label_count, session_labels)); + turn_rx_bytes = pcr_must_register_metric(prom_counter_new( + "rx_bytes","Bytes received from the turn client or peer.", + session_label_count, session_labels)); + turn_tx_bytes = pcr_must_register_metric(prom_counter_new( + "tx_bytes","Bytes sent to the turn client or peer.", + session_label_count, session_labels)); - if (turn_params.prometheus_username_labels) { - label[1] = "user"; - nlabels++; - } + // Create total allocations number gauge metric + // use peer as total label + str = session_labels[session_label_count - 1]; + session_labels[session_label_count - 1] = LABEL_ALLOCATIONS; + turn_allocations = pcr_must_register_metric(prom_gauge_new( + "allocations", "Current allocations", + session_label_count, session_labels)); + session_labels[session_label_count - 1] = str; + + if (use_sid_labels) + turn_lifetime = pcr_must_register_metric(prom_gauge_new( + "lifetime", "The life time of a client's allocation.", + session_label_count - 1, session_labels)); // Create STUN counters - stun_binding_request = pcr_must_register_metric(prom_counter_new( - "bind_requests","Incoming STUN Binding requests", 0, NULL)); - stun_binding_response = pcr_must_register_metric(prom_counter_new( - "bind_responses","Outgoing STUN Binding responses", 0, NULL)); - stun_binding_error = pcr_must_register_metric(prom_counter_new( - "bind_errors","STUN Binding errors", 0, NULL)); + if (!turn_params.no_stun) { + stun_binding_request = pcr_must_register_metric(prom_counter_new( + "bind_requests","Valid STUN Binding requests received.", + session_label_count - 1, session_labels)); + stun_binding_response = pcr_must_register_metric(prom_counter_new( + "bind_responses","STUN Binding responses sent.", + session_label_count - 1, session_labels)); + + // use peer as error label + str = session_labels[session_label_count - 1]; + session_labels[session_label_count - 1] = LABEL_STUN_ERR; + stun_binding_error = pcr_must_register_metric(prom_counter_new( + "bind_errors","STUN Binding errors", + session_label_count, session_labels)); + session_labels[session_label_count - 1] = str; + } - // Create TURN traffic counter metrics - // see tcp_client_input_handler_rfc6062data() - turn_traffic_rcvp = pcr_must_register_metric(prom_counter_new( - "rx_msgs","Messages received in a session from the turn client.", nlabels, label)); - turn_traffic_peer_sentp = pcr_must_register_metric(prom_counter_new( - "peer_tx_msgs","Messages sent in a session to the turn client.", nlabels, label)); - turn_traffic_rcvb = pcr_must_register_metric(prom_counter_new( - "rx_bytes","Bytes received in a session from the turn client.", nlabels, label)); - turn_traffic_peer_sentb = pcr_must_register_metric(prom_counter_new( - "peer_tx_bytes","Bytes sent in a session to the turn client.", nlabels, label)); - - // Create finished sessions traffic for peers counter metrics - // see tcp_peer_input_handler() - turn_traffic_peer_rcvp = pcr_must_register_metric(prom_counter_new( - "peer_rx_pkts","Messages received in a session from the peer.", nlabels, label)); - turn_traffic_sentp = pcr_must_register_metric(prom_counter_new( - "tx_msgs","Messages sent in a session to the peer.", nlabels, label)); - turn_traffic_peer_rcvb = pcr_must_register_metric(prom_counter_new( - "peer_rx_bytes","Bytes received in a session from peer.", nlabels, label)); - turn_traffic_sentb = pcr_must_register_metric(prom_counter_new( - "tx_bytes","Bytes sent in a session to the peer.", nlabels, label)); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "Metrics initialized.\n"); +} - // Create total allocations number gauge metric - const char *typeLabel[] = {"type"}; - turn_total_allocations = pcr_must_register_metric(prom_gauge_new( - "allocations", "Current allocations", 1, typeLabel)); +void start_prometheus_server(void) { + PROM_INIT_FLAGS features = PROM_PROCESS|PROM_SCRAPETIME_ALL; + if (turn_params.prom == 0) { + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "Metric collector disabled, not started.\n"); + return; + } + + if (turn_params.prom_compact) + features |= PROM_COMPACT; + if (pcr_init(features, "coturn_")) { + TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, "Metric collector disabled - init failed.\n"); + turn_params.prom = 0; + return; + } promhttp_set_active_collector_registry(NULL); @@ -104,94 +201,131 @@ void start_prometheus_server(void) { flags |= MHD_USE_DEBUG; // same as MHD_USE_ERROR_LOG #endif - struct MHD_Daemon *daemon = promhttp_start_daemon(flags, turn_params.prometheus_port, NULL, NULL); + struct MHD_Daemon *daemon = promhttp_start_daemon(flags, turn_params.prom_port, NULL, NULL); if (daemon == NULL) { - TURN_LOG_FUNC(TURN_LOG_LEVEL_ERROR, "could not start prometheus collector\n"); + TURN_LOG_FUNC(TURN_LOG_LEVEL_ERROR, "Could not start metric exporter.\n"); return; } + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "Metric exporter started successfully.\n"); - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "prometheus collector started successfully\n"); - + init_metrics(); + is_prom_enabled = true; return; } -// This is total non-sense right now, because called at the end of a session, -// only. So would only appear as a tiny spike in a timeseries visualization. -void prom_set_finished_traffic(const char *realm, const char *user, unsigned long rsvp, unsigned long rsvb, - unsigned long sentp, unsigned long sentb, bool peer) { - if (turn_params.prometheus == 1) { - - const char *label[] = {realm, NULL}; - if (turn_params.prometheus_username_labels) { - label[1] = user; - } - - if (peer) { - prom_counter_add(turn_traffic_peer_rcvp, rsvp, label); - prom_counter_add(turn_traffic_peer_rcvb, rsvb, label); - prom_counter_add(turn_traffic_peer_sentp, sentp, label); - prom_counter_add(turn_traffic_peer_sentb, sentb, label); - } else { - prom_counter_add(turn_traffic_rcvp, rsvp, label); - prom_counter_add(turn_traffic_rcvb, rsvb, label); - prom_counter_add(turn_traffic_sentp, sentp, label); - prom_counter_add(turn_traffic_sentb, sentb, label); - } - } + //bool peer, int32_t tid, int32_t sid, uint64_t usid) +#define PREPARE_TID_SID_PEER_LABELS(DST, TID, SID, USID, PEER) \ + char tidstr[12]; \ + char sidstr[12]; \ + int n = 0; \ + \ + sprintf(tidstr, "%d", TID); \ + DST[n++] = tidstr; \ + if (use_sid_labels) { \ + if (turn_params.prom_rsid) { \ + sprintf(sidstr, "%d", SID); \ + } else if (turn_params.prom_usid) { \ + /* uint64_t l = USID/TURN_SESSION_ID_FACTOR; sprintf(tidstr,"%ld",l);*/ \ + /* sprintf(sidstr, "%lld", USID - (l * TURN_SESSION_ID_FACTOR)); */ \ + sprintf(sidstr, "%lld", USID - (TID * TURN_SESSION_ID_FACTOR)); \ + } \ + DST[n++] = sidstr; \ + } \ + DST[n++] = PEER ? "1" : "0"; \ + +pms_t * +get_state_sample(int32_t tid, int32_t sid, uint64_t usid, + char *realm, char *user) +{ + const char *vals[state_label_count + 1]; // + dummy for peer + pms_t *res = NULL; + + if (state_label_count == 0 || prom_disabled()) + return NULL; + + PREPARE_TID_SID_PEER_LABELS(vals, tid, sid, usid, false); + if (turn_params.prom_realm) + vals[state_label_count-2] = realm; + if (turn_params.prom_usernames) + vals[state_label_count-1] = user; + + res = pms_from_labels(turn_state, vals); + return res; } -void prom_inc_allocation(SOCKET_TYPE type) { - if (turn_params.prometheus == 1) { - const char *label[] = {socket_type_name(type)}; - prom_gauge_inc(turn_total_allocations, label); - } +pms_t * +get_session_sample(session_metric_t type, bool peer, + int32_t tid, int32_t sid, uint64_t usid) +{ + const char *vals[session_label_count]; + + if (prom_disabled()) + return NULL; + + PREPARE_TID_SID_PEER_LABELS(vals, tid, sid, usid, peer); + + switch (type) { + case METRIC_RX_MSGS: + return pms_from_labels(turn_rx_msgs, vals); + case METRIC_TX_MSGS: + return pms_from_labels(turn_tx_msgs, vals); + case METRIC_RX_BYTES: + return pms_from_labels(turn_rx_bytes, vals); + case METRIC_TX_BYTES: + return pms_from_labels(turn_tx_bytes, vals); + case METRIC_LIFETIME: + return turn_lifetime ? pms_from_labels(turn_lifetime, vals) : NULL; + case METRIC_ALLOCATIONS_RUNNING: + vals[session_label_count-1] = "0"; + return pms_from_labels(turn_allocations, vals); + case METRIC_ALLOCATIONS_CREATED: + vals[session_label_count-1] = "1"; + return pms_from_labels(turn_allocations, vals); + case METRIC_STUN_REQUEST: + vals[session_label_count-1] = NULL; + return stun_binding_request + ? pms_from_labels(stun_binding_request, vals) + : NULL; + case METRIC_STUN_RESPONSE: + vals[session_label_count-1] = NULL; + return stun_binding_response + ? pms_from_labels(stun_binding_response, vals) + : NULL; + case METRIC_STUN_ERROR: + vals[session_label_count-1] = NULL; + // we do not know all errors, don't want to maintain the list. + // So on error the little bit slower way via + // prom_binding_error(...) and the metric will be used. + break; + default: TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, + "Session metric %d is not yet supported.\n", type); + } + return NULL; } -void prom_dec_allocation(SOCKET_TYPE type) { - if (turn_params.prometheus == 1) { - const char *label[] = {socket_type_name(type)}; - prom_gauge_dec(turn_total_allocations, label); - } -} +void prom_binding_error(int32_t tid, int32_t sid, uint64_t usid, + int err) +{ + const char *vals[session_label_count]; + char buf[12]; -void prom_inc_stun_binding_request(void) { - if (turn_params.prometheus == 1) { - prom_counter_add(stun_binding_request, 1, NULL); - } -} + if (!stun_binding_error) + return; -void prom_inc_stun_binding_response(void) { - if (turn_params.prometheus == 1) { - prom_counter_add(stun_binding_response, 1, NULL); - } -} - -void prom_inc_stun_binding_error(void) { - if (turn_params.prometheus == 1) { - prom_counter_add(stun_binding_error, 1, NULL); - } + PREPARE_TID_SID_PEER_LABELS(vals, tid, sid, usid, false); + sprintf(buf, "%d", err); + vals[session_label_count - 1] = buf; + prom_counter_add(stun_binding_error, 1, vals); } #else +bool prom_disabled(void) { return true; } +bool prom_rsids(void) { return false; } + void start_prometheus_server(void) { - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "turnserver compiled without prometheus support\n"); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "turnserver compiled without metric support.\n"); return; } -void prom_set_finished_traffic(const char *realm, const char *user, unsigned long rsvp, unsigned long rsvb, - unsigned long sentp, unsigned long sentb, bool peer) { - UNUSED_ARG(realm); - UNUSED_ARG(user); - UNUSED_ARG(rsvp); - UNUSED_ARG(rsvb); - UNUSED_ARG(sentp); - UNUSED_ARG(sentb); - UNUSED_ARG(peer); -} - -void prom_inc_allocation(SOCKET_TYPE type) { UNUSED_ARG(type); } - -void prom_dec_allocation(SOCKET_TYPE type) { UNUSED_ARG(type); } - #endif /* TURN_NO_PROMETHEUS */ diff --git a/src/apps/relay/prom_server.h b/src/apps/relay/prom_server.h index c766ff8b8..f30f806a8 100644 --- a/src/apps/relay/prom_server.h +++ b/src/apps/relay/prom_server.h @@ -5,7 +5,8 @@ #include "ns_turn_ioalib.h" #include -#define DEFAULT_PROM_SERVER_PORT (9641) +#define DEFAULT_PROM_SERVER_PORT 9641 +#define DEFAULT_PROM_SID_RETAIN 60 #define TURN_ALLOC_STR_MAX_SIZE (20) #if !defined(TURN_NO_PROMETHEUS) @@ -26,31 +27,27 @@ extern prom_counter_t *stun_binding_request; extern prom_counter_t *stun_binding_response; extern prom_counter_t *stun_binding_error; -extern prom_counter_t *turn_new_allocation; -extern prom_counter_t *turn_refreshed_allocation; -extern prom_counter_t *turn_deleted_allocation; +extern prom_counter_t *turn_rx_msgs; +extern prom_counter_t *turn_rx_bytes; +extern prom_counter_t *turn_tx_msgs; +extern prom_counter_t *turn_tx_bytes; +extern prom_gauge_t *turn_lifetime; +extern prom_gauge_t *turn_allocations; + +typedef enum { + METRIC_RX_MSGS, + METRIC_TX_MSGS, + METRIC_RX_BYTES, + METRIC_TX_BYTES, + METRIC_LIFETIME, + METRIC_ALLOCATIONS_RUNNING, + METRIC_ALLOCATIONS_CREATED, + METRIC_STUN_REQUEST, + METRIC_STUN_RESPONSE, + METRIC_STUN_ERROR, + METRIC_MAX +} session_metric_t; -extern prom_counter_t *turn_traffic_rcvp; -extern prom_counter_t *turn_traffic_rcvb; -extern prom_counter_t *turn_traffic_sentp; -extern prom_counter_t *turn_traffic_sentb; - -extern prom_counter_t *turn_traffic_peer_rcvp; -extern prom_counter_t *turn_traffic_peer_rcvb; -extern prom_counter_t *turn_traffic_peer_sentp; -extern prom_counter_t *turn_traffic_peer_sentb; - -extern prom_counter_t *turn_total_traffic_rcvp; -extern prom_counter_t *turn_total_traffic_rcvb; -extern prom_counter_t *turn_total_traffic_sentp; -extern prom_counter_t *turn_total_traffic_sentb; - -extern prom_counter_t *turn_total_traffic_peer_rcvp; -extern prom_counter_t *turn_total_traffic_peer_rcvb; -extern prom_counter_t *turn_total_traffic_peer_sentp; -extern prom_counter_t *turn_total_traffic_peer_sentb; - -extern prom_gauge_t *turn_total_allocations_number; #ifdef __cplusplus extern "C" { @@ -58,27 +55,20 @@ extern "C" { void start_prometheus_server(void); -void prom_set_finished_traffic(const char *realm, const char *user, unsigned long rsvp, unsigned long rsvb, - unsigned long sentp, unsigned long sentb, bool peer); +pms_t *get_state_sample(int32_t tid, int32_t sid, uint64_t usid, char *realm, char *user); +pms_t * get_session_sample(session_metric_t type, bool peer, int32_t tid, int32_t sid, uint64_t usid); -void prom_inc_allocation(SOCKET_TYPE type); -void prom_dec_allocation(SOCKET_TYPE type); - -void prom_inc_stun_binding_request(void); -void prom_inc_stun_binding_response(void); -void prom_inc_stun_binding_error(void); +void prom_binding_error(int32_t tid, int32_t sid, uint64_t usid, int error); #else void start_prometheus_server(void); -void prom_set_finished_traffic(const char *realm, const char *user, unsigned long rsvp, unsigned long rsvb, - unsigned long sentp, unsigned long sentb, bool peer); +#endif /* TURN_NO_PROMETHEUS */ -void prom_inc_allocation(SOCKET_TYPE type); -void prom_dec_allocation(SOCKET_TYPE type); +bool prom_disabled(void); +bool prom_rsids(void); -#endif /* TURN_NO_PROMETHEUS */ #ifdef __cplusplus } diff --git a/src/apps/relay/turn_admin_server.c b/src/apps/relay/turn_admin_server.c index 9967c7326..1ccb10e4d 100644 --- a/src/apps/relay/turn_admin_server.c +++ b/src/apps/relay/turn_admin_server.c @@ -188,6 +188,9 @@ struct toggleable_command tcmds[] = {{"stale-nonce", &turn_params.stale_nonce}, {"no-multicast-peers", &turn_params.no_multicast_peers}, {"allow-loopback-peers", &turn_params.allow_loopback_peers}, {"mobility", &turn_params.mobility}, +#if !defined(TURN_NO_PROMETHEUS) + {"log-ip", &turn_params.log_ip}, +#endif {NULL, NULL}}; /////////////////////////////// @@ -387,6 +390,12 @@ static void change_cli_param(struct cli_session *cs, const char *pn) { cli_max_output_sessions = atoi(pn + strlen("cli-max-output-sessions")); cli_print_uint(cs, (unsigned long)cli_max_output_sessions, "cli-max-output-sessions", 2); return; + } else if (strstr(pn, "prom-sid-retain") == pn) { + int t = atoi(pn + strlen("prom-sid-retain")); + if (t >= 0) + turn_params.prom_sid_retain = t; + cli_print_uint(cs, turn_params.prom_sid_retain, "prom-sid-retain", 2); + return; } myprintf(cs, "\n"); @@ -464,8 +473,8 @@ static int print_session(ur_map_key_type key, ur_map_value_type value, void *arg } if (cs->f || (unsigned long)csarg->counter < (unsigned long)cli_max_output_sessions) { myprintf(cs, "\n"); - myprintf(cs, " %lu) id=%018llu, user <%s>:\n", (unsigned long)(csarg->counter + 1), - (unsigned long long)tsi->id, tsi->username); + myprintf(cs, " %lu) id=%012llu, tid: %d, rsid: %d, user <%s>:\n", (unsigned long)(csarg->counter + 1), + (unsigned long long)tsi->id, (int) (tsi->id % TURN_SESSION_ID_FACTOR), tsi->rsid, tsi->username); if (tsi->realm[0]) myprintf(cs, " realm: %s\n", tsi->realm); if (tsi->origin[0]) @@ -779,6 +788,20 @@ static void cli_print_configuration(struct cli_session *cs) { myprintf(cs, "\n"); +#if !defined(TURN_NO_PROMETHEUS) + cli_print_flag(cs, turn_params.prom, "prom", 0); + cli_print_uint(cs, turn_params.prom_port, "prom-port", 0); + cli_print_flag(cs, turn_params.prom_compact, "prom-compact", 0); + cli_print_flag(cs, turn_params.prom_realm, "prom-realm", 0); + cli_print_flag(cs, turn_params.prom_usernames, "prom-usernames", 0); + cli_print_flag(cs, turn_params.prom_usid, "prom-usid", 0); + cli_print_flag(cs, turn_params.prom_rsid, "prom-sid", 0); + cli_print_uint(cs, turn_params.prom_sid_retain, "prom-sid-retain", 2); + cli_print_flag(cs, turn_params.log_ip, "log-ip", 1); + + myprintf(cs, "\n"); +#endif + cli_print_uint(cs, (unsigned long)cs->rp->status.total_current_allocs, "total-current-allocs", 0); myprintf(cs, "\n"); @@ -2153,6 +2176,20 @@ static void write_pc_page(ioa_socket_handle s) { https_print_empty_row(sb, 2); +#if !defined(TURN_NO_PROMETHEUS) + https_print_flag(sb, turn_params.prom, "prom", NULL); + https_print_uint(sb, turn_params.prom_port, "prom-port", "prom-port"); + https_print_flag(sb, turn_params.prom_compact, "prom-compact", NULL); + https_print_flag(sb, turn_params.prom_realm, "prom-realm", NULL); + https_print_flag(sb, turn_params.prom_usernames, "prom-usernames", NULL); + https_print_flag(sb, turn_params.prom_usid, "prom-usid", NULL); + https_print_flag(sb, turn_params.prom_rsid, "prom-sid", NULL); + https_print_uint(sb, turn_params.prom_sid_retain, "prom-sid-retain", "prom-sid-retain"); + https_print_flag(sb, turn_params.log_ip, "log-ip", "log-ip"); + + https_print_empty_row(sb, 2); +#endif + https_print_uint(sb, (unsigned long)rp->status.total_current_allocs, "total-current-allocs", 0); https_print_empty_row(sb, 2); @@ -2243,11 +2280,11 @@ static int https_print_session(ur_map_key_type key, ur_map_value_type value, voi str_buffer_append(sb, ""); str_buffer_append_sz(sb, (size_t)(csarg->counter + 1)); str_buffer_append(sb, ""); - str_buffer_append_sid(sb, tsi->id); + str_buffer_append_sid(sb, tsi->id, tsi->rsid); str_buffer_append(sb, "
cancel"); str_buffer_append(sb, ""); str_buffer_append(sb, (char *)tsi->username); diff --git a/src/apps/relay/turn_admin_server.h b/src/apps/relay/turn_admin_server.h index 69465618e..0b4b150bc 100644 --- a/src/apps/relay/turn_admin_server.h +++ b/src/apps/relay/turn_admin_server.h @@ -81,11 +81,11 @@ extern struct admin_server adminserver; extern int use_cli; -#define CLI_DEFAULT_IP ("127.0.0.1") +#define CLI_DEFAULT_IP "127.0.0.1" extern ioa_addr cli_addr; extern int cli_addr_set; -#define CLI_DEFAULT_PORT (5766) +#define CLI_DEFAULT_PORT 5766 extern int cli_port; #define CLI_PASSWORD_LENGTH (129) @@ -96,11 +96,11 @@ extern int cli_max_output_sessions; extern int use_web_admin; -#define WEB_ADMIN_DEFAULT_IP ("127.0.0.1") +#define WEB_ADMIN_DEFAULT_IP "127.0.0.1" extern ioa_addr web_admin_addr; extern int web_admin_addr_set; -#define WEB_ADMIN_DEFAULT_PORT (8080) +#define WEB_ADMIN_DEFAULT_PORT 8080 extern int web_admin_port; //////////////////////////////////////////// diff --git a/src/apps/relay/turn_ports.h b/src/apps/relay/turn_ports.h index b3bd9e306..e3867e9be 100644 --- a/src/apps/relay/turn_ports.h +++ b/src/apps/relay/turn_ports.h @@ -41,8 +41,8 @@ extern "C" { ////////////////////////////////////////////////// -#define LOW_DEFAULT_PORTS_BOUNDARY (49152) -#define HIGH_DEFAULT_PORTS_BOUNDARY (65535) +#define LOW_DEFAULT_PORTS_BOUNDARY 49152 +#define HIGH_DEFAULT_PORTS_BOUNDARY 65535 ////////////////////////////////////////////////// diff --git a/src/client/ns_turn_msg_defs.h b/src/client/ns_turn_msg_defs.h index 214deb96e..b4cf0d866 100644 --- a/src/client/ns_turn_msg_defs.h +++ b/src/client/ns_turn_msg_defs.h @@ -60,12 +60,12 @@ #define GET_STUN_ERR_RESP(msg_type) (msg_type | 0x0110) /* Lifetimes: */ -#define STUN_DEFAULT_ALLOCATE_LIFETIME (600) +#define STUN_DEFAULT_ALLOCATE_LIFETIME 600 #define STUN_MIN_ALLOCATE_LIFETIME STUN_DEFAULT_ALLOCATE_LIFETIME -#define STUN_DEFAULT_MAX_ALLOCATE_LIFETIME (3600) -#define STUN_DEFAULT_CHANNEL_LIFETIME (600) -#define STUN_DEFAULT_NONCE_EXPIRATION_TIME (600) -#define STUN_DEFAULT_PERMISSION_LIFETIME (300) +#define STUN_DEFAULT_MAX_ALLOCATE_LIFETIME 3600 +#define STUN_DEFAULT_CHANNEL_LIFETIME 600 +#define STUN_DEFAULT_NONCE_EXPIRATION_TIME 600 +#define STUN_DEFAULT_PERMISSION_LIFETIME 300 /**/ #define STUN_METHOD_BINDING (0x0001) diff --git a/src/ns_turn_defs.h b/src/ns_turn_defs.h index 0f92807f2..6c9ee30db 100644 --- a/src/ns_turn_defs.h +++ b/src/ns_turn_defs.h @@ -173,8 +173,8 @@ typedef uint32_t turn_time_t; #define NONCE_LENGTH_32BITS (4) -#define DEFAULT_STUN_PORT (3478) -#define DEFAULT_STUN_TLS_PORT (5349) +#define DEFAULT_STUN_PORT 3478 +#define DEFAULT_STUN_TLS_PORT 5349 #if BYTE_ORDER == LITTLE_ENDIAN #define DEFAULT_STUN_PORT_NBO (0x960D) diff --git a/src/server/ns_turn_allocation.c b/src/server/ns_turn_allocation.c index 0e87f690a..b657faafd 100644 --- a/src/server/ns_turn_allocation.c +++ b/src/server/ns_turn_allocation.c @@ -171,7 +171,7 @@ void turn_permission_clean(turn_permission_info *tinfo) { if (tinfo->verbose) { char s[257] = "\0"; addr_to_string(&(tinfo->addr), (uint8_t *)s); - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %018llu: peer %s deleted\n", tinfo->session_id, s); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %012llu.%d: peer %s deleted\n", tinfo->session_id, tinfo->session_rid, s); } if (!(tinfo->lifetime_ev)) { diff --git a/src/server/ns_turn_allocation.h b/src/server/ns_turn_allocation.h index bb5dea238..fc7841587 100644 --- a/src/server/ns_turn_allocation.h +++ b/src/server/ns_turn_allocation.h @@ -146,6 +146,7 @@ typedef struct _turn_permission_info { void *owner; // a int verbose; unsigned long long session_id; + uint32_t session_rid; } turn_permission_info; typedef struct _turn_permission_slot { diff --git a/src/server/ns_turn_ioalib.h b/src/server/ns_turn_ioalib.h index aba0db547..cf3d475f5 100644 --- a/src/server/ns_turn_ioalib.h +++ b/src/server/ns_turn_ioalib.h @@ -205,6 +205,20 @@ void ioa_network_buffer_delete(ioa_engine_handle e, ioa_network_buffer_handle nb /* * Status reporting functions */ +void attach_samples(ts_ur_super_session *ss); +void acquire_recyclable_session_id(ts_ur_super_session *ss); +void release_recyclable_session_id(ts_ur_super_session *ss); + +typedef enum { + SESSION_STATE_CLOSED, + SESSION_STATE_DEALLOCATED, + SESSION_STATE_CLOSING, + SESSION_STATE_OPEN, + SESSION_STATE_ALLOCATED, + SESSION_STATE_REFRESH, + SESSION_STATE_MAX +} session_state_t; + enum _STUN_PROMETHEUS_METRIC_TYPE { STUN_PROMETHEUS_METRIC_TYPE_REQUEST, STUN_PROMETHEUS_METRIC_TYPE_RESPONSE, @@ -213,7 +227,7 @@ enum _STUN_PROMETHEUS_METRIC_TYPE { }; typedef enum _STUN_PROMETHEUS_METRIC_TYPE STUN_PROMETHEUS_METRIC_TYPE; -void stun_report_binding(void *session, STUN_PROMETHEUS_METRIC_TYPE type); +void stun_report_binding(ts_ur_super_session *session, STUN_PROMETHEUS_METRIC_TYPE type, int err_code); void turn_report_allocation_set(void *a, turn_time_t lifetime, int refresh); void turn_report_allocation_delete(void *a, SOCKET_TYPE socket_type); diff --git a/src/server/ns_turn_server.c b/src/server/ns_turn_server.c index f92f4211a..6bc0a65b4 100644 --- a/src/server/ns_turn_server.c +++ b/src/server/ns_turn_server.c @@ -34,9 +34,9 @@ #include "ns_turn_allocation.h" #include "ns_turn_ioalib.h" #include "ns_turn_utils.h" +#include /////////////////////////////////////////// - #define FUNCSTART \ if (server && eve(server->verbose)) \ TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "%s:%d:start\n", __FUNCTION__, __LINE__) @@ -86,26 +86,26 @@ static inline void log_method(ts_ur_super_session *ss, const char *method, int e if (!err_code) { if (ss->origin[0]) { TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, - "session %018llu: origin <%s> realm <%s> user <%s>: incoming packet %s processed, success\n", - (unsigned long long)(ss->id), (const char *)(ss->origin), (const char *)(ss->realm_options.name), + "session %012llu.%d: origin <%s> realm <%s> user <%s>: incoming packet %s processed, success\n", + (unsigned long long)(ss->id), ss->rsid, (const char *)(ss->origin), (const char *)(ss->realm_options.name), (const char *)(ss->username), method); } else { TURN_LOG_FUNC( - TURN_LOG_LEVEL_INFO, "session %018llu: realm <%s> user <%s>: incoming packet %s processed, success\n", - (unsigned long long)(ss->id), (const char *)(ss->realm_options.name), (const char *)(ss->username), method); + TURN_LOG_LEVEL_INFO, "session %012llu.%d: realm <%s> user <%s>: incoming packet %s processed, success\n", + (unsigned long long)(ss->id), ss->rsid, (const char *)(ss->realm_options.name), (const char *)(ss->username), method); } } else { if (!reason) reason = get_default_reason(err_code); if (ss->origin[0]) { TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, - "session %018llu: origin <%s> realm <%s> user <%s>: incoming packet %s processed, error %d: %s\n", - (unsigned long long)(ss->id), (const char *)(ss->origin), (const char *)(ss->realm_options.name), + "session %012llu.%d: origin <%s> realm <%s> user <%s>: incoming packet %s processed, error %d: %s\n", + (unsigned long long)(ss->id), ss->rsid, (const char *)(ss->origin), (const char *)(ss->realm_options.name), (const char *)(ss->username), method, err_code, reason); } else { TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, - "session %018llu: realm <%s> user <%s>: incoming packet %s processed, error %d: %s\n", - (unsigned long long)(ss->id), (const char *)(ss->realm_options.name), + "session %012llu.%d: realm <%s> user <%s>: incoming packet %s processed, error %d: %s\n", + (unsigned long long)(ss->id), ss->rsid, (const char *)(ss->realm_options.name), (const char *)(ss->username), method, err_code, reason); } } @@ -431,6 +431,7 @@ int turn_session_info_copy_from(struct turn_session_info *tsi, ts_ur_super_sessi if (tsi && ss) { tsi->id = ss->id; + tsi->rsid = ss->rsid; tsi->bps = ss->bps; tsi->start_time = ss->start_time; tsi->valid = is_allocation_valid(&(ss->alloc)) && !(ss->to_be_closed) && (ss->quota_used); @@ -705,6 +706,8 @@ static void put_session_into_map(ts_ur_super_session *ss) { ss->id = (turnsession_id)((turnsession_id)server->id * TURN_SESSION_ID_FACTOR); ss->id += ++(server->session_id_counter); ss->start_time = server->ctime; + acquire_recyclable_session_id(ss); + attach_samples(ss); } ur_map_put(server->sessions_map, (ur_map_key_type)(ss->id), (ur_map_value_type)ss); put_session_into_mobile_map(ss); @@ -726,6 +729,7 @@ static void delete_session_from_map(ts_ur_super_session *ss) { if (ss && ss->server) { turn_turnserver *server = (turn_turnserver *)(ss->server); ur_map_del(server->sessions_map, (ur_map_key_type)(ss->id), NULL); + release_recyclable_session_id(ss); delete_session_from_mobile_map(ss); } } @@ -745,7 +749,7 @@ void turn_cancel_session(turn_turnserver *server, turnsession_id sid) { if (server) { ts_ur_super_session *ts = get_session_from_map(server, sid); if (ts) { - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "Session %018llu to be forcefully canceled\n", (unsigned long long)sid); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "Session %012llu.%d to be forcefully canceled\n", (unsigned long long)sid, ts->rsid); shutdown_client_connection(server, ts, 0, "Forceful shutdown"); } } @@ -869,10 +873,11 @@ static int update_turn_permission_lifetime(ts_ur_super_session *ss, turn_permiss if (server->verbose) { tinfo->verbose = 1; tinfo->session_id = ss->id; + tinfo->session_rid = ss->rsid; char s[257] = "\0"; addr_to_string(&(tinfo->addr), (uint8_t *)s); - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %018llu: peer %s lifetime updated: %lu\n", - (unsigned long long)ss->id, s, (unsigned long)time_delta); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %012llu.%d: peer %s lifetime updated: %lu\n", + (unsigned long long)ss->id, ss->rsid, s, (unsigned long)time_delta); } return 0; @@ -1848,6 +1853,12 @@ static void tcp_deliver_delayed_buffer(unsent_buffer *ub, ioa_socket_handle s, t } else { ++(ss->sent_packets); ss->sent_bytes += bytes; +#if !defined(TURN_NO_PROMETHEUS) + if (ss->sample_tx_msgs) { + pms_add(ss->sample_tx_msgs, 1); + pms_add(ss->sample_tx_bytes, bytes); + } +#endif turn_report_session_usage(ss, 0); } pop_unsent_buffer(ub); @@ -1884,6 +1895,12 @@ static void tcp_peer_input_handler(ioa_socket_handle s, int event_type, ioa_net_ if (ss) { ++(ss->peer_received_packets); ss->peer_received_bytes += bytes; +#if !defined(TURN_NO_PROMETHEUS) + if (ss->sample_peer_rx_msgs) { + pms_add(ss->sample_peer_rx_msgs, 1); + pms_add(ss->sample_peer_rx_bytes, bytes); + } +#endif } int ret = send_data_from_ioa_socket_nbh(tc->client_s, NULL, nbh, TTL_IGNORE, TOS_IGNORE, NULL); @@ -1892,6 +1909,12 @@ static void tcp_peer_input_handler(ioa_socket_handle s, int event_type, ioa_net_ } else if (ss) { ++(ss->sent_packets); ss->sent_bytes += bytes; +#if !defined(TURN_NO_PROMETHEUS) + if (ss->sample_tx_msgs) { + pms_add(ss->sample_tx_msgs, 1); + pms_add(ss->sample_tx_bytes, bytes); + } +#endif } if (ss) { @@ -1927,6 +1950,12 @@ static void tcp_client_input_handler_rfc6062data(ioa_socket_handle s, int event_ if (ss) { ++(ss->received_packets); ss->received_bytes += bytes; +#if !defined(TURN_NO_PROMETHEUS) + if (ss->sample_rx_msgs) { + pms_add(ss->sample_rx_msgs, 1); + pms_add(ss->sample_rx_bytes, bytes); + } +#endif } int skip = 0; @@ -1938,6 +1967,12 @@ static void tcp_client_input_handler_rfc6062data(ioa_socket_handle s, int event_ if (!skip && ss) { ++(ss->peer_sent_packets); ss->peer_sent_bytes += bytes; +#if !defined(TURN_NO_PROMETHEUS) + if (ss->sample_peer_tx_msgs) { + pms_add(ss->sample_peer_tx_msgs, 1); + pms_add(ss->sample_peer_tx_bytes, bytes); + } +#endif } if (ss) @@ -2787,12 +2822,11 @@ static int handle_turn_binding(turn_turnserver *server, ts_ur_super_session *ss, if (*ua_num > 0) { *err_code = 420; - stun_report_binding(ss, STUN_PROMETHEUS_METRIC_TYPE_ERROR); + stun_report_binding(ss, STUN_PROMETHEUS_METRIC_TYPE_ERROR, *err_code); } else if (*err_code) { - stun_report_binding(ss, STUN_PROMETHEUS_METRIC_TYPE_ERROR); - + stun_report_binding(ss, STUN_PROMETHEUS_METRIC_TYPE_ERROR, *err_code); } else if (ss->client_socket && get_remote_addr_from_ioa_socket(ss->client_socket)) { - stun_report_binding(ss, STUN_PROMETHEUS_METRIC_TYPE_REQUEST); + stun_report_binding(ss, STUN_PROMETHEUS_METRIC_TYPE_REQUEST, 0); size_t len = ioa_network_buffer_get_size(nbh); if (stun_set_binding_response_str(ioa_network_buffer_data(nbh), &len, tid, @@ -2965,6 +2999,12 @@ static int handle_turn_send(turn_turnserver *server, ts_ur_super_session *ss, in if (!skip) { ++(ss->peer_sent_packets); ss->peer_sent_bytes += len; +#if !defined(TURN_NO_PROMETHEUS) + if (ss->sample_peer_tx_msgs) { + pms_add(ss->sample_peer_tx_msgs, 1); + pms_add(ss->sample_peer_tx_bytes, len); + } +#endif turn_report_session_usage(ss, 0); } in_buffer->nbh = NULL; @@ -3732,7 +3772,7 @@ static int handle_turn_command(turn_turnserver *server, ts_ur_super_session *ss, } send_turn_message_to(server, nbh, &response_origin, &response_destination); - stun_report_binding(ss, STUN_PROMETHEUS_METRIC_TYPE_RESPONSE); + stun_report_binding(ss, STUN_PROMETHEUS_METRIC_TYPE_RESPONSE, 0); no_response = 1; } @@ -3913,7 +3953,7 @@ static int handle_old_stun_command(turn_turnserver *server, ts_ur_super_session } send_turn_message_to(server, nbh, &response_origin, &response_destination); - + stun_report_binding(ss, STUN_PROMETHEUS_METRIC_TYPE_RESPONSE, 0); no_response = 1; } } @@ -4012,8 +4052,15 @@ static int write_to_peerchannel(ts_ur_super_session *ss, uint16_t chnum, ioa_net in_buffer->recv_ttl - 1, in_buffer->recv_tos, &skip); if (!skip && rc > -1) { + uint32_t bytes = (uint32_t)ioa_network_buffer_get_size(in_buffer->nbh); ++(ss->peer_sent_packets); - ss->peer_sent_bytes += (uint32_t)ioa_network_buffer_get_size(in_buffer->nbh); + ss->peer_sent_bytes += bytes; +#if !defined(TURN_NO_PROMETHEUS) + if (ss->sample_peer_tx_msgs) { + pms_add(ss->sample_peer_tx_msgs, 1); + pms_add(ss->sample_peer_tx_bytes, bytes); + } +#endif turn_report_session_usage(ss, 0); } @@ -4058,8 +4105,8 @@ int shutdown_client_connection(turn_turnserver *server, ts_ur_super_session *ss, TURN_LOG_FUNC( TURN_LOG_LEVEL_INFO, - "session %018llu: closed (1st stage), user <%s> realm <%s> origin <%s>, local %s, remote %s, reason: %s\n", - (unsigned long long)(ss->id), (char *)ss->username, (char *)ss->realm_options.name, (char *)ss->origin, + "session %012llu.%d: closed (1st stage), user <%s> realm <%s> origin <%s>, local %s, remote %s, reason: %s\n", + (unsigned long long)(ss->id), ss->rsid, (char *)ss->username, (char *)ss->realm_options.name, (char *)ss->origin, sladdr, sraddr, reason); } @@ -4075,6 +4122,8 @@ int shutdown_client_connection(turn_turnserver *server, ts_ur_super_session *ss, (long)ss->client_socket, (long)get_ioa_socket_session(ss->client_socket)); } + if (ss->sample_session_state) + pms_set(ss->sample_session_state, SESSION_STATE_CLOSING); if (server->disconnect) server->disconnect(ss); @@ -4087,8 +4136,8 @@ int shutdown_client_connection(turn_turnserver *server, ts_ur_super_session *ss, TURN_LOG_FUNC( TURN_LOG_LEVEL_INFO, - "session %018llu: closed (2nd stage), user <%s> realm <%s> origin <%s>, local %s, remote %s, reason: %s\n", - (unsigned long long)(ss->id), (char *)ss->username, (char *)ss->realm_options.name, (char *)ss->origin, sladdr, + "session %012llu.%d: closed (2nd stage), user <%s> realm <%s> origin <%s>, local %s, remote %s, reason: %s\n", + (unsigned long long)(ss->id), ss->rsid, (char *)ss->username, (char *)ss->realm_options.name, (char *)ss->origin, sladdr, sraddr, reason); } @@ -4101,6 +4150,8 @@ int shutdown_client_connection(turn_turnserver *server, ts_ur_super_session *ss, } turn_server_remove_all_from_ur_map_ss(ss, socket_type); + if (ss->sample_session_state) + pms_set(ss->sample_session_state, SESSION_STATE_CLOSED); FUNCEND; @@ -4173,8 +4224,15 @@ static int write_client_connection(turn_turnserver *server, ts_ur_super_session int ret = send_data_from_ioa_socket_nbh(ss->client_socket, NULL, nbh, ttl, tos, &skip); if (!skip && ret > -1) { + uint32_t bytes = (uint32_t)ioa_network_buffer_get_size(nbh); ++(ss->sent_packets); - ss->sent_bytes += (uint32_t)ioa_network_buffer_get_size(nbh); + ss->sent_bytes += bytes; +#if !defined(TURN_NO_PROMETHEUS) + if (ss->sample_tx_msgs) { + pms_add(ss->sample_tx_msgs, 1); + pms_add(ss->sample_tx_bytes, bytes); + } +#endif turn_report_session_usage(ss, 0); } @@ -4376,8 +4434,15 @@ static int read_client_connection(turn_turnserver *server, ts_ur_super_session * } if (count_usage) { + uint32_t bytes = (uint32_t)ioa_network_buffer_get_size(in_buffer->nbh); ++(ss->received_packets); - ss->received_bytes += (uint32_t)ioa_network_buffer_get_size(in_buffer->nbh); + ss->received_bytes += bytes; +#if !defined(TURN_NO_PROMETHEUS) + if (ss->sample_rx_msgs) { + pms_add(ss->sample_rx_msgs, 1); + pms_add(ss->sample_rx_bytes, bytes); + } +#endif turn_report_session_usage(ss, 0); } @@ -4604,6 +4669,7 @@ static int attach_socket_to_session(turn_turnserver *server, ioa_socket_handle s int open_client_connection_session(turn_turnserver *server, struct socket_message *sm) { int ret = 0; + FUNCSTART; if (!server) return -1; @@ -4615,6 +4681,17 @@ int open_client_connection_session(turn_turnserver *server, struct socket_messag ss->client_socket = sm->s; + if (*(server->log_ip)) { + // log IPs early to be able to correlate metrics if needed + char sraddr[129] = "\0"; + char sladdr[129] = "\0"; + addr_to_string(get_remote_addr_from_ioa_socket(ss->client_socket), (uint8_t *)sraddr); + addr_to_string(get_local_addr_from_ioa_socket(ss->client_socket), (uint8_t *)sladdr); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, + "new session %012llu tid: %d rsid: %d: local %s, remote %s\n", + (unsigned long long)(ss->id), server->id, ss->rsid, sladdr, sraddr); + } + if (register_callback_on_ioa_socket(server->e, ss->client_socket, IOA_EV_READ, client_input_handler, ss, 0) < 0) { ret = -1; } @@ -4685,6 +4762,12 @@ static void peer_input_handler(ioa_socket_handle s, int event_type, ioa_net_data if (ilen >= 0) { ++(ss->peer_received_packets); ss->peer_received_bytes += ilen; +#if !defined(TURN_NO_PROMETHEUS) + if (ss->sample_peer_rx_msgs) { + pms_add(ss->sample_peer_rx_msgs, 1); + pms_add(ss->sample_peer_rx_bytes, ilen); + } +#endif turn_report_session_usage(ss, 0); allocation *a = get_allocation_ss(ss); @@ -4781,8 +4864,8 @@ static void client_input_handler(ioa_socket_handle s, int event_type, ioa_net_da if (ss->to_be_closed) { if (server->verbose) { - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %018llu: client socket to be closed in client handler: ss=0x%lx\n", - (unsigned long long)(ss->id), (long)ss); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %012llu.%d: client socket to be closed in client handler: ss=0x%lx\n", + (unsigned long long)(ss->id), ss->rsid, (long)ss); } set_ioa_socket_tobeclosed(s); } @@ -4796,6 +4879,10 @@ void init_turn_server(turn_turnserver *server, turnserver_id id, int verbose, io release_allocation_quota_cb raqcb, ioa_addr *external_ip, vintp check_origin, vintp no_tcp_relay, vintp no_udp_relay, vintp stale_nonce, vintp max_allocate_lifetime, vintp channel_lifetime, vintp permission_lifetime, vintp stun_only, vintp no_stun, vintp no_software_attribute, +#if !defined(TURN_NO_PROMETHEUS) + vintp retain, + vintp log_ip, +#endif vintp web_admin_listen_on_workers, turn_server_addrs_list_t *alternate_servers_list, turn_server_addrs_list_t *tls_alternate_servers_list, turn_server_addrs_list_t *aux_servers_list, int self_udp_balance, vintp no_multicast_peers, vintp allow_loopback_peers, @@ -4811,13 +4898,17 @@ void init_turn_server(turn_turnserver *server, turnserver_id id, int verbose, io return; memset(server, 0, sizeof(turn_turnserver)); - server->e = e; server->id = id; server->ctime = turn_time(); server->session_id_counter = 0; server->sessions_map = ur_map_create(); server->tcp_relay_connections = ur_map_create(); +#if !defined(TURN_NO_PROMETHEUS) + server->rsid_pool = (id_pool_t *) calloc(sizeof(id_pool_t), 1); + server->sid_retain = retain; + server->log_ip = log_ip; +#endif server->ct = ct; server->userkeycb = userkeycb; server->chquotacb = chquotacb; @@ -4836,7 +4927,7 @@ void init_turn_server(turn_turnserver *server, turnserver_id id, int verbose, io server->mobile_connections_map = ur_map_create(); server->acme_redirect = acme_redirect; - TURN_LOG_FUNC(TURN_LOG_LEVEL_DEBUG, "turn server id=%d created\n", (int)id); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "turn server id=%d created.\n", (int)id); server->check_origin = check_origin; server->no_tcp_relay = no_tcp_relay; diff --git a/src/server/ns_turn_server.h b/src/server/ns_turn_server.h index 526ab1554..7f5c88b7b 100644 --- a/src/server/ns_turn_server.h +++ b/src/server/ns_turn_server.h @@ -40,7 +40,10 @@ extern "C" { //////////// defines ////////////// -#define TURN_SESSION_ID_FACTOR (1000000000000000LL) +// Even if a single threaded app handles on average about 1 session/s +// this means, the counter lasts ~ 31.7 years before it wraps to 0 and even +// than it is unlikely, that a session last that long and a collision occures. +#define TURN_SESSION_ID_FACTOR (1000000000LL) //////////// ALTERNATE-SERVER ///////////// @@ -90,6 +93,19 @@ typedef enum { DONT_FRAGMENT_SUPPORT_EMULATED } dont_fragment_option_t; +#if !defined(TURN_NO_PROMETHEUS) +/** pool used to maintain recyclable session IDs for a single turn server. */ +typedef struct { + int32_t *id; // array with the IDs of closed sessions + time_t *release; // time from when the ID can be re-used again. + int32_t capacity; // number of IDs, which fit into the id and release. + int32_t len; // number of recyclable IDs in id and timestamps in release; + int32_t start; // the index where to start looking for a free ID + int32_t max_id; // the highest ID used so far + int32_t recycled; // how many times an ID of the pool got re-used +} id_pool_t; +#endif + struct _turn_turnserver; typedef struct _turn_turnserver turn_turnserver; @@ -115,6 +131,11 @@ struct _turn_turnserver { turnsession_id session_id_counter; ur_map *sessions_map; +#if !defined(TURN_NO_PROMETHEUS) + id_pool_t *rsid_pool; // the pool for recyclable session IDs + vintp sid_retain; // retain IDs of closed sessions at least N seconds before reuse + vintp log_ip; // log the endpoint IPs on each session setup +#endif turn_time_t ctime; @@ -210,6 +231,10 @@ void init_turn_server( check_new_allocation_quota_cb chquotacb, release_allocation_quota_cb raqcb, ioa_addr *external_addr, vintp check_origin, vintp no_tcp_relay, vintp no_udp_relay, vintp stale_nonce, vintp max_allocate_lifetime, vintp channel_lifetime, vintp permission_lifetime, vintp stun_only, vintp no_stun, vintp no_software_attribute, +#if !defined(TURN_NO_PROMETHEUS) + vintp retain, + vintp log_ip, +#endif vintp web_admin_listen_on_workers, turn_server_addrs_list_t *alternate_servers_list, turn_server_addrs_list_t *tls_alternate_servers_list, turn_server_addrs_list_t *aux_servers_list, int self_udp_balance, vintp no_multicast_peers, vintp allow_loopback_peers, ip_range_list_t *ip_whitelist, diff --git a/src/server/ns_turn_session.h b/src/server/ns_turn_session.h index a1cdab8d4..b65735c33 100644 --- a/src/server/ns_turn_session.h +++ b/src/server/ns_turn_session.h @@ -35,6 +35,9 @@ #include "ns_turn_ioalib.h" #include "ns_turn_maps.h" #include "ns_turn_utils.h" +#if !defined(TURN_NO_PROMETHEUS) +#include +#endif #ifdef __cplusplus extern "C" { @@ -90,6 +93,32 @@ struct _ts_ur_super_session { int origin_set; char origin[STUN_MAX_ORIGIN_SIZE + 1]; /* Stats */ + int32_t rsid; // recyclable ID (*server instance scope). Actually for metrics, + // but would clutter logging code, if one would always need to + // check for TURN_NO_PROMETHEUS ... +#if !defined(TURN_NO_PROMETHEUS) + // to avoid metric sample lookups on each counter/gauge op, we store and use + // the reference to the samples directly. The sample ops are just atomic + // inc/dev/set ops and thus really fast and safe (at least wrt. to add and + // set ;-)). This way explicit thread locking. key construction + hashing + // and bucket lookup are skipped, which are BTW pretty fast, too (e.g. the + // bucket lookup on a 2.6 GHz Xeon E5-2690v4 takes about 30 ns for the sample + // 'peer_tx_bytes{tid="10"_sid="20"_realm="nice_to_meet_you"}'). + pms_t *sample_rx_msgs; + pms_t *sample_tx_msgs; + pms_t *sample_rx_bytes; + pms_t *sample_tx_bytes; + pms_t *sample_peer_rx_msgs; + pms_t *sample_peer_tx_msgs; + pms_t *sample_peer_rx_bytes; + pms_t *sample_peer_tx_bytes; + pms_t *sample_allocations_running; + pms_t *sample_allocations_created; + pms_t *sample_lifetime; + pms_t *sample_session_state; + pms_t *sample_stun_req; + pms_t *sample_stun_resp; +#endif uint32_t received_packets; uint32_t sent_packets; uint32_t received_bytes; @@ -147,6 +176,7 @@ struct turn_session_info { uint8_t username[STUN_MAX_USERNAME_SIZE + 1]; int enforce_fingerprints; /* Stats */ + int32_t rsid; uint64_t received_packets; uint64_t sent_packets; uint64_t received_bytes; From 121b31977b0316a0e7b53beed1ac2956bf0f4ff7 Mon Sep 17 00:00:00 2001 From: Jens Elkner Date: Wed, 6 Dec 2023 18:35:43 +0100 Subject: [PATCH 04/12] do not use digitalocean/prometheus-client-c but libprom --- docs/Prometheus.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/Prometheus.md b/docs/Prometheus.md index 94860033a..643cd1be4 100644 --- a/docs/Prometheus.md +++ b/docs/Prometheus.md @@ -1,8 +1,8 @@ -# Prometheus setup +# Metrics setup It is need the following libraries: -- prometheus-client-c +- libprom - libmicrohttpd ## Ubuntu @@ -23,12 +23,12 @@ Download from https://git.gnunet.org/libmicrohttpd.git git clone https://git.gnunet.org/libmicrohttpd.git ``` -- [prometheus-client-c](https://github.com/digitalocean/prometheus-client-c) +- [libprom](https://github.com/jelmd/libprom) -Download from https://github.com/digitalocean/prometheus-client-c.git +Download from https://github.com/jelmd/libprom.git ``` -git clone https://github.com/digitalocean/prometheus-client-c.git +git clone https://github.com/jelmd/libprom.git ``` ## Build @@ -44,10 +44,10 @@ cd libmicrohttpd make install ``` -- Build prometheus-client-c from source code +- Build libprom from source code ``` -git clone https://github.com/digitalocean/prometheus-client-c.git -cd prometheus-client-c +git clone https://github.com/jelmd/libprom.git +cd libprom make ``` From 3584653db6a24b8c72cdfeb6d421fae89f89b4c6 Mon Sep 17 00:00:00 2001 From: Jens Elkner Date: Thu, 7 Dec 2023 08:05:09 +0100 Subject: [PATCH 05/12] Adjust GHA to use libprom --- .github/workflows/cmake.yaml | 10 ++++++---- .github/workflows/mingw.yml | 11 ++++++++--- .github/workflows/tests.yml | 19 +++++++++++-------- cmake/FindPrometheus.cmake | 4 ++-- 4 files changed, 27 insertions(+), 17 deletions(-) diff --git a/.github/workflows/cmake.yaml b/.github/workflows/cmake.yaml index 0a5b383a8..a1ecb228b 100644 --- a/.github/workflows/cmake.yaml +++ b/.github/workflows/cmake.yaml @@ -7,6 +7,8 @@ on: env: BUILD_TYPE: Release + LIBPROM_VERS: '1.2.0-1' + LIBPROM_URL: 'https://pkg.cs.ovgu.de/LNF/linux/ubuntu/20.04' jobs: build: @@ -28,10 +30,10 @@ jobs: - uses: actions/checkout@v4 - name: Prometheus support run: | - wget https://github.com/digitalocean/prometheus-client-c/releases/download/v0.1.3/libprom-dev-0.1.3-Linux.deb && \ - wget https://github.com/digitalocean/prometheus-client-c/releases/download/v0.1.3/libpromhttp-dev-0.1.3-Linux.deb && \ - sudo apt install ./libprom-dev-0.1.3-Linux.deb ./libpromhttp-dev-0.1.3-Linux.deb && \ - rm ./libprom-dev-0.1.3-Linux.deb ./libpromhttp-dev-0.1.3-Linux.deb + wget ${LIBPROM_URL}/libprom-dev-${LIBPROM_VERS}.deb && \ + wget ${LIBPROM_URL}/libprom-${LIBPROM_VERS}.deb && \ + sudo apt install ./libprom-${LIBPROM_VERS}.deb ./libprom-dev-${LIBPROM_VERS}.deb && \ + rm -f ./libprom-*.deb - name: Configure CMake run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} - name: Build diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index b2264b18e..615770383 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -2,6 +2,11 @@ name: mingw on: [push] +env: + LIBPROM_REPO: 'https://github.com/jelmd/libprom' + # GHA does not support ksh/bash parameter expansion like ${LIBPROM_REPO##*/} + LIBPROM_DIR: 'libprom' + jobs: build: name: build @@ -69,8 +74,8 @@ jobs: PATH: C:\msys64\mingw64\bin;C:\msys64\usr\bin run: | cd ${{env.SOURCE_DIR}} - git clone https://github.com/digitalocean/prometheus-client-c.git - cd prometheus-client-c/prom + git clone ${LIBPROM_REPO} + cd ${LIBPROM_DIR}/prom mkdir build cd build cmake .. -G"MinGW Makefiles" ^ @@ -79,7 +84,7 @@ jobs: -DCMAKE_INSTALL_PREFIX=${{env.INSTALL_DIR}} cmake --build . --config ${{matrix.BUILD_TYPE}} cmake --build . --config ${{matrix.BUILD_TYPE}} --target install - cd ${{env.SOURCE_DIR}}/prometheus-client-c/promhttp + cd ${{env.SOURCE_DIR}}/${LIBPROM_DIR}/promhttp mkdir build cd build cmake .. -G"MinGW Makefiles" ^ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0b676d099..e0f96051b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,12 +6,16 @@ on: pull_request: types: [ opened, reopened, synchronize ] +env: + LIBPROM_VERS: '1.2.0-1' + LIBPROM_URL: 'https://pkg.cs.ovgu.de/LNF/linux/ubuntu/20.04' + jobs: builds: strategy: fail-fast: false matrix: - os: [ 'ubuntu:16.04', 'ubuntu:20.04', 'ubuntu:22.04' ] + os: [ 'ubuntu:20.04', 'ubuntu:22.04' ] runs-on: ubuntu-latest container: ${{ matrix.os }} steps: @@ -30,13 +34,12 @@ jobs: libpq-dev libsqlite3-dev \ libhiredis-dev \ libmongoc-dev \ - libmicrohttpd-dev - if [ ${{ matrix.os }} = 'ubuntu:16.04' ]; then apt-get install -y libmariadb-client-lgpl-dev; fi - if [ ${{ matrix.os }} != 'ubuntu:16.04' ]; then apt-get install -y libmariadb-dev; fi - wget https://github.com/digitalocean/prometheus-client-c/releases/download/v0.1.3/libprom-dev-0.1.3-Linux.deb && \ - wget https://github.com/digitalocean/prometheus-client-c/releases/download/v0.1.3/libpromhttp-dev-0.1.3-Linux.deb && \ - apt install ./libprom-dev-0.1.3-Linux.deb ./libpromhttp-dev-0.1.3-Linux.deb && \ - rm ./libprom-dev-0.1.3-Linux.deb ./libpromhttp-dev-0.1.3-Linux.deb + libmicrohttpd-dev \ + libmariadb-dev + wget ${LIBPROM_URL}/libprom-dev-${LIBPROM_VERS}.deb && \ + wget ${LIBPROM_URL}/libprom-${LIBPROM_VERS}.deb && \ + apt install ./libprom-${LIBPROM_VERS}.deb ./libprom-dev-${LIBPROM_VERS}.deb && \ + rm -f ./libprom-*.deb - uses: actions/checkout@v3 - name: configure run: ./configure diff --git a/cmake/FindPrometheus.cmake b/cmake/FindPrometheus.cmake index f99abffcc..f8c01680d 100644 --- a/cmake/FindPrometheus.cmake +++ b/cmake/FindPrometheus.cmake @@ -34,7 +34,7 @@ find_library( PATH_SUFFIXES lib ${CMAKE_INSTALL_LIBDIR}) find_path(prom_INCLUDE_DIR - NAMES prom.h + NAMES prom.h libprom/prom.h HINTS ${Prometheus_DIR} ${Prometheus_ROOT} ${PC_prom_INCLUDE_DIRS} /usr PATHS $ENV{Prometheus_DIR} $ENV{Prometheus_ROOT} PATH_SUFFIXES include @@ -48,7 +48,7 @@ find_library( PATH_SUFFIXES lib ${CMAKE_INSTALL_LIBDIR}) find_path(promhttp_INCLUDE_DIR - NAMES promhttp.h + NAMES promhttp.h libprom/promhttp.h HINTS ${Prometheus_DIR} ${Prometheus_ROOT} ${PC_promhttp_INCLUDE_DIRS} /usr PATHS $ENV{Prometheus_DIR} $ENV{Prometheus_ROOT} PATH_SUFFIXES include From 99695119a148911ffcc5f5731b74296440109418 Mon Sep 17 00:00:00 2001 From: Jens Elkner Date: Thu, 7 Dec 2023 08:18:53 +0100 Subject: [PATCH 06/12] add missing TURN_NO_PROMETHEUS guards --- src/apps/relay/mainrelay.c | 2 ++ src/apps/relay/turn_admin_server.c | 2 ++ src/server/ns_turn_server.c | 8 ++++++++ 3 files changed, 12 insertions(+) diff --git a/src/apps/relay/mainrelay.c b/src/apps/relay/mainrelay.c index 6cf8ba3e1..9472717ad 100644 --- a/src/apps/relay/mainrelay.c +++ b/src/apps/relay/mainrelay.c @@ -1809,7 +1809,9 @@ static int get_bool_value(const char *s) { } static void set_option(int c, char *value) { +#if !defined(TURN_NO_PROMETHEUS) int n; +#endif if (value && value[0] == '=') { TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, diff --git a/src/apps/relay/turn_admin_server.c b/src/apps/relay/turn_admin_server.c index 1ccb10e4d..a6c072960 100644 --- a/src/apps/relay/turn_admin_server.c +++ b/src/apps/relay/turn_admin_server.c @@ -390,12 +390,14 @@ static void change_cli_param(struct cli_session *cs, const char *pn) { cli_max_output_sessions = atoi(pn + strlen("cli-max-output-sessions")); cli_print_uint(cs, (unsigned long)cli_max_output_sessions, "cli-max-output-sessions", 2); return; +#if !defined(TURN_NO_PROMETHEUS) } else if (strstr(pn, "prom-sid-retain") == pn) { int t = atoi(pn + strlen("prom-sid-retain")); if (t >= 0) turn_params.prom_sid_retain = t; cli_print_uint(cs, turn_params.prom_sid_retain, "prom-sid-retain", 2); return; +#endif } myprintf(cs, "\n"); diff --git a/src/server/ns_turn_server.c b/src/server/ns_turn_server.c index 6bc0a65b4..74ff8eb36 100644 --- a/src/server/ns_turn_server.c +++ b/src/server/ns_turn_server.c @@ -4122,8 +4122,11 @@ int shutdown_client_connection(turn_turnserver *server, ts_ur_super_session *ss, (long)ss->client_socket, (long)get_ioa_socket_session(ss->client_socket)); } +#if !defined(TURN_NO_PROMETHEUS) if (ss->sample_session_state) pms_set(ss->sample_session_state, SESSION_STATE_CLOSING); +#endif + if (server->disconnect) server->disconnect(ss); @@ -4150,8 +4153,11 @@ int shutdown_client_connection(turn_turnserver *server, ts_ur_super_session *ss, } turn_server_remove_all_from_ur_map_ss(ss, socket_type); + +#if !defined(TURN_NO_PROMETHEUS) if (ss->sample_session_state) pms_set(ss->sample_session_state, SESSION_STATE_CLOSED); +#endif FUNCEND; @@ -4681,6 +4687,7 @@ int open_client_connection_session(turn_turnserver *server, struct socket_messag ss->client_socket = sm->s; +#if !defined(TURN_NO_PROMETHEUS) if (*(server->log_ip)) { // log IPs early to be able to correlate metrics if needed char sraddr[129] = "\0"; @@ -4691,6 +4698,7 @@ int open_client_connection_session(turn_turnserver *server, struct socket_messag "new session %012llu tid: %d rsid: %d: local %s, remote %s\n", (unsigned long long)(ss->id), server->id, ss->rsid, sladdr, sraddr); } +#endif if (register_callback_on_ioa_socket(server->e, ss->client_socket, IOA_EV_READ, client_input_handler, ss, 0) < 0) { ret = -1; From 5ad19b5dafe97489cccb9e6fea8775120d32b55e Mon Sep 17 00:00:00 2001 From: Jens Elkner Date: Thu, 7 Dec 2023 16:42:35 +0100 Subject: [PATCH 07/12] fix typo for !TURN_NO_HIREDIS --- src/apps/relay/ns_ioalib_engine_impl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/relay/ns_ioalib_engine_impl.c b/src/apps/relay/ns_ioalib_engine_impl.c index dd403950b..3328c51e0 100644 --- a/src/apps/relay/ns_ioalib_engine_impl.c +++ b/src/apps/relay/ns_ioalib_engine_impl.c @@ -3928,7 +3928,7 @@ void turn_report_session_usage(void *session, int force_invalid) { (unsigned long)(ss->peer_sent_packets), (unsigned long)(ss->peer_sent_bytes)); } #if !defined(TURN_NO_HIREDIS) - if (e->chr) { + if (e->rch) { char key[1024]; if (ss->realm_options.name[0]) { snprintf(key, sizeof(key), "turn/realm/%s/user/%s/allocation/%012llu/traffic", ss->realm_options.name, From 0d9a79caa36dc8fd6f076109e90d2945d7eeee9c Mon Sep 17 00:00:00 2001 From: Jens Elkner Date: Thu, 7 Dec 2023 20:35:49 +0100 Subject: [PATCH 08/12] docker stuff should use libprom, too. --- README.md | 4 ++-- docker/coturn/alpine/Dockerfile | 28 +++++++++++++--------------- docker/coturn/debian/Dockerfile | 28 +++++++++++++--------------- docker/coturn/tests/main.bats | 4 ++-- 4 files changed, 30 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 17aa3bda1..03815b550 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ coturn requires following dependencies to be installed first Optional - openssl (to support TLS and DTLS, authorized STUN and TURN) -- libmicrohttp and [prometheus-client-c](https://github.com/digitalocean/prometheus-client-c) (prometheus interface) +- libmicrohttp and [libprom](https://github.com/jelmd/libprom) (libprom interface) - MySQL (user database) - [Hiredis](https://github.com/redis/hiredis) (user database, monitoring) - SQLite (user database) @@ -111,7 +111,7 @@ Management interfaces: Monitoring: * Redis can be used for status and statistics storage and notification - * [prometheus](https://prometheus.io/) interface (unavailable on apt package) + * [libprom](https://jelmd.github.io/libprom/) interface (unavailable on apt package) Message integrity digest algorithms: diff --git a/docker/coturn/alpine/Dockerfile b/docker/coturn/alpine/Dockerfile index 764c02978..4712b264c 100644 --- a/docker/coturn/alpine/Dockerfile +++ b/docker/coturn/alpine/Dockerfile @@ -8,10 +8,10 @@ ARG alpine_ver=3.18.5 # -# Stage 'dist-libprom' creates prometheus-client-c distribution. +# Stage 'dist-libprom' creates libprom distribution. # -# We compile prometheus-client-c from sources, because Alpine doesn't provide +# We compile libprom from sources, because Alpine doesn't provide # it as its package yet. # # TODO: Re-check this to be present in packages on next Alpine major version update. @@ -25,15 +25,15 @@ RUN apk update \ ca-certificates cmake g++ git make \ && update-ca-certificates -# Install prometheus-client-c build dependencies. +# Install libprom build dependencies. RUN apk add --no-cache \ libmicrohttpd-dev -# Prepare prometheus-client-c sources for building. -ARG prom_ver=0.1.3 +# Prepare libprom sources for building. +ARG prom_ver=1.2.0 RUN mkdir -p /build/ && cd /build/ \ && git init \ - && git remote add origin https://github.com/digitalocean/prometheus-client-c \ + && git remote add origin https://github.com/jelmd/libprom \ && git fetch --depth=1 origin "v${prom_ver}" \ && git checkout FETCH_HEAD @@ -49,8 +49,6 @@ RUN mkdir -p /build/prom/build/ && cd /build/prom/build/ \ # Build libpromhttp.so from sources. RUN mkdir -p /build/promhttp/build/ && cd /build/promhttp/build/ \ # Fix compiler warning: -Werror=incompatible-pointer-types - && sed -i 's/\&promhttp_handler/(MHD_AccessHandlerCallback)\&promhttp_handler/' \ - /build/promhttp/src/promhttp.c \ && TEST=0 cmake -G "Unix Makefiles" \ -DCMAKE_INSTALL_PREFIX=/usr \ -DCMAKE_SKIP_BUILD_RPATH=TRUE \ @@ -58,19 +56,19 @@ RUN mkdir -p /build/promhttp/build/ && cd /build/promhttp/build/ \ .. \ && make VERBOSE=1 -# Install prometheus-client-c. +# Install libprom. RUN LIBS_DIR=/out/$(dirname $(find /usr/ -name libc.so)) \ && mkdir -p $LIBS_DIR/ \ && cp -rf /build/prom/build/libprom.so \ /build/promhttp/build/libpromhttp.so \ $LIBS_DIR/ \ - && mkdir -p /out/usr/include/ \ + && mkdir -p /out/usr/include/libprom \ && cp -rf /build/prom/include/* \ /build/promhttp/include/* \ - /out/usr/include/ \ + /out/usr/include/libprom/ \ # Preserve license file. - && mkdir -p /out/usr/share/licenses/prometheus-client-c/ \ - && cp /build/LICENSE /out/usr/share/licenses/prometheus-client-c/ + && mkdir -p /out/usr/share/licenses/libprom/ \ + && cp /build/LICENSE /out/usr/share/licenses/libprom/ @@ -98,7 +96,7 @@ RUN apk add --no-cache \ mongo-c-driver-dev \ libmicrohttpd-dev -# Install prometheus-client-c distribution. +# Install libprom distribution. COPY --from=dist-libprom /out/ / # Prepare local Coturn sources for building. @@ -160,7 +158,7 @@ RUN ln -s /usr/local/bin/detect-external-ip.sh \ /out/usr/local/bin/detect-external-ip RUN chown -R nobody:nogroup /out/var/lib/coturn/ -# Re-export prometheus-client-c distribution. +# Re-export libprom distribution. COPY --from=dist-libprom /out/ /out/ diff --git a/docker/coturn/debian/Dockerfile b/docker/coturn/debian/Dockerfile index 1011dfe3c..39e83d16d 100644 --- a/docker/coturn/debian/Dockerfile +++ b/docker/coturn/debian/Dockerfile @@ -8,10 +8,10 @@ ARG debian_ver=bookworm # -# Stage 'dist-libprom' creates prometheus-client-c distribution. +# Stage 'dist-libprom' creates libprom distribution. # -# We compile prometheus-client-c from sources, because Debian doesn't provide +# We compile libprom from sources, because Debian doesn't provide # it as its package yet. # # TODO: Re-check this to be present in packages on next Debian major version update. @@ -25,15 +25,15 @@ RUN apt-get update \ ca-certificates cmake g++ git make \ && update-ca-certificates -# Install prometheus-client-c build dependencies. +# Install libprom build dependencies. RUN apt-get install -y --no-install-recommends --no-install-suggests \ libmicrohttpd-dev -# Prepare prometheus-client-c sources for building. -ARG prom_ver=0.1.3 +# Prepare libprom sources for building. +ARG prom_ver=1.2.0 RUN mkdir -p /build/ && cd /build/ \ && git init \ - && git remote add origin https://github.com/digitalocean/prometheus-client-c \ + && git remote add origin https://github.com/jelmd/libprom \ && git fetch --depth=1 origin "v${prom_ver}" \ && git checkout FETCH_HEAD @@ -49,8 +49,6 @@ RUN mkdir -p /build/prom/build/ && cd /build/prom/build/ \ # Build libpromhttp.so from sources. RUN mkdir -p /build/promhttp/build/ && cd /build/promhttp/build/ \ # Fix compiler warning: -Werror=incompatible-pointer-types - && sed -i 's/\&promhttp_handler/(MHD_AccessHandlerCallback)\&promhttp_handler/' \ - /build/promhttp/src/promhttp.c \ && TEST=0 cmake -G "Unix Makefiles" \ -DCMAKE_INSTALL_PREFIX=/usr \ -DCMAKE_SKIP_BUILD_RPATH=TRUE \ @@ -58,19 +56,19 @@ RUN mkdir -p /build/promhttp/build/ && cd /build/promhttp/build/ \ .. \ && make VERBOSE=1 -# Install prometheus-client-c. +# Install libprom. RUN LIBS_DIR=/out/$(dirname $(find /usr/ -name libc.so)) \ && mkdir -p $LIBS_DIR/ \ && cp -rf /build/prom/build/libprom.so \ /build/promhttp/build/libpromhttp.so \ $LIBS_DIR/ \ - && mkdir -p /out/usr/include/ \ + && mkdir -p /out/usr/include/libprom \ && cp -rf /build/prom/include/* \ /build/promhttp/include/* \ - /out/usr/include/ \ + /out/usr/include/libprom \ # Preserve license file. - && mkdir -p /out/usr/share/licenses/prometheus-client-c/ \ - && cp /build/LICENSE /out/usr/share/licenses/prometheus-client-c/ + && mkdir -p /out/usr/share/licenses/libprom \ + && cp /build/LICENSE /out/usr/share/licenses/libprom/ @@ -97,7 +95,7 @@ RUN apt-get install -y --no-install-recommends --no-install-suggests \ libmongoc-dev \ libmicrohttpd-dev -# Install prometheus-client-c distribution. +# Install libprom distribution. COPY --from=dist-libprom /out/ / # Prepare local Coturn sources for building. @@ -159,7 +157,7 @@ RUN ln -s /usr/local/bin/detect-external-ip.sh \ /out/usr/local/bin/detect-external-ip RUN chown -R nobody:nogroup /out/var/lib/coturn/ -# Re-export prometheus-client-c distribution. +# Re-export libprom distribution. COPY --from=dist-libprom /out/ /out/ diff --git a/docker/coturn/tests/main.bats b/docker/coturn/tests/main.bats index afc0c27ee..66b4f9790 100644 --- a/docker/coturn/tests/main.bats +++ b/docker/coturn/tests/main.bats @@ -127,10 +127,10 @@ @test "Prometheus supported" { # Support of Prometheus is not displayed in the output, - # but using --prometheus flag does the job. + # but using --prom flag does the job. run docker run --rm --pull never --platform $PLATFORM \ --entrypoint sh $IMAGE -c \ - "turnserver -o --log-file=stdout --prometheus | grep 'Version Coturn'" + "turnserver -o --log-file=stdout --prom | grep 'Version Coturn'" [ "$status" -eq 0 ] [ ! "$output" = '' ] } From 0711a35ccd6744af07ca6c22f0367e707feaef0e Mon Sep 17 00:00:00 2001 From: Jens Elkner Date: Fri, 8 Dec 2023 08:07:51 +0100 Subject: [PATCH 09/12] docker stuff seems to be buggy - try to find/fix problems --- configure | 16 +++++++++++++++- docker/coturn/alpine/Dockerfile | 33 ++++----------------------------- docker/coturn/debian/Dockerfile | 33 ++++----------------------------- 3 files changed, 23 insertions(+), 59 deletions(-) diff --git a/configure b/configure index e14f97732..5a85f21f8 100755 --- a/configure +++ b/configure @@ -1,5 +1,7 @@ #!/bin/sh +DEBUG= + # Proprietary configure script of Coturn project cleanup() { @@ -19,6 +21,12 @@ testlibraw() { ER=$? if ! [ ${ER} -eq 0 ] ; then ${ECHO_CMD} "Library option -${1} cannot be used" + if [ -n ${DEBUG} ]; then + ${ECHO_CMD} "Command: ${CC} ${TMPCPROGC} -o ${TMPCPROGB} ${OSCFLAGS} ${OSLIBS} -${1}" + ${ECHO_CMD} "${TMPCPROGC}:\n---start---" + cat ${TMPCPROGC} + ${ECHO_CMD} "---end---" + fi return 0 else OSLIBS="${OSLIBS} -${1}" @@ -843,6 +851,7 @@ fi if [ -z "${TURN_NO_PROMETHEUS}" ] ; then +DEBUG="1" testlib prom ER=$? if ! [ ${ER} -eq 0 ] ; then @@ -878,13 +887,18 @@ if [ -z "${TURN_NO_PROMETHEUS}" ] ; then fi else ${ECHO_CMD} + if [ -n ${DEBUG} ]; then + ls -al /usr/lib/*/libprom* + ls -al /usr/lib/libprom* + ${ECHO_CMD} + fi ${ECHO_CMD} "Warning: Libprom development libraries are not installed properly in required location." ${ECHO_CMD} "Prometheus support will be disabled." ${ECHO_CMD} "See the docs/Prometheus.md file." ${ECHO_CMD} OSCFLAGS="${OSCFLAGS} -DTURN_NO_PROMETHEUS" fi - +DEBUG= else OSCFLAGS="${OSCFLAGS} -DTURN_NO_PROMETHEUS" fi diff --git a/docker/coturn/alpine/Dockerfile b/docker/coturn/alpine/Dockerfile index 4712b264c..032741593 100644 --- a/docker/coturn/alpine/Dockerfile +++ b/docker/coturn/alpine/Dockerfile @@ -37,42 +37,17 @@ RUN mkdir -p /build/ && cd /build/ \ && git fetch --depth=1 origin "v${prom_ver}" \ && git checkout FETCH_HEAD -# Build libprom.so from sources. -RUN mkdir -p /build/prom/build/ && cd /build/prom/build/ \ - && TEST=0 cmake -G "Unix Makefiles" \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DCMAKE_SKIP_BUILD_RPATH=TRUE \ - -DCMAKE_C_FLAGS="-DPROM_LOG_ENABLE -g -O3" \ - .. \ - && make - -# Build libpromhttp.so from sources. -RUN mkdir -p /build/promhttp/build/ && cd /build/promhttp/build/ \ - # Fix compiler warning: -Werror=incompatible-pointer-types - && TEST=0 cmake -G "Unix Makefiles" \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DCMAKE_SKIP_BUILD_RPATH=TRUE \ - -DCMAKE_C_FLAGS="-g -O3" \ - .. \ - && make VERBOSE=1 +# Build libprom.so and libpromhttp.so from sources. +RUN cd /build && make VERBOSE=1 # Install libprom. -RUN LIBS_DIR=/out/$(dirname $(find /usr/ -name libc.so)) \ - && mkdir -p $LIBS_DIR/ \ - && cp -rf /build/prom/build/libprom.so \ - /build/promhttp/build/libpromhttp.so \ - $LIBS_DIR/ \ - && mkdir -p /out/usr/include/libprom \ - && cp -rf /build/prom/include/* \ - /build/promhttp/include/* \ - /out/usr/include/libprom/ \ +RUN cd /build/prom/build && DESTDIR=/out make install \ + && cd /build/promhttp/build && DESTDIR=/out make install \ # Preserve license file. && mkdir -p /out/usr/share/licenses/libprom/ \ && cp /build/LICENSE /out/usr/share/licenses/libprom/ - - # # Stage 'dist-coturn' creates Coturn distribution. # diff --git a/docker/coturn/debian/Dockerfile b/docker/coturn/debian/Dockerfile index 39e83d16d..a488ea1ac 100644 --- a/docker/coturn/debian/Dockerfile +++ b/docker/coturn/debian/Dockerfile @@ -37,42 +37,17 @@ RUN mkdir -p /build/ && cd /build/ \ && git fetch --depth=1 origin "v${prom_ver}" \ && git checkout FETCH_HEAD -# Build libprom.so from sources. -RUN mkdir -p /build/prom/build/ && cd /build/prom/build/ \ - && TEST=0 cmake -G "Unix Makefiles" \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DCMAKE_SKIP_BUILD_RPATH=TRUE \ - -DCMAKE_C_FLAGS="-DPROM_LOG_ENABLE -g -O3" \ - .. \ - && make - -# Build libpromhttp.so from sources. -RUN mkdir -p /build/promhttp/build/ && cd /build/promhttp/build/ \ - # Fix compiler warning: -Werror=incompatible-pointer-types - && TEST=0 cmake -G "Unix Makefiles" \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DCMAKE_SKIP_BUILD_RPATH=TRUE \ - -DCMAKE_C_FLAGS="-g -O3" \ - .. \ - && make VERBOSE=1 +# Build libprom.so and libpromhttp.so from sources. +RUN cd /build && make VERBOSE=1 # Install libprom. -RUN LIBS_DIR=/out/$(dirname $(find /usr/ -name libc.so)) \ - && mkdir -p $LIBS_DIR/ \ - && cp -rf /build/prom/build/libprom.so \ - /build/promhttp/build/libpromhttp.so \ - $LIBS_DIR/ \ - && mkdir -p /out/usr/include/libprom \ - && cp -rf /build/prom/include/* \ - /build/promhttp/include/* \ - /out/usr/include/libprom \ +RUN cd /build/prom/build && DESTDIR=/out make install \ + && cd /build/promhttp/build && DESTDIR=/out make install \ # Preserve license file. && mkdir -p /out/usr/share/licenses/libprom \ && cp /build/LICENSE /out/usr/share/licenses/libprom/ - - # # Stage 'dist-coturn' creates Coturn distribution. # From 40ac7c0adf621a6a34b7ccf818fd0da458af5065 Mon Sep 17 00:00:00 2001 From: Jens Elkner Date: Fri, 8 Dec 2023 15:20:25 +0100 Subject: [PATCH 10/12] fix clang-format usage =8-( and make it happy --- Makefile.in | 6 +- src/apps/common/win/getopt.h | 8 +- src/apps/relay/mainrelay.c | 20 +- src/apps/relay/netengine.c | 32 +- src/apps/relay/ns_ioalib_engine_impl.c | 431 ++++++++++++------------- src/apps/relay/prom_server.c | 372 ++++++++++----------- src/apps/relay/prom_server.h | 26 +- src/apps/relay/turn_admin_server.c | 2 +- src/server/ns_turn_allocation.c | 3 +- src/server/ns_turn_ioalib.h | 14 +- src/server/ns_turn_server.c | 93 +++--- src/server/ns_turn_server.h | 23 +- src/server/ns_turn_session.h | 6 +- 13 files changed, 501 insertions(+), 535 deletions(-) diff --git a/Makefile.in b/Makefile.in index 88570e4a0..42b61b533 100755 --- a/Makefile.in +++ b/Makefile.in @@ -46,10 +46,12 @@ check: bin/turnutils_rfc5769check bin/turnutils_rfc5769check format: - find . -iname "*.c" -o -iname "*.h" | xargs clang-format -i + find src -iname "*.c" -o -iname "*.h" | \ + xargs clang-format -i lint: - find . -iname "*.c" -o -iname "*.h" | xargs clang-format --dry-run -Werror + find src -iname "*.c" -o -iname "*.h" | \ + xargs clang-format --dry-run -Werror include/turn/ns_turn_defs.h: src/ns_turn_defs.h ${RMCMD} include diff --git a/src/apps/common/win/getopt.h b/src/apps/common/win/getopt.h index 2ebc68e1e..9987add20 100644 --- a/src/apps/common/win/getopt.h +++ b/src/apps/common/win/getopt.h @@ -77,12 +77,14 @@ struct option /* specification for a long form option... */ int val; /* its associated status value */ }; -enum /* permitted values for its `has_arg' field... */ -{ - no_argument = 0, /* option never takes an argument */ +// clang-format off +// different clang-format versions have different opinions on this and prevent PRs to succeed +enum /* permitted values for its `has_arg' field... */ +{ no_argument = 0, /* option never takes an argument */ required_argument, /* option always requires an argument */ optional_argument /* option may take an argument */ }; +// clang-format on extern int getopt_long(int nargc, char *const *nargv, const char *options, const struct option *long_options, int *idx); extern int getopt_long_only(int nargc, char *const *nargv, const char *options, const struct option *long_options, diff --git a/src/apps/relay/mainrelay.c b/src/apps/relay/mainrelay.c index 9472717ad..b4f0914fe 100644 --- a/src/apps/relay/mainrelay.c +++ b/src/apps/relay/mainrelay.c @@ -212,15 +212,15 @@ turn_params_t turn_params = { 0, /* total_quota */ 0, /* user_quota */ #if !defined(TURN_NO_PROMETHEUS) - 0, /* prom disabled by default */ - DEFAULT_PROM_SERVER_PORT, /* prom port */ - 0, /* prom compact output */ - 0, /* prom realm labels */ - 0, /* prom username labels */ - 0, /* prom unique sessionID labels */ - 0, /* prom sessionID labels, reusable */ - DEFAULT_PROM_SID_RETAIN, /* prom retain sessionID secs before reuse */ - 0, /* log ip on session create */ + 0, /* prom disabled by default */ + DEFAULT_PROM_SERVER_PORT, /* prom port */ + 0, /* prom compact output */ + 0, /* prom realm labels */ + 0, /* prom username labels */ + 0, /* prom unique sessionID labels */ + 0, /* prom sessionID labels, reusable */ + DEFAULT_PROM_SID_RETAIN, /* prom retain sessionID secs before reuse */ + 0, /* log ip on session create */ #endif ///////////// Users DB ////////////// {(TURN_USERDB_TYPE)0, {"\0", "\0"}, {0, NULL, {NULL, 0}}}, @@ -867,6 +867,7 @@ int get_a_local_relay(int family, ioa_addr *relay_addr) { } ////////////////////////////////////////////////// +// clang-format off #define _STRVAL(s) _STR(s) #define _STR(s) #s static char Usage[] = @@ -1287,6 +1288,7 @@ static char Usage[] = " --version Print version (and exit).\n" " -h Help\n" "\n"; +// clang-format on static char AdminUsage[] = "Usage: turnadmin [command] [options]\n" diff --git a/src/apps/relay/netengine.c b/src/apps/relay/netengine.c index 2f0ce2d54..2917d2fc8 100644 --- a/src/apps/relay/netengine.c +++ b/src/apps/relay/netengine.c @@ -1622,24 +1622,24 @@ static void setup_relay_server(struct relay_server *rs, ioa_engine_handle e, int bufferevent_setcb(rs->auth_in_buf, relay_receive_auth_message, NULL, NULL, rs); bufferevent_enable(rs->auth_in_buf, EV_READ); - init_turn_server( - &(rs->server), rs->id, turn_params.verbose, rs->ioa_eng, turn_params.ct, 0, turn_params.fingerprint, - DONT_FRAGMENT_SUPPORTED, start_user_check, check_new_allocation_quota, release_allocation_quota, - turn_params.external_ip, &turn_params.check_origin, &turn_params.no_tcp_relay, &turn_params.no_udp_relay, - &turn_params.stale_nonce, &turn_params.max_allocate_lifetime, &turn_params.channel_lifetime, - &turn_params.permission_lifetime, &turn_params.stun_only, &turn_params.no_stun, &turn_params.no_software_attribute, + init_turn_server(&(rs->server), rs->id, turn_params.verbose, rs->ioa_eng, turn_params.ct, 0, turn_params.fingerprint, + DONT_FRAGMENT_SUPPORTED, start_user_check, check_new_allocation_quota, release_allocation_quota, + turn_params.external_ip, &turn_params.check_origin, &turn_params.no_tcp_relay, + &turn_params.no_udp_relay, &turn_params.stale_nonce, &turn_params.max_allocate_lifetime, + &turn_params.channel_lifetime, &turn_params.permission_lifetime, &turn_params.stun_only, + &turn_params.no_stun, &turn_params.no_software_attribute, #if !defined(TURN_NO_PROMETHEUS) - &turn_params.prom_sid_retain, - &turn_params.log_ip, + &turn_params.prom_sid_retain, &turn_params.log_ip, #endif - &turn_params.web_admin_listen_on_workers, &turn_params.alternate_servers_list, - &turn_params.tls_alternate_servers_list, &turn_params.aux_servers_list, turn_params.udp_self_balance, - &turn_params.no_multicast_peers, &turn_params.allow_loopback_peers, &turn_params.ip_whitelist, - &turn_params.ip_blacklist, send_socket_to_relay, &turn_params.secure_stun, &turn_params.mobility, - turn_params.server_relay, send_turn_session_info, send_https_socket, allocate_bps, turn_params.oauth, - turn_params.oauth_server_name, turn_params.acme_redirect, turn_params.allocation_default_address_family, - &turn_params.log_binding, &turn_params.no_stun_backward_compatibility, - &turn_params.response_origin_only_with_rfc5780, &turn_params.respond_http_unsupported); + &turn_params.web_admin_listen_on_workers, &turn_params.alternate_servers_list, + &turn_params.tls_alternate_servers_list, &turn_params.aux_servers_list, turn_params.udp_self_balance, + &turn_params.no_multicast_peers, &turn_params.allow_loopback_peers, &turn_params.ip_whitelist, + &turn_params.ip_blacklist, send_socket_to_relay, &turn_params.secure_stun, &turn_params.mobility, + turn_params.server_relay, send_turn_session_info, send_https_socket, allocate_bps, turn_params.oauth, + turn_params.oauth_server_name, turn_params.acme_redirect, + turn_params.allocation_default_address_family, &turn_params.log_binding, + &turn_params.no_stun_backward_compatibility, &turn_params.response_origin_only_with_rfc5780, + &turn_params.respond_http_unsupported); if (to_set_rfc5780) { set_rfc5780(&(rs->server), get_alt_addr, send_message_from_listener_to_client); diff --git a/src/apps/relay/ns_ioalib_engine_impl.c b/src/apps/relay/ns_ioalib_engine_impl.c index 3328c51e0..91795242d 100644 --- a/src/apps/relay/ns_ioalib_engine_impl.c +++ b/src/apps/relay/ns_ioalib_engine_impl.c @@ -183,12 +183,12 @@ static void log_socket_event(ioa_socket_handle s, const char *msg, int error) { if (!msg) msg = "General socket event"; turnsession_id id = 0; - int32_t rsid = 0; + int32_t rsid = 0; { ts_ur_super_session *ss = s->session; if (ss) { id = ss->id; - rsid = ss->rsid; + rsid = ss->rsid; } else { return; } @@ -210,8 +210,8 @@ static void log_socket_event(ioa_socket_handle s, const char *msg, int error) { TURN_LOG_FUNC(ll, "session %012llu.%d: %s: %s (local %s, remote %s)\n", (unsigned long long)id, rsid, msg, evutil_socket_error_to_string(EVUTIL_SOCKET_ERROR()), sladdr, sraddr); } else { - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %012llu.%d: %s (local %s, remote %s)\n", (unsigned long long)id, rsid, msg, - sladdr, sraddr); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %012llu.%d: %s (local %s, remote %s)\n", (unsigned long long)id, + rsid, msg, sladdr, sraddr); } } } @@ -3528,65 +3528,67 @@ const char *get_ioa_socket_ssl_method(ioa_socket_handle s) { * @param ss The sessions, which needs to get a recyclable ID. */ #ifdef POOL_TRACE - #define _TRACE TURN_LOG_FUNC +#define _TRACE TURN_LOG_FUNC #else - // dirty talk - is for developers, only - #define _TRACE(level, fmt, ...) +// dirty talk - is for developers, only +#define _TRACE(level, fmt, ...) #endif -void -acquire_recyclable_session_id(ts_ur_super_session *ss) { +void acquire_recyclable_session_id(ts_ur_super_session *ss) { #if defined(TURN_NO_PROMETHEUS) - UNUSED_ARG(ss); + UNUSED_ARG(ss); #else - if (!prom_rsids()) - return; - - // gets called by initialized servers, only. So no ts->var NULL checks. - turn_turnserver *server = (turn_turnserver *)(ss->server); - id_pool_t *pool = server->rsid_pool; - _TRACE(TURN_LOG_LEVEL_DEBUG, "tid: %d usid: %d wants a sid (has %d) start: %d\n", - server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, pool->start); - - if (pool->id == NULL || pool->len == 0) { - (pool->max_id)++; - _TRACE(TURN_LOG_LEVEL_DEBUG,"tid: %d usid: %d <= newId: %d " - "capacity: %d len: %d\n", - server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), pool->max_id, - pool->capacity, pool->len); - ss->rsid = pool->max_id; - return; - } - if (pool->release[pool->start] < server->ctime) { - ss->rsid = pool->id[pool->start]; - // not really needed but - pool->release[pool->start] = INT_MAX; - pool->id[pool->start] = -1; - - (pool->start)++; - if (pool->start == pool->capacity) - pool->start = 0; - (pool->len)--; - (pool->recycled)++; - _TRACE(TURN_LOG_LEVEL_DEBUG, "tid: %d usid: %d <= newId: %d " - "capacity: %d len: %d recycledSids: %d startNow: %d\n", - server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, - pool->capacity, pool->len, pool->recycled, pool->start); - return; - } - // all IDs in use - (pool->max_id)++; - _TRACE(TURN_LOG_LEVEL_DEBUG, "tid: %d usid: %d <= newId: %d " - "capacity: %d len: %d\n", - server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), pool->max_id, - pool->capacity,pool->len); - ss->rsid = pool->max_id; + if (!prom_rsids()) + return; + + // gets called by initialized servers, only. So no ts->var NULL checks. + turn_turnserver *server = (turn_turnserver *)(ss->server); + id_pool_t *pool = server->rsid_pool; + _TRACE(TURN_LOG_LEVEL_DEBUG, "tid: %d usid: %d wants a sid (has %d) start: %d\n", server->id, + (int)(ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, pool->start); + + if (pool->id == NULL || pool->len == 0) { + (pool->max_id)++; + _TRACE(TURN_LOG_LEVEL_DEBUG, + "tid: %d usid: %d <= newId: %d " + "capacity: %d len: %d\n", + server->id, (int)(ss->id % TURN_SESSION_ID_FACTOR), pool->max_id, pool->capacity, pool->len); + ss->rsid = pool->max_id; + return; + } + if (pool->release[pool->start] < server->ctime) { + ss->rsid = pool->id[pool->start]; + // not really needed but + pool->release[pool->start] = INT_MAX; + pool->id[pool->start] = -1; + + (pool->start)++; + if (pool->start == pool->capacity) + pool->start = 0; + (pool->len)--; + (pool->recycled)++; + _TRACE(TURN_LOG_LEVEL_DEBUG, + "tid: %d usid: %d <= newId: %d " + "capacity: %d len: %d recycledSids: %d startNow: %d\n", + server->id, (int)(ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, pool->capacity, pool->len, pool->recycled, + pool->start); + return; + } + // all IDs in use + (pool->max_id)++; + _TRACE(TURN_LOG_LEVEL_DEBUG, + "tid: %d usid: %d <= newId: %d " + "capacity: %d len: %d\n", + server->id, (int)(ss->id % TURN_SESSION_ID_FACTOR), pool->max_id, pool->capacity, pool->len); + ss->rsid = pool->max_id; #endif } #define INITIAL_ID_POOL_SZ 4 -#define ID_POOL_NOMEM TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, \ - "Turnserver %d: Not enough memory to maintain the pool for " \ - "recyclable session IDs.\n", server->id); +#define ID_POOL_NOMEM \ + TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, \ + "Turnserver %d: Not enough memory to maintain the pool for " \ + "recyclable session IDs.\n", \ + server->id); /** * Delete the recyclable ID of the given session and and return it to the @@ -3595,161 +3597,146 @@ acquire_recyclable_session_id(ts_ur_super_session *ss) { * @param ss The sessions, which needs to give its recyclable ID back to the * server's rsid_pool. */ -void -release_recyclable_session_id(ts_ur_super_session *ss) { +void release_recyclable_session_id(ts_ur_super_session *ss) { #if defined(TURN_NO_PROMETHEUS) - UNUSED_ARG(ss); + UNUSED_ARG(ss); #else - int n, m; - - if (!prom_rsids()) - return; - - turn_turnserver *server = (turn_turnserver *)(ss->server); - id_pool_t *pool = server->rsid_pool; - _TRACE(TURN_LOG_LEVEL_DEBUG, "tid: %d usid: %d returns sid %d%s\n", - server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, - ss->rsid < 1 ? " INVALID" : ""); - - if (ss->rsid < 1) - return; - - _TRACE(TURN_LOG_LEVEL_DEBUG,"tid: %d usid: %d rsid: %d recycle before: " - "capacity: %d start: %d len: %d maxId: %d\n", - server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, - pool->capacity, pool->start, pool->len, pool->max_id); - - if (pool->id == NULL) { - int32_t *ia = (int32_t *) calloc(sizeof(int32_t), INITIAL_ID_POOL_SZ); - if (ia == NULL) { - ID_POOL_NOMEM - return; - } - time_t *t = (time_t *) calloc(sizeof(time_t), INITIAL_ID_POOL_SZ); - if (t == NULL) { - free(ia); - ID_POOL_NOMEM - return; - } - pool->id = ia; - pool->release = t; - pool->capacity = INITIAL_ID_POOL_SZ; - } else if (pool->len == pool->capacity) { - // grow slowly - n = pool->capacity + (pool->capacity >> 1); - // scrapetime for 1 session report takes probably ~ 160K CPU cycles - // to avoid complexity we assume a 2.6 GHz machine - double max = n * relay_servers_in_use * 16.0/260000; - // 0.8 s/report should be ok =~ 16K sessions =~ 256K metrics - TURN_LOG_LEVEL l = max < 0.8 ? TURN_LOG_LEVEL_INFO : TURN_LOG_LEVEL_WARNING; - int32_t *ia = (int32_t *) calloc(sizeof(int32_t), n); - if (ia == NULL) { - ID_POOL_NOMEM - return; - } - time_t *ta = (time_t *) calloc(sizeof(time_t), n); - if (ta == NULL) { - free(ia); - ID_POOL_NOMEM - return; - } - if (pool->start == 0) { - TURN_LOG_FUNC(l, "tid: %d usid: %d rsid: %d grew rsid pool from " - "%d to %d start: %d len: %d maxSid: %d\n", - server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, - pool->capacity, n, pool->start, pool->len, pool->max_id); - memcpy(ia, pool->id, pool->len * sizeof(int32_t)); - memcpy(ta, pool->release, pool->len * sizeof(time_t)); - } else { - m = pool->capacity - pool->start; - TURN_LOG_FUNC(l, "tid: %d usid: %d rsid: %d grew sid pool from " - "%d to %d start: %d len: %d/end: %d len: %d maxSid: %d\n", - server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, - pool->capacity,n, pool->start,m, 0, pool->len - m, pool->max_id); - memcpy(ia, &(pool->id[pool->start]), m * sizeof(int32_t)); - memcpy(ta, &(pool->release[pool->start]), m * sizeof(time_t)); - memcpy(&(ia[m]), pool->id, (pool->len - m) * sizeof(int32_t)); - memcpy(&(ta[m]), pool->release, (pool->len - m) * sizeof(time_t)); - pool->start = 0; - } - free(pool->id); - free(pool->release); - pool->id = ia; - pool->release = ta; - pool->capacity = n; - } - n = pool->start + pool->len; - if (n >= pool->capacity) - n -= pool->capacity; - pool->id[n] = ss->rsid; - pool->release[n] = server->ctime + *(server->sid_retain); - _TRACE(TURN_LOG_LEVEL_DEBUG, "tid: %d usid: %d rsid: %d idx: %d %d %ld\n", - server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, - n, pool->id[n], pool->release[n]); - (pool->len)++; - ss->rsid *= -1; - _TRACE(TURN_LOG_LEVEL_DEBUG, "tid: %d usid: %d rsid: %d recycle after: " - "capacity: %d start: %d len: %d maxId: %d\n", - server->id, (int) (ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, - pool->capacity, pool->start, pool->len, pool->max_id); + int n, m; + + if (!prom_rsids()) + return; + + turn_turnserver *server = (turn_turnserver *)(ss->server); + id_pool_t *pool = server->rsid_pool; + _TRACE(TURN_LOG_LEVEL_DEBUG, "tid: %d usid: %d returns sid %d%s\n", server->id, + (int)(ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, ss->rsid < 1 ? " INVALID" : ""); + + if (ss->rsid < 1) + return; + + _TRACE(TURN_LOG_LEVEL_DEBUG, + "tid: %d usid: %d rsid: %d recycle before: " + "capacity: %d start: %d len: %d maxId: %d\n", + server->id, (int)(ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, pool->capacity, pool->start, pool->len, + pool->max_id); + + if (pool->id == NULL) { + int32_t *ia = (int32_t *)calloc(sizeof(int32_t), INITIAL_ID_POOL_SZ); + if (ia == NULL) { + ID_POOL_NOMEM + return; + } + time_t *t = (time_t *)calloc(sizeof(time_t), INITIAL_ID_POOL_SZ); + if (t == NULL) { + free(ia); + ID_POOL_NOMEM + return; + } + pool->id = ia; + pool->release = t; + pool->capacity = INITIAL_ID_POOL_SZ; + } else if (pool->len == pool->capacity) { + // grow slowly + n = pool->capacity + (pool->capacity >> 1); + // scrapetime for 1 session report takes probably ~ 160K CPU cycles + // to avoid complexity we assume a 2.6 GHz machine + double max = n * relay_servers_in_use * 16.0 / 260000; + // 0.8 s/report should be ok =~ 16K sessions =~ 256K metrics + TURN_LOG_LEVEL l = max < 0.8 ? TURN_LOG_LEVEL_INFO : TURN_LOG_LEVEL_WARNING; + int32_t *ia = (int32_t *)calloc(sizeof(int32_t), n); + if (ia == NULL) { + ID_POOL_NOMEM + return; + } + time_t *ta = (time_t *)calloc(sizeof(time_t), n); + if (ta == NULL) { + free(ia); + ID_POOL_NOMEM + return; + } + if (pool->start == 0) { + TURN_LOG_FUNC(l, + "tid: %d usid: %d rsid: %d grew rsid pool from " + "%d to %d start: %d len: %d maxSid: %d\n", + server->id, (int)(ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, pool->capacity, n, pool->start, + pool->len, pool->max_id); + memcpy(ia, pool->id, pool->len * sizeof(int32_t)); + memcpy(ta, pool->release, pool->len * sizeof(time_t)); + } else { + m = pool->capacity - pool->start; + TURN_LOG_FUNC(l, + "tid: %d usid: %d rsid: %d grew sid pool from " + "%d to %d start: %d len: %d/end: %d len: %d maxSid: %d\n", + server->id, (int)(ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, pool->capacity, n, pool->start, m, 0, + pool->len - m, pool->max_id); + memcpy(ia, &(pool->id[pool->start]), m * sizeof(int32_t)); + memcpy(ta, &(pool->release[pool->start]), m * sizeof(time_t)); + memcpy(&(ia[m]), pool->id, (pool->len - m) * sizeof(int32_t)); + memcpy(&(ta[m]), pool->release, (pool->len - m) * sizeof(time_t)); + pool->start = 0; + } + free(pool->id); + free(pool->release); + pool->id = ia; + pool->release = ta; + pool->capacity = n; + } + n = pool->start + pool->len; + if (n >= pool->capacity) + n -= pool->capacity; + pool->id[n] = ss->rsid; + pool->release[n] = server->ctime + *(server->sid_retain); + _TRACE(TURN_LOG_LEVEL_DEBUG, "tid: %d usid: %d rsid: %d idx: %d %d %ld\n", server->id, + (int)(ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, n, pool->id[n], pool->release[n]); + (pool->len)++; + ss->rsid *= -1; + _TRACE(TURN_LOG_LEVEL_DEBUG, + "tid: %d usid: %d rsid: %d recycle after: " + "capacity: %d start: %d len: %d maxId: %d\n", + server->id, (int)(ss->id % TURN_SESSION_ID_FACTOR), ss->rsid, pool->capacity, pool->start, pool->len, + pool->max_id); #endif } void attach_samples(ts_ur_super_session *ss) { #if defined(TURN_NO_PROMETHEUS) - UNUSED_ARG(ss); + UNUSED_ARG(ss); #else - if (ss == NULL || prom_disabled() || ss->server == NULL) - return; - - if (ss->rsid < 0) - ss->rsid = -1; - uint32_t tid = ((turn_turnserver *) ss->server)->id; - - // no reflection in C, so ... - ss->sample_session_state = get_state_sample(tid, ss->rsid, ss->id, - ss->realm_options.name, (char *)ss->username); - if (ss->sample_session_state) - pms_set(ss->sample_session_state, SESSION_STATE_OPEN); - - ss->sample_rx_msgs = get_session_sample(METRIC_RX_MSGS, false, - tid, ss->rsid, ss->id); - ss->sample_tx_msgs = get_session_sample(METRIC_TX_MSGS, false, - tid, ss->rsid, ss->id); - ss->sample_rx_bytes = get_session_sample(METRIC_RX_BYTES, false, - tid, ss->rsid, ss->id); - ss->sample_tx_bytes = get_session_sample(METRIC_TX_BYTES, false, - tid, ss->rsid, ss->id); - - ss->sample_peer_rx_msgs = get_session_sample(METRIC_RX_MSGS, true, - tid, ss->rsid, ss->id); - ss->sample_peer_tx_msgs = get_session_sample(METRIC_TX_MSGS, true, - tid, ss->rsid, ss->id); - ss->sample_peer_rx_bytes = get_session_sample(METRIC_RX_BYTES, true, - tid, ss->rsid, ss->id); - ss->sample_peer_tx_bytes = get_session_sample(METRIC_TX_BYTES, true, - tid, ss->rsid, ss->id); - - ss->sample_lifetime = get_session_sample(METRIC_LIFETIME, false, - tid, ss->rsid, ss->id); - if (ss->sample_lifetime) - pms_set(ss->sample_lifetime, 0); - ss->sample_allocations_running = - get_session_sample(METRIC_ALLOCATIONS_RUNNING, false, - tid, ss->rsid, ss->id); - ss->sample_allocations_created = - get_session_sample(METRIC_ALLOCATIONS_CREATED, false, - tid, ss->rsid, ss->id); - ss->sample_stun_req = get_session_sample(METRIC_STUN_REQUEST, false, - tid, ss->rsid, ss->id); - ss->sample_stun_resp = get_session_sample(METRIC_STUN_RESPONSE, false, - tid, ss->rsid, ss->id); + if (ss == NULL || prom_disabled() || ss->server == NULL) + return; + + if (ss->rsid < 0) + ss->rsid = -1; + uint32_t tid = ((turn_turnserver *)ss->server)->id; + + // no reflection in C, so ... + ss->sample_session_state = get_state_sample(tid, ss->rsid, ss->id, ss->realm_options.name, (char *)ss->username); + if (ss->sample_session_state) + pms_set(ss->sample_session_state, SESSION_STATE_OPEN); + + ss->sample_rx_msgs = get_session_sample(METRIC_RX_MSGS, false, tid, ss->rsid, ss->id); + ss->sample_tx_msgs = get_session_sample(METRIC_TX_MSGS, false, tid, ss->rsid, ss->id); + ss->sample_rx_bytes = get_session_sample(METRIC_RX_BYTES, false, tid, ss->rsid, ss->id); + ss->sample_tx_bytes = get_session_sample(METRIC_TX_BYTES, false, tid, ss->rsid, ss->id); + + ss->sample_peer_rx_msgs = get_session_sample(METRIC_RX_MSGS, true, tid, ss->rsid, ss->id); + ss->sample_peer_tx_msgs = get_session_sample(METRIC_TX_MSGS, true, tid, ss->rsid, ss->id); + ss->sample_peer_rx_bytes = get_session_sample(METRIC_RX_BYTES, true, tid, ss->rsid, ss->id); + ss->sample_peer_tx_bytes = get_session_sample(METRIC_TX_BYTES, true, tid, ss->rsid, ss->id); + + ss->sample_lifetime = get_session_sample(METRIC_LIFETIME, false, tid, ss->rsid, ss->id); + if (ss->sample_lifetime) + pms_set(ss->sample_lifetime, 0); + ss->sample_allocations_running = get_session_sample(METRIC_ALLOCATIONS_RUNNING, false, tid, ss->rsid, ss->id); + ss->sample_allocations_created = get_session_sample(METRIC_ALLOCATIONS_CREATED, false, tid, ss->rsid, ss->id); + ss->sample_stun_req = get_session_sample(METRIC_STUN_REQUEST, false, tid, ss->rsid, ss->id); + ss->sample_stun_resp = get_session_sample(METRIC_STUN_RESPONSE, false, tid, ss->rsid, ss->id); #endif } void stun_report_binding(ts_ur_super_session *ss, STUN_PROMETHEUS_METRIC_TYPE type, int err_code) { #if !defined(TURN_NO_PROMETHEUS) - if (! ss->sample_stun_req) + if (!ss->sample_stun_req) return; switch (type) { @@ -3760,7 +3747,7 @@ void stun_report_binding(ts_ur_super_session *ss, STUN_PROMETHEUS_METRIC_TYPE ty pms_add(ss->sample_stun_resp, 1); break; case STUN_PROMETHEUS_METRIC_TYPE_ERROR: - prom_binding_error(((turn_turnserver *) ss->server)->id, ss->rsid, ss->id, err_code); + prom_binding_error(((turn_turnserver *)ss->server)->id, ss->rsid, ss->id, err_code); break; default: break; @@ -3783,19 +3770,20 @@ void turn_report_allocation_set(void *a, turn_time_t lifetime, int refresh) { if (server) { ioa_engine_handle e = turn_server_get_engine(server); if (e && e->verbose && ss->client_socket) { - SOCKET_TYPE socket_type = get_ioa_socket_type(ss->client_socket); - const char *socket_type_str = socket_type_name(socket_type); + SOCKET_TYPE socket_type = get_ioa_socket_type(ss->client_socket); + const char *socket_type_str = socket_type_name(socket_type); if (ss->client_socket->ssl) { - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, - "session %012llu.%d: %s, realm=<%s>, username=<%s>, lifetime=%lu, cipher=%s, method=%s, socket=%s\n", - (unsigned long long)ss->id, ss->rsid, status, (char *)ss->realm_options.name, (char *)ss->username, - (unsigned long)lifetime, SSL_get_cipher(ss->client_socket->ssl), - turn_get_ssl_method(ss->client_socket->ssl, "UNKNOWN"), - socket_type_str); + TURN_LOG_FUNC( + TURN_LOG_LEVEL_INFO, + "session %012llu.%d: %s, realm=<%s>, username=<%s>, lifetime=%lu, cipher=%s, method=%s, socket=%s\n", + (unsigned long long)ss->id, ss->rsid, status, (char *)ss->realm_options.name, (char *)ss->username, + (unsigned long)lifetime, SSL_get_cipher(ss->client_socket->ssl), + turn_get_ssl_method(ss->client_socket->ssl, "UNKNOWN"), socket_type_str); } else { - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %012llu.%d: %s, realm=<%s>, username=<%s>, lifetime=%lu, socket=%s\n", - (unsigned long long)ss->id, ss->rsid, status, (char *)ss->realm_options.name, (char *)ss->username, - (unsigned long)lifetime, socket_type_str); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, + "session %012llu.%d: %s, realm=<%s>, username=<%s>, lifetime=%lu, socket=%s\n", + (unsigned long long)ss->id, ss->rsid, status, (char *)ss->realm_options.name, + (char *)ss->username, (unsigned long)lifetime, socket_type_str); } } @@ -3804,18 +3792,16 @@ void turn_report_allocation_set(void *a, turn_time_t lifetime, int refresh) { if (!refresh) { pms_add(ss->sample_allocations_running, 1); pms_add(ss->sample_allocations_created, 1); - } + } if (ss->sample_session_state) { - pms_set(ss->sample_session_state, refresh - ? SESSION_STATE_REFRESH - : SESSION_STATE_ALLOCATED); + pms_set(ss->sample_session_state, refresh ? SESSION_STATE_REFRESH : SESSION_STATE_ALLOCATED); pms_set(ss->sample_lifetime, lifetime); } } #endif #if !defined(TURN_NO_HIREDIS) - if (e->rch) { + if (e->rch) { char key[1024]; if (ss->realm_options.name[0]) { snprintf(key, sizeof(key), "turn/realm/%s/user/%s/allocation/%012llu/status", ss->realm_options.name, @@ -3853,20 +3839,21 @@ void turn_report_allocation_delete(void *a, SOCKET_TYPE socket_type) { if (e && e->verbose) { const char *socket_type_str = socket_type_name(socket_type); TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %012llu.%d: delete: realm=<%s>, username=<%s>, socket=%s\n", - (unsigned long long)ss->id, ss->rsid, (char *)ss->realm_options.name, (char *)ss->username, socket_type_str); + (unsigned long long)ss->id, ss->rsid, (char *)ss->realm_options.name, (char *)ss->username, + socket_type_str); } #if !defined(TURN_NO_PROMETHEUS) - if (ss->sample_allocations_running) - pms_sub(ss->sample_allocations_running, 1); - if (ss->sample_session_state) { - pms_set(ss->sample_session_state, SESSION_STATE_DEALLOCATED); - pms_set(ss->sample_lifetime, -1); - } + if (ss->sample_allocations_running) + pms_sub(ss->sample_allocations_running, 1); + if (ss->sample_session_state) { + pms_set(ss->sample_session_state, SESSION_STATE_DEALLOCATED); + pms_set(ss->sample_lifetime, -1); + } #endif #if !defined(TURN_NO_HIREDIS) - if (e->rch) { + if (e->rch) { char key[1024]; if (ss->realm_options.name[0]) { snprintf(key, sizeof(key), "turn/realm/%s/user/%s/allocation/%012llu/status", ss->realm_options.name, diff --git a/src/apps/relay/prom_server.c b/src/apps/relay/prom_server.c index 0802ea474..829eb75df 100644 --- a/src/apps/relay/prom_server.c +++ b/src/apps/relay/prom_server.c @@ -1,8 +1,8 @@ #include -#include "prom_server.h" #include "mainrelay.h" #include "ns_turn_utils.h" +#include "prom_server.h" #if !defined(TURN_NO_PROMETHEUS) @@ -15,7 +15,7 @@ prom_counter_t *turn_rx_bytes = NULL; prom_counter_t *turn_tx_msgs = NULL; prom_counter_t *turn_tx_bytes = NULL; -prom_gauge_t *turn_lifetime = NULL; // -1 .. closed, otherwise lifetime [s] +prom_gauge_t *turn_lifetime = NULL; // -1 .. closed, otherwise lifetime [s] prom_gauge_t *turn_allocations = NULL; prom_gauge_t *turn_state = NULL; @@ -40,121 +40,107 @@ static const char **state_labels; static size_t state_label_count; static bool use_sid_labels = true; -static const char *session_labels[] = - { LABEL_TURNSERVER_ID, LABEL_SESSION_ID, LABEL_PEER }; +static const char *session_labels[] = {LABEL_TURNSERVER_ID, LABEL_SESSION_ID, LABEL_PEER}; static size_t session_label_count = 3; -static bool -init_state_labels(void) { - size_t n; - - if (turn_params.prom_usid && turn_params.prom_rsid) { - TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, "prom-sid takes precedence " - "over prom-usid. Metrics get tagged with dynamic session IDs to save" - "resources and hopefully keep your timeseries DB happy.\n"); - turn_params.prom_usid = 0; - } - if (!turn_params.prom_rsid && !turn_params.prom_usid) { - use_sid_labels = false; - if (!turn_params.prom_usernames) { - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "Neither prom-usid nor " - "prom-sid nor prom-usernames is given (makes your server safe " - "wrt. too many metrics attacks): session_state and lifetime " - "metrics get not enabled.\n"); - } - state_label_count = 0; // use as indicator for state n/a - return false; - } - - state_label_count = turn_params.prom_realm + turn_params.prom_usernames - + (use_sid_labels ? 2 : 1); - - state_labels = (const char **) - malloc(state_label_count * sizeof(const char *)); - if (state_labels == NULL) { - TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, - "Memory problem - metric collector disabled - init failed.\n"); - state_label_count = 0; - turn_params.prom = 0; - return false; - } - n = 0; - state_labels[n++] = LABEL_TURNSERVER_ID; - if (use_sid_labels) - state_labels[n++] = LABEL_SESSION_ID; - if (turn_params.prom_realm) - state_labels[n++] = LABEL_REALM; - if (turn_params.prom_usernames) - state_labels[n++] = LABEL_USER; - return true; +static bool init_state_labels(void) { + size_t n; + + if (turn_params.prom_usid && turn_params.prom_rsid) { + TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, "prom-sid takes precedence " + "over prom-usid. Metrics get tagged with dynamic session IDs to save" + "resources and hopefully keep your timeseries DB happy.\n"); + turn_params.prom_usid = 0; + } + if (!turn_params.prom_rsid && !turn_params.prom_usid) { + use_sid_labels = false; + if (!turn_params.prom_usernames) { + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "Neither prom-usid nor " + "prom-sid nor prom-usernames is given (makes your server safe " + "wrt. too many metrics attacks): session_state and lifetime " + "metrics get not enabled.\n"); + } + state_label_count = 0; // use as indicator for state n/a + return false; + } + + state_label_count = turn_params.prom_realm + turn_params.prom_usernames + (use_sid_labels ? 2 : 1); + + state_labels = (const char **)malloc(state_label_count * sizeof(const char *)); + if (state_labels == NULL) { + TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, "Memory problem - metric collector disabled - init failed.\n"); + state_label_count = 0; + turn_params.prom = 0; + return false; + } + n = 0; + state_labels[n++] = LABEL_TURNSERVER_ID; + if (use_sid_labels) + state_labels[n++] = LABEL_SESSION_ID; + if (turn_params.prom_realm) + state_labels[n++] = LABEL_REALM; + if (turn_params.prom_usernames) + state_labels[n++] = LABEL_USER; + return true; } -static void -init_metrics(void) { +static void init_metrics(void) { const char *str; - prom_metric_t *version = pcr_must_register_metric(prom_gauge_new( - "version","TURN server version.", 2,(const char *[]) {"name", "release"})); - prom_gauge_set(version, 1, (const char *[]) {"Coturn",TURN_SERVER_VERSION}); + prom_metric_t *version = pcr_must_register_metric( + prom_gauge_new("version", "TURN server version.", 2, (const char *[]){"name", "release"})); + prom_gauge_set(version, 1, (const char *[]){"Coturn", TURN_SERVER_VERSION}); if (init_state_labels()) { - turn_state = pcr_must_register_metric(prom_gauge_new( - "session_state", "The state of a client or peer session. " - "0 .. closed, 1 .. allocation deleted, 2 .. closing, 3 .. open, " - "4 .. allocation created, 5 .. allocation refresh seen.", - state_label_count, state_labels)); + turn_state = + pcr_must_register_metric(prom_gauge_new("session_state", + "The state of a client or peer session. " + "0 .. closed, 1 .. allocation deleted, 2 .. closing, 3 .. open, " + "4 .. allocation created, 5 .. allocation refresh seen.", + state_label_count, state_labels)); } if (!use_sid_labels) { session_labels[--session_label_count] = NULL; session_labels[session_label_count - 1] = LABEL_PEER; } else if (turn_params.prom_rsid) { - use_rsids = true; + use_rsids = true; } // Create TURN traffic counter metrics - turn_rx_msgs = pcr_must_register_metric(prom_counter_new( - "rx_msgs","Messages received from the turn client or peer.", - session_label_count, session_labels)); - turn_tx_msgs = pcr_must_register_metric(prom_counter_new( - "tx_msgs","Messages sent to the turn client or peer.", - session_label_count, session_labels)); - turn_rx_bytes = pcr_must_register_metric(prom_counter_new( - "rx_bytes","Bytes received from the turn client or peer.", - session_label_count, session_labels)); - turn_tx_bytes = pcr_must_register_metric(prom_counter_new( - "tx_bytes","Bytes sent to the turn client or peer.", - session_label_count, session_labels)); + turn_rx_msgs = pcr_must_register_metric(prom_counter_new("rx_msgs", "Messages received from the turn client or peer.", + session_label_count, session_labels)); + turn_tx_msgs = pcr_must_register_metric( + prom_counter_new("tx_msgs", "Messages sent to the turn client or peer.", session_label_count, session_labels)); + turn_rx_bytes = pcr_must_register_metric(prom_counter_new("rx_bytes", "Bytes received from the turn client or peer.", + session_label_count, session_labels)); + turn_tx_bytes = pcr_must_register_metric( + prom_counter_new("tx_bytes", "Bytes sent to the turn client or peer.", session_label_count, session_labels)); // Create total allocations number gauge metric // use peer as total label str = session_labels[session_label_count - 1]; session_labels[session_label_count - 1] = LABEL_ALLOCATIONS; - turn_allocations = pcr_must_register_metric(prom_gauge_new( - "allocations", "Current allocations", - session_label_count, session_labels)); + turn_allocations = pcr_must_register_metric( + prom_gauge_new("allocations", "Current allocations", session_label_count, session_labels)); session_labels[session_label_count - 1] = str; if (use_sid_labels) - turn_lifetime = pcr_must_register_metric(prom_gauge_new( - "lifetime", "The life time of a client's allocation.", - session_label_count - 1, session_labels)); + turn_lifetime = pcr_must_register_metric( + prom_gauge_new("lifetime", "The life time of a client's allocation.", session_label_count - 1, session_labels)); // Create STUN counters if (!turn_params.no_stun) { stun_binding_request = pcr_must_register_metric(prom_counter_new( - "bind_requests","Valid STUN Binding requests received.", - session_label_count - 1, session_labels)); - stun_binding_response = pcr_must_register_metric(prom_counter_new( - "bind_responses","STUN Binding responses sent.", - session_label_count - 1, session_labels)); + "bind_requests", "Valid STUN Binding requests received.", session_label_count - 1, session_labels)); + stun_binding_response = pcr_must_register_metric( + prom_counter_new("bind_responses", "STUN Binding responses sent.", session_label_count - 1, session_labels)); // use peer as error label str = session_labels[session_label_count - 1]; session_labels[session_label_count - 1] = LABEL_STUN_ERR; - stun_binding_error = pcr_must_register_metric(prom_counter_new( - "bind_errors","STUN Binding errors", - session_label_count, session_labels)); + stun_binding_error = pcr_must_register_metric( + prom_counter_new("bind_errors", "STUN Binding errors", session_label_count, session_labels)); session_labels[session_label_count - 1] = str; } @@ -162,18 +148,18 @@ init_metrics(void) { } void start_prometheus_server(void) { - PROM_INIT_FLAGS features = PROM_PROCESS|PROM_SCRAPETIME_ALL; + PROM_INIT_FLAGS features = PROM_PROCESS | PROM_SCRAPETIME_ALL; if (turn_params.prom == 0) { TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "Metric collector disabled, not started.\n"); return; } if (turn_params.prom_compact) - features |= PROM_COMPACT; + features |= PROM_COMPACT; if (pcr_init(features, "coturn_")) { - TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, "Metric collector disabled - init failed.\n"); - turn_params.prom = 0; - return; + TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, "Metric collector disabled - init failed.\n"); + turn_params.prom = 0; + return; } promhttp_set_active_collector_registry(NULL); @@ -181,24 +167,24 @@ void start_prometheus_server(void) { // some flags appeared first in microhttpd v0.9.53 unsigned int flags; #ifdef MHD_USE_AUTO - flags = MHD_USE_AUTO; - // EITHER -# ifdef MHD_USE_INTERNAL_POLLING_THREAD - flags |= MHD_USE_INTERNAL_POLLING_THREAD; // EPOLL if avail or POLL -# endif - /* OR + flags = MHD_USE_AUTO; + // EITHER +#ifdef MHD_USE_INTERNAL_POLLING_THREAD + flags |= MHD_USE_INTERNAL_POLLING_THREAD; // EPOLL if avail or POLL +#endif + /* OR # ifdef MHD_USE_THREAD_PER_CONNECTION - flags |= MHD_USE_THREAD_PER_CONNECTION; // implies POLL + flags |= MHD_USE_THREAD_PER_CONNECTION; // implies POLL # endif - */ + */ #else - flags = MHD_USE_POLL_INTERNALLY; // internal polling thread - /* OR - flags = MHD_USE_THREAD_PER_CONNECTION; // implies POLL - */ + flags = MHD_USE_POLL_INTERNALLY; // internal polling thread + /* OR + flags = MHD_USE_THREAD_PER_CONNECTION; // implies POLL + */ #endif #ifdef MHD_USE_DEBUG - flags |= MHD_USE_DEBUG; // same as MHD_USE_ERROR_LOG + flags |= MHD_USE_DEBUG; // same as MHD_USE_ERROR_LOG #endif struct MHD_Daemon *daemon = promhttp_start_daemon(flags, turn_params.prom_port, NULL, NULL); @@ -213,109 +199,97 @@ void start_prometheus_server(void) { return; } - //bool peer, int32_t tid, int32_t sid, uint64_t usid) -#define PREPARE_TID_SID_PEER_LABELS(DST, TID, SID, USID, PEER) \ - char tidstr[12]; \ - char sidstr[12]; \ - int n = 0; \ - \ - sprintf(tidstr, "%d", TID); \ - DST[n++] = tidstr; \ - if (use_sid_labels) { \ - if (turn_params.prom_rsid) { \ - sprintf(sidstr, "%d", SID); \ - } else if (turn_params.prom_usid) { \ - /* uint64_t l = USID/TURN_SESSION_ID_FACTOR; sprintf(tidstr,"%ld",l);*/ \ - /* sprintf(sidstr, "%lld", USID - (l * TURN_SESSION_ID_FACTOR)); */ \ - sprintf(sidstr, "%lld", USID - (TID * TURN_SESSION_ID_FACTOR)); \ - } \ - DST[n++] = sidstr; \ - } \ - DST[n++] = PEER ? "1" : "0"; \ - -pms_t * -get_state_sample(int32_t tid, int32_t sid, uint64_t usid, - char *realm, char *user) -{ - const char *vals[state_label_count + 1]; // + dummy for peer - pms_t *res = NULL; - - if (state_label_count == 0 || prom_disabled()) - return NULL; - - PREPARE_TID_SID_PEER_LABELS(vals, tid, sid, usid, false); - if (turn_params.prom_realm) - vals[state_label_count-2] = realm; - if (turn_params.prom_usernames) - vals[state_label_count-1] = user; - - res = pms_from_labels(turn_state, vals); - return res; +// bool peer, int32_t tid, int32_t sid, uint64_t usid) +#define PREPARE_TID_SID_PEER_LABELS(DST, TID, SID, USID, PEER) \ + char tidstr[12]; \ + char sidstr[12]; \ + int n = 0; \ + \ + sprintf(tidstr, "%d", TID); \ + DST[n++] = tidstr; \ + if (use_sid_labels) { \ + if (turn_params.prom_rsid) { \ + sprintf(sidstr, "%d", SID); \ + } else if (turn_params.prom_usid) { \ + /* uint64_t l = USID/TURN_SESSION_ID_FACTOR; sprintf(tidstr,"%ld",l);*/ \ + /* sprintf(sidstr, "%lld", USID - (l * TURN_SESSION_ID_FACTOR)); */ \ + sprintf(sidstr, "%lld", USID - (TID * TURN_SESSION_ID_FACTOR)); \ + } \ + DST[n++] = sidstr; \ + } \ + DST[n++] = PEER ? "1" : "0"; + +pms_t *get_state_sample(int32_t tid, int32_t sid, uint64_t usid, char *realm, char *user) { + const char *vals[state_label_count + 1]; // + dummy for peer + pms_t *res = NULL; + + if (state_label_count == 0 || prom_disabled()) + return NULL; + + PREPARE_TID_SID_PEER_LABELS(vals, tid, sid, usid, false); + if (turn_params.prom_realm) + vals[state_label_count - 2] = realm; + if (turn_params.prom_usernames) + vals[state_label_count - 1] = user; + + res = pms_from_labels(turn_state, vals); + return res; } -pms_t * -get_session_sample(session_metric_t type, bool peer, - int32_t tid, int32_t sid, uint64_t usid) -{ - const char *vals[session_label_count]; - - if (prom_disabled()) - return NULL; - - PREPARE_TID_SID_PEER_LABELS(vals, tid, sid, usid, peer); - - switch (type) { - case METRIC_RX_MSGS: - return pms_from_labels(turn_rx_msgs, vals); - case METRIC_TX_MSGS: - return pms_from_labels(turn_tx_msgs, vals); - case METRIC_RX_BYTES: - return pms_from_labels(turn_rx_bytes, vals); - case METRIC_TX_BYTES: - return pms_from_labels(turn_tx_bytes, vals); - case METRIC_LIFETIME: - return turn_lifetime ? pms_from_labels(turn_lifetime, vals) : NULL; - case METRIC_ALLOCATIONS_RUNNING: - vals[session_label_count-1] = "0"; - return pms_from_labels(turn_allocations, vals); - case METRIC_ALLOCATIONS_CREATED: - vals[session_label_count-1] = "1"; - return pms_from_labels(turn_allocations, vals); - case METRIC_STUN_REQUEST: - vals[session_label_count-1] = NULL; - return stun_binding_request - ? pms_from_labels(stun_binding_request, vals) - : NULL; - case METRIC_STUN_RESPONSE: - vals[session_label_count-1] = NULL; - return stun_binding_response - ? pms_from_labels(stun_binding_response, vals) - : NULL; - case METRIC_STUN_ERROR: - vals[session_label_count-1] = NULL; - // we do not know all errors, don't want to maintain the list. - // So on error the little bit slower way via - // prom_binding_error(...) and the metric will be used. - break; - default: TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, - "Session metric %d is not yet supported.\n", type); - } - return NULL; +pms_t *get_session_sample(session_metric_t type, bool peer, int32_t tid, int32_t sid, uint64_t usid) { + const char *vals[session_label_count]; + + if (prom_disabled()) + return NULL; + + PREPARE_TID_SID_PEER_LABELS(vals, tid, sid, usid, peer); + + switch (type) { + case METRIC_RX_MSGS: + return pms_from_labels(turn_rx_msgs, vals); + case METRIC_TX_MSGS: + return pms_from_labels(turn_tx_msgs, vals); + case METRIC_RX_BYTES: + return pms_from_labels(turn_rx_bytes, vals); + case METRIC_TX_BYTES: + return pms_from_labels(turn_tx_bytes, vals); + case METRIC_LIFETIME: + return turn_lifetime ? pms_from_labels(turn_lifetime, vals) : NULL; + case METRIC_ALLOCATIONS_RUNNING: + vals[session_label_count - 1] = "0"; + return pms_from_labels(turn_allocations, vals); + case METRIC_ALLOCATIONS_CREATED: + vals[session_label_count - 1] = "1"; + return pms_from_labels(turn_allocations, vals); + case METRIC_STUN_REQUEST: + vals[session_label_count - 1] = NULL; + return stun_binding_request ? pms_from_labels(stun_binding_request, vals) : NULL; + case METRIC_STUN_RESPONSE: + vals[session_label_count - 1] = NULL; + return stun_binding_response ? pms_from_labels(stun_binding_response, vals) : NULL; + case METRIC_STUN_ERROR: + vals[session_label_count - 1] = NULL; + // we do not know all errors, don't want to maintain the list. + // So on error the little bit slower way via + // prom_binding_error(...) and the metric will be used. + break; + default: + TURN_LOG_FUNC(TURN_LOG_LEVEL_WARNING, "Session metric %d is not yet supported.\n", type); + } + return NULL; } -void prom_binding_error(int32_t tid, int32_t sid, uint64_t usid, - int err) -{ - const char *vals[session_label_count]; - char buf[12]; +void prom_binding_error(int32_t tid, int32_t sid, uint64_t usid, int err) { + const char *vals[session_label_count]; + char buf[12]; - if (!stun_binding_error) - return; + if (!stun_binding_error) + return; - PREPARE_TID_SID_PEER_LABELS(vals, tid, sid, usid, false); - sprintf(buf, "%d", err); - vals[session_label_count - 1] = buf; - prom_counter_add(stun_binding_error, 1, vals); + PREPARE_TID_SID_PEER_LABELS(vals, tid, sid, usid, false); + sprintf(buf, "%d", err); + vals[session_label_count - 1] = buf; + prom_counter_add(stun_binding_error, 1, vals); } #else diff --git a/src/apps/relay/prom_server.h b/src/apps/relay/prom_server.h index f30f806a8..7170ddcc3 100644 --- a/src/apps/relay/prom_server.h +++ b/src/apps/relay/prom_server.h @@ -35,20 +35,19 @@ extern prom_gauge_t *turn_lifetime; extern prom_gauge_t *turn_allocations; typedef enum { - METRIC_RX_MSGS, - METRIC_TX_MSGS, - METRIC_RX_BYTES, - METRIC_TX_BYTES, - METRIC_LIFETIME, - METRIC_ALLOCATIONS_RUNNING, - METRIC_ALLOCATIONS_CREATED, - METRIC_STUN_REQUEST, - METRIC_STUN_RESPONSE, - METRIC_STUN_ERROR, - METRIC_MAX + METRIC_RX_MSGS, + METRIC_TX_MSGS, + METRIC_RX_BYTES, + METRIC_TX_BYTES, + METRIC_LIFETIME, + METRIC_ALLOCATIONS_RUNNING, + METRIC_ALLOCATIONS_CREATED, + METRIC_STUN_REQUEST, + METRIC_STUN_RESPONSE, + METRIC_STUN_ERROR, + METRIC_MAX } session_metric_t; - #ifdef __cplusplus extern "C" { #endif @@ -56,7 +55,7 @@ extern "C" { void start_prometheus_server(void); pms_t *get_state_sample(int32_t tid, int32_t sid, uint64_t usid, char *realm, char *user); -pms_t * get_session_sample(session_metric_t type, bool peer, int32_t tid, int32_t sid, uint64_t usid); +pms_t *get_session_sample(session_metric_t type, bool peer, int32_t tid, int32_t sid, uint64_t usid); void prom_binding_error(int32_t tid, int32_t sid, uint64_t usid, int error); @@ -69,7 +68,6 @@ void start_prometheus_server(void); bool prom_disabled(void); bool prom_rsids(void); - #ifdef __cplusplus } #endif /* __clplusplus */ diff --git a/src/apps/relay/turn_admin_server.c b/src/apps/relay/turn_admin_server.c index a6c072960..724de15d3 100644 --- a/src/apps/relay/turn_admin_server.c +++ b/src/apps/relay/turn_admin_server.c @@ -476,7 +476,7 @@ static int print_session(ur_map_key_type key, ur_map_value_type value, void *arg if (cs->f || (unsigned long)csarg->counter < (unsigned long)cli_max_output_sessions) { myprintf(cs, "\n"); myprintf(cs, " %lu) id=%012llu, tid: %d, rsid: %d, user <%s>:\n", (unsigned long)(csarg->counter + 1), - (unsigned long long)tsi->id, (int) (tsi->id % TURN_SESSION_ID_FACTOR), tsi->rsid, tsi->username); + (unsigned long long)tsi->id, (int)(tsi->id % TURN_SESSION_ID_FACTOR), tsi->rsid, tsi->username); if (tsi->realm[0]) myprintf(cs, " realm: %s\n", tsi->realm); if (tsi->origin[0]) diff --git a/src/server/ns_turn_allocation.c b/src/server/ns_turn_allocation.c index b657faafd..2a5df3b1e 100644 --- a/src/server/ns_turn_allocation.c +++ b/src/server/ns_turn_allocation.c @@ -171,7 +171,8 @@ void turn_permission_clean(turn_permission_info *tinfo) { if (tinfo->verbose) { char s[257] = "\0"; addr_to_string(&(tinfo->addr), (uint8_t *)s); - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %012llu.%d: peer %s deleted\n", tinfo->session_id, tinfo->session_rid, s); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %012llu.%d: peer %s deleted\n", tinfo->session_id, tinfo->session_rid, + s); } if (!(tinfo->lifetime_ev)) { diff --git a/src/server/ns_turn_ioalib.h b/src/server/ns_turn_ioalib.h index cf3d475f5..84e5e43b3 100644 --- a/src/server/ns_turn_ioalib.h +++ b/src/server/ns_turn_ioalib.h @@ -210,13 +210,13 @@ void acquire_recyclable_session_id(ts_ur_super_session *ss); void release_recyclable_session_id(ts_ur_super_session *ss); typedef enum { - SESSION_STATE_CLOSED, - SESSION_STATE_DEALLOCATED, - SESSION_STATE_CLOSING, - SESSION_STATE_OPEN, - SESSION_STATE_ALLOCATED, - SESSION_STATE_REFRESH, - SESSION_STATE_MAX + SESSION_STATE_CLOSED, + SESSION_STATE_DEALLOCATED, + SESSION_STATE_CLOSING, + SESSION_STATE_OPEN, + SESSION_STATE_ALLOCATED, + SESSION_STATE_REFRESH, + SESSION_STATE_MAX } session_state_t; enum _STUN_PROMETHEUS_METRIC_TYPE { diff --git a/src/server/ns_turn_server.c b/src/server/ns_turn_server.c index 74ff8eb36..aa75ae3e6 100644 --- a/src/server/ns_turn_server.c +++ b/src/server/ns_turn_server.c @@ -87,21 +87,23 @@ static inline void log_method(ts_ur_super_session *ss, const char *method, int e if (ss->origin[0]) { TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %012llu.%d: origin <%s> realm <%s> user <%s>: incoming packet %s processed, success\n", - (unsigned long long)(ss->id), ss->rsid, (const char *)(ss->origin), (const char *)(ss->realm_options.name), - (const char *)(ss->username), method); + (unsigned long long)(ss->id), ss->rsid, (const char *)(ss->origin), + (const char *)(ss->realm_options.name), (const char *)(ss->username), method); } else { - TURN_LOG_FUNC( - TURN_LOG_LEVEL_INFO, "session %012llu.%d: realm <%s> user <%s>: incoming packet %s processed, success\n", - (unsigned long long)(ss->id), ss->rsid, (const char *)(ss->realm_options.name), (const char *)(ss->username), method); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, + "session %012llu.%d: realm <%s> user <%s>: incoming packet %s processed, success\n", + (unsigned long long)(ss->id), ss->rsid, (const char *)(ss->realm_options.name), + (const char *)(ss->username), method); } } else { if (!reason) reason = get_default_reason(err_code); if (ss->origin[0]) { - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, - "session %012llu.%d: origin <%s> realm <%s> user <%s>: incoming packet %s processed, error %d: %s\n", - (unsigned long long)(ss->id), ss->rsid, (const char *)(ss->origin), (const char *)(ss->realm_options.name), - (const char *)(ss->username), method, err_code, reason); + TURN_LOG_FUNC( + TURN_LOG_LEVEL_INFO, + "session %012llu.%d: origin <%s> realm <%s> user <%s>: incoming packet %s processed, error %d: %s\n", + (unsigned long long)(ss->id), ss->rsid, (const char *)(ss->origin), (const char *)(ss->realm_options.name), + (const char *)(ss->username), method, err_code, reason); } else { TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "session %012llu.%d: realm <%s> user <%s>: incoming packet %s processed, error %d: %s\n", @@ -749,7 +751,8 @@ void turn_cancel_session(turn_turnserver *server, turnsession_id sid) { if (server) { ts_ur_super_session *ts = get_session_from_map(server, sid); if (ts) { - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "Session %012llu.%d to be forcefully canceled\n", (unsigned long long)sid, ts->rsid); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "Session %012llu.%d to be forcefully canceled\n", (unsigned long long)sid, + ts->rsid); shutdown_client_connection(server, ts, 0, "Forceful shutdown"); } } @@ -1854,10 +1857,10 @@ static void tcp_deliver_delayed_buffer(unsent_buffer *ub, ioa_socket_handle s, t ++(ss->sent_packets); ss->sent_bytes += bytes; #if !defined(TURN_NO_PROMETHEUS) - if (ss->sample_tx_msgs) { - pms_add(ss->sample_tx_msgs, 1); - pms_add(ss->sample_tx_bytes, bytes); - } + if (ss->sample_tx_msgs) { + pms_add(ss->sample_tx_msgs, 1); + pms_add(ss->sample_tx_bytes, bytes); + } #endif turn_report_session_usage(ss, 0); } @@ -1911,9 +1914,9 @@ static void tcp_peer_input_handler(ioa_socket_handle s, int event_type, ioa_net_ ss->sent_bytes += bytes; #if !defined(TURN_NO_PROMETHEUS) if (ss->sample_tx_msgs) { - pms_add(ss->sample_tx_msgs, 1); - pms_add(ss->sample_tx_bytes, bytes); - } + pms_add(ss->sample_tx_msgs, 1); + pms_add(ss->sample_tx_bytes, bytes); + } #endif } @@ -1968,10 +1971,10 @@ static void tcp_client_input_handler_rfc6062data(ioa_socket_handle s, int event_ ++(ss->peer_sent_packets); ss->peer_sent_bytes += bytes; #if !defined(TURN_NO_PROMETHEUS) - if (ss->sample_peer_tx_msgs) { - pms_add(ss->sample_peer_tx_msgs, 1); - pms_add(ss->sample_peer_tx_bytes, bytes); - } + if (ss->sample_peer_tx_msgs) { + pms_add(ss->sample_peer_tx_msgs, 1); + pms_add(ss->sample_peer_tx_bytes, bytes); + } #endif } @@ -3000,10 +3003,10 @@ static int handle_turn_send(turn_turnserver *server, ts_ur_super_session *ss, in ++(ss->peer_sent_packets); ss->peer_sent_bytes += len; #if !defined(TURN_NO_PROMETHEUS) - if (ss->sample_peer_tx_msgs) { - pms_add(ss->sample_peer_tx_msgs, 1); - pms_add(ss->sample_peer_tx_bytes, len); - } + if (ss->sample_peer_tx_msgs) { + pms_add(ss->sample_peer_tx_msgs, 1); + pms_add(ss->sample_peer_tx_bytes, len); + } #endif turn_report_session_usage(ss, 0); } @@ -3953,7 +3956,7 @@ static int handle_old_stun_command(turn_turnserver *server, ts_ur_super_session } send_turn_message_to(server, nbh, &response_origin, &response_destination); - stun_report_binding(ss, STUN_PROMETHEUS_METRIC_TYPE_RESPONSE, 0); + stun_report_binding(ss, STUN_PROMETHEUS_METRIC_TYPE_RESPONSE, 0); no_response = 1; } } @@ -4052,14 +4055,14 @@ static int write_to_peerchannel(ts_ur_super_session *ss, uint16_t chnum, ioa_net in_buffer->recv_ttl - 1, in_buffer->recv_tos, &skip); if (!skip && rc > -1) { - uint32_t bytes = (uint32_t)ioa_network_buffer_get_size(in_buffer->nbh); + uint32_t bytes = (uint32_t)ioa_network_buffer_get_size(in_buffer->nbh); ++(ss->peer_sent_packets); ss->peer_sent_bytes += bytes; #if !defined(TURN_NO_PROMETHEUS) - if (ss->sample_peer_tx_msgs) { - pms_add(ss->sample_peer_tx_msgs, 1); - pms_add(ss->sample_peer_tx_bytes, bytes); - } + if (ss->sample_peer_tx_msgs) { + pms_add(ss->sample_peer_tx_msgs, 1); + pms_add(ss->sample_peer_tx_bytes, bytes); + } #endif turn_report_session_usage(ss, 0); } @@ -4106,8 +4109,8 @@ int shutdown_client_connection(turn_turnserver *server, ts_ur_super_session *ss, TURN_LOG_FUNC( TURN_LOG_LEVEL_INFO, "session %012llu.%d: closed (1st stage), user <%s> realm <%s> origin <%s>, local %s, remote %s, reason: %s\n", - (unsigned long long)(ss->id), ss->rsid, (char *)ss->username, (char *)ss->realm_options.name, (char *)ss->origin, - sladdr, sraddr, reason); + (unsigned long long)(ss->id), ss->rsid, (char *)ss->username, (char *)ss->realm_options.name, + (char *)ss->origin, sladdr, sraddr, reason); } IOA_CLOSE_SOCKET(ss->client_socket); @@ -4140,8 +4143,8 @@ int shutdown_client_connection(turn_turnserver *server, ts_ur_super_session *ss, TURN_LOG_FUNC( TURN_LOG_LEVEL_INFO, "session %012llu.%d: closed (2nd stage), user <%s> realm <%s> origin <%s>, local %s, remote %s, reason: %s\n", - (unsigned long long)(ss->id), ss->rsid, (char *)ss->username, (char *)ss->realm_options.name, (char *)ss->origin, sladdr, - sraddr, reason); + (unsigned long long)(ss->id), ss->rsid, (char *)ss->username, (char *)ss->realm_options.name, + (char *)ss->origin, sladdr, sraddr, reason); } IOA_CLOSE_SOCKET(ss->client_socket); @@ -4234,10 +4237,10 @@ static int write_client_connection(turn_turnserver *server, ts_ur_super_session ++(ss->sent_packets); ss->sent_bytes += bytes; #if !defined(TURN_NO_PROMETHEUS) - if (ss->sample_tx_msgs) { - pms_add(ss->sample_tx_msgs, 1); - pms_add(ss->sample_tx_bytes, bytes); - } + if (ss->sample_tx_msgs) { + pms_add(ss->sample_tx_msgs, 1); + pms_add(ss->sample_tx_bytes, bytes); + } #endif turn_report_session_usage(ss, 0); } @@ -4694,9 +4697,8 @@ int open_client_connection_session(turn_turnserver *server, struct socket_messag char sladdr[129] = "\0"; addr_to_string(get_remote_addr_from_ioa_socket(ss->client_socket), (uint8_t *)sraddr); addr_to_string(get_local_addr_from_ioa_socket(ss->client_socket), (uint8_t *)sladdr); - TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, - "new session %012llu tid: %d rsid: %d: local %s, remote %s\n", - (unsigned long long)(ss->id), server->id, ss->rsid, sladdr, sraddr); + TURN_LOG_FUNC(TURN_LOG_LEVEL_INFO, "new session %012llu tid: %d rsid: %d: local %s, remote %s\n", + (unsigned long long)(ss->id), server->id, ss->rsid, sladdr, sraddr); } #endif @@ -4771,7 +4773,7 @@ static void peer_input_handler(ioa_socket_handle s, int event_type, ioa_net_data ++(ss->peer_received_packets); ss->peer_received_bytes += ilen; #if !defined(TURN_NO_PROMETHEUS) - if (ss->sample_peer_rx_msgs) { + if (ss->sample_peer_rx_msgs) { pms_add(ss->sample_peer_rx_msgs, 1); pms_add(ss->sample_peer_rx_bytes, ilen); } @@ -4888,8 +4890,7 @@ void init_turn_server(turn_turnserver *server, turnserver_id id, int verbose, io vintp no_udp_relay, vintp stale_nonce, vintp max_allocate_lifetime, vintp channel_lifetime, vintp permission_lifetime, vintp stun_only, vintp no_stun, vintp no_software_attribute, #if !defined(TURN_NO_PROMETHEUS) - vintp retain, - vintp log_ip, + vintp retain, vintp log_ip, #endif vintp web_admin_listen_on_workers, turn_server_addrs_list_t *alternate_servers_list, turn_server_addrs_list_t *tls_alternate_servers_list, turn_server_addrs_list_t *aux_servers_list, @@ -4913,7 +4914,7 @@ void init_turn_server(turn_turnserver *server, turnserver_id id, int verbose, io server->sessions_map = ur_map_create(); server->tcp_relay_connections = ur_map_create(); #if !defined(TURN_NO_PROMETHEUS) - server->rsid_pool = (id_pool_t *) calloc(sizeof(id_pool_t), 1); + server->rsid_pool = (id_pool_t *)calloc(sizeof(id_pool_t), 1); server->sid_retain = retain; server->log_ip = log_ip; #endif diff --git a/src/server/ns_turn_server.h b/src/server/ns_turn_server.h index 7f5c88b7b..564e97431 100644 --- a/src/server/ns_turn_server.h +++ b/src/server/ns_turn_server.h @@ -96,13 +96,13 @@ typedef enum { #if !defined(TURN_NO_PROMETHEUS) /** pool used to maintain recyclable session IDs for a single turn server. */ typedef struct { - int32_t *id; // array with the IDs of closed sessions - time_t *release; // time from when the ID can be re-used again. - int32_t capacity; // number of IDs, which fit into the id and release. - int32_t len; // number of recyclable IDs in id and timestamps in release; - int32_t start; // the index where to start looking for a free ID - int32_t max_id; // the highest ID used so far - int32_t recycled; // how many times an ID of the pool got re-used + int32_t *id; // array with the IDs of closed sessions + time_t *release; // time from when the ID can be re-used again. + int32_t capacity; // number of IDs, which fit into the id and release. + int32_t len; // number of recyclable IDs in id and timestamps in release; + int32_t start; // the index where to start looking for a free ID + int32_t max_id; // the highest ID used so far + int32_t recycled; // how many times an ID of the pool got re-used } id_pool_t; #endif @@ -132,9 +132,9 @@ struct _turn_turnserver { turnsession_id session_id_counter; ur_map *sessions_map; #if !defined(TURN_NO_PROMETHEUS) - id_pool_t *rsid_pool; // the pool for recyclable session IDs - vintp sid_retain; // retain IDs of closed sessions at least N seconds before reuse - vintp log_ip; // log the endpoint IPs on each session setup + id_pool_t *rsid_pool; // the pool for recyclable session IDs + vintp sid_retain; // retain IDs of closed sessions at least N seconds before reuse + vintp log_ip; // log the endpoint IPs on each session setup #endif turn_time_t ctime; @@ -232,8 +232,7 @@ void init_turn_server( vintp check_origin, vintp no_tcp_relay, vintp no_udp_relay, vintp stale_nonce, vintp max_allocate_lifetime, vintp channel_lifetime, vintp permission_lifetime, vintp stun_only, vintp no_stun, vintp no_software_attribute, #if !defined(TURN_NO_PROMETHEUS) - vintp retain, - vintp log_ip, + vintp retain, vintp log_ip, #endif vintp web_admin_listen_on_workers, turn_server_addrs_list_t *alternate_servers_list, turn_server_addrs_list_t *tls_alternate_servers_list, turn_server_addrs_list_t *aux_servers_list, diff --git a/src/server/ns_turn_session.h b/src/server/ns_turn_session.h index b65735c33..8547e2c0c 100644 --- a/src/server/ns_turn_session.h +++ b/src/server/ns_turn_session.h @@ -93,9 +93,9 @@ struct _ts_ur_super_session { int origin_set; char origin[STUN_MAX_ORIGIN_SIZE + 1]; /* Stats */ - int32_t rsid; // recyclable ID (*server instance scope). Actually for metrics, - // but would clutter logging code, if one would always need to - // check for TURN_NO_PROMETHEUS ... + int32_t rsid; // recyclable ID (*server instance scope). Actually for metrics, + // but would clutter logging code, if one would always need to + // check for TURN_NO_PROMETHEUS ... #if !defined(TURN_NO_PROMETHEUS) // to avoid metric sample lookups on each counter/gauge op, we store and use // the reference to the samples directly. The sample ops are just atomic From 6980c5b15148d90d5364c1c9ee3b25b7d59835fd Mon Sep 17 00:00:00 2001 From: Jens Elkner Date: Fri, 8 Dec 2023 15:35:54 +0100 Subject: [PATCH 11/12] GHA: ditch the s390x bloat - never succeeds Actually the s390x bloat never succeed and prevents a priori all PRs from being accepted. Never heard, that anybody is running coturn on a non-{Linux,Solaris,BSD} platform. Having tests for all possible OS/arches is one thing, actually testing against all is a completely different thing - not a green at all and wastes a lot of resources. So IMHO the test matrix should be revised to reflect, what is really needed. --- .github/workflows/docker.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 25d5e15d3..1b0bb018e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -31,7 +31,6 @@ jobs: - arm64v8 - i386 - ppc64le - - s390x runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -94,7 +93,6 @@ jobs: - arm64v8 - i386 - ppc64le - - s390x runs-on: ${{ (matrix.dist == 'alpine' && matrix.arch == 's390x' && 'macos') || 'ubuntu' }}-latest steps: @@ -203,8 +201,7 @@ jobs: arm32v7 \ arm64v8 \ i386 \ - ppc64le \ - s390x + ppc64le do make docker.untar \ from-file=.cache/${{ matrix.dist }}-$arch-${{ github.run_number }}/image.tar @@ -225,8 +222,7 @@ jobs: ${{ steps.docker.outputs.tag }}-arm32v7 ${{ steps.docker.outputs.tag }}-arm64v8 ${{ steps.docker.outputs.tag }}-i386 - ${{ steps.docker.outputs.tag }}-ppc64le - ${{ steps.docker.outputs.tag }}-s390x' + ${{ steps.docker.outputs.tag }}-ppc64le' tags=${{ (!startsWith(github.ref, 'refs/tags/') && steps.docker.outputs.tag) || '' }} From ed41ff268873be4c92a993d2f0384728f7962ff9 Mon Sep 17 00:00:00 2001 From: Jens Elkner Date: Tue, 6 Feb 2024 05:29:02 +0100 Subject: [PATCH 12/12] fix missing unit in comment --- src/server/ns_turn_server.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/ns_turn_server.h b/src/server/ns_turn_server.h index 564e97431..f44c7cb60 100644 --- a/src/server/ns_turn_server.h +++ b/src/server/ns_turn_server.h @@ -40,7 +40,7 @@ extern "C" { //////////// defines ////////////// -// Even if a single threaded app handles on average about 1 session/s +// Even if a single threaded app handles on average about 1k session/s // this means, the counter lasts ~ 31.7 years before it wraps to 0 and even // than it is unlikely, that a session last that long and a collision occures. #define TURN_SESSION_ID_FACTOR (1000000000LL)