From 48f0e2d5adacda9a7f85e8abc9c96e18c97b8715 Mon Sep 17 00:00:00 2001 From: Tobias Hintze Date: Tue, 3 Oct 2023 22:00:05 +0200 Subject: [PATCH 1/7] add missing newlines (\n) for TUNE Help strings for TUNE command were lacking the trailing newline character. This causes the FLUSH syntax printed in the same line as TUNE's. --- src/rrd_daemon.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rrd_daemon.c b/src/rrd_daemon.c index 0bc296bbc..4e70d153e 100644 --- a/src/rrd_daemon.c +++ b/src/rrd_daemon.c @@ -2883,8 +2883,8 @@ static command_t list_of_commands[] = { /* {{{ */ "TUNE", handle_request_tune, CMD_CONTEXT_CLIENT, - "TUNE [options]", - "Tunes the given file, takes the parameters as defined in rrdtool"}, + "TUNE [options]\n", + "Tunes the given file, takes the parameters as defined in rrdtool.\n"}, { "FLUSH", handle_request_flush, From 890301008a57e68b826777eadfcf086757467e07 Mon Sep 17 00:00:00 2001 From: Tobias Hintze Date: Tue, 10 Oct 2023 23:03:09 +0200 Subject: [PATCH 2/7] fix sendall edge cases The `sendall` func is implemented as a loop over multiple send() calls until all bytes are sent. Previously on subsequent calls the incorrect buffer was used for sending. This change also forbids `allow_retry` when at least one byte was successfully sent already. This would otherwise mess up the protocol, when retrying from the start. --- src/rrd_client.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rrd_client.c b/src/rrd_client.c index 51ba91d66..c31812b4f 100644 --- a/src/rrd_client.c +++ b/src/rrd_client.c @@ -653,10 +653,11 @@ static int sendall( return -1; while (ret != -1 && len > 0) { - ret = send(client->sd, msg, len, 0); + ret = send(client->sd, bufp, len, 0); if (ret > 0) { bufp += ret; len -= ret; + allow_retry = 0; // partial read forbids retry } } From bdd16db212ea07981e7c3f1aab1ba40ff4093cb2 Mon Sep 17 00:00:00 2001 From: Tobias Hintze Date: Tue, 3 Oct 2023 23:37:49 +0200 Subject: [PATCH 3/7] rrcached: implement DUMP --- CHANGES | 1 + src/rrd_client.c | 135 ++++++++++++++++++++++++++++++++++++++++++++++- src/rrd_client.h | 6 +++ src/rrd_daemon.c | 87 +++++++++++++++++++++++++++++- src/rrd_dump.c | 47 +++++++++++++---- 5 files changed, 263 insertions(+), 13 deletions(-) diff --git a/CHANGES b/CHANGES index e8d3239df..1b6f4114f 100644 --- a/CHANGES +++ b/CHANGES @@ -39,6 +39,7 @@ Features rrd_updatex_r(filename, tmplt, RRD_FLAGS_LOCKING_MODE_BLOCK, ...); +* Add (remote) dump support to rrdcached RRDtool 1.8.0 - 2022-03-13 ========================== diff --git a/src/rrd_client.c b/src/rrd_client.c index c31812b4f..cb6fa8c39 100644 --- a/src/rrd_client.c +++ b/src/rrd_client.c @@ -2060,6 +2060,125 @@ int rrdc_fetch( return status; } /* }}} int rrdc_fetch */ +int rrd_client_dump( + rrd_client_t *client, + const char *filename, /* {{{ */ + const char *opt_header, + rrd_output_callback_t output_cb, + void *cb_userdata) +{ + char buffer[RRD_CMD_MAX]; + char *buffer_ptr; + size_t buffer_free; + size_t buffer_size; + int status; + char *file_path; + char resp_buffer[256]; + + if (client == NULL) return -1; + if (filename == NULL) { + rrd_set_error("rrdc_dump: no input filename specified"); + return -1; + } + + memset(buffer, 0, sizeof(buffer)); + buffer_ptr = &buffer[0]; + buffer_free = sizeof(buffer); + + status = buffer_add_string("dump", &buffer_ptr, &buffer_free); + if (status != 0) { + rrd_set_error("rrdc_dump: out of memory"); + return -1; + } + + file_path = get_path(client, filename); + if (file_path == NULL) { + return -1; + } + + status = buffer_add_string(file_path, &buffer_ptr, &buffer_free); + free(file_path); + if (status != 0) { + rrd_set_error("rrdc_dump: out of memory"); + return -1; + } + + if (opt_header) { + status = buffer_add_string(opt_header, &buffer_ptr, &buffer_free); + if (status != 0) { + rrd_set_error("rrdc_dump: out of memory"); + return -1; + } + } + + /* buffer ready to send? */ + assert(buffer_free < sizeof(buffer)); + buffer_size = sizeof(buffer) - buffer_free; + assert(buffer[buffer_size - 1] == ' '); + buffer[buffer_size - 1] = '\n'; + + /* send request to rrdcached */ + status = sendall(client, buffer, buffer_size, 1); + if (status == -1) { + rrd_set_error("rrdc_dump: socket error (%s) while talking to rrdcached", + rrd_strerror(errno)); + close_connection(client); + return -1; + } + + /* receive response from rrdcached, relay to output_cb */ + ssize_t received, written; + ssize_t response_len = 0; + while (1) { + received = recv(client->sd, buffer, sizeof(buffer), 0); + if (received == -1L) { + rrd_set_error("rrdc_dump: failed to recv from rrdcached: %s", + rrd_strerror(errno)); + close_connection(client); + return -1; + } + if (received == 0) { + close_connection(client); + break; // EOF + } + written = output_cb(buffer, received, cb_userdata); + if (written != received) { + rrd_set_error("rrdc_dump: unexpected number of bytes (%d) " + "written (output_cb)", written); + close_connection(client); + return -1; + } + + // gather the first response bytes to detect XML response or status + size_t remaining_response_len = + ((signed) sizeof(resp_buffer) - response_len) > written ? written + : (signed)sizeof(resp_buffer) - response_len; + if (remaining_response_len > 0) { // continuously append to response buffer + memcpy(resp_buffer+response_len, buffer, remaining_response_len); + response_len += remaining_response_len; + } + + if (response_len < 1) continue; // unlikely empty write + + // handle non-xml response (error) + if (resp_buffer[0] != '<') { + char *nl = (char *) memchr((void *) resp_buffer, '\n', response_len); + if (nl == NULL) { + continue; // we did not get a line (yet) + } + *nl = '\0'; // \0 terminate at newline + chomp(resp_buffer); // chomp away possible \r too + rrd_set_error("rrdc_dump: failed to dump: %s", resp_buffer); + close_connection(client); + return -1; + } + // if the response starts with `<` an XML payload is assumed and the connection + // will be shutdown from the daemon to indicate EOF. + } + + return 0; +} /* }}} int rrd_client_dump */ + int rrd_client_tune( rrd_client_t *client, const char *filename, /* {{{ */ @@ -2152,6 +2271,20 @@ int rrdc_tune( return status; } /* }}} int rrdc_tune */ +int rrdc_dump( + const char *filename, /* {{{ */ + const char *opt_header, + rrd_output_callback_t output_cb, + void *cb_userdata) +{ + mutex_lock(&lock); + int status = + rrd_client_dump(&default_client, filename, opt_header, + output_cb, cb_userdata); + mutex_unlock(&lock); + return status; +} /* }}} int rrdc_tune */ + /* convenience function; if there is a daemon specified, or if we can * detect one from the environment, then flush the file. Otherwise, no-op */ @@ -2358,5 +2491,5 @@ void rrdc_stats_free( } /* }}} void rrdc_stats_free */ /* - * vim: set sw=2 sts=2 ts=8 et fdm=marker : + * vim: set sw=4 sts=4 ts=4 et fdm=marker : */ diff --git a/src/rrd_client.h b/src/rrd_client.h index 1ccb7487b..207c80313 100644 --- a/src/rrd_client.h +++ b/src/rrd_client.h @@ -107,6 +107,9 @@ int rrd_client_tune(rrd_client_t *client, const char *filename, int argc, const char **argv); +int rrd_client_dump(rrd_client_t *client, const char *filename, const char *opt_header, + rrd_output_callback_t output_cb, void *cb_userdata); + int rrd_client_stats_get(rrd_client_t *client, rrdc_stats_t **ret_stats); /* @@ -152,6 +155,9 @@ int rrdc_tune (const char *filename, int argc, const char **argv); +int rrdc_dump (const char *filename, const char *opt_header, + rrd_output_callback_t output_cb, void *cb_userdata); + int rrdc_fetch (const char *filename, const char *cf, time_t *ret_start, time_t *ret_end, diff --git a/src/rrd_daemon.c b/src/rrd_daemon.c index 4e70d153e..751798aa1 100644 --- a/src/rrd_daemon.c +++ b/src/rrd_daemon.c @@ -884,6 +884,35 @@ static int send_response( return 0; } /* }}} */ +/* send a chunk of bytes directly to the socket without buffering. + * the socket is passed as `void *user` parameter. + * this can be used as a callback for writes. + * rrd_dump_cb_r is an example use-case. + * returns number of bytes written on success, -1 on error + */ +static size_t send_unbuffered( + const void *data, + size_t len, + void *user) +{ /* {{{ */ + size_t bytes_written=0; + if (!user) { + RRDD_LOG(LOG_INFO, "send_unbuffered: missing user pointer"); + return -1; + } + listen_socket_t *sock = (listen_socket_t*)user; + + while (bytes_written < len) { + ssize_t rc = write(sock->fd, (char*)data + bytes_written, len - bytes_written); + if (rc <= 0) { + RRDD_LOG(LOG_INFO, "send_unbuffered: could not write data (%d)", errno); + return -1; + } + bytes_written += rc; + } + return bytes_written; +} /* }}} */ + static void wipe_ci_values( cache_item_t *ci, time_t when) @@ -1793,6 +1822,56 @@ static int handle_request_update( return rc; } /* }}} int handle_request_update */ +static int handle_request_dump( + HANDLER_PROTO) +{ /* {{{ */ + char *filename; + char *filepath; + int rc; + + rc = buffer_get_field(&buffer, &buffer_size, &filename); + if (rc != 0) return syntax_error(sock, cmd); + filepath = get_abs_path(filename); /* absolute filename */ + if (filepath == NULL) { + return send_response(sock, RESP_ERR, "%s\n", rrd_strerror(ENOMEM)); + } + + struct stat statbuf; + memset(&statbuf, 0, sizeof(statbuf)); + if (stat(filepath, &statbuf) != 0 || !S_ISREG(statbuf.st_mode)) { + free(filepath); + return send_response(sock, RESP_ERR, "%s: failed to stat: %s\n", + filename, rrd_strerror(errno)); + } + + rc = flush_file(filepath); + switch (rc) { + case 0: + break; // success + case ENOENT: + break; // success - nothing to flush + default: + free(filepath); + return send_response(sock, RESP_ERR, "%s: failed to flush\n", filename); + } + + rc = rrd_dump_cb_r(filepath, 1, send_unbuffered, (void*)sock); + if (rc != 0) { + RRDD_LOG(LOG_WARNING, "rrddump request for %s: failed to relay dump: %s", filepath, rrd_get_error()); + free(filepath); + return send_response(sock, RESP_ERR, "%s: failed to relay dump: %s\n", filename, rrd_get_error()); + } + + RRDD_LOG(LOG_INFO, "rrddump request for %s succeeded", filepath); + free(filepath); + + /* + * We return -1 here to indicate a bogus "failure". + * This will cause the connection to be closed and this conveys the + * end of the dumped XML file. + */ + return -1; +} /* }}} int handle_request_dump */ static int handle_request_tune( HANDLER_PROTO) @@ -2885,6 +2964,12 @@ static command_t list_of_commands[] = { /* {{{ */ CMD_CONTEXT_CLIENT, "TUNE [options]\n", "Tunes the given file, takes the parameters as defined in rrdtool.\n"}, + { + "DUMP", + handle_request_dump, + CMD_CONTEXT_CLIENT, + "DUMP [-h none|xsd|dtd]\n", + "Dumps the specified RRD to XML.\n"}, { "FLUSH", handle_request_flush, @@ -5019,5 +5104,5 @@ int main( } /* int main */ /* - * vim: set sw=2 sts=2 ts=8 et fdm=marker : + * vim: set sw=4 sts=2 ts=4 et fdm=marker : */ diff --git a/src/rrd_dump.c b/src/rrd_dump.c index 264442947..a4490d594 100644 --- a/src/rrd_dump.c +++ b/src/rrd_dump.c @@ -464,6 +464,29 @@ int rrd_dump_cb_r( } +static char *str_opt_xmlheader(int header) { + switch (header) { + case 1: + return "dtd"; + case 2: + return "xsd"; + default: + return "none"; + } +} + +static int parse_opt_xmlheader(const char *header) { + if (strcmp(header, "dtd") == 0) { + return 1; + } else if (strcmp(header, "xsd") == 0) { + return 2; + } else if (strcmp(header, "none") == 0) { + return 0; + } + return -1; +} + + static size_t rrd_dump_opt_cb_fileout( const void *data, size_t len, @@ -489,7 +512,13 @@ int rrd_dump_opt_r( out_file = stdout; } - res = rrd_dump_cb_r(filename, opt_noheader, rrd_dump_opt_cb_fileout, (void *)out_file); + if (rrdc_is_any_connected()) { + res = rrdc_dump(filename, str_opt_xmlheader(opt_noheader), + rrd_dump_opt_cb_fileout, (void *)out_file); + } else { + res = rrd_dump_cb_r(filename, opt_noheader, + rrd_dump_opt_cb_fileout, (void *)out_file); + } if (fflush(out_file) != 0) { rrd_set_error("error flushing output: %s", rrd_strerror(errno)); @@ -543,8 +572,7 @@ int rrd_dump( free (opt_daemon); } opt_daemon = strdup(options.optarg); - if (opt_daemon == NULL) - { + if (opt_daemon == NULL) { rrd_set_error ("strdup failed."); return (-1); } @@ -555,14 +583,8 @@ int rrd_dump( break; case 'h': - if (strcmp(options.optarg, "dtd") == 0) { - opt_header = 1; - } else if (strcmp(options.optarg, "xsd") == 0) { - opt_header = 2; - } else if (strcmp(options.optarg, "none") == 0) { - opt_header = 0; - } - break; + opt_header = parse_opt_xmlheader(options.optarg); + break; default: rrd_set_error("usage rrdtool %s [--header|-h {none,xsd,dtd}]\n" @@ -602,3 +624,6 @@ int rrd_dump( return rc; } +/* + * vim: set sw=4 sts=4 ts=4 et fdm=marker : + */ From d2dc89d0156bc1de1abb7b950e19508176c76a1f Mon Sep 17 00:00:00 2001 From: Tobias Hintze Date: Thu, 12 Oct 2023 16:26:03 +0200 Subject: [PATCH 4/7] update tests for remote dump support Previously, tests replaced the daemon-based dump use with a local use. This is changed in this commit as dump is now supported via daemon. --- tests/functions | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/functions b/tests/functions index 392239247..c88d2b6ad 100644 --- a/tests/functions +++ b/tests/functions @@ -159,13 +159,6 @@ function run_cached { ARGS=( $(sed "s#${BUILDDIR}#${BASEDIR}#" <<< "${ARGS[@]}") ) fi - # rrdcached does not support remote dump - if [ $1 == "dump" ]; then - RRDCACHED_STRIPPATH=${BUILDDIR} $RRDTOOL_ORIG flushcached $2 || fail flushcached - ARGS=( $(sed "s#${BUILDDIR}#${BASEDIR}#" <<< "${ARGS[@]}") ) - RRDCACHED_STRIPPATH=${BASEDIR} - fi - # rrdcached does not support remote restore if [ $1 == "restore" ]; then ARGS=( "restore" "$2" $(sed "s#${BUILDDIR}#${BASEDIR}#" <<< $3) ) From 1586178ea55d8b6049e0c6fe4824b104f83bda15 Mon Sep 17 00:00:00 2001 From: Tobias Hintze Date: Thu, 12 Oct 2023 22:29:00 +0200 Subject: [PATCH 5/7] update doc/rrddump --- doc/rrddump.pod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/rrddump.pod b/doc/rrddump.pod index 3aaa49077..49a924ecb 100644 --- a/doc/rrddump.pod +++ b/doc/rrddump.pod @@ -48,6 +48,8 @@ S<--no-header> option since 1.2 cannot deal with xml headers. Address of the L daemon. If specified, a C command is sent to the server before reading the RRD files. This allows B to return fresh data even if the daemon is configured to cache values for a long time. +When specified the RRD filename signifies a server side file, but the output +(XML) filename refers to the local side. For a list of accepted formats, see the B<-l> option in the L manual. rrdtool dump --daemon unix:/var/run/rrdcached.sock /var/lib/rrd/foo.rrd From 2888988d7a917bf1716e962bcd714d596c4ca35e Mon Sep 17 00:00:00 2001 From: Tobias Hintze Date: Sun, 15 Oct 2023 23:28:53 +0200 Subject: [PATCH 6/7] fix format argument type --- src/rrd_client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rrd_client.c b/src/rrd_client.c index cb6fa8c39..f4cec0d7c 100644 --- a/src/rrd_client.c +++ b/src/rrd_client.c @@ -2143,7 +2143,7 @@ int rrd_client_dump( } written = output_cb(buffer, received, cb_userdata); if (written != received) { - rrd_set_error("rrdc_dump: unexpected number of bytes (%d) " + rrd_set_error("rrdc_dump: unexpected number of bytes (%ld) " "written (output_cb)", written); close_connection(client); return -1; From 6c63597dd16077de48ca90ba52bb29fa7e1681f7 Mon Sep 17 00:00:00 2001 From: Tobias Hintze Date: Fri, 20 Oct 2023 17:08:06 +0200 Subject: [PATCH 7/7] update to latest windows vcpkg tag --- .github/workflows/ci-workflow.yml | 6 +++--- .github/workflows/release-windows.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index c1d35ea38..d69b6ba4b 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -51,14 +51,14 @@ jobs: include: - os: windows-2022 triplet: x64-windows - # https://github.com/microsoft/vcpkg/commit/501db0f17ef6df184fcdbfbe0f87cde2313b6ab1 - vcpkgCommitId: '501db0f17ef6df184fcdbfbe0f87cde2313b6ab1' + # https://github.com/microsoft/vcpkg/commit/8eb57355a4ffb410a2e94c07b4dca2dffbee8e50 + vcpkgCommitId: '8eb57355a4ffb410a2e94c07b4dca2dffbee8e50' vcpkgPackages: 'cairo expat fontconfig freetype gettext glib libpng libxml2 pango pcre zlib' configuration: 'x64' nmake_configuration: 'USE_64BIT=1' - os: windows-2022 triplet: x86-windows - vcpkgCommitId: '501db0f17ef6df184fcdbfbe0f87cde2313b6ab1' + vcpkgCommitId: '8eb57355a4ffb410a2e94c07b4dca2dffbee8e50' vcpkgPackages: 'cairo expat fontconfig freetype gettext glib libpng libxml2 pango pcre zlib' configuration: 'x86' nmake_configuration: '' diff --git a/.github/workflows/release-windows.yml b/.github/workflows/release-windows.yml index bee860482..b8d99650f 100644 --- a/.github/workflows/release-windows.yml +++ b/.github/workflows/release-windows.yml @@ -25,14 +25,14 @@ jobs: include: - os: windows-2022 triplet: x64-windows - # https://github.com/microsoft/vcpkg/commit/501db0f17ef6df184fcdbfbe0f87cde2313b6ab1 - vcpkgCommitId: '501db0f17ef6df184fcdbfbe0f87cde2313b6ab1' + # https://github.com/microsoft/vcpkg/commit/8eb57355a4ffb410a2e94c07b4dca2dffbee8e50 + vcpkgCommitId: '8eb57355a4ffb410a2e94c07b4dca2dffbee8e50' vcpkgPackages: 'cairo expat fontconfig freetype gettext glib libpng libxml2 pango pcre zlib' configuration: 'x64' nmake_configuration: 'USE_64BIT=1' - os: windows-2022 triplet: x86-windows - vcpkgCommitId: '501db0f17ef6df184fcdbfbe0f87cde2313b6ab1' + vcpkgCommitId: '8eb57355a4ffb410a2e94c07b4dca2dffbee8e50' vcpkgPackages: 'cairo expat fontconfig freetype gettext glib libpng libxml2 pango pcre zlib' configuration: 'x86' nmake_configuration: ''