rrdcached dump support

.github/workflows/ci-workflow.yml

@@ -51,14 +51,14 @@ jobs:
         include:
           - os: windows-2022
             triplet: x64-windows
-            # https://github.com/microsoft/vcpkg/commit/501db0f17ef6df184fcdbfbe0f87cde2313b6ab1
-            vcpkgCommitId: '501db0f17ef6df184fcdbfbe0f87cde2313b6ab1'
+            # https://github.com/microsoft/vcpkg/commit/8eb57355a4ffb410a2e94c07b4dca2dffbee8e50
+            vcpkgCommitId: '8eb57355a4ffb410a2e94c07b4dca2dffbee8e50'
             vcpkgPackages: 'cairo expat fontconfig freetype gettext glib libpng libxml2 pango pcre zlib'
             configuration: 'x64'
             nmake_configuration: 'USE_64BIT=1'
           - os: windows-2022
             triplet: x86-windows
-            vcpkgCommitId: '501db0f17ef6df184fcdbfbe0f87cde2313b6ab1'
+            vcpkgCommitId: '8eb57355a4ffb410a2e94c07b4dca2dffbee8e50'
             vcpkgPackages: 'cairo expat fontconfig freetype gettext glib libpng libxml2 pango pcre zlib'
             configuration: 'x86'
             nmake_configuration: ''

.github/workflows/release-windows.yml

@@ -25,14 +25,14 @@ jobs:
         include:
           - os: windows-2022
             triplet: x64-windows
-            # https://github.com/microsoft/vcpkg/commit/501db0f17ef6df184fcdbfbe0f87cde2313b6ab1
-            vcpkgCommitId: '501db0f17ef6df184fcdbfbe0f87cde2313b6ab1'
+            # https://github.com/microsoft/vcpkg/commit/8eb57355a4ffb410a2e94c07b4dca2dffbee8e50
+            vcpkgCommitId: '8eb57355a4ffb410a2e94c07b4dca2dffbee8e50'
             vcpkgPackages: 'cairo expat fontconfig freetype gettext glib libpng libxml2 pango pcre zlib'
             configuration: 'x64'
             nmake_configuration: 'USE_64BIT=1'
           - os: windows-2022
             triplet: x86-windows
-            vcpkgCommitId: '501db0f17ef6df184fcdbfbe0f87cde2313b6ab1'
+            vcpkgCommitId: '8eb57355a4ffb410a2e94c07b4dca2dffbee8e50'
             vcpkgPackages: 'cairo expat fontconfig freetype gettext glib libpng libxml2 pango pcre zlib'
             configuration: 'x86'
             nmake_configuration: ''

CHANGES

@@ -39,6 +39,7 @@ Features
 
   rrd_updatex_r(filename, tmplt, RRD_FLAGS_LOCKING_MODE_BLOCK, ...);
 
+* Add (remote) dump support to rrdcached <Tobias Hintze>
 
 RRDtool 1.8.0 - 2022-03-13
 ==========================

doc/rrddump.pod

@@ -48,6 +48,8 @@ S<--no-header> option since 1.2 cannot deal with xml headers.
 Address of the L<rrdcached> daemon. If specified, a C<flush> command is sent
 to the server before reading the RRD files. This allows B<rrdtool> to return
 fresh data even if the daemon is configured to cache values for a long time.
+When specified the RRD filename signifies a server side file, but the output
+(XML) filename refers to the local side.
 For a list of accepted formats, see the B<-l> option in the L<rrdcached> manual.
 
  rrdtool dump --daemon unix:/var/run/rrdcached.sock /var/lib/rrd/foo.rrd

src/rrd_client.c

@@ -653,10 +653,11 @@
         return -1;
 
     while (ret != -1 && len > 0) {
-        ret = send(client->sd, msg, len, 0);
+        ret = send(client->sd, bufp, len, 0);
         if (ret > 0) {
             bufp += ret;
             len -= ret;
+            allow_retry = 0; // partial read forbids retry
         }
     }
 
@@ -2059,6 +2060,125 @@
     return status;
 }                       /* }}} int rrdc_fetch */
 
+int rrd_client_dump(
+    rrd_client_t *client,
+    const char *filename,   /* {{{ */
+    const char *opt_header,
+    rrd_output_callback_t output_cb,
+    void *cb_userdata)
+{
+    char    buffer[RRD_CMD_MAX];
+    char   *buffer_ptr;
+    size_t  buffer_free;
+    size_t  buffer_size;
+    int     status;
+    char   *file_path;
+    char    resp_buffer[256];
+
+    if (client == NULL) return -1;
+    if (filename == NULL) {
+        rrd_set_error("rrdc_dump: no input filename specified");
+        return -1;
+    }
+
+    memset(buffer, 0, sizeof(buffer));
+    buffer_ptr = &buffer[0];
+    buffer_free = sizeof(buffer);
+
+    status = buffer_add_string("dump", &buffer_ptr, &buffer_free);
+    if (status != 0) {
+        rrd_set_error("rrdc_dump: out of memory");
+        return -1;
+    }
+
+    file_path = get_path(client, filename);
+    if (file_path == NULL) {
+        return -1;
+    }
+
+    status = buffer_add_string(file_path, &buffer_ptr, &buffer_free);
+    free(file_path);
+    if (status != 0) {
+        rrd_set_error("rrdc_dump: out of memory");
+        return -1;
+    }
+
+    if (opt_header) {
+        status = buffer_add_string(opt_header, &buffer_ptr, &buffer_free);
+        if (status != 0) {
+            rrd_set_error("rrdc_dump: out of memory");
+            return -1;
+        }
+    }
+
+    /* buffer ready to send? */
+    assert(buffer_free < sizeof(buffer));
+    buffer_size = sizeof(buffer) - buffer_free;
+    assert(buffer[buffer_size - 1] == ' ');
+    buffer[buffer_size - 1] = '\n';
+
+    /* send request to rrdcached */
+    status = sendall(client, buffer, buffer_size, 1);
+    if (status == -1) {
+        rrd_set_error("rrdc_dump: socket error (%s) while talking to rrdcached", 
+                rrd_strerror(errno));
+        close_connection(client);
+        return -1;
+    }
+
+    /* receive response from rrdcached, relay to output_cb */
+    ssize_t received, written;
+    ssize_t response_len = 0;
+    while (1) {
+        received = recv(client->sd, buffer, sizeof(buffer), 0);
+        if (received == -1L) {
+            rrd_set_error("rrdc_dump: failed to recv from rrdcached: %s",
+                    rrd_strerror(errno));
+            close_connection(client);
+            return -1;
+        }
+        if (received == 0) {
+            close_connection(client);
+            break; // EOF
+        }
+        written = output_cb(buffer, received, cb_userdata);
+        if (written != received) {
+            rrd_set_error("rrdc_dump: unexpected number of bytes (%ld) "
+                    "written (output_cb)", written);
+            close_connection(client);
+            return -1;
+        }
+
+        // gather the first response bytes to detect XML response or status
+        size_t remaining_response_len =
+            ((signed) sizeof(resp_buffer) - response_len) > written ? written
+            : (signed)sizeof(resp_buffer) - response_len;
+        if (remaining_response_len > 0) { // continuously append to response buffer
+            memcpy(resp_buffer+response_len, buffer, remaining_response_len);
+            response_len += remaining_response_len;
+        }
+
+        if (response_len < 1) continue; // unlikely empty write
+
+        // handle non-xml response (error)
+        if (resp_buffer[0] != '<') {
+            char *nl = (char *) memchr((void *) resp_buffer, '\n', response_len);
+            if (nl == NULL) {
+                continue; // we did not get a line (yet)
+            }
+            *nl = '\0'; // \0 terminate at newline
+            chomp(resp_buffer); // chomp away possible \r too
+            rrd_set_error("rrdc_dump: failed to dump: %s", resp_buffer);
+            close_connection(client);
+            return -1;
+        }
+        // if the response starts with `<` an XML payload is assumed and the connection
+        // will be shutdown from the daemon to indicate EOF.
+    }
+
+    return 0;
+}                       /* }}} int rrd_client_dump */
+
 int rrd_client_tune(
     rrd_client_t *client,
     const char *filename,   /* {{{ */
@@ -2151,6 +2271,20 @@
     return status;
 }                       /* }}} int rrdc_tune */
 
+int rrdc_dump(
+    const char *filename,   /* {{{ */
+    const char *opt_header,
+    rrd_output_callback_t output_cb,
+    void *cb_userdata)
+{
+    mutex_lock(&lock);
+    int status =
+        rrd_client_dump(&default_client, filename, opt_header,
+                output_cb, cb_userdata);
+    mutex_unlock(&lock);
+    return status;
+}                       /* }}} int rrdc_tune */
+
 /* convenience function; if there is a daemon specified, or if we can
  * detect one from the environment, then flush the file.  Otherwise, no-op
  */
@@ -2357,5 +2491,5 @@
 }                       /* }}} void rrdc_stats_free */
 
 /*
- * vim: set sw=2 sts=2 ts=8 et fdm=marker :
+ * vim: set sw=4 sts=4 ts=4 et fdm=marker :
  */

src/rrd_client.h

@@ -107,6 +107,9 @@ int rrd_client_tune(rrd_client_t *client, const char *filename,
     int argc,
     const char **argv);
 
+int rrd_client_dump(rrd_client_t *client, const char *filename, const char *opt_header,
+    rrd_output_callback_t output_cb, void *cb_userdata);
+
 int rrd_client_stats_get(rrd_client_t *client, rrdc_stats_t **ret_stats);
 
 /*
@@ -152,6 +155,9 @@ int rrdc_tune (const char *filename,
     int argc,
     const char **argv);
 
+int rrdc_dump (const char *filename, const char *opt_header,
+    rrd_output_callback_t output_cb, void *cb_userdata);
+
 int rrdc_fetch (const char *filename,
     const char *cf,
     time_t *ret_start, time_t *ret_end,

src/rrd_daemon.c

@@ -884,6 +884,35 @@
     return 0;
 }                       /* }}} */
 
+/* send a chunk of bytes directly to the socket without buffering.
+ * the socket is passed as `void *user` parameter.
+ * this can be used as a callback for writes.
+ * rrd_dump_cb_r is an example use-case.
+ * returns number of bytes written on success, -1 on error
+ */
+static size_t send_unbuffered(
+    const void *data,
+    size_t len,
+    void *user)
+{                       /* {{{ */
+    size_t bytes_written=0;
+    if (!user) {
+        RRDD_LOG(LOG_INFO, "send_unbuffered: missing user pointer");
+        return -1;
+    }
+    listen_socket_t *sock = (listen_socket_t*)user;
+
+    while (bytes_written < len) {
+        ssize_t rc = write(sock->fd, (char*)data + bytes_written, len - bytes_written);
+        if (rc <= 0) {
+            RRDD_LOG(LOG_INFO, "send_unbuffered: could not write data (%d)", errno);
+            return -1;
+        }
+        bytes_written += rc;
+    }
+    return bytes_written;
+}                       /* }}} */
+
 static void wipe_ci_values(
     cache_item_t *ci,
     time_t when)
@@ -1793,6 +1822,56 @@
     return rc;
 }                       /* }}} int handle_request_update */
 
+static int handle_request_dump(
+    HANDLER_PROTO)
+{                       /* {{{ */
+    char *filename;
+    char *filepath;
+    int rc;
+
+    rc = buffer_get_field(&buffer, &buffer_size, &filename);
+    if (rc != 0) return syntax_error(sock, cmd);
+    filepath = get_abs_path(filename); /* absolute filename */
+    if (filepath == NULL) {
+        return send_response(sock, RESP_ERR, "%s\n", rrd_strerror(ENOMEM));
+    }
+
+    struct stat statbuf;
+    memset(&statbuf, 0, sizeof(statbuf));
+    if (stat(filepath, &statbuf) != 0 || !S_ISREG(statbuf.st_mode)) {
+        free(filepath);
+        return send_response(sock, RESP_ERR, "%s: failed to stat: %s\n",
+                filename, rrd_strerror(errno));
+    }
+
+    rc = flush_file(filepath);
+    switch (rc) {
+        case 0:
+            break; // success
+        case ENOENT:
+            break; // success - nothing to flush
+        default:
+            free(filepath);
+            return send_response(sock, RESP_ERR, "%s: failed to flush\n", filename);
+    }
+
+    rc = rrd_dump_cb_r(filepath, 1, send_unbuffered, (void*)sock);
+    if (rc != 0) {
+        RRDD_LOG(LOG_WARNING, "rrddump request for %s: failed to relay dump: %s", filepath, rrd_get_error());
+        free(filepath);
+        return send_response(sock, RESP_ERR, "%s: failed to relay dump: %s\n", filename, rrd_get_error());
+    }
+
+    RRDD_LOG(LOG_INFO, "rrddump request for %s succeeded", filepath);
+    free(filepath);
+
+    /*
+     * We return -1 here to indicate a bogus "failure".
+     * This will cause the connection to be closed and this conveys the
+     * end of the dumped XML file.
+     */
+    return -1;
+}                       /* }}} int handle_request_dump */
 
 static int handle_request_tune(
     HANDLER_PROTO)
@@ -2883,8 +2962,14 @@
      "TUNE",
      handle_request_tune,
      CMD_CONTEXT_CLIENT,
-     "TUNE <filename> [options]",
-     "Tunes the given file, takes the parameters as defined in rrdtool"},
+     "TUNE <filename> [options]\n",
+     "Tunes the given file, takes the parameters as defined in rrdtool.\n"},
+    {
+     "DUMP",
+     handle_request_dump,
+     CMD_CONTEXT_CLIENT,
+     "DUMP <filename> [-h none|xsd|dtd]\n",
+     "Dumps the specified RRD to XML.\n"},
     {
      "FLUSH",
      handle_request_flush,
@@ -5019,5 +5104,5 @@
 }                       /* int main */
 
 /*
- * vim: set sw=2 sts=2 ts=8 et fdm=marker :
+ * vim: set sw=4 sts=2 ts=4 et fdm=marker :
  */