From 5b6f33e394eb6b8de682ba185e5cb169cb4c6728 Mon Sep 17 00:00:00 2001 From: Joerg Steffens Date: Tue, 10 Apr 2018 18:54:24 +0200 Subject: [PATCH] bareos-storage-droplet: improve error handling Previos version did not always return an error, if data could not be written. Especially the load_chunk ignored EIO errors, properly because of a typo. As the droplet_device in iothread mode relies on asynchronious write-backs, the new device method flush() has been introduced. If a droplet_device is configured to use iothreads and unlimited retries, this will do busy waiting until all data is written to the droplet backend. In case of a connection problems to the droplet_device, this will be forever. Note that a bconsole "status storage=..." command will inform about "Pending IO flush requests". Fixes #892: bareos-storage-droplet: if configured with unreachable S3 system, backup will terminate with OK --- src/stored/acquire.c | 20 ++++- src/stored/backends/Makefile.in | 10 +-- src/stored/backends/chunked_device.c | 116 +++++++++++++++++++++++++-- src/stored/backends/chunked_device.h | 11 +++ src/stored/backends/droplet_device.c | 96 +++++++++++++++++++++- src/stored/backends/droplet_device.h | 5 +- src/stored/dev.c | 2 +- src/stored/dev.h | 4 +- 8 files changed, 246 insertions(+), 18 deletions(-) diff --git a/src/stored/acquire.c b/src/stored/acquire.c index b0cb8a993bd..9e787a33643 100644 --- a/src/stored/acquire.c +++ b/src/stored/acquire.c @@ -3,7 +3,7 @@ Copyright (C) 2002-2013 Free Software Foundation Europe e.V. Copyright (C) 2011-2012 Planets Communications B.V. - Copyright (C) 2013-2013 Bareos GmbH & Co. KG + Copyright (C) 2013-2018 Bareos GmbH & Co. KG This program is Free Software; you can redistribute it and/or modify it under the terms of version three of the GNU Affero General Public @@ -496,6 +496,24 @@ bool release_device(DCR *dcr) now = (utime_t)time(NULL); update_job_statistics(jcr, now); + /* + * Some devices do cache write operations (e.g. droplet_device). + * Therefore flushing the cache is required to determine + * if a job have been written successfully. + * As a flush operation can take quite a long time, + * this must be done before acquiring locks. + * A previous implementation did the flush inside dev->close(), + * which resulted in various locking problems. + */ + if (!job_canceled(jcr)) { + Jmsg(jcr, M_INFO, 0, "Flushing device %s.\n", dev->print_name()); + if (!dev->flush(dcr)) { + Jmsg(jcr, M_FATAL, 0, "Failed to flush device %s.\n", dev->print_name()); + } else { + Jmsg(jcr, M_INFO, 0, "Device %s flushed.\n", dev->print_name()); + } + } + dev->Lock(); if (!dev->is_blocked()) { block_device(dev, BST_RELEASING); diff --git a/src/stored/backends/Makefile.in b/src/stored/backends/Makefile.in index d714a0517bb..03d085c2f8c 100644 --- a/src/stored/backends/Makefile.in +++ b/src/stored/backends/Makefile.in @@ -80,27 +80,27 @@ STORED_RESTYPES = autochanger device director ndmp messages storage $(NO_ECHO)$(LIBTOOL_COMPILE) $(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(INCLUDES) $(DINCLUDE) $(CXXFLAGS) $< if [ -d "$(@:.lo=.d)" ]; then $(MKDIR) $(CONF_EXTRA_DIR); $(CP) -r $(@:.lo=.d)/. $(CONF_EXTRA_DIR)/.; fi -$(CHEPHFS_LOBJS): +$(CHEPHFS_LOBJS): $(CHEPHFS_SRCS) @echo "Compiling $(@:.lo=.c)" $(NO_ECHO)$(LIBTOOL_COMPILE) $(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(INCLUDES) $(CEPHFS_INC) $(DINCLUDE) $(CXXFLAGS) $(@:.lo=.c) if [ -d "$(@:.lo=.d)" ]; then $(MKDIR) $(CONF_EXTRA_DIR); $(CP) -r $(@:.lo=.d)/. $(CONF_EXTRA_DIR)/.; fi -$(DROPLET_LOBJS): +$(DROPLET_LOBJS): $(DROPLET_SRCS) @echo "Compiling $(@:.lo=.c)" $(NO_ECHO)$(LIBTOOL_COMPILE) $(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(INCLUDES) $(DROPLET_INC) $(DINCLUDE) $(CXXFLAGS) $(@:.lo=.c) if [ -d "$(@:.lo=.d)" ]; then $(MKDIR) $(CONF_EXTRA_DIR); $(CP) -r $(@:.lo=.d)/. $(CONF_EXTRA_DIR)/.; fi -$(ELASTO_LOBJS): +$(ELASTO_LOBJS): $(ELASTO_SRCS) @echo "Compiling $(@:.lo=.c)" $(NO_ECHO)$(LIBTOOL_COMPILE) $(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(INCLUDES) $(ELASTO_INC) $(DINCLUDE) $(CXXFLAGS) $(@:.lo=.c) if [ -d "$(@:.lo=.d)" ]; then $(MKDIR) $(CONF_EXTRA_DIR); $(CP) -r $(@:.lo=.d)/. $(CONF_EXTRA_DIR)/.; fi -$(GFAPI_LOBJS): +$(GFAPI_LOBJS): $(GFAPI_SRCS) @echo "Compiling $(@:.lo=.c)" $(NO_ECHO)$(LIBTOOL_COMPILE) $(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(INCLUDES) $(GLUSTER_INC) $(DINCLUDE) $(CXXFLAGS) $(@:.lo=.c) if [ -d "$(@:.lo=.d)" ]; then $(MKDIR) $(CONF_EXTRA_DIR); $(CP) -r $(@:.lo=.d)/. $(CONF_EXTRA_DIR)/.; fi -$(RADOS_LOBJS): +$(RADOS_LOBJS): $(RADOS_SRCS) @echo "Compiling $(@:.lo=.c)" $(NO_ECHO)$(LIBTOOL_COMPILE) $(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(INCLUDES) $(RADOS_INC) $(DINCLUDE) $(CXXFLAGS) $(@:.lo=.c) if [ -d "$(@:.lo=.d)" ]; then $(MKDIR) $(CONF_EXTRA_DIR); $(CP) -r $(@:.lo=.d)/. $(CONF_EXTRA_DIR)/.; fi diff --git a/src/stored/backends/chunked_device.c b/src/stored/backends/chunked_device.c index 4c56a22c572..a2bc9baa1eb 100644 --- a/src/stored/backends/chunked_device.c +++ b/src/stored/backends/chunked_device.c @@ -2,6 +2,7 @@ BAREOS® - Backup Archiving REcovery Open Sourced Copyright (C) 2015-2017 Planets Communications B.V. + Copyright (C) 2017-2018 Bareos GmbH & Co. KG This program is Free Software; you can redistribute it and/or modify it under the terms of version three of the GNU Affero General Public @@ -71,7 +72,7 @@ static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; * * flush_remote_chunk() - Flush a chunk to the remote backing store. * read_remote_chunk() - Read a chunk from the remote backing store. - * chunked_remote_volume_size - Return the current size of a volume. + * chunked_remote_volume_size() - Return the current size of a volume. * truncate_remote_chunked_volume() - Truncate a chunked volume on the * remote backing store. */ @@ -657,16 +658,26 @@ bool chunked_device::read_chunk() /* * Setup a chunked volume for reading or writing. + * return: + * -1: failure + * 0: success */ int chunked_device::setup_chunk(const char *pathname, int flags, int mode) { + int retval = -1; /* * If device is (re)opened and we are put into readonly mode because * of problems flushing chunks to the backing store we return EROFS * to the upper layers. */ if ((flags & O_RDWR) && m_readonly) { - dev_errno = EROFS; + dev_errno = EROFS; /**< Read-only file system */ + return -1; + } + + if (!check_remote()) { + Dmsg0(100, "setup_chunk failed, as remote device is not available\n"); + dev_errno = EIO; /**< I/O error */ return -1; } @@ -698,7 +709,6 @@ int chunked_device::setup_chunk(const char *pathname, int flags, int mode) m_current_chunk->writing = true; } - m_current_chunk->opened = true; m_current_chunk->chunk_setup = false; /* @@ -727,7 +737,23 @@ int chunked_device::setup_chunk(const char *pathname, int flags, int mode) m_current_volname = bstrdup(getVolCatName()); - return 0; + /* + * in principle it is not required to load_chunk(), + * but we need a secure way to determine, + * if the chunk already exists. + */ + if (load_chunk()) { + m_current_chunk->opened = true; + retval = 0; + } else if (flags & O_CREAT) { + /* create a chunk */ + if (flush_chunk(false /* release */, false /* move_to_next_chunk */)) { + m_current_chunk->opened = true; + retval = 0; + } + } + + return retval; } /* @@ -1013,6 +1039,7 @@ int chunked_device::close_chunk() m_current_chunk->buflen = 0; m_current_chunk->start_offset = -1; m_current_chunk->end_offset = -1; + } else { errno = EBADF; } @@ -1151,6 +1178,62 @@ ssize_t chunked_device::chunked_volume_size() return chunked_remote_volume_size(); } + +bool chunked_device::is_written() +{ + /* + * See if we are using io-threads or not and the ordered circbuf is created. + * We try to make sure that nothing of the volume being requested is still inflight as then + * the chunked_remote_volume_size() method will fail to determine the size of the data as + * its not fully stored on the backing store yet. + */ + + /* + * Make sure there is also nothing inflight to the backing store anymore. + */ + if (nr_inflight_chunks() > 0) { + Dmsg0(100, "is_written = false, as there are inflight chunks\n"); + return false; + } + + if (m_io_threads > 0 && m_cb) { + + if (!m_cb->empty()) { + + chunk_io_request *request; + + /* + * Peek on the ordered circular queue if there are any pending IO-requests + * for this volume. If there are use that as the indication of the size of + * the volume and don't contact the remote storage as there is still data + * inflight and as such we need to look at the last chunk that is still not + * uploaded of the volume. + */ + request = (chunk_io_request *)m_cb->peek(PEEK_FIRST, m_current_volname, compare_volume_name); + if (request) { + free(request); + Dmsg0(100, "is_written = false, as there are queued write requests\n"); + return false; + } + } + } + + return true; +} + + +/* + * Busy waits until write buffer is empty. + */ +bool chunked_device::wait_until_chunks_written() +{ + while (!is_written()) { + bmicrosleep(DEFAULT_RECHECK_INTERVAL_WRITE_BUFFER, 0); + } + return true; +} + + static int clone_io_request(void *item1, void *item2) { chunk_io_request *src = (chunk_io_request *)item1; @@ -1271,6 +1354,7 @@ bool chunked_device::load_chunk() if (m_current_chunk->writing) { m_current_chunk->end_offset = start_offset + (m_current_chunk->chunk_size - 1); } + return false; break; default: return false; @@ -1290,7 +1374,7 @@ static int list_io_request(void *request, void *data) bsdDevStatTrig *dst = (bsdDevStatTrig *)data; POOL_MEM status(PM_MESSAGE); - status.bsprintf(" /%s/%04d - %ld\n", io_request->volname, io_request->chunk, io_request->wbuflen); + status.bsprintf(" /%s/%04d - %ld (try=%d)\n", io_request->volname, io_request->chunk, io_request->wbuflen, io_request->tries); dst->status_length = pm_strcat(dst->status, status.c_str()); return 0; @@ -1304,20 +1388,36 @@ bool chunked_device::device_status(bsdDevStatTrig *dst) /* * See if we are using io-threads or not and the ordered circbuf is created and not empty. */ + bool pending = false; + POOL_MEM inflights(PM_MESSAGE); + dst->status_length = 0; + if (check_remote()) { + dst->status_length = pm_strcpy(dst->status, _("Backend connection is working.\n")); + } else { + dst->status_length = pm_strcpy(dst->status, _("Backend connection is not working.\n")); + } if (m_io_threads > 0 && m_cb) { + if (nr_inflight_chunks() > 0) { + pending = true; + inflights.bsprintf("Inflight chunks: %d\n", nr_inflight_chunks()); + dst->status_length = pm_strcat(dst->status, inflights.c_str()); + } if (!m_cb->empty()) { - dst->status_length = pm_strcpy(dst->status, _("Pending IO flush requests:\n")); + pending = true; + dst->status_length = pm_strcat(dst->status, _("Pending IO flush requests:\n")); /* * Peek on the ordered circular queue and list all pending requests. */ m_cb->peek(PEEK_LIST, dst, list_io_request); - } else { - dst->status_length = pm_strcpy(dst->status, _("No Pending IO flush requests\n")); } } + if (!pending) { + dst->status_length += pm_strcat(dst->status, _("No Pending IO flush requests.\n")); + } + return (dst->status_length > 0); } diff --git a/src/stored/backends/chunked_device.h b/src/stored/backends/chunked_device.h index 957106790a3..0329b7e8d2c 100644 --- a/src/stored/backends/chunked_device.h +++ b/src/stored/backends/chunked_device.h @@ -2,6 +2,7 @@ BAREOS® - Backup Archiving REcovery Open Sourced Copyright (C) 2015-2017 Planets Communications B.V. + Copyright (C) 2018-2018 Bareos GmbH & Co. KG This program is Free Software; you can redistribute it and/or modify it under the terms of version three of the GNU Affero General Public @@ -32,6 +33,12 @@ */ #define DEFAULT_RECHECK_INTERVAL 300 +/* + * Recheck interval when waiting that buffer gets written + * (write buffer is empty). + */ +#define DEFAULT_RECHECK_INTERVAL_WRITE_BUFFER 10 + /* * Chunk the volume into chunks of this size. * This is the lower limit used the exact chunksize is @@ -112,6 +119,7 @@ class chunked_device: public DEVICE { bool enqueue_chunk(chunk_io_request *request); bool flush_chunk(bool release_chunk, bool move_to_next_chunk); bool read_chunk(); + bool is_written(); protected: /* @@ -138,10 +146,13 @@ class chunked_device: public DEVICE { bool truncate_chunked_volume(DCR *dcr); ssize_t chunked_volume_size(); bool load_chunk(); + bool wait_until_chunks_written(); /* * Methods implemented by inheriting class. */ + virtual bool check_remote() = 0; + virtual bool remote_chunked_volume_exists() = 0; virtual bool flush_remote_chunk(chunk_io_request *request) = 0; virtual bool read_remote_chunk(chunk_io_request *request) = 0; virtual ssize_t chunked_remote_volume_size() = 0; diff --git a/src/stored/backends/droplet_device.c b/src/stored/backends/droplet_device.c index b6a3faa9ef0..f05f18282e9 100644 --- a/src/stored/backends/droplet_device.c +++ b/src/stored/backends/droplet_device.c @@ -2,7 +2,7 @@ BAREOS® - Backup Archiving REcovery Open Sourced Copyright (C) 2014-2017 Planets Communications B.V. - Copyright (C) 2014-2014 Bareos GmbH & Co. KG + Copyright (C) 2014-2018 Bareos GmbH & Co. KG This program is Free Software; you can redistribute it and/or modify it under the terms of version three of the GNU Affero General Public @@ -140,6 +140,9 @@ static inline int droplet_errno_to_system_errno(dpl_status_t status) case DPL_EPERM: errno = EPERM; break; + case DPL_FAILURE: /**< General failure */ + errno = EIO; + break; default: errno = EINVAL; break; @@ -270,6 +273,91 @@ static bool walk_dpl_directory(dpl_ctx_t *ctx, const char *dirname, t_call_back return true; } + +/* + * + * Checks is connection to backend storage system is possible. + * + * Returns true - if connection can be established + * false - otherwise + * + * FIXME: currently, check_remote() returns true, + * after an initial connection could be made, + * even if the system is now no more reachable. + * Seams to be some caching effect. + */ +bool droplet_device::check_remote() +{ + bool retval = false; + dpl_status_t status; + dpl_sysmd_t *sysmd = NULL; + + if (!m_ctx) { + if (!initialize()) { + return false; + } + } + + sysmd = dpl_sysmd_dup(&m_sysmd); + status = dpl_getattr(m_ctx, /* context */ + "", /* locator */ + NULL, /* metadata */ + sysmd); /* sysmd */ + + switch (status) { + case DPL_SUCCESS: + Dmsg0(100, "check_remote: ok\n"); + retval = true; + break; + case DPL_ENOENT: + case DPL_FAILURE: + default: + Dmsg0(100, "check_remote: failed\n"); + break; + } + + return retval; +} + + + +bool droplet_device::remote_chunked_volume_exists() +{ + bool retval = false; + dpl_status_t status; + dpl_sysmd_t *sysmd = NULL; + POOL_MEM chunk_dir(PM_FNAME); + + if (!check_remote()) { + return false; + } + + Mmsg(chunk_dir, "/%s", getVolCatName()); + + Dmsg1(100, "checking remote_chunked_volume_exists %s\n", chunk_dir.c_str()); + + sysmd = dpl_sysmd_dup(&m_sysmd); + status = dpl_getattr(m_ctx, /* context */ + chunk_dir.c_str(), /* locator */ + NULL, /* metadata */ + sysmd); /* sysmd */ + + switch (status) { + case DPL_SUCCESS: + Dmsg1(100, "remote_chunked_volume %s exists\n", chunk_dir.c_str()); + retval = true; + break; + case DPL_ENOENT: + case DPL_FAILURE: + default: + Dmsg1(100, "remote_chunked_volume %s does not exists\n", chunk_dir.c_str()); + break; + } + + return retval; +} + + /* * Internal method for flushing a chunk to the backing store. * This does the real work either by being called from a @@ -514,6 +602,12 @@ bool droplet_device::truncate_remote_chunked_volume(DCR *dcr) return true; } + +bool droplet_device::d_flush(DCR *dcr) +{ + return wait_until_chunks_written(); +}; + /* * Initialize backend. */ diff --git a/src/stored/backends/droplet_device.h b/src/stored/backends/droplet_device.h index 7e1b25df3a0..1ecb15e17ad 100644 --- a/src/stored/backends/droplet_device.h +++ b/src/stored/backends/droplet_device.h @@ -2,7 +2,7 @@ BAREOS® - Backup Archiving REcovery Open Sourced Copyright (C) 2014-2017 Planets Communications B.V. - Copyright (C) 2014-2014 Bareos GmbH & Co. KG + Copyright (C) 2014-2018 Bareos GmbH & Co. KG This program is Free Software; you can redistribute it and/or modify it under the terms of version three of the GNU Affero General Public @@ -53,6 +53,8 @@ class droplet_device: public chunked_device { /* * Interface from chunked_device */ + bool check_remote(); + bool remote_chunked_volume_exists(); bool flush_remote_chunk(chunk_io_request *request); bool read_remote_chunk(chunk_io_request *request); ssize_t chunked_remote_volume_size(); @@ -75,5 +77,6 @@ class droplet_device: public chunked_device { ssize_t d_read(int fd, void *buffer, size_t count); ssize_t d_write(int fd, const void *buffer, size_t count); bool d_truncate(DCR *dcr); + bool d_flush(DCR *dcr); }; #endif /* OBJECTSTORE_DEVICE_H */ diff --git a/src/stored/dev.c b/src/stored/dev.c index 141e8d4f80c..cb67e29375b 100644 --- a/src/stored/dev.c +++ b/src/stored/dev.c @@ -704,7 +704,7 @@ bool DEVICE::rewind(DCR *dcr) if (lseek(dcr, (boffset_t)0, SEEK_SET) < 0) { berrno be; dev_errno = errno; - Mmsg2(errmsg, _("lseek error on %s. ERR=%s.\n"), print_name(), be.bstrerror()); + Mmsg2(errmsg, _("lseek error on %s. ERR=%s"), print_name(), be.bstrerror()); return false; } diff --git a/src/stored/dev.h b/src/stored/dev.h index 4a87b9f6203..940c25635bf 100644 --- a/src/stored/dev.h +++ b/src/stored/dev.h @@ -3,7 +3,7 @@ Copyright (C) 2000-2012 Free Software Foundation Europe e.V. Copyright (C) 2011-2012 Planets Communications B.V. - Copyright (C) 2013-2017 Bareos GmbH & Co. KG + Copyright (C) 2013-2018 Bareos GmbH & Co. KG This program is Free Software; you can redistribute it and/or modify it under the terms of version three of the GNU Affero General Public @@ -517,6 +517,7 @@ class DEVICE: public SMARTALLOC { virtual bool device_status(bsdDevStatTrig *dst) { return false; }; boffset_t lseek(DCR *dcr, boffset_t offset, int whence) { return d_lseek(dcr, offset, whence); }; bool truncate(DCR *dcr) { return d_truncate(dcr); }; + bool flush(DCR *dcr) { return d_flush(dcr); }; /* * Low level operations @@ -528,6 +529,7 @@ class DEVICE: public SMARTALLOC { virtual ssize_t d_write(int fd, const void *buffer, size_t count) = 0; virtual boffset_t d_lseek(DCR *dcr, boffset_t offset, int whence) = 0; virtual bool d_truncate(DCR *dcr) = 0; + virtual bool d_flush(DCR *dcr) { return true; }; /* * Locking and blocking calls