droplet: adapted to bareos-18.2 after merge

core/src/stored/backends/chunked_device.cc

@@ -680,6 +680,12 @@ int chunked_device::SetupChunk(const char *pathname, int flags, int mode)
       return -1;
    }
 
+   if (!check_remote()) {
+      Dmsg0(100, "setup_chunk failed, as remote device is not available\n");
+      dev_errno = EIO; /**< I/O error */
+      return -1;
+   }
+
    if (!current_chunk_) {
       current_chunk_ = (chunk_descriptor *)malloc(sizeof(chunk_descriptor));
       memset(current_chunk_, 0, sizeof(chunk_descriptor));
@@ -1190,6 +1196,88 @@ ssize_t chunked_device::ChunkedVolumeSize()
    return chunked_remote_volume_size();
 }
 
+bool chunked_device::is_written()
+{
+   /*
+    * See if we are using io-threads or not and the ordered circbuf is created.
+    * We try to make sure that nothing of the volume being requested is still inflight as then
+    * the chunked_remote_volume_size() method will fail to determine the size of the data as
+    * its not fully stored on the backing store yet.
+    */
+
+   if (current_chunk_->need_flushing) {
+      Dmsg1(100, "volume %s is pending, as current chunk needs flushing\n", current_volname_);
+      return false;
+   }
+
+   /*
+    * Make sure there is also nothing inflight to the backing store anymore.
+    */
+   int inflight_chunks = NrInflightChunks();
+   if (inflight_chunks > 0) {
+      Dmsg2(100, "volume %s is pending, as there are %d inflight chunks\n", current_volname_, inflight_chunks);
+      return false;
+   }
+
+   if (io_threads_ > 0 && cb_) {
+
+      if (!cb_->empty()) {
+
+         chunk_io_request *request;
+
+         /*
+          * Peek on the ordered circular queue if there are any pending IO-requests
+          * for this volume. If there are use that as the indication of the size of
+          * the volume and don't contact the remote storage as there is still data
+          * inflight and as such we need to look at the last chunk that is still not
+          * uploaded of the volume.
+          */
+         request = (chunk_io_request *)cb_->peek(PEEK_FIRST, current_volname_, compare_volume_name);
+         if (request) {
+            free(request);
+            Dmsg1(100, "volume %s is pending, as there are queued write requests\n", current_volname_);
+            return false;
+         }
+      }
+   }
+
+   /* compare expected to written volume size */
+   ssize_t remote_volume_size = chunked_remote_volume_size();
+   Dmsg3(100, "volume: %s, chunked_remote_volume_size = %lld, VolCatInfo.VolCatBytes = %lld\n",
+         current_volname_, remote_volume_size, VolCatInfo.VolCatBytes);
+
+   if (remote_volume_size < VolCatInfo.VolCatBytes) {
+      Dmsg3(100, "volume %s is pending, as 'remote volume size' = %lld < 'catalog volume size' = %lld\n",
+            current_volname_, remote_volume_size, VolCatInfo.VolCatBytes);
+      return false;
+   }
+
+   return true;
+}
+
+
+/*
+ * Busy waits until write buffer is empty.
+ */
+bool chunked_device::wait_until_chunks_written()
+{
+   bool retval = true;
+
+   if (current_chunk_->need_flushing) {
+      if (!FlushChunk(false /* release */, false /* move_to_next_chunk */)) {
+         dev_errno = EIO;
+         retval = false;
+      }
+   }
+
+   while (!is_written()) {
+      bmicrosleep(DEFAULT_RECHECK_INTERVAL_WRITE_BUFFER, 0);
+   }
+
+   return retval;
+}
+
+
 static int CloneIoRequest(void *item1, void *item2)
 {
    chunk_io_request *src = (chunk_io_request *)item1;
@@ -1348,9 +1436,20 @@ bool chunked_device::DeviceStatus(bsdDevStatTrig *dst)
    POOL_MEM inflights(PM_MESSAGE);
 
    dst->status_length = 0;
-   if (io_threads_ > 0 && cb_) {
-      if (!cb_->empty()) {
-         dst->status_length = PmStrcpy(dst->status, _("Pending IO flush requests:\n"));
+   if (check_remote()) {
+      dst->status_length = pm_strcpy(dst->status, _("Backend connection is working.\n"));
+   } else {
+      dst->status_length = pm_strcpy(dst->status, _("Backend connection is not working.\n"));
+   }
+   if (m_io_threads > 0 && m_cb) {
+      if (nr_inflight_chunks() > 0) {
+         pending = true;
+         inflights.bsprintf("Inflight chunks: %d\n", nr_inflight_chunks());
+         dst->status_length = pm_strcat(dst->status, inflights.c_str());
+      }
+      if (!m_cb->empty()) {
+         pending = true;
+         dst->status_length = pm_strcat(dst->status, _("Pending IO flush requests:\n"));
 
          /*
           * Peek on the ordered circular queue and list all pending requests.

core/src/stored/backends/chunked_device.h

@@ -121,6 +121,7 @@ class chunked_device: public Device {
    bool EnqueueChunk(chunk_io_request *request);
    bool FlushChunk(bool release_chunk, bool move_to_next_chunk);
    bool ReadChunk();
+   bool is_written();
 
 protected:
    /*
@@ -151,6 +152,8 @@ class chunked_device: public Device {
    /*
     * Methods implemented by inheriting class.
     */
+   virtual bool check_remote() = 0;
+   virtual bool remote_chunked_volume_exists() = 0;
    virtual bool FlushRemoteChunk(chunk_io_request *request) = 0;
    virtual bool ReadRemoteChunk(chunk_io_request *request) = 0;
    virtual ssize_t chunked_remote_volume_size() = 0;

core/src/stored/backends/droplet_device.cc

@@ -159,15 +159,6 @@ static inline int DropletErrnoToSystemErrno(dpl_status_t status)
    return errno;
 }
 
-/**
- * Generic callback for the WalkDplDirectory() function.
- *
- * Returns true - abort loop
- *         false - continue loop
- */
-typedef bool (*t_call_back)(dpl_dirent_t *dirent, dpl_ctx_t *ctx,
-                            const char *dirname, void *data);
-
 /*
  * Callback for getting the total size of a chunked volume.
  */
@@ -221,7 +212,7 @@ static dpl_status_t chunked_volume_truncate_callback(dpl_dirent_t *dirent, dpl_c
 /*
  * Callback for getting the total size of a chunked volume.
  */
-static bool WalkDplDirectory(dpl_ctx_t *ctx, const char *dirname, t_call_back callback, void *data)
+static dpl_status_t chunked_volume_size_callback(dpl_sysmd_t *sysmd, dpl_ctx_t *ctx, const char *chunkpath, void *data)
 {
    dpl_status_t status = DPL_SUCCESS;
    ssize_t *volumesize = (ssize_t *)data;
@@ -444,7 +435,7 @@ bool droplet_device::FlushRemoteChunk(chunk_io_request *request)
    dpl_option_t dpl_options;
    dpl_sysmd_t *sysmd = NULL;
    PoolMem chunk_dir(PM_FNAME),
-            chunk_name(PM_FNAME);
+           chunk_name(PM_FNAME);
 
    Mmsg(chunk_dir, "/%s", request->volname);
    Mmsg(chunk_name, "%s/%04d", chunk_dir.c_str(), request->chunk);
@@ -670,7 +661,9 @@ bool droplet_device::TruncateRemoteChunkedVolume(DeviceControlRecord *dcr)
 
    Dmsg1(100, "truncate_remote_chunked_volume(%s) start.\n", getVolCatName());
    Mmsg(chunk_dir, "/%s", getVolCatName());
-   if (!WalkDplDirectory(ctx_, chunk_dir.c_str(), chunked_volume_truncate_callback, NULL)) {
+   bool ignore_gaps = true;
+   if (!walk_chunks(chunk_dir.c_str(), chunked_volume_truncate_callback, NULL, ignore_gaps)) {
+      /* errno already set in walk_chunks. */
       return false;
    }
    Dmsg1(100, "truncate_remote_chunked_volume(%s) finished.\n", getVolCatName());
@@ -976,7 +969,7 @@ ssize_t droplet_device::chunked_remote_volume_size()
    /*
     * FIXME: With the current version of libdroplet a dpl_getattr() on a directory
     *        fails with DPL_ENOENT even when the directory does exist. All other
-    *        operations succeed and as WalkDplDirectory() does a dpl_chdir() anyway
+    *        operations succeed and as walk_chunks() does a dpl_chdir() anyway
     *        that will fail if the directory doesn't exist for now we should be
     *        mostly fine.
     */
@@ -1009,7 +1002,9 @@ ssize_t droplet_device::chunked_remote_volume_size()
    }
 #endif
 
-   if (!WalkDplDirectory(ctx_, chunk_dir.c_str(), chunked_volume_size_callback, &volumesize)) {
+   Dmsg1(100, "get chunked_remote_volume_size(%s)\n", getVolCatName());
+   if (!walk_chunks(chunk_dir.c_str(), chunked_volume_size_callback, &volumesize)) {
+      /* errno is already set in walk_chunks */
       volumesize = -1;
       goto bail_out;
    }

core/src/stored/backends/droplet_device.h

@@ -32,12 +32,26 @@
 #include <droplet/vfs.h>
 
 namespace storagedaemon {
+/*
+ * Generic callback for the droplet_device::walk_directory() function.
+ *
+ * Returns DPL_SUCCESS - success
+ *         other dpl_status_t value: failure
+ */
+typedef dpl_status_t (*t_dpl_walk_directory_call_back)(dpl_dirent_t *dirent, dpl_ctx_t *ctx,
+                                                       const char *dirname, void *data);
+typedef dpl_status_t (*t_dpl_walk_chunks_call_back)(dpl_sysmd_t *sysmd, dpl_ctx_t *ctx, const char *chunkpath, void *data);
+
+
+
 
 class droplet_device: public chunked_device {
 private:
    /*
     * Private Members
     */
+   /* maximun number of chunks in a volume (0000 to 9999) */
+   const int m_max_chunks = 10000;
    char *configstring_;
    const char *profile_;
    const char *location_;
@@ -56,6 +70,8 @@ class droplet_device: public chunked_device {
    /*
     * Interface from chunked_device
     */
+   bool CheckRemote();
+   bool remote_chunked_volume_exists();
    bool FlushRemoteChunk(chunk_io_request *request);
    bool ReadRemoteChunk(chunk_io_request *request);
    ssize_t chunked_remote_volume_size();