librbd: send FLUSH_SOURCE_INTERNAL when do copy/deep_copy.

copy/deep_copy use object_map to judge whether object exist.
If w/ librbdo pwl cache, flush can't flush data to osd which
change objectmap state. So we should send flush w/ FLUSH_SOURCE_INTERNAL
to make data flush to osd.

Fixes:https://tracker.ceph.com/issues/53057

Signed-off-by: Jianpeng Ma <jianpeng.ma@intel.com>

src/librbd/api/Image.cc

@@ -26,6 +26,8 @@
 #include "librbd/image/PreRemoveRequest.h"
 #include "librbd/io/ImageDispatcherInterface.h"
 #include "librbd/io/ObjectDispatcherInterface.h"
+#include "librbd/io/AioCompletion.h"
+#include "librbd/io/ImageDispatchSpec.h"
 #include <boost/scope_exit.hpp>
 
 #define dout_subsys ceph_subsys_rbd
@@ -680,6 +682,21 @@ int Image<I>::deep_copy(I *src, librados::IoCtx& dest_md_ctx,
 template <typename I>
 int Image<I>::deep_copy(I *src, I *dest, bool flatten,
                         ProgressContext &prog_ctx) {
+  C_SaferCond flush_ctx;
+  {
+    std::shared_lock owner_locker{src->owner_lock};
+    auto aio_comp = io::AioCompletion::create_and_start(&flush_ctx, src,
+                                                        io::AIO_TYPE_FLUSH);
+    auto req = io::ImageDispatchSpec::create_flush(
+      *src, io::IMAGE_DISPATCH_LAYER_INTERNAL_START,
+      aio_comp, io::FLUSH_SOURCE_INTERNAL, {});
+    req->send();
+  }
+  int r = flush_ctx.wait();
+  if (r < 0) {
+    return r;
+  }
+
   librados::snap_t snap_id_start = 0;
   librados::snap_t snap_id_end;
   {
@@ -696,7 +713,7 @@ int Image<I>::deep_copy(I *src, I *dest, bool flatten,
     src, dest, snap_id_start, snap_id_end, 0U, flatten, boost::none,
     asio_engine.get_work_queue(), &snap_seqs, &progress_handler, &cond);
   req->send();
-  int r = cond.wait();
+  r = cond.wait();
   if (r < 0) {
     return r;
   }

src/librbd/internal.cc

@@ -1281,12 +1281,26 @@ int validate_pool(IoCtx &io_ctx, CephContext *cct) {
       return -EINVAL;
     }
 
+    C_SaferCond flush_ctx;
+    {
+      auto aio_comp = io::AioCompletion::create_and_start(&flush_ctx, src,
+	  io::AIO_TYPE_FLUSH);
+      auto req = io::ImageDispatchSpec::create_flush(
+	  *src, io::IMAGE_DISPATCH_LAYER_INTERNAL_START,
+	  aio_comp, io::FLUSH_SOURCE_INTERNAL, {});
+      req->send();
+    }
+    int r = flush_ctx.wait();
+    if (r < 0) {
+      return r;
+    }
+
     C_SaferCond ctx;
     auto req = deep_copy::MetadataCopyRequest<>::create(
       src, dest, &ctx);
     req->send();
 
-    int r = ctx.wait();
+    r = ctx.wait();
     if (r < 0) {
       lderr(cct) << "failed to copy metadata: " << cpp_strerror(r) << dendl;
       return r;