Merge pull request #15442 from Vicente-Cheng/wip-20025-jewel

jewel: cephfs: osdc/Filer: truncate large file party by party

Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
Reviewed-by: Yan, Zheng <zyan@redhat.com>

src/common/config_opts.h

@@ -430,6 +430,8 @@ OPTION(objecter_inject_no_watch_ping, OPT_BOOL, false)   // suppress watch pings
 
 // Max number of deletes at once in a single Filer::purge call
 OPTION(filer_max_purge_ops, OPT_U32, 10)
+// Max number of truncate at once in a single Filer::truncate call
+OPTION(filer_max_truncate_ops, OPT_U32, 128)
 
 OPTION(journaler_allow_split_entries, OPT_BOOL, true)
 OPTION(journaler_write_head_interval, OPT_INT, 15)

src/mds/MDCache.cc

@@ -6249,9 +6249,8 @@ void MDCache::_truncate_inode(CInode *in, LogSegment *ls)
   filer.truncate(in->inode.ino, &in->inode.layout, *snapc,
 		 pi->truncate_size, pi->truncate_from-pi->truncate_size,
 		 pi->truncate_seq, ceph::real_time::min(), 0,
-		 0, new C_OnFinisher(new C_IO_MDC_TruncateFinish(this, in,
-								       ls),
-					   mds->finisher));
+		 new C_OnFinisher(new C_IO_MDC_TruncateFinish(this, in, ls),
+				  mds->finisher));
 }
 
 struct C_MDC_TruncateLogged : public MDCacheContext {

src/osdc/Filer.cc

@@ -383,3 +383,110 @@ void Filer::_do_purge_range(PurgeRange *pr, int fin)
 		     new C_OnFinisher(new C_PurgeRange(this, pr), finisher));
   }
 }
+
+// -----------------------
+struct TruncRange {
+  std::mutex lock;
+  typedef std::lock_guard<std::mutex> lock_guard;
+  typedef std::unique_lock<std::mutex> unique_lock;
+  inodeno_t ino;
+  file_layout_t layout;
+  SnapContext snapc;
+  ceph::real_time mtime;
+  int flags;
+  Context *oncommit;
+  int uncommitted;
+  uint64_t offset;
+  uint64_t length;
+  uint32_t truncate_seq;
+  TruncRange(inodeno_t i, const file_layout_t& l, const SnapContext& sc,
+	     ceph::real_time t, int fl, Context *fin,
+	     uint64_t off, uint64_t len, uint32_t ts)
+    : ino(i), layout(l), snapc(sc), mtime(t), flags(fl), oncommit(fin),
+      uncommitted(0), offset(off), length(len), truncate_seq(ts) {}
+};
+
+void Filer::truncate(inodeno_t ino,
+		     file_layout_t *layout,
+		     const SnapContext& snapc,
+		     uint64_t offset,
+		     uint64_t len,
+		     __u32 truncate_seq,
+		     ceph::real_time mtime,
+		     int flags,
+		     Context *oncommit)
+{
+  uint64_t period = layout->get_period();
+  uint64_t num_objs = Striper::get_num_objects(*layout, len + (offset % period));
+  if (num_objs == 1) {
+    vector<ObjectExtent> extents;
+    Striper::file_to_extents(cct, ino, layout, offset, len, 0, extents);
+    vector<OSDOp> ops(1);
+    ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC;
+    ops[0].op.extent.truncate_seq = truncate_seq;
+    ops[0].op.extent.truncate_size = extents[0].offset;
+    objecter->_modify(extents[0].oid, extents[0].oloc, ops, mtime, snapc,
+		      flags, NULL, oncommit);
+    return;
+  }
+
+  if (len > 0 && (offset + len) % period)
+    len += period - ((offset + len) % period);
+
+  TruncRange *tr = new TruncRange(ino, *layout, snapc, mtime, flags, oncommit,
+				  offset, len, truncate_seq);
+  _do_truncate_range(tr, 0);
+
+  return;
+}
+
+struct C_TruncRange : public Context {
+  Filer *filer;
+  TruncRange *tr;
+  C_TruncRange(Filer *f, TruncRange *t) : filer(f), tr(t) {}
+  void finish(int r) override {
+    filer->_do_truncate_range(tr, 1);
+  }
+};
+
+void Filer::_do_truncate_range(TruncRange *tr, int fin)
+{
+  TruncRange::unique_lock trl(tr->lock);
+  tr->uncommitted -= fin;
+  ldout(cct, 10) << "_do_truncate_range " << tr->ino << " objects " << tr->offset
+		 << "~" << tr->length << " uncommitted " << tr->uncommitted
+		 << dendl;
+
+  if (tr->length == 0 && tr->uncommitted == 0) {
+    tr->oncommit->complete(0);
+    trl.unlock();
+    delete tr;
+    return;
+  }
+
+  vector<ObjectExtent> extents;
+
+  int max = cct->_conf->filer_max_truncate_ops - tr->uncommitted;
+  if (max > 0 && tr->length > 0) {
+    uint64_t len = tr->layout.get_period() * max;
+    if (len > tr->length)
+      len = tr->length;
+
+    uint64_t offset = tr->offset + tr->length - len;
+    Striper::file_to_extents(cct, tr->ino, &tr->layout, offset, len, 0, extents);
+    tr->uncommitted += extents.size();
+    tr->length -= len;
+  }
+
+  trl.unlock();
+
+  // Issue objecter ops outside tr->lock to avoid lock dependency loop
+  for (const auto& p : extents) {
+    vector<OSDOp> ops(1);
+    ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC;
+    ops[0].op.extent.truncate_size = p.offset;
+    ops[0].op.extent.truncate_seq = tr->truncate_seq;
+    objecter->_modify(p.oid, p.oloc, ops, tr->mtime, tr->snapc, tr->flags, NULL,
+		      new C_OnFinisher(new C_TruncRange(this, tr), finisher));
+  }
+}

src/osdc/Filer.h

@@ -192,44 +192,16 @@ class Filer {
     return 0;
   }
 
-  int truncate(inodeno_t ino,
+  void truncate(inodeno_t ino,
 	       file_layout_t *layout,
 	       const SnapContext& snapc,
 	       uint64_t offset,
 	       uint64_t len,
 	       __u32 truncate_seq,
 	       ceph::real_time mtime,
 	       int flags,
-	       Context *onack,
-	       Context *oncommit) {
-    vector<ObjectExtent> extents;
-    Striper::file_to_extents(cct, ino, layout, offset, len, 0, extents);
-    if (extents.size() == 1) {
-      vector<OSDOp> ops(1);
-      ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC;
-      ops[0].op.extent.truncate_seq = truncate_seq;
-      ops[0].op.extent.truncate_size = extents[0].offset;
-      objecter->_modify(extents[0].oid, extents[0].oloc, ops, mtime, snapc,
-			flags, onack, oncommit);
-    } else {
-      C_GatherBuilder gack(cct, onack);
-      C_GatherBuilder gcom(cct, oncommit);
-      for (vector<ObjectExtent>::iterator p = extents.begin();
-	   p != extents.end();
-	   ++p) {
-	vector<OSDOp> ops(1);
-	ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC;
-	ops[0].op.extent.truncate_size = p->offset;
-	ops[0].op.extent.truncate_seq = truncate_seq;
-	objecter->_modify(p->oid, p->oloc, ops, mtime, snapc, flags,
-			  onack ? gack.new_sub():0,
-			  oncommit ? gcom.new_sub():0);
-      }
-      gack.activate();
-      gcom.activate();
-    }
-    return 0;
-  }
+	       Context *oncommit);
+  void _do_truncate_range(struct TruncRange *pr, int fin);
 
   int zero(inodeno_t ino,
 	   file_layout_t *layout,