From c49e249fb5a5232dc25b8250f4b7d2b4e3d47bef Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Tue, 25 Apr 2017 16:21:24 +0800 Subject: [PATCH] osdc/Filer: truncate large file party by party Fixes: http://tracker.ceph.com/issues/19755 Signed-off-by: "Yan, Zheng" (cherry picked from commit 5fab215e461e5ecc36c0f9d9ea867f6c45e80263) Conflicts: src/osdc/Filer.h src/osdc/Filer.cc - add parameter to fit _modify (need onack parameter) src/mds/MDCache.cc - make truncate() consistency --- src/common/config_opts.h | 2 + src/mds/MDCache.cc | 5 +- src/osdc/Filer.cc | 107 +++++++++++++++++++++++++++++++++++++++ src/osdc/Filer.h | 34 ++----------- 4 files changed, 114 insertions(+), 34 deletions(-) diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 60ff9772e5392..f005d761058e1 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -430,6 +430,8 @@ OPTION(objecter_inject_no_watch_ping, OPT_BOOL, false) // suppress watch pings // Max number of deletes at once in a single Filer::purge call OPTION(filer_max_purge_ops, OPT_U32, 10) +// Max number of truncate at once in a single Filer::truncate call +OPTION(filer_max_truncate_ops, OPT_U32, 128) OPTION(journaler_allow_split_entries, OPT_BOOL, true) OPTION(journaler_write_head_interval, OPT_INT, 15) diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index d3e9336c36800..01eb90acd519e 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -6249,9 +6249,8 @@ void MDCache::_truncate_inode(CInode *in, LogSegment *ls) filer.truncate(in->inode.ino, &in->inode.layout, *snapc, pi->truncate_size, pi->truncate_from-pi->truncate_size, pi->truncate_seq, ceph::real_time::min(), 0, - 0, new C_OnFinisher(new C_IO_MDC_TruncateFinish(this, in, - ls), - mds->finisher)); + new C_OnFinisher(new C_IO_MDC_TruncateFinish(this, in, ls), + mds->finisher)); } struct C_MDC_TruncateLogged : public MDCacheContext { diff --git a/src/osdc/Filer.cc b/src/osdc/Filer.cc index f64a122534841..5aef11610ac17 100644 --- a/src/osdc/Filer.cc +++ b/src/osdc/Filer.cc @@ -383,3 +383,110 @@ void Filer::_do_purge_range(PurgeRange *pr, int fin) new C_OnFinisher(new C_PurgeRange(this, pr), finisher)); } } + +// ----------------------- +struct TruncRange { + std::mutex lock; + typedef std::lock_guard lock_guard; + typedef std::unique_lock unique_lock; + inodeno_t ino; + file_layout_t layout; + SnapContext snapc; + ceph::real_time mtime; + int flags; + Context *oncommit; + int uncommitted; + uint64_t offset; + uint64_t length; + uint32_t truncate_seq; + TruncRange(inodeno_t i, const file_layout_t& l, const SnapContext& sc, + ceph::real_time t, int fl, Context *fin, + uint64_t off, uint64_t len, uint32_t ts) + : ino(i), layout(l), snapc(sc), mtime(t), flags(fl), oncommit(fin), + uncommitted(0), offset(off), length(len), truncate_seq(ts) {} +}; + +void Filer::truncate(inodeno_t ino, + file_layout_t *layout, + const SnapContext& snapc, + uint64_t offset, + uint64_t len, + __u32 truncate_seq, + ceph::real_time mtime, + int flags, + Context *oncommit) +{ + uint64_t period = layout->get_period(); + uint64_t num_objs = Striper::get_num_objects(*layout, len + (offset % period)); + if (num_objs == 1) { + vector extents; + Striper::file_to_extents(cct, ino, layout, offset, len, 0, extents); + vector ops(1); + ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC; + ops[0].op.extent.truncate_seq = truncate_seq; + ops[0].op.extent.truncate_size = extents[0].offset; + objecter->_modify(extents[0].oid, extents[0].oloc, ops, mtime, snapc, + flags, NULL, oncommit); + return; + } + + if (len > 0 && (offset + len) % period) + len += period - ((offset + len) % period); + + TruncRange *tr = new TruncRange(ino, *layout, snapc, mtime, flags, oncommit, + offset, len, truncate_seq); + _do_truncate_range(tr, 0); + + return; +} + +struct C_TruncRange : public Context { + Filer *filer; + TruncRange *tr; + C_TruncRange(Filer *f, TruncRange *t) : filer(f), tr(t) {} + void finish(int r) override { + filer->_do_truncate_range(tr, 1); + } +}; + +void Filer::_do_truncate_range(TruncRange *tr, int fin) +{ + TruncRange::unique_lock trl(tr->lock); + tr->uncommitted -= fin; + ldout(cct, 10) << "_do_truncate_range " << tr->ino << " objects " << tr->offset + << "~" << tr->length << " uncommitted " << tr->uncommitted + << dendl; + + if (tr->length == 0 && tr->uncommitted == 0) { + tr->oncommit->complete(0); + trl.unlock(); + delete tr; + return; + } + + vector extents; + + int max = cct->_conf->filer_max_truncate_ops - tr->uncommitted; + if (max > 0 && tr->length > 0) { + uint64_t len = tr->layout.get_period() * max; + if (len > tr->length) + len = tr->length; + + uint64_t offset = tr->offset + tr->length - len; + Striper::file_to_extents(cct, tr->ino, &tr->layout, offset, len, 0, extents); + tr->uncommitted += extents.size(); + tr->length -= len; + } + + trl.unlock(); + + // Issue objecter ops outside tr->lock to avoid lock dependency loop + for (const auto& p : extents) { + vector ops(1); + ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC; + ops[0].op.extent.truncate_size = p.offset; + ops[0].op.extent.truncate_seq = tr->truncate_seq; + objecter->_modify(p.oid, p.oloc, ops, tr->mtime, tr->snapc, tr->flags, NULL, + new C_OnFinisher(new C_TruncRange(this, tr), finisher)); + } +} diff --git a/src/osdc/Filer.h b/src/osdc/Filer.h index 96c11b80461a0..9cf8c92df77bb 100644 --- a/src/osdc/Filer.h +++ b/src/osdc/Filer.h @@ -192,7 +192,7 @@ class Filer { return 0; } - int truncate(inodeno_t ino, + void truncate(inodeno_t ino, file_layout_t *layout, const SnapContext& snapc, uint64_t offset, @@ -200,36 +200,8 @@ class Filer { __u32 truncate_seq, ceph::real_time mtime, int flags, - Context *onack, - Context *oncommit) { - vector extents; - Striper::file_to_extents(cct, ino, layout, offset, len, 0, extents); - if (extents.size() == 1) { - vector ops(1); - ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC; - ops[0].op.extent.truncate_seq = truncate_seq; - ops[0].op.extent.truncate_size = extents[0].offset; - objecter->_modify(extents[0].oid, extents[0].oloc, ops, mtime, snapc, - flags, onack, oncommit); - } else { - C_GatherBuilder gack(cct, onack); - C_GatherBuilder gcom(cct, oncommit); - for (vector::iterator p = extents.begin(); - p != extents.end(); - ++p) { - vector ops(1); - ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC; - ops[0].op.extent.truncate_size = p->offset; - ops[0].op.extent.truncate_seq = truncate_seq; - objecter->_modify(p->oid, p->oloc, ops, mtime, snapc, flags, - onack ? gack.new_sub():0, - oncommit ? gcom.new_sub():0); - } - gack.activate(); - gcom.activate(); - } - return 0; - } + Context *oncommit); + void _do_truncate_range(struct TruncRange *pr, int fin); int zero(inodeno_t ino, file_layout_t *layout,