From 7e8706bada034ce428716963a5e3151723f83126 Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Fri, 30 Oct 2015 17:21:11 +0800 Subject: [PATCH 1/3] mds: choose EXCL state for filelock when client has Fb capability Signed-off-by: Yan, Zheng (cherry picked from commit 655ae79dd266dfd488aff67642aceab4ab383d01) --- src/mds/CInode.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc index cb5d8f283c3b6..2c384225b89d0 100644 --- a/src/mds/CInode.cc +++ b/src/mds/CInode.cc @@ -2675,7 +2675,7 @@ void CInode::choose_lock_state(SimpleLock *lock, int allissued) if (lock->is_xlocked()) { // do nothing here } else if (lock->get_state() != LOCK_MIX) { - if (issued & CEPH_CAP_GEXCL) + if (issued & (CEPH_CAP_GEXCL | CEPH_CAP_GBUFFER)) lock->set_state(LOCK_EXCL); else if (issued & CEPH_CAP_GWR) lock->set_state(LOCK_MIX); From 3fcf905dd65d6db514d403e2829b039df7bd9645 Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Fri, 30 Oct 2015 17:28:02 +0800 Subject: [PATCH 2/3] client: cancel revoking caps when reconnecting the mds Signed-off-by: Yan, Zheng (cherry picked from commit 9ab61b2779740d9038b169da0de616b89b07b813) --- src/client/Client.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/client/Client.cc b/src/client/Client.cc index 546066a1a619d..b1de66a225623 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -2487,14 +2487,16 @@ void Client::send_reconnect(MetaSession *session) bufferlist flockbl; _encode_filelocks(in, flockbl); - in->caps[mds]->seq = 0; // reset seq. - in->caps[mds]->issue_seq = 0; // reset seq. - in->caps[mds]->mseq = 0; // reset seq. + Cap *cap = in->caps[mds]; + cap->seq = 0; // reset seq. + cap->issue_seq = 0; // reset seq. + cap->mseq = 0; // reset seq. + cap->issued = cap->implemented; m->add_cap(p->first.ino, - in->caps[mds]->cap_id, + cap->cap_id, path.get_ino(), path.get_path(), // ino in->caps_wanted(), // wanted - in->caps[mds]->issued, // issued + cap->issued, // issued in->snaprealm->ino, flockbl); From 6c68971d1cfc27d30f92a83119e922180b389b43 Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Fri, 30 Oct 2015 21:04:04 +0800 Subject: [PATCH 3/3] mds: consider client's flushing caps when choosing lock states Client may flush and drop caps at the same time. If client need to send cap reconnect before the caps get flushed. The issued caps in the cap reconnect does not include the flushing caps. When choosing lock states, MDS only consider the issued caps in cap reconnect, it may choose wrong states. Fixes: #11482 Signed-off-by: Yan, Zheng (cherry picked from commit ce9a596dcaf95dd4af0a3a9e28871462a6bcb930) --- src/mds/CInode.cc | 4 ++-- src/mds/CInode.h | 2 +- src/mds/Locker.cc | 5 +++++ src/mds/MDCache.cc | 13 +++++++++++-- src/mds/MDCache.h | 4 ++++ 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc index 2c384225b89d0..489ccf6970c97 100644 --- a/src/mds/CInode.cc +++ b/src/mds/CInode.cc @@ -2694,9 +2694,9 @@ void CInode::choose_lock_state(SimpleLock *lock, int allissued) } } -void CInode::choose_lock_states() +void CInode::choose_lock_states(int dirty_caps) { - int issued = get_caps_issued(); + int issued = get_caps_issued() | dirty_caps; if (is_auth() && (issued & (CEPH_CAP_ANY_EXCL|CEPH_CAP_ANY_WR)) && choose_ideal_loner() >= 0) try_set_loner(); diff --git a/src/mds/CInode.h b/src/mds/CInode.h index a7abba43c2c9f..53283f73926b1 100644 --- a/src/mds/CInode.h +++ b/src/mds/CInode.h @@ -812,7 +812,7 @@ class CInode : public MDSCacheObject, public InodeStoreBase { // choose new lock state during recovery, based on issued caps void choose_lock_state(SimpleLock *lock, int allissued); - void choose_lock_states(); + void choose_lock_states(int dirty_caps); int count_nonstale_caps() { int n = 0; diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc index fb548a56e6d63..987ef82f80d26 100644 --- a/src/mds/Locker.cc +++ b/src/mds/Locker.cc @@ -2463,6 +2463,11 @@ void Locker::handle_client_caps(MClientCaps *m) << " op " << ceph_cap_op_name(m->get_op()) << dendl; if (!mds->is_clientreplay() && !mds->is_active() && !mds->is_stopping()) { + if (mds->is_reconnect() && + m->get_dirty() && m->get_client_tid() > 0 && + session->have_completed_flush(m->get_client_tid())) { + mdcache->set_reconnect_dirty_caps(m->get_ino(), m->get_dirty()); + } mds->wait_for_replay(new C_MDS_RetryMessage(mds, m)); return; } diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index 3ec852a97d136..d7b0bdebfaabb 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -5374,7 +5374,11 @@ void MDCache::choose_lock_states_and_reconnect_caps() if (in->is_auth() && !in->is_base() && in->inode.is_dirty_rstat()) in->mark_dirty_rstat(); - in->choose_lock_states(); + int dirty_caps = 0; + map::iterator it = cap_imports_dirty.find(in->ino()); + if (it != cap_imports_dirty.end()) + dirty_caps = it->second; + in->choose_lock_states(dirty_caps); dout(15) << " chose lock states on " << *in << dendl; SnapRealm *realm = in->find_snaprealm(); @@ -5520,6 +5524,7 @@ void MDCache::export_remaining_imported_caps() } cap_imports.clear(); + cap_imports_dirty.clear(); if (warn_str.peek() != EOF) { mds->clog->warn() << "failed to reconnect caps for missing inodes:" << "\n"; @@ -5542,7 +5547,11 @@ void MDCache::try_reconnect_cap(CInode *in, Session *session) if (in->is_replicated()) { mds->locker->try_eval(in, CEPH_CAP_LOCKS); } else { - in->choose_lock_states(); + int dirty_caps = 0; + map::iterator it = cap_imports_dirty.find(in->ino()); + if (it != cap_imports_dirty.end()) + dirty_caps = it->second; + in->choose_lock_states(dirty_caps); dout(15) << " chose lock states on " << *in << dendl; } } diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h index c369acdbf8d8c..30411b1689cf9 100644 --- a/src/mds/MDCache.h +++ b/src/mds/MDCache.h @@ -492,6 +492,7 @@ class MDCache { map > > cap_imports; // ino -> client -> frommds -> capex map cap_import_paths; + map cap_imports_dirty; set cap_imports_missing; int cap_imports_num_opening; @@ -549,6 +550,9 @@ class MDCache { assert(cap_imports[ino][client].size() == 1); cap_imports.erase(ino); } + void set_reconnect_dirty_caps(inodeno_t ino, int dirty) { + cap_imports_dirty[ino] |= dirty; + } // [reconnect/rejoin caps] map > reconnected_caps; // inode -> client -> realmino