Skip to content

Commit

Permalink
mds: fix the lock order
Browse files Browse the repository at this point in the history
  • Loading branch information
@lxbsz
lxbsz committed Sep 12, 2023
1 parent 91c5e9d commit 54bd96e
Show file tree
Hide file tree
Showing 2 changed files with 75 additions and 15 deletions.
88 changes: 73 additions & 15 deletions src/mds/MDCache.cc
View file
Expand Up @@ -8264,6 +8264,28 @@ void MDCache::dispatch(const cref_t<Message> &m)
}
}

int MDCache::check_fscrypt_access(MDRequestRef& mdr, CInode *in, bool acquire_lock)
{
const cref_t<MClientRequest> &req = mdr->client_request;

if (acquire_lock) {
MutationImpl::LockOpVec lov;
/* We need 'As' caps for the fscrypt context */
lov.add_rdlock(&in->authlock);
if (!mds->locker->acquire_locks(mdr, lov)) {
dout(10) << "traverse: failed to rdlock" << dendl;
return 1; /* XXX */
}
}
if (!in->get_inode()->fscrypt_auth.empty()) {
dout(10) << "blocking '" << ceph_mds_op_name(req->get_op())
<< "' operation in encrypted node" << dendl;
return -CEPHFS_EROFS;
}

return 0;
}

int MDCache::path_traverse(MDRequestRef& mdr, MDSContextFactory& cf,
const filepath& path, int flags,
vector<CDentry*> *pdnvec, CInode **pin)
Expand All @@ -8278,6 +8300,7 @@ int MDCache::path_traverse(MDRequestRef& mdr, MDSContextFactory& cf,
bool rdlock_path = (flags & MDS_TRAVERSE_RDLOCK_PATH);
bool xlock_dentry = (flags & MDS_TRAVERSE_XLOCK_DENTRY);
bool rdlock_authlock = (flags & MDS_TRAVERSE_RDLOCK_AUTHLOCK);
bool fscrypt_rdlock_authlock = false;

if (forward)
ceph_assert(mdr); // forward requires a request
Expand Down Expand Up @@ -8331,18 +8354,7 @@ int MDCache::path_traverse(MDRequestRef& mdr, MDSContextFactory& cf,
req->get_op() != CEPH_MDS_OP_READDIR &&
req->get_op() != CEPH_MDS_OP_UNLINK &&
req->get_op() != CEPH_MDS_OP_RMDIR) {
MutationImpl::LockOpVec lov;
/* We need 'As' caps for the fscrypt context */
lov.add_rdlock(&cur->authlock);
if (!mds->locker->acquire_locks(mdr, lov)) {
dout(10) << "traverse: failed to rdlock" << dendl;
return 1; /* XXX */
}
if (!cur->get_inode()->fscrypt_auth.empty()) {
dout(10) << "blocking '" << ceph_mds_op_name(req->get_op())
<< "' operation in encrypted node" << dendl;
return -CEPHFS_EROFS;
}
fscrypt_rdlock_authlock = true;
}
} else if (rdlock_snap) {
int n = (flags & MDS_TRAVERSE_RDLOCK_SNAP2) ? 1 : 0;
Expand Down Expand Up @@ -8478,32 +8490,53 @@ int MDCache::path_traverse(MDRequestRef& mdr, MDSContextFactory& cf,
if (depth > 0 || !mdr->lock_cache) {
lov.add_wrlock(&cur->filelock);
lov.add_wrlock(&cur->nestlock);
if (rdlock_authlock)
if (rdlock_authlock || (!depth && fscrypt_rdlock_authlock))
lov.add_rdlock(&cur->authlock);
}
lov.add_xlock(&dn->lock);
} else {
// force client to flush async dir operation if necessary
if (cur->filelock.is_cached())
lov.add_wrlock(&cur->filelock);
if (!depth && fscrypt_rdlock_authlock)
lov.add_rdlock(&cur->authlock);
lov.add_rdlock(&dn->lock);
}
if (!mds->locker->acquire_locks(mdr, lov)) {
dout(10) << "traverse: failed to rdlock " << dn->lock << " " << *dn << dendl;
return 1;
}
if (!depth && fscrypt_rdlock_authlock) {
r = check_fscrypt_access(mdr, cur);
if (r)
return r;
}
} else if (!path_locked &&
!dn->lock.can_read(client) &&
!(dn->lock.is_xlocked() && dn->lock.get_xlock_by() == mdr)) {
dout(10) << "traverse: non-readable dentry at " << *dn << dendl;

if (!depth && fscrypt_rdlock_authlock) {
r = check_fscrypt_access(mdr, cur, true);
if (r)
return r;
}

dn->lock.add_waiter(SimpleLock::WAIT_RD, cf.build());
if (mds->logger)
mds->logger->inc(l_mds_traverse_lock);
if (dn->is_auth() && dn->lock.is_unstable_and_locked())
mds->mdlog->flush();
return 1;
} else {
if (!depth && fscrypt_rdlock_authlock) {
r = check_fscrypt_access(mdr, cur, true);
if (r)
return r;
}
}


if (pdnvec)
pdnvec->push_back(dn);

Expand Down Expand Up @@ -8579,6 +8612,8 @@ int MDCache::path_traverse(MDRequestRef& mdr, MDSContextFactory& cf,
!curdir->is_in_bloom(path[depth]))) {
// file not found
if (pdnvec) {
bool need_lock = true;

// instantiate a null dn?
if (depth < path.depth() - 1) {
dout(20) << " didn't traverse full path; not returning pdnvec" << dendl;
Expand All @@ -8599,22 +8634,30 @@ int MDCache::path_traverse(MDRequestRef& mdr, MDSContextFactory& cf,
if (depth > 0 || !mdr->lock_cache) {
lov.add_wrlock(&cur->filelock);
lov.add_wrlock(&cur->nestlock);
if (rdlock_authlock)
if (rdlock_authlock || (!depth && fscrypt_rdlock_authlock))
lov.add_rdlock(&cur->authlock);
}
lov.add_xlock(&dn->lock);
} else {
// force client to flush async dir operation if necessary
if (cur->filelock.is_cached())
lov.add_wrlock(&cur->filelock);
if (!depth && fscrypt_rdlock_authlock)
lov.add_rdlock(&cur->authlock);
lov.add_rdlock(&dn->lock);
}
if (!mds->locker->acquire_locks(mdr, lov)) {
dout(10) << "traverse: failed to rdlock " << dn->lock << " " << *dn << dendl;
return 1;
}
need_lock = false;
}
}
if (!depth && fscrypt_rdlock_authlock) {
r = check_fscrypt_access(mdr, cur, need_lock);
if (r)
return r;
}
if (dn) {
pdnvec->push_back(dn);
if (want_dentry)
Expand All @@ -8634,6 +8677,11 @@ int MDCache::path_traverse(MDRequestRef& mdr, MDSContextFactory& cf,
return -CEPHFS_EIO;
}

if (!depth && fscrypt_rdlock_authlock) {
r = check_fscrypt_access(mdr, cur, true);
if (r)
return r;
}
// directory isn't complete; reload
dout(7) << "traverse: incomplete dir contents for " << *cur << ", fetching" << dendl;
touch_inode(cur);
Expand All @@ -8642,8 +8690,13 @@ int MDCache::path_traverse(MDRequestRef& mdr, MDSContextFactory& cf,
return 1;
}
} else {
// dirfrag/dentry is not mine.
if (!depth && fscrypt_rdlock_authlock) {
r = check_fscrypt_access(mdr, cur, true);
if (r)
return r;
}

// dirfrag/dentry is not mine.
if (forward &&
mdr && mdr->client_request &&
(int)depth < mdr->client_request->get_num_fwd()){
Expand Down Expand Up @@ -8682,6 +8735,11 @@ int MDCache::path_traverse(MDRequestRef& mdr, MDSContextFactory& cf,
if (want_dentry && !want_inode) {
return -CEPHFS_ENOENT;
}
if (fscrypt_rdlock_authlock) {
r = check_fscrypt_access(mdr, cur, true);
if (r)
return r;
}
target_inode = cur;
}

Expand Down
2 changes: 2 additions & 0 deletions src/mds/MDCache.h
View file
Expand Up @@ -801,6 +801,8 @@ class MDCache {
void open_foreign_mdsdir(inodeno_t ino, MDSContext *c);
CDir *get_stray_dir(CInode *in);

int check_fscrypt_access(MDRequestRef& mdr, CInode *in, bool acquire_lock=false);

/**
* Find the given dentry (and whether it exists or not), its ancestors,
* and get them all into memory and usable on this MDS. This function
Expand Down

0 comments on commit 54bd96e

Please sign in to comment.