From 59b26bad1667956e92d59f610afb6a1f9976df7c Mon Sep 17 00:00:00 2001 From: dparmar18 Date: Mon, 7 Nov 2022 17:39:55 +0530 Subject: [PATCH 1/6] mds: add new feature to block clients from establishing sessions Fixes: https://tracker.ceph.com/issues/57090 Signed-off-by: Dhairya Parmar --- src/include/ceph_fs.h | 3 ++- src/mds/MDSMap.cc | 3 +++ src/mds/MDSMap.h | 3 ++- src/mds/Server.cc | 10 ++++++++++ src/mon/FSCommands.cc | 32 ++++++++++++++++++++++++++++++++ src/mon/MonCommands.h | 3 ++- 6 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h index c83ffa4eec63e..10f13a5577730 100644 --- a/src/include/ceph_fs.h +++ b/src/include/ceph_fs.h @@ -286,7 +286,8 @@ struct ceph_mon_subscribe_ack { #define CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS (1<<4) /* cluster alllowed to enable MULTIMDS and SNAPS at the same time */ #define CEPH_MDSMAP_ALLOW_STANDBY_REPLAY (1<<5) /* cluster alllowed to enable MULTIMDS */ - +#define CEPH_MDSMAP_REFUSE_CLIENT_SESSION (1<<6) /* cluster allowed to refuse client session + request */ #define CEPH_MDSMAP_DEFAULTS (CEPH_MDSMAP_ALLOW_SNAPS | \ CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS) diff --git a/src/mds/MDSMap.cc b/src/mds/MDSMap.cc index d2a0a83ecc293..6433d08c5d256 100644 --- a/src/mds/MDSMap.cc +++ b/src/mds/MDSMap.cc @@ -234,6 +234,7 @@ void MDSMap::dump_flags_state(Formatter *f) const f->dump_bool(flag_display.at(CEPH_MDSMAP_ALLOW_SNAPS), allows_snaps()); f->dump_bool(flag_display.at(CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS), allows_multimds_snaps()); f->dump_bool(flag_display.at(CEPH_MDSMAP_ALLOW_STANDBY_REPLAY), allows_standby_replay()); + f->dump_bool(flag_display.at(CEPH_MDSMAP_REFUSE_CLIENT_SESSION), test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION)); f->close_section(); } @@ -373,6 +374,8 @@ void MDSMap::print_flags(std::ostream& out) const { out << " " << flag_display.at(CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS); if (allows_standby_replay()) out << " " << flag_display.at(CEPH_MDSMAP_ALLOW_STANDBY_REPLAY); + if (test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION)) + out << " " << flag_display.at(CEPH_MDSMAP_REFUSE_CLIENT_SESSION); } void MDSMap::get_health(list >& summary, diff --git a/src/mds/MDSMap.h b/src/mds/MDSMap.h index 0ce99bd3e1511..75c44e27c5f00 100644 --- a/src/mds/MDSMap.h +++ b/src/mds/MDSMap.h @@ -663,7 +663,8 @@ class MDSMap { {CEPH_MDSMAP_NOT_JOINABLE, "joinable"}, //inverse for user display {CEPH_MDSMAP_ALLOW_SNAPS, "allow_snaps"}, {CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS, "allow_multimds_snaps"}, - {CEPH_MDSMAP_ALLOW_STANDBY_REPLAY, "allow_standby_replay"} + {CEPH_MDSMAP_ALLOW_STANDBY_REPLAY, "allow_standby_replay"}, + {CEPH_MDSMAP_REFUSE_CLIENT_SESSION, "refuse_client_session"} }; }; WRITE_CLASS_ENCODER_FEATURES(MDSMap::mds_info_t) diff --git a/src/mds/Server.cc b/src/mds/Server.cc index bbfa55144f570..f0a4b62648dca 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -580,6 +580,16 @@ void Server::handle_client_session(const cref_t &m) uint64_t sseq = 0; switch (m->get_op()) { case CEPH_SESSION_REQUEST_OPEN: + if(mds->mdsmap->test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION)) { + dout(0) << "new sessions are not permitted, enable again via" + "`ceph fs set refuse_client_session false`" << dendl; + auto reply = make_message(CEPH_SESSION_REJECT); + reply->metadata["error_string"] = "new sessions are not permitted," + " enable again via `ceph fs set" + " refuse_client_session false`"; + mds->send_message(reply, m->get_connection()); + return; + } if (session->is_opening() || session->is_open() || session->is_stale() || diff --git a/src/mon/FSCommands.cc b/src/mon/FSCommands.cc index 5fd1f4a21de9c..0d9948fe1515b 100644 --- a/src/mon/FSCommands.cc +++ b/src/mon/FSCommands.cc @@ -681,6 +681,38 @@ class SetHandler : public FileSystemCommandHandler fs->mds_map.set_min_compat_client(vno); }; fsmap.modify_filesystem(fs->fscid, std::move(f)); + } else if (var == "refuse_client_session") { + bool refuse_session = false; + int r = parse_bool(val, &refuse_session, ss); + if (r != 0) { + return r; + } + + if (refuse_session) { + if (!(fs->mds_map.test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION))) { + fsmap.modify_filesystem( + fs->fscid, + [](std::shared_ptr fs) + { + fs->mds_map.set_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION); + }); + ss << "client(s) blocked from establishing new session(s)"; + } else { + ss << "client(s) already blocked from establishing new session(s)"; + } + } else { + if (fs->mds_map.test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION)) { + fsmap.modify_filesystem( + fs->fscid, + [](std::shared_ptr fs) + { + fs->mds_map.clear_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION); + }); + ss << "client(s) allowed to establish new session(s)"; + } else { + ss << "client(s) already allowed to establish new session(s)"; + } + } } else { ss << "unknown variable " << var; return -EINVAL; diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index 29e7b21275a20..c80ee21dde6ad 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -377,7 +377,8 @@ COMMAND("fs set " "name=var,type=CephChoices,strings=max_mds|max_file_size" "|allow_new_snaps|inline_data|cluster_down|allow_dirfrags|balancer" "|standby_count_wanted|session_timeout|session_autoclose" - "|allow_standby_replay|down|joinable|min_compat_client|bal_rank_mask " + "|allow_standby_replay|down|joinable|min_compat_client|bal_rank_mask" + "|refuse_client_session " "name=val,type=CephString " "name=yes_i_really_mean_it,type=CephBool,req=false " "name=yes_i_really_really_mean_it,type=CephBool,req=false", From 86d1a4922620af2d1916cbe97c981af6c53eaf2e Mon Sep 17 00:00:00 2001 From: dparmar18 Date: Fri, 11 Nov 2022 20:46:28 +0530 Subject: [PATCH 2/6] mds: do not reconnect when refuse_client_session is set Fixes: https://tracker.ceph.com/issues/57090 Signed-off-by: Dhairya Parmar --- src/mds/Server.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/mds/Server.cc b/src/mds/Server.cc index f0a4b62648dca..6e9b9bbfa8f72 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -1447,6 +1447,18 @@ void Server::handle_client_reconnect(const cref_t &m) return; } + if(mds->mdsmap->test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION)) { + mds->clog->warn() << "client could not reconnect as" + " file system flag refuse_client_session is set"; + dout(0) << "client cannot reconnect when file system flag" + " refuse_client_session is set" << dendl; + auto reply = make_message(CEPH_SESSION_CLOSE); + reply->metadata["error_string"] = "client cannot reconnect when file system flag" + " refuse_client_session is set"; + mds->send_message(reply, m->get_connection()); + return; + } + if (!session->is_open()) { dout(0) << " ignoring msg from not-open session" << *m << dendl; auto reply = make_message(CEPH_SESSION_CLOSE); From 0abff678d2085dc98b64e36238480b74ccafbda2 Mon Sep 17 00:00:00 2001 From: dparmar18 Date: Thu, 17 Nov 2022 11:05:46 +0530 Subject: [PATCH 3/6] client: do not initiate session if flag refuse_client_session is set Fixes: https://tracker.ceph.com/issues/57090 Signed-off-by: Dhairya Parmar --- src/client/Client.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/client/Client.cc b/src/client/Client.cc index 4c180ef754da4..7939bb9621206 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -6372,6 +6372,12 @@ int Client::mount(const std::string &mount_root, const UserPerm& perms, } } + if(mdsmap->test_flag(CEPH_MDSMAP_REFUSE_CLIENT_SESSION)) { + lderr(cct) << "connections cannot be made while" + " the flag refuse_client_session is set" << dendl; + return -CEPHFS_EACCES; + } + populate_metadata(mount_root.empty() ? "/" : mount_root); filepath fp(CEPH_INO_ROOT); From 87cfd0b379d18590a09bbd5a341ac1804d36caa4 Mon Sep 17 00:00:00 2001 From: dparmar18 Date: Thu, 10 Nov 2022 00:27:24 +0530 Subject: [PATCH 4/6] qa: added two testcases - test_reject_client_session - test_reject_client_session_on_reconnect Also added function set_reject_client_session(bool) in filesystem.py Fixes: https://tracker.ceph.com/issues/57090 Signed-off-by: Dhairya Parmar --- qa/tasks/cephfs/filesystem.py | 3 +++ qa/tasks/cephfs/test_client_recovery.py | 31 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/qa/tasks/cephfs/filesystem.py b/qa/tasks/cephfs/filesystem.py index 96af44e6cdf7c..8dbb576972b97 100644 --- a/qa/tasks/cephfs/filesystem.py +++ b/qa/tasks/cephfs/filesystem.py @@ -598,6 +598,9 @@ def set_allow_new_snaps(self, yes): def set_bal_rank_mask(self, bal_rank_mask): self.set_var("bal_rank_mask", bal_rank_mask) + def set_refuse_client_session(self, yes): + self.set_var("refuse_client_session", yes) + def compat(self, *args): a = map(lambda x: str(x).lower(), args) self.mon_manager.raw_cluster_cmd("fs", "compat", self.name, *a) diff --git a/qa/tasks/cephfs/test_client_recovery.py b/qa/tasks/cephfs/test_client_recovery.py index 8c1a6c21e4d98..f8a50ad37a9b6 100644 --- a/qa/tasks/cephfs/test_client_recovery.py +++ b/qa/tasks/cephfs/test_client_recovery.py @@ -724,3 +724,34 @@ def test_reconnect_after_blocklisted(self): rproc.wait() self.assertEqual(rproc.exitstatus, 0) + + def test_refuse_client_session(self): + """ + Test that client cannot start session when file system flag + refuse_client_session is set + """ + + self.mount_a.umount_wait() + self.fs.set_refuse_client_session(True) + with self.assertRaises(CommandFailedError): + self.mount_a.mount_wait() + + def test_refuse_client_session_on_reconnect(self): + """ + Test that client cannot reconnect when filesystem comes online and + file system flag refuse_client_session is set + """ + + self.mount_a.create_files() + self.mount_a.check_files() + + self.fs.fail() + self.fs.set_refuse_client_session(True) + self.fs.set_joinable() + with self.assert_cluster_log('client could not reconnect as' + ' file system flag' + ' refuse_client_session is set'): + time.sleep(self.fs.get_var("session_timeout") * 1.5) + self.assertEqual(len(self.fs.mds_tell(["session", "ls"])), 0) + self.mount_a.umount_wait(force=True) + From c71583d8a202736f582f7db061b61ab9b36bde66 Mon Sep 17 00:00:00 2001 From: dparmar18 Date: Mon, 7 Nov 2022 18:53:41 +0530 Subject: [PATCH 5/6] PendingReleaseNotes: noted new MDSMap field refuse_client_session Signed-off-by: Dhairya Parmar --- PendingReleaseNotes | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/PendingReleaseNotes b/PendingReleaseNotes index e216861ba6fdd..6df3a02fe6661 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -157,3 +157,8 @@ of the feature, refer this link on how to perform it: https://docs.ceph.com/en/quincy/cephadm/upgrade/#staggered-upgrade Relevant tracker: https://tracker.ceph.com/issues/55715 +* Introduced a new file system flag `refuse_client_session` that can be set using the +`fs set` command. This flag allows blocking any incoming session +request from client(s). This can be useful during some recovery situations +where it's desirable to bring MDS up but have no client workload. +Relevant tracker: https://tracker.ceph.com/issues/57090 From 2ecd8d64900aac60578964f4011156c46c797e5d Mon Sep 17 00:00:00 2001 From: dparmar18 Date: Mon, 2 Jan 2023 17:54:50 +0530 Subject: [PATCH 6/6] qa: fix api failure issue caused during a dashboard test 2022-12-28 10:16:47,812.812 INFO:__main__: File "/home/jenkins-build/build/workspace/ceph-api/qa/tasks/mgr/dashboard/helper.py", line 712, in 2022-12-28 10:16:47,812.812 INFO:__main__: _validate_json(val[key], sub_schema, path + [key]) 2022-12-28 10:16:47,812.812 INFO:__main__: File "/home/jenkins-build/build/workspace/ceph-api/qa/tasks/mgr/dashboard/helper.py", line 711, in _validate_json 2022-12-28 10:16:47,812.812 INFO:__main__: result = all( 2022-12-28 10:16:47,812.812 INFO:__main__: File "/home/jenkins-build/build/workspace/ceph-api/qa/tasks/mgr/dashboard/helper.py", line 712, in 2022-12-28 10:16:47,812.812 INFO:__main__: _validate_json(val[key], sub_schema, path + [key]) 2022-12-28 10:16:47,812.812 INFO:__main__: File "/home/jenkins-build/build/workspace/ceph-api/qa/tasks/mgr/dashboard/helper.py", line 710, in _validate_json 2022-12-28 10:16:47,813.813 INFO:__main__: raise _ValError('unknown keys: {}'.format(unknown_keys), path) 2022-12-28 10:16:47,813.813 INFO:__main__:tasks.mgr.dashboard.helper._ValError: In `input['fs_map']['filesystems'][0]['mdsmap']['flags_state']`: unknown keys: {'refuse_client_session'} Fixes the above failure Fixes: https://tracker.ceph.com/issues/57090 Signed-off-by: Dhairya Parmar --- qa/tasks/mgr/dashboard/test_health.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qa/tasks/mgr/dashboard/test_health.py b/qa/tasks/mgr/dashboard/test_health.py index ae922d2b354e4..b6ffade4c9641 100644 --- a/qa/tasks/mgr/dashboard/test_health.py +++ b/qa/tasks/mgr/dashboard/test_health.py @@ -56,7 +56,8 @@ class HealthTest(DashboardTestCase): 'joinable': bool, 'allow_snaps': bool, 'allow_multimds_snaps': bool, - 'allow_standby_replay': bool + 'allow_standby_replay': bool, + 'refuse_client_session': bool }), 'ever_allowed_features': int, 'root': int