From 20a4c0c4831a9dcd49d8ad34b99369effbda9ab8 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 23 Sep 2015 10:58:13 -0400 Subject: [PATCH 1/2] mon/MonitorDBStore: assert/crash if there is a write error Do this globally intead of relying on teh zillion mon callers to check the error code. There are no cases where we want to tolerate a commit failure. Fixes: #13089 Signed-off-by: Sage Weil (cherry picked from commit 2fb7b1f0e33ada7c9a1be3de2f7708eb0760fcef) --- src/mon/MonitorDBStore.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/mon/MonitorDBStore.h b/src/mon/MonitorDBStore.h index e37b95db90085..2e66a301383dd 100644 --- a/src/mon/MonitorDBStore.h +++ b/src/mon/MonitorDBStore.h @@ -291,6 +291,8 @@ class MonitorDBStore db->compact_range_async(compact.front().first, compact.front().second.first, compact.front().second.second); compact.pop_front(); } + } else { + assert(0 == "failed to write to db"); } return r; } @@ -564,7 +566,8 @@ class MonitorDBStore for (iter = prefixes.begin(); iter != prefixes.end(); ++iter) { dbt->rmkeys_by_prefix((*iter)); } - db->submit_transaction_sync(dbt); + int r = db->submit_transaction_sync(dbt); + assert(r >= 0); } int open(ostream &out) { From d4e4d85c005ff68cb0461e9d7f0cbe5e6cf2f073 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 23 Sep 2015 10:58:01 -0400 Subject: [PATCH 2/2] mon/Elector: do a trivial write on every election cycle Currently we already do a small write when the *first* election in a round happens (to update the election epoch). If the backend happens to fail while we are already in the midst of elections, however, we may continue to call elections without verifying we are still writeable. Signed-off-by: Sage Weil (cherry picked from commit ef909ccbdc303cce8a39edef255325127832ff16) --- src/mon/Elector.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/mon/Elector.cc b/src/mon/Elector.cc index 6f54cb4bbc700..79d66de618959 100644 --- a/src/mon/Elector.cc +++ b/src/mon/Elector.cc @@ -78,8 +78,15 @@ void Elector::start() init(); // start by trying to elect me - if (epoch % 2 == 0) + if (epoch % 2 == 0) { bump_epoch(epoch+1); // odd == election cycle + } else { + // do a trivial db write just to ensure it is writeable. + MonitorDBStore::TransactionRef t(new MonitorDBStore::Transaction); + t->put(Monitor::MONITOR_NAME, "election_writeable_test", rand()); + int r = mon->store->apply_transaction(t); + assert(r >= 0); + } start_stamp = ceph_clock_now(g_ceph_context); electing_me = true; acked_me[mon->rank] = CEPH_FEATURES_ALL;