Skip to content

Commit

Permalink
osd: Implement asynchronous scrub sleep
Browse files Browse the repository at this point in the history
Rather than blocking the main op queue just do an async sleep.

Fixes: http://tracker.ceph.com/issues/19497

Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
  • Loading branch information
badone committed May 4, 2017
1 parent 77a4357 commit 7af3e86
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 10 deletions.
38 changes: 28 additions & 10 deletions src/osd/PG.cc
Expand Up @@ -324,7 +324,9 @@ PG::PG(OSDService *o, OSDMapRef curmap,
peer_features(CEPH_FEATURES_SUPPORTED_DEFAULT),
acting_features(CEPH_FEATURES_SUPPORTED_DEFAULT),
upacting_features(CEPH_FEATURES_SUPPORTED_DEFAULT),
last_epoch(0)
last_epoch(0),
scrub_sleep_lock("PG::scrub_sleep_lock"),
scrub_sleep_timer(o->cct, scrub_sleep_lock, false /* relax locking */)
{
#ifdef PG_DEBUG_REFS
osd->add_pgid(p, this);
Expand All @@ -335,6 +337,8 @@ PG::PG(OSDService *o, OSDMapRef curmap,
PG::~PG()
{
pgstate_history.set_pg_in_destructor();
Mutex::Locker l(scrub_sleep_lock);
scrub_sleep_timer.shutdown();
#ifdef PG_DEBUG_REFS
osd->remove_pgid(info.pgid, this);
#endif
Expand Down Expand Up @@ -2821,6 +2825,8 @@ void PG::init(
dirty_info = true;
dirty_big_info = true;
write_if_dirty(*t);

scrub_sleep_timer.init();
}

#pragma GCC diagnostic ignored "-Wpragmas"
Expand Down Expand Up @@ -4208,22 +4214,34 @@ void PG::scrub(epoch_t queued, ThreadPool::TPHandle &handle)
{
if (cct->_conf->osd_scrub_sleep > 0 &&
(scrubber.state == PG::Scrubber::NEW_CHUNK ||
scrubber.state == PG::Scrubber::INACTIVE)) {
scrubber.state == PG::Scrubber::INACTIVE) && scrubber.needs_sleep) {
ceph_assert(!scrubber.sleeping);
dout(20) << __func__ << " state is INACTIVE|NEW_CHUNK, sleeping" << dendl;
unlock();
utime_t t;
t.set_from_double(cct->_conf->osd_scrub_sleep);
handle.suspend_tp_timeout();
t.sleep();
handle.reset_tp_timeout();
lock();
dout(20) << __func__ << " slept for " << t << dendl;
// Do an async sleep so we don't block the op queue
auto scrub_requeue_callback = new FunctionContext([this](int r) {
lock();
scrubber.sleeping = false;
scrubber.needs_sleep = false;
dout(20) << __func__ << " slept for "
<< ceph_clock_now() - scrubber.sleep_start
<< ", re-queuing scrub" << dendl;
scrub_queued = false;
requeue_scrub();
scrubber.sleep_start = utime_t();
unlock();
});
Mutex::Locker l(scrub_sleep_lock);
scrub_sleep_timer.add_event_after(cct->_conf->osd_scrub_sleep, scrub_requeue_callback);
scrubber.sleeping = true;
scrubber.sleep_start = ceph_clock_now();
return;
}
if (pg_has_reset_since(queued)) {
return;
}
assert(scrub_queued);
scrub_queued = false;
scrubber.needs_sleep = true;

if (!is_primary() || !is_active() || !is_clean() || !is_scrubbing()) {
dout(10) << "scrub -- not primary or active or not clean" << dendl;
Expand Down
12 changes: 12 additions & 0 deletions src/osd/PG.h
Expand Up @@ -36,6 +36,7 @@
#include "include/xlist.h"
#include "SnapMapper.h"
#include "Session.h"
#include "common/Timer.h"

#include "PGLog.h"
#include "OSDMap.h"
Expand Down Expand Up @@ -1202,6 +1203,11 @@ class PG : public DoutPrefixProvider {
OpRequestRef active_rep_scrub;
utime_t scrub_reg_stamp; // stamp we registered for

// For async sleep
bool sleeping = false;
bool needs_sleep = true;
utime_t sleep_start;

// flags to indicate explicitly requested scrubs (by admin)
bool must_scrub, must_deep_scrub, must_repair;

Expand Down Expand Up @@ -1316,6 +1322,9 @@ class PG : public DoutPrefixProvider {
authoritative.clear();
num_digest_updates_pending = 0;
cleaned_meta_map = ScrubMap();
sleeping = false;
needs_sleep = true;
sleep_start = utime_t();
}

void create_results(const hobject_t& obj);
Expand Down Expand Up @@ -2189,6 +2198,9 @@ class PG : public DoutPrefixProvider {

epoch_t last_epoch;

Mutex scrub_sleep_lock;
SafeTimer scrub_sleep_timer;

public:
const spg_t& get_pgid() const { return pg_id; }

Expand Down

0 comments on commit 7af3e86

Please sign in to comment.