Skip to content

Commit

Permalink
rgw: store realm epoch with sync status markers
Browse files Browse the repository at this point in the history 
sync status markers can't be compared between periods, so we need to
record the current period's realm epoch with its markers. when the
rgw_meta_sync_info.realm_epoch is more recent than the marker's
realm_epoch, we must treat the marker as empty

Signed-off-by: Casey Bodley <cbodley@redhat.com>
  • Loading branch information
@cbodley
cbodley committed Apr 26, 2017
1 parent a898fb7 commit 574ff5f
Show file tree
Hide file tree
Showing 3 changed files with 45 additions and 17 deletions.
2 changes: 2 additions & 0 deletions src/rgw/rgw_json_enc.cc
View file
Expand Up @@ -1290,6 +1290,7 @@ void rgw_meta_sync_marker::decode_json(JSONObj *obj)
utime_t ut;
JSONDecoder::decode_json("timestamp", ut, obj);
timestamp = ut.to_real_time();
JSONDecoder::decode_json("realm_epoch", realm_epoch, obj);
}

void rgw_meta_sync_marker::dump(Formatter *f) const
Expand All @@ -1300,6 +1301,7 @@ void rgw_meta_sync_marker::dump(Formatter *f) const
encode_json("total_entries", total_entries, f);
encode_json("pos", pos, f);
encode_json("timestamp", utime_t(timestamp), f);
encode_json("realm_epoch", realm_epoch, f);
}

void rgw_meta_sync_status::decode_json(JSONObj *obj)
Expand Down
9 changes: 7 additions & 2 deletions src/rgw/rgw_meta_sync_status.h
View file
Expand Up @@ -55,28 +55,33 @@ struct rgw_meta_sync_marker {
uint64_t total_entries;
uint64_t pos;
real_time timestamp;
epoch_t realm_epoch{0}; //< realm_epoch of period marker

rgw_meta_sync_marker() : state(FullSync), total_entries(0), pos(0) {}

void encode(bufferlist& bl) const {
ENCODE_START(1, 1, bl);
ENCODE_START(2, 1, bl);
::encode(state, bl);
::encode(marker, bl);
::encode(next_step_marker, bl);
::encode(total_entries, bl);
::encode(pos, bl);
::encode(timestamp, bl);
::encode(realm_epoch, bl);
ENCODE_FINISH(bl);
}

void decode(bufferlist::iterator& bl) {
DECODE_START(1, bl);
DECODE_START(2, bl);
::decode(state, bl);
::decode(marker, bl);
::decode(next_step_marker, bl);
::decode(total_entries, bl);
::decode(pos, bl);
::decode(timestamp, bl);
if (struct_v >= 2) {
::decode(realm_epoch, bl);
}
DECODE_FINISH(bl);
}

Expand Down
51 changes: 36 additions & 15 deletions src/rgw/rgw_sync.cc
View file
Expand Up @@ -1104,7 +1104,7 @@ class RGWMetaSyncShardMarkerTrack : public RGWSyncShardMarkerTrack<string, strin
sync_marker.timestamp = timestamp;
}

ldout(sync_env->cct, 20) << __func__ << "(): updating marker marker_oid=" << marker_oid << " marker=" << new_marker << dendl;
ldout(sync_env->cct, 20) << __func__ << "(): updating marker marker_oid=" << marker_oid << " marker=" << new_marker << " realm_epoch=" << sync_marker.realm_epoch << dendl;
RGWRados *store = sync_env->store;
return new RGWSimpleRadosWriteCR<rgw_meta_sync_marker>(sync_env->async_rados,
store,
Expand Down Expand Up @@ -1245,6 +1245,7 @@ class RGWMetaSyncShardCR : public RGWCoroutine {

const rgw_pool& pool;
const std::string& period; //< currently syncing period id
const epoch_t realm_epoch; //< realm_epoch of period
RGWMetadataLog* mdlog; //< log of syncing period
uint32_t shard_id;
rgw_meta_sync_marker& sync_marker;
Expand Down Expand Up @@ -1292,11 +1293,13 @@ class RGWMetaSyncShardCR : public RGWCoroutine {

public:
RGWMetaSyncShardCR(RGWMetaSyncEnv *_sync_env, const rgw_pool& _pool,
const std::string& period, RGWMetadataLog* mdlog,
uint32_t _shard_id, rgw_meta_sync_marker& _marker,
const std::string& period, epoch_t realm_epoch,
RGWMetadataLog* mdlog, uint32_t _shard_id,
rgw_meta_sync_marker& _marker,
const std::string& period_marker, bool *_reset_backoff)
: RGWCoroutine(_sync_env->cct), sync_env(_sync_env), pool(_pool),
period(period), mdlog(mdlog), shard_id(_shard_id), sync_marker(_marker),
period(period), realm_epoch(realm_epoch), mdlog(mdlog),
shard_id(_shard_id), sync_marker(_marker),
period_marker(period_marker), inc_lock("RGWMetaSyncShardCR::inc_lock"),
reset_backoff(_reset_backoff) {
*reset_backoff = false;
Expand Down Expand Up @@ -1479,7 +1482,8 @@ class RGWMetaSyncShardCR : public RGWCoroutine {
temp_marker->state = rgw_meta_sync_marker::IncrementalSync;
temp_marker->marker = std::move(temp_marker->next_step_marker);
temp_marker->next_step_marker.clear();
ldout(sync_env->cct, 0) << *this << ": saving marker pos=" << temp_marker->marker << dendl;
temp_marker->realm_epoch = realm_epoch;
ldout(sync_env->cct, 0) << *this << ": saving marker pos=" << temp_marker->marker << " realm_epoch=" << realm_epoch << dendl;

using WriteMarkerCR = RGWSimpleRadosWriteCR<rgw_meta_sync_marker>;
yield call(new WriteMarkerCR(sync_env->async_rados, sync_env->store,
Expand Down Expand Up @@ -1550,6 +1554,14 @@ class RGWMetaSyncShardCR : public RGWCoroutine {
yield;
}
}
// if the period has advanced, we can't use the existing marker
if (sync_marker.realm_epoch < realm_epoch) {
ldout(sync_env->cct, 0) << "clearing marker=" << sync_marker.marker
<< " from old realm_epoch=" << sync_marker.realm_epoch
<< " (now " << realm_epoch << ')' << dendl;
sync_marker.realm_epoch = realm_epoch;
sync_marker.marker.clear();
}
mdlog_marker = sync_marker.marker;
set_marker_tracker(new RGWMetaSyncShardMarkerTrack(sync_env,
sync_env->shard_obj_name(shard_id),
Expand All @@ -1571,7 +1583,8 @@ class RGWMetaSyncShardCR : public RGWCoroutine {
}
#define INCREMENTAL_MAX_ENTRIES 100
ldout(sync_env->cct, 20) << __func__ << ":" << __LINE__ << ": shard_id=" << shard_id << " mdlog_marker=" << mdlog_marker << " sync_marker.marker=" << sync_marker.marker << " period_marker=" << period_marker << dendl;
if (!period_marker.empty() && period_marker <= marker) {
if (!period_marker.empty() && period_marker <= mdlog_marker) {
ldout(cct, 10) << "mdlog_marker past period_marker=" << period_marker << dendl;
done_with_period = true;
break;
}
Expand Down Expand Up @@ -1638,6 +1651,7 @@ class RGWMetaSyncShardCR : public RGWCoroutine {
ldout(sync_env->cct, 20) << __func__ << ":" << __LINE__ << ": shard_id=" << shard_id << " mdlog_marker=" << mdlog_marker << " max_marker=" << max_marker << " sync_marker.marker=" << sync_marker.marker << " period_marker=" << period_marker << dendl;
if (done_with_period) {
// return control to RGWMetaSyncCR and advance to the next period
ldout(sync_env->cct, 10) << *this << ": done with period" << dendl;
break;
}
if (mdlog_marker == max_marker && can_adjust_marker) {
Expand All @@ -1662,6 +1676,8 @@ class RGWMetaSyncShardCR : public RGWCoroutine {
if (!can_adjust_marker) {
return -EAGAIN;
}

return set_cr_done();
}
/* TODO */
return 0;
Expand All @@ -1674,6 +1690,7 @@ class RGWMetaSyncShardControlCR : public RGWBackoffControlCR

const rgw_pool& pool;
const std::string& period;
epoch_t realm_epoch;
RGWMetadataLog* mdlog;
uint32_t shard_id;
rgw_meta_sync_marker sync_marker;
Expand All @@ -1682,16 +1699,18 @@ class RGWMetaSyncShardControlCR : public RGWBackoffControlCR
static constexpr bool exit_on_error = false; // retry on all errors
public:
RGWMetaSyncShardControlCR(RGWMetaSyncEnv *_sync_env, const rgw_pool& _pool,
const std::string& period, RGWMetadataLog* mdlog,
uint32_t _shard_id, const rgw_meta_sync_marker& _marker,
const std::string& period, epoch_t realm_epoch,
RGWMetadataLog* mdlog, uint32_t _shard_id,
const rgw_meta_sync_marker& _marker,
std::string&& period_marker)
: RGWBackoffControlCR(_sync_env->cct, exit_on_error), sync_env(_sync_env),
pool(_pool), period(period), mdlog(mdlog), shard_id(_shard_id),
sync_marker(_marker), period_marker(std::move(period_marker)) {}
pool(_pool), period(period), realm_epoch(realm_epoch), mdlog(mdlog),
shard_id(_shard_id), sync_marker(_marker),
period_marker(std::move(period_marker)) {}

RGWCoroutine *alloc_cr() override {
return new RGWMetaSyncShardCR(sync_env, pool, period, mdlog, shard_id,
sync_marker, period_marker, backoff_ptr());
return new RGWMetaSyncShardCR(sync_env, pool, period, realm_epoch, mdlog,
shard_id, sync_marker, period_marker, backoff_ptr());
}

RGWCoroutine *alloc_finisher_cr() override {
Expand Down Expand Up @@ -1750,6 +1769,7 @@ class RGWMetaSyncCR : public RGWCoroutine {
yield {
// get the mdlog for the current period (may be empty)
auto& period_id = sync_status.sync_info.period;
auto realm_epoch = sync_status.sync_info.realm_epoch;
auto mdlog = sync_env->store->meta_mgr->get_log(period_id);

// prevent wakeup() from accessing shard_crs while we're spawning them
Expand All @@ -1772,9 +1792,10 @@ class RGWMetaSyncCR : public RGWCoroutine {
}
}

auto cr = new RGWMetaSyncShardControlCR(sync_env, pool, period_id,
mdlog, shard_id, marker,
std::move(period_marker));
using ShardCR = RGWMetaSyncShardControlCR;
auto cr = new ShardCR(sync_env, pool, period_id, realm_epoch,
mdlog, shard_id, marker,
std::move(period_marker));
auto stack = spawn(cr, false);
shard_crs[shard_id] = RefPair{cr, stack};
}
Expand Down

0 comments on commit 574ff5f

Please sign in to comment.