diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc index 067ad6fadbbbb..2a183b5919583 100644 --- a/src/rgw/rgw_json_enc.cc +++ b/src/rgw/rgw_json_enc.cc @@ -1290,6 +1290,7 @@ void rgw_meta_sync_marker::decode_json(JSONObj *obj) utime_t ut; JSONDecoder::decode_json("timestamp", ut, obj); timestamp = ut.to_real_time(); + JSONDecoder::decode_json("realm_epoch", realm_epoch, obj); } void rgw_meta_sync_marker::dump(Formatter *f) const @@ -1300,6 +1301,7 @@ void rgw_meta_sync_marker::dump(Formatter *f) const encode_json("total_entries", total_entries, f); encode_json("pos", pos, f); encode_json("timestamp", utime_t(timestamp), f); + encode_json("realm_epoch", realm_epoch, f); } void rgw_meta_sync_status::decode_json(JSONObj *obj) diff --git a/src/rgw/rgw_meta_sync_status.h b/src/rgw/rgw_meta_sync_status.h index e913e8ffb2101..e34bb05be1d7c 100644 --- a/src/rgw/rgw_meta_sync_status.h +++ b/src/rgw/rgw_meta_sync_status.h @@ -55,28 +55,33 @@ struct rgw_meta_sync_marker { uint64_t total_entries; uint64_t pos; real_time timestamp; + epoch_t realm_epoch{0}; //< realm_epoch of period marker rgw_meta_sync_marker() : state(FullSync), total_entries(0), pos(0) {} void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 1, bl); ::encode(state, bl); ::encode(marker, bl); ::encode(next_step_marker, bl); ::encode(total_entries, bl); ::encode(pos, bl); ::encode(timestamp, bl); + ::encode(realm_epoch, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); + DECODE_START(2, bl); ::decode(state, bl); ::decode(marker, bl); ::decode(next_step_marker, bl); ::decode(total_entries, bl); ::decode(pos, bl); ::decode(timestamp, bl); + if (struct_v >= 2) { + ::decode(realm_epoch, bl); + } DECODE_FINISH(bl); } diff --git a/src/rgw/rgw_sync.cc b/src/rgw/rgw_sync.cc index d8de8e5d2cfe2..cac266f2278c3 100644 --- a/src/rgw/rgw_sync.cc +++ b/src/rgw/rgw_sync.cc @@ -1104,7 +1104,7 @@ class RGWMetaSyncShardMarkerTrack : public RGWSyncShardMarkerTrackcct, 20) << __func__ << "(): updating marker marker_oid=" << marker_oid << " marker=" << new_marker << dendl; + ldout(sync_env->cct, 20) << __func__ << "(): updating marker marker_oid=" << marker_oid << " marker=" << new_marker << " realm_epoch=" << sync_marker.realm_epoch << dendl; RGWRados *store = sync_env->store; return new RGWSimpleRadosWriteCR(sync_env->async_rados, store, @@ -1245,6 +1245,7 @@ class RGWMetaSyncShardCR : public RGWCoroutine { const rgw_pool& pool; const std::string& period; //< currently syncing period id + const epoch_t realm_epoch; //< realm_epoch of period RGWMetadataLog* mdlog; //< log of syncing period uint32_t shard_id; rgw_meta_sync_marker& sync_marker; @@ -1292,11 +1293,13 @@ class RGWMetaSyncShardCR : public RGWCoroutine { public: RGWMetaSyncShardCR(RGWMetaSyncEnv *_sync_env, const rgw_pool& _pool, - const std::string& period, RGWMetadataLog* mdlog, - uint32_t _shard_id, rgw_meta_sync_marker& _marker, + const std::string& period, epoch_t realm_epoch, + RGWMetadataLog* mdlog, uint32_t _shard_id, + rgw_meta_sync_marker& _marker, const std::string& period_marker, bool *_reset_backoff) : RGWCoroutine(_sync_env->cct), sync_env(_sync_env), pool(_pool), - period(period), mdlog(mdlog), shard_id(_shard_id), sync_marker(_marker), + period(period), realm_epoch(realm_epoch), mdlog(mdlog), + shard_id(_shard_id), sync_marker(_marker), period_marker(period_marker), inc_lock("RGWMetaSyncShardCR::inc_lock"), reset_backoff(_reset_backoff) { *reset_backoff = false; @@ -1479,7 +1482,8 @@ class RGWMetaSyncShardCR : public RGWCoroutine { temp_marker->state = rgw_meta_sync_marker::IncrementalSync; temp_marker->marker = std::move(temp_marker->next_step_marker); temp_marker->next_step_marker.clear(); - ldout(sync_env->cct, 0) << *this << ": saving marker pos=" << temp_marker->marker << dendl; + temp_marker->realm_epoch = realm_epoch; + ldout(sync_env->cct, 0) << *this << ": saving marker pos=" << temp_marker->marker << " realm_epoch=" << realm_epoch << dendl; using WriteMarkerCR = RGWSimpleRadosWriteCR; yield call(new WriteMarkerCR(sync_env->async_rados, sync_env->store, @@ -1550,6 +1554,14 @@ class RGWMetaSyncShardCR : public RGWCoroutine { yield; } } + // if the period has advanced, we can't use the existing marker + if (sync_marker.realm_epoch < realm_epoch) { + ldout(sync_env->cct, 0) << "clearing marker=" << sync_marker.marker + << " from old realm_epoch=" << sync_marker.realm_epoch + << " (now " << realm_epoch << ')' << dendl; + sync_marker.realm_epoch = realm_epoch; + sync_marker.marker.clear(); + } mdlog_marker = sync_marker.marker; set_marker_tracker(new RGWMetaSyncShardMarkerTrack(sync_env, sync_env->shard_obj_name(shard_id), @@ -1571,7 +1583,8 @@ class RGWMetaSyncShardCR : public RGWCoroutine { } #define INCREMENTAL_MAX_ENTRIES 100 ldout(sync_env->cct, 20) << __func__ << ":" << __LINE__ << ": shard_id=" << shard_id << " mdlog_marker=" << mdlog_marker << " sync_marker.marker=" << sync_marker.marker << " period_marker=" << period_marker << dendl; - if (!period_marker.empty() && period_marker <= marker) { + if (!period_marker.empty() && period_marker <= mdlog_marker) { + ldout(cct, 10) << "mdlog_marker past period_marker=" << period_marker << dendl; done_with_period = true; break; } @@ -1638,6 +1651,7 @@ class RGWMetaSyncShardCR : public RGWCoroutine { ldout(sync_env->cct, 20) << __func__ << ":" << __LINE__ << ": shard_id=" << shard_id << " mdlog_marker=" << mdlog_marker << " max_marker=" << max_marker << " sync_marker.marker=" << sync_marker.marker << " period_marker=" << period_marker << dendl; if (done_with_period) { // return control to RGWMetaSyncCR and advance to the next period + ldout(sync_env->cct, 10) << *this << ": done with period" << dendl; break; } if (mdlog_marker == max_marker && can_adjust_marker) { @@ -1662,6 +1676,8 @@ class RGWMetaSyncShardCR : public RGWCoroutine { if (!can_adjust_marker) { return -EAGAIN; } + + return set_cr_done(); } /* TODO */ return 0; @@ -1674,6 +1690,7 @@ class RGWMetaSyncShardControlCR : public RGWBackoffControlCR const rgw_pool& pool; const std::string& period; + epoch_t realm_epoch; RGWMetadataLog* mdlog; uint32_t shard_id; rgw_meta_sync_marker sync_marker; @@ -1682,16 +1699,18 @@ class RGWMetaSyncShardControlCR : public RGWBackoffControlCR static constexpr bool exit_on_error = false; // retry on all errors public: RGWMetaSyncShardControlCR(RGWMetaSyncEnv *_sync_env, const rgw_pool& _pool, - const std::string& period, RGWMetadataLog* mdlog, - uint32_t _shard_id, const rgw_meta_sync_marker& _marker, + const std::string& period, epoch_t realm_epoch, + RGWMetadataLog* mdlog, uint32_t _shard_id, + const rgw_meta_sync_marker& _marker, std::string&& period_marker) : RGWBackoffControlCR(_sync_env->cct, exit_on_error), sync_env(_sync_env), - pool(_pool), period(period), mdlog(mdlog), shard_id(_shard_id), - sync_marker(_marker), period_marker(std::move(period_marker)) {} + pool(_pool), period(period), realm_epoch(realm_epoch), mdlog(mdlog), + shard_id(_shard_id), sync_marker(_marker), + period_marker(std::move(period_marker)) {} RGWCoroutine *alloc_cr() override { - return new RGWMetaSyncShardCR(sync_env, pool, period, mdlog, shard_id, - sync_marker, period_marker, backoff_ptr()); + return new RGWMetaSyncShardCR(sync_env, pool, period, realm_epoch, mdlog, + shard_id, sync_marker, period_marker, backoff_ptr()); } RGWCoroutine *alloc_finisher_cr() override { @@ -1750,6 +1769,7 @@ class RGWMetaSyncCR : public RGWCoroutine { yield { // get the mdlog for the current period (may be empty) auto& period_id = sync_status.sync_info.period; + auto realm_epoch = sync_status.sync_info.realm_epoch; auto mdlog = sync_env->store->meta_mgr->get_log(period_id); // prevent wakeup() from accessing shard_crs while we're spawning them @@ -1772,9 +1792,10 @@ class RGWMetaSyncCR : public RGWCoroutine { } } - auto cr = new RGWMetaSyncShardControlCR(sync_env, pool, period_id, - mdlog, shard_id, marker, - std::move(period_marker)); + using ShardCR = RGWMetaSyncShardControlCR; + auto cr = new ShardCR(sync_env, pool, period_id, realm_epoch, + mdlog, shard_id, marker, + std::move(period_marker)); auto stack = spawn(cr, false); shard_crs[shard_id] = RefPair{cr, stack}; }