Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #15531 from liewegas/wip-osd-pglog
osd: rename osd -> osd_pglog; include pglog-related bufferlists

Reviewed-by: Kefu Chai <kchai@redhat.com>
Reviewed-by: Josh Durgin <jdurgin@redhat.com>
  • Loading branch information
liewegas committed Jun 7, 2017
2 parents 8f5a5a9 + 85887a7 commit 99332b9
Show file tree
Hide file tree
Showing 13 changed files with 58 additions and 54 deletions.
1 change: 1 addition & 0 deletions src/include/mempool.h
Expand Up @@ -150,6 +150,7 @@ namespace mempool {
f(buffer_meta) \
f(osd) \
f(osd_mapbl) \
f(osd_pglog) \
f(osdmap) \
f(osdmap_mapping) \
f(pgmap) \
Expand Down
4 changes: 2 additions & 2 deletions src/messages/MOSDPGUpdateLogMissing.h
Expand Up @@ -29,7 +29,7 @@ class MOSDPGUpdateLogMissing : public MOSDFastDispatchOp {
spg_t pgid;
shard_id_t from;
ceph_tid_t rep_tid;
mempool::osd::list<pg_log_entry_t> entries;
mempool::osd_pglog::list<pg_log_entry_t> entries;

epoch_t get_epoch() const { return map_epoch; }
spg_t get_pgid() const { return pgid; }
Expand All @@ -50,7 +50,7 @@ class MOSDPGUpdateLogMissing : public MOSDFastDispatchOp {
: MOSDFastDispatchOp(MSG_OSD_PG_UPDATE_LOG_MISSING, HEAD_VERSION,
COMPAT_VERSION) { }
MOSDPGUpdateLogMissing(
const mempool::osd::list<pg_log_entry_t> &entries,
const mempool::osd_pglog::list<pg_log_entry_t> &entries,
spg_t pgid,
shard_id_t from,
epoch_t epoch,
Expand Down
1 change: 1 addition & 0 deletions src/osd/ECTransaction.cc
Expand Up @@ -169,6 +169,7 @@ void ECTransaction::generate_transactions(
bufferlist bl(op.updated_snaps->second.size() * 8 + 8);
::encode(op.updated_snaps->second, bl);
entry->snaps.swap(bl);
entry->snaps.reassign_to_mempool(mempool::mempool_osd_pglog);
}

ldpp_dout(dpp, 20) << "generate_transactions: "
Expand Down
4 changes: 2 additions & 2 deletions src/osd/PG.cc
Expand Up @@ -4928,7 +4928,7 @@ void PG::share_pg_info()
}

bool PG::append_log_entries_update_missing(
const mempool::osd::list<pg_log_entry_t> &entries,
const mempool::osd_pglog::list<pg_log_entry_t> &entries,
ObjectStore::Transaction &t)
{
assert(!entries.empty());
Expand All @@ -4955,7 +4955,7 @@ bool PG::append_log_entries_update_missing(


void PG::merge_new_log_entries(
const mempool::osd::list<pg_log_entry_t> &entries,
const mempool::osd_pglog::list<pg_log_entry_t> &entries,
ObjectStore::Transaction &t)
{
dout(10) << __func__ << " " << entries << dendl;
Expand Down
4 changes: 2 additions & 2 deletions src/osd/PG.h
Expand Up @@ -2382,15 +2382,15 @@ class PG : public DoutPrefixProvider {


bool append_log_entries_update_missing(
const mempool::osd::list<pg_log_entry_t> &entries,
const mempool::osd_pglog::list<pg_log_entry_t> &entries,
ObjectStore::Transaction &t);

/**
* Merge entries updating missing as necessary on all
* actingbackfill logs and missings (also missing_loc)
*/
void merge_new_log_entries(
const mempool::osd::list<pg_log_entry_t> &entries,
const mempool::osd_pglog::list<pg_log_entry_t> &entries,
ObjectStore::Transaction &t);

void reset_interval_flush();
Expand Down
2 changes: 1 addition & 1 deletion src/osd/PGLog.cc
Expand Up @@ -368,7 +368,7 @@ void PGLog::merge_log(pg_info_t &oinfo, pg_log_t &olog, pg_shard_t fromosd,
}
log.roll_forward_to(log.head, rollbacker);

mempool::osd::list<pg_log_entry_t> new_entries;
mempool::osd_pglog::list<pg_log_entry_t> new_entries;
new_entries.splice(new_entries.end(), olog.log, from, to);
append_log_entries_update_missing(
info.last_backfill,
Expand Down
41 changes: 18 additions & 23 deletions src/osd/PGLog.h
Expand Up @@ -97,7 +97,7 @@ struct PGLog : DoutPrefixProvider {
* It's a reverse_iterator because rend() is a natural representation for
* tail, and rbegin() works nicely for head.
*/
mempool::osd::list<pg_log_entry_t>::reverse_iterator
mempool::osd_pglog::list<pg_log_entry_t>::reverse_iterator
rollback_info_trimmed_to_riter;

template <typename F>
Expand Down Expand Up @@ -179,7 +179,7 @@ struct PGLog : DoutPrefixProvider {
advance_can_rollback_to(head, [&](const pg_log_entry_t &entry) {});
}

mempool::osd::list<pg_log_entry_t> rewind_from_head(eversion_t newhead) {
mempool::osd_pglog::list<pg_log_entry_t> rewind_from_head(eversion_t newhead) {
auto divergent = pg_log_t::rewind_from_head(newhead);
index();
reset_rollback_info_trimmed_to_riter();
Expand Down Expand Up @@ -284,8 +284,7 @@ struct PGLog : DoutPrefixProvider {
}
p = extra_caller_ops.find(r);
if (p != extra_caller_ops.end()) {
for (vector<pair<osd_reqid_t, version_t> >::const_iterator i =
p->second->extra_reqids.begin();
for (auto i = p->second->extra_reqids.begin();
i != p->second->extra_reqids.end();
++i) {
if (i->first == r) {
Expand All @@ -302,7 +301,7 @@ struct PGLog : DoutPrefixProvider {

/// get a (bounded) list of recent reqids for the given object
void get_object_reqids(const hobject_t& oid, unsigned max,
vector<pair<osd_reqid_t, version_t> > *pls) const {
mempool::osd_pglog::vector<pair<osd_reqid_t, version_t> > *pls) const {
// make sure object is present at least once before we do an
// O(n) search.
if (!(indexed_data & PGLOG_INDEXED_OBJECTS)) {
Expand Down Expand Up @@ -351,8 +350,7 @@ struct PGLog : DoutPrefixProvider {
}

if (to_index & PGLOG_INDEXED_EXTRA_CALLER_OPS) {
for (vector<pair<osd_reqid_t, version_t> >::const_iterator j =
i->extra_reqids.begin();
for (auto j = i->extra_reqids.begin();
j != i->extra_reqids.end();
++j) {
extra_caller_ops.insert(
Expand Down Expand Up @@ -389,8 +387,7 @@ struct PGLog : DoutPrefixProvider {
}
}
if (indexed_data & PGLOG_INDEXED_EXTRA_CALLER_OPS) {
for (vector<pair<osd_reqid_t, version_t> >::const_iterator j =
e.extra_reqids.begin();
for (auto j = e.extra_reqids.begin();
j != e.extra_reqids.end();
++j) {
extra_caller_ops.insert(make_pair(j->first, &e));
Expand All @@ -417,8 +414,7 @@ struct PGLog : DoutPrefixProvider {
}
}
if (indexed_data & PGLOG_INDEXED_EXTRA_CALLER_OPS) {
for (vector<pair<osd_reqid_t, version_t> >::const_iterator j =
e.extra_reqids.begin();
for (auto j = e.extra_reqids.begin();
j != e.extra_reqids.end();
++j) {
for (ceph::unordered_multimap<osd_reqid_t,pg_log_entry_t*>::iterator k =
Expand Down Expand Up @@ -465,8 +461,7 @@ struct PGLog : DoutPrefixProvider {
}

if (indexed_data & PGLOG_INDEXED_EXTRA_CALLER_OPS) {
for (vector<pair<osd_reqid_t, version_t> >::const_iterator j =
e.extra_reqids.begin();
for (auto j = e.extra_reqids.begin();
j != e.extra_reqids.end();
++j) {
extra_caller_ops.insert(make_pair(j->first, &(log.back())));
Expand Down Expand Up @@ -712,10 +707,10 @@ struct PGLog : DoutPrefixProvider {

protected:
static void split_by_object(
mempool::osd::list<pg_log_entry_t> &entries,
map<hobject_t, mempool::osd::list<pg_log_entry_t>> *out_entries) {
mempool::osd_pglog::list<pg_log_entry_t> &entries,
map<hobject_t, mempool::osd_pglog::list<pg_log_entry_t>> *out_entries) {
while (!entries.empty()) {
mempool::osd::list<pg_log_entry_t> &out_list = (*out_entries)[entries.front().soid];
auto &out_list = (*out_entries)[entries.front().soid];
out_list.splice(out_list.end(), entries, entries.begin());
}
}
Expand Down Expand Up @@ -744,7 +739,7 @@ struct PGLog : DoutPrefixProvider {
static void _merge_object_divergent_entries(
const IndexedLog &log, ///< [in] log to merge against
const hobject_t &hoid, ///< [in] object we are merging
const mempool::osd::list<pg_log_entry_t> &entries, ///< [in] entries for hoid to merge
const mempool::osd_pglog::list<pg_log_entry_t> &entries, ///< [in] entries for hoid to merge
const pg_info_t &info, ///< [in] info for merging entries
eversion_t olog_can_rollback_to, ///< [in] rollback boundary
missing_type &missing, ///< [in,out] missing to adjust, use
Expand Down Expand Up @@ -926,16 +921,16 @@ struct PGLog : DoutPrefixProvider {
template <typename missing_type>
static void _merge_divergent_entries(
const IndexedLog &log, ///< [in] log to merge against
mempool::osd::list<pg_log_entry_t> &entries, ///< [in] entries to merge
mempool::osd_pglog::list<pg_log_entry_t> &entries, ///< [in] entries to merge
const pg_info_t &oinfo, ///< [in] info for merging entries
eversion_t olog_can_rollback_to, ///< [in] rollback boundary
missing_type &omissing, ///< [in,out] missing to adjust, use
LogEntryHandler *rollbacker, ///< [in] optional rollbacker object
const DoutPrefixProvider *dpp ///< [in] logging provider
) {
map<hobject_t, mempool::osd::list<pg_log_entry_t> > split;
map<hobject_t, mempool::osd_pglog::list<pg_log_entry_t> > split;
split_by_object(entries, &split);
for (map<hobject_t, mempool::osd::list<pg_log_entry_t>>::iterator i = split.begin();
for (map<hobject_t, mempool::osd_pglog::list<pg_log_entry_t>>::iterator i = split.begin();
i != split.end();
++i) {
_merge_object_divergent_entries(
Expand All @@ -959,7 +954,7 @@ struct PGLog : DoutPrefixProvider {
const pg_log_entry_t& oe,
const pg_info_t& info,
LogEntryHandler *rollbacker) {
mempool::osd::list<pg_log_entry_t> entries;
mempool::osd_pglog::list<pg_log_entry_t> entries;
entries.push_back(oe);
_merge_object_divergent_entries(
log,
Expand Down Expand Up @@ -988,7 +983,7 @@ struct PGLog : DoutPrefixProvider {
static bool append_log_entries_update_missing(
const hobject_t &last_backfill,
bool last_backfill_bitwise,
const mempool::osd::list<pg_log_entry_t> &entries,
const mempool::osd_pglog::list<pg_log_entry_t> &entries,
bool maintain_rollback,
IndexedLog *log,
missing_type &missing,
Expand Down Expand Up @@ -1024,7 +1019,7 @@ struct PGLog : DoutPrefixProvider {
bool append_new_log_entries(
const hobject_t &last_backfill,
bool last_backfill_bitwise,
const mempool::osd::list<pg_log_entry_t> &entries,
const mempool::osd_pglog::list<pg_log_entry_t> &entries,
LogEntryHandler *rollbacker) {
bool invalidate_stats = append_log_entries_update_missing(
last_backfill,
Expand Down
6 changes: 3 additions & 3 deletions src/osd/PrimaryLogPG.cc
Expand Up @@ -2358,7 +2358,7 @@ void PrimaryLogPG::record_write_error(OpRequestRef op, const hobject_t &soid,
assert(op->may_write());
const osd_reqid_t &reqid = static_cast<const MOSDOp*>(op->get_req())->get_reqid();
ObjectContextRef obc;
mempool::osd::list<pg_log_entry_t> entries;
mempool::osd_pglog::list<pg_log_entry_t> entries;
entries.push_back(pg_log_entry_t(pg_log_entry_t::ERROR, soid,
get_next_version(), eversion_t(), 0,
reqid, utime_t(), r));
Expand Down Expand Up @@ -9212,7 +9212,7 @@ void PrimaryLogPG::simple_opc_submit(OpContextUPtr ctx)


void PrimaryLogPG::submit_log_entries(
const mempool::osd::list<pg_log_entry_t> &entries,
const mempool::osd_pglog::list<pg_log_entry_t> &entries,
ObcLockManager &&manager,
boost::optional<std::function<void(void)> > &&_on_complete,
OpRequestRef op,
Expand Down Expand Up @@ -10382,7 +10382,7 @@ void PrimaryLogPG::mark_all_unfound_lost(
pg_log.get_log().print(*_dout);
*_dout << dendl;

mempool::osd::list<pg_log_entry_t> log_entries;
mempool::osd_pglog::list<pg_log_entry_t> log_entries;

utime_t mtime = ceph_clock_now();
map<hobject_t, pg_missing_item>::const_iterator m =
Expand Down
6 changes: 3 additions & 3 deletions src/osd/PrimaryLogPG.h
Expand Up @@ -91,7 +91,7 @@ class PrimaryLogPG : public PG, public PGBackend::Listener {
uint32_t flags; // object_copy_data_t::FLAG_*
uint32_t source_data_digest, source_omap_digest;
uint32_t data_digest, omap_digest;
vector<pair<osd_reqid_t, version_t> > reqids; // [(reqid, user_version)]
mempool::osd_pglog::vector<pair<osd_reqid_t, version_t> > reqids; // [(reqid, user_version)]
map<string, bufferlist> attrs; // xattrs
uint64_t truncate_seq;
uint64_t truncate_size;
Expand Down Expand Up @@ -530,7 +530,7 @@ class PrimaryLogPG : public PG, public PGBackend::Listener {
int num_read; ///< count read ops
int num_write; ///< count update ops

vector<pair<osd_reqid_t, version_t> > extra_reqids;
mempool::osd_pglog::vector<pair<osd_reqid_t, version_t> > extra_reqids;

CopyFromCallback *copy_cb;

Expand Down Expand Up @@ -864,7 +864,7 @@ class PrimaryLogPG : public PG, public PGBackend::Listener {
* Also used to store error log entries for dup detection.
*/
void submit_log_entries(
const mempool::osd::list<pg_log_entry_t> &entries,
const mempool::osd_pglog::list<pg_log_entry_t> &entries,
ObcLockManager &&manager,
boost::optional<std::function<void(void)> > &&on_complete,
OpRequestRef op = OpRequestRef(),
Expand Down
1 change: 1 addition & 0 deletions src/osd/ReplicatedBackend.cc
Expand Up @@ -361,6 +361,7 @@ void generate_transaction(
bufferlist bl(oiter->second.updated_snaps->second.size() * 8 + 8);
::encode(oiter->second.updated_snaps->second, bl);
le.snaps.swap(bl);
le.snaps.reassign_to_mempool(mempool::mempool_osd_pglog);
}
}

Expand Down
7 changes: 4 additions & 3 deletions src/osd/osd_types.cc
Expand Up @@ -3878,6 +3878,7 @@ void ObjectModDesc::decode(bufferlist::iterator &_bl)
::decode(bl, _bl);
// ensure bl does not pin a larger buffer in memory
bl.rebuild();
bl.reassign_to_mempool(mempool::mempool_osd_pglog);
DECODE_FINISH(_bl);
}

Expand Down Expand Up @@ -3981,6 +3982,7 @@ void pg_log_entry_t::decode(bufferlist::iterator &bl)
::decode(snaps, bl);
// ensure snaps does not pin a larger buffer in memory
snaps.rebuild();
snaps.reassign_to_mempool(mempool::mempool_osd_pglog);
}

if (struct_v >= 8)
Expand All @@ -4007,8 +4009,7 @@ void pg_log_entry_t::dump(Formatter *f) const
f->dump_stream("prior_version") << prior_version;
f->dump_stream("reqid") << reqid;
f->open_array_section("extra_reqids");
for (vector<pair<osd_reqid_t, version_t> >::const_iterator p =
extra_reqids.begin();
for (auto p = extra_reqids.begin();
p != extra_reqids.end();
++p) {
f->open_object_section("extra_reqid");
Expand Down Expand Up @@ -4442,7 +4443,7 @@ void object_copy_data_t::dump(Formatter *f) const
f->dump_unsigned("snap", *p);
f->close_section();
f->open_array_section("reqids");
for (vector<pair<osd_reqid_t, version_t> >::const_iterator p = reqids.begin();
for (auto p = reqids.begin();
p != reqids.end();
++p) {
f->open_object_section("extra_reqid");
Expand Down

0 comments on commit 99332b9

Please sign in to comment.