Skip to content

Commit

Permalink
Merge pull request #39972 from singuliere/wip-49687-pacific
Browse files Browse the repository at this point in the history 
pacific: mds: add opened files/inodes and pinned i_caps metric payload support

Reviewed-by: Patrick Donnelly <pdonnell@redhat.com>
  • Loading branch information
@yuriw
yuriw committed Mar 25, 2021
2 parents 9823929 + 289d61e commit 44d482d
Show file tree
Hide file tree
Showing 13 changed files with 412 additions and 4 deletions.
33 changes: 33 additions & 0 deletions src/client/Client.cc
View file
Expand Up @@ -4219,6 +4219,8 @@ void Client::add_update_cap(Inode *in, MetaSession *mds_session, uint64_t cap_id
issued |= cap.issued;
flags |= CEPH_CAP_FLAG_AUTH;
}
} else {
inc_pinned_icaps();
}

check_cap_issue(in, issued);
Expand Down Expand Up @@ -4284,8 +4286,11 @@ void Client::remove_cap(Cap *cap, bool queue_release)
cap->issue_seq,
cap->mseq,
cap_epoch_barrier);
} else {
dec_pinned_icaps();
}


if (in.auth_cap == cap) {
if (in.flushing_cap_item.is_on_list()) {
ldout(cct, 10) << " removing myself from flushing_cap list" << dendl;
Expand Down Expand Up @@ -6463,11 +6468,14 @@ void Client::abort_conn()

void Client::flush_cap_releases()
{
uint64_t nr_caps = 0;

// send any cap releases
for (auto &p : mds_sessions) {
auto &session = p.second;
if (session.release && mdsmap->is_clientreplay_or_active_or_stopping(
p.first)) {
nr_caps += session.release->caps.size();
if (cct->_conf->client_inject_release_failure) {
ldout(cct, 20) << __func__ << " injecting failure to send cap release message" << dendl;
} else {
Expand All @@ -6476,6 +6484,10 @@ void Client::flush_cap_releases()
session.release.reset();
}
}

if (nr_caps > 0) {
dec_pinned_icaps(nr_caps);
}
}

void Client::renew_and_flush_cap_releases()
Expand Down Expand Up @@ -6629,6 +6641,27 @@ void Client::collect_and_send_global_metrics() {
metric = ClientMetricMessage(DentryLeasePayload(dlease_hits, dlease_misses, nr));
message.push_back(metric);

// opened files
{
auto [opened_files, total_inodes] = get_opened_files_rates();
metric = ClientMetricMessage(OpenedFilesPayload(opened_files, total_inodes));
}
message.push_back(metric);

// pinned i_caps
{
auto [pinned_icaps, total_inodes] = get_pinned_icaps_rates();
metric = ClientMetricMessage(PinnedIcapsPayload(pinned_icaps, total_inodes));
}
message.push_back(metric);

// opened inodes
{
auto [opened_inodes, total_inodes] = get_opened_inodes_rates();
metric = ClientMetricMessage(OpenedInodesPayload(opened_inodes, total_inodes));
}
message.push_back(metric);

session->con->send_message2(make_message<MClientMetrics>(std::move(message)));
}

Expand Down
34 changes: 34 additions & 0 deletions src/client/Client.h
View file
Expand Up @@ -803,6 +803,36 @@ class Client : public Dispatcher, public md_config_obs_t {
return std::make_pair(cap_hits, cap_misses);
}

void inc_opened_files() {
++opened_files;
}
void dec_opened_files() {
--opened_files;
}
std::pair<uint64_t, uint64_t> get_opened_files_rates() {
return std::make_pair(opened_files, inode_map.size());
}

void inc_pinned_icaps() {
++pinned_icaps;
}
void dec_pinned_icaps(uint64_t nr=1) {
pinned_icaps -= nr;
}
std::pair<uint64_t, uint64_t> get_pinned_icaps_rates() {
return std::make_pair(pinned_icaps, inode_map.size());
}

void inc_opened_inodes() {
++opened_inodes;
}
void dec_opened_inodes() {
--opened_inodes;
}
std::pair<uint64_t, uint64_t> get_opened_inodes_rates() {
return std::make_pair(opened_inodes, inode_map.size());
}

xlist<Inode*> &get_dirty_list() { return dirty_list; }

/* timer_lock for 'timer' */
Expand Down Expand Up @@ -1484,6 +1514,10 @@ class Client : public Dispatcher, public md_config_obs_t {
uint64_t cap_hits = 0;
uint64_t cap_misses = 0;

uint64_t opened_files = 0;
uint64_t pinned_icaps = 0;
uint64_t opened_inodes = 0;

ceph::spinlock delay_i_lock;
std::map<Inode*,int> delay_i_release;
};
Expand Down
8 changes: 7 additions & 1 deletion src/client/Inode.cc
View file
Expand Up @@ -137,6 +137,9 @@ void Inode::make_nosnap_relative_path(filepath& p)

void Inode::get_open_ref(int mode)
{
client->inc_opened_files();
if (open_by_mode.count(mode) == 0)
client->inc_opened_inodes();
open_by_mode[mode]++;
break_deleg(!(mode & CEPH_FILE_MODE_WR));
}
Expand All @@ -146,8 +149,11 @@ bool Inode::put_open_ref(int mode)
//cout << "open_by_mode[" << mode << "] " << open_by_mode[mode] << " -> " << (open_by_mode[mode]-1) << std::endl;
auto& ref = open_by_mode.at(mode);
ceph_assert(ref > 0);
if (--ref == 0)
client->dec_opened_files();
if (--ref == 0) {
client->dec_opened_inodes();
return true;
}
return false;
}

Expand Down
138 changes: 138 additions & 0 deletions src/include/cephfs/metrics/Types.h
View file
Expand Up @@ -22,6 +22,9 @@ enum ClientMetricType {
CLIENT_METRIC_TYPE_WRITE_LATENCY,
CLIENT_METRIC_TYPE_METADATA_LATENCY,
CLIENT_METRIC_TYPE_DENTRY_LEASE,
CLIENT_METRIC_TYPE_OPENED_FILES,
CLIENT_METRIC_TYPE_PINNED_ICAPS,
CLIENT_METRIC_TYPE_OPENED_INODES,
};
inline std::ostream &operator<<(std::ostream &os, const ClientMetricType &type) {
switch(type) {
Expand All @@ -40,6 +43,15 @@ inline std::ostream &operator<<(std::ostream &os, const ClientMetricType &type)
case ClientMetricType::CLIENT_METRIC_TYPE_DENTRY_LEASE:
os << "DENTRY_LEASE";
break;
case ClientMetricType::CLIENT_METRIC_TYPE_OPENED_FILES:
os << "OPENED_FILES";
break;
case ClientMetricType::CLIENT_METRIC_TYPE_PINNED_ICAPS:
os << "PINNED_ICAPS";
break;
case ClientMetricType::CLIENT_METRIC_TYPE_OPENED_INODES:
os << "OPENED_INODES";
break;
default:
ceph_abort();
}
Expand Down Expand Up @@ -232,6 +244,120 @@ struct DentryLeasePayload {
}
};

struct OpenedFilesPayload {
static const ClientMetricType METRIC_TYPE = ClientMetricType::CLIENT_METRIC_TYPE_OPENED_FILES;

uint64_t opened_files = 0;
uint64_t total_inodes = 0;

OpenedFilesPayload() { }
OpenedFilesPayload(uint64_t opened_files, uint64_t total_inodes)
: opened_files(opened_files), total_inodes(total_inodes) {
}

void encode(bufferlist &bl) const {
using ceph::encode;
ENCODE_START(1, 1, bl);
encode(opened_files, bl);
encode(total_inodes, bl);
ENCODE_FINISH(bl);
}

void decode(bufferlist::const_iterator &iter) {
using ceph::decode;
DECODE_START(1, iter);
decode(opened_files, iter);
decode(total_inodes, iter);
DECODE_FINISH(iter);
}

void dump(Formatter *f) const {
f->dump_int("opened_files", opened_files);
f->dump_int("total_inodes", total_inodes);
}

void print(ostream *out) const {
*out << "opened_files: " << opened_files << " "
<< "total_inodes: " << total_inodes;
}
};

struct PinnedIcapsPayload {
static const ClientMetricType METRIC_TYPE = ClientMetricType::CLIENT_METRIC_TYPE_PINNED_ICAPS;

uint64_t pinned_icaps = 0;
uint64_t total_inodes = 0;

PinnedIcapsPayload() { }
PinnedIcapsPayload(uint64_t pinned_icaps, uint64_t total_inodes)
: pinned_icaps(pinned_icaps), total_inodes(total_inodes) {
}

void encode(bufferlist &bl) const {
using ceph::encode;
ENCODE_START(1, 1, bl);
encode(pinned_icaps, bl);
encode(total_inodes, bl);
ENCODE_FINISH(bl);
}

void decode(bufferlist::const_iterator &iter) {
using ceph::decode;
DECODE_START(1, iter);
decode(pinned_icaps, iter);
decode(total_inodes, iter);
DECODE_FINISH(iter);
}

void dump(Formatter *f) const {
f->dump_int("pinned_icaps", pinned_icaps);
f->dump_int("total_inodes", total_inodes);
}

void print(ostream *out) const {
*out << "pinned_icaps: " << pinned_icaps << " "
<< "total_inodes: " << total_inodes;
}
};

struct OpenedInodesPayload {
static const ClientMetricType METRIC_TYPE = ClientMetricType::CLIENT_METRIC_TYPE_OPENED_INODES;

uint64_t opened_inodes = 0;
uint64_t total_inodes = 0;

OpenedInodesPayload() { }
OpenedInodesPayload(uint64_t opened_inodes, uint64_t total_inodes)
: opened_inodes(opened_inodes), total_inodes(total_inodes) {
}

void encode(bufferlist &bl) const {
using ceph::encode;
ENCODE_START(1, 1, bl);
encode(opened_inodes, bl);
encode(total_inodes, bl);
ENCODE_FINISH(bl);
}

void decode(bufferlist::const_iterator &iter) {
using ceph::decode;
DECODE_START(1, iter);
decode(opened_inodes, iter);
decode(total_inodes, iter);
DECODE_FINISH(iter);
}

void dump(Formatter *f) const {
f->dump_int("opened_inodes", opened_inodes);
f->dump_int("total_inodes", total_inodes);
}

void print(ostream *out) const {
*out << "opened_inodes: " << opened_inodes << " "
<< "total_inodes: " << total_inodes;
}
};

struct UnknownPayload {
static const ClientMetricType METRIC_TYPE = static_cast<ClientMetricType>(-1);

Expand All @@ -255,6 +381,9 @@ typedef boost::variant<CapInfoPayload,
WriteLatencyPayload,
MetadataLatencyPayload,
DentryLeasePayload,
OpenedFilesPayload,
PinnedIcapsPayload,
OpenedInodesPayload,
UnknownPayload> ClientMetricPayload;

// metric update message sent by clients
Expand Down Expand Up @@ -354,6 +483,15 @@ struct ClientMetricMessage {
case ClientMetricType::CLIENT_METRIC_TYPE_DENTRY_LEASE:
payload = DentryLeasePayload();
break;
case ClientMetricType::CLIENT_METRIC_TYPE_OPENED_FILES:
payload = OpenedFilesPayload();
break;
case ClientMetricType::CLIENT_METRIC_TYPE_PINNED_ICAPS:
payload = PinnedIcapsPayload();
break;
case ClientMetricType::CLIENT_METRIC_TYPE_OPENED_INODES:
payload = OpenedInodesPayload();
break;
default:
payload = UnknownPayload();
break;
Expand Down

0 comments on commit 44d482d

Please sign in to comment.