Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

pacific: mds: add opened files/inodes and pinned i_caps metric payload support #39972

Merged
merged 1 commit into from Mar 25, 2021
Merged

pacific: mds: add opened files/inodes and pinned i_caps metric payload support #39972

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
33 changes: 33 additions & 0 deletions src/client/Client.cc
View file
Open in desktop
Expand Up @@ -4219,6 +4219,8 @@ void Client::add_update_cap(Inode *in, MetaSession *mds_session, uint64_t cap_id
issued |= cap.issued;
flags |= CEPH_CAP_FLAG_AUTH;
}
} else {
inc_pinned_icaps();
}

check_cap_issue(in, issued);
Expand Down Expand Up @@ -4284,8 +4286,11 @@ void Client::remove_cap(Cap *cap, bool queue_release)
cap->issue_seq,
cap->mseq,
cap_epoch_barrier);
} else {
dec_pinned_icaps();
}


if (in.auth_cap == cap) {
if (in.flushing_cap_item.is_on_list()) {
ldout(cct, 10) << " removing myself from flushing_cap list" << dendl;
Expand Down Expand Up @@ -6463,11 +6468,14 @@ void Client::abort_conn()

void Client::flush_cap_releases()
{
uint64_t nr_caps = 0;

// send any cap releases
for (auto &p : mds_sessions) {
auto &session = p.second;
if (session.release && mdsmap->is_clientreplay_or_active_or_stopping(
p.first)) {
nr_caps += session.release->caps.size();
if (cct->_conf->client_inject_release_failure) {
ldout(cct, 20) << __func__ << " injecting failure to send cap release message" << dendl;
} else {
Expand All @@ -6476,6 +6484,10 @@ void Client::flush_cap_releases()
session.release.reset();
}
}

if (nr_caps > 0) {
dec_pinned_icaps(nr_caps);
}
}

void Client::renew_and_flush_cap_releases()
Expand Down Expand Up @@ -6629,6 +6641,27 @@ void Client::collect_and_send_global_metrics() {
metric = ClientMetricMessage(DentryLeasePayload(dlease_hits, dlease_misses, nr));
message.push_back(metric);

// opened files
{
auto [opened_files, total_inodes] = get_opened_files_rates();
metric = ClientMetricMessage(OpenedFilesPayload(opened_files, total_inodes));
}
message.push_back(metric);

// pinned i_caps
{
auto [pinned_icaps, total_inodes] = get_pinned_icaps_rates();
metric = ClientMetricMessage(PinnedIcapsPayload(pinned_icaps, total_inodes));
}
message.push_back(metric);

// opened inodes
{
auto [opened_inodes, total_inodes] = get_opened_inodes_rates();
metric = ClientMetricMessage(OpenedInodesPayload(opened_inodes, total_inodes));
}
message.push_back(metric);

session->con->send_message2(make_message<MClientMetrics>(std::move(message)));
}

Expand Down
34 changes: 34 additions & 0 deletions src/client/Client.h
View file
Open in desktop
Expand Up @@ -802,6 +802,36 @@ class Client : public Dispatcher, public md_config_obs_t {
return std::make_pair(cap_hits, cap_misses);
}

void inc_opened_files() {
++opened_files;
}
void dec_opened_files() {
--opened_files;
}
std::pair<uint64_t, uint64_t> get_opened_files_rates() {
return std::make_pair(opened_files, inode_map.size());
}

void inc_pinned_icaps() {
++pinned_icaps;
}
void dec_pinned_icaps(uint64_t nr=1) {
pinned_icaps -= nr;
}
std::pair<uint64_t, uint64_t> get_pinned_icaps_rates() {
return std::make_pair(pinned_icaps, inode_map.size());
}

void inc_opened_inodes() {
++opened_inodes;
}
void dec_opened_inodes() {
--opened_inodes;
}
std::pair<uint64_t, uint64_t> get_opened_inodes_rates() {
return std::make_pair(opened_inodes, inode_map.size());
}

xlist<Inode*> &get_dirty_list() { return dirty_list; }

/* timer_lock for 'timer' */
Expand Down Expand Up @@ -1483,6 +1513,10 @@ class Client : public Dispatcher, public md_config_obs_t {
uint64_t cap_hits = 0;
uint64_t cap_misses = 0;

uint64_t opened_files = 0;
uint64_t pinned_icaps = 0;
uint64_t opened_inodes = 0;

ceph::spinlock delay_i_lock;
std::map<Inode*,int> delay_i_release;
};
Expand Down
8 changes: 7 additions & 1 deletion src/client/Inode.cc
View file
Open in desktop
Expand Up @@ -137,6 +137,9 @@ void Inode::make_nosnap_relative_path(filepath& p)

void Inode::get_open_ref(int mode)
{
client->inc_opened_files();
if (open_by_mode.count(mode) == 0)
client->inc_opened_inodes();
open_by_mode[mode]++;
break_deleg(!(mode & CEPH_FILE_MODE_WR));
}
Expand All @@ -146,8 +149,11 @@ bool Inode::put_open_ref(int mode)
//cout << "open_by_mode[" << mode << "] " << open_by_mode[mode] << " -> " << (open_by_mode[mode]-1) << std::endl;
auto& ref = open_by_mode.at(mode);
ceph_assert(ref > 0);
if (--ref == 0)
client->dec_opened_files();
if (--ref == 0) {
client->dec_opened_inodes();
return true;
}
return false;
}

Expand Down
138 changes: 138 additions & 0 deletions src/include/cephfs/metrics/Types.h
View file
Open in desktop
Expand Up @@ -22,6 +22,9 @@ enum ClientMetricType {
CLIENT_METRIC_TYPE_WRITE_LATENCY,
CLIENT_METRIC_TYPE_METADATA_LATENCY,
CLIENT_METRIC_TYPE_DENTRY_LEASE,
CLIENT_METRIC_TYPE_OPENED_FILES,
CLIENT_METRIC_TYPE_PINNED_ICAPS,
CLIENT_METRIC_TYPE_OPENED_INODES,
};
inline std::ostream &operator<<(std::ostream &os, const ClientMetricType &type) {
switch(type) {
Expand All @@ -40,6 +43,15 @@ inline std::ostream &operator<<(std::ostream &os, const ClientMetricType &type)
case ClientMetricType::CLIENT_METRIC_TYPE_DENTRY_LEASE:
os << "DENTRY_LEASE";
break;
case ClientMetricType::CLIENT_METRIC_TYPE_OPENED_FILES:
os << "OPENED_FILES";
break;
case ClientMetricType::CLIENT_METRIC_TYPE_PINNED_ICAPS:
os << "PINNED_ICAPS";
break;
case ClientMetricType::CLIENT_METRIC_TYPE_OPENED_INODES:
os << "OPENED_INODES";
break;
default:
ceph_abort();
}
Expand Down Expand Up @@ -232,6 +244,120 @@ struct DentryLeasePayload {
}
};

struct OpenedFilesPayload {
static const ClientMetricType METRIC_TYPE = ClientMetricType::CLIENT_METRIC_TYPE_OPENED_FILES;

uint64_t opened_files = 0;
uint64_t total_inodes = 0;

OpenedFilesPayload() { }
OpenedFilesPayload(uint64_t opened_files, uint64_t total_inodes)
: opened_files(opened_files), total_inodes(total_inodes) {
}

void encode(bufferlist &bl) const {
using ceph::encode;
ENCODE_START(1, 1, bl);
encode(opened_files, bl);
encode(total_inodes, bl);
ENCODE_FINISH(bl);
}

void decode(bufferlist::const_iterator &iter) {
using ceph::decode;
DECODE_START(1, iter);
decode(opened_files, iter);
decode(total_inodes, iter);
DECODE_FINISH(iter);
}

void dump(Formatter *f) const {
f->dump_int("opened_files", opened_files);
f->dump_int("total_inodes", total_inodes);
}

void print(ostream *out) const {
*out << "opened_files: " << opened_files << " "
<< "total_inodes: " << total_inodes;
}
};

struct PinnedIcapsPayload {
static const ClientMetricType METRIC_TYPE = ClientMetricType::CLIENT_METRIC_TYPE_PINNED_ICAPS;

uint64_t pinned_icaps = 0;
uint64_t total_inodes = 0;

PinnedIcapsPayload() { }
PinnedIcapsPayload(uint64_t pinned_icaps, uint64_t total_inodes)
: pinned_icaps(pinned_icaps), total_inodes(total_inodes) {
}

void encode(bufferlist &bl) const {
using ceph::encode;
ENCODE_START(1, 1, bl);
encode(pinned_icaps, bl);
encode(total_inodes, bl);
ENCODE_FINISH(bl);
}

void decode(bufferlist::const_iterator &iter) {
using ceph::decode;
DECODE_START(1, iter);
decode(pinned_icaps, iter);
decode(total_inodes, iter);
DECODE_FINISH(iter);
}

void dump(Formatter *f) const {
f->dump_int("pinned_icaps", pinned_icaps);
f->dump_int("total_inodes", total_inodes);
}

void print(ostream *out) const {
*out << "pinned_icaps: " << pinned_icaps << " "
<< "total_inodes: " << total_inodes;
}
};

struct OpenedInodesPayload {
static const ClientMetricType METRIC_TYPE = ClientMetricType::CLIENT_METRIC_TYPE_OPENED_INODES;

uint64_t opened_inodes = 0;
uint64_t total_inodes = 0;

OpenedInodesPayload() { }
OpenedInodesPayload(uint64_t opened_inodes, uint64_t total_inodes)
: opened_inodes(opened_inodes), total_inodes(total_inodes) {
}

void encode(bufferlist &bl) const {
using ceph::encode;
ENCODE_START(1, 1, bl);
encode(opened_inodes, bl);
encode(total_inodes, bl);
ENCODE_FINISH(bl);
}

void decode(bufferlist::const_iterator &iter) {
using ceph::decode;
DECODE_START(1, iter);
decode(opened_inodes, iter);
decode(total_inodes, iter);
DECODE_FINISH(iter);
}

void dump(Formatter *f) const {
f->dump_int("opened_inodes", opened_inodes);
f->dump_int("total_inodes", total_inodes);
}

void print(ostream *out) const {
*out << "opened_inodes: " << opened_inodes << " "
<< "total_inodes: " << total_inodes;
}
};

struct UnknownPayload {
static const ClientMetricType METRIC_TYPE = static_cast<ClientMetricType>(-1);

Expand All @@ -255,6 +381,9 @@ typedef boost::variant<CapInfoPayload,
WriteLatencyPayload,
MetadataLatencyPayload,
DentryLeasePayload,
OpenedFilesPayload,
PinnedIcapsPayload,
OpenedInodesPayload,
UnknownPayload> ClientMetricPayload;

// metric update message sent by clients
Expand Down Expand Up @@ -354,6 +483,15 @@ struct ClientMetricMessage {
case ClientMetricType::CLIENT_METRIC_TYPE_DENTRY_LEASE:
payload = DentryLeasePayload();
break;
case ClientMetricType::CLIENT_METRIC_TYPE_OPENED_FILES:
payload = OpenedFilesPayload();
break;
case ClientMetricType::CLIENT_METRIC_TYPE_PINNED_ICAPS:
payload = PinnedIcapsPayload();
break;
case ClientMetricType::CLIENT_METRIC_TYPE_OPENED_INODES:
payload = OpenedInodesPayload();
break;
default:
payload = UnknownPayload();
break;
Expand Down