Skip to content

Commit

Permalink
In Histogram metrics, track mean instead of sum, also add sum_squared…
Browse files Browse the repository at this point in the history 
…_deviation.

Use a spinlock around updates.

Remove redundancies in collected metric representation.
Run-length encode buckets in protobuf representation.

PiperOrigin-RevId: 522686144
Change-Id: Id31bc9e8078fc131c2906bca523e471232a7e281
  • Loading branch information
@laramiel
laramiel authored and Copybara-Service committed Apr 7, 2023
1 parent 9977a25 commit d5b12de
Show file tree
Hide file tree
Showing 14 changed files with 381 additions and 689 deletions.
133 changes: 95 additions & 38 deletions BUILD.bazel
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair")
load("@bazel_skylib//lib:selects.bzl", "selects")

package(default_visibility = ["//visibility:public"])
Expand All @@ -14,32 +13,104 @@ exports_files(glob(["*"]))
# actual = "@com_google_tensorstore//:{target}".format(target = name),
# )
# for name in [
# "linux_x86_64_clang",
# "linux_arm64_clang",
# "linux_x86_64_gcc",
# "linux_arm64_gcc",
# "macos_x86_64",
# "macos_arm64",
# "linux_x86_64_clang",
# "linux_x86_64_gcc",
# "windows_x86_64_msvc",
# "windows_x86_64_mingw_gcc",
# "linux_x86_64",
# "macos_x86_64",
# "macos_arm64",
# "windows_x86_64",
# "linux_clang",
# "linux_gcc",
# "windows_mingw_gcc",
# "windows_msvc",
# "macos_clang",
# "compiler_msvc",
# "compiler_clang",
# "compiler_regular_gcc",
# "compiler_mingw_gcc",
# "compiler_unknown",
# "compiler_gcc",
# "arm64",
# "x86_64",
# "arm64",
# "linux",
# "macos",
# "windows",
#
# ]
#]
#

# Combined os / cpu / compiler settings
#
# :linux_arm64_clang
# :linux_arm64_gcc
# :linux_x86_64_clang
# :linux_x86_64_gcc
# :windows_x86_64_msvc
# :windows_x86_64_mingw_gcc
selects.config_setting_group(
name = "linux_arm64_clang",
match_all = [
":compiler_clang",
":linux",
":arm64",
],
)

selects.config_setting_group(
name = "linux_arm64_gcc",
match_all = [
":compiler_gcc",
":linux",
":arm64",
],
)

selects.config_setting_group(
name = "linux_x86_64_clang",
match_all = [
":compiler_clang",
":linux",
":x86_64",
],
)

selects.config_setting_group(
name = "linux_x86_64_gcc",
match_all = [
":compiler_gcc",
":linux",
":x86_64",
],
)

selects.config_setting_group(
name = "windows_x86_64_msvc",
match_all = [
":compiler_msvc",
":windows",
":x86_64",
],
)

selects.config_setting_group(
name = "windows_x86_64_mingw_gcc",
match_all = [
":compiler_mingw_gcc",
":x86_64",
],
)

# Combined os / cpu settings
#
# :linux_x86_64
# :macos_x86_64
# :macos_arm64
# :windows_x86_64

selects.config_setting_group(
name = "linux_x86_64",
match_all = [
Expand Down Expand Up @@ -72,74 +143,60 @@ selects.config_setting_group(
],
)

# Combined os / cpu / compiler settings
#
# :linux_arm64_clang
# :linux_x86_64_clang
# :linux_arm64_gcc
# :linux_x86_64_gcc
# :windows_x86_64_msvc
# :windows_x86_64_mingw_gcc
# Combined os / compiler settings
# :linux_clang
# :linux_gcc
# :windows_mingw_gcc
# :windows_msvc
# :macos_clang

selects.config_setting_group(
name = "linux_arm64_clang",
name = "linux_clang",
match_all = [
":compiler_clang",
":linux",
":arm64",
],
)

selects.config_setting_group(
name = "linux_x86_64_clang",
match_all = [
":compiler_clang",
":linux",
":x86_64",
],
)

selects.config_setting_group(
name = "linux_arm64_gcc",
name = "linux_gcc",
match_all = [
":compiler_gcc",
":linux",
":arm64",
],
)

selects.config_setting_group(
name = "linux_x86_64_gcc",
name = "windows_mingw_gcc",
match_all = [
":compiler_gcc",
":compiler_mingw_gcc",
":linux",
":x86_64",
],
)

selects.config_setting_group(
name = "windows_x86_64_msvc",
name = "windows_msvc",
match_all = [
":compiler_msvc",
":windows",
":x86_64",
],
)

selects.config_setting_group(
name = "windows_x86_64_mingw_gcc",
name = "macos_clang",
match_all = [
":compiler_mingw_gcc",
":x86_64",
":compiler_clang",
":macos",
],
)

# compiler selection
# Compiler selection
#
# :compiler_msvc
# :compiler_clang
# :compiler_regular_gcc
# :compiler_mingw_gcc
# :compiler_gcc

config_setting(
name = "compiler_msvc",
flag_values = {
Expand Down
1 change: 0 additions & 1 deletion tensorstore/internal/metrics/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ tensorstore_cc_library(
hdrs = ["collect.h"],
deps = [
":metadata",
"//tensorstore/util:str_cat",
"@com_github_nlohmann_json//:nlohmann_json",
"@com_google_absl//absl/functional:function_ref",
"@com_google_absl//absl/strings",
Expand Down
127 changes: 53 additions & 74 deletions tensorstore/internal/metrics/collect.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,31 +19,24 @@

namespace tensorstore {
namespace internal_metrics {
namespace {
struct IsNonZero {
bool operator()(int64_t x) { return x != 0; }
bool operator()(double x) { return x != 0; }
bool operator()(const std::string& x) { return !x.empty(); }
bool operator()(std::monostate) { return false; }
};
} // namespace

bool IsCollectedMetricNonZero(const CollectedMetric& metric) {
struct IsNonZero {
bool operator()(int64_t x) { return x != 0; }
bool operator()(double x) { return x != 0; }
bool operator()(const std::string& x) { return !x.empty(); }
};

if (!metric.gauges.empty()) {
for (const auto& v : metric.gauges) {
if (std::visit(IsNonZero{}, v.value)) return true;
if (std::visit(IsNonZero{}, v.max_value)) return true;
}
} else if (!metric.values.empty()) {
if (!metric.values.empty()) {
for (const auto& v : metric.values) {
if (std::visit(IsNonZero{}, v.value)) return true;
}
} else if (!metric.counters.empty()) {
for (const auto& v : metric.counters) {
if (std::visit(IsNonZero{}, v.value)) return true;
if (std::visit(IsNonZero{}, v.max_value)) return true;
}
} else if (!metric.histograms.empty()) {
for (const auto& v : metric.histograms) {
if (v.count != 0) return true;
if (v.sum != 0) return true;
}
}
return false;
Expand All @@ -63,40 +56,43 @@ void FormatCollectedMetric(
absl::StrJoin(v.fields, ", "), "]");
};

if (!metric.counters.empty()) {
for (const auto& v : metric.counters) {
std::visit(
[&](auto x) {
handle_line(
/*has_value=*/x != 0,
absl::StrCat(metric_name_with_fields(v), "=", x));
},
v.value);
}
}
if (!metric.gauges.empty()) {
for (const auto& v : metric.gauges) {
if (!metric.values.empty()) {
for (auto& v : metric.values) {
bool has_value = false;
std::string line = metric_name_with_fields(v);
std::visit(
[&](auto x) {
has_value = (x != 0);
absl::StrAppend(&line, "={value=", x);
},
v.value);
std::visit(
[&](auto x) {
has_value |= (x != 0);
absl::StrAppend(&line, ", max=", x, "}");
},
v.max_value);
if (std::holds_alternative<std::monostate>(v.max_value)) {
// A Normal value.
std::visit(
[&](auto x) {
has_value |= IsNonZero{}(x);
absl::StrAppend(&line, "=", x);
},
v.value);
} else {
// A Gauge-like value.
std::visit(
[&](auto x) {
has_value |= IsNonZero{}(x);
absl::StrAppend(&line, "={value=", x);
},
v.value);
std::visit(
[&](auto x) {
has_value |= IsNonZero{}(x);
if constexpr (!std::is_same<decltype(x), std::monostate>::value) {
absl::StrAppend(&line, ", max=", x, "}");
}
},
v.max_value);
}
handle_line(has_value, std::move(line));
}
}
if (!metric.histograms.empty()) {
for (auto& v : metric.histograms) {
std::string line = metric_name_with_fields(v);
absl::StrAppend(&line, "={count=", v.count, " sum=", v.sum, " buckets=[");
absl::StrAppend(&line, "={count=", v.count, " mean=", v.mean,
" buckets=[");

// find the last bucket with data.
size_t end = v.buckets.size();
Expand All @@ -115,19 +111,7 @@ void FormatCollectedMetric(
i = j;
}
absl::StrAppend(&line, "]}");
handle_line(/*has_value=*/v.count || v.sum, std::move(line));
}
}
if (!metric.values.empty()) {
for (auto& v : metric.values) {
std::visit(
[&](auto x) {
decltype(x) d{};
handle_line(
/*has_value=*/x != d,
absl::StrCat(metric_name_with_fields(v), "=", x));
},
v.value);
handle_line(/*has_value=*/v.count, std::move(line));
}
}
}
Expand All @@ -152,34 +136,29 @@ ::nlohmann::json CollectedMetricToJson(const CollectedMetric& metric) {
};

std::vector<::nlohmann::json> values;
if (!metric.gauges.empty()) {
for (const auto& v : metric.gauges) {
::nlohmann::json::object_t tmp{};
set_field_keys(v, tmp);
std::visit([&](auto x) { tmp["value"] = x; }, v.value);
std::visit([&](auto x) { tmp["max_value"] = x; }, v.max_value);
values.push_back(std::move(tmp));
}
} else if (!metric.values.empty()) {
if (!metric.values.empty()) {
for (const auto& v : metric.values) {
::nlohmann::json::object_t tmp{};
set_field_keys(v, tmp);
std::visit([&](auto x) { tmp["value"] = x; }, v.value);
values.push_back(std::move(tmp));
}
} else if (!metric.counters.empty()) {
for (const auto& v : metric.counters) {
::nlohmann::json::object_t tmp{};
set_field_keys(v, tmp);
std::visit([&](auto x) { tmp["count"] = x; }, v.value);
std::visit([&tmp](auto x) { tmp["value"] = x; }, v.value);
if (!std::holds_alternative<std::monostate>(v.max_value)) {
std::visit(
[&tmp](auto x) {
if constexpr (!std::is_same<decltype(x), std::monostate>::value) {
tmp["max_value"] = x;
}
},
v.max_value);
}
values.push_back(std::move(tmp));
}
} else if (!metric.histograms.empty()) {
for (const auto& v : metric.histograms) {
::nlohmann::json::object_t tmp{};
set_field_keys(v, tmp);
tmp["count"] = v.count;
tmp["sum"] = v.sum;
tmp["mean"] = v.mean;
tmp["sum_of_squared_deviation"] = v.sum_of_squared_deviation;

size_t end = v.buckets.size();
while (end > 0 && v.buckets[end - 1] == 0) end--;
Expand Down
Loading

0 comments on commit d5b12de

Please sign in to comment.