-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This change adds the following new metrics, which can be used to provide feedback on where a system is currently constrained. The metrics are collected for both EC2 instances and Titus containers, with the exception of the full:cpu metric, which is meaningless on EC2 instances. * name=sys.pressure.some,id=[cpu|io|memory] counter unit=seconds/second * name=sys.pressure.full,id=[cpu|io|memory] counter unit=seconds/second The `some` pressure stall category represents the amount of time that SOME tasks on the system are unable to make progress due to a resource constraint. The `full` pressure stall category represents the amount of time that ALL runnable tasks are unable to make progress due to a resource constraint. For example, the some:cpu category represents the case where a task is sitting in the CPU run queue, ready to run, but it cannot make forward progress because all CPUs are busy. The io:full category represents a situation where every runnable task on the system is unable to make progress because they are all stalled on IO. The counters report the `total` field for each pressure stall category, which represents the total number of seconds/second that a system is in a given stall state.
- Loading branch information
1 parent
71d1e82
commit 2dd921d
Showing
25 changed files
with
246 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| #pragma once | ||
|
|
||
| #include "absl/strings/str_split.h" | ||
| #include "tagging_registry.h" | ||
| #include "util.h" | ||
|
|
||
| namespace atlasagent { | ||
|
|
||
| using spectator::Id; | ||
| using spectator::IdPtr; | ||
| using spectator::Tags; | ||
|
|
||
| template <typename Reg = TaggingRegistry> | ||
| class PressureStall { | ||
| public: | ||
| explicit PressureStall(Reg* registry, std::string path_prefix = "/proc/pressure") noexcept | ||
| : registry_(registry), | ||
| path_prefix_(std::move(path_prefix)) {} | ||
|
|
||
| void set_prefix(std::string new_prefix) noexcept { path_prefix_ = std::move(new_prefix); } | ||
|
|
||
| void update_stats() noexcept { | ||
| auto lines = read_lines_fields(path_prefix_, "cpu"); | ||
| if (lines.size() == 2) { | ||
| auto some = registry_->GetMonotonicCounter(Id::of("sys.pressure.some", Tags{{"id", "cpu"}})); | ||
| auto usecs = std::strtoul(lines[0][4].substr(6).c_str(), nullptr, 10); | ||
| some->Set(usecs / MICROS); | ||
| } | ||
|
|
||
| lines = read_lines_fields(path_prefix_, "io"); | ||
| if (lines.size() == 2) { | ||
| auto some = registry_->GetMonotonicCounter(Id::of("sys.pressure.some", Tags{{"id", "io"}})); | ||
| auto usecs = std::strtoul(lines[0][4].substr(6).c_str(), nullptr, 10); | ||
| some->Set(usecs / MICROS); | ||
|
|
||
| auto full = registry_->GetMonotonicCounter(Id::of("sys.pressure.full", Tags{{"id", "io"}})); | ||
| usecs = std::strtoul(lines[1][4].substr(6).c_str(), nullptr, 10); | ||
| full->Set(usecs / MICROS); | ||
| } | ||
|
|
||
| lines = read_lines_fields(path_prefix_, "memory"); | ||
| if (lines.size() == 2) { | ||
| auto some = registry_->GetMonotonicCounter(Id::of("sys.pressure.some", Tags{{"id", "memory"}})); | ||
| auto usecs = std::strtoul(lines[0][4].substr(6).c_str(), nullptr, 10); | ||
| some->Set(usecs / MICROS); | ||
|
|
||
| auto full = registry_->GetMonotonicCounter(Id::of("sys.pressure.full", Tags{{"id", "memory"}})); | ||
| usecs = std::strtoul(lines[1][4].substr(6).c_str(), nullptr, 10); | ||
| full->Set(usecs / MICROS); | ||
| } | ||
| } | ||
|
|
||
| private: | ||
| Reg* registry_; | ||
| std::string path_prefix_; | ||
| static constexpr double MICROS = 1000 * 1000.0; | ||
| }; | ||
| } // namespace atlasagent |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| #include "measurement_utils.h" | ||
| #include "pressure_stall.h" | ||
| #include <gtest/gtest.h> | ||
|
|
||
| namespace { | ||
|
|
||
| using atlasagent::Logger; | ||
| using atlasagent::PressureStall; | ||
| using Registry = spectator::TestRegistry; | ||
| using spectator::Tags; | ||
|
|
||
| class PressureStallTest : public PressureStall<Registry> { | ||
| public: | ||
| explicit PressureStallTest(Registry* registry, std::string path_prefix = "/proc/pressure") | ||
| : PressureStall{registry, std::move(path_prefix)} {} | ||
|
|
||
| void stats() { | ||
| PressureStall::update_stats(); | ||
| } | ||
| }; | ||
|
|
||
| TEST(PressureStall, UpdateStats) { | ||
| Registry registry; | ||
| PressureStallTest pressure{®istry, "testdata/resources/proc/pressure"}; | ||
|
|
||
| pressure.stats(); | ||
| auto ms = registry.Measurements(); | ||
| EXPECT_EQ(ms.size(), 0); | ||
|
|
||
| // we need two samples, because these are all monotonic counters | ||
| pressure.set_prefix("testdata/resources/proc2/pressure"); | ||
| pressure.stats(); | ||
| ms = registry.Measurements(); | ||
| EXPECT_EQ(ms.size(), 5); | ||
|
|
||
| auto map = measurements_to_map(ms, ""); | ||
| std::unordered_map<std::string, double> expected = { | ||
| {"sys.pressure.some|count|cpu", 10}, | ||
| {"sys.pressure.some|count|io", 10}, | ||
| {"sys.pressure.some|count|memory", 10}, | ||
| {"sys.pressure.full|count|io", 20}, | ||
| {"sys.pressure.full|count|memory", 20}}; | ||
| EXPECT_EQ(map, expected); | ||
| } | ||
| } // namespace |
Oops, something went wrong.