Skip to content

get all policies at once #136

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 2, 2021
Merged

get all policies at once #136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
37a6864
namespace json output to policy name
weyrick Nov 1, 2021
8fa0de0
namespace json output to policy name
weyrick Nov 1, 2021
d717717
allow getting all policy json output at once with special "__all" policy
weyrick Nov 1, 2021
1eb2dd7
default json output to objects
weyrick Nov 1, 2021
94b2656
handle PeriodException
weyrick Nov 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions golang/pkg/client/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,13 +104,13 @@ func (c *client) getMetrics(url string, payload interface{}) error {
}

func (c *client) GetStats() (*StatSnapshot, error) {
var rawStats map[string]map[string]interface{}
var rawStats map[string]map[string]map[string]interface{}
err := c.getMetrics(fmt.Sprintf("%s://%s:%d/api/v1/policies/%s/metrics/window/5", c.config.Protocol, c.config.Host, c.config.Port, c.config.DefaultPolicy), &rawStats)
if err != nil {
return nil, err
}
stats := StatSnapshot{}
for _, handlerData := range rawStats {
for _, handlerData := range rawStats[c.config.DefaultPolicy] {
if data, ok := handlerData["pcap"]; ok {
err := mapstructure.Decode(data, &stats.Pcap)
if err != nil {
Expand Down
9 changes: 9 additions & 0 deletions src/AbstractManager.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,15 @@ class AbstractManager
{
}

std::vector<std::string> module_get_keys() const {
std::shared_lock lock(_map_mutex);
std::vector<std::string> result;
for (auto &kv : _map) {
result.emplace_back(kv.first);
}
return result;
}

auto module_get_all_locked()
{
struct retVals {
Expand Down
79 changes: 38 additions & 41 deletions src/CoreServer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config)
});
// Policies
_svr.Get(R"(/api/v1/policies)", [&]([[maybe_unused]] const httplib::Request &req, httplib::Response &res) {
json j;
json j = json::object();
try {
auto [policy_modules, hm_lock] = _registry.policy_manager()->module_get_all_locked();
for (auto &[name, mod] : policy_modules) {
Expand All @@ -220,7 +220,7 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config)
}
});
_svr.Post(R"(/api/v1/policies)", [&](const httplib::Request &req, httplib::Response &res) {
json j;
json j = json::object();
if (!req.has_header("Content-Type")) {
res.status = 400;
j["error"] = "must include Content-Type header";
Expand All @@ -247,7 +247,7 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config)
}
});
_svr.Get(fmt::format("/api/v1/policies/({})", AbstractModule::MODULE_ID_REGEX).c_str(), [&](const httplib::Request &req, httplib::Response &res) {
json j;
json j = json::object();
auto name = req.matches[1];
if (!_registry.policy_manager()->module_exists(name)) {
res.status = 404;
Expand All @@ -257,7 +257,7 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config)
}
try {
auto [policy, lock] = _registry.policy_manager()->module_get_locked(name);
policy->info_json(j);
policy->info_json(j[name]);
res.set_content(j.dump(), "text/json");
} catch (const std::exception &e) {
res.status = 500;
Expand All @@ -266,7 +266,7 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config)
}
});
_svr.Delete(fmt::format("/api/v1/policies/({})", AbstractModule::MODULE_ID_REGEX).c_str(), [&](const httplib::Request &req, httplib::Response &res) {
json j;
json j = json::object();
auto name = req.matches[1];
if (!_registry.policy_manager()->module_exists(name)) {
res.status = 404;
Expand All @@ -287,54 +287,51 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config)
res.set_content(j.dump(), "text/json");
}
});
_svr.Get(fmt::format("/api/v1/policies/({})/metrics/bucket/(\\d+)", AbstractModule::MODULE_ID_REGEX).c_str(), [&](const httplib::Request &req, httplib::Response &res) {
json j;
_svr.Get(fmt::format("/api/v1/policies/({})/metrics/(window|bucket)/(\\d+)", AbstractModule::MODULE_ID_REGEX).c_str(), [&](const httplib::Request &req, httplib::Response &res) {
json j = json::object();
auto name = req.matches[1];
if (!_registry.policy_manager()->module_exists(name)) {
std::vector<std::string> plist;
if (name == "__all") {
// special route to get all policy metrics in one call, for scraping performance reasons
plist = _registry.policy_manager()->module_get_keys();
} else if (!_registry.policy_manager()->module_exists(name)) {
res.status = 404;
j["error"] = "policy does not exist";
res.set_content(j.dump(), "text/json");
return;
} else {
plist.emplace_back(name);
}
try {
auto [policy, lock] = _registry.policy_manager()->module_get_locked(name);
uint64_t period(std::stol(req.matches[2]));
for (auto &mod : policy->modules()) {
auto hmod = dynamic_cast<StreamHandler *>(mod);
if (hmod) {
spdlog::stopwatch sw;
hmod->window_json(j[hmod->name()], period, false);
_logger->debug("{} bucket window_json elapsed time: {}", hmod->name(), sw);
for (const auto &p_mname : plist) {
spdlog::stopwatch psw;
auto [policy, lock] = _registry.policy_manager()->module_get_locked(p_mname);
uint64_t period(std::stol(req.matches[3]));
for (auto &mod : policy->modules()) {
auto hmod = dynamic_cast<StreamHandler *>(mod);
assert(hmod);
try {
spdlog::stopwatch hsw;
hmod->window_json(j[policy->name()][hmod->name()], period, req.matches[2] == "window");
_logger->debug("{} handler bucket json elapsed time: {}", hmod->name(), hsw);
} catch (const PeriodException &e) {
// if period is bad for a single policy in __all mode, skip it. otherwise fail
if (name == "__all") {
_logger->warn("{} handler for policy {} had a PeriodException, skipping: {}", hmod->name(), policy->name(), e.what());
j.erase(policy->name());
continue;
} else {
throw e;
}
}
}
_logger->debug("{} policy json metrics elapsed time: {}", policy->name(), psw);
}
res.set_content(j.dump(), "text/json");
} catch (const std::exception &e) {
res.status = 500;
} catch (const PeriodException &e) {
res.status = 425; // 425 Too Early
j["error"] = e.what();
res.set_content(j.dump(), "text/json");
}
});
_svr.Get(fmt::format("/api/v1/policies/({})/metrics/window/(\\d+)", AbstractModule::MODULE_ID_REGEX).c_str(), [&](const httplib::Request &req, httplib::Response &res) {
json j;
auto name = req.matches[1];
if (!_registry.policy_manager()->module_exists(name)) {
res.status = 404;
j["error"] = "policy does not exist";
res.set_content(j.dump(), "text/json");
return;
}
try {
auto [policy, lock] = _registry.policy_manager()->module_get_locked(name);
uint64_t period(std::stol(req.matches[2]));
for (auto &mod : policy->modules()) {
auto hmod = dynamic_cast<StreamHandler *>(mod);
if (hmod) {
spdlog::stopwatch sw;
hmod->window_json(j[hmod->name()], period, true);
_logger->debug("{} bucket window_json elapsed time: {}", hmod->name(), sw);
}
}
res.set_content(j.dump(), "text/json");
} catch (const std::exception &e) {
res.status = 500;
j["error"] = e.what();
Expand Down