diff --git a/golang/pkg/client/client.go b/golang/pkg/client/client.go index bc843dd9e..6644943b3 100644 --- a/golang/pkg/client/client.go +++ b/golang/pkg/client/client.go @@ -104,13 +104,13 @@ func (c *client) getMetrics(url string, payload interface{}) error { } func (c *client) GetStats() (*StatSnapshot, error) { - var rawStats map[string]map[string]interface{} + var rawStats map[string]map[string]map[string]interface{} err := c.getMetrics(fmt.Sprintf("%s://%s:%d/api/v1/policies/%s/metrics/window/5", c.config.Protocol, c.config.Host, c.config.Port, c.config.DefaultPolicy), &rawStats) if err != nil { return nil, err } stats := StatSnapshot{} - for _, handlerData := range rawStats { + for _, handlerData := range rawStats[c.config.DefaultPolicy] { if data, ok := handlerData["pcap"]; ok { err := mapstructure.Decode(data, &stats.Pcap) if err != nil { diff --git a/src/AbstractManager.h b/src/AbstractManager.h index 7ec3ff127..d9ee0ce47 100644 --- a/src/AbstractManager.h +++ b/src/AbstractManager.h @@ -55,6 +55,15 @@ class AbstractManager { } + std::vector module_get_keys() const { + std::shared_lock lock(_map_mutex); + std::vector result; + for (auto &kv : _map) { + result.emplace_back(kv.first); + } + return result; + } + auto module_get_all_locked() { struct retVals { diff --git a/src/CoreServer.cpp b/src/CoreServer.cpp index 88343406b..bd4acbabd 100644 --- a/src/CoreServer.cpp +++ b/src/CoreServer.cpp @@ -203,7 +203,7 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config) }); // Policies _svr.Get(R"(/api/v1/policies)", [&]([[maybe_unused]] const httplib::Request &req, httplib::Response &res) { - json j; + json j = json::object(); try { auto [policy_modules, hm_lock] = _registry.policy_manager()->module_get_all_locked(); for (auto &[name, mod] : policy_modules) { @@ -220,7 +220,7 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config) } }); _svr.Post(R"(/api/v1/policies)", [&](const httplib::Request &req, httplib::Response &res) { - json j; + json j = json::object(); if (!req.has_header("Content-Type")) { res.status = 400; j["error"] = "must include Content-Type header"; @@ -247,7 +247,7 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config) } }); _svr.Get(fmt::format("/api/v1/policies/({})", AbstractModule::MODULE_ID_REGEX).c_str(), [&](const httplib::Request &req, httplib::Response &res) { - json j; + json j = json::object(); auto name = req.matches[1]; if (!_registry.policy_manager()->module_exists(name)) { res.status = 404; @@ -257,7 +257,7 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config) } try { auto [policy, lock] = _registry.policy_manager()->module_get_locked(name); - policy->info_json(j); + policy->info_json(j[name]); res.set_content(j.dump(), "text/json"); } catch (const std::exception &e) { res.status = 500; @@ -266,7 +266,7 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config) } }); _svr.Delete(fmt::format("/api/v1/policies/({})", AbstractModule::MODULE_ID_REGEX).c_str(), [&](const httplib::Request &req, httplib::Response &res) { - json j; + json j = json::object(); auto name = req.matches[1]; if (!_registry.policy_manager()->module_exists(name)) { res.status = 404; @@ -287,54 +287,51 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config) res.set_content(j.dump(), "text/json"); } }); - _svr.Get(fmt::format("/api/v1/policies/({})/metrics/bucket/(\\d+)", AbstractModule::MODULE_ID_REGEX).c_str(), [&](const httplib::Request &req, httplib::Response &res) { - json j; + _svr.Get(fmt::format("/api/v1/policies/({})/metrics/(window|bucket)/(\\d+)", AbstractModule::MODULE_ID_REGEX).c_str(), [&](const httplib::Request &req, httplib::Response &res) { + json j = json::object(); auto name = req.matches[1]; - if (!_registry.policy_manager()->module_exists(name)) { + std::vector plist; + if (name == "__all") { + // special route to get all policy metrics in one call, for scraping performance reasons + plist = _registry.policy_manager()->module_get_keys(); + } else if (!_registry.policy_manager()->module_exists(name)) { res.status = 404; j["error"] = "policy does not exist"; res.set_content(j.dump(), "text/json"); return; + } else { + plist.emplace_back(name); } try { - auto [policy, lock] = _registry.policy_manager()->module_get_locked(name); - uint64_t period(std::stol(req.matches[2])); - for (auto &mod : policy->modules()) { - auto hmod = dynamic_cast(mod); - if (hmod) { - spdlog::stopwatch sw; - hmod->window_json(j[hmod->name()], period, false); - _logger->debug("{} bucket window_json elapsed time: {}", hmod->name(), sw); + for (const auto &p_mname : plist) { + spdlog::stopwatch psw; + auto [policy, lock] = _registry.policy_manager()->module_get_locked(p_mname); + uint64_t period(std::stol(req.matches[3])); + for (auto &mod : policy->modules()) { + auto hmod = dynamic_cast(mod); + assert(hmod); + try { + spdlog::stopwatch hsw; + hmod->window_json(j[policy->name()][hmod->name()], period, req.matches[2] == "window"); + _logger->debug("{} handler bucket json elapsed time: {}", hmod->name(), hsw); + } catch (const PeriodException &e) { + // if period is bad for a single policy in __all mode, skip it. otherwise fail + if (name == "__all") { + _logger->warn("{} handler for policy {} had a PeriodException, skipping: {}", hmod->name(), policy->name(), e.what()); + j.erase(policy->name()); + continue; + } else { + throw e; + } + } } + _logger->debug("{} policy json metrics elapsed time: {}", policy->name(), psw); } res.set_content(j.dump(), "text/json"); - } catch (const std::exception &e) { - res.status = 500; + } catch (const PeriodException &e) { + res.status = 425; // 425 Too Early j["error"] = e.what(); res.set_content(j.dump(), "text/json"); - } - }); - _svr.Get(fmt::format("/api/v1/policies/({})/metrics/window/(\\d+)", AbstractModule::MODULE_ID_REGEX).c_str(), [&](const httplib::Request &req, httplib::Response &res) { - json j; - auto name = req.matches[1]; - if (!_registry.policy_manager()->module_exists(name)) { - res.status = 404; - j["error"] = "policy does not exist"; - res.set_content(j.dump(), "text/json"); - return; - } - try { - auto [policy, lock] = _registry.policy_manager()->module_get_locked(name); - uint64_t period(std::stol(req.matches[2])); - for (auto &mod : policy->modules()) { - auto hmod = dynamic_cast(mod); - if (hmod) { - spdlog::stopwatch sw; - hmod->window_json(j[hmod->name()], period, true); - _logger->debug("{} bucket window_json elapsed time: {}", hmod->name(), sw); - } - } - res.set_content(j.dump(), "text/json"); } catch (const std::exception &e) { res.status = 500; j["error"] = e.what();