From 989a317b637a417ec4f7b1981b2c4b84e4e660e9 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Tue, 29 Oct 2013 13:44:43 +0100 Subject: [PATCH] Livestatus: Add log table (WIP). refs #4433 --- components/livestatus/listener.cpp | 6 +- components/livestatus/logtable.cpp | 130 +++++++++++++++++++++++++--- components/livestatus/logtable.h | 13 ++- components/livestatus/query.cpp | 21 +++-- components/livestatus/query.h | 5 +- components/livestatus/table.cpp | 4 +- components/livestatus/table.h | 3 +- test/livestatus/queries/log/log | 4 + test/livestatus/queries/log/minimal | 5 ++ test/livestatus/run_queries | 2 +- 10 files changed, 164 insertions(+), 29 deletions(-) create mode 100644 test/livestatus/queries/log/log create mode 100644 test/livestatus/queries/log/minimal diff --git a/components/livestatus/listener.cpp b/components/livestatus/listener.cpp index a508a46977d..421de615025 100644 --- a/components/livestatus/listener.cpp +++ b/components/livestatus/listener.cpp @@ -28,10 +28,10 @@ #include "base/networkstream.h" #include "base/application.h" #include "base/scriptfunction.h" +#include "base/convert.h" #include #include - using namespace icinga; using namespace livestatus; @@ -145,6 +145,7 @@ void LivestatusListener::ClientThreadProc(const Socket::Ptr& client) } } + void LivestatusListener::ValidateSocketType(const String& location, const Dictionary::Ptr& attrs) { Value socket_type = attrs->Get("socket_type"); @@ -153,5 +154,4 @@ void LivestatusListener::ValidateSocketType(const String& location, const Dictio ConfigCompilerContext::GetInstance()->AddMessage(true, "Validation failed for " + location + ": Socket type '" + socket_type + "' is invalid."); } -} - +} \ No newline at end of file diff --git a/components/livestatus/logtable.cpp b/components/livestatus/logtable.cpp index 9e46bc11775..3f7193188cb 100644 --- a/components/livestatus/logtable.cpp +++ b/components/livestatus/logtable.cpp @@ -20,13 +20,83 @@ #include "livestatus/logtable.h" #include "icinga/icingaapplication.h" #include "icinga/cib.h" +#include "base/convert.h" +#include "base/utility.h" +#include "base/logger_fwd.h" +#include "base/application.h" +#include "base/objectlock.h" #include +#include +#include +#include +#include using namespace icinga; using namespace livestatus; -LogTable::LogTable(void) +LogTable::LogTable(const unsigned long& from, const unsigned long& until) { + Log(LogInformation, "livestatus", "Pre-selecting log file from " + Convert::ToString(from) + " until " + Convert::ToString(until)); + + /* store from & until for FetchRows */ + m_TimeFrom = from; + m_TimeUntil = until; + + /* create log file index - TODO config option */ + CreateLogIndex(Application::GetLocalStateDir() + "/log/icinga2/compat"); + + /* m_LogFileIndex map tells which log files are involved ordered by their start timestamp */ + unsigned long ts; + unsigned long line_count = 0; + BOOST_FOREACH(boost::tie(ts, boost::tuples::ignore), m_LogFileIndex) { + if (ts < m_TimeFrom || ts > m_TimeUntil) //this is not entirely correct - logfile index only holds the start time! TODO + continue; + + String log_file = m_LogFileIndex[ts]; + + std::ifstream fp; + fp.exceptions(std::ifstream::badbit); + fp.open(log_file.CStr(), std::ifstream::in); + + while (fp.good()) { + std::string line; + std::getline(fp, line); + + if (line.empty()) + continue; /* Ignore empty lines */ + /* + * [1379025342] SERVICE NOTIFICATION: testconfig-admin;1066localhost;cert[exp]: thomas-1.office.crt;WARNING;true;sudo: Kein TTY vorhanden und kein »askpass«-Programm angegeben + */ + unsigned long time = atoi(line.substr(1, 11).c_str()); + + size_t colon = line.find_first_of(':'); + size_t colon_offset = colon - 13; + + std::string type = line.substr(13, colon_offset); + std::string options = line.substr(colon + 1); + + /* TODO - find class for log entry */ + + Dictionary::Ptr bag = boost::make_shared(); + bag->Set("time", time); + bag->Set("lineno", line_count); + bag->Set("class", 0); /* 0 is the default */ + bag->Set("message", String(line)); /* complete line */ + bag->Set("type", String(type)); + bag->Set("options", String(options)); + + /* TODO: comment, plugin_output, state, state_type, attempt */ + { + boost::mutex::scoped_lock lock(m_Mutex); + m_RowsCache[line_count] = bag; + } + + line_count++; + } + + fp.close(); + } + AddColumns(this); } @@ -59,22 +129,22 @@ String LogTable::GetName(void) const void LogTable::FetchRows(const AddRowFunction& addRowFn) { - Object::Ptr obj = boost::make_shared(); + unsigned long lineno; - /* Return a fake row. */ - addRowFn(obj); + BOOST_FOREACH(boost::tie(lineno, boost::tuples::ignore), m_RowsCache) { + /* pass a dictionary with "lineno" as key */ + addRowFn(m_RowsCache[lineno]); + } } Value LogTable::TimeAccessor(const Value& row) { - /* not supported */ - return Empty; + return static_cast(row)->Get("time"); } Value LogTable::LinenoAccessor(const Value& row) { - /* not supported */ - return Empty; + return static_cast(row)->Get("lineno"); } Value LogTable::ClassAccessor(const Value& row) @@ -85,20 +155,17 @@ Value LogTable::ClassAccessor(const Value& row) Value LogTable::MessageAccessor(const Value& row) { - /* not supported */ - return Empty; + return static_cast(row)->Get("message"); } Value LogTable::TypeAccessor(const Value& row) { - /* not supported */ - return Empty; + return static_cast(row)->Get("type"); } Value LogTable::OptionsAccessor(const Value& row) { - /* not supported */ - return Empty; + return static_cast(row)->Get("options"); } Value LogTable::CommentAccessor(const Value& row) @@ -154,3 +221,38 @@ Value LogTable::CommandNameAccessor(const Value& row) /* not supported */ return Empty; } + +void LogTable::CreateLogIndex(const String& path) +{ + Utility::Glob(path + "/icinga.log", boost::bind(&LogTable::CreateLogIndexFileHandler, _1, boost::ref(m_LogFileIndex))); + Utility::Glob(path + "/archives/*.log", boost::bind(&LogTable::CreateLogIndexFileHandler, _1, boost::ref(m_LogFileIndex))); +} + +void LogTable::CreateLogIndexFileHandler(const String& path, std::map& index) +{ + std::ifstream stream; + stream.open(path.CStr(), std::ifstream::in); + + if (!stream) + BOOST_THROW_EXCEPTION(std::runtime_error("Could not open log file: " + path)); + + /* read the first bytes to get the timestamp: [123456789] */ + char buffer[12]; + + stream.read(buffer, 12); + + if (buffer[0] != '[' || buffer[11] != ']') { + /* this can happen for directories too, silently ignore them */ + return; + } + + /* extract timestamp */ + buffer[11] = 0; + unsigned int ts_start = atoi(buffer+1); + + stream.close(); + + Log(LogInformation, "livestatus", "Indexing log file: '" + path + "' with timestamp start: '" + Convert::ToString(ts_start) + "'."); + + index[ts_start] = path; +} diff --git a/components/livestatus/logtable.h b/components/livestatus/logtable.h index f94b8ae6730..255e77100f1 100644 --- a/components/livestatus/logtable.h +++ b/components/livestatus/logtable.h @@ -21,6 +21,7 @@ #define LOGTABLE_H #include "livestatus/table.h" +#include using namespace icinga; @@ -35,7 +36,7 @@ class LogTable : public Table public: DECLARE_PTR_TYPEDEFS(LogTable); - LogTable(void); + LogTable(const unsigned long& from, const unsigned long& until); static void AddColumns(Table *table, const String& prefix = String(), const Column::ObjectAccessor& objectAccessor = Column::ObjectAccessor()); @@ -60,6 +61,16 @@ class LogTable : public Table static Value HostNameAccessor(const Value& row); static Value ContactNameAccessor(const Value& row); static Value CommandNameAccessor(const Value& row); + +private: + std::map m_LogFileIndex; + std::map m_RowsCache; + unsigned long m_TimeFrom; + unsigned long m_TimeUntil; + boost::mutex m_Mutex; + + void CreateLogIndex(const String& path); + static void CreateLogIndexFileHandler(const String& path, std::map& index); }; } diff --git a/components/livestatus/query.cpp b/components/livestatus/query.cpp index 7181ac400c6..f4a9a100ef2 100644 --- a/components/livestatus/query.cpp +++ b/components/livestatus/query.cpp @@ -36,6 +36,7 @@ #include "base/objectlock.h" #include "base/logger_fwd.h" #include "base/exception.h" +#include "base/utility.h" #include #include #include @@ -48,7 +49,8 @@ static int l_ExternalCommands = 0; static boost::mutex l_QueryMutex; Query::Query(const std::vector& lines) - : m_KeepAlive(false), m_OutputFormat("csv"), m_ColumnHeaders(true), m_Limit(-1) + : m_KeepAlive(false), m_OutputFormat("csv"), m_ColumnHeaders(true), m_Limit(-1), + m_LogTimeFrom(0), m_LogTimeUntil(static_cast(Utility::GetTime())) { if (lines.size() == 0) { m_Verb = "ERROR"; @@ -120,7 +122,7 @@ Query::Query(const std::vector& lines) else if (header == "ColumnHeaders") m_ColumnHeaders = (params == "on"); else if (header == "Filter") { - Filter::Ptr filter = ParseFilter(params); + Filter::Ptr filter = ParseFilter(params, m_LogTimeFrom, m_LogTimeUntil); if (!filter) { m_Verb = "ERROR"; @@ -162,7 +164,7 @@ Query::Query(const std::vector& lines) } else if (aggregate_arg == "avginv") { aggregator = boost::make_shared(aggregate_attr); } else { - filter = ParseFilter(params); + filter = ParseFilter(params, m_LogTimeFrom, m_LogTimeUntil); if (!filter) { m_Verb = "ERROR"; @@ -249,7 +251,7 @@ int Query::GetExternalCommands(void) return l_ExternalCommands; } -Filter::Ptr Query::ParseFilter(const String& params) +Filter::Ptr Query::ParseFilter(const String& params, unsigned long& from, unsigned long& until) { std::vector tokens; boost::algorithm::split(tokens, params, boost::is_any_of(" ")); @@ -282,6 +284,15 @@ Filter::Ptr Query::ParseFilter(const String& params) if (negate) filter = boost::make_shared(filter); + /* pre-filter log time duration */ + if (tokens[0] == "time") { + if (op == "<" || op == "<=") { + until = Convert::ToLong(tokens[2]); + } else if (op == ">" || op == ">=") { + from = Convert::ToLong(tokens[2]); + } + } + return filter; } @@ -350,7 +361,7 @@ void Query::ExecuteGetHelper(const Stream::Ptr& stream) { Log(LogInformation, "livestatus", "Table: " + m_Table); - Table::Ptr table = Table::GetByName(m_Table); + Table::Ptr table = Table::GetByName(m_Table, m_LogTimeFrom, m_LogTimeUntil); if (!table) { SendResponse(stream, LivestatusErrorNotFound, "Table '" + m_Table + "' does not exist."); diff --git a/components/livestatus/query.h b/components/livestatus/query.h index d92104797e0..14e85eb9415 100644 --- a/components/livestatus/query.h +++ b/components/livestatus/query.h @@ -78,6 +78,9 @@ class Query : public Object /* Parameters for invalid queries. */ int m_ErrorCode; String m_ErrorMessage; + + unsigned long m_LogTimeFrom; + unsigned long m_LogTimeUntil; void PrintResultSet(std::ostream& fp, const std::vector& columns, const Array::Ptr& rs); void PrintCsvArray(std::ostream& fp, const Array::Ptr& array, int level); @@ -89,7 +92,7 @@ class Query : public Object void SendResponse(const Stream::Ptr& stream, int code, const String& data); void PrintFixed16(const Stream::Ptr& stream, int code, const String& data); - static Filter::Ptr ParseFilter(const String& params); + static Filter::Ptr ParseFilter(const String& params, unsigned long& from, unsigned long& until); }; } diff --git a/components/livestatus/table.cpp b/components/livestatus/table.cpp index 1ba47029541..b34f8a15d43 100644 --- a/components/livestatus/table.cpp +++ b/components/livestatus/table.cpp @@ -44,7 +44,7 @@ using namespace livestatus; Table::Table(void) { } -Table::Ptr Table::GetByName(const String& name) +Table::Ptr Table::GetByName(const String& name, const unsigned long& from, const unsigned long& until) { if (name == "status") return boost::make_shared(); @@ -69,7 +69,7 @@ Table::Ptr Table::GetByName(const String& name) else if (name == "timeperiods") return boost::make_shared(); else if (name == "log") - return boost::make_shared(); + return boost::make_shared(from, until); return Table::Ptr(); } diff --git a/components/livestatus/table.h b/components/livestatus/table.h index db8433a3925..08b5273edb2 100644 --- a/components/livestatus/table.h +++ b/components/livestatus/table.h @@ -40,8 +40,7 @@ class Table : public Object typedef boost::function AddRowFunction; - static Table::Ptr GetByName(const String& name); - + static Table::Ptr GetByName(const String& name, const unsigned long& from = 0, const unsigned long& until = 0); virtual String GetName(void) const = 0; diff --git a/test/livestatus/queries/log/log b/test/livestatus/queries/log/log new file mode 100644 index 00000000000..f69f2f44809 --- /dev/null +++ b/test/livestatus/queries/log/log @@ -0,0 +1,4 @@ +GET log +Filter: time >= 1348657741 +ResponseHeader: fixed16 + diff --git a/test/livestatus/queries/log/minimal b/test/livestatus/queries/log/minimal new file mode 100644 index 00000000000..9ed3ffc2796 --- /dev/null +++ b/test/livestatus/queries/log/minimal @@ -0,0 +1,5 @@ +GET log +Columns: host_name service_description time lineno class type options +Filter: time >= 1348657741 +ResponseHeader: fixed16 + diff --git a/test/livestatus/run_queries b/test/livestatus/run_queries index 0c5eea427c2..f0c36c8037a 100755 --- a/test/livestatus/run_queries +++ b/test/livestatus/run_queries @@ -1,6 +1,6 @@ #!/bin/bash -LIVESTATUSHOST="10.0.10.18" +LIVESTATUSHOST="127.0.0.1" LIVESTATUSPORT="6558" LIVESTATUSQUERIES="./queries"