Skip to content

Commit

Permalink
Rework most of the troubleshoot cli feature.
Browse files Browse the repository at this point in the history 
TODO:
  config validation
  cluster support
  hints based on enabled/disabled features (ex. no debug log enabled)
  check livestatus, DB IDO
  check system health maybe? (free disk space, load etc.)

refs #3446
  • Loading branch information
@Crunsher
Crunsher committed Feb 12, 2015
1 parent 2f6d0cc commit 7ce2e9d
Showing 1 changed file with 161 additions and 67 deletions.
228 changes: 161 additions & 67 deletions lib/cli/troubleshootcollectcommand.cpp
View file
Expand Up @@ -31,6 +31,7 @@
#include <boost/filesystem.hpp>
#include <boost/circular_buffer.hpp>
#include <boost/foreach.hpp>
#include <boost/algorithm/string/join.hpp>
#include <iomanip>
#include <iostream>
#include <time.h>
Expand All @@ -48,6 +49,32 @@ namespace po = boost::program_options;

REGISTER_CLICOMMAND("troubleshoot/collect", TroubleshootCollectCommand);

/*
//std::setw(int) is not sticky, a custom operator<< fixes this.
template<typename T>
class indented_os
{
private:
T& os;
public:
indented_os(T os) :
os(os) {};
template<typename T>
indented_os<T>& operator<< (const &T output)
{
os << std::setw(4) << output;
return *this;
}
indented_os& operator<<(std::ofstream& (*func) (std::ostream&))
{
func(os);
return *this;
}
};
*/
String TroubleshootCollectCommand::GetDescription(void) const
{
return "Collect logs and other relevant information for troubleshooting purposes.";
Expand All @@ -58,7 +85,7 @@ String TroubleshootCollectCommand::GetShortDescription(void) const
return "Collect information for troubleshooting";
}

static void getLatestReport(const String& filename, time_t& bestTimestamp, String& bestFilename)
static void GetLatestReport(const String& filename, time_t& bestTimestamp, String& bestFilename)
{
#ifdef _WIN32
struct _stat buf;
Expand All @@ -76,23 +103,23 @@ static void getLatestReport(const String& filename, time_t& bestTimestamp, Strin
}

/*Print the latest crash report to *os* */
static void printCrashReports(std::ostream& os)
static void PrintCrashReports(std::ostream& os)
{
String spath = Application::GetLocalStateDir() + "/log/icinga2/crash/report.*";
time_t bestTimestamp = 0;
String bestFilename;

try {
Utility::Glob(spath,
boost::bind(&getLatestReport, _1, boost::ref(bestTimestamp), boost::ref(bestFilename)), GlobFile);
boost::bind(&GetLatestReport, _1, boost::ref(bestTimestamp), boost::ref(bestFilename)), GlobFile);
}

#ifdef _WIN32
catch (win32_error &ex) {
if (int const * err = boost::get_error_info<errinfo_win32_error>(ex)) {
if (*err != 3) //Error code for path does not exist
throw ex;
os << Application::GetLocalStateDir() + "/log/icinga2/crash/ does not exist" << std::endl;
os << Application::GetLocalStateDir() + "/log/icinga2/crash/ does not exist\n";
} else {
throw ex;
}
Expand All @@ -105,15 +132,15 @@ static void printCrashReports(std::ostream& os)


if (!bestTimestamp)
os << "\n\tNo crash logs found in " << Application::GetLocalStateDir().CStr() << "/log/icinga2/crash/\n";
os << "\nNo crash logs found in " << Application::GetLocalStateDir().CStr() << "/log/icinga2/crash/\n";
else {
const std::tm tm = Utility::LocalTime(bestTimestamp);
char *tmBuf = new char[200]; //Should always be enough
const char *fmt = "%Y-%m-%d %H:%M:%S" ;
if (!strftime(tmBuf, 199, fmt, &tm))
return;
os << "\n\tLatest crash report is from " << tmBuf
<< "\n\tFile: " << bestFilename << std::endl;
os << "\nLatest crash report is from " << tmBuf
<< "\nFile: " << bestFilename << '\n';
TroubleshootCollectCommand::tail(bestFilename, 20, os);
}
}
Expand All @@ -139,20 +166,78 @@ int TroubleshootCollectCommand::tail(const String& file, int numLines, std::ostr
numLines = lines;

for (int k = 0; k < numLines; k++)
os << '\t' << ringBuf[k] << std::endl;
os << '\t' << ringBuf[k] << '\n';;

text.close();
return numLines;
}

/*Find all FileLoggers and print out their last 10 lines*/
static void printLogTail(const String& objectfile, std::ostream& os)
static void PrintIcingaConf(std::ostream& os)
{
String path = Application::GetSysconfDir() + "/icinga2/icinga2.conf";

std::ifstream text;
text.open(path.CStr(), std::ifstream::in);
std::string line;

os << "\nFound main Icinga2 configuration file at " << path << '\n';
while (std::getline(text, line)) {
os << '\t' << line << '\n';
}
}

static void PrintZonesConf(std::ostream& os)
{
String path = Application::GetSysconfDir() + "/icinga2/zones.conf";

std::ifstream text;
text.open(path.CStr(), std::ifstream::in);
std::string line;

os << "\nFound zones configuration file at " << path << '\n';
while (std::getline(text, line)) {
os << '\t' << line << '\n';
}
}

static void CheckFeatures(std::ostream& os)
{
Dictionary::Ptr features = new Dictionary;
std::vector<String> disabled_features;
std::vector<String> enabled_features;

if (!FeatureUtility::GetFeatures(disabled_features, true)
|| !FeatureUtility::GetFeatures(enabled_features, false)) {
os << "Failed to collect enabled and/or disabled features. Check\n"
<< FeatureUtility::GetFeaturesAvailablePath() << '\n'
<< FeatureUtility::GetFeaturesEnabledPath() << '\n';
return;
}

BOOST_FOREACH(const String feature, disabled_features)
features->Set(feature, 0);
BOOST_FOREACH(const String feature, disabled_features)
features->Set(feature, 1);

os << "Icinga2 feature list\n"
<< "Enabled features:\n\t" << boost::algorithm::join(enabled_features, " ") << '\n'
<< "Disabled features:\n\t" << boost::algorithm::join(disabled_features, " ") << '\n';
}

static void CheckObjectFile(const String& objectfile, std::ostream& os)
{

os << "Checking object file from " << objectfile << '\n';

std::fstream fp;
std::set<String> configSet;
Dictionary::Ptr typeCount = new Dictionary();
Dictionary::Ptr logPath = new Dictionary();
fp.open(objectfile.CStr(), std::ios_base::in);

StdioStream::Ptr sfp = new StdioStream(&fp, false);
String message;
int typeL = 0, countTotal = 0;

while (NetString::ReadStringFromStream(sfp, &message)) {
Dictionary::Ptr object = JsonDecode(message);
Expand All @@ -161,60 +246,59 @@ static void printLogTail(const String& objectfile, std::ostream& os)
String name = object->Get("name");
String type = object->Get("type");

//Find longest typename for padding
typeL = type.GetLength() > typeL ? type.GetLength() : typeL;
countTotal++;

if (!typeCount->Contains(type))
typeCount->Set(type, 1);
else
typeCount->Set(type, typeCount->Get(type)+1);

Array::Ptr debug_info = object->Get("debug_info");
if (debug_info) {
configSet.insert(debug_info->Get(0));
}

if (Utility::Match(type, "FileLogger")) {
Dictionary::Ptr debug_hints = object->Get("debug_hints");
Dictionary::Ptr properties = object->Get("properties");

ObjectLock olock(properties);
BOOST_FOREACH(const Dictionary::Pair& kv, properties)
{
String key = kv.first;
Value val = kv.second;
if (Utility::Match(key, "path")) {
os << "\nFound Log " << object->Get("name") << " at path: " << val << "\n\n";
if (!TroubleshootCollectCommand::tail(val, 10, os))
os << '\t' << val << " either does not exist or is empty\n";
}
BOOST_FOREACH(const Dictionary::Pair& kv, properties) {
if (Utility::Match(kv.first, "path"))
logPath->Set(name, kv.second);
}
}
}

printCrashReports(os);
}

static void printConfigTree(const String& objectfile, std::ostream& os)
{
std::fstream fp;
std::set<String> configSet;
fp.open(objectfile.CStr(), std::ios_base::in);

StdioStream::Ptr sfp = new StdioStream(&fp, false);
String message;

while (NetString::ReadStringFromStream(sfp, &message)) {
Dictionary::Ptr object = JsonDecode(message);
Array::Ptr debug_info = object->Get("debug_info");
if (debug_info)
configSet.insert(debug_info->Get(0));
//Print objects with count
os << "Found the following objects:\n"
<< "\tType" << std::string(typeL-4, ' ') << " : Count\n";
ObjectLock olock(typeCount);
BOOST_FOREACH(const Dictionary::Pair& kv, typeCount) {
os << '\t' << kv.first << std::string(typeL - kv.first.GetLength(), ' ')
<< " : " << kv.second << '\n';
}

os << "Found the following config files used by objects:\n";
//Print location of .config files
os << '\n' << countTotal << " objects in total, originating from these files:\n";
for (std::set<String>::iterator it = configSet.begin();
it != configSet.end(); it++)
os << '\t' << *it << '\n';
os << std::endl;
}

static void ConfigValid(std::ostream& os)
{
std::string message = (ConfigItem::CommitItems() ?
"Failed to validate config, check the log for further information\n" :
"Successfuly validated config, check the log for additional information\n");
os << message;
it != configSet.end(); it++)
os << '\t' << *it << '\n';

//Print tail of file loggers
os << "\nGetting the last 20 lines of all found FileLogger objects.\n";
ObjectLock ulock(logPath);
BOOST_FOREACH(const Dictionary::Pair& kv, logPath) {
os << "\nLogger " << kv.first << " at path: " << kv.second << "\n";
if (!TroubleshootCollectCommand::tail(kv.second, 20, os))
os << "\t" << kv.second << " either does not exist or is empty\n";
}
}

void TroubleshootCollectCommand::InitParameters(boost::program_options::options_description& visibleDesc,
boost::program_options::options_description& hiddenDesc) const
boost::program_options::options_description& hiddenDesc) const
{
visibleDesc.add_options()
("console,c", "print to console instead of file")
Expand All @@ -232,44 +316,54 @@ int TroubleshootCollectCommand::Run(const boost::program_options::variables_map&
os.open((Application::GetLocalStateDir() + "/log/icinga2/troubleshooting.log").CStr(),
std::ios::out | std::ios::trunc);
if (!os.is_open()) {
std::cout << "Could not open " << (Application::GetLocalStateDir() + "/log/icinga2/troubleshooting.log") << " to write"
<< std::endl;
std::cout << "Could not open " << (Application::GetLocalStateDir() + "/log/icinga2/troubleshooting.log") << " to write\n";
return 3;
}
}


String appName = Utility::BaseName(Application::GetArgV()[0]);

os << appName << " -- Troubleshooting help:" << std::endl
<< "Should you run in problems with icinga please add this file to your help request\n";
<< "Should you run into problems with Icinga please add this file to your help request\n\n";

if (appName.GetLength() > 3 && appName.SubStr(0, 3) == "lt-")
appName = appName.SubStr(3, appName.GetLength() - 3);

os << std::endl;
Application::DisplayInfoMessage(os);
os << std::endl;
FeatureUtility::ListFeatures(os);
os << std::endl;

ConfigValid(os);
//Application::DisplayInfoMessage() but formatted
os << "\tApplication version: " << GetVersion() << "\n"
<< "\tInstallation root: " << Application::GetPrefixDir() << "\n"
<< "\tSysconf directory: " << Application::GetSysconfDir() << "\n"
<< "\tRun directory: " << Application::GetRunDir() << "\n"
<< "\tLocal state directory: " << Application::GetLocalStateDir() << "\n"
<< "\tPackage data directory: " << Application::GetPkgDataDir() << "\n"
<< "\tState path: " << Application::GetStatePath() << "\n"
<< "\tObjects path: " << Application::GetObjectsPath() << "\n"
<< "\tVars path: " << Application::GetVarsPath() << "\n"
<< "\tPID path: " << Application::GetPidPath() << "\n"
<< "\tApplication type: " << Application::GetApplicationType() << "\n";

os << '\n';
CheckFeatures(os);
os << '\n';

String objectfile = Application::GetObjectsPath();

if (!Utility::PathExists(objectfile)) {
os << "Cannot open objects file '" << Application::GetObjectsPath() << "'."
os << "Cannot open object file '" << objectfile << "'."
<< "Run 'icinga2 daemon -C' to validate config and generate the cache file.\n";
} else {

printConfigTree(objectfile, os);
printLogTail(objectfile, os);
}
} else
CheckObjectFile(objectfile, os);

os << "\nA collection of important configuration files follows, please make sure to censor your sensible data\n";
PrintIcingaConf(os);
os << '\n';
PrintZonesConf(os);
os << std::endl;

std::cout << "Finished collection";
if (!vm.count("console")) {
os.close();
std::cout << ", printed to \"" << Application::GetLocalStateDir() + "/log/icinga2/troubleshooting.log\"";
std::cout << ", see \"" << Application::GetLocalStateDir() + "/log/icinga2/troubleshooting.log\"";
}
std::cout << std::endl;

Expand Down

0 comments on commit 7ce2e9d

Please sign in to comment.