Permalink
Fetching contributors…
Cannot retrieve contributors at this time
463 lines (408 sloc) 13.3 KB
/** @file
Function defs for the Alarms keeper.
@section license License
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#include "ts/ink_platform.h"
#include "ts/ink_string.h"
#include "ts/ink_file.h"
#include "ts/ink_time.h"
#include "LocalManager.h"
#include "MgmtUtils.h"
#include "Alarms.h"
#include "ts/Diags.h"
#include "P_RecCore.h"
const char *alarmText[] = {
"Unknown Alarm",
"[TrafficManager] Traffic Server process was reset.",
"[TrafficManager] Traffic Server process established.",
"New Peer",
"Peer Died",
"Invalid Configuration",
"System Error",
"Log Space Crisis",
"Cache Error",
"Cache Warning",
"Logging Error",
"Logging Warning",
"Mgmt Debugging Alarm",
"Configuration File Update Failed",
"Unable to Establish Manager User-Interface Services",
"Ping Failure",
"",
"Add OEM Alarm",
"",
"HTTP Origin Server is Congested",
"Congested HTTP Origin Server is now Alleviated",
"", /* congested server */
"" /* alleviated server */
};
const int alarmTextNum = sizeof(alarmText) / sizeof(char *);
// Return the alarm script directory. Use proxy.config.alarm.abs_path if it is
// set, falling back to proxy.config.bin_path otherwise.
static char *
alarm_script_dir()
{
char *path;
path = REC_readString("proxy.config.alarm.abs_path", nullptr);
if (path && *path) {
return path;
}
return ats_stringdup(RecConfigReadBinDir());
}
Alarms::Alarms()
{
cur_cb = 0;
cblist = ink_hash_table_create(InkHashTableKeyType_String);
local_alarms = ink_hash_table_create(InkHashTableKeyType_String);
remote_alarms = ink_hash_table_create(InkHashTableKeyType_String);
ink_mutex_init(&mutex);
alarmOEMcount = minOEMkey;
} /* End Alarms::Alarms */
Alarms::~Alarms()
{
ink_hash_table_destroy(cblist);
ink_hash_table_destroy_and_free_values(local_alarms);
ink_hash_table_destroy_and_free_values(remote_alarms);
ink_mutex_destroy(&mutex);
} /* End Alarms::Alarms */
void
Alarms::registerCallback(AlarmCallbackFunc func)
{
char cb_buf[80];
ink_mutex_acquire(&mutex);
snprintf(cb_buf, sizeof(cb_buf), "%d", cur_cb++);
Debug("alarm", "[Alarms::registerCallback] Registering Alarms callback");
ink_hash_table_insert(cblist, cb_buf, (void *)func);
ink_mutex_release(&mutex);
} /* End Alarms::registerCallback */
bool
Alarms::isCurrentAlarm(alarm_t a, char *ip)
{
bool ret = false;
char buf[80];
InkHashTableValue hash_value;
ink_mutex_acquire(&mutex);
if (!ip) {
snprintf(buf, sizeof(buf), "%d", a);
} else {
snprintf(buf, sizeof(buf), "%d-%s", a, ip);
}
if (!ip && ink_hash_table_lookup(local_alarms, buf, &hash_value) != 0) {
ret = true;
} else if (ip && ink_hash_table_lookup(remote_alarms, buf, &hash_value) != 0) {
ret = true;
}
ink_mutex_release(&mutex);
return ret;
} /* End Alarms::isCurrentAlarm */
void
Alarms::resolveAlarm(alarm_t a, char *ip)
{
char buf[80];
InkHashTableValue hash_value;
ink_mutex_acquire(&mutex);
if (!ip) {
snprintf(buf, sizeof(buf), "%d", a);
} else {
snprintf(buf, sizeof(buf), "%d-%s", a, ip);
}
if (!ip && ink_hash_table_lookup(local_alarms, buf, &hash_value) != 0) {
ink_hash_table_delete(local_alarms, buf);
ats_free(((Alarm *)hash_value)->description);
ats_free(hash_value);
} else if (ip && ink_hash_table_lookup(remote_alarms, buf, &hash_value) != 0) {
char buf2[1024];
snprintf(buf2, sizeof(buf2), "aresolv: %d\n", a);
ink_hash_table_delete(remote_alarms, buf);
ats_free(hash_value);
}
ink_mutex_release(&mutex);
return;
} /* End Alarms::resolveAlarm */
void
Alarms::signalAlarm(alarm_t a, const char *desc, const char *ip)
{
static time_t last_sent = 0;
static char prev_alarm_text[2048] = "";
int priority;
char buf[80];
Alarm *atmp;
InkHashTableValue hash_value;
InkHashTableEntry *entry;
InkHashTableIteratorState iterator_state;
/* Assign correct priorities */
switch (a) {
case MGMT_ALARM_PROXY_CACHE_ERROR:
priority = 1; // INKqa07595
break;
case MGMT_ALARM_PROXY_CACHE_WARNING:
return;
case MGMT_ALARM_PROXY_PEER_BORN:
priority = 3;
break;
case MGMT_ALARM_PROXY_PEER_DIED:
priority = 3;
break;
case MGMT_ALARM_PING_FAILURE:
priority = 1;
break;
case MGMT_ALARM_PROXY_PROCESS_DIED:
priority = 1;
break;
case MGMT_ALARM_PROXY_PROCESS_BORN:
mgmt_log("[Alarms::signalAlarm] Server Process born\n");
return;
case MGMT_ALARM_ADD_ALARM:
priority = 2;
break;
case MGMT_ALARM_PROXY_HTTP_CONGESTED_SERVER:
case MGMT_ALARM_PROXY_HTTP_ALLEVIATED_SERVER:
return;
default:
priority = 2;
break;
}
/* Quick hack to buffer repeat alarms and only send every 15 min */
if (desc && (priority == 1 || priority == 2) && !ip) {
if (strcmp(prev_alarm_text, desc) == 0) { /* a repeated alarm */
time_t time_delta = time(nullptr) - last_sent;
if (time_delta < 900) {
mgmt_log("[Alarms::signalAlarm] Skipping Alarm: '%s'\n", desc);
return;
} else {
last_sent = time(nullptr);
}
} else {
ink_strlcpy(prev_alarm_text, desc, sizeof(prev_alarm_text));
last_sent = time(nullptr);
}
}
Debug("alarm", "[Alarms::signalAlarm] Sending Alarm: '%s'", desc);
if (!desc) {
desc = (char *)getAlarmText(a);
}
/*
* Exec alarm bin for priority alarms everytime, regardless if they are
* potentially duplicates. However, only exec this for you own alarms,
* don't want every node in the cluster reporting the same alarm.
*/
if (priority == 1 && !ip) {
execAlarmBin(desc);
}
ink_mutex_acquire(&mutex);
if (!ip) {
// if an OEM alarm, then must create the unique key alarm type;
// this key is used to hash the new OEM alarm descritption in the hash table
if (a == MGMT_ALARM_ADD_ALARM) {
a = (alarmOEMcount - minOEMkey) % (maxOEMkey - minOEMkey) + minOEMkey;
alarmOEMcount++;
}
snprintf(buf, sizeof(buf), "%d", a);
if (ink_hash_table_lookup(local_alarms, buf, &hash_value) != 0) {
ink_mutex_release(&mutex);
return;
}
} else {
snprintf(buf, sizeof(buf), "%d-%s", a, ip);
if (ink_hash_table_lookup(remote_alarms, buf, &hash_value) != 0) {
// Reset the seen flag so that we know the remote alarm is
// still active
atmp = (Alarm *)hash_value;
atmp->seen = true;
ink_mutex_release(&mutex);
return;
}
}
atmp = (Alarm *)ats_malloc(sizeof(Alarm));
atmp->type = a;
atmp->linger = true;
atmp->seen = true;
atmp->priority = priority;
atmp->description = nullptr;
if (!ip) {
atmp->local = true;
atmp->inet_address = 0;
ink_hash_table_insert(local_alarms, (InkHashTableKey)(buf), (atmp));
} else {
atmp->local = false;
atmp->inet_address = inet_addr(ip);
ink_hash_table_insert(remote_alarms, (InkHashTableKey)(buf), (atmp));
}
// Swap desc with time-stamped description. Kinda hackish
// Temporary until we get a new
// alarm system in place. TS 5.0.0, 02/08/2001
time_t my_time_t;
char my_ctime_str[32];
time(&my_time_t);
ink_ctime_r(&my_time_t, my_ctime_str);
char *p = my_ctime_str;
while (*p != '\n' && *p != '\0') {
p++;
}
if (*p == '\n') {
*p = '\0';
}
const size_t sz = sizeof(char) * (strlen(desc) + strlen(my_ctime_str) + 4);
ats_free(atmp->description);
atmp->description = (char *)ats_malloc(sz);
snprintf(atmp->description, sz, "[%s] %s", my_ctime_str, desc);
ink_mutex_release(&mutex);
for (entry = ink_hash_table_iterator_first(cblist, &iterator_state); entry != nullptr;
entry = ink_hash_table_iterator_next(cblist, &iterator_state)) {
AlarmCallbackFunc func = (AlarmCallbackFunc)ink_hash_table_entry_value(remote_alarms, entry);
Debug("alarm", "[Alarms::signalAlarm] invoke callback for %d", a);
(*(func))(a, ip, desc);
}
/* Priority 2 alarms get signalled if they are the first unsolved occurence. */
if (priority == 2 && !ip) {
execAlarmBin(desc);
}
} /* End Alarms::signalAlarm */
/*
* resetSeenFlag(...)
* Function resets the "seen" flag for a given peer's alarms. This allows
* us to flush alarms that may have expired naturally or were dealt.
*/
void
Alarms::resetSeenFlag(char *ip)
{
InkHashTableEntry *entry;
InkHashTableIteratorState iterator_state;
ink_mutex_acquire(&mutex);
for (entry = ink_hash_table_iterator_first(remote_alarms, &iterator_state); entry != nullptr;
entry = ink_hash_table_iterator_next(remote_alarms, &iterator_state)) {
char *key = (char *)ink_hash_table_entry_key(remote_alarms, entry);
Alarm *tmp = (Alarm *)ink_hash_table_entry_value(remote_alarms, entry);
if (strstr(key, ip)) {
tmp->seen = false;
}
}
ink_mutex_release(&mutex);
return;
} /* End Alarms::resetSeenFlag */
/*
* clearUnSeen(...)
* This function is a sweeper functionto clean up those alarms that have
* been taken care of through otehr local managers or at the peer itself.
*/
void
Alarms::clearUnSeen(char *ip)
{
InkHashTableEntry *entry;
InkHashTableIteratorState iterator_state;
ink_mutex_acquire(&mutex);
for (entry = ink_hash_table_iterator_first(remote_alarms, &iterator_state); entry != nullptr;
entry = ink_hash_table_iterator_next(remote_alarms, &iterator_state)) {
char *key = (char *)ink_hash_table_entry_key(remote_alarms, entry);
Alarm *tmp = (Alarm *)ink_hash_table_entry_value(remote_alarms, entry);
if (strstr(key, ip)) { /* Make sure alarm is for correct ip */
if (!tmp->seen) { /* Make sure we did not see it in peer's report */
ink_hash_table_delete(remote_alarms, key); /* Safe in iterator? */
ats_free(tmp->description);
ats_free(tmp);
}
}
}
ink_mutex_release(&mutex);
return;
} /* End Alarms::clearUnSeen */
/*
* checkSystemNAlert(...)
* This function should test the system and signal local alarms. Sending
* out remote notification commands if necessary.
*/
void
Alarms::checkSystemNAlert()
{
return;
} /* End Alarms::checkSystenNAlert */
void
Alarms::execAlarmBin(const char *desc)
{
ats_scoped_str bindir(alarm_script_dir());
char cmd_line[MAXPATHLEN];
ats_scoped_str alarm_bin(REC_readString("proxy.config.alarm.bin", nullptr));
ats_scoped_str alarm_email_from_name;
ats_scoped_str alarm_email_from_addr;
ats_scoped_str alarm_email_to_addr;
pid_t pid;
// If there's no alarm script configured, don't even bother.
if (!alarm_bin || *alarm_bin == '\0') {
return;
}
// get email info
alarm_email_from_name = REC_readString("proxy.config.product_name", nullptr);
alarm_email_from_addr = REC_readString("proxy.config.admin.admin_user", nullptr);
alarm_email_to_addr = REC_readString("proxy.config.alarm_email", nullptr);
ink_filepath_make(cmd_line, sizeof(cmd_line), bindir, alarm_bin);
#ifdef POSIX_THREAD
if ((pid = fork()) < 0)
#else
if ((pid = fork1()) < 0)
#endif
{
mgmt_elog(errno, "[Alarms::execAlarmBin] Unable to fork1 process\n");
} else if (pid > 0) { /* Parent */
int status;
bool script_done = false;
time_t timeout = (time_t)REC_readInteger("proxy.config.alarm.script_runtime", nullptr);
if (!timeout) {
timeout = 5; // default time = 5 secs
}
time_t time_delta = 0;
time_t first_time = time(nullptr);
while (time_delta <= timeout) {
// waitpid will return child's pid if status is available
// or -1 if there is some problem; returns 0 if child status
// is not available
if (waitpid(pid, &status, WNOHANG) != 0) {
Debug("alarm", "[Alarms::execAlarmBin] child pid %" PRId64 " has status", (int64_t)pid);
script_done = true;
break;
}
time_delta = time(nullptr) - first_time;
}
// need to kill the child script process if it's not complete
if (!script_done) {
Debug("alarm", "[Alarms::execAlarmBin] kill child pid %" PRId64 "", (int64_t)pid);
kill(pid, SIGKILL);
waitpid(pid, &status, 0); // to reap the thread
}
} else {
int res;
if (alarm_email_from_name && alarm_email_from_addr && alarm_email_to_addr) {
res = execl(cmd_line, (const char *)alarm_bin, desc, (const char *)alarm_email_from_name, (const char *)alarm_email_from_addr,
(const char *)alarm_email_to_addr, (char *)nullptr);
} else {
res = execl(cmd_line, (const char *)alarm_bin, desc, (char *)nullptr);
}
_exit(res);
}
}
//
// getAlarmText
//
// returns the corresponding text for the alarm id
//
const char *
Alarms::getAlarmText(alarm_t id)
{
if (id < alarmTextNum) {
return alarmText[id];
} else {
return alarmText[0]; // "Unknown Alarm";
}
}