Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

action logs collector feed enhancements #633

Merged
merged 7 commits into from
Oct 6, 2023
Merged

action logs collector feed enhancements #633

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a8820a6
Report to collector a wrap-up status based on the exitcode
cvaroqui Oct 4, 2023
70ab5d0
Report the sid in action logs send to the collector
cvaroqui Oct 5, 2023
3262ae2
Fix report of daemon sid instead of the action sid
cvaroqui Oct 5, 2023
6d02a9f
Obfuscate --value value in wrap-up action log
cvaroqui Oct 5, 2023
9e1696b
Report the rid and subset in their own svcactions columns
cvaroqui Oct 5, 2023
79ef507
Fix the rid reported to the collector svcactions
cvaroqui Oct 5, 2023
3b1bfdb
Fix agent the log line coalescing
cvaroqui Oct 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
79 changes: 47 additions & 32 deletions opensvc/core/collector/rpc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,28 +311,36 @@ def reinit(self):
return True
return False

def begin_action(self, svcname, action, version, begin, cron):
def begin_action(self, svcname, action, version, begin, cron, sid, argv):
from subprocess import list2cmdline
args = [
['svcname',
'action',
'hostname',
'sid',
'version',
'begin',
'status_log',
'cron'],
[str(svcname),
str(action),
str(Env.nodename),
sid,
str(version),
str(begin),
list2cmdline(argv),
'1' if cron else '0']
]
args += [(self.node.collector_env.uuid, Env.nodename)]
self.proxy.begin_action(*args)

def end_action(self, path, action, begin, end, cron, alogfile):
err = 'ok'
res = None
res_err = None
def end_action(self, path, action, begin, end, cron, sid, alogfile, err):
if err == 0:
err = "ok"
else:
err = "err"
rid = None
rid_err = None
pid = None
msg = None
name, namespace, kind = split_path(path)
Expand All @@ -348,13 +356,16 @@ def end_action(self, path, action, begin, end, cron, alogfile):
2009-11-11 01:03:25,252;;DISK.VG;;INFO;;unxtstsvc01_data is already up;;10200;;EOL
"""
vars = ["svcname",
"action",
"action", # 1
"hostname",
"pid",
"sid",
"pid", # 4
"rid", # 5
"subset",
"begin",
"end",
"status_log",
"status",
"status_log", # 9
"status", # 10
"cron"]
vals = []
last = []
Expand All @@ -363,13 +374,24 @@ def end_action(self, path, action, begin, end, cron, alogfile):
continue
if ";;status_history;;" in line:
continue
date = line.split(";;")[0]
rid_err = "ok"
date, rid, lvl, msg, pid = line.split(";;")
rid = rid.lower()
rid = rid.replace(Env.nodename+"."+kind+"."+name, "")
rid = rid.replace(Env.nodename+"."+name, "")
rid = rid.replace(Env.nodename, "")
rid = rid.lstrip(".")
subset = ""

# container:front#nginx
# container#nginx
# task
if ":" in rid:
rgrp, rid = rid.split(":")
if "#" in rid:
subset, rname = rid.split("#")
rid = rgrp + "#" + rname

res_err = "ok"
date, res, lvl, msg, pid = line.split(";;")
res = res.lower().replace(Env.nodename+"."+kind+"."+name, "").replace(Env.nodename, "").lstrip(".")
res_action = res + " " + action
res_action = res_action.strip()
date = date.split(",")[0]

# database overflow protection
Expand All @@ -384,17 +406,14 @@ def end_action(self, path, action, begin, end, cron, alogfile):
if lvl is None or lvl == "DEBUG":
continue
elif lvl == "ERROR":
err = "err"
res_err = "err"
elif lvl == "WARNING" and err != "err":
err = "warn"
elif lvl == "WARNING" and res_err != "err":
res_err = "warn"
rid_err = "err"
elif lvl == "WARNING":
rid_err = "warn"

try:
if last:
if last[3] == pid and last[1] == res_action and last[7] == res_err:
last[6] += "\n"+msg
if last[1] == action and last[4] == pid and last[5] == rid and last[10] == rid_err:
last[9] += "\n"+msg
continue
else:
vals.append(last)
Expand All @@ -404,13 +423,16 @@ def end_action(self, path, action, begin, end, cron, alogfile):

last = [
path,
res_action,
action,
Env.nodename,
sid,
pid,
rid,
subset,
date,
"",
msg,
res_err,
rid_err,
"1" if cron else "0"
]

Expand All @@ -425,18 +447,12 @@ def end_action(self, path, action, begin, end, cron, alogfile):
"""Complete the wrap-up database entry
"""

""" If logfile is empty, default to current process pid
"""
if len(pids) == 0:
pids = {os.getpid()}

duration = datetime.strptime(end, "%Y-%m-%d %H:%M:%S") - \
datetime.strptime(begin, "%Y-%m-%d %H:%M:%S")
args = [
['svcname',
'action',
'hostname',
'pid',
'begin',
'end',
'time',
Expand All @@ -445,7 +461,6 @@ def end_action(self, path, action, begin, end, cron, alogfile):
[str(path),
str(action),
str(Env.nodename),
','.join(map(str, pids)),
begin,
end,
str(duration.seconds),
Expand Down
19 changes: 12 additions & 7 deletions opensvc/core/objects/svc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
from utilities.lazy import lazy, set_lazy, unset_all_lazy, unset_lazy
from utilities.naming import (fmt_path, resolve_path, svc_pathcf, svc_pathetc,
svc_pathlog, svc_pathtmp, svc_pathvar, new_id, factory, split_path)
from utilities.proc import (action_triggers, drop_option, find_editor,
from utilities.proc import (action_triggers, drop_option, has_option, find_editor,
init_locale, justcall, lcall, vcall)
from utilities.storage import Storage
from utilities.string import is_string
Expand Down Expand Up @@ -1318,15 +1318,15 @@ def rollback_handler(self, action):
except ex.Error:
self.log.error("rollback %s failed", action)

def dblogger(self, action, begin, end, actionlogfile):
def dblogger(self, action, begin, end, actionlogfile, err):
"""
Send to the collector the service status after an action, and
the action log.
"""
try:
self.node.daemon_collector_xmlrpc("end_action", self.path, action,
begin, end, self.options.cron,
actionlogfile)
begin, end, self.options.cron, Env.session_uuid,
actionlogfile, err)
except Exception as exc:
self.log.warning("failed to send logs to the collector: %s", exc)

Expand All @@ -1346,9 +1346,14 @@ def do_logged_action(self, action, options):

# Provision a database entry to store action log later
try:
argv = sys.argv[1:]
if has_option("--value", argv):
drop_option("--value", argv, drop_value=True)
argv.append("--value=xxx")
self.node.daemon_collector_xmlrpc("begin_action", self.path,
action, self.node.agent_version,
begin, self.options.cron)
begin, self.options.cron, Env.session_uuid,
argv)
except Exception as exc:
self.log.warning("failed to init logs on the collector: %s", exc)
self.log_action_header(action, options)
Expand All @@ -1373,7 +1378,7 @@ def do_logged_action(self, action, options):
actionlogfilehandler.close()
self.logger.removeHandler(actionlogfilehandler)
end = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
self.dblogger(action, begin, end, actionlogfile)
self.dblogger(action, begin, end, actionlogfile, err)
return err

def log_action_obfuscate_secret(self, options):
Expand All @@ -1387,7 +1392,7 @@ def log_action_obfuscate_secret(self, options):
if self.kind not in ("usr", "sec"):
return data
for k, v in data.items():
if k == "value":
if k == "value" and data[k]:
data["value"] = "xxx"
return data

Expand Down