From 485a67230b2ea3bb9714c196ec5b6b0085aa18ae Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Mon, 3 Oct 2022 17:21:08 +0200 Subject: [PATCH 1/4] Add perflog_ignore attribute in httpjson handler --- reframe/core/logging.py | 19 +++++++++++++++---- reframe/frontend/cli.py | 2 +- reframe/schemas/config.json | 7 ++++++- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/reframe/core/logging.py b/reframe/core/logging.py index 4dc5519351..80936ca003 100644 --- a/reframe/core/logging.py +++ b/reframe/core/logging.py @@ -371,7 +371,8 @@ def _create_httpjson_handler(site_config, config_prefix): return None extras = site_config.get(f'{config_prefix}/extras') - return HTTPJSONHandler(url, extras) + perflog_ignore = site_config.get(f'{config_prefix}/perflog_ignore') + return HTTPJSONHandler(url, extras, perflog_ignore) class HTTPJSONHandler(logging.Handler): @@ -386,20 +387,30 @@ class HTTPJSONHandler(logging.Handler): 'stack_info', 'thread', 'threadName', 'exc_text' } - def __init__(self, url, extras=None): + def __init__(self, url, extras=None, perflog_ignore=None): super().__init__() self._url = url self._extras = extras + self._perflog_ignore = perflog_ignore def _record_to_json(self, record): + def _log_record(key): + if key.startswith('_') or key in HTTPJSONHandler.LOG_ATTRS: + return False + + if self._perflog_ignore and key in self._perflog_ignore: + return False + + return True + json_record = { k: v for k, v in record.__dict__.items() - if not (k.startswith('_') or k in HTTPJSONHandler.LOG_ATTRS) + if _log_record(k) } if self._extras: json_record.update({ k: v for k, v in self._extras.items() - if not (k.startswith('_') or k in HTTPJSONHandler.LOG_ATTRS) + if _log_record(k) }) return _xfmt(json_record).encode('utf-8') diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 7514f3bf78..be453c8b7b 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -600,7 +600,7 @@ def main(): envvar='RFM_IGNORE_REQNODENOTAVAIL', configvar='schedulers/ignore_reqnodenotavail', action='store_true', - help='Graylog server address' + help='Ignore ReqNodeNotAvail Slurm error' ) argparser.add_argument( dest='dump_pipeline_progress', diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index b8e1561f9e..00162b35c3 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -146,7 +146,11 @@ { "properties": { "url": {"type": "string"}, - "extras": {"type": "object"} + "extras": {"type": "object"}, + "perflog_ignore": { + "type": "array", + "items": {"type": "string"} + } }, "required": ["url"] } @@ -551,6 +555,7 @@ "logging/handlers*/filelog_basedir": "./perflogs", "logging/handlers*/graylog_extras": {}, "logging/handlers*/httpjson_extras": {}, + "logging/handlers*/httpjson_perflog_ignore": [], "logging/handlers*/stream_name": "stdout", "logging/handlers*/syslog_socktype": "udp", "logging/handlers*/syslog_facility": "user", From 66d5778fb53300bc82f4e11024864603b719acd9 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Mon, 3 Oct 2022 17:38:53 +0200 Subject: [PATCH 2/4] Add documentation for perflog_ignore --- docs/config_reference.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 356a97cec9..f128daf82e 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -1179,6 +1179,15 @@ The additional properties for the ``httpjson`` handler are the following: A set of optional key/value pairs to be passed with each log record to the server. These may depend on the server configuration. +.. js:attribute:: .logging[].handlers[].perflog_ignore + +.. object:: .logging[].handlers_perflog[].perflog_ignore + + :required: No + :default: ``[]`` + + These keys will be excluded from the log record that will be sent to the server. + The ``httpjson`` handler sends log messages in JSON format using an HTTP POST request to the specified URL. @@ -1193,7 +1202,8 @@ An example configuration of this handler for performance logging is shown here: 'extras': { 'facility': 'reframe', 'data-version': '1.0' - } + }, + 'perflog_ignore': ['check_perfvalues'] } From c2902f8a409f77a2c6dd8f0faf7fa058d7a72271 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Tue, 4 Oct 2022 08:44:13 +0200 Subject: [PATCH 3/4] Small fixes --- reframe/core/logging.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/reframe/core/logging.py b/reframe/core/logging.py index 80936ca003..e0946ee49f 100644 --- a/reframe/core/logging.py +++ b/reframe/core/logging.py @@ -394,23 +394,18 @@ def __init__(self, url, extras=None, perflog_ignore=None): self._perflog_ignore = perflog_ignore def _record_to_json(self, record): - def _log_record(key): - if key.startswith('_') or key in HTTPJSONHandler.LOG_ATTRS: - return False - - if self._perflog_ignore and key in self._perflog_ignore: - return False - - return True + def _can_send(key): + return not ( + key.startswith('_') or key in HTTPJSONHandler.LOG_ATTRS + or (self._perflog_ignore and key in self._perflog_ignore) + ) json_record = { - k: v for k, v in record.__dict__.items() - if _log_record(k) + k: v for k, v in record.__dict__.items() if _can_send(k) } if self._extras: json_record.update({ - k: v for k, v in self._extras.items() - if _log_record(k) + k: v for k, v in self._extras.items() if _can_send(k) }) return _xfmt(json_record).encode('utf-8') From f3e413c2f8e9e298b50c79a58a66e915a3e83569 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Tue, 4 Oct 2022 08:48:12 +0200 Subject: [PATCH 4/4] Rename perflog_ignore to ignore_keys --- docs/config_reference.rst | 6 +++--- reframe/core/logging.py | 10 +++++----- reframe/schemas/config.json | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index f128daf82e..b643399b76 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -1179,9 +1179,9 @@ The additional properties for the ``httpjson`` handler are the following: A set of optional key/value pairs to be passed with each log record to the server. These may depend on the server configuration. -.. js:attribute:: .logging[].handlers[].perflog_ignore +.. js:attribute:: .logging[].handlers[].ignore_keys -.. object:: .logging[].handlers_perflog[].perflog_ignore +.. object:: .logging[].handlers_perflog[].ignore_keys :required: No :default: ``[]`` @@ -1203,7 +1203,7 @@ An example configuration of this handler for performance logging is shown here: 'facility': 'reframe', 'data-version': '1.0' }, - 'perflog_ignore': ['check_perfvalues'] + 'ignore_keys': ['check_perfvalues'] } diff --git a/reframe/core/logging.py b/reframe/core/logging.py index e0946ee49f..10de121bd1 100644 --- a/reframe/core/logging.py +++ b/reframe/core/logging.py @@ -371,8 +371,8 @@ def _create_httpjson_handler(site_config, config_prefix): return None extras = site_config.get(f'{config_prefix}/extras') - perflog_ignore = site_config.get(f'{config_prefix}/perflog_ignore') - return HTTPJSONHandler(url, extras, perflog_ignore) + ignore_keys = site_config.get(f'{config_prefix}/ignore_keys') + return HTTPJSONHandler(url, extras, ignore_keys) class HTTPJSONHandler(logging.Handler): @@ -387,17 +387,17 @@ class HTTPJSONHandler(logging.Handler): 'stack_info', 'thread', 'threadName', 'exc_text' } - def __init__(self, url, extras=None, perflog_ignore=None): + def __init__(self, url, extras=None, ignore_keys=None): super().__init__() self._url = url self._extras = extras - self._perflog_ignore = perflog_ignore + self._ignore_keys = ignore_keys def _record_to_json(self, record): def _can_send(key): return not ( key.startswith('_') or key in HTTPJSONHandler.LOG_ATTRS - or (self._perflog_ignore and key in self._perflog_ignore) + or (self._ignore_keys and key in self._ignore_keys) ) json_record = { diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 00162b35c3..ae0abd98c3 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -147,7 +147,7 @@ "properties": { "url": {"type": "string"}, "extras": {"type": "object"}, - "perflog_ignore": { + "ignore_keys": { "type": "array", "items": {"type": "string"} } @@ -555,7 +555,7 @@ "logging/handlers*/filelog_basedir": "./perflogs", "logging/handlers*/graylog_extras": {}, "logging/handlers*/httpjson_extras": {}, - "logging/handlers*/httpjson_perflog_ignore": [], + "logging/handlers*/httpjson_ignore_keys": [], "logging/handlers*/stream_name": "stdout", "logging/handlers*/syslog_socktype": "udp", "logging/handlers*/syslog_facility": "user",