From c796318b6e7062c58098152c90d651b0967d6968 Mon Sep 17 00:00:00 2001 From: Steve Keay Date: Wed, 12 Feb 2025 10:40:34 +0000 Subject: [PATCH 1/5] When logging API error responses, parse any JSON payload Don't log the raw JSON payload, parse it and print it in python format. If it can't be parsed, dump the string. --- .../neutron_understack/nautobot.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/python/neutron-understack/neutron_understack/nautobot.py b/python/neutron-understack/neutron_understack/nautobot.py index 3ad956923..11e9a54e6 100644 --- a/python/neutron-understack/neutron_understack/nautobot.py +++ b/python/neutron-understack/neutron_understack/nautobot.py @@ -56,16 +56,18 @@ def make_api_request( except Exception as e: raise NautobotOSError(err=e) from e + if response.content: + try: + response_data = response.json() + except requests.exceptions.JSONDecodeError: + response_data = {"body": response.content} + else: + response_data = {"status_code": response.status_code} + if response.status_code >= 300: raise NautobotRequestError( - code=response.status_code, url=full_url, body=response.content + code=response.status_code, url=full_url, body=response_data ) - if not response.content: - response_data = {"status_code": response.status_code} - try: - response_data = response.json() - except requests.exceptions.JSONDecodeError: - response_data = {"body": response.content} caller_function = inspect.stack()[1].function LOG.debug( From 38c0caaef7963613408ccdfeae25a705135e19b6 Mon Sep 17 00:00:00 2001 From: Steve Keay Date: Wed, 12 Feb 2025 10:54:53 +0000 Subject: [PATCH 2/5] Explicitly log the empty body of API response with zero content-length --- python/neutron-understack/neutron_understack/nautobot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/neutron-understack/neutron_understack/nautobot.py b/python/neutron-understack/neutron_understack/nautobot.py index 11e9a54e6..e6161cb46 100644 --- a/python/neutron-understack/neutron_understack/nautobot.py +++ b/python/neutron-understack/neutron_understack/nautobot.py @@ -62,7 +62,7 @@ def make_api_request( except requests.exceptions.JSONDecodeError: response_data = {"body": response.content} else: - response_data = {"status_code": response.status_code} + response_data = {"status_code": response.status_code, "body": ""} if response.status_code >= 300: raise NautobotRequestError( From 41a92ee9c2ba15d1b5027c425ebb40ff927a360a Mon Sep 17 00:00:00 2001 From: Steve Keay Date: Wed, 12 Feb 2025 10:55:05 +0000 Subject: [PATCH 3/5] Only log the error part of an API response containing an "error" key If the body consists of a json payload with a top-level key "error" then we want to log only the contents of that key, otherwise log the whole response. --- python/neutron-understack/neutron_understack/nautobot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/neutron-understack/neutron_understack/nautobot.py b/python/neutron-understack/neutron_understack/nautobot.py index e6161cb46..114bb1dc1 100644 --- a/python/neutron-understack/neutron_understack/nautobot.py +++ b/python/neutron-understack/neutron_understack/nautobot.py @@ -65,6 +65,8 @@ def make_api_request( response_data = {"status_code": response.status_code, "body": ""} if response.status_code >= 300: + response_data = response_data.get("error", response_data) + raise NautobotRequestError( code=response.status_code, url=full_url, body=response_data ) From 68859621ca09d444d16a013ae1eb0fe4feffe637 Mon Sep 17 00:00:00 2001 From: Steve Keay Date: Wed, 12 Feb 2025 11:05:59 +0000 Subject: [PATCH 4/5] When logging API error responses, include request method and payload This could potentially leak sensitive information into the logs, however we don't use the Nautobot API client to post any sensitive data and API authentication is done via HTTP headers that are not logged. --- python/neutron-understack/neutron_understack/nautobot.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/neutron-understack/neutron_understack/nautobot.py b/python/neutron-understack/neutron_understack/nautobot.py index 114bb1dc1..555f0f190 100644 --- a/python/neutron-understack/neutron_understack/nautobot.py +++ b/python/neutron-understack/neutron_understack/nautobot.py @@ -11,7 +11,7 @@ class NautobotRequestError(exc.NeutronException): - message = "Nautobot API returned error %(code)s for %(url)s: %(body)s" + message = "Nautobot API ERROR %(code)s for %(url)s %(method)s %(payload)s: %(body)s" class NautobotOSError(exc.NeutronException): @@ -68,7 +68,11 @@ def make_api_request( response_data = response_data.get("error", response_data) raise NautobotRequestError( - code=response.status_code, url=full_url, body=response_data + code=response.status_code, + url=full_url, + method=method, + payload=payload, + body=response_data, ) caller_function = inspect.stack()[1].function From 23eff586178fb87cad9bd1ffab5f4987a0911f3b Mon Sep 17 00:00:00 2001 From: Steve Keay Date: Wed, 12 Feb 2025 12:32:12 +0000 Subject: [PATCH 5/5] When logging raw payload of large API errors, truncate the payload This is so we don't get pages of HTML in our container logs --- .../neutron-understack/neutron_understack/nautobot.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/python/neutron-understack/neutron_understack/nautobot.py b/python/neutron-understack/neutron_understack/nautobot.py index 555f0f190..a074489ba 100644 --- a/python/neutron-understack/neutron_understack/nautobot.py +++ b/python/neutron-understack/neutron_understack/nautobot.py @@ -26,6 +26,14 @@ class NautobotCustomFieldNotFoundError(exc.NeutronException): message = "Custom field with name %(cf_name)s not found for %(obj)s" +def _truncated(message: str | bytes, maxlen=200) -> str: + input = str(message) + if len(input) <= maxlen: + return input + + return f"{input[:maxlen]}...{len(input) - maxlen} more chars" + + class Nautobot: """Basic Nautobot wrapper because pynautobot doesn't expose plugin APIs.""" @@ -60,7 +68,8 @@ def make_api_request( try: response_data = response.json() except requests.exceptions.JSONDecodeError: - response_data = {"body": response.content} + response_data = {"body": _truncated(response.content)} + else: response_data = {"status_code": response.status_code, "body": ""}