Permalink
Cannot retrieve contributors at this time
from pymacaron.log import pymlogger | |
import json | |
import uuid | |
import os | |
import inspect | |
import sys | |
import traceback | |
from functools import wraps | |
from pprint import pformat | |
from flask import request, Response | |
from pymacaron_core.swagger.apipool import ApiPool | |
from pymacaron.config import get_config | |
from pymacaron.utils import timenow, is_ec2_instance | |
from pymacaron.exceptions import UnhandledServerError | |
log = pymlogger(__name__) | |
try: | |
from flask import _app_ctx_stack as stack | |
except ImportError: | |
from flask import _request_ctx_stack as stack | |
def function_name(f): | |
return "%s.%s" % (inspect.getmodule(f).__name__, f.__name__) | |
# | |
# Default error reporting | |
# | |
def default_error_reporter(title, message): | |
"""By default, error messages are just logged""" | |
log.error("error: %s" % title) | |
log.error("details:\n%s" % message) | |
error_reporter = default_error_reporter | |
def set_error_reporter(callback=None): | |
"""Here you can override the default crash reporter (and send yourself | |
emails, sms, slacks...)""" | |
global error_reporter | |
if error_reporter: | |
error_reporter = callback | |
def report_error(title=None, data={}, caught=None, is_fatal=False): | |
"""Format a crash report and send it somewhere relevant. There are two | |
types of crashes: fatal crashes (backend errors) or non-fatal ones (just | |
reporting a glitch, but the api call did not fail)""" | |
# Don't report errors if NO_ERROR_REPORTING set to 1 (set by run_acceptance_tests) | |
if os.environ.get('DO_REPORT_ERROR', None): | |
# Force error reporting | |
pass | |
elif os.environ.get('NO_ERROR_REPORTING', '') == '1': | |
log.info("NO_ERROR_REPORTING is set: not reporting error!") | |
return | |
elif 'is_ec2_instance' in data: | |
if not data['is_ec2_instance']: | |
# Not running on amazon: no reporting | |
log.info("DATA[is_ec2_instance] is False: not reporting error!") | |
return | |
elif not is_ec2_instance(): | |
log.info("Not running on an EC2 instance: not reporting error!") | |
return | |
# Fill error report with tons of usefull data | |
if 'user' not in data: | |
populate_error_report(data) | |
# Add the message | |
data['title'] = title | |
data['is_fatal_error'] = is_fatal | |
# Add the error caught, if any: | |
if caught: | |
data['error_caught'] = "%s" % caught | |
# Add a trace - Formatting traceback may raise a UnicodeDecodeError... | |
data['stack'] = [] | |
try: | |
data['stack'] = [l for l in traceback.format_stack()] | |
except Exception: | |
data['stack'] = 'Skipped trace - contained non-ascii chars' | |
# inspect may raise a UnicodeDecodeError... | |
fname = '' | |
try: | |
fname = inspect.stack()[1][3] | |
except Exception as e: | |
fname = 'unknown-method' | |
# Format the error's title | |
status, code = 'unknown_status', 'unknown_error_code' | |
app_name = get_config().name | |
if 'response' in data: | |
status = data['response'].get('status', status) | |
code = data['response'].get('error_code', code) | |
title_details = "%s %s %s" % (app_name, status, code) | |
else: | |
title_details = "%s %s()" % (app_name, fname) | |
if is_fatal: | |
title_details = 'FATAL ERROR %s' % title_details | |
else: | |
title_details = 'NON-FATAL ERROR %s' % title_details | |
if title: | |
title = "%s: %s" % (title_details, title) | |
else: | |
title = title_details | |
global error_reporter | |
log.info("Reporting crash...") | |
try: | |
error_reporter(title, json.dumps(data, sort_keys=True, indent=4)) | |
except Exception as e: | |
# Don't block on replying to api caller | |
log.error("Failed to send email report: %s" % str(e)) | |
def populate_error_report(data): | |
"""Add generic stats to the error report""" | |
# Did pymacaron_core set a call_id and call_path? | |
call_id, call_path = '', '' | |
if hasattr(stack.top, 'call_id'): | |
call_id = stack.top.call_id | |
if hasattr(stack.top, 'call_path'): | |
call_path = stack.top.call_path | |
# Unique ID associated to all responses associated to a given | |
# call to apis, across all micro-services | |
data['call_id'] = call_id | |
data['call_path'] = call_path | |
# Are we in aws? | |
data['is_ec2_instance'] = is_ec2_instance() | |
# If user is authenticated, get her id | |
user_data = { | |
'id': '', | |
'is_auth': 0, | |
'ip': '', | |
} | |
if stack.top: | |
# We are in a request context | |
user_data['ip'] = request.remote_addr | |
if 'X-Forwarded-For' in request.headers: | |
user_data['forwarded_ip'] = request.headers.get('X-Forwarded-For', '') | |
if 'User-Agent' in request.headers: | |
user_data['user_agent'] = request.headers.get('User-Agent', '') | |
if hasattr(stack.top, 'current_user'): | |
user_data['is_auth'] = 1 | |
user_data['id'] = stack.top.current_user.get('sub', '') | |
for k in ('name', 'email', 'is_expert', 'is_admin', 'is_support', 'is_tester', 'language'): | |
v = stack.top.current_user.get(k, None) | |
if v: | |
user_data[k] = v | |
data['user'] = user_data | |
# Is the current code running as a server? | |
if ApiPool().current_server_api: | |
# Server info | |
server = request.base_url | |
server = server.replace('http://', '') | |
server = server.replace('https://', '') | |
server = server.split('/')[0] | |
parts = server.split(':') | |
fqdn = parts[0] | |
port = parts[1] if len(parts) == 2 else '' | |
data['server'] = { | |
'fqdn': fqdn, | |
'port': port, | |
'api_name': ApiPool().current_server_name, | |
'api_version': ApiPool().current_server_api.get_version(), | |
} | |
# Endpoint data | |
data['endpoint'] = { | |
'id': "%s %s %s" % (ApiPool().current_server_name, request.method, request.path), | |
'url': request.url, | |
'base_url': request.base_url, | |
'path': request.path, | |
'method': request.method | |
} | |
def generate_crash_handler_decorator(error_decorator=None): | |
"""Return the crash_handler to pass to pymacaron_core, with optional error decoration""" | |
def crash_handler(f): | |
"""Return a decorator that reports failed api calls via the error_reporter, | |
for use on every server endpoint""" | |
@wraps(f) | |
def wrapper(*args, **kwargs): | |
"""Generate a report of this api call, and if the call failed or was too slow, | |
forward this report via the error_reporter""" | |
data = {} | |
t0 = timenow() | |
exception_string = '' | |
# Call endpoint and log execution time | |
try: | |
res = f(*args, **kwargs) | |
except Exception as e: | |
# An unhandled exception occured! | |
exception_string = str(e) | |
exc_type, exc_value, exc_traceback = sys.exc_info() | |
trace = traceback.format_exception(exc_type, exc_value, exc_traceback, 30) | |
data['trace'] = trace | |
# If it is a PyMacaronException, just call its http_reply() | |
if hasattr(e, 'http_reply'): | |
res = e.http_reply() | |
else: | |
# Otherwise, forge a Response | |
e = UnhandledServerError(exception_string) | |
log.error("UNHANDLED EXCEPTION: %s" % '\n'.join(trace)) | |
res = e.http_reply() | |
t1 = timenow() | |
# Is the response an Error instance? | |
response_type = type(res).__name__ | |
status_code = 200 | |
is_an_error = 0 | |
error = '' | |
error_description = '' | |
error_user_message = '' | |
error_id = '' | |
if isinstance(res, Response): | |
# Got a flask.Response object | |
res_data = None | |
status_code = str(res.status_code) | |
if str(status_code) == '200': | |
# It could be any valid json response, but it could also be an Error model | |
# that pymacaron_core handled as a status 200 because it does not know of | |
# pymacaron Errors | |
if res.content_type == 'application/json': | |
s = str(res.data) | |
if '"error":' in s and '"error_description":' in s and '"status":' in s: | |
# This looks like an error, let's decode it | |
res_data = res.get_data() | |
else: | |
# Assuming it is a PyMacaronException.http_reply() | |
res_data = res.get_data() | |
if res_data: | |
if type(res_data) is bytes: | |
res_data = res_data.decode("utf-8") | |
is_json = True | |
try: | |
j = json.loads(res_data) | |
except ValueError as e: | |
# This was a plain html response. Fake an error | |
is_json = False | |
j = {'error': res_data, 'status': status_code} | |
# Make sure that the response gets the same status as the PyMacaron Error it contained | |
status_code = j['status'] | |
res.status_code = int(status_code) | |
# Patch Response to contain a unique id | |
if is_json: | |
if 'error_id' not in j: | |
# If the error is forwarded by multiple micro-services, we | |
# want the error_id to be set only on the original error | |
error_id = str(uuid.uuid4()) | |
j['error_id'] = error_id | |
res.set_data(json.dumps(j)) | |
if error_decorator: | |
# Apply error_decorator, if any defined | |
res.set_data(json.dumps(error_decorator(j))) | |
# And extract data from this error | |
error = j.get('error', 'NO_ERROR_IN_JSON') | |
error_description = j.get('error_description', res_data) | |
if error_description == '': | |
error_description = res_data | |
if not exception_string: | |
exception_string = error_description | |
error_user_message = j.get('user_message', '') | |
is_an_error = 1 | |
request_args = [] | |
if len(args): | |
request_args.append(args) | |
if kwargs: | |
request_args.append(kwargs) | |
data.update({ | |
# Set only on the original error, not on forwarded ones, not on | |
# success responses | |
'error_id': error_id, | |
# Call results | |
'time': { | |
'start': t0.isoformat(), | |
'end': t1.isoformat(), | |
'microsecs': (t1.timestamp() - t0.timestamp()) * 1000000, | |
}, | |
# Response details | |
'response': { | |
'type': response_type, | |
'status': str(status_code), | |
'is_error': is_an_error, | |
'error_code': error, | |
'error_description': error_description, | |
'user_message': error_user_message, | |
}, | |
# Request details | |
'request': { | |
'params': pformat(request_args), | |
}, | |
}) | |
populate_error_report(data) | |
# inspect may raise a UnicodeDecodeError... | |
fname = function_name(f) | |
# | |
# Should we report this call? | |
# | |
# If it is an internal errors, report it | |
if data['response']['status'] and int(data['response']['status']) >= 500: | |
report_error( | |
title="%s(): %s" % (fname, exception_string), | |
data=data, | |
is_fatal=True | |
) | |
log.info("") | |
log.info(" <= Done!") | |
log.info("") | |
return res | |
return wrapper | |
return crash_handler | |
# | |
# Generic crash-handler as a decorator | |
# | |
def crash_handler(f): | |
"""Decorate method with pymacaron's generic crash handler""" | |
return generate_crash_handler_decorator(None)(f) |