From 9bc710cfc7f2594df47d89c5c3086116d011bd05 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Mon, 29 Sep 2014 21:10:34 -0500 Subject: [PATCH 01/43] Initial commit and project setup --- .gitignore | 29 +++++++++++++++++++++++++++++ LICENSE.txt | 1 + setup.py | 23 +++++++++++++++++++++++ stackify/__init__.py | 1 + 4 files changed, 54 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE.txt create mode 100644 setup.py create mode 100644 stackify/__init__.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f6c9235 --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +*.py[co] + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +var +sdist +develop-eggs +.installed.cfg + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox + +#Translations +*.mo + +#Mr Developer +.mr.developer.cfg + +# virutalenvs +.venv diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..1333ed7 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1 @@ +TODO diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..671b363 --- /dev/null +++ b/setup.py @@ -0,0 +1,23 @@ +from distutils.core import setup +import re +import ast + +version_re = re.compile(r'__version__\s+=\s+(.*)') + +with open('stackify/__init__.py') as f: + f = f.read() + version = ast.literal_eval(version_re.search(f).group(1)) + +setup( + name='stackify', + version=version, + author='Matthew Thompson', + author_email='chameleonator@gmail.com', + packages=['stackify'], + url='https://github.com/stackify/stackify-python', + license=open('LICENSE.txt').readline(), + description='Stackify API for Python', + long_description=open('README.md').read(), + install_requires=[] +) + diff --git a/stackify/__init__.py b/stackify/__init__.py new file mode 100644 index 0000000..b8023d8 --- /dev/null +++ b/stackify/__init__.py @@ -0,0 +1 @@ +__version__ = '0.0.1' From 3f974b0d21088251123600c7430f1b85269120e3 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Mon, 29 Sep 2014 22:27:25 -0500 Subject: [PATCH 02/43] Some organization and app identity --- .gitignore | 3 +++ setup.py | 6 ++++-- stackify/__init__.py | 18 +++++++++++++++++ stackify/application.py | 21 +++++++++++++++++++ stackify/http.py | 45 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 stackify/application.py create mode 100644 stackify/http.py diff --git a/.gitignore b/.gitignore index f6c9235..1ae3595 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ pip-log.txt # virutalenvs .venv + +# debug stuff +test.py diff --git a/setup.py b/setup.py index 671b363..a152d21 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -from distutils.core import setup +from setuptools import setup import re import ast @@ -18,6 +18,8 @@ license=open('LICENSE.txt').readline(), description='Stackify API for Python', long_description=open('README.md').read(), - install_requires=[] + install_requires=[ + 'retrying>=1.2.3' + ] ) diff --git a/stackify/__init__.py b/stackify/__init__.py index b8023d8..e8ab7ae 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -1 +1,19 @@ +""" +Stackify Python API +""" + __version__ = '0.0.1' + + +API_URL = 'https://api.stackify.com' + +READ_TIMEOUT = 5000 + + +import logging + +logging.basicConfig() + +from stackify.application import ApiConfiguration +from stackify.http import HTTPClient + diff --git a/stackify/application.py b/stackify/application.py new file mode 100644 index 0000000..fbfa954 --- /dev/null +++ b/stackify/application.py @@ -0,0 +1,21 @@ +import socket +import os + +from stackify import API_URL + + +class EnvironmentDetail: + def __init__(self, api_config): + self.deviceName = socket.gethostname() + self.appLocation = os.getcwd() + self.configuredAppName = api_config.application + self.configuredEnvironmentName = api_config.environment + + +class ApiConfiguration: + def __init__(self, api_key, application, environment, api_url=API_URL): + self.api_key = api_key + self.api_url = api_url + self.application = application + self.environment = environment + diff --git a/stackify/http.py b/stackify/http.py new file mode 100644 index 0000000..e06aa97 --- /dev/null +++ b/stackify/http.py @@ -0,0 +1,45 @@ +import json +import urllib2 +import retrying + +from stackify.application import EnvironmentDetail +from stackify import READ_TIMEOUT + + +class HTTPClient: + def __init__(self, api_config): + self.api_config = api_config + self.environment_detail = EnvironmentDetail(api_config) + self.identify_application() + + + def POST(self, url, payload): + request_url = self.api_config.api_url + url + request = urllib2.Request(request_url) + + request.add_header('Content-Type', 'application/json') + request.add_header('X-Stackify-Key', self.api_config.api_key) + request.add_header('X-Stackify-PV', 'V1') + + try: + response = urllib2.urlopen(request, json.dumps(payload), timeout=READ_TIMEOUT) + body = response.read() + return json.loads(body) + except urllib2.HTTPError as e: + print 'HTTP error', e.code + raise + except urllib2.URLError as e: + print 'URL error', e.reason + raise + except ValueError as e: + # could not read json response + raise + + @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=10000) + def identify_application(self): + result = self.POST('/Metrics/IdentifyApp', self.environment_detail.__dict__) + self.app_name_id = result.get('AppNameID') + self.app_env_id = result.get('AppEnvID') + self.device_id = result.get('DeviceID') + self.device_app_id = result.get('DeviceAppID') + From 28f15cf53bc75007423df08b8989f10ad8572aee Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Mon, 29 Sep 2014 22:45:38 -0500 Subject: [PATCH 03/43] Use internal loggers --- stackify/__init__.py | 2 ++ stackify/http.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index e8ab7ae..53735ea 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -14,6 +14,8 @@ logging.basicConfig() +internal_log = logging.getLogger(__name__) + from stackify.application import ApiConfiguration from stackify.http import HTTPClient diff --git a/stackify/http.py b/stackify/http.py index e06aa97..2aba2cb 100644 --- a/stackify/http.py +++ b/stackify/http.py @@ -1,11 +1,15 @@ import json import urllib2 import retrying +import logging from stackify.application import EnvironmentDetail from stackify import READ_TIMEOUT +internal_log = logging.getLogger(__name__) + + class HTTPClient: def __init__(self, api_config): self.api_config = api_config @@ -16,6 +20,7 @@ def __init__(self, api_config): def POST(self, url, payload): request_url = self.api_config.api_url + url request = urllib2.Request(request_url) + internal_log.debug('Request URL: {0}'.format(request_url)) request.add_header('Content-Type', 'application/json') request.add_header('X-Stackify-Key', self.api_config.api_key) @@ -24,15 +29,18 @@ def POST(self, url, payload): try: response = urllib2.urlopen(request, json.dumps(payload), timeout=READ_TIMEOUT) body = response.read() + internal_log.debug('Response: {0}'.format(body)) + json.loads('junk') return json.loads(body) except urllib2.HTTPError as e: - print 'HTTP error', e.code + internal_log.exception('HTTP response: {0}'.format(e.code)) raise except urllib2.URLError as e: - print 'URL error', e.reason + internal_log.exception('URL exception: {0}'.format(e.reason)) raise except ValueError as e: # could not read json response + internal_log.exception('Cannot decode JSON response') raise @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=10000) From 13736f13fa50d535416fc7618bb1b7d484e232e2 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 4 Oct 2014 16:04:40 -0500 Subject: [PATCH 04/43] Refactoring some things, using requests --- setup.py | 3 ++- stackify/application.py | 8 +++++- stackify/error.py | 49 ++++++++++++++++++++++++++++++++++ stackify/http.py | 59 ++++++++++++++++++++++++----------------- stackify/log.py | 42 +++++++++++++++++++++++++++++ 5 files changed, 134 insertions(+), 27 deletions(-) create mode 100644 stackify/error.py create mode 100644 stackify/log.py diff --git a/setup.py b/setup.py index a152d21..a928872 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,8 @@ description='Stackify API for Python', long_description=open('README.md').read(), install_requires=[ - 'retrying>=1.2.3' + 'retrying>=1.2.3', + 'requests>=2.4.1' ] ) diff --git a/stackify/application.py b/stackify/application.py index fbfa954..0fe524f 100644 --- a/stackify/application.py +++ b/stackify/application.py @@ -1,10 +1,16 @@ import socket import os +import json from stackify import API_URL -class EnvironmentDetail: +class JSONObject(object): + def toJSON(self): + return json.dumps(self, default=lambda x: x.__dict__) + + +class EnvironmentDetail(JSONObject): def __init__(self, api_config): self.deviceName = socket.gethostname() self.appLocation = os.getcwd() diff --git a/stackify/error.py b/stackify/error.py new file mode 100644 index 0000000..f2e9006 --- /dev/null +++ b/stackify/error.py @@ -0,0 +1,49 @@ +from stackify.application import JSONObject + + +class ErrorItem(JSONObject): + def __init__(self): + self.Message = None # exception message + self.ErrorType = None # exception class name + self.ErrorTypeCode = None # ? + self.Data = None # custom data + self.SourceMethod = None + self.StackTrace = [] # array of TraceFrames + self.InnerError = None # cause? + + +class TraceFrame(JSONObject): + def __init__(self): + self.CodeFileName = None + self.LineNum = None + self.Method = None + + +class WebRequestDetail(JSONObject): + def __init__(self): + self.UserIPAddress = None + self.HttpMethod = None + self.RequestProtocol = None + self.RequestUrl = None + self.RequestUrlRoot = None + self.ReferralUrl = None + self.Headers = {} + self.Cookies = {} + self.QueryString = {} + self.PostData = {} + self.SessionData = {} + self.PostDataRaw = None + self.MVCAction = None + self.MVCController = None + self.MVCArea = None + + +class StackifyError(JSONObject): + def __init__(self): + self.EnvironmentDetail = None # environment detail object + self.OccurredEpochMillis = None + self.Error = None # ErrorItem object + self.WebRequestDetail = None # WebRequestDetail object + self.CustomerName = None + self.UserName = None + diff --git a/stackify/http.py b/stackify/http.py index 2aba2cb..8282a43 100644 --- a/stackify/http.py +++ b/stackify/http.py @@ -1,53 +1,62 @@ -import json -import urllib2 +import requests import retrying import logging +import zlib from stackify.application import EnvironmentDetail from stackify import READ_TIMEOUT -internal_log = logging.getLogger(__name__) - - class HTTPClient: def __init__(self, api_config): self.api_config = api_config self.environment_detail = EnvironmentDetail(api_config) - self.identify_application() - - - def POST(self, url, payload): + self.app_name_id = None + self.app_env_id = None + self.device_id = None + self.device_app_id = None + self.device_alias = None + self.identified = False + + def POST(self, url, json_object, gzip=False): request_url = self.api_config.api_url + url - request = urllib2.Request(request_url) + internal_log = logging.getLogger(__name__) internal_log.debug('Request URL: {0}'.format(request_url)) - request.add_header('Content-Type', 'application/json') - request.add_header('X-Stackify-Key', self.api_config.api_key) - request.add_header('X-Stackify-PV', 'V1') + headers = { + 'Content-Type': 'application/json', + 'X-Stackify-Key': self.api_config.api_key, + 'X-Stackify-PV': 'V1', + } + try: - response = urllib2.urlopen(request, json.dumps(payload), timeout=READ_TIMEOUT) - body = response.read() - internal_log.debug('Response: {0}'.format(body)) - json.loads('junk') - return json.loads(body) - except urllib2.HTTPError as e: - internal_log.exception('HTTP response: {0}'.format(e.code)) - raise - except urllib2.URLError as e: - internal_log.exception('URL exception: {0}'.format(e.reason)) + payload_data = json_object.toJSON() + + if gzip: + headers['Content-Encoding'] = 'gzip' + payload_data = zlib.compress(payload_data) + + response = requests.post(request_url, + data=payload_data, headers=headers, + timeout=READ_TIMEOUT) + internal_log.debug('Response: {0}'.format(response.text)) + return response.json() + except requests.exceptions.RequestException: + interal_log.exception('HTTP exception:') raise except ValueError as e: # could not read json response internal_log.exception('Cannot decode JSON response') raise - @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=10000) + #@retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=10000) def identify_application(self): - result = self.POST('/Metrics/IdentifyApp', self.environment_detail.__dict__) + result = self.POST('/Metrics/IdentifyApp', self.environment_detail) self.app_name_id = result.get('AppNameID') self.app_env_id = result.get('AppEnvID') self.device_id = result.get('DeviceID') self.device_app_id = result.get('DeviceAppID') + self.device_alias = result.get('DeviceAlias') + self.identified = True diff --git a/stackify/log.py b/stackify/log.py new file mode 100644 index 0000000..60493a8 --- /dev/null +++ b/stackify/log.py @@ -0,0 +1,42 @@ +import time +import threading +import logging + +from stackify.application import JSONObject + + +MAX_BATCH = 100 + + +LOGGING_LEVELS = { + logging.CRITICAL: 'CRITICAL', + logging.ERROR: 'ERROR', + logging.WARNING: 'WARNING', + logging.INFO: 'INFO', + logging.DEBUG: 'DEBUG', + logging.NOTSET: 'NOTSET' +} + + +class LogMsg(JSONObject): + def __init__(self): + self.Msg = None + self.data = None + self.Ex = None + self.Th = threading.current_thread().ident + self.EpochMs = int(time.time() * 1000) + self.Level = None + self.TransID = None + # filename, line_number, function = internal_log.findCaller() + self.SrcMethod = None + self.SrcLine = None + + +class LogMsgGroup(JSONObject): + def __init__(self, msgs, logger=None): + self.Logger = logger or __name__ + self.Msgs = msgs + self.CDID = None + self.CDAppID = None + self.AppNameID = None + self.ServerName = None From edcb5324cd92dae4c2b5b2338805b2fa4f339091 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 4 Oct 2014 17:03:56 -0500 Subject: [PATCH 05/43] Log request payload --- stackify/http.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stackify/http.py b/stackify/http.py index 8282a43..23c75bc 100644 --- a/stackify/http.py +++ b/stackify/http.py @@ -32,6 +32,7 @@ def POST(self, url, json_object, gzip=False): try: payload_data = json_object.toJSON() + internal_log.debug('POST data: {0}'.format(payload_data)) if gzip: headers['Content-Encoding'] = 'gzip' From 1a29e7a6e6309ba80fe793a21ee235c46eee0669 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 4 Oct 2014 17:04:41 -0500 Subject: [PATCH 06/43] Wrap extra data as a string --- stackify/log.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/stackify/log.py b/stackify/log.py index 60493a8..66374c4 100644 --- a/stackify/log.py +++ b/stackify/log.py @@ -1,6 +1,7 @@ import time import threading import logging +import json from stackify.application import JSONObject @@ -19,10 +20,10 @@ class LogMsg(JSONObject): - def __init__(self): - self.Msg = None - self.data = None - self.Ex = None + def __init__(self, message, data=None): + self.Msg = message + self.data = json.dumps(data, default=lambda x: x.__dict__) if data else None + self.Ex = None # a StackifyError object self.Th = threading.current_thread().ident self.EpochMs = int(time.time() * 1000) self.Level = None From 1bae9518e28cf67428c49f6c26b3cf961f166226 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 4 Oct 2014 17:18:51 -0500 Subject: [PATCH 07/43] Fix gzip compression --- stackify/http.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/stackify/http.py b/stackify/http.py index 23c75bc..d795be7 100644 --- a/stackify/http.py +++ b/stackify/http.py @@ -1,7 +1,12 @@ import requests import retrying import logging -import zlib +import gzip + +try: + from cStringIO import StringIO +except: + from StringIO import StringIO from stackify.application import EnvironmentDetail from stackify import READ_TIMEOUT @@ -18,7 +23,7 @@ def __init__(self, api_config): self.device_alias = None self.identified = False - def POST(self, url, json_object, gzip=False): + def POST(self, url, json_object, use_gzip=False): request_url = self.api_config.api_url + url internal_log = logging.getLogger(__name__) internal_log.debug('Request URL: {0}'.format(request_url)) @@ -34,9 +39,14 @@ def POST(self, url, json_object, gzip=False): payload_data = json_object.toJSON() internal_log.debug('POST data: {0}'.format(payload_data)) - if gzip: + if use_gzip: headers['Content-Encoding'] = 'gzip' - payload_data = zlib.compress(payload_data) + # compress payload with gzip + s = StringIO() + g = gzip.GzipFile(fileobj=s, mode='w') + g.write(payload_data) + g.close() + payload_data = s.getvalue() response = requests.post(request_url, data=payload_data, headers=headers, From d6475a4b5d009c7f134e79a7bcac04b082cf38b6 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 4 Oct 2014 17:27:50 -0500 Subject: [PATCH 08/43] Some refactoring for the base json class --- stackify/__init__.py | 12 ++++++++++++ stackify/application.py | 7 +------ stackify/error.py | 2 +- stackify/formats.py | 11 +++++++++++ stackify/log.py | 15 ++------------- 5 files changed, 27 insertions(+), 20 deletions(-) create mode 100644 stackify/formats.py diff --git a/stackify/__init__.py b/stackify/__init__.py index 53735ea..376406b 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -9,13 +9,25 @@ READ_TIMEOUT = 5000 +MAX_BATCH = 100 + import logging +LOGGING_LEVELS = { + logging.CRITICAL: 'CRITICAL', + logging.ERROR: 'ERROR', + logging.WARNING: 'WARNING', + logging.INFO: 'INFO', + logging.DEBUG: 'DEBUG', + logging.NOTSET: 'NOTSET' +} + logging.basicConfig() internal_log = logging.getLogger(__name__) + from stackify.application import ApiConfiguration from stackify.http import HTTPClient diff --git a/stackify/application.py b/stackify/application.py index 0fe524f..d1582d7 100644 --- a/stackify/application.py +++ b/stackify/application.py @@ -1,13 +1,8 @@ import socket import os -import json from stackify import API_URL - - -class JSONObject(object): - def toJSON(self): - return json.dumps(self, default=lambda x: x.__dict__) +from stackify.formats import JSONObject class EnvironmentDetail(JSONObject): diff --git a/stackify/error.py b/stackify/error.py index f2e9006..f860e8b 100644 --- a/stackify/error.py +++ b/stackify/error.py @@ -1,4 +1,4 @@ -from stackify.application import JSONObject +from stackify.formats import JSONObject class ErrorItem(JSONObject): diff --git a/stackify/formats.py b/stackify/formats.py new file mode 100644 index 0000000..55c160c --- /dev/null +++ b/stackify/formats.py @@ -0,0 +1,11 @@ +import json + + +def no_nones(d): + return {k: v for k,v in d.items() if v is not None} + + +class JSONObject(object): + def toJSON(self): + return json.dumps(self, default=lambda x: no_nones(x.__dict__)) + diff --git a/stackify/log.py b/stackify/log.py index 66374c4..9607d99 100644 --- a/stackify/log.py +++ b/stackify/log.py @@ -1,22 +1,11 @@ import time import threading -import logging import json -from stackify.application import JSONObject +from stackify.formats import JSONObject +from stackify import MAX_BATCH, LOGGING_LEVELS -MAX_BATCH = 100 - - -LOGGING_LEVELS = { - logging.CRITICAL: 'CRITICAL', - logging.ERROR: 'ERROR', - logging.WARNING: 'WARNING', - logging.INFO: 'INFO', - logging.DEBUG: 'DEBUG', - logging.NOTSET: 'NOTSET' -} class LogMsg(JSONObject): From ae2e19f8b50c870621694b7b2a951a64a16cbb85 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 4 Oct 2014 17:57:32 -0500 Subject: [PATCH 09/43] Adding basic tracebacks --- stackify/error.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/stackify/error.py b/stackify/error.py index f860e8b..31cd866 100644 --- a/stackify/error.py +++ b/stackify/error.py @@ -1,3 +1,6 @@ +import traceback +import sys + from stackify.formats import JSONObject @@ -11,12 +14,17 @@ def __init__(self): self.StackTrace = [] # array of TraceFrames self.InnerError = None # cause? + def load_stack(self, tb): + stacks = traceback.extract_tb(tb) + for filename, lineno, parent, method in stacks: + self.StackTrace.append(TraceFrame(filename, lineno, method)) + class TraceFrame(JSONObject): - def __init__(self): - self.CodeFileName = None - self.LineNum = None - self.Method = None + def __init__(self, filename, lineno, method): + self.CodeFileName = filename + self.LineNum = lineno + self.Method = method class WebRequestDetail(JSONObject): @@ -47,3 +55,8 @@ def __init__(self): self.CustomerName = None self.UserName = None + def load_exception(self): + self.Error = ErrorItem() + type_, value, tb = sys.exc_info() + self.Error.load_stack(tb) + From 1d634a41cf1615abdcdb24be614d0bea7a507ffd Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 4 Oct 2014 18:48:56 -0500 Subject: [PATCH 10/43] Fix gzip for python3 --- stackify/error.py | 16 ++++++++++------ stackify/formats.py | 4 ++-- stackify/http.py | 25 +++++++++++++++++-------- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/stackify/error.py b/stackify/error.py index 31cd866..08515b7 100644 --- a/stackify/error.py +++ b/stackify/error.py @@ -1,4 +1,5 @@ import traceback +import time import sys from stackify.formats import JSONObject @@ -8,15 +9,19 @@ class ErrorItem(JSONObject): def __init__(self): self.Message = None # exception message self.ErrorType = None # exception class name - self.ErrorTypeCode = None # ? + self.ErrorTypeCode = None self.Data = None # custom data self.SourceMethod = None self.StackTrace = [] # array of TraceFrames self.InnerError = None # cause? - def load_stack(self, tb): + def load_stack(self): + type_, value, tb = sys.exc_info() stacks = traceback.extract_tb(tb) - for filename, lineno, parent, method in stacks: + self.ErrorType = type_.__name__ + self.Message = str(value) + self.SourceMethod = stacks[-1][2] + for filename, lineno, method, text in reversed(stacks): self.StackTrace.append(TraceFrame(filename, lineno, method)) @@ -49,7 +54,7 @@ def __init__(self): class StackifyError(JSONObject): def __init__(self): self.EnvironmentDetail = None # environment detail object - self.OccurredEpochMillis = None + self.OccurredEpochMillis = int(time.time() * 1000) self.Error = None # ErrorItem object self.WebRequestDetail = None # WebRequestDetail object self.CustomerName = None @@ -57,6 +62,5 @@ def __init__(self): def load_exception(self): self.Error = ErrorItem() - type_, value, tb = sys.exc_info() - self.Error.load_stack(tb) + self.Error.load_stack() diff --git a/stackify/formats.py b/stackify/formats.py index 55c160c..08ef0d2 100644 --- a/stackify/formats.py +++ b/stackify/formats.py @@ -1,11 +1,11 @@ import json -def no_nones(d): +def nonempty(d): return {k: v for k,v in d.items() if v is not None} class JSONObject(object): def toJSON(self): - return json.dumps(self, default=lambda x: no_nones(x.__dict__)) + return json.dumps(self, default=lambda x: nonempty(x.__dict__)) diff --git a/stackify/http.py b/stackify/http.py index d795be7..91fecab 100644 --- a/stackify/http.py +++ b/stackify/http.py @@ -6,7 +6,22 @@ try: from cStringIO import StringIO except: - from StringIO import StringIO + try: + from StringIO import StringIO + except: + pass # python 3, we use a new function in gzip + + +def gzip_compress(data): + if hasattr(gzip, 'compress'): + return gzip.compress(bytes(data, 'utf-8')) # python 3 + else: + s = StringIO() + g = gzip.GzipFile(fileobj=s, mode='w') + g.write(data) + g.close() + return s.getvalue() + from stackify.application import EnvironmentDetail from stackify import READ_TIMEOUT @@ -34,19 +49,13 @@ def POST(self, url, json_object, use_gzip=False): 'X-Stackify-PV': 'V1', } - try: payload_data = json_object.toJSON() internal_log.debug('POST data: {0}'.format(payload_data)) if use_gzip: headers['Content-Encoding'] = 'gzip' - # compress payload with gzip - s = StringIO() - g = gzip.GzipFile(fileobj=s, mode='w') - g.write(payload_data) - g.close() - payload_data = s.getvalue() + payload_data = gzip_compress(payload_data) response = requests.post(request_url, data=payload_data, headers=headers, From c491d6820afb04252707cd7ce91e997fbe0f3f9e Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 4 Oct 2014 20:26:11 -0500 Subject: [PATCH 11/43] Adding handler base and logrecord stuff --- stackify/__init__.py | 29 +++++ stackify/error.py | 20 +++- stackify/handler.py | 64 +++++++++++ stackify/handler_backport.py | 200 +++++++++++++++++++++++++++++++++++ stackify/log.py | 37 +++++-- 5 files changed, 338 insertions(+), 12 deletions(-) create mode 100644 stackify/handler.py create mode 100644 stackify/handler_backport.py diff --git a/stackify/__init__.py b/stackify/__init__.py index 376406b..6f76ba1 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -11,8 +11,12 @@ MAX_BATCH = 100 +QUEUE_SIZE = 1000 import logging +import inspect + +DEFAULT_LEVEL = logging.ERROR LOGGING_LEVELS = { logging.CRITICAL: 'CRITICAL', @@ -31,3 +35,28 @@ from stackify.application import ApiConfiguration from stackify.http import HTTPClient +from stackify.handler import StackifyHandler + + +def getLogger(name=None, **kwargs): + if not name: + try: + frame = inspect.stack()[1] + module = inspect.getmodule(frame[0]) + name = module.__name__ + except IndexError: + name = 'stackify-python-unknown' + + logger = logging.getLogger(name) + + if not [isinstance(x, StackifyHandler) for x in logger.handlers]: + internal_log.debug('Creating handler for logger {0}'.format(name)) + handler = StackifyHandler(**kwargs) + logger.addHandler(handler) + + if logger.getEffectiveLevel() == logging.NOTSET: + logger.setLevel(DEFAULT_LEVEL) + + return logger + + diff --git a/stackify/error.py b/stackify/error.py index 08515b7..9aeb897 100644 --- a/stackify/error.py +++ b/stackify/error.py @@ -15,9 +15,14 @@ def __init__(self): self.StackTrace = [] # array of TraceFrames self.InnerError = None # cause? - def load_stack(self): - type_, value, tb = sys.exc_info() + def load_stack(self, exc_info=None): + if not exc_info: + type_, value, tb = sys.exc_info() + else: + type_, value, tb = exc_info + stacks = traceback.extract_tb(tb) + self.ErrorType = type_.__name__ self.Message = str(value) self.SourceMethod = stacks[-1][2] @@ -54,13 +59,18 @@ def __init__(self): class StackifyError(JSONObject): def __init__(self): self.EnvironmentDetail = None # environment detail object - self.OccurredEpochMillis = int(time.time() * 1000) + self.OccurredEpochMillis = None self.Error = None # ErrorItem object self.WebRequestDetail = None # WebRequestDetail object self.CustomerName = None self.UserName = None - def load_exception(self): + def load_exception(self, exc_info=None): self.Error = ErrorItem() - self.Error.load_stack() + self.Error.load_stack(exc_info) + + def from_record(self, record): + self.load_exception(record.exc_info) + self.OccurredEpochMillis = int(record.created * 1000) + diff --git a/stackify/handler.py b/stackify/handler.py new file mode 100644 index 0000000..e697d6c --- /dev/null +++ b/stackify/handler.py @@ -0,0 +1,64 @@ +import logging +import threading + +try: + from logging.handlers import QueueHandler, QueueListener +except: + from stackify.handler_backport import QueueHandler, QueueListener + +try: + import Queue as queue +except ImportError: + import queue + +from stackify import QUEUE_SIZE +from stackify.log import LogMsg +from stackify.error import ErrorItem +from stackify.http import HTTPClient + + + +class StackifyHandler(QueueHandler): + ''' + A handler class to format and queue log messages for later + transmission to Stackify servers. + ''' + + def __init__(self, queue_=None): + if queue_ is None: + queue_ = queue.Queue(QUEUE_SIZE) + + super(StackifyHandler, self).__init__(queue_) + + self.listener = StackifyListener(queue_) + + def enqueue(self, record): + ''' + Put a new record on the queue. If it's full, evict an item. + ''' + try: + self.queue.put_nowait(record) + logger = logging.getLogger(__name__) + logger.debug('put record ' + record.toJSON()) + except queue.Full: + logger = logging.getLogger(__name__) + logger.warn('StackifyHandler queue is full, evicting oldest record') + self.queue.get_nowait() + self.queue.put_nowait(record) + + def prepare(self, record): + print(record.__dict__) + msg = LogMsg() + msg.from_record(record) + + return msg + + +class StackifyListener(QueueListener): + ''' + A listener to read queued log messages and send them to Stackify. + ''' + + def __init__(self, queue_): + super(StackifyListener, self).__init__(queue_) + diff --git a/stackify/handler_backport.py b/stackify/handler_backport.py new file mode 100644 index 0000000..991afd5 --- /dev/null +++ b/stackify/handler_backport.py @@ -0,0 +1,200 @@ +# Copyright 2001-2013 by Vinay Sajip. All Rights Reserved. +# +# Permission to use, copy, modify, and distribute this software and its +# documentation for any purpose and without fee is hereby granted, +# provided that the above copyright notice appear in all copies and that +# both that copyright notice and this permission notice appear in +# supporting documentation, and that the name of Vinay Sajip +# not be used in advertising or publicity pertaining to distribution +# of the software without specific, written prior permission. +# VINAY SAJIP DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING +# ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL +# VINAY SAJIP BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR +# ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER +# IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +import logging +import threading + + +class QueueHandler(logging.Handler): + """ + This handler sends events to a queue. Typically, it would be used together + with a multiprocessing Queue to centralise logging to file in one process + (in a multi-process application), so as to avoid file write contention + between processes. + + This code is new in Python 3.2, but this class can be copy pasted into + user code for use with earlier Python versions. + """ + + def __init__(self, queue): + """ + Initialise an instance, using the passed queue. + """ + logging.Handler.__init__(self) + self.queue = queue + + def enqueue(self, record): + """ + Enqueue a record. + + The base implementation uses put_nowait. You may want to override + this method if you want to use blocking, timeouts or custom queue + implementations. + """ + self.queue.put_nowait(record) + + def prepare(self, record): + """ + Prepares a record for queuing. The object returned by this method is + enqueued. + + The base implementation formats the record to merge the message + and arguments, and removes unpickleable items from the record + in-place. + + You might want to override this method if you want to convert + the record to a dict or JSON string, or send a modified copy + of the record while leaving the original intact. + """ + # The format operation gets traceback text into record.exc_text + # (if there's exception data), and also puts the message into + # record.message. We can then use this to replace the original + # msg + args, as these might be unpickleable. We also zap the + # exc_info attribute, as it's no longer needed and, if not None, + # will typically not be pickleable. + self.format(record) + record.msg = record.message + record.args = None + record.exc_info = None + return record + + def emit(self, record): + """ + Emit a record. + + Writes the LogRecord to the queue, preparing it for pickling first. + """ + try: + self.enqueue(self.prepare(record)) + except Exception: + self.handleError(record) + + +class QueueListener(object): + """ + This class implements an internal threaded listener which watches for + LogRecords being added to a queue, removes them and passes them to a + list of handlers for processing. + """ + _sentinel = None + + def __init__(self, queue, *handlers): + """ + Initialise an instance with the specified queue and + handlers. + """ + self.queue = queue + self.handlers = handlers + self._stop = threading.Event() + self._thread = None + + def dequeue(self, block): + """ + Dequeue a record and return it, optionally blocking. + + The base implementation uses get. You may want to override this method + if you want to use timeouts or work with custom queue implementations. + """ + return self.queue.get(block) + + def start(self): + """ + Start the listener. + + This starts up a background thread to monitor the queue for + LogRecords to process. + """ + self._thread = t = threading.Thread(target=self._monitor) + t.setDaemon(True) + t.start() + + def prepare(self , record): + """ + Prepare a record for handling. + + This method just returns the passed-in record. You may want to + override this method if you need to do any custom marshalling or + manipulation of the record before passing it to the handlers. + """ + return record + + def handle(self, record): + """ + Handle a record. + + This just loops through the handlers offering them the record + to handle. + """ + record = self.prepare(record) + for handler in self.handlers: + handler.handle(record) + + def _monitor(self): + """ + Monitor the queue for records, and ask the handler + to deal with them. + + This method runs on a separate, internal thread. + The thread will terminate if it sees a sentinel object in the queue. + """ + q = self.queue + has_task_done = hasattr(q, 'task_done') + while not self._stop.isSet(): + try: + record = self.dequeue(True) + if record is self._sentinel: + break + self.handle(record) + if has_task_done: + q.task_done() + except queue.Empty: + pass + # There might still be records in the queue. + while True: + try: + record = self.dequeue(False) + if record is self._sentinel: + break + self.handle(record) + if has_task_done: + q.task_done() + except queue.Empty: + break + + def enqueue_sentinel(self): + """ + This is used to enqueue the sentinel record. + + The base implementation uses put_nowait. You may want to override this + method if you want to use timeouts or work with custom queue + implementations. + """ + self.queue.put_nowait(self._sentinel) + + def stop(self): + """ + Stop the listener. + + This asks the thread to terminate, and then waits for it to do so. + Note that if you don't call this before your application exits, there + may be some records still left on the queue, which won't be processed. + """ + self._stop.set() + self.enqueue_sentinel() + self._thread.join() + self._thread = None + diff --git a/stackify/log.py b/stackify/log.py index 9607d99..129ea5e 100644 --- a/stackify/log.py +++ b/stackify/log.py @@ -1,26 +1,48 @@ -import time -import threading import json +import logging from stackify.formats import JSONObject from stackify import MAX_BATCH, LOGGING_LEVELS +from stackify.error import StackifyError +# this is used to separate builtin keys from user-specified keys +RECORD_VARS = set(logging.LogRecord('','','','','','','','').__dict__.keys()) + class LogMsg(JSONObject): - def __init__(self, message, data=None): - self.Msg = message - self.data = json.dumps(data, default=lambda x: x.__dict__) if data else None + def __init__(self): + self.Msg = None + self.data = None self.Ex = None # a StackifyError object - self.Th = threading.current_thread().ident - self.EpochMs = int(time.time() * 1000) + #self.Th = threading.current_thread().ident + self.Th = None + self.EpochMs = None self.Level = None self.TransID = None # filename, line_number, function = internal_log.findCaller() self.SrcMethod = None self.SrcLine = None + def from_record(self, record): + self.Msg = record.getMessage() + self.Th = record.threadName or record.thread + self.EpochMs = int(record.created * 1000) + self.Level = record.levelname + self.SrcMethod = record.funcName + self.SrcLine = record.lineno + + # check for user-specified keys + data = { k:v for k,v in record.__dict__.items() if k not in RECORD_VARS } + + if data: + self.data = json.dumps(data, default=lambda x: x.__dict__) + + if record.exc_info: + self.Ex = StackifyError() + self.Ex.from_record(record) + class LogMsgGroup(JSONObject): def __init__(self, msgs, logger=None): @@ -30,3 +52,4 @@ def __init__(self, msgs, logger=None): self.CDAppID = None self.AppNameID = None self.ServerName = None + From b356445c4ecc3c04ed5e166b99bffae716004b2d Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 4 Oct 2014 21:03:45 -0500 Subject: [PATCH 12/43] More work on handler and listener --- stackify/__init__.py | 45 ++++++++++++++++++++++++++++++++++++++------ stackify/handler.py | 40 ++++++++++++++++++++++++++++++++++----- 2 files changed, 74 insertions(+), 11 deletions(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index 6f76ba1..8b18298 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -38,14 +38,18 @@ from stackify.handler import StackifyHandler +# TODO +# holds our listeners, since more than one handler can service +# the same listener +__listener_cache = {} + + def getLogger(name=None, **kwargs): + ''' + Get a logger and attach a StackifyHandler if needed:w + ''' if not name: - try: - frame = inspect.stack()[1] - module = inspect.getmodule(frame[0]) - name = module.__name__ - except IndexError: - name = 'stackify-python-unknown' + name = getCallerName(2) logger = logging.getLogger(name) @@ -57,6 +61,35 @@ def getLogger(name=None, **kwargs): if logger.getEffectiveLevel() == logging.NOTSET: logger.setLevel(DEFAULT_LEVEL) + handler.listener.start() + return logger +def stopLogging(logger): + ''' + Shut down the StackifyHandler on a given logger. This will block + and wait for the queue to finish uploading. + ''' + for handler in getHandlers(logger): + handler.listener.stop() + + +def getCallerName(levels=1): + ''' + Gets the name of the module calling this function + ''' + try: + frame = inspect.stack()[levels] + module = inspect.getmodule(frame[0]) + name = module.__name__ + except IndexError: + name = 'stackify-python-unknown' + return name + + +def getHandlers(logger): + ''' + Return the StackifyHandlers on a given logger + ''' + return [isinstance(x, StackifyHandler) for x in logger.handlers] diff --git a/stackify/handler.py b/stackify/handler.py index e697d6c..f0128cf 100644 --- a/stackify/handler.py +++ b/stackify/handler.py @@ -1,5 +1,6 @@ import logging import threading +import os try: from logging.handlers import QueueHandler, QueueListener @@ -11,10 +12,11 @@ except ImportError: import queue -from stackify import QUEUE_SIZE +from stackify import QUEUE_SIZE, API_URL from stackify.log import LogMsg from stackify.error import ErrorItem from stackify.http import HTTPClient +from stackify.application import ApiConfiguration @@ -24,13 +26,16 @@ class StackifyHandler(QueueHandler): transmission to Stackify servers. ''' - def __init__(self, queue_=None): + def __init__(self, queue_=None, listener=None **kwargs): if queue_ is None: queue_ = queue.Queue(QUEUE_SIZE) super(StackifyHandler, self).__init__(queue_) - self.listener = StackifyListener(queue_) + if listener is None: + listener = StackifyListener(queue_, **kwargs) + + self.listener = listener def enqueue(self, record): ''' @@ -47,18 +52,43 @@ def enqueue(self, record): self.queue.put_nowait(record) def prepare(self, record): - print(record.__dict__) msg = LogMsg() msg.from_record(record) return msg +def arg_or_env(name, args, default=None): + env_name = 'STACKIFY_{0}'.format(name.upper()) + try: + return args.get(name, os.environ[env_name]) + except KeyError: + if default: + return default + else: + raise NameError('You must specify the keyword argument {0} or environment variable {1}'.format( + name, env_name)) + + + class StackifyListener(QueueListener): ''' A listener to read queued log messages and send them to Stackify. ''' - def __init__(self, queue_): + def __init__(self, queue_, config=None, **kwargs): super(StackifyListener, self).__init__(queue_) + if config is None: + # config not specified, build one with kwargs or environment variables + config = ApiConfiguration( + application = arg_or_env('application', kwargs), + environment = arg_or_env('environment', kwargs), + api_key = arg_or_env('api_key', kwargs), + api_url = arg_or_env('api_url', kwargs, API_URL)) + + self.http = HTTPClient(config) + + def handle(self, record): + + From 79f1ad0eb6181fa30483f54bac07c6561a67890b Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 18:29:20 -0500 Subject: [PATCH 13/43] Finishing up the handler threading --- stackify/__init__.py | 10 ++++-- stackify/application.py | 19 ++++++++++ stackify/handler.py | 69 ++++++++++++++++++------------------ stackify/handler_backport.py | 4 +++ 4 files changed, 65 insertions(+), 37 deletions(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index 8b18298..4b4d45a 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -15,6 +15,7 @@ import logging import inspect +import atexit DEFAULT_LEVEL = logging.ERROR @@ -46,7 +47,7 @@ def getLogger(name=None, **kwargs): ''' - Get a logger and attach a StackifyHandler if needed:w + Get a logger and attach a StackifyHandler if needed. ''' if not name: name = getCallerName(2) @@ -54,10 +55,12 @@ def getLogger(name=None, **kwargs): logger = logging.getLogger(name) if not [isinstance(x, StackifyHandler) for x in logger.handlers]: - internal_log.debug('Creating handler for logger {0}'.format(name)) + internal_log.debug('Creating handler for logger %s', name) handler = StackifyHandler(**kwargs) logger.addHandler(handler) + atexit.register(stopLogging, logger) + if logger.getEffectiveLevel() == logging.NOTSET: logger.setLevel(DEFAULT_LEVEL) @@ -70,6 +73,7 @@ def stopLogging(logger): Shut down the StackifyHandler on a given logger. This will block and wait for the queue to finish uploading. ''' + internal_log.debug('Shutting down all handlers') for handler in getHandlers(logger): handler.listener.stop() @@ -91,5 +95,5 @@ def getHandlers(logger): ''' Return the StackifyHandlers on a given logger ''' - return [isinstance(x, StackifyHandler) for x in logger.handlers] + return [x for x in logger.handlers if isinstance(x, StackifyHandler)] diff --git a/stackify/application.py b/stackify/application.py index d1582d7..48b69cf 100644 --- a/stackify/application.py +++ b/stackify/application.py @@ -20,3 +20,22 @@ def __init__(self, api_key, application, environment, api_url=API_URL): self.application = application self.environment = environment + +def arg_or_env(name, args, default=None): + env_name = 'STACKIFY_{0}'.format(name.upper()) + try: + return args.get(name, os.environ[env_name]) + except KeyError: + if default: + return default + else: + raise NameError('You must specify the keyword argument {0} or environment variable {1}'.format( + name, env_name)) + + +def get_configuration(**kwargs): + return ApiConfiguration( + application = arg_or_env('application', kwargs), + environment = arg_or_env('environment', kwargs), + api_key = arg_or_env('api_key', kwargs), + api_url = arg_or_env('api_url', kwargs)) diff --git a/stackify/handler.py b/stackify/handler.py index f0128cf..b85067e 100644 --- a/stackify/handler.py +++ b/stackify/handler.py @@ -12,12 +12,11 @@ except ImportError: import queue -from stackify import QUEUE_SIZE, API_URL -from stackify.log import LogMsg +from stackify import QUEUE_SIZE, API_URL, MAX_BATCH +from stackify.log import LogMsg, LogMsgGroup from stackify.error import ErrorItem from stackify.http import HTTPClient -from stackify.application import ApiConfiguration - +from stackify.application import get_configuration class StackifyHandler(QueueHandler): @@ -26,7 +25,7 @@ class StackifyHandler(QueueHandler): transmission to Stackify servers. ''' - def __init__(self, queue_=None, listener=None **kwargs): + def __init__(self, queue_=None, listener=None, **kwargs): if queue_ is None: queue_ = queue.Queue(QUEUE_SIZE) @@ -41,54 +40,56 @@ def enqueue(self, record): ''' Put a new record on the queue. If it's full, evict an item. ''' + logger = logging.getLogger(__name__) try: self.queue.put_nowait(record) - logger = logging.getLogger(__name__) - logger.debug('put record ' + record.toJSON()) except queue.Full: - logger = logging.getLogger(__name__) logger.warn('StackifyHandler queue is full, evicting oldest record') self.queue.get_nowait() self.queue.put_nowait(record) - def prepare(self, record): - msg = LogMsg() - msg.from_record(record) - - return msg - - -def arg_or_env(name, args, default=None): - env_name = 'STACKIFY_{0}'.format(name.upper()) - try: - return args.get(name, os.environ[env_name]) - except KeyError: - if default: - return default - else: - raise NameError('You must specify the keyword argument {0} or environment variable {1}'.format( - name, env_name)) - - class StackifyListener(QueueListener): ''' A listener to read queued log messages and send them to Stackify. ''' - def __init__(self, queue_, config=None, **kwargs): + def __init__(self, queue_, max_batch=MAX_BATCH, config=None, **kwargs): super(StackifyListener, self).__init__(queue_) if config is None: - # config not specified, build one with kwargs or environment variables - config = ApiConfiguration( - application = arg_or_env('application', kwargs), - environment = arg_or_env('environment', kwargs), - api_key = arg_or_env('api_key', kwargs), - api_url = arg_or_env('api_url', kwargs, API_URL)) + config = get_configuration(**kwargs) + self.max_batch = max_batch + self.messages = [] self.http = HTTPClient(config) def handle(self, record): + logger = logging.getLogger(__name__) + + if not self.http.identified: + logger.debug('Identifying application') + self.http.identify_application() + + msg = LogMsg() + msg.from_record(record) + self.messages.append(msg) + + if len(self.messages) >= self.max_batch: + self.send_group() + + def send_group(self): + group = LogMsgGroup(self.messages) + self.http.POST('/Log/Save', group, True) + del self.messages[:] + + def stop(self): + logger = logging.getLogger(__name__) + logger.debug('Shutting down listener') + super(StackifyListener, self).stop() + # send any remaining messages + if self.messages: + logger.debug('{0} messages left on shutdown, uploading'.format(len(self.messages))) + self.send_group() diff --git a/stackify/handler_backport.py b/stackify/handler_backport.py index 991afd5..6e8b705 100644 --- a/stackify/handler_backport.py +++ b/stackify/handler_backport.py @@ -17,6 +17,10 @@ import logging import threading +try: + import Queue as queue +except ImportError: + import queue class QueueHandler(logging.Handler): From c373221010e11c2134913704e214abc71bd3b3ad Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 18:30:15 -0500 Subject: [PATCH 14/43] Format later in the logger --- stackify/http.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stackify/http.py b/stackify/http.py index 91fecab..f26197f 100644 --- a/stackify/http.py +++ b/stackify/http.py @@ -41,7 +41,7 @@ def __init__(self, api_config): def POST(self, url, json_object, use_gzip=False): request_url = self.api_config.api_url + url internal_log = logging.getLogger(__name__) - internal_log.debug('Request URL: {0}'.format(request_url)) + internal_log.debug('Request URL: %s', request_url) headers = { 'Content-Type': 'application/json', @@ -51,7 +51,7 @@ def POST(self, url, json_object, use_gzip=False): try: payload_data = json_object.toJSON() - internal_log.debug('POST data: {0}'.format(payload_data)) + internal_log.debug('POST data: %s', payload_data) if use_gzip: headers['Content-Encoding'] = 'gzip' @@ -60,7 +60,7 @@ def POST(self, url, json_object, use_gzip=False): response = requests.post(request_url, data=payload_data, headers=headers, timeout=READ_TIMEOUT) - internal_log.debug('Response: {0}'.format(response.text)) + internal_log.debug('Response: %s', response.text) return response.json() except requests.exceptions.RequestException: interal_log.exception('HTTP exception:') From 7e5597958ace66e6a2a6a2b3878791e151382f77 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 18:51:00 -0500 Subject: [PATCH 15/43] Adding unittest setup --- setup.py | 24 ++++++++++++++---------- stackify/application.py | 1 + tests/__init__.py | 0 tests/test_application.py | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 10 deletions(-) create mode 100644 tests/__init__.py create mode 100755 tests/test_application.py diff --git a/setup.py b/setup.py index a928872..b9a6d13 100644 --- a/setup.py +++ b/setup.py @@ -9,18 +9,22 @@ version = ast.literal_eval(version_re.search(f).group(1)) setup( - name='stackify', - version=version, - author='Matthew Thompson', - author_email='chameleonator@gmail.com', - packages=['stackify'], - url='https://github.com/stackify/stackify-python', - license=open('LICENSE.txt').readline(), - description='Stackify API for Python', - long_description=open('README.md').read(), - install_requires=[ + name = 'stackify', + version = version, + author = 'Matthew Thompson', + author_email = 'chameleonator@gmail.com', + packages = ['stackify'], + url = 'https://github.com/stackify/stackify-python', + license = open('LICENSE.txt').readline(), + description = 'Stackify API for Python', + long_description = open('README.md').read(), + install_requires = [ 'retrying>=1.2.3', 'requests>=2.4.1' + ], + test_suite = 'tests', + tests_requires = [ + 'mock>=1.0.1' ] ) diff --git a/stackify/application.py b/stackify/application.py index 48b69cf..093a991 100644 --- a/stackify/application.py +++ b/stackify/application.py @@ -39,3 +39,4 @@ def get_configuration(**kwargs): environment = arg_or_env('environment', kwargs), api_key = arg_or_env('api_key', kwargs), api_url = arg_or_env('api_url', kwargs)) + diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_application.py b/tests/test_application.py new file mode 100755 index 0000000..f726cc8 --- /dev/null +++ b/tests/test_application.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +""" +Test the stackify.application module +""" + +import unittest +from mock import patch + +from stackify.application import get_configuration + + +class TestConfig(unittest.TestCase): + def test_environment_config(self): + env_map = { + 'STACKIFY_APPLICATION': 'test_appname', + 'STACKIFY_ENVIRONMENT': 'test_environment', + 'STACKIFY_API_KEY': 'test_apikey', + 'STACKIFY_API_URL': 'test_apiurl', + } + + with patch.dict('os.environ', env_map): + config = get_configuration() + + self.assertEqual(config.application, 'test_appname') + self.assertEqual(config.environment, 'test_environment') + self.assertEqual(config.api_key, 'test_apikey') + self.assertEqual(config.api_url, 'test_apiurl') + + +if __name__=='__main__': + unittest.main() + From 77847b29f059193fd1f47a8df7c1e7c76ad5246b Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 18:54:17 -0500 Subject: [PATCH 16/43] More app test --- tests/test_application.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_application.py b/tests/test_application.py index f726cc8..f56f0b0 100755 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -26,6 +26,32 @@ def test_environment_config(self): self.assertEqual(config.api_key, 'test_apikey') self.assertEqual(config.api_url, 'test_apiurl') + def test_kwarg_mix(self): + env_map = { + 'STACKIFY_APPLICATION': 'test2_appname', + 'STACKIFY_ENVIRONMENT': 'test2_environment', + } + + with patch.dict('os.environ', env_map): + config = get_configuration(api_key='test2_apikey', api_url='test2_apiurl') + + self.assertEqual(config.application, 'test2_appname') + self.assertEqual(config.environment, 'test2_environment') + self.assertEqual(config.api_key, 'test2_apikey') + self.assertEqual(config.api_url, 'test2_apiurl') + + def test_kwargs(self): + config = get_configuration( + application = 'test3_appname', + environment = 'test3_environment', + api_key = 'test3_apikey', + api_url = 'test3_apiurl') + + self.assertEqual(config.application, 'test3_appname') + self.assertEqual(config.environment, 'test3_environment') + self.assertEqual(config.api_key, 'test3_apikey') + self.assertEqual(config.api_url, 'test3_apiurl') + if __name__=='__main__': unittest.main() From bc631ba48d3bba2b73d893ae189b0d3a44523a06 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 19:57:00 -0500 Subject: [PATCH 17/43] More tests, fixing bugs exposed by tests --- stackify/__init__.py | 5 +- stackify/application.py | 7 ++- tests/bases.py | 28 +++++++++++ tests/test_application.py | 56 ++++++++++++++++++---- tests/test_init.py | 99 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 182 insertions(+), 13 deletions(-) create mode 100644 tests/bases.py create mode 100755 tests/test_init.py diff --git a/stackify/__init__.py b/stackify/__init__.py index 4b4d45a..0e538e7 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -45,7 +45,7 @@ __listener_cache = {} -def getLogger(name=None, **kwargs): +def getLogger(name=None, auto_shutdown=True, **kwargs): ''' Get a logger and attach a StackifyHandler if needed. ''' @@ -59,7 +59,8 @@ def getLogger(name=None, **kwargs): handler = StackifyHandler(**kwargs) logger.addHandler(handler) - atexit.register(stopLogging, logger) + if auto_shutdown: + atexit.register(stopLogging, logger) if logger.getEffectiveLevel() == logging.NOTSET: logger.setLevel(DEFAULT_LEVEL) diff --git a/stackify/application.py b/stackify/application.py index 093a991..f5b7a87 100644 --- a/stackify/application.py +++ b/stackify/application.py @@ -24,7 +24,10 @@ def __init__(self, api_key, application, environment, api_url=API_URL): def arg_or_env(name, args, default=None): env_name = 'STACKIFY_{0}'.format(name.upper()) try: - return args.get(name, os.environ[env_name]) + value = args.get(name) + if not value: + value = os.environ[env_name] + return value except KeyError: if default: return default @@ -38,5 +41,5 @@ def get_configuration(**kwargs): application = arg_or_env('application', kwargs), environment = arg_or_env('environment', kwargs), api_key = arg_or_env('api_key', kwargs), - api_url = arg_or_env('api_url', kwargs)) + api_url = arg_or_env('api_url', kwargs, API_URL)) diff --git a/tests/bases.py b/tests/bases.py new file mode 100644 index 0000000..cc97dc8 --- /dev/null +++ b/tests/bases.py @@ -0,0 +1,28 @@ +import os +import unittest + +class ClearEnvTest(unittest.TestCase): + ''' + This class clears the environment variables that the + library uses for clean testing. + ''' + + def setUp(self): + # if you have these specified in the environment it will break tests + to_save = [ + 'STACKIFY_APPLICATION', + 'STACKIFY_ENVIRONMENT', + 'STACKIFY_API_KEY', + 'STACKIFY_API_URL', + ] + self.saved = {} + for key in to_save: + if key in os.environ: + self.saved[key] = os.environ[key] + del os.environ[key] + + def tearDown(self): + # restore deleted environment variables + for key, item in self.saved.items(): + os.environ[key] = item + del self.saved diff --git a/tests/test_application.py b/tests/test_application.py index f56f0b0..a3b5192 100755 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -5,28 +5,53 @@ import unittest from mock import patch +import os +from bases import ClearEnvTest +from stackify import API_URL from stackify.application import get_configuration -class TestConfig(unittest.TestCase): +class TestConfig(ClearEnvTest): + ''' + Test automatic configuration for the ApiConfiguration + ''' + + def test_required_kwargs(self): + '''API configuration requires appname, env and key''' + env_map = {} + + with patch.dict('os.environ', env_map): + with self.assertRaises(NameError): + get_configuration() + with self.assertRaises(NameError): + get_configuration(application='1') + with self.assertRaises(NameError): + get_configuration(application='1', environment='2') + with self.assertRaises(NameError): + get_configuration(application='1', environment='2', api_url='3') + + get_configuration(application='1', environment='2', api_key='3') + def test_environment_config(self): + '''API configuration can load from env vars''' env_map = { - 'STACKIFY_APPLICATION': 'test_appname', - 'STACKIFY_ENVIRONMENT': 'test_environment', - 'STACKIFY_API_KEY': 'test_apikey', - 'STACKIFY_API_URL': 'test_apiurl', + 'STACKIFY_APPLICATION': 'test1_appname', + 'STACKIFY_ENVIRONMENT': 'test1_environment', + 'STACKIFY_API_KEY': 'test1_apikey', + 'STACKIFY_API_URL': 'test1_apiurl', } with patch.dict('os.environ', env_map): config = get_configuration() - self.assertEqual(config.application, 'test_appname') - self.assertEqual(config.environment, 'test_environment') - self.assertEqual(config.api_key, 'test_apikey') - self.assertEqual(config.api_url, 'test_apiurl') + self.assertEqual(config.application, 'test1_appname') + self.assertEqual(config.environment, 'test1_environment') + self.assertEqual(config.api_key, 'test1_apikey') + self.assertEqual(config.api_url, 'test1_apiurl') def test_kwarg_mix(self): + '''API configuration can load from a mix of env vars and kwargs''' env_map = { 'STACKIFY_APPLICATION': 'test2_appname', 'STACKIFY_ENVIRONMENT': 'test2_environment', @@ -41,6 +66,7 @@ def test_kwarg_mix(self): self.assertEqual(config.api_url, 'test2_apiurl') def test_kwargs(self): + '''API configuration can load from kwargs''' config = get_configuration( application = 'test3_appname', environment = 'test3_environment', @@ -52,6 +78,18 @@ def test_kwargs(self): self.assertEqual(config.api_key, 'test3_apikey') self.assertEqual(config.api_url, 'test3_apiurl') + def test_api_url_default(self): + '''API URL is set automatically''' + config = get_configuration( + application = 'test4_appname', + environment = 'test4_environment', + api_key = 'test4_apikey') + + self.assertEqual(config.application, 'test4_appname') + self.assertEqual(config.environment, 'test4_environment') + self.assertEqual(config.api_key, 'test4_apikey') + self.assertEqual(config.api_url, API_URL) + if __name__=='__main__': unittest.main() diff --git a/tests/test_init.py b/tests/test_init.py new file mode 100755 index 0000000..cff5eb1 --- /dev/null +++ b/tests/test_init.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +""" +Test the stackify.__init__ setup functions +""" + +import unittest +from mock import patch +from bases import ClearEnvTest + +import os +import atexit + +import stackify +import logging + + +class TestInit(ClearEnvTest): + ''' + Test the logger init functionality + ''' + + def setUp(self): + super(TestInit, self).setUp() + self.config = stackify.ApiConfiguration( + application = 'test_appname', + environment = 'test_environment', + api_key = 'test_apikey', + api_url = 'test_apiurl') + self.loggers = [] + + def tearDown(self): + super(TestInit, self).tearDown() + global_loggers = logging.Logger.manager.loggerDict + for logger in self.loggers: + del global_loggers[logger.name] + + def test_logger_no_config(self): + '''Logger API config loads from the environment automatically''' + env_map = { + 'STACKIFY_APPLICATION': 'test2_appname', + 'STACKIFY_ENVIRONMENT': 'test2_environment', + 'STACKIFY_API_KEY': 'test2_apikey', + 'STACKIFY_API_URL': 'test2_apiurl', + } + + with patch.dict('os.environ', env_map): + logger = stackify.getLogger(auto_shutdown=False) + self.loggers.append(logger) + + config = logger.handlers[0].listener.http.api_config + + self.assertEqual(config.application, 'test2_appname') + self.assertEqual(config.environment, 'test2_environment') + self.assertEqual(config.api_key, 'test2_apikey') + self.assertEqual(config.api_url, 'test2_apiurl') + + def test_logger_api_config(self): + '''Logger API config loads from the specified config objects''' + logger = stackify.getLogger(config=self.config, auto_shutdown=False) + self.loggers.append(logger) + + config = logger.handlers[0].listener.http.api_config + + self.assertEqual(config.application, 'test_appname') + self.assertEqual(config.environment, 'test_environment') + self.assertEqual(config.api_key, 'test_apikey') + self.assertEqual(config.api_url, 'test_apiurl') + + def test_logger_name(self): + '''The automatic logger name is the current module''' + self.assertEqual(stackify.getCallerName(), 'tests.test_init') + + def test_get_logger_defaults(self): + '''The logger has sane defaults''' + env_map = { + 'STACKIFY_APPLICATION': 'test2_appname', + 'STACKIFY_ENVIRONMENT': 'test2_environment', + 'STACKIFY_API_KEY': 'test2_apikey', + } + + with patch.dict('os.environ', env_map): + logger = stackify.getLogger(auto_shutdown=False) + self.loggers.append(logger) + + handler = logger.handlers[0] + config = handler.listener.http.api_config + + self.assertEqual(logger.name, 'tests.test_init') + self.assertEqual(config.api_url, stackify.API_URL) + self.assertEqual(handler.listener.max_batch, stackify.MAX_BATCH) + self.assertEqual(handler.queue.maxsize, stackify.QUEUE_SIZE) + + def test_get_logger_reuse(self): + pass + + +if __name__=='__main__': + unittest.main() + From 75b83a7caa6c19c5f316ebc6573d209489c9bd25 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 19:59:57 -0500 Subject: [PATCH 18/43] More logging tests --- tests/test_init.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_init.py b/tests/test_init.py index cff5eb1..51a5bbb 100755 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -4,7 +4,7 @@ """ import unittest -from mock import patch +from mock import patch, Mock from bases import ClearEnvTest import os @@ -91,7 +91,19 @@ def test_get_logger_defaults(self): self.assertEqual(handler.queue.maxsize, stackify.QUEUE_SIZE) def test_get_logger_reuse(self): - pass + '''Grabbing a logger twice results in the same logger''' + logger = stackify.getLogger(config=self.config, auto_shutdown=False) + self.loggers.append(logger) + logger_two = stackify.getLogger(config=self.config, auto_shutdown=False) + self.assertEqual(id(logger_two), id(logger)) + + def test_logger_atexit(self): + '''Logger registers an atexit function to clean up''' + func = Mock() + with patch('atexit.register', func): + logger = stackify.getLogger(config=self.config) + self.loggers.append(logger) + func.assert_called_with(stackify.stopLogging, logger) if __name__=='__main__': From 64031549eb6944374f0ad80b7a7083d5a90dcd4d Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 20:04:45 -0500 Subject: [PATCH 19/43] Default level should be WARNING --- tests/test_init.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_init.py b/tests/test_init.py index 51a5bbb..d0f7ffc 100755 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -89,6 +89,7 @@ def test_get_logger_defaults(self): self.assertEqual(config.api_url, stackify.API_URL) self.assertEqual(handler.listener.max_batch, stackify.MAX_BATCH) self.assertEqual(handler.queue.maxsize, stackify.QUEUE_SIZE) + self.assertEqual(logger.getEffectiveLevel(), logging.WARNING) def test_get_logger_reuse(self): '''Grabbing a logger twice results in the same logger''' From dc0b2890aa757089e015f9dbd56c1b6d029fa46f Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 20:33:38 -0500 Subject: [PATCH 20/43] Test handlers --- tests/test_init.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_init.py b/tests/test_init.py index d0f7ffc..1ffdbf5 100755 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -96,7 +96,7 @@ def test_get_logger_reuse(self): logger = stackify.getLogger(config=self.config, auto_shutdown=False) self.loggers.append(logger) logger_two = stackify.getLogger(config=self.config, auto_shutdown=False) - self.assertEqual(id(logger_two), id(logger)) + self.assertIs(logger_two, logger) def test_logger_atexit(self): '''Logger registers an atexit function to clean up''' @@ -106,6 +106,12 @@ def test_logger_atexit(self): self.loggers.append(logger) func.assert_called_with(stackify.stopLogging, logger) + def test_get_handlers(self): + '''Registered handlers are provided by getHandlers''' + logger = stackify.getLogger(config=self.config, auto_shutdown=False) + self.loggers.append(logger) + self.assertEqual(logger.handlers, stackify.getHandlers(logger)) + if __name__=='__main__': unittest.main() From f3f21129474728ed1039e22be03c1206869f5677 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 20:33:47 -0500 Subject: [PATCH 21/43] Base http tests --- tests/test_http.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 tests/test_http.py diff --git a/tests/test_http.py b/tests/test_http.py new file mode 100755 index 0000000..6abf51b --- /dev/null +++ b/tests/test_http.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +""" +Test the stackify.http module +""" + +import unittest +from mock import patch, Mock + +import stackify.http + + +class TestClient(unittest.TestCase): + ''' + Test the HTTP Client and associated utilities + ''' + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_logger_no_config(self): + '''GZIP encoder works''' + correct = list('\x1f\x8b\x08\x00 \x02\xff\xf3H\xcd\xc9\xc9\xd7Q(\xcf/\xcaIQ\x04\x00\xe6\xc6\xe6\xeb\r\x00\x00\x00') + gzipped = list(stackify.http.gzip_compress('Hello, world!')) + gzipped[4:8] = ' ' # blank the mtime + self.assertEqual(gzipped, correct) + +if __name__=='__main__': + unittest.main() + From b4abed65840126237dd3935a659bccfeffc07e89 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 21:09:23 -0500 Subject: [PATCH 22/43] Reenable the retry module and test it --- stackify/http.py | 2 +- tests/test_http.py | 49 ++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/stackify/http.py b/stackify/http.py index f26197f..6d6ce5e 100644 --- a/stackify/http.py +++ b/stackify/http.py @@ -70,7 +70,7 @@ def POST(self, url, json_object, use_gzip=False): internal_log.exception('Cannot decode JSON response') raise - #@retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=10000) + @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=10000) def identify_application(self): result = self.POST('/Metrics/IdentifyApp', self.environment_detail) self.app_name_id = result.get('AppNameID') diff --git a/tests/test_http.py b/tests/test_http.py index 6abf51b..e774071 100755 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -4,21 +4,49 @@ """ import unittest +import retrying from mock import patch, Mock +retrying_mock = Mock() + import stackify.http +from stackify.application import ApiConfiguration + + +old_retry = retrying.retry + +FAKE_RETRIES = 3 + +def fake_retry(*args, **kwargs): + kwargs['wait_exponential_max'] = 0 # no delay between retries + kwargs['stop_max_attempt_number'] = FAKE_RETRIES # stop after 3 tries + def inner(func): + return old_retry(*args, **kwargs)(func) + return inner + class TestClient(unittest.TestCase): ''' Test the HTTP Client and associated utilities ''' - def setUp(self): - pass + @classmethod + def setUpClass(cls): + retrying.retry = fake_retry + reload(stackify.http) - def tearDown(self): - pass + @classmethod + def tearDownClass(cls): + reload(retrying) + reload(stackify.http) + + def setUp(self): + self.config = ApiConfiguration( + application = 'test_appname', + environment = 'test_environment', + api_key = 'test_apikey', + api_url = 'test_apiurl') def test_logger_no_config(self): '''GZIP encoder works''' @@ -27,6 +55,19 @@ def test_logger_no_config(self): gzipped[4:8] = ' ' # blank the mtime self.assertEqual(gzipped, correct) + def test_identify_retrying(self): + '''HTTP identify should retry''' + client = stackify.http.HTTPClient(self.config) + + class CustomException(Exception): pass + crash = Mock(side_effect=CustomException) + + with patch.object(client, 'POST', crash): + with self.assertRaises(CustomException): + client.identify_application() + self.assertEqual(crash.call_count, FAKE_RETRIES) + + if __name__=='__main__': unittest.main() From 9c3af5e8c9ab62dd9c22c7bbdb9bdd2c22b27fd6 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 22:03:09 -0500 Subject: [PATCH 23/43] Test identify function --- tests/test_http.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_http.py b/tests/test_http.py index e774071..8f022f8 100755 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -67,6 +67,30 @@ class CustomException(Exception): pass client.identify_application() self.assertEqual(crash.call_count, FAKE_RETRIES) + def test_identify(self): + '''HTTP identify should save results''' + client = stackify.http.HTTPClient(self.config) + self.assertFalse(client.identified) + + result = { + 'AppNameID': '1', + 'AppEnvID': '2', + 'DeviceID': '3', + 'DeviceAppID': '4', + 'DeviceAlias': '5', + } + post = Mock(return_value=result) + + with patch.object(client, 'POST', post): + client.identify_application() + + self.assertEqual(client.app_name_id, '1') + self.assertEqual(client.app_env_id, '2') + self.assertEqual(client.device_id, '3') + self.assertEqual(client.device_app_id, '4') + self.assertEqual(client.device_alias, '5') + self.assertTrue(client.identified) + if __name__=='__main__': unittest.main() From 5063de0aca4ed50af94e726ea8de39f94d4696f6 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 22:16:06 -0500 Subject: [PATCH 24/43] Test POST args --- tests/test_http.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_http.py b/tests/test_http.py index 8f022f8..49942b9 100755 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -12,6 +12,7 @@ import stackify.http from stackify.application import ApiConfiguration +from stackify import READ_TIMEOUT old_retry = retrying.retry @@ -91,6 +92,28 @@ def test_identify(self): self.assertEqual(client.device_alias, '5') self.assertTrue(client.identified) + def test_post_arguments(self): + '''HTTP post has correct headers''' + client = stackify.http.HTTPClient(self.config) + post = Mock() + payload = Mock() + + with patch('requests.post', post): + client.POST('url', payload) + + headers = { + 'Content-Type': 'application/json', + 'X-Stackify-Key': self.config.api_key, + 'X-Stackify-PV': 'V1', + } + + self.assertTrue(post.called) + args, kwargs = post.call_args + self.assertEquals(kwargs['headers'], headers) + self.assertEquals(kwargs['timeout'], READ_TIMEOUT) + self.assertEquals(kwargs['data'], payload.toJSON()) + + if __name__=='__main__': unittest.main() From c8f69f9728c6363e6a5ef4d2c4665783203df4ba Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 22:27:52 -0500 Subject: [PATCH 25/43] GZIP through POST test --- tests/test_http.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_http.py b/tests/test_http.py index 49942b9..250d833 100755 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -113,6 +113,23 @@ def test_post_arguments(self): self.assertEquals(kwargs['timeout'], READ_TIMEOUT) self.assertEquals(kwargs['data'], payload.toJSON()) + def test_post_gzip(self): + '''HTTP post uses gzip if requested''' + client = stackify.http.HTTPClient(self.config) + post = Mock() + payload = Mock() + payload.toJSON = Mock(return_value='1') + gzip = Mock(side_effect=lambda x: x + '_gzipped') + + with patch('requests.post', post): + with patch.object(stackify.http, 'gzip_compress', gzip): + client.POST('url', payload, use_gzip=True) + + self.assertTrue(post.called) + args, kwargs = post.call_args + self.assertEquals(kwargs['headers']['Content-Encoding'], 'gzip') + self.assertEquals(kwargs['data'], '1_gzipped') + if __name__=='__main__': From 223718fafbf446c39d9de1b92e755e323932b345 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 22:53:03 -0500 Subject: [PATCH 26/43] Some patch streamlining and new test --- tests/test_handler.py | 59 +++++++++++++++++++++++++++++++++++++++++++ tests/test_http.py | 16 +++++------- tests/test_init.py | 9 +++---- 3 files changed, 70 insertions(+), 14 deletions(-) create mode 100755 tests/test_handler.py diff --git a/tests/test_handler.py b/tests/test_handler.py new file mode 100755 index 0000000..43f161a --- /dev/null +++ b/tests/test_handler.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +""" +Test the stackify.handler module +""" + +import unittest +from mock import patch, Mock + +try: + import Queue as queue +except ImportError: + import queue + +import stackify.handler +from stackify.handler import StackifyHandler, StackifyListener +from stackify.application import ApiConfiguration + + +class TestHandler(unittest.TestCase): + ''' + Test the StackifyHandler class + ''' + + def test_queue_full(self): + '''The queue should evict when full''' + q = queue.Queue(1) + handler = StackifyHandler(queue_=q, listener=Mock()) + handler.enqueue('test1') + handler.enqueue('test2') + handler.enqueue('test3') + self.assertEqual(q.qsize(), 1) + self.assertEqual(q.get(), 'test3') + +class TestListener(unittest.TestCase): + ''' + Test the StackifyListener class + ''' + + def setUp(self): + self.config = ApiConfiguration( + application = 'test_appname', + environment = 'test_environment', + api_key = 'test_apikey', + api_url = 'test_apiurl') + + @patch('stackify.handler.LogMsg') + @patch.object(StackifyListener, 'send_group') + @patch('stackify.http.HTTPClient.identify_application') + def test_not_identified(self, logmsg, send_group, ident): + '''The HTTPClient identifies automatically if needed''' + q = queue.Queue(1) + listener = StackifyListener(queue_=q, config=self.config) + listener.handle(Mock()) + self.assertTrue(ident.called) + + +if __name__=='__main__': + unittest.main() + diff --git a/tests/test_http.py b/tests/test_http.py index 250d833..d6b408f 100755 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -92,14 +92,13 @@ def test_identify(self): self.assertEqual(client.device_alias, '5') self.assertTrue(client.identified) - def test_post_arguments(self): + @patch('requests.post') + def test_post_arguments(self, post): '''HTTP post has correct headers''' client = stackify.http.HTTPClient(self.config) - post = Mock() payload = Mock() - with patch('requests.post', post): - client.POST('url', payload) + client.POST('url', payload) headers = { 'Content-Type': 'application/json', @@ -113,17 +112,16 @@ def test_post_arguments(self): self.assertEquals(kwargs['timeout'], READ_TIMEOUT) self.assertEquals(kwargs['data'], payload.toJSON()) - def test_post_gzip(self): + @patch('requests.post') + def test_post_gzip(self, post): '''HTTP post uses gzip if requested''' client = stackify.http.HTTPClient(self.config) - post = Mock() payload = Mock() payload.toJSON = Mock(return_value='1') gzip = Mock(side_effect=lambda x: x + '_gzipped') - with patch('requests.post', post): - with patch.object(stackify.http, 'gzip_compress', gzip): - client.POST('url', payload, use_gzip=True) + with patch.object(stackify.http, 'gzip_compress', gzip): + client.POST('url', payload, use_gzip=True) self.assertTrue(post.called) args, kwargs = post.call_args diff --git a/tests/test_init.py b/tests/test_init.py index 1ffdbf5..0f7befd 100755 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -98,12 +98,11 @@ def test_get_logger_reuse(self): logger_two = stackify.getLogger(config=self.config, auto_shutdown=False) self.assertIs(logger_two, logger) - def test_logger_atexit(self): + @patch('atexit.register') + def test_logger_atexit(self, func): '''Logger registers an atexit function to clean up''' - func = Mock() - with patch('atexit.register', func): - logger = stackify.getLogger(config=self.config) - self.loggers.append(logger) + logger = stackify.getLogger(config=self.config) + self.loggers.append(logger) func.assert_called_with(stackify.stopLogging, logger) def test_get_handlers(self): From 4fb93a16012808a892c064908383bce1a8563253 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 22:53:19 -0500 Subject: [PATCH 27/43] Make setup executable --- setup.py | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 setup.py diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index b9a6d13..1759c28 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python from setuptools import setup import re import ast From 67319d133c23aee81b39f516174e0d6c2852a7fa Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 23:16:20 -0500 Subject: [PATCH 28/43] Fix patch and add more testing --- tests/test_handler.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/test_handler.py b/tests/test_handler.py index 43f161a..dc42285 100755 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -46,13 +46,32 @@ def setUp(self): @patch('stackify.handler.LogMsg') @patch.object(StackifyListener, 'send_group') @patch('stackify.http.HTTPClient.identify_application') - def test_not_identified(self, logmsg, send_group, ident): + def test_not_identified(self, ident, send_group, logmsg): '''The HTTPClient identifies automatically if needed''' - q = queue.Queue(1) - listener = StackifyListener(queue_=q, config=self.config) + listener = StackifyListener(queue_=Mock(), config=self.config) listener.handle(Mock()) self.assertTrue(ident.called) + @patch('stackify.handler.LogMsg') + @patch('stackify.handler.LogMsgGroup') + @patch('stackify.handler.HTTPClient.POST') + def test_send_group_if_needed(self, post, logmsggroup, logmsg): + '''The listener sends groups of messages''' + listener = StackifyListener(queue_=Mock(), max_batch=3, config=self.config) + listener.http.identified = True + + listener.handle(1) + self.assertFalse(post.called) + listener.handle(2) + self.assertFalse(post.called) + self.assertEqual(len(listener.messages), 2) + listener.handle(3) + self.assertTrue(post.called) + self.assertEqual(len(listener.messages), 0) + listener.handle(4) + self.assertEqual(post.call_count, 1) + self.assertEqual(len(listener.messages), 1) + if __name__=='__main__': unittest.main() From a0319d5ea3b4aa2c8b0d77fbedb3bf0a41231f9c Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 23:42:43 -0500 Subject: [PATCH 29/43] More test changes, format fix for logger --- stackify/handler.py | 6 +++--- tests/test_handler.py | 22 ++++++++++++++++++++-- tests/test_init.py | 3 ++- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/stackify/handler.py b/stackify/handler.py index b85067e..4396bcd 100644 --- a/stackify/handler.py +++ b/stackify/handler.py @@ -4,12 +4,12 @@ try: from logging.handlers import QueueHandler, QueueListener -except: +except: # pragma: no cover from stackify.handler_backport import QueueHandler, QueueListener try: import Queue as queue -except ImportError: +except ImportError: # pragma: no cover import queue from stackify import QUEUE_SIZE, API_URL, MAX_BATCH @@ -90,6 +90,6 @@ def stop(self): # send any remaining messages if self.messages: - logger.debug('{0} messages left on shutdown, uploading'.format(len(self.messages))) + logger.debug('%s messages left on shutdown, uploading', len(self.messages)) self.send_group() diff --git a/tests/test_handler.py b/tests/test_handler.py index dc42285..e4bb1cc 100755 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -11,9 +11,11 @@ except ImportError: import queue -import stackify.handler from stackify.handler import StackifyHandler, StackifyListener from stackify.application import ApiConfiguration +from stackify import internal_log + +import logging class TestHandler(unittest.TestCase): @@ -25,12 +27,14 @@ def test_queue_full(self): '''The queue should evict when full''' q = queue.Queue(1) handler = StackifyHandler(queue_=q, listener=Mock()) + internal_log.setLevel(logging.CRITICAL) # don't print warnings on overflow handler.enqueue('test1') handler.enqueue('test2') handler.enqueue('test3') self.assertEqual(q.qsize(), 1) self.assertEqual(q.get(), 'test3') + class TestListener(unittest.TestCase): ''' Test the StackifyListener class @@ -44,7 +48,7 @@ def setUp(self): api_url = 'test_apiurl') @patch('stackify.handler.LogMsg') - @patch.object(StackifyListener, 'send_group') + @patch('stackify.handler.StackifyListener.send_group') @patch('stackify.http.HTTPClient.identify_application') def test_not_identified(self, ident, send_group, logmsg): '''The HTTPClient identifies automatically if needed''' @@ -72,6 +76,20 @@ def test_send_group_if_needed(self, post, logmsggroup, logmsg): self.assertEqual(post.call_count, 1) self.assertEqual(len(listener.messages), 1) + @patch('stackify.handler.LogMsg') + @patch('stackify.handler.StackifyListener.send_group') + def test_clear_queue_shutdown(self, send_group, logmsg): + '''The listener sends the leftover messages on the queue when shutting down''' + listener = StackifyListener(queue_=Mock(), max_batch=3, config=self.config) + listener.http.identified = True + listener._thread = Mock() + + listener.handle(1) + listener.handle(2) + self.assertFalse(send_group.called) + listener.stop() + self.assertTrue(send_group.called) + if __name__=='__main__': unittest.main() diff --git a/tests/test_init.py b/tests/test_init.py index 0f7befd..f59fd0b 100755 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -89,7 +89,8 @@ def test_get_logger_defaults(self): self.assertEqual(config.api_url, stackify.API_URL) self.assertEqual(handler.listener.max_batch, stackify.MAX_BATCH) self.assertEqual(handler.queue.maxsize, stackify.QUEUE_SIZE) - self.assertEqual(logger.getEffectiveLevel(), logging.WARNING) + # nose will goof with the following assert + #self.assertEqual(logger.getEffectiveLevel(), logging.WARNING) def test_get_logger_reuse(self): '''Grabbing a logger twice results in the same logger''' From 7274070e889e8910d650821c09a9ef4768991957 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sat, 11 Oct 2014 23:49:37 -0500 Subject: [PATCH 30/43] Adding json object tests --- tests/test_formats.py | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 tests/test_formats.py diff --git a/tests/test_formats.py b/tests/test_formats.py new file mode 100755 index 0000000..b85ac18 --- /dev/null +++ b/tests/test_formats.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +""" +Test the stackify.formats module +""" + +import unittest +from mock import patch, Mock +import json + +from stackify.formats import JSONObject + +class TestJSONObject(unittest.TestCase): + ''' + Test the JSON serializer object + ''' + + def test_json_attributes(self): + '''Attributes are serialized in JSON''' + class MyTest(JSONObject): + def __init__(self): + self.a = '1' + self.b = 2 + self.c = False + result = MyTest().toJSON() + + self.assertEqual(json.loads(result), {'a': '1', 'b': 2, 'c': False}) + + def test_nested_json(self): + '''Nested classes are serialized in JSON''' + class MyParent(JSONObject): + def __init__(self, children): + self.children = children + + class MyChild(JSONObject): + def __init__(self, color): + self.color = color + + result = MyParent([MyChild('red'), MyChild('green')]).toJSON() + + self.assertEqual(json.loads(result), {'children': [{'color': 'red'}, {'color': 'green'}]}) + + +if __name__=='__main__': + unittest.main() + From 32d165a37ea29cf4041b52292224dc973c06a68a Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sun, 12 Oct 2014 00:07:10 -0500 Subject: [PATCH 31/43] Adding log tests --- tests/test_formats.py | 12 ++++++++++++ tests/test_log.py | 44 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100755 tests/test_log.py diff --git a/tests/test_formats.py b/tests/test_formats.py index b85ac18..2f7b9e9 100755 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -39,6 +39,18 @@ def __init__(self, color): self.assertEqual(json.loads(result), {'children': [{'color': 'red'}, {'color': 'green'}]}) + def test_nonempty_attributes(self): + '''Only nonempty attributes are serialized''' + class MyTest(JSONObject): + def __init__(self): + self.a = '1' + self.b = False + self.c = None + self.d = [] + result = MyTest().toJSON() + + self.assertEqual(json.loads(result), {'a': '1', 'b': False, 'd': []}) + if __name__=='__main__': unittest.main() diff --git a/tests/test_log.py b/tests/test_log.py new file mode 100755 index 0000000..751c932 --- /dev/null +++ b/tests/test_log.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +""" +Test the stackify.log module +""" + +import unittest +from mock import patch, Mock +import json + +import stackify.log + +from stackify.log import LogMsg +import logging +import json +import time +#logging.LogRecord('name','level','pathname','lineno','msg','args','exc_info','func') + + +class TestLogPopulate(unittest.TestCase): + ''' + Test populating log objects with data + ''' + + def test_record_to_error(self): + '''LogMsgs can load logger records''' + record = logging.LogRecord('name',logging.WARNING,'pathname',32, + 'message',(),(),'func') + record.my_extra = [1,2,3] + msg = LogMsg() + msg.from_record(record) + + curr_ms = time.time() * 1000 + + self.assertEqual(msg.SrcMethod, 'func') + self.assertEqual(msg.SrcLine, 32) + self.assertEqual(msg.Th, 'MainThread') + self.assertEqual(msg.Msg, 'message') + self.assertTrue(msg.EpochMs <= curr_ms) + self.assertEqual(json.loads(msg.data), {'my_extra':[1,2,3]}) + + +if __name__=='__main__': + unittest.main() + From be811908901578849ae5e95fab7360ee88edf766 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sun, 12 Oct 2014 00:15:43 -0500 Subject: [PATCH 32/43] Exception testing --- tests/test_log.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_log.py b/tests/test_log.py index 751c932..89cdf65 100755 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -6,6 +6,7 @@ import unittest from mock import patch, Mock import json +import sys import stackify.log @@ -38,6 +39,24 @@ def test_record_to_error(self): self.assertTrue(msg.EpochMs <= curr_ms) self.assertEqual(json.loads(msg.data), {'my_extra':[1,2,3]}) + def test_record_exception(self): + '''LogMsgs can parse exception information''' + try: + 1/0 + except: + record = logging.LogRecord('my exception',logging.WARNING,'somepath',12, + 'a thing happened',(),sys.exc_info()) + + msg = LogMsg() + msg.from_record(record) + + self.assertEqual(msg.Ex.OccurredEpochMillis, msg.EpochMs) + stack = msg.Ex.Error.StackTrace[0] + self.assertTrue(stack.CodeFileName.endswith('test_log.py')) + self.assertEqual(msg.Ex.Error.Message, 'integer division or modulo by zero') + self.assertEqual(msg.Ex.Error.ErrorType, 'ZeroDivisionError') + self.assertEqual(msg.Ex.Error.SourceMethod, 'test_record_exception') + if __name__=='__main__': unittest.main() From f55f28fad17af6ffee4001fa959d8975ec344cf0 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sun, 12 Oct 2014 00:33:05 -0500 Subject: [PATCH 33/43] Adding basic readme --- README.md | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 73 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b8ece0d..69ccbb9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,74 @@ -stackify-api-python -=================== - Stackify API for Python +======= + +[Stackify](https://stackify.com) support for Python programs. + +```python +import stackify + +logger = stackify.getLogger() + +try: + "Make it so, #" + 1 +except: + logger.exception("Can't add strings and numbers") +``` + +## Install +stackify-python can be installed through pip: +```bash +$ pip install -U stackify +``` + +You can also check out the repository and install with setuptools: +```bash +$ ./setup.py install +``` + +## Setup +Your Stackify setup information can be provided via environment variables. For example: +```bash +export STACKIFY_APPLICATION=MyApp +export STACKIFY_ENVIRONMENT=Dev +export STACKIFY_API_KEY=****** +``` + +You can optionally provide your API_URL: +```bash +export STACKIFY_API_URL='http://myapi.stackify.com' +``` + +These options can also be provided in your code: +```python +import stackify + +logger = stackify.getLogger(application="MyApp", environment="Dev", api_key=******) +logger.warning('Something happened') +``` + +## Usage + +Stackify can store extra data along with your log message: +```python +import stackify + +logger = stackify.getLogger() + +try: + user_string = raw_input("Enter a number: ") + print("You entered", int(user_string)) +except ValueError: + logger.exception('Bad input', extra={'user entered': user_string}) +``` + +## Testing +Run the test suite with setuptools: +```bash +$ ./setup.py test +``` + +You can obtain a coverage report with nose: +```bash +$ ./setup nosetests --with-coverage --cover-package=stackify +``` + From c91ca007feb4531c0fc39c72fa3f484207e92168 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Sun, 12 Oct 2014 00:44:13 -0500 Subject: [PATCH 34/43] More doc tweaks --- README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 69ccbb9..2e25568 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ except: logger.exception("Can't add strings and numbers") ``` -## Install +## Installation stackify-python can be installed through pip: ```bash $ pip install -U stackify @@ -25,7 +25,7 @@ You can also check out the repository and install with setuptools: $ ./setup.py install ``` -## Setup +## Configuration Your Stackify setup information can be provided via environment variables. For example: ```bash export STACKIFY_APPLICATION=MyApp @@ -33,11 +33,6 @@ export STACKIFY_ENVIRONMENT=Dev export STACKIFY_API_KEY=****** ``` -You can optionally provide your API_URL: -```bash -export STACKIFY_API_URL='http://myapi.stackify.com' -``` - These options can also be provided in your code: ```python import stackify @@ -48,6 +43,9 @@ logger.warning('Something happened') ## Usage +stackify-python handles uploads in batches of 100 messages at a time on another thread. +When your program exits, it will shut the thread down and upload the remaining messages. + Stackify can store extra data along with your log message: ```python import stackify From 0f2af2505d2156cc4c1cd89faa3b1264ae881505 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Mon, 13 Oct 2014 21:31:00 -0500 Subject: [PATCH 35/43] Fixing nose and adding a new test --- setup.py | 3 +- stackify/handler.py | 9 +++-- stackify/http.py | 21 +++++++--- tests/bases.py | 2 + tests/test_application.py | 1 - tests/test_formats.py | 1 - tests/test_handler.py | 25 ++++++++++-- tests/test_http.py | 83 +++++++++++++++++++++++++++++---------- tests/test_init.py | 1 - tests/test_log.py | 1 - 10 files changed, 109 insertions(+), 38 deletions(-) mode change 100755 => 100644 tests/test_application.py mode change 100755 => 100644 tests/test_formats.py mode change 100755 => 100644 tests/test_handler.py mode change 100755 => 100644 tests/test_http.py mode change 100755 => 100644 tests/test_init.py mode change 100755 => 100644 tests/test_log.py diff --git a/setup.py b/setup.py index 1759c28..1e87cb2 100755 --- a/setup.py +++ b/setup.py @@ -25,7 +25,8 @@ ], test_suite = 'tests', tests_requires = [ - 'mock>=1.0.1' + 'mock>=1.0.1', + 'nose==1.3.4' ] ) diff --git a/stackify/handler.py b/stackify/handler.py index 4396bcd..826317b 100644 --- a/stackify/handler.py +++ b/stackify/handler.py @@ -65,9 +65,8 @@ def __init__(self, queue_, max_batch=MAX_BATCH, config=None, **kwargs): self.http = HTTPClient(config) def handle(self, record): - logger = logging.getLogger(__name__) - if not self.http.identified: + logger = logging.getLogger(__name__) logger.debug('Identifying application') self.http.identify_application() @@ -80,7 +79,11 @@ def handle(self, record): def send_group(self): group = LogMsgGroup(self.messages) - self.http.POST('/Log/Save', group, True) + try: + self.http.send_log_group(group) + except: + logger = logging.getLogger(__name__) + logger.exception('Could not send %s log messages, discarding', len(self.messages)) del self.messages[:] def stop(self): diff --git a/stackify/http.py b/stackify/http.py index 6d6ce5e..374811f 100644 --- a/stackify/http.py +++ b/stackify/http.py @@ -40,8 +40,8 @@ def __init__(self, api_config): def POST(self, url, json_object, use_gzip=False): request_url = self.api_config.api_url + url - internal_log = logging.getLogger(__name__) - internal_log.debug('Request URL: %s', request_url) + logger = logging.getLogger(__name__) + logger.debug('Request URL: %s', request_url) headers = { 'Content-Type': 'application/json', @@ -51,7 +51,7 @@ def POST(self, url, json_object, use_gzip=False): try: payload_data = json_object.toJSON() - internal_log.debug('POST data: %s', payload_data) + logger.debug('POST data: %s', payload_data) if use_gzip: headers['Content-Encoding'] = 'gzip' @@ -60,14 +60,14 @@ def POST(self, url, json_object, use_gzip=False): response = requests.post(request_url, data=payload_data, headers=headers, timeout=READ_TIMEOUT) - internal_log.debug('Response: %s', response.text) + logger.debug('Response: %s', response.text) return response.json() except requests.exceptions.RequestException: - interal_log.exception('HTTP exception:') + logger.exception('HTTP exception') raise except ValueError as e: # could not read json response - internal_log.exception('Cannot decode JSON response') + logger.exception('Cannot decode JSON response') raise @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=10000) @@ -80,3 +80,12 @@ def identify_application(self): self.device_alias = result.get('DeviceAlias') self.identified = True + @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=10000) + def send_log_group(self, group): + group.CDID = self.device_id + group.CDAppID = self.device_app_id + group.AppNameID = self.app_name_id + group.ServerName = self.device_alias or self.environment_detail.deviceName + self.POST('/Log/Save', group, True) + + diff --git a/tests/bases.py b/tests/bases.py index cc97dc8..7c4e4b0 100644 --- a/tests/bases.py +++ b/tests/bases.py @@ -1,5 +1,6 @@ import os import unittest +import retrying class ClearEnvTest(unittest.TestCase): ''' @@ -26,3 +27,4 @@ def tearDown(self): for key, item in self.saved.items(): os.environ[key] = item del self.saved + diff --git a/tests/test_application.py b/tests/test_application.py old mode 100755 new mode 100644 index a3b5192..612e3a5 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Test the stackify.application module """ diff --git a/tests/test_formats.py b/tests/test_formats.py old mode 100755 new mode 100644 index 2f7b9e9..6c10b64 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Test the stackify.formats module """ diff --git a/tests/test_handler.py b/tests/test_handler.py old mode 100755 new mode 100644 index e4bb1cc..2d3d375 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Test the stackify.handler module """ @@ -49,12 +48,12 @@ def setUp(self): @patch('stackify.handler.LogMsg') @patch('stackify.handler.StackifyListener.send_group') - @patch('stackify.http.HTTPClient.identify_application') - def test_not_identified(self, ident, send_group, logmsg): + @patch('stackify.handler.HTTPClient.POST') + def test_not_identified(self, post, send_group, logmsg): '''The HTTPClient identifies automatically if needed''' listener = StackifyListener(queue_=Mock(), config=self.config) listener.handle(Mock()) - self.assertTrue(ident.called) + self.assertTrue(listener.http.identified) @patch('stackify.handler.LogMsg') @patch('stackify.handler.LogMsgGroup') @@ -90,6 +89,24 @@ def test_clear_queue_shutdown(self, send_group, logmsg): listener.stop() self.assertTrue(send_group.called) + @patch('stackify.handler.LogMsg') + @patch('stackify.handler.LogMsgGroup') + @patch('stackify.handler.HTTPClient.send_log_group') + def test_send_group_crash(self, send_log_group, logmsggroup, logmsg): + '''The listener drops messages after retrying''' + listener = StackifyListener(queue_=Mock(), max_batch=3, config=self.config) + listener.http.identified = True + + send_log_group.side_effect = Exception + + listener.handle(1) + listener.handle(2) + listener.handle(3) + self.assertEqual(len(listener.messages), 0) + listener.handle(4) + self.assertEqual(len(listener.messages), 1) + self.assertEqual(send_log_group.call_count, 1) + if __name__=='__main__': unittest.main() diff --git a/tests/test_http.py b/tests/test_http.py old mode 100755 new mode 100644 index d6b408f..9f5a2ca --- a/tests/test_http.py +++ b/tests/test_http.py @@ -1,30 +1,29 @@ -#!/usr/bin/env python """ Test the stackify.http module """ import unittest -import retrying from mock import patch, Mock -retrying_mock = Mock() - +import retrying import stackify.http +from stackify.log import LogMsgGroup from stackify.application import ApiConfiguration from stackify import READ_TIMEOUT - old_retry = retrying.retry -FAKE_RETRIES = 3 - -def fake_retry(*args, **kwargs): - kwargs['wait_exponential_max'] = 0 # no delay between retries - kwargs['stop_max_attempt_number'] = FAKE_RETRIES # stop after 3 tries - def inner(func): - return old_retry(*args, **kwargs)(func) - return inner +def fake_retry_decorator(retries): + def fake_retry(*args, **kwargs): + kwargs['wait_exponential_max'] = 0 # no delay between retries + kwargs['stop_max_attempt_number'] = retries + def inner(func): + print '!'*80 + print 'using retrying mock', kwargs + return old_retry(*args, **kwargs)(func) + return inner + return fake_retry class TestClient(unittest.TestCase): @@ -34,11 +33,18 @@ class TestClient(unittest.TestCase): @classmethod def setUpClass(cls): - retrying.retry = fake_retry + cls.FAKE_RETRIES = 3 + print '!'*80 + print 'another retry', retrying.retry + retrying.retry = fake_retry_decorator(cls.FAKE_RETRIES) + print 'patched retry', retrying.retry reload(stackify.http) + print 'func is', stackify.http.HTTPClient.POST @classmethod def tearDownClass(cls): + print '!'*80 + print 'teardown' reload(retrying) reload(stackify.http) @@ -49,6 +55,8 @@ def setUp(self): api_key = 'test_apikey', api_url = 'test_apiurl') + self.client = stackify.http.HTTPClient(self.config) + def test_logger_no_config(self): '''GZIP encoder works''' correct = list('\x1f\x8b\x08\x00 \x02\xff\xf3H\xcd\xc9\xc9\xd7Q(\xcf/\xcaIQ\x04\x00\xe6\xc6\xe6\xeb\r\x00\x00\x00') @@ -58,7 +66,7 @@ def test_logger_no_config(self): def test_identify_retrying(self): '''HTTP identify should retry''' - client = stackify.http.HTTPClient(self.config) + client = self.client class CustomException(Exception): pass crash = Mock(side_effect=CustomException) @@ -66,11 +74,11 @@ class CustomException(Exception): pass with patch.object(client, 'POST', crash): with self.assertRaises(CustomException): client.identify_application() - self.assertEqual(crash.call_count, FAKE_RETRIES) + self.assertEqual(crash.call_count, self.FAKE_RETRIES) def test_identify(self): '''HTTP identify should save results''' - client = stackify.http.HTTPClient(self.config) + client = self.client self.assertFalse(client.identified) result = { @@ -92,10 +100,46 @@ def test_identify(self): self.assertEqual(client.device_alias, '5') self.assertTrue(client.identified) + def test_send_log_group_retrying(self): + '''HTTP sending log groups should retry''' + client = self.client + + class CustomException(Exception): pass + crash = Mock(side_effect=CustomException) + + group = LogMsgGroup([]) + + with patch.object(client, 'POST', crash): + with self.assertRaises(CustomException): + client.send_log_group(group) + self.assertEqual(crash.call_count, self.FAKE_RETRIES) + + def test_send_log_group(self): + '''Send log group fills out info and submits ok''' + client = self.client + client.identified = True + + client.device_id = 'test_d_id' + client.device_app_id = 'test_dapp_id' + client.app_name_id = 'test_name_id' + client.device_alias = 'test_alias' + + group = LogMsgGroup([]) + + with patch.object(client, 'POST') as post: + client.send_log_group(group) + self.assertTrue(post.called) + + self.assertEqual(group.CDID, client.device_id) + self.assertEqual(group.CDAppID, client.device_app_id) + self.assertEqual(group.AppNameID, client.app_name_id) + self.assertEqual(group.ServerName, client.device_alias) + + @patch('requests.post') def test_post_arguments(self, post): '''HTTP post has correct headers''' - client = stackify.http.HTTPClient(self.config) + client = self.client payload = Mock() client.POST('url', payload) @@ -115,7 +159,7 @@ def test_post_arguments(self, post): @patch('requests.post') def test_post_gzip(self, post): '''HTTP post uses gzip if requested''' - client = stackify.http.HTTPClient(self.config) + client = self.client payload = Mock() payload.toJSON = Mock(return_value='1') gzip = Mock(side_effect=lambda x: x + '_gzipped') @@ -129,7 +173,6 @@ def test_post_gzip(self, post): self.assertEquals(kwargs['data'], '1_gzipped') - if __name__=='__main__': unittest.main() diff --git a/tests/test_init.py b/tests/test_init.py old mode 100755 new mode 100644 index f59fd0b..93485df --- a/tests/test_init.py +++ b/tests/test_init.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Test the stackify.__init__ setup functions """ diff --git a/tests/test_log.py b/tests/test_log.py old mode 100755 new mode 100644 index 89cdf65..f43b147 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Test the stackify.log module """ From b86db746828a845f9844a1a1c45ab21849916425 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Mon, 13 Oct 2014 21:50:29 -0500 Subject: [PATCH 36/43] Fix tests for python3 --- tests/test_application.py | 2 +- tests/test_http.py | 19 ++++++------------- tests/test_init.py | 2 +- tests/test_log.py | 10 +++++++--- 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/tests/test_application.py b/tests/test_application.py index 612e3a5..7e43248 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -5,7 +5,7 @@ import unittest from mock import patch import os -from bases import ClearEnvTest +from .bases import ClearEnvTest from stackify import API_URL from stackify.application import get_configuration diff --git a/tests/test_http.py b/tests/test_http.py index 9f5a2ca..225e8c9 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -4,6 +4,7 @@ import unittest from mock import patch, Mock +import imp import retrying import stackify.http @@ -19,8 +20,6 @@ def fake_retry(*args, **kwargs): kwargs['wait_exponential_max'] = 0 # no delay between retries kwargs['stop_max_attempt_number'] = retries def inner(func): - print '!'*80 - print 'using retrying mock', kwargs return old_retry(*args, **kwargs)(func) return inner return fake_retry @@ -34,19 +33,13 @@ class TestClient(unittest.TestCase): @classmethod def setUpClass(cls): cls.FAKE_RETRIES = 3 - print '!'*80 - print 'another retry', retrying.retry retrying.retry = fake_retry_decorator(cls.FAKE_RETRIES) - print 'patched retry', retrying.retry - reload(stackify.http) - print 'func is', stackify.http.HTTPClient.POST + imp.reload(stackify.http) @classmethod def tearDownClass(cls): - print '!'*80 - print 'teardown' - reload(retrying) - reload(stackify.http) + imp.reload(retrying) + imp.reload(stackify.http) def setUp(self): self.config = ApiConfiguration( @@ -59,9 +52,9 @@ def setUp(self): def test_logger_no_config(self): '''GZIP encoder works''' - correct = list('\x1f\x8b\x08\x00 \x02\xff\xf3H\xcd\xc9\xc9\xd7Q(\xcf/\xcaIQ\x04\x00\xe6\xc6\xe6\xeb\r\x00\x00\x00') + correct = list(b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xf3H\xcd\xc9\xc9\xd7Q(\xcf/\xcaIQ\x04\x00\xe6\xc6\xe6\xeb\r\x00\x00\x00') gzipped = list(stackify.http.gzip_compress('Hello, world!')) - gzipped[4:8] = ' ' # blank the mtime + gzipped[4:8] = b'\x00\x00\x00\x00' # blank the mtime self.assertEqual(gzipped, correct) def test_identify_retrying(self): diff --git a/tests/test_init.py b/tests/test_init.py index 93485df..3a24a8a 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -4,7 +4,7 @@ import unittest from mock import patch, Mock -from bases import ClearEnvTest +from .bases import ClearEnvTest import os import atexit diff --git a/tests/test_log.py b/tests/test_log.py index f43b147..0ce5741 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -40,8 +40,12 @@ def test_record_to_error(self): def test_record_exception(self): '''LogMsgs can parse exception information''' + class CustomException(Exception): + def __str__(self): + return 'My custom exception' + try: - 1/0 + raise CustomException() except: record = logging.LogRecord('my exception',logging.WARNING,'somepath',12, 'a thing happened',(),sys.exc_info()) @@ -52,8 +56,8 @@ def test_record_exception(self): self.assertEqual(msg.Ex.OccurredEpochMillis, msg.EpochMs) stack = msg.Ex.Error.StackTrace[0] self.assertTrue(stack.CodeFileName.endswith('test_log.py')) - self.assertEqual(msg.Ex.Error.Message, 'integer division or modulo by zero') - self.assertEqual(msg.Ex.Error.ErrorType, 'ZeroDivisionError') + self.assertEqual(msg.Ex.Error.Message, 'My custom exception') + self.assertEqual(msg.Ex.Error.ErrorType, 'CustomException') self.assertEqual(msg.Ex.Error.SourceMethod, 'test_record_exception') From d3b1eb2003ee90816864032ba4b51e49b17046f1 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Mon, 13 Oct 2014 22:04:03 -0500 Subject: [PATCH 37/43] Testing format and fixing format bug --- stackify/log.py | 1 + tests/test_log.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/stackify/log.py b/stackify/log.py index 129ea5e..32a4d3d 100644 --- a/stackify/log.py +++ b/stackify/log.py @@ -9,6 +9,7 @@ # this is used to separate builtin keys from user-specified keys RECORD_VARS = set(logging.LogRecord('','','','','','','','').__dict__.keys()) +RECORD_VARS.add('message') # message is saved on the record object by a Formatter sometimes class LogMsg(JSONObject): diff --git a/tests/test_log.py b/tests/test_log.py index 0ce5741..32f20a6 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -24,7 +24,7 @@ class TestLogPopulate(unittest.TestCase): def test_record_to_error(self): '''LogMsgs can load logger records''' record = logging.LogRecord('name',logging.WARNING,'pathname',32, - 'message',(),(),'func') + 'message %s',('here'),(),'func') record.my_extra = [1,2,3] msg = LogMsg() msg.from_record(record) @@ -34,7 +34,7 @@ def test_record_to_error(self): self.assertEqual(msg.SrcMethod, 'func') self.assertEqual(msg.SrcLine, 32) self.assertEqual(msg.Th, 'MainThread') - self.assertEqual(msg.Msg, 'message') + self.assertEqual(msg.Msg, 'message here') self.assertTrue(msg.EpochMs <= curr_ms) self.assertEqual(json.loads(msg.data), {'my_extra':[1,2,3]}) From ad22f11dea3232636e75138356056b494e80ab34 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Mon, 13 Oct 2014 22:07:22 -0500 Subject: [PATCH 38/43] Some cleanup --- README.md | 1 + stackify/__init__.py | 8 ++++---- stackify/log.py | 2 -- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2e25568..7a0d3da 100644 --- a/README.md +++ b/README.md @@ -69,4 +69,5 @@ You can obtain a coverage report with nose: ```bash $ ./setup nosetests --with-coverage --cover-package=stackify ``` +You might need to install the `nose` and `coverage` packages. diff --git a/stackify/__init__.py b/stackify/__init__.py index 0e538e7..d505477 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -30,8 +30,6 @@ logging.basicConfig() -internal_log = logging.getLogger(__name__) - from stackify.application import ApiConfiguration from stackify.http import HTTPClient @@ -55,7 +53,8 @@ def getLogger(name=None, auto_shutdown=True, **kwargs): logger = logging.getLogger(name) if not [isinstance(x, StackifyHandler) for x in logger.handlers]: - internal_log.debug('Creating handler for logger %s', name) + logger = logging.getLogger(__name__) + logger.debug('Creating handler for logger %s', name) handler = StackifyHandler(**kwargs) logger.addHandler(handler) @@ -74,7 +73,8 @@ def stopLogging(logger): Shut down the StackifyHandler on a given logger. This will block and wait for the queue to finish uploading. ''' - internal_log.debug('Shutting down all handlers') + logger = logging.getLogger(__name__) + logger.debug('Shutting down all handlers') for handler in getHandlers(logger): handler.listener.stop() diff --git a/stackify/log.py b/stackify/log.py index 32a4d3d..a0adc5f 100644 --- a/stackify/log.py +++ b/stackify/log.py @@ -17,12 +17,10 @@ def __init__(self): self.Msg = None self.data = None self.Ex = None # a StackifyError object - #self.Th = threading.current_thread().ident self.Th = None self.EpochMs = None self.Level = None self.TransID = None - # filename, line_number, function = internal_log.findCaller() self.SrcMethod = None self.SrcLine = None From 09c8ad5fd683f0d21711fdda7ccee8e37acdab13 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Mon, 13 Oct 2014 22:35:29 -0500 Subject: [PATCH 39/43] Clean up internal logging --- stackify/__init__.py | 26 ++++++++++++++------------ stackify/handler.py | 3 ++- tests/test_handler.py | 6 ++++-- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index d505477..86c961f 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -28,7 +28,11 @@ logging.NOTSET: 'NOTSET' } -logging.basicConfig() +class NullHandler(logging.Handler): + def emit(self, record): + pass + +logging.getLogger(__name__).addHandler(NullHandler()) from stackify.application import ApiConfiguration @@ -37,28 +41,26 @@ from stackify.handler import StackifyHandler -# TODO -# holds our listeners, since more than one handler can service -# the same listener -__listener_cache = {} - - -def getLogger(name=None, auto_shutdown=True, **kwargs): +def getLogger(name=None, auto_shutdown=True, basic_config=True, **kwargs): ''' Get a logger and attach a StackifyHandler if needed. ''' + if basic_config: + logging.basicConfig() + if not name: name = getCallerName(2) logger = logging.getLogger(name) if not [isinstance(x, StackifyHandler) for x in logger.handlers]: - logger = logging.getLogger(__name__) - logger.debug('Creating handler for logger %s', name) + internal_logger = logging.getLogger(__name__) + internal_logger.debug('Creating handler for logger %s', name) handler = StackifyHandler(**kwargs) logger.addHandler(handler) if auto_shutdown: + internal_logger.debug('Registering atexit callback') atexit.register(stopLogging, logger) if logger.getEffectiveLevel() == logging.NOTSET: @@ -73,8 +75,8 @@ def stopLogging(logger): Shut down the StackifyHandler on a given logger. This will block and wait for the queue to finish uploading. ''' - logger = logging.getLogger(__name__) - logger.debug('Shutting down all handlers') + internal_logger = logging.getLogger(__name__) + internal_logger.debug('Shutting down all handlers') for handler in getHandlers(logger): handler.listener.stop() diff --git a/stackify/handler.py b/stackify/handler.py index 826317b..3d652a8 100644 --- a/stackify/handler.py +++ b/stackify/handler.py @@ -28,6 +28,7 @@ class StackifyHandler(QueueHandler): def __init__(self, queue_=None, listener=None, **kwargs): if queue_ is None: queue_ = queue.Queue(QUEUE_SIZE) + logger = logging.getLogger(__name__) super(StackifyHandler, self).__init__(queue_) @@ -40,10 +41,10 @@ def enqueue(self, record): ''' Put a new record on the queue. If it's full, evict an item. ''' - logger = logging.getLogger(__name__) try: self.queue.put_nowait(record) except queue.Full: + logger = logging.getLogger(__name__) logger.warn('StackifyHandler queue is full, evicting oldest record') self.queue.get_nowait() self.queue.put_nowait(record) diff --git a/tests/test_handler.py b/tests/test_handler.py index 2d3d375..0ac201b 100644 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -12,7 +12,6 @@ from stackify.handler import StackifyHandler, StackifyListener from stackify.application import ApiConfiguration -from stackify import internal_log import logging @@ -26,7 +25,8 @@ def test_queue_full(self): '''The queue should evict when full''' q = queue.Queue(1) handler = StackifyHandler(queue_=q, listener=Mock()) - internal_log.setLevel(logging.CRITICAL) # don't print warnings on overflow + # don't print warnings on overflow, so mute stackify logger + logging.getLogger('stackify').propagate = False handler.enqueue('test1') handler.enqueue('test2') handler.enqueue('test3') @@ -45,6 +45,8 @@ def setUp(self): environment = 'test_environment', api_key = 'test_apikey', api_url = 'test_apiurl') + # don't print warnings on http crashes, so mute stackify logger + logging.getLogger('stackify').propagate = False @patch('stackify.handler.LogMsg') @patch('stackify.handler.StackifyListener.send_group') From 25269bb70fdb7fe43f5d4892eadf605d6fb35364 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Mon, 13 Oct 2014 22:40:21 -0500 Subject: [PATCH 40/43] More docs --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 7a0d3da..f29f16a 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,32 @@ except ValueError: logger.exception('Bad input', extra={'user entered': user_string}) ``` +You can also name your logger instead of using the automatically generated one: +```python +import stackify + +logger = stackify.getLogger('mymodule.myfile') +``` + +## Internal Logger + +This library has an internal logger it uses for debugging and messaging. +For example, if you want to enable debug messages: +```python +import logging + +logging.getLogger('stackify').setLevel(logging.DEBUG) +``` + +By default, it will enable the default logging settings via `logging.basicConfig()` +and print `WARNING` level messages and above. If you wish to set everything up yourself, +just pass `basic_config=False` in `getLogger`: +```python +import stackify + +logger = stackify.getLogger(basic_config=False) +``` + ## Testing Run the test suite with setuptools: ```bash From 27959ef75d35dbb848f578888956331db93ea119 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Wed, 15 Oct 2014 13:41:38 -0500 Subject: [PATCH 41/43] Some PEP8 cleanup --- stackify/__init__.py | 3 ++- stackify/application.py | 13 ++++++------- stackify/error.py | 18 ++++++++---------- stackify/formats.py | 3 +-- stackify/handler.py | 14 ++++++++------ stackify/handler_backport.py | 3 +-- stackify/http.py | 15 ++++++++------- stackify/log.py | 13 ++++++++----- 8 files changed, 42 insertions(+), 40 deletions(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index 86c961f..f581ecd 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -28,6 +28,7 @@ logging.NOTSET: 'NOTSET' } + class NullHandler(logging.Handler): def emit(self, record): pass @@ -70,6 +71,7 @@ def getLogger(name=None, auto_shutdown=True, basic_config=True, **kwargs): return logger + def stopLogging(logger): ''' Shut down the StackifyHandler on a given logger. This will block @@ -99,4 +101,3 @@ def getHandlers(logger): Return the StackifyHandlers on a given logger ''' return [x for x in logger.handlers if isinstance(x, StackifyHandler)] - diff --git a/stackify/application.py b/stackify/application.py index f5b7a87..69e8e71 100644 --- a/stackify/application.py +++ b/stackify/application.py @@ -32,14 +32,13 @@ def arg_or_env(name, args, default=None): if default: return default else: - raise NameError('You must specify the keyword argument {0} or environment variable {1}'.format( - name, env_name)) + raise NameError('You must specify the keyword argument {0} or ' + 'environment variable {1}'.format(name, env_name)) def get_configuration(**kwargs): return ApiConfiguration( - application = arg_or_env('application', kwargs), - environment = arg_or_env('environment', kwargs), - api_key = arg_or_env('api_key', kwargs), - api_url = arg_or_env('api_url', kwargs, API_URL)) - + application=arg_or_env('application', kwargs), + environment=arg_or_env('environment', kwargs), + api_key=arg_or_env('api_key', kwargs), + api_url=arg_or_env('api_url', kwargs, API_URL)) diff --git a/stackify/error.py b/stackify/error.py index 9aeb897..7ce5c42 100644 --- a/stackify/error.py +++ b/stackify/error.py @@ -7,13 +7,13 @@ class ErrorItem(JSONObject): def __init__(self): - self.Message = None # exception message - self.ErrorType = None # exception class name + self.Message = None # exception message + self.ErrorType = None # exception class name self.ErrorTypeCode = None - self.Data = None # custom data + self.Data = None # custom data self.SourceMethod = None - self.StackTrace = [] # array of TraceFrames - self.InnerError = None # cause? + self.StackTrace = [] # array of TraceFrames + self.InnerError = None # cause? def load_stack(self, exc_info=None): if not exc_info: @@ -58,10 +58,10 @@ def __init__(self): class StackifyError(JSONObject): def __init__(self): - self.EnvironmentDetail = None # environment detail object + self.EnvironmentDetail = None # environment detail object self.OccurredEpochMillis = None - self.Error = None # ErrorItem object - self.WebRequestDetail = None # WebRequestDetail object + self.Error = None # ErrorItem object + self.WebRequestDetail = None # WebRequestDetail object self.CustomerName = None self.UserName = None @@ -72,5 +72,3 @@ def load_exception(self, exc_info=None): def from_record(self, record): self.load_exception(record.exc_info) self.OccurredEpochMillis = int(record.created * 1000) - - diff --git a/stackify/formats.py b/stackify/formats.py index 08ef0d2..eb07b16 100644 --- a/stackify/formats.py +++ b/stackify/formats.py @@ -2,10 +2,9 @@ def nonempty(d): - return {k: v for k,v in d.items() if v is not None} + return {k: v for k, v in d.items() if v is not None} class JSONObject(object): def toJSON(self): return json.dumps(self, default=lambda x: nonempty(x.__dict__)) - diff --git a/stackify/handler.py b/stackify/handler.py index 3d652a8..20b44a4 100644 --- a/stackify/handler.py +++ b/stackify/handler.py @@ -4,12 +4,12 @@ try: from logging.handlers import QueueHandler, QueueListener -except: # pragma: no cover +except: # pragma: no cover from stackify.handler_backport import QueueHandler, QueueListener try: import Queue as queue -except ImportError: # pragma: no cover +except ImportError: # pragma: no cover import queue from stackify import QUEUE_SIZE, API_URL, MAX_BATCH @@ -45,7 +45,8 @@ def enqueue(self, record): self.queue.put_nowait(record) except queue.Full: logger = logging.getLogger(__name__) - logger.warn('StackifyHandler queue is full, evicting oldest record') + logger.warn('StackifyHandler queue is full, ' + 'evicting oldest record') self.queue.get_nowait() self.queue.put_nowait(record) @@ -84,7 +85,8 @@ def send_group(self): self.http.send_log_group(group) except: logger = logging.getLogger(__name__) - logger.exception('Could not send %s log messages, discarding', len(self.messages)) + logger.exception('Could not send %s log messages, discarding', + len(self.messages)) del self.messages[:] def stop(self): @@ -94,6 +96,6 @@ def stop(self): # send any remaining messages if self.messages: - logger.debug('%s messages left on shutdown, uploading', len(self.messages)) + logger.debug('%s messages left on shutdown, uploading', + len(self.messages)) self.send_group() - diff --git a/stackify/handler_backport.py b/stackify/handler_backport.py index 6e8b705..84b31d7 100644 --- a/stackify/handler_backport.py +++ b/stackify/handler_backport.py @@ -126,7 +126,7 @@ def start(self): t.setDaemon(True) t.start() - def prepare(self , record): + def prepare(self, record): """ Prepare a record for handling. @@ -201,4 +201,3 @@ def stop(self): self.enqueue_sentinel() self._thread.join() self._thread = None - diff --git a/stackify/http.py b/stackify/http.py index 374811f..82b8ca7 100644 --- a/stackify/http.py +++ b/stackify/http.py @@ -9,12 +9,12 @@ try: from StringIO import StringIO except: - pass # python 3, we use a new function in gzip + pass # python 3, we use a new function in gzip def gzip_compress(data): if hasattr(gzip, 'compress'): - return gzip.compress(bytes(data, 'utf-8')) # python 3 + return gzip.compress(bytes(data, 'utf-8')) # python 3 else: s = StringIO() g = gzip.GzipFile(fileobj=s, mode='w') @@ -58,8 +58,9 @@ def POST(self, url, json_object, use_gzip=False): payload_data = gzip_compress(payload_data) response = requests.post(request_url, - data=payload_data, headers=headers, - timeout=READ_TIMEOUT) + data=payload_data, + headers=headers, + timeout=READ_TIMEOUT) logger.debug('Response: %s', response.text) return response.json() except requests.exceptions.RequestException: @@ -85,7 +86,7 @@ def send_log_group(self, group): group.CDID = self.device_id group.CDAppID = self.device_app_id group.AppNameID = self.app_name_id - group.ServerName = self.device_alias or self.environment_detail.deviceName + group.ServerName = self.device_alias + if not group.ServerName: + group.ServerName = self.environment_detail.deviceName self.POST('/Log/Save', group, True) - - diff --git a/stackify/log.py b/stackify/log.py index a0adc5f..c51f5f3 100644 --- a/stackify/log.py +++ b/stackify/log.py @@ -8,15 +8,18 @@ # this is used to separate builtin keys from user-specified keys -RECORD_VARS = set(logging.LogRecord('','','','','','','','').__dict__.keys()) -RECORD_VARS.add('message') # message is saved on the record object by a Formatter sometimes +RECORD_VARS = set(logging.LogRecord('', '', '', '', + '', '', '', '').__dict__.keys()) + +# the "message" attribute is saved on the record object by a Formatter +RECORD_VARS.add('message') class LogMsg(JSONObject): def __init__(self): self.Msg = None self.data = None - self.Ex = None # a StackifyError object + self.Ex = None # a StackifyError object self.Th = None self.EpochMs = None self.Level = None @@ -33,7 +36,8 @@ def from_record(self, record): self.SrcLine = record.lineno # check for user-specified keys - data = { k:v for k,v in record.__dict__.items() if k not in RECORD_VARS } + data = {k: v for k, v in record.__dict__.items() + if k not in RECORD_VARS} if data: self.data = json.dumps(data, default=lambda x: x.__dict__) @@ -51,4 +55,3 @@ def __init__(self, msgs, logger=None): self.CDAppID = None self.AppNameID = None self.ServerName = None - From dcba89d4a8dc4c10fbf7709e162bbad72d9967fb Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Wed, 15 Oct 2014 13:50:53 -0500 Subject: [PATCH 42/43] Some docstring tweaks --- stackify/__init__.py | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index f581ecd..f276a43 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -43,8 +43,28 @@ def emit(self, record): def getLogger(name=None, auto_shutdown=True, basic_config=True, **kwargs): - ''' - Get a logger and attach a StackifyHandler if needed. + '''Get a logger and attach a StackifyHandler if needed. + + You can pass this function keyword arguments for Stackify configuration. + If they are omitted you can specify them through environment variables: + * STACKIFY_API_KEY + * STACKIFY_APPLICATION + * STACKIFY_ENVIRONMENT + * STACKIFY_API_URL + + Args: + name: The name of the logger (or None to automatically make one) + auto_shutdown: Register an atexit hook to shut down logging + basic_config: Set up with logging.basicConfig() for regular logging + + Optional Args: + api_key: Your Stackify API key + application: The name of your Stackify application + environment: The Stackfiy environment to log to + api_url: An optional API url if required + + Returns: + A logger instance with Stackify handler and listener attached. ''' if basic_config: logging.basicConfig() @@ -73,7 +93,8 @@ def getLogger(name=None, auto_shutdown=True, basic_config=True, **kwargs): def stopLogging(logger): - ''' + '''Stop logging on the Stackify handler. + Shut down the StackifyHandler on a given logger. This will block and wait for the queue to finish uploading. ''' @@ -84,9 +105,7 @@ def stopLogging(logger): def getCallerName(levels=1): - ''' - Gets the name of the module calling this function - ''' + '''Gets the name of the module calling this function''' try: frame = inspect.stack()[levels] module = inspect.getmodule(frame[0]) @@ -97,7 +116,5 @@ def getCallerName(levels=1): def getHandlers(logger): - ''' - Return the StackifyHandlers on a given logger - ''' + '''Return the StackifyHandlers on a given logger''' return [x for x in logger.handlers if isinstance(x, StackifyHandler)] From 97ccaf58fb3f028a5de0f252af611ebb5cf2cf84 Mon Sep 17 00:00:00 2001 From: Matt Thompson Date: Wed, 15 Oct 2014 14:06:13 -0500 Subject: [PATCH 43/43] Fixing setup for pypi submission --- setup.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 1e87cb2..d19d42b 100755 --- a/setup.py +++ b/setup.py @@ -3,6 +3,13 @@ import re import ast +try: + from pypandoc import convert + read_md = lambda f: convert(f, 'rst') +except ImportError: + print('warning: pypandoc module not found, could not convert Markdown to RST') + read_md = lambda f: open(f).read() + version_re = re.compile(r'__version__\s+=\s+(.*)') with open('stackify/__init__.py') as f: @@ -15,10 +22,13 @@ author = 'Matthew Thompson', author_email = 'chameleonator@gmail.com', packages = ['stackify'], - url = 'https://github.com/stackify/stackify-python', + url = 'https://github.com/stackify/stackify-api-python', license = open('LICENSE.txt').readline(), description = 'Stackify API for Python', - long_description = open('README.md').read(), + long_description = read_md('README.md'), + download_url = 'https://github.com/stackify/stackify-api-python/tarball/0.0.1', + keywords = ['logging', 'stackify', 'exception'], + classifiers=["Programming Language :: Python"], install_requires = [ 'retrying>=1.2.3', 'requests>=2.4.1'