From acaa34ee1920c8a66b81fa37d540f798abfd75d0 Mon Sep 17 00:00:00 2001 From: Elpedio Adoptante Jr Date: Fri, 15 Nov 2019 23:06:58 +0800 Subject: [PATCH 1/3] API - Add AGENT_HTTP Transport --- stackify/constants.py | 4 +- stackify/handler.py | 4 +- stackify/transport/__init__.py | 102 +++++-------------------- stackify/transport/agent/__init__.py | 47 +++++++++++- stackify/transport/application.py | 18 ++++- stackify/transport/base.py | 31 ++++++++ stackify/transport/default/__init__.py | 28 ++++++- stackify/utils.py | 4 +- tests/bases.py | 1 + tests/test_handler.py | 12 +-- tests/test_init.py | 6 +- tests/transport/agent/test_init.py | 87 +++++++++++++++++++++ tests/transport/default/test_init.py | 47 ++++++++++++ tests/transport/test_application.py | 73 ++++++++++++++++++ tests/transport/test_base.py | 84 ++++++++++++++++++++ tests/transport/test_init.py | 34 ++++----- 16 files changed, 464 insertions(+), 118 deletions(-) create mode 100644 stackify/transport/base.py create mode 100644 tests/transport/agent/test_init.py create mode 100644 tests/transport/default/test_init.py create mode 100644 tests/transport/test_base.py diff --git a/stackify/constants.py b/stackify/constants.py index 1d94e4a..3dce49e 100644 --- a/stackify/constants.py +++ b/stackify/constants.py @@ -7,8 +7,9 @@ # using `%2F` instead of `/` as per package documentation DEFAULT_SOCKET_FILE = '%2Fusr%2Flocal%2Fstackify%2Fstackify.sock' +DEFAULT_HTTP_ENDPOINT = 'https://localhost:10601' SOCKET_URL = 'http+unix://' + DEFAULT_SOCKET_FILE -SOCKET_LOG_URL = '/log' +AGENT_LOG_URL = '/log' API_REQUEST_INTERVAL_IN_SEC = 30 @@ -36,3 +37,4 @@ TRANSPORT_TYPE_DEFAULT = 'default' TRANSPORT_TYPE_AGENT_SOCKET = 'agent_socket' +TRANSPORT_TYPE_AGENT_HTTP = 'agent_http' diff --git a/stackify/handler.py b/stackify/handler.py index 9478f36..f957f8e 100644 --- a/stackify/handler.py +++ b/stackify/handler.py @@ -16,7 +16,7 @@ from stackify.constants import MAX_BATCH from stackify.constants import QUEUE_SIZE from stackify.timer import RepeatedTimer -from stackify.transport import Transport +from stackify.transport import configure_transport internal_logger = logging.getLogger(__name__) @@ -70,7 +70,7 @@ def __init__(self, queue_, max_batch=MAX_BATCH, config=None, **kwargs): self.max_batch = max_batch self.messages = [] - self.transport = Transport(config, **kwargs) + self.transport = configure_transport(config, **kwargs) self.timer = RepeatedTimer(API_REQUEST_INTERVAL_IN_SEC, self.send_group) self._started = False diff --git a/stackify/transport/__init__.py b/stackify/transport/__init__.py index 633bb2e..dce78c4 100644 --- a/stackify/transport/__init__.py +++ b/stackify/transport/__init__.py @@ -1,17 +1,14 @@ import logging -from stackify.constants import LOG_SAVE_URL -from stackify.constants import SOCKET_LOG_URL +from stackify.constants import TRANSPORT_TYPE_AGENT_HTTP from stackify.constants import TRANSPORT_TYPE_AGENT_SOCKET from stackify.constants import TRANSPORT_TYPE_DEFAULT -from stackify.transport.agent import AgentSocket -from stackify.transport.agent.message import Log -from stackify.transport.agent.message import LogGroup +from stackify.transport.agent import AgentSocketTransport +from stackify.transport.agent import AgentHTTPTransport from stackify.transport.application import get_configuration from stackify.transport.application import EnvironmentDetail -from stackify.transport.default import HTTPClient -from stackify.transport.default.log import LogMsg -from stackify.transport.default.log import LogMsgGroup +from stackify.transport.default import DefaultSocketTransport + internal_logger = logging.getLogger(__name__) @@ -28,89 +25,28 @@ class TransportTypes(object): DEFAULT = TRANSPORT_TYPE_DEFAULT AGENT_SOCKET = TRANSPORT_TYPE_AGENT_SOCKET + AGENT_HTTP = TRANSPORT_TYPE_AGENT_HTTP @classmethod def get_transport(self, api_config=None, env_details=None): # determine which transport to use depening on users config if api_config.transport == self.AGENT_SOCKET: internal_logger.debug('Setting Agent Socket Transport.') - api_config.transport = self.AGENT_SOCKET - return AgentSocket() + return AgentSocketTransport(api_config, env_details) + + if api_config.transport == self.AGENT_HTTP: + internal_logger.debug('Setting Agent HTTP Transport.') + return AgentHTTPTransport(api_config, env_details) internal_logger.debug('Setting Default Transport.') api_config.transport = self.DEFAULT - return HTTPClient(api_config, env_details) - - @classmethod - def create_message(self, record, api_config, env_details): - # create message depending on which transport - if api_config.transport == self.AGENT_SOCKET: - return Log(record, api_config, env_details).get_object() - - msg = LogMsg() - msg.from_record(record) - return msg - - @classmethod - def create_group_message(self, messages, api_config, env_details): - # create group message depending on which transport - if api_config.transport == self.AGENT_SOCKET: - return LogGroup(messages, api_config, env_details).get_object() - - return LogMsgGroup(messages) - - @classmethod - def get_log_url(self, api_config): - # return log url depending on which transport - if api_config.transport == self.AGENT_SOCKET: - return api_config.socket_url + SOCKET_LOG_URL - - return LOG_SAVE_URL - - @classmethod - def prepare_message(self, api_config, message): - # convert message depending on which transport - if api_config.transport == self.AGENT_SOCKET: - return message.SerializeToString() - - return message - - -class Transport(object): - """ - Transport base class - """ - - def __init__(self, config=None, **kwargs): - self.api_config = config or get_configuration(**kwargs) - self.env_details = EnvironmentDetail(self.api_config) - self._transport = TransportTypes.get_transport( - self.api_config, - self.env_details, - ) - - def create_message(self, record): - # create message from record - return TransportTypes.create_message( - record, - self.api_config, - self.env_details, - ) + return DefaultSocketTransport(api_config, env_details) - def create_group_message(self, messages): - # create group message from list of records - return TransportTypes.create_group_message( - messages, - self.api_config, - self.env_details, - ) - def send(self, group_message): - # send group message - try: - self._transport.send( - TransportTypes.get_log_url(self.api_config), - TransportTypes.prepare_message(self.api_config, group_message), - ) - except Exception as e: - internal_logger.error('Request error: {}'.format(e)) +def configure_transport(config=None, **kwargs): + api_config = config or get_configuration(**kwargs) + env_details = EnvironmentDetail(api_config) + return TransportTypes.get_transport( + api_config, + env_details, + ) diff --git a/stackify/transport/agent/__init__.py b/stackify/transport/agent/__init__.py index 4ba8479..83990b4 100644 --- a/stackify/transport/agent/__init__.py +++ b/stackify/transport/agent/__init__.py @@ -1 +1,46 @@ -from .agent_socket import AgentSocket # noqa +import logging +import requests +import retrying + +from stackify.constants import AGENT_LOG_URL +from stackify.transport.agent.agent_socket import AgentSocket +from stackify.transport.base import AgentBaseTransport + +internal_logger = logging.getLogger(__name__) + + +class AgentSocketTransport(AgentBaseTransport): + _transport = None + + def __init__(self, api_config, env_details): + super(AgentSocketTransport, self).__init__(api_config, env_details) + self._transport = AgentSocket() + + def send(self, group_message): + self._transport.send( + self._api_config.socket_url + AGENT_LOG_URL, + group_message.SerializeToString(), + ) + + +class AgentHTTPTransport(AgentBaseTransport): + def __init__(self, api_config, env_details): + super(AgentHTTPTransport, self).__init__(api_config, env_details) + + def send(self, group_message): + self._post( + self._api_config.http_endpoint + AGENT_LOG_URL, + group_message.SerializeToString(), + ) + + @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=32000) + def _post(self, url, payload): + headers = { + 'Content-Type': 'application/x-protobuf', + } + print('url: {}'.format(url)) + try: + return requests.post(url, payload, headers=headers) + except Exception as e: + internal_logger.debug('HTTP transport exception: {}.'.format(e)) + raise diff --git a/stackify/transport/application.py b/stackify/transport/application.py index 5813f88..ac11947 100644 --- a/stackify/transport/application.py +++ b/stackify/transport/application.py @@ -3,7 +3,9 @@ from stackify.utils import arg_or_env from stackify.constants import API_URL +from stackify.constants import DEFAULT_HTTP_ENDPOINT from stackify.constants import SOCKET_URL +from stackify.constants import TRANSPORT_TYPE_AGENT_HTTP from stackify.constants import TRANSPORT_TYPE_AGENT_SOCKET from stackify.constants import TRANSPORT_TYPE_DEFAULT from stackify.transport.default.formats import JSONObject @@ -27,12 +29,22 @@ class ApiConfiguration: ApiConfiguration class that stores application configurations """ - def __init__(self, api_key, application, environment, api_url=API_URL, socket_url=SOCKET_URL, transport=None): + def __init__( + self, + api_key, + application, + environment, + api_url=API_URL, + socket_url=SOCKET_URL, + transport=None, + http_endpoint=DEFAULT_HTTP_ENDPOINT, + ): self.api_key = api_key self.api_url = api_url self.application = application self.environment = environment self.socket_url = socket_url + self.http_endpoint = http_endpoint self.transport = transport @@ -43,7 +55,8 @@ def get_configuration(**kwargs): """ transport = arg_or_env('transport', kwargs, TRANSPORT_TYPE_DEFAULT) - if transport == TRANSPORT_TYPE_AGENT_SOCKET: + + if transport in [TRANSPORT_TYPE_AGENT_SOCKET, TRANSPORT_TYPE_AGENT_HTTP]: api_key = arg_or_env('api_key', kwargs, '') else: api_key = arg_or_env('api_key', kwargs) @@ -54,5 +67,6 @@ def get_configuration(**kwargs): api_key=api_key, api_url=arg_or_env('api_url', kwargs, API_URL), socket_url=arg_or_env('socket_url', kwargs, SOCKET_URL), + http_endpoint=arg_or_env('http_endpoint', kwargs, DEFAULT_HTTP_ENDPOINT, env_key='STACKIFY_TRANSPORT_HTTP_ENDPOINT'), transport=transport, ) diff --git a/stackify/transport/base.py b/stackify/transport/base.py new file mode 100644 index 0000000..15dad5f --- /dev/null +++ b/stackify/transport/base.py @@ -0,0 +1,31 @@ +from stackify.transport.agent.message import Log +from stackify.transport.agent.message import LogGroup + + +class BaseTransport(object): + def __init__(self, api_config, env_details): + self._api_config = api_config + self._env_details = env_details + + def create_message(self, record): + raise NotImplementedError + + def create_group_message(self, messages): + raise NotImplementedError + + def send(self, group_message): + raise NotImplementedError + + +class AgentBaseTransport(BaseTransport): + def __init__(self, api_config, env_details): + super(AgentBaseTransport, self).__init__(api_config, env_details) + + def create_message(self, record): + return Log(record, self._api_config, self._env_details).get_object() + + def create_group_message(self, messages): + return LogGroup(messages, self._api_config, self._env_details).get_object() + + def send(self, group_message): + raise NotImplementedError diff --git a/stackify/transport/default/__init__.py b/stackify/transport/default/__init__.py index 419fdfd..1790052 100644 --- a/stackify/transport/default/__init__.py +++ b/stackify/transport/default/__init__.py @@ -1 +1,27 @@ -from .http import HTTPClient # noqa +from stackify.constants import LOG_SAVE_URL +from stackify.transport.base import BaseTransport +from stackify.transport.default.http import HTTPClient +from stackify.transport.default.log import LogMsg +from stackify.transport.default.log import LogMsgGroup + + +class DefaultSocketTransport(BaseTransport): + _transport = None + + def __init__(self, api_config, env_details): + super(DefaultSocketTransport, self).__init__(api_config, env_details) + self._transport = HTTPClient(api_config, env_details) + + def create_message(self, record): + msg = LogMsg() + msg.from_record(record) + return msg + + def create_group_message(self, messages): + return LogMsgGroup(messages) + + def send(self, group_message): + self._transport.send( + LOG_SAVE_URL, + group_message, + ) diff --git a/stackify/utils.py b/stackify/utils.py index 5eb8c1e..3a6ba54 100644 --- a/stackify/utils.py +++ b/stackify/utils.py @@ -1,8 +1,8 @@ import os -def arg_or_env(name, args, default=None): - env_name = 'STACKIFY_{0}'.format(name.upper()) +def arg_or_env(name, args, default=None, env_key=None): + env_name = env_key or 'STACKIFY_{0}'.format(name.upper()) try: value = args.get(name) if not value: diff --git a/tests/bases.py b/tests/bases.py index d495602..69992c0 100644 --- a/tests/bases.py +++ b/tests/bases.py @@ -20,6 +20,7 @@ def setUp(self): 'STACKIFY_API_KEY', 'STACKIFY_API_URL', 'STACKIFY_TRANSPORT', + 'STACKIFY_TRANSPORT_HTTP_ENDPOINT', ] self.saved = {} for key in to_save: diff --git a/tests/test_handler.py b/tests/test_handler.py index d915a14..793cbd7 100644 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -49,7 +49,7 @@ def setUp(self): # don't print warnings on http crashes, so mute stackify logger logging.getLogger('stackify').propagate = False - @patch('stackify.transport.Transport.create_message') + @patch('stackify.transport.default.DefaultSocketTransport.create_message') @patch('stackify.transport.default.http.HTTPClient.POST') def test_not_identified(self, post, logmsg): '''The HTTPClient identifies automatically if needed''' @@ -58,8 +58,8 @@ def test_not_identified(self, post, logmsg): listener.send_group() self.assertTrue(listener.transport._transport.identified) - @patch('stackify.transport.Transport.create_message') - @patch('stackify.transport.Transport.create_group_message') + @patch('stackify.transport.default.DefaultSocketTransport.create_message') + @patch('stackify.transport.default.DefaultSocketTransport.create_group_message') @patch('stackify.transport.default.http.HTTPClient.POST') def test_send_group_if_needed(self, post, logmsggroup, logmsg): '''The listener sends groups of messages''' @@ -78,7 +78,7 @@ def test_send_group_if_needed(self, post, logmsggroup, logmsg): self.assertEqual(post.call_count, 1) self.assertEqual(len(listener.messages), 1) - @patch('stackify.transport.Transport.create_message') + @patch('stackify.transport.default.DefaultSocketTransport.create_message') @patch('stackify.handler.StackifyListener.send_group') def test_clear_queue_shutdown(self, send_group, logmsg): '''The listener sends the leftover messages on the queue when shutting down''' @@ -92,8 +92,8 @@ def test_clear_queue_shutdown(self, send_group, logmsg): listener.stop() self.assertTrue(send_group.called) - @patch('stackify.transport.Transport.create_message') - @patch('stackify.transport.Transport.create_group_message') + @patch('stackify.transport.default.DefaultSocketTransport.create_message') + @patch('stackify.transport.default.DefaultSocketTransport.create_group_message') @patch('stackify.transport.default.http.HTTPClient.send_log_group') def test_send_group_crash(self, send_log_group, logmsggroup, logmsg): '''The listener drops messages after retrying''' diff --git a/tests/test_init.py b/tests/test_init.py index 547f109..7e69676 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -43,7 +43,7 @@ def test_logger_no_config(self): logger = stackify.getLogger(auto_shutdown=False) self.loggers.append(logger) - config = logger.handlers[0].listener.transport.api_config + config = logger.handlers[0].listener.transport._api_config self.assertEqual(config.application, 'test2_appname') self.assertEqual(config.environment, 'test2_environment') @@ -55,7 +55,7 @@ def test_logger_api_config(self): logger = stackify.getLogger(config=self.config, auto_shutdown=False) self.loggers.append(logger) - config = logger.handlers[0].listener.transport.api_config + config = logger.handlers[0].listener.transport._api_config self.assertEqual(config.application, 'test_appname') self.assertEqual(config.environment, 'test_environment') @@ -79,7 +79,7 @@ def test_get_logger_defaults(self): self.loggers.append(logger) handler = logger.handlers[0] - config = handler.listener.transport.api_config + config = handler.listener.transport._api_config self.assertEqual(logger.name, 'tests.test_init') self.assertEqual(config.api_url, stackify.constants.API_URL) diff --git a/tests/transport/agent/test_init.py b/tests/transport/agent/test_init.py new file mode 100644 index 0000000..e3bea23 --- /dev/null +++ b/tests/transport/agent/test_init.py @@ -0,0 +1,87 @@ +import logging +from unittest import TestCase +from mock import patch + +from stackify.protos import stackify_agent_pb2 +from stackify.transport import application +from stackify.transport.agent import AgentHTTPTransport +from stackify.transport.agent import AgentSocketTransport + + +class AgentSocketTransportTest(TestCase): + def setUp(self): + self.config = application.ApiConfiguration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + self.env_details = application.EnvironmentDetail(self.config) + self.agent_socket_transport = AgentSocketTransport(self.config, self.env_details) + + def test_init(self): + assert self.agent_socket_transport._api_config == self.config + assert self.agent_socket_transport._env_details == self.env_details + + def test_create_message(self): + message = self.agent_socket_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + assert isinstance(message, stackify_agent_pb2.LogGroup.Log) + + def test_create_group_message(self): + message = self.agent_socket_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + group_message = self.agent_socket_transport.create_group_message([message]) + + assert isinstance(group_message, stackify_agent_pb2.LogGroup) + + @patch('requests_unixsocket.Session.post') + def test_send(self, mock_post): + message = self.agent_socket_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = self.agent_socket_transport.create_group_message([message]) + + self.agent_socket_transport.send(group_message) + + assert mock_post.called + assert mock_post.call_args_list[0][0][0] == 'http+unix://%2Fusr%2Flocal%2Fstackify%2Fstackify.sock/log' + assert mock_post.call_args_list[0][1]['headers']['Content-Type'] == 'application/x-protobuf' + + +class AgentHTTPTransportTest(TestCase): + + def setUp(self): + self.config = application.ApiConfiguration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + self.env_details = application.EnvironmentDetail(self.config) + self.agent_http_transport = AgentHTTPTransport(self.config, self.env_details) + + def test_init(self): + assert self.agent_http_transport._api_config == self.config + assert self.agent_http_transport._env_details == self.env_details + + def test_create_message(self): + message = self.agent_http_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + assert isinstance(message, stackify_agent_pb2.LogGroup.Log) + + def test_create_group_message(self): + message = self.agent_http_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + group_message = self.agent_http_transport.create_group_message([message]) + + assert isinstance(group_message, stackify_agent_pb2.LogGroup) + + @patch('requests.post') + def test_send(self, mock_post): + message = self.agent_http_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = self.agent_http_transport.create_group_message([message]) + + self.agent_http_transport.send(group_message) + + assert mock_post.called + assert mock_post.call_args_list[0][0][0] == 'https://localhost:10601/log' + assert mock_post.call_args_list[0][1]['headers']['Content-Type'] == 'application/x-protobuf' diff --git a/tests/transport/default/test_init.py b/tests/transport/default/test_init.py new file mode 100644 index 0000000..ca9b51a --- /dev/null +++ b/tests/transport/default/test_init.py @@ -0,0 +1,47 @@ +import logging +from unittest import TestCase +from mock import patch + +from stackify.transport import application +from stackify.transport.default import DefaultSocketTransport +from stackify.transport.default.log import LogMsg +from stackify.transport.default.log import LogMsgGroup + + +class AgentSocketTransportTest(TestCase): + def setUp(self): + self.config = application.ApiConfiguration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + self.env_details = application.EnvironmentDetail(self.config) + self.default_transport = DefaultSocketTransport(self.config, self.env_details) + + def test_init(self): + assert self.default_transport._api_config == self.config + assert self.default_transport._env_details == self.env_details + + def test_create_message(self): + message = self.default_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + assert isinstance(message, LogMsg) + + def test_create_group_message(self): + message = self.default_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + group_message = self.default_transport.create_group_message([message]) + + assert isinstance(group_message, LogMsgGroup) + + @patch('requests.post') + def test_send(self, mock_post): + message = self.default_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = self.default_transport.create_group_message([message]) + + self.default_transport.send(group_message) + + assert mock_post.called + assert mock_post.call_args_list[0][0][0] == 'test_apiurl/Metrics/IdentifyApp' + assert mock_post.call_args_list[0][1]['headers']['Content-Type'] == 'application/json' diff --git a/tests/transport/test_application.py b/tests/transport/test_application.py index 901872f..7b4dc62 100644 --- a/tests/transport/test_application.py +++ b/tests/transport/test_application.py @@ -3,10 +3,15 @@ """ import unittest +import os from mock import patch from tests.bases import ClearEnvTest from stackify.constants import API_URL +from stackify.constants import DEFAULT_HTTP_ENDPOINT +from stackify.constants import TRANSPORT_TYPE_AGENT_HTTP +from stackify.constants import TRANSPORT_TYPE_AGENT_SOCKET +from stackify.constants import TRANSPORT_TYPE_DEFAULT from stackify.transport.application import get_configuration @@ -145,5 +150,73 @@ def test_api_key_is_not_required_on_agent_socket_transport(self): self.assertEqual(config.transport, 'agent_socket') +class ConfigEnvironmentVariableTest(ClearEnvTest): + def test_transport_environment_variable_default(self): + os.environ["STACKIFY_TRANSPORT"] = "default" + + config = get_configuration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + + assert config.transport == TRANSPORT_TYPE_DEFAULT + + del os.environ["STACKIFY_TRANSPORT"] + + def test_transport_environment_variable_agent_socket(self): + os.environ["STACKIFY_TRANSPORT"] = "agent_socket" + + config = get_configuration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + + assert config.transport == TRANSPORT_TYPE_AGENT_SOCKET + + del os.environ["STACKIFY_TRANSPORT"] + + def test_transport_environment_variable_agent_http(self): + os.environ["STACKIFY_TRANSPORT"] = "agent_http" + + config = get_configuration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + + assert config.transport == TRANSPORT_TYPE_AGENT_HTTP + + del os.environ["STACKIFY_TRANSPORT"] + + def test_http_endpoint_environment_variable_default(self): + config = get_configuration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + + assert config.http_endpoint == DEFAULT_HTTP_ENDPOINT + + def test_http_endpoint_environment_variable(self): + os.environ["STACKIFY_TRANSPORT_HTTP_ENDPOINT"] = "test" + + config = get_configuration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + + assert config.http_endpoint == "test" + + del os.environ["STACKIFY_TRANSPORT_HTTP_ENDPOINT"] + + if __name__ == '__main__': unittest.main() diff --git a/tests/transport/test_base.py b/tests/transport/test_base.py new file mode 100644 index 0000000..6a4cba4 --- /dev/null +++ b/tests/transport/test_base.py @@ -0,0 +1,84 @@ +import logging + +from tests.bases import ClearEnvTest +from stackify.protos import stackify_agent_pb2 +from stackify.transport.base import BaseTransport +from stackify.transport.base import AgentBaseTransport +from stackify.transport.application import EnvironmentDetail +from stackify.transport.application import get_configuration + +CONFIG = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', +} + + +class BaseTransportTest(ClearEnvTest): + def test_init(self): + api_config = 'test_api_config' + env_details = 'test_env_details' + + base_transport = BaseTransport(api_config, env_details) + + assert base_transport._api_config == api_config + assert base_transport._env_details == env_details + + def test_create_message(self): + api_config = 'test_api_config' + env_details = 'test_env_details' + base_transport = BaseTransport(api_config, env_details) + + self.assertRaises(NotImplementedError, base_transport.create_message, 'test_record') + + def test_create_group_message(self): + api_config = 'test_api_config' + env_details = 'test_env_details' + base_transport = BaseTransport(api_config, env_details) + + self.assertRaises(NotImplementedError, base_transport.create_group_message, 'test_messages') + + def test_send(self): + api_config = 'test_api_config' + env_details = 'test_env_details' + base_transport = BaseTransport(api_config, env_details) + + self.assertRaises(NotImplementedError, base_transport.send, 'test_group_message') + + +class AgentBaseTransportTest(ClearEnvTest): + def test_init(self): + api_config = 'test_api_config' + env_details = 'test_env_details' + + agent_base_transport = AgentBaseTransport(api_config, env_details) + + assert agent_base_transport._api_config == api_config + assert agent_base_transport._env_details == env_details + + def test_create_message(self): + api_config = 'test_api_config' + env_details = 'test_env_details' + agent_base_transport = AgentBaseTransport(api_config, env_details) + + message = agent_base_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + assert isinstance(message, stackify_agent_pb2.LogGroup.Log) + + def test_create_group_message(self): + api_config = get_configuration(**CONFIG) + env_details = EnvironmentDetail(api_config) + agent_base_transport = AgentBaseTransport(api_config, env_details) + + message = agent_base_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = agent_base_transport.create_group_message([message]) + + assert isinstance(group_message, stackify_agent_pb2.LogGroup) + + def test_send(self): + api_config = 'test_api_config' + env_details = 'test_env_details' + agent_base_transport = AgentBaseTransport(api_config, env_details) + + self.assertRaises(NotImplementedError, agent_base_transport.send, 'test_group_message') diff --git a/tests/transport/test_init.py b/tests/transport/test_init.py index 5910a2c..00b74bb 100644 --- a/tests/transport/test_init.py +++ b/tests/transport/test_init.py @@ -3,9 +3,9 @@ from tests.bases import ClearEnvTest from stackify.protos import stackify_agent_pb2 -from stackify.transport import Transport -from stackify.transport.agent import AgentSocket -from stackify.transport.default import HTTPClient +from stackify.transport import configure_transport +from stackify.transport.agent import AgentSocketTransport +from stackify.transport.default import DefaultSocketTransport from stackify.transport.default.log import LogMsg from stackify.transport.default.log import LogMsgGroup @@ -20,9 +20,9 @@ def test_invalid_transport(self): 'transport': 'invalid', } - transport = Transport(**config) + transport = configure_transport(**config) - assert isinstance(transport._transport, HTTPClient) + assert isinstance(transport, DefaultSocketTransport) def test_default_transport(self): config = { @@ -32,9 +32,9 @@ def test_default_transport(self): 'api_url': 'test_apiurl', } - transport = Transport(**config) + transport = configure_transport(**config) - assert isinstance(transport._transport, HTTPClient) + assert isinstance(transport, DefaultSocketTransport) def test_default_create_message(self): config = { @@ -44,7 +44,7 @@ def test_default_create_message(self): 'api_url': 'test_apiurl', } - transport = Transport(**config) + transport = configure_transport(**config) message = transport.create_message(logging.makeLogRecord({'mgs': 'message'})) assert isinstance(message, LogMsg) @@ -57,7 +57,7 @@ def test_default_create_group_message(self): 'api_url': 'test_apiurl', } - transport = Transport(**config) + transport = configure_transport(**config) message = transport.create_message(logging.makeLogRecord({'mgs': 'message'})) group_message = transport.create_group_message([message]) @@ -72,7 +72,7 @@ def test_default_send_url(self, mock_send): 'api_url': 'test_apiurl', } - transport = Transport(**config) + transport = configure_transport(**config) message = transport.create_message(logging.makeLogRecord({'mgs': 'message'})) group_message = transport.create_group_message([message]) transport.send(group_message) @@ -90,9 +90,9 @@ def test_agent_socket_transport(self): 'transport': 'agent_socket', } - transport = Transport(**config) + transport = configure_transport(**config) - assert isinstance(transport._transport, AgentSocket) + assert isinstance(transport, AgentSocketTransport) def test_agent_socket_create_message(self): config = { @@ -104,10 +104,10 @@ def test_agent_socket_create_message(self): 'transport': 'agent_socket', } - transport = Transport(**config) + transport = configure_transport(**config) message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) - isinstance(message, stackify_agent_pb2.LogGroup.Log) + assert isinstance(message, stackify_agent_pb2.LogGroup.Log) def test_agent_socket_create_group_message(self): config = { @@ -119,7 +119,7 @@ def test_agent_socket_create_group_message(self): 'transport': 'agent_socket', } - transport = Transport(**config) + transport = configure_transport(**config) message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) group_message = transport.create_group_message([message]) @@ -136,7 +136,7 @@ def test_agent_socket_send_url(self, mock_send): 'transport': 'agent_socket', } - transport = Transport(**config) + transport = configure_transport(**config) message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) group_message = transport.create_group_message([message]) transport.send(group_message) @@ -154,7 +154,7 @@ def test_agent_socket_send_url_default(self, mock_send): 'transport': 'agent_socket', } - transport = Transport(**config) + transport = configure_transport(**config) message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) group_message = transport.create_group_message([message]) transport.send(group_message) From 54176738b949511dfdf2dc2684dcd3b3614b0e60 Mon Sep 17 00:00:00 2001 From: Elpedio Adoptante Jr Date: Mon, 18 Nov 2019 17:19:48 +0800 Subject: [PATCH 2/3] added documentation and rename --- stackify/transport/__init__.py | 6 ++++-- stackify/transport/agent/__init__.py | 7 +++++++ stackify/transport/base.py | 6 ++++++ stackify/transport/default/__init__.py | 7 +++++-- tests/test_handler.py | 12 ++++++------ tests/transport/default/test_init.py | 4 ++-- tests/transport/test_init.py | 6 +++--- 7 files changed, 33 insertions(+), 15 deletions(-) diff --git a/stackify/transport/__init__.py b/stackify/transport/__init__.py index dce78c4..1b57e5c 100644 --- a/stackify/transport/__init__.py +++ b/stackify/transport/__init__.py @@ -7,7 +7,7 @@ from stackify.transport.agent import AgentHTTPTransport from stackify.transport.application import get_configuration from stackify.transport.application import EnvironmentDetail -from stackify.transport.default import DefaultSocketTransport +from stackify.transport.default import DefaultTransport internal_logger = logging.getLogger(__name__) @@ -21,6 +21,7 @@ class TransportTypes(object): Types: * DEFAULT - HTTP transport that will directly send logs to the Platform * AGENT_SOCKET - HTTP warapped Unix Socket Domain that will send logs to the StackifyAgent + * AGENT_HTTP - HTTP transport that will send logs to the Agent using HTTP requests """ DEFAULT = TRANSPORT_TYPE_DEFAULT @@ -40,10 +41,11 @@ def get_transport(self, api_config=None, env_details=None): internal_logger.debug('Setting Default Transport.') api_config.transport = self.DEFAULT - return DefaultSocketTransport(api_config, env_details) + return DefaultTransport(api_config, env_details) def configure_transport(config=None, **kwargs): + # return which transport to use depending on users input api_config = config or get_configuration(**kwargs) env_details = EnvironmentDetail(api_config) return TransportTypes.get_transport( diff --git a/stackify/transport/agent/__init__.py b/stackify/transport/agent/__init__.py index 83990b4..10313b0 100644 --- a/stackify/transport/agent/__init__.py +++ b/stackify/transport/agent/__init__.py @@ -10,6 +10,9 @@ class AgentSocketTransport(AgentBaseTransport): + """ + Agent Socket Transport handles sending of logs using Unix Socket Domain + """ _transport = None def __init__(self, api_config, env_details): @@ -24,6 +27,10 @@ def send(self, group_message): class AgentHTTPTransport(AgentBaseTransport): + """ + Agent HTTP Transport handles sending of logs using HTTP requests + """ + def __init__(self, api_config, env_details): super(AgentHTTPTransport, self).__init__(api_config, env_details) diff --git a/stackify/transport/base.py b/stackify/transport/base.py index 15dad5f..45988cf 100644 --- a/stackify/transport/base.py +++ b/stackify/transport/base.py @@ -3,6 +3,9 @@ class BaseTransport(object): + """ + Base Transport + """ def __init__(self, api_config, env_details): self._api_config = api_config self._env_details = env_details @@ -18,6 +21,9 @@ def send(self, group_message): class AgentBaseTransport(BaseTransport): + """ + Base Transport for protobuf data + """ def __init__(self, api_config, env_details): super(AgentBaseTransport, self).__init__(api_config, env_details) diff --git a/stackify/transport/default/__init__.py b/stackify/transport/default/__init__.py index 1790052..e4711a9 100644 --- a/stackify/transport/default/__init__.py +++ b/stackify/transport/default/__init__.py @@ -5,11 +5,14 @@ from stackify.transport.default.log import LogMsgGroup -class DefaultSocketTransport(BaseTransport): +class DefaultTransport(BaseTransport): + """ + Default Transport handles sending of logs directly to platform + """ _transport = None def __init__(self, api_config, env_details): - super(DefaultSocketTransport, self).__init__(api_config, env_details) + super(DefaultTransport, self).__init__(api_config, env_details) self._transport = HTTPClient(api_config, env_details) def create_message(self, record): diff --git a/tests/test_handler.py b/tests/test_handler.py index 793cbd7..2f35d05 100644 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -49,7 +49,7 @@ def setUp(self): # don't print warnings on http crashes, so mute stackify logger logging.getLogger('stackify').propagate = False - @patch('stackify.transport.default.DefaultSocketTransport.create_message') + @patch('stackify.transport.default.DefaultTransport.create_message') @patch('stackify.transport.default.http.HTTPClient.POST') def test_not_identified(self, post, logmsg): '''The HTTPClient identifies automatically if needed''' @@ -58,8 +58,8 @@ def test_not_identified(self, post, logmsg): listener.send_group() self.assertTrue(listener.transport._transport.identified) - @patch('stackify.transport.default.DefaultSocketTransport.create_message') - @patch('stackify.transport.default.DefaultSocketTransport.create_group_message') + @patch('stackify.transport.default.DefaultTransport.create_message') + @patch('stackify.transport.default.DefaultTransport.create_group_message') @patch('stackify.transport.default.http.HTTPClient.POST') def test_send_group_if_needed(self, post, logmsggroup, logmsg): '''The listener sends groups of messages''' @@ -78,7 +78,7 @@ def test_send_group_if_needed(self, post, logmsggroup, logmsg): self.assertEqual(post.call_count, 1) self.assertEqual(len(listener.messages), 1) - @patch('stackify.transport.default.DefaultSocketTransport.create_message') + @patch('stackify.transport.default.DefaultTransport.create_message') @patch('stackify.handler.StackifyListener.send_group') def test_clear_queue_shutdown(self, send_group, logmsg): '''The listener sends the leftover messages on the queue when shutting down''' @@ -92,8 +92,8 @@ def test_clear_queue_shutdown(self, send_group, logmsg): listener.stop() self.assertTrue(send_group.called) - @patch('stackify.transport.default.DefaultSocketTransport.create_message') - @patch('stackify.transport.default.DefaultSocketTransport.create_group_message') + @patch('stackify.transport.default.DefaultTransport.create_message') + @patch('stackify.transport.default.DefaultTransport.create_group_message') @patch('stackify.transport.default.http.HTTPClient.send_log_group') def test_send_group_crash(self, send_log_group, logmsggroup, logmsg): '''The listener drops messages after retrying''' diff --git a/tests/transport/default/test_init.py b/tests/transport/default/test_init.py index ca9b51a..f3e410f 100644 --- a/tests/transport/default/test_init.py +++ b/tests/transport/default/test_init.py @@ -3,7 +3,7 @@ from mock import patch from stackify.transport import application -from stackify.transport.default import DefaultSocketTransport +from stackify.transport.default import DefaultTransport from stackify.transport.default.log import LogMsg from stackify.transport.default.log import LogMsgGroup @@ -17,7 +17,7 @@ def setUp(self): api_url='test_apiurl', ) self.env_details = application.EnvironmentDetail(self.config) - self.default_transport = DefaultSocketTransport(self.config, self.env_details) + self.default_transport = DefaultTransport(self.config, self.env_details) def test_init(self): assert self.default_transport._api_config == self.config diff --git a/tests/transport/test_init.py b/tests/transport/test_init.py index 00b74bb..d1054dc 100644 --- a/tests/transport/test_init.py +++ b/tests/transport/test_init.py @@ -5,7 +5,7 @@ from stackify.protos import stackify_agent_pb2 from stackify.transport import configure_transport from stackify.transport.agent import AgentSocketTransport -from stackify.transport.default import DefaultSocketTransport +from stackify.transport.default import DefaultTransport from stackify.transport.default.log import LogMsg from stackify.transport.default.log import LogMsgGroup @@ -22,7 +22,7 @@ def test_invalid_transport(self): transport = configure_transport(**config) - assert isinstance(transport, DefaultSocketTransport) + assert isinstance(transport, DefaultTransport) def test_default_transport(self): config = { @@ -34,7 +34,7 @@ def test_default_transport(self): transport = configure_transport(**config) - assert isinstance(transport, DefaultSocketTransport) + assert isinstance(transport, DefaultTransport) def test_default_create_message(self): config = { From 0445e69e14c3503d0c6360d018e3d49903754320 Mon Sep 17 00:00:00 2001 From: Elpedio Adoptante Jr Date: Mon, 18 Nov 2019 23:25:25 +0800 Subject: [PATCH 3/3] cleanup - refactor, added tests --- requirements.txt | 1 + setup.cfg | 5 +- stackify/timer.py | 2 + stackify/transport/agent/__init__.py | 35 ++------ stackify/transport/agent/agent_http.py | 26 ++++++ stackify/transport/agent/agent_socket.py | 2 +- stackify/transport/agent/message.py | 2 +- stackify/transport/base.py | 5 +- test.sh | 47 +++++++++++ tests/bases.py | 101 ++++++++++++++++++++++- tests/test_timer.py | 32 +++++++ tests/transport/agent/test_init.py | 24 ++++++ tests/transport/agent/test_message.py | 21 ++++- tests/transport/test_base.py | 2 +- tests/transport/test_init.py | 87 +++++++++++++++++++ 15 files changed, 355 insertions(+), 37 deletions(-) create mode 100644 stackify/transport/agent/agent_http.py create mode 100755 test.sh create mode 100644 tests/test_timer.py diff --git a/requirements.txt b/requirements.txt index 911174c..54d51bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +flake8 mock==2.0.0 protobuf==3.9.1 pytest==4.3.0 diff --git a/setup.cfg b/setup.cfg index 51d679c..470d4da 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,13 +11,16 @@ exclude = README.md, LICENSE.md, requirements.txt, + *protos*, [coverage:run] include = stackify/* omit = - *tests* + *tests*, + *handler_backport.py, + *protos*, [tool:pytest] python_files=tests.py test.py test_*.py *_test.py tests_*.py *_tests.py diff --git a/stackify/timer.py b/stackify/timer.py index 0bb0c0a..5dd6c13 100644 --- a/stackify/timer.py +++ b/stackify/timer.py @@ -27,10 +27,12 @@ def _time(self): def start(self): if not self._started: + self._started = True self.thread.setDaemon(True) self.thread.start() def stop(self): if self._started: + self._started = False self.event.set() self.thread.join() diff --git a/stackify/transport/agent/__init__.py b/stackify/transport/agent/__init__.py index 10313b0..7ebc987 100644 --- a/stackify/transport/agent/__init__.py +++ b/stackify/transport/agent/__init__.py @@ -1,9 +1,8 @@ import logging -import requests -import retrying from stackify.constants import AGENT_LOG_URL -from stackify.transport.agent.agent_socket import AgentSocket +from stackify.transport.agent import agent_http +from stackify.transport.agent import agent_socket from stackify.transport.base import AgentBaseTransport internal_logger = logging.getLogger(__name__) @@ -13,17 +12,11 @@ class AgentSocketTransport(AgentBaseTransport): """ Agent Socket Transport handles sending of logs using Unix Socket Domain """ - _transport = None def __init__(self, api_config, env_details): super(AgentSocketTransport, self).__init__(api_config, env_details) - self._transport = AgentSocket() - - def send(self, group_message): - self._transport.send( - self._api_config.socket_url + AGENT_LOG_URL, - group_message.SerializeToString(), - ) + self.url = api_config.socket_url + AGENT_LOG_URL + self._transport = agent_socket.AgentSocket() class AgentHTTPTransport(AgentBaseTransport): @@ -33,21 +26,5 @@ class AgentHTTPTransport(AgentBaseTransport): def __init__(self, api_config, env_details): super(AgentHTTPTransport, self).__init__(api_config, env_details) - - def send(self, group_message): - self._post( - self._api_config.http_endpoint + AGENT_LOG_URL, - group_message.SerializeToString(), - ) - - @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=32000) - def _post(self, url, payload): - headers = { - 'Content-Type': 'application/x-protobuf', - } - print('url: {}'.format(url)) - try: - return requests.post(url, payload, headers=headers) - except Exception as e: - internal_logger.debug('HTTP transport exception: {}.'.format(e)) - raise + self.url = api_config.http_endpoint + AGENT_LOG_URL + self._transport = agent_http.AgentHTTP() diff --git a/stackify/transport/agent/agent_http.py b/stackify/transport/agent/agent_http.py new file mode 100644 index 0000000..1950594 --- /dev/null +++ b/stackify/transport/agent/agent_http.py @@ -0,0 +1,26 @@ +import logging +import requests +import retrying + +internal_logger = logging.getLogger(__name__) + + +class AgentHTTP(object): + """ + AgentHTTP class that handles HTTP post requests + """ + + def _post(self, url, payload): + headers = { + 'Content-Type': 'application/x-protobuf', + } + try: + return requests.post(url, payload, headers=headers) + except Exception as e: + internal_logger.debug('HTTP transport exception: {}.'.format(e)) + raise + + @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=32000) + def send(self, url, payload): + # send payload through socket domain using _post method + return self._post(url, payload) diff --git a/stackify/transport/agent/agent_socket.py b/stackify/transport/agent/agent_socket.py index aef8adc..af0db72 100644 --- a/stackify/transport/agent/agent_socket.py +++ b/stackify/transport/agent/agent_socket.py @@ -42,4 +42,4 @@ def _post(self, url, payload): @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=32000) def send(self, url, payload): # send payload through socket domain using _post method - self._post(url, payload) + return self._post(url, payload) diff --git a/stackify/transport/agent/message.py b/stackify/transport/agent/message.py index 444fb5f..3d6c340 100644 --- a/stackify/transport/agent/message.py +++ b/stackify/transport/agent/message.py @@ -122,4 +122,4 @@ def __init__(self, messages, api_config, env_details, logger=None): log_group.application_location = env_details.appLocation log_group.logger = logger or __name__ log_group.platform = 'python' - log_group.logs.MergeFrom(messages) + log_group.logs.extend(messages) diff --git a/stackify/transport/base.py b/stackify/transport/base.py index 45988cf..15c897e 100644 --- a/stackify/transport/base.py +++ b/stackify/transport/base.py @@ -24,6 +24,9 @@ class AgentBaseTransport(BaseTransport): """ Base Transport for protobuf data """ + url = None + _transport = None + def __init__(self, api_config, env_details): super(AgentBaseTransport, self).__init__(api_config, env_details) @@ -34,4 +37,4 @@ def create_group_message(self, messages): return LogGroup(messages, self._api_config, self._env_details).get_object() def send(self, group_message): - raise NotImplementedError + return self._transport.send(self.url, group_message.SerializeToString()) diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..8967876 --- /dev/null +++ b/test.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -e + + +VERSIONS=('2.7' '3.4' '3.5' '3.6' '3.7' '3.8') + + +function runFlake8() { + echo '<--------------------------------------------->' + # run flake8 and exit on error + # it will check the code base against coding style (PEP8) and programming errors + echo "Running flake8..." + flake8 || { echo 'You increased the number of flak8 errors'; exit 1; } +} + +function runPyTest() { + echo '<--------------------------------------------->' + python_version=${1} + test_venv="venv_test_${python_version//.}" + + echo "Creating virtualenv ${test_venv}..." + virtualenv -p python${python_version} ${test_venv} + + echo "Activating virtualenv ${test_venv}..." + source ${test_venv}/bin/activate + + echo 'Installing dependencies...' + pip install -r requirements.txt + + runFlake8 + + echo 'Running pytest...' + py.test + + echo "Deactivating virtualenv..." + deactivate +} + +echo 'Removing all existing virtualenv' +rm -rf venv_test_* | true + +for i in "${VERSIONS[@]}" +do + runPyTest ${i} +done + +echo 'Done' diff --git a/tests/bases.py b/tests/bases.py index 69992c0..0b11585 100644 --- a/tests/bases.py +++ b/tests/bases.py @@ -1,9 +1,11 @@ +import collections +import logging import os import retrying import unittest - old_retry = retrying.retry +_LoggingWatcher = collections.namedtuple("_LoggingWatcher", ["records", "output"]) class ClearEnvTest(unittest.TestCase): @@ -44,3 +46,100 @@ def inner(func): return old_retry(*args, **kwargs)(func) return inner return fake_retry + + +class _BaseTestCaseContext(object): + + def __init__(self, test_case): + self.test_case = test_case + + def _raiseFailure(self, standardMsg): + msg = self.test_case._formatMessage(self.msg, standardMsg) + raise self.test_case.failureException(msg) + + +class _CapturingHandler(logging.Handler): + """ + A logging handler capturing all (raw and formatted) logging output. + """ + + def __init__(self): + logging.Handler.__init__(self) + self.watcher = _LoggingWatcher([], []) + + def flush(self): + pass + + def emit(self, record): + self.watcher.records.append(record) + msg = self.format(record) + self.watcher.output.append(msg) + + +class _AssertLogsContext(_BaseTestCaseContext): + """A context manager used to implement TestCase.assertLogs().""" + + LOGGING_FORMAT = "%(levelname)s:%(name)s:%(message)s" + + def __init__(self, test_case, logger_name, level): + _BaseTestCaseContext.__init__(self, test_case) + self.logger_name = logger_name + if level: + self.level = logging._levelNames.get(level, level) + else: + self.level = logging.INFO + self.msg = None + + def __enter__(self): + if isinstance(self.logger_name, logging.Logger): + logger = self.logger = self.logger_name + else: + logger = self.logger = logging.getLogger(self.logger_name) + formatter = logging.Formatter(self.LOGGING_FORMAT) + handler = _CapturingHandler() + handler.setFormatter(formatter) + self.watcher = handler.watcher + self.old_handlers = logger.handlers[:] + self.old_level = logger.level + self.old_propagate = logger.propagate + logger.handlers = [handler] + logger.setLevel(self.level) + logger.propagate = False + return handler.watcher + + def __exit__(self, exc_type, exc_value, tb): + self.logger.handlers = self.old_handlers + self.logger.propagate = self.old_propagate + self.logger.setLevel(self.old_level) + if exc_type is not None: + # let unexpected exceptions pass through + return False + if len(self.watcher.records) == 0: + self._raiseFailure( + "no logs of level {} or higher triggered on {}" + .format(logging.getLevelName(self.level), self.logger.name)) + + +class LogTestCase(unittest.TestCase): + + def assertLogs(self, logger=None, level=None): + """Fail unless a log message of level *level* or higher is emitted + on *logger_name* or its children. If omitted, *level* defaults to + INFO and *logger* defaults to the root logger. + + This method must be used as a context manager, and will yield + a recording object with two attributes: `output` and `records`. + At the end of the context manager, the `output` attribute will + be a list of the matching formatted log messages and the + `records` attribute will be a list of the corresponding LogRecord + objects. + + Example:: + + with self.assertLogs('foo', level='INFO') as cm: + logging.getLogger('foo').info('first message') + logging.getLogger('foo.bar').error('second message') + self.assertEqual(cm.output, ['INFO:foo:first message', + 'ERROR:foo.bar:second message']) + """ + return _AssertLogsContext(self, logger, level) diff --git a/tests/test_timer.py b/tests/test_timer.py new file mode 100644 index 0000000..09d8f9e --- /dev/null +++ b/tests/test_timer.py @@ -0,0 +1,32 @@ +import time +from unittest import TestCase +try: + from unittest import mock +except Exception: + import mock + +from stackify.timer import RepeatedTimer + + +class TimerTest(TestCase): + def setUp(self): + self.function_mock = mock.Mock() + self.timer = RepeatedTimer(0.1, self.function_mock) + self.timer.start() + + def shutDown(self): + self.timer.stop() + + def test_start(self): + assert self.timer._started + + def test_stop(self): + self.timer.stop() + + assert not self.timer._started + + def test_timer(self): + time.sleep(0.3) + + assert self.function_mock.called + assert self.function_mock.call_count >= 2 diff --git a/tests/transport/agent/test_init.py b/tests/transport/agent/test_init.py index e3bea23..369af49 100644 --- a/tests/transport/agent/test_init.py +++ b/tests/transport/agent/test_init.py @@ -1,11 +1,15 @@ +import imp import logging +import retrying from unittest import TestCase from mock import patch +import stackify from stackify.protos import stackify_agent_pb2 from stackify.transport import application from stackify.transport.agent import AgentHTTPTransport from stackify.transport.agent import AgentSocketTransport +from tests.bases import fake_retry_decorator class AgentSocketTransportTest(TestCase): @@ -48,6 +52,15 @@ def test_send(self, mock_post): class AgentHTTPTransportTest(TestCase): + @classmethod + def setUpClass(self): + retrying.retry = fake_retry_decorator(3) + imp.reload(stackify.transport.agent.agent_http) + + @classmethod + def tearDownClass(self): + imp.reload(retrying) + imp.reload(stackify.transport.agent.agent_http) def setUp(self): self.config = application.ApiConfiguration( @@ -85,3 +98,14 @@ def test_send(self, mock_post): assert mock_post.called assert mock_post.call_args_list[0][0][0] == 'https://localhost:10601/log' assert mock_post.call_args_list[0][1]['headers']['Content-Type'] == 'application/x-protobuf' + + @patch('requests.post') + def test_retry(self, mock_post): + mock_post.side_effect = Exception('some error') + message = self.agent_http_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = self.agent_http_transport.create_group_message([message]) + + with self.assertRaises(Exception): + self.agent_http_transport.send(group_message) + + assert mock_post.call_count == 3 diff --git a/tests/transport/agent/test_message.py b/tests/transport/agent/test_message.py index 7f73886..690f3a0 100644 --- a/tests/transport/agent/test_message.py +++ b/tests/transport/agent/test_message.py @@ -1,11 +1,16 @@ import logging -from unittest import TestCase +import sys from stackify.protos import stackify_agent_pb2 from stackify.transport import application from stackify.transport.agent.message import Log from stackify.transport.agent.message import LogGroup +if sys.version_info[0] == 2: + from tests.bases import LogTestCase as TestCase +else: + from unittest import TestCase + class TestLog(TestCase): def setUp(self): @@ -25,6 +30,18 @@ def test_get_object(self): assert isinstance(log, stackify_agent_pb2.LogGroup.Log) + def test_get_object_with_trans_id_and_log_id(self): + with self.assertLogs('foo', level='INFO') as logging_watcher: + logging.getLogger('foo').info('some log') + record = logging_watcher.records[0] + record.trans_id = 'trans_id' + record.log_id = 'log_id' + + log = Log(record, self.config, self.env_details).get_object() + + assert log.id == 'log_id' + assert log.transaction_id == 'trans_id' + def test_info_log_details(self): with self.assertLogs('foo', level='INFO') as logging_watcher: logging.getLogger('foo').info('some log') @@ -83,7 +100,7 @@ def test_info_exception_details(self): assert environment_detail.application_location == self.env_details.appLocation error_item = error.error_item - assert error_item.message == 'division by zero' + assert error_item.message in ['integer division or modulo by zero', 'division by zero'] assert error_item.error_type == 'ZeroDivisionError' assert error_item.source_method == 'test_info_exception_details' assert len(error_item.stacktrace) diff --git a/tests/transport/test_base.py b/tests/transport/test_base.py index 6a4cba4..d94c891 100644 --- a/tests/transport/test_base.py +++ b/tests/transport/test_base.py @@ -81,4 +81,4 @@ def test_send(self): env_details = 'test_env_details' agent_base_transport = AgentBaseTransport(api_config, env_details) - self.assertRaises(NotImplementedError, agent_base_transport.send, 'test_group_message') + self.assertRaises(AttributeError, agent_base_transport.send, 'test_group_message') diff --git a/tests/transport/test_init.py b/tests/transport/test_init.py index d1054dc..079d249 100644 --- a/tests/transport/test_init.py +++ b/tests/transport/test_init.py @@ -2,8 +2,11 @@ from mock import patch from tests.bases import ClearEnvTest +from stackify.constants import AGENT_LOG_URL +from stackify.constants import DEFAULT_HTTP_ENDPOINT from stackify.protos import stackify_agent_pb2 from stackify.transport import configure_transport +from stackify.transport.agent import AgentHTTPTransport from stackify.transport.agent import AgentSocketTransport from stackify.transport.default import DefaultTransport from stackify.transport.default.log import LogMsg @@ -161,3 +164,87 @@ def test_agent_socket_send_url_default(self, mock_send): assert mock_send.called assert mock_send.call_args_list[0][0][0] == 'http+unix://%2Fusr%2Flocal%2Fstackify%2Fstackify.sock/log' + + def test_agent_http_transport(self): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + 'http_endpoint': 'test.url', + 'transport': 'agent_http', + } + + transport = configure_transport(**config) + + assert isinstance(transport, AgentHTTPTransport) + + def test_agent_http_create_message(self): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + 'http_endpoint': 'test.url', + 'transport': 'agent_http', + } + + transport = configure_transport(**config) + message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + assert isinstance(message, stackify_agent_pb2.LogGroup.Log) + + def test_agent_http_create_group_message(self): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + 'http_endpoint': 'test.url', + 'transport': 'agent_http', + } + + transport = configure_transport(**config) + message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = transport.create_group_message([message]) + + assert isinstance(group_message, stackify_agent_pb2.LogGroup) + + @patch('stackify.transport.agent.agent_http.AgentHTTP.send') + def test_agent_http_send_url(self, mock_send): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + 'http_endpoint': 'test.url', + 'transport': 'agent_http', + } + + transport = configure_transport(**config) + assert isinstance(transport, AgentHTTPTransport) + message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = transport.create_group_message([message]) + transport.send(group_message) + + assert mock_send.called + assert mock_send.call_args_list[0][0][0] == 'test.url/log' + + @patch('stackify.transport.agent.agent_http.AgentHTTP.send') + def test_agent_http_send_url_default(self, mock_send): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + 'transport': 'agent_http', + } + + transport = configure_transport(**config) + assert isinstance(transport, AgentHTTPTransport) + message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = transport.create_group_message([message]) + transport.send(group_message) + + assert mock_send.called + assert mock_send.call_args_list[0][0][0] == DEFAULT_HTTP_ENDPOINT + AGENT_LOG_URL