diff --git a/CHANGELOG b/CHANGELOG index d1b1989..72f6f8c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,11 @@ Change Log ========== +v0.8.0 +------ + * Mongolog.find utility for searching the logs + * flake8 compliance (mostly) + v0.7.4 ------ * Added base paramter 'max_keep' that is used for embedded documents to determine how many records to keep diff --git a/build.py b/build.py index 134dbda..221813e 100644 --- a/build.py +++ b/build.py @@ -1,27 +1,24 @@ #!/usr/bin/env python import os import subprocess -import collections -COPY_PATH=os.path.expanduser( - os.path.abspath( - os.environ.get("MONGOLOG_COPY_PATH", "./") - ) +COPY_PATH = os.path.expanduser( + os.path.abspath( + os.environ.get("MONGOLOG_COPY_PATH", "./") + ) ) if not COPY_PATH: - raise ValueError("You muse set MONGOLOG_COPY_PATH") + raise ValueError("You muse set MONGOLOG_COPY_PATH") Commands = ( "rm -rf dist", - "python setup.py sdist", + "python setup.py sdist", ) - for cmd in Commands: - print "running:", " ".join(cmd.split()) - subprocess.check_output(cmd.split()) - + print("running:", " ".join(cmd.split())) + subprocess.check_output(cmd.split()) -## Now copy the file over +# Now copy the file over cmd = "cp ./dist/%s %s" % (os.listdir("dist")[0], COPY_PATH) subprocess.check_output(cmd.split()) diff --git a/mongolog/__init__.py b/mongolog/__init__.py index 84256db..8cfca61 100644 --- a/mongolog/__init__.py +++ b/mongolog/__init__.py @@ -17,7 +17,7 @@ """ from .handlers import ( # noqa BaseMongoLogHandler, - SimpleMongoLogHandler, - VerboseMongoLogHandler, + SimpleMongoLogHandler, + VerboseMongoLogHandler, HttpLogHandler, ) diff --git a/mongolog/exceptions.py b/mongolog/exceptions.py index aaed93d..6b26f36 100644 --- a/mongolog/exceptions.py +++ b/mongolog/exceptions.py @@ -1,5 +1,6 @@ #!/usr/bin/env python + class MissingConnectionError(ValueError): - def __init__(self, *args, **kwargs): - ValueError.__init__(self, *args, **kwargs) \ No newline at end of file + def __init__(self, *args, **kwargs): + ValueError.__init__(self, *args, **kwargs) diff --git a/mongolog/handlers.py b/mongolog/handlers.py index fcf4cde..b1fcd23 100644 --- a/mongolog/handlers.py +++ b/mongolog/handlers.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -#!/usr/bin/env python +# !/usr/bin/env python """ django-mongolog. Simple Mongo based logger for Django Copyright (C) 2015 - John Furr @@ -39,16 +39,19 @@ uuid_namespace = uuid.UUID('8296424f-28b7-5982-a434-e6ec8ef529b3') -def get_mongolog_handler(logger_name=None): +# TODO Move to mongolog.models.Mongolog +def get_mongolog_handler(logger_name=None, show_logger_names=False): """ - Return the first MongoLogHander found in the list of defined loggers. + Return the first MongoLogHander found in the list of defined loggers. NOTE: If more than one is defined, only the first one is used. """ if logger_name: logger_names = [logger_name] else: logger_names = [''] + list(logging.Logger.manager.loggerDict) - console.info("get_mongolog_handler(): Logger_names: %s", json.dumps(logger_names, indent=4, sort_keys=True)) + + if show_logger_names: + console.info("get_mongolog_handler(): Logger_names: %s", json.dumps(logger_names, indent=4, sort_keys=True, default=str)) for name in logger_names: logger = logging.getLogger(name) @@ -78,13 +81,13 @@ def __init__(self, level=NOTSET, connection=None, w=1, j=False, verbose=None, ti valid_record_types = [self.REFERENCE, self.EMBEDDED] if record_type not in valid_record_types: raise ValueError("record_type myst be one of %s" % valid_record_types) - + # The type of document we store self.record_type = record_type # number of dates to keep in embedded document self.max_keep = max_keep - + # The write concern self.w = w @@ -97,7 +100,6 @@ def __init__(self, level=NOTSET, connection=None, w=1, j=False, verbose=None, ti # If True will print each log_record to console before writing to mongo self.verbose = verbose - handler_type = str(type(self)) if self.connection: self.connect() @@ -139,11 +141,11 @@ def get_db(self): """ Return a handler to the database handler """ - return getattr(self, "db", None) + return getattr(self, "db", None) def get_timestamp_collection(self): return getattr(self, "timestamp", None) - + def get_collection(self): """ Return the collection being used by MongoLogHandler @@ -161,15 +163,15 @@ def connect_pymongo3(self, test=False): self.client = pymongo.MongoClient(self.connection, serverSelectionTimeoutMS=5, w=self.w) else: self.client = pymongo.MongoClient(self.connection, serverSelectionTimeoutMS=5, w=self.w, j=self.j) - + except pymongo.errors.ServerSelectionTimeoutError: msg = "Unable to connect to mongo with (%s)" % self.connection # NOTE: Trying to log here ends up with Duplicate Key errors on upsert in emit() print(msg) raise pymongo.errors.ServerSelectionTimeoutError(msg) - + return self.client - + def connect_pymongo2(self): # TODO Determine proper try/except logic for pymongo 2.7 driver self.client = pymongo.MongoClient(self.connection) @@ -201,10 +203,10 @@ def check_keys(self, record): if isinstance(v, dict): for old_key in record['msg'][k].keys(): - record['msg'][k][self.new_key(old_key)] = record['msg'][k].pop(old_key) + record['msg'][k][self.new_key(old_key)] = record['msg'][k].pop(old_key) return record - + def create_log_record(self, record): """ Convert the python LogRecord to a MongoLog Record. @@ -223,7 +225,7 @@ def create_log_record(self, record): uuid_key = str(record['msg']) + str(record['levelname']) else: uuid_key = (unicode(record['msg']) + unicode(record['levelname'])).encode('utf-8', 'replace') - + record.update({ 'uuid': uuid.uuid5(uuid_namespace, uuid_key).hex, # NOTE: if the user is using django and they have USE_TZ=True in their settings @@ -262,9 +264,9 @@ def emit(self, record): # TODO move this to a validate log_record method and add more validation log_record.get('uuid', ValueError("You must have a uuid in your LogRecord")) - + if self.verbose: - print(json.dumps(log_record, sort_keys=True, indent=4, default=str)) + print(json.dumps(log_record, sort_keys=True, indent=4, default=str)) if self.record_type == self.EMBEDDED: self.insert_embedded(log_record) @@ -317,7 +319,7 @@ def reference_log_pymongo_2(self, log_record): # remove the old document self.mongolog.find_and_modify(query, remove=True) - + # insert the new one self.mongolog.insert(log_record) @@ -337,7 +339,7 @@ def reference_log_pymongo_3(self, log_record): ) # Now update the timestamp collection - # We can do this with a lower write concern than the previous operation since + # We can do this with a lower write concern than the previous operation since # we can alway's retreive the last datetime from the mongolog collection self.timestamp.insert({ 'uuid': log_record['uuid'], @@ -379,7 +381,7 @@ def create_log_record(self, record): class VerboseMongoLogHandler(BaseMongoLogHandler): def create_log_record(self, record): - record = super(VerboseMongoLogHandler, self).create_log_record(record) + record = super(VerboseMongoLogHandler, self).create_log_record(record) mongolog_record = LogRecord({ 'name': record['name'], 'thread': { @@ -404,7 +406,7 @@ def create_log_record(self, record): }, 'uuid': record['uuid'], 'time': record['time'], - }) + }) if record['exc_info']: mongolog_record['exception'] = { @@ -416,7 +418,7 @@ def create_log_record(self, record): class HttpLogHandler(SimpleMongoLogHandler): - def __init__(self, level=NOTSET, client_auth='', timeout=3, verbose=False, time_zone="local", *args, **kwargs): + def __init__(self, level=NOTSET, client_auth='', timeout=3, verbose=False, time_zone="local", *args, **kwargs): # Make sure there is a trailing slash or reqests 2.8.1 will try a GET instead of POST self.client_auth = client_auth if client_auth.endswith('/') else "%s/" % client_auth @@ -426,7 +428,7 @@ def __init__(self, level=NOTSET, client_auth='', timeout=3, verbose=False, time_ self.time_zone = time_zone # Intentionally hard coded in HttpLogHandler - self.record_type='reference' + self.record_type = 'reference' # If True will print each log_record to console before writing to mongo self.verbose = verbose @@ -439,7 +441,7 @@ def __unicode__(self): return u'%s' % self.client_auth def emit(self, record): - """ + """ From python: type(record) == LogRecord https://github.com/certik/python-2.7/blob/master/Lib/logging/__init__.py#L230 """ @@ -452,5 +454,4 @@ def emit(self, record): r = requests.post(self.client_auth, json=json.dumps(log_record, default=str), timeout=self.timeout, proxies={'http':''}) # noqa # uncomment to debug - print ("Response:", json.dumps(r.json(), indent=4, sort_keys=True, default=str)) - + print("Response:", json.dumps(r.json(), indent=4, sort_keys=True, default=str)) diff --git a/mongolog/management/commands/analog.py b/mongolog/management/commands/analog.py index 7db8c1d..078a530 100644 --- a/mongolog/management/commands/analog.py +++ b/mongolog/management/commands/analog.py @@ -1,23 +1,21 @@ # -*- coding: utf-8 -*- -#!/usr/bin/env python +# !/usr/bin/env python from __future__ import print_function import logging import logging.config import django import json -import sys import pymongo pymongo_version = int(pymongo.version.split(".")[0]) if pymongo_version >= 3: - from pymongo.collection import ReturnDocument + from pymongo.collection import ReturnDocument # noqa: F40 from mongolog.handlers import get_mongolog_handler from django.core.management.base import BaseCommand -#logger = logging.getLogger(__name__) logger = logging.getLogger('console') @@ -34,13 +32,12 @@ class Command(BaseCommand): make_option( '-q', '--query', default=None, action='store', dest='query', help='Pass in a search query to mongo.'), - ) + ) def __init__(self, *args, **kwargs): super(Command, self).__init__(*args, **kwargs) self.prev_object_id = None - def add_arguments(self, parser): if django.VERSION[1] >= 7: parser.add_argument( @@ -58,12 +55,12 @@ def add_arguments(self, parser): def print_results(self, results): # older versions of pymongo didn't use a CommandCursor object to iterate over the results. - # We check by trying to convert to a list and if there is a TypeError we continue on + # We check by trying to convert to a list and if there is a TypeError we continue on # and try to iterate over 'results' as thought it were a Cursor object. try: results = list(results['result']) results.reverse() - except TypeError as e: + except TypeError: pass for r in results: @@ -78,7 +75,7 @@ def print_results(self, results): logger.debug(r) elif level == 'CRITICAL': logger.critical(r) - elif level == 'MONGOLOG-INTERNAL' or level == None: + elif level == 'MONGOLOG-INTERNAL' or level is None: # Print nothing if it's a mongolog internal log pass else: @@ -86,7 +83,7 @@ def print_results(self, results): def fetch_results(self, options): query = options['query'] if options['query'] else {} - proj = {'_id': 1, 'level': 1, 'msg': 1,} + proj = {'_id': 1, 'level': 1, 'msg': 1} limit = options['limit'] return self.collection.aggregate([ {"$match": query}, diff --git a/mongolog/models.py b/mongolog/models.py index 820c77c..91c858e 100644 --- a/mongolog/models.py +++ b/mongolog/models.py @@ -16,6 +16,64 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . """ +import pymongo +from pymongo import MongoClient +pymongo_version = int(pymongo.version.split(".")[0]) +if pymongo_version >= 3: + from pymongo.collection import ReturnDocument # noqa: F401 + + +class Mongolog(object): + """ + This class provides a set of queriable functions. + """ + # If LOGGER is None we wil try and find the first available logger + LOGGER = None + + @classmethod + def find(cls, logger=None, query=None, project=None, uuid=None, level=None, limit=None, **kwargs): + """ + return self.collection.aggregate([ + {"$match": query}, + {"$project": proj}, + {"$sort": {'created': pymongo.DESCENDING}}, + {"$limit": limit}, + ]) + """ + from mongolog.handlers import get_mongolog_handler + + logger = cls.LOGGER if cls.LOGGER else logger + + handler = get_mongolog_handler(logger_name=logger) + client = MongoClient(handler.connection) + db = client.mongolog + + aggregate_commands = [] + + if not query: + query = {} + + if uuid: + query.update({'uuid': uuid}) + + if level: + query.update({'level': level}) + + if logger: + query.update({'name': logger}) + + aggregate_commands.append({"$match": query}) + + if project: + aggregate_commands.append({"$project": project}) + + aggregate_commands.append({"$sort": {'created': pymongo.DESCENDING}}) + + if limit: + aggregate_commands.append({"$limit": limit}) + + results = db.mongolog.aggregate(aggregate_commands) + return results['result'] if isinstance(results, dict) else results class LogRecord(dict): @@ -35,7 +93,7 @@ class LogRecord(dict): "num" : NumberLong(140061052233472), "name" : "MainThread" }, - "level" : { + "level" : { "num" : 10, "name" : "DEBUG" }, @@ -49,4 +107,4 @@ class LogRecord(dict): } } """ - pass \ No newline at end of file + pass diff --git a/mongolog/tests.py b/mongolog/tests.py index f381512..01ea1d8 100644 --- a/mongolog/tests.py +++ b/mongolog/tests.py @@ -19,17 +19,19 @@ import unittest import logging from logging import config # noqa +import os import sys +import subprocess import time import json -from unittest import skip, skipIf +from unittest import skipIf from requests.exceptions import ConnectionError # Different imports for python2/3 try: from StringIO import StringIO except ImportError: - from io import StringIO + from io import StringIO # noqa: F401 import pymongo pymongo_major_version = int(pymongo.version.split(".")[0]) @@ -37,8 +39,7 @@ from mongolog.handlers import ( get_mongolog_handler, SimpleMongoLogHandler ) -from mongolog.exceptions import MissingConnectionError - +from mongolog.models import Mongolog from django.core.management import call_command from django.test import TestCase @@ -56,11 +57,11 @@ """ All log tests should log a dictionary with a 'test' key logger.info({ - 'test': True, + 'test': True, 'msg': 'special message' }) - This key will allow us to easily clean test entries out of the + This key will allow us to easily clean test entries out of the database. NOTE: You can add any other key you want for testing purposes @@ -80,10 +81,10 @@ 'Archaea': [], 'Eukaryota': [ { - 'name': 'Excavata', + 'name': 'Excavata', 'description': 'Various flagellate protozoa', }, - { + { 'name': 'Amoebozoa', 'descritpion': 'most lobose amoeboids and slime moulds', }, @@ -104,12 +105,12 @@ 'description': 'Land plants, green algae, red algae, and glaucophytes' }, ] - } + } } } -def raiseException(logger = None): +def raiseException(logger=None): """ Used to test logger.exceptions. Use like: @@ -130,7 +131,6 @@ def remove_test_entries(self, test_key="msg.test"): Called in setUp and tearDown. TODO: Remove entries from timestamp collection """ - #self.collection.remove({test_key: True}) if pymongo_major_version < 3 else self.collection.delete_many({test_key: True}) # Drop the timetamp collection completely self.handler.get_db().command("dropDatabase") @@ -142,7 +142,7 @@ class TestBaseMongoLogHandler(TestCase, TestRemoveEntriesMixin): def setUp(self): self.logger = logging.getLogger("test.base.reference") - self.handler = get_mongolog_handler("test.base.reference") + self.handler = get_mongolog_handler("test.base.reference", show_logger_names=True) console.error("self.handler(%s)" % self.handler) self.collection = self.handler.get_collection() @@ -158,9 +158,9 @@ def test_dot_in_key(self): console.debug(self) self.logger.info({ 'META': { - 'user.name': 'jfurr', + 'user.name': 'jfurr', 'user$name': 'jfurr' - }, + }, 'user.name': 'jfurr', 'user$name': 'jfurr' }) @@ -193,32 +193,32 @@ def test_basehandler_exception(self): raiseException(self.logger) records = self.collection.find({ - 'msg.test': True, + 'msg.test': True, 'msg.Life.Domain.Eukaryota.name': "Archaeplastida", 'levelname': 'ERROR' }) self.assertEqual(1, records.count()) expected_keys = [ - u'threadName', - u'name', - u'thread', - u'relativeCreated', + u'threadName', + u'name', + u'thread', + u'relativeCreated', u'process', - u'args', - u'filename', - u'module', - u'funcName', - u'levelno', - u'processName', - u'created', - u'msecs', - u'msg', - u'exc_info', - u'exc_text', - u'pathname', - u'_id', - u'levelname', + u'args', + u'filename', + u'module', + u'funcName', + u'levelno', + u'processName', + u'created', + u'msecs', + u'msg', + u'exc_info', + u'exc_text', + u'pathname', + u'_id', + u'levelname', u'lineno', u'time', u'uuid', @@ -226,7 +226,7 @@ def test_basehandler_exception(self): # To make test pass on python 3 version if sys.version_info[0] >= 3: expected_keys.append(u'stack_info') - + record = records[0] self.assertEqual( set(record.keys()), @@ -238,14 +238,14 @@ def test_basehandler_exception(self): # Now test that the nested ValueError was successfully converted to a unicode str. try: # unicode will throw a NameError in Python 3 - self.assertEqual(unicode, type(record['msg']['Life']['Domain']['Bacteria'][0]['name'])) + self.assertEqual(unicode, type(record['msg']['Life']['Domain']['Bacteria'][0]['name'])) except NameError: - self.assertEqual(str, type(record['msg']['Life']['Domain']['Bacteria'][0]['name'])) - + self.assertEqual(str, type(record['msg']['Life']['Domain']['Bacteria'][0]['name'])) + self.logger.info("Just some friendly info") self.logger.error("Just some friendly info") self.logger.debug("Just some friendly info") - + def test_str_unicode_mongologhandler(self): console.debug(self) self.assertEqual(self.handler.connection, u"%s" % self.handler) @@ -262,23 +262,23 @@ def setUp(self): self.remove_test_entries() self.expected_keys = set([ - 'dates', - 'uuid', - 'thread', - 'level', - 'process', - 'exception', - 'module', - 'filename', - 'func', - 'created', - 'msg', - 'path', - 'line', - '_id', + 'dates', + 'uuid', + 'thread', + 'level', + 'process', + 'exception', + 'module', + 'filename', + 'func', + 'created', + 'msg', + 'path', + 'line', + '_id', 'name', 'counter', - ]) + ]) def test_missing_connection_key(self): LOGGING['handlers']['simple_no_connection'] = { @@ -297,14 +297,13 @@ def test_missing_connection_key(self): del LOGGING['handlers']['simple_no_connection'] del LOGGING['loggers']['simple.no.connection'] - def test_exception(self): console.debug(self) with self.assertRaises(ValueError): raiseException(self.logger) records = self.collection.find({ - 'msg.test': True, + 'msg.test': True, 'msg.Life.Domain.Eukaryota.name': "Archaeplastida", 'level': 'ERROR' }) @@ -312,11 +311,11 @@ def test_exception(self): record = records[0] self.assertEqual( - set(record.keys()), + set(record.keys()), self.expected_keys ) - # Verify that the dates field has 1 item + # Verify that the dates field has 1 item self.assertEqual(1, len(record['dates'])) # now create anthe rduplicate entry and show that only one record is still present @@ -325,7 +324,7 @@ def test_exception(self): raiseException(self.logger) records = self.collection.find({ - 'msg.test': True, + 'msg.test': True, 'msg.Life.Domain.Eukaryota.name': "Archaeplastida", 'level': 'ERROR' }) @@ -348,7 +347,7 @@ class TestSimpleMongoLogHandler_Reference(unittest.TestCase, TestRemoveEntriesMi 1) mongolog that holds that latest record and a single 'date' 2) timestamp collection that holds that individual timestamps - These two collections are related to each other via the uuid key. + These two collections are related to each other via the uuid key. """ def setUp(self): self.logger = logging.getLogger('test.reference') @@ -368,8 +367,8 @@ def test_logstructure_simple_reference(self): # now test a serializable dict with an exception call log_msg = {'test': True, 'fruits': ['apple', 'orange'], 'error': str(ValueError), 'handler': str(SMH_Obj)} expected_keys = set([ - '_id', 'exception', 'name', 'thread', 'time', - 'process', 'level', 'msg', 'path', 'module', + '_id', 'exception', 'name', 'thread', 'time', + 'process', 'level', 'msg', 'path', 'module', 'line', 'func', 'filename', 'uuid' ]) @@ -377,10 +376,10 @@ def test_logstructure_simple_reference(self): raise ValueError except ValueError: self.logger.exception(log_msg) - + rec = self.collection.find_one({'msg.fruits': ['apple', 'orange']}) self.assertEqual(set(rec.keys()), expected_keys) - + # Python 2 duplicate entry test log_msg = {'test': True, 'fruits': ['apple', 'orange'], 'error': str(ValueError), 'handler': str(SMH_Obj)} try: @@ -411,8 +410,8 @@ def test_logstructure_simple_reference(self): rec = self.collection.find_one({'msg.fruits': {'$in': ['apple', 'orange']}}) self.assertEqual(set(rec.keys()), expected_keys) - - + + class TestVerboseMongoLogHandler(unittest.TestCase, TestRemoveEntriesMixin): def setUp(self): console.debug(self) @@ -431,7 +430,7 @@ def test_logstructure_verbose_exception(self): raiseException(self.logger) records = self.collection.find({ - 'info.msg.test': True, + 'info.msg.test': True, 'info.msg.Life.Domain.Eukaryota.name': "Archaeplastida", 'level.name': 'ERROR' }) @@ -464,7 +463,7 @@ def test_logstructure_verbose_debug_info_warn(self): self.handler.setLevel("WARNING") log_msg = {'test': True, 'msg': 'WARNING', 'msg2': 'DANGER'} query = { - 'info.msg.test': log_msg['test'], + 'info.msg.test': log_msg['test'], 'info.msg.msg': log_msg['msg'], 'info.msg.msg2': log_msg['msg2'], 'level.name': 'WARNING' @@ -474,7 +473,7 @@ def test_logstructure_verbose_debug_info_warn(self): rec = self.collection.find_one(query) self.assertEqual( - set(rec.keys()), + set(rec.keys()), set(['info', 'name', 'thread', 'level', 'process', 'time', '_id', 'uuid']) ) @@ -496,7 +495,7 @@ def setUp(self): self.handler = get_mongolog_handler("test.http") self.collection = self.handler.get_collection() console.error("self.collection(%s)" % self.collection) - + @skipIf(sys.version_info.major == 3, "SKipping TestHttpLogHandler.test_timeout because of python version") def test_timeout(self): console.debug(self) @@ -538,12 +537,12 @@ def test_analog(self): self.logger.error({'test': True, 'logger': 'Error'}) self.logger.critical({'test': True, 'logger': 'Critical'}) - query = '{"name": "root"}' call_command('analog', limit=10, query='{"name": "root"}') - with self.assertRaises(NotImplementedError) as cm: + with self.assertRaises(NotImplementedError): call_command('analog', limit=20, tail=True) + class TestPerformanceTests(unittest.TestCase, TestRemoveEntriesMixin): def setUp(self): self.handler = get_mongolog_handler('test.embedded') @@ -553,16 +552,17 @@ def setUp(self): self.timestamp = self.handler.get_timestamp_collection() self.remove_test_entries() + + # TODO Do you need this? self.remove_test_entries(test_key='msg.Test') self.iterations = 100 - def _check_embedded_results(self, results, iterations): self.assertEqual(1, results.count()) rec = results[0] self.assertEqual(iterations, rec['counter']) - expected_date_len = iterations + expected_date_len = iterations if iterations > self.handler.max_keep: expected_date_len = self.handler.max_keep @@ -573,22 +573,19 @@ def _check_embedded_results(self, results, iterations): def test_embedded(self): console.debug(self) - self.logger = logging.getLogger('test.embedded') - console.info("Starting embedded test: max_keep(%s) iteration(%s)", self.handler.max_keep, self.iterations) start = time.time() for i in range(self.iterations): self.logger.info({'Test': True, 'Test1': 1}) results = self.collection.find({'msg.Test': True, 'msg.Test1': 1}) - self._check_embedded_results(results, i+1) + self._check_embedded_results(results, i + 1) end = time.time() results = self.collection.find({'msg.Test': True}) self._check_embedded_results(results, self.iterations) - console.warn("Test time: %s", end-start) - + console.warn("Test time: %s", end - start) # rerun with larger max_keep max_keep = LOGGING['handlers']['test_embedded']['max_keep'] @@ -596,7 +593,6 @@ def test_embedded(self): logging.config.dictConfig(LOGGING) self.setUp() self.logger = logging.getLogger('test.embedded') - console.info("Starting embedded test: max_keep(%s) iteration(%s)", self.handler.max_keep, self.iterations) @@ -604,12 +600,11 @@ def test_embedded(self): for i in range(self.iterations): self.logger.info({'Test': True}) results = self.collection.find({'msg.Test': True}) - self._check_embedded_results(results, i+1) + self._check_embedded_results(results, i + 1) end = time.time() results = self.collection.find({'msg.Test': True}) self._check_embedded_results(results, self.iterations) - console.warn("Test time: %s", end-start) - + console.warn("Test time: %s", end - start) LOGGING['handlers']['test_embedded']['max_keep'] = max_keep logging.config.dictConfig(LOGGING) @@ -622,8 +617,7 @@ def _check_reference_results(self, results, i): uuid = results[0]['uuid'] ts_results = self.timestamp.find({'uuid': uuid}) td_results = list(ts_results) - self.assertEqual(i+1, len(td_results)) - + self.assertEqual(i + 1, len(td_results)) def test_reference(self): console.debug(self) @@ -636,3 +630,111 @@ def test_reference(self): results = list(self.collection.find({'msg.Test': True, 'msg.Test1': 1})) self._check_reference_results(results, i) + + end = time.time() + self.logger.info("Total time %s" % (end - start)) + + +class MongoLogUtilsTests(unittest.TestCase, TestRemoveEntriesMixin): + def setUp(self): + self.handler = get_mongolog_handler('test.embedded') + + # Get some pointers to the collections for easy querying + self.collection = self.handler.get_collection() + self.timestamp = self.handler.get_timestamp_collection() + self.logger = logging.getLogger('test.embedded') + + self.remove_test_entries() + + def test_find_for_embedded(self): + console.debug(self) + Mongolog.LOGGER = 'test.embedded' + info_items = [ + {'location': {'country': 'USA', 'state': 'WA', 'city': 'Puyallup'}}, + {'location': {'country': 'USA', 'state': 'WA', 'city': 'Seattle'}}, + {'location': {'country': 'USA', 'state': 'MA', 'city': 'Boston'}}, + {'location': {'country': 'USA', 'state': 'MS', 'city': 'Greenville'}}, + {'location': {'country': 'France', 'region': 'Normandy', 'city': 'Rouen'}}, + {'location': {'country': 'France', 'region': 'Île-de-France', 'city': 'Paris'}}, + ] + for item in info_items: + self.logger.info(item) + + warn_items = [ + {'location': {'country': 'France', 'region': 'Normandy', 'city': 'Caen'}}, + {'location': {'country': 'France', 'region': 'Normandy', 'city': 'Bayeux'}}, + {'location': {'country': 'USA', 'state': 'MA', 'city': 'Framingham'}}, + ] + for item in warn_items: + self.logger.warn(item) + + critical_items = [ + {'location': {'country': 'Canada', 'Province': 'Quebec Ontario', 'city': 'Quebec City'}}, + {'location': {'country': 'Canada', 'Province': 'Quebec Ontario', 'city': 'Toronto'}}, + ] + + for item in critical_items: + self.logger.critical(item) + + results = Mongolog.find(query={'msg.location': {'$exists': True}}) + self.assertEqual(11, len(list(results))) + + results = Mongolog.find(level='INFO') + self.assertEqual(6, len(list(results))) + + # Test limit + results = Mongolog.find(level='INFO', limit=3) + self.assertEqual(3, len(list(results))) + + # Test search by name + results = Mongolog.find(logger="test.embedded") + self.assertEqual(11, len(list(results))) + + results = Mongolog.find(query={'msg.location.region': {'$exists': True}}) + self.assertEqual(4, len(list(results))) + + results = Mongolog.find(query={'msg.location.region': {'$exists': True}}, level='ERROR') + self.assertEqual(0, len(list(results))) + + results = Mongolog.find(query={'msg.location.region': {'$exists': True}}, level='WARNING') + self.assertEqual(2, len(list(results))) + + results = Mongolog.find(level='CRITICAL') + self.assertEqual(2, len(list(results))) + + results = Mongolog.find(level='CRITICAL', query={'msg.location.city': 'Quebec City'}) + self.assertEqual(1, len(list(results))) + + # Test uuid path. + results = Mongolog.find(uuid='no-results') + self.assertEqual(0, len(list(results))) + + # Now test projection + query = {'msg.location.city': 'Quebec City'} + level = 'CRITICAL' + results = Mongolog.find(level=level, query=query, project={'msg': 1}) + results = list(results) + self.assertEqual(set(['_id', 'msg']), set(results[0].keys())) + + results = Mongolog.find(level=level, query=query, project={'msg': 1, '_id': 0}) + results = list(results) + self.assertEqual(set(['msg']), set(results[0].keys())) + + results = Mongolog.find(level=level, query=query, project={'msg': 1, '_id': 0, 'level': 1}) + results = list(results) + self.assertEqual(set(['msg', 'level']), set(results[0].keys())) + + +class FlakeTests(unittest.TestCase): + # flake8 --ignore=E402,F82 + def test_flak8(self): + cmd = 'flake8 --ignore=E402,F82' + logger = logging.getLogger("console") + logger.debug(self) + try: + subprocess.check_output([cmd], shell=True, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + logger.critical("----------------- Please Fix flake8 errors -------------------") + os.system(cmd) + logger.critical("--------------------------------------------------------------") + raise diff --git a/mongolog/urls.py b/mongolog/urls.py index 4e08d67..9623be6 100644 --- a/mongolog/urls.py +++ b/mongolog/urls.py @@ -18,6 +18,6 @@ from django.conf.urls import url from mongolog.views import HomeView urlpatterns = [ - #url(r'^admin/', include(admin.site.urls)), + # url(r'^admin/', include(admin.site.urls)), url(r'^home/', HomeView.as_view(), name="home"), ] diff --git a/requirements.txt b/requirements.txt index 03ee8ee..3881b37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,6 @@ django>=1.4 pymongo>=2.4 coverage==3.6 coveralls==1.1 +flake8==3.0.4 python-color-logger django-extensions diff --git a/settings/colorlog.py b/settings/colorlog.py index 368ed37..3a92710 100644 --- a/settings/colorlog.py +++ b/settings/colorlog.py @@ -3,6 +3,7 @@ from logutils.colorize import ColorizingStreamHandler + class ColorLogHandler(ColorizingStreamHandler): def __init__(self, *args, **kwargs): super(ColorLogHandler, self).__init__(*args, **kwargs) diff --git a/settings/settings.py b/settings/settings.py index 49276c3..6f06edd 100644 --- a/settings/settings.py +++ b/settings/settings.py @@ -12,7 +12,6 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) import os -import sys BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -44,9 +43,9 @@ # Interesting Note: requests 2.8.1 will turn this into a GET if it's missing a trailing slash # We automagically add the trailing slash 'client_auth': 'http://192.168.33.51/4e487f07a84011e5a3403c15c2bcc424', - 'verbose': True, + 'verbose': True, }, - ################## Test Handlers + # Test Handlers 'test_reference': { 'level': 'DEBUG', 'class': 'mongolog.SimpleMongoLogHandler', @@ -156,9 +155,9 @@ 'http': { 'level': 'DEBUG', 'handlers': ['http'], - 'propagate': True, + 'propagate': True, }, - ##################### Test Loggers + # Test Loggers 'test': { 'level': 'DEBUG', 'propagate': False, @@ -170,16 +169,16 @@ 'handlers': ['test_embedded'], }, 'test.base.reference': { - 'handlers': ['test_base_reference'], + 'handlers': ['test_base_reference'], }, 'test.base.reference.w0': { - 'handlers': ['test_base_reference_w0'], + 'handlers': ['test_base_reference_w0'], }, 'test.base.invalid': { - 'handlers': ['test_base_invalid'], + 'handlers': ['test_base_invalid'], }, 'test.verbose': { - 'handlers': ['test_verbose'], + 'handlers': ['test_verbose'], }, 'test.http': { 'level': 'DEBUG', @@ -192,6 +191,7 @@ SHELL_PLUS_PRE_IMPORTS = ( 'logging', ('logging', 'getLogger'), + ('mongolog.models', 'Mongolog'), ) # Quick-start development settings - unsuitable for production diff --git a/setup.py b/setup.py index 8532297..95ebe3b 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,7 @@ import os -import subprocess from setuptools import setup -VERSION = "0.7.4" +VERSION = "0.8.0" with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as readme: README = readme.read() @@ -26,7 +25,7 @@ description='A simple mongo based log handler for python/django', long_description=README, url='https://github.com/gnulnx/django-mongolog', - download_url='https://github.com/gnulnx/django-mongolog/tree/%s'%(VERSION), + download_url='https://github.com/gnulnx/django-mongolog/tree/%s' % (VERSION), author='John Furr', author_email='john.furr@gmail.com', classifiers=[