diff --git a/.travis.yml b/.travis.yml index 1c9dab9..74794dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,18 +4,15 @@ # https://docs.djangoproject.com/en/1.9/releases/1.9/#django-1-9-release-notes language: python python: -# - "2.6" # Just to old to support - "2.7" - "3.4" - "3.5" -# - "3.5-dev" # 3.5 development branch -# - "nightly" # currently points to 3.6-dev + - "3.6" + - "3.7-dev" env: # Test out support for lowest/highest pymongo/django combos - #- PYMONGO="==2.4" DJANGO="==1.8" - #- PYMONGO=">=3.0" DJANGO=">=1.9" - - PYMONGO="==2.4" - - PYMONGO=">=3.0" + - PYMONGO=">=3.0" DJANGO="==1.9.8" + - PYMONGO=">=3.0" DJANGO=">=2.0" # command to install dependencies services: - mongodb diff --git a/CHANGELOG b/CHANGELOG index 72f6f8c..4fee22d 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,12 @@ Change Log ========== +v0.9.0 +------ + * Removed support for pymongo < 3.0 + * Removed support for Django < 1.9.8 + * Added ml_purge command for cleaning out log documents + v0.8.0 ------ * Mongolog.find utility for searching the logs diff --git a/mongolog/management/commands/analog.py b/mongolog/management/commands/analog.py index 37a2886..c90d882 100644 --- a/mongolog/management/commands/analog.py +++ b/mongolog/management/commands/analog.py @@ -3,7 +3,6 @@ from __future__ import print_function import logging import logging.config -import django import json import pymongo @@ -19,38 +18,24 @@ class Command(BaseCommand): - if django.VERSION[1] <= 7: - from optparse import make_option - option_list = BaseCommand.option_list + ( - make_option( - '-l', '--limit', default=10, type=int, action='store', dest='limit', - help='Delete poll instead of closing it'), - make_option( - '-t', '--tail', default=False, action='store_true', dest='tail', - help='Tail the log file. By default it will limit to 10 results. Use --limit to change'), - make_option( - '-q', '--query', default=None, action='store', dest='query', - help='Pass in a search query to mongo.'), - ) def __init__(self, *args, **kwargs): super(Command, self).__init__(*args, **kwargs) self.prev_object_id = None def add_arguments(self, parser): - if django.VERSION[1] >= 7: - parser.add_argument( - '-l', '--limit', default=10, type=int, action='store', dest='limit', - help='Limit Results', - ) - parser.add_argument( - '-t', '--tail', default=False, action='store_true', dest='tail', - help='Tail the log file. By default it will limit to 10 results. Use --limit to change', - ) - parser.add_argument( - '-q', '--query', default=None, type=str, action='store', dest='query', - help='Pass in a search query to mongo.', - ) + parser.add_argument( + '-l', '--limit', default=10, type=int, action='store', dest='limit', + help='Limit Results', + ) + parser.add_argument( + '-t', '--tail', default=False, action='store_true', dest='tail', + help='Tail the log file. By default it will limit to 10 results. Use --limit to change', + ) + parser.add_argument( + '-q', '--query', default=None, type=str, action='store', dest='query', + help='Pass in a search query to mongo.', + ) def print_results(self, results): # older versions of pymongo didn't use a CommandCursor object to iterate over the results. diff --git a/mongolog/management/commands/ml_purge.py b/mongolog/management/commands/ml_purge.py new file mode 100644 index 0000000..40e8127 --- /dev/null +++ b/mongolog/management/commands/ml_purge.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +""" +Management command to handle deletion of previous mongolog records. + +Usage Examples: + +# Delete all records alder than 54 day's AFTER backing up the current collection 'bulkupload' + ./manage.py mongolog_purge -d 54 -b -c bulkupload + + # Delete all entries +./manage.py mongolog_purge -p -c bulkupload +""" +from __future__ import print_function +import sys +import logging +from datetime import timedelta +import subprocess +from pymongo import MongoClient + +from mongolog.handlers import get_mongolog_handler + +from django.utils import timezone +from django.core.management.base import BaseCommand + +console = logging.getLogger('mongolog-int') + + +class Command(BaseCommand): + + def add_arguments(self, parser): + parser.add_argument( + '-p', '--purge', default=False, action='store_true', dest='purge', + help='Remove all old results', + ) + parser.add_argument( + '-d', '--delete', default=14, type=int, action='store', dest='delete', + help='Delete documents more than -d={n} days old', + ) + parser.add_argument( + '-f', '--force', default=False, action='store_true', dest='force', + help='Do not prompt before removing documents', + ) + parser.add_argument( + '-b', '--backup', default=False, action='store_true', dest='backup', + help='Backup collection before deleting', + ) + parser.add_argument( + '-l', '--logger', default='mongolog', type=str, action='store', dest='logger', + help='Which mongolog logger to use. The collection defined in the log handler will be used.', + ) + + def __init__(self, *args, **kwargs): + super(Command, self).__init__(*args, **kwargs) + + def confirm(self, **options): + if not options['force']: + while 1: + console.warn("Would you like to proceed? Y/N") + ans = input().strip().lower() + + if ans not in ['y', 'yes', 'n', 'no']: + continue + elif ans[0] == 'n': + console.info("You chose not to continue. Bye!") + sys.exit(1) + elif ans[0] == 'y': + break + + return True + + def purge(self, **options): + """ Purge all records from the collection """ + console.warn("You are about to delete all mongolog documents!!!") + + if self.confirm(**options): + total = self.collection.find({}).count() + self.collection.delete_many({}) + console.warn("Total docs to remove: %s", total) + + def delete(self, **options): + """ Delete all records older than --days={n} """ + days = options['delete'] + console.warn("Looking for documents older than %s day's", days) + + query = { + 'created': { + '$lte': timezone.now() - timedelta(days=days) + } + } + + total = self.collection.find(query).count() + console.warn("Total docs to remove: %s", total) + + if self.confirm(**options): + self.collection.delete_many(query) + console.info("Total docs removed: %s", total) + + def backup(self): + """ Backup the collection before deleting """ + console.info("Backing up your documents...") + cmd = 'mongodump --db mongolog' + subprocess.check_call([cmd], shell=True) + + def handle(self, *args, **options): + """ Main processing handle """ + handler = get_mongolog_handler(logger_name=options['logger']) + client = MongoClient(handler.connection) + db = client.mongolog + self.collection = getattr(db, handler.collection) + + if options['backup']: + self.backup() + + if options['purge']: + self.purge(**options) + elif isinstance(options['delete'], int): + self.delete(**options) diff --git a/mongolog/models.py b/mongolog/models.py index 91c858e..5d55458 100644 --- a/mongolog/models.py +++ b/mongolog/models.py @@ -48,6 +48,11 @@ def find(cls, logger=None, query=None, project=None, uuid=None, level=None, limi client = MongoClient(handler.connection) db = client.mongolog + if logger: + collection = getattr(db, handler.collection) + else: + collection = db.mongolog + aggregate_commands = [] if not query: @@ -72,7 +77,7 @@ def find(cls, logger=None, query=None, project=None, uuid=None, level=None, limi if limit: aggregate_commands.append({"$limit": limit}) - results = db.mongolog.aggregate(aggregate_commands) + results = collection.aggregate(aggregate_commands) return results['result'] if isinstance(results, dict) else results diff --git a/mongolog/tests.py b/mongolog/tests.py index 01ea1d8..a3dc7bb 100644 --- a/mongolog/tests.py +++ b/mongolog/tests.py @@ -41,10 +41,18 @@ ) from mongolog.models import Mongolog +import django +django_version = django.VERSION[0] + from django.core.management import call_command from django.test import TestCase from django.test import Client -from django.core.urlresolvers import reverse + +if django_version < 2: + from django.core.urlresolvers import reverse +else: + from django.urls import reverse + from django.conf import settings LOGGING = settings.LOGGING console = logging.getLogger("console")