Merge 12ac1a2 into 5dbad27

.travis.yml

@@ -4,18 +4,15 @@
 # https://docs.djangoproject.com/en/1.9/releases/1.9/#django-1-9-release-notes
 language: python
 python:
-#  - "2.6"  # Just to old to support 
   - "2.7"
   - "3.4"
   - "3.5"
-#  - "3.5-dev" # 3.5 development branch
-#  - "nightly" # currently points to 3.6-dev
+  - "3.6"
+  - "3.7-dev"
 env:
   # Test out support for lowest/highest pymongo/django combos
-  #- PYMONGO="==2.4" DJANGO="==1.8"
-  #- PYMONGO=">=3.0" DJANGO=">=1.9" 
-  - PYMONGO="==2.4"
-  - PYMONGO=">=3.0"
+  - PYMONGO=">=3.0" DJANGO="==1.9.8"
+  - PYMONGO=">=3.0" DJANGO=">=2.0"
 # command to install dependencies
 services:
   - mongodb

AUTHORS

@@ -1,3 +1,4 @@
 If you contribute to mongolog please add your name to the list
 
 * John Furr
+* John Dowling

CHANGELOG

@@ -1,6 +1,13 @@
 Change Log
 ==========
 
+v0.9.0
+------
+    * Removed support for pymongo < 3.0
+    * Removed support for Django < 1.9.8
+    * Added ml_purge command for cleaning out log documents
+    * Log Handler now takes 'collection' keyword to control which mongo collection is used
+
 v0.8.0
 ------
     * Mongolog.find utility for searching the logs

README.rst

@@ -38,9 +38,16 @@ Quick start
                 'mongolog': {
                     'level': 'DEBUG',
                     'class': 'mongolog.SimpleMongoLogHandler',
-                    'connection': 'mongodb://localhost:27017'
+                    
+                    # Set the connection string to the mongo instance.  
+                    'connection': 'mongodb://localhost:27017',
+                    
+                    # define mongo collection the log handler should use.  Default is mongolog
+                    # This is useful if you want different handlers to use different collections
+                    'collection': 'mongolog' 
                 },
             },
+            # Define a logger for your handler.  We are using the root '' logger in this case
             'loggers': {
                 '': {
                     'handlers': ['mongolog'],
@@ -220,6 +227,20 @@ Quick start
             "trace" :
              null
         }
+        
+Management Commands (Django Only)
+---------------------------------
+
+1) ml_purge::
+
+The ml_urge command is used to clean up mongo collections. The command has two basic modes:  --purge and --delete. Purge will remove all documents and delete will remove documents older than {n} day's.
+
+To backup and PURGE all documents from the collection defined in mongolog handler
+    ./manage.py ml_purge --purge --backup -logger mongolog
+
+To remove all documents older than 14 days without backing up first
+    ./manage.py ml_purge --delete 14 -logger mongolog
+
 
 Future  Roadmap
 ---------------

mongolog/management/commands/analog.py

@@ -3,7 +3,6 @@
 from __future__ import print_function
 import logging
 import logging.config
-import django
 import json
 
 import pymongo
@@ -19,38 +18,24 @@
 
 
 class Command(BaseCommand):
-    if django.VERSION[1] <= 7:
-        from optparse import make_option
-        option_list = BaseCommand.option_list + (
-            make_option(
-                '-l', '--limit', default=10, type=int, action='store', dest='limit',
-                help='Delete poll instead of closing it'),
-            make_option(
-                '-t', '--tail', default=False, action='store_true', dest='tail',
-                help='Tail the log file.  By default it will limit to 10 results.  Use --limit to change'),
-            make_option(
-                '-q', '--query', default=None, action='store', dest='query',
-                help='Pass in a search query to mongo.'),
-        )
 
     def __init__(self, *args, **kwargs):
         super(Command, self).__init__(*args, **kwargs)
         self.prev_object_id = None
 
     def add_arguments(self, parser):
-        if django.VERSION[1] >= 7:
-            parser.add_argument(
-                '-l', '--limit', default=10, type=int, action='store', dest='limit',
-                help='Limit Results',
-            )
-            parser.add_argument(
-                '-t', '--tail', default=False, action='store_true', dest='tail',
-                help='Tail the log file.  By default it will limit to 10 results.  Use --limit to change',
-            )
-            parser.add_argument(
-                '-q', '--query', default=None, type=str, action='store', dest='query',
-                help='Pass in a search query to mongo.',
-            )
+        parser.add_argument(
+            '-l', '--limit', default=10, type=int, action='store', dest='limit',
+            help='Limit Results',
+        )
+        parser.add_argument(
+            '-t', '--tail', default=False, action='store_true', dest='tail',
+            help='Tail the log file.  By default it will limit to 10 results.  Use --limit to change',
+        )
+        parser.add_argument(
+            '-q', '--query', default=None, type=str, action='store', dest='query',
+            help='Pass in a search query to mongo.',
+        )
 
     def print_results(self, results):
         # older versions of pymongo didn't use a CommandCursor object to iterate over the results.

mongolog/management/commands/ml_copy.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+"""
+Management command to handle deletion of previous mongolog records.
+
+Usage Examples:
+
+# Used to migrate monoglog data from one collection to another
+ ./manage.py ml_migrate {from_collection} {to_collectioj}
+"""
+from __future__ import print_function
+import sys
+import logging
+from pymongo import MongoClient
+
+from mongolog.handlers import get_mongolog_handler
+
+from django.core.management.base import BaseCommand
+
+console = logging.getLogger('mongolog-int')
+
+
+class Command(BaseCommand):
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '-f', '--from', default='', type=str, action='store', dest='from',
+            help='From Collection',
+        )
+        parser.add_argument(
+            '-t', '--to', default='', type=str, action='store', dest='to',
+            help='To Collection',
+        )
+        parser.add_argument(
+            '-d', '--delete', default=False, action='store_true', dest='delete',
+            help='Delete the from collection',
+        )
+        parser.add_argument(
+            '--force', default=False, action='store_true', dest='force',
+            help='Force delete "from" collection without prompt',
+        )
+
+    def __init__(self, *args, **kwargs):
+        super(Command, self).__init__(*args, **kwargs)
+
+    def confirm(self, **options):
+        if not options['force']:
+            while 1:
+                console.warn("Would you like to proceed?  Y/N")
+                ans = input().strip().lower()
+
+                if ans not in ['y', 'yes', 'n', 'no']:
+                    continue
+                elif ans[0] == 'n':
+                    console.info("You chose not to continue.  Bye!")
+                    sys.exit(1)
+                elif ans[0] == 'y':
+                    break
+
+        return True
+
+    def handle(self, *args, **options):
+        """ Main processing handle """
+        handler = get_mongolog_handler()
+        client = MongoClient(handler.connection)
+        db = client.mongolog
+
+        from_collection = getattr(db, options['from'])
+        from_collection.copyTo(options['to'])
+
+        if options['delete']:
+            console.warn("Deleting from collection")
+            if self.confirm():
+                form_collection.delete_many({})

mongolog/management/commands/ml_purge.py

@@ -0,0 +1,117 @@
+# -*- coding: utf-8 -*-
+"""
+Management command to handle deletion of previous mongolog records.
+
+Usage Examples:
+
+# Delete all records alder than 54 day's AFTER backing up the current collection 'bulkupload'
+ ./manage.py ml_purge -d 54 -b -c bulkupload
+
+ # Delete all entries
+./manage.py ml_purge -p -c bulkupload
+"""
+from __future__ import print_function
+import sys
+import logging
+from datetime import timedelta
+import subprocess
+from pymongo import MongoClient
+
+from mongolog.handlers import get_mongolog_handler
+
+from django.utils import timezone
+from django.core.management.base import BaseCommand
+
+console = logging.getLogger('mongolog-int')
+
+
+class Command(BaseCommand):
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '-p', '--purge', default=False, action='store_true', dest='purge',
+            help='Remove all old results',
+        )
+        parser.add_argument(
+            '-d', '--delete', default=14, type=int, action='store', dest='delete',
+            help='Delete documents more than -d={n} days old',
+        )
+        parser.add_argument(
+            '-f', '--force', default=False, action='store_true', dest='force',
+            help='Do not prompt before removing documents',
+        )
+        parser.add_argument(
+            '-b', '--backup', default=False, action='store_true', dest='backup',
+            help='Backup collection before deleting',
+        )
+        parser.add_argument(
+            '-l', '--logger', default='mongolog', type=str, action='store', dest='logger',
+            help='Which mongolog logger to use.  The collection defined in the log handler will be used.',
+        )
+
+    def __init__(self, *args, **kwargs):
+        super(Command, self).__init__(*args, **kwargs)
+
+    def confirm(self, **options):
+        if not options['force']:
+            while 1:
+                console.warn("Would you like to proceed?  Y/N")
+                ans = input().strip().lower()
+
+                if ans not in ['y', 'yes', 'n', 'no']:
+                    continue
+                elif ans[0] == 'n':
+                    console.info("You chose not to continue.  Bye!")
+                    sys.exit(1)
+                elif ans[0] == 'y':
+                    break
+
+        return True
+
+    def purge(self, **options):
+        """ Purge all records from the collection """
+        console.warn("You are about to delete all mongolog documents!!!")
+
+        if self.confirm(**options):
+            total = self.collection.find({}).count()
+            self.collection.delete_many({})
+            console.warn("Total docs to remove: %s", total)
+
+    def delete(self, **options):
+        """ Delete all records older than --delete={n} """
+        days = options['delete']
+        console.warn("Looking for documents older than %s day's", days)
+
+        query = {
+            'created': {
+                '$lte': timezone.now() - timedelta(days=days)
+            }
+        }
+
+        total = self.collection.find(query).count()
+        console.warn("Total docs to remove: %s", total)
+
+        if self.confirm(**options):
+            self.collection.delete_many(query)
+            console.info("Total docs removed: %s", total)
+
+    def backup(self):
+        """ Backup the collection before deleting """
+        console.info("Backing up your documents...")
+        cmd = 'mongodump --db mongolog'
+        subprocess.check_call([cmd], shell=True)
+
+    def handle(self, *args, **options):
+        """ Main processing handle """
+        handler = get_mongolog_handler(logger_name=options['logger'])
+        client = MongoClient(handler.connection)
+        db = client.mongolog
+        self.collection = getattr(db, handler.collection)
+
+        if options['backup']:
+            self.backup()
+
+        if options['purge']:
+            self.purge(**options)
+        elif isinstance(options['delete'], int):
+            self.delete(**options)

mongolog/models.py

@@ -48,6 +48,11 @@ def find(cls, logger=None, query=None, project=None, uuid=None, level=None, limi
         client = MongoClient(handler.connection)
         db = client.mongolog
 
+        if logger:
+            collection = getattr(db, handler.collection)
+        else:
+            collection = db.mongolog
+
         aggregate_commands = []
 
         if not query:
@@ -72,7 +77,7 @@ def find(cls, logger=None, query=None, project=None, uuid=None, level=None, limi
         if limit:
             aggregate_commands.append({"$limit": limit})
 
-        results = db.mongolog.aggregate(aggregate_commands)
+        results = collection.aggregate(aggregate_commands)
         return results['result'] if isinstance(results, dict) else results
 
 

mongolog/tests.py

@@ -41,10 +41,18 @@
 )
 from mongolog.models import Mongolog
 
+import django
+django_version = django.VERSION[0]
+
 from django.core.management import call_command
 from django.test import TestCase
 from django.test import Client
-from django.core.urlresolvers import reverse
+
+if django_version < 2:
+    from django.core.urlresolvers import reverse
+else:
+    from django.urls import reverse
+
 from django.conf import settings
 LOGGING = settings.LOGGING
 console = logging.getLogger("console")