Crawler: added support for authentication.

test_utils/crawler/base.py

@@ -5,6 +5,7 @@
 
 from django.conf import settings
 from django.db import transaction
+from django.views.debug import cleanse_setting
 from django.test.client import Client
 from django.test.utils import setup_test_environment, teardown_test_environment
 
@@ -55,6 +56,8 @@ def __init__(self, base_url, conf_urls={}, verbosity=1, output_dir=None, ascend=
         self.verbosity = verbosity
         self.ascend = ascend
 
+        auth = kwargs.get('auth')
+
         if output_dir:
             assert os.path.isdir(output_dir)
             self.output_dir = os.path.realpath(output_dir)
@@ -68,6 +71,13 @@ def __init__(self, base_url, conf_urls={}, verbosity=1, output_dir=None, ascend=
 
         self.c = Client(REMOTE_ADDR='127.0.0.1')
 
+        if auth:
+            printable_auth = ', '.join(
+                '%s: %s' % (key, cleanse_setting(key.upper(), value))
+                for key, value in auth.items())
+            LOG.info('Log in with %s' % printable_auth)
+            self.c.login(**auth)
+
         self.plugins = []
         for plug in Plugin.__subclasses__():
             active = getattr(plug, 'active', True)

test_utils/management/commands/crawlurls.py

@@ -33,6 +33,8 @@ class Command(BaseCommand):
             help='If specified, store plugin output in the provided directory'),
         make_option('--no-parent', action='store_true', dest="no_parent", default=False,
             help='Do not crawl URLs which do not start with your base URL'),
+        make_option('-a', "--auth", action='store', dest='auth', default=None,
+            help='Authenticate (login:user,password:secret) before crawl')
     )
 
     help = "Displays all of the url matching routes for the project."
@@ -42,6 +44,8 @@ def handle(self, *args, **options):
         verbosity = int(options.get('verbosity', 1))
         depth = int(options.get('depth', 3))
 
+        auth = _parse_auth(options.get('auth'))
+
         if verbosity > 1:
             log_level = logging.DEBUG
         elif verbosity:
@@ -94,6 +98,7 @@ def handle(self, *args, **options):
             verbosity=verbosity,
             output_dir=options.get("output_dir"),
             ascend=not options.get("no_parent"),
+            auth=auth,
         )
 
         # Load plugins:
@@ -128,3 +133,19 @@ def handle(self, *args, **options):
             sys.exit(1)
         else:
             sys.exit(0)
+
+
+def _parse_auth(auth):
+    """
+    Parse auth string and return dict.
+
+    >>> _parse_auth('login:user,password:secret')
+    {'login': 'user', 'password': 'secret'}
+
+    >>> _parse_auth('name:user, token:top:secret')
+    {'name': 'user', 'token': 'top:secret'}
+    """
+    if not auth:
+        return None
+    items = auth.split(',')
+    return dict(i.strip().split(':', 1) for i in items)