Permalink
Browse files

Crawler: added support for authentication.

  • Loading branch information...
1 parent 8ab5364 commit 2f7c79a3bd2ae6896b37a27ea2a6f377aee13d47 Yury Yurevich committed with Sep 29, 2010
Showing with 31 additions and 0 deletions.
  1. +10 −0 test_utils/crawler/base.py
  2. +21 −0 test_utils/management/commands/crawlurls.py
View
@@ -5,6 +5,7 @@
from django.conf import settings
from django.db import transaction
+from django.views.debug import cleanse_setting
from django.test.client import Client
from django.test.utils import setup_test_environment, teardown_test_environment
@@ -55,6 +56,8 @@ def __init__(self, base_url, conf_urls={}, verbosity=1, output_dir=None, ascend=
self.verbosity = verbosity
self.ascend = ascend
+ auth = kwargs.get('auth')
+
if output_dir:
assert os.path.isdir(output_dir)
self.output_dir = os.path.realpath(output_dir)
@@ -68,6 +71,13 @@ def __init__(self, base_url, conf_urls={}, verbosity=1, output_dir=None, ascend=
self.c = Client(REMOTE_ADDR='127.0.0.1')
+ if auth:
+ printable_auth = ', '.join(
+ '%s: %s' % (key, cleanse_setting(key.upper(), value))
+ for key, value in auth.items())
+ LOG.info('Log in with %s' % printable_auth)
+ self.c.login(**auth)
+
self.plugins = []
for plug in Plugin.__subclasses__():
active = getattr(plug, 'active', True)
@@ -33,6 +33,8 @@ class Command(BaseCommand):
help='If specified, store plugin output in the provided directory'),
make_option('--no-parent', action='store_true', dest="no_parent", default=False,
help='Do not crawl URLs which do not start with your base URL'),
+ make_option('-a', "--auth", action='store', dest='auth', default=None,
+ help='Authenticate (login:user,password:secret) before crawl')
)
help = "Displays all of the url matching routes for the project."
@@ -42,6 +44,8 @@ def handle(self, *args, **options):
verbosity = int(options.get('verbosity', 1))
depth = int(options.get('depth', 3))
+ auth = _parse_auth(options.get('auth'))
+
if verbosity > 1:
log_level = logging.DEBUG
elif verbosity:
@@ -94,6 +98,7 @@ def handle(self, *args, **options):
verbosity=verbosity,
output_dir=options.get("output_dir"),
ascend=not options.get("no_parent"),
+ auth=auth,
)
# Load plugins:
@@ -128,3 +133,19 @@ def handle(self, *args, **options):
sys.exit(1)
else:
sys.exit(0)
+
+
+def _parse_auth(auth):
+ """
+ Parse auth string and return dict.
+
+ >>> _parse_auth('login:user,password:secret')
+ {'login': 'user', 'password': 'secret'}
+
+ >>> _parse_auth('name:user, token:top:secret')
+ {'name': 'user', 'token': 'top:secret'}
+ """
+ if not auth:
+ return None
+ items = auth.split(',')
+ return dict(i.strip().split(':', 1) for i in items)

0 comments on commit 2f7c79a

Please sign in to comment.