add integration tests + cleanup

Makefile

@@ -1,5 +1,8 @@
 test:
 	. aws-credentials && ./runtests.py 
 
+test-coverage:
+	. aws-credentials && coverage run --source collectfast ./runtests.py
+
 distribute:
 	python setup.py sdist bdist_wheel upload

collectfast/management/commands/collectstatic.py

@@ -4,6 +4,7 @@
 
 from django.contrib.staticfiles.management.commands import collectstatic
 from django.utils.encoding import smart_str
+from storages.backends.s3boto3 import S3Boto3Storage
 
 from collectfast.etag import should_copy_file
 from collectfast import settings
@@ -27,6 +28,7 @@ def add_arguments(self, parser):
 
     def __init__(self, *args, **kwargs):
         super(Command, self).__init__(*args, **kwargs)
+        self.num_copied_files = 0
         self.tasks = []
         self.etags = {}
         self.storage.preload_metadata = True
@@ -61,12 +63,25 @@ def collect(self):
             Pool(settings.threads).map(self.do_copy_file, self.tasks)
         return ret
 
+    def handle(self, **options):
+        """
+        Override handle to supress summary output
+        """
+        super(Command, self).handle(**options)
+        return "{} static file{} copied.".format(
+            self.num_copied_files,
+            '' if self.num_copied_files == 1 else 's')
+
     def do_copy_file(self, args):
         """
         Determine if file should be copied or not and handle exceptions.
         """
         path, prefixed_path, source_storage = args
 
+        if settings.threads and isinstance(self.storage, S3Boto3Storage):
+            # reset connection
+            self.storage._connection = None
+
         if self.collectfast_enabled and not self.dry_run:
             try:
                 if not should_copy_file(
@@ -80,6 +95,7 @@ def do_copy_file(self, args):
                     "Ignored error in Collectfast:\n%s\n--> Continuing using "
                     "default collectstatic." % e))
 
+        self.num_copied_files += 1
         return super(Command, self).copy_file(
             path, prefixed_path, source_storage)
 

collectfast/tests/test_command.py

@@ -1,20 +1,82 @@
+from functools import wraps
+import warnings
+
 from django.core.management import call_command
 from django.utils.six import StringIO
 
-from .utils import test, create_static_file
+from collectfast import settings
+from .utils import test, create_static_file, clean_static_dir
+
+
+def override_setting(name, value):
+    def decorator(fn):
+        @wraps(fn)
+        def wrapper(*args, **kwargs):
+            original = getattr(settings, name)
+            setattr(settings, name, value)
+            ret = fn(*args, **kwargs)
+            setattr(settings, name, original)
+            return ret
+        return wrapper
+    return decorator
 
 
-def call_collectstatic():
+def call_collectstatic(*args, **kwargs):
     out = StringIO()
-    call_command('collectstatic', interactive=False, stdout=out)
+    call_command(
+        'collectstatic', *args, interactive=False, stdout=out, **kwargs)
     return out.getvalue()
 
 
 @test
 def test_basics(case):
-    create_static_file('static/testfile.txt')
+    clean_static_dir()
+    create_static_file()
+    result = call_collectstatic()
+    case.assertIn("1 static file copied.", result)
+    # file state should now be cached
+    result = call_collectstatic()
+    case.assertIn("0 static files copied.", result)
+
+
+@test
+@override_setting("threads", 5)
+def test_threads(case):
+    clean_static_dir()
+    create_static_file()
     result = call_collectstatic()
     case.assertIn("1 static file copied.", result)
     # file state should now be cached
     result = call_collectstatic()
     case.assertIn("0 static files copied.", result)
+
+
+@test
+@override_setting("preload_metadata_enabled", False)
+def test_warn_preload_metadata(case):
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
+        call_collectstatic()
+        case.assertIn('AWS_PRELOAD_METADATA', str(w[0].message))
+
+
+@test
+@override_setting("enabled", False)
+def test_collectfast_disabled(case):
+    call_collectstatic()
+
+
+@test
+def test_disable_collectfast(case):
+    clean_static_dir()
+    create_static_file()
+    result = call_collectstatic(disable_collectfast=True)
+    case.assertIn("1 static file copied.", result)
+
+
+@test
+def test_ignore_etag_deprecated(case):
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
+        call_collectstatic(ignore_etag=True)
+        case.assertIn('ignore-etag is deprecated', str(w[0].message))

collectfast/tests/utils.py

@@ -1,5 +1,7 @@
 import random
 import unittest
+import uuid
+import os
 
 
 def test(func):
@@ -15,9 +17,18 @@ def test(func):
     return type(func.__name__, (unittest.TestCase, ), {func.__name__: func})
 
 
-def create_static_file(filename):
+def create_static_file():
+    filename = 'static/%s.txt' % uuid.uuid4()
     for i in range(3):
         with open(filename, 'w+') as f:
             for i in range(500):
                 f.write(chr(int(random.random() * 64)))
             f.close()
+
+
+def clean_static_dir():
+    d = 'static/'
+    for f in os.listdir(d):
+        path = os.path.join(d, f)
+        if os.path.isfile(path):
+            os.unlink(path)

runtests.py

@@ -34,6 +34,8 @@ def main():
         os.makedirs(staticroot_dir)
 
     settings.configure(**{
+        # Set USE_TZ to True to work around bug in django-storages
+        "USE_TZ": True,
         "DATABASES": {
             'default': {
                 "ENGINE": 'django.db.backends.%s' % options.DATABASE_ENGINE,

setup.py

@@ -14,7 +14,7 @@
     url='https://github.com/antonagestam/collectfast/',
     license='MIT License',
     include_package_data=True,
-    install_requires=['Django>=1.8'],
+    install_requires=['Django>=1.8', 'django-storages'],
     classifiers=[
         'Environment :: Web Environment',
         'Intended Audience :: Developers',