Merge b1c98a3 into 4bae2ca

docker-compose.yml

@@ -33,6 +33,7 @@ web:
     LORE_ADMIN_EMAIL:
     LORE_CAS_URL:
     CELERY_ALWAYS_EAGER: False
+    CELERY_RESULT_BACKEND: redis://redis:6379/4
     BROKER_URL: redis://redis:6379/4
     HAYSTACK_URL: elastic:9200
   ports:

exporter/api.py

@@ -0,0 +1,74 @@
+"""
+Functions for exporting learning resources
+"""
+
+from __future__ import unicode_literals
+
+from tempfile import mkdtemp, mkstemp
+import tarfile
+import os
+from shutil import rmtree
+
+from django.core.files.storage import default_storage
+from lore.settings import EXPORT_PATH_PREFIX
+
+
+def export_resources_to_directory(learning_resources, tempdir):
+    """
+    Create XML files of learning resource contents inside directory.
+
+    Args:
+        learning_resources (list of learningresources.models.LearningResource):
+            LearningResources to export in tarball
+    """
+    def sanitize(title):
+        """Sanitize title for use in filename."""
+        # Limit filename to 200 characters since limit is 256
+        # and we have a number and extension too.
+        return title.replace("/", "_")[:200]
+
+    for learning_resource in learning_resources:
+        filename = "{id}_{title}.xml".format(
+            id=learning_resource.id,
+            title=sanitize(learning_resource.title),
+        )
+        with open(os.path.join(tempdir, filename), 'w') as f:
+            f.write(learning_resource.content_xml)
+
+
+def export_resources_to_tarball(learning_resources, username):
+    """
+    Create tarball and put learning resource contents in it as XML files.
+
+    Args:
+        learning_resources (list of learningresources.models.LearningResource):
+            LearningResources to export in tarball
+        username (unicode): Name of user
+    Returns:
+        unicode: path of tarball within django-storage
+    """
+    tempdir = mkdtemp()
+    handle, tempfilepath = mkstemp()
+    os.close(handle)
+    try:
+        archive = tarfile.open(tempfilepath, mode='w:gz')
+
+        export_resources_to_directory(learning_resources, tempdir)
+
+        for name in os.listdir(tempdir):
+            abs_path = os.path.join(tempdir, name)
+            if os.path.isfile(abs_path):
+                archive.add(abs_path, arcname=name)
+
+        archive.close()
+        output_path = '{prefix}{username}_exports.tar.gz'.format(
+            prefix=EXPORT_PATH_PREFIX,
+            username=username
+        )
+
+        # Remove any old paths.
+        default_storage.delete(output_path)
+        return default_storage.save(output_path, open(tempfilepath, 'rb'))
+    finally:
+        os.unlink(tempfilepath)
+        rmtree(tempdir)

exporter/tasks.py

@@ -0,0 +1,23 @@
+"""
+Exporter tasks for LORE
+"""
+
+from __future__ import unicode_literals
+
+from lore.celery import async
+from exporter.api import export_resources_to_tarball
+
+
+@async.task
+def export_resources(learning_resources, username):
+    """
+    Asynchronously export learning resources as tarball.
+
+    Args:
+        learning_resources (list of learningresources.models.LearningResource):
+            LearningResources to export in tarball
+        username (unicode): Name of user
+    Returns:
+        unicode: Path of tarball in django-storage
+    """
+    return export_resources_to_tarball(learning_resources, username)

exporter/tests/test_export.py

@@ -0,0 +1,133 @@
+"""
+Tests for export
+"""
+
+from __future__ import unicode_literals
+from tempfile import mkdtemp
+from shutil import rmtree
+import os
+
+from archive import Archive
+from django.core.files.storage import default_storage
+
+from learningresources.tests.base import LoreTestCase
+from learningresources.models import (
+    LearningResource,
+    LearningResourceType,
+)
+from learningresources.api import update_description_path
+from exporter.api import (
+    export_resources_to_directory,
+    export_resources_to_tarball,
+)
+from exporter.tasks import export_resources
+from learningresources.api import create_resource
+from importer.tasks import import_file
+
+
+class TestExport(LoreTestCase):
+    """
+    Tests for export
+    """
+
+    def setUp(self):
+        super(TestExport, self).setUp()
+
+        # Add some LearningResources on top of the default to make things
+        # interesting.
+        tarball_file = self.get_course_single_tarball()
+        import_file(
+            tarball_file, self.repo.id, self.user.id)
+
+        # Add a resource with a '/' in the title and too many characters.
+        resource = create_resource(
+            course=self.repo.course_set.first(),
+            resource_type=LearningResourceType.objects.first().name,
+            title="//x"*300,
+            content_xml="",
+            mpath="",
+            url_name=None,
+            parent=None,
+            dpath=''
+        )
+        update_description_path(resource)
+
+    def assert_resource_directory(self, resources, tempdir):
+        """Assert that files are present with correct content."""
+        def sanitize(title):
+            """Sanitize title for use in filename."""
+            # Limit filename to 200 characters since limit is 256
+            # and we have a number and extension too.
+            return title.replace("/", "_")[:200]
+
+        def make_name(resource):
+            """Format expected filename."""
+            return "{id}_{title}.xml".format(
+                id=resource.id,
+                title=sanitize(resource.title),
+            )
+
+        for resource in resources:
+            abs_path = os.path.join(tempdir, make_name(resource))
+            self.assertTrue(os.path.exists(abs_path))
+            with open(abs_path) as f:
+                self.assertEqual(f.read(), resource.content_xml)
+
+    def test_export_resources_to_directory(self):
+        """Test exporting learning resources to directory."""
+        resources = LearningResource.objects.all()
+
+        tempdir = mkdtemp()
+        try:
+            export_resources_to_directory(resources, tempdir)
+            self.assert_resource_directory(resources, tempdir)
+        finally:
+            rmtree(tempdir)
+
+    def test_export_resources_to_tarball(self):
+        """Test exporting learning resources to tarball."""
+        resources = LearningResource.objects.all()
+
+        path = export_resources_to_tarball(resources, self.user.username)
+        tempdir = mkdtemp()
+
+        # HACK: Have to patch in "seekable" attribute for python3 and tar
+        # See: https://code.djangoproject.com/ticket/24963#ticket. Remove
+        # when updating to Django 1.9
+        def seekable():
+            """Hacked seekable for django storage to work in python3."""
+            return True
+        try:
+            resource_archive = default_storage.open(path)
+            resource_archive.seekable = seekable
+            Archive(resource_archive, ext='.tar.gz').extract(
+                to_path=tempdir, method='safe'
+            )
+            self.assert_resource_directory(resources, tempdir)
+        finally:
+            rmtree(tempdir)
+            default_storage.delete(path)
+
+    def test_export_task(self):
+        """Test exporting resources task."""
+        resources = LearningResource.objects.all()
+
+        path = export_resources.delay(resources, self.user.username).get()
+        tempdir = mkdtemp()
+
+        # HACK: Have to patch in "seekable" attribute for python3 and tar
+        # See: https://code.djangoproject.com/ticket/24963#ticket. Remove
+        # when updating to Django 1.9
+        def seekable():
+            """Hacked seekable for django storage to work in python3."""
+            return True
+        try:
+            resource_archive = default_storage.open(path)
+            resource_archive.seekable = seekable
+            Archive(resource_archive, ext='.tar.gz').extract(
+                to_path=tempdir, method='safe'
+            )
+            self.assert_resource_directory(resources, tempdir)
+        finally:
+            rmtree(tempdir)
+            default_storage.delete(path)

learningresources/models.py

@@ -24,7 +24,8 @@
 
 # defining the file path max length
 FILE_PATH_MAX_LENGTH = 900
-STATIC_ASSET_BASEPATH = 'assets/{org}/{course_number}/{run}/'
+STATIC_ASSET_PREFIX = 'assets'
+STATIC_ASSET_BASEPATH = STATIC_ASSET_PREFIX + '/{org}/{course_number}/{run}/'
 
 
 class FilePathLengthException(Exception):
@@ -65,7 +66,7 @@ def course_asset_basepath(course, filename):
         (unicode): forward slash separated path to use below
             ``settings.MEDIA_ROOT``.
     """
-    return 'assets/{org}/{course_number}/{run}/{filename}'.format(
+    return (STATIC_ASSET_BASEPATH + '{filename}').format(
         org=course.org,
         course_number=course.course_number,
         run=course.run,

lore/settings.py

@@ -26,7 +26,7 @@
 
 
 def load_fallback():
-    """Load optional yaml config"""
+    """Load optional yaml config."""
     fallback_config = {}
     config_file_path = None
     for config_path in CONFIG_PATHS:
@@ -42,7 +42,7 @@ def load_fallback():
 
 
 def get_var(name, default):
-    """Return the settings in a precedence way with default"""
+    """Return the settings in a precedence way with default."""
     try:
         value = os.environ.get(name, FALLBACK_CONFIG.get(name, default))
         return ast.literal_eval(value)
@@ -88,6 +88,7 @@ def get_var(name, default):
     'audit',
     'learningresources',
     'importer',
+    'exporter',
     'ui',
     'taxonomy',
     'rest',
@@ -194,6 +195,7 @@ def get_var(name, default):
 
 # Media and storage settings
 IMPORT_PATH_PREFIX = get_var('LORE_IMPORT_PATH_PREFIX', 'course_archives/')
+EXPORT_PATH_PREFIX = get_var('LORE_EXPORT_PATH_PREFIX', 'resource_exports/')
 MEDIA_ROOT = get_var('MEDIA_ROOT', '/tmp/')
 MEDIA_URL = '/media/'
 LORE_USE_S3 = get_var('LORE_USE_S3', False)
@@ -331,6 +333,8 @@ def get_var(name, default):
     "CELERY_RESULT_BACKEND", get_var("REDISCLOUD_URL", None)
 )
 CELERY_ALWAYS_EAGER = get_var("CELERY_ALWAYS_EAGER", True)
+CELERY_EAGER_PROPAGATES_EXCEPTIONS = get_var(
+    "CELERY_EAGER_PROPAGATES_EXCEPTIONS", True)
 
 # guardian specific settings
 ANONYMOUS_USER_ID = None