Merge pull request #8005 from rtibbles/remote_content

Remote content serving

kolibri/core/assets/src/core-app/urls.js

@@ -65,6 +65,13 @@ const urls = {
     }
     return generateUrl(this.__contentUrl, { url: `${filename[0]}/${filename[1]}/${filename}` });
   },
+  downloadUrl(fileId, extension) {
+    const filename = `${fileId}.${extension}`;
+    if (!this.__contentUrl) {
+      throw new ReferenceError('Content Url is not defined');
+    }
+    return generateUrl(this.__contentUrl, { url: `${filename[0]}/${filename[1]}/${filename}` });
+  },
 };
 
 export default urls;

kolibri/core/assets/src/views/ContentRenderer/DownloadButton.vue

@@ -11,6 +11,7 @@
 
 <script>
 
+  import urls from 'kolibri.urls';
   import { getFilePresetString } from './filePresetStrings';
 
   export default {
@@ -20,21 +21,55 @@
         type: Array,
         default: () => [],
       },
+      nodeTitle: {
+        type: String,
+        default: '',
+      },
     },
     computed: {
       fileOptions() {
-        return this.files.map(file => ({
-          label: getFilePresetString(file),
-          url: file.download_url,
-        }));
+        return this.files.map(file => {
+          const label = getFilePresetString(file);
+          return {
+            label,
+            url: urls.downloadUrl(file.checksum, file.extension),
+            fileName: this.$tr('downloadFilename', {
+              resourceTitle: this.nodeTitle,
+              fileExtension: file.extension,
+              fileId: file.checksum.slice(0, 6),
+            }),
+          };
+        });
       },
     },
     methods: {
       download(file) {
-        window.open(file.url, '_blank');
+        const req = new XMLHttpRequest();
+        req.open('GET', file.url, true);
+        req.responseType = 'blob';
+
+        req.onload = function() {
+          const blob = req.response;
+          const blobUrl = window.URL.createObjectURL(blob);
+          try {
+            const a = document.createElement('a');
+            a.download = file.fileName;
+            a.href = blobUrl;
+            document.body.appendChild(a);
+            a.click();
+            a.remove();
+          } catch (e) {
+            window.open(file.url, '_blank');
+          }
+        };
+
+        req.send();
       },
     },
-    $trs: { downloadContent: 'Download resource' },
+    $trs: {
+      downloadContent: 'Download resource',
+      downloadFilename: '{ resourceTitle } ({ fileId }).{ fileExtension }',
+    },
   };
 
 </script>

kolibri/core/assets/test/download-button.spec.js

@@ -2,17 +2,21 @@ import { mount } from '@vue/test-utils';
 import store from 'kolibri.coreVue.vuex.store';
 import DownloadButton from '../src/views/ContentRenderer/DownloadButton';
 
+jest.mock('kolibri.urls');
+
 describe('download-button Component', function() {
   const samplesFiles = [
     {
       file_size: 100000,
       preset: 'high_res_video',
-      download_url: '/downloadcontent/3893fd801427402ad07487c5d2d35119.mp4/Math_Low_Resolution.mp4',
+      extension: 'mp4',
+      checksum: '3893fd801427402ad07487c5d2d35119',
     },
     {
       file_size: 500,
       preset: 'thumbnail',
-      download_url: '/downloadcontent/187598e1f4596bf4492f5a205922b633.jpg/Math_Thumbnail.jpg',
+      extension: 'jpg',
+      checksum: '187598e1f4596bf4492f5a205922b633',
     },
   ];
 

kolibri/core/content/api.py

@@ -5,7 +5,6 @@
 
 import requests
 from django.core.cache import cache
-from django.core.urlresolvers import reverse
 from django.db.models import Exists
 from django.db.models import OuterRef
 from django.db.models import Q
@@ -52,7 +51,6 @@
     get_channel_stats_from_studio,
 )
 from kolibri.core.content.utils.paths import get_channel_lookup_url
-from kolibri.core.content.utils.paths import get_content_file_name
 from kolibri.core.content.utils.paths import get_info_url
 from kolibri.core.content.utils.paths import get_local_content_storage_file_url
 from kolibri.core.content.utils.stopwords import stopwords_set
@@ -231,25 +229,18 @@ def map_lang(obj):
     return output
 
 
-def map_file(file, obj):
-    url_lookup = {
-        "available": file["available"],
-        "id": file["checksum"],
-        "extension": file["extension"],
-    }
-    download_filename = models.get_download_filename(
-        obj["title"],
-        models.PRESET_LOOKUP.get(file["preset"], _("Unknown format")),
-        file["extension"],
-    )
-    file["download_url"] = reverse(
-        "kolibri:core:downloadcontent",
-        kwargs={
-            "filename": get_content_file_name(url_lookup),
-            "new_filename": download_filename,
-        },
+def map_file(file):
+    file["checksum"] = file.pop("local_file__id")
+    file["available"] = file.pop("local_file__available")
+    file["file_size"] = file.pop("local_file__file_size")
+    file["extension"] = file.pop("local_file__extension")
+    file["storage_url"] = get_local_content_storage_file_url(
+        {
+            "available": file["available"],
+            "id": file["checksum"],
+            "extension": file["extension"],
+        }
     )
-    file["storage_url"] = get_local_content_storage_file_url(url_lookup)
     file["lang"] = map_lang(file)
     return file
 
@@ -331,11 +322,7 @@ def consolidate(self, items, queryset):
             ):
                 if f["contentnode"] not in files:
                     files[f["contentnode"]] = []
-                f["checksum"] = f.pop("local_file__id")
-                f["available"] = f.pop("local_file__available")
-                f["file_size"] = f.pop("local_file__file_size")
-                f["extension"] = f.pop("local_file__extension")
-                files[f["contentnode"]].append(f)
+                files[f["contentnode"]].append(map_file(f))
 
             ancestors = queryset.get_ancestors().values(
                 "id", "title", "lft", "rght", "tree_id"
@@ -356,10 +343,8 @@ def consolidate(self, items, queryset):
 
             for item in items:
                 item["assessmentmetadata"] = assessmentmetadata.get(item["id"])
-                item["files"] = list(
-                    map(lambda x: map_file(x, item), files.get(item["id"], []))
-                )
                 item["tags"] = tags.get(item["id"], [])
+                item["files"] = files.get(item["id"], [])
 
                 lft = item.pop("lft")
                 rght = item.pop("rght")

kolibri/core/content/management/commands/exportcontent.py

@@ -1,6 +1,8 @@
 import logging
 import os
 
+from django.core.management.base import CommandError
+
 from ...utils import paths
 from ...utils import transfer
 from kolibri.core.content.errors import InvalidStorageFilenameError
@@ -60,6 +62,8 @@ def update_job_metadata(self, total_bytes_to_transfer, total_resource_count):
             job.save_meta()
 
     def handle_async(self, *args, **options):
+        if paths.using_remote_storage():
+            raise CommandError("Cannot export files when using remote file storage")
         channel_id = options["channel_id"]
         data_dir = os.path.realpath(options["destination"])
         node_ids = options["node_ids"]

kolibri/core/content/management/commands/importcontent.py

@@ -294,8 +294,10 @@ def _transfer(  # noqa: max-complexity=16
                     overall_progress_update(f.file_size)
                     continue
 
-                # if the file already exists, add its size to our overall progress, and skip
-                if os.path.isfile(dest) and os.path.getsize(dest) == f.file_size:
+                # if the file already exists, or we are using remote storage, add its size to our overall progress, and skip
+                if paths.using_remote_storage() or (
+                    os.path.isfile(dest) and os.path.getsize(dest) == f.file_size
+                ):
                     overall_progress_update(f.file_size)
                     file_checksums_to_annotate.append(f.id)
                     transferred_file_size += f.file_size

kolibri/core/content/models.py

@@ -25,14 +25,12 @@
 import os
 from gettext import gettext as _
 
-from django.core.urlresolvers import reverse
 from django.db import connection
 from django.db import models
 from django.db.models import Min
 from django.db.models import Q
 from django.db.models import QuerySet
 from django.utils.encoding import python_2_unicode_compatible
-from django.utils.text import get_valid_filename
 from le_utils.constants import content_kinds
 from le_utils.constants import format_presets
 from mptt.managers import TreeManager
@@ -194,15 +192,6 @@ def __str__(self):
         return self.lang_name or ""
 
 
-def get_download_filename(title, preset, extension):
-    """
-    Return a valid filename to be downloaded as.
-    """
-    filename = "{} ({}).{}".format(title, preset, extension)
-    valid_filename = get_valid_filename(filename)
-    return valid_filename
-
-
 class File(base_models.File):
     """
     The second to bottom layer of the contentDB schema, defines the basic building brick for content.
@@ -230,27 +219,6 @@ def get_preset(self):
         """
         return PRESET_LOOKUP.get(self.preset, _("Unknown format"))
 
-    def get_download_filename(self):
-        """
-        Return a valid filename to be downloaded as.
-        """
-        return get_download_filename(
-            self.contentnode.title, self.get_preset(), self.get_extension()
-        )
-
-    def get_download_url(self):
-        """
-        Return the download url.
-        """
-        new_filename = self.get_download_filename()
-        return reverse(
-            "kolibri:core:downloadcontent",
-            kwargs={
-                "filename": self.local_file.get_filename(),
-                "new_filename": new_filename,
-            },
-        )
-
 
 class LocalFileQueryset(models.QuerySet, FilterByUUIDQuerysetMixin):
     def delete_unused_files(self):

kolibri/core/content/serializers.py

@@ -133,7 +133,6 @@ class Meta:
 class FileSerializer(serializers.ModelSerializer):
     checksum = serializers.CharField(source="local_file_id")
     storage_url = serializers.SerializerMethodField()
-    download_url = serializers.SerializerMethodField()
     extension = serializers.SerializerMethodField()
     file_size = serializers.SerializerMethodField()
     lang = LanguageSerializer()
@@ -142,9 +141,6 @@ class FileSerializer(serializers.ModelSerializer):
     def get_storage_url(self, target_node):
         return target_node.get_storage_url()
 
-    def get_download_url(self, target_node):
-        return target_node.get_download_url()
-
     def get_extension(self, target_node):
         return target_node.get_extension()
 
@@ -165,7 +161,6 @@ class Meta:
             "lang",
             "supplementary",
             "thumbnail",
-            "download_url",
         )
 
 

kolibri/core/content/urls.py

@@ -1,14 +1,7 @@
 from django.conf.urls import url
 
 from .views import ContentPermalinkRedirect
-from .views import DownloadContentView
 
 urlpatterns = [
-    url(
-        r"^downloadcontent/(?P<filename>[^/]+)/(?P<new_filename>.*)",
-        DownloadContentView.as_view(),
-        {},
-        "downloadcontent",
-    ),
     url(r"^viewcontent$", ContentPermalinkRedirect.as_view(), name="contentpermalink"),
 ]