Merge b0012d4 into 90e861d

analytics_data_api/utils.py

@@ -2,8 +2,19 @@
 from importlib import import_module
 
 from django.db.models import Q
+from django.conf import settings
+from django.core.files.storage import default_storage
+from django.core.exceptions import SuspiciousFileOperation, SuspiciousOperation
 from rest_framework.authtoken.models import Token
 
+from analytics_data_api.v0.exceptions import (
+    ReportFileNotFoundError,
+    CannotCreateReportDownloadLinkError
+)
+
+AWS_S3_FILE_STORAGE_CLASS = 'storages.backends.s3boto.S3BotoStorage'
+ISO_8601_FORMAT_STRING = '%Y-%m-%dT%H:%M:%SZ'
+
 
 def delete_user_auth_token(username):
     """
@@ -84,3 +95,117 @@ def date_range(start_date, end_date, delta=datetime.timedelta(days=1)):
     while cur_date < end_date:
         yield cur_date
         cur_date += delta
+
+
+def get_course_report_download_details(course_id, report_name):
+    """
+    Determine the path that the report file should be located at,
+    then return metadata sufficient for downloading it.
+    """
+    report_location_template = getattr(
+        settings,
+        'COURSE_REPORT_FILE_LOCATION_TEMPLATE',
+        '/{course_id}/{report_name}'
+    )
+    report_location = report_location_template.format(
+        course_id=course_id,
+        report_name=report_name
+    )
+    try:
+        if not default_storage.exists(report_location):
+            raise ReportFileNotFoundError(course_id=course_id, report_name=report_name)
+    except (
+            AttributeError,
+            NotImplementedError,
+            ImportError,
+            SuspiciousFileOperation,
+            SuspiciousOperation
+    ):
+        # Error out if:
+        # - We don't have a method to determine file existence
+        # - Such a method isn't implemented
+        # - We can't import the specified storage class
+        # - We don't have privileges for the specified file location
+        raise CannotCreateReportDownloadLinkError
+
+    try:
+        last_modified = default_storage.modified_time(report_location)
+    except (NotImplementedError, AttributeError):
+        last_modified = None
+
+    try:
+        download_size = default_storage.size(report_location)
+    except (NotImplementedError, AttributeError):
+        download_size = None
+
+    # Note that course IDs contain characters that aren't valid for filenames in
+    # Windows. However, we'll trust the client to save them with the appropriate
+    # alternate naming scheme (replace ':' with '_' and so on)
+    download_filename = '{}-{}-{}.csv'.format(
+        course_id,
+        report_name,
+        # We need a date for the filename; if we don't know when it was last modified,
+        # use the current date and time to stamp the filename.
+        (last_modified or datetime.datetime.utcnow()).strftime('%Y%m%dT%H%M%SZ')
+    ).replace(
+        # Remove characters that aren't filename-friendly
+        ':', '-'
+    ).replace(
+        '+', '-'
+    )
+    url, expiration_date = get_file_object_url(report_location, download_filename)
+
+    details = {
+        'course_id': course_id,
+        'report_name': report_name,
+        'download_url': url
+    }
+    # These are all optional items that aren't guaranteed. The URL isn't guaranteed
+    # either, but we'll raise an exception earlier if we don't have it. Currently, support
+    # is only present for S3, which has all these data types, but this future-proofs
+    # us for scenarios where we might have a storage provider that doesn't support them.
+    if last_modified is not None:
+        details.update({'last_modified': last_modified.strftime(ISO_8601_FORMAT_STRING)})
+    if expiration_date is not None:
+        details.update({'expiration_date': expiration_date.strftime(ISO_8601_FORMAT_STRING)})
+    if download_size is not None:
+        details.update({'file_size': download_size})
+    return details
+
+
+def get_file_object_url(filename, download_filename):
+    """
+    Retrieve a download URL for the file, as well as a datetime object
+    indicating when the URL expires.
+
+    We need to pass extra details to the URL method, above and beyond just the
+    file location, to give us what we need.
+
+    Currently, this method only supports S3 storage, and will need to be changed to
+    support building URLs on other storage providers.
+    """
+    # Default to expiring the link after two minutes
+    expire_length = getattr(settings, 'COURSE_REPORT_DOWNLOAD_EXPIRY_TIME', 120)
+    expires_at = get_expiration_date(expire_length)
+    try:
+        url = default_storage.url(
+            name=filename,
+            response_headers={
+                'response-content-disposition': 'attachment; filename={}'.format(download_filename),
+                'response-content-type': 'text/csv',
+                # The Expires header requires a very particular timestamp format
+                'response-expires': expires_at.strftime('%a, %d %b %Y %H:%M:%S GMT')
+            },
+            expire=expire_length
+        )
+    except (AttributeError, TypeError, NotImplementedError):
+        # Either we can't find a .url() method, or we can't use it. Raise an exception.
+        raise CannotCreateReportDownloadLinkError
+    return url, expires_at
+
+
+def get_expiration_date(seconds):
+    """
+    Determine when a given link will expire, based on a given lifetime
+    """
+    return datetime.datetime.utcnow() + datetime.timedelta(seconds=seconds)

analytics_data_api/v0/exceptions.py

@@ -72,3 +72,26 @@ class ParameterValueError(BaseError):
     def __init__(self, message, *args, **kwargs):
         super(ParameterValueError, self).__init__(*args, **kwargs)
         self.message = message
+
+
+class ReportFileNotFoundError(BaseError):
+    """
+    Raise if we couldn't find the file we need to produce the report
+    """
+    def __init__(self, *args, **kwargs):
+        course_id = kwargs.pop('course_id')
+        report_name = kwargs.pop('report_name')
+        super(ReportFileNotFoundError, self).__init__(*args, **kwargs)
+        self.message = self.message_template.format(course_id=course_id, report_name=report_name)
+
+    @property
+    def message_template(self):
+        return 'Could not find report \'{report_name}\' for course {course_id}.'
+
+
+class CannotCreateReportDownloadLinkError(BaseError):
+    """
+    Raise if we cannot create a link for the file to be downloaded
+    """
+
+    message = 'Could not create a downloadable link to the report.'

analytics_data_api/v0/middleware.py

@@ -8,6 +8,8 @@
     LearnerEngagementTimelineNotFoundError,
     LearnerNotFoundError,
     ParameterValueError,
+    ReportFileNotFoundError,
+    CannotCreateReportDownloadLinkError,
 )
 
 
@@ -129,3 +131,39 @@ def error_code(self):
     @property
     def status_code(self):
         return status.HTTP_400_BAD_REQUEST
+
+
+class ReportFileNotFoundErrorMiddleware(BaseProcessErrorMiddleware):
+    """
+    Raise 404 if the report file isn't present
+    """
+
+    @property
+    def error(self):
+        return ReportFileNotFoundError
+
+    @property
+    def error_code(self):
+        return 'report_file_not_found'
+
+    @property
+    def status_code(self):
+        return status.HTTP_404_NOT_FOUND
+
+
+class CannotCreateDownloadLinkErrorMiddleware(BaseProcessErrorMiddleware):
+    """
+    Raise 501 if the filesystem doesn't support creating download links
+    """
+
+    @property
+    def error(self):
+        return CannotCreateReportDownloadLinkError
+
+    @property
+    def error_code(self):
+        return 'cannot_create_report_download_link'
+
+    @property
+    def status_code(self):
+        return status.HTTP_501_NOT_IMPLEMENTED

analytics_data_api/v0/tests/test_connections.py

@@ -33,7 +33,7 @@ def test_signing(self):
         self.assertTrue('my_access_key' in auth_header)
 
     def test_timeout(self):
-        def fake_connection(_address):
+        def fake_connection(_address, _timeout):
             raise socket.timeout('fake error')
         socket.create_connection = fake_connection
         connection = ESConnection('mockservice.cc-zone-1.amazonaws.com',

analytics_data_api/v0/tests/views/test_courses.py

@@ -12,6 +12,7 @@
 from django.conf import settings
 from django_dynamic_fixture import G
 import pytz
+from mock import patch, Mock
 
 from analytics_data_api.constants.country import get_country
 from analytics_data_api.v0 import models
@@ -781,3 +782,119 @@ def test_get(self):
     def test_get_404(self):
         response = self._get_data('foo/bar/course')
         self.assertEquals(response.status_code, 404)
+
+
+class CourseReportDownloadViewTests(DemoCourseMixin, TestCaseWithAuthentication):
+
+    path = '/api/v0/courses/{course_id}/reports/{report_name}'
+
+    @patch('django.core.files.storage.default_storage.exists', Mock(return_value=False))
+    def test_report_file_not_found(self):
+        response = self.authenticated_get(
+            self.path.format(
+                course_id=DEMO_COURSE_ID,
+                report_name='problem_response'
+            )
+        )
+        self.assertEqual(response.status_code, 404)
+
+    def test_report_not_supported(self):
+        response = self.authenticated_get(
+            self.path.format(
+                course_id=DEMO_COURSE_ID,
+                report_name='fake_problem_that_we_dont_support'
+            )
+        )
+        self.assertEqual(response.status_code, 404)
+
+    @patch('django.conf.settings.DEFAULT_FILE_STORAGE', 'storages.backends.ftp.FTPStorage')
+    def test_storage_provider_other_than_s3(self):
+        response = self.authenticated_get(
+            self.path.format(
+                course_id=DEMO_COURSE_ID,
+                report_name='problem_response'
+            )
+        )
+        self.assertEqual(response.status_code, 501)
+
+    @patch('django.core.files.storage.default_storage.exists', Mock(return_value=True))
+    @patch('django.core.files.storage.default_storage.url', Mock(return_value='http://fake'))
+    @patch(
+        'django.core.files.storage.default_storage.modified_time',
+        Mock(return_value=datetime.datetime(2014, 1, 1, tzinfo=pytz.utc))
+    )
+    @patch('django.core.files.storage.default_storage.size', Mock(return_value=1000))
+    @patch(
+        'analytics_data_api.utils.get_expiration_date',
+        Mock(return_value=datetime.datetime(2014, 1, 1, tzinfo=pytz.utc))
+    )
+    def test_make_working_link(self):
+        response = self.authenticated_get(
+            self.path.format(
+                course_id=DEMO_COURSE_ID,
+                report_name='problem_response'
+            )
+        )
+        self.assertEqual(response.status_code, 200)
+        expected = {
+            'course_id': DEMO_COURSE_ID,
+            'report_name': 'problem_response',
+            'download_url': 'http://fake',
+            'last_modified': '2014-01-01T00:00:00Z',
+            'expiration_date': datetime.datetime(2014, 1, 1, tzinfo=pytz.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
+            'file_size': 1000
+        }
+        self.assertEqual(response.data, expected)
+
+    @patch('django.core.files.storage.default_storage.exists', Mock(return_value=True))
+    @patch('django.core.files.storage.default_storage.url', Mock(return_value='http://fake'))
+    @patch(
+        'django.core.files.storage.default_storage.modified_time',
+        Mock(return_value=datetime.datetime(2014, 1, 1, tzinfo=pytz.utc))
+    )
+    @patch('django.core.files.storage.default_storage.size', Mock(side_effect=NotImplementedError()))
+    @patch(
+        'analytics_data_api.utils.get_expiration_date',
+        Mock(return_value=datetime.datetime(2014, 1, 1, tzinfo=pytz.utc))
+    )
+    def test_make_working_link_with_missing_size(self):
+        response = self.authenticated_get(
+            self.path.format(
+                course_id=DEMO_COURSE_ID,
+                report_name='problem_response'
+            )
+        )
+        self.assertEqual(response.status_code, 200)
+        expected = {
+            'course_id': DEMO_COURSE_ID,
+            'report_name': 'problem_response',
+            'download_url': 'http://fake',
+            'last_modified': '2014-01-01T00:00:00Z',
+            'expiration_date': datetime.datetime(2014, 1, 1, tzinfo=pytz.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+        }
+        self.assertEqual(response.data, expected)
+
+    @patch('django.core.files.storage.default_storage.exists', Mock(return_value=True))
+    @patch('django.core.files.storage.default_storage.url', Mock(return_value='http://fake'))
+    @patch('django.core.files.storage.default_storage.modified_time', Mock(side_effect=NotImplementedError()))
+    @patch('django.core.files.storage.default_storage.size', Mock(return_value=1000))
+    @patch(
+        'analytics_data_api.utils.get_expiration_date',
+        Mock(return_value=datetime.datetime(2014, 1, 1, tzinfo=pytz.utc))
+    )
+    def test_make_working_link_with_missing_last_modified_date(self):
+        response = self.authenticated_get(
+            self.path.format(
+                course_id=DEMO_COURSE_ID,
+                report_name='problem_response'
+            )
+        )
+        self.assertEqual(response.status_code, 200)
+        expected = {
+            'course_id': DEMO_COURSE_ID,
+            'report_name': 'problem_response',
+            'download_url': 'http://fake',
+            'file_size': 1000,
+            'expiration_date': datetime.datetime(2014, 1, 1, tzinfo=pytz.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+        }
+        self.assertEqual(response.data, expected)

analytics_data_api/v0/urls/courses.py

@@ -14,7 +14,8 @@
     ('enrollment/location', views.CourseEnrollmentByLocationView, 'enrollment_by_location'),
     ('problems', views.ProblemsListView, 'problems'),
     ('problems_and_tags', views.ProblemsAndTagsListView, 'problems_and_tags'),
-    ('videos', views.VideosListView, 'videos')
+    ('videos', views.VideosListView, 'videos'),
+    ('reports/(?P<report_name>[a-zA-Z0-9_]+)', views.ReportDownloadView, 'reports'),
 ]
 
 urlpatterns = []