Added last_updated field to learner endpoints.

analytics_data_api/v0/models.py

@@ -6,7 +6,6 @@
 # some fields (e.g. Float, Integer) are dynamic and your IDE may highlight them as unavailable
 from elasticsearch_dsl import Date, DocType, Float, Integer, Q, String
 
-
 from analytics_data_api.constants import country, engagement_entity_types, genders, learner
 
 
@@ -215,6 +214,20 @@ class Meta(BaseVideo.Meta):
         db_table = 'video'
 
 
+class RosterUpdate(DocType):
+
+    date = Date()
+
+    # pylint: disable=old-style-class
+    class Meta:
+        index = settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX
+        doc_type = 'marker'
+
+    @classmethod
+    def get_last_updated(cls):
+        return cls.search().query('term', target_index=settings.ELASTICSEARCH_LEARNERS_INDEX).execute()
+
+
 class RosterEntry(DocType):
 
     course_id = String()

analytics_data_api/v0/serializers.py

@@ -317,17 +317,20 @@ class Meta(object):
         )
 
 
+class LastUpdatedSerializer(serializers.Serializer):
+    last_updated = serializers.DateField(source='date', format=settings.DATE_FORMAT)
+
+
 class LearnerSerializer(serializers.Serializer, DefaultIfNoneMixin):
-    username = serializers.CharField()
-    enrollment_mode = serializers.CharField()
-    name = serializers.CharField()
+    username = serializers.CharField(source='username')
+    enrollment_mode = serializers.CharField(source='enrollment_mode')
+    name = serializers.CharField(source='name')
     account_url = serializers.SerializerMethodField('get_account_url')
-    email = serializers.CharField()
+    email = serializers.CharField(source='email')
     segments = serializers.Field(source='segments')
     engagements = serializers.SerializerMethodField('get_engagements')
-    enrollment_date = serializers.DateField(format=settings.DATE_FORMAT)
-    last_updated = serializers.DateField(format=settings.DATE_FORMAT)
-    cohort = serializers.CharField()
+    enrollment_date = serializers.DateField(source='enrollment_date', format=settings.DATE_FORMAT)
+    cohort = serializers.CharField(source='cohort')
 
     def get_account_url(self, obj):
         if settings.LMS_USER_ACCOUNT_BASE_URL:
@@ -426,20 +429,11 @@ def _get_range(self, metric_range):
 
 
 class CourseLearnerMetadataSerializer(serializers.Serializer):
-    enrollment_modes = serializers.SerializerMethodField('get_enrollment_modes')
-    segments = serializers.SerializerMethodField('get_segments')
-    cohorts = serializers.SerializerMethodField('get_cohorts')
+    enrollment_modes = serializers.Field(source='es_data.enrollment_modes')
+    segments = serializers.Field(source='es_data.segments')
+    cohorts = serializers.Field(source='es_data.cohorts')
     engagement_ranges = serializers.SerializerMethodField('get_engagement_ranges')
 
-    def get_enrollment_modes(self, obj):
-        return obj['es_data']['enrollment_modes']
-
-    def get_segments(self, obj):
-        return obj['es_data']['segments']
-
-    def get_cohorts(self, obj):
-        return obj['es_data']['cohorts']
-
     def get_engagement_ranges(self, obj):
         query_set = obj['engagement_ranges']
         engagement_ranges = {

analytics_data_api/v0/tests/views/test_learners.py

@@ -25,10 +25,16 @@ def setUp(self):
         """Creates the index and defines a mapping."""
         super(LearnerAPITestMixin, self).setUp()
         self._es = Elasticsearch([settings.ELASTICSEARCH_LEARNERS_HOST])
-        # delete index if for some reason the index wasn't deleted in tearDown
-        if self._es.indices.exists(index=settings.ELASTICSEARCH_LEARNERS_INDEX):
-            self._es.indices.delete(index=settings.ELASTICSEARCH_LEARNERS_INDEX)
-        self._es.indices.create(index=settings.ELASTICSEARCH_LEARNERS_INDEX)
+
+        for index in [settings.ELASTICSEARCH_LEARNERS_INDEX, settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX]:
+            # delete index if for some reason the index wasn't deleted
+            def delete_index(to_delete):
+                if self._es.indices.exists(index=to_delete):
+                    self._es.indices.delete(index=to_delete)
+            delete_index(index)
+            self.addCleanup(delete_index, index)
+            self._es.indices.create(index=index)
+
         self._es.indices.put_mapping(
             index=settings.ELASTICSEARCH_LEARNERS_INDEX,
             doc_type='roster_entry',
@@ -76,18 +82,25 @@ def setUp(self):
                     'enrollment_date': {
                         'type': 'date', 'doc_values': True
                     },
-                    'last_updated': {
+                }
+            }
+        )
+
+        self._es.indices.put_mapping(
+            index=settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX,
+            doc_type='marker',
+            body={
+                'properties': {
+                    'date': {
                         'type': 'date', 'doc_values': True
                     },
+                    'target_index': {
+                        'type': 'string'
+                    },
                 }
             }
         )
 
-    def tearDown(self):
-        """Remove the index after every test."""
-        super(LearnerAPITestMixin, self).tearDown()
-        self._es.indices.delete(index=settings.ELASTICSEARCH_LEARNERS_INDEX)
-
     def _create_learner(
             self,
             username,
@@ -104,7 +117,6 @@ def _create_learner(
             attempt_ratio_order=0,
             videos_viewed=0,
             enrollment_date='2015-01-28',
-            last_updated='2015-01-28'
     ):
         """Create a single learner roster entry in the elasticsearch index."""
         self._es.create(
@@ -125,7 +137,6 @@ def _create_learner(
                 'attempt_ratio_order': attempt_ratio_order,
                 'videos_viewed': videos_viewed,
                 'enrollment_date': enrollment_date,
-                'last_updated': last_updated
             }
         )
 
@@ -141,6 +152,20 @@ def create_learners(self, learners):
             self._create_learner(**learner)
         self._es.indices.refresh(index=settings.ELASTICSEARCH_LEARNERS_INDEX)
 
+    def create_index_update(self, date=None):
+        """
+        Created an index with the date of when the learner index was updated.
+        """
+        self._es.create(
+            index=settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX,
+            doc_type='marker',
+            body={
+                'date': date,
+                'target_index': settings.ELASTICSEARCH_LEARNERS_INDEX,
+            }
+        )
+        self._es.indices.refresh(index=settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX)
+
 
 @ddt.ddt
 class LearnerTests(VerifyCourseIdMixin, LearnerAPITestMixin, TestCaseWithAuthentication):
@@ -172,8 +197,8 @@ def test_get_user(self, username, name, course_id, enrollment_mode, segments=Non
             "problem_attempts_per_completed": problem_attempts_per_completed,
             "attempt_ratio_order": attempt_ratio_order,
             "enrollment_date": enrollment_date,
-            "last_updated": last_updated,
         }])
+        self.create_index_update(last_updated)
 
         response = self.authenticated_get(self.path_template.format(username, course_id))
         self.assertEquals(response.status_code, 200)
@@ -254,6 +279,7 @@ def assert_learners_returned(self, response, expected_learners):
     def test_all_learners(self):
         usernames = ['dan', 'dennis', 'victor', 'olga', 'gabe', 'brian', 'alison']
         self.create_learners([{'username': username, 'course_id': self.course_id} for username in usernames])
+        self.create_index_update()
         response = self._get(self.course_id)
         # Default ordering is by username
         self.assert_learners_returned(response, [{'username': username} for username in sorted(usernames)])
@@ -263,6 +289,7 @@ def test_course_id(self):
             {'username': 'user_1', 'course_id': self.course_id},
             {'username': 'user_2', 'course_id': 'other/course/id'}
         ])
+        self.create_index_update()
         response = self._get(self.course_id)
         self.assert_learners_returned(response, [{'username': 'user_1'}])
 
@@ -279,6 +306,7 @@ def test_data(self):
             "discussions_contributed": 0,
             "problem_attempts_per_completed": 23.14,
         }])
+        self.create_index_update('2015-09-28')
         response = self._get(self.course_id)
         self.assert_learners_returned(response, [{
             'username': 'user_1',
@@ -291,7 +319,8 @@ def test_data(self):
                 "videos_viewed": 6,
                 "discussions_contributed": 0,
                 "problem_attempts_per_completed": 23.14,
-            }
+            },
+            'last_updated': '2015-09-28',
         }])
 
     @ddt.data(
@@ -339,6 +368,7 @@ def test_filters(
         learner[attribute_name] = attribute_value
         self.create_learners([learner])
         learner.pop('course_id')
+        self.create_index_update()
         response = self._get(self.course_id, **{filter_key: filter_value})
         expected_learners = [learner] if expect_learner else None
         self.assert_learners_returned(response, expected_learners)
@@ -406,6 +436,7 @@ def test_sort(self, learners, order_by, sort_order, expected_users):
         for learner in learners:
             learner['course_id'] = self.course_id
         self.create_learners(learners)
+        self.create_index_update()
         params = dict()
         if order_by:
             params['order_by'] = order_by
@@ -419,6 +450,7 @@ def test_pagination(self):
         expected_page_url_template = 'http://testserver/api/v0/learners/?' \
             '{course_query}&page={page}&page_size={page_size}'
         self.create_learners([{'username': username, 'course_id': self.course_id} for username in usernames])
+        self.create_index_update()
 
         response = self._get(self.course_id, page_size=2)
         payload = json.loads(response.content)

analytics_data_api/v0/views/learners.py

@@ -1,6 +1,8 @@
 """
 API methods for module level data.
 """
+import logging
+
 from rest_framework import generics, status
 
 from analytics_data_api.constants import (
@@ -14,19 +16,38 @@
 from analytics_data_api.v0.models import (
     ModuleEngagement,
     ModuleEngagementMetricRanges,
-    RosterEntry
+    RosterEntry,
+    RosterUpdate,
 )
 from analytics_data_api.v0.serializers import (
     CourseLearnerMetadataSerializer,
     ElasticsearchDSLSearchSerializer,
     EngagementDaySerializer,
+    LastUpdatedSerializer,
     LearnerSerializer,
 )
 from analytics_data_api.v0.views import CourseViewMixin
 from analytics_data_api.v0.views.utils import split_query_argument
 
 
-class LearnerView(CourseViewMixin, generics.RetrieveAPIView):
+logger = logging.getLogger(__name__)
+
+
+class LastUpdateMixin(object):
+
+    @classmethod
+    def get_last_updated(cls):
+        """ Returns the serialized RosterUpdate last_updated field. """
+        roster_update = RosterUpdate.get_last_updated()
+        last_updated = {'date': None}
+        if len(roster_update) == 1:
+            last_updated = roster_update[0]
+        else:
+            logger.warn('RosterUpdate not found.')
+        return LastUpdatedSerializer(last_updated).data
+
+
+class LearnerView(LastUpdateMixin, CourseViewMixin, generics.RetrieveAPIView):
     """
     Get data for a particular learner in a particular course.
 
@@ -73,6 +94,14 @@ def get(self, request, *args, **kwargs):
         self.username = self.kwargs.get('username')
         return super(LearnerView, self).get(request, *args, **kwargs)
 
+    def retrieve(self, request, *args, **kwargs):
+        """
+        Adds the last_updated field to the result.
+        """
+        response = super(LearnerView, self).retrieve(request, args, kwargs)
+        response.data.update(self.get_last_updated())
+        return response
+
     def get_queryset(self):
         return RosterEntry.get_course_user(self.course_id, self.username)
 
@@ -83,7 +112,7 @@ def get_object(self, queryset=None):
         raise LearnerNotFoundError(username=self.username, course_id=self.course_id)
 
 
-class LearnerListView(CourseViewMixin, generics.ListAPIView):
+class LearnerListView(LastUpdateMixin, CourseViewMixin, generics.ListAPIView):
     """
     Get a paginated list of data for all learners in a course.
 
@@ -186,11 +215,20 @@ def _validate_query_params(self):
                 if page_size > self.max_paginate_by or page_size < 1:
                     raise ParameterValueError('Page size must be in the range [1, {}]'.format(self.max_paginate_by))
 
+    def list(self, request, *args, **kwargs):
+        """
+        Adds the last_updated field to the results.
+        """
+        response = super(LearnerListView, self).list(request, args, kwargs)
+        last_updated = self.get_last_updated()
+        for result in response.data['results']:
+            result.update(last_updated)
+        return response
+
     def get_queryset(self):
         """
-        Fetches the user list from elasticsearch.  Note that an
-        elasticsearch_dsl `Search` object is returned, not an actual
-        queryset.
+        Fetches the user list and last updated from elasticsearch returned returned
+        as a an array of dicts with fields "learner" and "last_updated".
         """
         self._validate_query_params()
         query_params = self.request.QUERY_PARAMS

analyticsdataserver/settings/base.py

@@ -54,6 +54,7 @@
 ########## ELASTICSEARCH CONFIGURATION
 ELASTICSEARCH_LEARNERS_HOST = environ.get('ELASTICSEARCH_LEARNERS_HOST', None)
 ELASTICSEARCH_LEARNERS_INDEX = environ.get('ELASTICSEARCH_LEARNERS_INDEX', None)
+ELASTICSEARCH_LEARNERS_UPDATE_INDEX = environ.get('ELASTICSEARCH_LEARNERS_UPDATE_INDEX', None)
 ########## END ELASTICSEARCH CONFIGURATION
 
 ########## GENERAL CONFIGURATION

analyticsdataserver/settings/test.py

@@ -25,3 +25,4 @@
 # Default elasticsearch port when running locally
 ELASTICSEARCH_LEARNERS_HOST = 'http://localhost:9200/'
 ELASTICSEARCH_LEARNERS_INDEX = 'roster_test'
+ELASTICSEARCH_LEARNERS_UPDATE_INDEX = 'index_update_test'