diff --git a/analytics_data_api/v0/models.py b/analytics_data_api/v0/models.py index acf97ed3..b861926d 100644 --- a/analytics_data_api/v0/models.py +++ b/analytics_data_api/v0/models.py @@ -6,7 +6,6 @@ # some fields (e.g. Float, Integer) are dynamic and your IDE may highlight them as unavailable from elasticsearch_dsl import Date, DocType, Float, Integer, Q, String - from analytics_data_api.constants import country, engagement_entity_types, genders, learner @@ -215,6 +214,20 @@ class Meta(BaseVideo.Meta): db_table = 'video' +class RosterUpdate(DocType): + + date = Date() + + # pylint: disable=old-style-class + class Meta: + index = settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX + doc_type = 'marker' + + @classmethod + def get_last_updated(cls): + return cls.search().query('term', target_index=settings.ELASTICSEARCH_LEARNERS_INDEX).execute() + + class RosterEntry(DocType): course_id = String() diff --git a/analytics_data_api/v0/serializers.py b/analytics_data_api/v0/serializers.py index d578b71d..b391efbc 100644 --- a/analytics_data_api/v0/serializers.py +++ b/analytics_data_api/v0/serializers.py @@ -317,17 +317,20 @@ class Meta(object): ) +class LastUpdatedSerializer(serializers.Serializer): + last_updated = serializers.DateField(source='date', format=settings.DATE_FORMAT) + + class LearnerSerializer(serializers.Serializer, DefaultIfNoneMixin): - username = serializers.CharField() - enrollment_mode = serializers.CharField() - name = serializers.CharField() + username = serializers.CharField(source='username') + enrollment_mode = serializers.CharField(source='enrollment_mode') + name = serializers.CharField(source='name') account_url = serializers.SerializerMethodField('get_account_url') - email = serializers.CharField() + email = serializers.CharField(source='email') segments = serializers.Field(source='segments') engagements = serializers.SerializerMethodField('get_engagements') - enrollment_date = serializers.DateField(format=settings.DATE_FORMAT) - last_updated = serializers.DateField(format=settings.DATE_FORMAT) - cohort = serializers.CharField() + enrollment_date = serializers.DateField(source='enrollment_date', format=settings.DATE_FORMAT) + cohort = serializers.CharField(source='cohort') def get_account_url(self, obj): if settings.LMS_USER_ACCOUNT_BASE_URL: @@ -426,20 +429,11 @@ def _get_range(self, metric_range): class CourseLearnerMetadataSerializer(serializers.Serializer): - enrollment_modes = serializers.SerializerMethodField('get_enrollment_modes') - segments = serializers.SerializerMethodField('get_segments') - cohorts = serializers.SerializerMethodField('get_cohorts') + enrollment_modes = serializers.Field(source='es_data.enrollment_modes') + segments = serializers.Field(source='es_data.segments') + cohorts = serializers.Field(source='es_data.cohorts') engagement_ranges = serializers.SerializerMethodField('get_engagement_ranges') - def get_enrollment_modes(self, obj): - return obj['es_data']['enrollment_modes'] - - def get_segments(self, obj): - return obj['es_data']['segments'] - - def get_cohorts(self, obj): - return obj['es_data']['cohorts'] - def get_engagement_ranges(self, obj): query_set = obj['engagement_ranges'] engagement_ranges = { diff --git a/analytics_data_api/v0/tests/views/test_learners.py b/analytics_data_api/v0/tests/views/test_learners.py index 5cec6854..7629dbd8 100644 --- a/analytics_data_api/v0/tests/views/test_learners.py +++ b/analytics_data_api/v0/tests/views/test_learners.py @@ -25,10 +25,16 @@ def setUp(self): """Creates the index and defines a mapping.""" super(LearnerAPITestMixin, self).setUp() self._es = Elasticsearch([settings.ELASTICSEARCH_LEARNERS_HOST]) - # delete index if for some reason the index wasn't deleted in tearDown - if self._es.indices.exists(index=settings.ELASTICSEARCH_LEARNERS_INDEX): - self._es.indices.delete(index=settings.ELASTICSEARCH_LEARNERS_INDEX) - self._es.indices.create(index=settings.ELASTICSEARCH_LEARNERS_INDEX) + + for index in [settings.ELASTICSEARCH_LEARNERS_INDEX, settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX]: + # delete index if for some reason the index wasn't deleted + def delete_index(to_delete): + if self._es.indices.exists(index=to_delete): + self._es.indices.delete(index=to_delete) + delete_index(index) + self.addCleanup(delete_index, index) + self._es.indices.create(index=index) + self._es.indices.put_mapping( index=settings.ELASTICSEARCH_LEARNERS_INDEX, doc_type='roster_entry', @@ -76,18 +82,25 @@ def setUp(self): 'enrollment_date': { 'type': 'date', 'doc_values': True }, - 'last_updated': { + } + } + ) + + self._es.indices.put_mapping( + index=settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX, + doc_type='marker', + body={ + 'properties': { + 'date': { 'type': 'date', 'doc_values': True }, + 'target_index': { + 'type': 'string' + }, } } ) - def tearDown(self): - """Remove the index after every test.""" - super(LearnerAPITestMixin, self).tearDown() - self._es.indices.delete(index=settings.ELASTICSEARCH_LEARNERS_INDEX) - def _create_learner( self, username, @@ -104,7 +117,6 @@ def _create_learner( attempt_ratio_order=0, videos_viewed=0, enrollment_date='2015-01-28', - last_updated='2015-01-28' ): """Create a single learner roster entry in the elasticsearch index.""" self._es.create( @@ -125,7 +137,6 @@ def _create_learner( 'attempt_ratio_order': attempt_ratio_order, 'videos_viewed': videos_viewed, 'enrollment_date': enrollment_date, - 'last_updated': last_updated } ) @@ -141,6 +152,20 @@ def create_learners(self, learners): self._create_learner(**learner) self._es.indices.refresh(index=settings.ELASTICSEARCH_LEARNERS_INDEX) + def create_index_update(self, date=None): + """ + Created an index with the date of when the learner index was updated. + """ + self._es.create( + index=settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX, + doc_type='marker', + body={ + 'date': date, + 'target_index': settings.ELASTICSEARCH_LEARNERS_INDEX, + } + ) + self._es.indices.refresh(index=settings.ELASTICSEARCH_LEARNERS_UPDATE_INDEX) + @ddt.ddt class LearnerTests(VerifyCourseIdMixin, LearnerAPITestMixin, TestCaseWithAuthentication): @@ -172,8 +197,8 @@ def test_get_user(self, username, name, course_id, enrollment_mode, segments=Non "problem_attempts_per_completed": problem_attempts_per_completed, "attempt_ratio_order": attempt_ratio_order, "enrollment_date": enrollment_date, - "last_updated": last_updated, }]) + self.create_index_update(last_updated) response = self.authenticated_get(self.path_template.format(username, course_id)) self.assertEquals(response.status_code, 200) @@ -254,6 +279,7 @@ def assert_learners_returned(self, response, expected_learners): def test_all_learners(self): usernames = ['dan', 'dennis', 'victor', 'olga', 'gabe', 'brian', 'alison'] self.create_learners([{'username': username, 'course_id': self.course_id} for username in usernames]) + self.create_index_update() response = self._get(self.course_id) # Default ordering is by username self.assert_learners_returned(response, [{'username': username} for username in sorted(usernames)]) @@ -263,6 +289,7 @@ def test_course_id(self): {'username': 'user_1', 'course_id': self.course_id}, {'username': 'user_2', 'course_id': 'other/course/id'} ]) + self.create_index_update() response = self._get(self.course_id) self.assert_learners_returned(response, [{'username': 'user_1'}]) @@ -279,6 +306,7 @@ def test_data(self): "discussions_contributed": 0, "problem_attempts_per_completed": 23.14, }]) + self.create_index_update('2015-09-28') response = self._get(self.course_id) self.assert_learners_returned(response, [{ 'username': 'user_1', @@ -291,7 +319,8 @@ def test_data(self): "videos_viewed": 6, "discussions_contributed": 0, "problem_attempts_per_completed": 23.14, - } + }, + 'last_updated': '2015-09-28', }]) @ddt.data( @@ -339,6 +368,7 @@ def test_filters( learner[attribute_name] = attribute_value self.create_learners([learner]) learner.pop('course_id') + self.create_index_update() response = self._get(self.course_id, **{filter_key: filter_value}) expected_learners = [learner] if expect_learner else None self.assert_learners_returned(response, expected_learners) @@ -406,6 +436,7 @@ def test_sort(self, learners, order_by, sort_order, expected_users): for learner in learners: learner['course_id'] = self.course_id self.create_learners(learners) + self.create_index_update() params = dict() if order_by: params['order_by'] = order_by @@ -419,6 +450,7 @@ def test_pagination(self): expected_page_url_template = 'http://testserver/api/v0/learners/?' \ '{course_query}&page={page}&page_size={page_size}' self.create_learners([{'username': username, 'course_id': self.course_id} for username in usernames]) + self.create_index_update() response = self._get(self.course_id, page_size=2) payload = json.loads(response.content) diff --git a/analytics_data_api/v0/views/learners.py b/analytics_data_api/v0/views/learners.py index 5f7f7ee0..381f4020 100644 --- a/analytics_data_api/v0/views/learners.py +++ b/analytics_data_api/v0/views/learners.py @@ -1,6 +1,8 @@ """ API methods for module level data. """ +import logging + from rest_framework import generics, status from analytics_data_api.constants import ( @@ -14,19 +16,38 @@ from analytics_data_api.v0.models import ( ModuleEngagement, ModuleEngagementMetricRanges, - RosterEntry + RosterEntry, + RosterUpdate, ) from analytics_data_api.v0.serializers import ( CourseLearnerMetadataSerializer, ElasticsearchDSLSearchSerializer, EngagementDaySerializer, + LastUpdatedSerializer, LearnerSerializer, ) from analytics_data_api.v0.views import CourseViewMixin from analytics_data_api.v0.views.utils import split_query_argument -class LearnerView(CourseViewMixin, generics.RetrieveAPIView): +logger = logging.getLogger(__name__) + + +class LastUpdateMixin(object): + + @classmethod + def get_last_updated(cls): + """ Returns the serialized RosterUpdate last_updated field. """ + roster_update = RosterUpdate.get_last_updated() + last_updated = {'date': None} + if len(roster_update) == 1: + last_updated = roster_update[0] + else: + logger.warn('RosterUpdate not found.') + return LastUpdatedSerializer(last_updated).data + + +class LearnerView(LastUpdateMixin, CourseViewMixin, generics.RetrieveAPIView): """ Get data for a particular learner in a particular course. @@ -73,6 +94,14 @@ def get(self, request, *args, **kwargs): self.username = self.kwargs.get('username') return super(LearnerView, self).get(request, *args, **kwargs) + def retrieve(self, request, *args, **kwargs): + """ + Adds the last_updated field to the result. + """ + response = super(LearnerView, self).retrieve(request, args, kwargs) + response.data.update(self.get_last_updated()) + return response + def get_queryset(self): return RosterEntry.get_course_user(self.course_id, self.username) @@ -83,7 +112,7 @@ def get_object(self, queryset=None): raise LearnerNotFoundError(username=self.username, course_id=self.course_id) -class LearnerListView(CourseViewMixin, generics.ListAPIView): +class LearnerListView(LastUpdateMixin, CourseViewMixin, generics.ListAPIView): """ Get a paginated list of data for all learners in a course. @@ -186,11 +215,20 @@ def _validate_query_params(self): if page_size > self.max_paginate_by or page_size < 1: raise ParameterValueError('Page size must be in the range [1, {}]'.format(self.max_paginate_by)) + def list(self, request, *args, **kwargs): + """ + Adds the last_updated field to the results. + """ + response = super(LearnerListView, self).list(request, args, kwargs) + last_updated = self.get_last_updated() + for result in response.data['results']: + result.update(last_updated) + return response + def get_queryset(self): """ - Fetches the user list from elasticsearch. Note that an - elasticsearch_dsl `Search` object is returned, not an actual - queryset. + Fetches the user list and last updated from elasticsearch returned returned + as a an array of dicts with fields "learner" and "last_updated". """ self._validate_query_params() query_params = self.request.QUERY_PARAMS diff --git a/analyticsdataserver/settings/base.py b/analyticsdataserver/settings/base.py index 0c16636b..4ca110df 100644 --- a/analyticsdataserver/settings/base.py +++ b/analyticsdataserver/settings/base.py @@ -54,6 +54,7 @@ ########## ELASTICSEARCH CONFIGURATION ELASTICSEARCH_LEARNERS_HOST = environ.get('ELASTICSEARCH_LEARNERS_HOST', None) ELASTICSEARCH_LEARNERS_INDEX = environ.get('ELASTICSEARCH_LEARNERS_INDEX', None) +ELASTICSEARCH_LEARNERS_UPDATE_INDEX = environ.get('ELASTICSEARCH_LEARNERS_UPDATE_INDEX', None) ########## END ELASTICSEARCH CONFIGURATION ########## GENERAL CONFIGURATION diff --git a/analyticsdataserver/settings/test.py b/analyticsdataserver/settings/test.py index eea4a7b9..0f705d92 100644 --- a/analyticsdataserver/settings/test.py +++ b/analyticsdataserver/settings/test.py @@ -25,3 +25,4 @@ # Default elasticsearch port when running locally ELASTICSEARCH_LEARNERS_HOST = 'http://localhost:9200/' ELASTICSEARCH_LEARNERS_INDEX = 'roster_test' +ELASTICSEARCH_LEARNERS_UPDATE_INDEX = 'index_update_test'