MAU Pipeline, models, and updated views

devsite/requirements/hawthorn.txt

@@ -66,3 +66,4 @@ pytest-mock==1.7.1
 pytest-pythonpath==0.7.2
 pytest-cov==2.6.0
 tox==3.7.0
+freezegun==0.3.12

devsite/requirements/hawthorn_appsembler.txt

@@ -70,3 +70,4 @@ pytest-mock==1.7.1
 pytest-pythonpath==0.7.2
 pytest-cov==2.6.0
 tox==3.7.0
+freezegun==0.3.12

figures/filters.py

@@ -23,7 +23,12 @@
 from student.models import CourseEnrollment
 
 from figures.pipeline.course_daily_metrics import get_enrolled_in_exclude_admins
-from figures.models import CourseDailyMetrics, SiteDailyMetrics
+from figures.models import (
+    CourseDailyMetrics,
+    SiteDailyMetrics,
+    CourseMauMetrics,
+    SiteMauMetrics,
+)
 
 
 def char_method_filter(method):
@@ -181,6 +186,39 @@ class Meta:
         fields = ['date_for', 'date']
 
 
+class CourseMauMetricsFilter(django_filters.FilterSet):
+    """Provides filtering for CourseMauMetrics model objects
+
+
+    Use ``date_for`` to retrieve a specific date
+    Use ``date_0`` and ``date_1`` for retrieving values in a date range, inclusive
+    each of these can be used singly to get:
+    * ``date_0`` to get records greater than or equal
+    * ``date_1`` to get records less than or equal
+    """
+    date = django_filters.DateFromToRangeFilter(name='date_for')
+
+    class Meta:
+        model = CourseMauMetrics
+        fields = ['date_for', 'date', 'course_id', ]
+
+
+class SiteMauMetricsFilter(django_filters.FilterSet):
+    """Provides filtering for SiteDailyMetrics model objects
+
+    Use ``date_for`` for retrieving a specific date
+    Use ``date_0`` and ``date_1`` for retrieving values in a date range, inclusive
+    each of these can be used singly to get:
+    * ``date_0`` to get records greater than or equal
+    * ``date_1`` to get records less than or equal
+    """
+    date = django_filters.DateFromToRangeFilter(name='date_for')
+
+    class Meta:
+        model = SiteMauMetrics
+        fields = ['date_for', 'date']
+
+
 class SiteFilterSet(django_filters.FilterSet):
     """
     Note: The Site filter has no knowledge of a default site, nor should it

figures/mau.py

@@ -1,3 +1,15 @@
+"""
+This module provides MAU metrics retrieval functionality
+"""
+
+from datetime import datetime
+
+from figures.models import CourseMauMetrics, SiteMauMetrics
+from figures.sites import (
+    get_course_keys_for_site,
+    get_student_modules_for_site,
+    get_student_modules_for_course_in_site,
+)
 
 
 def get_mau_from_student_modules(student_modules, year, month):
@@ -12,3 +24,75 @@ def get_mau_from_student_modules(student_modules, year, month):
     qs = student_modules.filter(modified__year=year,
                                 modified__month=month)
     return qs.values_list('student__id', flat=True).distinct()
+
+
+def retrieve_live_site_mau_data(site):
+    """
+    Used this when we need to retrieve unique active users for the
+    whole site
+    """
+    student_modules = get_student_modules_for_site(site)
+    today = datetime.utcnow()
+    users = get_mau_from_student_modules(student_modules=student_modules,
+                                         year=today.year,
+                                         month=today.month)
+    return dict(
+        count=users.count(),
+        month_for=today.date(),
+        domain=site.domain,
+    )
+
+
+def retrieve_live_course_mau_data(site, course_id):
+    """
+    Used this when we need to retrieve unique active users for a given course
+    in the site
+    """
+    student_modules = get_student_modules_for_course_in_site(site, course_id)
+    today = datetime.utcnow()
+    users = get_mau_from_student_modules(student_modules=student_modules,
+                                         year=today.year,
+                                         month=today.month)
+    return dict(
+        count=users.count(),
+        month_for=today.date(),
+        course_id=str(course_id),
+        domain=site.domain,
+    )
+
+
+def store_mau_metrics(site, overwrite=False):
+    """
+    Save "snapshot" of MAU metrics
+    """
+    today = datetime.utcnow()
+
+    # get site data
+    student_modules = get_student_modules_for_site(site)
+    site_mau = get_mau_from_student_modules(student_modules=student_modules,
+                                            year=today.year,
+                                            month=today.month)
+
+    # store site data
+    site_mau_obj, created = SiteMauMetrics.save_metrics(site=site,
+                                                        date_for=today.date(),
+                                                        data=dict(mau=site_mau.count()),
+                                                        overwrite=overwrite)
+    course_mau_objects = []
+    for course_key in get_course_keys_for_site(site):
+        course_student_modules = student_modules.filter(course_id=course_key)
+        course_mau = get_mau_from_student_modules(
+            student_modules=course_student_modules,
+            year=today.year,
+            month=today.month)
+        course_mau_obj, created = CourseMauMetrics.save_metrics(
+                site=site,
+                course_id=str(course_key),
+                date_for=today.date(),
+                data=dict(mau=course_mau.count()),
+                overwrite=overwrite)
+
+        course_mau_objects.append(course_mau_obj)
+
+    return dict(smo=site_mau_obj,
+                cmos=course_mau_objects)

figures/migrations/0009_mau_metrics.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.15 on 2019-11-15 17:34
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import figures.models
+import model_utils.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sites', '0002_alter_domain_unique'),
+        ('figures', '0008_cdm_meta_update'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CourseMauMetrics',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
+                ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
+                ('date_for', models.DateField()),
+                ('course_id', models.CharField(max_length=255)),
+                ('mau', models.IntegerField()),
+                ('site', models.ForeignKey(default=figures.models.default_site, on_delete=django.db.models.deletion.CASCADE, to='sites.Site')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='SiteMauMetrics',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
+                ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
+                ('date_for', models.DateField()),
+                ('mau', models.IntegerField()),
+                ('site', models.ForeignKey(default=figures.models.default_site, on_delete=django.db.models.deletion.CASCADE, to='sites.Site')),
+            ],
+        ),
+        migrations.AlterUniqueTogether(
+            name='sitemaumetrics',
+            unique_together=set([('site', 'date_for')]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='coursemaumetrics',
+            unique_together=set([('site', 'course_id', 'date_for')]),
+        ),
+    ]

figures/models.py

@@ -13,6 +13,14 @@
 from model_utils.models import TimeStampedModel
 
 
+def default_site():
+    """
+    Wrapper aroung `django.conf.settings.SITE_ID` so we do not have to create a
+    new migration if we change how we get the default site ID
+    """
+    return settings.SITE_ID
+
+
 @python_2_unicode_compatible
 class CourseDailyMetrics(TimeStampedModel):
     """Metrics data specific to an individual course
@@ -201,3 +209,117 @@ class Meta:
 
     def __str__(self):
         return "{}, {}, {}".format(self.id, self.created, self.error_type)
+
+
+class BaseDateMetricsModel(TimeStampedModel):
+    site = models.ForeignKey(Site, default=default_site)
+    date_for = models.DateField()
+
+    class Meta:
+        abstract = True
+        ordering = ['-date_for']
+
+    @property
+    def year(self):
+        return self.date_for.year
+
+    @property
+    def month(self):
+        return self.date_for.month
+
+
+class SiteMauMetricsManager(models.Manager):
+    """Custom model manager for SiteMauMMetrics model
+    """
+    def latest_for_site_month(self, site, year, month):
+        """Return the latest record for the given site, month, and year
+        If no record found, returns 'None'
+        """
+        return self.filter(site=site,
+                           date_for__year=year,
+                           date_for__month=month).order_by('-modified').first()
+
+
+@python_2_unicode_compatible
+class SiteMauMetrics(BaseDateMetricsModel):
+
+    mau = models.IntegerField()
+    objects = SiteMauMetricsManager()
+
+    class Meta:
+        unique_together = ('site', 'date_for',)
+
+    @classmethod
+    def save_metrics(cls, site, date_for, data, overwrite=False):
+        """
+        Convenience method to save metrics data with option to
+        overwrite an existing record
+        """
+        if not overwrite:
+            try:
+                obj = SiteMauMetrics.objects.get(site=site, date_for=date_for)
+                return (obj, False,)
+            except SiteMauMetrics.DoesNotExist:
+                pass
+
+        return SiteMauMetrics.objects.update_or_create(site=site,
+                                                       date_for=date_for,
+                                                       defaults=dict(
+                                                            mau=data['mau']))
+
+    def __str__(self):
+        return '{}, {}, {}, {}'.format(self.id,
+                                       self.site.domain,
+                                       self.date_for,
+                                       self.mau)
+
+
+class CourseMauMetricsManager(models.Manager):
+    """Custom model manager for CourseMauMetrics model
+    """
+    def latest_for_course_month(self, site, course_id, year, month):
+        """Return the latest record for the given site, course_id, month and year
+        If no record found, returns 'None'
+        """
+        return self.filter(site=site,
+                           course_id=course_id,
+                           date_for__year=year,
+                           date_for__month=month).order_by('-modified').first()
+
+
+@python_2_unicode_compatible
+class CourseMauMetrics(BaseDateMetricsModel):
+    course_id = models.CharField(max_length=255)
+    mau = models.IntegerField()
+    objects = CourseMauMetricsManager()
+
+    class Meta:
+        unique_together = ('site', 'course_id', 'date_for',)
+
+    @classmethod
+    def save_metrics(cls, site, course_id, date_for, data, overwrite=False):
+        """
+        Convenience method to save metrics data with option to
+        overwrite an existing record
+        """
+        if not overwrite:
+            try:
+                obj = CourseMauMetrics.objects.get(site=site,
+                                                   course_id=course_id,
+                                                   date_for=date_for)
+                return (obj, False,)
+            except CourseMauMetrics.DoesNotExist:
+                pass
+
+        return CourseMauMetrics.objects.update_or_create(site=site,
+                                                         course_id=course_id,
+                                                         date_for=date_for,
+                                                         defaults=dict(
+                                                            mau=data['mau']))
+
+    def __str__(self):
+        return '{}, {}, {}, {}, {}'.format(self.id,
+                                           self.site.domain,
+                                           self.course_id,
+                                           self.date_for,
+                                           self.mau)

figures/serializers.py

@@ -44,7 +44,9 @@
     )
 from figures.models import (
     CourseDailyMetrics,
+    CourseMauMetrics,
     SiteDailyMetrics,
+    SiteMauMetrics,
     LearnerCourseGradeMetrics,
     PipelineError,
     )
@@ -733,13 +735,31 @@ def get_profile_image(self, user):
             return None
 
 
-class MauSiteMonthMetricsSerializer(serializers.Serializer):
+class CourseMauMetricsSerializer(serializers.ModelSerializer):
+    domain = serializers.CharField(source='site.domain')
+
+    class Meta:
+        model = CourseMauMetrics
+        exclude = ()
+        fields = ['mau', 'date_for', 'domain', 'course_id']
+
+
+class SiteMauMetricsSerializer(serializers.ModelSerializer):
+    domain = serializers.CharField(source='site.domain')
+
+    class Meta:
+        model = SiteMauMetrics
+        fields = ['mau', 'date_for', 'domain']
+
+
+class SiteMauLiveMetricsSerializer(serializers.Serializer):
+
     month_for = serializers.DateField()
     count = serializers.IntegerField()
     domain = serializers.CharField()
 
 
-class MauCourseMonthMetricsSerializer(serializers.Serializer):
+class CourseMauLiveMetricsSerializer(serializers.Serializer):
     month_for = serializers.DateField()
     count = serializers.IntegerField()
     course_id = serializers.CharField()