[bug 1139713] Add received_ts field to hb Answer model

* add a received_ts field to the hb Answer model with a silly
  initial default value
* backfill the field with the data from updated_ts

fjord/alerts/models.py

@@ -1,12 +1,9 @@
-from django.conf import settings
-from django.core.exceptions import ValidationError
 from django.db import models
 
-import pytz
-from rest_framework import fields
 from rest_framework import serializers
 
 from fjord.api_auth.models import Token
+from fjord.base.api_utils import UTCDateTimeField
 from fjord.base.models import ModelBase
 
 
@@ -79,49 +76,6 @@ def __unicode__(self):
         return self.url
 
 
-class UTCDateTimeField(fields.DateTimeField):
-    """Like DateTimeField, except it is always in UTC in the API"""
-    def to_native(self, value):
-        """Convert outgoing datetimes into UTC
-
-        Input currently saves everything in Pacific time, so this
-        takes the datetimes and converts them from Pacific time to
-        UTC.
-
-        If this situation ever changes, then we'd change
-        settings.TIME_ZONE and this should continue to work.
-
-        """
-        if value is not None and value.tzinfo is None:
-            default_tzinfo = pytz.timezone(settings.TIME_ZONE)
-            value = default_tzinfo.localize(value)
-            value = value.astimezone(pytz.utc)
-        return super(UTCDateTimeField, self).to_native(value)
-
-    def from_native(self, value):
-        """Converts incoming strings to localtime
-
-        Input currently saves everything in Pacific time, so this
-        takes the datetime that super().from_native() produces,
-        converts it to localtime and if USE_TZ=False, drops the timezone.
-
-        If this situation ever changes, this should continue to work.
-
-        """
-        result = super(UTCDateTimeField, self).from_native(value)
-
-        if result is not None and result.tzinfo is not None:
-            # Convert from whatever timezone it's in to local
-            # time.
-            local_tzinfo = pytz.timezone(settings.TIME_ZONE)
-            result = result.astimezone(local_tzinfo)
-
-            # If USE_TZ = False, drop the timezone
-            if not settings.USE_TZ:
-                result = result.replace(tzinfo=None)
-        return result
-
-
 class LinkSerializer(serializers.ModelSerializer):
     class Meta:
         model = Link

fjord/analytics/analyzer_views.py

@@ -56,7 +56,7 @@
 @analyzer_required
 def hb_data(request, answerid=None):
     """View for hb data that shows one or all of the answers"""
-    VALID_SORTBY_FIELDS = ('id', 'updated_ts')
+    VALID_SORTBY_FIELDS = ('id', 'received_ts', 'updated_ts')
 
     sortby = 'id'
     answer = None

fjord/analytics/templates/analytics/analyzer/hb_data.html

@@ -66,6 +66,9 @@ <h2>Heartbeat v2 data</h2>
           <dt>id</dt>
           <dd>{{ answer.id }}</dd>
 
+          <dt>received_ts</dt>
+          <dd>{{ answer.received_ts }}</dd>
+
           <dt>updated_ts</dt>
           <dd>{{ answer.updated_ts }}</dd>
 
@@ -151,6 +154,7 @@ <h2>Total answers: {{ answers.paginator.count }}</h2>
         <table class="summarytable">
           <tr>
             <th><a href="{{ request.get_full_path()|urlparams(sortby='id') }}">id</a>{% if sortby == 'id' %} v{% endif %}</th>
+            <th><a href="{{ request.get_full_path()|urlparams(sortby='received_ts') }}">received</a>{% if sortby == 'received_ts' %} v{% endif %}</th>
             <th><a href="{{ request.get_full_path()|urlparams(sortby='updated_ts') }}">updated</a>{% if sortby == 'updated_ts' %} v{% endif %}</th>
             <th>experiment ver</th>
             <th>response ver</th>
@@ -176,6 +180,7 @@ <h2>Total answers: {{ answers.paginator.count }}</h2>
           {% for ans in answers %}
             <tr {% if ans.is_test %} class="test-answer" {% endif %}>
               <td><a href="{{ url('hb_data', answerid=ans.id) }}">{{ ans.id }}</a></td>
+              <td><span class="nowrap" title="{{ ans.received_ts }}">{{ ans.received_ts }}</span></td>
               <td><span class="nowrap" title="{{ ans.updated_ts }}">{{ fix_ts(ans.updated_ts) }}</span></td>
               <td>{{ ans.experiment_version }}</td>
               <td>{{ ans.response_version }}</td>

fjord/base/api_utils.py

@@ -0,0 +1,47 @@
+from django.conf import settings
+
+import pytz
+from rest_framework import fields
+
+
+class UTCDateTimeField(fields.DateTimeField):
+    """Like DateTimeField, except it is always in UTC in the API"""
+    def to_native(self, value):
+        """Convert outgoing datetimes into UTC
+
+        Input currently saves everything in Pacific time, so this
+        takes the datetimes and converts them from Pacific time to
+        UTC.
+
+        If this situation ever changes, then we'd change
+        settings.TIME_ZONE and this should continue to work.
+
+        """
+        if value is not None and value.tzinfo is None:
+            default_tzinfo = pytz.timezone(settings.TIME_ZONE)
+            value = default_tzinfo.localize(value)
+            value = value.astimezone(pytz.utc)
+        return super(UTCDateTimeField, self).to_native(value)
+
+    def from_native(self, value):
+        """Converts incoming strings to localtime
+
+        Input currently saves everything in Pacific time, so this
+        takes the datetime that super().from_native() produces,
+        converts it to localtime and if USE_TZ=False, drops the timezone.
+
+        If this situation ever changes, this should continue to work.
+
+        """
+        result = super(UTCDateTimeField, self).from_native(value)
+
+        if result is not None and result.tzinfo is not None:
+            # Convert from whatever timezone it's in to local
+            # time.
+            local_tzinfo = pytz.timezone(settings.TIME_ZONE)
+            result = result.astimezone(local_tzinfo)
+
+            # If USE_TZ = False, drop the timezone
+            if not settings.USE_TZ:
+                result = result.replace(tzinfo=None)
+        return result

fjord/heartbeat/migrations/0006_answer_received_ts.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+"""Adds "received_ts" field to Heartbeat Answer model.
+
+"""
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+import datetime
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('heartbeat', '0005_auto_20150225_1238'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='answer',
+            name='received_ts',
+            field=models.DateTimeField(default=datetime.datetime(2011, 9, 1, 9, 0), help_text='Time the server received the last update packet', auto_now=True),
+            preserve_default=False,
+        ),
+    ]

fjord/heartbeat/migrations/0007_auto_20150305_1119.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+"""Set received_ts using the value in updated_ts. It's not the same
+thing, but it's "close enough" for an initial value where we don't
+have anything better to base it on.
+
+"""
+from __future__ import unicode_literals
+
+from datetime import datetime
+
+from django.conf import settings
+from django.db import models, migrations
+
+import pytz
+
+
+def no_op(apps, schema_editor):
+    pass
+
+
+def set_received_ts(apps, schema_editor):
+    """Sets received_ts based on updated_ts"""
+    Answer = apps.get_model('heartbeat', 'Answer')
+
+    qs = Answer.objects.filter(received_ts=datetime(2011, 9, 1, 9, 0))
+    for ans in qs:
+        # Note: Answer.updated_ts is milliseconds since epoch. We're
+        # assuming it's in UTC, so we convert it to server time.
+        dt = datetime.fromtimestamp(ans.updated_ts / 1000)
+
+        # Apply UTC
+        utc_tz = pytz.timezone('UTC')
+        dt = utc_tz.localize(dt)
+
+        # Then switch to server time
+        local_tz = pytz.timezone(settings.TIME_ZONE)
+        dt = dt.astimezone(local_tz)
+
+        ans.received_ts = dt
+        ans.save()
+        break
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('heartbeat', '0006_answer_received_ts'),
+    ]
+
+    operations = [
+        # Note: This can't be backed out, but for the sake of testing
+        # and convenience, we provide a no-op.
+        migrations.RunPython(set_received_ts, reverse_code=no_op),
+    ]

fjord/heartbeat/models.py

@@ -3,6 +3,7 @@
 
 from rest_framework import serializers
 
+from fjord.base.api_utils import UTCDateTimeField
 from fjord.base.models import ModelBase, JSONObjectField
 
 
@@ -111,6 +112,9 @@ class Answer(ModelBase):
     # Whether or not this is test data.
     is_test = models.BooleanField(default=False, blank=True)
 
+    received_ts = models.DateTimeField(
+        auto_now=True,
+        help_text=u'Time the server received the last update packet')
 
     class Meta:
         unique_together = (
@@ -129,6 +133,8 @@ class AnswerSerializer(serializers.ModelSerializer):
     updated_ts = serializers.IntegerField(source='updated_ts', required=True)
     survey_id = serializers.SlugRelatedField(slug_field='name')
 
+    received_ts = UTCDateTimeField(read_only=True)
+
     class Meta:
         model = Answer