Merge pull request #1341 from glogiotatidis/remove-old-data-movldes

Remove unused metrics tables.

snippets/base/admin/__init__.py

@@ -27,8 +27,5 @@
 admin.site.register(models.Job, adminmodels.JobAdmin)
 admin.site.register(models.Distribution, adminmodels.DistributionAdmin)
 admin.site.register(models.DistributionBundle)
-admin.site.register(models.DailyJobMetrics, adminmodels.DailyJobMetricsAdmin)
-admin.site.register(models.DailyChannelMetrics, adminmodels.DailyChannelMetricsAdmin)
-admin.site.register(models.DailyCountryMetrics, adminmodels.DailyCountryMetricsAdmin)
 admin.site.register(models.JobDailyPerformance, adminmodels.JobDailyPerformanceAdmin)
 admin.site.register(models.DailyImpressions, adminmodels.DailyImpressionsAdmin)

snippets/base/admin/adminmodels.py

@@ -21,7 +21,7 @@
 from reversion.admin import VersionAdmin
 from taggit_helpers.admin import TaggitListFilter
 
-from snippets.base import etl, forms, models
+from snippets.base import forms, models
 from snippets.base.admin import actions, filters
 
 
@@ -1376,155 +1376,6 @@ class DistributionAdmin(admin.ModelAdmin):
     save_on_top = True
 
 
-class DailyJobMetricsAdmin(admin.ModelAdmin):
-    list_display = [
-        'id',
-        'job',
-        'date',
-        'impressions',
-        'clicks',
-        'blocks',
-        'data_fetched_on'
-    ]
-    search_fields = [
-        'job__id',
-        'job__snippet__name',
-        'job__snippet__id'
-    ]
-    readonly_fields = ['redash_link']
-    fieldsets = [
-        ('Metrics', {
-            'fields': (
-                'job',
-                'date',
-                'impressions',
-                'clicks',
-                'blocks',
-                'redash_link'
-            ),
-        }),
-    ]
-
-    def has_add_permission(self, request):
-        return False
-
-    def has_change_permission(self, request, obj=None):
-        return False
-
-    def has_delete_permission(self, request, obj=None):
-        return False
-
-    def redash_link(self, obj):
-        link_legacy = etl.redash_source_url(
-            'redshift-message-id', begin_date=obj.date, end_date=obj.date)
-        # bq needs later end_date due to use of timestamps
-        link_bigquery = etl.redash_source_url(
-            'bq-message-id', begin_date=obj.date,
-            end_date=obj.date + timedelta(days=1))
-
-        return format_html(f'<a href="{link_legacy}">Redshift</a> - '
-                           f'<a href="{link_bigquery}">BigQuery (Fx 72+)</a>')
-
-    redash_link.short_description = 'Explore in Redash'
-
-
-class DailyChannelMetricsAdmin(admin.ModelAdmin):
-    list_display = [
-        'id',
-        'channel',
-        'date',
-        'impressions',
-        'clicks',
-        'blocks',
-        'data_fetched_on'
-    ]
-    search_fields = ['channel']
-    readonly_fields = ['redash_link']
-    fieldsets = [
-        ('Metrics', {
-            'fields': (
-                'channel',
-                'date',
-                'impressions',
-                'clicks',
-                'blocks',
-                'redash_link',
-            ),
-        }),
-    ]
-
-    def has_add_permission(self, request):
-        return False
-
-    def has_change_permission(self, request, obj=None):
-        return False
-
-    def has_delete_permission(self, request, obj=None):
-        return False
-
-    def redash_link(self, obj):
-        link_legacy = etl.redash_source_url(
-            'redshift-channel', begin_date=obj.date, end_date=obj.date)
-        # bq needs later end_date due to use of timestamps
-        link_bigquery = etl.redash_source_url(
-            'bq-channel', begin_date=obj.date,
-            end_date=obj.date + timedelta(days=1))
-
-        return format_html(f'<a href="{link_legacy}">Redshift</a> - '
-                           f'<a href="{link_bigquery}">BigQuery (Fx 72+)</a>')
-
-    redash_link.short_description = 'Explore in Redash'
-
-
-class DailyCountryMetricsAdmin(admin.ModelAdmin):
-    list_display = [
-        'id',
-        'date',
-        'country',
-        'impressions',
-        'clicks',
-        'blocks',
-        'data_fetched_on'
-    ]
-    search_fields = ['country']
-    readonly_fields = ['redash_link']
-    fieldsets = [
-        ('Metrics', {
-            'fields': (
-                'country',
-                'date',
-                'impressions',
-                'clicks',
-                'blocks',
-                'redash_link',
-            ),
-        }),
-    ]
-
-    def has_add_permission(self, request):
-        return False
-
-    def has_change_permission(self, request, obj=None):
-        return False
-
-    def has_delete_permission(self, request, obj=None):
-        return False
-
-    def redash_link(self, obj):
-        link_legacy = etl.redash_source_url('redshift-country',
-                                            begin_date=obj.date,
-                                            end_date=obj.date)
-        # bq needs different end_date due to use of timestamps
-        link_bigquery = etl.redash_source_url('bq-country',
-                                              begin_date=obj.date,
-                                              end_date=obj.date + timedelta(days=1))
-
-        return format_html(f'<a href="{link_legacy}">Redshift</a> - '
-                           f'<a href="{link_bigquery}">BigQuery (Fx 72+)</a>')
-
-    redash_link.short_description = 'Explore in Redash'
-
-
 class JobDailyPerformanceAdmin(admin.ModelAdmin):
     list_display = [
         'id',

snippets/base/migrations/0034_auto_20200323_1036.py

@@ -0,0 +1,22 @@
+# Generated by Django 2.2.10 on 2020-03-23 10:36
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0033_auto_20200303_1328'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='DailyChannelMetrics',
+        ),
+        migrations.DeleteModel(
+            name='DailyCountryMetrics',
+        ),
+        migrations.DeleteModel(
+            name='DailyJobMetrics',
+        ),
+    ]

snippets/base/models.py

@@ -2302,47 +2302,3 @@ class DailyImpressions(models.Model):
     data_fetched_on = models.DateTimeField(auto_now_add=True)
     date = models.DateField(editable=False, db_index=True, unique=True)
     details = JSONField(default=dict, editable=False)
-
-
-class MetricsAbstractModel(models.Model):
-    data_fetched_on = models.DateTimeField(auto_now_add=True)
-    date = models.DateField(editable=False, db_index=True)
-    impressions = models.PositiveIntegerField(default=0, editable=False)
-    clicks = models.PositiveIntegerField(default=0, editable=False)
-    blocks = models.PositiveIntegerField(default=0, editable=False)
-
-    class Meta:
-        abstract = True
-
-
-class DailyJobMetrics(MetricsAbstractModel):
-    job = models.ForeignKey(Job, on_delete=models.PROTECT)
-
-    class Meta:
-        verbose_name_plural = 'Daily Job Metrics'
-        unique_together = [('job', 'date')]
-
-    def __str__(self):
-        return f'{self.date.strftime("%Y%m%d")} - {self.job.id}'
-
-
-class DailyChannelMetrics(MetricsAbstractModel):
-    channel = models.CharField(max_length=255, db_index=True)
-
-    class Meta:
-        verbose_name_plural = 'Daily Channel Metrics'
-        unique_together = [('channel', 'date')]
-
-    def __str__(self):
-        return f'{self.date.strftime("%Y%m%d")} - {self.channel}'
-
-
-class DailyCountryMetrics(MetricsAbstractModel):
-    country = models.CharField(max_length=255, db_index=True)
-
-    class Meta:
-        verbose_name_plural = 'Daily Country Metrics'
-        unique_together = [('country', 'date')]
-
-    def __str__(self):
-        return f'{self.date.strftime("%Y%m%d")} - {self.country}'

snippets/base/tests/test_admin.py

@@ -1,22 +1,15 @@
-from datetime import date
 from django.contrib.admin.sites import AdminSite
 from django.contrib.auth.models import User
 from django.test.client import RequestFactory
 
 from unittest.mock import DEFAULT as DEFAULT_MOCK, Mock, patch
 
-from snippets.base import etl
 from snippets.base.admin.adminmodels import (ASRSnippetAdmin,
-                                             DailyChannelMetricsAdmin,
-                                             DailyCountryMetricsAdmin,
-                                             DailyJobMetricsAdmin,
                                              JobAdmin,
                                              SnippetTemplateAdmin)
 from snippets.base.admin.legacy import SnippetAdmin
 from snippets.base.models import (STATUS_CHOICES, ASRSnippet,
-                                  DailyChannelMetrics,
-                                  DailyCountryMetrics,
-                                  DailyJobMetrics, Job, Snippet,
+                                  Job, Snippet,
                                   SnippetTemplate,
                                   SnippetTemplateVariable)
 from snippets.base.tests import (ASRSnippetFactory, JobFactory,
@@ -296,41 +289,3 @@ def test_action_delete_job(self):
         )
         self.assertTrue(message_mocks['warning'].called)
         self.assertTrue(message_mocks['success'].called)
-
-
-class DailyMetricsAdminTests(TestCase):
-    def test_channel_redash_link(self):
-        metrics_admin = DailyChannelMetricsAdmin(
-            DailyChannelMetrics, AdminSite())
-        metrics = DailyChannelMetrics(date=date(2019, 12, 26))
-        html = metrics_admin.redash_link(metrics)
-        bq_url = etl.redash_source_url(
-            'bq-channel', begin_date=metrics.date, end_date='2019-12-27')
-        assert f'href="{bq_url}"' in html
-        redshift_url = etl.redash_source_url(
-            'redshift-channel', begin_date=metrics.date, end_date=metrics.date)
-        assert f'href="{redshift_url}"' in html
-
-    def test_country_redash_link(self):
-        metrics_admin = DailyCountryMetricsAdmin(
-            DailyCountryMetrics, AdminSite())
-        metrics = DailyCountryMetrics(date=date(2019, 12, 26))
-        html = metrics_admin.redash_link(metrics)
-        bq_url = etl.redash_source_url(
-            'bq-country', begin_date=metrics.date, end_date='2019-12-27')
-        assert f'href="{bq_url}"' in html
-        redshift_url = etl.redash_source_url(
-            'redshift-country', begin_date=metrics.date, end_date=metrics.date)
-        assert f'href="{redshift_url}"' in html
-
-    def test_Job_redash_link(self):
-        metrics_admin = DailyJobMetricsAdmin(
-            DailyJobMetrics, AdminSite())
-        metrics = DailyJobMetrics(date=date(2019, 12, 26))
-        html = metrics_admin.redash_link(metrics)
-        bq_url = etl.redash_source_url(
-            'bq-message-id', begin_date=metrics.date, end_date='2019-12-27')
-        assert f'href="{bq_url}"' in html
-        redshift_url = etl.redash_source_url(
-            'redshift-message-id', begin_date=metrics.date, end_date=metrics.date)
-        assert f'href="{redshift_url}"' in html

snippets/settings.py

@@ -349,15 +349,6 @@
             'base.DailyImpressions',
         ]
     },
-    {
-        'app': 'base',
-        'label': 'Legacy Metrics',
-        'models': [
-            'base.DailyJobMetrics',
-            'base.DailyChannelMetrics',
-            'base.DailyCountryMetrics',
-        ]
-    },
     {
         'app': 'auth',
         'label': 'Admin',