Bug794261 organize matviews

config/crontabber.ini

@@ -3,15 +3,14 @@
     # doc: Note! the hour part (relevant on daily jobs) is in UTC time
     # converter: class_list_converter
     jobs='''socorro.cron.jobs.weekly_reports_partitions.WeeklyReportsPartitionsCronApp|7d
-            socorro.cron.jobs.nightly_builds.NightlyBuildsCronApp|1d|02:00
             socorro.cron.jobs.matviews.ProductVersionsCronApp|1d|02:00
             socorro.cron.jobs.matviews.SignaturesCronApp|1d|02:00
             socorro.cron.jobs.matviews.TCBSCronApp|1d|02:00
             socorro.cron.jobs.matviews.ADUCronApp|1d|02:00
             socorro.cron.jobs.matviews.HangReportCronApp|1d|02:00
             socorro.cron.jobs.matviews.NightlyBuildsCronApp|1d|02:00
-            socorro.cron.jobs.duplicates.DuplicatesCronApp|1h
-            socorro.cron.jobs.reports_clean.ReportsCleanCronApp|1h
+            socorro.cron.jobs.matviews.DuplicatesCronApp|1h
+            socorro.cron.jobs.matviews.ReportsCleanCronApp|1h
             socorro.cron.jobs.bugzilla.BugzillaCronApp|1h
             socorro.cron.jobs.matviews.BuildADUCronApp|1d|02:00
             socorro.cron.jobs.matviews.ProductADUCronApp|1d|02:00
@@ -55,4 +54,3 @@
     # doc: logging level for the log file (10 - DEBUG, 20 - INFO, 30 - WARNING, 40 - ERROR, 50 - CRITICAL)
     # converter: int
     syslog_error_logging_level='10'
-

socorro/cron/jobs/matviews.py

@@ -15,22 +15,26 @@ class _Base(object):
     def get_proc_name(self):
         return self.proc_name
 
+    def run_proc(self, connection, signature=None):
+        cursor = connection.cursor()
+        if signature:
+            cursor.callproc(self.get_proc_name(), signature)
+        else:
+            cursor.callproc(self.get_proc_name())
+        connection.commit()
+
 
 class _MatViewBase(PostgresCronApp, _Base):
 
     def run(self, connection):
-        cursor = connection.cursor()
-        cursor.callproc(self.get_proc_name())
-        connection.commit()
+        self.run_proc(connection)
 
 
 class _MatViewBackfillBase(PostgresBackfillCronApp, _Base):
 
     def run(self, connection, date):
-        cursor = connection.cursor()
         target_date = (date - datetime.timedelta(days=1)).date()
-        cursor.callproc(self.get_proc_name(), [target_date])
-        connection.commit()
+        self.run_proc(connection, [target_date])
 
 #------------------------------------------------------------------------------
 
@@ -138,3 +142,33 @@ class ExplosivenessCronApp(_MatViewBackfillBase):
         'build-adu-matview',
         'reports-clean'
     )
+
+
+class ReportsCleanCronApp(PostgresBackfillCronApp, _Base):
+    proc_name = 'update_reports_clean'
+    app_name = 'reports-clean'
+    app_version = '1.0'
+    app_description = ""
+    depends_on = (
+        'duplicates',
+    )
+
+    def run(self, connection, date):
+        date -= datetime.timedelta(hours=2)
+        self.run_proc(connection, [date])
+
+
+class DuplicatesCronApp(PostgresBackfillCronApp, _Base):
+    proc_name = 'update_reports_duplicates'
+    app_name = 'duplicates'
+    app_version = '1.0'
+    app_description = ""
+
+    def run(self, connection, date):
+        start_time = date - datetime.timedelta(hours=3)
+        end_time = start_time + datetime.timedelta(hours=1)
+        self.run_proc(connection, [start_time, end_time])
+
+        start_time += datetime.timedelta(minutes=30)
+        end_time = start_time + datetime.timedelta(hours=1)
+        self.run_proc(connection, [start_time, end_time])

socorro/unittest/cron/jobs/test_matviews.py

@@ -2,6 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import os
 import datetime
 import json
 import mock
@@ -101,6 +102,67 @@ def test_all_matviews(self):
             # regular jobs writing only once.
             self.assertEqual(self.psycopg2().commit.call_count, 14 * 2 + 2)
 
+    def test_reports_clean_dependency_prerequisite(self):
+        config_manager, json_file = self._setup_config_manager(
+          'socorro.cron.jobs.matviews.ReportsCleanCronApp|1d'
+        )
+
+        with config_manager.context() as config:
+            tab = crontabber.CronTabber(config)
+            tab.run_all()
+
+            # no file is created because it's unable to run anything
+            self.assertTrue(not os.path.isfile(json_file))
+
+    def test_reports_clean_with_dependency(self):
+        config_manager, json_file = self._setup_config_manager(
+          'socorro.cron.jobs.matviews.DuplicatesCronApp|1h\n'
+          'socorro.cron.jobs.matviews.ReportsCleanCronApp|1h'
+        )
+
+        with config_manager.context() as config:
+            tab = crontabber.CronTabber(config)
+            tab.run_all()
+
+            information = json.load(open(json_file))
+            assert information['reports-clean']
+            assert not information['reports-clean']['last_error']
+            assert information['reports-clean']['last_success']
+
+            # not a huge fan of this test because it's so specific
+            calls = self.psycopg2().cursor().callproc.mock_calls
+            call = calls[-1]
+            __, called, __ = list(call)
+            self.assertEqual(called[0], 'update_reports_clean')
+
+    def test_duplicates(self):
+        config_manager, json_file = self._setup_config_manager(
+          'socorro.cron.jobs.matviews.DuplicatesCronApp|1d'
+        )
+
+        with config_manager.context() as config:
+            tab = crontabber.CronTabber(config)
+            tab.run_all()
+
+            information = json.load(open(json_file))
+            assert information['duplicates']
+            assert not information['duplicates']['last_error']
+            assert information['duplicates']['last_success']
+
+            # not a huge fan of this test because it's so specific
+            proc_name = 'update_reports_duplicates'
+            calls = self.psycopg2().cursor().callproc.mock_calls
+            call1, call2 = calls
+            __, called, __ = call1
+            assert called[0] == proc_name, called[0]
+            start, end = called[1]
+            self.assertEqual(end - start, datetime.timedelta(hours=1))
+
+            __, called, __ = call2
+            assert called[0] == proc_name, called[0]
+            start, end = called[1]
+            self.assertEqual(end - start, datetime.timedelta(hours=1))
+
 
 class _Job(crontabber.BaseCronApp):