Merge pull request #53 from LCOGT/delete_old_observations

added task for deleting old observations

observation_portal/observations/models.py

@@ -5,7 +5,9 @@
 from datetime import timedelta
 
 from observation_portal.requestgroups.models import Request, Configuration
+import logging
 
+logger = logging.getLogger()
 
 class Observation(models.Model):
     STATE_CHOICES = (
@@ -72,6 +74,27 @@ def cancel(observations):
 
         return deleted_observations.get('observations.Observation', 0) + canceled + aborted
 
+    @staticmethod
+    def delete_old_observations(cutoff):
+        observations = Observation.objects.filter(start__lt=cutoff, end__lt=cutoff, state='CANCELED').exclude(
+            configuration_statuses__state__in=['ATTEMPTED', 'FAILED', 'COMPLETED']
+        )
+        logger.warning('There are {} observations to be deleted. Only the first 100,000 will be deleted this run'.format(len(observations)))
+        total_deleted = 0
+        total_obs_deleted = 0
+        total_cs_deleted = 0
+        total_sm_deleted = 0
+        for observation in observations[:100000]:
+            num_deleted, deleted_dict = observation.delete()
+            total_deleted += num_deleted
+            total_obs_deleted += deleted_dict.get('observations.Observation', 0)
+            total_cs_deleted += deleted_dict.get('observations.ConfigurationStatus', 0)
+            total_sm_deleted += deleted_dict.get('observations.Summary', 0)
+
+        logger.warning('Deleted {} objects: {} observations, {} configuration_statuses, and {} summaries'.format(
+            total_deleted, total_obs_deleted, total_cs_deleted, total_sm_deleted
+        ))
+
     def as_dict(self, no_request=False):
         ret_dict = model_to_dict(self)
         if no_request:

observation_portal/observations/tasks.py

@@ -0,0 +1,14 @@
+import dramatiq
+import logging
+from datetime import datetime, timedelta
+
+from observation_portal.observations.models import Observation
+
+logger = logging.getLogger(__name__)
+
+
+@dramatiq.actor()
+def delete_old_observations():
+    cutoff = datetime.utcnow() - timedelta(days=14)
+    logger.info(f'Deleting CANCELED observations before cutoff date {cutoff}')
+    Observation.delete_old_observations(cutoff)

observation_portal/observations/tests.py

@@ -11,7 +11,7 @@
 from io import StringIO
 from django.core.management import call_command
 
-from observation_portal.requestgroups.models import RequestGroup, Window, Location
+from observation_portal.requestgroups.models import RequestGroup, Window, Location, Request
 from observation_portal.observations.time_accounting import configuration_time_used
 from observation_portal.observations.models import Observation, ConfigurationStatus, Summary
 from observation_portal.proposals.models import Proposal, Membership, Semester, TimeAllocation
@@ -292,6 +292,33 @@ def test_post_observation_hour_angle_missing_required_fields(self):
         self.assertEqual(response.status_code, 400)
         self.assertIn('dec', str(response.content))
 
+    def test_delete_observation_leaves_request(self):
+        response = self.client.post(reverse('api:schedule-list'), data=self.observation)
+        self.assertEqual(response.status_code, 201)
+        obj_json = response.json()
+        observation = Observation.objects.get(pk=obj_json['id'])
+        observation.state = 'CANCELED'
+        observation.save()
+        Observation.delete_old_observations(datetime(2099, 1, 1))
+        request = Request.objects.get(id=obj_json['request']['id'])
+        self.assertEqual(request.id, obj_json['request']['id'])
+        with self.assertRaises(Observation.DoesNotExist):
+            observation = Observation.objects.get(pk=obj_json['id'])
+
+    def test_cant_delete_observation_with_started_configuration_statuses(self):
+        response = self.client.post(reverse('api:schedule-list'), data=self.observation)
+        self.assertEqual(response.status_code, 201)
+        obj_json = response.json()
+        observation = Observation.objects.get(pk=obj_json['id'])
+        observation.state = 'CANCELED'
+        observation.save()
+        configuration_status = observation.configuration_statuses.all()[0]
+        configuration_status.state = 'ATTEMPTED'
+        configuration_status.save()
+        Observation.delete_old_observations(datetime(2099, 1, 1))
+        observation = Observation.objects.get(pk=obj_json['id'])
+        self.assertEqual(observation.id, obj_json['id'])
+
 
 class TestPostScheduleMultiConfigApi(SetTimeMixin, APITestCase):
     def setUp(self):

observation_portal/task_scheduler.py

@@ -2,6 +2,7 @@
 from apscheduler.triggers.cron import CronTrigger
 
 from observation_portal.requestgroups.tasks import expire_requests
+from observation_portal.observations.tasks import delete_old_observations
 from observation_portal.accounts.tasks import expire_access_tokens
 
 def run():
@@ -10,6 +11,10 @@ def run():
         expire_requests.send,
         CronTrigger.from_crontab('*/5 * * * *')
     )
+    scheduler.add_job(
+        delete_old_observations.send,
+        CronTrigger.from_crontab('0 * * * *')
+    )
     scheduler.add_job(
         expire_access_tokens.send,
         CronTrigger.from_crontab('0 15 * * *')