Merge pull request #57 from LCOGT/observation_datetime_filters

Observation datetime filters
observatorycontrolsystem · Aug 1, 2019 · c73d927 · c73d927
2 parents 6876a84 + ab043ff
commit c73d927
Show file tree

Hide file tree

Showing 4 changed files with 104 additions and 77 deletions.
diff --git a/observation_portal/blocks/filters.py b/observation_portal/blocks/filters.py
@@ -1,29 +1,53 @@
 import django_filters
 from django import forms
-from dateutil import parser
-from distutils.util import strtobool
 
 from observation_portal.observations.models import Observation
 from observation_portal.common.configdb import configdb
+from observation_portal.common import mixins
 
 
-class PondBlockFilter(django_filters.FilterSet):
+class PondBlockFilter(mixins.CustomIsoDateTimeFilterMixin, django_filters.FilterSet):
     site = django_filters.MultipleChoiceFilter(choices=configdb.get_site_tuples())
     observatory = django_filters.MultipleChoiceFilter(choices=configdb.get_enclosure_tuples(), field_name='enclosure')
     telescope = django_filters.MultipleChoiceFilter(choices=configdb.get_telescope_tuples())
-    start_after = django_filters.CharFilter(field_name='start', method='filter_start_after', label='Start after',
-                                            widget=forms.TextInput(attrs={'class': 'input', 'type': 'date'}))
-    start_before = django_filters.CharFilter(field_name='start', method='filter_start_before', label='Start before',
-                                             widget=forms.TextInput(attrs={'class': 'input', 'type': 'date'}))
-    end_after = django_filters.CharFilter(field_name='end', method='filter_end_after', label='End after')
-    end_before = django_filters.CharFilter(field_name='end', method='filter_end_before', label='End before')
-    modified_after = django_filters.CharFilter(field_name='modified', method='filter_modified_after',
-                                               label='Modified After')
+    start_after = django_filters.IsoDateTimeFilter(
+        field_name='start',
+        lookup_expr='gte',
+        label='Start after',
+        widget=forms.TextInput(attrs={'class': 'input', 'type': 'date'})
+    )
+    start_before = django_filters.IsoDateTimeFilter(
+        field_name='start',
+        lookup_expr='lt',
+        label='Start before',
+        widget=forms.TextInput(attrs={'class': 'input', 'type': 'date'})
+    )
+    end_after = django_filters.IsoDateTimeFilter(
+        field_name='end',
+        lookup_expr='gte',
+        label='End after'
+    )
+    end_before = django_filters.IsoDateTimeFilter(
+        field_name='end',
+        lookup_expr='lt',
+        label='End before'
+    )
+    modified_after = django_filters.IsoDateTimeFilter(
+        field_name='modified',
+        lookup_expr='gte',
+        label='Modified After'
+    )
     request_num = django_filters.CharFilter(field_name='request__id')
     tracking_num = django_filters.CharFilter(field_name='request__request_group__id')
-    proposal = django_filters.CharFilter(field_name='request__request_group__proposal__id', distinct=True, lookup_expr='exact')
-    instrument_class = django_filters.ChoiceFilter(choices=configdb.get_instrument_type_tuples(),
-                                                   field_name='configuration_statuses__configuration__instrument_type')
+    proposal = django_filters.CharFilter(
+        field_name='request__request_group__proposal__id',
+        distinct=True,
+        lookup_expr='exact'
+    )
+    instrument_class = django_filters.ChoiceFilter(
+        choices=configdb.get_instrument_type_tuples(),
+        field_name='configuration_statuses__configuration__instrument_type'
+    )
     canceled = django_filters.BooleanFilter(method='filter_canceled')
     order = django_filters.OrderingFilter(fields=('start', 'modified'))
     time_span = django_filters.DateRangeFilter(field_name='start')
@@ -32,26 +56,6 @@ class Meta:
         model = Observation
         exclude = ['start', 'end', 'request', 'created', 'modified']
 
-    def filter_start_after(self, queryset, name, value):
-        start = parser.parse(value, ignoretz=True)
-        return queryset.filter(start__gte=start)
-
-    def filter_start_before(self, queryset, name, value):
-        start = parser.parse(value, ignoretz=True)
-        return queryset.filter(start__lt=start)
-
-    def filter_end_after(self, queryset, name, value):
-        end = parser.parse(value, ignoretz=True)
-        return queryset.filter(end__gte=end)
-
-    def filter_end_before(self, queryset, name, value):
-        end = parser.parse(value, ignoretz=True)
-        return queryset.filter(end__lt=end)
-
-    def filter_modified_after(self, queryset, name, value):
-        modified_after = parser.parse(value, ignoretz=True)
-        return queryset.filter(modified__gte=modified_after)
-
     def filter_canceled(self, queryset, name, value):
         if not value:
             return queryset.exclude(state='CANCELED')

diff --git a/observation_portal/common/mixins.py b/observation_portal/common/mixins.py
@@ -1,4 +1,6 @@
+from django_filters import fields, IsoDateTimeFilter
 from django.contrib.auth.mixins import UserPassesTestMixin
+from django.forms import DateTimeField
 
 
 class ListAsDictMixin(object):
@@ -21,3 +23,20 @@ def get_serializer(self, *args, **kwargs):
 class StaffRequiredMixin(UserPassesTestMixin):
     def test_func(self):
         return self.request.user.is_staff
+
+
+# Use the CustomIsoDateTimeFilterMixin in a FilterSet. Makes all IsoDateTimeFilters within the FilterSet able to parse
+# ISO 8601 datetimes, as well as all the other usual formats that the DateTimeFilter can do.
+# https://django-filter.readthedocs.io/en/master/ref/fields.html#isodatetimefield
+class CustomIsoDateTimeField(fields.IsoDateTimeField):
+    input_formats = [fields.IsoDateTimeField.ISO_8601] + list(DateTimeField.input_formats)
+
+
+class CustomIsoDateTimeFilterMixin(object):
+    @classmethod
+    def get_filters(cls):
+        filters = super().get_filters()
+        for f in filters.values():
+            if isinstance(f, IsoDateTimeFilter):
+                f.field_class = CustomIsoDateTimeField
+        return filters
diff --git a/observation_portal/observations/filters.py b/observation_portal/observations/filters.py
@@ -1,35 +1,59 @@
 import django_filters
 from django import forms
-from dateutil import parser
 
 from observation_portal.observations.models import Observation, ConfigurationStatus
 from observation_portal.requestgroups.models import RequestGroup, Request
 from observation_portal.common.configdb import configdb
+from observation_portal.common import mixins
 
 
-class ObservationFilter(django_filters.FilterSet):
+class ObservationFilter(mixins.CustomIsoDateTimeFilterMixin, django_filters.FilterSet):
     site = django_filters.MultipleChoiceFilter(choices=configdb.get_site_tuples())
     enclosure = django_filters.MultipleChoiceFilter(choices=configdb.get_enclosure_tuples())
     telescope = django_filters.MultipleChoiceFilter(choices=configdb.get_telescope_tuples())
-    start_after = django_filters.CharFilter(field_name='start', method='filter_start_after', label='Start after',
-                                            widget=forms.TextInput(attrs={'class': 'input', 'type': 'date'}))
-    start_before = django_filters.CharFilter(field_name='start', method='filter_start_before', label='Start before',
-                                             widget=forms.TextInput(attrs={'class': 'input', 'type': 'date'}))
-    end_after = django_filters.CharFilter(field_name='end', method='filter_end_after', label='End after',
-                                          widget=forms.TextInput(attrs={'class': 'input', 'type': 'date'}))
-    end_before = django_filters.CharFilter(field_name='end', method='filter_end_before', label='End before',
-                                           widget=forms.TextInput(attrs={'class': 'input', 'type': 'date'}))
-    modified_after = django_filters.CharFilter(field_name='modified', method='filter_modified_after',
-                                               label='Modified After',
-                                               widget=forms.TextInput(attrs={'class': 'input', 'type': 'date'}))
+    start_after = django_filters.IsoDateTimeFilter(
+        field_name='start',
+        lookup_expr='gte',
+        label='Start after',
+        widget=forms.TextInput(attrs={'class': 'input', 'type': 'date'})
+    )
+    start_before = django_filters.IsoDateTimeFilter(
+        field_name='start',
+        lookup_expr='lt',
+        label='Start before',
+        widget=forms.TextInput(attrs={'class': 'input', 'type': 'date'})
+    )
+    end_after = django_filters.IsoDateTimeFilter(
+        field_name='end',
+        lookup_expr='gte',
+        label='End after',
+        widget=forms.TextInput(attrs={'class': 'input', 'type': 'date'})
+    )
+    end_before = django_filters.IsoDateTimeFilter(
+        field_name='end',
+        lookup_expr='lt',
+        label='End before',
+        widget=forms.TextInput(attrs={'class': 'input', 'type': 'date'})
+    )
+    modified_after = django_filters.IsoDateTimeFilter(
+        field_name='modified',
+        lookup_expr='gte',
+        label='Modified After',
+        widget=forms.TextInput(attrs={'class': 'input', 'type': 'date'})
+    )
     request_id = django_filters.CharFilter(field_name='request__id')
     request_group_id = django_filters.CharFilter(field_name='request__request_group__id', label='Request Group ID')
     state = django_filters.MultipleChoiceFilter(choices=Observation.STATE_CHOICES, field_name='state')
-    observation_type = django_filters.MultipleChoiceFilter(choices=RequestGroup.OBSERVATION_TYPES,
-                                                           field_name='request__request_group__observation_type',
-                                                           label='Observation Type')
-    request_state = django_filters.MultipleChoiceFilter(choices=Request.STATE_CHOICES, field_name='request__state',
-                                                        label='Request State')
+    observation_type = django_filters.MultipleChoiceFilter(
+        choices=RequestGroup.OBSERVATION_TYPES,
+        field_name='request__request_group__observation_type',
+        label='Observation Type'
+    )
+    request_state = django_filters.MultipleChoiceFilter(
+        choices=Request.STATE_CHOICES,
+        field_name='request__state',
+        label='Request State'
+    )
     proposal = django_filters.CharFilter(field_name='request__request_group__proposal__id', label='Proposal')
     instrument_type = django_filters.MultipleChoiceFilter(
         choices=configdb.get_instrument_type_tuples(),
@@ -49,26 +73,6 @@ class Meta:
         model = Observation
         exclude = ['start', 'end', 'request', 'created', 'modified']
 
-    def filter_start_after(self, queryset, name, value):
-        start = parser.parse(value, ignoretz=True)
-        return queryset.filter(start__gte=start)
-
-    def filter_start_before(self, queryset, name, value):
-        start = parser.parse(value, ignoretz=True)
-        return queryset.filter(start__lt=start)
-
-    def filter_end_after(self, queryset, name, value):
-        end = parser.parse(value, ignoretz=True)
-        return queryset.filter(end__gte=end)
-
-    def filter_end_before(self, queryset, name, value):
-        end = parser.parse(value, ignoretz=True)
-        return queryset.filter(end__lt=end)
-
-    def filter_modified_after(self, queryset, name, value):
-        modified_after = parser.parse(value, ignoretz=True)
-        return queryset.filter(modified__gte=modified_after)
-
 
 class ConfigurationStatusFilter(django_filters.FilterSet):
     instrument_name = django_filters.ChoiceFilter(choices=configdb.get_instrument_name_tuples())

diff --git a/observation_portal/observations/tests.py b/observation_portal/observations/tests.py
@@ -299,7 +299,7 @@ def test_delete_observation_leaves_request(self):
         observation = Observation.objects.get(pk=obj_json['id'])
         observation.state = 'CANCELED'
         observation.save()
-        Observation.delete_old_observations(datetime(2099, 1, 1))
+        Observation.delete_old_observations(datetime(2099, 1, 1, tzinfo=timezone.utc))
         request = Request.objects.get(id=obj_json['request']['id'])
         self.assertEqual(request.id, obj_json['request']['id'])
         with self.assertRaises(Observation.DoesNotExist):
@@ -315,7 +315,7 @@ def test_cant_delete_observation_with_started_configuration_statuses(self):
         configuration_status = observation.configuration_statuses.all()[0]
         configuration_status.state = 'ATTEMPTED'
         configuration_status.save()
-        Observation.delete_old_observations(datetime(2099, 1, 1))
+        Observation.delete_old_observations(datetime(2099, 1, 1, tzinfo=timezone.utc))
         observation = Observation.objects.get(pk=obj_json['id'])
         self.assertEqual(observation.id, obj_json['id'])
 
@@ -1146,11 +1146,11 @@ def setUp(self):
 
     def _add_observation(self, state, time_completed):
         observation = Observation.objects.create(request=self.requestgroup.requests.first(), state=state, site='tst', enclosure='domb', telescope='1m0a',
-        start=datetime(2016,9,5,22,35,39), end=datetime(2016,9,5,23,35,40))
+        start=datetime(2016,9,5,22,35,39, tzinfo=timezone.utc), end=datetime(2016,9,5,23,35,40, tzinfo=timezone.utc))
         config_status = ConfigurationStatus.objects.create(observation=observation, configuration=self.requestgroup.requests.first().configurations.first(),
         state=state, instrument_name='xx03', guide_camera_name='xx03')
-        Summary.objects.create(configuration_status=config_status, start=datetime(2016,9,5,22,35,39),
-        end=datetime(2016,9,5,23,35,40), time_completed=time_completed, state=state)
+        Summary.objects.create(configuration_status=config_status, start=datetime(2016,9,5,22,35,39, tzinfo=timezone.utc),
+        end=datetime(2016,9,5,23,35,40, tzinfo=timezone.utc), time_completed=time_completed, state=state)
         return observation
 
     def test_with_no_obs_command_reports_no_time_used(self):