Merge 146cf0d into 3ce206a

observation_portal/requestgroups/migrations/0017_request_extra_params.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.24 on 2021-09-01 21:11
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('requestgroups', '0016_auto_20210525_0643'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='request',
+            name='extra_params',
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, help_text='Extra Request parameters', verbose_name='extra parameters'),
+        ),
+    ]

observation_portal/requestgroups/models.py

@@ -267,6 +267,12 @@ class Request(models.Model):
                   'acceptable to meet the science goal of the Request. Defaults to 100 for FLOYDS observations and '
                   '90 for all other observations.'
     )
+    extra_params = JSONField(
+        default=dict,
+        blank=True,
+        verbose_name='extra parameters',
+        help_text='Extra Request parameters'
+    )
 
     class Meta:
         ordering = ('id',)

observation_portal/requestgroups/pattern_expansion.py

@@ -1,6 +1,8 @@
 from math import radians, cos, sin, sqrt
 from copy import deepcopy
 
+from observation_portal import settings
+
 
 def expand_dither_pattern(expansion_details):
 
@@ -28,6 +30,14 @@ def expand_dither_pattern(expansion_details):
             final_instrument_configs.append(instrument_config_copy)
 
     configuration_dict['instrument_configs'] = final_instrument_configs
+    # Save some information about the dither pattern in the configuration extra params
+    saved_dither_params = expansion_details.copy()
+    saved_dither_params.pop('configuration', None)
+    dither_pattern = saved_dither_params.pop('pattern', settings.DITHER['custom_pattern_key'])
+    if 'extra_params' not in configuration_dict:
+        configuration_dict['extra_params'] = {}
+    configuration_dict['extra_params']['dither_pattern'] = dither_pattern
+    configuration_dict['extra_params']['dither_pattern_params'] = saved_dither_params
     return configuration_dict
 
 
@@ -54,11 +64,18 @@ def expand_mosaic_pattern(expansion_details):
         cos_dec = max(cos_dec, 10e-4)
         configuration_copy['target']['ra'] += (offset[0] / 3600.0 / cos_dec)
         extra_params = configuration_copy.get('extra_params', {})
-        extra_params['obs_recipe'] = 'Mosaic'
         configuration_copy['extra_params'] = extra_params
         configurations.append(configuration_copy)
 
     request_dict['configurations'] = configurations
+    # Save some information about the mosaic pattern inside the request
+    save_mosaic_params = expansion_details.copy()
+    save_mosaic_params.pop('request', None)
+    pattern = save_mosaic_params.pop('pattern', settings.MOSAIC['custom_pattern_key'])
+    if 'extra_params' not in request_dict:
+        request_dict['extra_params'] = {}
+    request_dict['extra_params']['mosaic_pattern'] = pattern
+    request_dict['extra_params']['mosaic_pattern_params'] = save_mosaic_params
     return request_dict
 
 

observation_portal/requestgroups/serializers.py

@@ -1,6 +1,6 @@
 import json
 import logging
-from math import isnan, cos, sin, radians
+from math import cos, sin, radians
 from json import JSONDecodeError
 from abc import ABC, abstractmethod
 
@@ -24,7 +24,7 @@
 from observation_portal.common.state_changes import debit_ipp_time, TimeAllocationError, validate_ipp
 from observation_portal.requestgroups.target_helpers import TARGET_TYPE_HELPER_MAP
 from observation_portal.common.mixins import ExtraParamsFormatter
-from observation_portal.common.configdb import configdb, ConfigDB, ConfigDBException
+from observation_portal.common.configdb import configdb, ConfigDB
 from observation_portal.requestgroups.duration_utils import (
     get_total_request_duration, get_requestgroup_duration, get_total_duration_dict,
     get_instrument_configuration_duration, get_semester_in
@@ -469,6 +469,34 @@ def validate(self, data):
                     'You may only specify a repeat_duration for REPEAT_* type configurations.'
                 ))
 
+        # Validate dither pattern
+
+        # First check that any dither pattern that is set is valid
+        dither_pattern_is_set = False
+        if 'extra_params' in data and 'dither_pattern' in data['extra_params']:
+            dither_pattern_is_set = True
+            pattern = data['extra_params']['dither_pattern']
+            valid_patterns = list(settings.DITHER['valid_expansion_patterns']) + [settings.DITHER['custom_pattern_key']]
+            if pattern not in valid_patterns:
+                raise serializers.ValidationError(_(
+                    f'Invalid dither pattern {pattern} set in the configuration extra_params, choose from {", ".join(valid_patterns)}'
+                ))
+
+        # Then, if a dither pattern is not yet set and there is at least one instrument_config in the configuration that has offsets applied,
+        # then it is part of a custom dither sequence and we need to set the custom dither pattern field.
+        if not dither_pattern_is_set:
+            is_dither_sequence = False
+            for instrument_config in data['instrument_configs']:
+                offset_ra = instrument_config.get('extra_params', {}).get('offset_ra', 0)
+                offset_dec = instrument_config.get('extra_params', {}).get('offset_dec', 0)
+                if offset_dec != 0 or offset_ra != 0:
+                    is_dither_sequence = True
+                    break
+            if is_dither_sequence:
+                if 'extra_params' not in data:
+                    data['extra_params'] = {}
+                data['extra_params']['dither_pattern'] = settings.DITHER['custom_pattern_key']
+
         return data
 
 
@@ -618,6 +646,14 @@ def validate(self, data):
                  for configuration in data['configurations']]
             )
 
+        if 'extra_params' in data and 'mosaic_pattern' in data['extra_params']:
+            pattern = data['extra_params']['mosaic_pattern']
+            valid_patterns = list(settings.MOSAIC['valid_expansion_patterns']) + [settings.MOSAIC['custom_pattern_key']]
+            if pattern not in valid_patterns:
+                raise serializers.ValidationError(_(
+                    f'Invalid mosaic pattern {pattern} set in the request extra_params, choose from {", ".join(valid_patterns)}'
+                ))
+
         # check that the requests window has enough rise_set visible time to accomodate the requests duration
         if data.get('windows'):
             duration = get_total_request_duration(data)
@@ -874,7 +910,7 @@ def validate(self, data):
 
 class MosaicSerializer(PatternExpansionSerializer):
     request = import_string(settings.SERIALIZERS['requestgroups']['Request'])()
-    pattern = serializers.ChoiceField(choices=('line', 'grid'), required=True)
+    pattern = serializers.ChoiceField(choices=settings.MOSAIC['valid_expansion_patterns'], required=True)
     point_overlap_percent = serializers.FloatField(required=False, validators=[MinValueValidator(0.0), MaxValueValidator(100.0)])
     line_overlap_percent = serializers.FloatField(required=False, validators=[MinValueValidator(0.0), MaxValueValidator(100.0)])
 
@@ -930,6 +966,7 @@ def validate(self, data):
 
 class DitherSerializer(PatternExpansionSerializer):
     configuration = import_string(settings.SERIALIZERS['requestgroups']['Configuration'])()
+    pattern = serializers.ChoiceField(choices=settings.DITHER['valid_expansion_patterns'], required=True)
     point_spacing = serializers.FloatField(required=True)
 
 

observation_portal/requestgroups/test/test_api.py

@@ -490,6 +490,22 @@ def test_post_requestgroup_default_acceptability_threshold(self):
         self.assertEqual(response.status_code, 201)
         self.assertEqual(response.json()['requests'][0]['acceptability_threshold'], 100)
 
+    def test_mosaic_pattern_valid(self):
+        data = self.generic_payload.copy()
+        data['requests'][0]['extra_params'] = {}
+        # Test that an invalid mosaic pattern is rejected
+        data['requests'][0]['extra_params']['mosaic_pattern'] = 'swirl'
+        response = self.client.post(reverse('api:request_groups-list'), data=data)
+        self.assertEqual(response.status_code, 400)
+        # Then check that a valid pattern is accepted
+        data['requests'][0]['extra_params']['mosaic_pattern'] = 'line'
+        response = self.client.post(reverse('api:request_groups-list'), data=data)
+        self.assertEqual(response.status_code, 201)
+        # Then check that the custom mosaic pattern key is accepted
+        data['requests'][0]['extra_params']['mosaic_pattern'] = 'custom'
+        response = self.client.post(reverse('api:request_groups-list'), data=data)
+        self.assertEqual(response.status_code, 201)
+
 
 class TestDisallowedMethods(APITestCase):
     def setUp(self):
@@ -927,6 +943,24 @@ def test_expansion_autofills_line_spacing_for_grid(self):
         self.assertEqual(offset_diff_ra, dither_data['point_spacing'])
         self.assertEqual(offset_diff_dec, dither_data['point_spacing'])
 
+    def test_expansion_saves_dither_params(self):
+        configuration = self.configuration.copy()
+        dither_data = {
+            'configuration': configuration,
+            'num_rows': 2,
+            'num_columns': 2,
+            'pattern': 'grid',
+            'point_spacing': 2,
+            'orientation': 90
+        }
+        response = self.client.post(reverse('api:configurations-dither'), data=dither_data)
+        self.assertEqual(response.status_code, 200)
+        expanded_configuration = response.json()
+        self.assertEqual(expanded_configuration['extra_params']['dither_pattern'], dither_data['pattern'])
+        self.assertTrue('configuration' not in expanded_configuration['extra_params']['dither_pattern_params'])
+        self.assertEqual(expanded_configuration['extra_params']['dither_pattern_params']['num_rows'], dither_data['num_rows'])
+        self.assertEqual(expanded_configuration['extra_params']['dither_pattern_params']['num_columns'], dither_data['num_columns'])
+
 
 class TestMosaicApi(SetTimeMixin, APITestCase):
     def setUp(self):
@@ -1109,6 +1143,24 @@ def test_expansion_10_percent_overlap_spacing_grid(self):
         self.assertEqual(target2['ra'], target1['ra'] + (ra_diff / 3600.0) / cos_dec)
         self.assertEqual(target2['dec'], target1['dec'] + (dec_diff / 3600.0))
 
+    def test_expansion_saves_mosaic_parameters(self):
+        request = self.request.copy()
+        mosaic_data = {
+            'request': request,
+            'num_points': 3,
+            'pattern': 'line',
+            'point_overlap_percent': 10.0,
+            'orientation': 90.0
+        }
+        response = self.client.post(reverse('api:requests-mosaic'), data=mosaic_data)
+        self.assertEqual(response.status_code, 200)
+        expanded_request = response.json()
+        self.assertEqual(len(expanded_request['configurations']), 3)
+        self.assertEqual(expanded_request['extra_params']['mosaic_pattern'], mosaic_data['pattern'])
+        self.assertTrue('request' not in expanded_request['extra_params']['mosaic_pattern_params'])
+        self.assertEqual(expanded_request['extra_params']['mosaic_pattern_params']['num_points'], mosaic_data['num_points'])
+        self.assertEqual(expanded_request['extra_params']['mosaic_pattern_params']['point_overlap_percent'], mosaic_data['point_overlap_percent'])
+
 
 class TestCadenceApi(SetTimeMixin, APITestCase):
     def setUp(self):
@@ -1629,6 +1681,52 @@ def test_must_have_at_least_one_instrument_config(self):
         self.assertEqual(response.status_code, 400)
         self.assertIn('must have at least one instrument configuration', str(response.content))
 
+    def test_dither_pattern_valid(self):
+        data = self.generic_payload.copy()
+        data['requests'][0]['configurations'][0]['extra_params'] = {}
+        # First check that an invalid pattern is not accepted
+        data['requests'][0]['configurations'][0]['extra_params']['dither_pattern'] = 'mountain'
+        response = self.client.post(reverse('api:request_groups-list'), data=data)
+        self.assertEqual(response.status_code, 400)
+        # Then check that a valid pattern is accepted
+        data['requests'][0]['configurations'][0]['extra_params']['dither_pattern'] = 'line'
+        response = self.client.post(reverse('api:request_groups-list'), data=data)
+        self.assertEqual(response.status_code, 201)
+        # Then check that the custom dither pattern key is accepted
+        data['requests'][0]['configurations'][0]['extra_params']['dither_pattern'] = 'custom'
+        response = self.client.post(reverse('api:request_groups-list'), data=data)
+        self.assertEqual(response.status_code, 201)
+
+    def test_custom_dither_pattern_set(self):
+        data = self.generic_payload.copy()
+        # First check that the custom dither pattern is not set on a configuration that has no
+        # instrument configs with offsets applied.
+        data['requests'][0]['configurations'][0]['instrument_configs'][0]['extra_params'] = {}
+        response = self.client.post(reverse('api:request_groups-list'), data=data)
+        self.assertEqual(response.status_code, 201)
+        self.assertTrue('dither_pattern' not in response.json()['requests'][0]['configurations'][0]['extra_params'])
+        # The check that when the only offset is 0, the custom dither pattern is not set
+        data['requests'][0]['configurations'][0]['instrument_configs'][0]['extra_params'] = {'offset_ra': 0, 'offset_dec': 0}
+        response = self.client.post(reverse('api:request_groups-list'), data=data)
+        self.assertEqual(response.status_code, 201)
+        self.assertTrue('dither_pattern' not in response.json()['requests'][0]['configurations'][0]['extra_params'])
+        # Then check that when some offset is applied, the custom dither pattern is set
+        data['requests'][0]['configurations'][0]['instrument_configs'][0]['extra_params'] = {'offset_ra': 1, 'offset_dec': 0}
+        response = self.client.post(reverse('api:request_groups-list'), data=data)
+        self.assertEqual(response.status_code, 201)
+        self.assertEqual(response.json()['requests'][0]['configurations'][0]['extra_params']['dither_pattern'], 'custom')
+        # Then check that when only 0 offsets are applied, the custom dither pattern is not set
+        data['requests'][0]['configurations'][0]['instrument_configs'][0]['extra_params'] = {'offset_ra': 0, 'offset_dec': 0}
+        response = self.client.post(reverse('api:request_groups-list'), data=data)
+        self.assertEqual(response.status_code, 201)
+        self.assertTrue('dither_pattern' not in response.json()['requests'][0]['configurations'][0]['extra_params'])
+        # Then check that if a valid pattern is already set, it is not overridden
+        valid_pattern = 'line'
+        data['requests'][0]['configurations'][0]['extra_params'] = {'dither_pattern': valid_pattern}
+        response = self.client.post(reverse('api:request_groups-list'), data=data)
+        self.assertEqual(response.status_code, 201)
+        self.assertEqual(response.json()['requests'][0]['configurations'][0]['extra_params']['dither_pattern'], valid_pattern)
+
     def test_extra_params_saved_as_float_not_string(self):
         good_data = self.generic_payload.copy()
         good_data['requests'][0]['configurations'][0]['extra_params'] = {'test_value': '1.15'}

observation_portal/settings.py

@@ -242,6 +242,14 @@ def get_list_from_env(variable, default=None):
 REQUEST_DETAIL_URL = os.getenv('REQUEST_DETAIL_URL', '')  # use {request_id} to have it substituted in
 SCIENCE_APPLICATION_DETAIL_URL = os.getenv('SCIENCE_APPLICATION_DETAIL_URL', '')  # use {scicapp_id} to have it substituted in
 MAX_FAILURES_PER_REQUEST = int(os.getenv('MAX_FAILURES_PER_REQUEST', 0))  # 0 means unlimited / no max
+DITHER = {
+    'custom_pattern_key': 'custom', # Key used to indicate a custom dither pattern was created
+    'valid_expansion_patterns': ('line', 'grid', 'spiral', )
+}
+MOSAIC = {
+    'custom_pattern_key': 'custom', # Key used to indicate a custom mosaic pattern was created
+    'valid_expansion_patterns': ('line', 'grid', )
+}
 
 REST_FRAMEWORK = {
     'TEST_REQUEST_DEFAULT_FORMAT': 'json',