From 25e848f742b09d61d7b887cea1cc17d441131525 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Mon, 15 May 2023 19:17:06 -0700 Subject: [PATCH 01/42] refactor sharing view (broken) --- tom_dataproducts/sharing.py | 98 +++++++++++++++++++++++++ tom_dataproducts/tests/tests.py | 54 ++++++++++++++ tom_dataproducts/views.py | 123 ++++++++------------------------ 3 files changed, 183 insertions(+), 92 deletions(-) create mode 100644 tom_dataproducts/sharing.py diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py new file mode 100644 index 000000000..b1bd45483 --- /dev/null +++ b/tom_dataproducts/sharing.py @@ -0,0 +1,98 @@ + +def share_data_with_hermes(share_destination, form_data, product_id=None, target_id=None, selected_data=None): + # Query relevant Reduced Datums Queryset + accepted_data_types = ['photometry'] + if product_id: + product = DataProduct.objects.get(pk=product_id) + reduced_datums = ReducedDatum.objects.filter(data_product=product) + elif selected_data: + reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) + elif target_id: + target = Target.objects.get(pk=target_id) + data_type = form_data['data_type'] + reduced_datums = ReducedDatum.objects.filter(target=target, data_type=data_type) + else: + reduced_datums = ReducedDatum.objects.none() + + reduced_datums.filter(data_type__in=accepted_data_types) + + # Build and submit hermes table from Reduced Datums + hermes_topic = share_destination.split(':')[1] + destination = share_destination.split(':')[0] + message_info = BuildHermesMessage(title=form_data['share_title'], + submitter=form_data['submitter'], + authors=form_data['share_authors'], + message=form_data['share_message'], + topic=hermes_topic + ) + # Run ReducedDatums Queryset through sharing protocols to make sure they are safe to share. + filtered_reduced_datums = get_share_safe_datums(destination, reduced_datums, topic=hermes_topic) + if filtered_reduced_datums.count() > 0: + response = publish_photometry_to_hermes(message_info, filtered_reduced_datums) + else: + messages.error(self.request, f'No Data to share. (Check sharing Protocol, note that data types must be ' + f'in {accepted_data_types})') + + +def share_data_with_tom(destination, datums, product=None): + """ + When sharing a DataProduct with another TOM we likely want to share the data product itself and let the other + TOM process it rather than share the Reduced Datums + :param destination: name of destination tom in settings.DATA_SHARING + :param datums: Queryset of ReducedDatum Instances + :param product: DataProduct model instance + :return: + """ + try: + destination_tom_base_url = settings.DATA_SHARING[destination]['BASE_URL'] + username = settings.DATA_SHARING[destination]['USERNAME'] + password = settings.DATA_SHARING[destination]['PASSWORD'] + except KeyError as err: + raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {destination}: Key {err} not found.') + auth = (username, password) + headers = {'Media-Type': 'application/json'} + target = product.target + serialized_target_data = TargetSerializer(target).data + targets_url = destination_tom_base_url + 'api/targets/' + # TODO: Make sure aliases are checked before creating new target + # Attempt to create Target in Destination TOM + # response = requests.post(targets_url, headers=headers, auth=auth, data=serialized_target_data) + # try: + # target_response = response.json() + # destination_target_id = target_response['id'] + # except KeyError: + # # If Target already exists at destination, find ID + # response = requests.get(targets_url, headers=headers, auth=auth, data=serialized_target_data) + # target_response = response.json() + # destination_target_id = target_response['results'][0]['id'] + + print(serialized_target_data) + + response = requests.get(f'{targets_url}?name={target.name}', headers=headers, auth=auth) + target_response = response.json() + if target_response['results']: + destination_target_id = target_response['results'][0]['id'] + else: + return response + print("------------------------") + print(target_response) + serialized_dataproduct_data = DataProductSerializer(product).data + serialized_dataproduct_data['target'] = destination_target_id + dataproducts_url = destination_tom_base_url + 'api/dataproducts/' + # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage + dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) + # Save DataProduct in Destination TOM + # with open(dataproduct_filename, 'rb') as dataproduct_filep: + # files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} + # headers = {'Media-Type': 'multipart/form-data'} + # response = requests.post(dataproducts_url, data=serialized_dataproduct_data, files=files, + # headers=headers, auth=auth) + return response + + +def check_for_share_safe_datums(): + return + + +def check_for_save_safe_datums(): + return diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index bf671c3c8..9c6679f7b 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -489,3 +489,57 @@ def test_create_thumbnail(self, mock_is_fits_image_file): 'ignore_missing_simple=True') self.assertIn(expected, logs.output) + + +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], + TARGET_PERMISSIONS_ONLY=True) +@patch('tom_dataproducts.views.run_data_processor') +class TestShareDataProducts(TestCase): + def setUp(self): + self.target = SiderealTargetFactory.create() + self.observation_record = ObservingRecordFactory.create( + target_id=self.target.id, + facility=FakeRoboticFacility.name, + parameters={} + ) + self.data_product = DataProduct.objects.create( + product_id='testproductid', + target=self.target, + observation_record=self.observation_record, + data=SimpleUploadedFile('afile.fits', b'somedata') + ) + self.user = User.objects.create_user(username='test', email='test@example.com') + assign_perm('tom_targets.view_target', self.user, self.target) + self.client.force_login(self.user) + + def test_share_dataproduct(self, run_data_processor_mock): + + response = self.client.post( + reverse('dataproducts:share', kwargs={'dp_pk': self.data_product.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': ['local_host'], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'] + }, + follow=True + ) + self.assertContains(response, 'TOM-TOM sharing is not yet supported.') + + def test_share_data_for_target(self, run_data_processor_mock): + + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': ['local_host'], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'] + }, + follow=True + ) + self.assertContains(response, 'TOM-TOM sharing is not yet supported.') \ No newline at end of file diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 73cb920a8..207b77c76 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -314,109 +314,48 @@ def post(self, request, *args, **kwargs): Handles Data Products and All the data of a type for a target as well as individual Reduced Datums. Submit to Hermes, or Share with TOM (soon). """ - data_share_form = DataShareForm(request.POST, request.FILES) - # Check if data points have been selected. - selected_data = request.POST.getlist("share-box") + if data_share_form.is_valid(): form_data = data_share_form.cleaned_data - # 1st determine if pk is data product, Reduced Datum, or Target. - # Then query relevant Reduced Datums Queryset + share_destination = form_data['share_destination'] product_id = kwargs.get('dp_pk', None) - if product_id: - product = DataProduct.objects.get(pk=product_id) - data_type = product.data_product_type - reduced_datums = ReducedDatum.objects.filter(data_product=product) + target_id = kwargs.get('tg_pk', None) + + + # Check if data points have been selected. + selected_data = request.POST.getlist("share-box") + + # Check Destination + if 'HERMES' in share_destination.upper(): + response = share_data_with_hermes(share_destination, form_data, product_id, target_id, selected_data) else: - target_id = kwargs.get('tg_pk', None) - target = Target.objects.get(pk=target_id) - data_type = form_data['data_type'] - if request.POST.get("share-box", None) is None: - reduced_datums = ReducedDatum.objects.filter(target=target, data_type=data_type) + response = share_data_with_tom(share_destination, form_data, product_id, target_id, selected_data) + try: + if 'message' in response.json(): + publish_feedback = response.json()['message'] else: - reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) - if data_type == 'photometry': - share_destination = form_data['share_destination'] - if 'HERMES' in share_destination.upper(): - # Build and submit hermes table from Reduced Datums - hermes_topic = share_destination.split(':')[1] - destination = share_destination.split(':')[0] - message_info = BuildHermesMessage(title=form_data['share_title'], - submitter=form_data['submitter'], - authors=form_data['share_authors'], - message=form_data['share_message'], - topic=hermes_topic - ) - # Run ReducedDatums Queryset through sharing protocols to make sure they are safe to share. - filtered_reduced_datums = self.get_share_safe_datums(destination, reduced_datums, - topic=hermes_topic) - if filtered_reduced_datums.count() > 0: - response = publish_photometry_to_hermes(message_info, filtered_reduced_datums) - else: - messages.error(self.request, 'No Data to share. (Check sharing Protocol.)') + publish_feedback = f"ERROR: {response.text}" + except ValueError: + publish_feedback = f"ERROR: Returned Response code {response.status_code}" + if "ERROR" in publish_feedback.upper(): + messages.error(self.request, publish_feedback) + else: + messages.success(self.request, publish_feedback) + return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) + + + if data_share_form.is_valid(): + form_data = data_share_form.cleaned_data + return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) else: - messages.error(self.request, 'TOM-TOM sharing is not yet supported.') + # messages.error(self.request, 'TOM-TOM sharing is not yet supported.') + response = self.share_with_tom(share_destination, product) return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) # response = self.share_with_tom(share_destination, product) - try: - if 'message' in response.json(): - publish_feedback = response.json()['message'] - else: - publish_feedback = f"ERROR: {response.text}" - except ValueError: - publish_feedback = f"ERROR: Returned Response code {response.status_code}" - if "ERROR" in publish_feedback.upper(): - messages.error(self.request, publish_feedback) - else: - messages.success(self.request, publish_feedback) - else: - messages.error(self.request, f'Publishing {data_type} data is not yet supported.') - return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - def share_with_tom(self, tom_name, product): - """ - When sharing a DataProduct with another TOM we likely want to share the data product itself and let the other - TOM process it rather than share the Reduced Datums - :param tom_name: name of destination tom in settings.DATA_SHARING - :param product: DataProduct model instance - :return: - """ - try: - destination_tom_base_url = settings.DATA_SHARING[tom_name]['BASE_URL'] - username = settings.DATA_SHARING[tom_name]['USERNAME'] - password = settings.DATA_SHARING[tom_name]['PASSWORD'] - except KeyError as err: - raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {tom_name}: Key {err} not found.') - auth = (username, password) - headers = {'Media-Type': 'application/json'} - target = product.target - serialized_target_data = TargetSerializer(target).data - targets_url = destination_tom_base_url + 'api/targets/' - # TODO: Make sure aliases are checked before creating new target - # Attempt to create Target in Destination TOM - response = requests.post(targets_url, headers=headers, auth=auth, data=serialized_target_data) - try: - target_response = response.json() - destination_target_id = target_response['id'] - except KeyError: - # If Target already exists at destination, find ID - response = requests.get(targets_url, headers=headers, auth=auth, data=serialized_target_data) - target_response = response.json() - destination_target_id = target_response['results'][0]['id'] - - serialized_dataproduct_data = DataProductSerializer(product).data - serialized_dataproduct_data['target'] = destination_target_id - dataproducts_url = destination_tom_base_url + 'api/dataproducts/' - # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage - dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) - # Save DataProduct in Destination TOM - with open(dataproduct_filename, 'rb') as dataproduct_filep: - files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} - headers = {'Media-Type': 'multipart/form-data'} - response = requests.post(dataproducts_url, data=serialized_dataproduct_data, files=files, - headers=headers, auth=auth) - return response + def get_share_safe_datums(self, destination, reduced_datums, **kwargs): """ From 035dea33018285def9ddafefc8910511d3c0ec18 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 16 May 2023 12:30:26 -0700 Subject: [PATCH 02/42] add hermes error message --- tom_dataproducts/sharing.py | 7 +++++-- tom_dataproducts/views.py | 22 +++++++++------------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index b1bd45483..a24adc80e 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -30,8 +30,11 @@ def share_data_with_hermes(share_destination, form_data, product_id=None, target if filtered_reduced_datums.count() > 0: response = publish_photometry_to_hermes(message_info, filtered_reduced_datums) else: - messages.error(self.request, f'No Data to share. (Check sharing Protocol, note that data types must be ' - f'in {accepted_data_types})') + def response(): + def json(): + return {'message': f'No Data to share. (Check sharing Protocol, note that data types must be in ' + f'{accepted_data_types})'} + return response def share_data_with_tom(destination, datums, product=None): diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 207b77c76..62e2ee849 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -322,7 +322,6 @@ def post(self, request, *args, **kwargs): product_id = kwargs.get('dp_pk', None) target_id = kwargs.get('tg_pk', None) - # Check if data points have been selected. selected_data = request.POST.getlist("share-box") @@ -344,18 +343,15 @@ def post(self, request, *args, **kwargs): messages.success(self.request, publish_feedback) return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - - if data_share_form.is_valid(): - form_data = data_share_form.cleaned_data - - return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - else: - # messages.error(self.request, 'TOM-TOM sharing is not yet supported.') - response = self.share_with_tom(share_destination, product) - return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - # response = self.share_with_tom(share_destination, product) - - + # if data_share_form.is_valid(): + # form_data = data_share_form.cleaned_data + # + # return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) + # else: + # # messages.error(self.request, 'TOM-TOM sharing is not yet supported.') + # response = self.share_with_tom(share_destination, product) + # return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) + # # response = self.share_with_tom(share_destination, product) def get_share_safe_datums(self, destination, reduced_datums, **kwargs): """ From 4da20e85a3811d6492e7ed49507933dbed12df6b Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 16 May 2023 15:48:05 -0700 Subject: [PATCH 03/42] fix some imports --- tom_dataproducts/sharing.py | 35 ++++++++++++++++++++++++----------- tom_dataproducts/views.py | 22 +++------------------- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index a24adc80e..f1c7ad9ad 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -1,3 +1,7 @@ +from tom_targets.models import Target +from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum +from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage + def share_data_with_hermes(share_destination, form_data, product_id=None, target_id=None, selected_data=None): # Query relevant Reduced Datums Queryset @@ -26,14 +30,12 @@ def share_data_with_hermes(share_destination, form_data, product_id=None, target topic=hermes_topic ) # Run ReducedDatums Queryset through sharing protocols to make sure they are safe to share. - filtered_reduced_datums = get_share_safe_datums(destination, reduced_datums, topic=hermes_topic) + filtered_reduced_datums = check_for_share_safe_datums(destination, reduced_datums, topic=hermes_topic) if filtered_reduced_datums.count() > 0: response = publish_photometry_to_hermes(message_info, filtered_reduced_datums) else: - def response(): - def json(): - return {'message': f'No Data to share. (Check sharing Protocol, note that data types must be in ' - f'{accepted_data_types})'} + return {'message': f'ERROR: No valid data to share. (Check Sharing Protocol. Note that data types must be in ' + f'{accepted_data_types})'} return response @@ -69,16 +71,12 @@ def share_data_with_tom(destination, datums, product=None): # target_response = response.json() # destination_target_id = target_response['results'][0]['id'] - print(serialized_target_data) - response = requests.get(f'{targets_url}?name={target.name}', headers=headers, auth=auth) target_response = response.json() if target_response['results']: destination_target_id = target_response['results'][0]['id'] else: return response - print("------------------------") - print(target_response) serialized_dataproduct_data = DataProductSerializer(product).data serialized_dataproduct_data['target'] = destination_target_id dataproducts_url = destination_tom_base_url + 'api/dataproducts/' @@ -93,8 +91,23 @@ def share_data_with_tom(destination, datums, product=None): return response -def check_for_share_safe_datums(): - return +def check_for_share_safe_datums(destination, reduced_datums, **kwargs): + """ + Custom sharing protocols used to determine when data is shared with a destination. + This example prevents sharing if a datum has already been published to the given Hermes topic. + :param destination: sharing destination string + :param reduced_datums: selected input datums + :return: queryset of reduced datums to be shared + """ + return reduced_datums + # if 'hermes' in destination: + # message_topic = kwargs.get('topic', None) + # # Remove data points previously shared to the given topic + # filtered_datums = reduced_datums.exclude(Q(message__exchange_status='published') + # & Q(message__topic=message_topic)) + # else: + # filtered_datums = reduced_datums + # return filtered_datums def check_for_save_safe_datums(): diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 62e2ee849..d6ac8dd80 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -32,10 +32,10 @@ from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm, DataShareForm from tom_dataproducts.filters import DataProductFilter from tom_dataproducts.data_processor import run_data_processor -from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class from tom_dataproducts.serializers import DataProductSerializer +from tom_dataproducts.sharing import share_data_with_hermes import requests @@ -335,6 +335,8 @@ def post(self, request, *args, **kwargs): publish_feedback = response.json()['message'] else: publish_feedback = f"ERROR: {response.text}" + except AttributeError: + publish_feedback = response['message'] except ValueError: publish_feedback = f"ERROR: Returned Response code {response.status_code}" if "ERROR" in publish_feedback.upper(): @@ -353,24 +355,6 @@ def post(self, request, *args, **kwargs): # return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) # # response = self.share_with_tom(share_destination, product) - def get_share_safe_datums(self, destination, reduced_datums, **kwargs): - """ - Custom sharing protocols used to determine when data is shared with a destination. - This example prevents sharing if a datum has already been published to the given Hermes topic. - :param destination: sharing destination string - :param reduced_datums: selected input datums - :return: queryset of reduced datums to be shared - """ - return reduced_datums - # if 'hermes' in destination: - # message_topic = kwargs.get('topic', None) - # # Remove data points previously shared to the given topic - # filtered_datums = reduced_datums.exclude(Q(message__exchange_status='published') - # & Q(message__topic=message_topic)) - # else: - # filtered_datums = reduced_datums - # return filtered_datums - class DataProductGroupDetailView(DetailView): """ From e801e815fbb39a0424ec1fe24c8e68cf861b2d11 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 17 May 2023 17:07:24 -0700 Subject: [PATCH 04/42] finish Dataproduct sharing --- tom_dataproducts/alertstreams/hermes.py | 2 +- tom_dataproducts/api_views.py | 1 + tom_dataproducts/sharing.py | 93 ++++++++++++++++--------- tom_dataproducts/views.py | 3 +- 4 files changed, 65 insertions(+), 34 deletions(-) diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index d2589af22..cd1c2742a 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -136,7 +136,7 @@ def get_hermes_topics(**kwargs): response = requests.get(url=submit_url, headers=headers) topics = response.json()['writable_topics'] - except KeyError: + except (KeyError, requests.exceptions.JSONDecodeError): topics = settings.DATA_SHARING['hermes']['USER_TOPICS'] return topics diff --git a/tom_dataproducts/api_views.py b/tom_dataproducts/api_views.py index 8f59138ae..964df7a82 100644 --- a/tom_dataproducts/api_views.py +++ b/tom_dataproducts/api_views.py @@ -38,6 +38,7 @@ def create(self, request, *args, **kwargs): response = super().create(request, *args, **kwargs) if response.status_code == status.HTTP_201_CREATED: + response.data['message'] = 'Data product successfully uploaded.' dp = DataProduct.objects.get(pk=response.data['id']) try: run_hook('data_product_post_upload', dp) diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index f1c7ad9ad..9bf67f4ec 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -1,9 +1,24 @@ +import requests +import os + +from django.conf import settings + from tom_targets.models import Target from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage +from tom_dataproducts.serializers import DataProductSerializer def share_data_with_hermes(share_destination, form_data, product_id=None, target_id=None, selected_data=None): + """ + + :param share_destination: + :param form_data: + :param product_id: + :param target_id: + :param selected_data: + :return: + """ # Query relevant Reduced Datums Queryset accepted_data_types = ['photometry'] if product_id: @@ -39,26 +54,61 @@ def share_data_with_hermes(share_destination, form_data, product_id=None, target return response -def share_data_with_tom(destination, datums, product=None): +def share_data_with_tom(share_destination, form_data, product_id=None, target_id=None, selected_data=None): """ - When sharing a DataProduct with another TOM we likely want to share the data product itself and let the other - TOM process it rather than share the Reduced Datums - :param destination: name of destination tom in settings.DATA_SHARING - :param datums: Queryset of ReducedDatum Instances - :param product: DataProduct model instance + + :param share_destination: + :param form_data: + :param product_id: + :param target_id: + :param selected_data: :return: """ try: - destination_tom_base_url = settings.DATA_SHARING[destination]['BASE_URL'] - username = settings.DATA_SHARING[destination]['USERNAME'] - password = settings.DATA_SHARING[destination]['PASSWORD'] + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + username = settings.DATA_SHARING[share_destination]['USERNAME'] + password = settings.DATA_SHARING[share_destination]['PASSWORD'] except KeyError as err: - raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {destination}: Key {err} not found.') + raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {share_destination}: Key {err} not found.') auth = (username, password) headers = {'Media-Type': 'application/json'} - target = product.target - serialized_target_data = TargetSerializer(target).data + + dataproducts_url = destination_tom_base_url + 'api/dataproducts/' targets_url = destination_tom_base_url + 'api/targets/' + reduced_datums = ReducedDatum.objects.none() + if product_id: + product = DataProduct.objects.get(pk=product_id) + target = product.target + serialized_data = DataProductSerializer(product).data + # elif selected_data: + # reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) + # elif target_id: + # target = Target.objects.get(pk=target_id) + # data_type = form_data['data_type'] + # reduced_datums = ReducedDatum.objects.filter(target=target, data_type=data_type) + else: + return {'message': f'ERROR: No valid data to share.'} + + # get destination Target + target_response = requests.get(f'{targets_url}?name={target.name}', headers=headers, auth=auth) + target_response_json = target_response.json() + if target_response_json['results']: + destination_target_id = target_response_json['results'][0]['id'] + else: + return target_response + + serialized_data['target'] = destination_target_id + + # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage + dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) + # Save DataProduct in Destination TOM + with open(dataproduct_filename, 'rb') as dataproduct_filep: + files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} + headers = {'Media-Type': 'multipart/form-data'} + response = requests.post(dataproducts_url, data=serialized_data, files=files, + headers=headers, auth=auth) + return response + # serialized_target_data = TargetSerializer(target).data # TODO: Make sure aliases are checked before creating new target # Attempt to create Target in Destination TOM # response = requests.post(targets_url, headers=headers, auth=auth, data=serialized_target_data) @@ -71,25 +121,6 @@ def share_data_with_tom(destination, datums, product=None): # target_response = response.json() # destination_target_id = target_response['results'][0]['id'] - response = requests.get(f'{targets_url}?name={target.name}', headers=headers, auth=auth) - target_response = response.json() - if target_response['results']: - destination_target_id = target_response['results'][0]['id'] - else: - return response - serialized_dataproduct_data = DataProductSerializer(product).data - serialized_dataproduct_data['target'] = destination_target_id - dataproducts_url = destination_tom_base_url + 'api/dataproducts/' - # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage - dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) - # Save DataProduct in Destination TOM - # with open(dataproduct_filename, 'rb') as dataproduct_filep: - # files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} - # headers = {'Media-Type': 'multipart/form-data'} - # response = requests.post(dataproducts_url, data=serialized_dataproduct_data, files=files, - # headers=headers, auth=auth) - return response - def check_for_share_safe_datums(destination, reduced_datums, **kwargs): """ diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index d6ac8dd80..093ef8a6a 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -34,8 +34,7 @@ from tom_dataproducts.data_processor import run_data_processor from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class -from tom_dataproducts.serializers import DataProductSerializer -from tom_dataproducts.sharing import share_data_with_hermes +from tom_dataproducts.sharing import share_data_with_hermes, share_data_with_tom import requests From 008ac5729bce52377186b055e404a442708db4d0 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 22 Jun 2023 14:27:24 -0700 Subject: [PATCH 05/42] add reduced datum sharing --- tom_dataproducts/api_views.py | 30 ++++++++++- tom_dataproducts/forms.py | 61 +++++++++++----------- tom_dataproducts/models.py | 11 +++- tom_dataproducts/serializers.py | 26 +++++++++- tom_dataproducts/sharing.py | 83 ++++++++++++++++++------------ tom_dataproducts/tests/test_api.py | 40 ++++++++++++++ tom_dataproducts/urls.py | 3 +- 7 files changed, 183 insertions(+), 71 deletions(-) diff --git a/tom_dataproducts/api_views.py b/tom_dataproducts/api_views.py index 964df7a82..5661098b8 100644 --- a/tom_dataproducts/api_views.py +++ b/tom_dataproducts/api_views.py @@ -4,7 +4,7 @@ from guardian.shortcuts import assign_perm, get_objects_for_user from rest_framework import status from rest_framework.mixins import CreateModelMixin, DestroyModelMixin, ListModelMixin -from rest_framework.parsers import MultiPartParser +from rest_framework.parsers import MultiPartParser, FormParser, JSONParser from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet @@ -12,7 +12,7 @@ from tom_dataproducts.data_processor import run_data_processor from tom_dataproducts.filters import DataProductFilter from tom_dataproducts.models import DataProduct, ReducedDatum -from tom_dataproducts.serializers import DataProductSerializer +from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer class DataProductViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet, PermissionListMixin): @@ -69,3 +69,29 @@ def get_queryset(self): ) else: return get_objects_for_user(self.request.user, 'tom_dataproducts.view_dataproduct') + + +class ReducedDatumViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet, PermissionListMixin): + """ + Viewset for ReducedDatum objects. Supports list, create, and delete. + + To view supported query parameters, please use the OPTIONS endpoint, which can be accessed through the web UI. + + **Please note that ``groups`` are an accepted query parameters for the ``CREATE`` endpoint. The groups parameter + will specify which ``groups`` can view the created ``DataProduct``. If no ``groups`` are specified, the + ``ReducedDatum`` will only be visible to the user that created the ``DataProduct``. Make sure to check your + ``groups``!!** + """ + queryset = ReducedDatum.objects.all() + serializer_class = ReducedDatumSerializer + filter_backends = (drf_filters.DjangoFilterBackend,) + permission_required = 'tom_dataproducts.view_reduceddatum' + parser_classes = [FormParser, JSONParser] + + def create(self, request, *args, **kwargs): + response = super().create(request, *args, **kwargs) + + if response.status_code == status.HTTP_201_CREATED: + response.data['message'] = 'Data successfully uploaded.' + + return response diff --git a/tom_dataproducts/forms.py b/tom_dataproducts/forms.py index db39c1e75..e0b615a96 100644 --- a/tom_dataproducts/forms.py +++ b/tom_dataproducts/forms.py @@ -8,36 +8,6 @@ from tom_dataproducts.alertstreams.hermes import get_hermes_topics -def get_sharing_destination_options(): - """ - Build the Display options and headers for the dropdown form for choosing sharing topics. - Customize for a different selection experience. - :return: Tuple: Possible Destinations and their Display Names - """ - choices = [] - try: - for destination, details in settings.DATA_SHARING.items(): - new_destination = [details.get('DISPLAY_NAME', destination)] - if details.get('USER_TOPICS', None): - # If topics exist for a destination (Such as HERMES) give topics as sub-choices - # for non-selectable Destination - if destination == "hermes": - destination_topics = get_hermes_topics() - else: - destination_topics = details['USER_TOPICS'] - topic_list = [(f'{destination}:{topic}', topic) for topic in destination_topics] - new_destination.append(tuple(topic_list)) - else: - # Otherwise just use destination as option - new_destination.insert(0, destination) - choices.append(tuple(new_destination)) - except AttributeError: - pass - return tuple(choices) - - -DESTINATION_OPTIONS = get_sharing_destination_options() - DATA_TYPE_OPTIONS = (('photometry', 'Photometry'), ('spectroscopy', 'Spectroscopy')) @@ -82,7 +52,7 @@ def __init__(self, *args, **kwargs): class DataShareForm(forms.Form): - share_destination = forms.ChoiceField(required=True, choices=DESTINATION_OPTIONS, label="Destination") + share_destination = forms.ChoiceField(required=True, choices=[], label="Destination") share_title = forms.CharField(required=False, label="Title") share_message = forms.CharField(required=False, label="Message", widget=forms.Textarea()) share_authors = forms.CharField(required=False, widget=forms.HiddenInput()) @@ -95,6 +65,33 @@ class DataShareForm(forms.Form): widget=forms.HiddenInput() ) + def get_sharing_destination_options(self): + """ + Build the Display options and headers for the dropdown form for choosing sharing topics. + Customize for a different selection experience. + :return: Tuple: Possible Destinations and their Display Names + """ + choices = [] + try: + for destination, details in settings.DATA_SHARING.items(): + new_destination = [details.get('DISPLAY_NAME', destination)] + if details.get('USER_TOPICS', None): + # If topics exist for a destination (Such as HERMES) give topics as sub-choices + # for non-selectable Destination + if destination == "hermes": + destination_topics = get_hermes_topics() + else: + destination_topics = details['USER_TOPICS'] + topic_list = [(f'{destination}:{topic}', topic) for topic in destination_topics] + new_destination.append(tuple(topic_list)) + else: + # Otherwise just use destination as option + new_destination.insert(0, destination) + choices.append(tuple(new_destination)) + except AttributeError: + pass + return tuple(choices) + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['share_destination'].choices = DESTINATION_OPTIONS + self.fields['share_destination'].choices = self.get_sharing_destination_options() diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index d8fbef1ae..6e2664471 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -331,7 +331,7 @@ class ReducedDatum(models.Model): """ target = models.ForeignKey(Target, null=False, on_delete=models.CASCADE) - data_product = models.ForeignKey(DataProduct, null=True, on_delete=models.CASCADE) + data_product = models.ForeignKey(DataProduct, null=True, blank=True, on_delete=models.CASCADE) data_type = models.CharField( max_length=100, default='' @@ -352,3 +352,12 @@ def save(self, *args, **kwargs): else: raise ValidationError('Not a valid DataProduct type.') return super().save() + + def validate_unique(self, *args, **kwargs): + super().validate_unique(*args, **kwargs) + model_dict = self.__dict__.copy() + del model_dict['_state'] + del model_dict['id'] + obs = ReducedDatum.objects.filter(**model_dict) + if obs: + raise ValidationError('Data point already exists.') diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py index 223747567..7d6ba9993 100644 --- a/tom_dataproducts/serializers.py +++ b/tom_dataproducts/serializers.py @@ -18,6 +18,8 @@ class Meta: class ReducedDatumSerializer(serializers.ModelSerializer): + target = TargetFilteredPrimaryKeyRelatedField(queryset=Target.objects.all()) + class Meta: model = ReducedDatum fields = ( @@ -26,9 +28,31 @@ class Meta: 'source_name', 'source_location', 'timestamp', - 'value' + 'value', + 'target' ) + def create(self, validated_data): + """DRF requires explicitly handling writeable nested serializers, + here we pop the groups data and save it using its serializer. + """ + groups = validated_data.pop('groups', []) + + rd = ReducedDatum(**validated_data) + rd.full_clean() + rd.save() + + # Save groups for this target + group_serializer = GroupSerializer(data=groups, many=True) + if group_serializer.is_valid() and settings.TARGET_PERMISSIONS_ONLY is False: + for group in groups: + group_instance = Group.objects.get(pk=group['id']) + assign_perm('tom_dataproducts.view_dataproduct', group_instance, rd) + assign_perm('tom_dataproducts.change_dataproduct', group_instance, rd) + assign_perm('tom_dataproducts.delete_dataproduct', group_instance, rd) + + return rd + class DataProductSerializer(serializers.ModelSerializer): target = TargetFilteredPrimaryKeyRelatedField(queryset=Target.objects.all()) diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index 9bf67f4ec..cf0ddab52 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -6,7 +6,7 @@ from tom_targets.models import Target from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage -from tom_dataproducts.serializers import DataProductSerializer +from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer def share_data_with_hermes(share_destination, form_data, product_id=None, target_id=None, selected_data=None): @@ -71,55 +71,70 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id except KeyError as err: raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {share_destination}: Key {err} not found.') auth = (username, password) - headers = {'Media-Type': 'application/json'} + headers = {'Content-Type': 'application/json', 'Accept': 'application/json'} dataproducts_url = destination_tom_base_url + 'api/dataproducts/' targets_url = destination_tom_base_url + 'api/targets/' + reduced_datums_url = destination_tom_base_url + 'api/reduceddatums/' reduced_datums = ReducedDatum.objects.none() + if product_id: product = DataProduct.objects.get(pk=product_id) target = product.target serialized_data = DataProductSerializer(product).data - # elif selected_data: - # reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) - # elif target_id: - # target = Target.objects.get(pk=target_id) - # data_type = form_data['data_type'] - # reduced_datums = ReducedDatum.objects.filter(target=target, data_type=data_type) + destination_target_id = get_destination_target(target, targets_url, headers, auth) + serialized_data['target'] = destination_target_id + # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage + dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) + # Save DataProduct in Destination TOM + with open(dataproduct_filename, 'rb') as dataproduct_filep: + files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} + headers = {'Media-Type': 'multipart/form-data'} + response = requests.post(dataproducts_url, data=serialized_data, files=files, headers=headers, auth=auth) + elif selected_data or target_id: + if selected_data: + reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) + targets = set(reduced_datum.target for reduced_datum in reduced_datums) + target_dict = {} + for target in targets: + # get destination Target + destination_target_id = get_destination_target(target, targets_url, headers, auth) + target_dict[target.name] = destination_target_id + else: + target = Target.objects.get(pk=target_id) + reduced_datums = ReducedDatum.objects.filter(target=target) + destination_target_id = get_destination_target(target, targets_url, headers, auth) + target_dict = {target.name: destination_target_id} + response_codes = [] + for datum in reduced_datums: + if target_dict[datum.target.name]: + serialized_data = ReducedDatumSerializer(datum).data + serialized_data['target'] = target_dict[datum.target.name] + serialized_data['data_product'] = '' + if not serialized_data['source_name']: + serialized_data['source_name'] = settings.TOM_NAME + serialized_data['source_location'] = "TOM-TOM Direct Sharing" + response = requests.post(reduced_datums_url, json=serialized_data, headers=headers, auth=auth) + response_codes.append(response.status_code) + failed_data_count = response_codes.count(500) + if failed_data_count < len(response_codes): + return {'message': f'{len(response_codes)-failed_data_count} of {len(response_codes)} datums successfully saved.'} + else: + return {'message': f'ERROR: No valid data shared. These data may already exist in target TOM.'} else: return {'message': f'ERROR: No valid data to share.'} - # get destination Target + return response + + +def get_destination_target(target, targets_url, headers, auth): target_response = requests.get(f'{targets_url}?name={target.name}', headers=headers, auth=auth) target_response_json = target_response.json() if target_response_json['results']: destination_target_id = target_response_json['results'][0]['id'] + return destination_target_id else: - return target_response - - serialized_data['target'] = destination_target_id - - # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage - dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) - # Save DataProduct in Destination TOM - with open(dataproduct_filename, 'rb') as dataproduct_filep: - files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} - headers = {'Media-Type': 'multipart/form-data'} - response = requests.post(dataproducts_url, data=serialized_data, files=files, - headers=headers, auth=auth) - return response - # serialized_target_data = TargetSerializer(target).data - # TODO: Make sure aliases are checked before creating new target - # Attempt to create Target in Destination TOM - # response = requests.post(targets_url, headers=headers, auth=auth, data=serialized_target_data) - # try: - # target_response = response.json() - # destination_target_id = target_response['id'] - # except KeyError: - # # If Target already exists at destination, find ID - # response = requests.get(targets_url, headers=headers, auth=auth, data=serialized_target_data) - # target_response = response.json() - # destination_target_id = target_response['results'][0]['id'] + return None def check_for_share_safe_datums(destination, reduced_datums, **kwargs): diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py index fb86fdecd..c1eb5636a 100644 --- a/tom_dataproducts/tests/test_api.py +++ b/tom_dataproducts/tests/test_api.py @@ -1,6 +1,8 @@ +from datetime import datetime from django.contrib.auth.models import Group, User from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse +from django.core.exceptions import ValidationError from guardian.shortcuts import assign_perm from rest_framework import status from rest_framework.test import APITestCase @@ -102,3 +104,41 @@ def test_data_product_list(self): response = self.client.get(reverse('api:dataproducts-list')) self.assertContains(response, dp.product_id, status_code=status.HTTP_200_OK) + + +class TestReducedDatumViewset(APITestCase): + def setUp(self): + self.user = User.objects.create(username='testuser') + self.client.force_login(self.user) + self.st = SiderealTargetFactory.create() + self.obsr = ObservingRecordFactory.create(target_id=self.st.id) + self.rd_data = { + 'data_product': '', + 'data_type': 'photometry', + 'source_name': 'TOM Toolkit', + 'source_location': 'TOM-TOM Direct Sharing', + 'value': {'magnitude': 15.582, 'filter': 'r', 'error': 0.005}, + 'target': self.st.id, + 'timestamp': '2012-02-12T01:40:47Z' + } + + assign_perm('tom_dataproducts.add_reduceddatum', self.user) + assign_perm('tom_targets.add_target', self.user, self.st) + assign_perm('tom_targets.view_target', self.user, self.st) + assign_perm('tom_targets.change_target', self.user, self.st) + + def test_upload_reduced_datum(self): + response = self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + self.assertContains(response, self.rd_data['source_name'], status_code=status.HTTP_201_CREATED) + + def test_upload_same_reduced_datum_twice(self): + """ + Test that identical data raises a validation error while similar but different JSON will make it through. + """ + response = self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + with self.assertRaises(ValidationError): + self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + self.rd_data['value'] = {'magnitude': 15.582, 'filter': 'B', 'error': 0.005} + response3 = self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + rd_queryset = ReducedDatum.objects.all() + self.assertEqual(rd_queryset.count(), 2) diff --git a/tom_dataproducts/urls.py b/tom_dataproducts/urls.py index ebd4b450b..c08dddde5 100644 --- a/tom_dataproducts/urls.py +++ b/tom_dataproducts/urls.py @@ -7,10 +7,11 @@ from tom_dataproducts.views import DataShareView from tom_common.api_router import SharedAPIRootRouter -from tom_dataproducts.api_views import DataProductViewSet +from tom_dataproducts.api_views import DataProductViewSet, ReducedDatumViewSet router = SharedAPIRootRouter() router.register(r'dataproducts', DataProductViewSet, 'dataproducts') +router.register(r'reduceddatums', ReducedDatumViewSet, 'reduceddatums') app_name = 'tom_dataproducts' From b758531241149338d55d67821fed2bcaa02f55f6 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 22 Jun 2023 14:45:57 -0700 Subject: [PATCH 06/42] fix some lint issues --- tom_dataproducts/sharing.py | 11 +++++++---- tom_dataproducts/tests/test_api.py | 5 ++--- tom_dataproducts/tests/tests.py | 2 +- tom_dataproducts/views.py | 6 ------ 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index cf0ddab52..04e38f408 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -2,9 +2,10 @@ import os from django.conf import settings +from django.core.exceptions import ImproperlyConfigured from tom_targets.models import Target -from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum +from tom_dataproducts.models import DataProduct, ReducedDatum from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer @@ -106,6 +107,7 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id destination_target_id = get_destination_target(target, targets_url, headers, auth) target_dict = {target.name: destination_target_id} response_codes = [] + reduced_datums = check_for_share_safe_datums(share_destination, reduced_datums) for datum in reduced_datums: if target_dict[datum.target.name]: serialized_data = ReducedDatumSerializer(datum).data @@ -118,11 +120,12 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id response_codes.append(response.status_code) failed_data_count = response_codes.count(500) if failed_data_count < len(response_codes): - return {'message': f'{len(response_codes)-failed_data_count} of {len(response_codes)} datums successfully saved.'} + return {'message': f'{len(response_codes)-failed_data_count} of {len(response_codes)} ' + 'datums successfully saved.'} else: - return {'message': f'ERROR: No valid data shared. These data may already exist in target TOM.'} + return {'message': 'ERROR: No valid data shared. These data may already exist in target TOM.'} else: - return {'message': f'ERROR: No valid data to share.'} + return {'message': 'ERROR: No valid data to share.'} return response diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py index c1eb5636a..5bb39b4d8 100644 --- a/tom_dataproducts/tests/test_api.py +++ b/tom_dataproducts/tests/test_api.py @@ -1,4 +1,3 @@ -from datetime import datetime from django.contrib.auth.models import Group, User from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse @@ -135,10 +134,10 @@ def test_upload_same_reduced_datum_twice(self): """ Test that identical data raises a validation error while similar but different JSON will make it through. """ - response = self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') with self.assertRaises(ValidationError): self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') self.rd_data['value'] = {'magnitude': 15.582, 'filter': 'B', 'error': 0.005} - response3 = self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') rd_queryset = ReducedDatum.objects.all() self.assertEqual(rd_queryset.count(), 2) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index 9c6679f7b..a00049391 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -542,4 +542,4 @@ def test_share_data_for_target(self, run_data_processor_mock): }, follow=True ) - self.assertContains(response, 'TOM-TOM sharing is not yet supported.') \ No newline at end of file + self.assertContains(response, 'TOM-TOM sharing is not yet supported.') diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 093ef8a6a..b9d489795 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -1,6 +1,5 @@ from io import StringIO import logging -import os from urllib.parse import urlencode, urlparse from django.conf import settings @@ -9,7 +8,6 @@ from django.contrib.auth.models import Group from django.core.cache import cache from django.core.cache.utils import make_template_fragment_key -from django.core.exceptions import ImproperlyConfigured from django.core.management import call_command from django.http import HttpResponseRedirect from django.shortcuts import redirect @@ -25,8 +23,6 @@ from tom_common.hooks import run_hook from tom_common.hints import add_hint from tom_common.mixins import Raise403PermissionRequiredMixin -from tom_targets.serializers import TargetSerializer -from tom_targets.models import Target from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum from tom_dataproducts.exceptions import InvalidFileFormatException from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm, DataShareForm @@ -36,8 +32,6 @@ from tom_observations.facility import get_service_class from tom_dataproducts.sharing import share_data_with_hermes, share_data_with_tom -import requests - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) From 89b08763126f7aeb834face75e6f60792455bf98 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 27 Jun 2023 12:59:43 -0700 Subject: [PATCH 07/42] add tests for no connection to target tom --- setup.py | 1 + tom_dataproducts/sharing.py | 17 +++-- tom_dataproducts/tests/tests.py | 112 ++++++++++++++++++++++++++++++-- 3 files changed, 120 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index ef8ff72b0..538655a77 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,7 @@ 'plotly~=5.0', 'python-dateutil~=2.8', 'requests~=2.25', + 'responses~=0.23', 'specutils~=1.8', ], extras_require={ diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index 04e38f408..8995bd39e 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -84,6 +84,8 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id target = product.target serialized_data = DataProductSerializer(product).data destination_target_id = get_destination_target(target, targets_url, headers, auth) + if destination_target_id is None: + return {'message': 'ERROR: No matching target found.'} serialized_data['target'] = destination_target_id # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) @@ -101,10 +103,14 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id # get destination Target destination_target_id = get_destination_target(target, targets_url, headers, auth) target_dict[target.name] = destination_target_id + if all(value is None for value in target_dict.values()): + return {'message': 'ERROR: No matching targets found.'} else: target = Target.objects.get(pk=target_id) reduced_datums = ReducedDatum.objects.filter(target=target) destination_target_id = get_destination_target(target, targets_url, headers, auth) + if destination_target_id is None: + return {'message': 'ERROR: No matching target found.'} target_dict = {target.name: destination_target_id} response_codes = [] reduced_datums = check_for_share_safe_datums(share_destination, reduced_datums) @@ -133,10 +139,13 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id def get_destination_target(target, targets_url, headers, auth): target_response = requests.get(f'{targets_url}?name={target.name}', headers=headers, auth=auth) target_response_json = target_response.json() - if target_response_json['results']: - destination_target_id = target_response_json['results'][0]['id'] - return destination_target_id - else: + try: + if target_response_json['results']: + destination_target_id = target_response_json['results'][0]['id'] + return destination_target_id + else: + return None + except KeyError: return None diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index a00049391..75a41f841 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -1,6 +1,7 @@ import os from http import HTTPStatus import tempfile +import responses from astropy import units from astropy.io import fits @@ -18,7 +19,7 @@ from tom_dataproducts.exceptions import InvalidFileFormatException from tom_dataproducts.forms import DataProductUploadForm -from tom_dataproducts.models import DataProduct, is_fits_image_file +from tom_dataproducts.models import DataProduct, is_fits_image_file, ReducedDatum from tom_dataproducts.processors.data_serializers import SpectrumSerializer from tom_dataproducts.processors.photometry_processor import PhotometryProcessor from tom_dataproducts.processors.spectroscopy_processor import SpectroscopyProcessor @@ -493,7 +494,6 @@ def test_create_thumbnail(self, mock_is_fits_image_file): @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True) -@patch('tom_dataproducts.views.run_data_processor') class TestShareDataProducts(TestCase): def setUp(self): self.target = SiderealTargetFactory.create() @@ -512,7 +512,40 @@ def setUp(self): assign_perm('tom_targets.view_target', self.user, self.target) self.client.force_login(self.user) - def test_share_dataproduct(self, run_data_processor_mock): + self.rd1 = ReducedDatum.objects.create( + target=self.target, + data_type='photometry', + value={'magnitude': 18.5, 'error': .5, 'filter': 'V'} + ) + self.rd2 = ReducedDatum.objects.create( + target=self.target, + data_type='photometry', + value={'magnitude': 19.5, 'error': .5, 'filter': 'B'} + ) + self.rd3 = ReducedDatum.objects.create( + target=self.target, + data_type='photometry', + value={'magnitude': 17.5, 'error': .5, 'filter': 'R'} + ) + + @responses.activate + def test_share_dataproduct_no_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"error": "not found"}, + status=500 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) response = self.client.post( reverse('dataproducts:share', kwargs={'dp_pk': self.data_product.id}), @@ -520,13 +553,80 @@ def test_share_dataproduct(self, run_data_processor_mock): 'share_authors': ['test_author'], 'target': self.target.id, 'submitter': ['test_submitter'], - 'share_destination': ['local_host'], + 'share_destination': [share_destination], 'share_title': ['Updated data for thingy.'], 'share_message': ['test_message'] }, follow=True ) - self.assertContains(response, 'TOM-TOM sharing is not yet supported.') + self.assertContains(response, 'ERROR: No matching target found.') + + @responses.activate + def test_share_reduceddatums_target_no_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"error": "not found"}, + status=500 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'] + }, + follow=True + ) + self.assertContains(response, 'ERROR: No matching target found.') + + @responses.activate + def test_share_reduced_datums_no_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"error": "not found"}, + status=500 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'], + 'share-box': [1, 2] + }, + follow=True + ) + self.assertContains(response, 'ERROR: No matching targets found.') def test_share_data_for_target(self, run_data_processor_mock): @@ -542,4 +642,4 @@ def test_share_data_for_target(self, run_data_processor_mock): }, follow=True ) - self.assertContains(response, 'TOM-TOM sharing is not yet supported.') + self.assertContains(response, 'Data successfully uploaded.') From 06d2f70846e973820573f0975ab707b682a00d75 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 27 Jun 2023 13:14:27 -0700 Subject: [PATCH 08/42] fix tests to spoof settings --- tom_dataproducts/tests/tests.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index 75a41f841..a00ad8589 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -493,7 +493,8 @@ def test_create_thumbnail(self, mock_is_fits_image_file): @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], - TARGET_PERMISSIONS_ONLY=True) + TARGET_PERMISSIONS_ONLY=True, + DATA_SHARING={'local_host': {'BASE_URL': 'fake.url/example'}}) class TestShareDataProducts(TestCase): def setUp(self): self.target = SiderealTargetFactory.create() @@ -628,18 +629,18 @@ def test_share_reduced_datums_no_valid_responses(self): ) self.assertContains(response, 'ERROR: No matching targets found.') - def test_share_data_for_target(self, run_data_processor_mock): - - response = self.client.post( - reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), - { - 'share_authors': ['test_author'], - 'target': self.target.id, - 'submitter': ['test_submitter'], - 'share_destination': ['local_host'], - 'share_title': ['Updated data for thingy.'], - 'share_message': ['test_message'] - }, - follow=True - ) - self.assertContains(response, 'Data successfully uploaded.') + # def test_share_data_for_target(self, run_data_processor_mock): + # + # response = self.client.post( + # reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + # { + # 'share_authors': ['test_author'], + # 'target': self.target.id, + # 'submitter': ['test_submitter'], + # 'share_destination': ['local_host'], + # 'share_title': ['Updated data for thingy.'], + # 'share_message': ['test_message'] + # }, + # follow=True + # ) + # self.assertContains(response, 'Data successfully uploaded.') From f4deef8204949fe0162128402543f6a6564b0c40 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 27 Jun 2023 13:22:55 -0700 Subject: [PATCH 09/42] add username/password to settings override --- tom_dataproducts/tests/tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index a00ad8589..2d90ab838 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -494,7 +494,9 @@ def test_create_thumbnail(self, mock_is_fits_image_file): @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True, - DATA_SHARING={'local_host': {'BASE_URL': 'fake.url/example'}}) + DATA_SHARING={'local_host': {'BASE_URL': 'fake.url/example', + 'USERNAME': 'fake_user', + 'PASSWORD': 'password'}}) class TestShareDataProducts(TestCase): def setUp(self): self.target = SiderealTargetFactory.create() From 71ef6acfe6b5008705eaf88d6df6bf600ccc4138 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 27 Jun 2023 13:35:59 -0700 Subject: [PATCH 10/42] make properly formatted fake url --- tom_dataproducts/tests/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index 2d90ab838..9832b0c1a 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -494,7 +494,7 @@ def test_create_thumbnail(self, mock_is_fits_image_file): @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True, - DATA_SHARING={'local_host': {'BASE_URL': 'fake.url/example', + DATA_SHARING={'local_host': {'BASE_URL': 'https://fake.url/example', 'USERNAME': 'fake_user', 'PASSWORD': 'password'}}) class TestShareDataProducts(TestCase): From 17a7a03ceec1da2c180d3fe7edbfedfef904a6e0 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 28 Jun 2023 16:19:04 -0700 Subject: [PATCH 11/42] add tests for succesful sharing --- tom_dataproducts/tests/tests.py | 134 ++++++++++++++++++++++++++++---- 1 file changed, 118 insertions(+), 16 deletions(-) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index 9832b0c1a..a06d39245 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -494,7 +494,7 @@ def test_create_thumbnail(self, mock_is_fits_image_file): @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], TARGET_PERMISSIONS_ONLY=True, - DATA_SHARING={'local_host': {'BASE_URL': 'https://fake.url/example', + DATA_SHARING={'local_host': {'BASE_URL': 'https://fake.url/example/', 'USERNAME': 'fake_user', 'PASSWORD': 'password'}}) class TestShareDataProducts(TestCase): @@ -631,18 +631,120 @@ def test_share_reduced_datums_no_valid_responses(self): ) self.assertContains(response, 'ERROR: No matching targets found.') - # def test_share_data_for_target(self, run_data_processor_mock): - # - # response = self.client.post( - # reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), - # { - # 'share_authors': ['test_author'], - # 'target': self.target.id, - # 'submitter': ['test_submitter'], - # 'share_destination': ['local_host'], - # 'share_title': ['Updated data for thingy.'], - # 'share_message': ['test_message'] - # }, - # follow=True - # ) - # self.assertContains(response, 'Data successfully uploaded.') + @responses.activate + def test_share_dataproduct_valid_target_found(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"results": [{'id': 1}]}, + status=200 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + responses.add( + responses.POST, + destination_tom_base_url + 'api/dataproducts/', + json={"message": "Data product successfully uploaded."}, + status=200, + ) + + response = self.client.post( + reverse('dataproducts:share', kwargs={'dp_pk': self.data_product.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'] + }, + follow=True + ) + self.assertContains(response, 'Data product successfully uploaded.') + + @responses.activate + def test_share_reduceddatums_target_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"results": [{'id': 1}]}, + status=200 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + responses.add( + responses.POST, + destination_tom_base_url + 'api/reduceddatums/', + json={}, + status=201, + ) + + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'] + }, + follow=True + ) + self.assertContains(response, '3 of 3 datums successfully saved.') + + @responses.activate + def test_share_reduced_datums_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"results": [{'id': 1}]}, + status=200 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + responses.add( + responses.POST, + destination_tom_base_url + 'api/reduceddatums/', + json={}, + status=201, + ) + + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'], + 'share-box': [1, 2] + }, + follow=True + ) + self.assertContains(response, '2 of 2 datums successfully saved.') From e6e7e9567a503e181dfd95f91d44c8cbbd8cbf69 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 5 Jul 2023 17:05:31 -0700 Subject: [PATCH 12/42] set up ui buttons for sharing --- tom_dataproducts/views.py | 10 ---- .../tom_targets/partials/target_buttons.html | 5 +- tom_targets/urls.py | 1 + tom_targets/views.py | 58 +++++++++++++++++++ 4 files changed, 62 insertions(+), 12 deletions(-) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index b9d489795..314519b07 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -338,16 +338,6 @@ def post(self, request, *args, **kwargs): messages.success(self.request, publish_feedback) return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - # if data_share_form.is_valid(): - # form_data = data_share_form.cleaned_data - # - # return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - # else: - # # messages.error(self.request, 'TOM-TOM sharing is not yet supported.') - # response = self.share_with_tom(share_destination, product) - # return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - # # response = self.share_with_tom(share_destination, product) - class DataProductGroupDetailView(DetailView): """ diff --git a/tom_targets/templates/tom_targets/partials/target_buttons.html b/tom_targets/templates/tom_targets/partials/target_buttons.html index 5cd86e4f4..3f5216c37 100644 --- a/tom_targets/templates/tom_targets/partials/target_buttons.html +++ b/tom_targets/templates/tom_targets/partials/target_buttons.html @@ -1,2 +1,3 @@ -Update Target -Delete Target \ No newline at end of file +Update +Delete +Share \ No newline at end of file diff --git a/tom_targets/urls.py b/tom_targets/urls.py index 33e98aa81..09bb8228f 100644 --- a/tom_targets/urls.py +++ b/tom_targets/urls.py @@ -24,6 +24,7 @@ path('name/', TargetNameSearchView.as_view(), name='name-search'), path('/update/', TargetUpdateView.as_view(), name='update'), path('/delete/', TargetDeleteView.as_view(), name='delete'), + path('/share/', TargetUpdateView.as_view(), name='share'), path('/', TargetDetailView.as_view(), name='detail'), path('targetgrouping//delete/', TargetGroupingDeleteView.as_view(), name='delete-group'), path('targetgrouping/create/', TargetGroupingCreateView.as_view(), name='create-group') diff --git a/tom_targets/views.py b/tom_targets/views.py index 500c1e472..d762e7e47 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -321,6 +321,64 @@ def get_form(self, *args, **kwargs): return form +# class TargetShareView(FormView): +# """ +# View that handles the sharing of data either through HERMES or with another TOM. +# """ +# +# form_class = DataShareForm +# +# def get_form(self, *args, **kwargs): +# # TODO: Add permissions +# form = super().get_form(*args, **kwargs) +# return form +# +# def form_invalid(self, form): +# """ +# Adds errors to Django messaging framework in the case of an invalid form and redirects to the previous page. +# """ +# # TODO: Format error messages in a more human-readable way +# messages.error(self.request, 'There was a problem sharing your Data: {}'.format(form.errors.as_json())) +# return redirect(form.cleaned_data.get('referrer', '/')) +# +# def post(self, request, *args, **kwargs): +# """ +# Method that handles the POST requests for sharing data. +# Handles Data Products and All the data of a type for a target as well as individual Reduced Datums. +# Submit to Hermes, or Share with TOM (soon). +# """ +# data_share_form = DataShareForm(request.POST, request.FILES) +# +# if data_share_form.is_valid(): +# form_data = data_share_form.cleaned_data +# share_destination = form_data['share_destination'] +# product_id = kwargs.get('dp_pk', None) +# target_id = kwargs.get('tg_pk', None) +# +# # Check if data points have been selected. +# selected_data = request.POST.getlist("share-box") +# +# # Check Destination +# if 'HERMES' in share_destination.upper(): +# response = share_data_with_hermes(share_destination, form_data, product_id, target_id, selected_data) +# else: +# response = share_data_with_tom(share_destination, form_data, product_id, target_id, selected_data) +# try: +# if 'message' in response.json(): +# publish_feedback = response.json()['message'] +# else: +# publish_feedback = f"ERROR: {response.text}" +# except AttributeError: +# publish_feedback = response['message'] +# except ValueError: +# publish_feedback = f"ERROR: Returned Response code {response.status_code}" +# if "ERROR" in publish_feedback.upper(): +# messages.error(self.request, publish_feedback) +# else: +# messages.success(self.request, publish_feedback) +# return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) + + class TargetDeleteView(Raise403PermissionRequiredMixin, DeleteView): """ View for deleting a target. Requires authorization. From d342af2b0f946af2e9ed7cae99437d03b0ce4bb4 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 13 Jul 2023 16:42:06 -0700 Subject: [PATCH 13/42] Build Target sharing form page --- tom_dataproducts/forms.py | 31 +----- tom_dataproducts/sharing.py | 30 +++++- .../photometry_datalist_for_target.html | 36 +++---- .../templatetags/dataproduct_extras.py | 5 +- tom_targets/forms.py | 16 +++ .../tom_targets/partials/target_buttons.html | 4 +- tom_targets/urls.py | 4 +- tom_targets/views.py | 101 +++++++----------- 8 files changed, 111 insertions(+), 116 deletions(-) diff --git a/tom_dataproducts/forms.py b/tom_dataproducts/forms.py index e0b615a96..43fbbf309 100644 --- a/tom_dataproducts/forms.py +++ b/tom_dataproducts/forms.py @@ -5,7 +5,7 @@ from tom_dataproducts.models import DataProductGroup, DataProduct from tom_observations.models import ObservationRecord from tom_targets.models import Target -from tom_dataproducts.alertstreams.hermes import get_hermes_topics +from tom_dataproducts.sharing import get_sharing_destination_options DATA_TYPE_OPTIONS = (('photometry', 'Photometry'), @@ -65,33 +65,6 @@ class DataShareForm(forms.Form): widget=forms.HiddenInput() ) - def get_sharing_destination_options(self): - """ - Build the Display options and headers for the dropdown form for choosing sharing topics. - Customize for a different selection experience. - :return: Tuple: Possible Destinations and their Display Names - """ - choices = [] - try: - for destination, details in settings.DATA_SHARING.items(): - new_destination = [details.get('DISPLAY_NAME', destination)] - if details.get('USER_TOPICS', None): - # If topics exist for a destination (Such as HERMES) give topics as sub-choices - # for non-selectable Destination - if destination == "hermes": - destination_topics = get_hermes_topics() - else: - destination_topics = details['USER_TOPICS'] - topic_list = [(f'{destination}:{topic}', topic) for topic in destination_topics] - new_destination.append(tuple(topic_list)) - else: - # Otherwise just use destination as option - new_destination.insert(0, destination) - choices.append(tuple(new_destination)) - except AttributeError: - pass - return tuple(choices) - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['share_destination'].choices = self.get_sharing_destination_options() + self.fields['share_destination'].choices = get_sharing_destination_options() diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index 8995bd39e..f6a7cda1e 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -6,7 +6,7 @@ from tom_targets.models import Target from tom_dataproducts.models import DataProduct, ReducedDatum -from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage +from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage, get_hermes_topics from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer @@ -170,3 +170,31 @@ def check_for_share_safe_datums(destination, reduced_datums, **kwargs): def check_for_save_safe_datums(): return + + +def get_sharing_destination_options(): + """ + Build the Display options and headers for the dropdown form for choosing sharing topics. + Customize for a different selection experience. + :return: Tuple: Possible Destinations and their Display Names + """ + choices = [] + try: + for destination, details in settings.DATA_SHARING.items(): + new_destination = [details.get('DISPLAY_NAME', destination)] + if details.get('USER_TOPICS', None): + # If topics exist for a destination (Such as HERMES) give topics as sub-choices + # for non-selectable Destination + if destination == "hermes": + destination_topics = get_hermes_topics() + else: + destination_topics = details['USER_TOPICS'] + topic_list = [(f'{destination}:{topic}', topic) for topic in destination_topics] + new_destination.append(tuple(topic_list)) + else: + # Otherwise just use destination as option + new_destination.insert(0, destination) + choices.append(tuple(new_destination)) + except AttributeError: + pass + return tuple(choices) diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html index 953af3b88..1a2e499f5 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html @@ -46,25 +46,25 @@ {% endfor %} -
-
- Share Selected Data -
- {% if sharing_destinations %} -
-
- {% bootstrap_field target_data_share_form.share_destination %} -
-
- -
-
- {% else %} - Not Configured - {% endif %} - - + {% if not target_share %} +
+
+ Share Selected Data +
+ {% if sharing_destinations %} +
+
+ {% bootstrap_field target_data_share_form.share_destination %} +
+
+ +
+
+ {% else %} + Not Configured + {% endif %}
+ {% endif %}
+ +

Query Forced Photometry Service

+{% for service in forced_photometry_services %} +{{ service }} +{% endfor %} diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index adbc76816..d57ce06d6 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -20,6 +20,7 @@ from tom_dataproducts.forms import DataProductUploadForm, DataShareForm from tom_dataproducts.models import DataProduct, ReducedDatum from tom_dataproducts.processors.data_serializers import SpectrumSerializer +from tom_dataproducts.forced_photometry.forced_photometry_service import get_service_classes from tom_observations.models import ObservationRecord from tom_targets.models import Target @@ -94,6 +95,13 @@ def dataproduct_list_all(context): 'products': products, } +@register.inclusion_tag('tom_dataproducts/partials/query_forced_photometry.html') +def query_forced_photometry(target): + services = get_service_classes().keys() + return {'forced_photometry_services': services, + 'target': target + } + @register.inclusion_tag('tom_dataproducts/partials/upload_dataproduct.html', takes_context=True) def upload_dataproduct(context, obj): diff --git a/tom_dataproducts/urls.py b/tom_dataproducts/urls.py index ebd4b450b..b47124344 100644 --- a/tom_dataproducts/urls.py +++ b/tom_dataproducts/urls.py @@ -4,7 +4,7 @@ from tom_dataproducts.views import DataProductDeleteView, DataProductGroupCreateView from tom_dataproducts.views import DataProductGroupDetailView, DataProductGroupDataView, DataProductGroupDeleteView from tom_dataproducts.views import DataProductUploadView, DataProductFeatureView, UpdateReducedDataView -from tom_dataproducts.views import DataShareView +from tom_dataproducts.views import DataShareView, ForcedPhotometryQueryView from tom_common.api_router import SharedAPIRootRouter from tom_dataproducts.api_views import DataProductViewSet @@ -23,6 +23,7 @@ path('data/group//delete/', DataProductGroupDeleteView.as_view(), name='group-delete'), path('data/upload/', DataProductUploadView.as_view(), name='upload'), path('data/reduced/update/', UpdateReducedDataView.as_view(), name='update-reduced-data'), + path('data/forced_photometry//query/', ForcedPhotometryQueryView.as_view(), name='forced-photometry-query'), path('data//delete/', DataProductDeleteView.as_view(), name='delete'), path('data//feature/', DataProductFeatureView.as_view(), name='feature'), path('data//share/', DataShareView.as_view(), name='share'), diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 233437278..85ce0a2b5 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -1,6 +1,7 @@ from io import StringIO import logging import os +from typing import Any from urllib.parse import urlencode, urlparse from django.conf import settings @@ -36,6 +37,7 @@ from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class from tom_dataproducts.serializers import DataProductSerializer +from tom_dataproducts.forced_photometry.forced_photometry_service import ForcedPhotometryServiceException, get_service_class import requests @@ -86,6 +88,86 @@ def post(self, request, *args, **kwargs): ) +class ForcedPhotometryQueryView(LoginRequiredMixin, FormView): + """ + View that handles queries for forced photometry services + """ + template_name = 'tom_dataproducts/forced_photometry_form.html' + + def get_target_id(self): + """ + Parses the target id from the query parameters. + """ + if self.request.method == 'GET': + return self.request.GET.get('target_id') + elif self.request.method == 'POST': + return self.request.POST.get('target_id') + + def get_target(self): + """ + Gets the target for observing from the database + + :returns: target for observing + :rtype: Target + """ + return Target.objects.get(pk=self.get_target_id()) + + def get_service(self): + """ + Gets the forced photometry service that you want to query + """ + return self.kwargs['service'] + + def get_service_class(self): + """ + Gets the forced photometry service class + """ + return get_service_class(self.get_service()) + + def get_form_class(self): + """ + Gets the forced photometry service form class + """ + return self.get_service_class()().get_form() + + def get_context_data(self, **kwargs: Any) -> dict[str, Any]: + """ + Adds the target to the context object. + """ + context = super().get_context_data(**kwargs) + context['target'] = self.get_target() + context['query_form'] = self.get_form_class()(initial=self.get_initial()) + return context + + def get_initial(self): + """ + Populates the form with initial data including service name and target id + """ + initial = super().get_initial() + if not self.get_target_id(): + raise Exception('Must provide target_id') + initial['target_id'] = self.get_target_id() + initial['service'] = self.get_service() + initial.update(self.request.GET.dict()) + return initial + + def post(self, request, *args, **kwargs): + form = self.get_form() + if form.is_valid(): + service = self.get_service_class()() + try: + service.query_service(form.cleaned_data) + except ForcedPhotometryServiceException as e: + form.add_error(f"Problem querying forced photometry service: {repr(e)}") + return self.form_invalid(form) + messages.info(self.request, service.get_success_message()) + return redirect( + reverse('tom_targets:detail', kwargs={'pk': self.get_target_id()}) + ) + else: + return self.form_invalid(form) + + class DataProductUploadView(LoginRequiredMixin, FormView): """ View that handles manual upload of DataProducts. Requires authentication. diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index b58bc942f..5fdac4eac 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -85,6 +85,11 @@

Observations

{% observation_list object %}
+ {% if user.is_authenticated %} + {% query_forced_photometry object %} +
+
+ {% endif %} {% if user.is_authenticated %} {% upload_dataproduct object %} {% endif %} From 9a1fde4f08cdc42867abd2851c049e370b2582fe Mon Sep 17 00:00:00 2001 From: Jon Date: Fri, 6 Oct 2023 07:58:23 +0000 Subject: [PATCH 34/42] fix some flake8 issues --- tom_dataproducts/forced_photometry/atlas.py | 59 ++++++++++++------- .../forced_photometry_service.py | 12 ++-- .../processors/atlas_processor.py | 2 +- tom_dataproducts/tasks.py | 11 ++-- .../templatetags/dataproduct_extras.py | 8 ++- tom_dataproducts/urls.py | 3 +- tom_dataproducts/views.py | 6 +- 7 files changed, 60 insertions(+), 41 deletions(-) diff --git a/tom_dataproducts/forced_photometry/atlas.py b/tom_dataproducts/forced_photometry/atlas.py index b0b3e6676..5bb327f73 100644 --- a/tom_dataproducts/forced_photometry/atlas.py +++ b/tom_dataproducts/forced_photometry/atlas.py @@ -1,21 +1,28 @@ from django import forms from django.conf import settings -from django.utils import timezone -from django.core.files.base import ContentFile from crispy_forms.layout import Div, HTML from astropy.time import Time -from tom_dataproducts.forced_photometry.forced_photometry_service import BaseForcedPhotometryQueryForm, BaseForcedPhotometryService, ForcedPhotometryServiceException -from tom_dataproducts.models import ReducedDatum, DataProduct -from tom_dataproducts.data_processor import run_data_processor -from tom_dataproducts.exceptions import InvalidFileFormatException +import tom_dataproducts.forced_photometry.forced_photometry_service as fps from tom_dataproducts.tasks import atlas_query from tom_targets.models import Target -class AtlasForcedPhotometryQueryForm(BaseForcedPhotometryQueryForm): - min_date = forms.CharField(label='Min date:', required=False, widget=forms.TextInput(attrs={'class': 'ml-2', 'type': 'datetime-local'})) - max_date = forms.CharField(label='Max date:', required=False, widget=forms.TextInput(attrs={'class': 'ml-2', 'type': 'datetime-local'})) - min_date_mjd = forms.FloatField(label='Min date (mjd):', required=False, widget=forms.NumberInput(attrs={'class': 'ml-2'})) - max_date_mjd = forms.FloatField(label='Max date (mjd):', required=False, widget=forms.NumberInput(attrs={'class': 'ml-2'})) +class AtlasForcedPhotometryQueryForm(fps.BaseForcedPhotometryQueryForm): + min_date = forms.CharField( + label='Min date:', required=False, + widget=forms.TextInput(attrs={'class': 'ml-2', 'type': 'datetime-local'}) + ) + max_date = forms.CharField( + label='Max date:', required=False, + widget=forms.TextInput(attrs={'class': 'ml-2', 'type': 'datetime-local'}) + ) + min_date_mjd = forms.FloatField( + label='Min date (mjd):', required=False, + widget=forms.NumberInput(attrs={'class': 'ml-2'}) + ) + max_date_mjd = forms.FloatField( + label='Max date (mjd):', required=False, + widget=forms.NumberInput(attrs={'class': 'ml-2'}) + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -65,12 +72,14 @@ def clean(self): return cleaned_data -class AtlasForcedPhotometryService(BaseForcedPhotometryService): +class AtlasForcedPhotometryService(fps.BaseForcedPhotometryService): name = 'Atlas' def __init__(self): super().__init__ - self.success_message = 'Asynchronous Atlas query is processing. Refresh the page once complete it will show up as a dataproduct in the "Manage Data" tab.' + self.success_message = ('Asynchronous Atlas query is processing. ' + 'Refresh the page once complete it will show ' + 'up as a dataproduct in the "Manage Data" tab.') def get_form(self): """ @@ -91,21 +100,31 @@ def query_service(self, query_parameters): if not max_date_mjd and query_parameters.get('max_date'): max_date_mjd = Time(query_parameters.get('max_date')).mjd if not Target.objects.filter(pk=query_parameters.get('target_id')).exists(): - raise ForcedPhotometryServiceException(f"Target {query_parameters.get('target_id')} does not exist") + raise fps.ForcedPhotometryServiceException(f"Target {query_parameters.get('target_id')} does not exist") if 'atlas' not in settings.FORCED_PHOTOMETRY_SERVICES: - raise ForcedPhotometryServiceException("Must specify 'atlas' settings in FORCED_PHOTOMETRY_SERVICES") + raise fps.ForcedPhotometryServiceException("Must specify 'atlas' settings in FORCED_PHOTOMETRY_SERVICES") if not settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('url'): - raise ForcedPhotometryServiceException("Must specify a 'url' under atlas settings in FORCED_PHOTOMETRY_SERVICES") + raise fps.ForcedPhotometryServiceException( + "Must specify a 'url' under atlas settings in FORCED_PHOTOMETRY_SERVICES" + ) if not settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('api_key'): - raise ForcedPhotometryServiceException("Must specify an 'api_key' under atlas settings in FORCED_PHOTOMETRY_SERVICES") + raise fps.ForcedPhotometryServiceException( + "Must specify an 'api_key' under atlas settings in FORCED_PHOTOMETRY_SERVICES" + ) if 'django_dramatiq' in settings.INSTALLED_APPS: - atlas_query.send(min_date_mjd, max_date_mjd, query_parameters.get('target_id'), self.get_data_product_type()) + atlas_query.send(min_date_mjd, max_date_mjd, + query_parameters.get('target_id'), + self.get_data_product_type()) else: - query_succeeded = atlas_query(min_date_mjd, max_date_mjd, query_parameters.get('target_id'), self.get_data_product_type()) + query_succeeded = atlas_query(min_date_mjd, max_date_mjd, + query_parameters.get('target_id'), + self.get_data_product_type()) if not query_succeeded: - raise ForcedPhotometryServiceException("Atlas query failed, check the server logs for more information") + raise fps.ForcedPhotometryServiceException( + "Atlas query failed, check the server logs for more information" + ) self.success_message = "Atlas query completed. View its data product in the 'Manage Data' tab" return True diff --git a/tom_dataproducts/forced_photometry/forced_photometry_service.py b/tom_dataproducts/forced_photometry/forced_photometry_service.py index 8ab2b2a9a..806345d90 100644 --- a/tom_dataproducts/forced_photometry/forced_photometry_service.py +++ b/tom_dataproducts/forced_photometry/forced_photometry_service.py @@ -1,18 +1,12 @@ from abc import ABC, abstractmethod -import copy import logging -import requests from crispy_forms.helper import FormHelper -from crispy_forms.layout import ButtonHolder, Layout, Submit, Div, HTML +from crispy_forms.layout import ButtonHolder, Layout, Submit from django import forms from django.conf import settings -from django.contrib.auth.models import Group -from django.core.files.base import ContentFile from django.utils.module_loading import import_string -from tom_targets.models import Target - logger = logging.getLogger(__name__) @@ -37,7 +31,9 @@ def get_service_class(name): try: return available_classes[name] except KeyError: - raise ImportError(f'Could not a find a forced photometry service with the name {name}. Did you add it to TOM_FORCED_PHOTOMETRY_CLASSES?') + raise ImportError(( + f'Could not a find a forced photometry service with the name {name}. ' + 'Did you add it to TOM_FORCED_PHOTOMETRY_CLASSES?')) class ForcedPhotometryServiceException(Exception): diff --git a/tom_dataproducts/processors/atlas_processor.py b/tom_dataproducts/processors/atlas_processor.py index 9e90ed1c9..9d452c932 100644 --- a/tom_dataproducts/processors/atlas_processor.py +++ b/tom_dataproducts/processors/atlas_processor.py @@ -39,7 +39,7 @@ def _process_photometry_from_plaintext(self, data_product): text file, as produced by the ATLAS forced photometry service at https://fallingstar-data.com/forcedphot The header looks like this: - ###MJD m dm uJy duJy F err chi/N RA Dec x y maj min phi apfit mag5sig Sky Obs + ###MJD m dm uJy duJy F err chi/N RA Dec x y maj min phi apfit mag5sig Sky Obs :param data_product: ATLAS Photometric DataProduct which will be processed into a list of dicts :type data_product: DataProduct diff --git a/tom_dataproducts/tasks.py b/tom_dataproducts/tasks.py index d29ac0b3b..f6ce1b2d9 100644 --- a/tom_dataproducts/tasks.py +++ b/tom_dataproducts/tasks.py @@ -23,12 +23,13 @@ def atlas_query(min_date_mjd, max_date_mjd, target_id, data_product_type): print("Calling atlas query!") target = Target.objects.get(pk=target_id) - headers = {"Authorization": f"Token {settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('api_key')}", "Accept": "application/json"} + headers = {"Authorization": f"Token {settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('api_key')}", + "Accept": "application/json"} base_url = settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('url') task_url = None while not task_url: with requests.Session() as s: - task_data = {"ra":target.ra, "dec": target.dec, "mjd_min": min_date_mjd, "send_email": False} + task_data = {"ra": target.ra, "dec": target.dec, "mjd_min": min_date_mjd, "send_email": False} if max_date_mjd: task_data['mjd_max'] = max_date_mjd resp = s.post( @@ -81,11 +82,11 @@ def atlas_query(min_date_mjd, max_date_mjd, target_id, data_product_type): dp_name = f"atlas_{Time(min_date_mjd, format='mjd').strftime('%Y_%m_%d')}" if max_date_mjd: dp_name += f"-{Time(max_date_mjd, format='mjd').strftime('%Y_%m_%d')}" - dp_name += f"_{urlparse(result_url)[2].rpartition('/')[2]}" + dp_name += f"_{urlparse(result_url)[2].rpartition('/')[2]}" file = ContentFile(results.content, name=dp_name) dp = DataProduct.objects.create( - product_id = dp_name, + product_id=dp_name, target=target, data=file, data_product_type=data_product_type, @@ -99,4 +100,4 @@ def atlas_query(min_date_mjd, max_date_mjd, target_id, data_product_type): logger.error(f"Error processing returned Atlas data into ReducedDatums: {repr(e)}") return False - return True \ No newline at end of file + return True diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index d57ce06d6..107670f5b 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -95,12 +95,14 @@ def dataproduct_list_all(context): 'products': products, } + @register.inclusion_tag('tom_dataproducts/partials/query_forced_photometry.html') def query_forced_photometry(target): services = get_service_classes().keys() - return {'forced_photometry_services': services, - 'target': target - } + return { + 'forced_photometry_services': services, + 'target': target + } @register.inclusion_tag('tom_dataproducts/partials/upload_dataproduct.html', takes_context=True) diff --git a/tom_dataproducts/urls.py b/tom_dataproducts/urls.py index b47124344..1f9b7ae73 100644 --- a/tom_dataproducts/urls.py +++ b/tom_dataproducts/urls.py @@ -23,7 +23,8 @@ path('data/group//delete/', DataProductGroupDeleteView.as_view(), name='group-delete'), path('data/upload/', DataProductUploadView.as_view(), name='upload'), path('data/reduced/update/', UpdateReducedDataView.as_view(), name='update-reduced-data'), - path('data/forced_photometry//query/', ForcedPhotometryQueryView.as_view(), name='forced-photometry-query'), + path('data/forced_photometry//query/', ForcedPhotometryQueryView.as_view(), + name='forced-photometry-query'), path('data//delete/', DataProductDeleteView.as_view(), name='delete'), path('data//feature/', DataProductFeatureView.as_view(), name='feature'), path('data//share/', DataShareView.as_view(), name='share'), diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 85ce0a2b5..0423103a8 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -37,7 +37,7 @@ from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class from tom_dataproducts.serializers import DataProductSerializer -from tom_dataproducts.forced_photometry.forced_photometry_service import ForcedPhotometryServiceException, get_service_class +import tom_dataproducts.forced_photometry.forced_photometry_service as fps import requests @@ -122,7 +122,7 @@ def get_service_class(self): """ Gets the forced photometry service class """ - return get_service_class(self.get_service()) + return fps.get_service_class(self.get_service()) def get_form_class(self): """ @@ -157,7 +157,7 @@ def post(self, request, *args, **kwargs): service = self.get_service_class()() try: service.query_service(form.cleaned_data) - except ForcedPhotometryServiceException as e: + except fps.ForcedPhotometryServiceException as e: form.add_error(f"Problem querying forced photometry service: {repr(e)}") return self.form_invalid(form) messages.info(self.request, service.get_success_message()) From 77427bfd8d42a4629dfed70d9fe2085384c674f5 Mon Sep 17 00:00:00 2001 From: Jon Date: Fri, 6 Oct 2023 17:00:28 +0000 Subject: [PATCH 35/42] Added documentation for forced photometry services --- docs/managing_data/forced_photometry.rst | 128 ++++++++++++++++++++ docs/managing_data/index.rst | 4 +- tom_dataproducts/forced_photometry/atlas.py | 1 + 3 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 docs/managing_data/forced_photometry.rst diff --git a/docs/managing_data/forced_photometry.rst b/docs/managing_data/forced_photometry.rst new file mode 100644 index 000000000..a9e9f7665 --- /dev/null +++ b/docs/managing_data/forced_photometry.rst @@ -0,0 +1,128 @@ +Integrating Forced Photometry Service Queries +--------------------------------------- + +The base TOM Toolkit comes with Atlas, panSTARRS, and ZTF query services. More services +can be added by extending the base ForcedPhotometryService implementation. + + +Integrating existing Forced Photometry Services +############################################### + +You must add certain configuration to your TOM's ``settings.py`` to setup the existing forced +photometry services. This configuration will go in the ``FORCED_PHOTOMETRY_SERVICES`` section +shown below: + +.. code:: python + FORCED_PHOTOMETRY_SERVICES = { + 'atlas': { + 'class': 'tom_dataproducts.forced_photometry.atlas.AtlasForcedPhotometryService', + 'url': "https://fallingstar-data.com/forcedphot", + 'api_key': os.getenv('ATLAS_FORCED_PHOTOMETRY_API_KEY', 'your atlas account api token') + }, + 'panstarrs': { + #TODO + }, + 'ztf': { + #TODO + } + } + + DATA_PRODUCT_TYPES = { + ... + 'atlas_photometry': ('atlas_photometry', 'Atlas Photometry'), + ... + } + + DATA_PROCESSORS = { + ... + 'atlas_photometry': 'tom_dataproducts.processors.atlas_processor.AtlasProcessor', + ... + } + + +Configuring your TOM to serve tasks asynchronously: +*************************************************** + +Several of the services are best suited to be queried asynchronously, especially if you plan to make large +queries that would take a long time. The TOM Toolkit is setup to use `dramatiq `_ +as an asynchronous task manager, but doing so requires you to run either a `redis `_ +or `rabbitmq `_ server to act as the task queue. To use dramatiq with +a redis server, you would add the following to your ``settings.py``: + +.. code:: python + INSTALLED_APPS = [ + ... + 'django_dramatiq', + ... + ] + + DRAMATIQ_BROKER = { + "BROKER": "dramatiq.brokers.redis.RedisBroker", + "OPTIONS": { + "url": "redis://your-redis-service-url:your-redis-port" + }, + "MIDDLEWARE": [ + "dramatiq.middleware.AgeLimit", + "dramatiq.middleware.TimeLimit", + "dramatiq.middleware.Callbacks", + "dramatiq.middleware.Retries", + "django_dramatiq.middleware.DbConnectionsMiddleware", + ] + } + +After adding the ``django_dramatiq`` installed app, you will need to run ``./manage.py migrate`` once to setup +its DB tables. If this configuration is set in your TOM, the existing services which support asynchronous queries, +Atlas and ZTF, should start querying asynchronously. If you do not add these settings, those services will still +function but will fall back to synchronous queries. + + +Adding a new Forced Photometry Service +###################################### + +The Forced Photometry services fulfill an interface defined in +`BaseForcedPhotometryService `_. +To implement your own Forced Photometry service, you need to do 3 things: +1. Subclass BaseForcedPhotometryService +2. Subclass BaseForcedPhotometryQueryForm +3. Subclass DataProcessor +Once those are implemented, don't forget to update your settings for ``FORCED_PHOTOMETRY_SERVICES``, +``DATA_PRODUCT_TYPES``, and ``DATA_PROCESSORS`` for your new service and its associated data product type. + + +Subclass BaseForcedPhotometryService: +************************************* + +The most important method here is the ``query_service`` method which is where you put your service's business logic +for making the query, given the form parameters and target. This method is expected to create a DataProduct in the database +at the end of the query, storing the result file or files. If queries to your service are expected to take a long time and +you would like to make them asynchronously (not blocking the UI while calling), then follow the example in the +`atlas implementation `_ and place your +actual asynchronous query method in your module's ``tasks.py`` file so it can be found by dramatiq. Like in the atlas implementation, +your code should check to see if ``django_dramatiq`` is in the settings ``INSTALLED_APPS`` before trying to enqueue it with dramatiq. + +The ``get_data_product_type`` method should return the name of your new data product type you are going to define a +DataProcessor for. This must match the name you add to ``DATA_PROCESSORS`` and ``DATA_PRODUCT_TYPES`` in your ``settings.py``. +You will also need to define a `DataProcessor ` +for this data type. + + +Subclass BaseForcedPhotometryQueryForm: +*************************************** + +This class defines the form users will need to fill out to query the service. It uses +`django-crispy-forms `_ to define the layout +programmatically. You first will add whatever form fields you need to the base of your +subclass, and then just fill in the ``layout()`` method with a django-crispy-forms layout +for your fields, and optionally the ``clean()`` method if you want to perform any field validation. +The values of the fields from this form will be available to you in your service class in the +``query_service`` method. + + +Subclass DataProcessor: +*********************** + +You must create a custom DataProcessor that knows how to convert data returned from your service into +a series of either photometry or spectroscopy datums. Without defining this step, your queries will still +result in a DataProduct file being stored from the service's ``query_service`` method, but those files will +not be parsed into photometry or spectroscopy datums. You can read more about how to implement a custom +DataProcessor `here <../customizing_data_processing>`_. \ No newline at end of file diff --git a/docs/managing_data/index.rst b/docs/managing_data/index.rst index 78979a9f5..5b4bcb46a 100644 --- a/docs/managing_data/index.rst +++ b/docs/managing_data/index.rst @@ -11,6 +11,7 @@ Managing Data customizing_data_processing tom_direct_sharing stream_pub_sub + forced_photometry :doc:`Creating Plots from TOM Data ` - Learn how to create plots using plot.ly and your TOM @@ -23,4 +24,5 @@ TOM from uploaded data products. :doc:`Publish and Subscribe to a Kafka Stream ` - Learn how to publish and subscribe to a Kafka stream topic. - +:doc:`Integrating Forced Photometry Service Queries ` - Learn how to integrate the existing Atlas, panSTARRS, and ZTF +forced photometry services into your TOM, and learn how to add new services. diff --git a/tom_dataproducts/forced_photometry/atlas.py b/tom_dataproducts/forced_photometry/atlas.py index 5bb327f73..062fcdce4 100644 --- a/tom_dataproducts/forced_photometry/atlas.py +++ b/tom_dataproducts/forced_photometry/atlas.py @@ -6,6 +6,7 @@ from tom_dataproducts.tasks import atlas_query from tom_targets.models import Target + class AtlasForcedPhotometryQueryForm(fps.BaseForcedPhotometryQueryForm): min_date = forms.CharField( label='Min date:', required=False, From 199cecc86e76e6fb0a42cbb9255f5d9849f7fcd1 Mon Sep 17 00:00:00 2001 From: Jon Date: Fri, 6 Oct 2023 17:19:33 +0000 Subject: [PATCH 36/42] Remove typing since thats breaking tests --- tom_dataproducts/views.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 0423103a8..0800908bb 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -1,7 +1,6 @@ from io import StringIO import logging import os -from typing import Any from urllib.parse import urlencode, urlparse from django.conf import settings @@ -130,11 +129,11 @@ def get_form_class(self): """ return self.get_service_class()().get_form() - def get_context_data(self, **kwargs: Any) -> dict[str, Any]: + def get_context_data(self, *args, **kwargs): """ Adds the target to the context object. """ - context = super().get_context_data(**kwargs) + context = super().get_context_data(*args, **kwargs) context['target'] = self.get_target() context['query_form'] = self.get_form_class()(initial=self.get_initial()) return context From 319f315d66486888f5c97bafa54c0ea2e52d53d0 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 11 Oct 2023 13:13:45 -0700 Subject: [PATCH 37/42] add fuzzy, multi-target filters --- tom_dataproducts/sharing.py | 11 ++++++++++- tom_targets/filters.py | 26 ++++++++++++++++++++++++-- tom_targets/sharing.py | 2 ++ tom_targets/tests/tests.py | 33 ++++++++++++++++++++++++++++++++- 4 files changed, 68 insertions(+), 4 deletions(-) diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index 543a7579d..656069d14 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -92,6 +92,8 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id destination_target_id, target_search_response = get_destination_target(target, targets_url, headers, auth) if destination_target_id is None: return {'message': 'ERROR: No matching target found.'} + elif isinstance(destination_target_id, list) and len(destination_target_id) > 1: + return {'message': 'ERROR: Multiple targets with matching name found in destination TOM.'} serialized_data['target'] = destination_target_id # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) @@ -111,6 +113,8 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id targets_url, headers, auth) + if isinstance(destination_target_id, list) and len(destination_target_id) > 1: + return {'message': 'ERROR: Multiple targets with matching name found in destination TOM.'} target_dict[target.name] = destination_target_id if all(value is None for value in target_dict.values()): return {'message': 'ERROR: No matching targets found.'} @@ -120,6 +124,8 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id destination_target_id, target_search_response = get_destination_target(target, targets_url, headers, auth) if destination_target_id is None: return {'message': 'ERROR: No matching target found.'} + elif isinstance(destination_target_id, list) and len(destination_target_id) > 1: + return {'message': 'ERROR: Multiple targets with matching name found in destination TOM.'} target_dict = {target.name: destination_target_id} response_codes = [] reduced_datums = check_for_share_safe_datums(share_destination, reduced_datums) @@ -156,10 +162,13 @@ def get_destination_target(target, targets_url, headers, auth): :param auth: :return: """ - target_response = requests.get(f'{targets_url}?name={target.name}', headers=headers, auth=auth) + target_names = ','.join(map(str, target.names)) + target_response = requests.get(f'{targets_url}?name_fuzzy={target_names}', headers=headers, auth=auth) target_response_json = target_response.json() try: if target_response_json['results']: + if len(target_response_json['results']) > 1: + return target_response_json['results'], target_response destination_target_id = target_response_json['results'][0]['id'] return destination_target_id, target_response else: diff --git a/tom_targets/filters.py b/tom_targets/filters.py index dafc564b3..c47e7bd47 100644 --- a/tom_targets/filters.py +++ b/tom_targets/filters.py @@ -2,7 +2,7 @@ from django.db.models import Q import django_filters -from tom_targets.models import Target, TargetList +from tom_targets.models import Target, TargetList, TargetMatchManager from tom_targets.utils import cone_search_filter @@ -55,7 +55,29 @@ def __init__(self, *args, **kwargs): name = django_filters.CharFilter(method='filter_name', label='Name') def filter_name(self, queryset, name, value): - return queryset.filter(Q(name__icontains=value) | Q(aliases__name__icontains=value)).distinct() + """ + Return a queryset for targets with names or aliases containing the given coma-separated list of terms. + """ + q_set = Q() + for term in value.split(','): + q_set |= Q(name__icontains=term) | Q(aliases__name__icontains=term) + return queryset.filter(q_set).distinct() + + name_fuzzy = django_filters.CharFilter(method='filter_name_fuzzy', label='Name (Fuzzy)') + + def filter_name_fuzzy(self, queryset, name, value): + """ + Return a queryset for targets with names or aliases fuzzy matching the given coma-separated list of terms. + A fuzzy match is determined by the `make_simple_name` method of the `TargetMatchManager` class. + """ + matching_names = [] + for term in value.split(','): + simple_name = TargetMatchManager.make_simple_name(self, term) + for target in Target.objects.all().prefetch_related('aliases'): + for alias in target.names: + if TargetMatchManager.make_simple_name(self, alias) == simple_name: + matching_names.append(target.name) + return queryset.filter(name__in=matching_names).distinct() cone_search = django_filters.CharFilter(method='filter_cone_search', label='Cone Search', help_text='RA, Dec, Search Radius (degrees)') diff --git a/tom_targets/sharing.py b/tom_targets/sharing.py index 13ec10568..977b27f28 100644 --- a/tom_targets/sharing.py +++ b/tom_targets/sharing.py @@ -32,6 +32,8 @@ def share_target_with_tom(share_destination, form_data, target_lists=()): auth) if target_search_response.status_code != 200: return target_search_response + elif isinstance(destination_target_id, list) and len(destination_target_id) > 1: + return {'message': 'ERROR: Multiple targets with matching name found in destination TOM.'} target_dict_list = [{'name': f'Imported From {settings.TOM_NAME}'}] for target_list in target_lists: diff --git a/tom_targets/tests/tests.py b/tom_targets/tests/tests.py index 059ed5806..a335694aa 100644 --- a/tom_targets/tests/tests.py +++ b/tom_targets/tests/tests.py @@ -1380,6 +1380,37 @@ def test_share_target_valid_connection_no_target_found(self): self.assertContains(response, 'Target successfully uploaded.') + @responses.activate + def test_share_target_valid_connection_multiple_target_found(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"results": [{'id': 1}, {'id': 2}]}, + status=200 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + + response = self.client.post( + reverse('targets:share', kwargs={'pk': self.target.id}), + { + 'submitter': ['test_submitter'], + 'target': self.target.id, + 'share_destination': [share_destination], + }, + follow=True + ) + + self.assertContains(response, 'ERROR: Multiple targets with matching name found in destination TOM.') + @responses.activate def test_share_reduceddatums_target_valid_responses(self): share_destination = 'local_host' @@ -1543,7 +1574,7 @@ def test_share_reduceddatums_group_valid_responses(self): rsp1 = responses.Response( method="GET", url=destination_tom_base_url + 'api/targets/', - json={"results": [{'id': 1}, {'id': 2}]}, + json={"results": [{'id': 1}]}, status=200 ) responses.add(rsp1) From 19cf5ca98ccfebc0565496fccb26cd706b3b630f Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 12 Oct 2023 12:55:10 -0700 Subject: [PATCH 38/42] add comments --- tom_dataproducts/sharing.py | 41 +++++++++++------- tom_targets/api_views.py | 2 + tom_targets/forms.py | 6 +++ tom_targets/serializers.py | 5 +++ tom_targets/sharing.py | 13 ++++-- .../tom_targets/partials/target_buttons.html | 2 +- .../templates/tom_targets/target_share.html | 2 +- tom_targets/tests/test_api.py | 5 ++- tom_targets/tests/tests.py | 4 ++ tom_targets/views.py | 42 ++++++++++--------- 10 files changed, 81 insertions(+), 41 deletions(-) diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index 656069d14..d85c3524c 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -13,12 +13,12 @@ def share_data_with_hermes(share_destination, form_data, product_id=None, target_id=None, selected_data=None): """ - - :param share_destination: - :param form_data: - :param product_id: - :param target_id: - :param selected_data: + Serialize and share data with Hermes (hermes.lco.global) + :param share_destination: Topic to share data to. (e.g. 'hermes.test') + :param form_data: Sharing Form data + :param product_id: DataProduct ID (if provided) + :param target_id: Target ID (if provided) + :param selected_data: List of ReducedDatum IDs (if provided) :return: """ # Query relevant Reduced Datums Queryset @@ -63,14 +63,15 @@ def share_data_with_hermes(share_destination, form_data, product_id=None, target def share_data_with_tom(share_destination, form_data, product_id=None, target_id=None, selected_data=None): """ - - :param share_destination: - :param form_data: - :param product_id: - :param target_id: - :param selected_data: + Serialize and share data with another TOM + :param share_destination: TOM to share data to as described in settings.DATA_SHARING. (e.g. 'mytom') + :param form_data: Sharing Form data + :param product_id: DataProduct ID (if provided) + :param target_id: Target ID (if provided) + :param selected_data: List of ReducedDatum IDs (if provided) :return: """ + # Build destination TOM headers and URL information try: destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] username = settings.DATA_SHARING[share_destination]['USERNAME'] @@ -85,10 +86,12 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id reduced_datums_url = destination_tom_base_url + 'api/reduceddatums/' reduced_datums = ReducedDatum.objects.none() + # If a DataProduct is provided, share that DataProduct if product_id: product = DataProduct.objects.get(pk=product_id) target = product.target serialized_data = DataProductSerializer(product).data + # Find matching target in destination TOM destination_target_id, target_search_response = get_destination_target(target, targets_url, headers, auth) if destination_target_id is None: return {'message': 'ERROR: No matching target found.'} @@ -103,6 +106,7 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id headers = {'Media-Type': 'multipart/form-data'} response = requests.post(dataproducts_url, data=serialized_data, files=files, headers=headers, auth=auth) elif selected_data or target_id: + # If ReducedDatums are provided, share those ReducedDatums if selected_data: reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) targets = set(reduced_datum.target for reduced_datum in reduced_datums) @@ -119,6 +123,8 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id if all(value is None for value in target_dict.values()): return {'message': 'ERROR: No matching targets found.'} else: + # If Target is provided, share all ReducedDatums for that Target + # (Will not create New Target in Destination TOM) target = Target.objects.get(pk=target_id) reduced_datums = ReducedDatum.objects.filter(target=target) destination_target_id, target_search_response = get_destination_target(target, targets_url, headers, auth) @@ -155,13 +161,14 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id def get_destination_target(target, targets_url, headers, auth): """ - Retrieve the target ID from a destination TOM + Retrieve the target ID from a destination TOM that is a fuzzy match the given target name and aliases :param target: Target Model :param targets_url: Destination API URL for TOM Target List - :param headers: - :param auth: + :param headers: TOM API headers + :param auth: TOM API authorization :return: """ + # Create coma separated list of target names plus aliases that can be recognized and parsed by the TOM API Filter target_names = ','.join(map(str, target.names)) target_response = requests.get(f'{targets_url}?name_fuzzy={target_names}', headers=headers, auth=auth) target_response_json = target_response.json() @@ -229,6 +236,10 @@ def get_sharing_destination_options(): def sharing_feedback_handler(response, request): + """ + Handle the response from a sharing request and prepare a message to the user + :return: + """ try: if 'message' in response.json(): publish_feedback = response.json()['message'] diff --git a/tom_targets/api_views.py b/tom_targets/api_views.py index 41d831515..861640595 100644 --- a/tom_targets/api_views.py +++ b/tom_targets/api_views.py @@ -57,6 +57,7 @@ def get_queryset(self): def create(self, request, *args, **kwargs): response = super().create(request, *args, **kwargs) + # Custom message for successful target creation if response.status_code == status.HTTP_201_CREATED: response.data['message'] = 'Target successfully uploaded.' return response @@ -64,6 +65,7 @@ def create(self, request, *args, **kwargs): def update(self, request, *args, **kwargs): response = super().update(request, *args, **kwargs) + # Custom message for successful target update if response.status_code == status.HTTP_200_OK: response.data['message'] = 'Target successfully updated.' return response diff --git a/tom_targets/forms.py b/tom_targets/forms.py index f22520750..be549285e 100644 --- a/tom_targets/forms.py +++ b/tom_targets/forms.py @@ -170,6 +170,9 @@ def clean(self): class TargetShareForm(forms.Form): + """ + Form for sharing a target with an outside destination such as another TOM Toolkit or Hermes + """ share_destination = forms.ChoiceField(required=True, choices=[], label="Destination") target = forms.ModelChoiceField( Target.objects.all(), @@ -183,6 +186,9 @@ def __init__(self, *args, **kwargs): class TargetListShareForm(forms.Form): + """ + Form for sharing a target list with an outside destination such as another TOM Toolkit or Hermes + """ share_destination = forms.ChoiceField(required=True, choices=[], label="Destination") target_list = forms.ModelChoiceField( TargetList.objects.all(), diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py index 2d2b9f00d..2c1e46eca 100644 --- a/tom_targets/serializers.py +++ b/tom_targets/serializers.py @@ -24,6 +24,9 @@ class Meta: class TargetListSerializer(serializers.ModelSerializer): + """ + TargetList serializer responsible for transforming models to/from + """ id = serializers.IntegerField(required=False) name = serializers.CharField(required=False) @@ -93,6 +96,7 @@ def create(self, validated_data): if tes.is_valid(): tes.save(target=target) + # Save target lists for this target tls = TargetListSerializer(data=target_lists, many=True) if tls.is_valid(): for target_list in target_lists: @@ -157,6 +161,7 @@ def update(self, instance, validated_data): if tes.is_valid(): tes.save(target=instance) + # Update target lists for this target tls = TargetListSerializer(data=target_lists, many=True) if tls.is_valid(): for target_list in target_lists: diff --git a/tom_targets/sharing.py b/tom_targets/sharing.py index 977b27f28..62b1b813b 100644 --- a/tom_targets/sharing.py +++ b/tom_targets/sharing.py @@ -9,10 +9,11 @@ def share_target_with_tom(share_destination, form_data, target_lists=()): """ - :param share_destination: - :param form_data: - :param target_lists: - :return: + Share a target with a remote TOM. + :param share_destination: The name of the destination TOM as defined in settings.DATA_SHARING + :param form_data: The form data from the target form + :param target_lists: The target lists to add the target to in the destination TOM + :return: The response from the destination TOM """ # Try to get destination tom authentication/URL information try: @@ -30,15 +31,18 @@ def share_target_with_tom(share_destination, form_data, target_lists=()): # Check if target already exists in destination DB destination_target_id, target_search_response = get_destination_target(form_data['target'], targets_url, headers, auth) + # Handle errors or multiple targets found if target_search_response.status_code != 200: return target_search_response elif isinstance(destination_target_id, list) and len(destination_target_id) > 1: return {'message': 'ERROR: Multiple targets with matching name found in destination TOM.'} + # Build list of targetlists to add target to in destination TOM target_dict_list = [{'name': f'Imported From {settings.TOM_NAME}'}] for target_list in target_lists: target_dict_list.append({'name': target_list.name}) + # Create or update target in destination TOM if destination_target_id is None: # If target is not in Destination, serialize and create new target. serialized_target = TargetSerializer(form_data['target']).data @@ -48,6 +52,7 @@ def share_target_with_tom(share_destination, form_data, target_lists=()): serialized_target['target_lists'] = target_dict_list target_create_response = requests.post(targets_url, json=serialized_target, headers=headers, auth=auth) else: + # Add target to target lists if it already exists in destination TOM update_target_data = {'target_lists': target_dict_list} update_target_url = targets_url + f'{destination_target_id}/' target_create_response = requests.patch(update_target_url, json=update_target_data, headers=headers, auth=auth) diff --git a/tom_targets/templates/tom_targets/partials/target_buttons.html b/tom_targets/templates/tom_targets/partials/target_buttons.html index 76af2cfed..a22408d10 100644 --- a/tom_targets/templates/tom_targets/partials/target_buttons.html +++ b/tom_targets/templates/tom_targets/partials/target_buttons.html @@ -1,3 +1,3 @@ Update Share -Delete \ No newline at end of file +Delete diff --git a/tom_targets/templates/tom_targets/target_share.html b/tom_targets/templates/tom_targets/target_share.html index e1222c13a..1e504b843 100644 --- a/tom_targets/templates/tom_targets/target_share.html +++ b/tom_targets/templates/tom_targets/target_share.html @@ -24,4 +24,4 @@

Include Data

Back {% endbuttons %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/tom_targets/tests/test_api.py b/tom_targets/tests/test_api.py index d0d975063..a05cfa6e9 100644 --- a/tom_targets/tests/test_api.py +++ b/tom_targets/tests/test_api.py @@ -53,6 +53,9 @@ def test_target_detail(self): self.assertEqual(response.json()['detail'], 'Not found.') def test_target_create(self): + """ + Test that a target can be created with all valid parameters through the API + """ collaborator = User.objects.create(username='test collaborator') group = Group.objects.create(name='bourgeoisie') group.user_set.add(self.user) @@ -203,7 +206,7 @@ def test_targetname_update(self): status_code=status.HTTP_400_BAD_REQUEST) def test_targetlist_update(self): - # Test both create new alias and update alias + # Test Add existing target to new target list target_list = TargetGroupingFactory.create(name='tl') target_list.targets.add(self.st) updates = { diff --git a/tom_targets/tests/tests.py b/tom_targets/tests/tests.py index a335694aa..3792ae11a 100644 --- a/tom_targets/tests/tests.py +++ b/tom_targets/tests/tests.py @@ -1288,6 +1288,10 @@ def test_persist_filter_empty(self): 'USERNAME': 'fake_user', 'PASSWORD': 'password'}}) class TestShareTargets(TestCase): + """ + Tests for the share_targets view. + Tests the behavior of the SENDING TOM and Mocks responses from receiving TOM. + """ def setUp(self): self.target = SiderealTargetFactory.create() self.observation_record = ObservingRecordFactory.create( diff --git a/tom_targets/views.py b/tom_targets/views.py index 26274a348..d7689b02f 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -334,15 +334,15 @@ class TargetDeleteView(Raise403PermissionRequiredMixin, DeleteView): class TargetShareView(FormView): """ - View for deleting a target. Requires authorization. + View for sharing a target. Requires authorization. """ template_name = 'tom_targets/target_share.html' - # permission_required = 'tom_targets.share_target' + permission_required = 'tom_targets.share_target' form_class = TargetShareForm def get_context_data(self, *args, **kwargs): """ - Adds the ``DataProductUploadForm`` to the context and prepopulates the hidden fields. + Adds the target information to the context. :returns: context object :rtype: dict """ @@ -353,6 +353,9 @@ def get_context_data(self, *args, **kwargs): return context def get_success_url(self): + """ + Redirect to target detail page for shared target + """ return reverse_lazy('targets:detail', kwargs={'pk': self.kwargs.get('pk', None)}) def form_invalid(self, form): @@ -364,29 +367,24 @@ def form_invalid(self, form): return redirect(self.get_success_url()) def form_valid(self, form): + """ + Shares the target with the selected destination(s) and redirects to the target detail page. + """ form_data = form.cleaned_data share_destination = form_data['share_destination'] target_id = self.kwargs.get('pk', None) selected_data = self.request.POST.getlist("share-box") if 'HERMES' in share_destination.upper(): response = share_data_with_hermes(share_destination, form_data, None, target_id, selected_data) + sharing_feedback_handler(response, self.request) else: + # Share Target with Destination TOM response = share_target_with_tom(share_destination, form_data) + sharing_feedback_handler(response, self.request) if selected_data: + # Share Data with Destination TOM response = share_data_with_tom(share_destination, form_data, selected_data=selected_data) - try: - if 'message' in response.json(): - publish_feedback = response.json()['message'] - else: - publish_feedback = f"ERROR: {response.text}" - except AttributeError: - publish_feedback = response['message'] - except ValueError: - publish_feedback = f"ERROR: Returned Response code {response.status_code}" - if "ERROR" in publish_feedback.upper(): - messages.error(self.request, publish_feedback) - else: - messages.success(self.request, publish_feedback) + sharing_feedback_handler(response, self.request) return redirect(self.get_success_url()) @@ -592,15 +590,15 @@ def form_valid(self, form): class TargetGroupingShareView(FormView): """ - View for deleting a target. Requires authorization. + View for sharing a TargetList. Requires authorization. """ template_name = 'tom_targets/target_group_share.html' - # permission_required = 'tom_targets.share_target' + permission_required = 'tom_targets.share_target' form_class = TargetListShareForm def get_context_data(self, *args, **kwargs): """ - Adds the ``DataProductUploadForm`` to the context and prepopulates the hidden fields. + Adds the ``TargetListShareForm`` to the context and prepopulates the hidden fields. :returns: context object :rtype: dict """ @@ -616,6 +614,9 @@ def get_context_data(self, *args, **kwargs): return context def get_success_url(self): + """ + Redirects to the target list page with the target list name as a query parameter. + """ return reverse_lazy('targets:list')+f'?targetlist__name={self.kwargs.get("pk", None)}' def form_invalid(self, form): @@ -632,15 +633,18 @@ def form_valid(self, form): selected_targets = self.request.POST.getlist('selected-target') data_switch = self.request.POST.get('dataSwitch', False) if 'HERMES' in share_destination.upper(): + # TODO: Implement Hermes sharing # response = share_data_with_hermes(share_destination, form_data, None, target_id, selected_data) messages.error(self.request, "Publishing Groups to Hermes is not yet supported.") return redirect(self.get_success_url()) else: for target in selected_targets: + # Share each target individually form_data['target'] = Target.objects.get(id=target) response = share_target_with_tom(share_destination, form_data, target_lists=[form_data['target_list']]) sharing_feedback_handler(response, self.request) if data_switch: + # If Data sharing request, share all data associated with the target response = share_data_with_tom(share_destination, form_data, target_id=target) sharing_feedback_handler(response, self.request) if not selected_targets: From 9750f2a44b5977728f1b28b7d95b3d97bfc7e0e5 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Fri, 13 Oct 2023 14:19:22 -0700 Subject: [PATCH 39/42] hide sharing options when unconfigured. --- tom_base/settings.py | 48 +++++++++---------- .../tom_targets/partials/target_buttons.html | 4 +- .../tom_targets/target_grouping.html | 8 +++- tom_targets/templatetags/targets_extras.py | 3 +- tom_targets/views.py | 10 ++++ 5 files changed, 45 insertions(+), 28 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index fe8754ce3..8eef5e1fb 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -237,30 +237,30 @@ } # Configuration for the TOM/Kafka Stream receiving data from this TOM -DATA_SHARING = { - 'hermes': { - 'DISPLAY_NAME': os.getenv('HERMES_DISPLAY_NAME', 'Hermes'), - 'BASE_URL': os.getenv('HERMES_BASE_URL', 'https://hermes.lco.global/'), - 'CREDENTIAL_USERNAME': os.getenv('SCIMMA_CREDENTIAL_USERNAME', - 'set SCIMMA_CREDENTIAL_USERNAME value in environment'), - 'CREDENTIAL_PASSWORD': os.getenv('SCIMMA_CREDENTIAL_PASSWORD', - 'set SCIMMA_CREDENTIAL_PASSWORD value in environment'), - 'USER_TOPICS': ['hermes.test', 'tomtoolkit.test'] - }, - 'tom-demo-dev': { - 'DISPLAY_NAME': os.getenv('TOM_DEMO_DISPLAY_NAME', 'TOM Demo Dev'), - 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), - 'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'), - 'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'), - }, - 'localhost-tom': { - # for testing; share with yourself - 'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'), - 'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'), - 'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'), - 'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'), - } -} +# DATA_SHARING = { +# 'hermes': { +# 'DISPLAY_NAME': os.getenv('HERMES_DISPLAY_NAME', 'Hermes'), +# 'BASE_URL': os.getenv('HERMES_BASE_URL', 'https://hermes.lco.global/'), +# 'CREDENTIAL_USERNAME': os.getenv('SCIMMA_CREDENTIAL_USERNAME', +# 'set SCIMMA_CREDENTIAL_USERNAME value in environment'), +# 'CREDENTIAL_PASSWORD': os.getenv('SCIMMA_CREDENTIAL_PASSWORD', +# 'set SCIMMA_CREDENTIAL_PASSWORD value in environment'), +# 'USER_TOPICS': ['hermes.test', 'tomtoolkit.test'] +# }, +# 'tom-demo-dev': { +# 'DISPLAY_NAME': os.getenv('TOM_DEMO_DISPLAY_NAME', 'TOM Demo Dev'), +# 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), +# 'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'), +# 'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'), +# }, +# 'localhost-tom': { +# # for testing; share with yourself +# 'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'), +# 'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'), +# 'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'), +# 'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'), +# } +# } TOM_CADENCE_STRATEGIES = [ 'tom_observations.cadences.retry_failed_observations.RetryFailedObservationsStrategy', diff --git a/tom_targets/templates/tom_targets/partials/target_buttons.html b/tom_targets/templates/tom_targets/partials/target_buttons.html index a22408d10..321bca67d 100644 --- a/tom_targets/templates/tom_targets/partials/target_buttons.html +++ b/tom_targets/templates/tom_targets/partials/target_buttons.html @@ -1,3 +1,5 @@ Update -Share +{% if sharing %} + Share +{% endif %} Delete diff --git a/tom_targets/templates/tom_targets/target_grouping.html b/tom_targets/templates/tom_targets/target_grouping.html index 647d0db44..b6d2ba9e5 100644 --- a/tom_targets/templates/tom_targets/target_grouping.html +++ b/tom_targets/templates/tom_targets/target_grouping.html @@ -17,7 +17,9 @@

Target Groupings

Group Total Targets - Share + {% if sharing %} + Share + {% endif %} Delete @@ -26,7 +28,9 @@

Target Groupings

{{ group.targets.count }} - Share + {% if sharing %} + Share + {% endif %} Delete {% empty %} diff --git a/tom_targets/templatetags/targets_extras.py b/tom_targets/templatetags/targets_extras.py index 140abe19c..2fe5b6d0a 100644 --- a/tom_targets/templatetags/targets_extras.py +++ b/tom_targets/templatetags/targets_extras.py @@ -50,7 +50,8 @@ def target_buttons(target): """ Displays the Update and Delete buttons for a target. """ - return {'target': target} + return {'target': target, + 'sharing': getattr(settings, "DATA_SHARING", None)} @register.inclusion_tag('tom_targets/partials/target_data.html') diff --git a/tom_targets/views.py b/tom_targets/views.py index d7689b02f..ac7d39bfb 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -555,6 +555,16 @@ class TargetGroupingView(PermissionListMixin, ListView): model = TargetList paginate_by = 25 + def get_context_data(self, *args, **kwargs): + """ + Adds ``settings.DATA_SHARING`` to the context to see if sharing has been configured. + :returns: context object + :rtype: dict + """ + context = super().get_context_data(*args, **kwargs) + context['sharing'] = getattr(settings, "DATA_SHARING", None) + return context + class TargetGroupingDeleteView(Raise403PermissionRequiredMixin, DeleteView): """ From d1b2b03945a1308f4701e1954b1bd55b3083ba1a Mon Sep 17 00:00:00 2001 From: Jon Date: Mon, 16 Oct 2023 20:44:02 +0000 Subject: [PATCH 40/42] Add newline at end of template --- .../templates/tom_dataproducts/forced_photometry_form.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_dataproducts/templates/tom_dataproducts/forced_photometry_form.html b/tom_dataproducts/templates/tom_dataproducts/forced_photometry_form.html index 212d926b7..bb78fd757 100644 --- a/tom_dataproducts/templates/tom_dataproducts/forced_photometry_form.html +++ b/tom_dataproducts/templates/tom_dataproducts/forced_photometry_form.html @@ -11,4 +11,4 @@

Query {{ form.service.value }} Forced Photometry Service

Target {{ target.name }} at RA {{ target.ra }}, DEC {{ target.dec }}


{% crispy query_form %} -{% endblock %} \ No newline at end of file +{% endblock %} From 9d873921c72a760ef0c7ebbd6e7ad889d6ae9119 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Mon, 16 Oct 2023 14:51:18 -0700 Subject: [PATCH 41/42] update readthedocs --- docs/managing_data/tom_direct_sharing.rst | 100 +++++++++++++++------- 1 file changed, 71 insertions(+), 29 deletions(-) diff --git a/docs/managing_data/tom_direct_sharing.rst b/docs/managing_data/tom_direct_sharing.rst index ee04cb73e..bc0edb2ca 100644 --- a/docs/managing_data/tom_direct_sharing.rst +++ b/docs/managing_data/tom_direct_sharing.rst @@ -1,32 +1,74 @@ Sharing Data with Other TOMs ############################ -TOM Toolkit does not yet support direct sharing between TOMs, however we hope to add this functionality soon. - - -.. Configuring your TOM to submit data to another TOM: -.. *************************************************** -.. -.. You will need to add a ``DATA_SHARING`` configuration dictionary to your ``settings.py`` that gives the credentials -.. for the various TOMs with which you wish to share data. -.. -.. .. code:: python -.. -.. # Define the valid data sharing destinations for your TOM. -.. DATA_SHARING = { -.. 'tom-demo-dev': { -.. 'DISPLAY_NAME': os.getenv('TOM_DEMO_DISPLAY_NAME', 'TOM Demo Dev'), -.. 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), -.. 'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'), -.. 'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'), -.. }, -.. 'localhost-tom': { -.. # for testing; share with yourself -.. 'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'), -.. 'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'), -.. 'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'), -.. 'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'), -.. } -.. -.. } -.. \ No newline at end of file +TOM Toolkit supports direct data sharing between TOMs. + + +Permissions: +************ +To save data to a destination TOM your TOM will need to have access to a user account on that TOM with the correct +permissions. This is handled by your TOM's administrator as described below. + +.. warning:: Any user who has permission to access the relevant target or data in your TOM will have permission to + submit that data to the destination TOM once DATA_SHARING is configured. + + +Configuring your TOM to submit data to another TOM: +*************************************************** + +You will need to add a ``DATA_SHARING`` configuration dictionary to your ``settings.py`` that gives the credentials +for the various TOMs with which you wish to share data. This should be the same ``DATA_SHARING`` dictionary that is used +to :doc:`/managing_data/stream_pub_sub` such as `Hermes `_. + +.. code:: python + + # Define the valid data sharing destinations for your TOM. + DATA_SHARING = { + 'not-my-tom': { + # For sharing data with another TOM + 'DISPLAY_NAME': os.getenv('NOT_MY_TOM_DISPLAY_NAME', 'Not My Tom'), + 'BASE_URL': os.getenv('NOT_MY_TOM_BASE_URL', 'http://notmytom.com/'), + 'USERNAME': os.getenv('NOT_MY_TOM_USERNAME', 'set NOT_MY_TOM_USERNAME value in environment'), + 'PASSWORD': os.getenv('NOT_MY_TOM_PASSWORD', 'set NOT_MY_TOM_PASSWORD value in environment'), + }, + 'localhost-tom': { + # for testing; share with yourself + 'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'), + 'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'), + 'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'), + 'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'), + } + } + +Receiving Shared Data: +********************** + +Reduced Datums: +--------------- +When your TOM receives a new ``ReducedDatum`` from another TOM it will be saved to your TOM's database with its source +set to the name of the TOM that submitted it. Currently, only Photometry data can be directly shared between +TOMS and a ``Target`` with a matching name or alias must exist in both TOMS for sharing to take place. + +Data Products: +-------------- +When your TOM receives a new ``DataProduct`` from another TOM it will be saved to your TOM's database / storage and run +through the appropriate :doc:`data_processor ` pipeline. Only data products +associated with a ``Target`` with a name or alias that matches that of a target in the destination TOM will be shared. + +Targets: +-------- +When your TOM receives a new ``Target`` from another TOM it will be saved to your TOM's database. If the target's name +or alias doesn't match that of a target that already exists in the database, a new target will be created and added to a +new ``TargetList`` called "Imported from ". + +Target Lists: +------------- +When your TOM receives a new ``TargetList`` from another TOM it will be saved to your TOM's database. If the targets in +the ``TargetList`` are also shared, but already exist in the destination TOM, they will be added to the new +``TargetList``. + + + + + + From af9284ef7cd07ae79713a7b246c5f1bed1b16af9 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Mon, 16 Oct 2023 16:57:01 -0700 Subject: [PATCH 42/42] some lint issues --- tom_dataproducts/views.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index d8b38bc61..b4d7f7776 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -31,10 +31,8 @@ from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class from tom_dataproducts.sharing import share_data_with_hermes, share_data_with_tom, sharing_feedback_handler -from tom_dataproducts.serializers import DataProductSerializer import tom_dataproducts.forced_photometry.forced_photometry_service as fps - -import requests +from tom_targets.models import Target logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG)