From 6620c5acae981212a24e2b60d48138041bb31e54 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Sat, 24 Sep 2022 01:02:09 +0000 Subject: [PATCH 01/72] define and add a preliminary DATA_SHARING configuration dict --- tom_base/settings.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index 8dcc61086..444d0c054 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -196,6 +196,16 @@ }, } +TOM_FACILITY_CLASSES = [ + 'tom_observations.facilities.lco.LCOFacility', + 'tom_observations.facilities.gemini.GEMFacility', + 'tom_observations.facilities.soar.SOARFacility', +] + +# +# tom_dataproducts configuration +# + # Define the valid data product types for your TOM. Be careful when removing items, as previously valid types will no # longer be valid, and may cause issues unless the offending records are modified. DATA_PRODUCT_TYPES = { @@ -210,11 +220,17 @@ 'spectroscopy': 'tom_dataproducts.processors.spectroscopy_processor.SpectroscopyProcessor', } -TOM_FACILITY_CLASSES = [ - 'tom_observations.facilities.lco.LCOFacility', - 'tom_observations.facilities.gemini.GEMFacility', - 'tom_observations.facilities.soar.SOARFacility', -] +DATA_SHARING = { + 'hermes': { + 'BASE_URL': os.getenv('HERMES_BASE_URL', 'http://hermes-dev.lco.gtn/'), + 'API_TOKEN': os.getenv('HERMES_API_TOKEN', 'set HERMES_API_TOKEN value in environment'), + }, + 'tomtoolkit': { + 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), + 'API_TOKEN': os.getenv('TOM_DEMO_API_TOKEN', 'set TOM_DEMO_API_TOKEN value in environment'), + } +} + TOM_CADENCE_STRATEGIES = [ 'tom_observations.cadences.retry_failed_observations.RetryFailedObservationsStrategy', From 02a9e12df556fbccb25f0729e92d02f03d851654 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Sat, 24 Sep 2022 01:04:21 +0000 Subject: [PATCH 02/72] add route to and draft impl of DataProductShareView --- tom_dataproducts/urls.py | 3 +- tom_dataproducts/views.py | 94 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/tom_dataproducts/urls.py b/tom_dataproducts/urls.py index e4b621712..198cd53da 100644 --- a/tom_dataproducts/urls.py +++ b/tom_dataproducts/urls.py @@ -3,7 +3,7 @@ from tom_dataproducts.views import DataProductListView, DataProductSaveView, DataProductGroupListView from tom_dataproducts.views import DataProductDeleteView, DataProductGroupCreateView from tom_dataproducts.views import DataProductGroupDetailView, DataProductGroupDataView, DataProductGroupDeleteView -from tom_dataproducts.views import DataProductUploadView, DataProductFeatureView, UpdateReducedDataView +from tom_dataproducts.views import DataProductUploadView, DataProductFeatureView, UpdateReducedDataView, DataProductShareView from tom_common.api_router import SharedAPIRootRouter from tom_dataproducts.api_views import DataProductViewSet @@ -24,5 +24,6 @@ path('data/reduced/update/', UpdateReducedDataView.as_view(), name='update-reduced-data'), path('data//delete/', DataProductDeleteView.as_view(), name='delete'), path('data//feature/', DataProductFeatureView.as_view(), name='feature'), + path('data//share/', DataProductShareView.as_view(), name='share'), path('/save/', DataProductSaveView.as_view(), name='save'), ] diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index fd647cf9d..df80ea5ab 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -1,4 +1,6 @@ +import datetime from io import StringIO +import logging from urllib.parse import urlencode, urlparse from django.conf import settings @@ -27,9 +29,14 @@ from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm from tom_dataproducts.filters import DataProductFilter from tom_dataproducts.data_processor import run_data_processor +from tom_dataproducts.processors.photometry_processor import PhotometryProcessor from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class +import requests + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) class DataProductSaveView(LoginRequiredMixin, View): """ @@ -276,6 +283,93 @@ def get(self, request, *args, **kwargs): ) +class DataProductShareView(View): + # TODO: update class docstring + """ + View that handles the featuring of ``DataProduct``s. A featured ``DataProduct`` is displayed on the + ``TargetDetailView``. + """ + + # TODO: refactor the general data shaing mechanism to make it more driven by the + # configuration in settings.py + def submit_to_hermes(self, data_to_share, message='From TOMToolkit'): + hermes_base_url = settings.DATA_SHARING['hermes']['BASE_URL'] + + # Get the csrf-token to include in header + #csrf_url = hermes_base_url + 'get-csrf-token/' + #csrf_headers = {'Content-Type': 'application/json'} + #csrf_response = requests.get(url=csrf_url, headers=csrf_headers) + + #logger.debug(f'dir(csrf_response): {dir(csrf_response)}') + #logger.debug(f'csrf_response.text: {csrf_response.text}') + #logger.debug(f'csrf_response.json(): {csrf_response.json()}') + + csrf_token = '' + #csrf_token = csrf_response.json()['token'] + + submit_url = hermes_base_url + 'submit/' + headers = { + #'X-CSRFToken': csrf_token, + #'Content-Type': 'application/json', + } + + # fields required by hopingest.py: topic, title, author, data, message_text + data = data_to_share + alert = { + 'topic': 'hermes.test', + 'title': 'TOM Toolkit test (Photometry)', + 'author': 'llindstrom@lco.global', + 'data': data_to_share, + 'message_text': f'Test alert from TOM Toolkit at {datetime.datetime.now()}', + } + + submit_response = requests.post(url=submit_url, data=alert, headers=headers) + + logger.debug(f'submit_to_hermes response.status_code: {submit_response.status_code}') + logger.debug(f'submit_to_hermes response.text: {submit_response.text}') + + + def get(self, request, *args, **kwargs): + # TODO: update get method docstring + """ + Method that handles the GET requests for this view. Sets all other ``DataProduct``s to unfeatured in the + database, and sets the specified ``DataProduct`` to featured. Caches the featured image. Deletes previously + featured images from the cache. + """ + product_id = kwargs.get('pk', None) + product = DataProduct.objects.get(pk=product_id) + + logger.debug(f'Sharing data product: {product} of type: {product.data_product_type}') + if product.data_product_type == 'photometry': + data = PhotometryProcessor().process_data(product) + # data is a list of tuples: + # [ + # (datetime.datetime(2012, 2, 2, 1, 40, 47, 999986, + # tzinfo=), + # { + # 'magnitude': 15.582, + # 'filter': 'r', + # 'error': 0.005 + # } + # ) + # ] + for datum in data: + logger.debug(f'datum: {datum}') + + # Turn the data into JSON to send to the HERMES /submit endpoint + self.submit_to_hermes(data) + + + # TODO: if a target_id came in with the request then redirect to its TargetDetail page + # otherwise stay on the same DataProductsListView page + # TODO: should give user feedback about the success/failure of the publishing + + return redirect(reverse( + 'tom_targets:detail', + kwargs={'pk': request.GET.get('target_id')}) + ) + + class DataProductGroupDetailView(DetailView): """ View that handles the viewing of a specific ``DataProductGroup``. From 53aa87bc0bfa41ba2cb05feff52486d58f387745 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Sat, 24 Sep 2022 01:05:29 +0000 Subject: [PATCH 03/72] add Publish button to items in dataproduct_list occurances Publish routes to the new DataProductShareView. --- .../templates/tom_dataproducts/dataproduct_list.html | 7 ++++++- .../partials/dataproduct_list_for_target.html | 10 +++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html b/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html index 7292bc5a0..cc9e7af95 100644 --- a/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html +++ b/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html @@ -21,6 +21,7 @@ Observation Groups Type + Share Thumbnail @@ -43,7 +44,11 @@ {% if product.data_product_type %} {{ product.get_type_display }} {% endif %} - + + + Publish + {% if product.get_file_extension == '.fz' or product.get_file_extension == '.fits' %} {% if product.get_preview %} diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html index b6c3d3295..301887bad 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html @@ -2,7 +2,14 @@ {% include 'tom_dataproducts/partials/js9_scripts.html' %}

Data

- + + + + + + + + {% for product in products %} @@ -23,6 +30,7 @@

Data

{% endif %} + {% endfor %} From a5697030776652fd541971c85798160eddf9040f Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Sat, 24 Sep 2022 01:26:25 +0000 Subject: [PATCH 04/72] flake8 compliance --- tom_dataproducts/urls.py | 3 ++- tom_dataproducts/views.py | 32 +++++++++++++++----------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/tom_dataproducts/urls.py b/tom_dataproducts/urls.py index 198cd53da..be0e0bc93 100644 --- a/tom_dataproducts/urls.py +++ b/tom_dataproducts/urls.py @@ -3,7 +3,8 @@ from tom_dataproducts.views import DataProductListView, DataProductSaveView, DataProductGroupListView from tom_dataproducts.views import DataProductDeleteView, DataProductGroupCreateView from tom_dataproducts.views import DataProductGroupDetailView, DataProductGroupDataView, DataProductGroupDeleteView -from tom_dataproducts.views import DataProductUploadView, DataProductFeatureView, UpdateReducedDataView, DataProductShareView +from tom_dataproducts.views import DataProductUploadView, DataProductFeatureView, UpdateReducedDataView +from tom_dataproducts.views import DataProductShareView from tom_common.api_router import SharedAPIRootRouter from tom_dataproducts.api_views import DataProductViewSet diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index df80ea5ab..b9f94b466 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -38,6 +38,7 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) + class DataProductSaveView(LoginRequiredMixin, View): """ View that handles saving a ``DataProduct`` generated by an observation. Requires authentication. @@ -294,27 +295,25 @@ class DataProductShareView(View): # configuration in settings.py def submit_to_hermes(self, data_to_share, message='From TOMToolkit'): hermes_base_url = settings.DATA_SHARING['hermes']['BASE_URL'] - - # Get the csrf-token to include in header - #csrf_url = hermes_base_url + 'get-csrf-token/' - #csrf_headers = {'Content-Type': 'application/json'} - #csrf_response = requests.get(url=csrf_url, headers=csrf_headers) - #logger.debug(f'dir(csrf_response): {dir(csrf_response)}') - #logger.debug(f'csrf_response.text: {csrf_response.text}') - #logger.debug(f'csrf_response.json(): {csrf_response.json()}') + # Get the csrf-token to include in header + # csrf_url = hermes_base_url + 'get-csrf-token/' + # csrf_headers = {'Content-Type': 'application/json'} + # csrf_response = requests.get(url=csrf_url, headers=csrf_headers) + # logger.debug(f'dir(csrf_response): {dir(csrf_response)}') + # logger.debug(f'csrf_response.text: {csrf_response.text}') + # logger.debug(f'csrf_response.json(): {csrf_response.json()}') - csrf_token = '' - #csrf_token = csrf_response.json()['token'] + # csrf_token = '' + # csrf_token = csrf_response.json()['token'] submit_url = hermes_base_url + 'submit/' headers = { - #'X-CSRFToken': csrf_token, - #'Content-Type': 'application/json', + # 'X-CSRFToken': csrf_token, + # 'Content-Type': 'application/json', } # fields required by hopingest.py: topic, title, author, data, message_text - data = data_to_share alert = { 'topic': 'hermes.test', 'title': 'TOM Toolkit test (Photometry)', @@ -328,14 +327,14 @@ def submit_to_hermes(self, data_to_share, message='From TOMToolkit'): logger.debug(f'submit_to_hermes response.status_code: {submit_response.status_code}') logger.debug(f'submit_to_hermes response.text: {submit_response.text}') - def get(self, request, *args, **kwargs): - # TODO: update get method docstring """ Method that handles the GET requests for this view. Sets all other ``DataProduct``s to unfeatured in the database, and sets the specified ``DataProduct`` to featured. Caches the featured image. Deletes previously featured images from the cache. """ + # TODO: update get method docstring + product_id = kwargs.get('pk', None) product = DataProduct.objects.get(pk=product_id) @@ -359,10 +358,9 @@ def get(self, request, *args, **kwargs): # Turn the data into JSON to send to the HERMES /submit endpoint self.submit_to_hermes(data) - # TODO: if a target_id came in with the request then redirect to its TargetDetail page # otherwise stay on the same DataProductsListView page - # TODO: should give user feedback about the success/failure of the publishing + # TODO: should give user feedback about the success/failure of the publishing return redirect(reverse( 'tom_targets:detail', From 72a708445351a9134386a6555528e310d0b94626 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 26 Sep 2022 22:28:28 +0000 Subject: [PATCH 05/72] correctly formatting for hermes /submit endpoint --- tom_dataproducts/views.py | 52 ++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index b9f94b466..c16c1483a 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -1,3 +1,4 @@ +import csv import datetime from io import StringIO import logging @@ -293,7 +294,7 @@ class DataProductShareView(View): # TODO: refactor the general data shaing mechanism to make it more driven by the # configuration in settings.py - def submit_to_hermes(self, data_to_share, message='From TOMToolkit'): + def submit_to_hermes(self, target_name, photometry_data, message='From TOMToolkit'): hermes_base_url = settings.DATA_SHARING['hermes']['BASE_URL'] # Get the csrf-token to include in header @@ -304,7 +305,6 @@ def submit_to_hermes(self, data_to_share, message='From TOMToolkit'): # logger.debug(f'csrf_response.text: {csrf_response.text}') # logger.debug(f'csrf_response.json(): {csrf_response.json()}') - # csrf_token = '' # csrf_token = csrf_response.json()['token'] submit_url = hermes_base_url + 'submit/' @@ -313,25 +313,41 @@ def submit_to_hermes(self, data_to_share, message='From TOMToolkit'): # 'Content-Type': 'application/json', } + # + # Map TOM Toolkit Photometry.csv fields to HERMES Photometry reporting form fields + # + hermes_photometry_data = [] + for tomtoolkit_photometry in photometry_data: + hermes_photometry_data.append({ + 'photometryId': target_name, + 'dateObs': tomtoolkit_photometry['time'], + 'band': tomtoolkit_photometry['filter'], + 'brightness': tomtoolkit_photometry['magnitude'], + 'brightnessError': tomtoolkit_photometry['error'], + 'brightnessUnit': 'AB mag', + }) + # fields required by hopingest.py: topic, title, author, data, message_text + # TODO: maybe throw up form to get these fields alert = { 'topic': 'hermes.test', 'title': 'TOM Toolkit test (Photometry)', 'author': 'llindstrom@lco.global', - 'data': data_to_share, + 'data': { + 'photometry_data': hermes_photometry_data, + }, 'message_text': f'Test alert from TOM Toolkit at {datetime.datetime.now()}', } + logger.debug(f'DataProductShareView.submit_to_hermes() alert: {alert}') - submit_response = requests.post(url=submit_url, data=alert, headers=headers) - - logger.debug(f'submit_to_hermes response.status_code: {submit_response.status_code}') - logger.debug(f'submit_to_hermes response.text: {submit_response.text}') + submit_response = requests.post(url=submit_url, json=alert, headers=headers) + logger.debug(f'DataProductShareView.submit_to_hermes response.status_code: {submit_response.status_code}') + logger.debug(f'DataProductShareView.submit_to_hermes response.text: {submit_response.text}') def get(self, request, *args, **kwargs): """ - Method that handles the GET requests for this view. Sets all other ``DataProduct``s to unfeatured in the - database, and sets the specified ``DataProduct`` to featured. Caches the featured image. Deletes previously - featured images from the cache. + Method that handles the GET requests for this view. + """ # TODO: update get method docstring @@ -340,8 +356,11 @@ def get(self, request, *args, **kwargs): logger.debug(f'Sharing data product: {product} of type: {product.data_product_type}') if product.data_product_type == 'photometry': - data = PhotometryProcessor().process_data(product) - # data is a list of tuples: + #data = PhotometryProcessor().process_data(product) + # NOTE: for PhotemeryProcessor + # * CSV headers assummed: time, magnitude, filter, error + # * time assumed to be astropy.time.Time (format='mjd') + # * result of process_data() is a list of tuples: # [ # (datetime.datetime(2012, 2, 2, 1, 40, 47, 999986, # tzinfo=), @@ -352,11 +371,14 @@ def get(self, request, *args, **kwargs): # } # ) # ] - for datum in data: - logger.debug(f'datum: {datum}') + # + # Alternatively just convert CSV in to python dict using csv.DictReader: + with open(product.data.path, newline='') as csvfile: + photometry_reader = csv.DictReader(csvfile, delimiter=',') + data = [row for row in photometry_reader] # Turn the data into JSON to send to the HERMES /submit endpoint - self.submit_to_hermes(data) + self.submit_to_hermes(product.target.name, data) # TODO: if a target_id came in with the request then redirect to its TargetDetail page # otherwise stay on the same DataProductsListView page From 3efa0c4a0a2c02fc183400f1cfa9f92ab85849a3 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 27 Sep 2022 16:50:20 +0000 Subject: [PATCH 06/72] be more specific with DATA_SHARING configuration keys --- tom_base/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index 444d0c054..d69b7b5dd 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -225,7 +225,8 @@ 'BASE_URL': os.getenv('HERMES_BASE_URL', 'http://hermes-dev.lco.gtn/'), 'API_TOKEN': os.getenv('HERMES_API_TOKEN', 'set HERMES_API_TOKEN value in environment'), }, - 'tomtoolkit': { + 'tom-demo-dev': { + # configuration for the TOM receiving data from this TOM 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), 'API_TOKEN': os.getenv('TOM_DEMO_API_TOKEN', 'set TOM_DEMO_API_TOKEN value in environment'), } From 26e6f044882130bcdcadcc1cd187945be996145a Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 27 Sep 2022 16:50:58 +0000 Subject: [PATCH 07/72] preparation for TOM-TOM data sharing via REST API --- tom_dataproducts/views.py | 42 ++++++++++++++------------------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index c16c1483a..b5dbc1901 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -30,7 +30,6 @@ from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm from tom_dataproducts.filters import DataProductFilter from tom_dataproducts.data_processor import run_data_processor -from tom_dataproducts.processors.photometry_processor import PhotometryProcessor from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class @@ -294,8 +293,8 @@ class DataProductShareView(View): # TODO: refactor the general data shaing mechanism to make it more driven by the # configuration in settings.py - def submit_to_hermes(self, target_name, photometry_data, message='From TOMToolkit'): - hermes_base_url = settings.DATA_SHARING['hermes']['BASE_URL'] + def submit_to_stream(self, stream_name, target_name, photometry_data, message='From TOMToolkit'): + hermes_base_url = settings.DATA_SHARING[stream_name]['BASE_URL'] # Get the csrf-token to include in header # csrf_url = hermes_base_url + 'get-csrf-token/' @@ -344,10 +343,17 @@ def submit_to_hermes(self, target_name, photometry_data, message='From TOMToolki logger.debug(f'DataProductShareView.submit_to_hermes response.status_code: {submit_response.status_code}') logger.debug(f'DataProductShareView.submit_to_hermes response.text: {submit_response.text}') + def share_with_tom(self, tom_name, target_name, photometry_data): + logger.debug(f'DataProductShareView.share_with_tom: {tom_name}') + tom_base_url = settings.DATA_SHARING[tom_name]['BASE_URL'] + + logger.debug(f'DataProductShareView.share_with_tom: {tom_name} at {tom_base_url}') + pass + def get(self, request, *args, **kwargs): """ - Method that handles the GET requests for this view. - + Method that handles the GET requests for this view. + """ # TODO: update get method docstring @@ -356,33 +362,15 @@ def get(self, request, *args, **kwargs): logger.debug(f'Sharing data product: {product} of type: {product.data_product_type}') if product.data_product_type == 'photometry': - #data = PhotometryProcessor().process_data(product) - # NOTE: for PhotemeryProcessor - # * CSV headers assummed: time, magnitude, filter, error - # * time assumed to be astropy.time.Time (format='mjd') - # * result of process_data() is a list of tuples: - # [ - # (datetime.datetime(2012, 2, 2, 1, 40, 47, 999986, - # tzinfo=), - # { - # 'magnitude': 15.582, - # 'filter': 'r', - # 'error': 0.005 - # } - # ) - # ] - # - # Alternatively just convert CSV in to python dict using csv.DictReader: + # Convert CSV into python dict with csv.DictReader: with open(product.data.path, newline='') as csvfile: photometry_reader = csv.DictReader(csvfile, delimiter=',') data = [row for row in photometry_reader] # Turn the data into JSON to send to the HERMES /submit endpoint - self.submit_to_hermes(product.target.name, data) - - # TODO: if a target_id came in with the request then redirect to its TargetDetail page - # otherwise stay on the same DataProductsListView page - # TODO: should give user feedback about the success/failure of the publishing + # TODO: rename these photometry-specfic methods to reflect that.. + self.submit_to_stream('hermes', product.target.name, data) + self.share_with_tom('tom-demo-dev', product.target.name, data) return redirect(reverse( 'tom_targets:detail', From e8eb592ec6f90bc777f50a94385da56489becd8c Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 3 Oct 2022 19:28:25 +0000 Subject: [PATCH 08/72] working draft of TOM-TOM sharing in share_with_tom() This is a draft version and works with TOM Toolkit-based TOMs --- tom_base/settings.py | 12 +++++++-- tom_dataproducts/views.py | 57 ++++++++++++++++++++++++++++++++++----- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index d69b7b5dd..0280fcc50 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -26,6 +26,10 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True +DEBUG = False + + + ALLOWED_HOSTS = [''] @@ -227,8 +231,12 @@ }, 'tom-demo-dev': { # configuration for the TOM receiving data from this TOM - 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), - 'API_TOKEN': os.getenv('TOM_DEMO_API_TOKEN', 'set TOM_DEMO_API_TOKEN value in environment'), + #'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), + 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://127.0.0.1:8000/'), # for testing share with yourself + # TODO: explain authentication mechanisms to TOM impolementers + #'API_TOKEN': os.getenv('TOM_DEMO_API_TOKEN', 'set TOM_DEMO_API_TOKEN value in environment'), + 'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'), + 'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'), } } diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index b5dbc1901..7dfac3bf4 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -2,6 +2,7 @@ import datetime from io import StringIO import logging +import os from urllib.parse import urlencode, urlparse from django.conf import settings @@ -10,6 +11,7 @@ from django.contrib.auth.models import Group from django.core.cache import cache from django.core.cache.utils import make_template_fragment_key +from django.core.exceptions import ImproperlyConfigured from django.core.management import call_command from django.http import HttpResponseRedirect from django.shortcuts import redirect @@ -293,6 +295,8 @@ class DataProductShareView(View): # TODO: refactor the general data shaing mechanism to make it more driven by the # configuration in settings.py + # TODO: consider passing DataProduct instance to submit_ and _share methods + # and refactor data manipulation to _helper methods. def submit_to_stream(self, stream_name, target_name, photometry_data, message='From TOMToolkit'): hermes_base_url = settings.DATA_SHARING[stream_name]['BASE_URL'] @@ -343,12 +347,48 @@ def submit_to_stream(self, stream_name, target_name, photometry_data, message='F logger.debug(f'DataProductShareView.submit_to_hermes response.status_code: {submit_response.status_code}') logger.debug(f'DataProductShareView.submit_to_hermes response.text: {submit_response.text}') - def share_with_tom(self, tom_name, target_name, photometry_data): - logger.debug(f'DataProductShareView.share_with_tom: {tom_name}') - tom_base_url = settings.DATA_SHARING[tom_name]['BASE_URL'] + def share_with_tom(self, tom_name, product: DataProduct): + """Construct and make a POST (create) request to the destination TOM /api/dataproducts/ endpoint. + + Theoritically, we should be able to simply serializer the DataProduct instance with the + DataProductSerializer (producing native python data types) and JSONRenderer().render() that + (producing JSON) and POST that to the destination TOM DRF API endpoint. + + * tom_name is the key in the settings.DATA_SHARING configuration dictionary + * product is the DataProduct instance to share + """ + #logger.debug(f'DataProductShareView.share_with_tom: DATA_SHARING key: {tom_name}') + try: + destination_tom_base_url = settings.DATA_SHARING[tom_name]['BASE_URL'] + username = settings.DATA_SHARING[tom_name]['USERNAME'] + password = settings.DATA_SHARING[tom_name]['PASSWORD'] + except KeyError as err: + raise ImproperlyConfigured(f'Please check DATA_SHARING configuration dictiionary for {tom_name}: Key {err} not found.') + + # TODO: query destination TOM for it's target.id for this target + targets_url = destination_tom_base_url + 'api/targets/' + destination_tom_target_id = 1 + # TODO: what if this target doesn't exist for the destination TOM? + + data_products_url = destination_tom_base_url + 'api/dataproducts/' + + # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage + dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) + with open(dataproduct_filename, 'rb') as dataproduct_filep: + files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} + data = { + 'target': destination_tom_target_id, + 'data_product_type': product.data_product_type + } + headers = {'Media-Type': 'multipart/form-data'} + auth= (username, password) + logger.debug(f'DataProductShareView.share_with_tom auth: {auth}') + response = requests.post(data_products_url, data=data, files=files, headers=headers, auth=auth) + + logger.debug(f'DataProductShareView.share_with_tom response.status_code: {response.status_code}') + logger.debug(f'DataProductShareView.share_with_tom response.text: {response.text}') + - logger.debug(f'DataProductShareView.share_with_tom: {tom_name} at {tom_base_url}') - pass def get(self, request, *args, **kwargs): """ @@ -369,8 +409,11 @@ def get(self, request, *args, **kwargs): # Turn the data into JSON to send to the HERMES /submit endpoint # TODO: rename these photometry-specfic methods to reflect that.. - self.submit_to_stream('hermes', product.target.name, data) - self.share_with_tom('tom-demo-dev', product.target.name, data) + # TODO: sort out where to share to (perhaps template info or FORM data?) + + #self.submit_to_stream('hermes', product.target.name, data) + + self.share_with_tom('tom-demo-dev', product) return redirect(reverse( 'tom_targets:detail', From a055dd2220bc033359544a51586f5d13a6e95e8c Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 4 Oct 2022 00:29:40 +0000 Subject: [PATCH 09/72] query destination TOM for Target PK and handle responses --- tom_dataproducts/views.py | 42 +++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 7dfac3bf4..217ad0146 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -357,19 +357,51 @@ def share_with_tom(self, tom_name, product: DataProduct): * tom_name is the key in the settings.DATA_SHARING configuration dictionary * product is the DataProduct instance to share """ - #logger.debug(f'DataProductShareView.share_with_tom: DATA_SHARING key: {tom_name}') try: destination_tom_base_url = settings.DATA_SHARING[tom_name]['BASE_URL'] username = settings.DATA_SHARING[tom_name]['USERNAME'] password = settings.DATA_SHARING[tom_name]['PASSWORD'] except KeyError as err: raise ImproperlyConfigured(f'Please check DATA_SHARING configuration dictiionary for {tom_name}: Key {err} not found.') + auth= (username, password) + target_name = product.target.name - # TODO: query destination TOM for it's target.id for this target + # + # Get this target's PK from the destination TOM + # targets_url = destination_tom_base_url + 'api/targets/' - destination_tom_target_id = 1 - # TODO: what if this target doesn't exist for the destination TOM? + target_params = {'name': target_name} + response = requests.get(targets_url, auth=auth, params=target_params) + + target_response = response.json() + if target_response['count'] == 1: + destination_tom_target_id = target_response['results'][0]['id'] + # TODO: handle target groups correctly + elif target_response['count'] == 0: + # Target not found in destination tom + logger.warning( + f'DataProductShareView.share_with_tom Target {target_name} not found on {tom_name}. ' + f'If target {target_name} does exist on the destination TOM, then this may an ' + f'authentication problem preventing access to the targets on {tom_name}.' + ) + # TODO: post message to UI + return # NOTE: early exit + elif target_response['count'] > 1: + # More than one target found; Target name must be amibiguous + msg = ( + f'Target name must be unique on destination TOM {tom_name}. ' + f'The following targets share a name or alias with {target_name}:\n' + ) + for target in target_response['results']: + aliases = ', '.join([alias['name'] for alias in target['aliases']]) # alias1, alias2, alias, + msg += f' Target: {target["name"]} Aliases: {aliases}\n' + logger.warning(msg) + # TODO: post message to UI + return # NOTE: early exit + # + # Now POST the DataProduct to the destination TOM + # data_products_url = destination_tom_base_url + 'api/dataproducts/' # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage @@ -381,8 +413,6 @@ def share_with_tom(self, tom_name, product: DataProduct): 'data_product_type': product.data_product_type } headers = {'Media-Type': 'multipart/form-data'} - auth= (username, password) - logger.debug(f'DataProductShareView.share_with_tom auth: {auth}') response = requests.post(data_products_url, data=data, files=files, headers=headers, auth=auth) logger.debug(f'DataProductShareView.share_with_tom response.status_code: {response.status_code}') From b4c7257423e305e6eec2b1e5b62f69fbbbb000ea Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Wed, 5 Oct 2022 20:46:23 +0000 Subject: [PATCH 10/72] clarify comment --- tom_dataproducts/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 217ad0146..d8299d2b2 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -367,7 +367,7 @@ def share_with_tom(self, tom_name, product: DataProduct): target_name = product.target.name # - # Get this target's PK from the destination TOM + # Get this DataProduct's target's PK from the destination TOM # targets_url = destination_tom_base_url + 'api/targets/' target_params = {'name': target_name} From 4bdbecb4857a70b09c066aa6d0816c013fe19bc8 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Wed, 5 Oct 2022 22:33:04 +0000 Subject: [PATCH 11/72] undoing this change that shouldn't have been commited --- tom_base/settings.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index 0280fcc50..96d7d838c 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -26,10 +26,6 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -DEBUG = False - - - ALLOWED_HOSTS = [''] From 4d8dff340a5dc4caedd7e9b3d57c583f305dde06 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Wed, 5 Oct 2022 22:33:34 +0000 Subject: [PATCH 12/72] expand DATA_SHARING options and clean up conifg dict --- tom_base/settings.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index 96d7d838c..180258e7d 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -220,23 +220,25 @@ 'spectroscopy': 'tom_dataproducts.processors.spectroscopy_processor.SpectroscopyProcessor', } +# Configuration for the TOM receiving data from this TOM DATA_SHARING = { 'hermes': { - 'BASE_URL': os.getenv('HERMES_BASE_URL', 'http://hermes-dev.lco.gtn/'), + 'BASE_URL': os.getenv('HERMES_BASE_URL', 'http://hermes.lco.global/'), 'API_TOKEN': os.getenv('HERMES_API_TOKEN', 'set HERMES_API_TOKEN value in environment'), }, 'tom-demo-dev': { - # configuration for the TOM receiving data from this TOM - #'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), - 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://127.0.0.1:8000/'), # for testing share with yourself - # TODO: explain authentication mechanisms to TOM impolementers - #'API_TOKEN': os.getenv('TOM_DEMO_API_TOKEN', 'set TOM_DEMO_API_TOKEN value in environment'), + 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), 'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'), 'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'), + }, + 'localhost-tom': { + # for testing; share with yourself + 'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'), + 'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'), + 'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'), } } - TOM_CADENCE_STRATEGIES = [ 'tom_observations.cadences.retry_failed_observations.RetryFailedObservationsStrategy', 'tom_observations.cadences.resume_cadence_after_failure.ResumeCadenceAfterFailureStrategy' From f0eb86c40e4543a6529e473cafc76b3e89437e24 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Wed, 5 Oct 2022 22:34:24 +0000 Subject: [PATCH 13/72] mainly flake8 compliance; also stub out sharing_destination logic submit_to_tom and submit_to_stream should probably be specified in the DATA_SHARING configuration dict --- tom_dataproducts/views.py | 41 +++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index d8299d2b2..56dace222 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -362,8 +362,8 @@ def share_with_tom(self, tom_name, product: DataProduct): username = settings.DATA_SHARING[tom_name]['USERNAME'] password = settings.DATA_SHARING[tom_name]['PASSWORD'] except KeyError as err: - raise ImproperlyConfigured(f'Please check DATA_SHARING configuration dictiionary for {tom_name}: Key {err} not found.') - auth= (username, password) + raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {tom_name}: Key {err} not found.') + auth = (username, password) target_name = product.target.name # @@ -387,13 +387,13 @@ def share_with_tom(self, tom_name, product: DataProduct): # TODO: post message to UI return # NOTE: early exit elif target_response['count'] > 1: - # More than one target found; Target name must be amibiguous + # More than one target found; Target name must be amibiguous msg = ( f'Target name must be unique on destination TOM {tom_name}. ' f'The following targets share a name or alias with {target_name}:\n' ) for target in target_response['results']: - aliases = ', '.join([alias['name'] for alias in target['aliases']]) # alias1, alias2, alias, + aliases = ', '.join([alias['name'] for alias in target['aliases']]) # alias1, alias2, alias msg += f' Target: {target["name"]} Aliases: {aliases}\n' logger.warning(msg) # TODO: post message to UI @@ -404,7 +404,7 @@ def share_with_tom(self, tom_name, product: DataProduct): # data_products_url = destination_tom_base_url + 'api/dataproducts/' - # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage + # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) with open(dataproduct_filename, 'rb') as dataproduct_filep: files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} @@ -418,8 +418,6 @@ def share_with_tom(self, tom_name, product: DataProduct): logger.debug(f'DataProductShareView.share_with_tom response.status_code: {response.status_code}') logger.debug(f'DataProductShareView.share_with_tom response.text: {response.text}') - - def get(self, request, *args, **kwargs): """ Method that handles the GET requests for this view. @@ -432,18 +430,23 @@ def get(self, request, *args, **kwargs): logger.debug(f'Sharing data product: {product} of type: {product.data_product_type}') if product.data_product_type == 'photometry': - # Convert CSV into python dict with csv.DictReader: - with open(product.data.path, newline='') as csvfile: - photometry_reader = csv.DictReader(csvfile, delimiter=',') - data = [row for row in photometry_reader] - - # Turn the data into JSON to send to the HERMES /submit endpoint - # TODO: rename these photometry-specfic methods to reflect that.. - # TODO: sort out where to share to (perhaps template info or FORM data?) - - #self.submit_to_stream('hermes', product.target.name, data) - - self.share_with_tom('tom-demo-dev', product) + # TODO: get DATA_SHARING config dict key from UI via kwargs or ??? + sharing_destination = 'hermes' + sharing_destination = 'localhost-tom' + if sharing_destination == 'hermes': + # Convert CSV into python dict with csv.DictReader: + with open(product.data.path, newline='') as csvfile: + photometry_reader = csv.DictReader(csvfile, delimiter=',') + data = [row for row in photometry_reader] + + # Turn the data into JSON to send to the HERMES /submit endpoint + # TODO: rename these photometry-specfic methods to reflect that.. + # TODO: sort out where to share to (perhaps template info or FORM data?) + + # TODO: pass product to submit method, open path, and get data there + self.submit_to_stream(sharing_destination, product.target.name, data) + else: + self.share_with_tom(sharing_destination, product) return redirect(reverse( 'tom_targets:detail', From 318e6c5c788147a729ca2128847a100c5ad44c7d Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Sat, 8 Oct 2022 01:34:06 +0000 Subject: [PATCH 14/72] if DATA_SHARING is not configured, don't offer to share data A list of data sharing destinations from settings.DATA_SHARING.keys() should be in the context passed to the templates listing DataProducts. If sharing_destinatinos evaluates to False the the Sharing UI should not be presented. --- .../templates/tom_dataproducts/dataproduct_list.html | 10 +++++++--- .../partials/dataproduct_list_for_target.html | 8 +++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html b/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html index cc9e7af95..19328358b 100644 --- a/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html +++ b/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html @@ -46,9 +46,13 @@ {% endif %} - + {% if sharing_destinations %} + Publish + {% else %} +

Not Configured {{sharing_destinations}}

+ {% endif %} + {% if product.get_file_extension == '.fz' or product.get_file_extension == '.fits' %} - + {% endfor %} From 2d8c99727a5ebf31318919befa603c95a2ef4bd0 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Sat, 8 Oct 2022 01:37:06 +0000 Subject: [PATCH 15/72] put sharing_destinations in the context for the DataProduct lists --- .../templatetags/dataproduct_extras.py | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index e1ba594ea..442552727 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -1,5 +1,7 @@ +import logging from urllib.parse import urlencode + from django import template from django.conf import settings from django.contrib.auth.models import Group @@ -20,8 +22,28 @@ from tom_observations.models import ObservationRecord from tom_targets.models import Target +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + register = template.Library() +def get_data_sharing_destinations(): + """ + Return a list of data sharing destinations from the DATA_SHARING configuration + dictionary in settings.py. + + This should be placed into the context of the inclusion tags that offer to share + DataProducts. Templates should know that None means that DATA_SHARING has not + been configured. + """ + try: + sharing_destinations = settings.DATA_SHARING.keys() + except Exception as ex: + logger.warning(f'{ex.__class__.__name__} while calling DATA_SHARING.keys(): {ex}') + sharing_destinations = None + + return sharing_destinations + @register.inclusion_tag('tom_dataproducts/partials/dataproduct_list_for_target.html', takes_context=True) def dataproduct_list_for_target(context, target): @@ -33,9 +55,11 @@ def dataproduct_list_for_target(context, target): else: target_products_for_user = get_objects_for_user( context['request'].user, 'tom_dataproducts.view_dataproduct', klass=target.dataproduct_set.all()) + return { 'products': target_products_for_user, - 'target': target + 'target': target, + 'sharing_destinations': get_data_sharing_destinations() } @@ -75,7 +99,11 @@ def dataproduct_list_all(context): products = DataProduct.objects.all().order_by('-created') else: products = get_objects_for_user(context['request'].user, 'tom_dataproducts.view_dataproduct') - return {'products': products} + + return { + 'products': products, + 'sharing_destinations': get_data_sharing_destinations() + } @register.inclusion_tag('tom_dataproducts/partials/upload_dataproduct.html', takes_context=True) From 905c4291ca48e574c9bd1fd1fcec4a9b636c1e2d Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Sat, 8 Oct 2022 01:37:06 +0000 Subject: [PATCH 16/72] put sharing_destinations in the context for the DataProduct lists --- .../templatetags/dataproduct_extras.py | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index e1ba594ea..2a230d38c 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -1,5 +1,7 @@ +import logging from urllib.parse import urlencode + from django import template from django.conf import settings from django.contrib.auth.models import Group @@ -20,9 +22,30 @@ from tom_observations.models import ObservationRecord from tom_targets.models import Target +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + register = template.Library() +def get_data_sharing_destinations(): + """ + Return a list of data sharing destinations from the DATA_SHARING configuration + dictionary in settings.py. + + This should be placed into the context of the inclusion tags that offer to share + DataProducts. Templates should know that None means that DATA_SHARING has not + been configured. + """ + try: + sharing_destinations = settings.DATA_SHARING.keys() + except Exception as ex: + logger.warning(f'{ex.__class__.__name__} while calling DATA_SHARING.keys(): {ex}') + sharing_destinations = None + + return sharing_destinations + + @register.inclusion_tag('tom_dataproducts/partials/dataproduct_list_for_target.html', takes_context=True) def dataproduct_list_for_target(context, target): """ @@ -33,9 +56,11 @@ def dataproduct_list_for_target(context, target): else: target_products_for_user = get_objects_for_user( context['request'].user, 'tom_dataproducts.view_dataproduct', klass=target.dataproduct_set.all()) + return { 'products': target_products_for_user, - 'target': target + 'target': target, + 'sharing_destinations': get_data_sharing_destinations() } @@ -75,7 +100,11 @@ def dataproduct_list_all(context): products = DataProduct.objects.all().order_by('-created') else: products = get_objects_for_user(context['request'].user, 'tom_dataproducts.view_dataproduct') - return {'products': products} + + return { + 'products': products, + 'sharing_destinations': get_data_sharing_destinations() + } @register.inclusion_tag('tom_dataproducts/partials/upload_dataproduct.html', takes_context=True) From 42298ee33d86c595c6a6534843cbb47be6b868c0 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 10 Oct 2022 22:20:12 +0000 Subject: [PATCH 17/72] wip: handle MARS alerts that don't have magnitude as value dict key 'limit'-type Alerts should be handled as soon as we know what they mean; awaiting an answer from an astronomer --- .../templatetags/dataproduct_extras.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 2a230d38c..f0ee5698b 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -132,7 +132,22 @@ def recent_photometry(target, limit=1): Displays a table of the most recent photometric points for a target. """ photometry = ReducedDatum.objects.filter(data_type='photometry', target=target).order_by('-timestamp')[:limit] - return {'data': [{'timestamp': rd.timestamp, 'magnitude': rd.value['magnitude']} for rd in photometry]} + + # Possibilities for reduced_datums from ZTF/MARS: + # reduced_datum.value: {'error': 0.0929680392146111, 'filter': 'r', 'magnitude': 18.2364940643311} + # reduced_datum.value: {'limit': 20.1023998260498, 'filter': 'g'} + + try: + # TODO: handle case where the value dict has no 'magnitude' key + # TODO: ask an Astronomyer about magnitude vs. limit ZTF alerts + context = {'data': [{'timestamp': rd.timestamp, 'magnitude': rd.value['magnitude']} for rd in photometry]} + except KeyError as err: + logger.error(f'KeyError: {err} not found in one or more ReducedDatum instances:') + for reduced_datum in photometry: + logger.error(f'reduced_datum: {reduced_datum}') + logger.error(f'reduced_datum.value: {reduced_datum.value}') + context = {} + return context @register.inclusion_tag('tom_dataproducts/partials/photometry_for_target.html', takes_context=True) From e211d0c4ebee4289a938ffda3ee3953d65b77d0d Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Mon, 10 Oct 2022 22:26:01 +0000 Subject: [PATCH 18/72] remove code that got duplicated during a recursive, non-ff merge --- .../templatetags/dataproduct_extras.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 5700d6b06..f0ee5698b 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -27,23 +27,6 @@ register = template.Library() -def get_data_sharing_destinations(): - """ - Return a list of data sharing destinations from the DATA_SHARING configuration - dictionary in settings.py. - - This should be placed into the context of the inclusion tags that offer to share - DataProducts. Templates should know that None means that DATA_SHARING has not - been configured. - """ - try: - sharing_destinations = settings.DATA_SHARING.keys() - except Exception as ex: - logger.warning(f'{ex.__class__.__name__} while calling DATA_SHARING.keys(): {ex}') - sharing_destinations = None - - return sharing_destinations - def get_data_sharing_destinations(): """ From 368cd05186beaa42363355dee25fc69bc8462e46 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 11 Oct 2022 00:17:17 +0000 Subject: [PATCH 19/72] Mock-up a (non-functional) sharing UI for gathering feedback --- .../tom_dataproducts/dataproduct_list.html | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html b/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html index 19328358b..0076aa06d 100644 --- a/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html +++ b/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html @@ -13,9 +13,36 @@ {% bootstrap_pagination page_obj extra=request.GET.urlencode %} + +
+
+
+
+ +
+
+ +
+
+
+
FilenameTypeDelete
FilenameTypeShareDelete
Publish Delete
- Publish {% if product.get_preview %} diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html index 301887bad..0b3ce15ee 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html @@ -30,7 +30,13 @@

Data

{% endif %} 		Publish + {% if sharing_destinations %} + Publish + {% else %} +

Not Configured

+ {% endif %} + 		Delete
+ @@ -28,6 +55,7 @@ {% for product in object_list %} + {% if product.observation_record.id %} From 2d80290a2fa31d1ddde62e3ee8dc1ea65e3bd975 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Tue, 11 Oct 2022 00:47:52 +0000 Subject: [PATCH 20/72] clean out previous experimental sharing UI header/column --- .../templates/tom_dataproducts/dataproduct_list.html | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html b/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html index 0076aa06d..c48688f47 100644 --- a/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html +++ b/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html @@ -27,7 +27,7 @@
- @@ -73,14 +72,6 @@ {{ product.get_type_display }} {% endif %} - {% if product.get_file_extension == '.fz' or product.get_file_extension == '.fits' %} - - {% endfor %}
File Target Observation
{{ product.get_file_name|truncatechars:40 }} {{ product.target.name|truncatechars:40 }}Observation Groups TypeShare Thumbnail
- {% if sharing_destinations %} - Publish - {% else %} -

Not Configured {{sharing_destinations}}

- {% endif %} - 		{% if product.get_preview %} From 1b53ead37ea24a40ed9289484321cca821212c06 Mon Sep 17 00:00:00 2001 From: Lindy Lindstrom Date: Wed, 12 Oct 2022 01:21:17 +0000 Subject: [PATCH 21/72] fix bug handling limit magnitudes in ZTF alert photometry --- .../partials/recent_photometry.html | 12 +++++++-- .../templatetags/dataproduct_extras.py | 26 ++++++++++++------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/recent_photometry.html b/tom_dataproducts/templates/tom_dataproducts/partials/recent_photometry.html index 80baa0c56..e69c40b28 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/recent_photometry.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/recent_photometry.html @@ -4,12 +4,20 @@ Recent Photometry - + + + + + {% for datum in data %} - + {% empty %} diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index f0ee5698b..63844c549 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -137,16 +137,22 @@ def recent_photometry(target, limit=1): # reduced_datum.value: {'error': 0.0929680392146111, 'filter': 'r', 'magnitude': 18.2364940643311} # reduced_datum.value: {'limit': 20.1023998260498, 'filter': 'g'} - try: - # TODO: handle case where the value dict has no 'magnitude' key - # TODO: ask an Astronomyer about magnitude vs. limit ZTF alerts - context = {'data': [{'timestamp': rd.timestamp, 'magnitude': rd.value['magnitude']} for rd in photometry]} - except KeyError as err: - logger.error(f'KeyError: {err} not found in one or more ReducedDatum instances:') - for reduced_datum in photometry: - logger.error(f'reduced_datum: {reduced_datum}') - logger.error(f'reduced_datum.value: {reduced_datum.value}') - context = {} + # for limit magnitudes, set the value of the limit key to True and + # the value of the magnitude key to the limit so the template and + # treat magnitudes as such and prepend a '>' to the limit magnitudes + # see recent_photometry.html + data = [] + for reduced_datum in photometry: + rd_data = {'timestamp': reduced_datum.timestamp} + if 'limit' in reduced_datum.value.keys(): + rd_data['magnitude'] = reduced_datum.value['limit'] + rd_data['limit'] = True + else: + rd_data['magnitude'] = reduced_datum.value['magnitude'] + rd_data['limit'] = False + data.append(rd_data) + + context = {'data': data} return context From 814c3bb9f810d742ce5cb4bdd2bc186e01e389ec Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Fri, 14 Oct 2022 11:21:03 -0700 Subject: [PATCH 22/72] add dropdown form --- tom_base/settings.py | 2 +- tom_dataproducts/forms.py | 11 +++++++++++ .../partials/dataproduct_list_for_target.html | 15 +++++++++++++-- .../templatetags/dataproduct_extras.py | 8 ++++++-- tom_dataproducts/views.py | 12 ++++++------ 5 files changed, 37 insertions(+), 11 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index c17dd88dc..8dbc40726 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -230,7 +230,7 @@ # Configuration for the TOM receiving data from this TOM DATA_SHARING = { 'hermes': { - 'BASE_URL': os.getenv('HERMES_BASE_URL', 'http://hermes.lco.global/'), + 'BASE_URL': os.getenv('HERMES_BASE_URL', 'https://hermes.lco.global/'), 'API_TOKEN': os.getenv('HERMES_API_TOKEN', 'set HERMES_API_TOKEN value in environment'), }, 'tom-demo-dev': { diff --git a/tom_dataproducts/forms.py b/tom_dataproducts/forms.py index 59a766004..5465fbc8d 100644 --- a/tom_dataproducts/forms.py +++ b/tom_dataproducts/forms.py @@ -7,6 +7,11 @@ from tom_targets.models import Target +DESTINATION_OPTIONS = (('hermes', 'Hermes'), + ('tom-demo-dev', '2nd best TOM'), + ('local_host', 'Best TOM (My TOM)')) + + class AddProductToGroupForm(forms.Form): products = forms.ModelMultipleChoiceField( DataProduct.objects.all(), @@ -46,3 +51,9 @@ def __init__(self, *args, **kwargs): self.fields['groups'] = forms.ModelMultipleChoiceField(Group.objects.none(), required=False, widget=forms.CheckboxSelectMultiple) + + +class DataProductShareForm(forms.Form): + share_destination = forms.ChoiceField(required=True, choices=DESTINATION_OPTIONS) + share_title = forms.CharField(required=False) + share_message = forms.CharField(required=False) diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html index 0b3ce15ee..50453a30a 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html @@ -28,16 +28,27 @@

Data

{% if product.data_product_type %} {{ product.get_type_display }} {% endif %} - + + + + {% endfor %}
TimestampMagnitude
TimestampMagnitude
{{ datum.timestamp }}{{ datum.magnitude|truncate_number }} + + {% if datum.limit %}>{% endif %} + {{ datum.magnitude|truncate_number }} +
{% if sharing_destinations %} - Publish + {% else %}

Not Configured

{% endif %} 		Delete
+
+ {% csrf_token %} + {% bootstrap_form data_product_share_form %} + {% buttons %} + + {% endbuttons %} +
+ 		Submit
diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 63844c549..ec570acd2 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -16,7 +16,7 @@ from PIL import Image, ImageDraw import base64 -from tom_dataproducts.forms import DataProductUploadForm +from tom_dataproducts.forms import DataProductUploadForm, DataProductShareForm from tom_dataproducts.models import DataProduct, ReducedDatum from tom_dataproducts.processors.data_serializers import SpectrumSerializer from tom_observations.models import ObservationRecord @@ -57,10 +57,14 @@ def dataproduct_list_for_target(context, target): target_products_for_user = get_objects_for_user( context['request'].user, 'tom_dataproducts.view_dataproduct', klass=target.dataproduct_set.all()) + initial = {} + form = DataProductShareForm(initial=initial) + return { 'products': target_products_for_user, 'target': target, - 'sharing_destinations': get_data_sharing_destinations() + 'sharing_destinations': get_data_sharing_destinations(), + 'data_product_share_form': form } diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 56dace222..008bee5b9 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -293,7 +293,7 @@ class DataProductShareView(View): ``TargetDetailView``. """ - # TODO: refactor the general data shaing mechanism to make it more driven by the + # TODO: refactor the general data sharing mechanism to make it more driven by the # configuration in settings.py # TODO: consider passing DataProduct instance to submit_ and _share methods # and refactor data manipulation to _helper methods. @@ -341,11 +341,11 @@ def submit_to_stream(self, stream_name, target_name, photometry_data, message='F }, 'message_text': f'Test alert from TOM Toolkit at {datetime.datetime.now()}', } - logger.debug(f'DataProductShareView.submit_to_hermes() alert: {alert}') + # logger.debug(f'DataProductShareView.submit_to_hermes() alert: {alert}') submit_response = requests.post(url=submit_url, json=alert, headers=headers) - logger.debug(f'DataProductShareView.submit_to_hermes response.status_code: {submit_response.status_code}') - logger.debug(f'DataProductShareView.submit_to_hermes response.text: {submit_response.text}') + # logger.debug(f'DataProductShareView.submit_to_hermes response.status_code: {submit_response.status_code}') + # logger.debug(f'DataProductShareView.submit_to_hermes response.text: {submit_response.text}') def share_with_tom(self, tom_name, product: DataProduct): """Construct and make a POST (create) request to the destination TOM /api/dataproducts/ endpoint. @@ -432,7 +432,7 @@ def get(self, request, *args, **kwargs): if product.data_product_type == 'photometry': # TODO: get DATA_SHARING config dict key from UI via kwargs or ??? sharing_destination = 'hermes' - sharing_destination = 'localhost-tom' + # sharing_destination = 'localhost-tom' if sharing_destination == 'hermes': # Convert CSV into python dict with csv.DictReader: with open(product.data.path, newline='') as csvfile: @@ -440,7 +440,7 @@ def get(self, request, *args, **kwargs): data = [row for row in photometry_reader] # Turn the data into JSON to send to the HERMES /submit endpoint - # TODO: rename these photometry-specfic methods to reflect that.. + # TODO: rename these photometry-specific methods to reflect that.. # TODO: sort out where to share to (perhaps template info or FORM data?) # TODO: pass product to submit method, open path, and get data there From 5f920b2e0997cb83e67411dc9b773b0e936c66f4 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 19 Oct 2022 11:10:20 -0700 Subject: [PATCH 23/72] add Dataproduct sharing form --- tom_dataproducts/forms.py | 16 ++++++++-- .../partials/dataproduct_list_for_target.html | 31 +++++++++++++------ .../templatetags/dataproduct_extras.py | 4 ++- tom_dataproducts/urls.py | 1 + tom_dataproducts/views.py | 29 +++++++++++++++-- 5 files changed, 66 insertions(+), 15 deletions(-) diff --git a/tom_dataproducts/forms.py b/tom_dataproducts/forms.py index 5465fbc8d..6fa779d03 100644 --- a/tom_dataproducts/forms.py +++ b/tom_dataproducts/forms.py @@ -1,6 +1,8 @@ from django import forms from django.contrib.auth.models import Group from django.conf import settings +from crispy_forms.helper import FormHelper +from crispy_forms.layout import HTML, Layout, Div, Fieldset, Row, Column from tom_dataproducts.models import DataProductGroup, DataProduct from tom_observations.models import ObservationRecord @@ -54,6 +56,14 @@ def __init__(self, *args, **kwargs): class DataProductShareForm(forms.Form): - share_destination = forms.ChoiceField(required=True, choices=DESTINATION_OPTIONS) - share_title = forms.CharField(required=False) - share_message = forms.CharField(required=False) + share_destination = forms.ChoiceField(required=True, choices=DESTINATION_OPTIONS, label="Destination") + share_title = forms.CharField(required=False, label="Title") + share_message = forms.CharField(required=False, label="Message", widget=forms.Textarea()) + share_authors = forms.CharField(required=False, widget=forms.HiddenInput()) + target = forms.ModelChoiceField( + Target.objects.all(), + widget=forms.HiddenInput(), + required=False) + submitter = forms.CharField( + widget=forms.HiddenInput() + ) diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html index 50453a30a..36d2616ef 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html @@ -39,16 +39,29 @@

Data

Delete
-
- {% csrf_token %} - {% bootstrap_form data_product_share_form %} - {% buttons %} - - {% endbuttons %} -
+ 		+
+ {% csrf_token %} +
+
+ {% bootstrap_field data_product_share_form.share_destination %} +
+
+ {% bootstrap_field data_product_share_form.share_title %} +
+
+
+
+ {% bootstrap_field data_product_share_form.share_message %} +
+
+ {% buttons %} + + {% endbuttons %} +
+
+
Submit
diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index ec570acd2..3d7a74e67 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -57,7 +57,9 @@ def dataproduct_list_for_target(context, target): target_products_for_user = get_objects_for_user( context['request'].user, 'tom_dataproducts.view_dataproduct', klass=target.dataproduct_set.all()) - initial = {} + initial = {'submitter': context['request'].user, + 'target': target, + 'share_title': f"Updated data for {target.name}."} form = DataProductShareForm(initial=initial) return { diff --git a/tom_dataproducts/urls.py b/tom_dataproducts/urls.py index be0e0bc93..5f807748f 100644 --- a/tom_dataproducts/urls.py +++ b/tom_dataproducts/urls.py @@ -25,6 +25,7 @@ path('data/reduced/update/', UpdateReducedDataView.as_view(), name='update-reduced-data'), path('data//delete/', DataProductDeleteView.as_view(), name='delete'), path('data//feature/', DataProductFeatureView.as_view(), name='feature'), + # path('data//share/', DataProductShareView.as_view(), name='share'), path('data//share/', DataProductShareView.as_view(), name='share'), path('/save/', DataProductSaveView.as_view(), name='save'), ] diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 008bee5b9..1bebd9a31 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -29,7 +29,7 @@ from tom_common.mixins import Raise403PermissionRequiredMixin from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum from tom_dataproducts.exceptions import InvalidFileFormatException -from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm +from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm, DataProductShareForm from tom_dataproducts.filters import DataProductFilter from tom_dataproducts.data_processor import run_data_processor from tom_observations.models import ObservationRecord @@ -286,13 +286,38 @@ def get(self, request, *args, **kwargs): ) -class DataProductShareView(View): +class DataProductShareView(FormView): # TODO: update class docstring """ View that handles the featuring of ``DataProduct``s. A featured ``DataProduct`` is displayed on the ``TargetDetailView``. """ + form_class = DataProductShareForm + + def get_form(self, *args, **kwargs): + form = super().get_form(*args, **kwargs) + return form + + def form_valid(self, form): + """ + Runs after ``DataProductUploadForm`` is validated. Saves each ``DataProduct`` and calls ``run_data_processor`` + on each saved file. Redirects to the previous page. + """ + + print(form.cleaned_data) + return redirect('/') + + def form_invalid(self, form): + """ + Adds errors to Django messaging framework in the case of an invalid form and redirects to the previous page. + """ + # TODO: Format error messages in a more human-readable way + messages.error(self.request, 'There was a problem sharing your Data: {}'.format(form.errors.as_json())) + return redirect(form.cleaned_data.get('referrer', '/')) + + +class DataProductShareViewOld(View): # TODO: refactor the general data sharing mechanism to make it more driven by the # configuration in settings.py # TODO: consider passing DataProduct instance to submit_ and _share methods From 85165d1008bf22b20895b77bb2bdf6ac51b30069 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 20 Oct 2022 13:39:45 -0700 Subject: [PATCH 24/72] get hermes submission working --- .../partials/dataproduct_list_for_target.html | 8 ++- tom_dataproducts/views.py | 68 ++++++++++++++++--- 2 files changed, 65 insertions(+), 11 deletions(-) diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html index 36d2616ef..d80310ea3 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html @@ -42,7 +42,11 @@

Data

{% csrf_token %} -
+ {% for hidden in data_product_share_form.hidden_fields %} + {{ hidden }} + {% endfor %} + {{ data_product_share_form.data_product.value }} +
{% bootstrap_field data_product_share_form.share_destination %}
@@ -50,7 +54,7 @@

Data

{% bootstrap_field data_product_share_form.share_title %}
-
+
{% bootstrap_field data_product_share_form.share_message %}
diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 1bebd9a31..dfb22b424 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -296,18 +296,10 @@ class DataProductShareView(FormView): form_class = DataProductShareForm def get_form(self, *args, **kwargs): + # TODO: Add permissions form = super().get_form(*args, **kwargs) return form - def form_valid(self, form): - """ - Runs after ``DataProductUploadForm`` is validated. Saves each ``DataProduct`` and calls ``run_data_processor`` - on each saved file. Redirects to the previous page. - """ - - print(form.cleaned_data) - return redirect('/') - def form_invalid(self, form): """ Adds errors to Django messaging framework in the case of an invalid form and redirects to the previous page. @@ -316,6 +308,64 @@ def form_invalid(self, form): messages.error(self.request, 'There was a problem sharing your Data: {}'.format(form.errors.as_json())) return redirect(form.cleaned_data.get('referrer', '/')) + def post(self, request, *args, **kwargs): + """ + Method that handles thePOST requests for this view. + """ + data_product_share_form = DataProductShareForm(request.POST, request.FILES) + if data_product_share_form.is_valid(): + form_data = data_product_share_form.cleaned_data + product_id = kwargs.get('pk', None) + product = DataProduct.objects.get(pk=product_id) + if product.data_product_type == 'photometry': + reduced_datums = ReducedDatum.objects.filter(data_product=product) + share_destination = form_data['share_destination'] + if share_destination == 'hermes': + # build hermes table from Reduced Datums + self.publish_photometry_to_stream(share_destination, form_data, reduced_datums) + else: + self.share_with_tom(share_destination, product) + return redirect('/') + + def publish_photometry_to_stream(self, destination, message_info, datums): + """ + For now this code submits a typical hermes photometry alert using the datums tied to the dataproduct being + shared. In the future this should instead send the user to a new tab with a populated hermes form. + :param destination: target stream (topic included?) + :param message_info: Dictionary of message information + :param datums: Reduced Datums to be built into table. + :return: + """ + stream_base_url = settings.DATA_SHARING[destination]['BASE_URL'] + submit_url = stream_base_url + 'submit/' + headers = {} + hermes_photometry_data = [] + for tomtoolkit_photometry in datums: + hermes_photometry_data.append({ + 'photometryId': tomtoolkit_photometry.target.name, + 'dateObs': tomtoolkit_photometry.timestamp.strftime('%x %X'), + 'band': tomtoolkit_photometry.value['filter'], + 'brightness': tomtoolkit_photometry.value['magnitude'], + 'brightnessError': tomtoolkit_photometry.value['error'], + 'brightnessUnit': 'AB mag', + }) + alert = { + 'topic': 'hermes.test', + 'title': message_info['share_title'], + 'author': message_info['submitter'], + '' + 'data': { + 'authors': message_info['share_authors'], + 'photometry_data': hermes_photometry_data, + }, + 'message_text': message_info['share_message'], + } + + submit_response = requests.post(url=submit_url, json=alert, headers=headers) + + def share_with_tom(self, destination, product): + print(destination, product) + class DataProductShareViewOld(View): # TODO: refactor the general data sharing mechanism to make it more driven by the From d158528e8c283fbcd602ce3cb41cea90647c8f78 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 20 Oct 2022 14:41:58 -0700 Subject: [PATCH 25/72] fix some typos --- tom_dataproducts/forms.py | 2 -- tom_dataproducts/urls.py | 1 - tom_dataproducts/views.py | 11 +++++++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tom_dataproducts/forms.py b/tom_dataproducts/forms.py index 6fa779d03..22a4952dd 100644 --- a/tom_dataproducts/forms.py +++ b/tom_dataproducts/forms.py @@ -1,8 +1,6 @@ from django import forms from django.contrib.auth.models import Group from django.conf import settings -from crispy_forms.helper import FormHelper -from crispy_forms.layout import HTML, Layout, Div, Fieldset, Row, Column from tom_dataproducts.models import DataProductGroup, DataProduct from tom_observations.models import ObservationRecord diff --git a/tom_dataproducts/urls.py b/tom_dataproducts/urls.py index 5f807748f..be0e0bc93 100644 --- a/tom_dataproducts/urls.py +++ b/tom_dataproducts/urls.py @@ -25,7 +25,6 @@ path('data/reduced/update/', UpdateReducedDataView.as_view(), name='update-reduced-data'), path('data//delete/', DataProductDeleteView.as_view(), name='delete'), path('data//feature/', DataProductFeatureView.as_view(), name='feature'), - # path('data//share/', DataProductShareView.as_view(), name='share'), path('data//share/', DataProductShareView.as_view(), name='share'), path('/save/', DataProductSaveView.as_view(), name='save'), ] diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index dfb22b424..3c2fbf9e7 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -361,9 +361,16 @@ def publish_photometry_to_stream(self, destination, message_info, datums): 'message_text': message_info['share_message'], } - submit_response = requests.post(url=submit_url, json=alert, headers=headers) + requests.post(url=submit_url, json=alert, headers=headers) def share_with_tom(self, destination, product): + """ + When sharing a DataProduct with another TOM we likely want to share the data product itself and let the other + TOM process it rather than share the Reduced Datums + :param destination: + :param product: + :return: + """ print(destination, product) @@ -418,7 +425,7 @@ def submit_to_stream(self, stream_name, target_name, photometry_data, message='F } # logger.debug(f'DataProductShareView.submit_to_hermes() alert: {alert}') - submit_response = requests.post(url=submit_url, json=alert, headers=headers) + requests.post(url=submit_url, json=alert, headers=headers) # logger.debug(f'DataProductShareView.submit_to_hermes response.status_code: {submit_response.status_code}') # logger.debug(f'DataProductShareView.submit_to_hermes response.text: {submit_response.text}') From 5964ea5844d976381cfae2798bc93313e5069569 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 25 Oct 2022 11:59:39 -0700 Subject: [PATCH 26/72] begin phot data table --- tom_dataproducts/templatetags/dataproduct_extras.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 3d7a74e67..bc85bc517 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -249,6 +249,12 @@ def photometry_for_target(context, target, width=700, height=600, background=Non fig = go.Figure(data=plot_data, layout=layout) fig.update_yaxes(showgrid=grid, color=label_color, showline=True, linecolor=label_color, mirror=True) fig.update_xaxes(showgrid=grid, color=label_color, showline=True, linecolor=label_color, mirror=True) + fig.update_layout(clickmode='event+select') + + # TODO: Build linked data table + tab = go.Table(header=dict(values=[]), + cells=dict(values=[])) + return { 'target': target, 'plot': offline.plot(fig, output_type='div', show_link=False) From bc51ba973e2b99d4c53a5cecf37ab992bda7cfa8 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 26 Oct 2022 14:39:36 -0700 Subject: [PATCH 27/72] begin work on tom-tom sharing --- tom_dataproducts/views.py | 23 +++++++++++++++++++---- tom_targets/serializers.py | 4 ++-- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 3c2fbf9e7..4cf46e0fe 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -23,10 +23,12 @@ from django.views.generic.edit import CreateView, DeleteView, FormView from django_filters.views import FilterView from guardian.shortcuts import assign_perm, get_objects_for_user +from rest_framework.renderers import JSONRenderer from tom_common.hooks import run_hook from tom_common.hints import add_hint from tom_common.mixins import Raise403PermissionRequiredMixin +from tom_targets.serializers import TargetSerializer from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum from tom_dataproducts.exceptions import InvalidFileFormatException from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm, DataProductShareForm @@ -363,15 +365,28 @@ def publish_photometry_to_stream(self, destination, message_info, datums): requests.post(url=submit_url, json=alert, headers=headers) - def share_with_tom(self, destination, product): + def share_with_tom(self, tom_name, product): """ When sharing a DataProduct with another TOM we likely want to share the data product itself and let the other TOM process it rather than share the Reduced Datums - :param destination: + :param tom_name: :param product: :return: """ - print(destination, product) + try: + destination_tom_base_url = settings.DATA_SHARING[tom_name]['BASE_URL'] + username = settings.DATA_SHARING[tom_name]['USERNAME'] + password = settings.DATA_SHARING[tom_name]['PASSWORD'] + except KeyError as err: + raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {tom_name}: Key {err} not found.') + auth = (username, password) + headers = {'Media-Type': 'application/json'} + target = product.target + serialized_target_data = TargetSerializer(target).data + target_json = JSONRenderer().render(serialized_target_data) + targets_url = destination_tom_base_url + 'api/targets/' + response = requests.post(targets_url, headers=headers, auth=auth, data=serialized_target_data) + print(response.text) class DataProductShareViewOld(View): @@ -432,7 +447,7 @@ def submit_to_stream(self, stream_name, target_name, photometry_data, message='F def share_with_tom(self, tom_name, product: DataProduct): """Construct and make a POST (create) request to the destination TOM /api/dataproducts/ endpoint. - Theoritically, we should be able to simply serializer the DataProduct instance with the + Theoretically, we should be able to simply serializer the DataProduct instance with the DataProductSerializer (producing native python data types) and JSONRenderer().render() that (producing JSON) and POST that to the destination TOM DRF API endpoint. diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py index 6a552f092..a39dec43c 100644 --- a/tom_targets/serializers.py +++ b/tom_targets/serializers.py @@ -28,8 +28,8 @@ class TargetSerializer(serializers.ModelSerializer): json (or other representations). See https://www.django-rest-framework.org/api-guide/serializers/#modelserializer """ - targetextra_set = TargetExtraSerializer(many=True) - aliases = TargetNameSerializer(many=True) + targetextra_set = TargetExtraSerializer(many=True, required=False) + aliases = TargetNameSerializer(many=True, required=False) groups = GroupSerializer(many=True, required=False) # TODO: return groups in detail and list class Meta: From 07ae2bf341f828a01b15a98e8277dbc0925785a5 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 26 Oct 2022 15:32:47 -0700 Subject: [PATCH 28/72] create message table --- tom_alerts/models.py | 47 ++++++++++++++++++++++++++++++++++++++ tom_dataproducts/models.py | 5 ++++ 2 files changed, 52 insertions(+) diff --git a/tom_alerts/models.py b/tom_alerts/models.py index 0acace442..6b97e8c5c 100644 --- a/tom_alerts/models.py +++ b/tom_alerts/models.py @@ -33,3 +33,50 @@ class BrokerQuery(models.Model): def __str__(self): return self.name + + +class AlertStreamMessage(models.Model): + """ + Class representing a streaming message containing data + :param topic: The destination or source of sharing for the message. + :type topic: str + + :param message_id: An external message identifier that can be used to locate the message within the given topic. + :type message_id: str + + :param date_shared: The date on which the message is shared. (Date created by default.) + :type date_shared: datetime + + :param exchange_status: Whether this message was sent or received. + :type exchange_status: str + """ + + EXCHANGE_STATUS_CHOICES = ( + ('published', 'Published'), + ('ingested', 'Ingested') + ) + + topic = models.CharField( + max_length=500, + verbose_name='Message Topic', + help_text='The destination or source of sharing for the message.' + ) + message_id = models.CharField( + max_length=50, + verbose_name='Message ID', + help_text='An external message identifier that can be used to locate the message within the given topic.' + ) + date_shared = models.DateTimeField( + auto_now_add=True, + verbose_name='Date Shared', + help_text='The date on which the message is shared. (Date created by default.)' + ) + exchange_status = models.CharField( + max_length=10, + verbose_name='Exchange Status', + choices=EXCHANGE_STATUS_CHOICES, + help_text='Whether this message was sent or received.' + ) + + def __str__(self): + return f'Message {self.message_id} on {self.topic}.' diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index 9ba5bd216..ae613894a 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -12,6 +12,7 @@ from PIL import Image from tom_targets.models import Target +from tom_alerts.models import AlertStreamMessage from tom_observations.models import ObservationRecord logger = logging.getLogger(__name__) @@ -322,6 +323,9 @@ class ReducedDatum(models.Model): } :type value: dict + :param message: Set of ``AlertStreamMessage`` objects this object is associated with. + :type message: ManyRelatedManager object + """ target = models.ForeignKey(Target, null=False, on_delete=models.CASCADE) @@ -334,6 +338,7 @@ class ReducedDatum(models.Model): source_location = models.CharField(max_length=200, default='') timestamp = models.DateTimeField(null=False, blank=False, default=datetime.now, db_index=True) value = models.JSONField(null=False, blank=False) + message = models.ManyToManyField(AlertStreamMessage) class Meta: get_latest_by = ('timestamp',) From 710f00475ee59518fdbb5bc122c4335487e3f3e2 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 27 Oct 2022 11:56:15 -0700 Subject: [PATCH 29/72] add migrations --- .../migrations/0005_alertstreammessage.py | 23 +++++++++++++++++++ ...lter_alertstreammessage_exchange_status.py | 18 +++++++++++++++ .../migrations/0011_reduceddatum_message.py | 19 +++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 tom_alerts/migrations/0005_alertstreammessage.py create mode 100644 tom_alerts/migrations/0006_alter_alertstreammessage_exchange_status.py create mode 100644 tom_dataproducts/migrations/0011_reduceddatum_message.py diff --git a/tom_alerts/migrations/0005_alertstreammessage.py b/tom_alerts/migrations/0005_alertstreammessage.py new file mode 100644 index 000000000..33c148cf7 --- /dev/null +++ b/tom_alerts/migrations/0005_alertstreammessage.py @@ -0,0 +1,23 @@ +# Generated by Django 4.1 on 2022-10-26 21:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tom_alerts', '0004_auto_20210204_2300'), + ] + + operations = [ + migrations.CreateModel( + name='AlertStreamMessage', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('topic', models.CharField(help_text='The destination or source of sharing for the message.', max_length=500, verbose_name='Message Topic')), + ('message_id', models.CharField(help_text='An external message identifier that can be used to locate the message within the given topic.', max_length=50, verbose_name='Message ID')), + ('date_shared', models.DateTimeField(auto_now_add=True, help_text='The date on which the message is shared. (Date created by default.)', verbose_name='Date Shared')), + ('exchange_status', models.CharField(default='', help_text='Whether this message was sent or received.', max_length=10, verbose_name='Exchange Status')), + ], + ), + ] diff --git a/tom_alerts/migrations/0006_alter_alertstreammessage_exchange_status.py b/tom_alerts/migrations/0006_alter_alertstreammessage_exchange_status.py new file mode 100644 index 000000000..688c604e7 --- /dev/null +++ b/tom_alerts/migrations/0006_alter_alertstreammessage_exchange_status.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1 on 2022-10-26 23:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tom_alerts', '0005_alertstreammessage'), + ] + + operations = [ + migrations.AlterField( + model_name='alertstreammessage', + name='exchange_status', + field=models.CharField(choices=[('published', 'Published'), ('ingested', 'Ingested')], help_text='Whether this message was sent or received.', max_length=10, verbose_name='Exchange Status'), + ), + ] diff --git a/tom_dataproducts/migrations/0011_reduceddatum_message.py b/tom_dataproducts/migrations/0011_reduceddatum_message.py new file mode 100644 index 000000000..246377019 --- /dev/null +++ b/tom_dataproducts/migrations/0011_reduceddatum_message.py @@ -0,0 +1,19 @@ +# Generated by Django 4.1 on 2022-10-26 21:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tom_alerts', '0005_alertstreammessage'), + ('tom_dataproducts', '0010_manual_20210305_fix_spectroscopy'), + ] + + operations = [ + migrations.AddField( + model_name='reduceddatum', + name='message', + field=models.ManyToManyField(to='tom_alerts.alertstreammessage'), + ), + ] From a47937d3f4be7e7d38efb4ac1f54705044131abd Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 27 Oct 2022 17:49:19 -0700 Subject: [PATCH 30/72] generalize hermes submission --- tom_dataproducts/hermes.py | 45 ++++++++++++++++++++++++++++++++++++++ tom_dataproducts/views.py | 41 +++------------------------------- 2 files changed, 48 insertions(+), 38 deletions(-) create mode 100644 tom_dataproducts/hermes.py diff --git a/tom_dataproducts/hermes.py b/tom_dataproducts/hermes.py new file mode 100644 index 000000000..6518eca0a --- /dev/null +++ b/tom_dataproducts/hermes.py @@ -0,0 +1,45 @@ +from django.conf import settings + +from tom_alerts.models import AlertStreamMessage + +import requests + + +def publish_photometry_to_hermes(destination, message_info, datums): + """ + For now this code submits a typical hermes photometry alert using the datums tied to the dataproduct being + shared. In the future this should instead send the user to a new tab with a populated hermes form. + :param destination: target stream (topic included?) + :param message_info: Dictionary of message information + :param datums: Reduced Datums to be built into table. + :return: + """ + stream_base_url = settings.DATA_SHARING[destination]['BASE_URL'] + submit_url = stream_base_url + 'submit/' + headers = {} + hermes_photometry_data = [] + hermes_alert = AlertStreamMessage(topic='hermes.test', exchange_status='published') + hermes_alert.save() + for tomtoolkit_photometry in datums: + tomtoolkit_photometry.message.add(hermes_alert) + hermes_photometry_data.append({ + 'photometryId': tomtoolkit_photometry.target.name, + 'dateObs': tomtoolkit_photometry.timestamp.strftime('%x %X'), + 'band': tomtoolkit_photometry.value['filter'], + 'brightness': tomtoolkit_photometry.value['magnitude'], + 'brightnessError': tomtoolkit_photometry.value['error'], + 'brightnessUnit': 'AB mag', + }) + alert = { + 'topic': 'hermes.test', + 'title': message_info['share_title'], + 'author': message_info['submitter'], + '' + 'data': { + 'authors': message_info['share_authors'], + 'photometry_data': hermes_photometry_data, + }, + 'message_text': message_info['share_message'], + } + + requests.post(url=submit_url, json=alert, headers=headers) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 4cf46e0fe..f8c930733 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -34,6 +34,7 @@ from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm, DataProductShareForm from tom_dataproducts.filters import DataProductFilter from tom_dataproducts.data_processor import run_data_processor +from tom_dataproducts.hermes import publish_photometry_to_hermes from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class @@ -324,47 +325,11 @@ def post(self, request, *args, **kwargs): share_destination = form_data['share_destination'] if share_destination == 'hermes': # build hermes table from Reduced Datums - self.publish_photometry_to_stream(share_destination, form_data, reduced_datums) + publish_photometry_to_hermes(share_destination, form_data, reduced_datums) else: self.share_with_tom(share_destination, product) return redirect('/') - def publish_photometry_to_stream(self, destination, message_info, datums): - """ - For now this code submits a typical hermes photometry alert using the datums tied to the dataproduct being - shared. In the future this should instead send the user to a new tab with a populated hermes form. - :param destination: target stream (topic included?) - :param message_info: Dictionary of message information - :param datums: Reduced Datums to be built into table. - :return: - """ - stream_base_url = settings.DATA_SHARING[destination]['BASE_URL'] - submit_url = stream_base_url + 'submit/' - headers = {} - hermes_photometry_data = [] - for tomtoolkit_photometry in datums: - hermes_photometry_data.append({ - 'photometryId': tomtoolkit_photometry.target.name, - 'dateObs': tomtoolkit_photometry.timestamp.strftime('%x %X'), - 'band': tomtoolkit_photometry.value['filter'], - 'brightness': tomtoolkit_photometry.value['magnitude'], - 'brightnessError': tomtoolkit_photometry.value['error'], - 'brightnessUnit': 'AB mag', - }) - alert = { - 'topic': 'hermes.test', - 'title': message_info['share_title'], - 'author': message_info['submitter'], - '' - 'data': { - 'authors': message_info['share_authors'], - 'photometry_data': hermes_photometry_data, - }, - 'message_text': message_info['share_message'], - } - - requests.post(url=submit_url, json=alert, headers=headers) - def share_with_tom(self, tom_name, product): """ When sharing a DataProduct with another TOM we likely want to share the data product itself and let the other @@ -383,8 +348,8 @@ def share_with_tom(self, tom_name, product): headers = {'Media-Type': 'application/json'} target = product.target serialized_target_data = TargetSerializer(target).data - target_json = JSONRenderer().render(serialized_target_data) targets_url = destination_tom_base_url + 'api/targets/' + # TODO: Make sure aliases are checked before creating new target response = requests.post(targets_url, headers=headers, auth=auth, data=serialized_target_data) print(response.text) From 6e52821c45103a3168bb3667b8410fadd453b1c1 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Mon, 31 Oct 2022 09:34:28 -0700 Subject: [PATCH 31/72] add message_builder object --- tom_dataproducts/hermes.py | 16 ++++++++++++---- tom_dataproducts/views.py | 10 +++++++--- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/tom_dataproducts/hermes.py b/tom_dataproducts/hermes.py index 6518eca0a..2856cb58b 100644 --- a/tom_dataproducts/hermes.py +++ b/tom_dataproducts/hermes.py @@ -5,6 +5,14 @@ import requests +class BuildHermesMessage(object): + def __init__(self, title='', submitter='', authors='', message=''): + self.title = title + self.submitter = submitter + self.authors = authors + self.message = message + + def publish_photometry_to_hermes(destination, message_info, datums): """ For now this code submits a typical hermes photometry alert using the datums tied to the dataproduct being @@ -32,14 +40,14 @@ def publish_photometry_to_hermes(destination, message_info, datums): }) alert = { 'topic': 'hermes.test', - 'title': message_info['share_title'], - 'author': message_info['submitter'], + 'title': message_info.title, + 'author': message_info.submitter, '' 'data': { - 'authors': message_info['share_authors'], + 'authors': message_info.authors, 'photometry_data': hermes_photometry_data, }, - 'message_text': message_info['share_message'], + 'message_text': message_info.message, } requests.post(url=submit_url, json=alert, headers=headers) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index f8c930733..91e3dedda 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -34,7 +34,7 @@ from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm, DataProductShareForm from tom_dataproducts.filters import DataProductFilter from tom_dataproducts.data_processor import run_data_processor -from tom_dataproducts.hermes import publish_photometry_to_hermes +from tom_dataproducts.hermes import publish_photometry_to_hermes, BuildHermesMessage from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class @@ -324,8 +324,12 @@ def post(self, request, *args, **kwargs): reduced_datums = ReducedDatum.objects.filter(data_product=product) share_destination = form_data['share_destination'] if share_destination == 'hermes': - # build hermes table from Reduced Datums - publish_photometry_to_hermes(share_destination, form_data, reduced_datums) + # build and submit hermes table from Reduced Datums + message_info = BuildHermesMessage(title=form_data['share_title'], + submitter=form_data['submitter'], + authors=form_data['share_authors'], + message=form_data['share_message']) + publish_photometry_to_hermes(share_destination, message_info, reduced_datums) else: self.share_with_tom(share_destination, product) return redirect('/') From a0f65cefc02ad791ab0769e871ddd3cfe287405d Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 1 Nov 2022 10:30:11 -0700 Subject: [PATCH 32/72] refactor table builder for generalization --- tom_dataproducts/hermes.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tom_dataproducts/hermes.py b/tom_dataproducts/hermes.py index 2856cb58b..5ff6722fc 100644 --- a/tom_dataproducts/hermes.py +++ b/tom_dataproducts/hermes.py @@ -13,7 +13,7 @@ def __init__(self, title='', submitter='', authors='', message=''): self.message = message -def publish_photometry_to_hermes(destination, message_info, datums): +def publish_photometry_to_hermes(destination, message_info, datums, **kwargs): """ For now this code submits a typical hermes photometry alert using the datums tied to the dataproduct being shared. In the future this should instead send the user to a new tab with a populated hermes form. @@ -30,19 +30,11 @@ def publish_photometry_to_hermes(destination, message_info, datums): hermes_alert.save() for tomtoolkit_photometry in datums: tomtoolkit_photometry.message.add(hermes_alert) - hermes_photometry_data.append({ - 'photometryId': tomtoolkit_photometry.target.name, - 'dateObs': tomtoolkit_photometry.timestamp.strftime('%x %X'), - 'band': tomtoolkit_photometry.value['filter'], - 'brightness': tomtoolkit_photometry.value['magnitude'], - 'brightnessError': tomtoolkit_photometry.value['error'], - 'brightnessUnit': 'AB mag', - }) + hermes_photometry_data.append(create_hermes_phot_table_row(tomtoolkit_photometry, **kwargs)) alert = { 'topic': 'hermes.test', 'title': message_info.title, 'author': message_info.submitter, - '' 'data': { 'authors': message_info.authors, 'photometry_data': hermes_photometry_data, @@ -51,3 +43,16 @@ def publish_photometry_to_hermes(destination, message_info, datums): } requests.post(url=submit_url, json=alert, headers=headers) + + +def create_hermes_phot_table_row(datum, **kwargs): + """Build a row for a Hermes Photometry Table using a TOM Photometry datum""" + table_row = { + 'photometryId': datum.target.name, + 'dateObs': datum.timestamp.strftime('%x %X'), + 'band': datum.value['filter'], + 'brightness': datum.value['magnitude'], + 'brightnessError': datum.value['error'], + 'brightnessUnit': 'AB mag', + } + return table_row From 942878ca95a51d77de42f0bd6e2ddc05b0156c7a Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 2 Nov 2022 15:47:46 -0700 Subject: [PATCH 33/72] TOM-TOM Dataproduct sharing --- tom_dataproducts/views.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 91e3dedda..62b0bd264 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -37,6 +37,7 @@ from tom_dataproducts.hermes import publish_photometry_to_hermes, BuildHermesMessage from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class +from tom_dataproducts.serializers import DataProductSerializer import requests @@ -354,8 +355,28 @@ def share_with_tom(self, tom_name, product): serialized_target_data = TargetSerializer(target).data targets_url = destination_tom_base_url + 'api/targets/' # TODO: Make sure aliases are checked before creating new target + # Attempt to create Target in Destination TOM response = requests.post(targets_url, headers=headers, auth=auth, data=serialized_target_data) - print(response.text) + try: + target_response = response.json() + destination_target_id = target_response['id'] + except KeyError: + # If Target already exists at destination, find ID + response = requests.get(targets_url, headers=headers, auth=auth, data=serialized_target_data) + target_response = response.json() + destination_target_id = target_response['results'][0]['id'] + + serialized_dataproduct_data = DataProductSerializer(product).data + serialized_dataproduct_data['target'] = destination_target_id + dataproducts_url = destination_tom_base_url + 'api/dataproducts/' + # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage + dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) + # Save DataProduct in Destination TOM + with open(dataproduct_filename, 'rb') as dataproduct_filep: + files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} + headers = {'Media-Type': 'multipart/form-data'} + response = requests.post(dataproducts_url, data=serialized_dataproduct_data, files=files, + headers=headers, auth=auth) class DataProductShareViewOld(View): From c0c1c99ba912cee83680c2daf134f744f8dcc7fe Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 3 Nov 2022 14:19:22 -0700 Subject: [PATCH 34/72] add extra info and allow for empty fields in photometry datum.value --- tom_dataproducts/hermes.py | 14 +++++++++----- tom_dataproducts/views.py | 4 ++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/tom_dataproducts/hermes.py b/tom_dataproducts/hermes.py index 5ff6722fc..0f211969f 100644 --- a/tom_dataproducts/hermes.py +++ b/tom_dataproducts/hermes.py @@ -6,11 +6,12 @@ class BuildHermesMessage(object): - def __init__(self, title='', submitter='', authors='', message=''): + def __init__(self, title='', submitter='', authors='', message='', **kwargs): self.title = title self.submitter = submitter self.authors = authors self.message = message + self.extra_info = kwargs def publish_photometry_to_hermes(destination, message_info, datums, **kwargs): @@ -41,6 +42,7 @@ def publish_photometry_to_hermes(destination, message_info, datums, **kwargs): }, 'message_text': message_info.message, } + alert['data'].update(message_info.extra_info) requests.post(url=submit_url, json=alert, headers=headers) @@ -50,9 +52,11 @@ def create_hermes_phot_table_row(datum, **kwargs): table_row = { 'photometryId': datum.target.name, 'dateObs': datum.timestamp.strftime('%x %X'), - 'band': datum.value['filter'], - 'brightness': datum.value['magnitude'], - 'brightnessError': datum.value['error'], - 'brightnessUnit': 'AB mag', + 'telescope': datum.value.get('telescope', ''), + 'instrument': datum.value.get('instrument', ''), + 'band': datum.value.get('filter', ''), + 'brightness': datum.value.get('magnitude', ''), + 'brightnessError': datum.value.get('error', ''), + 'brightnessUnit': datum.value.get('unit', 'AB mag'), } return table_row diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 62b0bd264..3d6e5feba 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -339,8 +339,8 @@ def share_with_tom(self, tom_name, product): """ When sharing a DataProduct with another TOM we likely want to share the data product itself and let the other TOM process it rather than share the Reduced Datums - :param tom_name: - :param product: + :param tom_name: name of destination tom in settings.DATA_SHARING + :param product: DataProduct model instance :return: """ try: From eceecfb4c83ed5b83bf12695d233bc80fe12dc08 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Mon, 7 Nov 2022 10:50:12 -0800 Subject: [PATCH 35/72] build plotly phot table --- .../partials/photometry_for_target.html | 1 + .../templatetags/dataproduct_extras.py | 17 ++++++++++++++--- tom_dataproducts/views.py | 1 - 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_for_target.html index 11b505923..34c290f40 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_for_target.html @@ -8,4 +8,5 @@

Photometry

{{ plot|safe }} + {{ table|safe }}
\ No newline at end of file diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index bc85bc517..a18501ac5 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -209,8 +209,12 @@ def photometry_for_target(context, target, width=700, height=600, background=Non photometry_data[datum.value['filter']].setdefault('error', []).append(datum.value.get('error')) photometry_data[datum.value['filter']].setdefault('limit', []).append(datum.value.get('limit')) + table_cols = {'time': [], 'filter': [], 'magnitude': [], 'error': [], 'limit': []} plot_data = [] for filter_name, filter_values in photometry_data.items(): + for col_header, col_data in filter_values.items(): + table_cols[col_header] += col_data + table_cols['filter'] += [filter_name]*len(filter_values['time']) if filter_values['magnitude']: series = go.Scatter( x=filter_values['time'], @@ -252,12 +256,19 @@ def photometry_for_target(context, target, width=700, height=600, background=Non fig.update_layout(clickmode='event+select') # TODO: Build linked data table - tab = go.Table(header=dict(values=[]), - cells=dict(values=[])) + tab = go.Figure(data=[go.Table(header=dict(values=[table_header for table_header in table_cols]), + cells=dict(values=[table_cols[table_header] for table_header in table_cols]))]) + + def selection_fn(trace, points, selector): + print(trace, points, selector) + + for plot in plot_data: + plot.on_selection(selection_fn) return { 'target': target, - 'plot': offline.plot(fig, output_type='div', show_link=False) + 'plot': offline.plot(fig, output_type='div', show_link=False), + 'table': offline.plot(tab, output_type='div', show_link=False) } diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 3d6e5feba..0df1014ee 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -23,7 +23,6 @@ from django.views.generic.edit import CreateView, DeleteView, FormView from django_filters.views import FilterView from guardian.shortcuts import assign_perm, get_objects_for_user -from rest_framework.renderers import JSONRenderer from tom_common.hooks import run_hook from tom_common.hints import add_hint From da1f30b932815bc9eacce3270efc696c9ed500ed Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Mon, 7 Nov 2022 10:51:30 -0800 Subject: [PATCH 36/72] remove plotly phot table --- .../partials/photometry_for_target.html | 1 - .../templatetags/dataproduct_extras.py | 15 --------------- 2 files changed, 16 deletions(-) diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_for_target.html index 34c290f40..11b505923 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_for_target.html @@ -8,5 +8,4 @@

Photometry

{{ plot|safe }} - {{ table|safe }}
\ No newline at end of file diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index a18501ac5..4373d777a 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -209,12 +209,8 @@ def photometry_for_target(context, target, width=700, height=600, background=Non photometry_data[datum.value['filter']].setdefault('error', []).append(datum.value.get('error')) photometry_data[datum.value['filter']].setdefault('limit', []).append(datum.value.get('limit')) - table_cols = {'time': [], 'filter': [], 'magnitude': [], 'error': [], 'limit': []} plot_data = [] for filter_name, filter_values in photometry_data.items(): - for col_header, col_data in filter_values.items(): - table_cols[col_header] += col_data - table_cols['filter'] += [filter_name]*len(filter_values['time']) if filter_values['magnitude']: series = go.Scatter( x=filter_values['time'], @@ -255,20 +251,9 @@ def photometry_for_target(context, target, width=700, height=600, background=Non fig.update_xaxes(showgrid=grid, color=label_color, showline=True, linecolor=label_color, mirror=True) fig.update_layout(clickmode='event+select') - # TODO: Build linked data table - tab = go.Figure(data=[go.Table(header=dict(values=[table_header for table_header in table_cols]), - cells=dict(values=[table_cols[table_header] for table_header in table_cols]))]) - - def selection_fn(trace, points, selector): - print(trace, points, selector) - - for plot in plot_data: - plot.on_selection(selection_fn) - return { 'target': target, 'plot': offline.plot(fig, output_type='div', show_link=False), - 'table': offline.plot(tab, output_type='div', show_link=False) } From 3ffd0d2299c5a84c2baa8fbddc18ddc1f7ff6778 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 8 Nov 2022 12:13:41 -0800 Subject: [PATCH 37/72] add full data sharing UI to target page --- tom_dataproducts/forms.py | 6 +++- .../partials/dataproduct_list_for_target.html | 3 +- .../partials/share_target_data.html | 25 +++++++++++++++ .../templatetags/dataproduct_extras.py | 22 +++++++++++-- tom_dataproducts/urls.py | 5 +-- tom_dataproducts/views.py | 31 ++++++++++++------- .../templates/tom_targets/target_detail.html | 1 + 7 files changed, 74 insertions(+), 19 deletions(-) create mode 100644 tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html diff --git a/tom_dataproducts/forms.py b/tom_dataproducts/forms.py index 22a4952dd..78cc5c52e 100644 --- a/tom_dataproducts/forms.py +++ b/tom_dataproducts/forms.py @@ -11,6 +11,9 @@ ('tom-demo-dev', '2nd best TOM'), ('local_host', 'Best TOM (My TOM)')) +DATA_TYPE_OPTIONS = (('photometry', 'Photometry'), + ('spectroscopy', 'Spectroscopy')) + class AddProductToGroupForm(forms.Form): products = forms.ModelMultipleChoiceField( @@ -53,11 +56,12 @@ def __init__(self, *args, **kwargs): widget=forms.CheckboxSelectMultiple) -class DataProductShareForm(forms.Form): +class DataShareForm(forms.Form): share_destination = forms.ChoiceField(required=True, choices=DESTINATION_OPTIONS, label="Destination") share_title = forms.CharField(required=False, label="Title") share_message = forms.CharField(required=False, label="Message", widget=forms.Textarea()) share_authors = forms.CharField(required=False, widget=forms.HiddenInput()) + data_type = forms.ChoiceField(required=False, choices=DATA_TYPE_OPTIONS, label="Type") target = forms.ModelChoiceField( Target.objects.all(), widget=forms.HiddenInput(), diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html index d80310ea3..7fed82617 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html @@ -40,12 +40,11 @@

Data

- + {% csrf_token %} {% for hidden in data_product_share_form.hidden_fields %} {{ hidden }} {% endfor %} - {{ data_product_share_form.data_product.value }}
{% bootstrap_field data_product_share_form.share_destination %} diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html b/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html new file mode 100644 index 000000000..164a62114 --- /dev/null +++ b/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html @@ -0,0 +1,25 @@ +{% load bootstrap4 %} +{% include 'tom_dataproducts/partials/js9_scripts.html' %} +{% load tom_common_extras %} +
+
+ Publish Data for {{target.name}} +
+ + {% csrf_token %} + {% for hidden in target_data_share_form.hidden_fields %} + {{ hidden }} + {% endfor %} +
+
+ {% bootstrap_field target_data_share_form.share_destination %} +
+
+ {% bootstrap_field target_data_share_form.data_type %} +
+
+ +
+
+ +
\ No newline at end of file diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 4373d777a..cf3eed624 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -3,6 +3,7 @@ from django import template +from django import forms from django.conf import settings from django.contrib.auth.models import Group from django.core.paginator import Paginator @@ -16,7 +17,7 @@ from PIL import Image, ImageDraw import base64 -from tom_dataproducts.forms import DataProductUploadForm, DataProductShareForm +from tom_dataproducts.forms import DataProductUploadForm, DataShareForm from tom_dataproducts.models import DataProduct, ReducedDatum from tom_dataproducts.processors.data_serializers import SpectrumSerializer from tom_observations.models import ObservationRecord @@ -60,7 +61,7 @@ def dataproduct_list_for_target(context, target): initial = {'submitter': context['request'].user, 'target': target, 'share_title': f"Updated data for {target.name}."} - form = DataProductShareForm(initial=initial) + form = DataShareForm(initial=initial) return { 'products': target_products_for_user, @@ -132,6 +133,23 @@ def upload_dataproduct(context, obj): return {'data_product_form': form} +@register.inclusion_tag('tom_dataproducts/partials/share_target_data.html', takes_context=True) +def share_data(context, target): + """ + Publish data to Hermes + """ + initial = {'submitter': context['request'].user, + 'target': target, + 'share_title': f"Updated data for {target.name}.", + } + form = DataShareForm(initial=initial) + form.fields['share_title'].widget = forms.HiddenInput() + + context = {'target': target, + 'target_data_share_form': form} + return context + + @register.inclusion_tag('tom_dataproducts/partials/recent_photometry.html') def recent_photometry(target, limit=1): """ diff --git a/tom_dataproducts/urls.py b/tom_dataproducts/urls.py index be0e0bc93..ebd4b450b 100644 --- a/tom_dataproducts/urls.py +++ b/tom_dataproducts/urls.py @@ -4,7 +4,7 @@ from tom_dataproducts.views import DataProductDeleteView, DataProductGroupCreateView from tom_dataproducts.views import DataProductGroupDetailView, DataProductGroupDataView, DataProductGroupDeleteView from tom_dataproducts.views import DataProductUploadView, DataProductFeatureView, UpdateReducedDataView -from tom_dataproducts.views import DataProductShareView +from tom_dataproducts.views import DataShareView from tom_common.api_router import SharedAPIRootRouter from tom_dataproducts.api_views import DataProductViewSet @@ -25,6 +25,7 @@ path('data/reduced/update/', UpdateReducedDataView.as_view(), name='update-reduced-data'), path('data//delete/', DataProductDeleteView.as_view(), name='delete'), path('data//feature/', DataProductFeatureView.as_view(), name='feature'), - path('data//share/', DataProductShareView.as_view(), name='share'), + path('data//share/', DataShareView.as_view(), name='share'), + path('target//share/', DataShareView.as_view(), name='share_all'), path('/save/', DataProductSaveView.as_view(), name='save'), ] diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 0df1014ee..9c86922fc 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -28,9 +28,10 @@ from tom_common.hints import add_hint from tom_common.mixins import Raise403PermissionRequiredMixin from tom_targets.serializers import TargetSerializer +from tom_targets.models import Target from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum from tom_dataproducts.exceptions import InvalidFileFormatException -from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm, DataProductShareForm +from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm, DataShareForm from tom_dataproducts.filters import DataProductFilter from tom_dataproducts.data_processor import run_data_processor from tom_dataproducts.hermes import publish_photometry_to_hermes, BuildHermesMessage @@ -289,14 +290,12 @@ def get(self, request, *args, **kwargs): ) -class DataProductShareView(FormView): - # TODO: update class docstring +class DataShareView(FormView): """ - View that handles the featuring of ``DataProduct``s. A featured ``DataProduct`` is displayed on the - ``TargetDetailView``. + View that handles the sharing of ``DataProduct``s either through HERMES or with another TOM. """ - form_class = DataProductShareForm + form_class = DataShareForm def get_form(self, *args, **kwargs): # TODO: Add permissions @@ -315,13 +314,21 @@ def post(self, request, *args, **kwargs): """ Method that handles thePOST requests for this view. """ - data_product_share_form = DataProductShareForm(request.POST, request.FILES) - if data_product_share_form.is_valid(): - form_data = data_product_share_form.cleaned_data - product_id = kwargs.get('pk', None) - product = DataProduct.objects.get(pk=product_id) - if product.data_product_type == 'photometry': + data_share_form = DataShareForm(request.POST, request.FILES) + if data_share_form.is_valid(): + form_data = data_share_form.cleaned_data + # determine if pk is data product, Reduced Datum, or Target. + product_id = kwargs.get('dp_pk', None) + if product_id: + product = DataProduct.objects.get(pk=product_id) + data_type = product.data_product_type reduced_datums = ReducedDatum.objects.filter(data_product=product) + else: + target_id = kwargs.get('tg_pk', None) + target = Target.objects.get(pk=target_id) + data_type = form_data['data_type'] + reduced_datums = ReducedDatum.objects.filter(target=target, data_type=data_type) + if data_type == 'photometry': share_destination = form_data['share_destination'] if share_destination == 'hermes': # build and submit hermes table from Reduced Datums diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 2b06a3def..0a29424a1 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -35,6 +35,7 @@ {% target_buttons object %} {% target_data object %} {% recent_photometry object limit=3 %} + {% share_data object %} {% if object.type == 'SIDEREAL' %} {% aladin object %} {% endif %} From 313ebe53a978dc64f343288db80b4990077fc46c Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 8 Nov 2022 14:19:13 -0800 Subject: [PATCH 38/72] pull sharing destinations from settings. --- tom_base/settings.py | 3 +++ tom_dataproducts/forms.py | 7 +++---- .../tom_dataproducts/partials/share_target_data.html | 2 +- tom_dataproducts/templatetags/dataproduct_extras.py | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index 8dbc40726..100bcc27c 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -230,16 +230,19 @@ # Configuration for the TOM receiving data from this TOM DATA_SHARING = { 'hermes': { + 'NICKNAME': os.getenv('HERMES_NICKNAME', 'Hermes'), 'BASE_URL': os.getenv('HERMES_BASE_URL', 'https://hermes.lco.global/'), 'API_TOKEN': os.getenv('HERMES_API_TOKEN', 'set HERMES_API_TOKEN value in environment'), }, 'tom-demo-dev': { + 'NICKNAME': os.getenv('TOM_DEMO_NICKNAME', 'TOM Demo Dev'), 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), 'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'), 'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'), }, 'localhost-tom': { # for testing; share with yourself + 'NICKNAME': os.getenv('LOCALHOST_TOM_NICKNAME', 'Local'), 'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'), 'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'), 'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'), diff --git a/tom_dataproducts/forms.py b/tom_dataproducts/forms.py index 78cc5c52e..b1873178e 100644 --- a/tom_dataproducts/forms.py +++ b/tom_dataproducts/forms.py @@ -7,9 +7,8 @@ from tom_targets.models import Target -DESTINATION_OPTIONS = (('hermes', 'Hermes'), - ('tom-demo-dev', '2nd best TOM'), - ('local_host', 'Best TOM (My TOM)')) +DESTINATION_OPTIONS = [(destination, details.get('NICKNAME', destination)) + for destination, details in settings.DATA_SHARING.items()] DATA_TYPE_OPTIONS = (('photometry', 'Photometry'), ('spectroscopy', 'Spectroscopy')) @@ -61,7 +60,7 @@ class DataShareForm(forms.Form): share_title = forms.CharField(required=False, label="Title") share_message = forms.CharField(required=False, label="Message", widget=forms.Textarea()) share_authors = forms.CharField(required=False, widget=forms.HiddenInput()) - data_type = forms.ChoiceField(required=False, choices=DATA_TYPE_OPTIONS, label="Type") + data_type = forms.ChoiceField(required=False, choices=DATA_TYPE_OPTIONS, label="Data Type") target = forms.ModelChoiceField( Target.objects.all(), widget=forms.HiddenInput(), diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html b/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html index 164a62114..a6c4ac273 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html @@ -1,7 +1,7 @@ {% load bootstrap4 %} {% include 'tom_dataproducts/partials/js9_scripts.html' %} {% load tom_common_extras %} -
+
Publish Data for {{target.name}}
diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index cf3eed624..dd3063392 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -140,7 +140,7 @@ def share_data(context, target): """ initial = {'submitter': context['request'].user, 'target': target, - 'share_title': f"Updated data for {target.name}.", + 'share_title': f"Updated data for {target.name} from {settings.TOM_NAME}.", } form = DataShareForm(initial=initial) form.fields['share_title'].widget = forms.HiddenInput() From f983e86d48271c5a83e545edfceb685ed83c23a2 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 8 Nov 2022 15:40:38 -0800 Subject: [PATCH 39/72] give feedback for successful message --- tom_dataproducts/hermes.py | 3 ++- tom_dataproducts/views.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tom_dataproducts/hermes.py b/tom_dataproducts/hermes.py index 0f211969f..022223f01 100644 --- a/tom_dataproducts/hermes.py +++ b/tom_dataproducts/hermes.py @@ -44,7 +44,8 @@ def publish_photometry_to_hermes(destination, message_info, datums, **kwargs): } alert['data'].update(message_info.extra_info) - requests.post(url=submit_url, json=alert, headers=headers) + response = requests.post(url=submit_url, json=alert, headers=headers) + return response def create_hermes_phot_table_row(datum, **kwargs): diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 9c86922fc..0ca75ed62 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -336,9 +336,12 @@ def post(self, request, *args, **kwargs): submitter=form_data['submitter'], authors=form_data['share_authors'], message=form_data['share_message']) - publish_photometry_to_hermes(share_destination, message_info, reduced_datums) + response = publish_photometry_to_hermes(share_destination, message_info, reduced_datums) else: - self.share_with_tom(share_destination, product) + response = self.share_with_tom(share_destination, product) + publish_feedback = response.json()["message"] + for feedback in publish_feedback: + messages.success(self.request, feedback) return redirect('/') def share_with_tom(self, tom_name, product): @@ -383,6 +386,7 @@ def share_with_tom(self, tom_name, product): headers = {'Media-Type': 'multipart/form-data'} response = requests.post(dataproducts_url, data=serialized_dataproduct_data, files=files, headers=headers, auth=auth) + return response class DataProductShareViewOld(View): From f866d184a74e12fa39c4c1dac609e1076642c8e5 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 8 Nov 2022 16:54:38 -0800 Subject: [PATCH 40/72] add specifics to error message --- tom_dataproducts/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 0ca75ed62..3327ba722 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -342,6 +342,8 @@ def post(self, request, *args, **kwargs): publish_feedback = response.json()["message"] for feedback in publish_feedback: messages.success(self.request, feedback) + else: + messages.error(self.request, f'Publishing {data_type} data is not yet supported.') return redirect('/') def share_with_tom(self, tom_name, product): From e1b023f573fcd7ea3a2fad5aa1b0d7c04651577a Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 9 Nov 2022 12:04:19 -0800 Subject: [PATCH 41/72] clean up some old code and disable TOM-TOM sharing --- tom_base/settings.py | 6 +- tom_dataproducts/forms.py | 2 +- .../tom_dataproducts/dataproduct_list.html | 19 -- tom_dataproducts/views.py | 166 +----------------- 4 files changed, 7 insertions(+), 186 deletions(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index 100bcc27c..87f332195 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -230,19 +230,19 @@ # Configuration for the TOM receiving data from this TOM DATA_SHARING = { 'hermes': { - 'NICKNAME': os.getenv('HERMES_NICKNAME', 'Hermes'), + 'DISPLAY_NAME': os.getenv('HERMES_DISPLAY_NAME', 'Hermes'), 'BASE_URL': os.getenv('HERMES_BASE_URL', 'https://hermes.lco.global/'), 'API_TOKEN': os.getenv('HERMES_API_TOKEN', 'set HERMES_API_TOKEN value in environment'), }, 'tom-demo-dev': { - 'NICKNAME': os.getenv('TOM_DEMO_NICKNAME', 'TOM Demo Dev'), + 'DISPLAY_NAME': os.getenv('TOM_DEMO_DISPLAY_NAME', 'TOM Demo Dev'), 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), 'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'), 'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'), }, 'localhost-tom': { # for testing; share with yourself - 'NICKNAME': os.getenv('LOCALHOST_TOM_NICKNAME', 'Local'), + 'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'), 'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'), 'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'), 'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'), diff --git a/tom_dataproducts/forms.py b/tom_dataproducts/forms.py index b1873178e..1f9369301 100644 --- a/tom_dataproducts/forms.py +++ b/tom_dataproducts/forms.py @@ -7,7 +7,7 @@ from tom_targets.models import Target -DESTINATION_OPTIONS = [(destination, details.get('NICKNAME', destination)) +DESTINATION_OPTIONS = [(destination, details.get('DISPLAY_NAME', destination)) for destination, details in settings.DATA_SHARING.items()] DATA_TYPE_OPTIONS = (('photometry', 'Photometry'), diff --git a/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html b/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html index c48688f47..06d7b607b 100644 --- a/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html +++ b/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html @@ -22,27 +22,9 @@
--> -
-
-
-
- -
-
- -
-
-
-
- @@ -54,7 +36,6 @@ {% for product in object_list %} - {% if product.observation_record.id %} diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 3327ba722..a366f2a2a 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -338,7 +338,9 @@ def post(self, request, *args, **kwargs): message=form_data['share_message']) response = publish_photometry_to_hermes(share_destination, message_info, reduced_datums) else: - response = self.share_with_tom(share_destination, product) + messages.error(self.request, f'TOM-TOM sharing is not yet supported.') + return redirect('/') + # response = self.share_with_tom(share_destination, product) publish_feedback = response.json()["message"] for feedback in publish_feedback: messages.success(self.request, feedback) @@ -391,168 +393,6 @@ def share_with_tom(self, tom_name, product): return response -class DataProductShareViewOld(View): - # TODO: refactor the general data sharing mechanism to make it more driven by the - # configuration in settings.py - # TODO: consider passing DataProduct instance to submit_ and _share methods - # and refactor data manipulation to _helper methods. - def submit_to_stream(self, stream_name, target_name, photometry_data, message='From TOMToolkit'): - hermes_base_url = settings.DATA_SHARING[stream_name]['BASE_URL'] - - # Get the csrf-token to include in header - # csrf_url = hermes_base_url + 'get-csrf-token/' - # csrf_headers = {'Content-Type': 'application/json'} - # csrf_response = requests.get(url=csrf_url, headers=csrf_headers) - # logger.debug(f'dir(csrf_response): {dir(csrf_response)}') - # logger.debug(f'csrf_response.text: {csrf_response.text}') - # logger.debug(f'csrf_response.json(): {csrf_response.json()}') - - # csrf_token = csrf_response.json()['token'] - - submit_url = hermes_base_url + 'submit/' - headers = { - # 'X-CSRFToken': csrf_token, - # 'Content-Type': 'application/json', - } - - # - # Map TOM Toolkit Photometry.csv fields to HERMES Photometry reporting form fields - # - hermes_photometry_data = [] - for tomtoolkit_photometry in photometry_data: - hermes_photometry_data.append({ - 'photometryId': target_name, - 'dateObs': tomtoolkit_photometry['time'], - 'band': tomtoolkit_photometry['filter'], - 'brightness': tomtoolkit_photometry['magnitude'], - 'brightnessError': tomtoolkit_photometry['error'], - 'brightnessUnit': 'AB mag', - }) - - # fields required by hopingest.py: topic, title, author, data, message_text - # TODO: maybe throw up form to get these fields - alert = { - 'topic': 'hermes.test', - 'title': 'TOM Toolkit test (Photometry)', - 'author': 'llindstrom@lco.global', - 'data': { - 'photometry_data': hermes_photometry_data, - }, - 'message_text': f'Test alert from TOM Toolkit at {datetime.datetime.now()}', - } - # logger.debug(f'DataProductShareView.submit_to_hermes() alert: {alert}') - - requests.post(url=submit_url, json=alert, headers=headers) - # logger.debug(f'DataProductShareView.submit_to_hermes response.status_code: {submit_response.status_code}') - # logger.debug(f'DataProductShareView.submit_to_hermes response.text: {submit_response.text}') - - def share_with_tom(self, tom_name, product: DataProduct): - """Construct and make a POST (create) request to the destination TOM /api/dataproducts/ endpoint. - - Theoretically, we should be able to simply serializer the DataProduct instance with the - DataProductSerializer (producing native python data types) and JSONRenderer().render() that - (producing JSON) and POST that to the destination TOM DRF API endpoint. - - * tom_name is the key in the settings.DATA_SHARING configuration dictionary - * product is the DataProduct instance to share - """ - try: - destination_tom_base_url = settings.DATA_SHARING[tom_name]['BASE_URL'] - username = settings.DATA_SHARING[tom_name]['USERNAME'] - password = settings.DATA_SHARING[tom_name]['PASSWORD'] - except KeyError as err: - raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {tom_name}: Key {err} not found.') - auth = (username, password) - target_name = product.target.name - - # - # Get this DataProduct's target's PK from the destination TOM - # - targets_url = destination_tom_base_url + 'api/targets/' - target_params = {'name': target_name} - response = requests.get(targets_url, auth=auth, params=target_params) - - target_response = response.json() - if target_response['count'] == 1: - destination_tom_target_id = target_response['results'][0]['id'] - # TODO: handle target groups correctly - elif target_response['count'] == 0: - # Target not found in destination tom - logger.warning( - f'DataProductShareView.share_with_tom Target {target_name} not found on {tom_name}. ' - f'If target {target_name} does exist on the destination TOM, then this may an ' - f'authentication problem preventing access to the targets on {tom_name}.' - ) - # TODO: post message to UI - return # NOTE: early exit - elif target_response['count'] > 1: - # More than one target found; Target name must be amibiguous - msg = ( - f'Target name must be unique on destination TOM {tom_name}. ' - f'The following targets share a name or alias with {target_name}:\n' - ) - for target in target_response['results']: - aliases = ', '.join([alias['name'] for alias in target['aliases']]) # alias1, alias2, alias - msg += f' Target: {target["name"]} Aliases: {aliases}\n' - logger.warning(msg) - # TODO: post message to UI - return # NOTE: early exit - - # - # Now POST the DataProduct to the destination TOM - # - data_products_url = destination_tom_base_url + 'api/dataproducts/' - - # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage - dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) - with open(dataproduct_filename, 'rb') as dataproduct_filep: - files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} - data = { - 'target': destination_tom_target_id, - 'data_product_type': product.data_product_type - } - headers = {'Media-Type': 'multipart/form-data'} - response = requests.post(data_products_url, data=data, files=files, headers=headers, auth=auth) - - logger.debug(f'DataProductShareView.share_with_tom response.status_code: {response.status_code}') - logger.debug(f'DataProductShareView.share_with_tom response.text: {response.text}') - - def get(self, request, *args, **kwargs): - """ - Method that handles the GET requests for this view. - - """ - # TODO: update get method docstring - - product_id = kwargs.get('pk', None) - product = DataProduct.objects.get(pk=product_id) - - logger.debug(f'Sharing data product: {product} of type: {product.data_product_type}') - if product.data_product_type == 'photometry': - # TODO: get DATA_SHARING config dict key from UI via kwargs or ??? - sharing_destination = 'hermes' - # sharing_destination = 'localhost-tom' - if sharing_destination == 'hermes': - # Convert CSV into python dict with csv.DictReader: - with open(product.data.path, newline='') as csvfile: - photometry_reader = csv.DictReader(csvfile, delimiter=',') - data = [row for row in photometry_reader] - - # Turn the data into JSON to send to the HERMES /submit endpoint - # TODO: rename these photometry-specific methods to reflect that.. - # TODO: sort out where to share to (perhaps template info or FORM data?) - - # TODO: pass product to submit method, open path, and get data there - self.submit_to_stream(sharing_destination, product.target.name, data) - else: - self.share_with_tom(sharing_destination, product) - - return redirect(reverse( - 'tom_targets:detail', - kwargs={'pk': request.GET.get('target_id')}) - ) - - class DataProductGroupDetailView(DetailView): """ View that handles the viewing of a specific ``DataProductGroup``. From 811c7244e2c18b7ae22c86ef2961733c56f0dbad Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 9 Nov 2022 13:48:12 -0800 Subject: [PATCH 42/72] account for no data_sharing in settings --- tom_dataproducts/forms.py | 8 ++-- .../tom_dataproducts/dataproduct_list.html | 9 ----- .../partials/share_target_data.html | 40 ++++++++++--------- 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/tom_dataproducts/forms.py b/tom_dataproducts/forms.py index 1f9369301..694c23ff0 100644 --- a/tom_dataproducts/forms.py +++ b/tom_dataproducts/forms.py @@ -6,9 +6,11 @@ from tom_observations.models import ObservationRecord from tom_targets.models import Target - -DESTINATION_OPTIONS = [(destination, details.get('DISPLAY_NAME', destination)) - for destination, details in settings.DATA_SHARING.items()] +try: + DESTINATION_OPTIONS = [(destination, details.get('DISPLAY_NAME', destination)) + for destination, details in settings.DATA_SHARING.items()] +except AttributeError: + DESTINATION_OPTIONS = [] DATA_TYPE_OPTIONS = (('photometry', 'Photometry'), ('spectroscopy', 'Spectroscopy')) diff --git a/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html b/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html index 06d7b607b..45366b7df 100644 --- a/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html +++ b/tom_dataproducts/templates/tom_dataproducts/dataproduct_list.html @@ -13,15 +13,6 @@ {% bootstrap_pagination page_obj extra=request.GET.urlencode %} -
File Target Observation
{{ product.get_file_name|truncatechars:40 }} {{ product.target.name|truncatechars:40 }}
diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html b/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html index a6c4ac273..60ecef955 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html @@ -3,23 +3,27 @@ {% load tom_common_extras %}
- Publish Data for {{target.name}} + Publish Data for {{target.name}}:
+ {% if sharing_destinations %}
- {% csrf_token %} - {% for hidden in target_data_share_form.hidden_fields %} - {{ hidden }} - {% endfor %} -
-
- {% bootstrap_field target_data_share_form.share_destination %} -
-
- {% bootstrap_field target_data_share_form.data_type %} -
-
- -
-
- -
\ No newline at end of file + {% csrf_token %} + {% for hidden in target_data_share_form.hidden_fields %} + {{ hidden }} + {% endfor %} +
+
+ {% bootstrap_field target_data_share_form.share_destination %} +
+
+ {% bootstrap_field target_data_share_form.data_type %} +
+
+ +
+
+ + {% else %} + Not Configured + {% endif %} + From 31af53db8dd2912dea728eba69bc36a8db794080 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 9 Nov 2022 14:25:09 -0800 Subject: [PATCH 43/72] fix some typos --- tom_dataproducts/templatetags/dataproduct_extras.py | 2 -- tom_dataproducts/views.py | 6 ++++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index dd3063392..82abbbba7 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -1,7 +1,6 @@ import logging from urllib.parse import urlencode - from django import template from django import forms from django.conf import settings @@ -110,7 +109,6 @@ def dataproduct_list_all(context): return { 'products': products, - 'sharing_destinations': get_data_sharing_destinations() } diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index a366f2a2a..6204728eb 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -292,7 +292,7 @@ def get(self, request, *args, **kwargs): class DataShareView(FormView): """ - View that handles the sharing of ``DataProduct``s either through HERMES or with another TOM. + View that handles the sharing of data either through HERMES or with another TOM. """ form_class = DataShareForm @@ -312,7 +312,9 @@ def form_invalid(self, form): def post(self, request, *args, **kwargs): """ - Method that handles thePOST requests for this view. + Method that handles the POST requests for sharing data. + Handles Data Products and All the data of a type for a target. + Submit to Hermes, or Share with TOM. """ data_share_form = DataShareForm(request.POST, request.FILES) if data_share_form.is_valid(): From ebe0848851d3743c51974815ae3d4f9adea925ed Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 9 Nov 2022 14:37:40 -0800 Subject: [PATCH 44/72] remove unused imports --- tom_dataproducts/views.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 6204728eb..3f3db15a0 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -1,5 +1,3 @@ -import csv -import datetime from io import StringIO import logging import os From 532d132e90a3e7a9dc2fc3beb853794cc7a3b867 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 9 Nov 2022 14:40:48 -0800 Subject: [PATCH 45/72] remove unnecessary f string --- tom_dataproducts/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 3f3db15a0..d22fdb06d 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -338,7 +338,7 @@ def post(self, request, *args, **kwargs): message=form_data['share_message']) response = publish_photometry_to_hermes(share_destination, message_info, reduced_datums) else: - messages.error(self.request, f'TOM-TOM sharing is not yet supported.') + messages.error(self.request, 'TOM-TOM sharing is not yet supported.') return redirect('/') # response = self.share_with_tom(share_destination, product) publish_feedback = response.json()["message"] From 59193f273d51d3b8e102581522603daef3aa457a Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 29 Nov 2022 17:51:37 -0800 Subject: [PATCH 46/72] add alertstream --- tom_base/settings.py | 1 + tom_dataproducts/forms.py | 20 ++++-- tom_dataproducts/hermes.py | 63 ------------------- .../templatetags/dataproduct_extras.py | 5 +- tom_dataproducts/views.py | 19 ++++-- 5 files changed, 33 insertions(+), 75 deletions(-) delete mode 100644 tom_dataproducts/hermes.py diff --git a/tom_base/settings.py b/tom_base/settings.py index 87f332195..1c6b8c6d8 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -56,6 +56,7 @@ 'tom_catalogs', 'tom_observations', 'tom_dataproducts', + 'tom_alertstreams', ] SITE_ID = 1 diff --git a/tom_dataproducts/forms.py b/tom_dataproducts/forms.py index 694c23ff0..e01ea30be 100644 --- a/tom_dataproducts/forms.py +++ b/tom_dataproducts/forms.py @@ -6,11 +6,21 @@ from tom_observations.models import ObservationRecord from tom_targets.models import Target -try: - DESTINATION_OPTIONS = [(destination, details.get('DISPLAY_NAME', destination)) - for destination, details in settings.DATA_SHARING.items()] -except AttributeError: - DESTINATION_OPTIONS = [] + +def get_sharing_destination_options(): + choices = [] + for destination, details in settings.DATA_SHARING.items(): + new_destination = [details.get('DISPLAY_NAME', destination)] + if details.get('USER_TOPICS', None): + topic_list = [(f'{destination}:{topic}', topic) for topic in details['USER_TOPICS']] + new_destination.append(tuple(topic_list)) + else: + new_destination.insert(0, destination) + choices.append(tuple(new_destination)) + return tuple(choices) + + +DESTINATION_OPTIONS = get_sharing_destination_options() DATA_TYPE_OPTIONS = (('photometry', 'Photometry'), ('spectroscopy', 'Spectroscopy')) diff --git a/tom_dataproducts/hermes.py b/tom_dataproducts/hermes.py deleted file mode 100644 index 022223f01..000000000 --- a/tom_dataproducts/hermes.py +++ /dev/null @@ -1,63 +0,0 @@ -from django.conf import settings - -from tom_alerts.models import AlertStreamMessage - -import requests - - -class BuildHermesMessage(object): - def __init__(self, title='', submitter='', authors='', message='', **kwargs): - self.title = title - self.submitter = submitter - self.authors = authors - self.message = message - self.extra_info = kwargs - - -def publish_photometry_to_hermes(destination, message_info, datums, **kwargs): - """ - For now this code submits a typical hermes photometry alert using the datums tied to the dataproduct being - shared. In the future this should instead send the user to a new tab with a populated hermes form. - :param destination: target stream (topic included?) - :param message_info: Dictionary of message information - :param datums: Reduced Datums to be built into table. - :return: - """ - stream_base_url = settings.DATA_SHARING[destination]['BASE_URL'] - submit_url = stream_base_url + 'submit/' - headers = {} - hermes_photometry_data = [] - hermes_alert = AlertStreamMessage(topic='hermes.test', exchange_status='published') - hermes_alert.save() - for tomtoolkit_photometry in datums: - tomtoolkit_photometry.message.add(hermes_alert) - hermes_photometry_data.append(create_hermes_phot_table_row(tomtoolkit_photometry, **kwargs)) - alert = { - 'topic': 'hermes.test', - 'title': message_info.title, - 'author': message_info.submitter, - 'data': { - 'authors': message_info.authors, - 'photometry_data': hermes_photometry_data, - }, - 'message_text': message_info.message, - } - alert['data'].update(message_info.extra_info) - - response = requests.post(url=submit_url, json=alert, headers=headers) - return response - - -def create_hermes_phot_table_row(datum, **kwargs): - """Build a row for a Hermes Photometry Table using a TOM Photometry datum""" - table_row = { - 'photometryId': datum.target.name, - 'dateObs': datum.timestamp.strftime('%x %X'), - 'telescope': datum.value.get('telescope', ''), - 'instrument': datum.value.get('instrument', ''), - 'band': datum.value.get('filter', ''), - 'brightness': datum.value.get('magnitude', ''), - 'brightnessError': datum.value.get('error', ''), - 'brightnessUnit': datum.value.get('unit', 'AB mag'), - } - return table_row diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 82abbbba7..d7e0baef4 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -16,6 +16,7 @@ from PIL import Image, ImageDraw import base64 +from tom_dataproducts.alertstreams.hermes import get_hermes_topics from tom_dataproducts.forms import DataProductUploadForm, DataShareForm from tom_dataproducts.models import DataProduct, ReducedDatum from tom_dataproducts.processors.data_serializers import SpectrumSerializer @@ -136,6 +137,7 @@ def share_data(context, target): """ Publish data to Hermes """ + initial = {'submitter': context['request'].user, 'target': target, 'share_title': f"Updated data for {target.name} from {settings.TOM_NAME}.", @@ -144,7 +146,8 @@ def share_data(context, target): form.fields['share_title'].widget = forms.HiddenInput() context = {'target': target, - 'target_data_share_form': form} + 'target_data_share_form': form, + 'sharing_destinations': settings.DATA_SHARING} return context diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index d22fdb06d..b7b90122b 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -32,7 +32,7 @@ from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm, DataShareForm from tom_dataproducts.filters import DataProductFilter from tom_dataproducts.data_processor import run_data_processor -from tom_dataproducts.hermes import publish_photometry_to_hermes, BuildHermesMessage +from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class from tom_dataproducts.serializers import DataProductSerializer @@ -314,6 +314,7 @@ def post(self, request, *args, **kwargs): Handles Data Products and All the data of a type for a target. Submit to Hermes, or Share with TOM. """ + data_share_form = DataShareForm(request.POST, request.FILES) if data_share_form.is_valid(): form_data = data_share_form.cleaned_data @@ -330,20 +331,26 @@ def post(self, request, *args, **kwargs): reduced_datums = ReducedDatum.objects.filter(target=target, data_type=data_type) if data_type == 'photometry': share_destination = form_data['share_destination'] - if share_destination == 'hermes': + if 'HERMES' in share_destination.upper(): # build and submit hermes table from Reduced Datums + hermes_topic = share_destination.split(':')[1] + destination = share_destination.split(':')[0] message_info = BuildHermesMessage(title=form_data['share_title'], submitter=form_data['submitter'], authors=form_data['share_authors'], - message=form_data['share_message']) - response = publish_photometry_to_hermes(share_destination, message_info, reduced_datums) + message=form_data['share_message'], + topic=hermes_topic + ) + response = publish_photometry_to_hermes(destination, message_info, reduced_datums) else: messages.error(self.request, 'TOM-TOM sharing is not yet supported.') return redirect('/') # response = self.share_with_tom(share_destination, product) publish_feedback = response.json()["message"] - for feedback in publish_feedback: - messages.success(self.request, feedback) + if "ERROR" in publish_feedback.upper(): + messages.error(self.request, publish_feedback) + else: + messages.success(self.request, publish_feedback) else: messages.error(self.request, f'Publishing {data_type} data is not yet supported.') return redirect('/') From 586e874124a73c008f798d44a4b40f5280850454 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 29 Nov 2022 18:50:21 -0800 Subject: [PATCH 47/72] lint fix --- tom_dataproducts/templatetags/dataproduct_extras.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index d7e0baef4..1beead30f 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -16,7 +16,6 @@ from PIL import Image, ImageDraw import base64 -from tom_dataproducts.alertstreams.hermes import get_hermes_topics from tom_dataproducts.forms import DataProductUploadForm, DataShareForm from tom_dataproducts.models import DataProduct, ReducedDatum from tom_dataproducts.processors.data_serializers import SpectrumSerializer From 5a5cf7f1e5298b666eb8f309f52c1d532b3d041f Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 29 Nov 2022 18:55:56 -0800 Subject: [PATCH 48/72] remove alertstreams from settings --- tom_base/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index 1c6b8c6d8..87f332195 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -56,7 +56,6 @@ 'tom_catalogs', 'tom_observations', 'tom_dataproducts', - 'tom_alertstreams', ] SITE_ID = 1 From c3de822dcb7659761be37ca8e61d42d6a5441063 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Tue, 29 Nov 2022 19:04:46 -0800 Subject: [PATCH 49/72] add new dataproducts/alertstreams directory --- tom_dataproducts/alertstreams/__init__.py | 0 tom_dataproducts/alertstreams/hermes.py | 135 ++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 tom_dataproducts/alertstreams/__init__.py create mode 100644 tom_dataproducts/alertstreams/hermes.py diff --git a/tom_dataproducts/alertstreams/__init__.py b/tom_dataproducts/alertstreams/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py new file mode 100644 index 000000000..d2d2a4332 --- /dev/null +++ b/tom_dataproducts/alertstreams/hermes.py @@ -0,0 +1,135 @@ +import logging +import json +from datetime import datetime + +from django.conf import settings + +from hop.models import JSONBlob +from hop.io import Metadata + +from tom_alerts.models import AlertStreamMessage +from tom_targets.models import Target +from tom_dataproducts.models import ReducedDatum + +import requests + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class BuildHermesMessage(object): + def __init__(self, title='', submitter='', authors='', message='', topic='hermes.test', **kwargs): + self.title = title + self.submitter = submitter + self.authors = authors + self.message = message + self.topic = topic + self.extra_info = kwargs + + +def publish_photometry_to_hermes(destination, message_info, datums, **kwargs): + """ + For now this code submits a typical hermes photometry alert using the datums tied to the dataproduct being + shared. In the future this should instead send the user to a new tab with a populated hermes form. + :param destination: target stream (topic included?) + :param message_info: Dictionary of message information + :param datums: Reduced Datums to be built into table. + :return: + """ + stream_base_url = settings.DATA_SHARING[destination]['BASE_URL'] + submit_url = stream_base_url + 'submit/' + headers = {'SCIMMA-API-Auth-Username': settings.DATA_SHARING['hermes']['CREDENTIAL_USERNAME'], + 'SCIMMA-API-Auth-Password': settings.DATA_SHARING['hermes']['CREDENTIAL_PASSWORD']} + hermes_photometry_data = [] + hermes_alert = AlertStreamMessage(topic=message_info.topic, exchange_status='published') + hermes_alert.save() + for tomtoolkit_photometry in datums: + tomtoolkit_photometry.message.add(hermes_alert) + hermes_photometry_data.append(create_hermes_phot_table_row(tomtoolkit_photometry, **kwargs)) + alert = { + 'topic': message_info.topic, + 'title': message_info.title, + 'author': message_info.submitter, + 'data': { + 'authors': message_info.authors, + 'photometry_data': hermes_photometry_data, + }, + 'message_text': message_info.message, + } + alert['data'].update(message_info.extra_info) + + response = requests.post(url=submit_url, json=alert, headers=headers) + return response + + +def create_hermes_phot_table_row(datum, **kwargs): + """Build a row for a Hermes Photometry Table using a TOM Photometry datum""" + table_row = { + 'photometryId': datum.target.name, + 'dateObs': datum.timestamp.strftime('%x %X'), + 'telescope': datum.value.get('telescope', ''), + 'instrument': datum.value.get('instrument', ''), + 'band': datum.value.get('filter', ''), + 'brightness': datum.value.get('magnitude', ''), + 'brightnessError': datum.value.get('error', ''), + 'brightnessUnit': datum.value.get('unit', 'AB mag'), + } + return table_row + + +def get_hermes_topics(): + stream_base_url = settings.DATA_SHARING['hermes']['BASE_URL'] + submit_url = stream_base_url + "api/v0/topics/" + headers = {'SCIMMA-API-Auth-Username': settings.DATA_SHARING['hermes']['CREDENTIAL_USERNAME'], + 'SCIMMA-API-Auth-Password': settings.DATA_SHARING['hermes']['CREDENTIAL_PASSWORD']} + user = settings.DATA_SHARING['hermes']['SCIMMA_AUTH_USERNAME'] + headers = {} + + # response = requests.get(url=submit_url, headers=headers) + topics = settings.DATA_SHARING['hermes']['USER_TOPICS'] + return topics + + +def hermes_alert_handler(alert, metadata: Metadata): + # logger.info(f'Alert received on topic {metadata.topic}: {alert}; metatdata: {metadata}') + alert_as_dict = alert.content + photometry_table = alert_as_dict['data'].get('photometry_data', None) + if photometry_table: + hermes_alert = AlertStreamMessage(topic=alert_as_dict['topic'], exchange_status='ingested') + hermes_alert.save() + for row in photometry_table: + try: + target = Target.objects.get(name=row['photometryId']) + except Target.DoesNotExist: + continue + + try: + obs_date = datetime.strptime(row['dateObs'], '%x %X') + except ValueError: + continue + + datum = { + 'target': target, + 'data_type': 'photometry', + 'source_name': alert_as_dict['topic'], + 'source_location': 'HERMES', + 'timestamp': obs_date, + 'value': get_hermes_phot_value(row) + } + new_rd, created = ReducedDatum.objects.get_or_create(**datum) + if created: + new_rd.message.add(hermes_alert) + new_rd.save() + + +def get_hermes_phot_value(phot_data): + data_dictionary = { + 'magnitude': phot_data['brightness'], + 'magnitude_error': phot_data['brightnessError'], + 'filter': phot_data['band'], + 'telescope': phot_data['telescope'], + 'instrument': phot_data['instrument'], + 'unit': phot_data['brightnessUnit'], + } + return data_dictionary + From 35330e2ad88a69cb28099b640ed3afe302a84a07 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 30 Nov 2022 10:52:52 -0800 Subject: [PATCH 50/72] fix lint issues in alertstreams.hermes --- tom_dataproducts/alertstreams/hermes.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index d2d2a4332..1f0f45127 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -1,10 +1,8 @@ import logging -import json from datetime import datetime from django.conf import settings -from hop.models import JSONBlob from hop.io import Metadata from tom_alerts.models import AlertStreamMessage @@ -78,12 +76,12 @@ def create_hermes_phot_table_row(datum, **kwargs): def get_hermes_topics(): - stream_base_url = settings.DATA_SHARING['hermes']['BASE_URL'] - submit_url = stream_base_url + "api/v0/topics/" - headers = {'SCIMMA-API-Auth-Username': settings.DATA_SHARING['hermes']['CREDENTIAL_USERNAME'], - 'SCIMMA-API-Auth-Password': settings.DATA_SHARING['hermes']['CREDENTIAL_PASSWORD']} - user = settings.DATA_SHARING['hermes']['SCIMMA_AUTH_USERNAME'] - headers = {} + # stream_base_url = settings.DATA_SHARING['hermes']['BASE_URL'] + # submit_url = stream_base_url + "api/v0/topics/" + # headers = {'SCIMMA-API-Auth-Username': settings.DATA_SHARING['hermes']['CREDENTIAL_USERNAME'], + # 'SCIMMA-API-Auth-Password': settings.DATA_SHARING['hermes']['CREDENTIAL_PASSWORD']} + # user = settings.DATA_SHARING['hermes']['SCIMMA_AUTH_USERNAME'] + # headers = {} # response = requests.get(url=submit_url, headers=headers) topics = settings.DATA_SHARING['hermes']['USER_TOPICS'] @@ -104,7 +102,7 @@ def hermes_alert_handler(alert, metadata: Metadata): continue try: - obs_date = datetime.strptime(row['dateObs'], '%x %X') + obs_date = datetime.strptime(row['dateObs'].strip(), '%x %X') except ValueError: continue @@ -132,4 +130,3 @@ def get_hermes_phot_value(phot_data): 'unit': phot_data['brightnessUnit'], } return data_dictionary - From b1c8091075966f6d43f9620f21ea22a20d3c7e92 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 30 Nov 2022 11:00:22 -0800 Subject: [PATCH 51/72] comment out currently unnecesary hop import --- tom_dataproducts/alertstreams/hermes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index 1f0f45127..333fdc373 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -3,7 +3,7 @@ from django.conf import settings -from hop.io import Metadata +# from hop.io import Metadata from tom_alerts.models import AlertStreamMessage from tom_targets.models import Target @@ -88,7 +88,7 @@ def get_hermes_topics(): return topics -def hermes_alert_handler(alert, metadata: Metadata): +def hermes_alert_handler(alert, metadata): # logger.info(f'Alert received on topic {metadata.topic}: {alert}; metatdata: {metadata}') alert_as_dict = alert.content photometry_table = alert_as_dict['data'].get('photometry_data', None) From 95607d823ca74816c456114e27f3545f39c5270c Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 1 Dec 2022 15:38:52 -0800 Subject: [PATCH 52/72] allow for no TOM_NAME in settings --- tom_dataproducts/templatetags/dataproduct_extras.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 1beead30f..aeb03ccec 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -139,7 +139,7 @@ def share_data(context, target): initial = {'submitter': context['request'].user, 'target': target, - 'share_title': f"Updated data for {target.name} from {settings.TOM_NAME}.", + 'share_title': f"Updated data for {target.name} from {getattr(settings, 'TOM_NAME', 'TOM Toolkit')}.", } form = DataShareForm(initial=initial) form.fields['share_title'].widget = forms.HiddenInput() From 9ff5a3171ce728bd59fc67fe57a50c46d11f4605 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 1 Dec 2022 16:54:08 -0800 Subject: [PATCH 53/72] fix magnitude_error in hermes message creation. --- tom_dataproducts/alertstreams/hermes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index 333fdc373..8d9094e67 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -69,7 +69,7 @@ def create_hermes_phot_table_row(datum, **kwargs): 'instrument': datum.value.get('instrument', ''), 'band': datum.value.get('filter', ''), 'brightness': datum.value.get('magnitude', ''), - 'brightnessError': datum.value.get('error', ''), + 'brightnessError': datum.value.get('magnitude_error', ''), 'brightnessUnit': datum.value.get('unit', 'AB mag'), } return table_row From 5225f473539cac12099061d204ebfcd910aa7a1e Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Mon, 5 Dec 2022 16:35:53 -0800 Subject: [PATCH 54/72] add sharing protocol function --- tom_dataproducts/alertstreams/hermes.py | 13 +++++++++++++ tom_dataproducts/views.py | 26 ++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index 8d9094e67..2d6ebf772 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -42,6 +42,7 @@ def publish_photometry_to_hermes(destination, message_info, datums, **kwargs): hermes_alert = AlertStreamMessage(topic=message_info.topic, exchange_status='published') hermes_alert.save() for tomtoolkit_photometry in datums: + tomtoolkit_photometry.message.add(hermes_alert) hermes_photometry_data.append(create_hermes_phot_table_row(tomtoolkit_photometry, **kwargs)) alert = { @@ -89,6 +90,13 @@ def get_hermes_topics(): def hermes_alert_handler(alert, metadata): + """Alert Handler to record data streamed through Hermes. + -- Only Reads Photometry Data + -- Only ingests Data if exact match for Target Name + -- Does not Ingest Data if exact match already exists + -- Requires 'tom_alertstreams' in settings.INSTALLED_APPS + -- Requires ALERT_STREAMS['topic_handlers'] in settings + """ # logger.info(f'Alert received on topic {metadata.topic}: {alert}; metatdata: {metadata}') alert_as_dict = alert.content photometry_table = alert_as_dict['data'].get('photometry_data', None) @@ -121,6 +129,11 @@ def hermes_alert_handler(alert, metadata): def get_hermes_phot_value(phot_data): + """ + Convert Hermes Message format for a row of Photometry table into parameters accepted by the Reduced Datum model + :param phot_data: + :return: Dictionary containing properly formatted parameters for Reduced_Datum + """ data_dictionary = { 'magnitude': phot_data['brightness'], 'magnitude_error': phot_data['brightnessError'], diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index b7b90122b..183d45d18 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -20,6 +20,7 @@ from django.views.generic.detail import DetailView from django.views.generic.edit import CreateView, DeleteView, FormView from django_filters.views import FilterView +from django.db.models import Q from guardian.shortcuts import assign_perm, get_objects_for_user from tom_common.hooks import run_hook @@ -341,7 +342,13 @@ def post(self, request, *args, **kwargs): message=form_data['share_message'], topic=hermes_topic ) - response = publish_photometry_to_hermes(destination, message_info, reduced_datums) + filtered_reduced_datums = self.get_share_safe_datums(destination, reduced_datums, + topic=hermes_topic) + if filtered_reduced_datums.count() > 0: + response = publish_photometry_to_hermes(destination, message_info, filtered_reduced_datums) + else: + messages.error(self.request, 'No Data to share. (Check sharing Protocol.)') + return redirect('/') else: messages.error(self.request, 'TOM-TOM sharing is not yet supported.') return redirect('/') @@ -399,6 +406,23 @@ def share_with_tom(self, tom_name, product): headers=headers, auth=auth) return response + def get_share_safe_datums(self, destination, reduced_datums, **kwargs): + """ + Custom sharing protocols used to determine when data is shared with a destination. + This example prevents sharing if a datum has already been published to the given Hermes topic. + :param destination: sharing destination string + :param reduced_datums: selected input datums + :return: queryset of reduced datums to be shared + """ + if 'hermes' in destination: + message_topic = kwargs.get('topic', None) + # Remove data points previously shared to the given topic + filtered_datums = reduced_datums.exclude(Q(message__exchange_status='published') + & Q(message__topic=message_topic)) + else: + filtered_datums = reduced_datums + return filtered_datums + class DataProductGroupDetailView(DetailView): """ From 14e1692e5376a07f3b787ff210e08e9c8c4a2fc8 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 8 Dec 2022 16:21:58 -0800 Subject: [PATCH 55/72] Add data table and ability to select individual points. --- tom_dataproducts/models.py | 4 +- .../partials/photometry_for_target.html | 2 +- .../templatetags/dataproduct_extras.py | 47 +++++++++++++++++++ tom_dataproducts/views.py | 6 ++- .../templates/tom_targets/target_detail.html | 1 + 5 files changed, 57 insertions(+), 3 deletions(-) diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index ae613894a..d8fbef1ae 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -312,7 +312,7 @@ class ReducedDatum(models.Model): 'error': .5 } - but could also contain a filter: + but could also contain a filter, a telescope, an instrument, and/or a unit: :: @@ -320,6 +320,8 @@ class ReducedDatum(models.Model): 'magnitude': 18.5, 'magnitude_error': .5, 'filter': 'r' + 'telescope': 'ELP.domeA.1m0a' + 'instrument': 'fa07' } :type value: dict diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_for_target.html index 11b505923..8cfabb63a 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_for_target.html @@ -8,4 +8,4 @@

Photometry

{{ plot|safe }} -
\ No newline at end of file + diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index aeb03ccec..467170e77 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -180,6 +180,53 @@ def recent_photometry(target, limit=1): return context +@register.inclusion_tag('tom_dataproducts/partials/photometry_datalist_for_target.html', takes_context=True) +def get_photometry_data(context, target): + """ + Displays a table of the all photometric points for a target. + """ + photometry = ReducedDatum.objects.filter(data_type='photometry', target=target).order_by('-timestamp') + + # Possibilities for reduced_datums from ZTF/MARS: + # reduced_datum.value: {'error': 0.0929680392146111, 'filter': 'r', 'magnitude': 18.2364940643311} + # reduced_datum.value: {'limit': 20.1023998260498, 'filter': 'g'} + + # for limit magnitudes, set the value of the limit key to True and + # the value of the magnitude key to the limit so the template and + # treat magnitudes as such and prepend a '>' to the limit magnitudes + # see recent_photometry.html + data = [] + for reduced_datum in photometry: + rd_data = {'id': reduced_datum.pk, + 'timestamp': reduced_datum.timestamp, + 'source': reduced_datum.source_name, + 'filter': reduced_datum.value.get('filter', ''), + 'telescope': reduced_datum.value.get('telescope', ''), + 'magnitude_error': reduced_datum.value.get('magnitude_error', '') + } + + if 'limit' in reduced_datum.value.keys(): + rd_data['magnitude'] = reduced_datum.value['limit'] + rd_data['limit'] = True + else: + rd_data['magnitude'] = reduced_datum.value['magnitude'] + rd_data['limit'] = False + data.append(rd_data) + + initial = {'submitter': context['request'].user, + 'target': target, + 'share_title': f"Updated data for {target.name} from {getattr(settings, 'TOM_NAME', 'TOM Toolkit')}.", + } + form = DataShareForm(initial=initial) + form.fields['share_title'].widget = forms.HiddenInput() + + context = {'data': data, + 'target': target, + 'target_data_share_form': form, + 'sharing_destinations': settings.DATA_SHARING} + return context + + @register.inclusion_tag('tom_dataproducts/partials/photometry_for_target.html', takes_context=True) def photometry_for_target(context, target, width=700, height=600, background=None, label_color=None, grid=True): """ diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 183d45d18..d9068e2aa 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -317,6 +317,7 @@ def post(self, request, *args, **kwargs): """ data_share_form = DataShareForm(request.POST, request.FILES) + selected_data = request.POST.getlist("share-box", None) if data_share_form.is_valid(): form_data = data_share_form.cleaned_data # determine if pk is data product, Reduced Datum, or Target. @@ -329,7 +330,10 @@ def post(self, request, *args, **kwargs): target_id = kwargs.get('tg_pk', None) target = Target.objects.get(pk=target_id) data_type = form_data['data_type'] - reduced_datums = ReducedDatum.objects.filter(target=target, data_type=data_type) + if selected_data is None: + reduced_datums = ReducedDatum.objects.filter(target=target, data_type=data_type) + else: + reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) if data_type == 'photometry': share_destination = form_data['share_destination'] if 'HERMES' in share_destination.upper(): diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index 0a29424a1..b58bc942f 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -95,6 +95,7 @@

Observations

{% photometry_for_target target %} + {% get_photometry_data object %}
{% spectroscopy_for_target target %} From 1f275eaaac03233df1a3aff4807fde93dd9745f9 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 8 Dec 2022 17:13:01 -0800 Subject: [PATCH 56/72] add select all option --- .../photometry_datalist_for_target.html | 79 +++++++++++++++++++ .../templatetags/dataproduct_extras.py | 2 + 2 files changed, 81 insertions(+) create mode 100644 tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html new file mode 100644 index 000000000..2260b05b3 --- /dev/null +++ b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html @@ -0,0 +1,79 @@ +{% load bootstrap4 %} +{% load tom_common_extras %} + +
+ {% csrf_token %} + {% for hidden in target_data_share_form.hidden_fields %} + {{ hidden }} + {% endfor %} +
+
+ Photometry Data +
+
+ + + + + + + + + + + + {% for datum in data %} + + + + + + + + + + {% empty %} + + + + {% endfor %} + +
+ + +
TimestampTelescopeFilterMagnitudeErrorSource
{{ datum.timestamp }}{{ datum.telescope }}{{ datum.filter }} + + {% if datum.limit %}>{% endif %} + {{ datum.magnitude|truncate_number }} + {{ datum.magnitude_error }}{{ datum.source }}
No Photometry Data.
+
+
+ Share Selected Data +
+ {% if sharing_destinations %} +
+
+ {% bootstrap_field target_data_share_form.share_destination %} +
+
+ +
+
+ {% else %} + Not Configured + {% endif %} + + +
+
+ + \ No newline at end of file diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 467170e77..54ee994f5 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -215,10 +215,12 @@ def get_photometry_data(context, target): initial = {'submitter': context['request'].user, 'target': target, + 'data_type': 'photometry', 'share_title': f"Updated data for {target.name} from {getattr(settings, 'TOM_NAME', 'TOM Toolkit')}.", } form = DataShareForm(initial=initial) form.fields['share_title'].widget = forms.HiddenInput() + form.fields['data_type'].widget = forms.HiddenInput() context = {'data': data, 'target': target, From 520391a301a49c768514866a10e61efa7998d5fa Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 15 Dec 2022 10:55:07 -0800 Subject: [PATCH 57/72] update comments, fix if no datasharing --- tom_alerts/models.py | 2 +- tom_base/settings.py | 3 +-- tom_dataproducts/alertstreams/hermes.py | 22 ++++++++++----- tom_dataproducts/forms.py | 27 +++++++++++++------ .../photometry_datalist_for_target.html | 2 +- .../templatetags/dataproduct_extras.py | 4 +-- 6 files changed, 39 insertions(+), 21 deletions(-) diff --git a/tom_alerts/models.py b/tom_alerts/models.py index 6b97e8c5c..6c6eac6a0 100644 --- a/tom_alerts/models.py +++ b/tom_alerts/models.py @@ -37,7 +37,7 @@ def __str__(self): class AlertStreamMessage(models.Model): """ - Class representing a streaming message containing data + Class representing a streaming message containing data sent/received either over Kafka or to/from another TOM :param topic: The destination or source of sharing for the message. :type topic: str diff --git a/tom_base/settings.py b/tom_base/settings.py index 87f332195..e01b265ef 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -212,7 +212,6 @@ # # tom_dataproducts configuration # - # Define the valid data product types for your TOM. Be careful when removing items, as previously valid types will no # longer be valid, and may cause issues unless the offending records are modified. DATA_PRODUCT_TYPES = { @@ -227,7 +226,7 @@ 'spectroscopy': 'tom_dataproducts.processors.spectroscopy_processor.SpectroscopyProcessor', } -# Configuration for the TOM receiving data from this TOM +# Configuration for the TOM/Kafka Stream receiving data from this TOM DATA_SHARING = { 'hermes': { 'DISPLAY_NAME': os.getenv('HERMES_DISPLAY_NAME', 'Hermes'), diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index 2d6ebf772..af69a0e7e 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -16,6 +16,9 @@ class BuildHermesMessage(object): + """ + A HERMES Message Object that can be submitted to HOP through HERMES + """ def __init__(self, title='', submitter='', authors='', message='', topic='hermes.test', **kwargs): self.title = title self.submitter = submitter @@ -27,10 +30,10 @@ def __init__(self, title='', submitter='', authors='', message='', topic='hermes def publish_photometry_to_hermes(destination, message_info, datums, **kwargs): """ - For now this code submits a typical hermes photometry alert using the datums tied to the dataproduct being - shared. In the future this should instead send the user to a new tab with a populated hermes form. - :param destination: target stream (topic included?) - :param message_info: Dictionary of message information + Submits a typical hermes photometry alert using the datums supplied to build a photometry table. + -- Stores an AlertStreamMessage connected to each datum to show that the datum has previously been shared. + :param destination: target stream + :param message_info: HERMES Message Object :param datums: Reduced Datums to be built into table. :return: """ @@ -42,7 +45,6 @@ def publish_photometry_to_hermes(destination, message_info, datums, **kwargs): hermes_alert = AlertStreamMessage(topic=message_info.topic, exchange_status='published') hermes_alert.save() for tomtoolkit_photometry in datums: - tomtoolkit_photometry.message.add(hermes_alert) hermes_photometry_data.append(create_hermes_phot_table_row(tomtoolkit_photometry, **kwargs)) alert = { @@ -62,7 +64,8 @@ def publish_photometry_to_hermes(destination, message_info, datums, **kwargs): def create_hermes_phot_table_row(datum, **kwargs): - """Build a row for a Hermes Photometry Table using a TOM Photometry datum""" + """Build a row for a Hermes Photometry Table using a TOM Photometry datum + """ table_row = { 'photometryId': datum.target.name, 'dateObs': datum.timestamp.strftime('%x %X'), @@ -77,6 +80,11 @@ def create_hermes_phot_table_row(datum, **kwargs): def get_hermes_topics(): + """ + Method to retrieve a list of available topics. + TODO: Retrieve list from HOP, currently unavailable due to authentication issues. + :return: List of topics available for users + """ # stream_base_url = settings.DATA_SHARING['hermes']['BASE_URL'] # submit_url = stream_base_url + "api/v0/topics/" # headers = {'SCIMMA-API-Auth-Username': settings.DATA_SHARING['hermes']['CREDENTIAL_USERNAME'], @@ -90,7 +98,7 @@ def get_hermes_topics(): def hermes_alert_handler(alert, metadata): - """Alert Handler to record data streamed through Hermes. + """Alert Handler to record data streamed through Hermes as a new ReducedDatum. -- Only Reads Photometry Data -- Only ingests Data if exact match for Target Name -- Does not Ingest Data if exact match already exists diff --git a/tom_dataproducts/forms.py b/tom_dataproducts/forms.py index e01ea30be..8576d3a94 100644 --- a/tom_dataproducts/forms.py +++ b/tom_dataproducts/forms.py @@ -8,15 +8,26 @@ def get_sharing_destination_options(): + """ + Build the Display options and headers for the dropdown form for choosing sharing topics. + Customize for a different selection experience. + :return: Tuple: Possible Destinations and their Display Names + """ choices = [] - for destination, details in settings.DATA_SHARING.items(): - new_destination = [details.get('DISPLAY_NAME', destination)] - if details.get('USER_TOPICS', None): - topic_list = [(f'{destination}:{topic}', topic) for topic in details['USER_TOPICS']] - new_destination.append(tuple(topic_list)) - else: - new_destination.insert(0, destination) - choices.append(tuple(new_destination)) + try: + for destination, details in settings.DATA_SHARING.items(): + new_destination = [details.get('DISPLAY_NAME', destination)] + if details.get('USER_TOPICS', None): + # If topics exist for a destination (Such as HERMES) give topics as sub-choices + # for non-selectable Destination + topic_list = [(f'{destination}:{topic}', topic) for topic in details['USER_TOPICS']] + new_destination.append(tuple(topic_list)) + else: + # Otherwise just use destination as option + new_destination.insert(0, destination) + choices.append(tuple(new_destination)) + except AttributeError: + pass return tuple(choices) diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html index 2260b05b3..bf046ab14 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html @@ -76,4 +76,4 @@ $('input[name=share-box]').prop('checked', false); } } - \ No newline at end of file + diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 54ee994f5..6f6e8b07f 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -146,7 +146,7 @@ def share_data(context, target): context = {'target': target, 'target_data_share_form': form, - 'sharing_destinations': settings.DATA_SHARING} + 'sharing_destinations': form.fields['share_destination'].choices} return context @@ -225,7 +225,7 @@ def get_photometry_data(context, target): context = {'data': data, 'target': target, 'target_data_share_form': form, - 'sharing_destinations': settings.DATA_SHARING} + 'sharing_destinations': form.fields['share_destination'].choices} return context From b1f00f7ad25b40636e0268a6b70e2f738dabddb0 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 15 Dec 2022 13:41:08 -0800 Subject: [PATCH 58/72] update DATA_SHARING in settings --- tom_base/settings.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tom_base/settings.py b/tom_base/settings.py index e01b265ef..f45380142 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -231,7 +231,11 @@ 'hermes': { 'DISPLAY_NAME': os.getenv('HERMES_DISPLAY_NAME', 'Hermes'), 'BASE_URL': os.getenv('HERMES_BASE_URL', 'https://hermes.lco.global/'), - 'API_TOKEN': os.getenv('HERMES_API_TOKEN', 'set HERMES_API_TOKEN value in environment'), + 'CREDENTIAL_USERNAME': os.getenv('SCIMMA_CREDENTIAL_USERNAME', + 'set SCIMMA_CREDENTIAL_USERNAME value in environment'), + 'CREDENTIAL_PASSWORD': os.getenv('SCIMMA_CREDENTIAL_PASSWORD', + 'set SCIMMA_CREDENTIAL_PASSWORD value in environment'), + 'USER_TOPICS': ['hermes.test', 'tomtoolkit.test'] }, 'tom-demo-dev': { 'DISPLAY_NAME': os.getenv('TOM_DEMO_DISPLAY_NAME', 'TOM Demo Dev'), From 5d08898c57cc9bc505c676ea5ed94e610f28ba54 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Thu, 15 Dec 2022 13:50:21 -0800 Subject: [PATCH 59/72] remove get_data_sharing_destinations --- .../templatetags/dataproduct_extras.py | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index 6f6e8b07f..25ed40714 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -28,24 +28,6 @@ register = template.Library() -def get_data_sharing_destinations(): - """ - Return a list of data sharing destinations from the DATA_SHARING configuration - dictionary in settings.py. - - This should be placed into the context of the inclusion tags that offer to share - DataProducts. Templates should know that None means that DATA_SHARING has not - been configured. - """ - try: - sharing_destinations = settings.DATA_SHARING.keys() - except Exception as ex: - logger.warning(f'{ex.__class__.__name__} while calling DATA_SHARING.keys(): {ex}') - sharing_destinations = None - - return sharing_destinations - - @register.inclusion_tag('tom_dataproducts/partials/dataproduct_list_for_target.html', takes_context=True) def dataproduct_list_for_target(context, target): """ @@ -65,7 +47,7 @@ def dataproduct_list_for_target(context, target): return { 'products': target_products_for_user, 'target': target, - 'sharing_destinations': get_data_sharing_destinations(), + 'sharing_destinations': form.fields['share_destination'].choices, 'data_product_share_form': form } From 1500db1e6437e5ed34f8c7cede0bf369d822d756 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Fri, 16 Dec 2022 08:17:21 -0800 Subject: [PATCH 60/72] update some docstrings. --- tom_dataproducts/alertstreams/hermes.py | 13 ++++++------- tom_dataproducts/views.py | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index af69a0e7e..6544a4329 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -28,16 +28,15 @@ def __init__(self, title='', submitter='', authors='', message='', topic='hermes self.extra_info = kwargs -def publish_photometry_to_hermes(destination, message_info, datums, **kwargs): +def publish_photometry_to_hermes(message_info, datums, **kwargs): """ Submits a typical hermes photometry alert using the datums supplied to build a photometry table. -- Stores an AlertStreamMessage connected to each datum to show that the datum has previously been shared. - :param destination: target stream - :param message_info: HERMES Message Object - :param datums: Reduced Datums to be built into table. - :return: + :param message_info: HERMES Message Object created with BuildHermesMessage + :param datums: Queryset of Reduced Datums to be built into table. + :return: response """ - stream_base_url = settings.DATA_SHARING[destination]['BASE_URL'] + stream_base_url = settings.DATA_SHARING['hermes']['BASE_URL'] submit_url = stream_base_url + 'submit/' headers = {'SCIMMA-API-Auth-Username': settings.DATA_SHARING['hermes']['CREDENTIAL_USERNAME'], 'SCIMMA-API-Auth-Password': settings.DATA_SHARING['hermes']['CREDENTIAL_PASSWORD']} @@ -139,7 +138,7 @@ def hermes_alert_handler(alert, metadata): def get_hermes_phot_value(phot_data): """ Convert Hermes Message format for a row of Photometry table into parameters accepted by the Reduced Datum model - :param phot_data: + :param phot_data: Dictionary containing Hermes Photometry table. :return: Dictionary containing properly formatted parameters for Reduced_Datum """ data_dictionary = { diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index d9068e2aa..2d395bcb5 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -349,7 +349,7 @@ def post(self, request, *args, **kwargs): filtered_reduced_datums = self.get_share_safe_datums(destination, reduced_datums, topic=hermes_topic) if filtered_reduced_datums.count() > 0: - response = publish_photometry_to_hermes(destination, message_info, filtered_reduced_datums) + response = publish_photometry_to_hermes(message_info, filtered_reduced_datums) else: messages.error(self.request, 'No Data to share. (Check sharing Protocol.)') return redirect('/') From e4b809c3fa6cb3eed67d38b59272debe01872b21 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Fri, 16 Dec 2022 10:25:37 -0800 Subject: [PATCH 61/72] add a few comments --- tom_dataproducts/views.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 2d395bcb5..a15b808d0 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -312,15 +312,17 @@ def form_invalid(self, form): def post(self, request, *args, **kwargs): """ Method that handles the POST requests for sharing data. - Handles Data Products and All the data of a type for a target. - Submit to Hermes, or Share with TOM. + Handles Data Products and All the data of a type for a target as well as individual Reduced Datums. + Submit to Hermes, or Share with TOM (soon). """ data_share_form = DataShareForm(request.POST, request.FILES) + # Check if data points have been selected. selected_data = request.POST.getlist("share-box", None) if data_share_form.is_valid(): form_data = data_share_form.cleaned_data - # determine if pk is data product, Reduced Datum, or Target. + # 1st determine if pk is data product, Reduced Datum, or Target. + # Then query relevant Reduced Datums Queryset product_id = kwargs.get('dp_pk', None) if product_id: product = DataProduct.objects.get(pk=product_id) @@ -346,6 +348,7 @@ def post(self, request, *args, **kwargs): message=form_data['share_message'], topic=hermes_topic ) + # Run ReducedDatums Queryset through sharing protocols to make sure they are safe to share. filtered_reduced_datums = self.get_share_safe_datums(destination, reduced_datums, topic=hermes_topic) if filtered_reduced_datums.count() > 0: From 7d2208244fd952409ee2de27da2375e4067bd4bd Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Fri, 16 Dec 2022 14:16:13 -0800 Subject: [PATCH 62/72] redirect after submit to target page --- tom_dataproducts/alertstreams/hermes.py | 4 +++- tom_dataproducts/views.py | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index 6544a4329..481279394 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -80,7 +80,9 @@ def create_hermes_phot_table_row(datum, **kwargs): def get_hermes_topics(): """ - Method to retrieve a list of available topics. + !CURRENTLY UNUSED! + Method to retrieve a list of available topics from HOP. + Intended to be called from forms when building topic list. TODO: Retrieve list from HOP, currently unavailable due to authentication issues. :return: List of topics available for users """ diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index a15b808d0..700797b72 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -339,7 +339,7 @@ def post(self, request, *args, **kwargs): if data_type == 'photometry': share_destination = form_data['share_destination'] if 'HERMES' in share_destination.upper(): - # build and submit hermes table from Reduced Datums + # Build and submit hermes table from Reduced Datums hermes_topic = share_destination.split(':')[1] destination = share_destination.split(':')[0] message_info = BuildHermesMessage(title=form_data['share_title'], @@ -355,10 +355,10 @@ def post(self, request, *args, **kwargs): response = publish_photometry_to_hermes(message_info, filtered_reduced_datums) else: messages.error(self.request, 'No Data to share. (Check sharing Protocol.)') - return redirect('/') + return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) else: messages.error(self.request, 'TOM-TOM sharing is not yet supported.') - return redirect('/') + return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) # response = self.share_with_tom(share_destination, product) publish_feedback = response.json()["message"] if "ERROR" in publish_feedback.upper(): @@ -367,7 +367,7 @@ def post(self, request, *args, **kwargs): messages.success(self.request, publish_feedback) else: messages.error(self.request, f'Publishing {data_type} data is not yet supported.') - return redirect('/') + return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) def share_with_tom(self, tom_name, product): """ From 4698dcf45f0f7e4ecf0f1cc9627c6043db5340c7 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Fri, 16 Dec 2022 16:10:57 -0800 Subject: [PATCH 63/72] update docs --- .../customizing_data_sharing.rst | 36 +++++++++++++++++++ docs/managing_data/index.rst | 3 ++ 2 files changed, 39 insertions(+) create mode 100644 docs/managing_data/customizing_data_sharing.rst diff --git a/docs/managing_data/customizing_data_sharing.rst b/docs/managing_data/customizing_data_sharing.rst new file mode 100644 index 000000000..dbee9b57b --- /dev/null +++ b/docs/managing_data/customizing_data_sharing.rst @@ -0,0 +1,36 @@ +Customizing Data Sharing +--------------------------- + +Data sharing is Possible currently only with HERMES. + +You will need to add ``DATA_SHARING`` to your ``settings.py`` that will give the proper credentials for the various +streams, TOMS, etc. with which you desire to share data. + +.. code:: python + + # Define the valid data sharing destinations for your TOM. + DATA_SHARING = { + 'hermes': { + 'DISPLAY_NAME': os.getenv('HERMES_DISPLAY_NAME', 'Hermes'), + 'BASE_URL': os.getenv('HERMES_BASE_URL', 'https://hermes.lco.global/'), + 'CREDENTIAL_USERNAME': os.getenv('SCIMMA_CREDENTIAL_USERNAME', + 'set SCIMMA_CREDENTIAL_USERNAME value in environment'), + 'CREDENTIAL_PASSWORD': os.getenv('SCIMMA_CREDENTIAL_PASSWORD', + 'set SCIMMA_CREDENTIAL_PASSWORD value in environment'), + 'USER_TOPICS': ['hermes.test', 'tomtoolkit.test'] + }, + 'tom-demo-dev': { + 'DISPLAY_NAME': os.getenv('TOM_DEMO_DISPLAY_NAME', 'TOM Demo Dev'), + 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), + 'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'), + 'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'), + }, + 'localhost-tom': { + # for testing; share with yourself + 'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'), + 'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'), + 'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'), + 'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'), + } + + } diff --git a/docs/managing_data/index.rst b/docs/managing_data/index.rst index 86a7108d4..37d511a29 100644 --- a/docs/managing_data/index.rst +++ b/docs/managing_data/index.rst @@ -9,6 +9,7 @@ Managing Data ../api/tom_dataproducts/views plotting_data customizing_data_processing + customizing_data_sharing :doc:`Creating Plots from TOM Data ` - Learn how to create plots using plot.ly and your TOM @@ -16,3 +17,5 @@ data to display anywhere in your TOM. :doc:`Adding Custom Data Processing ` - Learn how you can process data into your TOM from uploaded data products. + +:doc:`Adding Custom Data Sharing ` - Learn how you can share data from your TOM. From a57ce0dde1132798daf3c96f16a894a9c7df81d0 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Fri, 6 Jan 2023 16:32:05 -0800 Subject: [PATCH 64/72] update for new HERMES validation, disable selected_share button when nothing selected --- tom_dataproducts/alertstreams/hermes.py | 48 ++++++++++++------- .../photometry_datalist_for_target.html | 15 +++++- tom_dataproducts/views.py | 29 ++++++----- 3 files changed, 62 insertions(+), 30 deletions(-) diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index 481279394..004f3e916 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -1,5 +1,6 @@ import logging from datetime import datetime +from dateutil.parser import parse from django.conf import settings @@ -37,7 +38,7 @@ def publish_photometry_to_hermes(message_info, datums, **kwargs): :return: response """ stream_base_url = settings.DATA_SHARING['hermes']['BASE_URL'] - submit_url = stream_base_url + 'submit/' + submit_url = stream_base_url + 'api/v0/' + 'submit_photometry/' headers = {'SCIMMA-API-Auth-Username': settings.DATA_SHARING['hermes']['CREDENTIAL_USERNAME'], 'SCIMMA-API-Auth-Password': settings.DATA_SHARING['hermes']['CREDENTIAL_PASSWORD']} hermes_photometry_data = [] @@ -49,10 +50,10 @@ def publish_photometry_to_hermes(message_info, datums, **kwargs): alert = { 'topic': message_info.topic, 'title': message_info.title, - 'author': message_info.submitter, + 'submitter': message_info.submitter, 'data': { 'authors': message_info.authors, - 'photometry_data': hermes_photometry_data, + 'photometry': hermes_photometry_data, }, 'message_text': message_info.message, } @@ -66,15 +67,22 @@ def create_hermes_phot_table_row(datum, **kwargs): """Build a row for a Hermes Photometry Table using a TOM Photometry datum """ table_row = { - 'photometryId': datum.target.name, - 'dateObs': datum.timestamp.strftime('%x %X'), + 'target_name': datum.target.name, + 'ra': datum.target.ra, + 'dec': datum.target.dec, + 'date': datum.timestamp.strftime('%x %X'), 'telescope': datum.value.get('telescope', ''), 'instrument': datum.value.get('instrument', ''), 'band': datum.value.get('filter', ''), - 'brightness': datum.value.get('magnitude', ''), - 'brightnessError': datum.value.get('magnitude_error', ''), - 'brightnessUnit': datum.value.get('unit', 'AB mag'), + 'brightness_unit': datum.value.get('unit', 'AB mag'), } + if datum.value.get('magnitude', None): + table_row['brightness'] = datum.value['magnitude'] + else: + table_row['brightness'] = datum.value['limit'] + table_row['nondetection'] = True + if datum.value.get('magnitude_error', None): + table_row['brightness_error'] = datum.value['magnitude_error'] return table_row @@ -108,18 +116,18 @@ def hermes_alert_handler(alert, metadata): """ # logger.info(f'Alert received on topic {metadata.topic}: {alert}; metatdata: {metadata}') alert_as_dict = alert.content - photometry_table = alert_as_dict['data'].get('photometry_data', None) + print(alert_as_dict) + photometry_table = alert_as_dict['data'].get('photometry', None) if photometry_table: hermes_alert = AlertStreamMessage(topic=alert_as_dict['topic'], exchange_status='ingested') - hermes_alert.save() for row in photometry_table: try: - target = Target.objects.get(name=row['photometryId']) + target = Target.objects.get(name=row['target_name']) except Target.DoesNotExist: continue try: - obs_date = datetime.strptime(row['dateObs'].strip(), '%x %X') + obs_date = parse(row['date']) except ValueError: continue @@ -133,6 +141,7 @@ def hermes_alert_handler(alert, metadata): } new_rd, created = ReducedDatum.objects.get_or_create(**datum) if created: + hermes_alert.save() new_rd.message.add(hermes_alert) new_rd.save() @@ -144,11 +153,16 @@ def get_hermes_phot_value(phot_data): :return: Dictionary containing properly formatted parameters for Reduced_Datum """ data_dictionary = { - 'magnitude': phot_data['brightness'], - 'magnitude_error': phot_data['brightnessError'], + 'magnitude_error': phot_data.get('brightness_error', ''), 'filter': phot_data['band'], - 'telescope': phot_data['telescope'], - 'instrument': phot_data['instrument'], - 'unit': phot_data['brightnessUnit'], + 'telescope': phot_data.get('telescope', ''), + 'instrument': phot_data.get('instrument', ''), + 'unit': phot_data['brightness_unit'], } + + if not phot_data.get('nondetection', False): + data_dictionary['magnitude'] = phot_data['brightness'] + else: + data_dictionary['limit'] = phot_data['brightness'] + return data_dictionary diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html index bf046ab14..b59cee1f5 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html @@ -27,7 +27,7 @@ {% for datum in data %} - + {{ datum.timestamp }} {{ datum.telescope }} {{ datum.filter }} @@ -56,7 +56,7 @@ {% bootstrap_field target_data_share_form.share_destination %}
- +
{% else %} @@ -76,4 +76,15 @@ $('input[name=share-box]').prop('checked', false); } } + function check_selected() { + var share_boxes = document.getElementsByName("share-box"); + var submit_btn = document.getElementById('submit_selected'); + for (const box of share_boxes) { + if(box.checked == true) { + submit_btn.disabled = false; + return; + } + } + submit_btn.disabled = true; + } diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 700797b72..eb7f65e98 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -318,7 +318,7 @@ def post(self, request, *args, **kwargs): data_share_form = DataShareForm(request.POST, request.FILES) # Check if data points have been selected. - selected_data = request.POST.getlist("share-box", None) + selected_data = request.POST.getlist("share-box") if data_share_form.is_valid(): form_data = data_share_form.cleaned_data # 1st determine if pk is data product, Reduced Datum, or Target. @@ -332,7 +332,7 @@ def post(self, request, *args, **kwargs): target_id = kwargs.get('tg_pk', None) target = Target.objects.get(pk=target_id) data_type = form_data['data_type'] - if selected_data is None: + if request.POST.get("share-box", None) is None: reduced_datums = ReducedDatum.objects.filter(target=target, data_type=data_type) else: reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) @@ -360,7 +360,13 @@ def post(self, request, *args, **kwargs): messages.error(self.request, 'TOM-TOM sharing is not yet supported.') return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) # response = self.share_with_tom(share_destination, product) - publish_feedback = response.json()["message"] + try: + if 'message' in response.json(): + publish_feedback = response.json()['message'] + else: + publish_feedback = f"ERROR: {response.text}" + except ValueError: + publish_feedback = f"ERROR: Returned Response code {response.status_code}" if "ERROR" in publish_feedback.upper(): messages.error(self.request, publish_feedback) else: @@ -421,14 +427,15 @@ def get_share_safe_datums(self, destination, reduced_datums, **kwargs): :param reduced_datums: selected input datums :return: queryset of reduced datums to be shared """ - if 'hermes' in destination: - message_topic = kwargs.get('topic', None) - # Remove data points previously shared to the given topic - filtered_datums = reduced_datums.exclude(Q(message__exchange_status='published') - & Q(message__topic=message_topic)) - else: - filtered_datums = reduced_datums - return filtered_datums + return reduced_datums + # if 'hermes' in destination: + # message_topic = kwargs.get('topic', None) + # # Remove data points previously shared to the given topic + # filtered_datums = reduced_datums.exclude(Q(message__exchange_status='published') + # & Q(message__topic=message_topic)) + # else: + # filtered_datums = reduced_datums + # return filtered_datums class DataProductGroupDetailView(DetailView): From a313484cc8a9a156d85a493160da3f73fe8ae9cf Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Fri, 6 Jan 2023 16:34:02 -0800 Subject: [PATCH 65/72] fix small bug with select_all --- .../partials/photometry_datalist_for_target.html | 1 + 1 file changed, 1 insertion(+) diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html index b59cee1f5..953af3b88 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html @@ -75,6 +75,7 @@ } else { $('input[name=share-box]').prop('checked', false); } + check_selected() } function check_selected() { var share_boxes = document.getElementsByName("share-box"); From a88781e68ea781c820b28620d46bc17b8f05fb8c Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Sat, 7 Jan 2023 10:27:57 -0800 Subject: [PATCH 66/72] remove some unused imports --- tom_dataproducts/alertstreams/hermes.py | 1 - tom_dataproducts/views.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index 004f3e916..3c1191444 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -1,5 +1,4 @@ import logging -from datetime import datetime from dateutil.parser import parse from django.conf import settings diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index eb7f65e98..73cb920a8 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -20,7 +20,6 @@ from django.views.generic.detail import DetailView from django.views.generic.edit import CreateView, DeleteView, FormView from django_filters.views import FilterView -from django.db.models import Q from guardian.shortcuts import assign_perm, get_objects_for_user from tom_common.hooks import run_hook From fd6dfc9ece558941b7217092add31ea87d3c8b2d Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Mon, 9 Jan 2023 22:34:42 -0800 Subject: [PATCH 67/72] fix some typos and schema issues --- docs/managing_data/customizing_data_sharing.rst | 2 +- tom_dataproducts/alertstreams/hermes.py | 10 +++++----- .../tom_dataproducts/partials/share_target_data.html | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/managing_data/customizing_data_sharing.rst b/docs/managing_data/customizing_data_sharing.rst index dbee9b57b..f5fdace13 100644 --- a/docs/managing_data/customizing_data_sharing.rst +++ b/docs/managing_data/customizing_data_sharing.rst @@ -1,7 +1,7 @@ Customizing Data Sharing --------------------------- -Data sharing is Possible currently only with HERMES. +Currently, data sharing is only possible with HERMES. You will need to add ``DATA_SHARING`` to your ``settings.py`` that will give the proper credentials for the various streams, TOMS, etc. with which you desire to share data. diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index 3c1191444..3f7fff777 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -50,13 +50,13 @@ def publish_photometry_to_hermes(message_info, datums, **kwargs): 'topic': message_info.topic, 'title': message_info.title, 'submitter': message_info.submitter, + 'authors': message_info.authors, 'data': { - 'authors': message_info.authors, 'photometry': hermes_photometry_data, + 'extra_info': message_info.extra_info }, 'message_text': message_info.message, } - alert['data'].update(message_info.extra_info) response = requests.post(url=submit_url, json=alert, headers=headers) return response @@ -69,7 +69,7 @@ def create_hermes_phot_table_row(datum, **kwargs): 'target_name': datum.target.name, 'ra': datum.target.ra, 'dec': datum.target.dec, - 'date': datum.timestamp.strftime('%x %X'), + 'date': datum.timestamp.isoformat(), 'telescope': datum.value.get('telescope', ''), 'instrument': datum.value.get('instrument', ''), 'band': datum.value.get('filter', ''), @@ -78,7 +78,7 @@ def create_hermes_phot_table_row(datum, **kwargs): if datum.value.get('magnitude', None): table_row['brightness'] = datum.value['magnitude'] else: - table_row['brightness'] = datum.value['limit'] + table_row['brightness'] = datum.value.get('limit', None) table_row['nondetection'] = True if datum.value.get('magnitude_error', None): table_row['brightness_error'] = datum.value['magnitude_error'] @@ -94,7 +94,7 @@ def get_hermes_topics(): :return: List of topics available for users """ # stream_base_url = settings.DATA_SHARING['hermes']['BASE_URL'] - # submit_url = stream_base_url + "api/v0/topics/" + # submit_url = stream_base_url + "api/v0/profile/" # headers = {'SCIMMA-API-Auth-Username': settings.DATA_SHARING['hermes']['CREDENTIAL_USERNAME'], # 'SCIMMA-API-Auth-Password': settings.DATA_SHARING['hermes']['CREDENTIAL_PASSWORD']} # user = settings.DATA_SHARING['hermes']['SCIMMA_AUTH_USERNAME'] diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html b/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html index 60ecef955..bcccaabc4 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/share_target_data.html @@ -3,7 +3,7 @@ {% load tom_common_extras %}
- Publish Data for {{target.name}}: + Publish Data for {{ target.name }}:
{% if sharing_destinations %}
From de9bb593ac38e55e2abdac1a4f26dd0af73b6424 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Mon, 9 Jan 2023 22:37:17 -0800 Subject: [PATCH 68/72] missed a typo --- .../tom_dataproducts/partials/dataproduct_list_for_target.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html index 7fed82617..35f8b20ca 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html @@ -38,7 +38,7 @@

Data

Delete - + {% csrf_token %} From 26ca592b74801ef70ac56ee14b90388f2fad0b18 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Mon, 9 Jan 2023 22:38:17 -0800 Subject: [PATCH 69/72] missed another typo --- .../tom_dataproducts/partials/dataproduct_list_for_target.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html index 35f8b20ca..eb119fd8a 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/dataproduct_list_for_target.html @@ -31,7 +31,7 @@

Data

{% if sharing_destinations %} - + {% else %}

Not Configured

{% endif %} From 2a13bf82c1a6a9fd9754f12bbc4cfecd458bd695 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 11 Jan 2023 14:16:02 -0800 Subject: [PATCH 70/72] add TODO for source_location --- tom_dataproducts/alertstreams/hermes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index 3f7fff777..93cab237f 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -115,7 +115,6 @@ def hermes_alert_handler(alert, metadata): """ # logger.info(f'Alert received on topic {metadata.topic}: {alert}; metatdata: {metadata}') alert_as_dict = alert.content - print(alert_as_dict) photometry_table = alert_as_dict['data'].get('photometry', None) if photometry_table: hermes_alert = AlertStreamMessage(topic=alert_as_dict['topic'], exchange_status='ingested') @@ -134,7 +133,7 @@ def hermes_alert_handler(alert, metadata): 'target': target, 'data_type': 'photometry', 'source_name': alert_as_dict['topic'], - 'source_location': 'HERMES', + 'source_location': 'Hermes via HOP', # TODO Add message URL here once message ID's exist 'timestamp': obs_date, 'value': get_hermes_phot_value(row) } From 7e23b0a0a6950ff444919af77a22c37b876f9481 Mon Sep 17 00:00:00 2001 From: Joseph Chatelain Date: Wed, 11 Jan 2023 14:28:13 -0800 Subject: [PATCH 71/72] remove unused log message --- tom_dataproducts/alertstreams/hermes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index 93cab237f..a44095e0e 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -113,7 +113,6 @@ def hermes_alert_handler(alert, metadata): -- Requires 'tom_alertstreams' in settings.INSTALLED_APPS -- Requires ALERT_STREAMS['topic_handlers'] in settings """ - # logger.info(f'Alert received on topic {metadata.topic}: {alert}; metatdata: {metadata}') alert_as_dict = alert.content photometry_table = alert_as_dict['data'].get('photometry', None) if photometry_table: From a81f993782de21632df730e1f3f981401e968de9 Mon Sep 17 00:00:00 2001 From: Lindy Date: Fri, 27 Jan 2023 22:50:11 +0000 Subject: [PATCH 72/72] incomplete first draft of expanded data sharing docs (#590) * incomplete first draft of expanded data sharing docs * update sharing documentation * improved data sharing docs --------- Co-authored-by: Joseph Chatelain --- .../customizing_data_sharing.rst | 36 --------- docs/managing_data/index.rst | 9 ++- docs/managing_data/stream_pub_sub.rst | 78 +++++++++++++++++++ docs/managing_data/tom_direct_sharing.rst | 32 ++++++++ 4 files changed, 117 insertions(+), 38 deletions(-) delete mode 100644 docs/managing_data/customizing_data_sharing.rst create mode 100644 docs/managing_data/stream_pub_sub.rst create mode 100644 docs/managing_data/tom_direct_sharing.rst diff --git a/docs/managing_data/customizing_data_sharing.rst b/docs/managing_data/customizing_data_sharing.rst deleted file mode 100644 index f5fdace13..000000000 --- a/docs/managing_data/customizing_data_sharing.rst +++ /dev/null @@ -1,36 +0,0 @@ -Customizing Data Sharing ---------------------------- - -Currently, data sharing is only possible with HERMES. - -You will need to add ``DATA_SHARING`` to your ``settings.py`` that will give the proper credentials for the various -streams, TOMS, etc. with which you desire to share data. - -.. code:: python - - # Define the valid data sharing destinations for your TOM. - DATA_SHARING = { - 'hermes': { - 'DISPLAY_NAME': os.getenv('HERMES_DISPLAY_NAME', 'Hermes'), - 'BASE_URL': os.getenv('HERMES_BASE_URL', 'https://hermes.lco.global/'), - 'CREDENTIAL_USERNAME': os.getenv('SCIMMA_CREDENTIAL_USERNAME', - 'set SCIMMA_CREDENTIAL_USERNAME value in environment'), - 'CREDENTIAL_PASSWORD': os.getenv('SCIMMA_CREDENTIAL_PASSWORD', - 'set SCIMMA_CREDENTIAL_PASSWORD value in environment'), - 'USER_TOPICS': ['hermes.test', 'tomtoolkit.test'] - }, - 'tom-demo-dev': { - 'DISPLAY_NAME': os.getenv('TOM_DEMO_DISPLAY_NAME', 'TOM Demo Dev'), - 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), - 'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'), - 'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'), - }, - 'localhost-tom': { - # for testing; share with yourself - 'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'), - 'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'), - 'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'), - 'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'), - } - - } diff --git a/docs/managing_data/index.rst b/docs/managing_data/index.rst index 37d511a29..78979a9f5 100644 --- a/docs/managing_data/index.rst +++ b/docs/managing_data/index.rst @@ -9,7 +9,8 @@ Managing Data ../api/tom_dataproducts/views plotting_data customizing_data_processing - customizing_data_sharing + tom_direct_sharing + stream_pub_sub :doc:`Creating Plots from TOM Data ` - Learn how to create plots using plot.ly and your TOM @@ -18,4 +19,8 @@ data to display anywhere in your TOM. :doc:`Adding Custom Data Processing ` - Learn how you can process data into your TOM from uploaded data products. -:doc:`Adding Custom Data Sharing ` - Learn how you can share data from your TOM. +:doc:`TOM-TOM Direct Sharing ` - Learn how you can send and receive data between your TOM and another TOM-Toolkit TOM via an API. + +:doc:`Publish and Subscribe to a Kafka Stream ` - Learn how to publish and subscribe to a Kafka stream topic. + + diff --git a/docs/managing_data/stream_pub_sub.rst b/docs/managing_data/stream_pub_sub.rst new file mode 100644 index 000000000..2e072b5a3 --- /dev/null +++ b/docs/managing_data/stream_pub_sub.rst @@ -0,0 +1,78 @@ +Publish and Subscribe to a Kafka Stream +--------------------------------------- + +Publishing data to a stream and subscribing to a stream are handled independently and we describe each below. + + +Publish Data to a Kafka Topic +############################# + +TOM Toolkit supports publishing data to a Kafka stream such as `Hermes `_ (an interface to +`HOPSKOTCH `_) and `GCNClassicOverKafka `_. + +When sharing photometry data via Hermes, the TOM publishes the data to be shared to a topic on the HOPSKOTCH +Kafka stream. At this time, only photometry data is supported. + + +Configuring your TOM to Publish Data to a stream: +************************************************* + +You will need to add a ``DATA_SHARING`` configuration dictionary to your ``settings.py`` that gives the credentials +for the various streams with which you wish to share data. + +.. code:: python + + # Define the valid data sharing destinations for your TOM. + DATA_SHARING = { + 'hermes': { + 'DISPLAY_NAME': os.getenv('HERMES_DISPLAY_NAME', 'Hermes'), + 'BASE_URL': os.getenv('HERMES_BASE_URL', 'https://hermes.lco.global/'), + 'CREDENTIAL_USERNAME': os.getenv('SCIMMA_CREDENTIAL_USERNAME', + 'set SCIMMA_CREDENTIAL_USERNAME value in environment'), + 'CREDENTIAL_PASSWORD': os.getenv('SCIMMA_CREDENTIAL_PASSWORD', + 'set SCIMMA_CREDENTIAL_PASSWORD value in environment'), + 'USER_TOPICS': ['hermes.test', 'tomtoolkit.test'] + }, + } + +Subscribe to a Kafka Topic +########################## + +TOM Toolkit allows a TOM to subscribe to a topic on a Kafka stream, ingesting messages from that topic and handling the data. +This could involve simply logging the message or extracting the data from the message and saving it if it is properly formatted. + +Configuring your TOM to subscribe to a stream: +********************************************** + +First you will need to add ``tom_alertstreams`` to your list of ``INSTALLED_APPS`` in your ``settings.py``. + +.. code:: python + + INSTALLED_APPS = [ + ... + 'tom_alertstreams', + ] + +Then you will need to add an ``ALERT_STREAMS`` configuration dictionary to your ``settings.py``. This gives the credentials +for the various streams to which you wish to subscribe. Additionally, the ``TOPIC_HANDLERS`` section of the stream ``OPTIONS`` +will include a list of handlers for each topic. + +Some alert handlers are included as examples. Below we demonstrate how to connect to a Hermes Topic. You'll want to check +out the ``tom-alertstreams`` `README `_ for more details. + +.. code:: python + + ALERT_STREAMS = [ + { + 'ACTIVE': True, + 'NAME': 'tom_alertstreams.alertstreams.hopskotch.HopskotchAlertStream', + 'OPTIONS': { + 'URL': 'kafka://kafka.scimma.org/', + 'USERNAME': os.getenv('SCIMMA_CREDENTIAL_USERNAME', 'set SCIMMA_CREDENTIAL_USERNAME value in environment'), + 'PASSWORD': os.getenv('SCIMMA_CREDENTIAL_PASSWORD', 'set SCIMMA_CREDENTIAL_USERNAME value in environment'), + 'TOPIC_HANDLERS': { + 'tomtoolkit.test': 'tom_dataproducts.alertstreams.hermes.hermes_alert_handler', + }, + }, + }, + ] diff --git a/docs/managing_data/tom_direct_sharing.rst b/docs/managing_data/tom_direct_sharing.rst new file mode 100644 index 000000000..ee04cb73e --- /dev/null +++ b/docs/managing_data/tom_direct_sharing.rst @@ -0,0 +1,32 @@ +Sharing Data with Other TOMs +############################ + +TOM Toolkit does not yet support direct sharing between TOMs, however we hope to add this functionality soon. + + +.. Configuring your TOM to submit data to another TOM: +.. *************************************************** +.. +.. You will need to add a ``DATA_SHARING`` configuration dictionary to your ``settings.py`` that gives the credentials +.. for the various TOMs with which you wish to share data. +.. +.. .. code:: python +.. +.. # Define the valid data sharing destinations for your TOM. +.. DATA_SHARING = { +.. 'tom-demo-dev': { +.. 'DISPLAY_NAME': os.getenv('TOM_DEMO_DISPLAY_NAME', 'TOM Demo Dev'), +.. 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), +.. 'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'), +.. 'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'), +.. }, +.. 'localhost-tom': { +.. # for testing; share with yourself +.. 'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'), +.. 'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'), +.. 'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'), +.. 'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'), +.. } +.. +.. } +.. \ No newline at end of file