diff --git a/docs/managing_data/forced_photometry.rst b/docs/managing_data/forced_photometry.rst new file mode 100644 index 000000000..a9e9f7665 --- /dev/null +++ b/docs/managing_data/forced_photometry.rst @@ -0,0 +1,128 @@ +Integrating Forced Photometry Service Queries +--------------------------------------- + +The base TOM Toolkit comes with Atlas, panSTARRS, and ZTF query services. More services +can be added by extending the base ForcedPhotometryService implementation. + + +Integrating existing Forced Photometry Services +############################################### + +You must add certain configuration to your TOM's ``settings.py`` to setup the existing forced +photometry services. This configuration will go in the ``FORCED_PHOTOMETRY_SERVICES`` section +shown below: + +.. code:: python + FORCED_PHOTOMETRY_SERVICES = { + 'atlas': { + 'class': 'tom_dataproducts.forced_photometry.atlas.AtlasForcedPhotometryService', + 'url': "https://fallingstar-data.com/forcedphot", + 'api_key': os.getenv('ATLAS_FORCED_PHOTOMETRY_API_KEY', 'your atlas account api token') + }, + 'panstarrs': { + #TODO + }, + 'ztf': { + #TODO + } + } + + DATA_PRODUCT_TYPES = { + ... + 'atlas_photometry': ('atlas_photometry', 'Atlas Photometry'), + ... + } + + DATA_PROCESSORS = { + ... + 'atlas_photometry': 'tom_dataproducts.processors.atlas_processor.AtlasProcessor', + ... + } + + +Configuring your TOM to serve tasks asynchronously: +*************************************************** + +Several of the services are best suited to be queried asynchronously, especially if you plan to make large +queries that would take a long time. The TOM Toolkit is setup to use `dramatiq `_ +as an asynchronous task manager, but doing so requires you to run either a `redis `_ +or `rabbitmq `_ server to act as the task queue. To use dramatiq with +a redis server, you would add the following to your ``settings.py``: + +.. code:: python + INSTALLED_APPS = [ + ... + 'django_dramatiq', + ... + ] + + DRAMATIQ_BROKER = { + "BROKER": "dramatiq.brokers.redis.RedisBroker", + "OPTIONS": { + "url": "redis://your-redis-service-url:your-redis-port" + }, + "MIDDLEWARE": [ + "dramatiq.middleware.AgeLimit", + "dramatiq.middleware.TimeLimit", + "dramatiq.middleware.Callbacks", + "dramatiq.middleware.Retries", + "django_dramatiq.middleware.DbConnectionsMiddleware", + ] + } + +After adding the ``django_dramatiq`` installed app, you will need to run ``./manage.py migrate`` once to setup +its DB tables. If this configuration is set in your TOM, the existing services which support asynchronous queries, +Atlas and ZTF, should start querying asynchronously. If you do not add these settings, those services will still +function but will fall back to synchronous queries. + + +Adding a new Forced Photometry Service +###################################### + +The Forced Photometry services fulfill an interface defined in +`BaseForcedPhotometryService `_. +To implement your own Forced Photometry service, you need to do 3 things: +1. Subclass BaseForcedPhotometryService +2. Subclass BaseForcedPhotometryQueryForm +3. Subclass DataProcessor +Once those are implemented, don't forget to update your settings for ``FORCED_PHOTOMETRY_SERVICES``, +``DATA_PRODUCT_TYPES``, and ``DATA_PROCESSORS`` for your new service and its associated data product type. + + +Subclass BaseForcedPhotometryService: +************************************* + +The most important method here is the ``query_service`` method which is where you put your service's business logic +for making the query, given the form parameters and target. This method is expected to create a DataProduct in the database +at the end of the query, storing the result file or files. If queries to your service are expected to take a long time and +you would like to make them asynchronously (not blocking the UI while calling), then follow the example in the +`atlas implementation `_ and place your +actual asynchronous query method in your module's ``tasks.py`` file so it can be found by dramatiq. Like in the atlas implementation, +your code should check to see if ``django_dramatiq`` is in the settings ``INSTALLED_APPS`` before trying to enqueue it with dramatiq. + +The ``get_data_product_type`` method should return the name of your new data product type you are going to define a +DataProcessor for. This must match the name you add to ``DATA_PROCESSORS`` and ``DATA_PRODUCT_TYPES`` in your ``settings.py``. +You will also need to define a `DataProcessor ` +for this data type. + + +Subclass BaseForcedPhotometryQueryForm: +*************************************** + +This class defines the form users will need to fill out to query the service. It uses +`django-crispy-forms `_ to define the layout +programmatically. You first will add whatever form fields you need to the base of your +subclass, and then just fill in the ``layout()`` method with a django-crispy-forms layout +for your fields, and optionally the ``clean()`` method if you want to perform any field validation. +The values of the fields from this form will be available to you in your service class in the +``query_service`` method. + + +Subclass DataProcessor: +*********************** + +You must create a custom DataProcessor that knows how to convert data returned from your service into +a series of either photometry or spectroscopy datums. Without defining this step, your queries will still +result in a DataProduct file being stored from the service's ``query_service`` method, but those files will +not be parsed into photometry or spectroscopy datums. You can read more about how to implement a custom +DataProcessor `here <../customizing_data_processing>`_. \ No newline at end of file diff --git a/docs/managing_data/index.rst b/docs/managing_data/index.rst index 78979a9f5..5b4bcb46a 100644 --- a/docs/managing_data/index.rst +++ b/docs/managing_data/index.rst @@ -11,6 +11,7 @@ Managing Data customizing_data_processing tom_direct_sharing stream_pub_sub + forced_photometry :doc:`Creating Plots from TOM Data ` - Learn how to create plots using plot.ly and your TOM @@ -23,4 +24,5 @@ TOM from uploaded data products. :doc:`Publish and Subscribe to a Kafka Stream ` - Learn how to publish and subscribe to a Kafka stream topic. - +:doc:`Integrating Forced Photometry Service Queries ` - Learn how to integrate the existing Atlas, panSTARRS, and ZTF +forced photometry services into your TOM, and learn how to add new services. diff --git a/docs/managing_data/tom_direct_sharing.rst b/docs/managing_data/tom_direct_sharing.rst index ee04cb73e..bc0edb2ca 100644 --- a/docs/managing_data/tom_direct_sharing.rst +++ b/docs/managing_data/tom_direct_sharing.rst @@ -1,32 +1,74 @@ Sharing Data with Other TOMs ############################ -TOM Toolkit does not yet support direct sharing between TOMs, however we hope to add this functionality soon. - - -.. Configuring your TOM to submit data to another TOM: -.. *************************************************** -.. -.. You will need to add a ``DATA_SHARING`` configuration dictionary to your ``settings.py`` that gives the credentials -.. for the various TOMs with which you wish to share data. -.. -.. .. code:: python -.. -.. # Define the valid data sharing destinations for your TOM. -.. DATA_SHARING = { -.. 'tom-demo-dev': { -.. 'DISPLAY_NAME': os.getenv('TOM_DEMO_DISPLAY_NAME', 'TOM Demo Dev'), -.. 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), -.. 'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'), -.. 'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'), -.. }, -.. 'localhost-tom': { -.. # for testing; share with yourself -.. 'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'), -.. 'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'), -.. 'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'), -.. 'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'), -.. } -.. -.. } -.. \ No newline at end of file +TOM Toolkit supports direct data sharing between TOMs. + + +Permissions: +************ +To save data to a destination TOM your TOM will need to have access to a user account on that TOM with the correct +permissions. This is handled by your TOM's administrator as described below. + +.. warning:: Any user who has permission to access the relevant target or data in your TOM will have permission to + submit that data to the destination TOM once DATA_SHARING is configured. + + +Configuring your TOM to submit data to another TOM: +*************************************************** + +You will need to add a ``DATA_SHARING`` configuration dictionary to your ``settings.py`` that gives the credentials +for the various TOMs with which you wish to share data. This should be the same ``DATA_SHARING`` dictionary that is used +to :doc:`/managing_data/stream_pub_sub` such as `Hermes `_. + +.. code:: python + + # Define the valid data sharing destinations for your TOM. + DATA_SHARING = { + 'not-my-tom': { + # For sharing data with another TOM + 'DISPLAY_NAME': os.getenv('NOT_MY_TOM_DISPLAY_NAME', 'Not My Tom'), + 'BASE_URL': os.getenv('NOT_MY_TOM_BASE_URL', 'http://notmytom.com/'), + 'USERNAME': os.getenv('NOT_MY_TOM_USERNAME', 'set NOT_MY_TOM_USERNAME value in environment'), + 'PASSWORD': os.getenv('NOT_MY_TOM_PASSWORD', 'set NOT_MY_TOM_PASSWORD value in environment'), + }, + 'localhost-tom': { + # for testing; share with yourself + 'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'), + 'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'), + 'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'), + 'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'), + } + } + +Receiving Shared Data: +********************** + +Reduced Datums: +--------------- +When your TOM receives a new ``ReducedDatum`` from another TOM it will be saved to your TOM's database with its source +set to the name of the TOM that submitted it. Currently, only Photometry data can be directly shared between +TOMS and a ``Target`` with a matching name or alias must exist in both TOMS for sharing to take place. + +Data Products: +-------------- +When your TOM receives a new ``DataProduct`` from another TOM it will be saved to your TOM's database / storage and run +through the appropriate :doc:`data_processor ` pipeline. Only data products +associated with a ``Target`` with a name or alias that matches that of a target in the destination TOM will be shared. + +Targets: +-------- +When your TOM receives a new ``Target`` from another TOM it will be saved to your TOM's database. If the target's name +or alias doesn't match that of a target that already exists in the database, a new target will be created and added to a +new ``TargetList`` called "Imported from ". + +Target Lists: +------------- +When your TOM receives a new ``TargetList`` from another TOM it will be saved to your TOM's database. If the targets in +the ``TargetList`` are also shared, but already exist in the destination TOM, they will be added to the new +``TargetList``. + + + + + + diff --git a/setup.py b/setup.py index 7f382cfb0..d652e9af5 100644 --- a/setup.py +++ b/setup.py @@ -32,12 +32,14 @@ 'astroplan~=0.8', 'astropy>=5.0', 'beautifulsoup4~=4.9', + 'dramatiq[redis, watch]<2.0.0', 'django>=3.1,<5', # TOM Toolkit requires db math functions 'djangorestframework~=3.12', 'django-bootstrap4>=3,<24', 'django-contrib-comments~=2.0', # Earlier version are incompatible with Django >= 3.0 'django-crispy-forms~=2.0', 'crispy-bootstrap4~=2022.0', + 'django-dramatiq<1.0.0', 'django-extensions~=3.1', 'django-filter>=21,<24', 'django-gravatar2~=1.4', @@ -52,7 +54,8 @@ 'specutils~=1.8', ], extras_require={ - 'test': ['factory_boy>=3.2.1,<3.4.0'], + 'test': ['factory_boy>=3.2.1,<3.4.0', + 'responses~=0.23'], 'docs': [ 'recommonmark~=0.7', 'sphinx>=4,<8', diff --git a/tom_base/settings.py b/tom_base/settings.py index fe8754ce3..8eef5e1fb 100644 --- a/tom_base/settings.py +++ b/tom_base/settings.py @@ -237,30 +237,30 @@ } # Configuration for the TOM/Kafka Stream receiving data from this TOM -DATA_SHARING = { - 'hermes': { - 'DISPLAY_NAME': os.getenv('HERMES_DISPLAY_NAME', 'Hermes'), - 'BASE_URL': os.getenv('HERMES_BASE_URL', 'https://hermes.lco.global/'), - 'CREDENTIAL_USERNAME': os.getenv('SCIMMA_CREDENTIAL_USERNAME', - 'set SCIMMA_CREDENTIAL_USERNAME value in environment'), - 'CREDENTIAL_PASSWORD': os.getenv('SCIMMA_CREDENTIAL_PASSWORD', - 'set SCIMMA_CREDENTIAL_PASSWORD value in environment'), - 'USER_TOPICS': ['hermes.test', 'tomtoolkit.test'] - }, - 'tom-demo-dev': { - 'DISPLAY_NAME': os.getenv('TOM_DEMO_DISPLAY_NAME', 'TOM Demo Dev'), - 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), - 'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'), - 'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'), - }, - 'localhost-tom': { - # for testing; share with yourself - 'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'), - 'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'), - 'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'), - 'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'), - } -} +# DATA_SHARING = { +# 'hermes': { +# 'DISPLAY_NAME': os.getenv('HERMES_DISPLAY_NAME', 'Hermes'), +# 'BASE_URL': os.getenv('HERMES_BASE_URL', 'https://hermes.lco.global/'), +# 'CREDENTIAL_USERNAME': os.getenv('SCIMMA_CREDENTIAL_USERNAME', +# 'set SCIMMA_CREDENTIAL_USERNAME value in environment'), +# 'CREDENTIAL_PASSWORD': os.getenv('SCIMMA_CREDENTIAL_PASSWORD', +# 'set SCIMMA_CREDENTIAL_PASSWORD value in environment'), +# 'USER_TOPICS': ['hermes.test', 'tomtoolkit.test'] +# }, +# 'tom-demo-dev': { +# 'DISPLAY_NAME': os.getenv('TOM_DEMO_DISPLAY_NAME', 'TOM Demo Dev'), +# 'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'), +# 'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'), +# 'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'), +# }, +# 'localhost-tom': { +# # for testing; share with yourself +# 'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'), +# 'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'), +# 'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'), +# 'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'), +# } +# } TOM_CADENCE_STRATEGIES = [ 'tom_observations.cadences.retry_failed_observations.RetryFailedObservationsStrategy', diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index d2589af22..cd1c2742a 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -136,7 +136,7 @@ def get_hermes_topics(**kwargs): response = requests.get(url=submit_url, headers=headers) topics = response.json()['writable_topics'] - except KeyError: + except (KeyError, requests.exceptions.JSONDecodeError): topics = settings.DATA_SHARING['hermes']['USER_TOPICS'] return topics diff --git a/tom_dataproducts/api_views.py b/tom_dataproducts/api_views.py index 8f59138ae..5661098b8 100644 --- a/tom_dataproducts/api_views.py +++ b/tom_dataproducts/api_views.py @@ -4,7 +4,7 @@ from guardian.shortcuts import assign_perm, get_objects_for_user from rest_framework import status from rest_framework.mixins import CreateModelMixin, DestroyModelMixin, ListModelMixin -from rest_framework.parsers import MultiPartParser +from rest_framework.parsers import MultiPartParser, FormParser, JSONParser from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet @@ -12,7 +12,7 @@ from tom_dataproducts.data_processor import run_data_processor from tom_dataproducts.filters import DataProductFilter from tom_dataproducts.models import DataProduct, ReducedDatum -from tom_dataproducts.serializers import DataProductSerializer +from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer class DataProductViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet, PermissionListMixin): @@ -38,6 +38,7 @@ def create(self, request, *args, **kwargs): response = super().create(request, *args, **kwargs) if response.status_code == status.HTTP_201_CREATED: + response.data['message'] = 'Data product successfully uploaded.' dp = DataProduct.objects.get(pk=response.data['id']) try: run_hook('data_product_post_upload', dp) @@ -68,3 +69,29 @@ def get_queryset(self): ) else: return get_objects_for_user(self.request.user, 'tom_dataproducts.view_dataproduct') + + +class ReducedDatumViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet, PermissionListMixin): + """ + Viewset for ReducedDatum objects. Supports list, create, and delete. + + To view supported query parameters, please use the OPTIONS endpoint, which can be accessed through the web UI. + + **Please note that ``groups`` are an accepted query parameters for the ``CREATE`` endpoint. The groups parameter + will specify which ``groups`` can view the created ``DataProduct``. If no ``groups`` are specified, the + ``ReducedDatum`` will only be visible to the user that created the ``DataProduct``. Make sure to check your + ``groups``!!** + """ + queryset = ReducedDatum.objects.all() + serializer_class = ReducedDatumSerializer + filter_backends = (drf_filters.DjangoFilterBackend,) + permission_required = 'tom_dataproducts.view_reduceddatum' + parser_classes = [FormParser, JSONParser] + + def create(self, request, *args, **kwargs): + response = super().create(request, *args, **kwargs) + + if response.status_code == status.HTTP_201_CREATED: + response.data['message'] = 'Data successfully uploaded.' + + return response diff --git a/tom_dataproducts/data_processor.py b/tom_dataproducts/data_processor.py index 73854c19a..1ca859ed5 100644 --- a/tom_dataproducts/data_processor.py +++ b/tom_dataproducts/data_processor.py @@ -35,8 +35,9 @@ def run_data_processor(dp): data_processor = clazz() data = data_processor.process_data(dp) + data_type = data_processor.data_type_override() or dp.data_product_type - reduced_datums = [ReducedDatum(target=dp.target, data_product=dp, data_type=dp.data_product_type, + reduced_datums = [ReducedDatum(target=dp.target, data_product=dp, data_type=data_type, timestamp=datum[0], value=datum[1], source_name=datum[2]) for datum in data] ReducedDatum.objects.bulk_create(reduced_datums) @@ -65,3 +66,10 @@ def process_data(self, data_product): :rtype: list of 2-tuples """ return [] + + def data_type_override(self): + """ + Override for the ReducedDatum data type, if you want it to be different from the + DataProduct data_type. + """ + return '' diff --git a/tom_dataproducts/forced_photometry/__init__.py b/tom_dataproducts/forced_photometry/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tom_dataproducts/forced_photometry/atlas.py b/tom_dataproducts/forced_photometry/atlas.py new file mode 100644 index 000000000..062fcdce4 --- /dev/null +++ b/tom_dataproducts/forced_photometry/atlas.py @@ -0,0 +1,146 @@ +from django import forms +from django.conf import settings +from crispy_forms.layout import Div, HTML +from astropy.time import Time +import tom_dataproducts.forced_photometry.forced_photometry_service as fps +from tom_dataproducts.tasks import atlas_query +from tom_targets.models import Target + + +class AtlasForcedPhotometryQueryForm(fps.BaseForcedPhotometryQueryForm): + min_date = forms.CharField( + label='Min date:', required=False, + widget=forms.TextInput(attrs={'class': 'ml-2', 'type': 'datetime-local'}) + ) + max_date = forms.CharField( + label='Max date:', required=False, + widget=forms.TextInput(attrs={'class': 'ml-2', 'type': 'datetime-local'}) + ) + min_date_mjd = forms.FloatField( + label='Min date (mjd):', required=False, + widget=forms.NumberInput(attrs={'class': 'ml-2'}) + ) + max_date_mjd = forms.FloatField( + label='Max date (mjd):', required=False, + widget=forms.NumberInput(attrs={'class': 'ml-2'}) + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def layout(self): + return Div( + Div( + Div( + 'min_date', + css_class='col-md-4', + ), + Div( + HTML('OR'), + css_class='col-md-1' + ), + Div( + 'min_date_mjd', + css_class='col-md-5' + ), + css_class='form-row form-inline mb-2' + ), + Div( + Div( + 'max_date', + css_class='col-md-4', + ), + Div( + HTML('OR'), + css_class='col-md-1' + ), + Div( + 'max_date_mjd', + css_class='col-md-5' + ), + css_class='form-row form-inline mb-4' + ), + ) + + def clean(self): + cleaned_data = super().clean() + if not (cleaned_data.get('min_date') or cleaned_data.get('min_date_mjd')): + raise forms.ValidationError("Must supply a minimum date in either datetime or mjd format") + if cleaned_data.get('min_date') and cleaned_data.get('min_date_mjd'): + raise forms.ValidationError("Please specify the minimum date in either datetime or mjd format") + if cleaned_data.get('max_date') and cleaned_data.get('max_date_mjd'): + raise forms.ValidationError("Please specify the maximum date in either datetime or mjd format") + return cleaned_data + + +class AtlasForcedPhotometryService(fps.BaseForcedPhotometryService): + name = 'Atlas' + + def __init__(self): + super().__init__ + self.success_message = ('Asynchronous Atlas query is processing. ' + 'Refresh the page once complete it will show ' + 'up as a dataproduct in the "Manage Data" tab.') + + def get_form(self): + """ + This method returns the form for querying this service. + """ + return AtlasForcedPhotometryQueryForm + + def query_service(self, query_parameters): + """ + This method takes in the serialized data from the query form and actually + submits the query to the service + """ + print(f"Querying Atlas service with params: {query_parameters}") + min_date_mjd = query_parameters.get('min_date_mjd') + if not min_date_mjd: + min_date_mjd = Time(query_parameters.get('min_date')).mjd + max_date_mjd = query_parameters.get('max_date_mjd') + if not max_date_mjd and query_parameters.get('max_date'): + max_date_mjd = Time(query_parameters.get('max_date')).mjd + if not Target.objects.filter(pk=query_parameters.get('target_id')).exists(): + raise fps.ForcedPhotometryServiceException(f"Target {query_parameters.get('target_id')} does not exist") + + if 'atlas' not in settings.FORCED_PHOTOMETRY_SERVICES: + raise fps.ForcedPhotometryServiceException("Must specify 'atlas' settings in FORCED_PHOTOMETRY_SERVICES") + if not settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('url'): + raise fps.ForcedPhotometryServiceException( + "Must specify a 'url' under atlas settings in FORCED_PHOTOMETRY_SERVICES" + ) + if not settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('api_key'): + raise fps.ForcedPhotometryServiceException( + "Must specify an 'api_key' under atlas settings in FORCED_PHOTOMETRY_SERVICES" + ) + + if 'django_dramatiq' in settings.INSTALLED_APPS: + atlas_query.send(min_date_mjd, max_date_mjd, + query_parameters.get('target_id'), + self.get_data_product_type()) + else: + query_succeeded = atlas_query(min_date_mjd, max_date_mjd, + query_parameters.get('target_id'), + self.get_data_product_type()) + if not query_succeeded: + raise fps.ForcedPhotometryServiceException( + "Atlas query failed, check the server logs for more information" + ) + self.success_message = "Atlas query completed. View its data product in the 'Manage Data' tab" + + return True + + def validate_form(self, query_parameters): + """ + Same thing as query_service, but a dry run. You can + skip this in different modules by just using "pass" + + Typically called by the .is_valid() method. + """ + print(f"Validating Atlas service with params: {query_parameters}") + + def get_success_message(self): + return self.success_message + + def get_data_product_type(self): + return 'atlas_photometry' diff --git a/tom_dataproducts/forced_photometry/forced_photometry_service.py b/tom_dataproducts/forced_photometry/forced_photometry_service.py new file mode 100644 index 000000000..806345d90 --- /dev/null +++ b/tom_dataproducts/forced_photometry/forced_photometry_service.py @@ -0,0 +1,123 @@ +from abc import ABC, abstractmethod +import logging + +from crispy_forms.helper import FormHelper +from crispy_forms.layout import ButtonHolder, Layout, Submit +from django import forms +from django.conf import settings +from django.utils.module_loading import import_string + +logger = logging.getLogger(__name__) + + +def get_service_classes(): + try: + forced_photometry_services = settings.FORCED_PHOTOMETRY_SERVICES + except AttributeError: + return {} + + service_choices = {} + for service in forced_photometry_services.values(): + try: + clazz = import_string(service.get('class')) + except (ImportError, AttributeError): + raise ImportError(f'Could not import {service}. Did you provide the correct path?') + service_choices[clazz.name] = clazz + return service_choices + + +def get_service_class(name): + available_classes = get_service_classes() + try: + return available_classes[name] + except KeyError: + raise ImportError(( + f'Could not a find a forced photometry service with the name {name}. ' + 'Did you add it to TOM_FORCED_PHOTOMETRY_CLASSES?')) + + +class ForcedPhotometryServiceException(Exception): + pass + + +class BaseForcedPhotometryQueryForm(forms.Form): + """ + This is the class that is responsible for displaying the forced photometry request form. + This form is meant to be subclassed by more specific classes that represent a + form for a specific forced photometry service, including the query parameters it supports. + + For an implementation example please see + https://github.com/TOMToolkit/tom_base/blob/main/tom_dataproducts/forced_photometry/atlas.py + """ + service = forms.CharField(required=True, max_length=50, widget=forms.HiddenInput()) + target_id = forms.IntegerField(required=True, widget=forms.HiddenInput()) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.helper = FormHelper() + self.common_layout = Layout('service', 'target_id') + self.helper.layout = Layout( + self.common_layout, + self.layout(), + self.button_layout() + ) + + def layout(self): + return + + def button_layout(self): + return ButtonHolder( + Submit('submit', 'Submit'), + ) + + +class BaseForcedPhotometryService(ABC): + """ + This is the class that is responsible for defining the base forced photometry service class. + This form is meant to be subclassed by more specific classes that represent a + form for a particular forced photometry service. + """ + name = 'BaseForcedPhotometryService' + + @abstractmethod + def get_form(self): + """ + This method returns the form for querying this service. + """ + pass + + @abstractmethod + def query_service(self, query_parameters): + """ + This method takes in the serialized data from the query form and actually + submits the query to the service + """ + pass + + @abstractmethod + def validate_form(self, query_parameters): + """ + Same thing as query_service, but a dry run. You can + skip this in different modules by just using "pass" + + Typically called by the .is_valid() method. + """ + pass + + @abstractmethod + def get_success_message(self): + """ + This should return a message that shows up in the UI after making the query. + It should explain what is happening / next steps, i.e. if the results will be + emailed to you it should say that and that you must upload them once received. + """ + pass + + @abstractmethod + def get_data_product_type(self): + """ + This should return the data_product_type for data products produced by this service + Make sure to also add this type in your settings to DATA_PRODUCT_TYPES and + DATA_PROCESSORS. + """ + pass diff --git a/tom_dataproducts/forms.py b/tom_dataproducts/forms.py index db39c1e75..43fbbf309 100644 --- a/tom_dataproducts/forms.py +++ b/tom_dataproducts/forms.py @@ -5,39 +5,9 @@ from tom_dataproducts.models import DataProductGroup, DataProduct from tom_observations.models import ObservationRecord from tom_targets.models import Target -from tom_dataproducts.alertstreams.hermes import get_hermes_topics +from tom_dataproducts.sharing import get_sharing_destination_options -def get_sharing_destination_options(): - """ - Build the Display options and headers for the dropdown form for choosing sharing topics. - Customize for a different selection experience. - :return: Tuple: Possible Destinations and their Display Names - """ - choices = [] - try: - for destination, details in settings.DATA_SHARING.items(): - new_destination = [details.get('DISPLAY_NAME', destination)] - if details.get('USER_TOPICS', None): - # If topics exist for a destination (Such as HERMES) give topics as sub-choices - # for non-selectable Destination - if destination == "hermes": - destination_topics = get_hermes_topics() - else: - destination_topics = details['USER_TOPICS'] - topic_list = [(f'{destination}:{topic}', topic) for topic in destination_topics] - new_destination.append(tuple(topic_list)) - else: - # Otherwise just use destination as option - new_destination.insert(0, destination) - choices.append(tuple(new_destination)) - except AttributeError: - pass - return tuple(choices) - - -DESTINATION_OPTIONS = get_sharing_destination_options() - DATA_TYPE_OPTIONS = (('photometry', 'Photometry'), ('spectroscopy', 'Spectroscopy')) @@ -82,7 +52,7 @@ def __init__(self, *args, **kwargs): class DataShareForm(forms.Form): - share_destination = forms.ChoiceField(required=True, choices=DESTINATION_OPTIONS, label="Destination") + share_destination = forms.ChoiceField(required=True, choices=[], label="Destination") share_title = forms.CharField(required=False, label="Title") share_message = forms.CharField(required=False, label="Message", widget=forms.Textarea()) share_authors = forms.CharField(required=False, widget=forms.HiddenInput()) @@ -97,4 +67,4 @@ class DataShareForm(forms.Form): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['share_destination'].choices = DESTINATION_OPTIONS + self.fields['share_destination'].choices = get_sharing_destination_options() diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index d8fbef1ae..6e2664471 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -331,7 +331,7 @@ class ReducedDatum(models.Model): """ target = models.ForeignKey(Target, null=False, on_delete=models.CASCADE) - data_product = models.ForeignKey(DataProduct, null=True, on_delete=models.CASCADE) + data_product = models.ForeignKey(DataProduct, null=True, blank=True, on_delete=models.CASCADE) data_type = models.CharField( max_length=100, default='' @@ -352,3 +352,12 @@ def save(self, *args, **kwargs): else: raise ValidationError('Not a valid DataProduct type.') return super().save() + + def validate_unique(self, *args, **kwargs): + super().validate_unique(*args, **kwargs) + model_dict = self.__dict__.copy() + del model_dict['_state'] + del model_dict['id'] + obs = ReducedDatum.objects.filter(**model_dict) + if obs: + raise ValidationError('Data point already exists.') diff --git a/tom_dataproducts/processors/atlas_processor.py b/tom_dataproducts/processors/atlas_processor.py new file mode 100644 index 000000000..9d452c932 --- /dev/null +++ b/tom_dataproducts/processors/atlas_processor.py @@ -0,0 +1,71 @@ +import mimetypes + +from astropy import units +import astropy.io.ascii +from astropy.time import Time, TimezoneInfo + +from tom_dataproducts.data_processor import DataProcessor +from tom_dataproducts.exceptions import InvalidFileFormatException + + +class AtlasProcessor(DataProcessor): + + def data_type_override(self): + return 'photometry' + + def process_data(self, data_product): + """ + Routes a atlas processing call to a method specific to a file-format. + + :param data_product: Photometric DataProduct which will be processed into the specified format for database + ingestion + :type data_product: DataProduct + + :returns: python list of 2-tuples, each with a timestamp and corresponding data + :rtype: list + """ + + mimetype = mimetypes.guess_type(data_product.data.path)[0] + if mimetype in self.PLAINTEXT_MIMETYPES: + photometry = self._process_photometry_from_plaintext(data_product) + return [(datum.pop('timestamp'), datum, datum.pop('source', 'ATLAS')) for datum in photometry] + else: + raise InvalidFileFormatException('Unsupported file type') + + def _process_photometry_from_plaintext(self, data_product): + """ + Processes the photometric data from a plaintext file into a list of dicts. File is read using astropy as + specified in the below documentation. The file is expected to be a multi-column delimited space delimited + text file, as produced by the ATLAS forced photometry service at https://fallingstar-data.com/forcedphot + + The header looks like this: + ###MJD m dm uJy duJy F err chi/N RA Dec x y maj min phi apfit mag5sig Sky Obs + + :param data_product: ATLAS Photometric DataProduct which will be processed into a list of dicts + :type data_product: DataProduct + + :returns: python list containing the photometric data from the DataProduct + :rtype: list + """ + photometry = [] + + data = astropy.io.ascii.read(data_product.data.path) + if len(data) < 1: + raise InvalidFileFormatException('Empty table or invalid file type') + + try: + for datum in data: + time = Time(float(datum['##MJD']), format='mjd') + utc = TimezoneInfo(utc_offset=0*units.hour) + time.format = 'datetime' + value = { + 'timestamp': time.to_datetime(timezone=utc), + 'magnitude': float(datum['m']), + 'magnitude_error': float(datum['dm']), + 'filter': str(datum['F']) + } + photometry.append(value) + except Exception as e: + raise InvalidFileFormatException(e) + + return photometry diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py index 223747567..7d6ba9993 100644 --- a/tom_dataproducts/serializers.py +++ b/tom_dataproducts/serializers.py @@ -18,6 +18,8 @@ class Meta: class ReducedDatumSerializer(serializers.ModelSerializer): + target = TargetFilteredPrimaryKeyRelatedField(queryset=Target.objects.all()) + class Meta: model = ReducedDatum fields = ( @@ -26,9 +28,31 @@ class Meta: 'source_name', 'source_location', 'timestamp', - 'value' + 'value', + 'target' ) + def create(self, validated_data): + """DRF requires explicitly handling writeable nested serializers, + here we pop the groups data and save it using its serializer. + """ + groups = validated_data.pop('groups', []) + + rd = ReducedDatum(**validated_data) + rd.full_clean() + rd.save() + + # Save groups for this target + group_serializer = GroupSerializer(data=groups, many=True) + if group_serializer.is_valid() and settings.TARGET_PERMISSIONS_ONLY is False: + for group in groups: + group_instance = Group.objects.get(pk=group['id']) + assign_perm('tom_dataproducts.view_dataproduct', group_instance, rd) + assign_perm('tom_dataproducts.change_dataproduct', group_instance, rd) + assign_perm('tom_dataproducts.delete_dataproduct', group_instance, rd) + + return rd + class DataProductSerializer(serializers.ModelSerializer): target = TargetFilteredPrimaryKeyRelatedField(queryset=Target.objects.all()) diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py new file mode 100644 index 000000000..d85c3524c --- /dev/null +++ b/tom_dataproducts/sharing.py @@ -0,0 +1,256 @@ +import requests +import os + +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.contrib import messages + +from tom_targets.models import Target +from tom_dataproducts.models import DataProduct, ReducedDatum +from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage, get_hermes_topics +from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer + + +def share_data_with_hermes(share_destination, form_data, product_id=None, target_id=None, selected_data=None): + """ + Serialize and share data with Hermes (hermes.lco.global) + :param share_destination: Topic to share data to. (e.g. 'hermes.test') + :param form_data: Sharing Form data + :param product_id: DataProduct ID (if provided) + :param target_id: Target ID (if provided) + :param selected_data: List of ReducedDatum IDs (if provided) + :return: + """ + # Query relevant Reduced Datums Queryset + accepted_data_types = ['photometry'] + if product_id: + product = DataProduct.objects.get(pk=product_id) + target = product.target + reduced_datums = ReducedDatum.objects.filter(data_product=product) + elif selected_data: + reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) + target = reduced_datums[0].target + elif target_id: + target = Target.objects.get(pk=target_id) + data_type = form_data.get('data_type', 'photometry') + reduced_datums = ReducedDatum.objects.filter(target=target, data_type=data_type) + else: + reduced_datums = ReducedDatum.objects.none() + target = Target.objects.none() + + reduced_datums.filter(data_type__in=accepted_data_types) + + # Build and submit hermes table from Reduced Datums + hermes_topic = share_destination.split(':')[1] + destination = share_destination.split(':')[0] + message_info = BuildHermesMessage(title=form_data.get('share_title', + f"Updated data for {target.name} from " + f"{getattr(settings, 'TOM_NAME','TOM Toolkit')}."), + submitter=form_data.get('submitter'), + authors=form_data.get('share_authors', None), + message=form_data.get('share_message', None), + topic=hermes_topic + ) + # Run ReducedDatums Queryset through sharing protocols to make sure they are safe to share. + filtered_reduced_datums = check_for_share_safe_datums(destination, reduced_datums, topic=hermes_topic) + if filtered_reduced_datums.count() > 0: + response = publish_photometry_to_hermes(message_info, filtered_reduced_datums) + else: + return {'message': f'ERROR: No valid data to share. (Check Sharing Protocol. Note that data types must be in ' + f'{accepted_data_types})'} + return response + + +def share_data_with_tom(share_destination, form_data, product_id=None, target_id=None, selected_data=None): + """ + Serialize and share data with another TOM + :param share_destination: TOM to share data to as described in settings.DATA_SHARING. (e.g. 'mytom') + :param form_data: Sharing Form data + :param product_id: DataProduct ID (if provided) + :param target_id: Target ID (if provided) + :param selected_data: List of ReducedDatum IDs (if provided) + :return: + """ + # Build destination TOM headers and URL information + try: + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + username = settings.DATA_SHARING[share_destination]['USERNAME'] + password = settings.DATA_SHARING[share_destination]['PASSWORD'] + except KeyError as err: + raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {share_destination}: Key {err} not found.') + auth = (username, password) + headers = {'Content-Type': 'application/json', 'Accept': 'application/json'} + + dataproducts_url = destination_tom_base_url + 'api/dataproducts/' + targets_url = destination_tom_base_url + 'api/targets/' + reduced_datums_url = destination_tom_base_url + 'api/reduceddatums/' + reduced_datums = ReducedDatum.objects.none() + + # If a DataProduct is provided, share that DataProduct + if product_id: + product = DataProduct.objects.get(pk=product_id) + target = product.target + serialized_data = DataProductSerializer(product).data + # Find matching target in destination TOM + destination_target_id, target_search_response = get_destination_target(target, targets_url, headers, auth) + if destination_target_id is None: + return {'message': 'ERROR: No matching target found.'} + elif isinstance(destination_target_id, list) and len(destination_target_id) > 1: + return {'message': 'ERROR: Multiple targets with matching name found in destination TOM.'} + serialized_data['target'] = destination_target_id + # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage + dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) + # Save DataProduct in Destination TOM + with open(dataproduct_filename, 'rb') as dataproduct_filep: + files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} + headers = {'Media-Type': 'multipart/form-data'} + response = requests.post(dataproducts_url, data=serialized_data, files=files, headers=headers, auth=auth) + elif selected_data or target_id: + # If ReducedDatums are provided, share those ReducedDatums + if selected_data: + reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) + targets = set(reduced_datum.target for reduced_datum in reduced_datums) + target_dict = {} + for target in targets: + # get destination Target + destination_target_id, target_search_response = get_destination_target(target, + targets_url, + headers, + auth) + if isinstance(destination_target_id, list) and len(destination_target_id) > 1: + return {'message': 'ERROR: Multiple targets with matching name found in destination TOM.'} + target_dict[target.name] = destination_target_id + if all(value is None for value in target_dict.values()): + return {'message': 'ERROR: No matching targets found.'} + else: + # If Target is provided, share all ReducedDatums for that Target + # (Will not create New Target in Destination TOM) + target = Target.objects.get(pk=target_id) + reduced_datums = ReducedDatum.objects.filter(target=target) + destination_target_id, target_search_response = get_destination_target(target, targets_url, headers, auth) + if destination_target_id is None: + return {'message': 'ERROR: No matching target found.'} + elif isinstance(destination_target_id, list) and len(destination_target_id) > 1: + return {'message': 'ERROR: Multiple targets with matching name found in destination TOM.'} + target_dict = {target.name: destination_target_id} + response_codes = [] + reduced_datums = check_for_share_safe_datums(share_destination, reduced_datums) + if not reduced_datums: + return {'message': 'ERROR: No valid data to share.'} + for datum in reduced_datums: + if target_dict[datum.target.name]: + serialized_data = ReducedDatumSerializer(datum).data + serialized_data['target'] = target_dict[datum.target.name] + serialized_data['data_product'] = '' + if not serialized_data['source_name']: + serialized_data['source_name'] = settings.TOM_NAME + serialized_data['source_location'] = "TOM-TOM Direct Sharing" + response = requests.post(reduced_datums_url, json=serialized_data, headers=headers, auth=auth) + response_codes.append(response.status_code) + failed_data_count = response_codes.count(500) + if failed_data_count < len(response_codes): + return {'message': f'{len(response_codes)-failed_data_count} of {len(response_codes)} ' + 'datums successfully saved.'} + else: + return {'message': 'ERROR: No valid data shared. These data may already exist in target TOM.'} + else: + return {'message': 'ERROR: No valid data to share.'} + + return response + + +def get_destination_target(target, targets_url, headers, auth): + """ + Retrieve the target ID from a destination TOM that is a fuzzy match the given target name and aliases + :param target: Target Model + :param targets_url: Destination API URL for TOM Target List + :param headers: TOM API headers + :param auth: TOM API authorization + :return: + """ + # Create coma separated list of target names plus aliases that can be recognized and parsed by the TOM API Filter + target_names = ','.join(map(str, target.names)) + target_response = requests.get(f'{targets_url}?name_fuzzy={target_names}', headers=headers, auth=auth) + target_response_json = target_response.json() + try: + if target_response_json['results']: + if len(target_response_json['results']) > 1: + return target_response_json['results'], target_response + destination_target_id = target_response_json['results'][0]['id'] + return destination_target_id, target_response + else: + return None, target_response + except KeyError: + return None, target_response + + +def check_for_share_safe_datums(destination, reduced_datums, **kwargs): + """ + Custom sharing protocols used to determine when data is shared with a destination. + This example prevents sharing if a datum has already been published to the given Hermes topic. + :param destination: sharing destination string + :param reduced_datums: selected input datums + :return: queryset of reduced datums to be shared + """ + return reduced_datums + # if 'hermes' in destination: + # message_topic = kwargs.get('topic', None) + # # Remove data points previously shared to the given topic + # filtered_datums = reduced_datums.exclude(Q(message__exchange_status='published') + # & Q(message__topic=message_topic)) + # else: + # filtered_datums = reduced_datums + # return filtered_datums + + +def check_for_save_safe_datums(): + return + + +def get_sharing_destination_options(): + """ + Build the Display options and headers for the dropdown form for choosing sharing topics. + Customize for a different selection experience. + :return: Tuple: Possible Destinations and their Display Names + """ + choices = [] + try: + for destination, details in settings.DATA_SHARING.items(): + new_destination = [details.get('DISPLAY_NAME', destination)] + if details.get('USER_TOPICS', None): + # If topics exist for a destination (Such as HERMES) give topics as sub-choices + # for non-selectable Destination + if destination == "hermes": + destination_topics = get_hermes_topics() + else: + destination_topics = details['USER_TOPICS'] + topic_list = [(f'{destination}:{topic}', topic) for topic in destination_topics] + new_destination.append(tuple(topic_list)) + else: + # Otherwise just use destination as option + new_destination.insert(0, destination) + choices.append(tuple(new_destination)) + except AttributeError: + pass + return tuple(choices) + + +def sharing_feedback_handler(response, request): + """ + Handle the response from a sharing request and prepare a message to the user + :return: + """ + try: + if 'message' in response.json(): + publish_feedback = response.json()['message'] + else: + publish_feedback = f"ERROR: {response.text}" + except AttributeError: + publish_feedback = response['message'] + except ValueError: + publish_feedback = f"ERROR: Returned Response code {response.status_code}" + if "ERROR" in publish_feedback.upper(): + messages.error(request, publish_feedback) + else: + messages.success(request, publish_feedback) + return diff --git a/tom_dataproducts/tasks.py b/tom_dataproducts/tasks.py new file mode 100644 index 000000000..f6ce1b2d9 --- /dev/null +++ b/tom_dataproducts/tasks.py @@ -0,0 +1,103 @@ +# Place dramatiq asynchronous tasks here - they are auto-discovered + +import dramatiq +import requests +import time +import logging +import re +from astropy.time import Time +from urllib.parse import urlparse +from django.conf import settings +from django.utils import timezone +from django.core.files.base import ContentFile + +from tom_targets.models import Target +from tom_dataproducts.models import DataProduct +from tom_dataproducts.exceptions import InvalidFileFormatException +from tom_dataproducts.data_processor import run_data_processor + +logger = logging.getLogger(__name__) + + +@dramatiq.actor(max_retries=0) +def atlas_query(min_date_mjd, max_date_mjd, target_id, data_product_type): + print("Calling atlas query!") + target = Target.objects.get(pk=target_id) + headers = {"Authorization": f"Token {settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('api_key')}", + "Accept": "application/json"} + base_url = settings.FORCED_PHOTOMETRY_SERVICES.get('atlas', {}).get('url') + task_url = None + while not task_url: + with requests.Session() as s: + task_data = {"ra": target.ra, "dec": target.dec, "mjd_min": min_date_mjd, "send_email": False} + if max_date_mjd: + task_data['mjd_max'] = max_date_mjd + resp = s.post( + f"{base_url}/queue/", headers=headers, + data=task_data) + + if resp.status_code == 201: + task_url = resp.json()["url"] + print(f"The task url is {task_url}") + elif resp.status_code == 429: + message = resp.json()["detail"] + print(f"{resp.status_code} {message}") + t_sec = re.findall(r"available in (\d+) seconds", message) + t_min = re.findall(r"available in (\d+) minutes", message) + if t_sec: + waittime = int(t_sec[0]) + elif t_min: + waittime = int(t_min[0]) * 60 + else: + waittime = 10 + print(f"Waiting {waittime} seconds") + time.sleep(waittime) + else: + logger.error(f"Failed to queue Atlas task: HTTP Error {resp.status_code} - {resp.text}") + return False + + result_url = None + taskstarted_printed = False + while not result_url: + with requests.Session() as s: + resp = s.get(task_url, headers=headers) + + if resp.status_code == 200: + if resp.json()["finishtimestamp"]: + result_url = resp.json()["result_url"] # PART WHEN QUERY IS COMPLETE + print(f"Task is complete with results available at {result_url}") + elif resp.json()["starttimestamp"]: + if not taskstarted_printed: + print(f"Task is running (started at {resp.json()['starttimestamp']})") + taskstarted_printed = True + time.sleep(2) + else: + print(f"Waiting for job to start (queued at {resp.json()['timestamp']})") + time.sleep(4) + else: + logger.error(f"Failed to retrieve Atlas task status: HTTP Error {resp.status_code} - {resp.text}") + return False + + results = requests.get(result_url, headers=headers) + dp_name = f"atlas_{Time(min_date_mjd, format='mjd').strftime('%Y_%m_%d')}" + if max_date_mjd: + dp_name += f"-{Time(max_date_mjd, format='mjd').strftime('%Y_%m_%d')}" + dp_name += f"_{urlparse(result_url)[2].rpartition('/')[2]}" + file = ContentFile(results.content, name=dp_name) + + dp = DataProduct.objects.create( + product_id=dp_name, + target=target, + data=file, + data_product_type=data_product_type, + extra_data=f'Queried from Atlas within the TOM on {timezone.now().isoformat()}' + ) + logger.info(f"Created dataproduct {dp_name} from atlas query") + + try: + run_data_processor(dp) + except InvalidFileFormatException as e: + logger.error(f"Error processing returned Atlas data into ReducedDatums: {repr(e)}") + return False + + return True diff --git a/tom_dataproducts/templates/tom_dataproducts/forced_photometry_form.html b/tom_dataproducts/templates/tom_dataproducts/forced_photometry_form.html new file mode 100644 index 000000000..bb78fd757 --- /dev/null +++ b/tom_dataproducts/templates/tom_dataproducts/forced_photometry_form.html @@ -0,0 +1,14 @@ +{% extends 'tom_common/base.html' %} +{% load bootstrap4 static crispy_forms_tags %} +{% block title %}Query Forced Photometry{% endblock %} +{% block additional_css %} + +{% endblock %} +{% block content %} +{{ form|as_crispy_errors }} +

Query {{ form.service.value }} Forced Photometry Service

+
+

Target {{ target.name }} at RA {{ target.ra }}, DEC {{ target.dec }}

+
+{% crispy query_form %} +{% endblock %} diff --git a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html index 953af3b88..1a2e499f5 100644 --- a/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html +++ b/tom_dataproducts/templates/tom_dataproducts/partials/photometry_datalist_for_target.html @@ -46,25 +46,25 @@ {% endfor %} -
-
- Share Selected Data -
- {% if sharing_destinations %} -
-
- {% bootstrap_field target_data_share_form.share_destination %} -
-
- -
-
- {% else %} - Not Configured - {% endif %} - - + {% if not target_share %} +
+
+ Share Selected Data +
+ {% if sharing_destinations %} +
+
+ {% bootstrap_field target_data_share_form.share_destination %} +
+
+ +
+
+ {% else %} + Not Configured + {% endif %}
+ {% endif %}
+ +

Query Forced Photometry Service

+{% for service in forced_photometry_services %} +{{ service }} +{% endfor %} diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index adbc76816..59cbc1894 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -20,6 +20,7 @@ from tom_dataproducts.forms import DataProductUploadForm, DataShareForm from tom_dataproducts.models import DataProduct, ReducedDatum from tom_dataproducts.processors.data_serializers import SpectrumSerializer +from tom_dataproducts.forced_photometry.forced_photometry_service import get_service_classes from tom_observations.models import ObservationRecord from tom_targets.models import Target @@ -95,6 +96,15 @@ def dataproduct_list_all(context): } +@register.inclusion_tag('tom_dataproducts/partials/query_forced_photometry.html') +def query_forced_photometry(target): + services = get_service_classes().keys() + return { + 'forced_photometry_services': services, + 'target': target + } + + @register.inclusion_tag('tom_dataproducts/partials/upload_dataproduct.html', takes_context=True) def upload_dataproduct(context, obj): user = context['user'] @@ -164,7 +174,7 @@ def recent_photometry(target, limit=1): @register.inclusion_tag('tom_dataproducts/partials/photometry_datalist_for_target.html', takes_context=True) -def get_photometry_data(context, target): +def get_photometry_data(context, target, target_share=False): """ Displays a table of the all photometric points for a target. """ @@ -208,7 +218,8 @@ def get_photometry_data(context, target): context = {'data': data, 'target': target, 'target_data_share_form': form, - 'sharing_destinations': form.fields['share_destination'].choices} + 'sharing_destinations': form.fields['share_destination'].choices, + 'target_share': target_share} return context diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py index fb86fdecd..5bb39b4d8 100644 --- a/tom_dataproducts/tests/test_api.py +++ b/tom_dataproducts/tests/test_api.py @@ -1,6 +1,7 @@ from django.contrib.auth.models import Group, User from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse +from django.core.exceptions import ValidationError from guardian.shortcuts import assign_perm from rest_framework import status from rest_framework.test import APITestCase @@ -102,3 +103,41 @@ def test_data_product_list(self): response = self.client.get(reverse('api:dataproducts-list')) self.assertContains(response, dp.product_id, status_code=status.HTTP_200_OK) + + +class TestReducedDatumViewset(APITestCase): + def setUp(self): + self.user = User.objects.create(username='testuser') + self.client.force_login(self.user) + self.st = SiderealTargetFactory.create() + self.obsr = ObservingRecordFactory.create(target_id=self.st.id) + self.rd_data = { + 'data_product': '', + 'data_type': 'photometry', + 'source_name': 'TOM Toolkit', + 'source_location': 'TOM-TOM Direct Sharing', + 'value': {'magnitude': 15.582, 'filter': 'r', 'error': 0.005}, + 'target': self.st.id, + 'timestamp': '2012-02-12T01:40:47Z' + } + + assign_perm('tom_dataproducts.add_reduceddatum', self.user) + assign_perm('tom_targets.add_target', self.user, self.st) + assign_perm('tom_targets.view_target', self.user, self.st) + assign_perm('tom_targets.change_target', self.user, self.st) + + def test_upload_reduced_datum(self): + response = self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + self.assertContains(response, self.rd_data['source_name'], status_code=status.HTTP_201_CREATED) + + def test_upload_same_reduced_datum_twice(self): + """ + Test that identical data raises a validation error while similar but different JSON will make it through. + """ + self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + with self.assertRaises(ValidationError): + self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + self.rd_data['value'] = {'magnitude': 15.582, 'filter': 'B', 'error': 0.005} + self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') + rd_queryset = ReducedDatum.objects.all() + self.assertEqual(rd_queryset.count(), 2) diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index bf671c3c8..a06d39245 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -1,6 +1,7 @@ import os from http import HTTPStatus import tempfile +import responses from astropy import units from astropy.io import fits @@ -18,7 +19,7 @@ from tom_dataproducts.exceptions import InvalidFileFormatException from tom_dataproducts.forms import DataProductUploadForm -from tom_dataproducts.models import DataProduct, is_fits_image_file +from tom_dataproducts.models import DataProduct, is_fits_image_file, ReducedDatum from tom_dataproducts.processors.data_serializers import SpectrumSerializer from tom_dataproducts.processors.photometry_processor import PhotometryProcessor from tom_dataproducts.processors.spectroscopy_processor import SpectroscopyProcessor @@ -489,3 +490,261 @@ def test_create_thumbnail(self, mock_is_fits_image_file): 'ignore_missing_simple=True') self.assertIn(expected, logs.output) + + +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], + TARGET_PERMISSIONS_ONLY=True, + DATA_SHARING={'local_host': {'BASE_URL': 'https://fake.url/example/', + 'USERNAME': 'fake_user', + 'PASSWORD': 'password'}}) +class TestShareDataProducts(TestCase): + def setUp(self): + self.target = SiderealTargetFactory.create() + self.observation_record = ObservingRecordFactory.create( + target_id=self.target.id, + facility=FakeRoboticFacility.name, + parameters={} + ) + self.data_product = DataProduct.objects.create( + product_id='testproductid', + target=self.target, + observation_record=self.observation_record, + data=SimpleUploadedFile('afile.fits', b'somedata') + ) + self.user = User.objects.create_user(username='test', email='test@example.com') + assign_perm('tom_targets.view_target', self.user, self.target) + self.client.force_login(self.user) + + self.rd1 = ReducedDatum.objects.create( + target=self.target, + data_type='photometry', + value={'magnitude': 18.5, 'error': .5, 'filter': 'V'} + ) + self.rd2 = ReducedDatum.objects.create( + target=self.target, + data_type='photometry', + value={'magnitude': 19.5, 'error': .5, 'filter': 'B'} + ) + self.rd3 = ReducedDatum.objects.create( + target=self.target, + data_type='photometry', + value={'magnitude': 17.5, 'error': .5, 'filter': 'R'} + ) + + @responses.activate + def test_share_dataproduct_no_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"error": "not found"}, + status=500 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + + response = self.client.post( + reverse('dataproducts:share', kwargs={'dp_pk': self.data_product.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'] + }, + follow=True + ) + self.assertContains(response, 'ERROR: No matching target found.') + + @responses.activate + def test_share_reduceddatums_target_no_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"error": "not found"}, + status=500 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'] + }, + follow=True + ) + self.assertContains(response, 'ERROR: No matching target found.') + + @responses.activate + def test_share_reduced_datums_no_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"error": "not found"}, + status=500 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'], + 'share-box': [1, 2] + }, + follow=True + ) + self.assertContains(response, 'ERROR: No matching targets found.') + + @responses.activate + def test_share_dataproduct_valid_target_found(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"results": [{'id': 1}]}, + status=200 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + responses.add( + responses.POST, + destination_tom_base_url + 'api/dataproducts/', + json={"message": "Data product successfully uploaded."}, + status=200, + ) + + response = self.client.post( + reverse('dataproducts:share', kwargs={'dp_pk': self.data_product.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'] + }, + follow=True + ) + self.assertContains(response, 'Data product successfully uploaded.') + + @responses.activate + def test_share_reduceddatums_target_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"results": [{'id': 1}]}, + status=200 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + responses.add( + responses.POST, + destination_tom_base_url + 'api/reduceddatums/', + json={}, + status=201, + ) + + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'] + }, + follow=True + ) + self.assertContains(response, '3 of 3 datums successfully saved.') + + @responses.activate + def test_share_reduced_datums_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"results": [{'id': 1}]}, + status=200 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + responses.add( + responses.POST, + destination_tom_base_url + 'api/reduceddatums/', + json={}, + status=201, + ) + + response = self.client.post( + reverse('dataproducts:share_all', kwargs={'tg_pk': self.target.id}), + { + 'share_authors': ['test_author'], + 'target': self.target.id, + 'submitter': ['test_submitter'], + 'share_destination': [share_destination], + 'share_title': ['Updated data for thingy.'], + 'share_message': ['test_message'], + 'share-box': [1, 2] + }, + follow=True + ) + self.assertContains(response, '2 of 2 datums successfully saved.') diff --git a/tom_dataproducts/urls.py b/tom_dataproducts/urls.py index ebd4b450b..b38bcd9f4 100644 --- a/tom_dataproducts/urls.py +++ b/tom_dataproducts/urls.py @@ -4,13 +4,14 @@ from tom_dataproducts.views import DataProductDeleteView, DataProductGroupCreateView from tom_dataproducts.views import DataProductGroupDetailView, DataProductGroupDataView, DataProductGroupDeleteView from tom_dataproducts.views import DataProductUploadView, DataProductFeatureView, UpdateReducedDataView -from tom_dataproducts.views import DataShareView +from tom_dataproducts.views import DataShareView, ForcedPhotometryQueryView from tom_common.api_router import SharedAPIRootRouter -from tom_dataproducts.api_views import DataProductViewSet +from tom_dataproducts.api_views import DataProductViewSet, ReducedDatumViewSet router = SharedAPIRootRouter() router.register(r'dataproducts', DataProductViewSet, 'dataproducts') +router.register(r'reduceddatums', ReducedDatumViewSet, 'reduceddatums') app_name = 'tom_dataproducts' @@ -23,6 +24,8 @@ path('data/group//delete/', DataProductGroupDeleteView.as_view(), name='group-delete'), path('data/upload/', DataProductUploadView.as_view(), name='upload'), path('data/reduced/update/', UpdateReducedDataView.as_view(), name='update-reduced-data'), + path('data/forced_photometry//query/', ForcedPhotometryQueryView.as_view(), + name='forced-photometry-query'), path('data//delete/', DataProductDeleteView.as_view(), name='delete'), path('data//feature/', DataProductFeatureView.as_view(), name='feature'), path('data//share/', DataShareView.as_view(), name='share'), diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index 233437278..b4d7f7776 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -1,6 +1,5 @@ from io import StringIO import logging -import os from urllib.parse import urlencode, urlparse from django.conf import settings @@ -9,7 +8,6 @@ from django.contrib.auth.models import Group from django.core.cache import cache from django.core.cache.utils import make_template_fragment_key -from django.core.exceptions import ImproperlyConfigured from django.core.management import call_command from django.http import HttpResponseRedirect from django.shortcuts import redirect @@ -25,19 +23,16 @@ from tom_common.hooks import run_hook from tom_common.hints import add_hint from tom_common.mixins import Raise403PermissionRequiredMixin -from tom_targets.serializers import TargetSerializer -from tom_targets.models import Target from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum from tom_dataproducts.exceptions import InvalidFileFormatException from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm, DataShareForm from tom_dataproducts.filters import DataProductFilter from tom_dataproducts.data_processor import run_data_processor -from tom_dataproducts.alertstreams.hermes import publish_photometry_to_hermes, BuildHermesMessage from tom_observations.models import ObservationRecord from tom_observations.facility import get_service_class -from tom_dataproducts.serializers import DataProductSerializer - -import requests +from tom_dataproducts.sharing import share_data_with_hermes, share_data_with_tom, sharing_feedback_handler +import tom_dataproducts.forced_photometry.forced_photometry_service as fps +from tom_targets.models import Target logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -86,6 +81,86 @@ def post(self, request, *args, **kwargs): ) +class ForcedPhotometryQueryView(LoginRequiredMixin, FormView): + """ + View that handles queries for forced photometry services + """ + template_name = 'tom_dataproducts/forced_photometry_form.html' + + def get_target_id(self): + """ + Parses the target id from the query parameters. + """ + if self.request.method == 'GET': + return self.request.GET.get('target_id') + elif self.request.method == 'POST': + return self.request.POST.get('target_id') + + def get_target(self): + """ + Gets the target for observing from the database + + :returns: target for observing + :rtype: Target + """ + return Target.objects.get(pk=self.get_target_id()) + + def get_service(self): + """ + Gets the forced photometry service that you want to query + """ + return self.kwargs['service'] + + def get_service_class(self): + """ + Gets the forced photometry service class + """ + return fps.get_service_class(self.get_service()) + + def get_form_class(self): + """ + Gets the forced photometry service form class + """ + return self.get_service_class()().get_form() + + def get_context_data(self, *args, **kwargs): + """ + Adds the target to the context object. + """ + context = super().get_context_data(*args, **kwargs) + context['target'] = self.get_target() + context['query_form'] = self.get_form_class()(initial=self.get_initial()) + return context + + def get_initial(self): + """ + Populates the form with initial data including service name and target id + """ + initial = super().get_initial() + if not self.get_target_id(): + raise Exception('Must provide target_id') + initial['target_id'] = self.get_target_id() + initial['service'] = self.get_service() + initial.update(self.request.GET.dict()) + return initial + + def post(self, request, *args, **kwargs): + form = self.get_form() + if form.is_valid(): + service = self.get_service_class()() + try: + service.query_service(form.cleaned_data) + except fps.ForcedPhotometryServiceException as e: + form.add_error(f"Problem querying forced photometry service: {repr(e)}") + return self.form_invalid(form) + messages.info(self.request, service.get_success_message()) + return redirect( + reverse('tom_targets:detail', kwargs={'pk': self.get_target_id()}) + ) + else: + return self.form_invalid(form) + + class DataProductUploadView(LoginRequiredMixin, FormView): """ View that handles manual upload of DataProducts. Requires authentication. @@ -327,128 +402,25 @@ def post(self, request, *args, **kwargs): Handles Data Products and All the data of a type for a target as well as individual Reduced Datums. Submit to Hermes, or Share with TOM (soon). """ - data_share_form = DataShareForm(request.POST, request.FILES) - # Check if data points have been selected. - selected_data = request.POST.getlist("share-box") + if data_share_form.is_valid(): form_data = data_share_form.cleaned_data - # 1st determine if pk is data product, Reduced Datum, or Target. - # Then query relevant Reduced Datums Queryset + share_destination = form_data['share_destination'] product_id = kwargs.get('dp_pk', None) - if product_id: - product = DataProduct.objects.get(pk=product_id) - data_type = product.data_product_type - reduced_datums = ReducedDatum.objects.filter(data_product=product) - else: - target_id = kwargs.get('tg_pk', None) - target = Target.objects.get(pk=target_id) - data_type = form_data['data_type'] - if request.POST.get("share-box", None) is None: - reduced_datums = ReducedDatum.objects.filter(target=target, data_type=data_type) - else: - reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) - if data_type == 'photometry': - share_destination = form_data['share_destination'] - if 'HERMES' in share_destination.upper(): - # Build and submit hermes table from Reduced Datums - hermes_topic = share_destination.split(':')[1] - destination = share_destination.split(':')[0] - message_info = BuildHermesMessage(title=form_data['share_title'], - submitter=form_data['submitter'], - authors=form_data['share_authors'], - message=form_data['share_message'], - topic=hermes_topic - ) - # Run ReducedDatums Queryset through sharing protocols to make sure they are safe to share. - filtered_reduced_datums = self.get_share_safe_datums(destination, reduced_datums, - topic=hermes_topic) - if filtered_reduced_datums.count() > 0: - response = publish_photometry_to_hermes(message_info, filtered_reduced_datums) - else: - messages.error(self.request, 'No Data to share. (Check sharing Protocol.)') - return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - else: - messages.error(self.request, 'TOM-TOM sharing is not yet supported.') - return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - # response = self.share_with_tom(share_destination, product) - try: - if 'message' in response.json(): - publish_feedback = response.json()['message'] - else: - publish_feedback = f"ERROR: {response.text}" - except ValueError: - publish_feedback = f"ERROR: Returned Response code {response.status_code}" - if "ERROR" in publish_feedback.upper(): - messages.error(self.request, publish_feedback) - else: - messages.success(self.request, publish_feedback) + target_id = kwargs.get('tg_pk', None) + + # Check if data points have been selected. + selected_data = request.POST.getlist("share-box") + + # Check Destination + if 'HERMES' in share_destination.upper(): + response = share_data_with_hermes(share_destination, form_data, product_id, target_id, selected_data) else: - messages.error(self.request, f'Publishing {data_type} data is not yet supported.') + response = share_data_with_tom(share_destination, form_data, product_id, target_id, selected_data) + sharing_feedback_handler(response, self.request) return redirect(reverse('tom_targets:detail', kwargs={'pk': request.POST.get('target')})) - def share_with_tom(self, tom_name, product): - """ - When sharing a DataProduct with another TOM we likely want to share the data product itself and let the other - TOM process it rather than share the Reduced Datums - :param tom_name: name of destination tom in settings.DATA_SHARING - :param product: DataProduct model instance - :return: - """ - try: - destination_tom_base_url = settings.DATA_SHARING[tom_name]['BASE_URL'] - username = settings.DATA_SHARING[tom_name]['USERNAME'] - password = settings.DATA_SHARING[tom_name]['PASSWORD'] - except KeyError as err: - raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {tom_name}: Key {err} not found.') - auth = (username, password) - headers = {'Media-Type': 'application/json'} - target = product.target - serialized_target_data = TargetSerializer(target).data - targets_url = destination_tom_base_url + 'api/targets/' - # TODO: Make sure aliases are checked before creating new target - # Attempt to create Target in Destination TOM - response = requests.post(targets_url, headers=headers, auth=auth, data=serialized_target_data) - try: - target_response = response.json() - destination_target_id = target_response['id'] - except KeyError: - # If Target already exists at destination, find ID - response = requests.get(targets_url, headers=headers, auth=auth, data=serialized_target_data) - target_response = response.json() - destination_target_id = target_response['results'][0]['id'] - - serialized_dataproduct_data = DataProductSerializer(product).data - serialized_dataproduct_data['target'] = destination_target_id - dataproducts_url = destination_tom_base_url + 'api/dataproducts/' - # TODO: this should be updated when tom_dataproducts is updated to use django.core.storage - dataproduct_filename = os.path.join(settings.MEDIA_ROOT, product.data.name) - # Save DataProduct in Destination TOM - with open(dataproduct_filename, 'rb') as dataproduct_filep: - files = {'file': (product.data.name, dataproduct_filep, 'text/csv')} - headers = {'Media-Type': 'multipart/form-data'} - response = requests.post(dataproducts_url, data=serialized_dataproduct_data, files=files, - headers=headers, auth=auth) - return response - - def get_share_safe_datums(self, destination, reduced_datums, **kwargs): - """ - Custom sharing protocols used to determine when data is shared with a destination. - This example prevents sharing if a datum has already been published to the given Hermes topic. - :param destination: sharing destination string - :param reduced_datums: selected input datums - :return: queryset of reduced datums to be shared - """ - return reduced_datums - # if 'hermes' in destination: - # message_topic = kwargs.get('topic', None) - # # Remove data points previously shared to the given topic - # filtered_datums = reduced_datums.exclude(Q(message__exchange_status='published') - # & Q(message__topic=message_topic)) - # else: - # filtered_datums = reduced_datums - # return filtered_datums - class DataProductGroupDetailView(DetailView): """ diff --git a/tom_targets/api_views.py b/tom_targets/api_views.py index 6e87576ac..861640595 100644 --- a/tom_targets/api_views.py +++ b/tom_targets/api_views.py @@ -3,10 +3,11 @@ from guardian.shortcuts import get_objects_for_user from rest_framework.mixins import DestroyModelMixin, RetrieveModelMixin from rest_framework.viewsets import GenericViewSet, ModelViewSet +from rest_framework import status from tom_targets.filters import TargetFilter -from tom_targets.models import TargetExtra, TargetName -from tom_targets.serializers import TargetSerializer, TargetExtraSerializer, TargetNameSerializer +from tom_targets.models import TargetExtra, TargetName, TargetList +from tom_targets.serializers import TargetSerializer, TargetExtraSerializer, TargetNameSerializer, TargetListSerializer permissions_map = { # TODO: Use the built-in DRF mapping or just switch to DRF entirely. @@ -53,6 +54,22 @@ def get_queryset(self): permission_required = permissions_map.get(self.request.method) return get_objects_for_user(self.request.user, f'tom_targets.{permission_required}') + def create(self, request, *args, **kwargs): + response = super().create(request, *args, **kwargs) + + # Custom message for successful target creation + if response.status_code == status.HTTP_201_CREATED: + response.data['message'] = 'Target successfully uploaded.' + return response + + def update(self, request, *args, **kwargs): + response = super().update(request, *args, **kwargs) + + # Custom message for successful target update + if response.status_code == status.HTTP_200_OK: + response.data['message'] = 'Target successfully updated.' + return response + class TargetNameViewSet(DestroyModelMixin, PermissionListMixin, RetrieveModelMixin, GenericViewSet): """ @@ -82,3 +99,18 @@ def get_queryset(self): return TargetExtra.objects.filter( target__in=get_objects_for_user(self.request.user, f'tom_targets.{permission_required}') ) + + +class TargetListViewSet(DestroyModelMixin, PermissionListMixin, RetrieveModelMixin, GenericViewSet): + """ + Viewset for TargetList objects. Only ``GET`` and ``DELETE`` operations are permitted. + + To view available query parameters, please use the OPTIONS endpoint, which can be accessed through the web UI. + """ + serializer_class = TargetListSerializer + + def get_queryset(self): + permission_required = permissions_map.get(self.request.method) + return TargetList.objects.filter( + target__in=get_objects_for_user(self.request.user, f'tom_targets.{permission_required}') + ) diff --git a/tom_targets/filters.py b/tom_targets/filters.py index dafc564b3..c47e7bd47 100644 --- a/tom_targets/filters.py +++ b/tom_targets/filters.py @@ -2,7 +2,7 @@ from django.db.models import Q import django_filters -from tom_targets.models import Target, TargetList +from tom_targets.models import Target, TargetList, TargetMatchManager from tom_targets.utils import cone_search_filter @@ -55,7 +55,29 @@ def __init__(self, *args, **kwargs): name = django_filters.CharFilter(method='filter_name', label='Name') def filter_name(self, queryset, name, value): - return queryset.filter(Q(name__icontains=value) | Q(aliases__name__icontains=value)).distinct() + """ + Return a queryset for targets with names or aliases containing the given coma-separated list of terms. + """ + q_set = Q() + for term in value.split(','): + q_set |= Q(name__icontains=term) | Q(aliases__name__icontains=term) + return queryset.filter(q_set).distinct() + + name_fuzzy = django_filters.CharFilter(method='filter_name_fuzzy', label='Name (Fuzzy)') + + def filter_name_fuzzy(self, queryset, name, value): + """ + Return a queryset for targets with names or aliases fuzzy matching the given coma-separated list of terms. + A fuzzy match is determined by the `make_simple_name` method of the `TargetMatchManager` class. + """ + matching_names = [] + for term in value.split(','): + simple_name = TargetMatchManager.make_simple_name(self, term) + for target in Target.objects.all().prefetch_related('aliases'): + for alias in target.names: + if TargetMatchManager.make_simple_name(self, alias) == simple_name: + matching_names.append(target.name) + return queryset.filter(name__in=matching_names).distinct() cone_search = django_filters.CharFilter(method='filter_cone_search', label='Cone Search', help_text='RA, Dec, Search Radius (degrees)') diff --git a/tom_targets/forms.py b/tom_targets/forms.py index 149ec5cdb..be549285e 100644 --- a/tom_targets/forms.py +++ b/tom_targets/forms.py @@ -6,8 +6,9 @@ from django.contrib.auth.models import Group from guardian.shortcuts import assign_perm, get_groups_with_perms, remove_perm +from tom_dataproducts.sharing import get_sharing_destination_options from .models import ( - Target, TargetExtra, TargetName, SIDEREAL_FIELDS, NON_SIDEREAL_FIELDS, REQUIRED_SIDEREAL_FIELDS, + Target, TargetExtra, TargetName, TargetList, SIDEREAL_FIELDS, NON_SIDEREAL_FIELDS, REQUIRED_SIDEREAL_FIELDS, REQUIRED_NON_SIDEREAL_FIELDS, REQUIRED_NON_SIDEREAL_FIELDS_PER_SCHEME ) @@ -166,3 +167,35 @@ def clean(self): widgets={'value': forms.TextInput()}) TargetNamesFormset = inlineformset_factory(Target, TargetName, fields=('name',), validate_min=False, can_delete=True, extra=3) + + +class TargetShareForm(forms.Form): + """ + Form for sharing a target with an outside destination such as another TOM Toolkit or Hermes + """ + share_destination = forms.ChoiceField(required=True, choices=[], label="Destination") + target = forms.ModelChoiceField( + Target.objects.all(), + widget=forms.HiddenInput(), + required=True) + submitter = forms.CharField(widget=forms.HiddenInput()) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['share_destination'].choices = get_sharing_destination_options() + + +class TargetListShareForm(forms.Form): + """ + Form for sharing a target list with an outside destination such as another TOM Toolkit or Hermes + """ + share_destination = forms.ChoiceField(required=True, choices=[], label="Destination") + target_list = forms.ModelChoiceField( + TargetList.objects.all(), + widget=forms.HiddenInput(), + required=True) + submitter = forms.CharField(widget=forms.HiddenInput()) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['share_destination'].choices = get_sharing_destination_options() diff --git a/tom_targets/serializers.py b/tom_targets/serializers.py index a39dec43c..2c1e46eca 100644 --- a/tom_targets/serializers.py +++ b/tom_targets/serializers.py @@ -3,7 +3,7 @@ from rest_framework import serializers from tom_common.serializers import GroupSerializer -from tom_targets.models import Target, TargetExtra, TargetName +from tom_targets.models import Target, TargetExtra, TargetName, TargetList from tom_targets.validators import RequiredFieldsTogetherValidator @@ -23,13 +23,26 @@ class Meta: fields = ('id', 'key', 'value') +class TargetListSerializer(serializers.ModelSerializer): + """ + TargetList serializer responsible for transforming models to/from + """ + id = serializers.IntegerField(required=False) + name = serializers.CharField(required=False) + + class Meta: + model = TargetList + fields = ('id', 'name') + + class TargetSerializer(serializers.ModelSerializer): - """Target serializer responsbile for transforming models to/from + """Target serializer responsible for transforming models to/from json (or other representations). See https://www.django-rest-framework.org/api-guide/serializers/#modelserializer """ targetextra_set = TargetExtraSerializer(many=True, required=False) aliases = TargetNameSerializer(many=True, required=False) + target_lists = TargetListSerializer(many=True, required=False) groups = GroupSerializer(many=True, required=False) # TODO: return groups in detail and list class Meta: @@ -50,18 +63,21 @@ def create(self, validated_data): here we pop the alias/tag/group data and save it using their respective serializers """ - aliases = validated_data.pop('aliases', []) targetextras = validated_data.pop('targetextra_set', []) groups = validated_data.pop('groups', []) + target_lists = validated_data.pop('target_lists', []) target = Target.objects.create(**validated_data) - # Save groups for this target + # Save user groups for this target group_serializer = GroupSerializer(data=groups, many=True) if group_serializer.is_valid(): for group in groups: - group_instance = Group.objects.get(pk=group['id']) + try: + group_instance = Group.objects.get(pk=group['id']) + except KeyError: + group_instance, _ = Group.objects.get_or_create(name=group['name']) assign_perm('tom_targets.view_target', group_instance, target) assign_perm('tom_targets.change_target', group_instance, target) assign_perm('tom_targets.delete_target', group_instance, target) @@ -80,6 +96,13 @@ def create(self, validated_data): if tes.is_valid(): tes.save(target=target) + # Save target lists for this target + tls = TargetListSerializer(data=target_lists, many=True) + if tls.is_valid(): + for target_list in target_lists: + tl_instance, _ = TargetList.objects.get_or_create(name=target_list['name']) + tl_instance.targets.add(target) + return target def to_representation(self, instance): @@ -98,6 +121,7 @@ def update(self, instance, validated_data): aliases = validated_data.pop('aliases', []) targetextras = validated_data.pop('targetextra_set', []) groups = validated_data.pop('groups', []) + target_lists = validated_data.pop('target_lists', []) # Save groups for this target group_serializer = GroupSerializer(data=groups, many=True) @@ -137,6 +161,13 @@ def update(self, instance, validated_data): if tes.is_valid(): tes.save(target=instance) + # Update target lists for this target + tls = TargetListSerializer(data=target_lists, many=True) + if tls.is_valid(): + for target_list in target_lists: + tl_instance, _ = TargetList.objects.get_or_create(name=target_list['name']) + tl_instance.targets.add(instance) + fields_to_validate = ['name', 'type', 'ra', 'dec', 'epoch', 'parallax', 'pm_ra', 'pm_dec', 'galactic_lng', 'galactic_lat', 'distance', 'distance_err', 'scheme', 'epoch_of_elements', 'mean_anomaly', 'arg_of_perihelion', 'eccentricity', 'lng_asc_node', 'inclination', diff --git a/tom_targets/sharing.py b/tom_targets/sharing.py new file mode 100644 index 000000000..62b1b813b --- /dev/null +++ b/tom_targets/sharing.py @@ -0,0 +1,59 @@ +import requests + +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured + +from tom_targets.serializers import TargetSerializer +from tom_dataproducts.sharing import get_destination_target + + +def share_target_with_tom(share_destination, form_data, target_lists=()): + """ + Share a target with a remote TOM. + :param share_destination: The name of the destination TOM as defined in settings.DATA_SHARING + :param form_data: The form data from the target form + :param target_lists: The target lists to add the target to in the destination TOM + :return: The response from the destination TOM + """ + # Try to get destination tom authentication/URL information + try: + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + username = settings.DATA_SHARING[share_destination]['USERNAME'] + password = settings.DATA_SHARING[share_destination]['PASSWORD'] + except KeyError as err: + raise ImproperlyConfigured(f'Check DATA_SHARING configuration for {share_destination}: Key {err} not found.') + auth = (username, password) + headers = {'Content-Type': 'application/json', 'Accept': 'application/json'} + + # establish destination TOM URLs + targets_url = destination_tom_base_url + 'api/targets/' + + # Check if target already exists in destination DB + destination_target_id, target_search_response = get_destination_target(form_data['target'], targets_url, headers, + auth) + # Handle errors or multiple targets found + if target_search_response.status_code != 200: + return target_search_response + elif isinstance(destination_target_id, list) and len(destination_target_id) > 1: + return {'message': 'ERROR: Multiple targets with matching name found in destination TOM.'} + + # Build list of targetlists to add target to in destination TOM + target_dict_list = [{'name': f'Imported From {settings.TOM_NAME}'}] + for target_list in target_lists: + target_dict_list.append({'name': target_list.name}) + + # Create or update target in destination TOM + if destination_target_id is None: + # If target is not in Destination, serialize and create new target. + serialized_target = TargetSerializer(form_data['target']).data + # Remove local User Groups + serialized_target['groups'] = [] + # Add target lists + serialized_target['target_lists'] = target_dict_list + target_create_response = requests.post(targets_url, json=serialized_target, headers=headers, auth=auth) + else: + # Add target to target lists if it already exists in destination TOM + update_target_data = {'target_lists': target_dict_list} + update_target_url = targets_url + f'{destination_target_id}/' + target_create_response = requests.patch(update_target_url, json=update_target_data, headers=headers, auth=auth) + return target_create_response diff --git a/tom_targets/templates/tom_targets/partials/target_buttons.html b/tom_targets/templates/tom_targets/partials/target_buttons.html index 5cd86e4f4..321bca67d 100644 --- a/tom_targets/templates/tom_targets/partials/target_buttons.html +++ b/tom_targets/templates/tom_targets/partials/target_buttons.html @@ -1,2 +1,5 @@ -Update Target -Delete Target \ No newline at end of file +Update +{% if sharing %} + Share +{% endif %} +Delete diff --git a/tom_targets/templates/tom_targets/partials/target_table.html b/tom_targets/templates/tom_targets/partials/target_table.html index 725ea5e10..f6ff3b51a 100644 --- a/tom_targets/templates/tom_targets/partials/target_table.html +++ b/tom_targets/templates/tom_targets/partials/target_table.html @@ -1,7 +1,13 @@ - + {% if request.GET.type == 'SIDEREAL' %} @@ -15,7 +21,13 @@ {% for target in targets %} - + diff --git a/tom_targets/templates/tom_targets/target_detail.html b/tom_targets/templates/tom_targets/target_detail.html index b58bc942f..5fdac4eac 100644 --- a/tom_targets/templates/tom_targets/target_detail.html +++ b/tom_targets/templates/tom_targets/target_detail.html @@ -85,6 +85,11 @@

Observations

{% observation_list object %}
+ {% if user.is_authenticated %} + {% query_forced_photometry object %} +
+
+ {% endif %} {% if user.is_authenticated %} {% upload_dataproduct object %} {% endif %} diff --git a/tom_targets/templates/tom_targets/target_group_share.html b/tom_targets/templates/tom_targets/target_group_share.html new file mode 100644 index 000000000..cb6476a35 --- /dev/null +++ b/tom_targets/templates/tom_targets/target_group_share.html @@ -0,0 +1,41 @@ +{% extends 'tom_common/base.html' %} +{% load bootstrap4 static targets_extras %} +{% block title %}Share {{ target_list.name }} Target List{% endblock %} +{% block additional_css %} + + +{% endblock %} +{% block content %} +

Share {{ target_list.name }} Target List

+
+
+
+ {% csrf_token %} + {% bootstrap_form form %} +
+
+ +
+
+ +
+ {% select_target_js %} + +
+ + +
+ {% csrf_token %} +
+ + +
+ {% target_table target_list.targets.all True %} +
+ + +{% buttons %} + Back +{% endbuttons %} + +{% endblock %} \ No newline at end of file diff --git a/tom_targets/templates/tom_targets/target_grouping.html b/tom_targets/templates/tom_targets/target_grouping.html index d5889db1a..b6d2ba9e5 100644 --- a/tom_targets/templates/tom_targets/target_grouping.html +++ b/tom_targets/templates/tom_targets/target_grouping.html @@ -17,6 +17,9 @@

Target Groupings

+ {% if sharing %} + + {% endif %} @@ -25,6 +28,9 @@

Target Groupings

+ {% if sharing %} + + {% endif %} {% empty %} diff --git a/tom_targets/templates/tom_targets/target_share.html b/tom_targets/templates/tom_targets/target_share.html new file mode 100644 index 000000000..1e504b843 --- /dev/null +++ b/tom_targets/templates/tom_targets/target_share.html @@ -0,0 +1,27 @@ +{% extends 'tom_common/base.html' %} +{% load bootstrap4 dataproduct_extras static %} +{% block title %}Share Target {{ target.name }}{% endblock %} +{% block additional_css %} + + +{% endblock %} +{% block content %} +

Share {{ target.name }}

+ +
+
+ {% bootstrap_form form %} +
+
+ +
+
+

Include Data

+ {% with target_share=True %} + {% get_photometry_data target target_share %} + {% endwith %} + {% buttons %} + Back + {% endbuttons %} + +{% endblock %} diff --git a/tom_targets/templatetags/targets_extras.py b/tom_targets/templatetags/targets_extras.py index d5c8a40b1..2fe5b6d0a 100644 --- a/tom_targets/templatetags/targets_extras.py +++ b/tom_targets/templatetags/targets_extras.py @@ -50,7 +50,8 @@ def target_buttons(target): """ Displays the Update and Delete buttons for a target. """ - return {'target': target} + return {'target': target, + 'sharing': getattr(settings, "DATA_SHARING", None)} @register.inclusion_tag('tom_targets/partials/target_data.html') @@ -310,9 +311,10 @@ def aladin(target): @register.inclusion_tag('tom_targets/partials/target_table.html') -def target_table(targets): +def target_table(targets, all_checked=False): """ Returns a partial for a table of targets, used in the target_list.html template by default """ - return {'targets': targets} + + return {'targets': targets, 'all_checked': all_checked} diff --git a/tom_targets/tests/test_api.py b/tom_targets/tests/test_api.py index 7c2261319..a05cfa6e9 100644 --- a/tom_targets/tests/test_api.py +++ b/tom_targets/tests/test_api.py @@ -7,8 +7,8 @@ from rest_framework.test import APITestCase from tom_targets.tests.factories import SiderealTargetFactory, NonSiderealTargetFactory -from tom_targets.tests.factories import TargetExtraFactory, TargetNameFactory -from tom_targets.models import Target, TargetExtra, TargetName +from tom_targets.tests.factories import TargetExtraFactory, TargetNameFactory, TargetGroupingFactory +from tom_targets.models import Target, TargetExtra, TargetName, TargetList class TestTargetViewset(APITestCase): @@ -53,24 +53,32 @@ def test_target_detail(self): self.assertEqual(response.json()['detail'], 'Not found.') def test_target_create(self): + """ + Test that a target can be created with all valid parameters through the API + """ collaborator = User.objects.create(username='test collaborator') group = Group.objects.create(name='bourgeoisie') group.user_set.add(self.user) group.user_set.add(collaborator) + target_list = TargetList.objects.create(name="Test TargetList") target_data = { - 'name': 'test_target_name_wtf', + 'name': 'test_target_name', 'type': Target.SIDEREAL, 'ra': 123.456, 'dec': -32.1, 'groups': [ - {'id': group.id} + {'id': group.id}, {'name': 'test_group'} ], 'targetextra_set': [ {'key': 'foo', 'value': 5} ], 'aliases': [ {'name': 'alternative name'} + ], + 'target_lists': [ + {'name': target_list.name}, # Add to existing Target List + {'name': 'newer_tlist'} # Create new Target List ] } response = self.client.post(reverse('api:targets-list'), data=target_data) @@ -79,6 +87,9 @@ def test_target_create(self): self.assertEqual(response.json()['aliases'][0]['name'], target_data['aliases'][0]['name']) self.assertEqual(get_objects_for_user(collaborator, 'tom_targets.view_target').first().name, target_data['name']) # Test that group permissions are respected + target_list2 = TargetList.objects.get(name='newer_tlist') + self.assertEqual(target_list.targets.all().count(), 1) + self.assertEqual(target_list2.targets.all().count(), 1) # TODO: For whatever reason, in django-guardian, authenticated users have permission to create objects, # regardless of their row-level permissions. This should be addressed eventually--however, we don't provide a @@ -194,6 +205,38 @@ def test_targetname_update(self): f'Alias \'{self.st.name}\' conflicts with Target name \'{self.st.name}\'', status_code=status.HTTP_400_BAD_REQUEST) + def test_targetlist_update(self): + # Test Add existing target to new target list + target_list = TargetGroupingFactory.create(name='tl') + target_list.targets.add(self.st) + updates = { + 'target_lists': [ + {'id': target_list.id, 'name': 'update tl'}, + {'name': 'create tl'} + ] + } + response = self.client.patch(reverse('api:targets-detail', args=(self.st.id,)), data=updates) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.st.refresh_from_db() + target_list.refresh_from_db() + # Target list added by name, not ID, so new name = new TargetList + self.assertEqual(TargetList.objects.filter(targets=self.st).count(), 3) + self.assertEqual(target_list.name, 'tl') + + # Ensure proper handling when adding target to existing targetlist + target_list_s2 = TargetGroupingFactory.create(name='tl_s2') + target_list_s2.targets.add(self.st2) + updates = { + 'target_lists': [ + {'name': target_list_s2.name} + ] + } + response = self.client.patch(reverse('api:targets-detail', args=(self.st.id,)), data=updates) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.st.refresh_from_db() + target_list_s2.refresh_from_db() + self.assertEqual(target_list_s2.targets.count(), 2) + def test_target_delete(self): response = self.client.delete(reverse('api:targets-detail', args=(self.st.id,))) self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) diff --git a/tom_targets/tests/tests.py b/tom_targets/tests/tests.py index 661acff85..3792ae11a 100644 --- a/tom_targets/tests/tests.py +++ b/tom_targets/tests/tests.py @@ -1,16 +1,21 @@ import pytz from datetime import datetime +import responses from django.contrib.auth.models import User, Group from django.contrib.messages import get_messages from django.contrib.messages.constants import SUCCESS, WARNING from django.test import TestCase, override_settings +from django.conf import settings from django.urls import reverse from django.core.exceptions import ValidationError from .factories import SiderealTargetFactory, NonSiderealTargetFactory, TargetGroupingFactory, TargetNameFactory +from tom_observations.tests.utils import FakeRoboticFacility +from tom_observations.tests.factories import ObservingRecordFactory from tom_targets.models import Target, TargetExtra, TargetList, TargetName from tom_targets.utils import import_targets +from tom_dataproducts.models import ReducedDatum from guardian.shortcuts import assign_perm @@ -1275,3 +1280,361 @@ def test_persist_filter_empty(self): response = self.client.post(reverse('targets:add-remove-grouping'), data={}, follow=True) response_query_dict = response.context['filter'].data self.assertEqual(response_query_dict, expected_query_dict) + + +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], + TARGET_PERMISSIONS_ONLY=True, + DATA_SHARING={'local_host': {'BASE_URL': 'https://fake.url/example/', + 'USERNAME': 'fake_user', + 'PASSWORD': 'password'}}) +class TestShareTargets(TestCase): + """ + Tests for the share_targets view. + Tests the behavior of the SENDING TOM and Mocks responses from receiving TOM. + """ + def setUp(self): + self.target = SiderealTargetFactory.create() + self.observation_record = ObservingRecordFactory.create( + target_id=self.target.id, + facility=FakeRoboticFacility.name, + parameters={} + ) + self.user = User.objects.create_user(username='test', email='test@example.com') + assign_perm('tom_targets.view_target', self.user, self.target) + self.client.force_login(self.user) + self.rd1 = ReducedDatum.objects.create( + target=self.target, + data_type='photometry', + value={'magnitude': 18.5, 'error': .5, 'filter': 'V'} + ) + self.rd2 = ReducedDatum.objects.create( + target=self.target, + data_type='photometry', + value={'magnitude': 19.5, 'error': .5, 'filter': 'B'} + ) + self.rd3 = ReducedDatum.objects.create( + target=self.target, + data_type='photometry', + value={'magnitude': 17.5, 'error': .5, 'filter': 'R'} + ) + + @responses.activate + def test_share_target_no_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"error": "not found"}, + status=500 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + + response = self.client.post( + reverse('targets:share', kwargs={'pk': self.target.id}), + { + 'submitter': ['test_submitter'], + 'target': self.target.id, + 'share_destination': [share_destination], + }, + follow=True + ) + self.assertContains(response, "not found") + + @responses.activate + def test_share_target_valid_connection_no_target_found(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"results": []}, + status=200 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + responses.add( + responses.POST, + destination_tom_base_url + 'api/targets/', + json={"message": "Target successfully uploaded."}, + status=200, + ) + response = self.client.post( + reverse('targets:share', kwargs={'pk': self.target.id}), + { + 'submitter': ['test_submitter'], + 'target': self.target.id, + 'share_destination': [share_destination], + }, + follow=True + ) + + self.assertContains(response, 'Target successfully uploaded.') + + @responses.activate + def test_share_target_valid_connection_multiple_target_found(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"results": [{'id': 1}, {'id': 2}]}, + status=200 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + + response = self.client.post( + reverse('targets:share', kwargs={'pk': self.target.id}), + { + 'submitter': ['test_submitter'], + 'target': self.target.id, + 'share_destination': [share_destination], + }, + follow=True + ) + + self.assertContains(response, 'ERROR: Multiple targets with matching name found in destination TOM.') + + @responses.activate + def test_share_reduceddatums_target_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"results": [{'id': 1}]}, + status=200 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + responses.add( + responses.POST, + destination_tom_base_url + 'api/reduceddatums/', + json={}, + status=201, + ) + responses.add( + responses.PATCH, + destination_tom_base_url + 'api/targets/1/', + json={}, + status=200, + ) + + response = self.client.post( + reverse('targets:share', kwargs={'pk': self.target.id}), + { + 'submitter': ['test_submitter'], + 'target': self.target.id, + 'share_destination': [share_destination], + 'share-box': [1, 2] + }, + follow=True + ) + + self.assertContains(response, '2 of 2 datums successfully saved.') + + +@override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], + TARGET_PERMISSIONS_ONLY=True, + DATA_SHARING={'local_host': {'BASE_URL': 'https://fake.url/example/', + 'USERNAME': 'fake_user', + 'PASSWORD': 'password'}}) +class TestShareTargetList(TestCase): + def setUp(self): + self.target = SiderealTargetFactory.create() + self.target2 = SiderealTargetFactory.create() + self.target3 = SiderealTargetFactory.create() + self.observation_record = ObservingRecordFactory.create( + target_id=self.target.id, + facility=FakeRoboticFacility.name, + parameters={} + ) + self.target_list = TargetList.objects.create( + name="test_group", + ) + self.target_list.targets.set(Target.objects.all()) + self.user = User.objects.create_user(username='test', email='test@example.com') + assign_perm('tom_targets.view_target', self.user, self.target) + self.client.force_login(self.user) + self.rd1 = ReducedDatum.objects.create( + target=self.target, + data_type='photometry', + value={'magnitude': 18.5, 'error': .5, 'filter': 'V'} + ) + self.rd2 = ReducedDatum.objects.create( + target=self.target, + data_type='photometry', + value={'magnitude': 19.5, 'error': .5, 'filter': 'B'} + ) + self.rd3 = ReducedDatum.objects.create( + target=self.target, + data_type='photometry', + value={'magnitude': 17.5, 'error': .5, 'filter': 'R'} + ) + self.rd4 = ReducedDatum.objects.create( + target=self.target2, + data_type='photometry', + value={'magnitude': 17.5, 'error': .5, 'filter': 'R'} + ) + + @responses.activate + def test_share_group_no_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"error": "not found"}, + status=500 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + + response = self.client.post( + reverse('targets:share-group', kwargs={'pk': self.target_list.id}), + { + 'submitter': ['test_submitter'], + 'target_list': self.target_list.id, + 'share_destination': [share_destination], + }, + follow=True + ) + self.assertContains(response, "not found") + + @responses.activate + def test_share_group_valid_connection_selected_target_not_found(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"results": []}, + status=200 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + responses.add( + responses.POST, + destination_tom_base_url + 'api/targets/', + json={"message": "Target successfully uploaded."}, + status=200, + ) + response = self.client.post( + reverse('targets:share-group', kwargs={'pk': self.target_list.id}), + { + 'submitter': ['test_submitter'], + 'target_list': self.target_list.id, + 'share_destination': [share_destination], + 'selected-target': [self.target.id, self.target2.id] + }, + follow=True + ) + + self.assertContains(response, 'Target successfully uploaded.') + + @responses.activate + def test_share_reduceddatums_group_valid_responses(self): + share_destination = 'local_host' + destination_tom_base_url = settings.DATA_SHARING[share_destination]['BASE_URL'] + + rsp1 = responses.Response( + method="GET", + url=destination_tom_base_url + 'api/targets/', + json={"results": [{'id': 1}]}, + status=200 + ) + responses.add(rsp1) + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + responses.add( + responses.POST, + destination_tom_base_url + 'api/reduceddatums/', + json={}, + status=201, + ) + responses.add( + responses.PATCH, + destination_tom_base_url + 'api/targets/1/', + json={"message": "Target successfully uploaded."}, + status=200, + ) + + response = self.client.post( + reverse('targets:share-group', kwargs={'pk': self.target_list.id}), + { + 'submitter': ['test_submitter'], + 'target_list': self.target_list.id, + 'share_destination': [share_destination], + 'dataSwitch': 'on', + 'selected-target': [self.target.id, self.target2.id] + }, + follow=True + ) + + self.assertContains(response, '1 of 1 datums successfully saved.') + + @responses.activate + def test_share_empty_group(self): + share_destination = 'local_host' + + responses.add( + responses.GET, + "http://hermes-dev.lco.global/api/v0/profile/", + json={"error": "not found"}, + status=404, + ) + + response = self.client.post( + reverse('targets:share-group', kwargs={'pk': self.target_list.id}), + { + 'submitter': ['test_submitter'], + 'target_list': self.target_list.id, + 'share_destination': [share_destination], + 'dataSwitch': 'on', + 'selected-target': [] + }, + follow=True + ) + self.assertContains(response, 'No targets shared.') diff --git a/tom_targets/urls.py b/tom_targets/urls.py index 33e98aa81..136bc0066 100644 --- a/tom_targets/urls.py +++ b/tom_targets/urls.py @@ -1,16 +1,18 @@ from django.urls import path from .views import TargetCreateView, TargetUpdateView, TargetDetailView, TargetNameSearchView -from .views import TargetDeleteView, TargetListView, TargetImportView, TargetExportView +from .views import TargetDeleteView, TargetListView, TargetImportView, TargetExportView, TargetShareView from .views import TargetGroupingView, TargetGroupingDeleteView, TargetGroupingCreateView, TargetAddRemoveGroupingView +from .views import TargetGroupingShareView -from .api_views import TargetViewSet, TargetExtraViewSet, TargetNameViewSet +from .api_views import TargetViewSet, TargetExtraViewSet, TargetNameViewSet, TargetListViewSet from tom_common.api_router import SharedAPIRootRouter router = SharedAPIRootRouter() router.register(r'targets', TargetViewSet, 'targets') router.register(r'targetextra', TargetExtraViewSet, 'targetextra') router.register(r'targetname', TargetNameViewSet, 'targetname') +router.register(r'targetlist', TargetListViewSet, 'targetlist') app_name = 'tom_targets' @@ -24,7 +26,10 @@ path('name/', TargetNameSearchView.as_view(), name='name-search'), path('/update/', TargetUpdateView.as_view(), name='update'), path('/delete/', TargetDeleteView.as_view(), name='delete'), + path('/share/', TargetShareView.as_view(), name='share'), path('/', TargetDetailView.as_view(), name='detail'), path('targetgrouping//delete/', TargetGroupingDeleteView.as_view(), name='delete-group'), - path('targetgrouping/create/', TargetGroupingCreateView.as_view(), name='create-group') + path('targetgrouping/create/', TargetGroupingCreateView.as_view(), name='create-group'), + path('targetgrouping//share/', TargetGroupingShareView.as_view(), name='share-group') + ] diff --git a/tom_targets/views.py b/tom_targets/views.py index 500c1e472..ac7d39bfb 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -17,7 +17,7 @@ from django.urls import reverse_lazy, reverse from django.utils.text import slugify from django.utils.safestring import mark_safe -from django.views.generic.edit import CreateView, UpdateView, DeleteView +from django.views.generic.edit import CreateView, UpdateView, DeleteView, FormView from django.views.generic.detail import DetailView from django.views.generic.list import ListView from django.views.generic import RedirectView, TemplateView, View @@ -32,9 +32,11 @@ from tom_observations.observation_template import ApplyObservationTemplateForm from tom_observations.models import ObservationTemplate from tom_targets.filters import TargetFilter -from tom_targets.forms import ( - SiderealTargetCreateForm, NonSiderealTargetCreateForm, TargetExtraFormset, TargetNamesFormset -) +from tom_targets.forms import SiderealTargetCreateForm, NonSiderealTargetCreateForm, TargetExtraFormset +from tom_targets.forms import TargetNamesFormset, TargetShareForm, TargetListShareForm +from tom_targets.sharing import share_target_with_tom +from tom_dataproducts.sharing import share_data_with_hermes, share_data_with_tom, sharing_feedback_handler + from tom_targets.groups import ( add_all_to_grouping, add_selected_to_grouping, remove_all_from_grouping, remove_selected_from_grouping, move_all_to_grouping, move_selected_to_grouping @@ -330,6 +332,62 @@ class TargetDeleteView(Raise403PermissionRequiredMixin, DeleteView): model = Target +class TargetShareView(FormView): + """ + View for sharing a target. Requires authorization. + """ + template_name = 'tom_targets/target_share.html' + permission_required = 'tom_targets.share_target' + form_class = TargetShareForm + + def get_context_data(self, *args, **kwargs): + """ + Adds the target information to the context. + :returns: context object + :rtype: dict + """ + context = super().get_context_data(*args, **kwargs) + target_id = self.kwargs.get('pk', None) + context['target'] = Target.objects.get(id=target_id) + + return context + + def get_success_url(self): + """ + Redirect to target detail page for shared target + """ + return reverse_lazy('targets:detail', kwargs={'pk': self.kwargs.get('pk', None)}) + + def form_invalid(self, form): + """ + Adds errors to Django messaging framework in the case of an invalid form and redirects to the previous page. + """ + # TODO: Format error messages in a more human-readable way + messages.error(self.request, 'There was a problem sharing your Data: {}'.format(form.errors.as_json())) + return redirect(self.get_success_url()) + + def form_valid(self, form): + """ + Shares the target with the selected destination(s) and redirects to the target detail page. + """ + form_data = form.cleaned_data + share_destination = form_data['share_destination'] + target_id = self.kwargs.get('pk', None) + selected_data = self.request.POST.getlist("share-box") + if 'HERMES' in share_destination.upper(): + response = share_data_with_hermes(share_destination, form_data, None, target_id, selected_data) + sharing_feedback_handler(response, self.request) + else: + # Share Target with Destination TOM + response = share_target_with_tom(share_destination, form_data) + sharing_feedback_handler(response, self.request) + if selected_data: + # Share Data with Destination TOM + response = share_data_with_tom(share_destination, form_data, selected_data=selected_data) + sharing_feedback_handler(response, self.request) + return redirect(self.get_success_url()) + + class TargetDetailView(Raise403PermissionRequiredMixin, DetailView): """ View that handles the display of the target details. Requires authorization. @@ -497,6 +555,16 @@ class TargetGroupingView(PermissionListMixin, ListView): model = TargetList paginate_by = 25 + def get_context_data(self, *args, **kwargs): + """ + Adds ``settings.DATA_SHARING`` to the context to see if sharing has been configured. + :returns: context object + :rtype: dict + """ + context = super().get_context_data(*args, **kwargs) + context['sharing'] = getattr(settings, "DATA_SHARING", None) + return context + class TargetGroupingDeleteView(Raise403PermissionRequiredMixin, DeleteView): """ @@ -528,3 +596,68 @@ def form_valid(self, form): assign_perm('tom_targets.change_targetlist', self.request.user, obj) assign_perm('tom_targets.delete_targetlist', self.request.user, obj) return super().form_valid(form) + + +class TargetGroupingShareView(FormView): + """ + View for sharing a TargetList. Requires authorization. + """ + template_name = 'tom_targets/target_group_share.html' + permission_required = 'tom_targets.share_target' + form_class = TargetListShareForm + + def get_context_data(self, *args, **kwargs): + """ + Adds the ``TargetListShareForm`` to the context and prepopulates the hidden fields. + :returns: context object + :rtype: dict + """ + context = super().get_context_data(*args, **kwargs) + target_list_id = self.kwargs.get('pk', None) + target_list = TargetList.objects.get(id=target_list_id) + context['target_list'] = target_list + initial = {'submitter': self.request.user, + 'target_list': target_list, + 'share_title': f"Updated targets for group {target_list.name}."} + form = TargetListShareForm(initial=initial) + context['form'] = form + return context + + def get_success_url(self): + """ + Redirects to the target list page with the target list name as a query parameter. + """ + return reverse_lazy('targets:list')+f'?targetlist__name={self.kwargs.get("pk", None)}' + + def form_invalid(self, form): + """ + Adds errors to Django messaging framework in the case of an invalid form and redirects to the previous page. + """ + # TODO: Format error messages in a more human-readable way + messages.error(self.request, 'There was a problem sharing your Target List: {}'.format(form.errors.as_json())) + return redirect(self.get_success_url()) + + def form_valid(self, form): + form_data = form.cleaned_data + share_destination = form_data['share_destination'] + selected_targets = self.request.POST.getlist('selected-target') + data_switch = self.request.POST.get('dataSwitch', False) + if 'HERMES' in share_destination.upper(): + # TODO: Implement Hermes sharing + # response = share_data_with_hermes(share_destination, form_data, None, target_id, selected_data) + messages.error(self.request, "Publishing Groups to Hermes is not yet supported.") + return redirect(self.get_success_url()) + else: + for target in selected_targets: + # Share each target individually + form_data['target'] = Target.objects.get(id=target) + response = share_target_with_tom(share_destination, form_data, target_lists=[form_data['target_list']]) + sharing_feedback_handler(response, self.request) + if data_switch: + # If Data sharing request, share all data associated with the target + response = share_data_with_tom(share_destination, form_data, target_id=target) + sharing_feedback_handler(response, self.request) + if not selected_targets: + messages.error(self.request, f'No targets shared. {form.errors.as_json()}') + return redirect(self.get_success_url()) + return redirect(self.get_success_url())
+ {% if all_checked %} + + {% else %} + + {% endif %} + Name Type
+ {% if all_checked %} + + {% else %} + + {% endif %} + {{ target.names|join:", " }}
Group Total TargetsShareDelete
{{ group.targets.count }}ShareDelete