Merge pull request #659 from TOMToolkit/feature/tom2tom_data_sharing

tom2tom data sharing

docs/managing_data/tom_direct_sharing.rst

@@ -1,32 +1,74 @@
 Sharing Data with Other TOMs
 ############################
 
-TOM Toolkit does not yet support direct sharing between TOMs, however we hope to add this functionality soon.
-
-
-..  Configuring your TOM to submit data to another TOM:
-..  ***************************************************
-..  
-..  You will need to add a ``DATA_SHARING`` configuration dictionary to your ``settings.py`` that gives the credentials
-..  for the various TOMs with which you wish to share data.
-..  
-..  .. code:: python
-..  
-..     # Define the valid data sharing destinations for your TOM.
-..     DATA_SHARING = {
-..          'tom-demo-dev': {
-..              'DISPLAY_NAME': os.getenv('TOM_DEMO_DISPLAY_NAME', 'TOM Demo Dev'),
-..              'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'),
-..              'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'),
-..              'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'),
-..          },
-..          'localhost-tom': {
-..              # for testing; share with yourself
-..              'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'),
-..              'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'),
-..              'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'),
-..              'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'),
-..          }
-..  
-..     }
-..  
+TOM Toolkit supports direct data sharing between TOMs.
+
+
+Permissions:
+************
+To save data to a destination TOM your TOM will need to have access to a user account on that TOM with the correct
+permissions. This is handled by your TOM's administrator as described below.
+
+.. warning:: Any user who has permission to access the relevant target or data in your TOM will have permission to
+            submit that data to the destination TOM once DATA_SHARING is configured.
+
+
+Configuring your TOM to submit data to another TOM:
+***************************************************
+
+You will need to add a ``DATA_SHARING`` configuration dictionary to your ``settings.py`` that gives the credentials
+for the various TOMs with which you wish to share data. This should be the same ``DATA_SHARING`` dictionary that is used
+to :doc:`/managing_data/stream_pub_sub` such as `Hermes <https://hermes.lco.global>`_.
+
+.. code:: python
+
+    # Define the valid data sharing destinations for your TOM.
+    DATA_SHARING = {
+        'not-my-tom': {
+            # For sharing data with another TOM
+            'DISPLAY_NAME': os.getenv('NOT_MY_TOM_DISPLAY_NAME', 'Not My Tom'),
+            'BASE_URL': os.getenv('NOT_MY_TOM_BASE_URL', 'http://notmytom.com/'),
+            'USERNAME': os.getenv('NOT_MY_TOM_USERNAME', 'set NOT_MY_TOM_USERNAME value in environment'),
+            'PASSWORD': os.getenv('NOT_MY_TOM_PASSWORD', 'set NOT_MY_TOM_PASSWORD value in environment'),
+        },
+        'localhost-tom': {
+            # for testing; share with yourself
+            'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'),
+            'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'),
+            'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'),
+            'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'),
+        }
+    }
+
+Receiving Shared Data:
+**********************
+
+Reduced Datums:
+---------------
+When your TOM receives a new ``ReducedDatum`` from another TOM it will be saved to your TOM's database with its source
+set to the name of the TOM that submitted it. Currently, only Photometry data can be directly shared between
+TOMS and a ``Target`` with a matching name or alias must exist in both TOMS for sharing to take place.
+
+Data Products:
+--------------
+When your TOM receives a new ``DataProduct`` from another TOM it will be saved to your TOM's database / storage and run
+through the appropriate :doc:`data_processor </managing_data/customizing_data_processing>` pipeline. Only data products
+associated with a ``Target`` with a name or alias that matches that of a target in the destination TOM will be shared.
+
+Targets:
+--------
+When your TOM receives a new ``Target`` from another TOM it will be saved to your TOM's database. If the target's name
+or alias doesn't match that of a target that already exists in the database, a new target will be created and added to a
+new ``TargetList`` called "Imported from <TOM Name>".
+
+Target Lists:
+-------------
+When your TOM receives a new ``TargetList`` from another TOM it will be saved to your TOM's database. If the targets in
+the ``TargetList`` are also shared, but already exist in the destination TOM, they will be added to the new
+``TargetList``.
+
+
+
+
+
+

setup.py

@@ -54,7 +54,8 @@
         'specutils~=1.8',
     ],
     extras_require={
-        'test': ['factory_boy>=3.2.1,<3.4.0'],
+        'test': ['factory_boy>=3.2.1,<3.4.0',
+                 'responses~=0.23'],
         'docs': [
             'recommonmark~=0.7',
             'sphinx>=4,<8',

tom_base/settings.py

@@ -237,30 +237,30 @@
 }
 
 # Configuration for the TOM/Kafka Stream receiving data from this TOM
-DATA_SHARING = {
-    'hermes': {
-        'DISPLAY_NAME': os.getenv('HERMES_DISPLAY_NAME', 'Hermes'),
-        'BASE_URL': os.getenv('HERMES_BASE_URL', 'https://hermes.lco.global/'),
-        'CREDENTIAL_USERNAME': os.getenv('SCIMMA_CREDENTIAL_USERNAME',
-                                         'set SCIMMA_CREDENTIAL_USERNAME value in environment'),
-        'CREDENTIAL_PASSWORD': os.getenv('SCIMMA_CREDENTIAL_PASSWORD',
-                                         'set SCIMMA_CREDENTIAL_PASSWORD value in environment'),
-        'USER_TOPICS': ['hermes.test', 'tomtoolkit.test']
-    },
-    'tom-demo-dev': {
-        'DISPLAY_NAME': os.getenv('TOM_DEMO_DISPLAY_NAME', 'TOM Demo Dev'),
-        'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'),
-        'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'),
-        'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'),
-    },
-    'localhost-tom': {
-        # for testing; share with yourself
-        'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'),
-        'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'),
-        'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'),
-        'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'),
-    }
-}
+# DATA_SHARING = {
+#     'hermes': {
+#         'DISPLAY_NAME': os.getenv('HERMES_DISPLAY_NAME', 'Hermes'),
+#         'BASE_URL': os.getenv('HERMES_BASE_URL', 'https://hermes.lco.global/'),
+#         'CREDENTIAL_USERNAME': os.getenv('SCIMMA_CREDENTIAL_USERNAME',
+#                                          'set SCIMMA_CREDENTIAL_USERNAME value in environment'),
+#         'CREDENTIAL_PASSWORD': os.getenv('SCIMMA_CREDENTIAL_PASSWORD',
+#                                          'set SCIMMA_CREDENTIAL_PASSWORD value in environment'),
+#         'USER_TOPICS': ['hermes.test', 'tomtoolkit.test']
+#     },
+#     'tom-demo-dev': {
+#         'DISPLAY_NAME': os.getenv('TOM_DEMO_DISPLAY_NAME', 'TOM Demo Dev'),
+#         'BASE_URL': os.getenv('TOM_DEMO_BASE_URL', 'http://tom-demo-dev.lco.gtn/'),
+#         'USERNAME': os.getenv('TOM_DEMO_USERNAME', 'set TOM_DEMO_USERNAME value in environment'),
+#         'PASSWORD': os.getenv('TOM_DEMO_PASSWORD', 'set TOM_DEMO_PASSWORD value in environment'),
+#     },
+#     'localhost-tom': {
+#         # for testing; share with yourself
+#         'DISPLAY_NAME': os.getenv('LOCALHOST_TOM_DISPLAY_NAME', 'Local'),
+#         'BASE_URL': os.getenv('LOCALHOST_TOM_BASE_URL', 'http://127.0.0.1:8000/'),
+#         'USERNAME': os.getenv('LOCALHOST_TOM_USERNAME', 'set LOCALHOST_TOM_USERNAME value in environment'),
+#         'PASSWORD': os.getenv('LOCALHOST_TOM_PASSWORD', 'set LOCALHOST_TOM_PASSWORD value in environment'),
+#     }
+# }
 
 TOM_CADENCE_STRATEGIES = [
     'tom_observations.cadences.retry_failed_observations.RetryFailedObservationsStrategy',

tom_dataproducts/alertstreams/hermes.py

@@ -136,7 +136,7 @@ def get_hermes_topics(**kwargs):
         response = requests.get(url=submit_url, headers=headers)
 
         topics = response.json()['writable_topics']
-    except KeyError:
+    except (KeyError, requests.exceptions.JSONDecodeError):
         topics = settings.DATA_SHARING['hermes']['USER_TOPICS']
     return topics
 

tom_dataproducts/api_views.py

@@ -4,15 +4,15 @@
 from guardian.shortcuts import assign_perm, get_objects_for_user
 from rest_framework import status
 from rest_framework.mixins import CreateModelMixin, DestroyModelMixin, ListModelMixin
-from rest_framework.parsers import MultiPartParser
+from rest_framework.parsers import MultiPartParser, FormParser, JSONParser
 from rest_framework.response import Response
 from rest_framework.viewsets import GenericViewSet
 
 from tom_common.hooks import run_hook
 from tom_dataproducts.data_processor import run_data_processor
 from tom_dataproducts.filters import DataProductFilter
 from tom_dataproducts.models import DataProduct, ReducedDatum
-from tom_dataproducts.serializers import DataProductSerializer
+from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer
 
 
 class DataProductViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet, PermissionListMixin):
@@ -38,6 +38,7 @@ def create(self, request, *args, **kwargs):
         response = super().create(request, *args, **kwargs)
 
         if response.status_code == status.HTTP_201_CREATED:
+            response.data['message'] = 'Data product successfully uploaded.'
             dp = DataProduct.objects.get(pk=response.data['id'])
             try:
                 run_hook('data_product_post_upload', dp)
@@ -68,3 +69,29 @@ def get_queryset(self):
             )
         else:
             return get_objects_for_user(self.request.user, 'tom_dataproducts.view_dataproduct')
+
+
+class ReducedDatumViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet, PermissionListMixin):
+    """
+    Viewset for ReducedDatum objects. Supports list, create, and delete.
+
+    To view supported query parameters, please use the OPTIONS endpoint, which can be accessed through the web UI.
+
+    **Please note that ``groups`` are an accepted query parameters for the ``CREATE`` endpoint. The groups parameter
+    will specify which ``groups`` can view the created ``DataProduct``. If no ``groups`` are specified, the
+    ``ReducedDatum`` will only be visible to the user that created the ``DataProduct``. Make sure to check your
+    ``groups``!!**
+    """
+    queryset = ReducedDatum.objects.all()
+    serializer_class = ReducedDatumSerializer
+    filter_backends = (drf_filters.DjangoFilterBackend,)
+    permission_required = 'tom_dataproducts.view_reduceddatum'
+    parser_classes = [FormParser, JSONParser]
+
+    def create(self, request, *args, **kwargs):
+        response = super().create(request, *args, **kwargs)
+
+        if response.status_code == status.HTTP_201_CREATED:
+            response.data['message'] = 'Data successfully uploaded.'
+
+        return response

tom_dataproducts/forms.py

@@ -5,39 +5,9 @@
 from tom_dataproducts.models import DataProductGroup, DataProduct
 from tom_observations.models import ObservationRecord
 from tom_targets.models import Target
-from tom_dataproducts.alertstreams.hermes import get_hermes_topics
+from tom_dataproducts.sharing import get_sharing_destination_options
 
 
-def get_sharing_destination_options():
-    """
-    Build the Display options and headers for the dropdown form for choosing sharing topics.
-    Customize for a different selection experience.
-    :return: Tuple: Possible Destinations and their Display Names
-    """
-    choices = []
-    try:
-        for destination, details in settings.DATA_SHARING.items():
-            new_destination = [details.get('DISPLAY_NAME', destination)]
-            if details.get('USER_TOPICS', None):
-                # If topics exist for a destination (Such as HERMES) give topics as sub-choices
-                #   for non-selectable Destination
-                if destination == "hermes":
-                    destination_topics = get_hermes_topics()
-                else:
-                    destination_topics = details['USER_TOPICS']
-                topic_list = [(f'{destination}:{topic}', topic) for topic in destination_topics]
-                new_destination.append(tuple(topic_list))
-            else:
-                # Otherwise just use destination as option
-                new_destination.insert(0, destination)
-            choices.append(tuple(new_destination))
-    except AttributeError:
-        pass
-    return tuple(choices)
-
-
-DESTINATION_OPTIONS = get_sharing_destination_options()
-
 DATA_TYPE_OPTIONS = (('photometry', 'Photometry'),
                      ('spectroscopy', 'Spectroscopy'))
 
@@ -82,7 +52,7 @@ def __init__(self, *args, **kwargs):
 
 
 class DataShareForm(forms.Form):
-    share_destination = forms.ChoiceField(required=True, choices=DESTINATION_OPTIONS, label="Destination")
+    share_destination = forms.ChoiceField(required=True, choices=[], label="Destination")
     share_title = forms.CharField(required=False, label="Title")
     share_message = forms.CharField(required=False, label="Message", widget=forms.Textarea())
     share_authors = forms.CharField(required=False, widget=forms.HiddenInput())
@@ -97,4 +67,4 @@ class DataShareForm(forms.Form):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.fields['share_destination'].choices = DESTINATION_OPTIONS
+        self.fields['share_destination'].choices = get_sharing_destination_options()

tom_dataproducts/models.py

@@ -331,7 +331,7 @@ class ReducedDatum(models.Model):
     """
 
     target = models.ForeignKey(Target, null=False, on_delete=models.CASCADE)
-    data_product = models.ForeignKey(DataProduct, null=True, on_delete=models.CASCADE)
+    data_product = models.ForeignKey(DataProduct, null=True, blank=True, on_delete=models.CASCADE)
     data_type = models.CharField(
         max_length=100,
         default=''
@@ -352,3 +352,12 @@ def save(self, *args, **kwargs):
         else:
             raise ValidationError('Not a valid DataProduct type.')
         return super().save()
+
+    def validate_unique(self, *args, **kwargs):
+        super().validate_unique(*args, **kwargs)
+        model_dict = self.__dict__.copy()
+        del model_dict['_state']
+        del model_dict['id']
+        obs = ReducedDatum.objects.filter(**model_dict)
+        if obs:
+            raise ValidationError('Data point already exists.')

tom_dataproducts/serializers.py

@@ -18,6 +18,8 @@ class Meta:
 
 
 class ReducedDatumSerializer(serializers.ModelSerializer):
+    target = TargetFilteredPrimaryKeyRelatedField(queryset=Target.objects.all())
+
     class Meta:
         model = ReducedDatum
         fields = (
@@ -26,9 +28,31 @@ class Meta:
             'source_name',
             'source_location',
             'timestamp',
-            'value'
+            'value',
+            'target'
         )
 
+    def create(self, validated_data):
+        """DRF requires explicitly handling writeable nested serializers,
+        here we pop the groups data and save it using its serializer.
+        """
+        groups = validated_data.pop('groups', [])
+
+        rd = ReducedDatum(**validated_data)
+        rd.full_clean()
+        rd.save()
+
+        # Save groups for this target
+        group_serializer = GroupSerializer(data=groups, many=True)
+        if group_serializer.is_valid() and settings.TARGET_PERMISSIONS_ONLY is False:
+            for group in groups:
+                group_instance = Group.objects.get(pk=group['id'])
+                assign_perm('tom_dataproducts.view_dataproduct', group_instance, rd)
+                assign_perm('tom_dataproducts.change_dataproduct', group_instance, rd)
+                assign_perm('tom_dataproducts.delete_dataproduct', group_instance, rd)
+
+        return rd
+
 
 class DataProductSerializer(serializers.ModelSerializer):
     target = TargetFilteredPrimaryKeyRelatedField(queryset=Target.objects.all())