Merge pull request #825 from TOMToolkit/768-fix-atlas-forced-photomet…

…ry-limiting-mag-interpretation 768 fix atlas forced photometry limiting mag interpretation
TOMToolkit · Jan 26, 2024 · 6b8be92 · 6b8be92
2 parents ac9b50f + 2cbfc99
commit 6b8be92
Show file tree

Hide file tree

Showing 3 changed files with 100 additions and 4 deletions.
diff --git a/tom_dataproducts/processors/atlas_processor.py b/tom_dataproducts/processors/atlas_processor.py
@@ -21,7 +21,7 @@ def process_data(self, data_product):
         ingestion
         :type data_product: DataProduct
 
-        :returns: python list of 2-tuples, each with a timestamp and corresponding data
+        :returns: python list of 3-tuples, each with a timestamp and corresponding data, and source
         :rtype: list
         """
 
@@ -37,6 +37,7 @@ def _process_photometry_from_plaintext(self, data_product):
         Processes the photometric data from a plaintext file into a list of dicts. File is read using astropy as
         specified in the below documentation. The file is expected to be a multi-column delimited space delimited
         text file, as produced by the ATLAS forced photometry service at https://fallingstar-data.com/forcedphot
+        See https://fallingstar-data.com/forcedphot/resultdesc/ for a description of the output format.
 
         The header looks like this:
         ###MJD   m   dm  uJy   duJy F err chi/N   RA  Dec   x   y  maj  min   phi  apfit mag5sig Sky   Obs
@@ -48,6 +49,7 @@ def _process_photometry_from_plaintext(self, data_product):
         :rtype: list
         """
         photometry = []
+        signal_to_noise_cutoff = 3.0  # cutoff to turn magnitudes into non-detection limits
 
         data = astropy.io.ascii.read(data_product.data.path)
         if len(data) < 1:
@@ -60,10 +62,19 @@ def _process_photometry_from_plaintext(self, data_product):
                 time.format = 'datetime'
                 value = {
                     'timestamp': time.to_datetime(timezone=utc),
-                    'magnitude': float(datum['m']),
-                    'magnitude_error': float(datum['dm']),
-                    'filter': str(datum['F'])
+                    'filter': str(datum['F']),
+                    'error': float(datum['dm']),
+                    'telescope': 'ATLAS',
                 }
+                # If the signal is in the noise, set the non-detection limit to the
+                # absolute value of the reported magnitude.
+                # see https://fallingstar-data.com/forcedphot/resultdesc/
+                signal_to_noise = abs(float(datum['uJy']))/abs(float(datum['duJy']))
+                if signal_to_noise <= signal_to_noise_cutoff:
+                    value['limit'] = abs(float(datum['m']))
+                else:
+                    value['magnitude'] = abs(float(datum['m']))
+
                 photometry.append(value)
         except Exception as e:
             raise InvalidFileFormatException(e)

diff --git a/tom_dataproducts/tests/test_atlas.py b/tom_dataproducts/tests/test_atlas.py
@@ -0,0 +1,52 @@
+from unittest.mock import patch
+import logging
+
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.test import TestCase  # , override_settings
+
+from tom_dataproducts.models import DataProduct
+from tom_dataproducts.processors.atlas_processor import AtlasProcessor
+
+
+from tom_observations.tests.factories import SiderealTargetFactory
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+
+class TestAtlasProcessor(TestCase):
+    """Test the AtlasProcessor(DataProcessor) class.
+    """
+
+    def setUp(self):
+        self.target = SiderealTargetFactory.create()
+        self.data_product = DataProduct.objects.create(target=self.target)
+        self.data_product_filefield_data = SimpleUploadedFile('nonsense.csv', b'somedata')
+
+    @patch('tom_dataproducts.processors.atlas_processor.AtlasProcessor._process_photometry_from_plaintext')
+    def test_process_photometry_with_plaintext_file(self, mocked_method):
+        """Test that AtlasProcessor.process_data() calls AtlasProcessor._process_photometry_from_plaintext().
+        """
+        self.data_product.data.save('lightcurve.csv', self.data_product_filefield_data)
+
+        # this is the call under test
+        AtlasProcessor().process_data(self.data_product)
+        mocked_method.assert_called_with(self.data_product)
+
+    def test_mags_under_SN_cutoff_become_limits(self):
+        """Test that magnitudes become limits when the flux S/N given by uJy/duJy
+        is under the cutoff limit of 3.0. see https://fallingstar-data.com/forcedphot/resultdesc/
+
+        The test data is an ATLAS forced photometry query on NGC1566 over the dates
+        shown in the csv file.
+        """
+        # read the test data in as a data_product's data
+        with open('tom_dataproducts/tests/test_data/test_atlas_fp.csv') as atlas_fp_file:
+            self.data_product.data.save('test_data.csv', atlas_fp_file)
+
+        # this is the call under test
+        photometry = AtlasProcessor()._process_photometry_from_plaintext(self.data_product)
+
+        expected_non_detection_count = 9  # known apriori from test data in test_atlas_fp.csv
+        self.assertEqual(expected_non_detection_count,
+                         len([datum for datum in photometry if 'limit' in datum.keys()]))
diff --git a/tom_dataproducts/tests/test_data/test_atlas_fp.csv b/tom_dataproducts/tests/test_data/test_atlas_fp.csv
@@ -0,0 +1,33 @@
+###MJD          m      dm   uJy   duJy F err chi/N     RA       Dec        x        y     maj  min   phi  apfit mag5sig Sky   Obs
+60316.142545  17.349  0.089   417   37 o  0  28.03  65.00165 -54.93795  9674.06  2072.74 2.51 1.85  76.9 -0.504 19.34 20.82 04a60316o0180o
+60316.145763 -17.672  0.077  -310   24 o  0  29.48  65.00165 -54.93795  9692.04  2109.54 2.83 2.47 -29.0 -0.366 19.39 20.93 04a60316o0187o
+60316.151230 -17.743  0.088  -290   26 o  0  19.95  65.00165 -54.93795  9628.40  2026.13 2.77 2.45 -27.2 -0.361 19.42 20.94 04a60316o0199o
+60316.173900  17.477  0.085   371   32 o  0  22.12  65.00165 -54.93795  9599.61  2131.95 2.62 2.51 -41.2 -0.389 19.37 20.93 04a60316o0247o
+60320.133567  19.086  0.775    84   65 o  0   6.11  65.00165 -54.93795  3052.87  9916.23 2.17 1.83 -67.6 -0.440 18.94 20.29 04a60320o0237o
+60320.136786 -16.909  0.068  -626   43 o  0   6.81  65.00165 -54.93795  3068.57  9953.31 2.22 1.86 -67.6 -0.437 19.13 20.48 04a60320o0244o
+60320.142327  17.279  0.087   445   39 o  0  17.83  65.00165 -54.93795  3014.86  9872.64 2.01 1.78 -59.1 -0.478 19.40 20.80 04a60320o0256o
+60320.152398 -18.787  0.296  -111   33 o  0  14.29  65.00165 -54.93795  2991.84  9985.29 2.23 1.90 -63.9 -0.448 19.34 20.77 04a60320o0278o
+60321.868084 -17.765  0.067  -284   19 c  0  10.08  65.00165 -54.93795 10107.42 10303.45 2.94 2.44 -56.6 -0.420 19.75 21.82 03a60322o0201c
+60321.870492  21.153  1.437    13   18 c  0   7.97  65.00165 -54.93795   974.16  9960.67 2.79 2.48 -43.7 -0.403 19.75 21.82 03a60322o0206c
+60321.870948 -21.857  3.432    -7   22 c  0   8.10  65.00165 -54.93795 10126.86 10345.17 2.85 2.40 -57.4 -0.418 19.75 21.78 03a60322o0207c
+60321.873749 -20.216  0.732   -30   22 c  0  10.64  65.00165 -54.93795   991.68  9999.45 2.60 2.31 -47.0 -0.412 19.79 21.82 03a60322o0213c
+60321.876502 -19.000  0.221   -91   20 c  0  11.23  65.00165 -54.93795 10073.20 10261.39 2.78 2.31 -59.5 -0.420 19.76 21.76 03a60322o0219c
+60321.879306 -20.117  0.596   -33   19 c  0   8.07  65.00165 -54.93795   938.76  9922.96 2.63 2.33 -42.5 -0.417 19.77 21.80 03a60322o0225c
+60321.886315  18.340  0.123   168   21 c  0  14.92  65.00165 -54.93795 10052.86 10377.72 2.80 2.35 -57.9 -0.414 19.76 21.78 03a60322o0240c
+60321.890505 -18.524  0.129  -141   18 c  0  14.85  65.00165 -54.93795   918.79 10042.45 2.77 2.47 -47.6 -0.405 19.72 21.80 03a60322o0249c
+60324.116538  16.673  0.046   778   36 o  0  24.20  65.00165 -54.93795  1678.35  1911.75 2.00 1.90  57.0 -0.428 19.54 21.15 04a60324o0200o
+60324.119777  19.060  0.424    86   37 o  0   8.23  65.00165 -54.93795  1697.50  1947.32 1.99 1.88  59.2 -0.422 19.52 21.17 04a60324o0207o
+60324.125384  17.363  0.066   412   27 o  0  31.09  65.00165 -54.93795  1636.41  1867.76 2.02 2.01 -36.1 -0.416 19.49 21.17 04a60324o0219o
+60324.135990  17.412  0.110   394   43 o  0   8.24  65.00165 -54.93795  1610.32  1979.99 2.06 1.91  56.4 -0.421 19.50 21.17 04a60324o0242o
+60328.097204  17.193  0.068   482   33 o  0  16.18  65.00165 -54.93795  5612.41  3696.93 2.09 1.92 -19.3 -0.501 19.23 20.50 04a60328o0161o
+60328.100068 -17.946  0.129  -241   31 o  0  15.96  65.00165 -54.93795  5630.80  3733.42 2.16 2.01 -20.5 -0.507 19.23 20.47 04a60328o0167o
+60328.106077  19.414  0.597    62   37 o  0   8.05  65.00165 -54.93795  5571.33  3651.80 2.15 2.04 -18.0 -0.520 19.20 20.45 04a60328o0180o
+60328.117189  18.660  0.253   125   32 o  0  14.08  65.00165 -54.93795  5545.41  3761.56 2.08 1.93 -18.4 -0.483 19.25 20.48 04a60328o0203o
+60332.103237  17.445  0.103   382   39 o  0  24.91  65.00165 -54.93795  5481.07  5711.99 2.09 1.89 -16.1 -0.429 18.89 19.65 04a60332o0176o
+60332.106874  17.617  0.160   326   52 o  0  10.23  65.00165 -54.93795  5499.61  5748.95 2.07 1.87 -20.4 -0.415 18.88 19.65 04a60332o0184o
+60332.114281  17.921  0.164   246   40 o  0  16.81  65.00165 -54.93795  5438.94  5668.10 2.02 1.83 -18.1 -0.424 18.87 19.65 04a60332o0200o
+60332.124416  17.985  0.168   232   39 o  0  28.09  65.00165 -54.93795  5416.25  5779.68 2.03 1.84 -32.5 -0.425 18.89 19.64 04a60332o0222o
+60333.845249  19.153  0.406    79   32 o  0   8.20  65.00165 -54.93795  5682.83  1684.25 2.29 2.10 -57.4 -0.423 18.77 19.37 03a60334o0160o
+60333.848429  17.596  0.092   332   30 o  0  18.97  65.00165 -54.93795  5701.42  1723.75 2.22 2.02 -62.6 -0.434 18.79 19.38 03a60334o0167o
+60333.859097 -21.134  2.050   -13   26 o  0  23.63  65.00165 -54.93795  5645.71  1647.39 2.20 2.01 -54.4 -0.441 18.78 19.33 03a60334o0189o
+60333.866714  18.457  0.192   150   29 o  0  21.26  65.00165 -54.93795  5620.66  1762.65 2.24 2.07 -60.7 -0.430 18.76 19.33 03a60334o0205o