Support QLP changes in sectors 56+ (lightkurve#1392)

* QLP sector 56+: handle default flux_err column

* handle QLP-specific quality bitmask

* docstring updates for QLP sectors 56+

* add changelog [skip ci]

CHANGES.rst

@@ -2,6 +2,8 @@
 =====================
 
 - Fixed memory leak in reading Lightcurve / TargetPixel FITS files in v2.4.2 [#1390]
+- Added support for changes in QLP High Level Science Product
+  in TESS sectors 56 and later. [#1392]
 
 
 2.4.2 (2023-11-03)

src/lightkurve/io/qlp.py

@@ -9,23 +9,26 @@
 from .generic import read_generic_lightcurve
 
 
-def read_qlp_lightcurve(filename, flux_column="sap_flux", flux_err_column="kspsap_flux_err", quality_bitmask="default"):
+def read_qlp_lightcurve(filename, flux_column="sap_flux", flux_err_column=None, quality_bitmask="default"):
     """Returns a `~lightkurve.lightcurve.LightCurve` object given a light curve file from the MIT Quicklook Pipeline (QLP).
 
     By default, QLP's `sap_flux` column is used to populate the `flux` values,
-    and 'kspsap_flux_err' is used to populate `flux_err`. For a discussion
+    and `kspsap_flux_err` / `det_flux_err` is used to populate `flux_err`. For a discussion
     related to this choice, see https://github.com/lightkurve/lightkurve/issues/1083
 
+    For detrended flux, the columns are named with `kspsap_` prefix in sectors 1-55,
+    and `det_` prefix in sectors 56+. Column `sys_rm_flux` is available in sectors 56+.
+
     More information: https://archive.stsci.edu/hlsp/qlp
 
     Parameters
     ----------
     filename : str
         Local path or remote url of a QLP light curve FITS file.
-    flux_column : 'sap_flux', 'kspsap_flux', 'kspsap_flux_sml', 'kspsap_flux_lag', or 'sap_bkg'
+    flux_column : 'sap_flux', 'kspsap_flux', 'kspsap_flux_sml', 'kspsap_flux_lag', 'det_flux', 'det_flux_sml', 'det_flux_lag', 'sys_rm_flux', or 'sap_bkg'
         Which column in the FITS file contains the preferred flux data?
         By default the "Simple Aperture Photometry" flux (sap_flux) is used.
-    flux_err_column: 'kspsap_flux_err', or 'sap_bkg_err'
+    flux_err_column: 'kspsap_flux_err','det_flux_err', or 'sap_bkg_err'
       Which column in the FITS file contains the preferred flux_err data?
     quality_bitmask : str or int
         Bitmask (integer) which identifies the quality flag bitmask that should
@@ -39,9 +42,17 @@ def read_qlp_lightcurve(filename, flux_column="sap_flux", flux_err_column="kspsa
             * "hardest": removes all data that has been flagged.
               This mask is not recommended.
 
+        QLP-specific "Low precision points" (bit 13 in sectors 1-55, bit 31 in sectors 56+)
+        is included in "hard" and "hardest" bitmasks.
+
         See the `~lightkurve.utils.TessQualityFlags` class for details on the bitmasks.
     """
     lc = read_generic_lightcurve(filename, flux_column=flux_column, flux_err_column=flux_err_column, time_format="btjd")
+    if flux_err_column is None:
+        if lc.meta.get("SECTOR", 0) >= 56:
+            lc["flux_err"] = lc["det_flux_err"]
+        else:
+            lc["flux_err"] = lc["kspsap_flux_err"]
 
     # Filter out poor-quality data
     # NOTE: Unfortunately Astropy Table masking does not yet work for columns
@@ -50,6 +61,16 @@ def read_qlp_lightcurve(filename, flux_column="sap_flux", flux_err_column="kspsa
     quality_mask = TessQualityFlags.create_quality_mask(
         quality_array=lc["quality"], bitmask=quality_bitmask
     )
+    # QLP-specific quality_bitmask handling
+    if quality_bitmask in ["hardest", "hard"]:
+        if lc.meta.get("SECTOR", 0) >= 56:
+            qlp_low_precision_bitmask = 2 ** 30
+        else:
+            # https://archive.stsci.edu/hlsps/qlp/hlsp_qlp_tess_ffi_all_tess_v1_data-prod-desc.pdf
+            qlp_low_precision_bitmask = 2 ** 12
+        q_mask2 = TessQualityFlags.create_quality_mask(
+            quality_array=lc["quality"], bitmask=qlp_low_precision_bitmask)
+        quality_mask = quality_mask & q_mask2
     lc = lc[quality_mask]
 
     lc.meta["AUTHOR"] = "QLP"

tests/io/test_qlp.py

@@ -10,16 +10,43 @@
 
 
 @pytest.mark.remote_data
-def test_qlp():
+@pytest.mark.parametrize(
+    "url, flux_err_colname_expected, qlp_low_precision_bitmask", [
+        ("https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:HLSP/qlp/s0011/0000/0002/7755/4109/hlsp_qlp_tess_ffi_s0011-0000000277554109_tess_v01_llc.fits",
+         "KSPSAP_FLUX_ERR",  # for sectors 1 - 55
+         2**12,  # bit 13 for sectors 1 -55
+         ),
+        ("https://mast.stsci.edu/api/v0.1/Download/file/?uri=mast:HLSP/qlp/s0056/0000/0000/1054/9159/hlsp_qlp_tess_ffi_s0056-0000000010549159_tess_v01_llc.fits",
+         "DET_FLUX_ERR",  # for sectors 56+
+         2**30,  # bit 31 for sectors 56+
+         ),
+        ])
+def test_qlp(url, flux_err_colname_expected, qlp_low_precision_bitmask):
     """Can we read in QLP light curves?"""
-    url = "https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:HLSP/qlp/s0011/0000/0002/7755/4109/hlsp_qlp_tess_ffi_s0011-0000000277554109_tess_v01_llc.fits"    
     with fits.open(url, mode="readonly") as hdulist:
         # Can we auto-detect a QLP file?
         assert detect_filetype(hdulist) == "QLP"
         # Are the correct fluxes read in?
         lc = read_qlp_lightcurve(url, quality_bitmask=0)
         assert lc.meta["FLUX_ORIGIN"] == "sap_flux"
         assert_array_equal(lc.flux.value, hdulist[1].data["SAP_FLUX"])
+        assert_array_equal(lc.flux_err.value, hdulist[1].data[flux_err_colname_expected])
+
+        # Test handling of QLP-specific low-precision bitmask
+        # - the cadences marked as such will be masked out by "hard" / "hardest"
+
+        # first assure the test FITS file has cadence marked by QLP bit only
+        # to easily isolate the effects of the quality_bitmask
+        assert (lc["quality"] == qlp_low_precision_bitmask).any()
+
+        lc = read_qlp_lightcurve(url, quality_bitmask="default")
+        assert (lc["quality"] & (qlp_low_precision_bitmask)).any()
+
+        lc = read_qlp_lightcurve(url, quality_bitmask="hard")
+        assert not (lc["quality"] & (qlp_low_precision_bitmask)).any()
+
+        lc = read_qlp_lightcurve(url, quality_bitmask="hardest")
+        assert not (lc["quality"] & (qlp_low_precision_bitmask)).any()
 
 
 @pytest.mark.remote_data