Merge pull request #40 from lsst/tickets/DM-39583

DM-39583: Fix lots of deprecation warnings
lsst · Jul 19, 2023 · cf1727e · cf1727e
2 parents 7bf34a7 + e920223
commit cf1727e
Show file tree

Hide file tree

Showing 10 changed files with 193 additions and 194 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -16,6 +16,7 @@ jobs:
 
       - uses: conda-incubator/setup-miniconda@v2
         with:
+          python-version: "3.11"
           miniforge-variant: Mambaforge
           miniforge-version: latest
           show-channel-urls: true
@@ -25,9 +26,13 @@ jobs:
         # Might be quicker to install rubin-env plus any necessary additions.
         shell: bash -l {0}
         run: |
-          mamba install -y -q numpy>1.15 scipy matplotlib>3.1 pandas llvmlite numba astropy>=3.2 photutils>=0.7 astroquery coloredlogs scikit-image h5py emcee tqdm mpi4py schwimmbad iminuit>=2 coverage>=3.6 configparser coveralls deprecated pyyaml nose rubin-libradtran getCalspec>=2.0.0
+          mamba install -y  "numpy>1.15" scipy "matplotlib>3.1" pandas llvmlite numba "astropy>=3.2" "photutils>=0.7" astroquery coloredlogs scikit-image h5py emcee tqdm mpi4py schwimmbad "iminuit>=2" "coverage>=3.6" coveralls deprecated pyyaml nose rubin-libradtran "getCalspec>=2.0.0"
           # python -c "from getCalspec.rebuild import rebuild_tables; rebuild_tables()"
           mamba install astrometry
+
+      - name: Download test data
+        shell: bash -l {0}
+        run: |
           wget -r -q -nc http://data.astrometry.net/5000/index-5002-24.fits
           wget -r -q -nc http://data.astrometry.net/5000/index-5000-40.fits
           mv data.astrometry.net/5000/index-*.fits $CONDA_PREFIX/data/
@@ -42,7 +47,7 @@ jobs:
       - name: Build and install
         shell: bash -l {0}
         run: |
-          pip install -v -e .
+          pip install -v --no-deps -e .
 
       - name: Run full chain
         shell: bash -l {0}

diff --git a/spectractor/astrometry.py b/spectractor/astrometry.py
@@ -423,9 +423,8 @@ def get_quad_stars_pixel_positions(self):
 
         """
         coords = []
-        hdu = fits.open(self.match_file_name)
-        table = Table.read(hdu)
-        hdu.close()
+        with fits.open(self.match_file_name) as hdu:
+            table = Table.read(hdu)
         for k in range(0, len(table["QUADPIX"][0]), 2):
             coord = [float(table["QUADPIX"][0][k]), float(table["QUADPIX"][0][k+1])]
             if np.sum(coord) > 0:
@@ -1124,11 +1123,11 @@ def run_simple_astrometry(self, extent=None, sources=None):
         # The source file given to solve-field is understood as a FITS file with pixel origin value at 1,
         # whereas pixel coordinates comes from photutils using a numpy convention with pixel origin value at 0
         # To correct for this we shift the CRPIX center of 1 pixel
-        hdu = fits.open(self.wcs_file_name)
-        hdu[0].header['CRPIX1'] = float(hdu[0].header['CRPIX1']) + 1
-        hdu[0].header['CRPIX2'] = float(hdu[0].header['CRPIX2']) + 1
-        self.my_logger.info(f"\n\tWrite astrometry.net WCS solution in {self.wcs_file_name}...")
-        hdu.writeto(self.wcs_file_name, overwrite=True)
+        with fits.open(self.wcs_file_name) as hdu:
+            hdu[0].header['CRPIX1'] = float(hdu[0].header['CRPIX1']) + 1
+            hdu[0].header['CRPIX2'] = float(hdu[0].header['CRPIX2']) + 1
+            self.my_logger.info(f"\n\tWrite astrometry.net WCS solution in {self.wcs_file_name}...")
+            hdu.writeto(self.wcs_file_name, overwrite=True)
 
         # load quad stars
         self.quad_stars_pixel_positions = self.get_quad_stars_pixel_positions()
@@ -1273,14 +1272,14 @@ def run_gaia_astrometry(self, min_range=3 * u.arcsec, max_range=5 * u.arcmin, ma
         ddec_median = np.median(ddec.to(u.arcsec).value)
         dra_rms = np.std(dra.to(u.arcsec).value)
         ddec_rms = np.std(ddec.to(u.arcsec).value)
-        hdu = fits.open(self.wcs_file_name)
-        hdu[0].header['CRVAL1'] = self.wcs.wcs.crval[0]
-        hdu[0].header['CRVAL2'] = self.wcs.wcs.crval[1]
-        hdu[0].header['CRV1_MED'] = dra_median
-        hdu[0].header['CRV2_MED'] = ddec_median
-        hdu[0].header['CRV1_RMS'] = dra_rms
-        hdu[0].header['CRV2_RMS'] = ddec_rms
-        hdu.writeto(self.wcs_file_name, overwrite=True)
+        with fits.open(self.wcs_file_name) as hdu:
+            hdu[0].header['CRVAL1'] = self.wcs.wcs.crval[0]
+            hdu[0].header['CRVAL2'] = self.wcs.wcs.crval[1]
+            hdu[0].header['CRV1_MED'] = dra_median
+            hdu[0].header['CRV2_MED'] = ddec_median
+            hdu[0].header['CRV1_RMS'] = dra_rms
+            hdu[0].header['CRV2_RMS'] = ddec_rms
+            hdu.writeto(self.wcs_file_name, overwrite=True)
 
         # check histogram medians
         self.wcs = load_wcs_from_file(self.wcs_file_name)
@@ -1299,12 +1298,12 @@ def run_gaia_astrometry(self, min_range=3 * u.arcsec, max_range=5 * u.arcmin, ma
         ddec_median = np.median(ddec.to(u.arcsec).value)
         dra_rms = np.std(dra.to(u.arcsec).value)
         ddec_rms = np.std(ddec.to(u.arcsec).value)
-        hdu = fits.open(self.wcs_file_name)
-        hdu[0].header['CRV1_MED'] = dra_median
-        hdu[0].header['CRV2_MED'] = ddec_median
-        hdu[0].header['CRV1_RMS'] = dra_rms
-        hdu[0].header['CRV2_RMS'] = ddec_rms
-        hdu.writeto(self.wcs_file_name, overwrite=True)
+        with fits.open(self.wcs_file_name) as hdu:
+            hdu[0].header['CRV1_MED'] = dra_median
+            hdu[0].header['CRV2_MED'] = ddec_median
+            hdu[0].header['CRV1_RMS'] = dra_rms
+            hdu[0].header['CRV2_RMS'] = ddec_rms
+            hdu.writeto(self.wcs_file_name, overwrite=True)
 
         # if parameters.DEBUG:
         #     plot_shifts_histograms(dra, ddec)

diff --git a/spectractor/config.py b/spectractor/config.py
@@ -41,7 +41,7 @@ def from_config_to_parameters(config):
     for section in config.sections():
         for options in config.options(section):
             value = config.get(section, options)
-            if re.match("[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?", value):
+            if re.match(r"[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?", value):
                 if ' ' in value:
                     value = str(value)
                 elif '.' in value or 'e' in value:

diff --git a/spectractor/extractor/background.py b/spectractor/extractor/background.py
@@ -9,7 +9,7 @@
 from spectractor.tools import fit_poly1d_outlier_removal, fit_poly2d_outlier_removal, plot_image_simple
 
 from astropy.stats import SigmaClip
-from photutils import Background2D, SExtractorBackground
+from photutils.background import Background2D, SExtractorBackground
 from photutils.segmentation import SegmentationImage, detect_threshold, detect_sources
 
 from scipy import ndimage

diff --git a/spectractor/extractor/dispersers.py b/spectractor/extractor/dispersers.py
@@ -428,7 +428,8 @@ def load_files(self, verbose=False):
             self.ratio_order_3over2 = None
         filename = os.path.join(self.data_dir, self.label, "hologram_center.txt")
         if os.path.isfile(filename):
-            lines = [ll.rstrip('\n') for ll in open(filename)]
+            with open(filename) as f:
+                lines = [ll.rstrip('\n') for ll in f]
             self.theta_tilt = float(lines[1].split(' ')[2])
         else:
             self.theta_tilt = 0
@@ -798,7 +799,8 @@ def load_specs(self, verbose=True):
                 raise ValueError("To define an hologram, you must provide hologram_grooves_per_mm.txt or N.txt files.")
         filename = os.path.join(self.data_dir, self.label, "hologram_center.txt")
         if os.path.isfile(filename):
-            lines = [ll.rstrip('\n') for ll in open(filename)]
+            with open(filename) as f:
+                lines = [ll.rstrip('\n') for ll in f]
             self.holo_center = list(map(float, lines[1].split(' ')[:2]))
             self.theta_tilt = float(lines[1].split(' ')[2])
         else:

diff --git a/spectractor/extractor/images.py b/spectractor/extractor/images.py
@@ -622,49 +622,49 @@ def load_LPNHE_image(image):  # pragma: no cover
         The Image instance to fill with file data and header.
     """
     image.my_logger.info(f'\n\tLoading LPNHE image {image.file_name}...')
-    hdus = fits.open(image.file_name)
-    image.header = hdus[0].header
-    hdu1 = hdus["CHAN_14"]
-    hdu2 = hdus["CHAN_06"]
-    data1 = hdu1.data[imgslice(hdu1.header['DATASEC'])].astype(np.float64)
-    bias1 = np.median(hdu1.data[imgslice(hdu1.header['BIASSEC'])].astype(np.float64))
-    data1 -= bias1
-    detsecy, detsecx = imgslice(hdu1.header['DETSEC'])
-    if detsecy.start > detsecy.stop:
-        data1 = data1[:, ::-1]
-    if detsecx.start > detsecx.stop:
-        data1 = data1[::-1, :]
-    data2 = hdu2.data[imgslice(hdu2.header['DATASEC'])].astype(np.float64)
-    bias2 = np.median(hdu2.data[imgslice(hdu2.header['BIASSEC'])].astype(np.float64))
-    data2 -= bias2
-    detsecy, detsecx = imgslice(hdu2.header['DETSEC'])
-    if detsecy.start > detsecy.stop:
-        data2 = data2[:, ::-1]
-    if detsecx.start > detsecx.stop:
-        data2 = data2[::-1, :]
-    data = np.concatenate([data2, data1])
-    image.data = data[::-1, :].T
-    image.date_obs = image.header['DATE-OBS']
-    image.expo = float(image.header['EXPTIME'])
-    image.airmass = -1
-    parameters.DISTANCE2CCD -= float(hdus["XYZ"].header["ZPOS"])
-    if "mm" not in hdus["XYZ"].header.comments["ZPOS"]:
-        raise KeyError(f'mm is absent from ZPOS key in XYZ header. Had {hdus["XYZ"].header.comments["ZPOS"]}'
-                       f'Distances along Z axis must be in mm.')
-    image.my_logger.info(f'\n\tDistance to CCD adjusted to {parameters.DISTANCE2CCD} mm '
-                         f'considering XYZ platform is set at ZPOS={float(hdus["XYZ"].header["ZPOS"])} mm.')
-    # compute CCD gain map
-    image.gain = float(image.header['CCDGAIN']) * np.ones_like(image.data)
-    image.read_out_noise = float(image.header['CCDNOISE']) * np.ones_like(image.data)
-    parameters.CCD_IMSIZE = image.data.shape[1] // parameters.CCD_REBIN
-    # save xys platform position into main header
-    image.header["XPOS"] = hdus["XYZ"].header["XPOS"]
-    image.header.comments["XPOS"] = hdus["XYZ"].header.comments["XPOS"]
-    image.header["YPOS"] = hdus["XYZ"].header["YPOS"]
-    image.header.comments["YPOS"] = hdus["XYZ"].header.comments["YPOS"]
-    image.header["ZPOS"] = hdus["XYZ"].header["ZPOS"]
-    image.header.comments["ZPOS"] = hdus["XYZ"].header.comments["ZPOS"]
-    image.my_logger.info('\n\tImage loaded')
+    with fits.open(image.file_name) as hdus:
+        image.header = hdus[0].header
+        hdu1 = hdus["CHAN_14"]
+        hdu2 = hdus["CHAN_06"]
+        data1 = hdu1.data[imgslice(hdu1.header['DATASEC'])].astype(np.float64)
+        bias1 = np.median(hdu1.data[imgslice(hdu1.header['BIASSEC'])].astype(np.float64))
+        data1 -= bias1
+        detsecy, detsecx = imgslice(hdu1.header['DETSEC'])
+        if detsecy.start > detsecy.stop:
+            data1 = data1[:, ::-1]
+        if detsecx.start > detsecx.stop:
+            data1 = data1[::-1, :]
+        data2 = hdu2.data[imgslice(hdu2.header['DATASEC'])].astype(np.float64)
+        bias2 = np.median(hdu2.data[imgslice(hdu2.header['BIASSEC'])].astype(np.float64))
+        data2 -= bias2
+        detsecy, detsecx = imgslice(hdu2.header['DETSEC'])
+        if detsecy.start > detsecy.stop:
+            data2 = data2[:, ::-1]
+        if detsecx.start > detsecx.stop:
+            data2 = data2[::-1, :]
+        data = np.concatenate([data2, data1])
+        image.data = data[::-1, :].T
+        image.date_obs = image.header['DATE-OBS']
+        image.expo = float(image.header['EXPTIME'])
+        image.airmass = -1
+        parameters.DISTANCE2CCD -= float(hdus["XYZ"].header["ZPOS"])
+        if "mm" not in hdus["XYZ"].header.comments["ZPOS"]:
+            raise KeyError(f'mm is absent from ZPOS key in XYZ header. Had {hdus["XYZ"].header.comments["ZPOS"]}'
+                        f'Distances along Z axis must be in mm.')
+        image.my_logger.info(f'\n\tDistance to CCD adjusted to {parameters.DISTANCE2CCD} mm '
+                            f'considering XYZ platform is set at ZPOS={float(hdus["XYZ"].header["ZPOS"])} mm.')
+        # compute CCD gain map
+        image.gain = float(image.header['CCDGAIN']) * np.ones_like(image.data)
+        image.read_out_noise = float(image.header['CCDNOISE']) * np.ones_like(image.data)
+        parameters.CCD_IMSIZE = image.data.shape[1] // parameters.CCD_REBIN
+        # save xys platform position into main header
+        image.header["XPOS"] = hdus["XYZ"].header["XPOS"]
+        image.header.comments["XPOS"] = hdus["XYZ"].header.comments["XPOS"]
+        image.header["YPOS"] = hdus["XYZ"].header["YPOS"]
+        image.header.comments["YPOS"] = hdus["XYZ"].header.comments["YPOS"]
+        image.header["ZPOS"] = hdus["XYZ"].header["ZPOS"]
+        image.header.comments["ZPOS"] = hdus["XYZ"].header.comments["ZPOS"]
+        image.my_logger.info('\n\tImage loaded')
 
 
 def load_AUXTEL_image(image):  # pragma: no cover
@@ -676,10 +676,9 @@ def load_AUXTEL_image(image):  # pragma: no cover
         The Image instance to fill with file data and header.
     """
     image.my_logger.info(f'\n\tLoading AUXTEL image {image.file_name}...')
-    hdu_list = fits.open(image.file_name)
-    image.header = hdu_list[0].header
-    image.data = hdu_list[1].data.astype(np.float64)
-    hdu_list.close()  # need to free allocation for file descripto
+    with fits.open(image.file_name) as hdu_list:
+        image.header = hdu_list[0].header
+        image.data = hdu_list[1].data.astype(np.float64)
     image.date_obs = image.header['DATE']
     image.expo = float(image.header['EXPTIME'])
     if "empty" not in image.header['FILTER'].lower():
@@ -751,10 +750,9 @@ def load_STARDICE_image(image):  # pragma: no cover
     """
 
     image.my_logger.info(f'\n\tLoading STARDICE image {image.file_name}...')
-    hdu_list = fits.open(image.file_name)
-    image.header = hdu_list[0].header
-    image.data = hdu_list[0].data.astype(np.float64)
-    hdu_list.close()  # need to free allocation for file descripto
+    with fits.open(image.file_name) as hdu_list:
+        image.header = hdu_list[0].header
+        image.data = hdu_list[0].data.astype(np.float64)
     if "BZERO" in image.header:
         del image.header["BZERO"]
     if "BSCALE" in image.header: