Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DM-39583: Fix lots of deprecation warnings #40

Merged
merged 7 commits into from
Jul 19, 2023
Merged

DM-39583: Fix lots of deprecation warnings #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a413211
Fix some deprecation warnings from imports
timj Jul 19, 2023
90e239b
Use raw string for regex
timj Jul 19, 2023
f0eef94
Use "with fits.open" to ensure the file is closed
timj Jul 19, 2023
74a0462
Remove ftol option that is no longer used
timj Jul 19, 2023
d11f8da
Close file that was opened
timj Jul 19, 2023
d3f0457
Try python 3.11
timj Jul 19, 2023
e920223
Experimental changes to action
timj Jul 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 7 additions & 2 deletions .github/workflows/build.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ jobs:

- uses: conda-incubator/setup-miniconda@v2
with:
python-version: "3.11"
miniforge-variant: Mambaforge
miniforge-version: latest
show-channel-urls: true
Expand All @@ -25,9 +26,13 @@ jobs:
# Might be quicker to install rubin-env plus any necessary additions.
shell: bash -l {0}
run: |
mamba install -y -q numpy>1.15 scipy matplotlib>3.1 pandas llvmlite numba astropy>=3.2 photutils>=0.7 astroquery coloredlogs scikit-image h5py emcee tqdm mpi4py schwimmbad iminuit>=2 coverage>=3.6 configparser coveralls deprecated pyyaml nose rubin-libradtran getCalspec>=2.0.0
mamba install -y "numpy>1.15" scipy "matplotlib>3.1" pandas llvmlite numba "astropy>=3.2" "photutils>=0.7" astroquery coloredlogs scikit-image h5py emcee tqdm mpi4py schwimmbad "iminuit>=2" "coverage>=3.6" coveralls deprecated pyyaml nose rubin-libradtran "getCalspec>=2.0.0"
# python -c "from getCalspec.rebuild import rebuild_tables; rebuild_tables()"
mamba install astrometry

- name: Download test data
shell: bash -l {0}
run: |
wget -r -q -nc http://data.astrometry.net/5000/index-5002-24.fits
wget -r -q -nc http://data.astrometry.net/5000/index-5000-40.fits
mv data.astrometry.net/5000/index-*.fits $CONDA_PREFIX/data/
Expand All @@ -42,7 +47,7 @@ jobs:
- name: Build and install
shell: bash -l {0}
run: |
pip install -v -e .
pip install -v --no-deps -e .

- name: Run full chain
shell: bash -l {0}
Expand Down
43 changes: 21 additions & 22 deletions spectractor/astrometry.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -423,9 +423,8 @@ def get_quad_stars_pixel_positions(self):

"""
coords = []
hdu = fits.open(self.match_file_name)
table = Table.read(hdu)
hdu.close()
with fits.open(self.match_file_name) as hdu:
table = Table.read(hdu)
for k in range(0, len(table["QUADPIX"][0]), 2):
coord = [float(table["QUADPIX"][0][k]), float(table["QUADPIX"][0][k+1])]
if np.sum(coord) > 0:
Expand Down Expand Up @@ -1124,11 +1123,11 @@ def run_simple_astrometry(self, extent=None, sources=None):
# The source file given to solve-field is understood as a FITS file with pixel origin value at 1,
# whereas pixel coordinates comes from photutils using a numpy convention with pixel origin value at 0
# To correct for this we shift the CRPIX center of 1 pixel
hdu = fits.open(self.wcs_file_name)
hdu[0].header['CRPIX1'] = float(hdu[0].header['CRPIX1']) + 1
hdu[0].header['CRPIX2'] = float(hdu[0].header['CRPIX2']) + 1
self.my_logger.info(f"\n\tWrite astrometry.net WCS solution in {self.wcs_file_name}...")
hdu.writeto(self.wcs_file_name, overwrite=True)
with fits.open(self.wcs_file_name) as hdu:
hdu[0].header['CRPIX1'] = float(hdu[0].header['CRPIX1']) + 1
hdu[0].header['CRPIX2'] = float(hdu[0].header['CRPIX2']) + 1
self.my_logger.info(f"\n\tWrite astrometry.net WCS solution in {self.wcs_file_name}...")
hdu.writeto(self.wcs_file_name, overwrite=True)

# load quad stars
self.quad_stars_pixel_positions = self.get_quad_stars_pixel_positions()
Expand Down Expand Up @@ -1273,14 +1272,14 @@ def run_gaia_astrometry(self, min_range=3 * u.arcsec, max_range=5 * u.arcmin, ma
ddec_median = np.median(ddec.to(u.arcsec).value)
dra_rms = np.std(dra.to(u.arcsec).value)
ddec_rms = np.std(ddec.to(u.arcsec).value)
hdu = fits.open(self.wcs_file_name)
hdu[0].header['CRVAL1'] = self.wcs.wcs.crval[0]
hdu[0].header['CRVAL2'] = self.wcs.wcs.crval[1]
hdu[0].header['CRV1_MED'] = dra_median
hdu[0].header['CRV2_MED'] = ddec_median
hdu[0].header['CRV1_RMS'] = dra_rms
hdu[0].header['CRV2_RMS'] = ddec_rms
hdu.writeto(self.wcs_file_name, overwrite=True)
with fits.open(self.wcs_file_name) as hdu:
hdu[0].header['CRVAL1'] = self.wcs.wcs.crval[0]
hdu[0].header['CRVAL2'] = self.wcs.wcs.crval[1]
hdu[0].header['CRV1_MED'] = dra_median
hdu[0].header['CRV2_MED'] = ddec_median
hdu[0].header['CRV1_RMS'] = dra_rms
hdu[0].header['CRV2_RMS'] = ddec_rms
hdu.writeto(self.wcs_file_name, overwrite=True)

# check histogram medians
self.wcs = load_wcs_from_file(self.wcs_file_name)
Expand All @@ -1299,12 +1298,12 @@ def run_gaia_astrometry(self, min_range=3 * u.arcsec, max_range=5 * u.arcmin, ma
ddec_median = np.median(ddec.to(u.arcsec).value)
dra_rms = np.std(dra.to(u.arcsec).value)
ddec_rms = np.std(ddec.to(u.arcsec).value)
hdu = fits.open(self.wcs_file_name)
hdu[0].header['CRV1_MED'] = dra_median
hdu[0].header['CRV2_MED'] = ddec_median
hdu[0].header['CRV1_RMS'] = dra_rms
hdu[0].header['CRV2_RMS'] = ddec_rms
hdu.writeto(self.wcs_file_name, overwrite=True)
with fits.open(self.wcs_file_name) as hdu:
hdu[0].header['CRV1_MED'] = dra_median
hdu[0].header['CRV2_MED'] = ddec_median
hdu[0].header['CRV1_RMS'] = dra_rms
hdu[0].header['CRV2_RMS'] = ddec_rms
hdu.writeto(self.wcs_file_name, overwrite=True)

# if parameters.DEBUG:
# plot_shifts_histograms(dra, ddec)
Expand Down
2 changes: 1 addition & 1 deletion spectractor/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def from_config_to_parameters(config):
for section in config.sections():
for options in config.options(section):
value = config.get(section, options)
if re.match("[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?", value):
if re.match(r"[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?", value):
if ' ' in value:
value = str(value)
elif '.' in value or 'e' in value:
Expand Down
2 changes: 1 addition & 1 deletion spectractor/extractor/background.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from spectractor.tools import fit_poly1d_outlier_removal, fit_poly2d_outlier_removal, plot_image_simple

from astropy.stats import SigmaClip
from photutils import Background2D, SExtractorBackground
from photutils.background import Background2D, SExtractorBackground
from photutils.segmentation import SegmentationImage, detect_threshold, detect_sources

from scipy import ndimage
Expand Down
6 changes: 4 additions & 2 deletions spectractor/extractor/dispersers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,8 @@ def load_files(self, verbose=False):
self.ratio_order_3over2 = None
filename = os.path.join(self.data_dir, self.label, "hologram_center.txt")
if os.path.isfile(filename):
lines = [ll.rstrip('\n') for ll in open(filename)]
with open(filename) as f:
lines = [ll.rstrip('\n') for ll in f]
self.theta_tilt = float(lines[1].split(' ')[2])
else:
self.theta_tilt = 0
Expand Down Expand Up @@ -798,7 +799,8 @@ def load_specs(self, verbose=True):
raise ValueError("To define an hologram, you must provide hologram_grooves_per_mm.txt or N.txt files.")
filename = os.path.join(self.data_dir, self.label, "hologram_center.txt")
if os.path.isfile(filename):
lines = [ll.rstrip('\n') for ll in open(filename)]
with open(filename) as f:
lines = [ll.rstrip('\n') for ll in f]
self.holo_center = list(map(float, lines[1].split(' ')[:2]))
self.theta_tilt = float(lines[1].split(' ')[2])
else:
Expand Down
100 changes: 49 additions & 51 deletions spectractor/extractor/images.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -622,49 +622,49 @@ def load_LPNHE_image(image): # pragma: no cover
The Image instance to fill with file data and header.
"""
image.my_logger.info(f'\n\tLoading LPNHE image {image.file_name}...')
hdus = fits.open(image.file_name)
image.header = hdus[0].header
hdu1 = hdus["CHAN_14"]
hdu2 = hdus["CHAN_06"]
data1 = hdu1.data[imgslice(hdu1.header['DATASEC'])].astype(np.float64)
bias1 = np.median(hdu1.data[imgslice(hdu1.header['BIASSEC'])].astype(np.float64))
data1 -= bias1
detsecy, detsecx = imgslice(hdu1.header['DETSEC'])
if detsecy.start > detsecy.stop:
data1 = data1[:, ::-1]
if detsecx.start > detsecx.stop:
data1 = data1[::-1, :]
data2 = hdu2.data[imgslice(hdu2.header['DATASEC'])].astype(np.float64)
bias2 = np.median(hdu2.data[imgslice(hdu2.header['BIASSEC'])].astype(np.float64))
data2 -= bias2
detsecy, detsecx = imgslice(hdu2.header['DETSEC'])
if detsecy.start > detsecy.stop:
data2 = data2[:, ::-1]
if detsecx.start > detsecx.stop:
data2 = data2[::-1, :]
data = np.concatenate([data2, data1])
image.data = data[::-1, :].T
image.date_obs = image.header['DATE-OBS']
image.expo = float(image.header['EXPTIME'])
image.airmass = -1
parameters.DISTANCE2CCD -= float(hdus["XYZ"].header["ZPOS"])
if "mm" not in hdus["XYZ"].header.comments["ZPOS"]:
raise KeyError(f'mm is absent from ZPOS key in XYZ header. Had {hdus["XYZ"].header.comments["ZPOS"]}'
f'Distances along Z axis must be in mm.')
image.my_logger.info(f'\n\tDistance to CCD adjusted to {parameters.DISTANCE2CCD} mm '
f'considering XYZ platform is set at ZPOS={float(hdus["XYZ"].header["ZPOS"])} mm.')
# compute CCD gain map
image.gain = float(image.header['CCDGAIN']) * np.ones_like(image.data)
image.read_out_noise = float(image.header['CCDNOISE']) * np.ones_like(image.data)
parameters.CCD_IMSIZE = image.data.shape[1] // parameters.CCD_REBIN
# save xys platform position into main header
image.header["XPOS"] = hdus["XYZ"].header["XPOS"]
image.header.comments["XPOS"] = hdus["XYZ"].header.comments["XPOS"]
image.header["YPOS"] = hdus["XYZ"].header["YPOS"]
image.header.comments["YPOS"] = hdus["XYZ"].header.comments["YPOS"]
image.header["ZPOS"] = hdus["XYZ"].header["ZPOS"]
image.header.comments["ZPOS"] = hdus["XYZ"].header.comments["ZPOS"]
image.my_logger.info('\n\tImage loaded')
with fits.open(image.file_name) as hdus:
image.header = hdus[0].header
hdu1 = hdus["CHAN_14"]
hdu2 = hdus["CHAN_06"]
data1 = hdu1.data[imgslice(hdu1.header['DATASEC'])].astype(np.float64)
bias1 = np.median(hdu1.data[imgslice(hdu1.header['BIASSEC'])].astype(np.float64))
data1 -= bias1
detsecy, detsecx = imgslice(hdu1.header['DETSEC'])
if detsecy.start > detsecy.stop:
data1 = data1[:, ::-1]
if detsecx.start > detsecx.stop:
data1 = data1[::-1, :]
data2 = hdu2.data[imgslice(hdu2.header['DATASEC'])].astype(np.float64)
bias2 = np.median(hdu2.data[imgslice(hdu2.header['BIASSEC'])].astype(np.float64))
data2 -= bias2
detsecy, detsecx = imgslice(hdu2.header['DETSEC'])
if detsecy.start > detsecy.stop:
data2 = data2[:, ::-1]
if detsecx.start > detsecx.stop:
data2 = data2[::-1, :]
data = np.concatenate([data2, data1])
image.data = data[::-1, :].T
image.date_obs = image.header['DATE-OBS']
image.expo = float(image.header['EXPTIME'])
image.airmass = -1
parameters.DISTANCE2CCD -= float(hdus["XYZ"].header["ZPOS"])
if "mm" not in hdus["XYZ"].header.comments["ZPOS"]:
raise KeyError(f'mm is absent from ZPOS key in XYZ header. Had {hdus["XYZ"].header.comments["ZPOS"]}'
f'Distances along Z axis must be in mm.')
image.my_logger.info(f'\n\tDistance to CCD adjusted to {parameters.DISTANCE2CCD} mm '
f'considering XYZ platform is set at ZPOS={float(hdus["XYZ"].header["ZPOS"])} mm.')
# compute CCD gain map
image.gain = float(image.header['CCDGAIN']) * np.ones_like(image.data)
image.read_out_noise = float(image.header['CCDNOISE']) * np.ones_like(image.data)
parameters.CCD_IMSIZE = image.data.shape[1] // parameters.CCD_REBIN
# save xys platform position into main header
image.header["XPOS"] = hdus["XYZ"].header["XPOS"]
image.header.comments["XPOS"] = hdus["XYZ"].header.comments["XPOS"]
image.header["YPOS"] = hdus["XYZ"].header["YPOS"]
image.header.comments["YPOS"] = hdus["XYZ"].header.comments["YPOS"]
image.header["ZPOS"] = hdus["XYZ"].header["ZPOS"]
image.header.comments["ZPOS"] = hdus["XYZ"].header.comments["ZPOS"]
image.my_logger.info('\n\tImage loaded')


def load_AUXTEL_image(image): # pragma: no cover
Expand All @@ -676,10 +676,9 @@ def load_AUXTEL_image(image): # pragma: no cover
The Image instance to fill with file data and header.
"""
image.my_logger.info(f'\n\tLoading AUXTEL image {image.file_name}...')
hdu_list = fits.open(image.file_name)
image.header = hdu_list[0].header
image.data = hdu_list[1].data.astype(np.float64)
hdu_list.close() # need to free allocation for file descripto
with fits.open(image.file_name) as hdu_list:
image.header = hdu_list[0].header
image.data = hdu_list[1].data.astype(np.float64)
image.date_obs = image.header['DATE']
image.expo = float(image.header['EXPTIME'])
if "empty" not in image.header['FILTER'].lower():
Expand Down Expand Up @@ -751,10 +750,9 @@ def load_STARDICE_image(image): # pragma: no cover
"""

image.my_logger.info(f'\n\tLoading STARDICE image {image.file_name}...')
hdu_list = fits.open(image.file_name)
image.header = hdu_list[0].header
image.data = hdu_list[0].data.astype(np.float64)
hdu_list.close() # need to free allocation for file descripto
with fits.open(image.file_name) as hdu_list:
image.header = hdu_list[0].header
image.data = hdu_list[0].data.astype(np.float64)
if "BZERO" in image.header:
del image.header["BZERO"]
if "BSCALE" in image.header:
Expand Down