Merge pull request #26 from lsst/tickets/DM-35647

DM-35647: Resync with upstream again

.github/workflows/build.yaml

@@ -16,16 +16,23 @@ jobs:
 
       - uses: conda-incubator/setup-miniconda@v2
         with:
-          python-version: 3.8
+          python-version: "3.10"
+          mamba-version: "*"
           channels: conda-forge,defaults
           channel-priority: strict
           show-channel-urls: true
+          activate-environment: test
+
+      - name: Test Simbad URL is contactable
+        shell: bash -l {0}
+        run: |
+          curl -L http://simbad.u-strasbg.fr/ > /dev/null
 
       - name: Install dependencies
         # Might be quicker to install rubin-env plus any necessary additions.
         shell: bash -l {0}
         run: |
-          conda install -y -q numpy scipy matplotlib pandas llvmlite numba astropy photutils astroquery coloredlogs scikit-image h5py emcee tqdm schwimmbad iminuit coverage configparser coveralls pysynphot deprecated pyyaml nose
+          mamba install -y -q numpy>1.15 scipy matplotlib>3.1 pandas llvmlite numba astropy>=3.2 photutils>=0.7 astroquery coloredlogs scikit-image h5py emcee tqdm mpi4py schwimmbad iminuit>=2 coverage>=3.6 configparser coveralls deprecated pyyaml nose getCalspec
 
       - name: List installed packages
         shell: bash -l {0}
@@ -46,4 +53,4 @@ jobs:
       - name: Run full chain
         shell: bash -l {0}
         run: |
-          nosetests tests/run_full_chain.py --all --debug --detailed-errors --verbose --process-restartworker --with-coverage --cover-package=spectractor
+          nosetests tests/run_full_chain.py --all --debug --detailed-errors --verbose --process-restartworker --with-coverage --cover-package=spectractor

README.md

@@ -21,17 +21,17 @@ Some submodules complete the structures with generic functions:
 
 ## Installation
 
-Spectractor is written in Python 3.7. The dependencies are listed in the `requirements.txt` file. To install Spectractor, just run
+Spectractor is written in Python 3.9. The dependencies are listed in the `requirements.txt` file. To install Spectractor, just run
 ```
 pip install -r requirements.txt .
 ```
-Be careful, Spectractor can perform fits using the MCMC library [emcee](https://emcee.readthedocs.io/en/stable/) with [mpi4py](https://mpi4py.readthedocs.io/en/stable/) and [h5py](https://www.h5py.org/).  The latter might be better installed using `conda install ...` command to get their own dependencies (openmp and hdf5).
-
 For the simulation of spectra, Spectractor needs the following external libraries:
 - [libradtran](http://www.libradtran.org/doku.php) to simulate atmospheric transmission: it needs the installation of [netcdf](https://www.unidata.ucar.edu/software/netcdf/) and a python 2 environment (for the compilation only, not the usage); `uvpsec` executable must in the user `$PATH` or the user has to set an environmental variable `$LIBRADTRAN_DIR` pointing to the install directory.
-- [pysynphot](https://pysynphot.readthedocs.io/en/latest/) to get the CALSPEC star spectra: the HST CALSPEC calibration spectra must be downloaded and the environment variable `$PYSYN_CDBS` must be created.
+- [getCalspec](https://github.com/LSSTDESC/getCalspec) to get the CALSPEC star spectra;
 - [astrometry.net](https://astrometrynet.readthedocs.io/en/latest/) (optional): needed to create World Coordinate System files from the images; `solve-field` executable must in the user `$PATH` or the user has to set an environmental variable `$ASTROMETRYNET_DIR` pointing to the install directory. Version below or equal v0.78 should be used.
 
+Be careful, Spectractor can perform fits using the MCMC library [emcee](https://emcee.readthedocs.io/en/stable/) with [mpi4py](https://mpi4py.readthedocs.io/en/stable/) and [h5py](https://www.h5py.org/).  The latter might be better installed using `conda install ...` command to get their own dependencies (openmp and hdf5).
+
 Detailled command lines for the installation of Spectractor and the external dependencies can be found in the file `.travis.yml`.
 
 ## Basic extraction

config/auxtel.ini

@@ -13,6 +13,8 @@ SPECTRACTOR_COMPUTE_ROTATION_ANGLE = disperser
 SPECTRACTOR_DECONVOLUTION_PSF2D = False
 # deconvolve spectrogram with full forward model: False, True
 SPECTRACTOR_DECONVOLUTION_FFM = True
+# value of sigma clip parameter for the spectractor deconvolution process PSF2D and FFM
+SPECTRACTOR_DECONVOLUTION_SIGMA_CLIP = 100
 
 [instrument]
 # instrument name
@@ -39,7 +41,7 @@ OBS_QUANTUM_EFFICIENCY = calexp_2020031500162-EMPTY_ronchi90lpmm-det000_auxtel_t
 OBS_CAMERA_ROTATION = 0
 # Camera (x,y) flip signs with respect to (north-up, east-left) system
 OBS_CAMERA_DEC_FLIP_SIGN = 1
-OBS_CAMERA_RA_FLIP_SIGN = -1
+OBS_CAMERA_RA_FLIP_SIGN = 1
 
 [CCD]
 # size of the image in pixel  # MFL: this number is wrong, and the CCD is not square
@@ -57,9 +59,9 @@ CCD_REBIN = 2
 
 [spectrograph]
 # distance between hologram and CCD in mm
-DISTANCE2CCD = 175
+DISTANCE2CCD = 181
 # uncertainty on distance between hologram and CCD in mm
-DISTANCE2CCD_ERR = 0.75
+DISTANCE2CCD_ERR = 0.4
 # default value for order 2 over order 1 transmission ratio
 GRATING_ORDER_2OVER1 = 0.1
 

config/default.ini

@@ -13,6 +13,8 @@ SPECTRACTOR_COMPUTE_ROTATION_ANGLE = hessian
 SPECTRACTOR_DECONVOLUTION_PSF2D = True
 # deconvolve spectrogram with full forward model: False, True
 SPECTRACTOR_DECONVOLUTION_FFM = True
+# value of sigma clip parameter for the spectractor deconvolution process PSF2D and FFM
+SPECTRACTOR_DECONVOLUTION_SIGMA_CLIP = 20
 
 [instrument]
 # instrument name

requirements.txt

@@ -18,7 +18,8 @@ iminuit>=2
 coverage>=3.6 # <5
 configparser
 coveralls
-pysynphot
 deprecated
 pyyaml
 nose
+getCalspec
+pysynphot

spectractor/extractor/background.py

@@ -203,8 +203,11 @@ def extract_spectrogram_background_sextractor(data, err, ws=(20, 30), mask_signa
     #                    filter_size=((ws[1] - ws[0]) // 2, (ws[1] - ws[0]) // 2),
     #                    sigma_clip=sigma_clip, bkg_estimator=bkg_estimator,
     #                    mask=mask)
+    filter_size = parameters.PIXWIDTH_BOXSIZE // 2
+    if filter_size % 2 == 0:  # must be odd since photutils 1.5.0
+        filter_size += 1
     bkg = Background2D(data, (parameters.PIXWIDTH_BOXSIZE, parameters.PIXWIDTH_BOXSIZE),
-                       filter_size=(parameters.PIXWIDTH_BOXSIZE // 2, parameters.PIXWIDTH_BOXSIZE // 2),
+                       filter_size=(filter_size, filter_size),
                        sigma_clip=sigma_clip, bkg_estimator=bkg_estimator,
                        mask=mask)
     bgd_model_func = interp2d(np.arange(Nx), np.arange(Ny), bkg.background, kind='linear', bounds_error=False,

spectractor/extractor/chromaticpsf.py

@@ -1142,7 +1142,8 @@ def fit_chromatic_psf(self, data, bgd_model_func=None, data_errors=None, mode="1
             w.set_mask(poly_params=w.poly_params)
             # precise fit with sigma clipping
             run_minimisation_sigma_clipping(w, method="newton", ftol=1 / (w.Nx * w.Ny), xtol=1e-6, niter=50,
-                                            fix=w.fixed, sigma_clip=20, niter_clip=3, verbose=verbose)
+                                            fix=w.fixed, sigma_clip=parameters.SPECTRACTOR_DECONVOLUTION_SIGMA_CLIP,
+                                            niter_clip=3, verbose=verbose)
         else:
             raise ValueError(f"Unknown fitting mode={mode}. Must be '1D' or '2D'.")
 

spectractor/extractor/dispersers.py

@@ -647,6 +647,12 @@ def __init__(self, label, D=parameters.DISTANCE2CCD, data_dir=parameters.DISPERS
         self.is_hologram = True
         self.load_specs(verbose=verbose)
 
+    def theta_func(self, x, y):
+        return self.theta_tilt
+
+    def N_func(self, x, y):
+        return self.N_input
+
     def N(self, x):
         """Return the number of grooves per mm of the grating at position x. If the position is inside
         the data provided by the text files, this number is computed from an interpolation. If it lies outside,
@@ -749,11 +755,10 @@ def load_specs(self, verbose=True):
             filename = os.path.join(self.data_dir, self.label, "N.txt")
             if os.path.isfile(filename):
                 a = np.loadtxt(filename)
-
-                def N_func(x, y):
-                    return a[0]
-                self.N_interp = N_func
-                self.N_fit = N_func
+                self.N_input = a[0]
+                self.N_err = a[1]
+                self.N_interp = self.N_func
+                self.N_fit = self.N_func
             else:
                 raise ValueError("To define an hologram, you must provide hologram_grooves_per_mm.txt or N.txt files.")
         filename = os.path.join(self.data_dir, self.label, "hologram_center.txt")
@@ -773,9 +778,7 @@ def N_func(x, y):
                 self.theta_y /= parameters.CCD_REBIN
             self.theta_interp = interpolate.interp2d(self.theta_x, self.theta_y, self.theta_data, kind='cubic')
         else:
-            def theta_func(x, y):
-                return self.theta_tilt
-            self.theta_interp = theta_func
+            self.theta_interp = self.theta_func
         self.x_lines, self.line1, self.line2 = neutral_lines(self.holo_center[0], self.holo_center[1], self.theta_tilt)
         if verbose:
             if self.is_hologram:

spectractor/extractor/dispersers/holo4_003/NOTES

@@ -0,0 +1,2 @@
+- transmission.txt : measurements from LPNHE optical test bench using holo-4-003-diffraction-efficiencies-merged.npy
+- ratio_order_2over1.txt : chimera with LPNHE optical test bench measurements between 430 and 1000nm, then extrapolating with an hologram efficiency model fitted on data