Merge branch 'master' into enh/pattern_noise_efficiency

GeminiDRSoftware · Mar 14, 2023 · 929fb5f · 929fb5f
2 parents 401f4c6 + 4bd66d8
commit 929fb5f
Show file tree

Hide file tree

Showing 21 changed files with 226 additions and 69 deletions.
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: ['3.10']
 
     steps:
       - uses: actions/checkout@v2
@@ -53,4 +53,4 @@ jobs:
       - name: Run Tox
         shell: bash -l {0}
         run: |
-          tox -e py37-unit -v
+          tox -e py310-unit -v
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -68,16 +68,16 @@ pipeline {
                         echo "Running build #${env.BUILD_ID} on ${env.NODE_NAME}"
                         checkout scm
                         sh '.jenkins/scripts/setup_agent.sh'
-                        echo "Running tests with Python 3.7"
-                        sh 'tox -e py37-unit -v -r -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/unittests_results.xml ${TOX_ARGS}'
+                        echo "Running tests with Python 3.10"
+                        sh 'tox -e py310-unit -v -r -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/unittests_results.xml ${TOX_ARGS}'
                         echo "Reportint coverage to CodeCov"
                         sh 'tox -e codecov -- -F unit'
                     }
                     post {
                         always {
                             junit (
                                 allowEmptyResults: true,
-                                testResults: '.tmp/py37-unit/reports/*_results.xml'
+                                testResults: '.tmp/py310-unit/reports/*_results.xml'
                             )
                             echo "Deleting Unit tests workspace ${env.WORKSPACE}"
                             cleanWs()
@@ -108,15 +108,15 @@ pipeline {
                         echo "${env.PATH}"
                         sh '.jenkins/scripts/setup_agent.sh'
                         echo "Integration tests"
-                        sh 'tox -e py37-integ -v -r -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/integration_results.xml ${TOX_ARGS}'
+                        sh 'tox -e py310-integ -v -r -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/integration_results.xml ${TOX_ARGS}'
                         echo "Reporting coverage"
                         sh 'tox -e codecov -- -F integration'
                     } // end steps
                     post {
                         always {
                             junit (
                                 allowEmptyResults: true,
-                                testResults: '.tmp/py37-integ/reports/*_results.xml'
+                                testResults: '.tmp/py310-integ/reports/*_results.xml'
                             )
                             echo "Deleting Integration tests workspace ${env.WORKSPACE}"
                             cleanWs()
@@ -142,15 +142,15 @@ pipeline {
                         echo "${env.PATH}"
                         sh '.jenkins/scripts/setup_agent.sh'
                         echo "Regression tests"
-                        sh 'tox -e py37-reg -v -r -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/regression_results.xml ${TOX_ARGS}'
+                        sh 'tox -e py310-reg -v -r -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/regression_results.xml ${TOX_ARGS}'
                         echo "Reporting coverage"
                         sh 'tox -e codecov -- -F regression'
                     } // end steps
                     post {
                         always {
                             junit (
                                 allowEmptyResults: true,
-                                testResults: '.tmp/py37-reg/reports/*_results.xml'
+                                testResults: '.tmp/py310-reg/reports/*_results.xml'
                             )
                             echo "Deleting Regression Tests workspace ${env.WORKSPACE}"
                             cleanWs()
@@ -181,7 +181,7 @@ pipeline {
                 checkout scm
                 sh '.jenkins/scripts/setup_agent.sh'
                 echo "Running tests"
-                sh 'tox -e py37-f2 -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/f2_results.xml ${TOX_ARGS}'
+                sh 'tox -e py310-f2 -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/f2_results.xml ${TOX_ARGS}'
                 echo "Reporting coverage"
                 sh 'tox -e codecov -- -F f2'
             }  // end steps
@@ -191,7 +191,7 @@ pipeline {
                     archiveArtifacts artifacts: "plots/*", allowEmptyArchive: true
                     junit (
                         allowEmptyResults: true,
-                        testResults: '.tmp/py37-f2/reports/*_results.xml'
+                        testResults: '.tmp/py310-f2/reports/*_results.xml'
                     )
                     echo "Deleting F2 Tests workspace ${env.WORKSPACE}"
                     cleanWs()
@@ -219,7 +219,7 @@ pipeline {
                 checkout scm
                 sh '.jenkins/scripts/setup_agent.sh'
                 echo "Running tests"
-                sh 'tox -e py37-gsaoi -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/gsaoi_results.xml ${TOX_ARGS}'
+                sh 'tox -e py310-gsaoi -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/gsaoi_results.xml ${TOX_ARGS}'
                 echo "Reporting coverage"
                 sh 'tox -e codecov -- -F gsaoi'
             }  // end steps
@@ -229,7 +229,7 @@ pipeline {
                     archiveArtifacts artifacts: "plots/*", allowEmptyArchive: true
                     junit (
                         allowEmptyResults: true,
-                        testResults: '.tmp/py37-gsaoi/reports/*_results.xml'
+                        testResults: '.tmp/py310-gsaoi/reports/*_results.xml'
                     )
                     echo "Deleting GSAOI Tests workspace ${env.WORKSPACE}"
                     cleanWs()
@@ -257,7 +257,7 @@ pipeline {
                 checkout scm
                 sh '.jenkins/scripts/setup_agent.sh'
                 echo "Running tests"
-                sh 'tox -e py37-niri -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/niri_results.xml ${TOX_ARGS}'
+                sh 'tox -e py310-niri -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/niri_results.xml ${TOX_ARGS}'
                 echo "Reporting coverage"
                 sh 'tox -e codecov -- -F niri'
             }  // end steps
@@ -267,7 +267,7 @@ pipeline {
                     archiveArtifacts artifacts: "plots/*", allowEmptyArchive: true
                     junit (
                         allowEmptyResults: true,
-                        testResults: '.tmp/py37-niri/reports/*_results.xml'
+                        testResults: '.tmp/py310-niri/reports/*_results.xml'
                     )
                     echo "Deleting NIRI Tests workspace ${env.WORKSPACE}"
                     cleanWs()
@@ -295,7 +295,7 @@ pipeline {
                 checkout scm
                 sh '.jenkins/scripts/setup_agent.sh'
                 echo "Running tests"
-                sh 'tox -e py37-gnirs -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/gnirs_results.xml ${TOX_ARGS}'
+                sh 'tox -e py310-gnirs -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/gnirs_results.xml ${TOX_ARGS}'
                 echo "Reporting coverage"
                 sh 'tox -e codecov -- -F gnirs'
             }  // end steps
@@ -305,7 +305,7 @@ pipeline {
                     archiveArtifacts artifacts: "plots/*", allowEmptyArchive: true
                     junit (
                         allowEmptyResults: true,
-                        testResults: '.tmp/py37-gnirs/reports/*_results.xml'
+                        testResults: '.tmp/py310-gnirs/reports/*_results.xml'
                     )
                     echo "Deleting GNIRS Tests workspace ${env.WORKSPACE}"
                     cleanWs()
@@ -333,7 +333,7 @@ pipeline {
                 checkout scm
                 sh '.jenkins/scripts/setup_agent.sh'
                 echo "Running tests"
-                sh 'tox -e py37-wavecal -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/wavecal_results.xml ${TOX_ARGS}'
+                sh 'tox -e py310-wavecal -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/wavecal_results.xml ${TOX_ARGS}'
                 echo "Reporting coverage"
                 sh 'tox -e codecov -- -F wavecal'
             }  // end steps
@@ -343,7 +343,7 @@ pipeline {
                     archiveArtifacts artifacts: "plots/*", allowEmptyArchive: true
                     junit (
                         allowEmptyResults: true,
-                        testResults: '.tmp/py37-wavecal/reports/*_results.xml'
+                        testResults: '.tmp/py310-wavecal/reports/*_results.xml'
                     )
                     echo "Deleting WaveCal Tests workspace ${env.WORKSPACE}"
                     cleanWs()
@@ -374,7 +374,7 @@ pipeline {
                         checkout scm
                         sh '.jenkins/scripts/setup_agent.sh'
                         echo "Running tests"
-                        sh 'tox -e py37-gmosls -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/gmosls_results.xml ${TOX_ARGS}'
+                        sh 'tox -e py310-gmosls -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/gmosls_results.xml ${TOX_ARGS}'
                         echo "Reporting coverage"
                         sh 'tox -e codecov -- -F gmosls'
                     }  // end steps
@@ -384,7 +384,7 @@ pipeline {
                             archiveArtifacts artifacts: "plots/*", allowEmptyArchive: true
                             junit (
                                 allowEmptyResults: true,
-                                testResults: '.tmp/py37-gmosls/reports/*_results.xml'
+                                testResults: '.tmp/py310-gmosls/reports/*_results.xml'
                             )
                             echo "Deleting GMOS LS Tests workspace ${env.WORKSPACE}"
                             cleanWs()
@@ -413,15 +413,15 @@ pipeline {
                         echo "${env.PATH}"
                         sh '.jenkins/scripts/setup_agent.sh'
                         echo "Slow tests"
-                        sh 'tox -e py37-slow -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/slow_results.xml ${TOX_ARGS}'
+                        sh 'tox -e py310-slow -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/slow_results.xml ${TOX_ARGS}'
                         echo "Reporting coverage"
                         sh 'tox -e codecov -- -F slow'
                     } // end steps
                     post {
                         always {
                             junit (
                                 allowEmptyResults: true,
-                                testResults: '.tmp/py37-slow/reports/*_results.xml'
+                                testResults: '.tmp/py310-slow/reports/*_results.xml'
                             )
                             echo "Deleting GMOS LS Tests workspace ${env.WORKSPACE}"
                             cleanWs()

diff --git a/astrodata/wcs.py b/astrodata/wcs.py
@@ -882,6 +882,10 @@ def create_new_image_projection(transform, new_center):
     xc, yc = transform.inverse(*current_center)
     xcnew, ycnew = transform.inverse(*new_center)
     xpole, ypole = transform.inverse(0, 90)
+    # astropy >=5.2 returns NaNs for a point not in the same hemisphere as
+    # the projection centre, so use the Celestial South Pole if required
+    if np.isnan(xpole) or np.isnan(ypole):
+        xpole, ypole = transform.inverse(0, -90)
     angle1 = np.arctan2(xpole - xc, ypole - yc)
     angle2 = np.arctan2(xpole - xcnew, ypole - ycnew)
     rotation = (angle1 - angle2) * 180 / np.pi

diff --git a/geminidr/core/parameters_spect.py b/geminidr/core/parameters_spect.py
@@ -166,9 +166,10 @@ class distortionCorrectConfig(parameters_generic.calRequirementConfig):
 class extractSpectraConfig(config.Config):
     suffix = config.Field("Filename suffix", str, "_extracted", optional=True)
     method = config.ChoiceField("Extraction method", str,
-                                allowed={"standard": "no weighting",
-                                         "optimal": "optimal extraction"},
-                                default="standard")
+                                allowed={"aperture": "no weighting",
+                                         "optimal": "optimal extraction",
+                                         "default": "use 'optimal' for STANDARDs, and 'aperture' otherwise"},
+                                default="aperture")
     width = config.RangeField("Width of extraction aperture (pixels)", float, None, min=1, optional=True)
     grow = config.RangeField("Source aperture avoidance region (pixels)", float, 10, min=0, optional=True)
     subtract_sky = config.Field("Subtract sky spectra if the data have not been sky corrected?", bool, True)

diff --git a/geminidr/core/primitives_preprocess.py b/geminidr/core/primitives_preprocess.py
@@ -19,6 +19,11 @@
 from astropy import units as u
 from astropy.coordinates import SkyCoord
 from astropy.table import Table
+from gwcs import coordinate_frames as cf
+from gwcs.utils import CoordinateFrameError
+from scipy.interpolate import interp1d
+from scipy.ndimage import binary_dilation
+
 from geminidr import PrimitivesBASE
 from geminidr.gemini.lookups import DQ_definitions as DQ
 from gempy.gemini import gemini_tools as gt
@@ -27,8 +32,6 @@
 from gempy.library.filtering import ring_median_filter
 from recipe_system.utils.decorators import parameter_override, capture_provenance
 from recipe_system.utils.md5 import md5sum
-from scipy.interpolate import interp1d
-from scipy.ndimage import binary_dilation
 
 from . import parameters_preprocess
 
@@ -822,6 +825,23 @@ def flatCorrect(self, adinputs=None, suffix=None, flat=None, do_cal=True):
                          f"{flat.filename}{origin_str}")
             ad.divide(flat)
 
+            # Try to get a slit rectification model from the flat, and if one
+            # exists insert it before the pixels-to-world transform. Print a
+            # warning if one doesn't exist, but allow it to pass.
+            try:
+                rect_model = flat[0].wcs.get_transform('pixels', 'rectified')
+                rectified = True
+            except CoordinateFrameError:
+                log.warning(f"No rectification model found in {flat.filename}")
+                rectified = False
+
+            if rectified:
+                log.stdinfo(f"{ad.filename}: applying slit rectification model "
+                            f"from the flat {flat.filename}")
+                for ext in ad:
+                    ext.wcs.insert_frame(ext.wcs.input_frame, rect_model,
+                                         cf.Frame2D(name='rectified'))
+
             # Update the header and filename, copying QECORR keyword from flat
             ad.phu.set("FLATIM", flat.filename, self.keyword_comments["FLATIM"])
             try: