DM-15606: Move unit tests' retargetting a.net to configfile

setup.cfg

@@ -4,6 +4,7 @@ ignore = E133, E226, E228, N802, N803, N806
 exclude =
   **/*/__init__.py,
   tests/config/hsc-config.py,
+  tests/config/config.py,
   bin
   tests/data/
 

tests/config/config.py

@@ -0,0 +1,3 @@
+from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
+config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
+config.astrometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)

tests/config/hsc-config.py

@@ -1,3 +1,5 @@
+from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
+config.astrometryRefObjLoader.retarget(LoadIndexedReferenceObjectsTask)
 config.astrometryRefObjLoader.ref_dataset_name = "gaia"
 config.astrometryRefObjLoader.filterMap = {'u': 'phot_g_mean_mag',
     'g': 'phot_g_mean_mag',

tests/jointcalTestBase.py

@@ -62,6 +62,7 @@ def setUp_base(self, center, radius,
 
         # Individual tests may want to tweak the config that is passed to parseAndRun().
         self.config = None
+        self.configfiles = []
 
         # Append `msg` arguments to assert failures.
         self.longMessage = True
@@ -172,9 +173,14 @@ def _runJointcalTask(self, nCatalogs, caller, metrics=None):
         output_dir = os.path.join('.test', self.__class__.__name__, caller)
         if self.log_level is not None:
             self.other_args.extend(['--loglevel', 'jointcal=%s'%self.log_level])
+
+        #  Place default configfile first so that specific subclass configfiles are applied after
+        test_config = os.path.join(lsst.utils.getPackageDir('jointcal'), 'tests/config/config.py')
+        self.configfiles = [test_config] + self.configfiles
+
         args = [self.input_dir, '--output', output_dir,
                 '--clobber-versions', '--clobber-config',
-                '--doraise',
+                '--doraise', '--configfile', *self.configfiles,
                 '--id', 'visit=%s'%visits]
         args.extend(self.other_args)
         result = jointcal.JointcalTask.parseAndRun(args=args, doReturnResults=True, config=self.config)

tests/test_jointcal_cfht.py

@@ -7,7 +7,6 @@
 import lsst.afw.geom
 import lsst.utils
 import lsst.pex.exceptions
-from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
 
 import jointcalTestBase
 
@@ -50,10 +49,8 @@ def setUp(self):
 
     def test_jointcalTask_2_visits(self):
         self.config = lsst.jointcal.jointcal.JointcalConfig()
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.astrometryModel = "simple"
         self.config.photometryModel = "simpleFlux"
-        self.config.astrometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")
 
         # to test whether we got the expected chi2 contribution files.
@@ -96,7 +93,6 @@ def setup_jointcalTask_2_visits_constrainedAstrometry(self):
         the differences between them more obvious.
         """
         self.config = lsst.jointcal.jointcal.JointcalConfig()
-        self.config.astrometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.astrometryModel = "constrained"
         self.config.doPhotometry = False
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")
@@ -143,7 +139,6 @@ def setup_jointcalTask_2_visits_constrainedPhotometry(self):
         the differences between them more obvious.
         """
         self.config = lsst.jointcal.jointcal.JointcalConfig()
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.photometryModel = "constrainedFlux"
         self.config.doAstrometry = False
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")

tests/test_jointcal_cfht_minimal.py

@@ -6,7 +6,6 @@
 import lsst.afw.geom
 import lsst.utils
 import lsst.pex.exceptions
-from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
 
 import jointcalTestBase
 from lsst.jointcal import jointcal
@@ -47,7 +46,6 @@ def setUp(self):
     def test_jointcalTask_2_visits_photometry(self):
         self.config = lsst.jointcal.jointcal.JointcalConfig()
         self.config.photometryModel = "simpleFlux"
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.doAstrometry = False
         self.config.writeInitMatrix = True  # write Hessian/gradient files
         self.jointcalStatistics.do_astrometry = False
@@ -79,7 +77,6 @@ def test_jointcalTask_2_visits_photometry(self):
 
     def test_jointcalTask_2_visits_photometry_magnitude(self):
         self.config = lsst.jointcal.jointcal.JointcalConfig()
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.photometryModel = "simpleMagnitude"
         self.config.doAstrometry = False
         self.jointcalStatistics.do_astrometry = False
@@ -104,7 +101,6 @@ def test_jointcalTask_fails_raise(self):
         self.config = lsst.jointcal.jointcal.JointcalConfig()
         self.config.setDefaults()
         self.config.photometryModel = "simpleFlux"
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.sourceSelector['astrometry'].minSnr = 10000
         self.config.doAstrometry = False
 
@@ -113,8 +109,10 @@ def test_jointcalTask_fails_raise(self):
         nCatalogs = 2
         visits = '^'.join(str(v) for v in self.all_visits[:nCatalogs])
         output_dir = os.path.join('.test', self.__class__.__name__, caller)
+        test_config = os.path.join(lsst.utils.getPackageDir('jointcal'), 'tests/config/config.py')
+        self.configfiles = [test_config] + self.configfiles
         args = [self.input_dir, '--output', output_dir,
-                '--clobber-versions', '--clobber-config',
+                '--clobber-versions', '--clobber-config', '--configfile', *self.configfiles,
                 '--doraise',
                 '--id', 'visit=%s'%visits]
         args.extend(self.other_args)
@@ -126,7 +124,6 @@ def test_jointcalTask_fails_no_raise(self):
         self.config = lsst.jointcal.jointcal.JointcalConfig()
         self.config.setDefaults()
         self.config.photometryModel = "simpleFlux"
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.sourceSelector['astrometry'].minSnr = 10000
         self.config.doAstrometry = False
 
@@ -135,8 +132,10 @@ def test_jointcalTask_fails_no_raise(self):
         nCatalogs = 2
         visits = '^'.join(str(v) for v in self.all_visits[:nCatalogs])
         output_dir = os.path.join('.test', self.__class__.__name__, caller)
+        test_config = os.path.join(lsst.utils.getPackageDir('jointcal'), 'tests/config/config.py')
+        self.configfiles = [test_config] + self.configfiles
         args = [self.input_dir, '--output', output_dir,
-                '--clobber-versions', '--clobber-config',
+                '--clobber-versions', '--clobber-config', '--configfile', *self.configfiles,
                 '--noExit',  # have to specify noExit, otherwise the test quits
                 '--id', 'visit=%s'%visits]
         args.extend(self.other_args)
@@ -148,7 +147,6 @@ def test_jointcalTask_fails_no_raise_no_return_results(self):
         self.config = lsst.jointcal.jointcal.JointcalConfig()
         self.config.setDefaults()
         self.config.photometryModel = "simpleFlux"
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.sourceSelector['astrometry'].minSnr = 10000
         self.config.doAstrometry = False
 
@@ -157,8 +155,10 @@ def test_jointcalTask_fails_no_raise_no_return_results(self):
         nCatalogs = 2
         visits = '^'.join(str(v) for v in self.all_visits[:nCatalogs])
         output_dir = os.path.join('.test', self.__class__.__name__, caller)
+        test_config = os.path.join(lsst.utils.getPackageDir('jointcal'), 'tests/config/config.py')
+        self.configfiles = [test_config] + self.configfiles
         args = [self.input_dir, '--output', output_dir,
-                '--clobber-versions', '--clobber-config',
+                '--clobber-versions', '--clobber-config', '--configfile', *self.configfiles,
                 '--noExit',  # have to specify noExit, otherwise the test quits
                 '--id', 'visit=%s'%visits]
         args.extend(self.other_args)

tests/test_jointcal_decam.py

@@ -7,7 +7,6 @@
 import lsst.afw.geom
 import lsst.utils
 import lsst.pex.exceptions
-from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
 
 import jointcalTestBase
 
@@ -54,8 +53,6 @@ def test_jointcalTask_2_visits(self):
         self.config = lsst.jointcal.jointcal.JointcalConfig()
         self.config.astrometryModel = "simple"
         self.config.photometryModel = "simpleFlux"
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
-        self.config.astrometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")
 
         # See Readme for an explanation of these empirical values.
@@ -83,8 +80,6 @@ def setup_jointcalTask_2_visits_constrainedAstrometry_no_photometry(self):
         """Help keep the two constrainedAstrometry tests consistent and make
         the difference between them more obvious."""
         self.config = lsst.jointcal.jointcal.JointcalConfig()
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
-        self.config.astrometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.astrometryModel = "constrained"
         self.config.doPhotometry = False
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")
@@ -133,8 +128,6 @@ def setup_jointcalTask_2_visits_constrainedPhotometry_no_astrometry(self):
         the differences between them more obvious.
         """
         self.config = lsst.jointcal.jointcal.JointcalConfig()
-        self.config.astrometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.photometryModel = "constrainedFlux"
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")
         self.config.doAstrometry = False

tests/test_jointcal_hsc.py

@@ -8,8 +8,6 @@
 import lsst.afw.geom
 import lsst.utils
 import lsst.pex.exceptions
-from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
-from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
 
 import jointcalTestBase
 
@@ -51,8 +49,6 @@ def test_jointcalTask_2_visits(self):
         self.config = lsst.jointcal.jointcal.JointcalConfig()
         self.config.astrometryModel = "simple"
         self.config.photometryModel = "simpleFlux"
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
-        self.config.astrometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")
 
         # See Readme for an explanation of these empirical values.
@@ -82,7 +78,6 @@ def test_jointcalTask_11_visits_no_photometry(self):
 
         self.config = lsst.jointcal.jointcal.JointcalConfig()
         self.config.astrometryModel = "simple"
-        self.config.astrometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.doPhotometry = False
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")
         self.jointcalStatistics.do_photometry = False
@@ -105,7 +100,6 @@ def setup_jointcalTask_2_visits_simplePhotometry(self):
         the differences between them more obvious.
         """
         self.config = lsst.jointcal.jointcal.JointcalConfig()
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.photometryModel = "simpleFlux"
         self.config.doAstrometry = False
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")
@@ -149,7 +143,6 @@ def testJointcalTask_2_visits_no_photometry(self):
                    }
 
         self.config = lsst.jointcal.jointcal.JointcalConfig()
-        self.config.astrometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.astrometryModel = "simple"
         self.config.doPhotometry = False
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")
@@ -176,13 +169,10 @@ def test_jointcalTask_2_visits_gaia_refcat(self):
         self.config = lsst.jointcal.jointcal.JointcalConfig()
         self.config.astrometryModel = "simple"
         self.config.photometryModel = "simpleFlux"
-        self.config.astrometryRefObjLoader.retarget(LoadIndexedReferenceObjectsTask)
-        # use the a.net refcat for photometry.
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
+        # use the a.net refcat for photometry, gaia for astrometry
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")
-
         test_config = os.path.join(lsst.utils.getPackageDir('jointcal'), 'tests/config/hsc-config.py')
-        self.other_args.extend(['--configfile', test_config])
+        self.configfiles.extend([test_config])
         dist_rms_relative = 17e-3*u.arcsecond
 
         # See Readme for an explanation of these empirical values.
@@ -213,7 +203,6 @@ def test_jointcalTask_2_visits_gaia_refcat(self):
     def test_jointcalTask_2_visits_no_photometry_match_cut_10(self):
         self.config = lsst.jointcal.jointcal.JointcalConfig()
         self.config.astrometryModel = "simple"
-        self.config.astrometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.matchCut = 10.0  # TODO: once DM-6885 is fixed, we need to put `*lsst.afw.geom.arcseconds`
         self.config.doPhotometry = False
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")
@@ -236,7 +225,6 @@ def test_jointcalTask_3_visits_no_photometry(self):
         """3 visit, default config to compare with min_measurements_3 test."""
         self.config = lsst.jointcal.jointcal.JointcalConfig()
         self.config.astrometryModel = "simple"
-        self.config.astrometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.minMeasurements = 2
         self.config.doPhotometry = False
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")
@@ -260,7 +248,6 @@ def test_jointcalTask_3_visits_no_photometry_min_measurements_3(self):
         fitted stars (and thus the chisq and Ndof), but should not change the
         other values."""
         self.config = lsst.jointcal.jointcal.JointcalConfig()
-        self.config.astrometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.minMeasurements = 3
         self.config.astrometryModel = "simple"
         self.config.doPhotometry = False

tests/test_jointcal_lsstSim.py

@@ -9,7 +9,6 @@
 import lsst.afw.image
 import lsst.utils
 import lsst.pex.exceptions
-from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
 
 import jointcalTestBase
 import lsst.jointcal.jointcal
@@ -55,8 +54,6 @@ def setUp(self):
     @unittest.skip('jointcal currently fails (may segfault) if only given one catalog!')
     def testJointcalTask_1_visits(self):
         self.config = lsst.jointcal.jointcal.JointcalConfig()
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
-        self.config.astrometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")
 
         dist_rms_relative = 0*u.arcsecond  # there is no such thing as a "relative" test for 1 catalog.
@@ -73,8 +70,6 @@ def testJointcalTask_2_visits(self):
         pa1 = None
         self.config = lsst.jointcal.jointcal.JointcalConfig()
         self.config.astrometryModel = "simple"
-        self.config.astrometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")
         self.config.doPhotometry = False
         self.jointcalStatistics.do_photometry = False
@@ -105,8 +100,6 @@ def testJointcalTask_10_visits(self):
         pa1 = None
         self.config = lsst.jointcal.jointcal.JointcalConfig()
         self.config.astrometryModel = "simple"
-        self.config.astrometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.doPhotometry = False
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")
         self.jointcalStatistics.do_photometry = False
@@ -145,7 +138,6 @@ def testJointcalTask_2_visits_no_astrometry(self):
                    }
 
         self.config = lsst.jointcal.jointcal.JointcalConfig()
-        self.config.photometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.doAstrometry = False
         self.config.photometryModel = "simpleFlux"
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")
@@ -182,7 +174,6 @@ def testJointcalTask_2_visits_no_photometry(self):
                    }
 
         self.config = lsst.jointcal.jointcal.JointcalConfig()
-        self.config.astrometryRefObjLoader.retarget(LoadAstrometryNetObjectsTask)
         self.config.astrometryModel = "simple"
         self.config.doPhotometry = False
         self.config.sourceSelector['astrometry'].badFlags.append("base_PixelFlags_flag_interpolated")