DM-13851: Speed up ap_verify unit tests

SConstruct

@@ -1,4 +1,4 @@
 # -*- python -*-
 from lsst.sconsUtils import scripts
-scripts.BasicSConstruct("ap_verify")
+scripts.BasicSConstruct("ap_verify", disableCc=True)
 

python/lsst/ap/verify/ingestion.py

@@ -195,12 +195,12 @@ def _ingestRaws(self, dataset, workspace):
             self.log.info("Ingesting raw images...")
             dataFiles = _findMatchingFiles(dataset.rawLocation, self.config.dataFiles)
             if dataFiles:
-                self._doIngest(workspace.dataRepo, dataFiles, self.config.dataBadFiles)
+                self._doIngestRaws(workspace.dataRepo, dataFiles, self.config.dataBadFiles)
                 self.log.info("Images are now ingested in {0}".format(workspace.dataRepo))
             else:
                 raise RuntimeError("No raw files found at %s." % dataset.rawLocation)
 
-    def _doIngest(self, repo, dataFiles, badFiles):
+    def _doIngestRaws(self, repo, dataFiles, badFiles):
         """Ingest raw images into a repository.
 
         ``repo`` shall be populated with *links* to ``dataFiles``.
@@ -350,7 +350,7 @@ def _doIngestDefects(self, repo, calibRepo, defectTarball):
         # TODO: clean up implementation after DM-5467 resolved
         defectDir = os.path.join(calibRepo, "defects")
         with tarfile.open(defectTarball, "r") as opened:
-            if opened.getNames():
+            if opened.getnames():
                 pathlib.Path(defectDir).mkdir(parents=True, exist_ok=True)
                 opened.extractall(defectDir)
             else:

python/lsst/ap/verify/testUtils.py

@@ -0,0 +1,62 @@
+#
+# This file is part of ap_verify.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""Common code for ap_verify unit tests.
+"""
+
+import unittest
+
+import lsst.utils.tests
+
+import lsst.pex.exceptions as pexExcept
+from lsst.ap.verify.config import Config
+
+
+class DataTestCase(lsst.utils.tests.TestCase):
+    """Unit test class for tests that need to use the Dataset framework.
+
+    Unit tests based on this class will search for a designated dataset
+    (`testDataset`), and skip all tests if the dataset is not available.
+
+    Subclasses must call `DataTestCase.setUpClass()` if they override
+    ``setUpClass`` themselves.
+    """
+
+    testDataset = 'ap_verify_testdata'
+    """The EUPS package name of the dataset to use for testing (`str`).
+    """
+    datasetKey = 'test'
+    """The ap_verify dataset name that would be used on the command line (`str`).
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        try:
+            lsst.utils.getPackageDir(cls.testDataset)
+        except pexExcept.NotFoundError:
+            raise unittest.SkipTest(cls.testDataset + ' not set up')
+
+        # Hack the config for testing purposes
+        # Note that Config.instance is supposed to be immutable, so, depending on initialization order,
+        # this modification may cause other tests to see inconsistent config values
+        Config.instance._allInfo['datasets.' + cls.datasetKey] = cls.testDataset

tests/test_args.py

@@ -26,9 +26,11 @@
 
 import lsst.utils.tests
 import lsst.ap.verify.ap_verify as ap_verify
+import lsst.ap.verify.testUtils
 
 
-class CommandLineTestSuite(lsst.utils.tests.TestCase):
+class CommandLineTestSuite(lsst.ap.verify.testUtils.DataTestCase):
+    # DataTestCase's test dataset is needed for successful parsing of the --dataset argument
 
     def _parseString(self, commandLine, parser=None):
         """Tokenize and parse a command line string.
@@ -54,21 +56,21 @@ def _parseString(self, commandLine, parser=None):
     def testMissingMain(self):
         """Verify that a command line consisting missing required arguments is rejected.
         """
-        args = '--dataset HiTS2015 --output tests/output/foo'
+        args = '--dataset %s --output tests/output/foo' % CommandLineTestSuite.datasetKey
         with self.assertRaises(SystemExit):
             self._parseString(args)
 
     def testMissingIngest(self):
         """Verify that a command line consisting missing required arguments is rejected.
         """
-        args = '--dataset HiTS2015'
+        args = '--dataset %s' % CommandLineTestSuite.datasetKey
         with self.assertRaises(SystemExit):
             self._parseString(args, ap_verify._IngestOnlyParser())
 
     def testMinimumMain(self):
         """Verify that a command line consisting only of required arguments parses correctly.
         """
-        args = '--dataset HiTS2015 --output tests/output/foo --id "visit=54123"'
+        args = '--dataset %s --output tests/output/foo --id "visit=54123"' % CommandLineTestSuite.datasetKey
         parsed = self._parseString(args)
         self.assertIn('dataset', dir(parsed))
         self.assertIn('output', dir(parsed))
@@ -77,30 +79,31 @@ def testMinimumMain(self):
     def testMinimumIngest(self):
         """Verify that a command line consisting only of required arguments parses correctly.
         """
-        args = '--dataset HiTS2015 --output tests/output/foo'
+        args = '--dataset %s --output tests/output/foo' % CommandLineTestSuite.datasetKey
         parsed = self._parseString(args, ap_verify._IngestOnlyParser())
         self.assertIn('dataset', dir(parsed))
         self.assertIn('output', dir(parsed))
 
     def testRerun(self):
         """Verify that a command line with reruns is handled correctly.
         """
-        args = '--dataset HiTS2015 --rerun me --id "visit=54123"'
+        args = '--dataset %s --rerun me --id "visit=54123"' % CommandLineTestSuite.datasetKey
         parsed = self._parseString(args)
         out = ap_verify._getOutputDir('non_lsst_repo/', parsed.output, parsed.rerun)
         self.assertEqual(out, 'non_lsst_repo/rerun/me')
 
     def testRerunInput(self):
         """Verify that a command line trying to redirect input is rejected.
         """
-        args = '--dataset HiTS2015 --rerun from:to --id "visit=54123"'
+        args = '--dataset %s --rerun from:to --id "visit=54123"' % CommandLineTestSuite.datasetKey
         with self.assertRaises(SystemExit):
             self._parseString(args)
 
     def testTwoOutputs(self):
         """Verify that a command line with both --output and --rerun is rejected.
         """
-        args = '--dataset HiTS2015 --output tests/output/foo --rerun me --id "visit=54123"'
+        args = '--dataset %s --output tests/output/foo --rerun me --id "visit=54123"' \
+            % CommandLineTestSuite.datasetKey
         with self.assertRaises(SystemExit):
             self._parseString(args)
 
@@ -114,14 +117,16 @@ def testBadDataset(self):
     def testBadKeyMain(self):
         """Verify that a command line with unsupported arguments is rejected.
         """
-        args = '--dataset HiTS2015 --output tests/output/foo --id "visit=54123" --clobber'
+        args = '--dataset %s --output tests/output/foo --id "visit=54123" --clobber' \
+            % CommandLineTestSuite.datasetKey
         with self.assertRaises(SystemExit):
             self._parseString(args)
 
     def testBadKeyIngest(self):
         """Verify that a command line with unsupported arguments is rejected.
         """
-        args = '--dataset HiTS2015 --output tests/output/foo --id "visit=54123"'
+        args = '--dataset %s --output tests/output/foo --id "visit=54123"' \
+            % CommandLineTestSuite.datasetKey
         with self.assertRaises(SystemExit):
             self._parseString(args, ap_verify._IngestOnlyParser())
 

tests/test_association.py

@@ -22,24 +22,20 @@
 #
 
 import unittest
+from unittest.mock import NonCallableMock
 
 import astropy.units as u
 import numpy as np
 import os
-import shutil
 import sqlite3
-import tempfile
 
 import lsst.daf.persistence as dafPersist
 import lsst.afw.geom as afwGeom
 import lsst.afw.table as afwTable
 from lsst.ap.association import \
     make_minimal_dia_source_schema, \
     make_minimal_dia_object_schema, \
-    AssociationDBSqliteTask, \
-    AssociationDBSqliteConfig, \
     AssociationTask
-import lsst.obs.test as obsTest
 import lsst.pipe.base as pipeBase
 import lsst.utils.tests
 from lsst.verify import Measurement
@@ -119,14 +115,6 @@ def setUp(self):
         self.assocTask = AssociationTask()
 
         # Create a empty butler repository and put data in it.
-        self.testDir = tempfile.mkdtemp(
-            dir=ROOT, prefix="TestAssocMeasurements-")
-        outputRepoArgs = dafPersist.RepositoryArgs(
-            root=os.path.join(self.testDir, 'repoA'),
-            mapper=obsTest.TestMapper,
-            mode='rw')
-        self.butler = dafPersist.Butler(
-            outputs=outputRepoArgs)
         self.numTestSciSources = 10
         self.numTestDiaSources = 5
         testSources = createTestPoints(
@@ -135,42 +123,34 @@ def setUp(self):
         testDiaSources = createTestPoints(
             pointLocsDeg=[[idx, idx] for idx in
                           range(self.numTestDiaSources)])
-        self.butler.put(obj=testSources,
-                        datasetType='src',
-                        dataId=dataIdDict)
-        self.butler.put(obj=testDiaSources,
-                        datasetType='deepDiff_diaSrc',
-                        dataId=dataIdDict)
-
-        (self.tmpFile, self.dbFile) = tempfile.mkstemp(
-            dir=os.path.dirname(__file__))
-        assocDbConfig = AssociationDBSqliteConfig()
-        assocDbConfig.db_name = self.dbFile
-        assocDbConfig.filter_names = ['r']
-        assocDb = AssociationDBSqliteTask(config=assocDbConfig)
-        assocDb.create_tables()
+
+        # Fake Butler to avoid initialization and I/O overhead
+        def mockGet(datasetType, dataId=None):
+            """An emulator for `lsst.daf.persistence.Butler.get` that can only handle test data.
+            """
+            # Check whether dataIdDict is a subset of dataId
+            if dataIdDict.items() <= dataId.items():
+                if datasetType == 'src':
+                    return testSources
+                elif datasetType == 'deepDiff_diaSrc':
+                    return testDiaSources
+            raise dafPersist.NoResults("Dataset not found:", datasetType, dataId)
+        self.butler = NonCallableMock(spec=dafPersist.Butler, get=mockGet)
 
         self.numTestDiaObjects = 5
-        diaObjects = createTestPoints(
+        self.diaObjects = createTestPoints(
             pointLocsDeg=[[idx, idx] for idx in
                           range(self.numTestDiaObjects)],
             schema=make_minimal_dia_object_schema(['r']))
-        for diaObject in diaObjects:
+        for diaObject in self.diaObjects:
             diaObject['nDiaSources'] = 1
-        assocDb.store_dia_objects(diaObjects, True)
-        assocDb.close()
 
     def tearDown(self):
         del self.assocTask
 
-        if os.path.exists(self.testDir):
-            shutil.rmtree(self.testDir)
         if hasattr(self, "butler"):
             del self.butler
 
-        del self.tmpFile
-        os.remove(self.dbFile)
-
     def testValidFromMetadata(self):
         """Verify that association information can be recovered from metadata.
         """
@@ -230,7 +210,7 @@ def testValidFromButler(self):
         self.assertEqual(
             meas.metric_name,
             lsst.verify.Name(metric='ip_diffim.numSciSrc'))
-        self.assertEqual(meas.quantity, 10 * u.count)
+        self.assertEqual(meas.quantity, self.numTestSciSources * u.count)
 
         meas = measureFractionDiaSourcesToSciSources(
             self.butler,
@@ -240,12 +220,13 @@ def testValidFromButler(self):
         self.assertEqual(
             meas.metric_name,
             lsst.verify.Name(metric='ip_diffim.fracDiaSrcToSciSrc'))
-        # We put in half the number of DIASources as detected sources.
-        self.assertEqual(meas.quantity, 0.5 * u.dimensionless_unscaled)
+        self.assertEqual(meas.quantity,
+                         self.numTestDiaSources / self.numTestSciSources * u.dimensionless_unscaled)
 
     def testValidFromSqlite(self):
-        conn = sqlite3.connect(self.dbFile)
-        cursor = conn.cursor()
+        # Fake DB handle to avoid DB initialization overhead
+        cursor = NonCallableMock(spec=sqlite3.Cursor)
+        cursor.fetchall.return_value = [(len(self.diaObjects),)]
 
         meas = measureTotalUnassociatedDiaObjects(
             cursor,
@@ -255,7 +236,7 @@ def testValidFromSqlite(self):
             meas.metric_name,
             lsst.verify.Name(
                 metric='association.numTotalUnassociatedDiaObjects'))
-        self.assertEqual(meas.quantity, 5 * u.count)
+        self.assertEqual(meas.quantity, self.numTestDiaObjects * u.count)
 
     def testNoButlerData(self):
         """ Test attempting to create a measurement with data that the butler
@@ -298,17 +279,6 @@ def testMetadataNotCreated(self):
             "association.numNewDIAObjects")
         self.assertIsNone(meas)
 
-    def testInvalidDb(self):
-        """ Test that the measurement raises the correct error when given an
-        improper database.
-        """
-        conn = sqlite3.connect(":memory:")
-        cursor = conn.cursor()
-        with self.assertRaises(sqlite3.OperationalError):
-            measureTotalUnassociatedDiaObjects(
-                cursor,
-                metricName='association.numTotalUnassociatedDiaObjects')
-
     def testNoMetric(self):
         """Verify that trying to measure a nonexistent metric fails.
         """
@@ -337,8 +307,9 @@ def testNoMetric(self):
                 self.butler, dataId=dataIdDict,
                 metricName='foo.bar.FooBar')
 
-        conn = sqlite3.connect(self.dbFile)
-        cursor = conn.cursor()
+        # Fake DB handle to avoid DB initialization overhead
+        cursor = NonCallableMock(spec=sqlite3.Cursor)
+        cursor.fetchall.return_value = [(0,)]
         with self.assertRaises(TypeError):
             measureTotalUnassociatedDiaObjects(
                 cursor, metricName='foo.bar.FooBar')

tests/test_dataset.py

@@ -27,28 +27,18 @@
 import unittest
 
 import lsst.utils.tests
-import lsst.pex.exceptions as pexExcept
-from lsst.ap.verify.config import Config
 from lsst.ap.verify.dataset import Dataset
+from lsst.ap.verify.testUtils import DataTestCase
 
 
-class DatasetTestSuite(lsst.utils.tests.TestCase):
+class DatasetTestSuite(DataTestCase):
 
     @classmethod
     def setUpClass(cls):
-        cls.testDataset = 'ap_verify_testdata'
-        cls.datasetKey = 'test'
+        super().setUpClass()
+
         cls.obsPackage = 'obs_test'
         cls.camera = 'test'
-        try:
-            lsst.utils.getPackageDir(cls.testDataset)
-        except pexExcept.NotFoundError:
-            raise unittest.SkipTest(cls.testDataset + ' not set up')
-
-        # Hack the config for testing purposes
-        # Note that Config.instance is supposed to be immutable, so, depending on initialization order,
-        # this modification may cause other tests to see inconsistent config values
-        Config.instance._allInfo['datasets.' + cls.datasetKey] = cls.testDataset
 
     def setUp(self):
         self._testbed = Dataset(DatasetTestSuite.datasetKey)