Skip to content

Commit

Permalink
Use single butler for each test class
Browse files Browse the repository at this point in the history
Register instrument takes a while for some instruments
and it always does the same thing and can be reused for
each test.
  • Loading branch information
timj committed Oct 16, 2020
1 parent 6f4a0ff commit 10ebb70
Showing 1 changed file with 24 additions and 19 deletions.
43 changes: 24 additions & 19 deletions python/lsst/obs/base/ingest_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import unittest
import os
import shutil
import random

import lsst.afw.cameraGeom
from lsst.daf.butler import Butler
Expand Down Expand Up @@ -77,10 +78,6 @@ class IngestTestBase(metaclass=abc.ABCMeta):
observations).
"""

outputRun = "raw"
"""The name of the output run to use in tests.
"""

@property
@abc.abstractmethod
def instrumentClassName(self):
Expand Down Expand Up @@ -109,17 +106,23 @@ def instrumentName(self):
"""
return self.instrumentClass.getName()

def setUp(self):
@classmethod
def setUpClass(cls):
# Use a temporary working directory
self.root = tempfile.mkdtemp(dir=self.ingestDir)
self._createRepo()
cls.root = tempfile.mkdtemp(dir=cls.ingestDir)
cls._createRepo()

# Register the instrument and its static metadata
self._registerInstrument()
cls._registerInstrument()

def tearDown(self):
if os.path.exists(self.root):
shutil.rmtree(self.root, ignore_errors=True)
def setUp(self):
# Want a unique run name per test
self.outputRun = "raw_ingest_" + str(random.randrange(1_000_000_000))

@classmethod
def tearDownClass(cls):
if os.path.exists(cls.root):
shutil.rmtree(cls.root, ignore_errors=True)

def verifyIngest(self, files=None, cli=False, fullCheck=False):
"""
Expand All @@ -135,7 +138,7 @@ def verifyIngest(self, files=None, cli=False, fullCheck=False):
but do not read the entire raw exposure.
"""
butler = Butler(self.root, run=self.outputRun)
datasets = butler.registry.queryDatasets("raw", collections=...)
datasets = butler.registry.queryDatasets("raw", collections=self.outputRun)
self.assertEqual(len(list(datasets)), len(self.dataIds))

# Reading all the ingested test data can be expensive. The code paths
Expand Down Expand Up @@ -178,12 +181,13 @@ def checkRepo(self, files=None):
"""
pass

def _createRepo(self):
@classmethod
def _createRepo(cls):
"""Use the Click `testing` module to call the butler command line api
to create a repository."""
runner = LogCliRunner()
result = runner.invoke(butlerCli, ["create", self.root])
self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
result = runner.invoke(butlerCli, ["create", cls.root])
assert result.exit_code == 0, f"output: {result.output} exception: {result.exception}"

def _ingestRaws(self, transfer, file=None):
"""Use the Click `testing` module to call the butler command line api
Expand All @@ -206,12 +210,13 @@ def _ingestRaws(self, transfer, file=None):
"--ingest-task", self.rawIngestTask])
self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")

def _registerInstrument(self):
@classmethod
def _registerInstrument(cls):
"""Use the Click `testing` module to call the butler command line api
to register the instrument."""
runner = LogCliRunner()
result = runner.invoke(butlerCli, ["register-instrument", self.root, self.instrumentClassName])
self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
result = runner.invoke(butlerCli, ["register-instrument", cls.root, cls.instrumentClassName])
assert result.exit_code == 0, f"output: {result.output} exception: {result.exception}"

def _writeCuratedCalibrations(self):
"""Use the Click `testing` module to call the butler command line api
Expand Down Expand Up @@ -283,7 +288,7 @@ def testWriteCuratedCalibrations(self):
# Trying to load a camera with a data ID not known to the registry
# is an error, because we can't get any temporal information.
with self.assertRaises(LookupError):
lsst.obs.base.loadCamera(butler, self.dataIds[0], collections=collection)
lsst.obs.base.loadCamera(butler, {"exposure": 0}, collections=collection)

# Ingest raws in order to get some exposure records.
self._ingestRaws(transfer="auto")
Expand Down

0 comments on commit 10ebb70

Please sign in to comment.