Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DM-28355: Add failFast config option for ingest-raws #353

Merged
merged 1 commit into from
Feb 4, 2021
Merged

DM-28355: Add failFast config option for ingest-raws #353

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
19 changes: 17 additions & 2 deletions python/lsst/obs/base/ingest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
FileDataset,
Formatter,
)
from lsst.pex.config import Config, ChoiceField
from lsst.pex.config import Config, ChoiceField, Field
from lsst.pipe.base import Task

from ._instrument import Instrument, makeExposureRecordFromObsInfo
Expand Down Expand Up @@ -156,6 +156,12 @@ def makeTransferChoiceField(doc="How to transfer files (None for no transfer).",

class RawIngestConfig(Config):
transfer = makeTransferChoiceField()
failFast = Field(
dtype=bool,
default=False,
doc="If True, stop ingest as soon as any problem is encountered with any file. "
"Otherwise problems files will be skipped and logged and a report issued at completion.",
)


class RawIngestTask(Task):
Expand Down Expand Up @@ -243,16 +249,21 @@ def extractMetadata(self, filename: str) -> RawFileData:
datasets = []
FormatterClass = Formatter
instrument = None
if self.config.failFast:
raise RuntimeError(f"Problem extracting metadata from file {filename}") from e
else:
self.log.debug("Extracted metadata from file %s", filename)
# The data model currently assumes that whilst multiple datasets
# can be associated with a single file, they must all share the
# same formatter.
try:
instrument = Instrument.fromName(datasets[0].dataId["instrument"], self.butler.registry)
except LookupError:
except LookupError as e:
self.log.warning("Instrument %s for file %s not known to registry",
datasets[0].dataId["instrument"], filename)
if self.config.failFast:
raise RuntimeError(f"Instrument {datasets[0].dataId['instrument']} for"
f" file {filename} not known to registry") from e
datasets = []
FormatterClass = Formatter
instrument = None
Expand Down Expand Up @@ -544,6 +555,8 @@ def run(self, files, *, pool: Optional[Pool] = None, processes: int = 1, run: Op
n_exposures_failed += 1
self.log.warning("Exposure %s:%s could not be registered: %s",
exposure.record.instrument, exposure.record.obs_id, e)
if self.config.failFast:
raise e
continue

# Override default run if nothing specified explicitly
Expand All @@ -563,6 +576,8 @@ def run(self, files, *, pool: Optional[Pool] = None, processes: int = 1, run: Op
self.log.warning("Failed to ingest the following for reason: %s", e)
for f in exposure.files:
self.log.warning("- %s", f.filename)
if self.config.failFast:
raise e
continue

# Success for this exposure
Expand Down