Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DM-14765: Allow validateDrp to run w/o instrument, dataset_repo_url #82

Merged
merged 2 commits into from
Jun 27, 2018
Merged

DM-14765: Allow validateDrp to run w/o instrument, dataset_repo_url #82

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
624f01c
Guess reasonable 'instrument', 'dataset_repo_url' if not passed.
wmwv Jun 12, 2018
b71c871
Fix butler-> repo error.
wmwv Jun 26, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
36 changes: 28 additions & 8 deletions python/lsst/validate/drp/validate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@ def run(repo_or_json, metrics=None,


def runOneRepo(repo, dataIds=None, metrics=None, outputPrefix='', verbose=False,
instrument=None, dataset_repo_url=None,
metrics_package='verify_metrics', **kwargs):
"""Calculate statistics for all filters in a repo.

Expand All @@ -192,6 +193,12 @@ def runOneRepo(repo, dataIds=None, metrics=None, outputPrefix='', verbose=False,
The level of the specification to check: "design", "minimum", "stretch".
verbose : `bool`
Provide detailed output.
instrument : `str`
Name of the instrument. If None will be extracted from the Butler mapper.
dataset_repo_url : `str`
Location of the dataset used. If None will be set to the path of the repo.
metrics_package : `string`
Name of the metrics package to be used in the Jobs created.

Notes
-----
Expand All @@ -202,6 +209,22 @@ def runOneRepo(repo, dataIds=None, metrics=None, outputPrefix='', verbose=False,
The filter name is added to this prefix. If the filter name has spaces,
there will be annoyance and sadness as those spaces will appear in the filenames.
"""
from lsst.daf.persistence import Butler
if instrument is None:
instrument = extract_instrument_from_repo(repo)
if dataset_repo_url is None:
dataset_repo_url = repo

def extract_instrument_from_repo(repo):
"""Extract the last part of the mapper name from a Butler repo.
'lsst.obs.lsstSim.lsstSimMapper.LsstSimMapper' -> 'lsstSim'
'lsst.obs.cfht.megacamMapper.MegacamMapper' -> 'cfht'
'lsst.obs.decam.decamMapper.DecamMapper' -> 'decam'
'lsst.obs.hsc.hscMapper.HscMapper' -> 'hsc'
"""
mapper_class = Butler.getMapperClass(repo)
instrument = mapper_class.split('.')[2]
return instrument
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For what it's worth, we have (not uniformly unfortunately) been using all upper case for instrument names. Maybe you should return instrument.upper().

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd like to keep with the simpler beahvior.
I'm not trying to replicate the SQuaSH behavior.
For use with SQuaSH, the caller can specify the instrument name.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I didn't understand that 'instrument' was required now because the 'verify_metrics' are now only specified for defined instruments. And thus 'instrument' must be something that matches to something specified in a metrics YAML file. That's why it matters that it's upper-case.


allFilters = set([d['filter'] for d in dataIds])

Expand All @@ -216,6 +239,8 @@ def runOneRepo(repo, dataIds=None, metrics=None, outputPrefix='', verbose=False,
job = runOneFilter(repo, theseVisitDataIds, metrics,
outputPrefix=thisOutputPrefix,
verbose=verbose, filterName=filterName,
instrument=instrument,
dataset_repo_url=dataset_repo_url,
metrics_package=metrics_package, **kwargs)
jobs[filterName] = job

Expand All @@ -225,7 +250,9 @@ def runOneRepo(repo, dataIds=None, metrics=None, outputPrefix='', verbose=False,
def runOneFilter(repo, visitDataIds, metrics, brightSnr=100,
makeJson=True, filterName=None, outputPrefix='',
useJointCal=False, skipTEx=False, verbose=False,
metrics_package='verify_metrics', **kwargs):
metrics_package='verify_metrics',
instrument='Unknown', dataset_repo_url='./',
**kwargs):
"""Main executable for the case where there is just one filter.

Plot files and JSON files are generated in the local directory
Expand Down Expand Up @@ -264,12 +291,6 @@ def runOneFilter(repo, visitDataIds, metrics, brightSnr=100,
verbose : bool, optional
Output additional information on the analysis steps.
"""
try:
instrument = kwargs['instrument']
dataset_repo_url = kwargs['dataset_repo_url']
except KeyError:
raise ValueError("Instrument name and input dataset URL must be set in config file")

job = Job.load_metrics_package(meta={'instrument': instrument,
'filter_name': filterName,
'dataset_repo_url': dataset_repo_url},
Expand All @@ -281,7 +302,6 @@ def runOneFilter(repo, visitDataIds, metrics, brightSnr=100,
skipTEx=skipTEx)

photomModel = build_photometric_error_model(matchedDataset)

astromModel = build_astrometric_error_model(matchedDataset)

linkedBlobs = [matchedDataset, photomModel, astromModel]
Expand Down