Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DM-24937: move implementation of remaining butler commands to script folder #246

Merged
merged 5 commits into from
May 19, 2020
Merged

DM-24937: move implementation of remaining butler commands to script folder #246

Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3435c60
use command line interface instead of private impl
n8pease May 16, 2020
17a5835
add ingest_raws to __all__
n8pease May 18, 2020
7f08ddb
move ingest_raws to commands.py
n8pease May 17, 2020
06b3301
move write_curated_calibrations to commands.py
n8pease May 17, 2020
ed3e1b3
move register_instrument to commands.py
n8pease May 17, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
74 changes: 54 additions & 20 deletions python/lsst/obs/base/ingest_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,17 +25,17 @@
__all__ = ("IngestTestBase",)

import abc
import click.testing
import tempfile
import unittest
import os
import shutil

from lsst.daf.butler import Butler
from lsst.daf.butler.script import createRepo
from lsst.daf.butler.cli.butler import cli as butlerCli
import lsst.obs.base
from lsst.utils import doImport
from .utils import getInstrument
from .script import ingestRaws, registerInstrument, writeCuratedCalibrations


class IngestTestBase(metaclass=abc.ABCMeta):
Expand Down Expand Up @@ -110,10 +110,10 @@ def instrumentName(self):
def setUp(self):
# Use a temporary working directory
self.root = tempfile.mkdtemp(dir=self.ingestDir)
createRepo(self.root)
self._createRepo()

# Register the instrument and its static metadata
registerInstrument(self.root, self.instrumentClassName)
self._registerInstrument()

def tearDown(self):
if os.path.exists(self.root):
Expand Down Expand Up @@ -160,25 +160,62 @@ def checkRepo(self, files=None):
"""
pass

def _createRepo(self):
"""Use the Click `testing` module to call the butler command line api
to create a repository."""
runner = click.testing.CliRunner()
result = runner.invoke(butlerCli, ["create", self.root])
self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")

def _ingestRaws(self, transfer):
"""Use the Click `testing` module to call the butler command line api
to ingest raws.

Parameters
----------
transfer : `str`
The external data transfer type.
"""
runner = click.testing.CliRunner()
result = runner.invoke(butlerCli, ["ingest-raws", self.root,
"--output-run", self.outputRun,
"--file", self.file,
"--transfer", transfer,
"--ingest-task", self.rawIngestTask])
self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")
Copy link
Contributor Author

@n8pease n8pease May 18, 2020 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@timj, you had mentioned to mark the script functions as private api. I thought it would be better to just use the butler command in those locations instead, and get the extra mileage on testing it. (Let me know if you disagree)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This works for me.


def _registerInstrument(self):
"""Use the Click `testing` module to call the butler command line api
to register the instrument."""
runner = click.testing.CliRunner()
result = runner.invoke(butlerCli, ["register-instrument", self.root,
"--instrument", self.instrumentClassName])
self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")

def _writeCuratedCalibrations(self):
"""Use the Click `testing` module to call the butler command line api
to write curated calibrations."""
runner = click.testing.CliRunner()
result = runner.invoke(butlerCli, ["write-curated-calibrations", self.root,
"--instrument", self.instrumentName,
"--output-run", self.outputRun])
self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}")

def testLink(self):
ingestRaws(self.root, self.outputRun, file=self.file, transfer="link",
ingest_task=self.rawIngestTask)
self._ingestRaws(transfer="link")
self.verifyIngest()

def testSymLink(self):
ingestRaws(self.root, self.outputRun, file=self.file, transfer="symlink",
ingest_task=self.rawIngestTask)
self._ingestRaws(transfer="symlink")
self.verifyIngest()

def testCopy(self):
ingestRaws(self.root, self.outputRun, file=self.file, transfer="copy",
ingest_task=self.rawIngestTask)
self._ingestRaws(transfer="copy")
self.verifyIngest()

def testHardLink(self):
try:
ingestRaws(self.root, self.outputRun, file=self.file, transfer="hardlink",
ingest_task=self.rawIngestTask)
self._ingestRaws(transfer="hardlink")
self.verifyIngest()
except PermissionError as err:
raise unittest.SkipTest("Skipping hard-link test because input data"
Expand All @@ -192,24 +229,22 @@ def testInPlace(self):
butler = Butler(self.root, run=self.outputRun)
newPath = os.path.join(butler.datastore.root, os.path.basename(self.file))
os.symlink(os.path.abspath(self.file), newPath)
ingestRaws(self.root, self.outputRun, file=newPath, transfer=None, ingest_task=self.rawIngestTask)
self._ingestRaws(transfer=None)
self.verifyIngest()

def testFailOnConflict(self):
"""Re-ingesting the same data into the repository should fail.
"""
ingestRaws(self.root, self.outputRun, file=self.file, transfer="symlink",
ingest_task=self.rawIngestTask)
self._ingestRaws(transfer="symlink")
with self.assertRaises(Exception):
ingestRaws(self.root, self.outputRun, file=self.file, transfer="symlink",
ingest_task=self.rawIngestTask)
self._ingestRaws(transfer="symlink")

def testWriteCuratedCalibrations(self):
"""Test that we can ingest the curated calibrations"""
if self.curatedCalibrationDatasetTypes is None:
raise unittest.SkipTest("Class requests disabling of writeCuratedCalibrations test")

writeCuratedCalibrations(self.root, self.instrumentName, self.outputRun)
self._writeCuratedCalibrations()

dataId = {"instrument": self.instrumentName}
butler = Butler(self.root, run=self.outputRun)
Expand All @@ -222,8 +257,7 @@ def testWriteCuratedCalibrations(self):
def testDefineVisits(self):
if self.visits is None:
self.skipTest("Expected visits were not defined.")
ingestRaws(self.root, self.outputRun, file=self.file, transfer="link",
ingest_task=self.rawIngestTask)
self._ingestRaws(transfer="link")

config = self.defineVisitsTask.ConfigClass()
butler = Butler(self.root, run=self.outputRun)
Expand Down