Skip to content

Commit

Permalink
Remove conform=False in datastore tests since we do now conform
Browse files Browse the repository at this point in the history
  • Loading branch information
@timj
timj committed Jun 15, 2023
1 parent b2ee4b4 commit 8e43c55
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 45 deletions.
59 changes: 27 additions & 32 deletions tests/test_datastore.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,15 +151,15 @@ def testConfigurationValidation(self) -> None:

dimensions = self.universe.extract(("visit", "physical_filter"))
dataId = dict({"instrument": "dummy", "visit": 52, "physical_filter": "V", "band": "v"})
ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
ref = self.makeDatasetRef("metric", dimensions, sc, dataId)
datastore.validateConfiguration([ref])

def testParameterValidation(self) -> None:
"""Check that parameters are validated"""
sc = self.storageClassFactory.getStorageClass("ThingOne")
dimensions = self.universe.extract(("visit", "physical_filter"))
dataId = dict({"instrument": "dummy", "visit": 52, "physical_filter": "V", "band": "v"})
ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
ref = self.makeDatasetRef("metric", dimensions, sc, dataId)
datastore = self.makeDatastore()
data = {1: 2, 3: 4}
datastore.put(data, ref)
Expand All @@ -183,8 +183,8 @@ def testBasicPutGet(self) -> None:
dataId2 = dict({"instrument": "dummy", "visit": 53, "physical_filter": "V", "band": "v"})

for sc in storageClasses:
ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
ref2 = self.makeDatasetRef("metric", dimensions, sc, dataId2, conform=False)
ref = self.makeDatasetRef("metric", dimensions, sc, dataId)
ref2 = self.makeDatasetRef("metric", dimensions, sc, dataId2)

# Make sure that using getManyURIs without predicting before the
# dataset has been put raises.
Expand Down Expand Up @@ -242,7 +242,7 @@ def testBasicPutGet(self) -> None:
# get it back as None
metricsNone = makeExampleMetrics(use_none=True)
dataIdNone = {"instrument": "dummy", "visit": 54, "physical_filter": "V", "band": "v"}
refNone = self.makeDatasetRef("metric", dimensions, sc, dataIdNone, conform=False)
refNone = self.makeDatasetRef("metric", dimensions, sc, dataIdNone)
datastore.put(metricsNone, refNone)

comp = "data"
Expand Down Expand Up @@ -302,7 +302,7 @@ def testTrustGetRequest(self) -> None:

dataId = dict({"instrument": "dummy", "visit": 52 + i, "physical_filter": "V", "band": "v"})

ref = self.makeDatasetRef(datasetTypeName, dimensions, sc, dataId, conform=False)
ref = self.makeDatasetRef(datasetTypeName, dimensions, sc, dataId)
datastore.put(metrics, ref)

# Does it exist?
Expand Down Expand Up @@ -454,7 +454,7 @@ def testDisassembly(self) -> None:
# Create a different dataset type each time round
# so that a test failure in this subtest does not trigger
# a cascade of tests because of file clashes
ref = self.makeDatasetRef(f"metric_comp_{i}", dimensions, sc, dataId, conform=False)
ref = self.makeDatasetRef(f"metric_comp_{i}", dimensions, sc, dataId)

disassembled = sc.name not in {"StructuredData", "StructuredCompositeReadCompNoDisassembly"}

Expand Down Expand Up @@ -503,7 +503,7 @@ def prepDeleteTest(self, n_refs: int = 1) -> tuple[Datastore, tuple[DatasetRef,
refs = []
for i in range(n_refs):
dataId = {"instrument": "dummy", "visit": 638 + i, "physical_filter": "U", "band": "u"}
ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
ref = self.makeDatasetRef("metric", dimensions, sc, dataId)
datastore.put(metrics, ref)

# Does it exist?
Expand Down Expand Up @@ -566,7 +566,7 @@ def testTransfer(self) -> None:
dataId = dict({"instrument": "dummy", "visit": 2048, "physical_filter": "Uprime", "band": "u"})

sc = self.storageClassFactory.getStorageClass("StructuredData")
ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
ref = self.makeDatasetRef("metric", dimensions, sc, dataId)

inputDatastore = self.makeDatastore("test_input_datastore")
outputDatastore = self.makeDatastore("test_output_datastore")
Expand All @@ -588,7 +588,7 @@ def testBasicTransaction(self) -> None:
]
data = [
(
self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False),
self.makeDatasetRef("metric", dimensions, storageClass, dataId),
makeExampleMetrics(),
)
for dataId in dataIds
Expand Down Expand Up @@ -631,16 +631,16 @@ def testNestedTransaction(self) -> None:
metrics = makeExampleMetrics()

dataId = dict({"instrument": "dummy", "visit": 0, "physical_filter": "V", "band": "v"})
refBefore = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False)
refBefore = self.makeDatasetRef("metric", dimensions, storageClass, dataId)
datastore.put(metrics, refBefore)
with self.assertRaises(TransactionTestError):
with datastore.transaction():
dataId = dict({"instrument": "dummy", "visit": 1, "physical_filter": "V", "band": "v"})
refOuter = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False)
refOuter = self.makeDatasetRef("metric", dimensions, storageClass, dataId)
datastore.put(metrics, refOuter)
with datastore.transaction():
dataId = dict({"instrument": "dummy", "visit": 2, "physical_filter": "V", "band": "v"})
refInner = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False)
refInner = self.makeDatasetRef("metric", dimensions, storageClass, dataId)
datastore.put(metrics, refInner)
# All datasets should exist
for ref in (refBefore, refOuter, refInner):
Expand All @@ -662,7 +662,7 @@ def _prepareIngestTest(self) -> tuple[MetricsExample, DatasetRef]:
dimensions = self.universe.extract(("visit", "physical_filter"))
metrics = makeExampleMetrics()
dataId = dict({"instrument": "dummy", "visit": 0, "physical_filter": "V", "band": "v"})
ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False)
ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId)
return metrics, ref

def runIngestTest(
Expand Down Expand Up @@ -817,7 +817,7 @@ def _populate_export_datastore(self, name: str) -> tuple[Datastore, list[Dataset
refs = []
for visit in (2048, 2049, 2050):
dataId = {"instrument": "dummy", "visit": visit, "physical_filter": "Uprime", "band": "u"}
ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
ref = self.makeDatasetRef("metric", dimensions, sc, dataId)
datastore.put(metrics, ref)
refs.append(ref)
return datastore, refs
Expand Down Expand Up @@ -878,7 +878,7 @@ def testExport(self) -> None:
sc = self.storageClassFactory.getStorageClass("ThingOne")
dimensions = self.universe.extract(("visit", "physical_filter"))
dataId = dict({"instrument": "dummy", "visit": 52, "physical_filter": "V", "band": "v"})
ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
ref = self.makeDatasetRef("metric", dimensions, sc, dataId)
with self.assertRaises(FileNotFoundError):
list(datastore.export(refs + [ref], transfer=None))

Expand Down Expand Up @@ -968,7 +968,7 @@ def testAtomicWrite(self) -> None:
metrics = makeExampleMetrics()

dataId = dict({"instrument": "dummy", "visit": 0, "physical_filter": "V", "band": "v"})
ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False)
ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId)

with self.assertLogs("lsst.resources", "DEBUG") as cm:
datastore.put(metrics, ref)
Expand Down Expand Up @@ -996,8 +996,8 @@ def testCanNotDeterminePutFormatterLocation(self) -> None:
dimensions = self.universe.extract(("visit", "physical_filter"))
dataId = dict({"instrument": "dummy", "visit": 52, "physical_filter": "V", "band": "v"})

ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False)
compRef = self.makeDatasetRef("metric", dimensions, compositeStorageClass, dataId, conform=False)
ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId)
compRef = self.makeDatasetRef("metric", dimensions, compositeStorageClass, dataId)

def raiser(ref):
raise DatasetTypeNotSupportedError()
Expand Down Expand Up @@ -1030,7 +1030,7 @@ def testChecksum(self) -> None:
metrics = makeExampleMetrics()

dataId = dict({"instrument": "dummy", "visit": 0, "physical_filter": "V", "band": "v"})
ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False)
ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId)

# Configuration should have disabled checksum calculation
datastore.put(metrics, ref)
Expand Down Expand Up @@ -1109,7 +1109,7 @@ def testCleanup(self) -> None:
dimensions = self.universe.extract(("visit", "physical_filter"))
dataId = dict({"instrument": "dummy", "visit": 52, "physical_filter": "V", "band": "v"})

ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False)
ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId)

# Determine where the file will end up (we assume Formatters use
# the same file extension)
Expand Down Expand Up @@ -1202,7 +1202,7 @@ def testConstraints(self) -> None:
testfile = testfile_j if sc.name.endswith("Json") else testfile_y

with self.subTest(datasetTypeName=datasetTypeName, storageClass=sc.name, file=testfile.name):
ref = self.makeDatasetRef(datasetTypeName, dimensions, sc, dataId, conform=False)
ref = self.makeDatasetRef(datasetTypeName, dimensions, sc, dataId)
if accepted:
datastore.put(metrics, ref)
self.assertTrue(datastore.exists(ref))
Expand Down Expand Up @@ -1301,7 +1301,7 @@ def testConstraints(self) -> None:
testfile = testfile_j if sc.name.endswith("Json") else testfile_y

with self.subTest(datasetTypeName=typeName, dataId=dataId, sc=sc.name):
ref = self.makeDatasetRef(typeName, dimensions, sc, dataId, conform=False)
ref = self.makeDatasetRef(typeName, dimensions, sc, dataId)
if any(accept):
datastore.put(metrics, ref)
self.assertTrue(datastore.exists(ref))
Expand Down Expand Up @@ -1374,10 +1374,7 @@ def setUp(self) -> None:

# Create list of refs and list of temporary files
n_datasets = 10
self.refs = [
self.makeDatasetRef(f"metric{n}", dimensions, sc, dataId, conform=False)
for n in range(n_datasets)
]
self.refs = [self.makeDatasetRef(f"metric{n}", dimensions, sc, dataId) for n in range(n_datasets)]

root_uri = ResourcePath(self.root, forceDirectory=True)
self.files = [root_uri.join(f"file{n}.txt") for n in range(n_datasets)]
Expand All @@ -1388,9 +1385,7 @@ def setUp(self) -> None:

# Create some composite refs with component files.
sc = self.storageClassFactory.getStorageClass("StructuredData")
self.composite_refs = [
self.makeDatasetRef(f"composite{n}", dimensions, sc, dataId, conform=False) for n in range(3)
]
self.composite_refs = [self.makeDatasetRef(f"composite{n}", dimensions, sc, dataId) for n in range(3)]
self.comp_files = []
self.comp_refs = []
for n, ref in enumerate(self.composite_refs):
Expand Down Expand Up @@ -1774,7 +1769,7 @@ class StoredFileInfoTestCase(DatasetTestHelper, unittest.TestCase):

def test_StoredFileInfo(self) -> None:
storageClass = self.storageClassFactory.getStorageClass("StructuredDataDict")
ref = self.makeDatasetRef("metric", DimensionUniverse().extract(()), storageClass, {}, conform=False)
ref = self.makeDatasetRef("metric", DimensionUniverse().extract(()), storageClass, {})

record = dict(
storage_class="StructuredDataDict",
Expand All @@ -1790,7 +1785,7 @@ def test_StoredFileInfo(self) -> None:
self.assertEqual(info.dataset_id, ref.id)
self.assertEqual(info.to_record(), record)

ref2 = self.makeDatasetRef("metric", DimensionUniverse().extract(()), storageClass, {}, conform=False)
ref2 = self.makeDatasetRef("metric", DimensionUniverse().extract(()), storageClass, {})
rebased = info.rebase(ref2)
self.assertEqual(rebased.dataset_id, ref2.id)
self.assertEqual(rebased.rebase(ref), info)
Expand Down
26 changes: 13 additions & 13 deletions tests/test_templates.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,58 +83,58 @@ def testBasic(self):
self.assertTemplate(
tmplstr,
"run2/calexp/00052/Most_Amazing_U_Filter_Ever",
self.makeDatasetRef("calexp", conform=False),
self.makeDatasetRef("calexp"),
)
tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail"
self.assertTemplate(
tmplstr,
"run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail",
self.makeDatasetRef("calexp", conform=False),
self.makeDatasetRef("calexp"),
)

tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
self.assertTemplate(
tmplstr,
"run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2",
self.makeDatasetRef("calexp", conform=False),
self.makeDatasetRef("calexp"),
)
self.assertTemplate(
tmplstr,
"run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
self.makeDatasetRef("calexp", run="run/2", conform=False),
self.makeDatasetRef("calexp", run="run/2"),
)

# Check that the id is sufficient without any other information.
self.assertTemplate("{id}", "1", self.makeDatasetRef("calexp", run="run2", conform=False))
self.assertTemplate("{id}", "1", self.makeDatasetRef("calexp", run="run2"))

self.assertTemplate("{run}/{id}", "run2/1", self.makeDatasetRef("calexp", run="run2", conform=False))
self.assertTemplate("{run}/{id}", "run2/1", self.makeDatasetRef("calexp", run="run2"))

self.assertTemplate(
"fixed/{id}",
"fixed/1",
self.makeDatasetRef("calexp", run="run2", conform=False),
self.makeDatasetRef("calexp", run="run2"),
)

self.assertTemplate(
"fixed/{id}_{physical_filter}",
"fixed/1_Most_Amazing_U_Filter_Ever",
self.makeDatasetRef("calexp", run="run2", conform=False),
self.makeDatasetRef("calexp", run="run2"),
)

# Retain any "/" in run
tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
self.assertTemplate(
tmplstr,
"run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
self.makeDatasetRef("calexp", run="run/2", conform=False),
self.makeDatasetRef("calexp", run="run/2"),
)

# Check that "." are replaced in the file basename, but not directory.
dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"}
self.assertTemplate(
tmplstr,
"run.2/calexp/00052/g_10-trail-run_2",
self.makeDatasetRef("calexp", run="run.2", dataId=dataId, conform=False),
self.makeDatasetRef("calexp", run="run.2", dataId=dataId),
)

with self.assertRaises(FileTemplateValidationError):
Expand Down Expand Up @@ -165,16 +165,16 @@ def testNoRecord(self):

def testOptional(self):
"""Optional units in templates."""
ref = self.makeDatasetRef("calexp", conform=False)
ref = self.makeDatasetRef("calexp")
tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}"
self.assertTemplate(
tmplstr,
"run2/calexp/v00052_fMost_Amazing_U_Filter_Ever",
self.makeDatasetRef("calexp", conform=False),
self.makeDatasetRef("calexp"),
)

du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"}
self.assertTemplate(tmplstr, "run2/calexpT/v00048", self.makeDatasetRef("calexpT", du, conform=False))
self.assertTemplate(tmplstr, "run2/calexpT/v00048", self.makeDatasetRef("calexpT", du))

# Ensure that this returns a relative path even if the first field
# is optional
Expand Down

0 comments on commit 8e43c55

Please sign in to comment.