Skip to content

Commit

Permalink
Remove print statements
Browse files Browse the repository at this point in the history
  • Loading branch information
timj committed Apr 29, 2020
1 parent bedbc6c commit cee542b
Show file tree
Hide file tree
Showing 5 changed files with 8 additions and 29 deletions.
4 changes: 2 additions & 2 deletions python/lsst/daf/butler/_butler.py
Original file line number Diff line number Diff line change
Expand Up @@ -697,14 +697,14 @@ def getDirect(self, ref: DatasetRef, *, parameters: Optional[Dict[str, Any]] = N
if self.datastore.exists(ref):
return self.datastore.get(ref, parameters=parameters)
elif ref.isComposite() and ref.components:
print(f"%%%%%% %%%%% %%%%% %%%%% {ref} in getDirect %%% %%% %%% %%%")
# The presence of components indicates that this dataset
# was disassembled at the registry level.
# Check that we haven't got any unknown parameters
ref.datasetType.storageClass.validateParameters(parameters)
# Reconstruct the composite
usedParams = set()
components = {}
for compName, compRef in ref.components.items():
print(f"{compName}: {compRef}")
# make a dictionary of parameters containing only the subset
# supported by the StorageClass of the components
compParams = compRef.datasetType.storageClass.filterParameters(parameters)
Expand Down
22 changes: 3 additions & 19 deletions python/lsst/daf/butler/datastores/fileLikeDatastore.py
Original file line number Diff line number Diff line change
Expand Up @@ -288,12 +288,10 @@ def addStoredItemInfo(self, refs, infos):
storage_class=info.storageClass.name, component=component,
checksum=info.checksum, file_size=info.file_size)
)
print(f"Inserting opqaue record {records[-1]} for ref {ref}")
self.registry.insertOpaqueData(self._tableName, *records)

def getStoredItemInfo(self, ref):
# Docstring inherited from GenericBaseDatastore
print(f"@@@@@@@@@@@@@@@@@ Looking for itemInfo for ref {ref}")

where = {"dataset_id": ref.id}

Expand All @@ -316,7 +314,6 @@ def getStoredItemInfo(self, ref):
# Look for the dataset_id -- there might be multiple matches
# if we have disassembled the dataset.
records = list(self.registry.fetchOpaqueData(self._tableName, **where))
print(f"All the records: {records}")
if len(records) == 0:
raise KeyError(f"Unable to retrieve location associated with dataset {ref}.")

Expand All @@ -331,17 +328,14 @@ def getStoredItemInfo(self, ref):
records_by_component = {}
for r in records:
this_component = r["component"] if r["component"] else None
print(f"Found component {this_component} in records!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
records_by_component[this_component] = r

# Look for component by name else fall back to the parent
for lookup in (component, None):
if lookup in records_by_component:
record = records_by_component[lookup]
print(f";;;;;;;;;;;;;;;;;;;;;; Match record {record} with lookup {lookup}")
break
else:
print(f"Failed to find a location for {ref}")
raise KeyError(f"Unable to retrieve location for component {component} associated with "
f"dataset {ref}.")

Expand All @@ -357,7 +351,6 @@ def getStoredItemInfo(self, ref):

def getStoredItemsInfo(self, ref):
# Docstring inherited from GenericBaseDatastore
print(f"YYYYYXXXXXX@@@@@@@@@@@@@@@@@ Looking for itemInfos for ref {ref}")

# Look for the dataset_id -- there might be multiple matches
# if we have disassembled the dataset.
Expand All @@ -376,7 +369,7 @@ def getStoredItemsInfo(self, ref):
checksum=record["checksum"],
file_size=record["file_size"])
results.append(info)
print(f"Found {len(results)} itemInfos")

return results

def _registered_refs_per_artifact(self, pathInStore):
Expand Down Expand Up @@ -466,8 +459,6 @@ def _can_remove_dataset_artifact(self, ref, location):
True if the artifact can be safely removed.
"""

print(f"Asking for can remove info {ref}")

# Get all entries associated with this path
allRefs = self._registered_refs_per_artifact(location.pathInStore)
if not allRefs:
Expand Down Expand Up @@ -719,13 +710,12 @@ def exists(self, ref):
`True` if the entity exists in the `Datastore`.
"""
fileLocations = self._get_dataset_locations_info(ref)
print(f"<???????? Ref {ref} at {fileLocations}")
if not fileLocations:
return False
for location, _ in fileLocations:
if not self._artifact_exists(location):
return False
print("DID MANAGE TO FIND DATASET", ref)

return True

def getUri(self, ref, predict=False):
Expand Down Expand Up @@ -835,7 +825,6 @@ def get(self, ref, parameters=None):
# a component though because it is really reading a
# standalone dataset -- always tell reader it is not a
# component.
print(f"Reading component {component} from {getInfo.location}")
components[component] = self._read_artifact_into_memory(getInfo, ref, isComponent=False)

inMemoryDataset = ref.datasetType.storageClass.assembler().assemble(components)
Expand All @@ -851,7 +840,7 @@ def get(self, ref, parameters=None):
parameters=unusedParams)

else:
# Single file request or component
# Single file request or component from that composite file
allComponents = {i.component: i for i in allGetInfo}
for lookup in (refComponent, None):
if lookup in allComponents:
Expand Down Expand Up @@ -935,7 +924,6 @@ def trash(self, ref, ignore_errors=True):
"""
# Get file metadata and internal metadata
log.debug("Trashing %s in datastore %s", ref, self.name)
print("CALLING TRASH", ref)

fileLocations = self._get_dataset_locations_info(ref)

Expand All @@ -959,7 +947,6 @@ def trash(self, ref, ignore_errors=True):

# Mark dataset as trashed
try:
print(f"############ TRASHING {ref}")
self._move_to_trash_in_registry(ref)
except Exception as e:
if ignore_errors:
Expand All @@ -985,10 +972,8 @@ def emptyTrash(self, ignore_errors=True):

for ref in trashed:
fileLocations = self._get_dataset_locations_info(ref)
print(f"Found {len(fileLocations)} locations to remove for ref {ref}")

for location, _ in fileLocations:
print(f"* * * * * * * Trying to empty trash for dataset {location} $$$$$$$$%%%%%%%%%%%%%%%%%")

if location is None:
err_msg = f"Requested dataset ({ref}) does not exist in datastore {self.name}"
Expand All @@ -1010,7 +995,6 @@ def emptyTrash(self, ignore_errors=True):
# to the file from untrashed dataset refs.
if self._can_remove_dataset_artifact(ref, location):
# Point of no return for this artifact
print(f"Removing artifact {location.uri} from datasetore {self.name}")
log.debug("Removing artifact %s from datastore %s", location.uri, self.name)
try:
self._delete_artifact(location)
Expand Down
5 changes: 1 addition & 4 deletions python/lsst/daf/butler/datastores/genericDatastore.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,14 +122,11 @@ def _register_datasets(self, refsAndInfos):
# Need the main dataset and the components
expandedRefs.extend(ref.flatten([ref]))

all = "\n - ".join(str(f) for f in ref.flatten([ref]))
print(f"************* FLATTENED: {all}")

# Need one for the main ref and then one for each registered
# component
expandedItemInfos.extend([itemInfo] * (len(ref.components) + 1))

# Dataset location only cares about registry so if we have
# Dataset location only cares about registry ID so if we have
# disassembled in datastore we have to deduplicate. Since they
# will have different datasetTypes we can't use a set
registryRefs = {r.id: r for r in expandedRefs}
Expand Down
5 changes: 2 additions & 3 deletions python/lsst/daf/butler/registry/_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -586,12 +586,11 @@ def findDataset(self, datasetType: Union[DatasetType, str], dataId: Optional[Dat

# fallback to the parent if we got nothing and this was a component
if storage.datasetType.isComponent():
parentType, _ = datasetType.nameAndComponent()
parentType, _ = storage.datasetType.nameAndComponent()
parentRef = self.findDataset(parentType, dataId, collections=collections, **kwargs)
if parentRef is not None:
# Should already conform and we know no components
print(f">>>>>>>>>>>>>>>>>>>>>>>>> Got PARENT: {parentRef} for {datasetType}")
return DatasetRef(datasetType, parentRef.dataId, id=parentRef.id,
return DatasetRef(storage.datasetType, parentRef.dataId, id=parentRef.id,
run=parentRef.run, conform=False)

return None
Expand Down
1 change: 0 additions & 1 deletion tests/test_butler.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,6 @@ def runPutGetTest(self, storageClass, datasetTypeName):
# Registry shouldn't be able to find it by dataset_id anymore.
self.assertIsNone(butler.registry.getDataset(ref.id))

print(f"Putting dataset back: {refIn}")
# Put the dataset again, since the last thing we did was remove it.
ref = butler.put(metric, refIn)

Expand Down

0 comments on commit cee542b

Please sign in to comment.