Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DM-41321: Add try-except block to manifest checker so that failure to get butler logs does not make error reporting impossible #384

Merged
merged 2 commits into from
Nov 7, 2023
Merged

DM-41321: Add try-except block to manifest checker so that failure to get butler logs does not make error reporting impossible #384

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a485bc6
Address FileNotFoundError associated with missing butler logs
eigerx Oct 25, 2023
e084f20
Rename bool logs to do_store_logs for clarity
eigerx Oct 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
23 changes: 14 additions & 9 deletions python/lsst/pipe/base/execution_reports.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,14 +230,14 @@
else:
dataset_type_report.handle_produced_dataset(output_ref, status_graph)

def to_summary_dict(self, butler: Butler, logs: bool = True) -> dict[str, Any]:
def to_summary_dict(self, butler: Butler, do_store_logs: bool = True) -> dict[str, Any]:
"""Summarize the results of the TaskExecutionReport in a dictionary.

Parameters
----------
butler : `lsst.daf.butler.Butler`
The Butler used for this report.
logs : `bool`
do_store_logs : `bool`
Store the logs in the summary dictionary.

Returns
Expand All @@ -257,11 +257,13 @@
failed_quanta = {}
for node_id, log_ref in self.failed.items():
quantum_info: dict[str, Any] = {"data_id": log_ref.dataId.byName()}
if logs:
if do_store_logs:
try:
log = butler.get(log_ref)
except LookupError:
quantum_info["error"] = []
except FileNotFoundError:
quantum_info["error"] = None

Check warning on line 266 in python/lsst/pipe/base/execution_reports.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/pipe/base/execution_reports.py#L265-L266 

Added lines #L265 - L266 were not covered by tests
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might be worth printing a message here that you did miss some files just so people know that it happened, unless that's dealt with elsewhere.

else:
quantum_info["error"] = [
record.message for record in log if record.levelno >= logging.ERROR
Expand Down Expand Up @@ -305,15 +307,15 @@
"""A dictionary of TaskExecutionReports by task label (`dict`).
"""

def to_summary_dict(self, butler: Butler, logs: bool = True) -> dict[str, Any]:
def to_summary_dict(self, butler: Butler, do_store_logs: bool = True) -> dict[str, Any]:
"""Summarize the results of the `QuantumGraphExecutionReport` in a
dictionary.

Parameters
----------
butler : `lsst.daf.butler.Butler`
The Butler used for this report.
logs : `bool`
do_store_logs : `bool`
Store the logs in the summary dictionary.

Returns
Expand All @@ -322,9 +324,12 @@
A dictionary containing a summary of a `TaskExecutionReport` for
each task in the quantum graph.
"""
return {task: report.to_summary_dict(butler, logs=logs) for task, report in self.tasks.items()}
return {
task: report.to_summary_dict(butler, do_store_logs=do_store_logs)
for task, report in self.tasks.items()
}

def write_summary_yaml(self, butler: Butler, filename: str, logs: bool = True) -> None:
def write_summary_yaml(self, butler: Butler, filename: str, do_store_logs: bool = True) -> None:
"""Take the dictionary from
`QuantumGraphExecutionReport.to_summary_dict` and store its contents in
a yaml file.
Expand All @@ -335,11 +340,11 @@
The Butler used for this report.
filename : `str`
The name to be used for the summary yaml file.
logs : `bool`
do_store_logs : `bool`
Store the logs in the summary dictionary.
"""
with open(filename, "w") as stream:
yaml.safe_dump(self.to_summary_dict(butler, logs=logs), stream)
yaml.safe_dump(self.to_summary_dict(butler, do_store_logs=do_store_logs), stream)

Check warning on line 347 in python/lsst/pipe/base/execution_reports.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/pipe/base/execution_reports.py#L347 

Added line #L347 was not covered by tests

@classmethod
def make_reports(
Expand Down
2 changes: 1 addition & 1 deletion tests/test_execution_reports.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def test_make_reports(self) -> None:
# make a simple qgraph to make an execution report on
butler, qgraph = simpleQGraph.makeSimpleQGraph(root=root)
report = QuantumGraphExecutionReport.make_reports(butler, qgraph)
dict_of_expected_failures = report.to_summary_dict(butler, logs=False)
dict_of_expected_failures = report.to_summary_dict(butler, do_store_logs=False)
self.assertIsNotNone(dict_of_expected_failures["task0"]["failed_quanta"])
self.assertEqual(
dict_of_expected_failures["task1"]["outputs"]["add_dataset2"]["missing_upstream_failed"], 1
Expand Down