Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove dump-triggers, enable ctapipe-process to store only trigger and simulation data #2375

Merged
merged 2 commits into from
Jul 6, 2023
Merged

Remove dump-triggers, enable ctapipe-process to store only trigger and simulation data #2375

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ce1bc0c
Allow only writing simulation info and trigger info in datawriter, re…
maxnoe Jul 5, 2023
ea996fe
Remove writer argument from internal functions
maxnoe Jul 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
99 changes: 47 additions & 52 deletions ctapipe/io/datawriter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
from .eventsource import EventSource
from .hdf5tableio import HDF5TableWriter
from .simteleventsource import SimTelEventSource
from .tableio import FixedPointColumnTransform, TableWriter, TelListToMaskTransform
from .tableio import FixedPointColumnTransform, TelListToMaskTransform

__all__ = ["DataWriter", "DATA_MODEL_VERSION", "write_reference_metadata_headers"]

Expand Down Expand Up @@ -298,15 +298,11 @@ def __call__(self, event: ArrayEventContainer):
Write a single event to the output file.
"""
self._at_least_one_event = True
self.log.debug("WRITING EVENT %s", event.index)

# Write subarray event data
self._write_subarray_pointing(event, writer=self._writer)
self._write_subarray_pointing(event)
self._write_trigger(event)

self.log.debug("WRITING EVENT %s", event.index)
self._writer.write(
table_name="dl1/event/subarray/trigger",
containers=[event.index, event.trigger],
)
if event.simulation is not None and event.simulation.shower is not None:
self._writer.write(
table_name="simulation/event/subarray/shower",
Expand All @@ -322,21 +318,21 @@ def __call__(self, event: ArrayEventContainer):
)

if self.write_waveforms:
self._write_r1_telescope_events(self._writer, event)
self._write_r1_telescope_events(event)

if self.write_raw_waveforms:
self._write_r0_telescope_events(self._writer, event)
self._write_r0_telescope_events(event)

# write telescope event data
self._write_dl1_telescope_events(self._writer, event)
self._write_dl1_telescope_events(event)

# write DL2 info if requested
if self.write_showers:
self._write_dl2_telescope_events(self._writer, event)
self._write_dl2_stereo_event(self._writer, event)
self._write_dl2_telescope_events(event)
self._write_dl2_stereo_event(event)

if self.write_muon_parameters:
self._write_muon_telescope_events(self._writer, event)
self._write_muon_telescope_events(event)

def finish(self):
"""called after all events are done"""
Expand All @@ -358,7 +354,6 @@ def finish(self):
)

self._write_context_metadata_headers()

self._writer.close()

@property
Expand Down Expand Up @@ -415,8 +410,9 @@ def _setup_output_path(self):
self.write_muon_parameters,
]
if not any(writable_things):
raise ToolConfigurationError(
"DataWriter configured to write no information"
self.log.warning(
"No processing results were selected for writing"
", only writing trigger and simulation information"
)

def _setup_writer(self):
Expand Down Expand Up @@ -507,13 +503,13 @@ def _setup_writer(self):
self._writer = writer
self.log.debug("Writer initialized: %s", self._writer)

def _write_subarray_pointing(self, event: ArrayEventContainer, writer: TableWriter):
def _write_subarray_pointing(self, event: ArrayEventContainer):
"""store subarray pointing info in a monitoring table"""
pnt = event.pointing
current_pointing = (pnt.array_azimuth, pnt.array_altitude)
if current_pointing != self._last_pointing:
pnt.prefix = ""
writer.write("dl1/monitoring/subarray/pointing", [event.trigger, pnt])
self._writer.write("dl1/monitoring/subarray/pointing", [event.trigger, pnt])
self._last_pointing = current_pointing

def _write_scheduling_and_observation_blocks(self):
Expand Down Expand Up @@ -616,49 +612,50 @@ def table_name(self, tel_id):
"""construct dataset table names depending on chosen split method"""
return f"tel_{tel_id:03d}"

def _write_r1_telescope_events(
self, writer: TableWriter, event: ArrayEventContainer
):
def _write_trigger(self, event: ArrayEventContainer):
"""
Write trigger information
"""
self._writer.write(
table_name="dl1/event/subarray/trigger",
containers=[event.index, event.trigger],
)

for tel_id, trigger in event.trigger.tel.items():
self._writer.write(
"dl1/event/telescope/trigger", (_get_tel_index(event, tel_id), trigger)
)

def _write_r1_telescope_events(self, event: ArrayEventContainer):
for tel_id, r1_tel in event.r1.tel.items():

tel_index = _get_tel_index(event, tel_id)
table_name = self.table_name(tel_id)

r1_tel.prefix = ""
writer.write(f"r1/event/telescope/{table_name}", [tel_index, r1_tel])
self._writer.write(f"r1/event/telescope/{table_name}", [tel_index, r1_tel])

def _write_r0_telescope_events(
self, writer: TableWriter, event: ArrayEventContainer
):
def _write_r0_telescope_events(self, event: ArrayEventContainer):
for tel_id, r0_tel in event.r0.tel.items():

tel_index = _get_tel_index(event, tel_id)
table_name = self.table_name(tel_id)

r0_tel.prefix = ""
writer.write(f"r0/event/telescope/{table_name}", [tel_index, r0_tel])
self._writer.write(f"r0/event/telescope/{table_name}", [tel_index, r0_tel])

def _write_dl1_telescope_events(
self, writer: TableWriter, event: ArrayEventContainer
):
def _write_dl1_telescope_events(self, event: ArrayEventContainer):
"""
add entries to the event/telescope tables for each telescope in a single
event
"""

# write the telescope tables
# trigger info
for tel_id, trigger in event.trigger.tel.items():
writer.write(
"dl1/event/telescope/trigger", [_get_tel_index(event, tel_id), trigger]
)

# pointing info
for tel_id, pnt in event.pointing.tel.items():
current_pointing = (pnt.azimuth, pnt.altitude)
if current_pointing != self._last_pointing_tel[tel_id]:
pnt.prefix = ""
writer.write(
self._writer.write(
f"dl1/monitoring/telescope/pointing/tel_{tel_id:03d}",
[event.trigger.tel[tel_id], pnt],
)
Expand All @@ -674,7 +671,7 @@ def _write_dl1_telescope_events(
table_name = self.table_name(tel_id)

if self.write_parameters:
writer.write(
self._writer.write(
table_name=f"dl1/event/telescope/parameters/{table_name}",
containers=[tel_index, *dl1_camera.parameters.values()],
)
Expand All @@ -685,14 +682,14 @@ def _write_dl1_telescope_events(
"DataWriter.write_images is True but event does not contain image"
)

writer.write(
self._writer.write(
table_name=f"dl1/event/telescope/images/{table_name}",
containers=[tel_index, dl1_camera],
)

if self._is_simulation:
# always write this, so that at least the sum is included
writer.write(
self._writer.write(
f"simulation/event/telescope/images/{table_name}",
[tel_index, event.simulation.tel[tel_id]],
)
Expand All @@ -705,7 +702,7 @@ def _write_dl1_telescope_events(
true_parameters = event.simulation.tel[tel_id].true_parameters
# only write the available containers, no peak time related
# features for true image available.
writer.write(
self._writer.write(
f"simulation/event/telescope/parameters/{table_name}",
[
tel_index,
Expand All @@ -717,21 +714,17 @@ def _write_dl1_telescope_events(
],
)

def _write_muon_telescope_events(
self, writer: TableWriter, event: ArrayEventContainer
):
def _write_muon_telescope_events(self, event: ArrayEventContainer):

for tel_id, muon in event.muon.tel.items():
table_name = self.table_name(tel_id)
tel_index = _get_tel_index(event, tel_id)
writer.write(
self._writer.write(
f"dl1/event/telescope/muon/{table_name}",
[tel_index, muon.ring, muon.parameters, muon.efficiency],
)

def _write_dl2_telescope_events(
self, writer: TableWriter, event: ArrayEventContainer
):
def _write_dl2_telescope_events(self, event: ArrayEventContainer):
"""
write per-telescope DL2 shower information.

Expand All @@ -749,9 +742,11 @@ def _write_dl2_telescope_events(
f"dl2/event/telescope/{container_name}/{algorithm}/{table_name}"
)

writer.write(table_name=name, containers=[tel_index, container])
self._writer.write(
table_name=name, containers=[tel_index, container]
)

def _write_dl2_stereo_event(self, writer: TableWriter, event: ArrayEventContainer):
def _write_dl2_stereo_event(self, event: ArrayEventContainer):
"""
write per-telescope DL2 shower information to e.g.
`/dl2/event/stereo/{geometry,energy,classification}/<algorithm_name>`
Expand All @@ -762,7 +757,7 @@ def _write_dl2_stereo_event(self, writer: TableWriter, event: ArrayEventContaine
# note this will only write info if the particular algorithm
# generated it (otherwise the algorithm map is empty, and no
# data will be written)
writer.write(
self._writer.write(
table_name=f"dl2/event/subarray/{container_name}/{algorithm}",
containers=[event.index, container],
)
Expand Down
142 changes: 0 additions & 142 deletions ctapipe/tools/dump_triggers.py
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions ctapipe/tools/process.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,7 @@ def start(self):
"""
Process events
"""
self.log.info("applying calibration: %s", self.should_calibrate)
self.log.info("(re)compute DL1: %s", self.should_compute_dl1)
self.log.info("(re)compute DL2: %s", self.should_compute_dl2)
self.log.info(
Expand Down