Skip to content

Commit

Permalink
Merge pull request #2481 from cta-observatory/fixed_pointing_table_lo…
Browse files Browse the repository at this point in the history 
…ader

Load constant pointing in TableLoader
  • Loading branch information
@kosack
kosack committed Jan 18, 2024
2 parents 448fd62 + 492b0b2 commit 17c1ae4
Show file tree
Hide file tree
Showing 5 changed files with 169 additions and 61 deletions.
22 changes: 19 additions & 3 deletions ctapipe/io/astropy_helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
"""
import os
from contextlib import ExitStack
from uuid import uuid4

import numpy as np
import tables
Expand All @@ -23,7 +24,7 @@
TimeColumnTransform,
)

__all__ = ["read_table", "join_allow_empty"]
__all__ = ["read_table", "write_table", "join_allow_empty"]


def read_table(
Expand Down Expand Up @@ -227,7 +228,7 @@ def _parse_hdf5_attrs(table):
return transforms, descriptions, meta


def join_allow_empty(left, right, keys, join_type="left", **kwargs):
def join_allow_empty(left, right, keys, join_type="left", keep_order=False, **kwargs):
"""
Join two astropy tables, allowing both sides to be empty tables.
Expand Down Expand Up @@ -262,4 +263,19 @@ def join_allow_empty(left, right, keys, join_type="left", **kwargs):
if right_empty:
return left.copy()

return join(left, right, keys, join_type=join_type, **kwargs)
sort_key = None
if keep_order:
sort_key = str(uuid4())
if join_type == "left":
left[sort_key] = np.arange(len(left))
elif join_type == "right":
right[sort_key] = np.arange(len(left))
else:
raise ValueError("keep_order is only supported for left and right joins")

joined = join(left, right, keys, join_type=join_type, **kwargs)
if sort_key is not None:
joined.sort(sort_key)
del joined[sort_key]

return joined
128 changes: 79 additions & 49 deletions ctapipe/io/tableloader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
SIMULATION_CONFIG_TABLE = "/configuration/simulation/run"
SHOWER_DISTRIBUTION_TABLE = "/simulation/service/shower_distribution"
OBSERVATION_TABLE = "/configuration/observation/observation_block"
FIXED_POINTING_GROUP = "configuration/telescope/pointing"

DL2_SUBARRAY_GROUP = "/dl2/event/subarray"
DL2_TELESCOPE_GROUP = "/dl2/event/telescope"
Expand Down Expand Up @@ -201,6 +202,12 @@ class TableLoader(Component):
observation_info = traits.Bool(
False, help="join observation information to each event"
).tag(config=True)

pointing = traits.Bool(
True,
help="Load pointing information and join to events",
).tag(config=True)

focal_length_choice = traits.UseEnum(
FocalLengthKind,
default_value=FocalLengthKind.EFFECTIVE,
Expand Down Expand Up @@ -357,22 +364,14 @@ def _join_observation_info(self, table):
# casts the obs_id in the joint result to float.
obs_table["obs_id"] = obs_table["obs_id"].astype(table["obs_id"].dtype)

# to be able to sort to original table order
index_added = self._add_index_if_needed(table)

joint = join_allow_empty(
return join_allow_empty(
table,
obs_table,
keys="obs_id",
join_type="left",
keep_order=True,
)

# sort back to original order and remove index col
self._sort_to_original_order(joint, keep_index=not index_added)
if index_added:
table.remove_column("__index__")
return joint

def read_subarray_events(
self,
start=None,
Expand Down Expand Up @@ -403,13 +402,14 @@ def read_subarray_events(
table: astropy.io.Table
Table with primary index columns "obs_id" and "event_id".
"""
updated_attributes = self._check_args(
dl2=dl2, simulated=simulated, observation_info=observation_info
updated_args = self._check_args(
dl2=dl2,
simulated=simulated,
observation_info=observation_info,
)

dl2 = updated_attributes["dl2"]
simulated = updated_attributes["simulated"]
observation_info = updated_attributes["observation_info"]
dl2 = updated_args["dl2"]
simulated = updated_args["simulated"]
observation_info = updated_args["observation_info"]

table = read_table(self.h5file, TRIGGER_TABLE, start=start, stop=stop)
if keep_order:
Expand Down Expand Up @@ -469,6 +469,7 @@ def _read_telescope_events_for_id(
true_images,
true_parameters,
instrument,
pointing,
start=None,
stop=None,
):
Expand Down Expand Up @@ -500,6 +501,8 @@ def _read_telescope_events_for_id(
load image parameters obtained from true images
instrument: bool
join subarray instrument information to each event
pointing: bool
join pointing information to each event
Returns
-------
Expand Down Expand Up @@ -593,6 +596,16 @@ def _read_telescope_events_for_id(
)
table = _join_telescope_events(table, impacts)

if len(table) > 0 and pointing and FIXED_POINTING_GROUP in self.h5file.root:
pointing = read_table(
self.h5file, f"{FIXED_POINTING_GROUP}/tel_{tel_id:03d}"
)
# we know that we only have the tel_id we are looking for, remove optimize joining
del pointing["tel_id"]
table = join_allow_empty(
table, pointing, ["obs_id"], "left", keep_order=True
)

return table

def _read_telescope_events_for_ids(
Expand All @@ -606,6 +619,7 @@ def _read_telescope_events_for_ids(
true_images,
true_parameters,
instrument,
pointing,
start=None,
stop=None,
):
Expand All @@ -620,6 +634,7 @@ def _read_telescope_events_for_ids(
true_images=true_images,
true_parameters=true_parameters,
instrument=instrument,
pointing=pointing,
start=start,
stop=stop,
)
Expand Down Expand Up @@ -687,6 +702,7 @@ def read_telescope_events(
true_parameters=None,
instrument=None,
observation_info=None,
pointing=None,
):
"""
Read telescope-based event information.
Expand Down Expand Up @@ -727,13 +743,15 @@ def read_telescope_events(
join subarray instrument information to each event
observation_info: bool
join observation information to each event
pointing: bool
join pointing information to each event
Returns
-------
events: astropy.io.Table
Table with primary index columns "obs_id", "event_id" and "tel_id".
"""
updated_attributes = self._check_args(
updated_args = self._check_args(
dl1_images=dl1_images,
dl1_parameters=dl1_parameters,
dl1_muons=dl1_muons,
Expand All @@ -743,17 +761,18 @@ def read_telescope_events(
true_parameters=true_parameters,
instrument=instrument,
observation_info=observation_info,
pointing=pointing,
)

dl1_images = updated_attributes["dl1_images"]
dl1_parameters = updated_attributes["dl1_parameters"]
dl1_muons = updated_attributes["dl1_muons"]
dl2 = updated_attributes["dl2"]
simulated = updated_attributes["simulated"]
true_images = updated_attributes["true_images"]
true_parameters = updated_attributes["true_parameters"]
instrument = updated_attributes["instrument"]
observation_info = updated_attributes["observation_info"]
dl1_images = updated_args["dl1_images"]
dl1_parameters = updated_args["dl1_parameters"]
dl1_muons = updated_args["dl1_muons"]
dl2 = updated_args["dl2"]
simulated = updated_args["simulated"]
true_images = updated_args["true_images"]
true_parameters = updated_args["true_parameters"]
instrument = updated_args["instrument"]
observation_info = updated_args["observation_info"]
pointing = updated_args["pointing"]

if telescopes is None:
tel_ids = tuple(self.subarray.tel.keys())
Expand All @@ -770,6 +789,7 @@ def read_telescope_events(
true_images=true_images,
true_parameters=true_parameters,
instrument=instrument,
pointing=pointing,
start=start,
stop=stop,
)
Expand Down Expand Up @@ -853,6 +873,7 @@ def read_telescope_events_by_type(
true_parameters=None,
instrument=None,
observation_info=None,
pointing=None,
) -> Dict[str, Table]:
"""Read subarray-based event information.
Expand All @@ -878,14 +899,16 @@ def read_telescope_events_by_type(
join subarray instrument information to each event
observation_info: bool
join observation information to each event
pointing: bool
join pointing information to each event
Returns
-------
tables: dict(astropy.io.Table)
Dictionary of tables organized by telescope types
Table with primary index columns "obs_id", "event_id" and "tel_id".
"""
updated_attributes = self._check_args(
updated_args = self._check_args(
dl1_images=dl1_images,
dl1_parameters=dl1_parameters,
dl1_muons=dl1_muons,
Expand All @@ -895,17 +918,18 @@ def read_telescope_events_by_type(
true_parameters=true_parameters,
instrument=instrument,
observation_info=observation_info,
pointing=pointing,
)

dl1_images = updated_attributes["dl1_images"]
dl1_parameters = updated_attributes["dl1_parameters"]
dl1_muons = updated_attributes["dl1_muons"]
dl2 = updated_attributes["dl2"]
simulated = updated_attributes["simulated"]
true_images = updated_attributes["true_images"]
true_parameters = updated_attributes["true_parameters"]
instrument = updated_attributes["instrument"]
observation_info = updated_attributes["observation_info"]
dl1_images = updated_args["dl1_images"]
dl1_parameters = updated_args["dl1_parameters"]
dl1_muons = updated_args["dl1_muons"]
dl2 = updated_args["dl2"]
simulated = updated_args["simulated"]
true_images = updated_args["true_images"]
true_parameters = updated_args["true_parameters"]
instrument = updated_args["instrument"]
observation_info = updated_args["observation_info"]
pointing = updated_args["pointing"]

if telescopes is None:
tel_ids = tuple(self.subarray.tel.keys())
Expand Down Expand Up @@ -937,6 +961,7 @@ def read_telescope_events_by_type(
true_images=true_images,
true_parameters=true_parameters,
instrument=instrument,
pointing=pointing,
)

if len(table) > 0:
Expand Down Expand Up @@ -990,6 +1015,7 @@ def read_telescope_events_by_id(
true_parameters=None,
instrument=None,
observation_info=None,
pointing=None,
) -> Dict[int, Table]:
"""Read subarray-based event information.
Expand All @@ -1015,14 +1041,16 @@ def read_telescope_events_by_id(
join subarray instrument information to each event
observation_info: bool
join observation information to each event
pointing: bool
join pointing information to each event
Returns
-------
tables: dict(astropy.io.Table)
Dictionary of tables organized by telescope ids
Table with primary index columns "obs_id", "event_id" and "tel_id".
"""
updated_attributes = self._check_args(
updated_args = self._check_args(
dl1_images=dl1_images,
dl1_parameters=dl1_parameters,
dl1_muons=dl1_muons,
Expand All @@ -1032,17 +1060,18 @@ def read_telescope_events_by_id(
true_parameters=true_parameters,
instrument=instrument,
observation_info=observation_info,
pointing=pointing,
)

dl1_images = updated_attributes["dl1_images"]
dl1_parameters = updated_attributes["dl1_parameters"]
dl1_muons = updated_attributes["dl1_muons"]
dl2 = updated_attributes["dl2"]
simulated = updated_attributes["simulated"]
true_images = updated_attributes["true_images"]
true_parameters = updated_attributes["true_parameters"]
instrument = updated_attributes["instrument"]
observation_info = updated_attributes["observation_info"]
dl1_images = updated_args["dl1_images"]
dl1_parameters = updated_args["dl1_parameters"]
dl1_muons = updated_args["dl1_muons"]
dl2 = updated_args["dl2"]
simulated = updated_args["simulated"]
true_images = updated_args["true_images"]
true_parameters = updated_args["true_parameters"]
instrument = updated_args["instrument"]
observation_info = updated_args["observation_info"]
pointing = updated_args["pointing"]

if telescopes is None:
tel_ids = tuple(self.subarray.tel.keys())
Expand Down Expand Up @@ -1074,6 +1103,7 @@ def read_telescope_events_by_id(
true_images=true_images,
true_parameters=true_parameters,
instrument=instrument,
pointing=pointing,
)
if len(table) > 0:
by_id[tel_id] = table
Expand Down

0 comments on commit 17c1ae4

Please sign in to comment.