Skip to content

Commit

Permalink
Change API parameter for temporal objects in Pandas, to include all t…
Browse files Browse the repository at this point in the history
…emporal objects (#164)
  • Loading branch information
martin-traverse committed Aug 25, 2022
1 parent 4d67840 commit 6da8469
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 17 deletions.
12 changes: 6 additions & 6 deletions tracdap-runtime/python/src/tracdap/rt/_exec/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ class TracContextImpl(_api.TracContext):
Output views will contain schemas but no data.
"""

__DEFAULT_DATES_AS_OBJECTS = False
__DEFAULT_TEMPORAL_OBJECTS = False

def __init__(self,
model_def: _meta.ModelDefinition,
Expand Down Expand Up @@ -100,9 +100,9 @@ def get_schema(self, dataset_name: str) -> _meta.SchemaDefinition:

return data_view.trac_schema

def get_pandas_table(self, dataset_name: str, dates_as_objects: tp.Optional[bool] = None) -> pd.DataFrame:
def get_pandas_table(self, dataset_name: str, use_temporal_objects: tp.Optional[bool] = None) -> pd.DataFrame:

_api_hook.ApiGuard.validate_signature(self.get_pandas_table, dataset_name, dates_as_objects)
_api_hook.ApiGuard.validate_signature(self.get_pandas_table, dataset_name, use_temporal_objects)

part_key = _data.DataPartKey.for_root()

Expand All @@ -115,10 +115,10 @@ def get_pandas_table(self, dataset_name: str, dates_as_objects: tp.Optional[bool

data_view = self.__data[dataset_name]

if dates_as_objects is None:
dates_as_objects = self.__DEFAULT_DATES_AS_OBJECTS
if use_temporal_objects is None:
use_temporal_objects = self.__DEFAULT_TEMPORAL_OBJECTS

return _data.DataMapping.view_to_pandas(data_view, part_key, dates_as_objects)
return _data.DataMapping.view_to_pandas(data_view, part_key, use_temporal_objects)

def put_pandas_table(self, dataset_name: str, dataset: pd.DataFrame):

Expand Down
18 changes: 9 additions & 9 deletions tracdap-runtime/python/src/tracdap/rt/_impl/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ def pandas_datetime_type(cls):
return cls.__PANDAS_DATETIME_TYPE

@classmethod
def view_to_pandas(cls, view: DataView, part: DataPartKey, date_as_object: bool) -> pd.DataFrame:
def view_to_pandas(cls, view: DataView, part: DataPartKey, temporal_objects_flag: bool) -> pd.DataFrame:

deltas = view.parts.get(part)

Expand All @@ -235,7 +235,7 @@ def view_to_pandas(cls, view: DataView, part: DataPartKey, date_as_object: bool)
raise _ex.ETracInternal(f"Data view for part [{part.opaque_key}] does not contain any items")

if len(deltas) == 1:
return cls.item_to_pandas(deltas[0], date_as_object)
return cls.item_to_pandas(deltas[0], temporal_objects_flag)

batches = {
batch
Expand All @@ -246,30 +246,30 @@ def view_to_pandas(cls, view: DataView, part: DataPartKey, date_as_object: bool)
else delta.table.to_batches())}

table = pa.Table.from_batches(batches) # noqa
return cls.arrow_to_pandas(table, date_as_object)
return cls.arrow_to_pandas(table, temporal_objects_flag)

@classmethod
def item_to_pandas(cls, item: DataItem, date_as_object: bool) -> pd.DataFrame:
def item_to_pandas(cls, item: DataItem, temporal_objects_flag: bool) -> pd.DataFrame:

if item.pandas is not None:
return item.pandas.copy()

if item.table is not None:
return cls.arrow_to_pandas(item.table, date_as_object)
return cls.arrow_to_pandas(item.table, temporal_objects_flag)

if item.batches is not None:
table = pa.Table.from_batches(item.batches, item.schema) # noqa
return cls.arrow_to_pandas(table, date_as_object)
return cls.arrow_to_pandas(table, temporal_objects_flag)

raise _ex.ETracInternal(f"Data item does not contain any usable data")

@classmethod
def arrow_to_pandas(cls, table: pa.Table, date_as_object: bool = False) -> pd.DataFrame:
def arrow_to_pandas(cls, table: pa.Table, temporal_objects_flag: bool = False) -> pd.DataFrame:

return table.to_pandas(
ignore_metadata=True, # noqa
date_as_object=date_as_object, # noqa
timestamp_as_object=date_as_object, # noqa
date_as_object=temporal_objects_flag, # noqa
timestamp_as_object=temporal_objects_flag, # noqa
types_mapper=cls.__ARROW_TO_PANDAS_TYPE_MAPPING.get)

@classmethod
Expand Down
4 changes: 2 additions & 2 deletions tracdap-runtime/python/src/tracdap/rt/api/model_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def get_schema(self, dataset_name: str) -> SchemaDefinition:
pass

@_abc.abstractmethod
def get_pandas_table(self, dataset_name: str, dates_as_objects: _tp.Optional[bool] = None) -> pandas.DataFrame:
def get_pandas_table(self, dataset_name: str, use_temporal_objects: _tp.Optional[bool] = None) -> pandas.DataFrame:

"""
Get the data for a model input or output as a Pandas dataframe
Expand All @@ -124,7 +124,7 @@ def get_pandas_table(self, dataset_name: str, dates_as_objects: _tp.Optional[boo
other models.
:param dataset_name: The name of the model input or output to get data for
:param dates_as_objects: Use Python objects for date/time fields instead of the NumPy *datetime64* type
:param use_temporal_objects: Use Python objects for date/time fields instead of the NumPy *datetime64* type
:return: A pandas dataframe containing the data for the named dataset
:raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`
"""
Expand Down

0 comments on commit 6da8469

Please sign in to comment.