Skip to content

Commit

Permalink
Replace a couple of deprecations with updated methods (#21)
Browse files Browse the repository at this point in the history
  • Loading branch information
j-bennet committed Jun 9, 2023
1 parent 618fc05 commit 61b9e0f
Showing 1 changed file with 11 additions and 12 deletions.
23 changes: 11 additions & 12 deletions dask_deltatable/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,17 @@ def __init__(
self.fs, self.fs_token, _ = get_fs_token_paths(
path, storage_options=storage_options
)
self.schema = self.dt.pyarrow_schema()
self.schema = self.dt.schema().to_pyarrow()

def read_delta_dataset(self, f: str, **kwargs: Dict[any, any]):
schema = kwargs.pop("schema", None) or self.schema
filter = kwargs.pop("filter", None)
if filter:
filter_expression = pq._filters_to_expression(filter)
try:
filter_expression = pq.filters_to_expression(filter)
except AttributeError:
# fallback to older internal method
filter_expression = pq._filters_to_expression(filter)
else:
filter_expression = None
return (
Expand All @@ -65,15 +69,10 @@ def read_delta_dataset(self, f: str, **kwargs: Dict[any, any]):
)

def _make_meta_from_schema(self) -> Dict[str, str]:
meta = dict()

for field in self.schema:
if self.columns:
if field.name in self.columns:
meta[field.name] = field.type.to_pandas_dtype()
else:
meta[field.name] = field.type.to_pandas_dtype()
return meta
meta = self.schema.empty_table().to_pandas()
if self.columns:
meta = meta[self.columns]
return meta.dtypes.to_dict()

def _history_helper(self, log_file_name: str):
log = self.fs.cat(log_file_name).decode().split("\n")
Expand Down Expand Up @@ -142,7 +141,7 @@ def vacuum(self, retention_hours: int = 168, dry_run: bool = True) -> None:
)(f)
for f in tombstones
]
dask.compute(parts)[0]
return dask.compute(parts)[0]

def get_pq_files(self) -> List[str]:
"""
Expand Down

0 comments on commit 61b9e0f

Please sign in to comment.