Skip to content
This repository was archived by the owner on May 17, 2024. It is now read-only.

Fix precision fetching for DataBricks #826

Merged
merged 1 commit into from
Dec 30, 2023
Merged

Fix precision fetching for DataBricks #826

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions data_diff/databases/databricks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ def query_table_schema(self, path: DbPath) -> Dict[str, RawColumnInfo]:

d = {
r.COLUMN_NAME: RawColumnInfo(
column_name=r.COLUMN_NAME, data_type=r.TYPE_NAME, datetime_precision=r.DECIMAL_DIGITS
column_name=r.COLUMN_NAME, data_type=r.TYPE_NAME, numeric_precision=r.DECIMAL_DIGITS
)
for r in rows
}
Expand Down Expand Up @@ -194,7 +194,7 @@ def _process_table_schema(
info = attrs.evolve(info, numeric_scale=0)

elif issubclass(type_cls, Float):
numeric_precision = math.ceil(info[2] / math.log(2, 10))
numeric_precision = math.ceil(info.numeric_precision / math.log(2, 10))
info = attrs.evolve(info, numeric_precision=numeric_precision)

elif issubclass(type_cls, Decimal):
Expand All @@ -207,7 +207,14 @@ def _process_table_schema(
)

elif issubclass(type_cls, Timestamp):
info = attrs.evolve(info, datetime_precision=info.datetime_precision)
info = attrs.evolve(
info,
datetime_precision=info.numeric_precision,
numeric_precision=None,
)

else:
info = attrs.evolve(info, numeric_precision=None)

resulted_rows.append(info)

Expand Down