Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 29 additions & 23 deletions data_diff/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -259,11 +259,13 @@ def main(conf, run, **kw):

try:
if kw["dbt"]:
dbt_diff(
diff = dbt_diff(
profiles_dir_override=kw["dbt_profiles_dir"],
project_dir_override=kw["dbt_project_dir"],
is_cloud=kw["cloud"],
)
render_diff(diff, kw["limit"], kw["stats"], kw["json_output"])

else:
return _data_diff(**kw)
except Exception as e:
Expand All @@ -272,6 +274,31 @@ def main(conf, run, **kw):
raise


def render_diff(diff_iter, limit, stats, json_output):
if limit:
assert not stats
diff_iter = islice(diff_iter, int(limit))

if stats:
if json_output:
rich.print(json.dumps(diff_iter.get_stats_dict()))
else:
rich.print(diff_iter.get_stats_string())

else:
for op, values in diff_iter:
color = COLOR_SCHEME[op]

if json_output:
jsonl = json.dumps([op, list(values)])
rich.print(f"[{color}]{jsonl}[/{color}]")
else:
text = f"{op} {', '.join(map(str, values))}"
rich.print(f"[{color}]{text}[/{color}]")

sys.stdout.flush()


def _data_diff(
database1,
table1,
Expand Down Expand Up @@ -444,28 +471,7 @@ def _data_diff(

diff_iter = differ.diff_tables(*segments)

if limit:
assert not stats
diff_iter = islice(diff_iter, int(limit))

if stats:
if json_output:
rich.print(json.dumps(diff_iter.get_stats_dict()))
else:
rich.print(diff_iter.get_stats_string())

else:
for op, values in diff_iter:
color = COLOR_SCHEME[op]

if json_output:
jsonl = json.dumps([op, list(values)])
rich.print(f"[{color}]{jsonl}[/{color}]")
else:
text = f"{op} {', '.join(map(str, values))}"
rich.print(f"[{color}]{text}[/{color}]")

sys.stdout.flush()
render_diff(diff_iter, limit, stats, json_output)

end = time.monotonic()

Expand Down
25 changes: 2 additions & 23 deletions data_diff/dbt.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ def dbt_diff(
)

if not is_cloud and len(diff_vars.primary_keys) == 1:
_local_diff(diff_vars)
return _local_diff(diff_vars)
elif not is_cloud:
rich.print(
"[red]"
Expand Down Expand Up @@ -152,28 +152,7 @@ def _local_diff(diff_vars: DiffVars) -> None:
extra_columns = tuple(mutual_set)

diff = diff_tables(table1, table2, threaded=True, algorithm=Algorithm.JOINDIFF, extra_columns=extra_columns)

if list(diff):
rich.print(
"[red]"
+ dev_qualified_string
+ " <> "
+ prod_qualified_string
+ "[/] \n"
+ column_diffs_str
+ diff.get_stats_string()
+ "\n"
)
else:
rich.print(
"[red]"
+ dev_qualified_string
+ " <> "
+ prod_qualified_string
+ "[/] \n"
+ column_diffs_str
+ "[green]No row differences[/] \n"
)
return diff


def _cloud_diff(diff_vars: DiffVars) -> None:
Expand Down