chore(ingest/presto-on-hive) Set enable_properties_merge to True by default

metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py

@@ -130,8 +130,8 @@ class HiveMetastore(BasicSQLAlchemyConfig):
     )
 
     enable_properties_merge: bool = Field(
-        default=False,
-        description="By default, the connector overwrites properties every time. Set this to True to enable merging of properties with what exists on the server.",
+        default=True,
+        description="By default, the connector enables merging of properties with what exists on the server. Set this to False to enable the default connector behavior of overwriting properties on each ingestion.",
     )
 
     simplify_nested_field_paths: bool = Field(

metadata-ingestion/src/datahub/testing/compare_metadata_json.py

@@ -5,6 +5,7 @@
 import os
 import pathlib
 import pprint
+import re
 import shutil
 import tempfile
 from typing import Any, Dict, List, Sequence, Union
@@ -40,6 +41,7 @@ def assert_metadata_files_equal(
     update_golden: bool,
     copy_output: bool,
     ignore_paths: Sequence[str] = (),
+    ignore_paths_v2: Sequence[str] = (),
     ignore_order: bool = True,
 ) -> None:
     golden_exists = os.path.isfile(golden_path)
@@ -70,6 +72,16 @@ def assert_metadata_files_equal(
                 logger.info(f"Error reformatting golden file as MCP/MCEs: {e}")
                 golden = load_json_file(golden_path)
 
+    if ignore_paths_v2:
+        golden_json = load_json_file(golden_path)
+        for i, obj in enumerate(golden_json):
+            aspect_json = obj.get("aspect", {}).get("json", [])
+            for j, item in enumerate(aspect_json):
+                if isinstance(item, dict):
+                    if item.get("path") in ignore_paths_v2:
+                        json_path = f"root[{i}]['aspect']['json'][{j}]['value']"
+                        ignore_paths = (*ignore_paths, re.escape(json_path))
+
     diff = diff_metadata_json(output, golden, ignore_paths, ignore_order=ignore_order)
     if diff and update_golden:
         if isinstance(diff, MCPDiff):