Merge pull request #823 from datafold/fix-numeric-precision-recognition-bq-pg

Fix precision recognition

Author: vvkh

data_diff/databases/base.py

@@ -202,6 +202,7 @@ class BaseDialect(abc.ABC):
     SUPPORTS_INDEXES: ClassVar[bool] = False
     PREVENT_OVERFLOW_WHEN_CONCAT: ClassVar[bool] = False
     TYPE_CLASSES: ClassVar[Dict[str, Type[ColType]]] = {}
+    DEFAULT_NUMERIC_PRECISION: ClassVar[int] = 0  # effective precision when type is just "NUMERIC"
 
     PLACEHOLDER_TABLE = None  # Used for Oracle
 

data_diff/databases/bigquery.py

@@ -76,6 +76,11 @@ class Dialect(BaseDialect):
     }
     TYPE_ARRAY_RE = re.compile(r"ARRAY<(.+)>")
     TYPE_STRUCT_RE = re.compile(r"STRUCT<(.+)>")
+    # [BIG]NUMERIC, [BIG]NUMERIC(precision, scale), [BIG]NUMERIC(precision)
+    TYPE_NUMERIC_RE = re.compile(r"^((BIG)?NUMERIC)(?:\((\d+)(?:, (\d+))?\))?$")
+    # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#parameterized_decimal_type
+    # The default scale is 9, which means a number can have up to 9 digits after the decimal point.
+    DEFAULT_NUMERIC_PRECISION = 9
 
     def random(self) -> str:
         return "RAND()"
@@ -94,21 +99,43 @@ def type_repr(self, t) -> str:
 
     def parse_type(self, table_path: DbPath, info: RawColumnInfo) -> ColType:
         col_type = super().parse_type(table_path, info)
-        if isinstance(col_type, UnknownColType):
-            m = self.TYPE_ARRAY_RE.fullmatch(info.data_type)
-            if m:
-                item_info = attrs.evolve(info, data_type=m.group(1))
-                item_type = self.parse_type(table_path, item_info)
-                col_type = Array(item_type=item_type)
-
-            # We currently ignore structs' structure, but later can parse it too. Examples:
-            # - STRUCT<INT64, STRING(10)> (unnamed)
-            # - STRUCT<foo INT64, bar STRING(10)> (named)
-            # - STRUCT<foo INT64, bar ARRAY<INT64>> (with complex fields)
-            # - STRUCT<foo INT64, bar STRUCT<a INT64, b INT64>> (nested)
-            m = self.TYPE_STRUCT_RE.fullmatch(info.data_type)
-            if m:
-                col_type = Struct()
+        if not isinstance(col_type, UnknownColType):
+            return col_type
+
+        m = self.TYPE_ARRAY_RE.fullmatch(info.data_type)
+        if m:
+            item_info = attrs.evolve(info, data_type=m.group(1))
+            item_type = self.parse_type(table_path, item_info)
+            col_type = Array(item_type=item_type)
+            return col_type
+
+        # We currently ignore structs' structure, but later can parse it too. Examples:
+        # - STRUCT<INT64, STRING(10)> (unnamed)
+        # - STRUCT<foo INT64, bar STRING(10)> (named)
+        # - STRUCT<foo INT64, bar ARRAY<INT64>> (with complex fields)
+        # - STRUCT<foo INT64, bar STRUCT<a INT64, b INT64>> (nested)
+        m = self.TYPE_STRUCT_RE.fullmatch(info.data_type)
+        if m:
+            col_type = Struct()
+            return col_type
+
+        m = self.TYPE_NUMERIC_RE.fullmatch(info.data_type)
+        if m:
+            precision = int(m.group(3)) if m.group(3) else None
+            scale = int(m.group(4)) if m.group(4) else None
+
+            if scale is not None:
+                # NUMERIC(..., scale) — scale is set explicitly
+                effective_precision = scale
+            elif precision is not None:
+                # NUMERIC(...) — scale is missing but precision is set
+                # effectively the same as NUMERIC(..., 0)
+                effective_precision = 0
+            else:
+                # NUMERIC → default scale is 9
+                effective_precision = 9
+            col_type = Decimal(precision=effective_precision)
+            return col_type
 
         return col_type
 

data_diff/databases/duckdb.py

@@ -46,6 +46,10 @@ class Dialect(BaseDialect):
     SUPPORTS_PRIMARY_KEY = True
     SUPPORTS_INDEXES = True
 
+    # https://duckdb.org/docs/sql/data_types/numeric#fixed-point-decimals
+    # The default WIDTH and SCALE is DECIMAL(18, 3), if none are specified.
+    DEFAULT_NUMERIC_PRECISION = 3
+
     TYPE_CLASSES = {
         # Timestamps
         "TIMESTAMP WITH TIME ZONE": TimestampTZ,

data_diff/databases/postgresql.py

@@ -45,6 +45,12 @@ class PostgresqlDialect(BaseDialect):
     SUPPORTS_PRIMARY_KEY: ClassVar[bool] = True
     SUPPORTS_INDEXES = True
 
+    # https://www.postgresql.org/docs/current/datatype-numeric.html#DATATYPE-NUMERIC-DECIMAL
+    # without any precision or scale creates an “unconstrained numeric” column
+    # in which numeric values of any length can be stored, up to the implementation limits.
+    # https://www.postgresql.org/docs/current/datatype-numeric.html#DATATYPE-NUMERIC-TABLE
+    DEFAULT_NUMERIC_PRECISION = 16383
+
     TYPE_CLASSES: ClassVar[Dict[str, Type[ColType]]] = {
         # Timestamps
         "timestamp with time zone": TimestampTZ,
@@ -185,10 +191,21 @@ def select_table_schema(self, path: DbPath) -> str:
         if database:
             info_schema_path.insert(0, database)
 
-        return (
-            f"SELECT column_name, data_type, datetime_precision, numeric_precision, numeric_scale FROM {'.'.join(info_schema_path)} "
-            f"WHERE table_name = '{table}' AND table_schema = '{schema}'"
-        )
+        return f"""SELECT column_name, data_type, datetime_precision,
+                    -- see comment for DEFAULT_NUMERIC_PRECISION
+                    CASE
+                        WHEN data_type = 'numeric'
+                            THEN coalesce(numeric_precision, 131072 + {self.dialect.DEFAULT_NUMERIC_PRECISION})
+                        ELSE numeric_precision
+                    END AS numeric_precision,
+                    CASE
+                        WHEN data_type = 'numeric'
+                            THEN coalesce(numeric_scale, {self.dialect.DEFAULT_NUMERIC_PRECISION})
+                        ELSE numeric_scale
+                    END AS numeric_scale
+                    FROM {'.'.join(info_schema_path)}
+                    WHERE table_name = '{table}' AND table_schema = '{schema}'
+            """
 
     def select_table_unique_columns(self, path: DbPath) -> str:
         database, schema, table = self._normalize_table_path(path)

data_diff/databases/vertica.py

@@ -57,6 +57,9 @@ class Dialect(BaseDialect):
         "boolean": Boolean,
     }
 
+    # https://www.vertica.com/docs/9.3.x/HTML/Content/Authoring/SQLReferenceManual/DataTypes/Numeric/NUMERIC.htm#Default
+    DEFAULT_NUMERIC_PRECISION = 15
+
     def quote(self, s: str):
         return f'"{s}"'
 

tests/test_database.py

@@ -134,3 +134,36 @@ def test_three_part_support(self):
             d = db.query_table_schema(part.path)
             assert len(d) == 1
             db.query(part.drop())
+
+
+@test_each_database
+class TestNumericPrecisionParsing(unittest.TestCase):
+    def test_specified_precision(self):
+        name = "tbl_" + random_table_suffix()
+        db = get_conn(self.db_cls)
+        tbl = table(name, schema={"value": "DECIMAL(10, 2)"})
+        db.query(tbl.create())
+        t = table(name)
+        raw_schema = db.query_table_schema(t.path)
+        schema = db._process_table_schema(t.path, raw_schema)
+        self.assertEqual(schema["value"].precision, 2)
+
+    def test_specified_zero_precision(self):
+        name = "tbl_" + random_table_suffix()
+        db = get_conn(self.db_cls)
+        tbl = table(name, schema={"value": "DECIMAL(10)"})
+        db.query(tbl.create())
+        t = table(name)
+        raw_schema = db.query_table_schema(t.path)
+        schema = db._process_table_schema(t.path, raw_schema)
+        self.assertEqual(schema["value"].precision, 0)
+
+    def test_default_precision(self):
+        name = "tbl_" + random_table_suffix()
+        db = get_conn(self.db_cls)
+        tbl = table(name, schema={"value": "DECIMAL"})
+        db.query(tbl.create())
+        t = table(name)
+        raw_schema = db.query_table_schema(t.path)
+        schema = db._process_table_schema(t.path, raw_schema)
+        self.assertEqual(schema["value"].precision, db.dialect.DEFAULT_NUMERIC_PRECISION)