From 93da5b0cfc4b012bd6614a14d1c90a772b3541bc Mon Sep 17 00:00:00 2001 From: Katherine Bargar Date: Thu, 23 Apr 2026 18:17:40 +0000 Subject: [PATCH 1/2] Switch to using a regex for datetime detection --- src/ldlite/database/_expansion/fixed_nodes.py | 9 +- tests/test_expansion.py | 93 ++++++++++++++++++- 2 files changed, 95 insertions(+), 7 deletions(-) diff --git a/src/ldlite/database/_expansion/fixed_nodes.py b/src/ldlite/database/_expansion/fixed_nodes.py index 7758a8f..2fe0102 100644 --- a/src/ldlite/database/_expansion/fixed_nodes.py +++ b/src/ldlite/database/_expansion/fixed_nodes.py @@ -107,7 +107,7 @@ def specify_type(self, conn: Conn) -> None: if self.json_type == "string": with conn.cursor() as cur: - specify = cte + sql.SQL(""" + specify = cte + sql.SQL(r""" SELECT NOT EXISTS( SELECT 1 FROM string_values @@ -119,11 +119,8 @@ def specify_type(self, conn: Conn) -> None: SELECT 1 FROM string_values WHERE string_value IS NOT NULL AND - ( - string_value NOT LIKE '____-__-__T__:__:__.___' AND - string_value NOT LIKE '____-__-__T__:__:__.___+__:__' - ) - ) AS is_uuid;""") + string_value !~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]{1,9})?(Z|[+-][0-9]{2}(:?[0-9]{2})?)$' + ) AS is_datetime;""") # noqa: E501 cur.execute(specify.as_string()) if row := cur.fetchone(): diff --git a/tests/test_expansion.py b/tests/test_expansion.py index 9efed25..fb13915 100644 --- a/tests/test_expansion.py +++ b/tests/test_expansion.py @@ -61,7 +61,7 @@ def case_typed_columns() -> ExpansionTC: b""" { "id": "id2", - "timestamptz": "2025-06-20T17:37:58.675", + "timestamptz": "2025-06-20T17:37:58.675Z", "integer": 2, "numeric": 2, "bigint": 2, @@ -94,6 +94,97 @@ def case_typed_columns() -> ExpansionTC: ) +@parametrize( + "isodate", + [ + ("z_plain", "2026-01-20T06:29:11Z"), + ("z_ms", "2026-01-20T06:29:11.973Z"), + ("z_us", "2026-01-20T06:29:11.973553Z"), + ("z_ns", "2026-01-20T06:29:11.123456789Z"), + ("2offset_plain", "2026-01-20T06:29:11+00"), + ("2offset_ms", "2026-01-20T06:29:11.973+01"), + ("2offset_us", "2026-01-20T06:29:11.973553+04"), + ("2offset_ns", "2026-01-20T06:29:11.123456789+04"), + ("4offset_plain", "2026-01-20T06:29:11+0000"), + ("4offset_ms", "2026-01-20T06:29:11.973+0100"), + ("4offset_us", "2026-01-20T06:29:11.973553+0430"), + ("4offset_ns", "2026-01-20T06:29:11.123456789+0430"), + ("2:2offset_plain", "2026-01-20T06:29:11+00:00"), + ("2:2offset_ms", "2026-01-20T06:29:11.973+01:00"), + ("2:2offset_us", "2026-01-20T06:29:11.973553+04:30"), + ("2:2offset_ns", "2026-01-20T06:29:11.123456789+04:30"), + ("2-offset_plain", "2026-01-20T06:29:11-01"), + ("2-offset_ms", "2026-01-20T06:29:11.973-01"), + ("2-offset_us", "2026-01-20T06:29:11.973553-04"), + ("2-offset_ns", "2026-01-20T06:29:11.123456789-04"), + ("4-offset_plain", "2026-01-20T06:29:11-0100"), + ("4-offset_ms", "2026-01-20T06:29:11.973-0100"), + ("4-offset_us", "2026-01-20T06:29:11.973553-0430"), + ("4-offset_ns", "2026-01-20T06:29:11.123456789-0430"), + ("2:2-offset_plain", "2026-01-20T06:29:11-01:00"), + ("2:2-offset_ms", "2026-01-20T06:29:11.973-01:00"), + ("2:2-offset_us", "2026-01-20T06:29:11.973553-04:30"), + ("2:2-offset_ns", "2026-01-20T06:29:11.123456789-04:30"), + ], + idgen="{isodate[0]}", +) +def case_iso8601(isodate: tuple[str, str]) -> ExpansionTC: + return ExpansionTC( + records=[ + f"""{{ "{isodate[0]}": "{isodate[1]}" }}""".encode(), + f"""{{ "{isodate[0]}": "{isodate[1].replace("T", " ")}" }}""".encode(), + ], + assertions=[ + Assertion( + """ +SELECT DATA_TYPE +FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_NAME = 'prefix__t' AND COLUMN_NAME <> '__id' +""", + exp_pg="timestamp with time zone", + exp_duck="TIMESTAMP WITH TIME ZONE", + ), + ], + ) + + +@parametrize( + "isodate", + [ + ("no_tz_plain", "2026-01-20T06:29:11"), + ("no_tz_ms", "2026-01-20T06:29:11.973"), + ("no_tz_us", "2026-01-20T06:29:11.973553"), + ("no_tz_ns", "2026-01-20T06:29:11.123456789"), + ("z_dot", "2026-01-20T06:29:11.Z"), + ("2offset_dot", "2026-01-20T06:29:11.+01"), + ("4offset_dot", "2026-01-20T06:29:11.+0100"), + ("2:2offset_dot", "2026-01-20T06:29:11.+01:00"), + ("2-offset_dot", "2026-01-20T06:29:11.-01"), + ("4-offset_dot", "2026-01-20T06:29:11.-0100"), + ("2:2-offset_dot", "2026-01-20T06:29:11.-01:00"), + ], + idgen="{isodate[0]}", +) +def case_not_iso8601(isodate: tuple[str, str]) -> ExpansionTC: + return ExpansionTC( + records=[ + f"""{{ "{isodate[0]}": "{isodate[1]}" }}""".encode(), + f"""{{ "{isodate[0]}": "{isodate[1].replace("T", " ")}" }}""".encode(), + ], + assertions=[ + Assertion( + """ +SELECT DATA_TYPE +FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_NAME = 'prefix__t' AND COLUMN_NAME <> '__id' +""", + exp_pg="text", + exp_duck="VARCHAR", + ), + ], + ) + + def case_camel() -> ExpansionTC: return ExpansionTC( records=[ From d40378b18111294e2c3670ecd17678a8bcf67e4e Mon Sep 17 00:00:00 2001 From: Katherine Bargar Date: Fri, 24 Apr 2026 13:05:20 +0000 Subject: [PATCH 2/2] Update changelog --- CHANGELOG.md | 5 +++++ pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85db87a..9a991d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ Please see [MIGRATING.md](./MIGRATING.md) for information on breaking changes. ### Removed +## [4.0.1] - April + +### Changed +* Relaxed datetime detection from ~rfc3339 to ~iso8601 + ## [4.0.0] - April ### Fixed diff --git a/pyproject.toml b/pyproject.toml index 26f791d..8151874 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "pdm.backend" [project] name = "ldlite" -version = "4.0.0" +version = "4.0.1" description = "Lightweight analytics tool for FOLIO services" authors = [ { name = "Katherine Bargar", email = "kbargar@fivecolleges.edu" },