Merge pull request #36 from Breaka84/fix_string_to_timestamp

Fix long overflow in extended_string_to_timestamp

.github/workflows/pytest_on_label.yml

@@ -15,8 +15,8 @@ jobs:
         include:
           - pyspark-version: 3.0.1
             pyspark-pip-package: "pyspark==3.0.1"
-#          - pyspark-version: 2.4.3
-#            pyspark-pip-package: "pyspark==2.4.3"
+          - pyspark-version: 3.1.2
+            pyspark-pip-package: "pyspark==3.1.2"
     steps:
     - uses: actions/checkout@v2
     - name: Set up Java 1.8

CHANGELOG.rst

@@ -2,6 +2,10 @@
 Changelog
 =========
 
+3.3.7 (2022-03-15)
+-------------------
+* [FIX] Fix long overflow in extended_string_to_timestamp
+
 3.3.6 (2021-11-19)
 -------------------
 * [FIX] Fix Cleaners logs in case of field type different than string

spooq/_version.py

@@ -1 +1 @@
-__version__ = "3.3.6"
+__version__ = "3.3.7"

spooq/transformer/mapper_custom_data_types.py

@@ -824,6 +824,10 @@ def _generate_select_expression_for_extended_string_to_timestamp(source_column,
             F.abs(F.trim(source_column).cast(T.LongType())).between(0, MAX_TIMESTAMP_SEC),
             F.trim(source_column).cast(T.LongType()).cast(T.TimestampType()),
         )
+        .when(
+            F.abs(F.trim(source_column).cast(T.LongType())) > MAX_TIMESTAMP_MS,
+            F.lit(None),
+        )
         .when(
             F.abs(F.trim(source_column).cast(T.LongType())) > MAX_TIMESTAMP_SEC,
             (F.trim(source_column) / 1000).cast(T.TimestampType()),

tests/data/test_fixtures/mapper_custom_data_types_fixtures.py

@@ -287,6 +287,10 @@
     ("nil",                        None),
     ("Hello",                      None),
     ("2k",                         None),
+    ("-5355957305054330880",       None),
+    ("5355957305054330880",        None),
+    (-5355957305054330880,         None),
+    ("1.547035982469E12",          None),
 ]
 
 fixtures_for_extended_string_unix_timestamp_ms_to_timestamp_spark2 = [