From 80e0b53de8841edff308dcb76dc17cf3cbf8560b Mon Sep 17 00:00:00 2001 From: Xu He Date: Mon, 19 Feb 2024 22:16:05 -0800 Subject: [PATCH 1/2] update oracle to datetime --- .../destinations/oracledb/__init__.py | 8 ++++++- .../destinations/oracledb/utils.py | 24 +++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/mage_integrations/mage_integrations/destinations/oracledb/__init__.py b/mage_integrations/mage_integrations/destinations/oracledb/__init__.py index a060b28e395..adc167e70d5 100644 --- a/mage_integrations/mage_integrations/destinations/oracledb/__init__.py +++ b/mage_integrations/mage_integrations/destinations/oracledb/__init__.py @@ -12,6 +12,7 @@ clean_column_name, convert_column_to_type, convert_column_type, + convert_datetime, ) from mage_integrations.destinations.sql.base import Destination, main from mage_integrations.destinations.sql.utils import ( @@ -161,6 +162,7 @@ def build_insert_commands( columns=columns, records=records, convert_column_to_type_func=convert_column_to_type, + convert_datetime_func=convert_datetime, string_parse_func=lambda x, y: x.replace("'", "''").replace('\\', '\\\\') if COLUMN_TYPE_OBJECT == y['type'] else x, use_lowercase=self.use_lowercase, @@ -178,7 +180,11 @@ def build_insert_commands( # Build update command insert_value_without_left_parenthesis = re.sub(r"^\(", "", insert_value) insert_value_strip = re.sub(r"\)$", "", insert_value_without_left_parenthesis) - updated_values = insert_value_strip.split(',') + insert_value_strip = re.sub(r"\,$", "", insert_value_strip) + splitted_values = insert_value_strip.split('),') + updated_values = [s + ")" if i < len(splitted_values) - 1 else s + for i, s in enumerate(splitted_values)] + updated_command = ', '.join([f'{col} = {updated_values[idx].strip()}' for idx, col in enumerate(columns_cleaned)]) update_command_constraint = "" diff --git a/mage_integrations/mage_integrations/destinations/oracledb/utils.py b/mage_integrations/mage_integrations/destinations/oracledb/utils.py index b5d95e3c6b1..6538da0ff18 100644 --- a/mage_integrations/mage_integrations/destinations/oracledb/utils.py +++ b/mage_integrations/mage_integrations/destinations/oracledb/utils.py @@ -1,3 +1,4 @@ +from datetime import datetime from typing import Dict, List from mage_integrations.destinations.sql.constants import SQL_RESERVED_WORDS @@ -54,6 +55,24 @@ def convert_column_to_type(value, column_type) -> str: return f"CAST('{value}' AS {column_type})" +def is_valid_datetime_format(date_string, date_format): + try: + datetime.strptime(date_string, date_format) + return True + except ValueError: + return False + + +def convert_datetime(value: str, column_type_dict: Dict) -> str: + if is_valid_datetime_format(value, "%Y-%m-%d %H:%M:%S.%f"): + # Oracle stores only the fractions up to second in a DATE field. + # Use TO_TIMESTAMP for milliseconds and microseconds. + return f"TO_TIMESTAMP('{value}', 'yyyy-mm-dd hh24:mi:ss.ff6')" + if is_valid_datetime_format(value, "%Y-%m-%d"): + return f"TO_DATE('{value}', 'yyyy-mm-dd')" + return 'CHAR(255)' + + def build_create_table_command( column_type_mapping: Dict, columns: List[str], @@ -108,7 +127,8 @@ def convert_column_type(column_type: str, column_settings: Dict, **kwargs) -> st return 'NCLOB' elif COLUMN_TYPE_STRING == column_type: if COLUMN_FORMAT_DATETIME == column_settings.get('format'): - # Twice as long as the number of characters in ISO date format - return 'CHAR(52)' + return 'DATE' + elif COLUMN_FORMAT_DATETIME == column_type: + return 'DATE' return 'CHAR(255)' From 610b1f2cd71451f777dd29c0ed25b636de1d6878 Mon Sep 17 00:00:00 2001 From: Xu He Date: Mon, 18 Mar 2024 23:48:28 -0700 Subject: [PATCH 2/2] address comment to add comment and example --- .../destinations/oracledb/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mage_integrations/mage_integrations/destinations/oracledb/__init__.py b/mage_integrations/mage_integrations/destinations/oracledb/__init__.py index adc167e70d5..ff322d310b7 100644 --- a/mage_integrations/mage_integrations/destinations/oracledb/__init__.py +++ b/mage_integrations/mage_integrations/destinations/oracledb/__init__.py @@ -178,6 +178,20 @@ def build_insert_commands( if unique_constraints and unique_conflict_method: if UNIQUE_CONFLICT_METHOD_UPDATE == unique_conflict_method: # Build update command + # Convert insert_value from format: + # CAST('1' AS NUMBER), TO_DATE('2023-09-10', 'yyyy-mm-dd'), + # to_nclob('{\"customer\": \"John Doe\", \"items\": {\"product\": \"Beer\"}'), + + # to format: + # ["col1 = CAST('1' AS NUMBER)", "col2 = TO_DATE('2023-09-10', 'yyyy-mm-dd')", + # 'col3 = to_nclob(\'{"customer": "John Doe", "items": + # {"product": "Beer"}}\')'] + + # Steps: + # - First remove unnecessary punctuation + # (1) ( at beginning (2) ) at the end (3), at the end + # - split by ")," + # - Add back ) at the end of each value insert_value_without_left_parenthesis = re.sub(r"^\(", "", insert_value) insert_value_strip = re.sub(r"\)$", "", insert_value_without_left_parenthesis) insert_value_strip = re.sub(r"\,$", "", insert_value_strip)