diff --git a/README.md b/README.md index 317a596..15cfa3a 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ The destination.type value controls both the data reader type and the destinatio | numeric | float | sqlalchemy.Numeric | numeric | Stores whole and decimal numbers | | guid | str | sqlalchemy.dialects.postgresql.UUID | uuid | | | bigint | int | sqlalchemy.BigInteger | BigInt | Relies on 64big python. Limited to largest number of ~2147483647121212| +| boolean | bool | sqlalchemy.Boolean | Boolean | | These are implemented in Column_Type_Resolver.py diff --git a/integration_tests/csv_source/assertions/column_test_full_refresh_assertions.sql b/integration_tests/csv_source/assertions/column_test_full_refresh_assertions.sql index b5fed84..dc37743 100644 --- a/integration_tests/csv_source/assertions/column_test_full_refresh_assertions.sql +++ b/integration_tests/csv_source/assertions/column_test_full_refresh_assertions.sql @@ -3,20 +3,20 @@ SET client_encoding TO 'UTF8'; DROP TABLE IF EXISTS results; CREATE TEMPORARY TABLE results AS - WITH expected(id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1) AS ( - SELECT 1, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e'::UUID,2147483647121212 + WITH expected(id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1, bool_column_1) AS ( + SELECT 1, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e'::UUID,2147483647121212, True UNION ALL - SELECT 2, NULL, NULL, NULL, NULL, NULL, NULL, NULL + SELECT 2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL UNION ALL - SELECT 3, 333.0, '2001-01-01', 33.333, NULL, 'This Text Has a Quote Before "Dave', NULL, NULL + SELECT 3, 333.0, '2001-01-01', 33.333, NULL, 'This Text Has a Quote Before "Dave', NULL, NULL, True UNION ALL - SELECT 4, NULL, NULL, NULL, NULL, 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ','aabc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL + SELECT 4, NULL, NULL, NULL, NULL, 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ','aabc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL, False UNION ALL - SELECT 5, NULL, NULL, NULL, NULL, 'This row will be updated in the incremental review test', NULL, NULL + SELECT 5, NULL, NULL, NULL, NULL, 'This row will be updated in the incremental review test', NULL, NULL, NULL ), actual AS ( - SELECT id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1,guid_column_1,big_int_column_1 + SELECT id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1,guid_column_1,big_int_column_1,bool_column_1 FROM rdl_integration_tests.load_source_data ) diff --git a/integration_tests/csv_source/assertions/column_test_incremental_refresh_assertions.sql b/integration_tests/csv_source/assertions/column_test_incremental_refresh_assertions.sql index 1eb3fd0..ba4c807 100644 --- a/integration_tests/csv_source/assertions/column_test_incremental_refresh_assertions.sql +++ b/integration_tests/csv_source/assertions/column_test_incremental_refresh_assertions.sql @@ -2,24 +2,24 @@ SET client_encoding TO 'UTF8'; DROP TABLE IF EXISTS results; CREATE TEMPORARY TABLE results AS - WITH expected(id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1) AS ( - SELECT 1, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e'::UUID,2147483647121212 + WITH expected(id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1,bool_column_1) AS ( + SELECT 1, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e'::UUID,2147483647121212, True UNION ALL - SELECT 2, NULL, NULL, NULL, NULL, NULL, NULL, NULL + SELECT 2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL UNION ALL - SELECT 3, 333.0, '2001-01-01', 33.333, NULL, 'This Text Has a Quote Before "Dave', NULL, NULL + SELECT 3, 333.0, '2001-01-01', 33.333, NULL, 'This Text Has a Quote Before "Dave', NULL, NULL, True UNION ALL - SELECT 4, NULL, NULL, NULL, NULL, 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ', 'aabc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL + SELECT 4, NULL, NULL, NULL, NULL, 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ', 'aabc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL, False UNION ALL - SELECT 5, NULL, NULL, NULL, NULL, 'This row WAS updated in the incremental review test', NULL, NULL + SELECT 5, NULL, NULL, NULL, NULL, 'This row WAS updated in the incremental review test', NULL, NULL, True UNION ALL - SELECT 6, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL + SELECT 6, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL, False UNION ALL - SELECT 7, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'Another Basic String', NULL, NULL + SELECT 7, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'Another Basic String', NULL, NULL, True ), actual AS ( - SELECT id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1 + SELECT id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1,bool_column_1 FROM rdl_integration_tests.load_source_data ) diff --git a/integration_tests/csv_source/config/ColumnTest.json b/integration_tests/csv_source/config/ColumnTest.json index eeff9ae..37e994f 100644 --- a/integration_tests/csv_source/config/ColumnTest.json +++ b/integration_tests/csv_source/config/ColumnTest.json @@ -77,10 +77,19 @@ "type": "bigint", "nullable": true } + }, + { + "source_name": "BoolColumn1", + "destination": { + "name": "bool_column_1", + "type": "boolean", + "nullable": true + } } + ] } \ No newline at end of file diff --git a/integration_tests/csv_source/full_refresh_data/ColumnTest.csv b/integration_tests/csv_source/full_refresh_data/ColumnTest.csv index fcc595d..14085db 100644 --- a/integration_tests/csv_source/full_refresh_data/ColumnTest.csv +++ b/integration_tests/csv_source/full_refresh_data/ColumnTest.csv @@ -1,6 +1,6 @@ -id,StringColumn1,IntColumn1,DecimalColumn1,DateColumn1,DateTimeColumn1,GuidColumn1,BigIntColumn1 -1,"A Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,57BC8093-FE4C-477A-BBD7-FB5C02055A7E,2147483647121212 -2,,,,,,, -3,"This Text Has a Quote Before ""Dave", 333,33.333, 01-01-01,,, -4,"ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ",,,,,AABC8093-FE4C-477A-BBD7-FB5C02055A7E, +id,StringColumn1,IntColumn1,DecimalColumn1,DateColumn1,DateTimeColumn1,GuidColumn1,BigIntColumn1,BoolColumn1 +1,"A Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,57BC8093-FE4C-477A-BBD7-FB5C02055A7E,2147483647121212,1 +2,,,,,,,, +3,"This Text Has a Quote Before ""Dave", 333,33.333, 01-01-01,,,,1 +4,"ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ",,,,,AABC8093-FE4C-477A-BBD7-FB5C02055A7E,,0 5,"This row will be updated in the incremental review test",, \ No newline at end of file diff --git a/integration_tests/csv_source/incremental_refresh_data/ColumnTest.csv b/integration_tests/csv_source/incremental_refresh_data/ColumnTest.csv index 03eb5eb..5bf5caa 100644 --- a/integration_tests/csv_source/incremental_refresh_data/ColumnTest.csv +++ b/integration_tests/csv_source/incremental_refresh_data/ColumnTest.csv @@ -1,4 +1,4 @@ -id,StringColumn1,IntColumn1,DecimalColumn1,DateColumn1,DateTimeColumn1,GuidColumn1,BigIntColumn1 -5,"This row WAS updated in the incremental review test",,,,,, -6,"A Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,57BC8093-FE4C-477A-BBD7-FB5C02055A7E, -7,"Another Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,, +id,StringColumn1,IntColumn1,DecimalColumn1,DateColumn1,DateTimeColumn1,GuidColumn1,BigIntColumn1,BoolColumn1 +5,"This row WAS updated in the incremental review test",,,,,,,1 +6,"A Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,57BC8093-FE4C-477A-BBD7-FB5C02055A7E,,0 +7,"Another Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,,,1 \ No newline at end of file diff --git a/integration_tests/mssql_source/config/LargeTableTest.json b/integration_tests/mssql_source/config/LargeTableTest.json index 96bc387..7c305d7 100644 --- a/integration_tests/mssql_source/config/LargeTableTest.json +++ b/integration_tests/mssql_source/config/LargeTableTest.json @@ -68,7 +68,16 @@ "type": "guid", "nullable": true } + }, + { + "source_name": "BoolColumn", + "destination": { + "name": "bool_column_1", + "type": "boolean", + "nullable": true + } } + ] } \ No newline at end of file diff --git a/integration_tests/mssql_source/dockerfile b/integration_tests/mssql_source/dockerfile deleted file mode 100644 index 0bdb54e..0000000 --- a/integration_tests/mssql_source/dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM microsoft/mssql-server-linux:2017-latest - -ENV ACCEPT_EULA=Y diff --git a/integration_tests/mssql_source/source_database_setup/create_large_table.sql b/integration_tests/mssql_source/source_database_setup/create_large_table.sql index dd31e22..b33f85d 100644 --- a/integration_tests/mssql_source/source_database_setup/create_large_table.sql +++ b/integration_tests/mssql_source/source_database_setup/create_large_table.sql @@ -6,7 +6,8 @@ IF object_id('LargeTable') IS NULL DateColumn2 DATE, IntColumn1 INT, StringColumn2 NVARCHAR(100), - GuidColumn UNIQUEIDENTIFIER) + GuidColumn UNIQUEIDENTIFIER, + BoolColumn BIT) ELSE TRUNCATE TABLE LargeTable @@ -29,7 +30,8 @@ INSERT LargeTable DateColumn2, IntColumn1, StringColumn2, - GuidColumn + GuidColumn, + BoolColumn ) SELECT n, CASE WHEN n % 3 = 0 THEN NULL ELSE 'Row Number ' + CAST(n as varchar) END, @@ -37,7 +39,9 @@ SELECT n, CASE WHEN n % 7 = 0 THEN NULL ELSE DateAdd(hour, n, '2000-01-1') END, CASE WHEN n % 9 = 0 THEN NULL ELSE n * 1000 END, CASE WHEN n % 11 = 0 THEN NULL ELSE N'काचं शक्नोम्यत्तुम् । नोपहिनस्ति माम् ॥' END, - CASE WHEN n % 13 = 0 THEN NULL ELSE newid() END + CASE WHEN n % 13 = 0 THEN NULL ELSE newid() END, + CASE WHEN n % 3 = 0 THEN NULL ELSE 1 END + FROM Numbers diff --git a/modules/ColumnTypeResolver.py b/modules/ColumnTypeResolver.py index b9f4993..79f7c77 100644 --- a/modules/ColumnTypeResolver.py +++ b/modules/ColumnTypeResolver.py @@ -1,5 +1,5 @@ import citext -from sqlalchemy import DateTime, Numeric, Integer, BigInteger +from sqlalchemy import DateTime, Numeric, Integer, BigInteger, Boolean from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.dialects.postgresql import UUID @@ -10,7 +10,8 @@ class ColumnTypeResolver(object): 'json': str, 'numeric': float, 'guid': str, - 'bigint': int} + 'bigint': int, + 'boolean': bool} POSTGRES_TYPE_MAP = {'string': citext.CIText, 'datetime': DateTime, @@ -18,7 +19,8 @@ class ColumnTypeResolver(object): 'numeric': Numeric, 'guid': UUID, 'int': Integer, - 'bigint': BigInteger} + 'bigint': BigInteger, + 'boolean': Boolean} def resolve_postgres_type(self, column): return self.POSTGRES_TYPE_MAP[column['type']]