Added bool support

README.md

@@ -46,6 +46,7 @@ The destination.type value controls both the data reader type and the destinatio
 | numeric                     | float       | sqlalchemy.Numeric                    | numeric        | Stores whole and decimal numbers                 |
 | guid                        | str         | sqlalchemy.dialects.postgresql.UUID   | uuid           | |
 | bigint                      | int         | sqlalchemy.BigInteger                 | BigInt         | Relies on 64big python. Limited to largest number of ~2147483647121212|
+| boolean                     | bool        | sqlalchemy.Boolean                    | Boolean         | |
 
 
 These are implemented in Column_Type_Resolver.py

integration_tests/csv_source/assertions/column_test_full_refresh_assertions.sql

@@ -3,20 +3,20 @@ SET client_encoding TO 'UTF8';
 DROP TABLE IF EXISTS results;
 
 CREATE TEMPORARY TABLE results AS
-  WITH expected(id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1) AS (
-    SELECT 1, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e'::UUID,2147483647121212
+   WITH expected(id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1, bool_column_1) AS (
+    SELECT 1, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e'::UUID,2147483647121212, True
     UNION ALL
-    SELECT 2, NULL, NULL, NULL, NULL, NULL, NULL, NULL
+    SELECT 2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL
     UNION ALL
-    SELECT 3, 333.0, '2001-01-01', 33.333, NULL, 'This Text Has a Quote Before "Dave', NULL, NULL
+    SELECT 3, 333.0, '2001-01-01', 33.333, NULL, 'This Text Has a Quote Before "Dave', NULL, NULL, True
     UNION ALL
-    SELECT 4, NULL, NULL, NULL, NULL, 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ','aabc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL
+    SELECT 4, NULL, NULL, NULL, NULL, 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ','aabc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL, False
     UNION ALL
-    SELECT 5, NULL, NULL, NULL, NULL, 'This row will be updated in the incremental review test', NULL, NULL
+    SELECT 5, NULL, NULL, NULL, NULL, 'This row will be updated in the incremental review test', NULL, NULL, NULL
   ),
 
   actual AS (
-    SELECT  id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1,guid_column_1,big_int_column_1
+    SELECT  id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1,guid_column_1,big_int_column_1,bool_column_1
     FROM    rdl_integration_tests.load_source_data
   )
 

integration_tests/csv_source/assertions/column_test_incremental_refresh_assertions.sql

@@ -2,24 +2,24 @@ SET client_encoding TO 'UTF8';
 DROP TABLE IF EXISTS results;
 
 CREATE TEMPORARY TABLE results AS
-  WITH expected(id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1) AS (
-    SELECT 1, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e'::UUID,2147483647121212
+  WITH expected(id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1,bool_column_1) AS (
+    SELECT 1, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e'::UUID,2147483647121212, True
     UNION ALL
-    SELECT 2, NULL, NULL, NULL, NULL, NULL, NULL, NULL
+    SELECT 2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL
     UNION ALL
-    SELECT 3, 333.0, '2001-01-01', 33.333, NULL, 'This Text Has a Quote Before "Dave', NULL, NULL
+    SELECT 3, 333.0, '2001-01-01', 33.333, NULL, 'This Text Has a Quote Before "Dave', NULL, NULL, True
     UNION ALL
-    SELECT 4, NULL, NULL, NULL, NULL, 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ', 'aabc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL
+    SELECT 4, NULL, NULL, NULL, NULL, 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ', 'aabc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL, False
     UNION ALL
-    SELECT 5, NULL, NULL, NULL, NULL, 'This row WAS updated in the incremental review test', NULL, NULL
+    SELECT 5, NULL, NULL, NULL, NULL, 'This row WAS updated in the incremental review test', NULL, NULL, True
     UNION ALL
-    SELECT 6, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL
+    SELECT 6, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL, False
     UNION ALL
-    SELECT 7, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'Another Basic String', NULL, NULL
+    SELECT 7, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'Another Basic String', NULL, NULL, True
   ),
 
   actual AS (
-    SELECT  id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1
+    SELECT  id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1,bool_column_1
     FROM    rdl_integration_tests.load_source_data
   )
 

integration_tests/csv_source/config/ColumnTest.json

@@ -77,10 +77,19 @@
         "type": "bigint",
         "nullable": true
       }
+    },
+     {
+      "source_name": "BoolColumn1",
+      "destination": {
+        "name": "bool_column_1",
+        "type": "boolean",
+        "nullable": true
+      }
     }
 
 
 
 
+
   ]
 }

integration_tests/csv_source/full_refresh_data/ColumnTest.csv

@@ -1,6 +1,6 @@
-id,StringColumn1,IntColumn1,DecimalColumn1,DateColumn1,DateTimeColumn1,GuidColumn1,BigIntColumn1
-1,"A Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,57BC8093-FE4C-477A-BBD7-FB5C02055A7E,2147483647121212
-2,,,,,,,
-3,"This Text Has a Quote Before ""Dave", 333,33.333, 01-01-01,,,
-4,"ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ",,,,,AABC8093-FE4C-477A-BBD7-FB5C02055A7E,
+id,StringColumn1,IntColumn1,DecimalColumn1,DateColumn1,DateTimeColumn1,GuidColumn1,BigIntColumn1,BoolColumn1
+1,"A Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,57BC8093-FE4C-477A-BBD7-FB5C02055A7E,2147483647121212,1
+2,,,,,,,,
+3,"This Text Has a Quote Before ""Dave", 333,33.333, 01-01-01,,,,1
+4,"ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ",,,,,AABC8093-FE4C-477A-BBD7-FB5C02055A7E,,0
 5,"This row will be updated in the incremental review test",,

integration_tests/csv_source/incremental_refresh_data/ColumnTest.csv

@@ -1,4 +1,4 @@
-id,StringColumn1,IntColumn1,DecimalColumn1,DateColumn1,DateTimeColumn1,GuidColumn1,BigIntColumn1
-5,"This row WAS updated in the incremental review test",,,,,,
-6,"A Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,57BC8093-FE4C-477A-BBD7-FB5C02055A7E,
-7,"Another Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,,
+id,StringColumn1,IntColumn1,DecimalColumn1,DateColumn1,DateTimeColumn1,GuidColumn1,BigIntColumn1,BoolColumn1
+5,"This row WAS updated in the incremental review test",,,,,,,1
+6,"A Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,57BC8093-FE4C-477A-BBD7-FB5C02055A7E,,0
+7,"Another Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,,,1

integration_tests/mssql_source/config/LargeTableTest.json

@@ -68,7 +68,16 @@
         "type": "guid",
         "nullable": true
       }
+    },
+     {
+      "source_name": "BoolColumn",
+      "destination": {
+        "name": "bool_column_1",
+        "type": "boolean",
+        "nullable": true
+      }
     }
 
+
   ]
 }

integration_tests/mssql_source/source_database_setup/create_large_table.sql

@@ -6,7 +6,8 @@ IF object_id('LargeTable') IS NULL
       DateColumn2	DATE,
       IntColumn1 INT,
       StringColumn2 NVARCHAR(100),
-      GuidColumn UNIQUEIDENTIFIER)
+      GuidColumn UNIQUEIDENTIFIER,
+      BoolColumn BIT)
 ELSE
 	TRUNCATE TABLE LargeTable
 
@@ -29,15 +30,18 @@ INSERT LargeTable
         DateColumn2,
         IntColumn1,
         StringColumn2,
-        GuidColumn
+        GuidColumn,
+        BoolColumn
 )
 SELECT	n,
         CASE WHEN n % 3 = 0 THEN NULL ELSE 'Row Number ' + CAST(n as varchar) END,
         CASE WHEN n % 5 = 0 THEN NULL ELSE DateAdd(hour, -n, '2000-01-1') END,
         CASE WHEN n % 7 = 0 THEN NULL ELSE DateAdd(hour, n, '2000-01-1') END,
         CASE WHEN n % 9 = 0 THEN NULL ELSE n * 1000 END,
         CASE WHEN n % 11 = 0 THEN NULL ELSE N'काचं शक्नोम्यत्तुम् । नोपहिनस्ति माम् ॥' END,
-        CASE WHEN n % 13 = 0 THEN NULL ELSE newid() END
+        CASE WHEN n % 13 = 0 THEN NULL ELSE newid() END,
+        CASE WHEN n % 3 = 0 THEN NULL ELSE 1 END
+
 FROM    Numbers
 
 

modules/ColumnTypeResolver.py

@@ -1,5 +1,5 @@
 import citext
-from sqlalchemy import DateTime, Numeric, Integer, BigInteger
+from sqlalchemy import DateTime, Numeric, Integer, BigInteger, Boolean
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.dialects.postgresql import UUID
 
@@ -10,15 +10,17 @@ class ColumnTypeResolver(object):
                        'json': str,
                        'numeric': float,
                        'guid': str,
-                       'bigint': int}
+                       'bigint': int,
+                       'boolean': bool}
 
     POSTGRES_TYPE_MAP = {'string': citext.CIText,
                          'datetime': DateTime,
                          'json': JSONB,
                          'numeric': Numeric,
                          'guid': UUID,
                          'int': Integer,
-                         'bigint': BigInteger}
+                         'bigint': BigInteger,
+                         'boolean': Boolean}
 
     def resolve_postgres_type(self, column):
         return self.POSTGRES_TYPE_MAP[column['type']]