Add csv encoding option and fix a few spelling errors in example

villebro · Nov 6, 2019 · 046f826 · 046f826
1 parent 7d14237
commit 046f826
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 6 deletions.
diff --git a/example/README.md b/example/README.md
@@ -79,12 +79,12 @@ echo "select * from fact_customer_dq;"  | sqlite3 --column --header target.db
 |report_date|customer_id|rowid|source|priority|category|column_name|message|
 | --- | --- | --- | --- | --- | --- | --- | --- |
 2019-06-30|Terminator|2019-11-06 05:59:52.380884|source|high|incorrect|blood_group|Invalid blood group: Liquid Metal
-2019-06-30|Terminator|2019-11-06 05:59:52.380825|transform|medium|missing|age|Age is undefined due to undefined
+2019-06-30|Terminator|2019-11-06 05:59:52.380825|transform|medium|missing|age|Age is undefined due to undefined birthdate
 2019-06-30|Terminator|2019-11-06 05:59:52.380767|source|high|incorrect|birthdate|Birthdate in future: 2095-01-01
 2019-06-30|Peter Impossible|2019-11-06 05:59:52.380575|source|high|incorrect|blood_group|Invalid blood group: X+
-2019-06-30|Peter Impossible|2019-11-06 05:59:52.380516|transform|medium|missing|age|Age is undefined due to undefined's blood group table
+2019-06-30|Peter Impossible|2019-11-06 05:59:52.380516|transform|medium|missing|age|Age is undefined due to undefined birthdate
 2019-06-30|Peter Impossible|2019-11-06 05:59:52.380459|source|high|incorrect|birthdate|Cannot parse birthdate: 1980-13-01
-2019-06-30|Mary Null|2019-11-06 05:59:52.380341|source|medium|missing|blood_group|Blood group undefined in customer's blood group table
+2019-06-30|Mary Null|2019-11-06 05:59:52.380341|source|medium|missing|blood_group|Blood group undefined in customer blood group table
 2019-06-30|Mary Null|2019-11-06 05:59:52.380280|transform|medium|missing|age|Age is undefined due to undefined birthdate
 2019-06-30|Mary Null|2019-11-06 05:59:52.380219|source|medium|missing|birthdate|Missing birthdate
 2019-06-30|John Connor|2019-11-06 05:59:52.378454|transform|medium|missing|age|Age is undefined due to undefined birthdate

diff --git a/example/tasks/fact_customer_task.py b/example/tasks/fact_customer_task.py
@@ -146,7 +146,7 @@ def transform(self) -> None:
                     source=dq.Source.SOURCE,
                     priority=dq.Priority.MEDIUM,
                     category=dq.Category.MISSING,
-                    message="Blood group undefined in customer's blood group table"
+                    message="Blood group undefined in customer blood group table"
                 )
             elif not valid_blood_group:
                 row.log_dq(

diff --git a/sqltask/sources/csv.py b/sqltask/sources/csv.py
@@ -13,20 +13,23 @@ class CsvRowSource(BaseRowSource):
     def __init__(self,
                  file_path: str,
                  name: Optional[str] = None,
-                 delimiter: str = ","):
+                 delimiter: str = ",",
+                 encoding: Optional[str] = None):
         """
         :param name: name of data source.
         :param file_path: path to the csv file.
         :param delimiter: csv file delimiter.
+        :param encoding: Character encoding of csv file.
         """
 
         super().__init__(name)
         self.file_path = file_path
         self.delimiter = delimiter
+        self.encoding = encoding
 
         # populate column names
         self.columns: List[str] = []
-        with open(self.file_path, newline="") as csvfile:
+        with open(self.file_path, newline="", encoding=encoding) as csvfile:
             csvreader = csv.reader(csvfile, delimiter=self.delimiter)
             row = next(csvreader)
             for column in row: