diff --git a/rdl/BatchDataLoader.py b/rdl/BatchDataLoader.py index e727738..4c700c0 100644 --- a/rdl/BatchDataLoader.py +++ b/rdl/BatchDataLoader.py @@ -61,6 +61,10 @@ def load_batch(self, batch_key_tracker): batch_key_tracker.has_more_data = False return + # replacing unicode null characters because postgres doesn't support null characters in text fields + # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.replace.html + data_frame = data_frame.replace(regex=r"\x00", value="") + data_frame = self.attach_column_transformers(data_frame) self.write_data_frame_to_table(data_frame) batch_tracker.load_completed_successfully() diff --git a/rdl/data_sources/AWSLambdaDataSource.py b/rdl/data_sources/AWSLambdaDataSource.py index da4f33b..4b25ebd 100644 --- a/rdl/data_sources/AWSLambdaDataSource.py +++ b/rdl/data_sources/AWSLambdaDataSource.py @@ -141,9 +141,7 @@ def __get_table_data( return result["ColumnNames"], data def __get_data_frame(self, data: [[]], column_names: []): - return pandas.DataFrame(data=data, columns=column_names).replace( - regex=r"\x00", value="" - ) + return pandas.DataFrame(data=data, columns=column_names) def __invoke_lambda(self, pay_load): self.logger.debug("\nRequest being sent to Lambda:")