Merge pull request #69 from pageuppeople-opensource/task/SP-337_remove-null-character

[SP-337] Remove null characters

Author: ChintanRaval

rdl/BatchDataLoader.py

@@ -61,6 +61,10 @@ def load_batch(self, batch_key_tracker):
             batch_key_tracker.has_more_data = False
             return
 
+        # replacing unicode null characters because postgres doesn't support null characters in text fields
+        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.replace.html
+        data_frame = data_frame.replace(regex=r"\x00", value="")
+
         data_frame = self.attach_column_transformers(data_frame)
         self.write_data_frame_to_table(data_frame)
         batch_tracker.load_completed_successfully()

rdl/data_sources/AWSLambdaDataSource.py

@@ -145,9 +145,7 @@ def __get_table_data(
         return result["ColumnNames"], data
 
     def __get_data_frame(self, data: [[]], column_names: []):
-        return pandas.DataFrame(data=data, columns=column_names).replace(
-            regex=r"\x00", value=""
-        )
+        return pandas.DataFrame(data=data, columns=column_names)
 
     def __invoke_lambda(self, pay_load):
         self.logger.debug("\nRequest being sent to Lambda:")