From 42b087a8e39982e130b388b1cfd9ed3169380e94 Mon Sep 17 00:00:00 2001 From: igncampa Date: Mon, 26 Mar 2018 18:47:29 -0600 Subject: [PATCH 1/7] added quote character logic to copy_to --- postgres_copy/copy_to.py | 5 +++-- postgres_copy/managers.py | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/postgres_copy/copy_to.py b/postgres_copy/copy_to.py index cc7be1e..cbb3bc5 100644 --- a/postgres_copy/copy_to.py +++ b/postgres_copy/copy_to.py @@ -49,12 +49,13 @@ def execute_sql(self, csv_path=None): # compile the SELECT query select_sql = self.as_sql()[0] % adapted_params # then the COPY TO query - copy_to_sql = "COPY ({}) TO STDOUT DELIMITER '{}' CSV {} {}" + copy_to_sql = "COPY ({}) TO STDOUT DELIMITER '{}' CSV {} {} {}" copy_to_sql = copy_to_sql.format( select_sql, self.query.copy_to_delimiter, self.query.copy_to_header, - self.query.copy_to_null_string + self.query.copy_to_null_string, + self.query.copy_to_quote_char, ) # then execute logger.debug(copy_to_sql) diff --git a/postgres_copy/managers.py b/postgres_copy/managers.py index 93e3351..cb1dd0a 100644 --- a/postgres_copy/managers.py +++ b/postgres_copy/managers.py @@ -172,6 +172,10 @@ def to_csv(self, csv_path=None, *fields, **kwargs): null_string = kwargs.get('null', None) query.copy_to_null_string = "" if null_string is None else "NULL '{}'".format(null_string) + # Quote character + quote_char = kwargs.get('quote', None) + query.copy_to_quote_char = "QUOTE '{}'".format(quote_char) if quote_char else "" + # Run the query compiler = query.get_compiler(self.db, connection=connection) data = compiler.execute_sql(csv_path) From fe6eefde13716015d93b8d84e92ca41b3ec48a3c Mon Sep 17 00:00:00 2001 From: igncampa Date: Mon, 26 Mar 2018 21:44:12 -0600 Subject: [PATCH 2/7] added force quote logic --- postgres_copy/copy_to.py | 3 ++- postgres_copy/managers.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/postgres_copy/copy_to.py b/postgres_copy/copy_to.py index cbb3bc5..b25dd6e 100644 --- a/postgres_copy/copy_to.py +++ b/postgres_copy/copy_to.py @@ -49,13 +49,14 @@ def execute_sql(self, csv_path=None): # compile the SELECT query select_sql = self.as_sql()[0] % adapted_params # then the COPY TO query - copy_to_sql = "COPY ({}) TO STDOUT DELIMITER '{}' CSV {} {} {}" + copy_to_sql = "COPY ({}) TO STDOUT DELIMITER '{}' CSV {} {} {} {}" copy_to_sql = copy_to_sql.format( select_sql, self.query.copy_to_delimiter, self.query.copy_to_header, self.query.copy_to_null_string, self.query.copy_to_quote_char, + self.query.copy_to_force_quote, ) # then execute logger.debug(copy_to_sql) diff --git a/postgres_copy/managers.py b/postgres_copy/managers.py index cb1dd0a..3905a6c 100644 --- a/postgres_copy/managers.py +++ b/postgres_copy/managers.py @@ -176,6 +176,12 @@ def to_csv(self, csv_path=None, *fields, **kwargs): quote_char = kwargs.get('quote', None) query.copy_to_quote_char = "QUOTE '{}'".format(quote_char) if quote_char else "" + # Force quote on columns + force_quote = kwargs.get('force_quote', None) + query.copy_to_force_quote = + "FORCE QUOTE {}".format(", ".join(column for column in force_quote)) + if force_quote else "" + # Run the query compiler = query.get_compiler(self.db, connection=connection) data = compiler.execute_sql(csv_path) From 7bd86b10dad180b59080d89a3c2fe58b8daeecc9 Mon Sep 17 00:00:00 2001 From: igncampa Date: Mon, 26 Mar 2018 21:48:25 -0600 Subject: [PATCH 3/7] helps in reading copy_to_sql --- postgres_copy/copy_to.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/postgres_copy/copy_to.py b/postgres_copy/copy_to.py index b25dd6e..c050018 100644 --- a/postgres_copy/copy_to.py +++ b/postgres_copy/copy_to.py @@ -49,14 +49,15 @@ def execute_sql(self, csv_path=None): # compile the SELECT query select_sql = self.as_sql()[0] % adapted_params # then the COPY TO query - copy_to_sql = "COPY ({}) TO STDOUT DELIMITER '{}' CSV {} {} {} {}" + copy_to_sql = + "COPY ({0}) TO STDOUT DELIMITER '{1}' CSV {2} {3} {4} {5}" copy_to_sql = copy_to_sql.format( - select_sql, - self.query.copy_to_delimiter, - self.query.copy_to_header, - self.query.copy_to_null_string, - self.query.copy_to_quote_char, - self.query.copy_to_force_quote, + select_sql, #0 + self.query.copy_to_delimiter, #1 + self.query.copy_to_header, #2 + self.query.copy_to_null_string, #3 + self.query.copy_to_quote_char, #4 + self.query.copy_to_force_quote, #5 ) # then execute logger.debug(copy_to_sql) From b35e75664309defcc04b12064c7a09f003aebc16 Mon Sep 17 00:00:00 2001 From: igncampa Date: Mon, 26 Mar 2018 21:55:30 -0600 Subject: [PATCH 4/7] added encoding logic to copy_to --- postgres_copy/copy_to.py | 3 ++- postgres_copy/managers.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/postgres_copy/copy_to.py b/postgres_copy/copy_to.py index c050018..f5cc0e4 100644 --- a/postgres_copy/copy_to.py +++ b/postgres_copy/copy_to.py @@ -50,7 +50,7 @@ def execute_sql(self, csv_path=None): select_sql = self.as_sql()[0] % adapted_params # then the COPY TO query copy_to_sql = - "COPY ({0}) TO STDOUT DELIMITER '{1}' CSV {2} {3} {4} {5}" + "COPY ({0}) TO STDOUT DELIMITER '{1}' CSV {2} {3} {4} {5} {6}" copy_to_sql = copy_to_sql.format( select_sql, #0 self.query.copy_to_delimiter, #1 @@ -58,6 +58,7 @@ def execute_sql(self, csv_path=None): self.query.copy_to_null_string, #3 self.query.copy_to_quote_char, #4 self.query.copy_to_force_quote, #5 + self.query.copy_to_encoding, #6 ) # then execute logger.debug(copy_to_sql) diff --git a/postgres_copy/managers.py b/postgres_copy/managers.py index 3905a6c..3743926 100644 --- a/postgres_copy/managers.py +++ b/postgres_copy/managers.py @@ -182,6 +182,10 @@ def to_csv(self, csv_path=None, *fields, **kwargs): "FORCE QUOTE {}".format(", ".join(column for column in force_quote)) if force_quote else "" + # Encoding + set_encoding = kwargs.get('encoding', None) + query.copy_to_encoding = "ENCODING '{}'".format(set_encoding) if set_encoding else "" + # Run the query compiler = query.get_compiler(self.db, connection=connection) data = compiler.execute_sql(csv_path) From bcee83e33ea79b6144af57c7afd1b5b21900a9dd Mon Sep 17 00:00:00 2001 From: igncampa Date: Mon, 26 Mar 2018 22:11:00 -0600 Subject: [PATCH 5/7] added escape character logic to copy_to --- postgres_copy/copy_to.py | 3 ++- postgres_copy/managers.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/postgres_copy/copy_to.py b/postgres_copy/copy_to.py index f5cc0e4..59ecd09 100644 --- a/postgres_copy/copy_to.py +++ b/postgres_copy/copy_to.py @@ -50,7 +50,7 @@ def execute_sql(self, csv_path=None): select_sql = self.as_sql()[0] % adapted_params # then the COPY TO query copy_to_sql = - "COPY ({0}) TO STDOUT DELIMITER '{1}' CSV {2} {3} {4} {5} {6}" + "COPY ({0}) TO STDOUT DELIMITER '{1}' CSV {2} {3} {4} {5} {6} {7}" copy_to_sql = copy_to_sql.format( select_sql, #0 self.query.copy_to_delimiter, #1 @@ -59,6 +59,7 @@ def execute_sql(self, csv_path=None): self.query.copy_to_quote_char, #4 self.query.copy_to_force_quote, #5 self.query.copy_to_encoding, #6 + self.query.copy_to_escape, #7 ) # then execute logger.debug(copy_to_sql) diff --git a/postgres_copy/managers.py b/postgres_copy/managers.py index 3743926..c379bb2 100644 --- a/postgres_copy/managers.py +++ b/postgres_copy/managers.py @@ -186,6 +186,10 @@ def to_csv(self, csv_path=None, *fields, **kwargs): set_encoding = kwargs.get('encoding', None) query.copy_to_encoding = "ENCODING '{}'".format(set_encoding) if set_encoding else "" + # Escape character + escape_char = kwargs.get('escape', None) + query.copy_to_escape = "ESCAPE '{}'".format(escape_char) if escape_char else "" + # Run the query compiler = query.get_compiler(self.db, connection=connection) data = compiler.execute_sql(csv_path) From cd4db4c0733fc78ecaceb87c9e98d9b8b0c4d139 Mon Sep 17 00:00:00 2001 From: igncampa Date: Mon, 26 Mar 2018 23:06:21 -0600 Subject: [PATCH 6/7] fixed bugs for newly added logic --- postgres_copy/copy_to.py | 2 +- postgres_copy/managers.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/postgres_copy/copy_to.py b/postgres_copy/copy_to.py index 59ecd09..ef76fcc 100644 --- a/postgres_copy/copy_to.py +++ b/postgres_copy/copy_to.py @@ -49,7 +49,7 @@ def execute_sql(self, csv_path=None): # compile the SELECT query select_sql = self.as_sql()[0] % adapted_params # then the COPY TO query - copy_to_sql = + copy_to_sql = \ "COPY ({0}) TO STDOUT DELIMITER '{1}' CSV {2} {3} {4} {5} {6} {7}" copy_to_sql = copy_to_sql.format( select_sql, #0 diff --git a/postgres_copy/managers.py b/postgres_copy/managers.py index c379bb2..eb69211 100644 --- a/postgres_copy/managers.py +++ b/postgres_copy/managers.py @@ -178,9 +178,14 @@ def to_csv(self, csv_path=None, *fields, **kwargs): # Force quote on columns force_quote = kwargs.get('force_quote', None) - query.copy_to_force_quote = - "FORCE QUOTE {}".format(", ".join(column for column in force_quote)) - if force_quote else "" + if force_quote: + if type(force_quote) == list: + query.copy_to_force_quote = \ + "FORCE QUOTE {}".format(", ".join(column for column in force_quote)) + else: + query.copy_to_force_quote = "FORCE QUOTE {}".format(force_quote) + else: + query.copy_to_force_quote = "" # Encoding set_encoding = kwargs.get('encoding', None) From a500e4b244e0493b59d5f3c28ffb8f06427b17c4 Mon Sep 17 00:00:00 2001 From: igncampa Date: Tue, 27 Mar 2018 12:40:00 -0600 Subject: [PATCH 7/7] fixed inline comments to flake8 specifications --- postgres_copy/copy_to.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/postgres_copy/copy_to.py b/postgres_copy/copy_to.py index ef76fcc..594a769 100644 --- a/postgres_copy/copy_to.py +++ b/postgres_copy/copy_to.py @@ -52,14 +52,14 @@ def execute_sql(self, csv_path=None): copy_to_sql = \ "COPY ({0}) TO STDOUT DELIMITER '{1}' CSV {2} {3} {4} {5} {6} {7}" copy_to_sql = copy_to_sql.format( - select_sql, #0 - self.query.copy_to_delimiter, #1 - self.query.copy_to_header, #2 - self.query.copy_to_null_string, #3 - self.query.copy_to_quote_char, #4 - self.query.copy_to_force_quote, #5 - self.query.copy_to_encoding, #6 - self.query.copy_to_escape, #7 + select_sql, # 0 + self.query.copy_to_delimiter, # 1 + self.query.copy_to_header, # 2 + self.query.copy_to_null_string, # 3 + self.query.copy_to_quote_char, # 4 + self.query.copy_to_force_quote, # 5 + self.query.copy_to_encoding, # 6 + self.query.copy_to_escape, # 7 ) # then execute logger.debug(copy_to_sql)