Permalink
Browse files

added documentation; fix up style

  • Loading branch information...
1 parent c1ff465 commit 2ebe1ad09c80a186703008459e9ab454f07ac147 @p5k6 p5k6 committed Apr 26, 2016
Showing with 7 additions and 4 deletions.
  1. +2 −1 dataduct/pipeline/s3_node.py
  2. +3 −3 dataduct/steps/load_redshift.py
  3. +2 −0 docs/steps.rst
@@ -60,7 +60,8 @@ def __init__(self,
else:
additional_args['filePath'] = s3_object
- if (compression == "gzip"):
+ ### only gzip and none supported here, per aws s3datanode reqs
+ if compression == "gzip":
additional_args['compression'] = "gzip"
else:
additional_args['compression'] = "none"
@@ -44,11 +44,11 @@ def __init__(self,
command_options = ["DELIMITER '\t' ESCAPE TRUNCATECOLUMNS"]
command_options.append("NULL AS 'NULL' ")
- if (compression == "gzip"):
+ if compression == "gzip":
command_options.append("GZIP")
- elif (compression == "bzip2"):
+ elif compression == "bzip2":
command_options.append("BZIP2")
- elif (compression == "lzo"):
+ elif compression == "lzo":
command_options.append("lzop")
if max_errors:
command_options.append('MAXERROR %d' % int(max_errors))
View
@@ -235,6 +235,8 @@ Properties
load
- ``replace_invalid_char``: Character to replace non-utf8 characters
with
+- ``compression``: accepts one of [gzip, bzip2, lzo]. Allows redshift to
+ load compressed data. Leaving unspecified will load uncompressed data.
Example
^^^^^^^

0 comments on commit 2ebe1ad

Please sign in to comment.