From 4af64fe721056d1667611f262ce37ba76bd5b9be Mon Sep 17 00:00:00 2001 From: Sourabh Bajaj Date: Mon, 16 Nov 2015 16:15:59 -0800 Subject: [PATCH] Fix URI --- dataduct/s3/s3_file.py | 6 +++--- dataduct/steps/create_update_sql.py | 4 +++- dataduct/steps/executors/runner.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dataduct/s3/s3_file.py b/dataduct/s3/s3_file.py index 4654a7c..6cb8c73 100644 --- a/dataduct/s3/s3_file.py +++ b/dataduct/s3/s3_file.py @@ -1,11 +1,11 @@ """ Base class for storing a S3 File """ +from ..utils.exceptions import ETLInputError +from ..utils.helpers import parse_path from .s3_path import S3Path -from .utils import upload_to_s3 from .utils import read_from_s3 -from ..utils.helpers import parse_path -from ..utils.exceptions import ETLInputError +from .utils import upload_to_s3 DEFAULT_FILE_NAME = 'file' diff --git a/dataduct/steps/create_update_sql.py b/dataduct/steps/create_update_sql.py index 03bbf74..8c6ad91 100644 --- a/dataduct/steps/create_update_sql.py +++ b/dataduct/steps/create_update_sql.py @@ -38,6 +38,8 @@ def __init__(self, update_script = SqlScript(filename=parse_path(script)) else: update_script = SqlScript(command) + sql_script = self.create_script(S3File(text=update_script.sql())) + sql_script.upload_to_s3() dest = Table(SqlScript(filename=parse_path(table_definition))) @@ -46,7 +48,7 @@ def __init__(self, arguments = [ '--table_definition=%s' % dest.sql().sql(), - '--sql=%s' % self.create_script(S3File(text=update_script)) + '--sql=%s' % sql_script.s3_path.uri ] if analyze_table: diff --git a/dataduct/steps/executors/runner.py b/dataduct/steps/executors/runner.py index 715ee89..bd1ae5a 100644 --- a/dataduct/steps/executors/runner.py +++ b/dataduct/steps/executors/runner.py @@ -68,7 +68,7 @@ def sql_runner(): sql_query = args.sql if sql_query.startswith('s3://'): - sql_query = S3File(s3_path=S3Path(uri=args.sql)).text() + sql_query = S3File(s3_path=S3Path(uri=args.sql)).text table = Table(SqlStatement(args.table_definition)) connection = redshift_connection()