Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add post_cmd_args() interface to jobs #450

Merged
merged 1 commit into from
Jan 9, 2017
Merged

add post_cmd_args() interface to jobs #450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 25 additions & 1 deletion genie-client/src/main/python/pygenie/jobs/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ def __init__(self, conf=None):
self._job_name = None
self._job_version = 'NA'
self._parameters = dict()
self._post_cmd_args = list()
self._setup_file = None
self._tags = list()
self._timeout = None
Expand Down Expand Up @@ -797,6 +798,30 @@ def parameters(self, **kwargs):

return self

@unicodify
@arg_list
@add_to_repr('append')
def post_cmd_args(self, _post_cmd_args):
"""
Add arguments, options, etc to the end of the constructed command line.

Example:
>>> job = HiveJob() \\
... .script('select * from dual') \\
... .headers() \\
... .hiveconf('a', 'b') \\
... .post_cmd_args('some_arg_1') \\
... .post_cmd_args(['arg_2', 'arg_3'])
>>> job.cmd_args
"--hiveconf a=b --hiveconf hive.cli.print.header=true -f script.hive some_arg_1 arg_2 arg_3"

Args:
tag (str, list): A tag for the job.

Returns:
:py:class:`GenieJob`: self
"""

@unicodify
@arg_list
@add_to_repr('append')
Expand All @@ -819,7 +844,6 @@ def tags(self, _tags):
:py:class:`GenieJob`: self
"""


def to_dict(self):
"""
Get a mapping of attributes to values for the object.
Expand Down
5 changes: 3 additions & 2 deletions genie-client/src/main/python/pygenie/jobs/hadoop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,11 @@ def cmd_args(self):
if self._property_file \
else ''

return '{prop_file} {props} {cmd}' \
return '{prop_file} {props} {cmd} {post_cmd_args}' \
.format(prop_file=prop_file_str,
props=props_str,
cmd=self._script or '') \
cmd=self._script or '',
post_cmd_args=' '.join(self._post_cmd_args)) \
.strip()

def command(self, script):
Expand Down
5 changes: 3 additions & 2 deletions genie-client/src/main/python/pygenie/jobs/hive.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,11 +81,12 @@ def cmd_args(self):
if self._property_file \
else ''

return '{prop_file} {props} {params} -f {filename}' \
return '{prop_file} {props} {params} -f {filename} {post_cmd_args}' \
.format(prop_file=prop_file_str,
props=props_str,
filename=filename,
params='-i _hive_parameters.txt' if param_str else '') \
params='-i _hive_parameters.txt' if param_str else '',
post_cmd_args=' '.join(self._post_cmd_args)) \
.strip()

@property
Expand Down
5 changes: 3 additions & 2 deletions genie-client/src/main/python/pygenie/jobs/pig.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,12 +88,13 @@ def cmd_args(self):
if self._property_file \
else ''

return '{props} {prop_file} {param_files} {params} -f {filename}' \
return '{props} {prop_file} {param_files} {params} -f {filename} {post_cmd_args}' \
.format(prop_file=prop_file_str,
props=props_str,
filename=filename,
param_files=param_files_str,
params='-param_file _pig_parameters.txt' if param_str else '') \
params='-param_file _pig_parameters.txt' if param_str else '',
post_cmd_args=' '.join(self._post_cmd_args)) \
.strip()

@property
Expand Down
5 changes: 3 additions & 2 deletions genie-client/src/main/python/pygenie/jobs/presto.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,10 +84,11 @@ def cmd_args(self):
'--session {}={}'.format(k, v) \
for k, v in self._command_options.get('--session', {}).items()])

return '{sessions} {options} -f {filename}' \
return '{sessions} {options} -f {filename} {post_cmd_args}' \
.format(sessions=sessions_str,
options=options_str,
filename=filename) \
filename=filename,
post_cmd_args=' '.join(self._post_cmd_args)) \
.strip()

def headers(self):
Expand Down
5 changes: 3 additions & 2 deletions genie-client/src/main/python/pygenie/jobs/sqoop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,10 @@ def cmd_args(self):
'data': self._options_file
})

return '{cmd} {hadoop_opts} --options-file _sqoop_options.txt' \
return '{cmd} {hadoop_opts} --options-file _sqoop_options.txt {post_cmd_args}' \
.format(cmd=self._cmd,
hadoop_opts=hadoop_opts) \
hadoop_opts=hadoop_opts,
post_cmd_args=' '.join(self._post_cmd_args)) \
.strip()

@property
Expand Down
2 changes: 1 addition & 1 deletion genie-client/src/main/python/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@

setup(
name='nflx-genie-client',
version='3.0.48',
version='3.1.0',
author='Netflix Inc.',
author_email='genieoss@googlegroups.com',
keywords='genie hadoop cloud netflix client bigdata presto',
Expand Down
58 changes: 31 additions & 27 deletions genie-client/src/main/python/tests/job_tests/test_geniejob.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,39 +81,43 @@ def test_repr(self, is_file):
.parameter('param1', 'pval1') \
.parameter('param2', 'pval2') \
.parameters(param3='pval3', param4='pval4') \
.post_cmd_args('post1') \
.post_cmd_args(['post2', 'post3']) \
.tags('tag1') \
.tags('tag2')

assert_equals(
str(job),
'.'.join([
u'GenieJob()',
u'applications("app1")',
u'applications("app2")',
u'archive(False)',
u'cluster_tags("cluster1")',
u'cluster_tags("cluster2")',
u'command_arguments("genie job repr args")',
u'command_tags("cmd1")',
u'command_tags("cmd2")',
u'dependencies("/dep1")',
u'dependencies("/dep2")',
u'description("description")',
u'genie_email("jsmith@email.com")',
u'genie_setup_file("/setup.sh")',
u'genie_timeout(999)',
u'genie_url("http://asdfasdf")',
u'genie_username("jsmith")',
u'group("group1")',
u'job_id("geniejob_repr")',
u'job_name("geniejob_repr")',
u'job_version("1.1.1")',
u'parameter("param1", "pval1")',
u'parameter("param2", "pval2")',
u'parameter("param3", "pval3")',
u'parameter("param4", "pval4")',
u'tags("tag1")',
u'tags("tag2")'
'GenieJob()',
'applications("app1")',
'applications("app2")',
'archive(False)',
'cluster_tags("cluster1")',
'cluster_tags("cluster2")',
'command_arguments("genie job repr args")',
'command_tags("cmd1")',
'command_tags("cmd2")',
'dependencies("/dep1")',
'dependencies("/dep2")',
'description("description")',
'genie_email("jsmith@email.com")',
'genie_setup_file("/setup.sh")',
'genie_timeout(999)',
'genie_url("http://asdfasdf")',
'genie_username("jsmith")',
'group("group1")',
'job_id("geniejob_repr")',
'job_name("geniejob_repr")',
'job_version("1.1.1")',
'parameter("param1", "pval1")',
'parameter("param2", "pval2")',
'parameter("param3", "pval3")',
'parameter("param4", "pval4")',
'post_cmd_args("post1")',
"post_cmd_args([u'post2', u'post3'])",
'tags("tag1")',
'tags("tag2")'
])
)

Expand Down
29 changes: 26 additions & 3 deletions genie-client/src/main/python/tests/job_tests/test_hadoopjob.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,12 @@ def test_cmd_args_explicit(self):

job = pygenie.jobs.HadoopJob() \
.command_arguments('explicitly stating command args') \
.script("""this should not be taken""")
.script("""this should not be taken""") \
.post_cmd_args('should not be used')

assert_equals(
job.cmd_args,
u'explicitly stating command args'
'explicitly stating command args'
)

def test_cmd_args_constructed_script_code(self):
Expand All @@ -62,6 +63,28 @@ def test_cmd_args_constructed_script_code(self):
u'fs -ls /test'
)

def test_cmd_args_post_cmd_args(self):
"""Test HadoopJob constructed cmd args with post cmd args."""

job = pygenie.jobs.HadoopJob() \
.command("fs -ls /test") \
.property('prop1', 'v1') \
.property('prop2', 'v2') \
.property_file('/Users/hadoop/props.conf') \
.post_cmd_args('a') \
.post_cmd_args(['a', 'b', 'c']) \
.post_cmd_args('d e f')

assert_equals(
' '.join([
'-conf props.conf',
'-Dprop1=v1 -Dprop2=v2',
'fs -ls /test',
'a b c d e f'
]),
job.cmd_args
)


@patch.dict('os.environ', {'GENIE_BYPASS_HOME_CONFIG': '1'})
class TestingHadoopJobRepr(unittest.TestCase):
Expand Down Expand Up @@ -99,7 +122,7 @@ def test_repr(self, is_file):
.property('mapred.1', 'p1') \
.property('mapred.2', 'p2') \
.property_file('/Users/hadoop/test.conf') \
.script('jar testing.jar') \
.script('jar testing.jar')

assert_equals(
str(job),
Expand Down
26 changes: 26 additions & 0 deletions genie-client/src/main/python/tests/job_tests/test_hivejob.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,32 @@ def test_cmd_args_constructed_script_file(self, is_file):
])
)

@patch('pygenie.jobs.hive.is_file')
def test_cmd_args_post_cmd_args(self, is_file):
"""Test HiveJob constructed cmd args with post cmd args."""

is_file.return_value = True

job = pygenie.jobs.HiveJob() \
.script('/Users/hive/test.hql') \
.parameter('hello', 'hi') \
.parameter('goodbye', 'bye') \
.property('p1', 'v1') \
.property('p2', 'v2') \
.post_cmd_args('a') \
.post_cmd_args(['a', 'b', 'c']) \
.post_cmd_args('d e f')

assert_equals(
" ".join([
"--hiveconf p2=v2 --hiveconf p1=v1",
"-i _hive_parameters.txt",
"-f test.hql",
"a b c d e f"
]),
job.cmd_args
)


@patch.dict('os.environ', {'GENIE_BYPASS_HOME_CONFIG': '1'})
class TestingHiveJobParameters(unittest.TestCase):
Expand Down
29 changes: 29 additions & 0 deletions genie-client/src/main/python/tests/job_tests/test_pigjob.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,35 @@ def test_cmd_args_constructed_script_file(self, is_file):
])
)

@patch('pygenie.jobs.pig.is_file')
def test_cmd_args_post_cmd_args(self, is_file):
"""Test PigJob constructed cmd args with post cmd args."""

is_file.return_value = True

job = pygenie.jobs.PigJob() \
.script('/Users/pig/test.pig') \
.parameter('p', 'v') \
.parameter_file('/Users/pig/p.params') \
.property('p1', 'v1') \
.property('p2', 'v2') \
.property_file('/Users/pig/p.conf') \
.post_cmd_args('a') \
.post_cmd_args(['a', 'b', 'c']) \
.post_cmd_args('d e f')

assert_equals(
" ".join([
"-Dp2=v2 -Dp1=v1",
"-P p.conf",
"-param_file p.params",
"-param_file _pig_parameters.txt",
"-f test.pig",
"a b c d e f"
]),
job.cmd_args
)


@patch.dict('os.environ', {'GENIE_BYPASS_HOME_CONFIG': '1'})
class TestingPigParameterFile(unittest.TestCase):
Expand Down
22 changes: 22 additions & 0 deletions genie-client/src/main/python/tests/job_tests/test_prestojob.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,28 @@ def test_cmd_args_constructed_script_file(self, is_file):
u'--session s2=v2 --session s1=v1 --debug --source tester --opt1 val1 -f test.presto'
)

@patch('pygenie.jobs.presto.is_file')
def test_cmd_args_post_cmd_args(self, is_file):
"""Test PrestoJob constructed cmd args with post cmd args."""

is_file.return_value = True

job = pygenie.jobs.PrestoJob() \
.script('/Users/presto/test.presto') \
.option('source', 'tester') \
.option('opt1', 'val1') \
.option('debug') \
.session('s1', 'v1') \
.session('s2', 'v2') \
.post_cmd_args('a') \
.post_cmd_args(['a', 'b', 'c']) \
.post_cmd_args('d e f')

assert_equals(
'--session s2=v2 --session s1=v1 --debug --source tester --opt1 val1 -f test.presto a b c d e f',
job.cmd_args
)


@patch.dict('os.environ', {'GENIE_BYPASS_HOME_CONFIG': '1'})
class TestingPrestoJobRepr(unittest.TestCase):
Expand Down
30 changes: 30 additions & 0 deletions genie-client/src/main/python/tests/job_tests/test_sqoopjob.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,36 @@ def test_cmd_args_constructed_script_code(self):
])
)

def test_cmd_args_post_cmd_args(self):
"""Test SqoopJob constructed cmd args with post cmd args."""

job = pygenie.jobs.SqoopJob() \
.cmd('export') \
.option('option1', 'value1') \
.option('option2', 'value2') \
.option('option3', 'value3') \
.option('verbose') \
.property('prop1', 'value1') \
.property('prop2', 'value2', flag='-X') \
.connect('jdbc://test') \
.password('passw0rd') \
.split_by('col1') \
.table('mytable') \
.target_dir('/path/to/output') \
.post_cmd_args('a') \
.post_cmd_args(['a', 'b', 'c']) \
.post_cmd_args('d e f')

assert_equals(
' '.join([
'export',
'-Dprop1=value1',
'--options-file _sqoop_options.txt',
'a b c d e f'
]),
job.cmd_args
)


@patch.dict('os.environ', {'GENIE_BYPASS_HOME_CONFIG': '1'})
class TestingSqoopJobRepr(unittest.TestCase):
Expand Down