add post_cmd_args() interface to jobs (#450)

genie-client/src/main/python/pygenie/jobs/core.py

@@ -163,6 +163,7 @@ def __init__(self, conf=None):
         self._job_name = None
         self._job_version = 'NA'
         self._parameters = dict()
+        self._post_cmd_args = list()
         self._setup_file = None
         self._tags = list()
         self._timeout = None
@@ -797,6 +798,30 @@ def parameters(self, **kwargs):
 
         return self
 
+    @unicodify
+    @arg_list
+    @add_to_repr('append')
+    def post_cmd_args(self, _post_cmd_args):
+        """
+        Add arguments, options, etc to the end of the constructed command line.
+
+        Example:
+            >>> job = HiveJob() \\
+            ...     .script('select * from dual') \\
+            ...     .headers() \\
+            ...     .hiveconf('a', 'b') \\
+            ...     .post_cmd_args('some_arg_1') \\
+            ...     .post_cmd_args(['arg_2', 'arg_3'])
+            >>> job.cmd_args
+            "--hiveconf a=b --hiveconf hive.cli.print.header=true -f script.hive some_arg_1 arg_2 arg_3"
+
+        Args:
+            tag (str, list): A tag for the job.
+
+        Returns:
+            :py:class:`GenieJob`: self
+        """
+
     @unicodify
     @arg_list
     @add_to_repr('append')
@@ -819,7 +844,6 @@ def tags(self, _tags):
             :py:class:`GenieJob`: self
         """
 
-
     def to_dict(self):
         """
         Get a mapping of attributes to values for the object.

genie-client/src/main/python/pygenie/jobs/hadoop.py

@@ -49,10 +49,11 @@ def cmd_args(self):
             if self._property_file \
             else ''
 
-        return '{prop_file} {props} {cmd}' \
+        return '{prop_file} {props} {cmd} {post_cmd_args}' \
             .format(prop_file=prop_file_str,
                     props=props_str,
-                    cmd=self._script or '') \
+                    cmd=self._script or '',
+                    post_cmd_args=' '.join(self._post_cmd_args)) \
             .strip()
 
     def command(self, script):

genie-client/src/main/python/pygenie/jobs/hive.py

@@ -81,11 +81,12 @@ def cmd_args(self):
             if self._property_file \
             else ''
 
-        return '{prop_file} {props} {params} -f {filename}' \
+        return '{prop_file} {props} {params} -f {filename} {post_cmd_args}' \
             .format(prop_file=prop_file_str,
                     props=props_str,
                     filename=filename,
-                    params='-i _hive_parameters.txt' if param_str else '') \
+                    params='-i _hive_parameters.txt' if param_str else '',
+                    post_cmd_args=' '.join(self._post_cmd_args)) \
             .strip()
 
     @property

genie-client/src/main/python/pygenie/jobs/pig.py

@@ -88,12 +88,13 @@ def cmd_args(self):
             if self._property_file \
             else ''
 
-        return '{props} {prop_file} {param_files} {params} -f {filename}' \
+        return '{props} {prop_file} {param_files} {params} -f {filename} {post_cmd_args}' \
             .format(prop_file=prop_file_str,
                     props=props_str,
                     filename=filename,
                     param_files=param_files_str,
-                    params='-param_file _pig_parameters.txt' if param_str else '') \
+                    params='-param_file _pig_parameters.txt' if param_str else '',
+                    post_cmd_args=' '.join(self._post_cmd_args)) \
             .strip()
 
     @property

genie-client/src/main/python/pygenie/jobs/presto.py

@@ -84,10 +84,11 @@ def cmd_args(self):
             '--session {}={}'.format(k, v) \
             for k, v in self._command_options.get('--session', {}).items()])
 
-        return '{sessions} {options} -f {filename}' \
+        return '{sessions} {options} -f {filename} {post_cmd_args}' \
             .format(sessions=sessions_str,
                     options=options_str,
-                    filename=filename) \
+                    filename=filename,
+                    post_cmd_args=' '.join(self._post_cmd_args)) \
             .strip()
 
     def headers(self):

genie-client/src/main/python/pygenie/jobs/sqoop.py

@@ -68,9 +68,10 @@ def cmd_args(self):
             'data': self._options_file
         })
 
-        return '{cmd} {hadoop_opts} --options-file _sqoop_options.txt' \
+        return '{cmd} {hadoop_opts} --options-file _sqoop_options.txt {post_cmd_args}' \
             .format(cmd=self._cmd,
-                    hadoop_opts=hadoop_opts) \
+                    hadoop_opts=hadoop_opts,
+                    post_cmd_args=' '.join(self._post_cmd_args)) \
             .strip()
 
     @property

genie-client/src/main/python/setup.py

@@ -26,7 +26,7 @@
 
 setup(
     name='nflx-genie-client',
-    version='3.0.48',
+    version='3.1.0',
     author='Netflix Inc.',
     author_email='genieoss@googlegroups.com',
     keywords='genie hadoop cloud netflix client bigdata presto',

genie-client/src/main/python/tests/job_tests/test_geniejob.py

@@ -81,39 +81,43 @@ def test_repr(self, is_file):
             .parameter('param1', 'pval1') \
             .parameter('param2', 'pval2') \
             .parameters(param3='pval3', param4='pval4') \
+            .post_cmd_args('post1') \
+            .post_cmd_args(['post2', 'post3']) \
             .tags('tag1') \
             .tags('tag2')
 
         assert_equals(
             str(job),
             '.'.join([
-                u'GenieJob()',
-                u'applications("app1")',
-                u'applications("app2")',
-                u'archive(False)',
-                u'cluster_tags("cluster1")',
-                u'cluster_tags("cluster2")',
-                u'command_arguments("genie job repr args")',
-                u'command_tags("cmd1")',
-                u'command_tags("cmd2")',
-                u'dependencies("/dep1")',
-                u'dependencies("/dep2")',
-                u'description("description")',
-                u'genie_email("jsmith@email.com")',
-                u'genie_setup_file("/setup.sh")',
-                u'genie_timeout(999)',
-                u'genie_url("http://asdfasdf")',
-                u'genie_username("jsmith")',
-                u'group("group1")',
-                u'job_id("geniejob_repr")',
-                u'job_name("geniejob_repr")',
-                u'job_version("1.1.1")',
-                u'parameter("param1", "pval1")',
-                u'parameter("param2", "pval2")',
-                u'parameter("param3", "pval3")',
-                u'parameter("param4", "pval4")',
-                u'tags("tag1")',
-                u'tags("tag2")'
+                'GenieJob()',
+                'applications("app1")',
+                'applications("app2")',
+                'archive(False)',
+                'cluster_tags("cluster1")',
+                'cluster_tags("cluster2")',
+                'command_arguments("genie job repr args")',
+                'command_tags("cmd1")',
+                'command_tags("cmd2")',
+                'dependencies("/dep1")',
+                'dependencies("/dep2")',
+                'description("description")',
+                'genie_email("jsmith@email.com")',
+                'genie_setup_file("/setup.sh")',
+                'genie_timeout(999)',
+                'genie_url("http://asdfasdf")',
+                'genie_username("jsmith")',
+                'group("group1")',
+                'job_id("geniejob_repr")',
+                'job_name("geniejob_repr")',
+                'job_version("1.1.1")',
+                'parameter("param1", "pval1")',
+                'parameter("param2", "pval2")',
+                'parameter("param3", "pval3")',
+                'parameter("param4", "pval4")',
+                'post_cmd_args("post1")',
+                "post_cmd_args([u'post2', u'post3'])",
+                'tags("tag1")',
+                'tags("tag2")'
             ])
         )
 

genie-client/src/main/python/tests/job_tests/test_hadoopjob.py

@@ -39,11 +39,12 @@ def test_cmd_args_explicit(self):
 
         job = pygenie.jobs.HadoopJob() \
             .command_arguments('explicitly stating command args') \
-            .script("""this should not be taken""")
+            .script("""this should not be taken""") \
+            .post_cmd_args('should not be used')
 
         assert_equals(
             job.cmd_args,
-            u'explicitly stating command args'
+            'explicitly stating command args'
         )
 
     def test_cmd_args_constructed_script_code(self):
@@ -62,6 +63,28 @@ def test_cmd_args_constructed_script_code(self):
             u'fs -ls /test'
         )
 
+    def test_cmd_args_post_cmd_args(self):
+        """Test HadoopJob constructed cmd args with post cmd args."""
+
+        job = pygenie.jobs.HadoopJob() \
+            .command("fs -ls /test") \
+            .property('prop1', 'v1') \
+            .property('prop2', 'v2') \
+            .property_file('/Users/hadoop/props.conf') \
+            .post_cmd_args('a') \
+            .post_cmd_args(['a', 'b', 'c']) \
+            .post_cmd_args('d e f')
+
+        assert_equals(
+            ' '.join([
+                '-conf props.conf',
+                '-Dprop1=v1 -Dprop2=v2',
+                'fs -ls /test',
+                'a b c d e f'
+            ]),
+            job.cmd_args
+        )
+
 
 @patch.dict('os.environ', {'GENIE_BYPASS_HOME_CONFIG': '1'})
 class TestingHadoopJobRepr(unittest.TestCase):
@@ -99,7 +122,7 @@ def test_repr(self, is_file):
             .property('mapred.1', 'p1') \
             .property('mapred.2', 'p2') \
             .property_file('/Users/hadoop/test.conf') \
-            .script('jar testing.jar') \
+            .script('jar testing.jar')
 
         assert_equals(
             str(job),

genie-client/src/main/python/tests/job_tests/test_hivejob.py

@@ -89,6 +89,32 @@ def test_cmd_args_constructed_script_file(self, is_file):
             ])
         )
 
+    @patch('pygenie.jobs.hive.is_file')
+    def test_cmd_args_post_cmd_args(self, is_file):
+        """Test HiveJob constructed cmd args with post cmd args."""
+
+        is_file.return_value = True
+
+        job = pygenie.jobs.HiveJob() \
+            .script('/Users/hive/test.hql') \
+            .parameter('hello', 'hi') \
+            .parameter('goodbye', 'bye') \
+            .property('p1', 'v1') \
+            .property('p2', 'v2') \
+            .post_cmd_args('a') \
+            .post_cmd_args(['a', 'b', 'c']) \
+            .post_cmd_args('d e f')
+
+        assert_equals(
+            " ".join([
+                "--hiveconf p2=v2 --hiveconf p1=v1",
+                "-i _hive_parameters.txt",
+                "-f test.hql",
+                "a b c d e f"
+            ]),
+            job.cmd_args
+        )
+
 
 @patch.dict('os.environ', {'GENIE_BYPASS_HOME_CONFIG': '1'})
 class TestingHiveJobParameters(unittest.TestCase):

genie-client/src/main/python/tests/job_tests/test_pigjob.py

@@ -98,6 +98,35 @@ def test_cmd_args_constructed_script_file(self, is_file):
             ])
         )
 
+    @patch('pygenie.jobs.pig.is_file')
+    def test_cmd_args_post_cmd_args(self, is_file):
+        """Test PigJob constructed cmd args with post cmd args."""
+
+        is_file.return_value = True
+
+        job = pygenie.jobs.PigJob() \
+            .script('/Users/pig/test.pig') \
+            .parameter('p', 'v') \
+            .parameter_file('/Users/pig/p.params') \
+            .property('p1', 'v1') \
+            .property('p2', 'v2') \
+            .property_file('/Users/pig/p.conf') \
+            .post_cmd_args('a') \
+            .post_cmd_args(['a', 'b', 'c']) \
+            .post_cmd_args('d e f')
+
+        assert_equals(
+            " ".join([
+                "-Dp2=v2 -Dp1=v1",
+                "-P p.conf",
+                "-param_file p.params",
+                "-param_file _pig_parameters.txt",
+                "-f test.pig",
+                "a b c d e f"
+            ]),
+            job.cmd_args
+        )
+
 
 @patch.dict('os.environ', {'GENIE_BYPASS_HOME_CONFIG': '1'})
 class TestingPigParameterFile(unittest.TestCase):

genie-client/src/main/python/tests/job_tests/test_prestojob.py

@@ -81,6 +81,28 @@ def test_cmd_args_constructed_script_file(self, is_file):
             u'--session s2=v2 --session s1=v1 --debug --source tester --opt1 val1 -f test.presto'
         )
 
+    @patch('pygenie.jobs.presto.is_file')
+    def test_cmd_args_post_cmd_args(self, is_file):
+        """Test PrestoJob constructed cmd args with post cmd args."""
+
+        is_file.return_value = True
+
+        job = pygenie.jobs.PrestoJob() \
+            .script('/Users/presto/test.presto') \
+            .option('source', 'tester') \
+            .option('opt1', 'val1') \
+            .option('debug') \
+            .session('s1', 'v1') \
+            .session('s2', 'v2') \
+            .post_cmd_args('a') \
+            .post_cmd_args(['a', 'b', 'c']) \
+            .post_cmd_args('d e f')
+
+        assert_equals(
+            '--session s2=v2 --session s1=v1 --debug --source tester --opt1 val1 -f test.presto a b c d e f',
+            job.cmd_args
+        )
+
 
 @patch.dict('os.environ', {'GENIE_BYPASS_HOME_CONFIG': '1'})
 class TestingPrestoJobRepr(unittest.TestCase):

genie-client/src/main/python/tests/job_tests/test_sqoopjob.py

@@ -72,6 +72,36 @@ def test_cmd_args_constructed_script_code(self):
             ])
         )
 
+    def test_cmd_args_post_cmd_args(self):
+        """Test SqoopJob constructed cmd args with post cmd args."""
+
+        job = pygenie.jobs.SqoopJob() \
+            .cmd('export') \
+            .option('option1', 'value1') \
+            .option('option2', 'value2') \
+            .option('option3', 'value3') \
+            .option('verbose') \
+            .property('prop1', 'value1') \
+            .property('prop2', 'value2', flag='-X') \
+            .connect('jdbc://test') \
+            .password('passw0rd') \
+            .split_by('col1') \
+            .table('mytable') \
+            .target_dir('/path/to/output') \
+            .post_cmd_args('a') \
+            .post_cmd_args(['a', 'b', 'c']) \
+            .post_cmd_args('d e f')
+
+        assert_equals(
+            ' '.join([
+                'export',
+                '-Dprop1=value1',
+                '--options-file _sqoop_options.txt',
+                'a b c d e f'
+            ]),
+            job.cmd_args
+        )
+
 
 @patch.dict('os.environ', {'GENIE_BYPASS_HOME_CONFIG': '1'})
 class TestingSqoopJobRepr(unittest.TestCase):