Merge pull request #303 from irontable/develop

add request JSON data to RunningJob in Python client

genie-client/src/main/python/pygenie/__init__.py

@@ -28,16 +28,15 @@
 import logging
 import pkg_resources
 
-from .adapter.adapter import (execute_job,
-                              get_adapter_for_version)
-from .jobs import core
-from .jobs import running
-
-
 __version__ = pkg_resources.get_distribution('nflx-genie-client').version
 
 # get around circular imports
 # adapter imports jobs, jobs need to import execute_job
 # adapter imports RunningJob, RunningJob needs to import get_adapter_for_version
+from .adapter.adapter import (execute_job,
+                              get_adapter_for_version)
+from .jobs import core
+from .jobs import running
+
 core.execute_job = execute_job
 running.get_adapter_for_version = get_adapter_for_version

genie-client/src/main/python/pygenie/adapter/genie_2.py

@@ -177,15 +177,15 @@ def construct_base_payload(job):
 
         return payload
 
-    def get(self, job_id, path=None):
+    def get(self, job_id, path=None, **kwargs):
         """Get information."""
 
         url = self.__url_for_job(job_id)
         if path:
             url = '{}/{}'.format(url, path.lstrip('/'))
 
         try:
-            return call(method='get', url=url, timeout=10).json()
+            return call(method='get', url=url, **kwargs).json()
         except GenieHTTPError as err:
             if err.response.status_code == 404:
                 raise GenieJobNotFoundError("job not found at {}".format(url))
@@ -196,7 +196,7 @@ def get_info_for_rj(self, job_id, *args, **kwargs):
         Get information for RunningJob object.
         """
 
-        data = self.get(job_id)
+        data = self.get(job_id, timeout=10)
 
         file_dependencies = data.get('fileDependencies')
 
@@ -224,6 +224,7 @@ def get_info_for_rj(self, job_id, *args, **kwargs):
             'kill_uri': data.get('killURI'),
             'name': data.get('name'),
             'output_uri': data.get('outputURI'),
+            'request_data': dict(),
             'setup_file': data.get('envPropFile'),
             'started': data.get('started'),
             'status': data.get('status'),
@@ -243,7 +244,7 @@ def get_genie_log(self, job_id, **kwargs):
     def get_status(self, job_id):
         """Get job status."""
 
-        return self.get(job_id, path='status').get('status')
+        return self.get(job_id, path='status', timeout=10).get('status')
 
     def get_stderr(self, job_id, **kwargs):
         """Get a stderr log for a job."""

genie-client/src/main/python/pygenie/adapter/genie_3.py

@@ -165,7 +165,7 @@ def construct_base_payload(job):
 
         return payload
 
-    def get(self, job_id, path=None, if_not_found=None):
+    def get(self, job_id, path=None, if_not_found=None, **kwargs):
         """
         Get information for a job.
 
@@ -188,8 +188,9 @@ def get(self, job_id, path=None, if_not_found=None):
         try:
             return call(method='get',
                         url=url,
-                        timeout=60,
-                        auth_handler=self.auth_handler).json()
+                        auth_handler=self.auth_handler,
+                        **kwargs) \
+                   .json()
         except GenieHTTPError as err:
             if err.response.status_code == 404:
                 msg = "job not found at {}".format(url)
@@ -204,20 +205,24 @@ def get_info_for_rj(self, job_id, *args, **kwargs):
         Get information for RunningJob object.
         """
 
-        data = self.get(job_id)
-        request_data = self.get(job_id, path='request')
+        data = self.get(job_id, timeout=60)
+        request_data = self.get(job_id, path='request', timeout=60)
         application_data = self.get(job_id,
                                     path='applications',
-                                    if_not_found=list())
+                                    if_not_found=list(),
+                                    timeout=60)
         cluster_data = self.get(job_id,
                                 path='cluster',
-                                if_not_found=dict())
+                                if_not_found=dict(),
+                                timeout=60)
         command_data = self.get(job_id,
                                 path='command',
-                                if_not_found=dict())
+                                if_not_found=dict(),
+                                timeout=60)
         execution_data = self.get(job_id,
                                   path='execution',
-                                  if_not_found=dict())
+                                  if_not_found=dict(),
+                                  timeout=60)
 
         link = data.get('_links', {}).get('self', {}).get('href')
         link_parts = urlparse(link)
@@ -255,6 +260,7 @@ def get_info_for_rj(self, job_id, *args, **kwargs):
             'kill_uri': link,
             'name': data.get('name'),
             'output_uri': output_link,
+            'request_data': request_data,
             'setup_file': request_data.get('setupFile'),
             'started': data.get('started'),
             'status': data.get('status'),
@@ -273,7 +279,7 @@ def get_genie_log(self, job_id, **kwargs):
     def get_status(self, job_id):
         """Get job status."""
 
-        return self.get(job_id, path='status').get('status')
+        return self.get(job_id, path='status', timeout=10).get('status')
 
     def get_stderr(self, job_id, **kwargs):
         """Get a stderr log for a job."""

genie-client/src/main/python/pygenie/jobs/running.py

@@ -340,14 +340,30 @@ def output_uri(self):
         Get the output uri for the job.
 
         Example:
-            >>> print running_job.output_uri
+            >>> running_job.output_uri
             'http://localhost/genie/1234-abcd/output'
 
         Returns:
             str: The output URI.
         """
+
         return self.info.get('output_uri')
 
+    @property
+    def request_data(self):
+        """
+        Get the JSON of the job submission request sent to Genie.
+
+        Example:
+            >>> running_job.request_data
+            {...}
+
+        Returns:
+            dict: JSON of the job submission request.
+        """
+
+        return self.info.get('request_data')
+
     @property
     def start_time(self):
         """

genie-client/src/main/python/setup.py

@@ -26,7 +26,7 @@
 
 setup(
     name='nflx-genie-client',
-    version='3.0.10',
+    version='3.0.11',
     author='Netflix Inc.',
     author_email='genieoss@googlegroups.com',
     keywords='genie hadoop cloud netflix client bigdata presto',