From fb093038f182367f6b49c3fb292ced6310879382 Mon Sep 17 00:00:00 2001 From: Aneil Mallavarapu Date: Fri, 19 Sep 2014 15:23:51 -0700 Subject: [PATCH 1/3] Ensure wait_for_job accepts a job_id as well as a resource --- bigquery/client.py | 3 +- bigquery/tests/test_client.py | 76 ++++++++++++++++++++++------------- 2 files changed, 50 insertions(+), 29 deletions(-) diff --git a/bigquery/client.py b/bigquery/client.py index 5dc360f..4f6c949 100644 --- a/bigquery/client.py +++ b/bigquery/client.py @@ -649,6 +649,7 @@ def wait_for_job(self, job, interval=5, timeout=None): Waits until the job indicated by job_resource is done or has failed Args: job: dict, representing a BigQuery job resource + or str, representing a BigQuery job id interval: optional float polling interval in seconds, default = 5 timeout: optional float timeout in seconds, default = None Returns: @@ -659,7 +660,7 @@ def wait_for_job(self, job, interval=5, timeout=None): JobExecutingException on http/auth failures or error in result """ complete = False - job_id = job['jobReference']['jobId'] + job_id = job if isinstance(job, (str,unicode)) else job['jobReference']['jobId'] job_resource = None start_time = time() diff --git a/bigquery/tests/test_client.py b/bigquery/tests/test_client.py index 00cfc36..27f8a15 100644 --- a/bigquery/tests/test_client.py +++ b/bigquery/tests/test_client.py @@ -434,6 +434,26 @@ def side_effect(*args, **kwargs): self.assertEqual(self.api_mock.jobs().get().execute.call_count, 2) self.assertIsInstance(job_resource, dict) + def test_accepts_job_id(self): + """Ensure it accepts a job Id rather than a full job resource""" + + return_values = [{'status': {'state': u'RUNNING'}, + 'jobReference': {'jobId': "testJob"}}, + {'status': {'state': u'DONE'}, + 'jobReference': {'jobId': "testJob"}}] + + def side_effect(*args, **kwargs): + return return_values.pop(0) + + self.api_mock.jobs().get().execute.side_effect = side_effect + + job_resource = self.client.wait_for_job("testJob", + interval=.01, + timeout=None) + + self.assertEqual(self.api_mock.jobs().get().execute.call_count, 2) + self.assertIsInstance(job_resource, dict) + def test_respects_timeout(self): """Ensure that the wait can be timed out""" incomplete_job = {'status': {'state': u'RUNNING'}, @@ -470,10 +490,10 @@ def test_wait_job_http_error(self): expected_result = { "error": { "errors": [{ - "domain": "global", - "reason": "required", - "message": "Required parameter is missing" - }], + "domain": "global", + "reason": "required", + "message": "Required parameter is missing" + }], "code": 400, "message": "Required parameter is missing" } @@ -718,10 +738,10 @@ def test_import_http_error(self): expected_result = { "error": { "errors": [{ - "domain": "global", - "reason": "required", - "message": "Required parameter is missing" - }], + "domain": "global", + "reason": "required", + "message": "Required parameter is missing" + }], "code": 400, "message": "Required parameter is missing" } @@ -779,7 +799,7 @@ def test_export(self, mock_generate_hex): "jobReference": { "projectId": self.project_id, "jobId": "%s-%s-destinationuri" % - (self.dataset_id, self.table_id) + (self.dataset_id, self.table_id) }, "configuration": { "extract": { @@ -815,10 +835,10 @@ def test_export_http_error(self): expected_result = { "error": { "errors": [{ - "domain": "global", - "reason": "required", - "message": "Required parameter is missing" - }], + "domain": "global", + "reason": "required", + "message": "Required parameter is missing" + }], "code": 400, "message": "Required parameter is missing" } @@ -906,10 +926,10 @@ def test_write_http_error(self): expected_result = { "error": { "errors": [{ - "domain": "global", - "reason": "required", - "message": "Required parameter is missing" - }], + "domain": "global", + "reason": "required", + "message": "Required parameter is missing" + }], "code": 400, "message": "Required parameter is missing" } @@ -953,12 +973,12 @@ def test_multi_inside_range(self): bq = client.BigQueryClient(None, 'project') tables = bq._filter_tables_by_time({ - 'Spider-Man': 1370002001, - 'Daenerys Targaryen': 1370001999, - 'Gordon Freeman': 1369999999, - 'William Shatner': 1370001000, - 'Heavy Weapons Guy': 0 - }, 1370002000, 1370000000) + 'Spider-Man': 1370002001, + 'Daenerys Targaryen': 1370001999, + 'Gordon Freeman': 1369999999, + 'William Shatner': 1370001000, + 'Heavy Weapons Guy': 0 + }, 1370002000, 1370000000) self.assertEqual( ['Daenerys Targaryen', 'William Shatner', 'Gordon Freeman'], @@ -973,11 +993,11 @@ def test_not_inside_range(self): bq = client.BigQueryClient(None, 'project') tables = bq._filter_tables_by_time({ - 'John Snow': 9001, - 'Adam West': 100000000000000, - 'Glados': -1, - 'Potato': 0, - }, 1370002000, 1370000000) + 'John Snow': 9001, + 'Adam West': 100000000000000, + 'Glados': -1, + 'Potato': 0, + }, 1370002000, 1370000000) self.assertEqual([], tables) From 280d8a59dbe1effdd63f79b1b72144f607ddaccd Mon Sep 17 00:00:00 2001 From: Aneil Mallavarapu Date: Fri, 19 Sep 2014 15:25:57 -0700 Subject: [PATCH 2/3] Fix flake8 --- bigquery/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigquery/client.py b/bigquery/client.py index 4f6c949..37bd657 100644 --- a/bigquery/client.py +++ b/bigquery/client.py @@ -660,7 +660,8 @@ def wait_for_job(self, job, interval=5, timeout=None): JobExecutingException on http/auth failures or error in result """ complete = False - job_id = job if isinstance(job, (str,unicode)) else job['jobReference']['jobId'] + job_id = job if isinstance(job, (str, unicode)) else \ + job['jobReference']['jobId'] job_resource = None start_time = time() From d29f5ba0e8ca181453ea174257d336210392e046 Mon Sep 17 00:00:00 2001 From: Aneil Mallavarapu Date: Wed, 24 Sep 2014 10:26:02 -0700 Subject: [PATCH 3/3] Fix flake8 --- bigquery/client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bigquery/client.py b/bigquery/client.py index 37bd657..c600b57 100644 --- a/bigquery/client.py +++ b/bigquery/client.py @@ -666,11 +666,11 @@ def wait_for_job(self, job, interval=5, timeout=None): start_time = time() elapsed_time = 0 - while not (complete - or (timeout is not None and elapsed_time > timeout)): + while not (complete or (timeout is not None + and elapsed_time > timeout)): sleep(interval) - request = self.bigquery.jobs().get(projectId=self.project_id, - jobId=job_id) + request = self.bigquery.jobs().get( + projectId=self.project_id, jobId=job_id) job_resource = request.execute() self._raise_executing_exception_if_error(job_resource) complete = job_resource.get('status').get('state') == u'DONE'