Skip to content

Ensure wait_for_job accepts a job_id as well as a resource #29

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

Ensure wait_for_job accepts a job_id as well as a resource #29

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fb09303
Ensure wait_for_job accepts a job_id as well as a resource
aneilbaboo Sep 19, 2014
280d8a5
Fix flake8
aneilbaboo Sep 19, 2014
d29f5ba
Fix flake8
aneilbaboo Sep 24, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions bigquery/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -649,6 +649,7 @@ def wait_for_job(self, job, interval=5, timeout=None):
Waits until the job indicated by job_resource is done or has failed
Args:
job: dict, representing a BigQuery job resource
or str, representing a BigQuery job id
interval: optional float polling interval in seconds, default = 5
timeout: optional float timeout in seconds, default = None
Returns:
Expand All @@ -659,16 +660,17 @@ def wait_for_job(self, job, interval=5, timeout=None):
JobExecutingException on http/auth failures or error in result
"""
complete = False
job_id = job['jobReference']['jobId']
job_id = job if isinstance(job, (str, unicode)) else \
job['jobReference']['jobId']
job_resource = None

start_time = time()
elapsed_time = 0
while not (complete
or (timeout is not None and elapsed_time > timeout)):
while not (complete or (timeout is not None
and elapsed_time > timeout)):
sleep(interval)
request = self.bigquery.jobs().get(projectId=self.project_id,
jobId=job_id)
request = self.bigquery.jobs().get(
projectId=self.project_id, jobId=job_id)
job_resource = request.execute()
self._raise_executing_exception_if_error(job_resource)
complete = job_resource.get('status').get('state') == u'DONE'
Expand Down
76 changes: 48 additions & 28 deletions bigquery/tests/test_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -434,6 +434,26 @@ def side_effect(*args, **kwargs):
self.assertEqual(self.api_mock.jobs().get().execute.call_count, 2)
self.assertIsInstance(job_resource, dict)

def test_accepts_job_id(self):
"""Ensure it accepts a job Id rather than a full job resource"""

return_values = [{'status': {'state': u'RUNNING'},
'jobReference': {'jobId': "testJob"}},
{'status': {'state': u'DONE'},
'jobReference': {'jobId': "testJob"}}]

def side_effect(*args, **kwargs):
return return_values.pop(0)

self.api_mock.jobs().get().execute.side_effect = side_effect

job_resource = self.client.wait_for_job("testJob",
interval=.01,
timeout=None)

self.assertEqual(self.api_mock.jobs().get().execute.call_count, 2)
self.assertIsInstance(job_resource, dict)

def test_respects_timeout(self):
"""Ensure that the wait can be timed out"""
incomplete_job = {'status': {'state': u'RUNNING'},
Expand Down Expand Up @@ -470,10 +490,10 @@ def test_wait_job_http_error(self):
expected_result = {
"error": {
"errors": [{
"domain": "global",
"reason": "required",
"message": "Required parameter is missing"
}],
"domain": "global",
"reason": "required",
"message": "Required parameter is missing"
}],
"code": 400,
"message": "Required parameter is missing"
}
Expand Down Expand Up @@ -718,10 +738,10 @@ def test_import_http_error(self):
expected_result = {
"error": {
"errors": [{
"domain": "global",
"reason": "required",
"message": "Required parameter is missing"
}],
"domain": "global",
"reason": "required",
"message": "Required parameter is missing"
}],
"code": 400,
"message": "Required parameter is missing"
}
Expand Down Expand Up @@ -779,7 +799,7 @@ def test_export(self, mock_generate_hex):
"jobReference": {
"projectId": self.project_id,
"jobId": "%s-%s-destinationuri" %
(self.dataset_id, self.table_id)
(self.dataset_id, self.table_id)
},
"configuration": {
"extract": {
Expand Down Expand Up @@ -815,10 +835,10 @@ def test_export_http_error(self):
expected_result = {
"error": {
"errors": [{
"domain": "global",
"reason": "required",
"message": "Required parameter is missing"
}],
"domain": "global",
"reason": "required",
"message": "Required parameter is missing"
}],
"code": 400,
"message": "Required parameter is missing"
}
Expand Down Expand Up @@ -906,10 +926,10 @@ def test_write_http_error(self):
expected_result = {
"error": {
"errors": [{
"domain": "global",
"reason": "required",
"message": "Required parameter is missing"
}],
"domain": "global",
"reason": "required",
"message": "Required parameter is missing"
}],
"code": 400,
"message": "Required parameter is missing"
}
Expand Down Expand Up @@ -953,12 +973,12 @@ def test_multi_inside_range(self):
bq = client.BigQueryClient(None, 'project')

tables = bq._filter_tables_by_time({
'Spider-Man': 1370002001,
'Daenerys Targaryen': 1370001999,
'Gordon Freeman': 1369999999,
'William Shatner': 1370001000,
'Heavy Weapons Guy': 0
}, 1370002000, 1370000000)
'Spider-Man': 1370002001,
'Daenerys Targaryen': 1370001999,
'Gordon Freeman': 1369999999,
'William Shatner': 1370001000,
'Heavy Weapons Guy': 0
}, 1370002000, 1370000000)

self.assertEqual(
['Daenerys Targaryen', 'William Shatner', 'Gordon Freeman'],
Expand All @@ -973,11 +993,11 @@ def test_not_inside_range(self):
bq = client.BigQueryClient(None, 'project')

tables = bq._filter_tables_by_time({
'John Snow': 9001,
'Adam West': 100000000000000,
'Glados': -1,
'Potato': 0,
}, 1370002000, 1370000000)
'John Snow': 9001,
'Adam West': 100000000000000,
'Glados': -1,
'Potato': 0,
}, 1370002000, 1370000000)

self.assertEqual([], tables)

Expand Down