Skip to content
This repository has been archived by the owner on Aug 25, 2023. It is now read-only.

YACHT-1124: Async copy job handle 403 errors #99

Merged
merged 2 commits into from
Dec 4, 2018
Merged

YACHT-1124: Async copy job handle 403 errors #99

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8fffbf8
YACHT-1124: Async copy job handle 403 errors
przemyslaw-jasinski Dec 3, 2018
b218970
YACHT-1124: CopyJobService handles AccessDenied and throws all other …
przemyslaw-jasinski Dec 4, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/commons/big_query/copy_job_async/copy_job/copy_job_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,10 @@ def __schedule(source_big_query_table, target_big_query_table, job_id,
logging.info("Successfully insert: %s", job_reference)
return job_reference
except HttpError as bq_error:
if bq_error.resp.status == 404:
if bq_error.resp.status == 403:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

403 could be also quota exceeded, which should be retried

logging.exception('403 while creating Copy Job from %s to %s' % (source_big_query_table, target_big_query_table))
return None
elif bq_error.resp.status == 404:
logging.exception('404 while creating Copy Job from %s to %s' % (source_big_query_table, target_big_query_table))
return None
elif bq_error.resp.status == 409:
Expand Down
53 changes: 53 additions & 0 deletions tests/commons/big_query/copy_job_async/copy_job/test_copy_job_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,59 @@ def test_that_copy_table_should_create_correct_post_copy_action_if_404_http_erro
}
)

@patch.object(BigQuery, 'insert_job')
@patch.object(TaskCreator, 'create_post_copy_action')
def test_that_copy_table_should_create_correct_post_copy_action_if_403_http_error_thrown_on_copy_job_creation(
self, create_post_copy_action, insert_job):
# given
insert_job.side_effect = HttpError(Mock(status=403), 'Forbidden')
post_copy_action_request = PostCopyActionRequest(url='/my/url', data={
'key1': 'value1'})
request = CopyJobRequest(
task_name_suffix='task_name_suffix',
copy_job_type_id='test-process',
source_big_query_table=self.example_source_bq_table,
target_big_query_table=self.example_target_bq_table,
create_disposition="CREATE_IF_NEEDED",
write_disposition="WRITE_EMPTY",
retry_count=0,
post_copy_action_request=post_copy_action_request
)

# when
CopyJobService().run_copy_job_request(request)

# then
create_post_copy_action.assert_called_once_with(
copy_job_type_id='test-process',
post_copy_action_request=post_copy_action_request,
job_json={
'status': {
'state': 'DONE',
'errors': [
{
'reason': 'invalid',
'message': 'Job not scheduled'
}
]
},
'configuration': {
'copy': {
'sourceTable': {
'projectId': self.example_source_bq_table.get_project_id(),
'tableId': self.example_source_bq_table.get_table_id(),
'datasetId': self.example_source_bq_table.get_dataset_id()
},
'destinationTable': {
'projectId': self.example_target_bq_table.get_project_id(),
'tableId': self.example_target_bq_table.get_table_id(),
'datasetId': self.example_target_bq_table.get_dataset_id()
}
}
}
}
)

@patch('src.commons.big_query.big_query_table_metadata.BigQueryTableMetadata')
@patch.object(TaskCreator, 'create_copy_job_result_check')
@patch.object(CopyJobService, '_create_random_job_id',
Expand Down