From 8fffbf8d0b42b90d4a54981d41e4502ac839d551 Mon Sep 17 00:00:00 2001
From: Przemyslaw Jasinski
Date: Mon, 3 Dec 2018 10:55:54 +0100
Subject: [PATCH 1/2] YACHT-1124: Async copy job handle 403 errors
---
.../copy_job/copy_job_service.py | 5 +-
.../copy_job/test_copy_job_service.py | 53 +++++++++++++++++++
2 files changed, 57 insertions(+), 1 deletion(-)
diff --git a/src/commons/big_query/copy_job_async/copy_job/copy_job_service.py b/src/commons/big_query/copy_job_async/copy_job/copy_job_service.py
index 89c67ad..00bb4e4 100644
--- a/src/commons/big_query/copy_job_async/copy_job/copy_job_service.py
+++ b/src/commons/big_query/copy_job_async/copy_job/copy_job_service.py
@@ -79,7 +79,10 @@ def __schedule(source_big_query_table, target_big_query_table, job_id,
logging.info("Successfully insert: %s", job_reference)
return job_reference
except HttpError as bq_error:
- if bq_error.resp.status == 404:
+ if bq_error.resp.status == 403:
+ logging.exception('403 while creating Copy Job from %s to %s' % (source_big_query_table, target_big_query_table))
+ return None
+ elif bq_error.resp.status == 404:
logging.exception('404 while creating Copy Job from %s to %s' % (source_big_query_table, target_big_query_table))
return None
elif bq_error.resp.status == 409:
diff --git a/tests/commons/big_query/copy_job_async/copy_job/test_copy_job_service.py b/tests/commons/big_query/copy_job_async/copy_job/test_copy_job_service.py
index 13bd127..c5f6c0f 100644
--- a/tests/commons/big_query/copy_job_async/copy_job/test_copy_job_service.py
+++ b/tests/commons/big_query/copy_job_async/copy_job/test_copy_job_service.py
@@ -218,6 +218,59 @@ def test_that_copy_table_should_create_correct_post_copy_action_if_404_http_erro
}
)
+ @patch.object(BigQuery, 'insert_job')
+ @patch.object(TaskCreator, 'create_post_copy_action')
+ def test_that_copy_table_should_create_correct_post_copy_action_if_403_http_error_thrown_on_copy_job_creation(
+ self, create_post_copy_action, insert_job):
+ # given
+ insert_job.side_effect = HttpError(Mock(status=403), 'Forbidden')
+ post_copy_action_request = PostCopyActionRequest(url='/my/url', data={
+ 'key1': 'value1'})
+ request = CopyJobRequest(
+ task_name_suffix='task_name_suffix',
+ copy_job_type_id='test-process',
+ source_big_query_table=self.example_source_bq_table,
+ target_big_query_table=self.example_target_bq_table,
+ create_disposition="CREATE_IF_NEEDED",
+ write_disposition="WRITE_EMPTY",
+ retry_count=0,
+ post_copy_action_request=post_copy_action_request
+ )
+
+ # when
+ CopyJobService().run_copy_job_request(request)
+
+ # then
+ create_post_copy_action.assert_called_once_with(
+ copy_job_type_id='test-process',
+ post_copy_action_request=post_copy_action_request,
+ job_json={
+ 'status': {
+ 'state': 'DONE',
+ 'errors': [
+ {
+ 'reason': 'invalid',
+ 'message': 'Job not scheduled'
+ }
+ ]
+ },
+ 'configuration': {
+ 'copy': {
+ 'sourceTable': {
+ 'projectId': self.example_source_bq_table.get_project_id(),
+ 'tableId': self.example_source_bq_table.get_table_id(),
+ 'datasetId': self.example_source_bq_table.get_dataset_id()
+ },
+ 'destinationTable': {
+ 'projectId': self.example_target_bq_table.get_project_id(),
+ 'tableId': self.example_target_bq_table.get_table_id(),
+ 'datasetId': self.example_target_bq_table.get_dataset_id()
+ }
+ }
+ }
+ }
+ )
+
@patch('src.commons.big_query.big_query_table_metadata.BigQueryTableMetadata')
@patch.object(TaskCreator, 'create_copy_job_result_check')
@patch.object(CopyJobService, '_create_random_job_id',
From b21897046e714c2c251f2933fce78823d00a9f57 Mon Sep 17 00:00:00 2001
From: Przemyslaw Jasinski
Date: Tue, 4 Dec 2018 13:57:50 +0100
Subject: [PATCH 2/2] YACHT-1124: CopyJobService handles AccessDenied and
throws all other 403s
---
.../copy_job_async/copy_job/copy_job_service.py | 6 +++---
.../copy_job_async/copy_job/test_copy_job_service.py | 11 +++++++++--
2 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/src/commons/big_query/copy_job_async/copy_job/copy_job_service.py b/src/commons/big_query/copy_job_async/copy_job/copy_job_service.py
index 00bb4e4..c44bef8 100644
--- a/src/commons/big_query/copy_job_async/copy_job/copy_job_service.py
+++ b/src/commons/big_query/copy_job_async/copy_job/copy_job_service.py
@@ -79,9 +79,9 @@ def __schedule(source_big_query_table, target_big_query_table, job_id,
logging.info("Successfully insert: %s", job_reference)
return job_reference
except HttpError as bq_error:
- if bq_error.resp.status == 403:
- logging.exception('403 while creating Copy Job from %s to %s' % (source_big_query_table, target_big_query_table))
- return None
+ if bq_error.resp.status == 403 and bq_error._get_reason().startswith('Access Denied'):
+ logging.exception('403 while creating Copy Job from %s to %s' % (source_big_query_table, target_big_query_table))
+ return None
elif bq_error.resp.status == 404:
logging.exception('404 while creating Copy Job from %s to %s' % (source_big_query_table, target_big_query_table))
return None
diff --git a/tests/commons/big_query/copy_job_async/copy_job/test_copy_job_service.py b/tests/commons/big_query/copy_job_async/copy_job/test_copy_job_service.py
index c5f6c0f..63e42f9 100644
--- a/tests/commons/big_query/copy_job_async/copy_job/test_copy_job_service.py
+++ b/tests/commons/big_query/copy_job_async/copy_job/test_copy_job_service.py
@@ -220,10 +220,17 @@ def test_that_copy_table_should_create_correct_post_copy_action_if_404_http_erro
@patch.object(BigQuery, 'insert_job')
@patch.object(TaskCreator, 'create_post_copy_action')
- def test_that_copy_table_should_create_correct_post_copy_action_if_403_http_error_thrown_on_copy_job_creation(
+ def test_that_copy_table_should_create_correct_post_copy_action_if_access_denied_http_error_thrown_on_copy_job_creation(
self, create_post_copy_action, insert_job):
# given
- insert_job.side_effect = HttpError(Mock(status=403), 'Forbidden')
+ http_error_content = "{\"error\": " \
+ " {\"errors\": [" \
+ " {\"reason\": \"Access Denied\"," \
+ " \"message\": \"Access Denied\"" \
+ " }]," \
+ " \"code\": 403,\"" \
+ " message\": \"Access Denied\"}}"
+ insert_job.side_effect = HttpError(Mock(status=403), http_error_content)
post_copy_action_request = PostCopyActionRequest(url='/my/url', data={
'key1': 'value1'})
request = CopyJobRequest(