Merge b218970 into fe468a7

ocadotechnology · Dec 4, 2018 · fec47ba · fec47ba
2 parents fe468a7 + b218970
commit fec47ba
Show file tree

Hide file tree

Showing 2 changed files with 64 additions and 1 deletion.
diff --git a/src/commons/big_query/copy_job_async/copy_job/copy_job_service.py b/src/commons/big_query/copy_job_async/copy_job/copy_job_service.py
@@ -79,7 +79,10 @@ def __schedule(source_big_query_table, target_big_query_table, job_id,
             logging.info("Successfully insert: %s", job_reference)
             return job_reference
         except HttpError as bq_error:
-            if bq_error.resp.status == 404:
+            if bq_error.resp.status == 403 and bq_error._get_reason().startswith('Access Denied'):
+                    logging.exception('403 while creating Copy Job from %s to %s' % (source_big_query_table, target_big_query_table))
+                    return None
+            elif bq_error.resp.status == 404:
                 logging.exception('404 while creating Copy Job from %s to %s' % (source_big_query_table, target_big_query_table))
                 return None
             elif bq_error.resp.status == 409:

diff --git a/tests/commons/big_query/copy_job_async/copy_job/test_copy_job_service.py b/tests/commons/big_query/copy_job_async/copy_job/test_copy_job_service.py
@@ -218,6 +218,66 @@ def test_that_copy_table_should_create_correct_post_copy_action_if_404_http_erro
             }
         )
 
+    @patch.object(BigQuery, 'insert_job')
+    @patch.object(TaskCreator, 'create_post_copy_action')
+    def test_that_copy_table_should_create_correct_post_copy_action_if_access_denied_http_error_thrown_on_copy_job_creation(
+            self, create_post_copy_action, insert_job):
+        # given
+        http_error_content = "{\"error\": " \
+                             "  {\"errors\": [" \
+                             "    {\"reason\": \"Access Denied\"," \
+                             "     \"message\": \"Access Denied\"" \
+                             "  }]," \
+                             "  \"code\": 403,\"" \
+                             "  message\": \"Access Denied\"}}"
+        insert_job.side_effect = HttpError(Mock(status=403), http_error_content)
+        post_copy_action_request = PostCopyActionRequest(url='/my/url', data={
+            'key1': 'value1'})
+        request = CopyJobRequest(
+            task_name_suffix='task_name_suffix',
+            copy_job_type_id='test-process',
+            source_big_query_table=self.example_source_bq_table,
+            target_big_query_table=self.example_target_bq_table,
+            create_disposition="CREATE_IF_NEEDED",
+            write_disposition="WRITE_EMPTY",
+            retry_count=0,
+            post_copy_action_request=post_copy_action_request
+        )
+
+        # when
+        CopyJobService().run_copy_job_request(request)
+
+        # then
+        create_post_copy_action.assert_called_once_with(
+            copy_job_type_id='test-process',
+            post_copy_action_request=post_copy_action_request,
+            job_json={
+                'status': {
+                    'state': 'DONE',
+                    'errors': [
+                        {
+                            'reason': 'invalid',
+                            'message': 'Job not scheduled'
+                        }
+                    ]
+                },
+                'configuration': {
+                    'copy': {
+                        'sourceTable': {
+                            'projectId': self.example_source_bq_table.get_project_id(),
+                            'tableId': self.example_source_bq_table.get_table_id(),
+                            'datasetId': self.example_source_bq_table.get_dataset_id()
+                        },
+                        'destinationTable': {
+                            'projectId': self.example_target_bq_table.get_project_id(),
+                            'tableId': self.example_target_bq_table.get_table_id(),
+                            'datasetId': self.example_target_bq_table.get_dataset_id()
+                        }
+                    }
+                }
+            }
+        )
+
     @patch('src.commons.big_query.big_query_table_metadata.BigQueryTableMetadata')
     @patch.object(TaskCreator, 'create_copy_job_result_check')
     @patch.object(CopyJobService, '_create_random_job_id',