Merge pull request #159 from Morgenz/YACHT-1259

YACHT-1259 Add exception handling for on-demand table backup, code re…

src/backup/abstract_backup_predicate.py

@@ -12,13 +12,15 @@ def test(self, big_query_table_metadata, table_entity):
     def _is_possible_to_copy_table(big_query_table_metadata):
         if not big_query_table_metadata.table_exists():
             logging.info('Table not found (404)')
-            return False
+            return False, "Table not found"
         if not big_query_table_metadata.is_schema_defined():
             logging.info('This table is without schema')
-            return False
+            return False, "This table is without schema"
         if big_query_table_metadata.is_external_or_view_type():
-            return False
+            logging.info('This table is external or view type')
+            return False, "This table is external or view type"
         if big_query_table_metadata.is_empty():
             logging.info('This table is empty')
+            return True, "This table is empty"
 
-        return True
+        return True, "This table is valid"

src/backup/default_backup_predicate.py

@@ -2,13 +2,15 @@
 
 from src.backup.abstract_backup_predicate import \
     AbstractBackupPredicate
+from src.commons.exceptions import ParameterValidationException, NotFoundException
 
 
 class DefaultBackupPredicate(AbstractBackupPredicate):
     TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S'
 
     def test(self, big_query_table_metadata, table_entity):
-        if not self._is_possible_to_copy_table(big_query_table_metadata):
+        table_validation_status, table_validation_message = self._is_possible_to_copy_table(big_query_table_metadata)
+        if not table_validation_status:
             return False
 
         last_backup = self.__get_last_table_backup_if_any(table_entity)

src/backup/on_demand/on_demand_backup_predicate.py

@@ -2,17 +2,26 @@
 
 from src.backup.abstract_backup_predicate import \
     AbstractBackupPredicate
+from src.commons.exceptions import ParameterValidationException, NotFoundException
 
 
 class OnDemandBackupPredicate(AbstractBackupPredicate):
 
     def test(self, big_query_table_metadata, table_entity):
-        if not self._is_possible_to_copy_table(big_query_table_metadata):
-            return False
+        if big_query_table_metadata.is_daily_partitioned() and not big_query_table_metadata.is_partition():
+            raise ParameterValidationException("Partition id is required for partitioned table in on-demand mode")
+
+        table_validation_status, table_validation_message = self._is_possible_to_copy_table(big_query_table_metadata)
+
+        if not table_validation_status:
+            if table_validation_message == "Table not found":
+                raise NotFoundException(table_validation_message)
+            else:
+                raise ParameterValidationException(table_validation_message)
 
         logging.info("Performing on-demand backup for %s."
                      "It is performed without checking "
-                     "if table aready has up to date backup",
+                     "if table already has up to date backup",
                      big_query_table_metadata.table_reference())
 
         return True

src/backup/on_demand/on_demand_table_backup.py

@@ -1,6 +1,6 @@
 from src.backup.backup_process import BackupProcess
 from src.backup.on_demand.on_demand_backup_predicate import \
-  OnDemandBackupPredicate
+    OnDemandBackupPredicate
 from src.commons.big_query.big_query import BigQuery
 from src.commons.big_query.big_query_table_metadata import BigQueryTableMetadata
 from src.commons.exceptions import ParameterValidationException
@@ -16,9 +16,6 @@ class OnDemandTableBackup(object):
     def start(table_reference):
         big_query_table_metadata = BigQueryTableMetadata.get_table_by_reference(table_reference)
 
-        if big_query_table_metadata.is_daily_partitioned() and not big_query_table_metadata.is_partition():
-            raise ParameterValidationException("Partition id is required for partitioned table in on-demand mode")
-
         BackupProcess(table_reference=table_reference,
                       big_query=BigQuery(),
                       big_query_table_metadata=big_query_table_metadata,

tests/backup/on_demand/test_on_demand_backup_predicate.py

@@ -10,6 +10,7 @@
 from src.backup.on_demand.on_demand_backup_predicate import \
     OnDemandBackupPredicate
 from src.commons.big_query.big_query_table_metadata import BigQueryTableMetadata
+from src.commons.exceptions import ParameterValidationException, NotFoundException
 
 
 class TestOnDemandBackupPredicate(unittest.TestCase):
@@ -48,46 +49,40 @@ def tearDown(self):
         patch.stopall()
 
     @patch.object(BigQueryTableMetadata, 'is_schema_defined', return_value=False)
-    def test_should_return_false_if_schema_is_not_defined(self, _):
+    def test_should_raise_ParameterValidationException_if_schema_is_not_defined(self, _):
         # given
         predicate = OnDemandBackupPredicate()
-        # when
-        result = predicate.test(self.big_query_table_metadata, self.table)
-        # then
-        self.assertFalse(result, "OnDemandShouldBackupPredicate should return FALSE "
-                                "if table has no schema")
-
-    @patch.object(BigQueryTableMetadata, 'is_empty', return_value=True)
-    def test_should_return_true_if_table_is_empty(self, _):
-        # given
-        predicate = OnDemandBackupPredicate()
-        # when
-        result = predicate.test(self.big_query_table_metadata, self.table)
-        # then
-        self.assertTrue(result, "OnDemandShouldBackupPredicate should return TRUE "
-                                "if table is empty")
+        # when-then
+        with self.assertRaises(ParameterValidationException):
+            predicate.test(self.big_query_table_metadata, self.table)
 
     @patch.object(BigQueryTableMetadata, 'table_exists',
                   return_value=False)
-    def test_should_return_false_if_table_not_exist_anymore(self, _):
+    def test_should_raise_NotFoundException_if_table_not_exist_anymore(self, _):
         # given
         predicate = OnDemandBackupPredicate()
-        # when
-        result = predicate.test(self.big_query_table_metadata, self.table)
-        # then
-        self.assertFalse(result, "OnDemandShouldBackupPredicate should return FALSE "
-                                 "if table not exists")
+        # when-then
+        with self.assertRaises(NotFoundException):
+            predicate.test(self.big_query_table_metadata, self.table)
 
     @patch.object(BigQueryTableMetadata, 'is_external_or_view_type',
                   return_value=True)
-    def test_should_return_false_if_table_is_external_or_view_type(self, _):
+    def test_should_raise_ParameterValidationException_if_table_is_external_or_view_type(self, _):
+        # given
+        predicate = OnDemandBackupPredicate()
+        # when-then
+        with self.assertRaises(ParameterValidationException):
+            predicate.test(self.big_query_table_metadata, self.table)
+
+    @patch.object(BigQueryTableMetadata, 'is_empty', return_value=True)
+    def test_should_return_true_if_table_is_empty(self, _):
         # given
         predicate = OnDemandBackupPredicate()
         # when
         result = predicate.test(self.big_query_table_metadata, self.table)
         # then
-        self.assertFalse(result, "OnDemandShouldBackupPredicate should return FALSE "
-                                 "if object is table or external type")
+        self.assertTrue(result, "OnDemandBackupPredicate should return TRUE "
+                                "if table is empty")
 
     @patch.object(BigQueryTableMetadata, 'get_last_modified_datetime',
                   return_value=datetime(2016, 11, 13, 15, 00))
@@ -119,7 +114,8 @@ def test_should_return_true_if_table_was_changed_after_last_backup(self, _):
 
     @patch.object(BigQueryTableMetadata, 'get_last_modified_datetime',
                   return_value=datetime(2016, 11, 13, 15, 00))
-    def test_should_return_true_if_table_was_changed_at_the_same_time_when_last_backup(self, _):    # nopep8 pylint: disable=C0301
+    def test_should_return_true_if_table_was_changed_at_the_same_time_when_last_backup(self,
+                                                                                       _):  # nopep8 pylint: disable=C0301
         # given
         backup = Backup(
             parent=self.table.key,
@@ -132,8 +128,8 @@ def test_should_return_true_if_table_was_changed_at_the_same_time_when_last_back
         result = predicate.test(self.big_query_table_metadata, self.table)
         # then
         self.assertTrue(result, "OnDemandShouldBackupPredicate should return False "
-                                 "if table was change at the same time when "
-                                 "last backup was made")
+                                "if table was change at the same time when "
+                                "last backup was made")
 
     @patch.object(BigQueryTableMetadata, 'get_last_modified_datetime',
                   return_value=datetime(2016, 11, 13, 14, 00))
@@ -150,5 +146,5 @@ def test_should_return_true_if_table_was_changed_before_last_backup(self, _):  #
         result = predicate.test(self.big_query_table_metadata, self.table)
         # then
         self.assertTrue(result, "OnDemandShouldBackupPredicate should return FALSE "
-                                 "if table was changed before "
-                                 "last backup was made")
+                                "if table was changed before "
+                                "last backup was made")

tests/backup/on_demand/test_on_demand_table_backup.py

@@ -0,0 +1,35 @@
+import unittest
+
+from mock import patch
+from src.backup.datastore.Table import Table
+from src.backup.on_demand.on_demand_table_backup import OnDemandTableBackup
+from src.commons.big_query.big_query_table_metadata import BigQueryTableMetadata
+from src.commons.exceptions import ParameterValidationException
+from src.commons.table_reference import TableReference
+
+
+class TestOnDemandTableBackup(unittest.TestCase):
+
+    @patch.object(Table, "get_table", return_value=None)
+    @patch('src.commons.big_query.big_query_table_metadata.BigQueryTableMetadata.get_table_by_reference',
+           return_value=BigQueryTableMetadata(
+               {"tableReference": {
+                   "projectId": "test-project",
+                   "datasetId": "test-dataset",
+                   "tableId": "test-table-without-partition"
+               },
+                   "timePartitioning": {
+                       "type": "DAY"
+                   }}
+           ))
+    def test_should_throw_parameter_validation_exception_if_table_is_partitioned_but_partition_number_was_not_given(
+            self, _1, _2):
+        # given
+        table_reference = TableReference(project_id="test-project",
+                                         dataset_id="test-dataset",
+                                         table_id="test-table",
+                                         partition_id="")
+
+        # when-then
+        with self.assertRaises(ParameterValidationException):
+            OnDemandTableBackup.start(table_reference)

tests/backup/on_demand/test_on_demand_table_backup_handler.py

@@ -28,7 +28,7 @@ def tearDown(self):
 
     @patch.object(OnDemandTableBackup, 'start')
     def test_on_demand_request_for_partitioned_table_is_properly_parsed(
-        self, on_demand_table_backup_start):
+            self, on_demand_table_backup_start):
         # given
         table_reference = TableReference('example-proj-name',
                                          'example-dataset-name',
@@ -48,11 +48,11 @@ def test_on_demand_request_for_partitioned_table_is_properly_parsed(
 
     @patch.object(OnDemandTableBackup, 'start')
     def test_on_demand_request_for_non_partitioned_table_is_properly_parsed(
-        self, on_demand_table_backup_start):
+            self, on_demand_table_backup_start):
         # given
         table_reference = TableReference('example-proj-name',
-                                           'example-dataset-name',
-                                           'example-table-name')
+                                         'example-dataset-name',
+                                         'example-table-name')
         url = '/tasks/backups/on_demand/table/{}/{}/{}'.format(
             table_reference.get_project_id(),
             table_reference.get_dataset_id(),
@@ -65,19 +65,19 @@ def test_on_demand_request_for_non_partitioned_table_is_properly_parsed(
         on_demand_table_backup_start.assert_called_with(table_reference)
 
     @patch.object(OnDemandTableBackup, 'start', side_effect=ParameterValidationException("error msg"))
-    def test_on_demand_request_for_partitioned_but_without_passing_partition_should_casue_400(
-        self, on_demand_table_backup_start):
-      # given
-      table_reference = TableReference('example-proj-name',
-                                       'example-dataset-name',
-                                       'example-table-name')
-      url = '/tasks/backups/on_demand/table/{}/{}/{}'.format(
-          table_reference.get_project_id(),
-          table_reference.get_dataset_id(),
-          table_reference.get_table_id())
+    def test_should_return_400_if_parameter_validation_exception(
+            self, on_demand_table_backup_start):
+        # given
+        table_reference = TableReference('example-proj-name',
+                                         'example-dataset-name',
+                                         'example-table-name')
+        url = '/tasks/backups/on_demand/table/{}/{}/{}'.format(
+            table_reference.get_project_id(),
+            table_reference.get_dataset_id(),
+            table_reference.get_table_id())
 
-      # when
-      response = self.under_test.get(url, expect_errors=True)
+        # when
+        response = self.under_test.get(url, expect_errors=True)
 
-      # then
-      self.assertEquals(400, response.status_int)
+        # then
+        self.assertEquals(400, response.status_int)