Merge 3eaeaa5 into 36b1f2d

src/big_query/big_query.py

@@ -67,6 +67,7 @@ def for_each_table(self, project_id, dataset_id, func):
         for table_id in self.list_table_ids(project_id, dataset_id):
             func(project_id, dataset_id, table_id)
 
+    @retry(HttpError, tries=3, delay=2, backoff=2)
     def list_table_ids(self, project_id, dataset_id):
         request = self.service.tables().list(
             projectId=project_id, datasetId=dataset_id
@@ -79,8 +80,7 @@ def list_table_ids(self, project_id, dataset_id):
                     logging.info("Dataset '%s:%s' is not found", project_id,
                                  dataset_id)
                     return
-                else:
-                    raise ex
+                raise ex
 
             if 'tables' in tables:
                 for table in tables['tables']:

tests/big_query/test_big_query.py

@@ -1,5 +1,6 @@
 import unittest
 
+from apiclient.errors import HttpError
 from apiclient.http import HttpMockSequence
 from google.appengine.ext import testbed
 from mock import patch
@@ -60,6 +61,17 @@ def test_iterating_tables(self):
         # then
         self.assertEqual(self.count(tables_ids), 5)
 
+    def test_iterating_tables_should_retry_if_gets_http_503_response_once(self):
+        # given
+        self._create_http.return_value = self.__create_tables_list_responses_with_503()
+
+        # when
+        with self.assertRaises(HttpError) as context:
+            tables_ids = BigQuery().list_table_ids("project1233", "dataset_id")
+        # then
+            self.assertEqual(self.count(tables_ids), 5)
+
+
     def test_when_dataset_not_exist_then_iterating_tables_should_not_return_any_table(self):
         # given
         self._create_http.return_value = self.__create_dataset_not_found_during_tables_list_responses()
@@ -177,6 +189,19 @@ def __create_tables_list_responses():
              content('tests/json_samples/bigquery_table_list_page_last.json'))
         ])
 
+    @staticmethod
+    def __create_tables_list_responses_with_503():
+        return HttpMockSequence([
+            ({'status': '200'},
+             content('tests/json_samples/bigquery_v2_test_schema.json')),
+            ({'status': '503'},
+             content('tests/json_samples/bigquery_table_list_503_error.json')),
+            ({'status': '200'},
+             content('tests/json_samples/bigquery_table_list_page_1.json')),
+            ({'status': '200'},
+             content('tests/json_samples/bigquery_table_list_page_last.json'))
+        ])
+
     @staticmethod
     def __create_dataset_not_found_during_tables_list_responses():
         return HttpMockSequence([

tests/json_samples/bigquery_table_list_503_error.json

@@ -0,0 +1,13 @@
+{
+  "error": {
+    "errors": [
+      {
+        "domain": "",
+        "reason": "",
+        "message": "Error encountered during execution. Retrying may solve the problem."
+      }
+    ],
+    "code": 503,
+    "message": "Error encountered during execution. Retrying may solve the problem."
+  }
+}