Merge f1fbde3 into 3a0881d

src/commons/big_query/big_query.py

@@ -111,15 +111,15 @@ def execute_query(self, query, use_legacy_sql=True):
         page_token = None
 
         while True:
-            page = self.service.jobs().getQueryResults(
+            response = self.service.jobs().getQueryResults(
                 pageToken=page_token,
                 **query_job['jobReference']).execute(num_retries=2)
-
-            results.extend(page.get('rows', []))
-
-            page_token = page.get('pageToken')
-            if not page_token:
-                break
+            job_complete = response.get('jobComplete')
+            if job_complete:
+                results.extend(response.get('rows', []))
+                page_token = response.get('pageToken')
+                if not page_token:
+                    break
 
         return results
 

terraform/SLI_history_dataset.tf

@@ -26,7 +26,7 @@ resource "google_bigquery_table" "SLI_backup_creation_latency_view" {
   table_id = "SLI_backup_creation_latency_view"
 
   view {
-    query = "SELECT * FROM [project-bbq:SLI_history.SLI_backup_creation_latency]"
+    query = "SELECT * FROM [${var.SLI_views_destination_project}:${var.SLI_history_dataset}.SLI_backup_creation_latency]"
     use_legacy_sql = true
   }
 

tests/commons/big_query/test_big_query.py

@@ -57,7 +57,8 @@ def func_for_test(self, project_id, dataset_id, table_id):
 
     @patch('time.sleep', return_value=None)
     @patch.object(TestClass, "func_for_test")
-    def test_iterating_tables_should_retry_if_gets_http_503_response_once(self, func,_):
+    def test_iterating_tables_should_retry_if_gets_http_503_response_once(
+            self, func, _):
         # given
         self._create_http.return_value = self.__create_tables_list_responses_with_503()
 
@@ -67,7 +68,8 @@ def test_iterating_tables_should_retry_if_gets_http_503_response_once(self, func
         # then
         self.assertEquals(5, func.call_count)
 
-    def test_when_dataset_not_exist_then_iterating_tables_should_not_return_any_table(self):
+    def test_when_dataset_not_exist_then_iterating_tables_should_not_return_any_table(
+            self):
         # given
         self._create_http.return_value = self.__create_dataset_not_found_during_tables_list_responses()
 
@@ -104,7 +106,6 @@ def test_should_fetch_single_result_from_random_table_query(self):
         self.assertEqual('O_PRODUCT_SUPPLIER_20151127',
                          random_table.get_table_id())
 
-
     def test_get_dataset_cached_should_only_call_bq_once_but_response_is_cached(
             self):
         # given
@@ -184,6 +185,48 @@ def __create_tables_list_responses():
              content('tests/json_samples/bigquery_table_list_page_last.json'))
         ])
 
+    def test_execute_query_when_executing_long_query(self):
+        # given
+        self._create_http.return_value = self.__execute_long_query_responses()
+        # when
+        result = BigQuery().execute_query("SELECT * FROM tableXYZ")
+        # then
+        self.assertEqual(result, [
+            {
+                "f": [
+                    {
+                        "v": "a-gcp-project2"
+                    },
+                    {
+                        "v": "test1"
+                    }
+                ]
+            },
+            {
+                "f": [
+                    {
+                        "v": "a-gcp-project3"
+                    },
+                    {
+                        "v": "smoke_test_US"
+                    }
+                ]
+            }
+        ])
+
+    @staticmethod
+    def __execute_long_query_responses():
+        return HttpMockSequence([
+            ({'status': '200'}, content(
+                'tests/json_samples/bigquery_v2_test_schema.json')),
+            ({'status': '200'},
+             content('tests/json_samples/big_query/query_response.json')),
+            ({'status': '200'}, content(
+                'tests/json_samples/big_query/get_query_results_job_not_completed.json')),
+            ({'status': '200'}, content(
+                'tests/json_samples/big_query/get_query_results_job_completed.json'))
+        ])
+
     @staticmethod
     def __create_tables_list_responses_with_503():
         return HttpMockSequence([

tests/json_samples/big_query/get_query_results_job_completed.json

@@ -0,0 +1,49 @@
+{
+  "kind": "bigquery#getQueryResultsResponse",
+  "etag": "wMuFW2dF3iB9tEUKtgpe7A==",
+  "schema": {
+    "fields": [
+      {
+        "name": "projectId",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "datasetId",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      }
+    ]
+  },
+  "jobReference": {
+    "projectId": "a-gcp-project-name",
+    "jobId": "job_6h2TY5uZbZao5rKXkRRGgUvEgApI",
+    "location": "EU"
+  },
+  "totalRows": "28",
+  "rows": [
+    {
+      "f": [
+        {
+          "v": "a-gcp-project2"
+        },
+        {
+          "v": "test1"
+        }
+      ]
+    },
+    {
+      "f": [
+        {
+          "v": "a-gcp-project3"
+        },
+        {
+          "v": "smoke_test_US"
+        }
+      ]
+    }
+  ],
+  "totalBytesProcessed": "5294435269",
+  "jobComplete": true,
+  "cacheHit": false
+}

tests/json_samples/big_query/get_query_results_job_not_completed.json

@@ -0,0 +1,10 @@
+{
+  "kind": "bigquery#getQueryResultsResponse",
+  "etag": "7VSjCR5BwV54v+5+bmZCzQ==",
+  "jobReference": {
+    "projectId": "a-gcp-project-name",
+    "jobId": "job_6h2TY5uZbZao5rKXkRRGgUvEgApI",
+    "location": "EU"
+  },
+  "jobComplete": false
+}

tests/json_samples/big_query/query_response.json

@@ -0,0 +1,9 @@
+{
+ "kind": "bigquery#queryResponse",
+ "jobReference": {
+  "projectId": "a-gcp-project-name",
+  "jobId": "job_6h2TY5uZbZao5rKXkRRGgUvEgApI",
+  "location": "EU"
+ },
+ "jobComplete": false
+}