From 49d947e523aca06ea8bdabdb2fe29ea4b14b83c2 Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Fri, 22 Mar 2019 13:07:59 -0700 Subject: [PATCH 1/7] Set a timeout when waiting for query job to finish. --- .../standard_python37/bigquery/.gcloudignore | 19 +++++++ appengine/standard_python37/bigquery/main.py | 10 +++- .../standard_python37/bigquery/main_test.py | 31 +++++++++-- .../bigquery/templates/query_result.html | 52 +++++++++++-------- .../bigquery/templates/timeout.html | 20 +++++++ 5 files changed, 107 insertions(+), 25 deletions(-) create mode 100644 appengine/standard_python37/bigquery/.gcloudignore create mode 100644 appengine/standard_python37/bigquery/templates/timeout.html diff --git a/appengine/standard_python37/bigquery/.gcloudignore b/appengine/standard_python37/bigquery/.gcloudignore new file mode 100644 index 00000000000..a987f1123d8 --- /dev/null +++ b/appengine/standard_python37/bigquery/.gcloudignore @@ -0,0 +1,19 @@ +# This file specifies files that are *not* uploaded to Google Cloud Platform +# using gcloud. It follows the same syntax as .gitignore, with the addition of +# "#!include" directives (which insert the entries of the given .gitignore-style +# file at that point). +# +# For more information, run: +# $ gcloud topic gcloudignore +# +.gcloudignore +# If you would like to upload your .git directory, .gitignore file or files +# from your .gitignore file, remove the corresponding line +# below: +.git +.gitignore + +# Python pycache: +__pycache__/ +# Ignored by the build system +/setup.cfg \ No newline at end of file diff --git a/appengine/standard_python37/bigquery/main.py b/appengine/standard_python37/bigquery/main.py index e6cf2b8dd3e..e8d35ad414f 100644 --- a/appengine/standard_python37/bigquery/main.py +++ b/appengine/standard_python37/bigquery/main.py @@ -13,9 +13,12 @@ # limitations under the License. # [START gae_python37_bigquery] +import concurrent.futures + from flask import Flask, render_template from google.cloud import bigquery + app = Flask(__name__) bigquery_client = bigquery.Client() @@ -34,7 +37,12 @@ def main(): LIMIT 10 """) - results = query_job.result() + try: + # Set a timeout because queries could take longer than one minute. + results = query_job.result(timeout=30) + except concurrent.futures.TimeoutError: + return render_template('timeout.html', job_id=query_job.job_id) + return render_template('query_result.html', results=results) diff --git a/appengine/standard_python37/bigquery/main_test.py b/appengine/standard_python37/bigquery/main_test.py index 727f0c1533c..83806860cf1 100644 --- a/appengine/standard_python37/bigquery/main_test.py +++ b/appengine/standard_python37/bigquery/main_test.py @@ -12,13 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. +import concurrent.futures +from unittest import mock -def test_main(): +from google.cloud import bigquery +import pytest + + +@pytest.fixture +def flask_client(): import main main.app.testing = True - client = main.app.test_client() + return main.app.test_client() + - r = client.get('/') +def test_main(flask_client): + r = flask_client.get('/') assert r.status_code == 200 assert 'Query Result' in r.data.decode('utf-8') + + +def test_main_timeout(flask_client, monkeypatch): + import main + + fake_job = mock.create_autospec(bigquery.QueryJob) + fake_job.result.side_effect = concurrent.futures.TimeoutError() + + def fake_query(query): + return fake_job + + monkeypatch.setattr(main.bigquery_client, 'query', fake_query) + + r = flask_client.get('/') + assert r.status_code == 200 + assert 'Query Timeout' in r.data.decode('utf-8') diff --git a/appengine/standard_python37/bigquery/templates/query_result.html b/appengine/standard_python37/bigquery/templates/query_result.html index 0213cb2c6cf..aa727e5baea 100644 --- a/appengine/standard_python37/bigquery/templates/query_result.html +++ b/appengine/standard_python37/bigquery/templates/query_result.html @@ -1,21 +1,31 @@ - - - - - Query Result - - - - - - - - {% for result in results %} - - - - - {% endfor %} -
URLView Count
{{ result[0] }}{{ result[1] }}
- - \ No newline at end of file + +{# +Copyright 2019 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + +Query Result + + + + + + + {% for result in results %} + + + + + {% endfor %} +
URLView Count
{{ result[0] }}{{ result[1] }}
diff --git a/appengine/standard_python37/bigquery/templates/timeout.html b/appengine/standard_python37/bigquery/templates/timeout.html new file mode 100644 index 00000000000..b59a22a97bc --- /dev/null +++ b/appengine/standard_python37/bigquery/templates/timeout.html @@ -0,0 +1,20 @@ + +{# +Copyright 2019 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + +Query Timeout + +

Query job {{ job_id }} timed out. From c74c454bdd3e030a378311ca0a2ae087eb803ea0 Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Mon, 25 Mar 2019 14:08:31 -0700 Subject: [PATCH 2/7] Transferred results from main function into it's own route. Added redirect from main. --- appengine/standard_python37/bigquery/main.py | 24 +++++++++++++++---- .../standard_python37/bigquery/main_test.py | 5 ++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/appengine/standard_python37/bigquery/main.py b/appengine/standard_python37/bigquery/main.py index e8d35ad414f..d3b6c8ee9aa 100644 --- a/appengine/standard_python37/bigquery/main.py +++ b/appengine/standard_python37/bigquery/main.py @@ -14,12 +14,12 @@ # [START gae_python37_bigquery] import concurrent.futures +import flask -from flask import Flask, render_template from google.cloud import bigquery -app = Flask(__name__) +app = flask.Flask(__name__) bigquery_client = bigquery.Client() @@ -37,13 +37,27 @@ def main(): LIMIT 10 """) + return flask.redirect(flask.url_for('results', project_id=query_job.project, + job_id=query_job.job_id, + location=query_job.location)) + + +@app.route('/results') +def results(): + project_id = flask.request.args.get('project_id') + job_id = flask.request.args.get('job_id') + location = flask.request.args.get('location') + + bigquery_client = bigquery.Client(project=project_id, location=location) + job = bigquery_client.get_job(job_id) + try: # Set a timeout because queries could take longer than one minute. - results = query_job.result(timeout=30) + results = job.result(timeout=30) except concurrent.futures.TimeoutError: - return render_template('timeout.html', job_id=query_job.job_id) + return flask.render_template('timeout.html', job_id=query_job.job_id) - return render_template('query_result.html', results=results) + return flask.render_template('query_result.html', results=results) if __name__ == '__main__': diff --git a/appengine/standard_python37/bigquery/main_test.py b/appengine/standard_python37/bigquery/main_test.py index 83806860cf1..abcf49596ff 100644 --- a/appengine/standard_python37/bigquery/main_test.py +++ b/appengine/standard_python37/bigquery/main_test.py @@ -47,3 +47,8 @@ def fake_query(query): r = flask_client.get('/') assert r.status_code == 200 assert 'Query Timeout' in r.data.decode('utf-8') + +def test_results(flask_client): + r = flask_client.get('/') + assert r.status_code == 200 + assert 'Query Result' in r.data.decode('utf-8') From 3175316134123199d0925a63a8cc9e86df594407 Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Mon, 1 Apr 2019 16:32:46 -0700 Subject: [PATCH 3/7] Made changes to tests per feedback --- appengine/standard_python37/bigquery/main.py | 39 ++++++++++------- .../standard_python37/bigquery/main_test.py | 42 +++++++++++++------ 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/appengine/standard_python37/bigquery/main.py b/appengine/standard_python37/bigquery/main.py index fdf099ef989..fa5e19c1567 100644 --- a/appengine/standard_python37/bigquery/main.py +++ b/appengine/standard_python37/bigquery/main.py @@ -23,9 +23,10 @@ bigquery_client = bigquery.Client() -@app.route('/') +@app.route("/") def main(): - query_job = bigquery_client.query(""" + query_job = bigquery_client.query( + """ SELECT CONCAT( 'https://stackoverflow.com/questions/', @@ -35,34 +36,40 @@ def main(): WHERE tags like '%google-bigquery%' ORDER BY view_count DESC LIMIT 10 - """) + """ + ) - return flask.redirect(flask.url_for('results', project_id=query_job.project, - job_id=query_job.job_id, - location=query_job.location)) + return flask.redirect( + flask.url_for( + "results", + project_id=query_job.project, + job_id=query_job.job_id, + location=query_job.location, + ) + ) -@app.route('/results') +@app.route("/results") def results(): - project_id = flask.request.args.get('project_id') - job_id = flask.request.args.get('job_id') - location = flask.request.args.get('location') + project_id = flask.request.args.get("project_id") + job_id = flask.request.args.get("job_id") + location = flask.request.args.get("location") bigquery_client = bigquery.Client(project=project_id, location=location) - job = bigquery_client.get_job(job_id) + query_job = bigquery_client.get_job(job_id) try: # Set a timeout because queries could take longer than one minute. - results = job.result(timeout=30) + results = query_job.result(timeout=30) except concurrent.futures.TimeoutError: - return flask.render_template('timeout.html', job_id=query_job.job_id) + return flask.render_template("timeout.html", job_id=query_job.job_id) - return flask.render_template('query_result.html', results=results) + return flask.render_template("query_result.html", results=results) -if __name__ == '__main__': +if __name__ == "__main__": # This is used when running locally only. When deploying to Google App # Engine, a webserver process such as Gunicorn will serve the app. This # can be configured by adding an `entrypoint` to app.yaml. - app.run(host='127.0.0.1', port=8080, debug=True) + app.run(host="127.0.0.1", port=8080, debug=True) # [END gae_python37_bigquery] diff --git a/appengine/standard_python37/bigquery/main_test.py b/appengine/standard_python37/bigquery/main_test.py index 56a186d8fd1..aa4a57833bf 100644 --- a/appengine/standard_python37/bigquery/main_test.py +++ b/appengine/standard_python37/bigquery/main_test.py @@ -28,28 +28,46 @@ def flask_client(): def test_main(flask_client): - r = flask_client.get('/') - assert r.status_code == 200 - assert 'Query Result' in r.data.decode('utf-8') + r = flask_client.get("/") + assert r.status_code == 302 + assert "/results" in r.headers.get("location", "") -def test_main_timeout(flask_client, monkeypatch): +def test_results(flask_client, monkeypatch): import main fake_job = mock.create_autospec(bigquery.QueryJob) - fake_job.result.side_effect = concurrent.futures.TimeoutError() + fake_rows = [("example1.com", "42"), ("example2.com", "38")] + fake_job.result.return_value = fake_rows - def fake_query(query): + def fake_get_job(self, job_id): return fake_job - monkeypatch.setattr(main.bigquery_client, 'query', fake_query) + monkeypatch.setattr(main.bigquery.Client, "get_job", fake_get_job) + + r = flask_client.get( + "/results?project_id=123&job_id=456&location=my_location", follow_redirects=True + ) + response_body = r.data.decode("utf-8") - r = flask_client.get('/') assert r.status_code == 200 - assert 'Query Timeout' in r.data.decode('utf-8') + assert "Query Result" in response_body # verifies header + assert "example2.com" in response_body + assert "42" in response_body + + +def test_results_timeout(flask_client, monkeypatch): + import main + + fake_job = mock.create_autospec(bigquery.QueryJob) + fake_job.result.side_effect = concurrent.futures.TimeoutError() + + def fake_get_job(self, job_id): + return fake_job + + monkeypatch.setattr(main.bigquery.Client, "get_job", fake_get_job) + r = flask_client.get("/results", follow_redirects=True) -def test_results(flask_client): - r = flask_client.get('/') assert r.status_code == 200 - assert 'Query Result' in r.data.decode('utf-8') + assert "Query Timeout" in r.data.decode("utf-8") From 65fff9ac73c675f8859aafd599580181f7fa530f Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 2 Apr 2019 09:07:23 -0700 Subject: [PATCH 4/7] Flake8 fixes. --- appengine/standard_python37/bigquery/main_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appengine/standard_python37/bigquery/main_test.py b/appengine/standard_python37/bigquery/main_test.py index aa4a57833bf..2cd71fae86f 100644 --- a/appengine/standard_python37/bigquery/main_test.py +++ b/appengine/standard_python37/bigquery/main_test.py @@ -46,7 +46,7 @@ def fake_get_job(self, job_id): monkeypatch.setattr(main.bigquery.Client, "get_job", fake_get_job) r = flask_client.get( - "/results?project_id=123&job_id=456&location=my_location", follow_redirects=True + "/results?project_id=123&job_id=456&location=my_location" ) response_body = r.data.decode("utf-8") From cf115d24d244cfb9be0cecb6e0edf39aee73e702 Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Tue, 2 Apr 2019 13:06:16 -0700 Subject: [PATCH 5/7] Updated get_job arguments per feedback. --- appengine/standard_python37/bigquery/main.py | 3 +-- appengine/standard_python37/bigquery/main_test.py | 8 +++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/appengine/standard_python37/bigquery/main.py b/appengine/standard_python37/bigquery/main.py index fa5e19c1567..8a514ac6d3a 100644 --- a/appengine/standard_python37/bigquery/main.py +++ b/appengine/standard_python37/bigquery/main.py @@ -55,8 +55,7 @@ def results(): job_id = flask.request.args.get("job_id") location = flask.request.args.get("location") - bigquery_client = bigquery.Client(project=project_id, location=location) - query_job = bigquery_client.get_job(job_id) + query_job = bigquery_client.get_job(job_id, project=project_id, location=location) try: # Set a timeout because queries could take longer than one minute. diff --git a/appengine/standard_python37/bigquery/main_test.py b/appengine/standard_python37/bigquery/main_test.py index 2cd71fae86f..fc23e82eb34 100644 --- a/appengine/standard_python37/bigquery/main_test.py +++ b/appengine/standard_python37/bigquery/main_test.py @@ -40,14 +40,12 @@ def test_results(flask_client, monkeypatch): fake_rows = [("example1.com", "42"), ("example2.com", "38")] fake_job.result.return_value = fake_rows - def fake_get_job(self, job_id): + def fake_get_job(self, job_id, **kwargs): return fake_job monkeypatch.setattr(main.bigquery.Client, "get_job", fake_get_job) - r = flask_client.get( - "/results?project_id=123&job_id=456&location=my_location" - ) + r = flask_client.get("/results?project_id=123&job_id=456&location=my_location") response_body = r.data.decode("utf-8") assert r.status_code == 200 @@ -62,7 +60,7 @@ def test_results_timeout(flask_client, monkeypatch): fake_job = mock.create_autospec(bigquery.QueryJob) fake_job.result.side_effect = concurrent.futures.TimeoutError() - def fake_get_job(self, job_id): + def fake_get_job(self, job_id, **kwargs): return fake_job monkeypatch.setattr(main.bigquery.Client, "get_job", fake_get_job) From b63488a0ae0c8d2bde799131093f0dbe734ba6cc Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 2 Apr 2019 15:11:03 -0700 Subject: [PATCH 6/7] Fix flake8 lint error in main.py --- appengine/standard_python37/bigquery/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/appengine/standard_python37/bigquery/main.py b/appengine/standard_python37/bigquery/main.py index 8a514ac6d3a..fc2ec8d4ad0 100644 --- a/appengine/standard_python37/bigquery/main.py +++ b/appengine/standard_python37/bigquery/main.py @@ -55,7 +55,11 @@ def results(): job_id = flask.request.args.get("job_id") location = flask.request.args.get("location") - query_job = bigquery_client.get_job(job_id, project=project_id, location=location) + query_job = bigquery_client.get_job( + job_id, + project=project_id, + location=location, + ) try: # Set a timeout because queries could take longer than one minute. From 32f16423cff6fef9c8663abd1d7aad92a8c5df95 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 2 Apr 2019 15:11:37 -0700 Subject: [PATCH 7/7] Fix flake8 lint error in main_test.py --- appengine/standard_python37/bigquery/main_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/appengine/standard_python37/bigquery/main_test.py b/appengine/standard_python37/bigquery/main_test.py index fc23e82eb34..b00f7cd8aac 100644 --- a/appengine/standard_python37/bigquery/main_test.py +++ b/appengine/standard_python37/bigquery/main_test.py @@ -45,7 +45,9 @@ def fake_get_job(self, job_id, **kwargs): monkeypatch.setattr(main.bigquery.Client, "get_job", fake_get_job) - r = flask_client.get("/results?project_id=123&job_id=456&location=my_location") + r = flask_client.get( + "/results?project_id=123&job_id=456&location=my_location" + ) response_body = r.data.decode("utf-8") assert r.status_code == 200