From f0408a8977167de9b1801e4fa6ff167c9b50e701 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 27 Oct 2016 14:42:59 -0700 Subject: [PATCH 01/12] Add basic noxfile to orchestrate system tests --- .gitignore | 1 + system_tests/nox.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ tox.ini | 8 ++++++-- 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 system_tests/nox.py diff --git a/.gitignore b/.gitignore index cd35d1544..2c7b7226d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ dist/ docs/_build # Test files +.nox/ .tox/ .cache/ diff --git a/system_tests/nox.py b/system_tests/nox.py new file mode 100644 index 000000000..41dc361fe --- /dev/null +++ b/system_tests/nox.py @@ -0,0 +1,44 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Noxfile for automating system tests. + +This file handles setting up environments needed by the system tests. This +separates the tests from their environment configuration. +""" + + +def session_service_account(session): + session.virtualenv = False + session.run('pytest', 'test_service_account.py') + + +def session_oauth2_credentials(session): + session.virtualenv = False + session.run('pytest', 'test_oauth2_credentials.py') + + +def session_compute_engine(session): + session.virtualenv = False + session.run('pytest', 'test_compute_engine.py') + + +def session_app_engine(session): + session.virtualenv = False + session.run('pytest', 'app_engine/test_app_engine.py') + + +def session_default(session): + session.virtualenv = False + session.run('pytest', 'test_default.py') diff --git a/tox.ini b/tox.ini index 6864ddee7..14f11661f 100644 --- a/tox.ini +++ b/tox.ini @@ -23,19 +23,23 @@ deps = [testenv:py35-system] basepython = python3.5 +changedir = {toxinidir}/system_tests commands = - py.test --ignore system_tests/app_engine/app {posargs:system_tests} + nox {posargs} deps = {[testenv]deps} + nox-automation passenv = SKIP_APP_ENGINE_SYSTEM_TEST [testenv:py27-system] basepython = python2.7 +changedir = {toxinidir}/system_tests commands = - py.test --ignore system_tests/app_engine/app {posargs:system_tests} + nox {posargs} deps = {[testenv]deps} + nox-automation passenv = SKIP_APP_ENGINE_SYSTEM_TEST From c04985bf962211a1cda8e24ed19a07f61e5ce222 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 27 Oct 2016 15:01:45 -0700 Subject: [PATCH 02/12] Move explicit system tests to nox --- system_tests/conftest.py | 21 +++++++++- system_tests/nox.py | 31 +++++++++++++++ system_tests/test_default.py | 57 ++------------------------- system_tests/test_default_explicit.py | 30 ++++++++++++++ 4 files changed, 84 insertions(+), 55 deletions(-) create mode 100644 system_tests/test_default_explicit.py diff --git a/system_tests/conftest.py b/system_tests/conftest.py index 5969beeec..f92ac6541 100644 --- a/system_tests/conftest.py +++ b/system_tests/conftest.py @@ -23,6 +23,8 @@ HERE = os.path.dirname(__file__) DATA_DIR = os.path.join(HERE, 'data') +SERVICE_ACCOUNT_FILE = os.path.join(DATA_DIR, 'service_account.json') +AUTHORIZED_USER_FILE = os.path.join(DATA_DIR, 'authorized_user.json') HTTP = urllib3.PoolManager() TOKEN_INFO_URL = 'https://www.googleapis.com/oauth2/v3/tokeninfo' @@ -30,13 +32,13 @@ @pytest.fixture def service_account_file(): """The full path to a valid service account key file.""" - yield os.path.join(DATA_DIR, 'service_account.json') + yield SERVICE_ACCOUNT_FILE @pytest.fixture def authorized_user_file(): """The full path to a valid authorized user file.""" - yield os.path.join(DATA_DIR, 'authorized_user.json') + yield AUTHORIZED_USER_FILE @pytest.fixture @@ -67,6 +69,21 @@ def _token_info(access_token=None, id_token=None): yield _token_info +@pytest.fixture +def verify_refresh(http_request): + """Returns a function that verifies that credentials can be refreshed.""" + def _verify_refresh(credentials): + if credentials.requires_scopes: + credentials = credentials.with_scopes(['email', 'profile']) + + credentials.refresh(http_request) + + assert credentials.token + assert credentials.valid + + yield _verify_refresh + + def verify_environment(): """Checks to make sure that requisite data files are available.""" if not os.path.isdir(DATA_DIR): diff --git a/system_tests/nox.py b/system_tests/nox.py index 41dc361fe..df18f3682 100644 --- a/system_tests/nox.py +++ b/system_tests/nox.py @@ -16,8 +16,18 @@ This file handles setting up environments needed by the system tests. This separates the tests from their environment configuration. + +See the `nox docs`_ for details on how this file works: + +.. _nox docs: http://nox.readthedocs.io/en/latest/ """ +import os + +HERE = os.path.dirname(__file__) +DATA_DIR = os.path.join(HERE, 'data') +SERVICE_ACCOUNT_FILE = os.path.join(DATA_DIR, 'service_account.json') +AUTHORIZED_USER_FILE = os.path.join(DATA_DIR, 'authorized_user.json') def session_service_account(session): session.virtualenv = False @@ -42,3 +52,24 @@ def session_app_engine(session): def session_default(session): session.virtualenv = False session.run('pytest', 'test_default.py') + + +def session_default_explicit_service_account(session): + session.virtualenv = False + session.env['GOOGLE_APPLICATION_CREDENTIALS'] = SERVICE_ACCOUNT_FILE + session.env['EXPECT_PROJECT_ID'] = '1' + session.run('pytest', 'test_default_explicit.py') + + +def session_default_explicit_authorized_user(session): + session.virtualenv = False + session.env['GOOGLE_APPLICATION_CREDENTIALS'] = AUTHORIZED_USER_FILE + session.run('pytest', 'test_default_explicit.py') + + +def session_default_explicit_authorized_user_explicit_project(session): + session.virtualenv = False + session.env['GOOGLE_APPLICATION_CREDENTIALS'] = AUTHORIZED_USER_FILE + session.env['GOOGLE_CLOUD_PROJECT'] = 'example-project' + session.env['EXPECT_PROJECT_ID'] = '1' + session.run('pytest', 'test_default_explicit.py') diff --git a/system_tests/test_default.py b/system_tests/test_default.py index 02242e94f..2c7749b23 100644 --- a/system_tests/test_default.py +++ b/system_tests/test_default.py @@ -22,55 +22,6 @@ from google.oauth2 import service_account -def validate_refresh(credentials, http_request): - if credentials.requires_scopes: - credentials = credentials.with_scopes(['email', 'profile']) - - credentials.refresh(http_request) - - assert credentials.token - assert credentials.valid - - -def test_explicit_credentials_service_account( - monkeypatch, service_account_file, http_request): - monkeypatch.setitem( - os.environ, environment_vars.CREDENTIALS, service_account_file) - - credentials, project_id = google.auth.default() - - assert isinstance(credentials, service_account.Credentials) - assert project_id is not None - - validate_refresh(credentials, http_request) - - -def test_explicit_credentials_authorized_user( - monkeypatch, authorized_user_file, http_request): - monkeypatch.setitem( - os.environ, environment_vars.CREDENTIALS, authorized_user_file) - - credentials, project_id = google.auth.default() - - assert isinstance(credentials, google.oauth2.credentials.Credentials) - assert project_id is None - - validate_refresh(credentials, http_request) - - -def test_explicit_credentials_explicit_project_id( - monkeypatch, service_account_file, http_request): - project = 'system-test-project' - monkeypatch.setitem( - os.environ, environment_vars.CREDENTIALS, service_account_file) - monkeypatch.setitem( - os.environ, environment_vars.PROJECT, project) - - _, project_id = google.auth.default() - - assert project_id == project - - def generate_cloud_sdk_config( tmpdir, credentials_file, active_config='default', project=None): tmpdir.join('active_config').write( @@ -87,7 +38,7 @@ def generate_cloud_sdk_config( def test_cloud_sdk_credentials_service_account( - tmpdir, monkeypatch, service_account_file, http_request): + tmpdir, monkeypatch, service_account_file, verify_refresh): # Create the Cloud SDK configuration tree project = 'system-test-project' generate_cloud_sdk_config(tmpdir, service_account_file, project=project) @@ -102,11 +53,11 @@ def test_cloud_sdk_credentials_service_account( # account file, not the project in the config. assert project_id is not project - validate_refresh(credentials, http_request) + verify_refresh(credentials) def test_cloud_sdk_credentials_authorized_user( - tmpdir, monkeypatch, authorized_user_file, http_request): + tmpdir, monkeypatch, authorized_user_file, verify_refresh): # Create the Cloud SDK configuration tree project = 'system-test-project' generate_cloud_sdk_config(tmpdir, authorized_user_file, project=project) @@ -118,4 +69,4 @@ def test_cloud_sdk_credentials_authorized_user( assert isinstance(credentials, google.oauth2.credentials.Credentials) assert project_id == project - validate_refresh(credentials, http_request) + verify_refresh(credentials) diff --git a/system_tests/test_default_explicit.py b/system_tests/test_default_explicit.py new file mode 100644 index 000000000..0e0bcc6c0 --- /dev/null +++ b/system_tests/test_default_explicit.py @@ -0,0 +1,30 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import google.auth + +EXPECT_PROJECT_ID = os.environ.get('EXPECT_PROJECT_ID') + + +def test_explicit_credentials(verify_refresh): + credentials, project_id = google.auth.default() + + if EXPECT_PROJECT_ID is not None: + assert project_id is not None + else: + assert project_id is None + + verify_refresh(credentials) From 92e24d3732ae13171acace049d3f53df1cff1c57 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 27 Oct 2016 16:34:13 -0700 Subject: [PATCH 03/12] Add cloud sdk environment automation --- .travis.yml | 1 + google/auth/_cloud_sdk.py | 9 +- system_tests/nox.py | 149 +++++++++++++++++++++++--- system_tests/test_default.py | 54 ++-------- system_tests/test_default_explicit.py | 30 ------ tox.ini | 2 + 6 files changed, 148 insertions(+), 97 deletions(-) delete mode 100644 system_tests/test_default_explicit.py diff --git a/.travis.yml b/.travis.yml index aa39f9a4a..fe9f9ed92 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,3 +39,4 @@ deploy: env: global: secure: s6GdhJklftl8w/9WoETwLtvtKL4ledPA/TuBuqCXQxSuYWaPuTdRVcvoejGkHJpp7i/7v2T/0etYl+5koyskKm5+QZZweaaL7MAyjPGp+hmIaIlWQRz6w481NOf3i9uSmoQycssT0mNmwScNIqo+igbA2y14mr/e9aBuOcxNNzNzFQp2vaRMEju6q7xZMjYdcudUWL48vq9CoNa3X2ZArpqjkApR/TfYlG7glOj43NxuVDN4z9wIyUjaMHBfPgEhjaOaRyEFgEYITRwX1qDoXqcZdTVIq4Cn0uCH+Mvrz6Y+oUJGTJqH1k7N/DhzbSN9lJnVYaQW/yuvGHiGAwbb6Tcxiq2UqqhA9MfbPpmstDECs46v9Z3BT252KvYEQY7Q1v9g2gFhHvFGWISUxs80rnnPhEYfa11JoLvj2t8cowkE4pvj4OH32Eoyvc5H07hW3F5xpuF7Jt7N09TNZkUrpmiRJEhfrVNgjsrWO77/q5h8mXGd+9vYmz++yzKu+63x8x1MpeigGCG73Dpu9Otm5eydOZfpJ39ZfZWUb7G2JahgHaGweM9dmnpJtzHQgijmHjjfAx9jgnQ8IQz9nkFmyMI8H7HouwalnrJtpSSbvMqOQ0kiZhMzdBKH5pD3tjLgSlgA0pKelBwlooY6jGlj4LrtbDAxa6cZyXiFoqWpT1w= + CLOUD_SDK_ROOT: ${HOME}/.cache/cloud-sdk diff --git a/google/auth/_cloud_sdk.py b/google/auth/_cloud_sdk.py index f51d825aa..1e851c87d 100644 --- a/google/auth/_cloud_sdk.py +++ b/google/auth/_cloud_sdk.py @@ -131,13 +131,14 @@ def get_project_id(): try: config.read(config_file) + + if config.has_section(_PROJECT_CONFIG_SECTION): + return config.get( + _PROJECT_CONFIG_SECTION, _PROJECT_CONFIG_KEY) + except configparser.Error: return None - if config.has_section(_PROJECT_CONFIG_SECTION): - return config.get( - _PROJECT_CONFIG_SECTION, _PROJECT_CONFIG_KEY) - def load_authorized_user_credentials(info): """Loads an authorized user credential. diff --git a/system_tests/nox.py b/system_tests/nox.py index df18f3682..ea566596c 100644 --- a/system_tests/nox.py +++ b/system_tests/nox.py @@ -24,10 +24,105 @@ import os +from nox.command import Command +import py.path + + HERE = os.path.dirname(__file__) DATA_DIR = os.path.join(HERE, 'data') SERVICE_ACCOUNT_FILE = os.path.join(DATA_DIR, 'service_account.json') AUTHORIZED_USER_FILE = os.path.join(DATA_DIR, 'authorized_user.json') +CLOUD_SDK_CONFIG_ENV = 'CLOUDSDK_CONFIG' + +# If set, this is where the environment setup will store the Cloud SDK. +# If unset, it will download the SDK to a temporary directory. +CLOUD_SDK_ROOT = os.environ.get('CLOUD_SDK_ROOT') +if CLOUD_SDK_ROOT: + CLOUD_SDK_ROOT = py.path.local(CLOUD_SDK_ROOT) + CLOUD_SDK_ROOT.ensure(dir=True) +if not CLOUD_SDK_ROOT: + CLOUD_SDK_ROOT = py.path.local.mkdtemp() + + +# Helper functions + + +def prerun(*args, **kwargs): + """Runs a command before the session.""" + kwargs.setdefault('silent', True) + env = os.environ.copy() + env.update(kwargs.pop('env', {})) + Command(args, env=env, **kwargs).run() + + +# Cloud SDK helpers + + +def setup_cloud_sdk(): + """Downloads and installs the Google Cloud SDK.""" + + # If the sdk already exists, we don't need to do anything else. + if CLOUD_SDK_ROOT.join('google-cloud-sdk').exists(): + return + + tar_file = 'google-cloud-sdk.tar.gz' + tar_path = CLOUD_SDK_ROOT.join(tar_file) + + # Download the release. + prerun( + 'wget', 'https://dl.google.com/dl/cloudsdk/release/{}'.format( + tar_file), + '-O', str(tar_path)) + + # Extract the release. + prerun('tar', 'xzf', str(tar_path), '-C', str(CLOUD_SDK_ROOT)) + tar_path.remove() + + # Run the install script. + prerun( + str(CLOUD_SDK_ROOT.join('google-cloud-sdk', 'install.sh')), + '--usage-reporting', 'false', + '--path-update', 'false', + '--command-completion', 'false', + env={CLOUD_SDK_CONFIG_ENV: str(CLOUD_SDK_ROOT)}) + + return CLOUD_SDK_ROOT + + +def gcloud(*args, **kwargs): + """Calls the Cloud SDK CLI.""" + bin = str(CLOUD_SDK_ROOT.join('google-cloud-sdk', 'bin', 'gcloud')) + env = {CLOUD_SDK_CONFIG_ENV: str(CLOUD_SDK_ROOT)} + return prerun(bin, *args, env=env, **kwargs) + + +def configure_cloud_sdk(application_default_credentials, project=False): + """Configures the Cloud SDK with the given application default + credentials. + + If project is True, then a project will be set in the active config. + If it is false, this will ensure no project is set. + """ + + if project: + gcloud('config', 'set', 'project', 'example-project') + else: + gcloud('config', 'unset', 'project') + + # Copy the credentials file to the config root. This is needed because + # unfortunately gcloud doesn't provide a clean way to tell it to use + # a particular set of credentials. However, this does verify that gcloud + # also considers the credentials valid by calling application-default + # print-access-token + dest = CLOUD_SDK_ROOT.join('application_default_credentials.json') + dest.remove() + py.path.local(application_default_credentials).copy(dest) + + gcloud('auth', 'application-default', 'print-access-token') + + +# Test sesssions + def session_service_account(session): session.virtualenv = False @@ -39,37 +134,61 @@ def session_oauth2_credentials(session): session.run('pytest', 'test_oauth2_credentials.py') -def session_compute_engine(session): +def session_default_explicit_service_account(session): session.virtualenv = False - session.run('pytest', 'test_compute_engine.py') + session.env['GOOGLE_APPLICATION_CREDENTIALS'] = SERVICE_ACCOUNT_FILE + session.env['EXPECT_PROJECT_ID'] = '1' + session.run('pytest', 'test_default.py') -def session_app_engine(session): +def session_default_explicit_authorized_user(session): session.virtualenv = False - session.run('pytest', 'app_engine/test_app_engine.py') + session.env['GOOGLE_APPLICATION_CREDENTIALS'] = AUTHORIZED_USER_FILE + session.run('pytest', 'test_default.py') -def session_default(session): +def session_default_explicit_authorized_user_explicit_project(session): session.virtualenv = False + session.env['GOOGLE_APPLICATION_CREDENTIALS'] = AUTHORIZED_USER_FILE + session.env['GOOGLE_CLOUD_PROJECT'] = 'example-project' + session.env['EXPECT_PROJECT_ID'] = '1' session.run('pytest', 'test_default.py') -def session_default_explicit_service_account(session): +def session_default_cloud_sdk_service_account(session): session.virtualenv = False - session.env['GOOGLE_APPLICATION_CREDENTIALS'] = SERVICE_ACCOUNT_FILE + setup_cloud_sdk() + configure_cloud_sdk(SERVICE_ACCOUNT_FILE) + + session.env[CLOUD_SDK_CONFIG_ENV] = str(CLOUD_SDK_ROOT) session.env['EXPECT_PROJECT_ID'] = '1' - session.run('pytest', 'test_default_explicit.py') + session.run('pytest', 'test_default.py') -def session_default_explicit_authorized_user(session): +def session_default_cloud_sdk_authorized_user(session): session.virtualenv = False - session.env['GOOGLE_APPLICATION_CREDENTIALS'] = AUTHORIZED_USER_FILE - session.run('pytest', 'test_default_explicit.py') + setup_cloud_sdk() + configure_cloud_sdk(AUTHORIZED_USER_FILE) + session.env[CLOUD_SDK_CONFIG_ENV] = str(CLOUD_SDK_ROOT) + session.run('pytest', '--pdb', 'test_default.py') -def session_default_explicit_authorized_user_explicit_project(session): + +def session_default_cloud_sdk_authorized_user_configured_project(session): session.virtualenv = False - session.env['GOOGLE_APPLICATION_CREDENTIALS'] = AUTHORIZED_USER_FILE - session.env['GOOGLE_CLOUD_PROJECT'] = 'example-project' + setup_cloud_sdk() + configure_cloud_sdk(AUTHORIZED_USER_FILE, project=True) + + session.env[CLOUD_SDK_CONFIG_ENV] = str(CLOUD_SDK_ROOT) session.env['EXPECT_PROJECT_ID'] = '1' - session.run('pytest', 'test_default_explicit.py') + session.run('pytest', 'test_default.py') + + +def session_compute_engine(session): + session.virtualenv = False + session.run('pytest', 'test_compute_engine.py') + + +def session_app_engine(session): + session.virtualenv = False + session.run('pytest', 'app_engine/test_app_engine.py') diff --git a/system_tests/test_default.py b/system_tests/test_default.py index 2c7749b23..0e0bcc6c0 100644 --- a/system_tests/test_default.py +++ b/system_tests/test_default.py @@ -14,59 +14,17 @@ import os -import py - import google.auth -from google.auth import environment_vars -import google.oauth2.credentials -from google.oauth2 import service_account - - -def generate_cloud_sdk_config( - tmpdir, credentials_file, active_config='default', project=None): - tmpdir.join('active_config').write( - '{}\n'.format(active_config), ensure=True) - - if project is not None: - config_file = tmpdir.join( - 'configurations', 'config_{}'.format(active_config)) - config_file.write( - '[core]\nproject = {}'.format(project), ensure=True) - - py.path.local(credentials_file).copy( - tmpdir.join('application_default_credentials.json')) - - -def test_cloud_sdk_credentials_service_account( - tmpdir, monkeypatch, service_account_file, verify_refresh): - # Create the Cloud SDK configuration tree - project = 'system-test-project' - generate_cloud_sdk_config(tmpdir, service_account_file, project=project) - monkeypatch.setitem( - os.environ, environment_vars.CLOUD_SDK_CONFIG_DIR, str(tmpdir)) - - credentials, project_id = google.auth.default() - - assert isinstance(credentials, service_account.Credentials) - assert project_id is not None - # The project ID should be the project ID specified in the the service - # account file, not the project in the config. - assert project_id is not project - - verify_refresh(credentials) +EXPECT_PROJECT_ID = os.environ.get('EXPECT_PROJECT_ID') -def test_cloud_sdk_credentials_authorized_user( - tmpdir, monkeypatch, authorized_user_file, verify_refresh): - # Create the Cloud SDK configuration tree - project = 'system-test-project' - generate_cloud_sdk_config(tmpdir, authorized_user_file, project=project) - monkeypatch.setitem( - os.environ, environment_vars.CLOUD_SDK_CONFIG_DIR, str(tmpdir)) +def test_explicit_credentials(verify_refresh): credentials, project_id = google.auth.default() - assert isinstance(credentials, google.oauth2.credentials.Credentials) - assert project_id == project + if EXPECT_PROJECT_ID is not None: + assert project_id is not None + else: + assert project_id is None verify_refresh(credentials) diff --git a/system_tests/test_default_explicit.py b/system_tests/test_default_explicit.py deleted file mode 100644 index 0e0bcc6c0..000000000 --- a/system_tests/test_default_explicit.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import google.auth - -EXPECT_PROJECT_ID = os.environ.get('EXPECT_PROJECT_ID') - - -def test_explicit_credentials(verify_refresh): - credentials, project_id = google.auth.default() - - if EXPECT_PROJECT_ID is not None: - assert project_id is not None - else: - assert project_id is None - - verify_refresh(credentials) diff --git a/tox.ini b/tox.ini index 14f11661f..427f2b726 100644 --- a/tox.ini +++ b/tox.ini @@ -31,6 +31,7 @@ deps = nox-automation passenv = SKIP_APP_ENGINE_SYSTEM_TEST + CLOUD_SDK_ROOT [testenv:py27-system] basepython = python2.7 @@ -42,6 +43,7 @@ deps = nox-automation passenv = SKIP_APP_ENGINE_SYSTEM_TEST + CLOUD_SDK_ROOT [testenv:docgen] basepython = python3.5 From 6d8addd2e6fa2a080248f1343176ad3a278f054c Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 27 Oct 2016 16:35:10 -0700 Subject: [PATCH 04/12] Fix lint error --- system_tests/nox.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system_tests/nox.py b/system_tests/nox.py index ea566596c..ef027179b 100644 --- a/system_tests/nox.py +++ b/system_tests/nox.py @@ -91,9 +91,9 @@ def setup_cloud_sdk(): def gcloud(*args, **kwargs): """Calls the Cloud SDK CLI.""" - bin = str(CLOUD_SDK_ROOT.join('google-cloud-sdk', 'bin', 'gcloud')) + prog = str(CLOUD_SDK_ROOT.join('google-cloud-sdk', 'bin', 'gcloud')) env = {CLOUD_SDK_CONFIG_ENV: str(CLOUD_SDK_ROOT)} - return prerun(bin, *args, env=env, **kwargs) + return prerun(prog, *args, env=env, **kwargs) def configure_cloud_sdk(application_default_credentials, project=False): From 1ba1b3781044d2ef867a97ac684a66f6314ff4e6 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 27 Oct 2016 16:37:32 -0700 Subject: [PATCH 05/12] Fix travis.yaml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index fe9f9ed92..f7d255407 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,5 +38,5 @@ deploy: repo: GoogleCloudPlatform/google-auth-library-python env: global: - secure: s6GdhJklftl8w/9WoETwLtvtKL4ledPA/TuBuqCXQxSuYWaPuTdRVcvoejGkHJpp7i/7v2T/0etYl+5koyskKm5+QZZweaaL7MAyjPGp+hmIaIlWQRz6w481NOf3i9uSmoQycssT0mNmwScNIqo+igbA2y14mr/e9aBuOcxNNzNzFQp2vaRMEju6q7xZMjYdcudUWL48vq9CoNa3X2ZArpqjkApR/TfYlG7glOj43NxuVDN4z9wIyUjaMHBfPgEhjaOaRyEFgEYITRwX1qDoXqcZdTVIq4Cn0uCH+Mvrz6Y+oUJGTJqH1k7N/DhzbSN9lJnVYaQW/yuvGHiGAwbb6Tcxiq2UqqhA9MfbPpmstDECs46v9Z3BT252KvYEQY7Q1v9g2gFhHvFGWISUxs80rnnPhEYfa11JoLvj2t8cowkE4pvj4OH32Eoyvc5H07hW3F5xpuF7Jt7N09TNZkUrpmiRJEhfrVNgjsrWO77/q5h8mXGd+9vYmz++yzKu+63x8x1MpeigGCG73Dpu9Otm5eydOZfpJ39ZfZWUb7G2JahgHaGweM9dmnpJtzHQgijmHjjfAx9jgnQ8IQz9nkFmyMI8H7HouwalnrJtpSSbvMqOQ0kiZhMzdBKH5pD3tjLgSlgA0pKelBwlooY6jGlj4LrtbDAxa6cZyXiFoqWpT1w= - CLOUD_SDK_ROOT: ${HOME}/.cache/cloud-sdk + - CLOUD_SDK_ROOT: ${HOME}/.cache/cloud-sdk + - secure: s6GdhJklftl8w/9WoETwLtvtKL4ledPA/TuBuqCXQxSuYWaPuTdRVcvoejGkHJpp7i/7v2T/0etYl+5koyskKm5+QZZweaaL7MAyjPGp+hmIaIlWQRz6w481NOf3i9uSmoQycssT0mNmwScNIqo+igbA2y14mr/e9aBuOcxNNzNzFQp2vaRMEju6q7xZMjYdcudUWL48vq9CoNa3X2ZArpqjkApR/TfYlG7glOj43NxuVDN4z9wIyUjaMHBfPgEhjaOaRyEFgEYITRwX1qDoXqcZdTVIq4Cn0uCH+Mvrz6Y+oUJGTJqH1k7N/DhzbSN9lJnVYaQW/yuvGHiGAwbb6Tcxiq2UqqhA9MfbPpmstDECs46v9Z3BT252KvYEQY7Q1v9g2gFhHvFGWISUxs80rnnPhEYfa11JoLvj2t8cowkE4pvj4OH32Eoyvc5H07hW3F5xpuF7Jt7N09TNZkUrpmiRJEhfrVNgjsrWO77/q5h8mXGd+9vYmz++yzKu+63x8x1MpeigGCG73Dpu9Otm5eydOZfpJ39ZfZWUb7G2JahgHaGweM9dmnpJtzHQgijmHjjfAx9jgnQ8IQz9nkFmyMI8H7HouwalnrJtpSSbvMqOQ0kiZhMzdBKH5pD3tjLgSlgA0pKelBwlooY6jGlj4LrtbDAxa6cZyXiFoqWpT1w= From a876b175e7cc6b0cf7fd1a801904b4da97f2dc42 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 27 Oct 2016 16:41:13 -0700 Subject: [PATCH 06/12] (Maybe) fix travis secrets --- .travis.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f7d255407..14f22c75d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,9 +17,9 @@ matrix: - python: 3.5 env: TOXENV=cover - python: 3.5 - env: TOXENV=py35-system SYSTEM_TEST=1 SKIP_APP_ENGINE_SYSTEM_TEST=1 + env: TOXENV=py35-system SYSTEM_TEST=1 SKIP_APP_ENGINE_SYSTEM_TEST=1 CLOUD_SDK_ROOT=${HOME}/.cache/cloud-sdk - python: 2.7 - env: TOXENV=py27-system SYSTEM_TEST=1 SKIP_APP_ENGINE_SYSTEM_TEST=1 + env: TOXENV=py27-system SYSTEM_TEST=1 SKIP_APP_ENGINE_SYSTEM_TEST=1 CLOUD_SDK_ROOT=${HOME}/.cache/cloud-sdk cache: directories: - ${HOME}/.cache @@ -38,5 +38,4 @@ deploy: repo: GoogleCloudPlatform/google-auth-library-python env: global: - - CLOUD_SDK_ROOT: ${HOME}/.cache/cloud-sdk - - secure: s6GdhJklftl8w/9WoETwLtvtKL4ledPA/TuBuqCXQxSuYWaPuTdRVcvoejGkHJpp7i/7v2T/0etYl+5koyskKm5+QZZweaaL7MAyjPGp+hmIaIlWQRz6w481NOf3i9uSmoQycssT0mNmwScNIqo+igbA2y14mr/e9aBuOcxNNzNzFQp2vaRMEju6q7xZMjYdcudUWL48vq9CoNa3X2ZArpqjkApR/TfYlG7glOj43NxuVDN4z9wIyUjaMHBfPgEhjaOaRyEFgEYITRwX1qDoXqcZdTVIq4Cn0uCH+Mvrz6Y+oUJGTJqH1k7N/DhzbSN9lJnVYaQW/yuvGHiGAwbb6Tcxiq2UqqhA9MfbPpmstDECs46v9Z3BT252KvYEQY7Q1v9g2gFhHvFGWISUxs80rnnPhEYfa11JoLvj2t8cowkE4pvj4OH32Eoyvc5H07hW3F5xpuF7Jt7N09TNZkUrpmiRJEhfrVNgjsrWO77/q5h8mXGd+9vYmz++yzKu+63x8x1MpeigGCG73Dpu9Otm5eydOZfpJ39ZfZWUb7G2JahgHaGweM9dmnpJtzHQgijmHjjfAx9jgnQ8IQz9nkFmyMI8H7HouwalnrJtpSSbvMqOQ0kiZhMzdBKH5pD3tjLgSlgA0pKelBwlooY6jGlj4LrtbDAxa6cZyXiFoqWpT1w= + secure: s6GdhJklftl8w/9WoETwLtvtKL4ledPA/TuBuqCXQxSuYWaPuTdRVcvoejGkHJpp7i/7v2T/0etYl+5koyskKm5+QZZweaaL7MAyjPGp+hmIaIlWQRz6w481NOf3i9uSmoQycssT0mNmwScNIqo+igbA2y14mr/e9aBuOcxNNzNzFQp2vaRMEju6q7xZMjYdcudUWL48vq9CoNa3X2ZArpqjkApR/TfYlG7glOj43NxuVDN4z9wIyUjaMHBfPgEhjaOaRyEFgEYITRwX1qDoXqcZdTVIq4Cn0uCH+Mvrz6Y+oUJGTJqH1k7N/DhzbSN9lJnVYaQW/yuvGHiGAwbb6Tcxiq2UqqhA9MfbPpmstDECs46v9Z3BT252KvYEQY7Q1v9g2gFhHvFGWISUxs80rnnPhEYfa11JoLvj2t8cowkE4pvj4OH32Eoyvc5H07hW3F5xpuF7Jt7N09TNZkUrpmiRJEhfrVNgjsrWO77/q5h8mXGd+9vYmz++yzKu+63x8x1MpeigGCG73Dpu9Otm5eydOZfpJ39ZfZWUb7G2JahgHaGweM9dmnpJtzHQgijmHjjfAx9jgnQ8IQz9nkFmyMI8H7HouwalnrJtpSSbvMqOQ0kiZhMzdBKH5pD3tjLgSlgA0pKelBwlooY6jGlj4LrtbDAxa6cZyXiFoqWpT1w= From 5fac77e28f02366479e9c036087ab5d773aae2f3 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 27 Oct 2016 16:45:09 -0700 Subject: [PATCH 07/12] Fix call to py.path.remove --- system_tests/nox.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/system_tests/nox.py b/system_tests/nox.py index ef027179b..6bd9c0fc7 100644 --- a/system_tests/nox.py +++ b/system_tests/nox.py @@ -115,7 +115,8 @@ def configure_cloud_sdk(application_default_credentials, project=False): # also considers the credentials valid by calling application-default # print-access-token dest = CLOUD_SDK_ROOT.join('application_default_credentials.json') - dest.remove() + if dest.exists(): + dest.remove() py.path.local(application_default_credentials).copy(dest) gcloud('auth', 'application-default', 'print-access-token') From 2becbf45d5b3589e1f4861c995959939db9e2cc1 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 27 Oct 2016 21:42:53 -0700 Subject: [PATCH 08/12] Address review comments --- .travis.yml | 7 ++- system_tests/nox.py | 145 ++++++++++++++++++++++++-------------------- 2 files changed, 83 insertions(+), 69 deletions(-) diff --git a/.travis.yml b/.travis.yml index 14f22c75d..642101b6d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,9 +17,9 @@ matrix: - python: 3.5 env: TOXENV=cover - python: 3.5 - env: TOXENV=py35-system SYSTEM_TEST=1 SKIP_APP_ENGINE_SYSTEM_TEST=1 CLOUD_SDK_ROOT=${HOME}/.cache/cloud-sdk + env: TOXENV=py35-system SYSTEM_TEST=1 SKIP_APP_ENGINE_SYSTEM_TEST=1 - python: 2.7 - env: TOXENV=py27-system SYSTEM_TEST=1 SKIP_APP_ENGINE_SYSTEM_TEST=1 CLOUD_SDK_ROOT=${HOME}/.cache/cloud-sdk + env: TOXENV=py27-system SYSTEM_TEST=1 SKIP_APP_ENGINE_SYSTEM_TEST=1 cache: directories: - ${HOME}/.cache @@ -38,4 +38,5 @@ deploy: repo: GoogleCloudPlatform/google-auth-library-python env: global: - secure: s6GdhJklftl8w/9WoETwLtvtKL4ledPA/TuBuqCXQxSuYWaPuTdRVcvoejGkHJpp7i/7v2T/0etYl+5koyskKm5+QZZweaaL7MAyjPGp+hmIaIlWQRz6w481NOf3i9uSmoQycssT0mNmwScNIqo+igbA2y14mr/e9aBuOcxNNzNzFQp2vaRMEju6q7xZMjYdcudUWL48vq9CoNa3X2ZArpqjkApR/TfYlG7glOj43NxuVDN4z9wIyUjaMHBfPgEhjaOaRyEFgEYITRwX1qDoXqcZdTVIq4Cn0uCH+Mvrz6Y+oUJGTJqH1k7N/DhzbSN9lJnVYaQW/yuvGHiGAwbb6Tcxiq2UqqhA9MfbPpmstDECs46v9Z3BT252KvYEQY7Q1v9g2gFhHvFGWISUxs80rnnPhEYfa11JoLvj2t8cowkE4pvj4OH32Eoyvc5H07hW3F5xpuF7Jt7N09TNZkUrpmiRJEhfrVNgjsrWO77/q5h8mXGd+9vYmz++yzKu+63x8x1MpeigGCG73Dpu9Otm5eydOZfpJ39ZfZWUb7G2JahgHaGweM9dmnpJtzHQgijmHjjfAx9jgnQ8IQz9nkFmyMI8H7HouwalnrJtpSSbvMqOQ0kiZhMzdBKH5pD3tjLgSlgA0pKelBwlooY6jGlj4LrtbDAxa6cZyXiFoqWpT1w= + - secure: s6GdhJklftl8w/9WoETwLtvtKL4ledPA/TuBuqCXQxSuYWaPuTdRVcvoejGkHJpp7i/7v2T/0etYl+5koyskKm5+QZZweaaL7MAyjPGp+hmIaIlWQRz6w481NOf3i9uSmoQycssT0mNmwScNIqo+igbA2y14mr/e9aBuOcxNNzNzFQp2vaRMEju6q7xZMjYdcudUWL48vq9CoNa3X2ZArpqjkApR/TfYlG7glOj43NxuVDN4z9wIyUjaMHBfPgEhjaOaRyEFgEYITRwX1qDoXqcZdTVIq4Cn0uCH+Mvrz6Y+oUJGTJqH1k7N/DhzbSN9lJnVYaQW/yuvGHiGAwbb6Tcxiq2UqqhA9MfbPpmstDECs46v9Z3BT252KvYEQY7Q1v9g2gFhHvFGWISUxs80rnnPhEYfa11JoLvj2t8cowkE4pvj4OH32Eoyvc5H07hW3F5xpuF7Jt7N09TNZkUrpmiRJEhfrVNgjsrWO77/q5h8mXGd+9vYmz++yzKu+63x8x1MpeigGCG73Dpu9Otm5eydOZfpJ39ZfZWUb7G2JahgHaGweM9dmnpJtzHQgijmHjjfAx9jgnQ8IQz9nkFmyMI8H7HouwalnrJtpSSbvMqOQ0kiZhMzdBKH5pD3tjLgSlgA0pKelBwlooY6jGlj4LrtbDAxa6cZyXiFoqWpT1w= + - CLOUD_SDK_ROOT: ${HOME}/.cache/cloud-sdk diff --git a/system_tests/nox.py b/system_tests/nox.py index 6bd9c0fc7..881ee1901 100644 --- a/system_tests/nox.py +++ b/system_tests/nox.py @@ -24,7 +24,7 @@ import os -from nox.command import Command +from nox.command import which import py.path @@ -32,94 +32,116 @@ DATA_DIR = os.path.join(HERE, 'data') SERVICE_ACCOUNT_FILE = os.path.join(DATA_DIR, 'service_account.json') AUTHORIZED_USER_FILE = os.path.join(DATA_DIR, 'authorized_user.json') +EXPLICIT_CREDENTIALS_ENV = 'GOOGLE_APPLICATION_CREDENTIALS' +EXPLICIT_PROJECT_ENV = 'GOOGLE_CLOUD_PROJECT' +EXPECT_PROJECT_ENV = 'EXPECT_PROJECT_ID' + +# The download location for the Cloud SDK +CLOUD_SDK_DIST_FILENAME = 'google-cloud-sdk.tar.gz' +CLOUD_SDK_DOWNLOAD_URL = ( + 'https://dl.google.com/dl/cloudsdk/release/{}'.format( + CLOUD_SDK_DIST_FILENAME)) + +# This environment variable is recognized by the Cloud SDK and overrides +# the location of the SDK's configuration files (which is usually at +# ${HOME}/.config). CLOUD_SDK_CONFIG_ENV = 'CLOUDSDK_CONFIG' -# If set, this is where the environment setup will store the Cloud SDK. +# If set, this is where the environment setup will install the Cloud SDK. # If unset, it will download the SDK to a temporary directory. CLOUD_SDK_ROOT = os.environ.get('CLOUD_SDK_ROOT') -if CLOUD_SDK_ROOT: + +if CLOUD_SDK_ROOT is not None: CLOUD_SDK_ROOT = py.path.local(CLOUD_SDK_ROOT) - CLOUD_SDK_ROOT.ensure(dir=True) -if not CLOUD_SDK_ROOT: + CLOUD_SDK_ROOT.ensure(dir=True) # Makes sure the directory exists. +else: CLOUD_SDK_ROOT = py.path.local.mkdtemp() +# The full path the cloud sdk install directory +CLOUD_SDK_INSTALL_DIR = CLOUD_SDK_ROOT.join('google-cloud-sdk') -# Helper functions - - -def prerun(*args, **kwargs): - """Runs a command before the session.""" - kwargs.setdefault('silent', True) - env = os.environ.copy() - env.update(kwargs.pop('env', {})) - Command(args, env=env, **kwargs).run() +# The full path to the gcloud cli executable. +GCLOUD = str(CLOUD_SDK_INSTALL_DIR.join('bin', 'gcloud')) +# gcloud requires Python 2.7 and doesn't work on 3.x, so we need to tell it +# where to find 2.7 when we're running in a 3.x environment. +CLOUD_SDK_PYTHON_ENV = 'GCLOUD_PYTHON' +CLOUD_SDK_PYTHON = which('python2.7', None) # Cloud SDK helpers -def setup_cloud_sdk(): +def install_cloud_sdk(session): """Downloads and installs the Google Cloud SDK.""" + # Configure environment variables needed by the SDK. + # This sets the config root to the tests' config root. This prevents + # our tests from clobbering a developer's configuration when running + # these tests locally. + session.env[CLOUD_SDK_CONFIG_ENV] = str(CLOUD_SDK_ROOT) + # This tells gcloud which Python interpreter to use (always use 2.7) + session.env[CLOUD_SDK_PYTHON_ENV] = CLOUD_SDK_PYTHON # If the sdk already exists, we don't need to do anything else. - if CLOUD_SDK_ROOT.join('google-cloud-sdk').exists(): + # Note that because of this we do not attempt to update the sdk - + # if the CLOUD_SDK_ROOT is cached, it will need to be periodically cleared. + if CLOUD_SDK_INSTALL_DIR.exists(): return - tar_file = 'google-cloud-sdk.tar.gz' - tar_path = CLOUD_SDK_ROOT.join(tar_file) + tar_path = CLOUD_SDK_ROOT.join(CLOUD_SDK_DIST_FILENAME) # Download the release. - prerun( - 'wget', 'https://dl.google.com/dl/cloudsdk/release/{}'.format( - tar_file), - '-O', str(tar_path)) + session.run( + 'wget', CLOUD_SDK_DOWNLOAD_URL, '-O', str(tar_path), silent=True) # Extract the release. - prerun('tar', 'xzf', str(tar_path), '-C', str(CLOUD_SDK_ROOT)) - tar_path.remove() + session.run( + 'tar', 'xzf', str(tar_path), '-C', str(CLOUD_SDK_ROOT)) + session.run(tar_path.remove) # Run the install script. - prerun( - str(CLOUD_SDK_ROOT.join('google-cloud-sdk', 'install.sh')), + session.run( + str(CLOUD_SDK_INSTALL_DIR.join('install.sh')), '--usage-reporting', 'false', '--path-update', 'false', '--command-completion', 'false', - env={CLOUD_SDK_CONFIG_ENV: str(CLOUD_SDK_ROOT)}) + silent=True) - return CLOUD_SDK_ROOT - -def gcloud(*args, **kwargs): - """Calls the Cloud SDK CLI.""" - prog = str(CLOUD_SDK_ROOT.join('google-cloud-sdk', 'bin', 'gcloud')) - env = {CLOUD_SDK_CONFIG_ENV: str(CLOUD_SDK_ROOT)} - return prerun(prog, *args, env=env, **kwargs) - - -def configure_cloud_sdk(application_default_credentials, project=False): - """Configures the Cloud SDK with the given application default +def configure_cloud_sdk( + session, application_default_credentials, project=False): + """Installs and configures the Cloud SDK with the given application default credentials. If project is True, then a project will be set in the active config. If it is false, this will ensure no project is set. """ + install_cloud_sdk(session) if project: - gcloud('config', 'set', 'project', 'example-project') + session.run(GCLOUD, 'config', 'set', 'project', 'example-project') else: - gcloud('config', 'unset', 'project') + session.run(GCLOUD, 'config', 'unset', 'project') # Copy the credentials file to the config root. This is needed because # unfortunately gcloud doesn't provide a clean way to tell it to use # a particular set of credentials. However, this does verify that gcloud # also considers the credentials valid by calling application-default # print-access-token - dest = CLOUD_SDK_ROOT.join('application_default_credentials.json') - if dest.exists(): - dest.remove() - py.path.local(application_default_credentials).copy(dest) + def copy_credentials(): + dest = CLOUD_SDK_ROOT.join('application_default_credentials.json') + if dest.exists(): + dest.remove() + py.path.local(application_default_credentials).copy(dest) + + session.run(copy_credentials) - gcloud('auth', 'application-default', 'print-access-token') + # Calling this forces the Cloud SDK to read the credentials we just wrote + # and obtain a new access token with those credentials. This validates + # that our credentials matches the format expected by gcloud. + # Silent is set to True to prevent leaking secrets in test logs. + session.run( + GCLOUD, 'auth', 'application-default', 'print-access-token', + silent=True) # Test sesssions @@ -137,51 +159,42 @@ def session_oauth2_credentials(session): def session_default_explicit_service_account(session): session.virtualenv = False - session.env['GOOGLE_APPLICATION_CREDENTIALS'] = SERVICE_ACCOUNT_FILE - session.env['EXPECT_PROJECT_ID'] = '1' + session.env[EXPLICIT_CREDENTIALS_ENV] = SERVICE_ACCOUNT_FILE + session.env[EXPECT_PROJECT_ENV] = '1' session.run('pytest', 'test_default.py') def session_default_explicit_authorized_user(session): session.virtualenv = False - session.env['GOOGLE_APPLICATION_CREDENTIALS'] = AUTHORIZED_USER_FILE + session.env[EXPLICIT_CREDENTIALS_ENV] = AUTHORIZED_USER_FILE session.run('pytest', 'test_default.py') def session_default_explicit_authorized_user_explicit_project(session): session.virtualenv = False - session.env['GOOGLE_APPLICATION_CREDENTIALS'] = AUTHORIZED_USER_FILE - session.env['GOOGLE_CLOUD_PROJECT'] = 'example-project' - session.env['EXPECT_PROJECT_ID'] = '1' + session.env[EXPLICIT_CREDENTIALS_ENV] = AUTHORIZED_USER_FILE + session.env[EXPLICIT_PROJECT_ENV] = 'example-project' + session.env[EXPECT_PROJECT_ENV] = '1' session.run('pytest', 'test_default.py') def session_default_cloud_sdk_service_account(session): session.virtualenv = False - setup_cloud_sdk() - configure_cloud_sdk(SERVICE_ACCOUNT_FILE) - - session.env[CLOUD_SDK_CONFIG_ENV] = str(CLOUD_SDK_ROOT) - session.env['EXPECT_PROJECT_ID'] = '1' + configure_cloud_sdk(session, SERVICE_ACCOUNT_FILE) + session.env[EXPECT_PROJECT_ENV] = '1' session.run('pytest', 'test_default.py') def session_default_cloud_sdk_authorized_user(session): session.virtualenv = False - setup_cloud_sdk() - configure_cloud_sdk(AUTHORIZED_USER_FILE) - - session.env[CLOUD_SDK_CONFIG_ENV] = str(CLOUD_SDK_ROOT) - session.run('pytest', '--pdb', 'test_default.py') + configure_cloud_sdk(session, AUTHORIZED_USER_FILE) + session.run('pytest', 'test_default.py') def session_default_cloud_sdk_authorized_user_configured_project(session): session.virtualenv = False - setup_cloud_sdk() - configure_cloud_sdk(AUTHORIZED_USER_FILE, project=True) - - session.env[CLOUD_SDK_CONFIG_ENV] = str(CLOUD_SDK_ROOT) - session.env['EXPECT_PROJECT_ID'] = '1' + configure_cloud_sdk(session, AUTHORIZED_USER_FILE, project=True) + session.env[EXPECT_PROJECT_ENV] = '1' session.run('pytest', 'test_default.py') From b2ca56be3847973411c02d1e604e17c71124dc77 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 27 Oct 2016 22:29:56 -0700 Subject: [PATCH 09/12] Address review comments --- system_tests/nox.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/system_tests/nox.py b/system_tests/nox.py index 881ee1901..1679c1414 100644 --- a/system_tests/nox.py +++ b/system_tests/nox.py @@ -22,6 +22,7 @@ .. _nox docs: http://nox.readthedocs.io/en/latest/ """ +import functools import os from nox.command import which @@ -65,7 +66,7 @@ # gcloud requires Python 2.7 and doesn't work on 3.x, so we need to tell it # where to find 2.7 when we're running in a 3.x environment. -CLOUD_SDK_PYTHON_ENV = 'GCLOUD_PYTHON' +CLOUD_SDK_PYTHON_ENV = 'CLOUDSDK_PYTHON' CLOUD_SDK_PYTHON = which('python2.7', None) # Cloud SDK helpers @@ -107,6 +108,15 @@ def install_cloud_sdk(session): silent=True) +def copy_credentials(credentials_path): + """Copies credentials into the SDK root as the application default + credentials.""" + dest = CLOUD_SDK_ROOT.join('application_default_credentials.json') + if dest.exists(): + dest.remove() + py.path.local(credentials_path).copy(dest) + + def configure_cloud_sdk( session, application_default_credentials, project=False): """Installs and configures the Cloud SDK with the given application default @@ -127,13 +137,8 @@ def configure_cloud_sdk( # a particular set of credentials. However, this does verify that gcloud # also considers the credentials valid by calling application-default # print-access-token - def copy_credentials(): - dest = CLOUD_SDK_ROOT.join('application_default_credentials.json') - if dest.exists(): - dest.remove() - py.path.local(application_default_credentials).copy(dest) - - session.run(copy_credentials) + session.run( + functools.partial(copy_credentials, application_default_credentials)) # Calling this forces the Cloud SDK to read the credentials we just wrote # and obtain a new access token with those credentials. This validates From 294792c573ddd726d71de0fc53952b213e910eff Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 27 Oct 2016 22:50:20 -0700 Subject: [PATCH 10/12] Fixing function call --- system_tests/nox.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/system_tests/nox.py b/system_tests/nox.py index 1679c1414..16f0255d8 100644 --- a/system_tests/nox.py +++ b/system_tests/nox.py @@ -22,7 +22,6 @@ .. _nox docs: http://nox.readthedocs.io/en/latest/ """ -import functools import os from nox.command import which @@ -64,10 +63,11 @@ # The full path to the gcloud cli executable. GCLOUD = str(CLOUD_SDK_INSTALL_DIR.join('bin', 'gcloud')) -# gcloud requires Python 2.7 and doesn't work on 3.x, so we need to tell it -# where to find 2.7 when we're running in a 3.x environment. +# gcloud requires Python 2 and doesn't work on 3, so we need to tell it +# where to find 2 when we're running in a 3 environment. CLOUD_SDK_PYTHON_ENV = 'CLOUDSDK_PYTHON' -CLOUD_SDK_PYTHON = which('python2.7', None) +CLOUD_SDK_PYTHON = which('python2', None) +print('Python 2: ', CLOUD_SDK_PYTHON) # Cloud SDK helpers @@ -137,8 +137,7 @@ def configure_cloud_sdk( # a particular set of credentials. However, this does verify that gcloud # also considers the credentials valid by calling application-default # print-access-token - session.run( - functools.partial(copy_credentials, application_default_credentials)) + session.run(copy_credentials, application_default_credentials) # Calling this forces the Cloud SDK to read the credentials we just wrote # and obtain a new access token with those credentials. This validates From f82ba10758b8dba2c488bc09a7e857fdbf7e5883 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 27 Oct 2016 23:07:18 -0700 Subject: [PATCH 11/12] Testing travis --- system_tests/nox.py | 5 ++--- system_tests/test_compute_engine.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/system_tests/nox.py b/system_tests/nox.py index 16f0255d8..5478240c5 100644 --- a/system_tests/nox.py +++ b/system_tests/nox.py @@ -67,7 +67,6 @@ # where to find 2 when we're running in a 3 environment. CLOUD_SDK_PYTHON_ENV = 'CLOUDSDK_PYTHON' CLOUD_SDK_PYTHON = which('python2', None) -print('Python 2: ', CLOUD_SDK_PYTHON) # Cloud SDK helpers @@ -82,10 +81,10 @@ def install_cloud_sdk(session): # This tells gcloud which Python interpreter to use (always use 2.7) session.env[CLOUD_SDK_PYTHON_ENV] = CLOUD_SDK_PYTHON - # If the sdk already exists, we don't need to do anything else. + # If the glcoud already exists, we don't need to do anything else. # Note that because of this we do not attempt to update the sdk - # if the CLOUD_SDK_ROOT is cached, it will need to be periodically cleared. - if CLOUD_SDK_INSTALL_DIR.exists(): + if GCLOUD.exists(): return tar_path = CLOUD_SDK_ROOT.join(CLOUD_SDK_DIST_FILENAME) diff --git a/system_tests/test_compute_engine.py b/system_tests/test_compute_engine.py index 7fd31b454..ceba0f787 100644 --- a/system_tests/test_compute_engine.py +++ b/system_tests/test_compute_engine.py @@ -21,7 +21,7 @@ @pytest.fixture(autouse=True) def check_gce_environment(http_request): - if not _metadata.ping(http_request): + if not _metadata.ping(http_request, timeout=1): pytest.skip('Compute Engine metadata service is not available.') From ee0049cd564de94ff3f94a10f8ba2847673599a6 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 27 Oct 2016 23:09:52 -0700 Subject: [PATCH 12/12] Testing travis --- system_tests/conftest.py | 2 +- system_tests/nox.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/system_tests/conftest.py b/system_tests/conftest.py index f92ac6541..afa78545e 100644 --- a/system_tests/conftest.py +++ b/system_tests/conftest.py @@ -25,7 +25,7 @@ DATA_DIR = os.path.join(HERE, 'data') SERVICE_ACCOUNT_FILE = os.path.join(DATA_DIR, 'service_account.json') AUTHORIZED_USER_FILE = os.path.join(DATA_DIR, 'authorized_user.json') -HTTP = urllib3.PoolManager() +HTTP = urllib3.PoolManager(retries=False) TOKEN_INFO_URL = 'https://www.googleapis.com/oauth2/v3/tokeninfo' diff --git a/system_tests/nox.py b/system_tests/nox.py index 5478240c5..67f99004d 100644 --- a/system_tests/nox.py +++ b/system_tests/nox.py @@ -84,7 +84,7 @@ def install_cloud_sdk(session): # If the glcoud already exists, we don't need to do anything else. # Note that because of this we do not attempt to update the sdk - # if the CLOUD_SDK_ROOT is cached, it will need to be periodically cleared. - if GCLOUD.exists(): + if py.path.local(GCLOUD).exists(): return tar_path = CLOUD_SDK_ROOT.join(CLOUD_SDK_DIST_FILENAME)