From 3aeda3a2838457e22fe16a39c8ddf3e294c1f2d2 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Wed, 3 Feb 2021 02:35:33 +0000 Subject: [PATCH 1/3] feat: support self-signed-jwt in requests and urllib3 transports --- google/auth/transport/requests.py | 18 ++++++++ google/auth/transport/urllib3.py | 18 ++++++++ system_tests/noxfile.py | 17 +++++++ system_tests/system_tests_sync/test_grpc.py | 3 +- .../system_tests_sync/test_requests.py | 40 +++++++++++++++++ .../system_tests_sync/test_urllib3.py | 44 +++++++++++++++++++ tests/transport/test_requests.py | 23 ++++++++++ tests/transport/test_urllib3.py | 22 ++++++++++ 8 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 system_tests/system_tests_sync/test_requests.py create mode 100644 system_tests/system_tests_sync/test_urllib3.py diff --git a/google/auth/transport/requests.py b/google/auth/transport/requests.py index 9a2f3afc7..92205bd0b 100644 --- a/google/auth/transport/requests.py +++ b/google/auth/transport/requests.py @@ -44,6 +44,7 @@ from google.auth import environment_vars from google.auth import exceptions from google.auth import transport +from google.oauth2 import service_account import google.auth.transport._mtls_helper _LOGGER = logging.getLogger(__name__) @@ -313,6 +314,9 @@ def my_cert_callback(): refreshing credentials. If not passed, an instance of :class:`~google.auth.transport.requests.Request` is created. + default_host (Optional[str]): A host like "pubsub.googleapis.com". + This is used when a self-signed JWT is created from service + account credentials. """ def __init__( @@ -322,6 +326,7 @@ def __init__( max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS, refresh_timeout=None, auth_request=None, + default_host=None, ): super(AuthorizedSession, self).__init__() self.credentials = credentials @@ -329,6 +334,7 @@ def __init__( self._max_refresh_attempts = max_refresh_attempts self._refresh_timeout = refresh_timeout self._is_mtls = False + self._default_host = default_host if auth_request is None: auth_request_session = requests.Session() @@ -347,6 +353,18 @@ def __init__( # credentials.refresh). self._auth_request = auth_request + # https://google.aip.dev/auth/4111 + # Attempt to use self-signed JWTs when a service account is used. + # A default host must be explicitly provided. + if ( + isinstance(self.credentials, service_account.Credentials) + and self._default_host + ): + self.credentials._create_self_signed_jwt( + "https://{}/".format(self._default_host) + ) + + def configure_mtls_channel(self, client_cert_callback=None): """Configure the client certificate and key for SSL connection. diff --git a/google/auth/transport/urllib3.py b/google/auth/transport/urllib3.py index 209fc51bc..6c4b5ef29 100644 --- a/google/auth/transport/urllib3.py +++ b/google/auth/transport/urllib3.py @@ -49,6 +49,7 @@ from google.auth import environment_vars from google.auth import exceptions from google.auth import transport +from google.oauth2 import service_account _LOGGER = logging.getLogger(__name__) @@ -262,6 +263,9 @@ def my_cert_callback(): retried. max_refresh_attempts (int): The maximum number of times to attempt to refresh the credentials and retry the request. + default_host (Optional[str]): A host like "pubsub.googleapis.com". + This is used when a self-signed JWT is created from service + account credentials. """ def __init__( @@ -270,6 +274,7 @@ def __init__( http=None, refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES, max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS, + default_host=None, ): if http is None: self.http = _make_default_http() @@ -281,10 +286,23 @@ def __init__( self.credentials = credentials self._refresh_status_codes = refresh_status_codes self._max_refresh_attempts = max_refresh_attempts + self._default_host = default_host # Request instance used by internal methods (for example, # credentials.refresh). self._request = Request(self.http) + # https://google.aip.dev/auth/4111 + # Attempt to use self-signed JWTs when a service account is used. + # A default host must be explicitly provided. + if ( + isinstance(self.credentials, service_account.Credentials) + and self._default_host + ): + self.credentials._create_self_signed_jwt( + "https://{}/".format(self._default_host) + ) + + super(AuthorizedHttp, self).__init__() def configure_mtls_channel(self, client_cert_callback=None): diff --git a/system_tests/noxfile.py b/system_tests/noxfile.py index 5d0014bc8..3693b4275 100644 --- a/system_tests/noxfile.py +++ b/system_tests/noxfile.py @@ -293,6 +293,22 @@ def grpc(session): session.run("pytest", "system_tests_sync/test_grpc.py") +@nox.session(python=PYTHON_VERSIONS_SYNC) +def requests(session): + session.install(LIBRARY_DIR) + session.install(*TEST_DEPENDENCIES_SYNC) + session.env[EXPLICIT_CREDENTIALS_ENV] = SERVICE_ACCOUNT_FILE + session.run("pytest", "system_tests_sync/test_requests.py") + + +@nox.session(python=PYTHON_VERSIONS_SYNC) +def urllib3(session): + session.install(LIBRARY_DIR) + session.install(*TEST_DEPENDENCIES_SYNC) + session.env[EXPLICIT_CREDENTIALS_ENV] = SERVICE_ACCOUNT_FILE + session.run("pytest", "system_tests_sync/test_requests.py") + + @nox.session(python=PYTHON_VERSIONS_SYNC) def mtls_http(session): session.install(LIBRARY_DIR) @@ -300,6 +316,7 @@ def mtls_http(session): session.env[EXPLICIT_CREDENTIALS_ENV] = SERVICE_ACCOUNT_FILE session.run("pytest", "system_tests_sync/test_mtls_http.py") + #ASYNC SYSTEM TESTS @nox.session(python=PYTHON_VERSIONS_ASYNC) diff --git a/system_tests/system_tests_sync/test_grpc.py b/system_tests/system_tests_sync/test_grpc.py index da2eb71fb..7f548ec0e 100644 --- a/system_tests/system_tests_sync/test_grpc.py +++ b/system_tests/system_tests_sync/test_grpc.py @@ -57,8 +57,9 @@ def test_grpc_request_with_regular_credentials_and_self_signed_jwt(http_request) list_topics_iter = client.list_topics(project="projects/{}".format(project_id)) list(list_topics_iter) - # Check that self-signed JWT was created + # Check that self-signed JWT was created and is being used assert credentials._jwt_credentials is not None + assert credentials._jwt_credentials.token == credentials.token def test_grpc_request_with_jwt_credentials(): diff --git a/system_tests/system_tests_sync/test_requests.py b/system_tests/system_tests_sync/test_requests.py new file mode 100644 index 000000000..3ac9179b5 --- /dev/null +++ b/system_tests/system_tests_sync/test_requests.py @@ -0,0 +1,40 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import google.auth +import google.auth.credentials +import google.auth.transport.requests +from google.oauth2 import service_account + + +def test_authorized_session_with_service_account_and_self_signed_jwt(): + credentials, project_id = google.auth.default() + + credentials = credentials.with_scopes( + scopes=[], + default_scopes=["https://www.googleapis.com/auth/pubsub"], + ) + + session = google.auth.transport.requests.AuthorizedSession( + credentials=credentials, default_host="pubsub.googleapis.com" + ) + + # List Pub/Sub Topics through the REST API + # https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/list + response = session.get("https://pubsub.googleapis.com/v1/projects/{}/topics".format(project_id)) + response.raise_for_status() + + # Check that self-signed JWT was created and is being used + assert credentials._jwt_credentials is not None + assert credentials._jwt_credentials.token == credentials.token diff --git a/system_tests/system_tests_sync/test_urllib3.py b/system_tests/system_tests_sync/test_urllib3.py new file mode 100644 index 000000000..1932e1913 --- /dev/null +++ b/system_tests/system_tests_sync/test_urllib3.py @@ -0,0 +1,44 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import google.auth +import google.auth.credentials +import google.auth.transport.requests +from google.oauth2 import service_account + + +def test_authorized_session_with_service_account_and_self_signed_jwt(): + credentials, project_id = google.auth.default() + + credentials = credentials.with_scopes( + scopes=[], + default_scopes=["https://www.googleapis.com/auth/pubsub"], + ) + + http = google.auth.transport.urllib3.AuthorizedHttp( + credentials=credentials, default_host="pubsub.googleapis.com" + ) + + # List Pub/Sub Topics through the REST API + # https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/list + response = http.urlopen( + method="GET", + url="https://pubsub.googleapis.com/v1/projects/{}/topics".format(project_id) + ) + + assert response.status == 200 + + # Check that self-signed JWT was created and is being used + assert credentials._jwt_credentials is not None + assert credentials._jwt_credentials.token == credentials.token diff --git a/tests/transport/test_requests.py b/tests/transport/test_requests.py index d56c2be55..1f5011236 100644 --- a/tests/transport/test_requests.py +++ b/tests/transport/test_requests.py @@ -27,6 +27,7 @@ from google.auth import environment_vars from google.auth import exceptions +from google.oauth2 import service_account import google.auth.credentials import google.auth.transport._mtls_helper import google.auth.transport.requests @@ -372,6 +373,28 @@ def test_request_timeout_w_refresh_timeout_timeout_error(self, frozen_time): "GET", self.TEST_URL, timeout=60, max_allowed_time=2.9 ) + def test_authorized_session_without_default_host(self): + credentials = mock.create_autospec(service_account.Credentials) + + authed_session = google.auth.transport.requests.AuthorizedSession( + credentials, + ) + + authed_session.credentials._create_self_signed_jwt.assert_not_called() + + def test_authorized_session_with_default_host(self): + default_host = "pubsub.googleapis.com" + credentials = mock.create_autospec(service_account.Credentials) + + authed_session = google.auth.transport.requests.AuthorizedSession( + credentials, + default_host=default_host + ) + + authed_session.credentials._create_self_signed_jwt.assert_called_once_with( + "https://{}/".format(default_host) + ) + def test_configure_mtls_channel_with_callback(self): mock_callback = mock.Mock() mock_callback.return_value = ( diff --git a/tests/transport/test_urllib3.py b/tests/transport/test_urllib3.py index 29561f6d6..a34600fc2 100644 --- a/tests/transport/test_urllib3.py +++ b/tests/transport/test_urllib3.py @@ -23,6 +23,7 @@ from google.auth import environment_vars from google.auth import exceptions +from google.oauth2 import service_account import google.auth.credentials import google.auth.transport._mtls_helper import google.auth.transport.urllib3 @@ -157,6 +158,27 @@ def test_urlopen_refresh(self): ("GET", self.TEST_URL, None, {"authorization": "token"}, {}), ("GET", self.TEST_URL, None, {"authorization": "token1"}, {}), ] + + def test_urlopen_no_default_host(self): + credentials = mock.create_autospec(service_account.Credentials) + + authed_http = google.auth.transport.urllib3.AuthorizedHttp( + credentials + ) + + authed_http.credentials._create_self_signed_jwt.assert_not_called() + + def test_urlopen_with_default_host(self): + default_host = "pubsub.googleapis.com" + credentials = mock.create_autospec(service_account.Credentials) + + authed_http = google.auth.transport.urllib3.AuthorizedHttp( + credentials, default_host=default_host + ) + + authed_http.credentials._create_self_signed_jwt.assert_called_once_with( + "https://{}/".format(default_host) + ) def test_proxies(self): http = mock.create_autospec(urllib3.PoolManager) From 7b2432ed3a3fe4511287e8d10d4dde41191edd06 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Wed, 3 Feb 2021 02:38:12 +0000 Subject: [PATCH 2/3] chore: lint --- google/auth/transport/requests.py | 3 +-- google/auth/transport/urllib3.py | 1 - tests/transport/test_requests.py | 9 +++------ tests/transport/test_urllib3.py | 8 +++----- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/google/auth/transport/requests.py b/google/auth/transport/requests.py index 92205bd0b..ef973fce4 100644 --- a/google/auth/transport/requests.py +++ b/google/auth/transport/requests.py @@ -44,8 +44,8 @@ from google.auth import environment_vars from google.auth import exceptions from google.auth import transport -from google.oauth2 import service_account import google.auth.transport._mtls_helper +from google.oauth2 import service_account _LOGGER = logging.getLogger(__name__) @@ -364,7 +364,6 @@ def __init__( "https://{}/".format(self._default_host) ) - def configure_mtls_channel(self, client_cert_callback=None): """Configure the client certificate and key for SSL connection. diff --git a/google/auth/transport/urllib3.py b/google/auth/transport/urllib3.py index 6c4b5ef29..aadd116e8 100644 --- a/google/auth/transport/urllib3.py +++ b/google/auth/transport/urllib3.py @@ -302,7 +302,6 @@ def __init__( "https://{}/".format(self._default_host) ) - super(AuthorizedHttp, self).__init__() def configure_mtls_channel(self, client_cert_callback=None): diff --git a/tests/transport/test_requests.py b/tests/transport/test_requests.py index 1f5011236..3fdd17c3e 100644 --- a/tests/transport/test_requests.py +++ b/tests/transport/test_requests.py @@ -27,10 +27,10 @@ from google.auth import environment_vars from google.auth import exceptions -from google.oauth2 import service_account import google.auth.credentials import google.auth.transport._mtls_helper import google.auth.transport.requests +from google.oauth2 import service_account from tests.transport import compliance @@ -376,9 +376,7 @@ def test_request_timeout_w_refresh_timeout_timeout_error(self, frozen_time): def test_authorized_session_without_default_host(self): credentials = mock.create_autospec(service_account.Credentials) - authed_session = google.auth.transport.requests.AuthorizedSession( - credentials, - ) + authed_session = google.auth.transport.requests.AuthorizedSession(credentials) authed_session.credentials._create_self_signed_jwt.assert_not_called() @@ -387,8 +385,7 @@ def test_authorized_session_with_default_host(self): credentials = mock.create_autospec(service_account.Credentials) authed_session = google.auth.transport.requests.AuthorizedSession( - credentials, - default_host=default_host + credentials, default_host=default_host ) authed_session.credentials._create_self_signed_jwt.assert_called_once_with( diff --git a/tests/transport/test_urllib3.py b/tests/transport/test_urllib3.py index a34600fc2..7c0693476 100644 --- a/tests/transport/test_urllib3.py +++ b/tests/transport/test_urllib3.py @@ -23,10 +23,10 @@ from google.auth import environment_vars from google.auth import exceptions -from google.oauth2 import service_account import google.auth.credentials import google.auth.transport._mtls_helper import google.auth.transport.urllib3 +from google.oauth2 import service_account from tests.transport import compliance @@ -158,13 +158,11 @@ def test_urlopen_refresh(self): ("GET", self.TEST_URL, None, {"authorization": "token"}, {}), ("GET", self.TEST_URL, None, {"authorization": "token1"}, {}), ] - + def test_urlopen_no_default_host(self): credentials = mock.create_autospec(service_account.Credentials) - authed_http = google.auth.transport.urllib3.AuthorizedHttp( - credentials - ) + authed_http = google.auth.transport.urllib3.AuthorizedHttp(credentials) authed_http.credentials._create_self_signed_jwt.assert_not_called() From bd441d8ebe3fae05d64d6ed8735a95451010d7de Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Wed, 3 Feb 2021 15:57:21 -0700 Subject: [PATCH 3/3] test: fix filename for urllib3 system test --- system_tests/noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/noxfile.py b/system_tests/noxfile.py index 3693b4275..4ba7cc3ef 100644 --- a/system_tests/noxfile.py +++ b/system_tests/noxfile.py @@ -306,7 +306,7 @@ def urllib3(session): session.install(LIBRARY_DIR) session.install(*TEST_DEPENDENCIES_SYNC) session.env[EXPLICIT_CREDENTIALS_ENV] = SERVICE_ACCOUNT_FILE - session.run("pytest", "system_tests_sync/test_requests.py") + session.run("pytest", "system_tests_sync/test_urllib3.py") @nox.session(python=PYTHON_VERSIONS_SYNC)