From 89d6f35c54b0a9b81c9b5f580d2e9eb87352ed93 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Wed, 21 Apr 2021 12:55:19 -0600 Subject: [PATCH] feat: support self-signed JWT flow for service accounts (#774) See [RFC (internal only)](https://docs.google.com/document/d/1SNCVTmW6Rtr__u-_V7nsT9PhSzjj1z0P9fAD3YUgRoc/edit#) and https://aip.dev/auth/4111. Support the self-signed JWT flow for service accounts by passing `default_scopes` and `default_host` in calls to the auth library and `create_channel`. This depends on features exposed in the following PRs: https://github.com/googleapis/python-api-core/pull/134, https://github.com/googleapis/google-auth-library-python/pull/665. It may be easier to look at https://github.com/googleapis/python-translate/pull/107/files for a diff on a real library. This change is written so that the library is (temporarily) compatible with older `google-api-core` and `google-auth` versions. Because of this it not possible to reach 100% coverage on a single unit test run. `pytest` runs twice in two of the `nox` sessions. Miscellaneous changes: - sprinkled in `__init__.py` files in subdirs of the `test/` directory, as otherwise pytest-cov seems to fail to collect coverage properly in some instances. - new dependency on `packaging` for Version comparison https://pypi.org/project/packaging/ Co-authored-by: Brent Shaffer --- .../services/%service/transports/base.py.j2 | 96 +++++-- .../services/%service/transports/grpc.py.j2 | 8 +- .../%service/transports/grpc_asyncio.py.j2 | 10 +- .../services/%service/transports/rest.py.j2 | 9 +- gapic/templates/.coveragerc.j2 | 1 - gapic/templates/noxfile.py.j2 | 62 +++++ gapic/templates/setup.py.j2 | 3 +- gapic/templates/tests/__init__.py.j2 | 2 + gapic/templates/tests/unit/__init__.py.j2 | 2 + .../%name_%version/%sub/test_%service.py.j2 | 245 ++++++++++++++++-- .../templates/tests/unit/gapic/__init__.py.j2 | 2 + noxfile.py | 80 ++++-- 12 files changed, 458 insertions(+), 62 deletions(-) create mode 100644 gapic/templates/tests/__init__.py.j2 create mode 100644 gapic/templates/tests/unit/__init__.py.j2 create mode 100644 gapic/templates/tests/unit/gapic/__init__.py.j2 diff --git a/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/base.py.j2 b/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/base.py.j2 index fabd1769f7..f5d9ee5dc6 100644 --- a/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/base.py.j2 +++ b/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/base.py.j2 @@ -2,10 +2,12 @@ {% block content %} import abc -import typing +from typing import Awaitable, Callable, Dict, Optional, Sequence, Union +import packaging.version import pkg_resources from google import auth # type: ignore +import google.api_core # type: ignore from google.api_core import exceptions # type: ignore from google.api_core import gapic_v1 # type: ignore from google.api_core import retry as retries # type: ignore @@ -34,6 +36,18 @@ try: except pkg_resources.DistributionNotFound: DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() +try: + # google.auth.__version__ was added in 1.26.0 + _GOOGLE_AUTH_VERSION = auth.__version__ +except AttributeError: + try: # try pkg_resources if it is available + _GOOGLE_AUTH_VERSION = pkg_resources.get_distribution("google-auth").version + except pkg_resources.DistributionNotFound: # pragma: NO COVER + _GOOGLE_AUTH_VERSION = None + +_API_CORE_VERSION = google.api_core.__version__ + + class {{ service.name }}Transport(abc.ABC): """Abstract transport class for {{ service.name }}.""" @@ -43,13 +57,15 @@ class {{ service.name }}Transport(abc.ABC): {%- endfor %} ) + DEFAULT_HOST: str = {% if service.host %}'{{ service.host }}'{% else %}{{ '' }}{% endif %} + def __init__( self, *, - host: str{% if service.host %} = '{{ service.host }}'{% endif %}, + host: str = DEFAULT_HOST, credentials: credentials.Credentials = None, - credentials_file: typing.Optional[str] = None, - scopes: typing.Optional[typing.Sequence[str]] = AUTH_SCOPES, - quota_project_id: typing.Optional[str] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, **kwargs, ) -> None: @@ -66,7 +82,7 @@ class {{ service.name }}Transport(abc.ABC): credentials_file (Optional[str]): A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is mutually exclusive with credentials. - scope (Optional[Sequence[str]]): A list of scopes. + scopes (Optional[Sequence[str]]): A list of scopes. quota_project_id (Optional[str]): An optional project to use for billing and quota. client_info (google.api_core.gapic_v1.client_info.ClientInfo): @@ -80,6 +96,8 @@ class {{ service.name }}Transport(abc.ABC): host += ':443' self._host = host + scopes_kwargs = self._get_scopes_kwargs(self._host, scopes) + # Save the scopes. self._scopes = scopes or self.AUTH_SCOPES @@ -91,17 +109,59 @@ class {{ service.name }}Transport(abc.ABC): if credentials_file is not None: credentials, _ = auth.load_credentials_from_file( credentials_file, - scopes=self._scopes, + **scopes_kwargs, quota_project_id=quota_project_id ) elif credentials is None: - credentials, _ = auth.default(scopes=self._scopes, quota_project_id=quota_project_id) + credentials, _ = auth.default(**scopes_kwargs, quota_project_id=quota_project_id) # Save the credentials. self._credentials = credentials + # TODO(busunkim): These two class methods are in the base transport + # to avoid duplicating code across the transport classes. These functions + # should be deleted once the minimum required versions of google-api-core + # and google-auth are increased. + + # TODO: Remove this function once google-auth >= 1.25.0 is required + @classmethod + def _get_scopes_kwargs(cls, host: str, scopes: Optional[Sequence[str]]) -> Dict[str, Optional[Sequence[str]]]: + """Returns scopes kwargs to pass to google-auth methods depending on the google-auth version""" + + scopes_kwargs = {} + + if _GOOGLE_AUTH_VERSION and ( + packaging.version.parse(_GOOGLE_AUTH_VERSION) + >= packaging.version.parse("1.25.0") + ): + scopes_kwargs = {"scopes": scopes, "default_scopes": cls.AUTH_SCOPES} + else: + scopes_kwargs = {"scopes": scopes or cls.AUTH_SCOPES} + + return scopes_kwargs + + # TODO: Remove this function once google-api-core >= 1.26.0 is required + @classmethod + def _get_self_signed_jwt_kwargs(cls, host: str, scopes: Optional[Sequence[str]]) -> Dict[str, Union[Optional[Sequence[str]], str]]: + """Returns kwargs to pass to grpc_helpers.create_channel depending on the google-api-core version""" + + self_signed_jwt_kwargs: Dict[str, Union[Optional[Sequence[str]], str]] = {} + + if _API_CORE_VERSION and ( + packaging.version.parse(_API_CORE_VERSION) + >= packaging.version.parse("1.26.0") + ): + self_signed_jwt_kwargs["default_scopes"] = cls.AUTH_SCOPES + self_signed_jwt_kwargs["scopes"] = scopes + self_signed_jwt_kwargs["default_host"] = cls.DEFAULT_HOST + else: + self_signed_jwt_kwargs["scopes"] = scopes or cls.AUTH_SCOPES + + return self_signed_jwt_kwargs + + def _prep_wrapped_messages(self, client_info): # Precompute the wrapped methods. self._wrapped_methods = { @@ -138,11 +198,11 @@ class {{ service.name }}Transport(abc.ABC): {%- for method in service.methods.values() %} @property - def {{ method.name|snake_case }}(self) -> typing.Callable[ + def {{ method.name|snake_case }}(self) -> Callable[ [{{ method.input.ident }}], - typing.Union[ + Union[ {{ method.output.ident }}, - typing.Awaitable[{{ method.output.ident }}] + Awaitable[{{ method.output.ident }}] ]]: raise NotImplementedError() {%- endfor %} @@ -152,29 +212,29 @@ class {{ service.name }}Transport(abc.ABC): @property def set_iam_policy( self, - ) -> typing.Callable[ + ) -> Callable[ [iam_policy.SetIamPolicyRequest], - typing.Union[policy.Policy, typing.Awaitable[policy.Policy]], + Union[policy.Policy, Awaitable[policy.Policy]], ]: raise NotImplementedError() @property def get_iam_policy( self, - ) -> typing.Callable[ + ) -> Callable[ [iam_policy.GetIamPolicyRequest], - typing.Union[policy.Policy, typing.Awaitable[policy.Policy]], + Union[policy.Policy, Awaitable[policy.Policy]], ]: raise NotImplementedError() @property def test_iam_permissions( self, - ) -> typing.Callable[ + ) -> Callable[ [iam_policy.TestIamPermissionsRequest], - typing.Union[ + Union[ iam_policy.TestIamPermissionsResponse, - typing.Awaitable[iam_policy.TestIamPermissionsResponse], + Awaitable[iam_policy.TestIamPermissionsResponse], ], ]: raise NotImplementedError() diff --git a/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc.py.j2 b/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc.py.j2 index 7d54941190..e7df35a1df 100644 --- a/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc.py.j2 +++ b/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc.py.j2 @@ -2,7 +2,7 @@ {% block content %} import warnings -from typing import Callable, Dict, Optional, Sequence, Tuple +from typing import Callable, Dict, Optional, Sequence, Tuple, Union from google.api_core import grpc_helpers # type: ignore {%- if service.has_lro %} @@ -202,13 +202,15 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport): google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` and ``credentials_file`` are passed. """ - scopes = scopes or cls.AUTH_SCOPES + + self_signed_jwt_kwargs = cls._get_self_signed_jwt_kwargs(host, scopes) + return grpc_helpers.create_channel( host, credentials=credentials, credentials_file=credentials_file, - scopes=scopes, quota_project_id=quota_project_id, + **self_signed_jwt_kwargs, **kwargs ) diff --git a/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc_asyncio.py.j2 b/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc_asyncio.py.j2 index 21d9311c4e..accb46fc91 100644 --- a/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc_asyncio.py.j2 +++ b/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc_asyncio.py.j2 @@ -2,7 +2,7 @@ {% block content %} import warnings -from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple +from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union from google.api_core import gapic_v1 # type: ignore from google.api_core import grpc_helpers_async # type: ignore @@ -12,6 +12,7 @@ from google.api_core import operations_v1 # type: ignore from google import auth # type: ignore from google.auth import credentials # type: ignore from google.auth.transport.grpc import SslCredentials # type: ignore +import packaging.version import grpc # type: ignore from grpc.experimental import aio # type: ignore @@ -75,13 +76,15 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport): Returns: aio.Channel: A gRPC AsyncIO channel object. """ - scopes = scopes or cls.AUTH_SCOPES + + self_signed_jwt_kwargs = cls._get_self_signed_jwt_kwargs(host, scopes) + return grpc_helpers_async.create_channel( host, credentials=credentials, credentials_file=credentials_file, - scopes=scopes, quota_project_id=quota_project_id, + **self_signed_jwt_kwargs, **kwargs ) @@ -163,7 +166,6 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport): # If a channel was explicitly provided, set it. self._grpc_channel = channel self._ssl_channel_credentials = None - else: if api_mtls_endpoint: host = api_mtls_endpoint diff --git a/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2 b/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2 index 9b2e1ff52f..4f30997ce4 100644 --- a/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2 +++ b/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2 @@ -81,12 +81,14 @@ class {{ service.name }}RestTransport({{ service.name }}Transport): """ # Run the base constructor # TODO(yon-mg): resolve other ctor params i.e. scopes, quota, etc. + # TODO: When custom host (api_endpoint) is set, `scopes` must *also* be set on the + # credentials object super().__init__( host=host, credentials=credentials, client_info=client_info, ) - self._session = AuthorizedSession(self._credentials) + self._session = AuthorizedSession(self._credentials, default_host=self.DEFAULT_HOST) {%- if service.has_lro %} self._operations_client = None {%- endif %} @@ -106,11 +108,14 @@ class {{ service.name }}RestTransport({{ service.name }}Transport): # Sanity check: Only create a new client if we do not already have one. if self._operations_client is None: from google.api_core import grpc_helpers + + self_signed_jwt_kwargs = cls._get_self_signed_jwt_kwargs(self._host, self._scopes) + self._operations_client = operations_v1.OperationsClient( grpc_helpers.create_channel( self._host, credentials=self._credentials, - scopes=self.AUTH_SCOPES, + **self_signed_jwt_kwargs, options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), diff --git a/gapic/templates/.coveragerc.j2 b/gapic/templates/.coveragerc.j2 index f2ac95dda9..6e2f585cbd 100644 --- a/gapic/templates/.coveragerc.j2 +++ b/gapic/templates/.coveragerc.j2 @@ -2,7 +2,6 @@ branch = True [report] -fail_under = 100 show_missing = True omit = {{ api.naming.module_namespace|join("/") }}/{{ api.naming.module_name }}/__init__.py diff --git a/gapic/templates/noxfile.py.j2 b/gapic/templates/noxfile.py.j2 index b6225d867d..0b3e167a73 100644 --- a/gapic/templates/noxfile.py.j2 +++ b/gapic/templates/noxfile.py.j2 @@ -2,10 +2,28 @@ {% block content %} import os +import pathlib import shutil +import subprocess +import sys + import nox # type: ignore +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + +LOWER_BOUND_CONSTRAINTS_FILE = CURRENT_DIRECTORY / "constraints.txt" +PACKAGE_NAME = subprocess.check_output([sys.executable, "setup.py", "--name"], encoding="utf-8") + + +nox.sessions = [ + "unit", + "cover", + "mypy", + "check_lower_bounds" + # exclude update_lower_bounds from default + "docs", +] @nox.session(python=['3.6', '3.7', '3.8', '3.9']) def unit(session): @@ -25,6 +43,18 @@ def unit(session): ) +@nox.session(python='3.7') +def cover(session): + """Run the final coverage report. + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ + session.install("coverage", "pytest-cov") + session.run("coverage", "report", "--show-missing", "--fail-under=100") + + session.run("coverage", "erase") + + @nox.session(python=['3.6', '3.7']) def mypy(session): """Run the type checker.""" @@ -40,6 +70,38 @@ def mypy(session): {%- endif %} ) + +@nox.session +def update_lower_bounds(session): + """Update lower bounds in constraints.txt to match setup.py""" + session.install('google-cloud-testutils') + session.install('.') + + session.run( + 'lower-bound-checker', + 'update', + '--package-name', + PACKAGE_NAME, + '--constraints-file', + str(LOWER_BOUND_CONSTRAINTS_FILE), + ) + + +@nox.session +def check_lower_bounds(session): + """Check lower bounds in setup.py are reflected in constraints file""" + session.install('google-cloud-testutils') + session.install('.') + + session.run( + 'lower-bound-checker', + 'check', + '--package-name', + PACKAGE_NAME, + '--constraints-file', + str(LOWER_BOUND_CONSTRAINTS_FILE), + ) + @nox.session(python='3.6') def docs(session): """Build the docs for this library.""" diff --git a/gapic/templates/setup.py.j2 b/gapic/templates/setup.py.j2 index f7ed0a9923..637c4fa97d 100644 --- a/gapic/templates/setup.py.j2 +++ b/gapic/templates/setup.py.j2 @@ -29,8 +29,9 @@ setuptools.setup( 'google-api-core[grpc] >= 1.22.2, < 2.0.0dev', 'libcst >= 0.2.5', 'proto-plus >= 1.15.0', + 'packaging >= 14.3', {%- if api.requires_package(('google', 'iam', 'v1')) or opts.add_iam_methods %} - 'grpc-google-iam-v1', + 'grpc-google-iam-v1 >= 0.12.3, < 0.13dev', {%- endif %} ), python_requires='>=3.6', diff --git a/gapic/templates/tests/__init__.py.j2 b/gapic/templates/tests/__init__.py.j2 new file mode 100644 index 0000000000..34200f2eca --- /dev/null +++ b/gapic/templates/tests/__init__.py.j2 @@ -0,0 +1,2 @@ + +{% extends '_base.py.j2' %} \ No newline at end of file diff --git a/gapic/templates/tests/unit/__init__.py.j2 b/gapic/templates/tests/unit/__init__.py.j2 new file mode 100644 index 0000000000..34200f2eca --- /dev/null +++ b/gapic/templates/tests/unit/__init__.py.j2 @@ -0,0 +1,2 @@ + +{% extends '_base.py.j2' %} \ No newline at end of file diff --git a/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 b/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 index f1d8685850..f7ae145b74 100644 --- a/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 +++ b/gapic/templates/tests/unit/gapic/%name_%version/%sub/test_%service.py.j2 @@ -3,6 +3,7 @@ {% block content %} import os import mock +import packaging.version import grpc from grpc.experimental import aio @@ -26,6 +27,8 @@ from {{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + ser from {{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }}.services.{{ service.name|snake_case }} import {{ service.async_client_name }} {%- endif %} from {{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }}.services.{{ service.name|snake_case }} import transports +from {{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }}.services.{{ service.name|snake_case }}.transports.base import _GOOGLE_AUTH_VERSION +from {{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }}.services.{{ service.name|snake_case }}.transports.base import _API_CORE_VERSION from google.api_core import client_options from google.api_core import exceptions from google.api_core import grpc_helpers @@ -51,6 +54,28 @@ from google.iam.v1 import policy_pb2 as policy # type: ignore {% endfilter %} +# TODO(busunkim): Once google-api-core >= 1.26.0 is required: +# - Delete all the api-core and auth "less than" test cases +# - Delete these pytest markers (Make the "greater than or equal to" tests the default). +requires_google_auth_lt_1_25_0 = pytest.mark.skipif( + packaging.version.parse(_GOOGLE_AUTH_VERSION) >= packaging.version.parse("1.25.0"), + reason="This test requires google-auth < 1.25.0", +) +requires_google_auth_gte_1_25_0 = pytest.mark.skipif( + packaging.version.parse(_GOOGLE_AUTH_VERSION) < packaging.version.parse("1.25.0"), + reason="This test requires google-auth >= 1.25.0", +) + +requires_api_core_lt_1_26_0 = pytest.mark.skipif( + packaging.version.parse(_API_CORE_VERSION) >= packaging.version.parse("1.26.0"), + reason="This test requires google-api-core < 1.26.0", +) + +requires_api_core_gte_1_26_0 = pytest.mark.skipif( + packaging.version.parse(_API_CORE_VERSION) < packaging.version.parse("1.26.0"), + reason="This test requires google-api-core >= 1.26.0", +) + def client_cert_source_callback(): return b"cert bytes", b"key bytes" @@ -1439,16 +1464,39 @@ def test_{{ service.name|snake_case }}_base_transport(): {% endif %} +@requires_google_auth_gte_1_25_0 def test_{{ service.name|snake_case }}_base_transport_with_credentials_file(): # Instantiate the base transport with a credentials file - with mock.patch.object(auth, 'load_credentials_from_file') as load_creds, mock.patch('{{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }}.services.{{ service.name|snake_case }}.transports.{{ service.name }}Transport._prep_wrapped_messages') as Transport: + with mock.patch.object(auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('{{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }}.services.{{ service.name|snake_case }}.transports.{{ service.name }}Transport._prep_wrapped_messages') as Transport: + Transport.return_value = None + load_creds.return_value = (credentials.AnonymousCredentials(), None) + transport = transports.{{ service.name }}Transport( + credentials_file="credentials.json", + quota_project_id="octopus", + ) + load_creds.assert_called_once_with("credentials.json", + scopes=None, + default_scopes=( + {%- for scope in service.oauth_scopes %} + '{{ scope }}', + {%- endfor %} + ), + quota_project_id="octopus", + ) + + +@requires_google_auth_lt_1_25_0 +def test_{{ service.name|snake_case }}_base_transport_with_credentials_file_old_google_auth(): + # Instantiate the base transport with a credentials file + with mock.patch.object(auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('{{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }}.services.{{ service.name|snake_case }}.transports.{{ service.name }}Transport._prep_wrapped_messages') as Transport: Transport.return_value = None load_creds.return_value = (credentials.AnonymousCredentials(), None) transport = transports.{{ service.name }}Transport( credentials_file="credentials.json", quota_project_id="octopus", ) - load_creds.assert_called_once_with("credentials.json", scopes=( + load_creds.assert_called_once_with("credentials.json", + scopes=( {%- for scope in service.oauth_scopes %} '{{ scope }}', {%- endfor %} @@ -1459,38 +1507,205 @@ def test_{{ service.name|snake_case }}_base_transport_with_credentials_file(): def test_{{ service.name|snake_case }}_base_transport_with_adc(): # Test the default credentials are used if credentials and credentials_file are None. - with mock.patch.object(auth, 'default') as adc, mock.patch('{{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }}.services.{{ service.name|snake_case }}.transports.{{ service.name }}Transport._prep_wrapped_messages') as Transport: + with mock.patch.object(auth, 'default', autospec=True) as adc, mock.patch('{{ (api.naming.module_namespace + (api.naming.versioned_module_name,) + service.meta.address.subpackage)|join(".") }}.services.{{ service.name|snake_case }}.transports.{{ service.name }}Transport._prep_wrapped_messages') as Transport: Transport.return_value = None adc.return_value = (credentials.AnonymousCredentials(), None) transport = transports.{{ service.name }}Transport() adc.assert_called_once() +@requires_google_auth_gte_1_25_0 def test_{{ service.name|snake_case }}_auth_adc(): # If no credentials are provided, we should use ADC credentials. - with mock.patch.object(auth, 'default') as adc: + with mock.patch.object(auth, 'default', autospec=True) as adc: adc.return_value = (credentials.AnonymousCredentials(), None) {{ service.client_name }}() - adc.assert_called_once_with(scopes=( - {%- for scope in service.oauth_scopes %} - '{{ scope }}', - {%- endfor %}), + adc.assert_called_once_with( + scopes=None, + default_scopes=( + {%- for scope in service.oauth_scopes %} + '{{ scope }}', + {%- endfor %}), quota_project_id=None, ) + +@requires_google_auth_lt_1_25_0 +def test_{{ service.name|snake_case }}_auth_adc_old_google_auth(): + # If no credentials are provided, we should use ADC credentials. + with mock.patch.object(auth, 'default', autospec=True) as adc: + adc.return_value = (credentials.AnonymousCredentials(), None) + {{ service.client_name }}() + adc.assert_called_once_with( + scopes=( + {%- for scope in service.oauth_scopes %} + '{{ scope }}', + {%- endfor %}), + quota_project_id=None, + ) + + {% if 'grpc' in opts.transport %} -def test_{{ service.name|snake_case }}_transport_auth_adc(): +@pytest.mark.parametrize( + "transport_class", + [ + transports.{{ service.name }}GrpcTransport, + transports.{{ service.name }}GrpcAsyncIOTransport, + ], +) +@requires_google_auth_gte_1_25_0 +def test_{{ service.name|snake_case }}_transport_auth_adc(transport_class): # If credentials and host are not provided, the transport class should use # ADC credentials. - with mock.patch.object(auth, 'default') as adc: + with mock.patch.object(auth, 'default', autospec=True) as adc: adc.return_value = (credentials.AnonymousCredentials(), None) - transports.{{ service.name }}GrpcTransport(host="squid.clam.whelk", quota_project_id="octopus") - adc.assert_called_once_with(scopes=( - {%- for scope in service.oauth_scopes %} - '{{ scope }}', - {%- endfor %}), + transport_class(quota_project_id="octopus", scopes=["1", "2"]) + adc.assert_called_once_with( + scopes=["1", "2"], + default_scopes=( + {%- for scope in service.oauth_scopes %} + '{{ scope }}', + {%- endfor %}), + quota_project_id="octopus", + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.{{ service.name }}GrpcTransport, + transports.{{ service.name }}GrpcAsyncIOTransport, + ], +) +@requires_google_auth_lt_1_25_0 +def test_{{ service.name|snake_case }}_transport_auth_adc_old_google_auth(transport_class): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(auth, "default", autospec=True) as adc: + adc.return_value = (credentials.AnonymousCredentials(), None) + transport_class(quota_project_id="octopus") + adc.assert_called_once_with( + scopes=( + {%- for scope in service.oauth_scopes %} + '{{ scope }}', + {%- endfor %}), + quota_project_id="octopus", + ) + + +@pytest.mark.parametrize( + "transport_class,grpc_helpers", + [ + (transports.{{ service.name }}GrpcTransport, grpc_helpers), + (transports.{{ service.name }}GrpcAsyncIOTransport, grpc_helpers_async) + ], +) +@requires_api_core_gte_1_26_0 +def test_{{ service.name|snake_case }}_transport_create_channel(transport_class, grpc_helpers): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(auth, "default", autospec=True) as adc, mock.patch.object( + grpc_helpers, "create_channel", autospec=True + ) as create_channel: + creds = credentials.AnonymousCredentials() + adc.return_value = (creds, None) + transport_class( + quota_project_id="octopus", + scopes=["1", "2"] + ) + + {% with host = (service.host|default('localhost', true)) -%} + create_channel.assert_called_with( + "{{ host }}", + credentials=creds, + credentials_file=None, quota_project_id="octopus", + default_scopes=( + {%- for scope in service.oauth_scopes %} + '{{ scope }}', + {%- endfor %}), + scopes=["1", "2"], + default_host="{{ host }}", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) + {% endwith %} + + +@pytest.mark.parametrize( + "transport_class,grpc_helpers", + [ + (transports.{{ service.name }}GrpcTransport, grpc_helpers), + (transports.{{ service.name }}GrpcAsyncIOTransport, grpc_helpers_async) + ], +) +@requires_api_core_lt_1_26_0 +def test_{{ service.name|snake_case }}_transport_create_channel_old_api_core(transport_class, grpc_helpers): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(auth, "default", autospec=True) as adc, mock.patch.object( + grpc_helpers, "create_channel", autospec=True + ) as create_channel: + creds = credentials.AnonymousCredentials() + adc.return_value = (creds, None) + transport_class(quota_project_id="octopus") + + {% with host = (service.host|default('localhost', true)) -%} + create_channel.assert_called_with( + "{{ host }}", + credentials=creds, + credentials_file=None, + quota_project_id="octopus", + scopes=( + {%- for scope in service.oauth_scopes %} + '{{ scope }}', + {%- endfor %}), + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + {% endwith %} + + +@pytest.mark.parametrize( + "transport_class,grpc_helpers", + [ + (transports.{{ service.name }}GrpcTransport, grpc_helpers), + (transports.{{ service.name }}GrpcAsyncIOTransport, grpc_helpers_async) + ], +) +@requires_api_core_lt_1_26_0 +def test_{{ service.name|snake_case }}_transport_create_channel_user_scopes(transport_class, grpc_helpers): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(auth, "default", autospec=True) as adc, mock.patch.object( + grpc_helpers, "create_channel", autospec=True + ) as create_channel: + creds = credentials.AnonymousCredentials() + adc.return_value = (creds, None) + {% with host = (service.host|default('localhost', true)) -%} + + transport_class(quota_project_id="octopus", scopes=["1", "2"]) + + create_channel.assert_called_with( + "{{ host }}", + credentials=creds, + credentials_file=None, + quota_project_id="octopus", + scopes=["1", "2"], + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + {% endwith %} + {% endif %} {% if 'grpc' in opts.transport %} diff --git a/gapic/templates/tests/unit/gapic/__init__.py.j2 b/gapic/templates/tests/unit/gapic/__init__.py.j2 new file mode 100644 index 0000000000..34200f2eca --- /dev/null +++ b/gapic/templates/tests/unit/gapic/__init__.py.j2 @@ -0,0 +1,2 @@ + +{% extends '_base.py.j2' %} \ No newline at end of file diff --git a/noxfile.py b/noxfile.py index 7dbe33ebc3..e17277a487 100644 --- a/noxfile.py +++ b/noxfile.py @@ -175,12 +175,7 @@ def showcase_mtls_alternative_templates(session): ) -@nox.session(python=["3.6", "3.7", "3.8", "3.9"]) -def showcase_unit( - session, templates="DEFAULT", other_opts: typing.Iterable[str] = (), -): - """Run the generated unit tests against the Showcase library.""" - +def run_showcase_unit_tests(session, fail_under=100): session.install( "coverage", "pytest", @@ -190,28 +185,77 @@ def showcase_unit( "pytest-asyncio", ) + # Run the tests. + session.run( + "py.test", + "-n=auto", + "--quiet", + "--cov=google", + "--cov-append", + f"--cov-fail-under={str(fail_under)}", + *(session.posargs or [path.join("tests", "unit")]), + ) + + +@nox.session(python=["3.6", "3.7", "3.8", "3.9"]) +def showcase_unit( + session, templates="DEFAULT", other_opts: typing.Iterable[str] = (), +): + """Run the generated unit tests against the Showcase library.""" + with showcase_library(session, templates=templates, other_opts=other_opts) as lib: session.chdir(lib) - - # Run the tests. - session.run( - "py.test", - "-n=auto", - "--quiet", - "--cov=google", - "--cov-report=term", - *(session.posargs or [path.join("tests", "unit")]), - ) + + # Unit tests are run twice with different dependencies to exercise + # all code paths. + # TODO(busunkim): remove when default templates require google-auth>=1.25.0 + + # 1. Run tests at lower bound of dependencies + session.install("nox") + session.run("nox", "-s", "update_lower_bounds") + session.install(".", "--force-reinstall", "-c", "constraints.txt") + # Some code paths require an older version of google-auth. + # google-auth is a transitive dependency so it isn't in the + # lower bound constraints file produced above. + session.install("google-auth==1.21.1") + run_showcase_unit_tests(session, fail_under=0) + + # 2. Run the tests again with latest version of dependencies + session.install(".", "--upgrade", "--force-reinstall") + # This time aggregate coverage should reach 100% + run_showcase_unit_tests(session, fail_under=100) @nox.session(python=["3.7", "3.8", "3.9"]) def showcase_unit_alternative_templates(session): - showcase_unit(session, templates=ADS_TEMPLATES, other_opts=("old-naming",)) + with showcase_library(session, templates=ADS_TEMPLATES, other_opts=("old-naming",)) as lib: + session.chdir(lib) + run_showcase_unit_tests(session) @nox.session(python=["3.8"]) def showcase_unit_add_iam_methods(session): - showcase_unit(session, other_opts=("add-iam-methods",)) + with showcase_library(session, other_opts=("add-iam-methods",)) as lib: + session.chdir(lib) + + # Unit tests are run twice with different dependencies to exercise + # all code paths. + # TODO(busunkim): remove when default templates require google-auth>=1.25.0 + + # 1. Run tests at lower bound of dependencies + session.install("nox") + session.run("nox", "-s", "update_lower_bounds") + session.install(".", "--force-reinstall", "-c", "constraints.txt") + # Some code paths require an older version of google-auth. + # google-auth is a transitive dependency so it isn't in the + # lower bound constraints file produced above. + session.install("google-auth==1.21.1") + run_showcase_unit_tests(session, fail_under=0) + + # 2. Run the tests again with latest version of dependencies + session.install(".", "--upgrade", "--force-reinstall") + # This time aggregate coverage should reach 100% + run_showcase_unit_tests(session, fail_under=100) @nox.session(python="3.8")