From d56ad12f197e9e379d2a4a0a38be108808985c23 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 9 Feb 2024 13:54:48 -0800 Subject: [PATCH] fix(diregapic): s/bazel/bazelisk/ in DIREGAPIC build GitHub action (#1064) Co-authored-by: Owl Bot Co-authored-by: mukund-ananthu <83691193+mukund-ananthu@users.noreply.github.com> Co-authored-by: Anthonios Partheniou --- google/cloud/pubsub_v1/publisher/client.py | 2 +- google/pubsub/__init__.py | 2 + google/pubsub_v1/__init__.py | 2 + .../services/publisher/async_client.py | 107 ++- google/pubsub_v1/services/publisher/client.py | 327 +++++++- .../services/publisher/transports/base.py | 8 +- .../services/publisher/transports/grpc.py | 5 +- .../publisher/transports/grpc_asyncio.py | 5 +- .../services/publisher/transports/rest.py | 6 +- .../services/schema_service/async_client.py | 107 ++- .../services/schema_service/client.py | 331 +++++++- .../schema_service/transports/base.py | 6 +- .../schema_service/transports/grpc.py | 2 +- .../schema_service/transports/grpc_asyncio.py | 2 +- .../schema_service/transports/rest.py | 6 +- .../services/subscriber/async_client.py | 155 +++- .../pubsub_v1/services/subscriber/client.py | 375 +++++++-- .../services/subscriber/transports/base.py | 8 +- .../services/subscriber/transports/grpc.py | 8 +- .../subscriber/transports/grpc_asyncio.py | 8 +- .../services/subscriber/transports/rest.py | 6 +- google/pubsub_v1/types/__init__.py | 2 + google/pubsub_v1/types/pubsub.py | 780 +++++++++++------- google/pubsub_v1/types/schema.py | 2 +- owlbot.py | 4 +- scripts/fixup_pubsub_v1_keywords.py | 2 +- testing/constraints-3.8.txt | 2 +- tests/unit/gapic/pubsub_v1/test_publisher.py | 482 ++++++++++- .../gapic/pubsub_v1/test_schema_service.py | 471 ++++++++++- tests/unit/gapic/pubsub_v1/test_subscriber.py | 457 +++++++++- .../publisher/test_publisher_client.py | 40 +- 31 files changed, 3112 insertions(+), 608 deletions(-) diff --git a/google/cloud/pubsub_v1/publisher/client.py b/google/cloud/pubsub_v1/publisher/client.py index 3e668533d..caf9fa180 100644 --- a/google/cloud/pubsub_v1/publisher/client.py +++ b/google/cloud/pubsub_v1/publisher/client.py @@ -399,7 +399,7 @@ def on_publish_done(future): transport = self._transport base_retry = transport._wrapped_methods[transport.publish]._retry retry = base_retry.with_deadline(2.0**32) - else: + elif retry is not None: retry = retry.with_deadline(2.0**32) # Delegate the publishing to the sequencer. diff --git a/google/pubsub/__init__.py b/google/pubsub/__init__.py index 453dca45f..ebcbb8271 100644 --- a/google/pubsub/__init__.py +++ b/google/pubsub/__init__.py @@ -41,6 +41,7 @@ from google.pubsub_v1.types.pubsub import GetSnapshotRequest from google.pubsub_v1.types.pubsub import GetSubscriptionRequest from google.pubsub_v1.types.pubsub import GetTopicRequest +from google.pubsub_v1.types.pubsub import IngestionDataSourceSettings from google.pubsub_v1.types.pubsub import ListSnapshotsRequest from google.pubsub_v1.types.pubsub import ListSnapshotsResponse from google.pubsub_v1.types.pubsub import ListSubscriptionsRequest @@ -112,6 +113,7 @@ "GetSnapshotRequest", "GetSubscriptionRequest", "GetTopicRequest", + "IngestionDataSourceSettings", "ListSnapshotsRequest", "ListSnapshotsResponse", "ListSubscriptionsRequest", diff --git a/google/pubsub_v1/__init__.py b/google/pubsub_v1/__init__.py index 08c6bc72a..731d03999 100644 --- a/google/pubsub_v1/__init__.py +++ b/google/pubsub_v1/__init__.py @@ -39,6 +39,7 @@ from .types.pubsub import GetSnapshotRequest from .types.pubsub import GetSubscriptionRequest from .types.pubsub import GetTopicRequest +from .types.pubsub import IngestionDataSourceSettings from .types.pubsub import ListSnapshotsRequest from .types.pubsub import ListSnapshotsResponse from .types.pubsub import ListSubscriptionsRequest @@ -113,6 +114,7 @@ "GetSnapshotRequest", "GetSubscriptionRequest", "GetTopicRequest", + "IngestionDataSourceSettings", "ListSchemaRevisionsRequest", "ListSchemaRevisionsResponse", "ListSchemasRequest", diff --git a/google/pubsub_v1/services/publisher/async_client.py b/google/pubsub_v1/services/publisher/async_client.py index ae632ea96..730d5c8a5 100644 --- a/google/pubsub_v1/services/publisher/async_client.py +++ b/google/pubsub_v1/services/publisher/async_client.py @@ -38,9 +38,9 @@ from google.oauth2 import service_account # type: ignore try: - OptionalRetry = Union[retries.AsyncRetry, gapic_v1.method._MethodDefault] + OptionalRetry = Union[retries.AsyncRetry, gapic_v1.method._MethodDefault, None] except AttributeError: # pragma: NO COVER - OptionalRetry = Union[retries.AsyncRetry, object] # type: ignore + OptionalRetry = Union[retries.AsyncRetry, object, None] # type: ignore from google.iam.v1 import iam_policy_pb2 # type: ignore from google.iam.v1 import policy_pb2 # type: ignore @@ -61,8 +61,12 @@ class PublisherAsyncClient: _client: PublisherClient + # Copy defaults from the synchronous client for use here. + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. DEFAULT_ENDPOINT = PublisherClient.DEFAULT_ENDPOINT DEFAULT_MTLS_ENDPOINT = PublisherClient.DEFAULT_MTLS_ENDPOINT + _DEFAULT_ENDPOINT_TEMPLATE = PublisherClient._DEFAULT_ENDPOINT_TEMPLATE + _DEFAULT_UNIVERSE = PublisherClient._DEFAULT_UNIVERSE schema_path = staticmethod(PublisherClient.schema_path) parse_schema_path = staticmethod(PublisherClient.parse_schema_path) @@ -167,6 +171,25 @@ def transport(self) -> PublisherTransport: """ return self._client.transport + @property + def api_endpoint(self): + """Return the API endpoint used by the client instance. + + Returns: + str: The API endpoint used by the client instance. + """ + return self._client._api_endpoint + + @property + def universe_domain(self) -> str: + """Return the universe domain used by the client instance. + + Returns: + str: The universe domain used + by the client instance. + """ + return self._client._universe_domain + get_transport_class = functools.partial( type(PublisherClient).get_transport_class, type(PublisherClient) ) @@ -179,7 +202,7 @@ def __init__( client_options: Optional[ClientOptions] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, ) -> None: - """Instantiates the publisher client. + """Instantiates the publisher async client. Args: credentials (Optional[google.auth.credentials.Credentials]): The @@ -190,23 +213,38 @@ def __init__( transport (Union[str, ~.PublisherTransport]): The transport to use. If set to None, a transport is chosen automatically. - client_options (ClientOptions): Custom options for the client. It - won't take effect if a ``transport`` instance is provided. - (1) The ``api_endpoint`` property can be used to override the - default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT - environment variable can also be used to override the endpoint: + client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): + Custom options for the client. + + 1. The ``api_endpoint`` property can be used to override the + default endpoint provided by the client when ``transport`` is + not explicitly provided. Only if this property is not set and + ``transport`` was not explicitly provided, the endpoint is + determined by the GOOGLE_API_USE_MTLS_ENDPOINT environment + variable, which have one of the following values: "always" (always use the default mTLS endpoint), "never" (always - use the default regular endpoint) and "auto" (auto switch to the - default mTLS endpoint if client certificate is present, this is - the default value). However, the ``api_endpoint`` property takes - precedence if provided. - (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + use the default regular endpoint) and "auto" (auto-switch to the + default mTLS endpoint if client certificate is present; this is + the default value). + + 2. If the GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable is "true", then the ``client_cert_source`` property can be used - to provide client certificate for mutual TLS transport. If + to provide a client certificate for mTLS transport. If not provided, the default SSL client certificate will be used if present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not set, no client certificate will be used. + 3. The ``universe_domain`` property can be used to override the + default "googleapis.com" universe. Note that ``api_endpoint`` + property still takes precedence; and ``universe_domain`` is + currently not supported for mTLS. + + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + Raises: google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport creation failed for any reason. @@ -324,6 +362,9 @@ async def sample_create_topic(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -345,7 +386,8 @@ async def update_topic( timeout: TimeoutType = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, str]] = (), ) -> pubsub.Topic: - r"""Updates an existing topic. Note that certain + r"""Updates an existing topic by updating the fields + specified in the update mask. Note that certain properties of a topic are not modifiable. .. code-block:: python @@ -452,6 +494,9 @@ async def sample_update_topic(): ), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -555,7 +600,7 @@ async def sample_publish(): default_retry=retries.AsyncRetry( initial=0.1, maximum=60.0, - multiplier=4.0, + multiplier=4, predicate=retries.if_exception_type( core_exceptions.Aborted, core_exceptions.Cancelled, @@ -577,6 +622,9 @@ async def sample_publish(): gapic_v1.routing_header.to_grpc_metadata((("topic", request.topic),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -688,6 +736,9 @@ async def sample_get_topic(): gapic_v1.routing_header.to_grpc_metadata((("topic", request.topic),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -804,6 +855,9 @@ async def sample_list_topics(): gapic_v1.routing_header.to_grpc_metadata((("project", request.project),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -931,6 +985,9 @@ async def sample_list_topic_subscriptions(): gapic_v1.routing_header.to_grpc_metadata((("topic", request.topic),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -1062,6 +1119,9 @@ async def sample_list_topic_snapshots(): gapic_v1.routing_header.to_grpc_metadata((("topic", request.topic),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -1178,6 +1238,9 @@ async def sample_delete_topic(): gapic_v1.routing_header.to_grpc_metadata((("topic", request.topic),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. await rpc( request, @@ -1271,6 +1334,9 @@ async def sample_detach_subscription(): ), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -1391,6 +1457,9 @@ async def set_iam_policy( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -1513,6 +1582,9 @@ async def get_iam_policy( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -1573,6 +1645,9 @@ async def test_iam_permissions( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, diff --git a/google/pubsub_v1/services/publisher/client.py b/google/pubsub_v1/services/publisher/client.py index 1a92362c5..37e7410f9 100644 --- a/google/pubsub_v1/services/publisher/client.py +++ b/google/pubsub_v1/services/publisher/client.py @@ -29,6 +29,7 @@ Union, cast, ) +import warnings from google.pubsub_v1 import gapic_version as package_version @@ -44,9 +45,9 @@ from google.oauth2 import service_account # type: ignore try: - OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault] + OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] except AttributeError: # pragma: NO COVER - OptionalRetry = Union[retries.Retry, object] # type: ignore + OptionalRetry = Union[retries.Retry, object, None] # type: ignore from google.iam.v1 import iam_policy_pb2 # type: ignore from google.iam.v1 import policy_pb2 # type: ignore @@ -133,6 +134,8 @@ def _get_default_mtls_endpoint(api_endpoint): return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. + # The scopes needed to make gRPC calls to all of the methods defined in # this service _DEFAULT_SCOPES = ( @@ -148,6 +151,9 @@ def _get_default_mtls_endpoint(api_endpoint): DEFAULT_ENDPOINT ) + _DEFAULT_ENDPOINT_TEMPLATE = "pubsub.{UNIVERSE_DOMAIN}" + _DEFAULT_UNIVERSE = "googleapis.com" + @classmethod def from_service_account_info(cls, info: dict, *args, **kwargs): """Creates an instance of this client using the provided credentials @@ -329,7 +335,7 @@ def parse_common_location_path(path: str) -> Dict[str, str]: def get_mtls_endpoint_and_cert_source( cls, client_options: Optional[client_options_lib.ClientOptions] = None ): - """Return the API endpoint and client cert source for mutual TLS. + """Deprecated. Return the API endpoint and client cert source for mutual TLS. The client cert source is determined in the following order: (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the @@ -359,6 +365,11 @@ def get_mtls_endpoint_and_cert_source( Raises: google.auth.exceptions.MutualTLSChannelError: If any errors happen. """ + + warnings.warn( + "get_mtls_endpoint_and_cert_source is deprecated. Use the api_endpoint property instead.", + DeprecationWarning, + ) if client_options is None: client_options = client_options_lib.ClientOptions() use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") @@ -392,6 +403,178 @@ def get_mtls_endpoint_and_cert_source( return api_endpoint, client_cert_source + @staticmethod + def _read_environment_variables(): + """Returns the environment variables used by the client. + + Returns: + Tuple[bool, str, str]: returns the GOOGLE_API_USE_CLIENT_CERTIFICATE, + GOOGLE_API_USE_MTLS_ENDPOINT, and GOOGLE_CLOUD_UNIVERSE_DOMAIN environment variables. + + Raises: + ValueError: If GOOGLE_API_USE_CLIENT_CERTIFICATE is not + any of ["true", "false"]. + google.auth.exceptions.MutualTLSChannelError: If GOOGLE_API_USE_MTLS_ENDPOINT + is not any of ["auto", "never", "always"]. + """ + use_client_cert = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_client_cert not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError( + "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + ) + return use_client_cert == "true", use_mtls_endpoint, universe_domain_env + + @staticmethod + def _get_client_cert_source(provided_cert_source, use_cert_flag): + """Return the client cert source to be used by the client. + + Args: + provided_cert_source (bytes): The client certificate source provided. + use_cert_flag (bool): A flag indicating whether to use the client certificate. + + Returns: + bytes or None: The client cert source to be used by the client. + """ + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif mtls.has_default_client_cert_source(): + client_cert_source = mtls.default_client_cert_source() + return client_cert_source + + @staticmethod + def _get_api_endpoint( + api_override, client_cert_source, universe_domain, use_mtls_endpoint + ): + """Return the API endpoint used by the client. + + Args: + api_override (str): The API endpoint override. If specified, this is always + the return value of this function and the other arguments are not used. + client_cert_source (bytes): The client certificate source used by the client. + universe_domain (str): The universe domain used by the client. + use_mtls_endpoint (str): How to use the mTLS endpoint, which depends also on the other parameters. + Possible values are "always", "auto", or "never". + + Returns: + str: The API endpoint to be used by the client. + """ + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or ( + use_mtls_endpoint == "auto" and client_cert_source + ): + _default_universe = PublisherClient._DEFAULT_UNIVERSE + if universe_domain != _default_universe: + raise MutualTLSChannelError( + f"mTLS is not supported in any universe other than {_default_universe}." + ) + api_endpoint = PublisherClient.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = PublisherClient._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=universe_domain + ) + return api_endpoint + + @staticmethod + def _get_universe_domain( + client_universe_domain: Optional[str], universe_domain_env: Optional[str] + ) -> str: + """Return the universe domain used by the client. + + Args: + client_universe_domain (Optional[str]): The universe domain configured via the client options. + universe_domain_env (Optional[str]): The universe domain configured via the "GOOGLE_CLOUD_UNIVERSE_DOMAIN" environment variable. + + Returns: + str: The universe domain to be used by the client. + + Raises: + ValueError: If the universe domain is an empty string. + """ + universe_domain = PublisherClient._DEFAULT_UNIVERSE + if client_universe_domain is not None: + universe_domain = client_universe_domain + elif universe_domain_env is not None: + universe_domain = universe_domain_env + if len(universe_domain.strip()) == 0: + raise ValueError("Universe Domain cannot be an empty string.") + return universe_domain + + @staticmethod + def _compare_universes( + client_universe: str, credentials: ga_credentials.Credentials + ) -> bool: + """Returns True iff the universe domains used by the client and credentials match. + + Args: + client_universe (str): The universe domain configured via the client options. + credentials (ga_credentials.Credentials): The credentials being used in the client. + + Returns: + bool: True iff client_universe matches the universe in credentials. + + Raises: + ValueError: when client_universe does not match the universe in credentials. + """ + + default_universe = PublisherClient._DEFAULT_UNIVERSE + credentials_universe = getattr(credentials, "universe_domain", default_universe) + + if client_universe != credentials_universe: + raise ValueError( + "The configured universe domain " + f"({client_universe}) does not match the universe domain " + f"found in the credentials ({credentials_universe}). " + "If you haven't configured the universe domain explicitly, " + f"`{default_universe}` is the default." + ) + return True + + def _validate_universe_domain(self): + """Validates client's and credentials' universe domains are consistent. + + Returns: + bool: True iff the configured universe domain is valid. + + Raises: + ValueError: If the configured universe domain is not valid. + """ + self._is_universe_domain_valid = ( + self._is_universe_domain_valid + or PublisherClient._compare_universes( + self.universe_domain, self.transport._credentials + ) + ) + return self._is_universe_domain_valid + + @property + def api_endpoint(self): + """Return the API endpoint used by the client instance. + + Returns: + str: The API endpoint used by the client instance. + """ + return self._api_endpoint + + @property + def universe_domain(self) -> str: + """Return the universe domain used by the client instance. + + Returns: + str: The universe domain used by the client instance. + """ + return self._universe_domain + def __init__( self, *, @@ -411,22 +594,32 @@ def __init__( transport (Union[str, PublisherTransport]): The transport to use. If set to None, a transport is chosen automatically. - client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): Custom options for the - client. It won't take effect if a ``transport`` instance is provided. - (1) The ``api_endpoint`` property can be used to override the - default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT - environment variable can also be used to override the endpoint: + client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): + Custom options for the client. + + 1. The ``api_endpoint`` property can be used to override the + default endpoint provided by the client when ``transport`` is + not explicitly provided. Only if this property is not set and + ``transport`` was not explicitly provided, the endpoint is + determined by the GOOGLE_API_USE_MTLS_ENDPOINT environment + variable, which have one of the following values: "always" (always use the default mTLS endpoint), "never" (always - use the default regular endpoint) and "auto" (auto switch to the - default mTLS endpoint if client certificate is present, this is - the default value). However, the ``api_endpoint`` property takes - precedence if provided. - (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + use the default regular endpoint) and "auto" (auto-switch to the + default mTLS endpoint if client certificate is present; this is + the default value). + + 2. If the GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable is "true", then the ``client_cert_source`` property can be used - to provide client certificate for mutual TLS transport. If + to provide a client certificate for mTLS transport. If not provided, the default SSL client certificate will be used if present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not set, no client certificate will be used. + + 3. The ``universe_domain`` property can be used to override the + default "googleapis.com" universe. Note that the ``api_endpoint`` + property still takes precedence; and ``universe_domain`` is + currently not supported for mTLS. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): The client info used to send a user-agent string along with API requests. If ``None``, then default info will be used. @@ -437,17 +630,34 @@ def __init__( google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport creation failed for any reason. """ - if isinstance(client_options, dict): - client_options = client_options_lib.from_dict(client_options) - if client_options is None: - client_options = client_options_lib.ClientOptions() - client_options = cast(client_options_lib.ClientOptions, client_options) + self._client_options = client_options + if isinstance(self._client_options, dict): + self._client_options = client_options_lib.from_dict(self._client_options) + if self._client_options is None: + self._client_options = client_options_lib.ClientOptions() + self._client_options = cast( + client_options_lib.ClientOptions, self._client_options + ) - api_endpoint, client_cert_source_func = self.get_mtls_endpoint_and_cert_source( - client_options + universe_domain_opt = getattr(self._client_options, "universe_domain", None) + + ( + self._use_client_cert, + self._use_mtls_endpoint, + self._universe_domain_env, + ) = PublisherClient._read_environment_variables() + self._client_cert_source = PublisherClient._get_client_cert_source( + self._client_options.client_cert_source, self._use_client_cert + ) + self._universe_domain = PublisherClient._get_universe_domain( + universe_domain_opt, self._universe_domain_env ) + self._api_endpoint = None # updated below, depending on `transport` - api_key_value = getattr(client_options, "api_key", None) + # Initialize the universe domain validation. + self._is_universe_domain_valid = False + + api_key_value = getattr(self._client_options, "api_key", None) if api_key_value and credentials: raise ValueError( "client_options.api_key and credentials are mutually exclusive" @@ -456,20 +666,30 @@ def __init__( # Save or instantiate the transport. # Ordinarily, we provide the transport, but allowing a custom transport # instance provides an extensibility point for unusual situations. - if isinstance(transport, PublisherTransport): + transport_provided = isinstance(transport, PublisherTransport) + if transport_provided: # transport is a PublisherTransport instance. - if credentials or client_options.credentials_file or api_key_value: + if credentials or self._client_options.credentials_file or api_key_value: raise ValueError( "When providing a transport instance, " "provide its credentials directly." ) - if client_options.scopes: + if self._client_options.scopes: raise ValueError( "When providing a transport instance, provide its scopes " "directly." ) - self._transport = transport - else: + self._transport = cast(PublisherTransport, transport) + self._api_endpoint = self._transport.host + + self._api_endpoint = self._api_endpoint or PublisherClient._get_api_endpoint( + self._client_options.api_endpoint, + self._client_cert_source, + self._universe_domain, + self._use_mtls_endpoint, + ) + + if not transport_provided: import google.auth._default # type: ignore if api_key_value and hasattr( @@ -479,7 +699,7 @@ def __init__( api_key_value ) - Transport = type(self).get_transport_class(transport) + Transport = type(self).get_transport_class(cast(str, transport)) emulator_host = os.environ.get("PUBSUB_EMULATOR_HOST") if emulator_host: @@ -491,14 +711,14 @@ def __init__( self._transport = Transport( credentials=credentials, - credentials_file=client_options.credentials_file, - host=api_endpoint, - scopes=client_options.scopes, - client_cert_source_for_mtls=client_cert_source_func, - quota_project_id=client_options.quota_project_id, + credentials_file=self._client_options.credentials_file, + host=self._api_endpoint, + scopes=self._client_options.scopes, + client_cert_source_for_mtls=self._client_cert_source, + quota_project_id=self._client_options.quota_project_id, client_info=client_info, always_use_jwt_access=True, - api_audience=client_options.api_audience, + api_audience=self._client_options.api_audience, ) def create_topic( @@ -598,6 +818,9 @@ def sample_create_topic(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -619,7 +842,8 @@ def update_topic( timeout: TimeoutType = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, str]] = (), ) -> pubsub.Topic: - r"""Updates an existing topic. Note that certain + r"""Updates an existing topic by updating the fields + specified in the update mask. Note that certain properties of a topic are not modifiable. .. code-block:: python @@ -717,6 +941,9 @@ def sample_update_topic(): ), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -827,6 +1054,9 @@ def sample_publish(): gapic_v1.routing_header.to_grpc_metadata((("topic", request.topic),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -927,6 +1157,9 @@ def sample_get_topic(): gapic_v1.routing_header.to_grpc_metadata((("topic", request.topic),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1032,6 +1265,9 @@ def sample_list_topics(): gapic_v1.routing_header.to_grpc_metadata((("project", request.project),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1148,6 +1384,9 @@ def sample_list_topic_subscriptions(): gapic_v1.routing_header.to_grpc_metadata((("topic", request.topic),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1268,6 +1507,9 @@ def sample_list_topic_snapshots(): gapic_v1.routing_header.to_grpc_metadata((("topic", request.topic),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1375,6 +1617,9 @@ def sample_delete_topic(): gapic_v1.routing_header.to_grpc_metadata((("topic", request.topic),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. rpc( request, @@ -1460,6 +1705,9 @@ def sample_detach_subscription(): ), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1594,6 +1842,9 @@ def set_iam_policy( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1716,6 +1967,9 @@ def get_iam_policy( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1776,6 +2030,9 @@ def test_iam_permissions( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, diff --git a/google/pubsub_v1/services/publisher/transports/base.py b/google/pubsub_v1/services/publisher/transports/base.py index e8caa080b..6c92dd8f9 100644 --- a/google/pubsub_v1/services/publisher/transports/base.py +++ b/google/pubsub_v1/services/publisher/transports/base.py @@ -63,7 +63,7 @@ def __init__( Args: host (Optional[str]): - The hostname to connect to. + The hostname to connect to (default: 'pubsub.googleapis.com'). credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none @@ -126,6 +126,10 @@ def __init__( host += ":443" self._host = host + @property + def host(self): + return self._host + def _prep_wrapped_messages(self, client_info): # Precompute the wrapped methods. self._wrapped_methods = { @@ -162,7 +166,7 @@ def _prep_wrapped_messages(self, client_info): default_retry=retries.Retry( initial=0.1, maximum=60.0, - multiplier=4.0, + multiplier=4, predicate=retries.if_exception_type( core_exceptions.Aborted, core_exceptions.Cancelled, diff --git a/google/pubsub_v1/services/publisher/transports/grpc.py b/google/pubsub_v1/services/publisher/transports/grpc.py index 268e687f0..ed6410fd4 100644 --- a/google/pubsub_v1/services/publisher/transports/grpc.py +++ b/google/pubsub_v1/services/publisher/transports/grpc.py @@ -68,7 +68,7 @@ def __init__( Args: host (Optional[str]): - The hostname to connect to. + The hostname to connect to (default: 'pubsub.googleapis.com'). credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none @@ -265,7 +265,8 @@ def create_topic(self) -> Callable[[pubsub.Topic], pubsub.Topic]: def update_topic(self) -> Callable[[pubsub.UpdateTopicRequest], pubsub.Topic]: r"""Return a callable for the update topic method over gRPC. - Updates an existing topic. Note that certain + Updates an existing topic by updating the fields + specified in the update mask. Note that certain properties of a topic are not modifiable. Returns: diff --git a/google/pubsub_v1/services/publisher/transports/grpc_asyncio.py b/google/pubsub_v1/services/publisher/transports/grpc_asyncio.py index 8d971fa32..c620cd4ae 100644 --- a/google/pubsub_v1/services/publisher/transports/grpc_asyncio.py +++ b/google/pubsub_v1/services/publisher/transports/grpc_asyncio.py @@ -113,7 +113,7 @@ def __init__( Args: host (Optional[str]): - The hostname to connect to. + The hostname to connect to (default: 'pubsub.googleapis.com'). credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none @@ -270,7 +270,8 @@ def update_topic( ) -> Callable[[pubsub.UpdateTopicRequest], Awaitable[pubsub.Topic]]: r"""Return a callable for the update topic method over gRPC. - Updates an existing topic. Note that certain + Updates an existing topic by updating the fields + specified in the update mask. Note that certain properties of a topic are not modifiable. Returns: diff --git a/google/pubsub_v1/services/publisher/transports/rest.py b/google/pubsub_v1/services/publisher/transports/rest.py index de7215153..d450cb6eb 100644 --- a/google/pubsub_v1/services/publisher/transports/rest.py +++ b/google/pubsub_v1/services/publisher/transports/rest.py @@ -36,9 +36,9 @@ import warnings try: - OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault] + OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] except AttributeError: # pragma: NO COVER - OptionalRetry = Union[retries.Retry, object] # type: ignore + OptionalRetry = Union[retries.Retry, object, None] # type: ignore from google.iam.v1 import iam_policy_pb2 # type: ignore @@ -427,7 +427,7 @@ def __init__( Args: host (Optional[str]): - The hostname to connect to. + The hostname to connect to (default: 'pubsub.googleapis.com'). credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none diff --git a/google/pubsub_v1/services/schema_service/async_client.py b/google/pubsub_v1/services/schema_service/async_client.py index 2d4b5d77b..44a4916e6 100644 --- a/google/pubsub_v1/services/schema_service/async_client.py +++ b/google/pubsub_v1/services/schema_service/async_client.py @@ -38,9 +38,9 @@ from google.oauth2 import service_account # type: ignore try: - OptionalRetry = Union[retries.AsyncRetry, gapic_v1.method._MethodDefault] + OptionalRetry = Union[retries.AsyncRetry, gapic_v1.method._MethodDefault, None] except AttributeError: # pragma: NO COVER - OptionalRetry = Union[retries.AsyncRetry, object] # type: ignore + OptionalRetry = Union[retries.AsyncRetry, object, None] # type: ignore from google.iam.v1 import iam_policy_pb2 # type: ignore from google.iam.v1 import policy_pb2 # type: ignore @@ -58,8 +58,12 @@ class SchemaServiceAsyncClient: _client: SchemaServiceClient + # Copy defaults from the synchronous client for use here. + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. DEFAULT_ENDPOINT = SchemaServiceClient.DEFAULT_ENDPOINT DEFAULT_MTLS_ENDPOINT = SchemaServiceClient.DEFAULT_MTLS_ENDPOINT + _DEFAULT_ENDPOINT_TEMPLATE = SchemaServiceClient._DEFAULT_ENDPOINT_TEMPLATE + _DEFAULT_UNIVERSE = SchemaServiceClient._DEFAULT_UNIVERSE schema_path = staticmethod(SchemaServiceClient.schema_path) parse_schema_path = staticmethod(SchemaServiceClient.parse_schema_path) @@ -166,6 +170,25 @@ def transport(self) -> SchemaServiceTransport: """ return self._client.transport + @property + def api_endpoint(self): + """Return the API endpoint used by the client instance. + + Returns: + str: The API endpoint used by the client instance. + """ + return self._client._api_endpoint + + @property + def universe_domain(self) -> str: + """Return the universe domain used by the client instance. + + Returns: + str: The universe domain used + by the client instance. + """ + return self._client._universe_domain + get_transport_class = functools.partial( type(SchemaServiceClient).get_transport_class, type(SchemaServiceClient) ) @@ -178,7 +201,7 @@ def __init__( client_options: Optional[ClientOptions] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, ) -> None: - """Instantiates the schema service client. + """Instantiates the schema service async client. Args: credentials (Optional[google.auth.credentials.Credentials]): The @@ -189,23 +212,38 @@ def __init__( transport (Union[str, ~.SchemaServiceTransport]): The transport to use. If set to None, a transport is chosen automatically. - client_options (ClientOptions): Custom options for the client. It - won't take effect if a ``transport`` instance is provided. - (1) The ``api_endpoint`` property can be used to override the - default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT - environment variable can also be used to override the endpoint: + client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): + Custom options for the client. + + 1. The ``api_endpoint`` property can be used to override the + default endpoint provided by the client when ``transport`` is + not explicitly provided. Only if this property is not set and + ``transport`` was not explicitly provided, the endpoint is + determined by the GOOGLE_API_USE_MTLS_ENDPOINT environment + variable, which have one of the following values: "always" (always use the default mTLS endpoint), "never" (always - use the default regular endpoint) and "auto" (auto switch to the - default mTLS endpoint if client certificate is present, this is - the default value). However, the ``api_endpoint`` property takes - precedence if provided. - (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + use the default regular endpoint) and "auto" (auto-switch to the + default mTLS endpoint if client certificate is present; this is + the default value). + + 2. If the GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable is "true", then the ``client_cert_source`` property can be used - to provide client certificate for mutual TLS transport. If + to provide a client certificate for mTLS transport. If not provided, the default SSL client certificate will be used if present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not set, no client certificate will be used. + 3. The ``universe_domain`` property can be used to override the + default "googleapis.com" universe. Note that ``api_endpoint`` + property still takes precedence; and ``universe_domain`` is + currently not supported for mTLS. + + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + Raises: google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport creation failed for any reason. @@ -285,7 +323,7 @@ async def sample_create_schema(): final component of the schema's resource name. See - https://cloud.google.com/pubsub/docs/admin#resource_names + https://cloud.google.com/pubsub/docs/pubsub-basics#resource_names for resource name constraints. This corresponds to the ``schema_id`` field @@ -345,6 +383,9 @@ async def sample_create_schema(): gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -453,6 +494,9 @@ async def sample_get_schema(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -566,6 +610,9 @@ async def sample_list_schemas(): gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -688,6 +735,9 @@ async def sample_list_schema_revisions(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -819,6 +869,9 @@ async def sample_commit_schema(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -942,6 +995,9 @@ async def sample_rollback_schema(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -1063,6 +1119,9 @@ async def sample_delete_schema_revision(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -1164,6 +1223,9 @@ async def sample_delete_schema(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. await rpc( request, @@ -1285,6 +1347,9 @@ async def sample_validate_schema(): gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -1374,6 +1439,9 @@ async def sample_validate_message(): gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -1493,6 +1561,9 @@ async def set_iam_policy( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -1614,6 +1685,9 @@ async def get_iam_policy( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -1673,6 +1747,9 @@ async def test_iam_permissions( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, diff --git a/google/pubsub_v1/services/schema_service/client.py b/google/pubsub_v1/services/schema_service/client.py index 3106a3292..d869e4dec 100644 --- a/google/pubsub_v1/services/schema_service/client.py +++ b/google/pubsub_v1/services/schema_service/client.py @@ -29,6 +29,7 @@ Union, cast, ) +import warnings from google.pubsub_v1 import gapic_version as package_version @@ -43,9 +44,9 @@ from google.oauth2 import service_account # type: ignore try: - OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault] + OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] except AttributeError: # pragma: NO COVER - OptionalRetry = Union[retries.Retry, object] # type: ignore + OptionalRetry = Union[retries.Retry, object, None] # type: ignore from google.iam.v1 import iam_policy_pb2 # type: ignore from google.iam.v1 import policy_pb2 # type: ignore @@ -129,11 +130,15 @@ def _get_default_mtls_endpoint(api_endpoint): return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. DEFAULT_ENDPOINT = "pubsub.googleapis.com" DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore DEFAULT_ENDPOINT ) + _DEFAULT_ENDPOINT_TEMPLATE = "pubsub.{UNIVERSE_DOMAIN}" + _DEFAULT_UNIVERSE = "googleapis.com" + @classmethod def from_service_account_info(cls, info: dict, *args, **kwargs): """Creates an instance of this client using the provided credentials @@ -279,7 +284,7 @@ def parse_common_location_path(path: str) -> Dict[str, str]: def get_mtls_endpoint_and_cert_source( cls, client_options: Optional[client_options_lib.ClientOptions] = None ): - """Return the API endpoint and client cert source for mutual TLS. + """Deprecated. Return the API endpoint and client cert source for mutual TLS. The client cert source is determined in the following order: (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the @@ -309,6 +314,11 @@ def get_mtls_endpoint_and_cert_source( Raises: google.auth.exceptions.MutualTLSChannelError: If any errors happen. """ + + warnings.warn( + "get_mtls_endpoint_and_cert_source is deprecated. Use the api_endpoint property instead.", + DeprecationWarning, + ) if client_options is None: client_options = client_options_lib.ClientOptions() use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") @@ -342,6 +352,178 @@ def get_mtls_endpoint_and_cert_source( return api_endpoint, client_cert_source + @staticmethod + def _read_environment_variables(): + """Returns the environment variables used by the client. + + Returns: + Tuple[bool, str, str]: returns the GOOGLE_API_USE_CLIENT_CERTIFICATE, + GOOGLE_API_USE_MTLS_ENDPOINT, and GOOGLE_CLOUD_UNIVERSE_DOMAIN environment variables. + + Raises: + ValueError: If GOOGLE_API_USE_CLIENT_CERTIFICATE is not + any of ["true", "false"]. + google.auth.exceptions.MutualTLSChannelError: If GOOGLE_API_USE_MTLS_ENDPOINT + is not any of ["auto", "never", "always"]. + """ + use_client_cert = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_client_cert not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError( + "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + ) + return use_client_cert == "true", use_mtls_endpoint, universe_domain_env + + @staticmethod + def _get_client_cert_source(provided_cert_source, use_cert_flag): + """Return the client cert source to be used by the client. + + Args: + provided_cert_source (bytes): The client certificate source provided. + use_cert_flag (bool): A flag indicating whether to use the client certificate. + + Returns: + bytes or None: The client cert source to be used by the client. + """ + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif mtls.has_default_client_cert_source(): + client_cert_source = mtls.default_client_cert_source() + return client_cert_source + + @staticmethod + def _get_api_endpoint( + api_override, client_cert_source, universe_domain, use_mtls_endpoint + ): + """Return the API endpoint used by the client. + + Args: + api_override (str): The API endpoint override. If specified, this is always + the return value of this function and the other arguments are not used. + client_cert_source (bytes): The client certificate source used by the client. + universe_domain (str): The universe domain used by the client. + use_mtls_endpoint (str): How to use the mTLS endpoint, which depends also on the other parameters. + Possible values are "always", "auto", or "never". + + Returns: + str: The API endpoint to be used by the client. + """ + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or ( + use_mtls_endpoint == "auto" and client_cert_source + ): + _default_universe = SchemaServiceClient._DEFAULT_UNIVERSE + if universe_domain != _default_universe: + raise MutualTLSChannelError( + f"mTLS is not supported in any universe other than {_default_universe}." + ) + api_endpoint = SchemaServiceClient.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = SchemaServiceClient._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=universe_domain + ) + return api_endpoint + + @staticmethod + def _get_universe_domain( + client_universe_domain: Optional[str], universe_domain_env: Optional[str] + ) -> str: + """Return the universe domain used by the client. + + Args: + client_universe_domain (Optional[str]): The universe domain configured via the client options. + universe_domain_env (Optional[str]): The universe domain configured via the "GOOGLE_CLOUD_UNIVERSE_DOMAIN" environment variable. + + Returns: + str: The universe domain to be used by the client. + + Raises: + ValueError: If the universe domain is an empty string. + """ + universe_domain = SchemaServiceClient._DEFAULT_UNIVERSE + if client_universe_domain is not None: + universe_domain = client_universe_domain + elif universe_domain_env is not None: + universe_domain = universe_domain_env + if len(universe_domain.strip()) == 0: + raise ValueError("Universe Domain cannot be an empty string.") + return universe_domain + + @staticmethod + def _compare_universes( + client_universe: str, credentials: ga_credentials.Credentials + ) -> bool: + """Returns True iff the universe domains used by the client and credentials match. + + Args: + client_universe (str): The universe domain configured via the client options. + credentials (ga_credentials.Credentials): The credentials being used in the client. + + Returns: + bool: True iff client_universe matches the universe in credentials. + + Raises: + ValueError: when client_universe does not match the universe in credentials. + """ + + default_universe = SchemaServiceClient._DEFAULT_UNIVERSE + credentials_universe = getattr(credentials, "universe_domain", default_universe) + + if client_universe != credentials_universe: + raise ValueError( + "The configured universe domain " + f"({client_universe}) does not match the universe domain " + f"found in the credentials ({credentials_universe}). " + "If you haven't configured the universe domain explicitly, " + f"`{default_universe}` is the default." + ) + return True + + def _validate_universe_domain(self): + """Validates client's and credentials' universe domains are consistent. + + Returns: + bool: True iff the configured universe domain is valid. + + Raises: + ValueError: If the configured universe domain is not valid. + """ + self._is_universe_domain_valid = ( + self._is_universe_domain_valid + or SchemaServiceClient._compare_universes( + self.universe_domain, self.transport._credentials + ) + ) + return self._is_universe_domain_valid + + @property + def api_endpoint(self): + """Return the API endpoint used by the client instance. + + Returns: + str: The API endpoint used by the client instance. + """ + return self._api_endpoint + + @property + def universe_domain(self) -> str: + """Return the universe domain used by the client instance. + + Returns: + str: The universe domain used by the client instance. + """ + return self._universe_domain + def __init__( self, *, @@ -361,22 +543,32 @@ def __init__( transport (Union[str, SchemaServiceTransport]): The transport to use. If set to None, a transport is chosen automatically. - client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): Custom options for the - client. It won't take effect if a ``transport`` instance is provided. - (1) The ``api_endpoint`` property can be used to override the - default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT - environment variable can also be used to override the endpoint: + client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): + Custom options for the client. + + 1. The ``api_endpoint`` property can be used to override the + default endpoint provided by the client when ``transport`` is + not explicitly provided. Only if this property is not set and + ``transport`` was not explicitly provided, the endpoint is + determined by the GOOGLE_API_USE_MTLS_ENDPOINT environment + variable, which have one of the following values: "always" (always use the default mTLS endpoint), "never" (always - use the default regular endpoint) and "auto" (auto switch to the - default mTLS endpoint if client certificate is present, this is - the default value). However, the ``api_endpoint`` property takes - precedence if provided. - (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + use the default regular endpoint) and "auto" (auto-switch to the + default mTLS endpoint if client certificate is present; this is + the default value). + + 2. If the GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable is "true", then the ``client_cert_source`` property can be used - to provide client certificate for mutual TLS transport. If + to provide a client certificate for mTLS transport. If not provided, the default SSL client certificate will be used if present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not set, no client certificate will be used. + + 3. The ``universe_domain`` property can be used to override the + default "googleapis.com" universe. Note that the ``api_endpoint`` + property still takes precedence; and ``universe_domain`` is + currently not supported for mTLS. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): The client info used to send a user-agent string along with API requests. If ``None``, then default info will be used. @@ -387,17 +579,34 @@ def __init__( google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport creation failed for any reason. """ - if isinstance(client_options, dict): - client_options = client_options_lib.from_dict(client_options) - if client_options is None: - client_options = client_options_lib.ClientOptions() - client_options = cast(client_options_lib.ClientOptions, client_options) + self._client_options = client_options + if isinstance(self._client_options, dict): + self._client_options = client_options_lib.from_dict(self._client_options) + if self._client_options is None: + self._client_options = client_options_lib.ClientOptions() + self._client_options = cast( + client_options_lib.ClientOptions, self._client_options + ) - api_endpoint, client_cert_source_func = self.get_mtls_endpoint_and_cert_source( - client_options + universe_domain_opt = getattr(self._client_options, "universe_domain", None) + + ( + self._use_client_cert, + self._use_mtls_endpoint, + self._universe_domain_env, + ) = SchemaServiceClient._read_environment_variables() + self._client_cert_source = SchemaServiceClient._get_client_cert_source( + self._client_options.client_cert_source, self._use_client_cert + ) + self._universe_domain = SchemaServiceClient._get_universe_domain( + universe_domain_opt, self._universe_domain_env ) + self._api_endpoint = None # updated below, depending on `transport` - api_key_value = getattr(client_options, "api_key", None) + # Initialize the universe domain validation. + self._is_universe_domain_valid = False + + api_key_value = getattr(self._client_options, "api_key", None) if api_key_value and credentials: raise ValueError( "client_options.api_key and credentials are mutually exclusive" @@ -406,20 +615,33 @@ def __init__( # Save or instantiate the transport. # Ordinarily, we provide the transport, but allowing a custom transport # instance provides an extensibility point for unusual situations. - if isinstance(transport, SchemaServiceTransport): + transport_provided = isinstance(transport, SchemaServiceTransport) + if transport_provided: # transport is a SchemaServiceTransport instance. - if credentials or client_options.credentials_file or api_key_value: + if credentials or self._client_options.credentials_file or api_key_value: raise ValueError( "When providing a transport instance, " "provide its credentials directly." ) - if client_options.scopes: + if self._client_options.scopes: raise ValueError( "When providing a transport instance, provide its scopes " "directly." ) - self._transport = transport - else: + self._transport = cast(SchemaServiceTransport, transport) + self._api_endpoint = self._transport.host + + self._api_endpoint = ( + self._api_endpoint + or SchemaServiceClient._get_api_endpoint( + self._client_options.api_endpoint, + self._client_cert_source, + self._universe_domain, + self._use_mtls_endpoint, + ) + ) + + if not transport_provided: import google.auth._default # type: ignore if api_key_value and hasattr( @@ -429,7 +651,7 @@ def __init__( api_key_value ) - Transport = type(self).get_transport_class(transport) + Transport = type(self).get_transport_class(cast(str, transport)) emulator_host = os.environ.get("PUBSUB_EMULATOR_HOST") if emulator_host: @@ -441,14 +663,14 @@ def __init__( self._transport = Transport( credentials=credentials, - credentials_file=client_options.credentials_file, - host=api_endpoint, - scopes=client_options.scopes, - client_cert_source_for_mtls=client_cert_source_func, - quota_project_id=client_options.quota_project_id, + credentials_file=self._client_options.credentials_file, + host=self._api_endpoint, + scopes=self._client_options.scopes, + client_cert_source_for_mtls=self._client_cert_source, + quota_project_id=self._client_options.quota_project_id, client_info=client_info, always_use_jwt_access=True, - api_audience=client_options.api_audience, + api_audience=self._client_options.api_audience, ) def create_schema( @@ -519,7 +741,7 @@ def sample_create_schema(): final component of the schema's resource name. See - https://cloud.google.com/pubsub/docs/admin#resource_names + https://cloud.google.com/pubsub/docs/pubsub-basics#resource_names for resource name constraints. This corresponds to the ``schema_id`` field @@ -570,6 +792,9 @@ def sample_create_schema(): gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -669,6 +894,9 @@ def sample_get_schema(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -773,6 +1001,9 @@ def sample_list_schemas(): gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -886,6 +1117,9 @@ def sample_list_schema_revisions(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1008,6 +1242,9 @@ def sample_commit_schema(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1122,6 +1359,9 @@ def sample_rollback_schema(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1234,6 +1474,9 @@ def sample_delete_schema_revision(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1326,6 +1569,9 @@ def sample_delete_schema(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. rpc( request, @@ -1438,6 +1684,9 @@ def sample_validate_schema(): gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1519,6 +1768,9 @@ def sample_validate_message(): gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1652,6 +1904,9 @@ def set_iam_policy( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1773,6 +2028,9 @@ def get_iam_policy( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1832,6 +2090,9 @@ def test_iam_permissions( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, diff --git a/google/pubsub_v1/services/schema_service/transports/base.py b/google/pubsub_v1/services/schema_service/transports/base.py index 08b370cf7..39151e397 100644 --- a/google/pubsub_v1/services/schema_service/transports/base.py +++ b/google/pubsub_v1/services/schema_service/transports/base.py @@ -64,7 +64,7 @@ def __init__( Args: host (Optional[str]): - The hostname to connect to. + The hostname to connect to (default: 'pubsub.googleapis.com'). credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none @@ -127,6 +127,10 @@ def __init__( host += ":443" self._host = host + @property + def host(self): + return self._host + def _prep_wrapped_messages(self, client_info): # Precompute the wrapped methods. self._wrapped_methods = { diff --git a/google/pubsub_v1/services/schema_service/transports/grpc.py b/google/pubsub_v1/services/schema_service/transports/grpc.py index c14a1d5f4..66b382bfb 100644 --- a/google/pubsub_v1/services/schema_service/transports/grpc.py +++ b/google/pubsub_v1/services/schema_service/transports/grpc.py @@ -68,7 +68,7 @@ def __init__( Args: host (Optional[str]): - The hostname to connect to. + The hostname to connect to (default: 'pubsub.googleapis.com'). credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none diff --git a/google/pubsub_v1/services/schema_service/transports/grpc_asyncio.py b/google/pubsub_v1/services/schema_service/transports/grpc_asyncio.py index 5b435aff6..94a872b56 100644 --- a/google/pubsub_v1/services/schema_service/transports/grpc_asyncio.py +++ b/google/pubsub_v1/services/schema_service/transports/grpc_asyncio.py @@ -113,7 +113,7 @@ def __init__( Args: host (Optional[str]): - The hostname to connect to. + The hostname to connect to (default: 'pubsub.googleapis.com'). credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none diff --git a/google/pubsub_v1/services/schema_service/transports/rest.py b/google/pubsub_v1/services/schema_service/transports/rest.py index 0586bbf9d..5e5d7d478 100644 --- a/google/pubsub_v1/services/schema_service/transports/rest.py +++ b/google/pubsub_v1/services/schema_service/transports/rest.py @@ -36,9 +36,9 @@ import warnings try: - OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault] + OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] except AttributeError: # pragma: NO COVER - OptionalRetry = Union[retries.Retry, object] # type: ignore + OptionalRetry = Union[retries.Retry, object, None] # type: ignore from google.iam.v1 import iam_policy_pb2 # type: ignore @@ -463,7 +463,7 @@ def __init__( Args: host (Optional[str]): - The hostname to connect to. + The hostname to connect to (default: 'pubsub.googleapis.com'). credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none diff --git a/google/pubsub_v1/services/subscriber/async_client.py b/google/pubsub_v1/services/subscriber/async_client.py index 13f30bd89..78ad7c9a2 100644 --- a/google/pubsub_v1/services/subscriber/async_client.py +++ b/google/pubsub_v1/services/subscriber/async_client.py @@ -42,9 +42,9 @@ from google.oauth2 import service_account # type: ignore try: - OptionalRetry = Union[retries.AsyncRetry, gapic_v1.method._MethodDefault] + OptionalRetry = Union[retries.AsyncRetry, gapic_v1.method._MethodDefault, None] except AttributeError: # pragma: NO COVER - OptionalRetry = Union[retries.AsyncRetry, object] # type: ignore + OptionalRetry = Union[retries.AsyncRetry, object, None] # type: ignore from google.iam.v1 import iam_policy_pb2 # type: ignore from google.iam.v1 import policy_pb2 # type: ignore @@ -67,8 +67,12 @@ class SubscriberAsyncClient: _client: SubscriberClient + # Copy defaults from the synchronous client for use here. + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. DEFAULT_ENDPOINT = SubscriberClient.DEFAULT_ENDPOINT DEFAULT_MTLS_ENDPOINT = SubscriberClient.DEFAULT_MTLS_ENDPOINT + _DEFAULT_ENDPOINT_TEMPLATE = SubscriberClient._DEFAULT_ENDPOINT_TEMPLATE + _DEFAULT_UNIVERSE = SubscriberClient._DEFAULT_UNIVERSE snapshot_path = staticmethod(SubscriberClient.snapshot_path) parse_snapshot_path = staticmethod(SubscriberClient.parse_snapshot_path) @@ -173,6 +177,25 @@ def transport(self) -> SubscriberTransport: """ return self._client.transport + @property + def api_endpoint(self): + """Return the API endpoint used by the client instance. + + Returns: + str: The API endpoint used by the client instance. + """ + return self._client._api_endpoint + + @property + def universe_domain(self) -> str: + """Return the universe domain used by the client instance. + + Returns: + str: The universe domain used + by the client instance. + """ + return self._client._universe_domain + get_transport_class = functools.partial( type(SubscriberClient).get_transport_class, type(SubscriberClient) ) @@ -185,7 +208,7 @@ def __init__( client_options: Optional[ClientOptions] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, ) -> None: - """Instantiates the subscriber client. + """Instantiates the subscriber async client. Args: credentials (Optional[google.auth.credentials.Credentials]): The @@ -196,23 +219,38 @@ def __init__( transport (Union[str, ~.SubscriberTransport]): The transport to use. If set to None, a transport is chosen automatically. - client_options (ClientOptions): Custom options for the client. It - won't take effect if a ``transport`` instance is provided. - (1) The ``api_endpoint`` property can be used to override the - default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT - environment variable can also be used to override the endpoint: + client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): + Custom options for the client. + + 1. The ``api_endpoint`` property can be used to override the + default endpoint provided by the client when ``transport`` is + not explicitly provided. Only if this property is not set and + ``transport`` was not explicitly provided, the endpoint is + determined by the GOOGLE_API_USE_MTLS_ENDPOINT environment + variable, which have one of the following values: "always" (always use the default mTLS endpoint), "never" (always - use the default regular endpoint) and "auto" (auto switch to the - default mTLS endpoint if client certificate is present, this is - the default value). However, the ``api_endpoint`` property takes - precedence if provided. - (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + use the default regular endpoint) and "auto" (auto-switch to the + default mTLS endpoint if client certificate is present; this is + the default value). + + 2. If the GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable is "true", then the ``client_cert_source`` property can be used - to provide client certificate for mutual TLS transport. If + to provide a client certificate for mTLS transport. If not provided, the default SSL client certificate will be used if present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not set, no client certificate will be used. + 3. The ``universe_domain`` property can be used to override the + default "googleapis.com" universe. Note that ``api_endpoint`` + property still takes precedence; and ``universe_domain`` is + currently not supported for mTLS. + + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + Raises: google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport creation failed for any reason. @@ -308,21 +346,21 @@ async def sample_create_subscription(): on the ``request`` instance; if ``request`` is provided, this should not be set. push_config (:class:`google.pubsub_v1.types.PushConfig`): - If push delivery is used with this - subscription, this field is used to - configure it. + Optional. If push delivery is used + with this subscription, this field is + used to configure it. This corresponds to the ``push_config`` field on the ``request`` instance; if ``request`` is provided, this should not be set. ack_deadline_seconds (:class:`int`): - The approximate amount of time (on a best-effort basis) - Pub/Sub waits for the subscriber to acknowledge receipt - before resending the message. In the interval after the - message is delivered and before it is acknowledged, it - is considered to be *outstanding*. During that time - period, the message will not be redelivered (on a - best-effort basis). + Optional. The approximate amount of time (on a + best-effort basis) Pub/Sub waits for the subscriber to + acknowledge receipt before resending the message. In the + interval after the message is delivered and before it is + acknowledged, it is considered to be *outstanding*. + During that time period, the message will not be + redelivered (on a best-effort basis). For pull subscriptions, this value is used as the initial value for the ack deadline. To override this @@ -406,6 +444,9 @@ async def sample_create_subscription(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -523,6 +564,9 @@ async def sample_get_subscription(): ), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -544,7 +588,8 @@ async def update_subscription( timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, str]] = (), ) -> pubsub.Subscription: - r"""Updates an existing subscription. Note that certain + r"""Updates an existing subscription by updating the + fields specified in the update mask. Note that certain properties of a subscription, such as its topic, are not modifiable. @@ -655,6 +700,9 @@ async def sample_update_subscription(): ), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -770,6 +818,9 @@ async def sample_list_subscriptions(): gapic_v1.routing_header.to_grpc_metadata((("project", request.project),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -888,6 +939,9 @@ async def sample_delete_subscription(): ), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. await rpc( request, @@ -965,8 +1019,8 @@ async def sample_modify_ack_deadline(): client. This typically results in an increase in the rate of message redeliveries (that is, duplicates). The minimum deadline you can specify is 0 seconds. The - maximum deadline you can specify is 600 seconds (10 - minutes). + maximum deadline you can specify in a single request is + 600 seconds (10 minutes). This corresponds to the ``ack_deadline_seconds`` field on the ``request`` instance; if ``request`` is provided, this @@ -1023,6 +1077,9 @@ async def sample_modify_ack_deadline(): ), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. await rpc( request, @@ -1143,6 +1200,9 @@ async def sample_acknowledge(): ), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. await rpc( request, @@ -1291,6 +1351,9 @@ async def sample_pull(): ), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -1383,7 +1446,7 @@ def request_generator(): default_retry=retries.AsyncRetry( initial=0.1, maximum=60.0, - multiplier=4.0, + multiplier=4, predicate=retries.if_exception_type( core_exceptions.Aborted, core_exceptions.DeadlineExceeded, @@ -1397,6 +1460,9 @@ def request_generator(): client_info=DEFAULT_CLIENT_INFO, ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = rpc( requests, @@ -1522,6 +1588,9 @@ async def sample_modify_push_config(): ), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. await rpc( request, @@ -1640,6 +1709,9 @@ async def sample_get_snapshot(): gapic_v1.routing_header.to_grpc_metadata((("snapshot", request.snapshot),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -1759,6 +1831,9 @@ async def sample_list_snapshots(): gapic_v1.routing_header.to_grpc_metadata((("project", request.project),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -1925,6 +2000,9 @@ async def sample_create_snapshot(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -1946,7 +2024,8 @@ async def update_snapshot( timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, str]] = (), ) -> pubsub.Snapshot: - r"""Updates an existing snapshot. Snapshots are used in + r"""Updates an existing snapshot by updating the fields specified in + the update mask. Snapshots are used in `Seek `__ operations, which allow you to manage message acknowledgments in bulk. That is, you can set the acknowledgment state of messages @@ -2056,6 +2135,9 @@ async def sample_update_snapshot(): ), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -2166,6 +2248,9 @@ async def sample_delete_snapshot(): gapic_v1.routing_header.to_grpc_metadata((("snapshot", request.snapshot),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. await rpc( request, @@ -2261,6 +2346,9 @@ async def sample_seek(): ), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -2380,6 +2468,9 @@ async def set_iam_policy( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -2501,6 +2592,9 @@ async def get_iam_policy( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, @@ -2560,6 +2654,9 @@ async def test_iam_permissions( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._client._validate_universe_domain() + # Send the request. response = await rpc( request, diff --git a/google/pubsub_v1/services/subscriber/client.py b/google/pubsub_v1/services/subscriber/client.py index f74e895a3..74fdb1d28 100644 --- a/google/pubsub_v1/services/subscriber/client.py +++ b/google/pubsub_v1/services/subscriber/client.py @@ -31,6 +31,7 @@ Union, cast, ) +import warnings import warnings from google.pubsub_v1 import gapic_version as package_version @@ -46,9 +47,9 @@ from google.oauth2 import service_account # type: ignore try: - OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault] + OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] except AttributeError: # pragma: NO COVER - OptionalRetry = Union[retries.Retry, object] # type: ignore + OptionalRetry = Union[retries.Retry, object, None] # type: ignore from google.iam.v1 import iam_policy_pb2 # type: ignore from google.iam.v1 import policy_pb2 # type: ignore @@ -137,6 +138,8 @@ def _get_default_mtls_endpoint(api_endpoint): return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. + # The scopes needed to make gRPC calls to all of the methods defined in # this service _DEFAULT_SCOPES = ( @@ -152,6 +155,9 @@ def _get_default_mtls_endpoint(api_endpoint): DEFAULT_ENDPOINT ) + _DEFAULT_ENDPOINT_TEMPLATE = "pubsub.{UNIVERSE_DOMAIN}" + _DEFAULT_UNIVERSE = "googleapis.com" + @classmethod def from_service_account_info(cls, info: dict, *args, **kwargs): """Creates an instance of this client using the provided credentials @@ -333,7 +339,7 @@ def parse_common_location_path(path: str) -> Dict[str, str]: def get_mtls_endpoint_and_cert_source( cls, client_options: Optional[client_options_lib.ClientOptions] = None ): - """Return the API endpoint and client cert source for mutual TLS. + """Deprecated. Return the API endpoint and client cert source for mutual TLS. The client cert source is determined in the following order: (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the @@ -363,6 +369,11 @@ def get_mtls_endpoint_and_cert_source( Raises: google.auth.exceptions.MutualTLSChannelError: If any errors happen. """ + + warnings.warn( + "get_mtls_endpoint_and_cert_source is deprecated. Use the api_endpoint property instead.", + DeprecationWarning, + ) if client_options is None: client_options = client_options_lib.ClientOptions() use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") @@ -396,6 +407,178 @@ def get_mtls_endpoint_and_cert_source( return api_endpoint, client_cert_source + @staticmethod + def _read_environment_variables(): + """Returns the environment variables used by the client. + + Returns: + Tuple[bool, str, str]: returns the GOOGLE_API_USE_CLIENT_CERTIFICATE, + GOOGLE_API_USE_MTLS_ENDPOINT, and GOOGLE_CLOUD_UNIVERSE_DOMAIN environment variables. + + Raises: + ValueError: If GOOGLE_API_USE_CLIENT_CERTIFICATE is not + any of ["true", "false"]. + google.auth.exceptions.MutualTLSChannelError: If GOOGLE_API_USE_MTLS_ENDPOINT + is not any of ["auto", "never", "always"]. + """ + use_client_cert = os.getenv( + "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false" + ).lower() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_client_cert not in ("true", "false"): + raise ValueError( + "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError( + "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + ) + return use_client_cert == "true", use_mtls_endpoint, universe_domain_env + + @staticmethod + def _get_client_cert_source(provided_cert_source, use_cert_flag): + """Return the client cert source to be used by the client. + + Args: + provided_cert_source (bytes): The client certificate source provided. + use_cert_flag (bool): A flag indicating whether to use the client certificate. + + Returns: + bytes or None: The client cert source to be used by the client. + """ + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif mtls.has_default_client_cert_source(): + client_cert_source = mtls.default_client_cert_source() + return client_cert_source + + @staticmethod + def _get_api_endpoint( + api_override, client_cert_source, universe_domain, use_mtls_endpoint + ): + """Return the API endpoint used by the client. + + Args: + api_override (str): The API endpoint override. If specified, this is always + the return value of this function and the other arguments are not used. + client_cert_source (bytes): The client certificate source used by the client. + universe_domain (str): The universe domain used by the client. + use_mtls_endpoint (str): How to use the mTLS endpoint, which depends also on the other parameters. + Possible values are "always", "auto", or "never". + + Returns: + str: The API endpoint to be used by the client. + """ + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or ( + use_mtls_endpoint == "auto" and client_cert_source + ): + _default_universe = SubscriberClient._DEFAULT_UNIVERSE + if universe_domain != _default_universe: + raise MutualTLSChannelError( + f"mTLS is not supported in any universe other than {_default_universe}." + ) + api_endpoint = SubscriberClient.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = SubscriberClient._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=universe_domain + ) + return api_endpoint + + @staticmethod + def _get_universe_domain( + client_universe_domain: Optional[str], universe_domain_env: Optional[str] + ) -> str: + """Return the universe domain used by the client. + + Args: + client_universe_domain (Optional[str]): The universe domain configured via the client options. + universe_domain_env (Optional[str]): The universe domain configured via the "GOOGLE_CLOUD_UNIVERSE_DOMAIN" environment variable. + + Returns: + str: The universe domain to be used by the client. + + Raises: + ValueError: If the universe domain is an empty string. + """ + universe_domain = SubscriberClient._DEFAULT_UNIVERSE + if client_universe_domain is not None: + universe_domain = client_universe_domain + elif universe_domain_env is not None: + universe_domain = universe_domain_env + if len(universe_domain.strip()) == 0: + raise ValueError("Universe Domain cannot be an empty string.") + return universe_domain + + @staticmethod + def _compare_universes( + client_universe: str, credentials: ga_credentials.Credentials + ) -> bool: + """Returns True iff the universe domains used by the client and credentials match. + + Args: + client_universe (str): The universe domain configured via the client options. + credentials (ga_credentials.Credentials): The credentials being used in the client. + + Returns: + bool: True iff client_universe matches the universe in credentials. + + Raises: + ValueError: when client_universe does not match the universe in credentials. + """ + + default_universe = SubscriberClient._DEFAULT_UNIVERSE + credentials_universe = getattr(credentials, "universe_domain", default_universe) + + if client_universe != credentials_universe: + raise ValueError( + "The configured universe domain " + f"({client_universe}) does not match the universe domain " + f"found in the credentials ({credentials_universe}). " + "If you haven't configured the universe domain explicitly, " + f"`{default_universe}` is the default." + ) + return True + + def _validate_universe_domain(self): + """Validates client's and credentials' universe domains are consistent. + + Returns: + bool: True iff the configured universe domain is valid. + + Raises: + ValueError: If the configured universe domain is not valid. + """ + self._is_universe_domain_valid = ( + self._is_universe_domain_valid + or SubscriberClient._compare_universes( + self.universe_domain, self.transport._credentials + ) + ) + return self._is_universe_domain_valid + + @property + def api_endpoint(self): + """Return the API endpoint used by the client instance. + + Returns: + str: The API endpoint used by the client instance. + """ + return self._api_endpoint + + @property + def universe_domain(self) -> str: + """Return the universe domain used by the client instance. + + Returns: + str: The universe domain used by the client instance. + """ + return self._universe_domain + def __init__( self, *, @@ -415,22 +598,32 @@ def __init__( transport (Union[str, SubscriberTransport]): The transport to use. If set to None, a transport is chosen automatically. - client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): Custom options for the - client. It won't take effect if a ``transport`` instance is provided. - (1) The ``api_endpoint`` property can be used to override the - default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT - environment variable can also be used to override the endpoint: + client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): + Custom options for the client. + + 1. The ``api_endpoint`` property can be used to override the + default endpoint provided by the client when ``transport`` is + not explicitly provided. Only if this property is not set and + ``transport`` was not explicitly provided, the endpoint is + determined by the GOOGLE_API_USE_MTLS_ENDPOINT environment + variable, which have one of the following values: "always" (always use the default mTLS endpoint), "never" (always - use the default regular endpoint) and "auto" (auto switch to the - default mTLS endpoint if client certificate is present, this is - the default value). However, the ``api_endpoint`` property takes - precedence if provided. - (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + use the default regular endpoint) and "auto" (auto-switch to the + default mTLS endpoint if client certificate is present; this is + the default value). + + 2. If the GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable is "true", then the ``client_cert_source`` property can be used - to provide client certificate for mutual TLS transport. If + to provide a client certificate for mTLS transport. If not provided, the default SSL client certificate will be used if present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not set, no client certificate will be used. + + 3. The ``universe_domain`` property can be used to override the + default "googleapis.com" universe. Note that the ``api_endpoint`` + property still takes precedence; and ``universe_domain`` is + currently not supported for mTLS. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): The client info used to send a user-agent string along with API requests. If ``None``, then default info will be used. @@ -441,17 +634,34 @@ def __init__( google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport creation failed for any reason. """ - if isinstance(client_options, dict): - client_options = client_options_lib.from_dict(client_options) - if client_options is None: - client_options = client_options_lib.ClientOptions() - client_options = cast(client_options_lib.ClientOptions, client_options) + self._client_options = client_options + if isinstance(self._client_options, dict): + self._client_options = client_options_lib.from_dict(self._client_options) + if self._client_options is None: + self._client_options = client_options_lib.ClientOptions() + self._client_options = cast( + client_options_lib.ClientOptions, self._client_options + ) - api_endpoint, client_cert_source_func = self.get_mtls_endpoint_and_cert_source( - client_options + universe_domain_opt = getattr(self._client_options, "universe_domain", None) + + ( + self._use_client_cert, + self._use_mtls_endpoint, + self._universe_domain_env, + ) = SubscriberClient._read_environment_variables() + self._client_cert_source = SubscriberClient._get_client_cert_source( + self._client_options.client_cert_source, self._use_client_cert + ) + self._universe_domain = SubscriberClient._get_universe_domain( + universe_domain_opt, self._universe_domain_env ) + self._api_endpoint = None # updated below, depending on `transport` - api_key_value = getattr(client_options, "api_key", None) + # Initialize the universe domain validation. + self._is_universe_domain_valid = False + + api_key_value = getattr(self._client_options, "api_key", None) if api_key_value and credentials: raise ValueError( "client_options.api_key and credentials are mutually exclusive" @@ -460,20 +670,30 @@ def __init__( # Save or instantiate the transport. # Ordinarily, we provide the transport, but allowing a custom transport # instance provides an extensibility point for unusual situations. - if isinstance(transport, SubscriberTransport): + transport_provided = isinstance(transport, SubscriberTransport) + if transport_provided: # transport is a SubscriberTransport instance. - if credentials or client_options.credentials_file or api_key_value: + if credentials or self._client_options.credentials_file or api_key_value: raise ValueError( "When providing a transport instance, " "provide its credentials directly." ) - if client_options.scopes: + if self._client_options.scopes: raise ValueError( "When providing a transport instance, provide its scopes " "directly." ) - self._transport = transport - else: + self._transport = cast(SubscriberTransport, transport) + self._api_endpoint = self._transport.host + + self._api_endpoint = self._api_endpoint or SubscriberClient._get_api_endpoint( + self._client_options.api_endpoint, + self._client_cert_source, + self._universe_domain, + self._use_mtls_endpoint, + ) + + if not transport_provided: import google.auth._default # type: ignore if api_key_value and hasattr( @@ -483,7 +703,7 @@ def __init__( api_key_value ) - Transport = type(self).get_transport_class(transport) + Transport = type(self).get_transport_class(cast(str, transport)) emulator_host = os.environ.get("PUBSUB_EMULATOR_HOST") if emulator_host: @@ -495,14 +715,14 @@ def __init__( self._transport = Transport( credentials=credentials, - credentials_file=client_options.credentials_file, - host=api_endpoint, - scopes=client_options.scopes, - client_cert_source_for_mtls=client_cert_source_func, - quota_project_id=client_options.quota_project_id, + credentials_file=self._client_options.credentials_file, + host=self._api_endpoint, + scopes=self._client_options.scopes, + client_cert_source_for_mtls=self._client_cert_source, + quota_project_id=self._client_options.quota_project_id, client_info=client_info, always_use_jwt_access=True, - api_audience=client_options.api_audience, + api_audience=self._client_options.api_audience, ) def create_subscription( @@ -589,21 +809,21 @@ def sample_create_subscription(): on the ``request`` instance; if ``request`` is provided, this should not be set. push_config (google.pubsub_v1.types.PushConfig): - If push delivery is used with this - subscription, this field is used to - configure it. + Optional. If push delivery is used + with this subscription, this field is + used to configure it. This corresponds to the ``push_config`` field on the ``request`` instance; if ``request`` is provided, this should not be set. ack_deadline_seconds (int): - The approximate amount of time (on a best-effort basis) - Pub/Sub waits for the subscriber to acknowledge receipt - before resending the message. In the interval after the - message is delivered and before it is acknowledged, it - is considered to be *outstanding*. During that time - period, the message will not be redelivered (on a - best-effort basis). + Optional. The approximate amount of time (on a + best-effort basis) Pub/Sub waits for the subscriber to + acknowledge receipt before resending the message. In the + interval after the message is delivered and before it is + acknowledged, it is considered to be *outstanding*. + During that time period, the message will not be + redelivered (on a best-effort basis). For pull subscriptions, this value is used as the initial value for the ack deadline. To override this @@ -676,6 +896,9 @@ def sample_create_subscription(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -782,6 +1005,9 @@ def sample_get_subscription(): ), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -803,7 +1029,8 @@ def update_subscription( timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, str]] = (), ) -> pubsub.Subscription: - r"""Updates an existing subscription. Note that certain + r"""Updates an existing subscription by updating the + fields specified in the update mask. Note that certain properties of a subscription, such as its topic, are not modifiable. @@ -905,6 +1132,9 @@ def sample_update_subscription(): ), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1009,6 +1239,9 @@ def sample_list_subscriptions(): gapic_v1.routing_header.to_grpc_metadata((("project", request.project),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1118,6 +1351,9 @@ def sample_delete_subscription(): ), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. rpc( request, @@ -1195,8 +1431,8 @@ def sample_modify_ack_deadline(): client. This typically results in an increase in the rate of message redeliveries (that is, duplicates). The minimum deadline you can specify is 0 seconds. The - maximum deadline you can specify is 600 seconds (10 - minutes). + maximum deadline you can specify in a single request is + 600 seconds (10 minutes). This corresponds to the ``ack_deadline_seconds`` field on the ``request`` instance; if ``request`` is provided, this @@ -1244,6 +1480,9 @@ def sample_modify_ack_deadline(): ), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. rpc( request, @@ -1355,6 +1594,9 @@ def sample_acknowledge(): ), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. rpc( request, @@ -1491,6 +1733,9 @@ def sample_pull(): ), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1585,6 +1830,9 @@ def request_generator(): # and friendly error handling. rpc = self._transport._wrapped_methods[self._transport.streaming_pull] + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( requests, @@ -1701,6 +1949,9 @@ def sample_modify_push_config(): ), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. rpc( request, @@ -1808,6 +2059,9 @@ def sample_get_snapshot(): gapic_v1.routing_header.to_grpc_metadata((("snapshot", request.snapshot),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -1916,6 +2170,9 @@ def sample_list_snapshots(): gapic_v1.routing_header.to_grpc_metadata((("project", request.project),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -2073,6 +2330,9 @@ def sample_create_snapshot(): gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -2094,7 +2354,8 @@ def update_snapshot( timeout: Union[float, object] = gapic_v1.method.DEFAULT, metadata: Sequence[Tuple[str, str]] = (), ) -> pubsub.Snapshot: - r"""Updates an existing snapshot. Snapshots are used in + r"""Updates an existing snapshot by updating the fields specified in + the update mask. Snapshots are used in `Seek `__ operations, which allow you to manage message acknowledgments in bulk. That is, you can set the acknowledgment state of messages @@ -2195,6 +2456,9 @@ def sample_update_snapshot(): ), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -2296,6 +2560,9 @@ def sample_delete_snapshot(): gapic_v1.routing_header.to_grpc_metadata((("snapshot", request.snapshot),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. rpc( request, @@ -2381,6 +2648,9 @@ def sample_seek(): ), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -2514,6 +2784,9 @@ def set_iam_policy( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -2635,6 +2908,9 @@ def get_iam_policy( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, @@ -2694,6 +2970,9 @@ def test_iam_permissions( gapic_v1.routing_header.to_grpc_metadata((("resource", request.resource),)), ) + # Validate the universe domain. + self._validate_universe_domain() + # Send the request. response = rpc( request, diff --git a/google/pubsub_v1/services/subscriber/transports/base.py b/google/pubsub_v1/services/subscriber/transports/base.py index d50b8baf6..cea627d76 100644 --- a/google/pubsub_v1/services/subscriber/transports/base.py +++ b/google/pubsub_v1/services/subscriber/transports/base.py @@ -63,7 +63,7 @@ def __init__( Args: host (Optional[str]): - The hostname to connect to. + The hostname to connect to (default: 'pubsub.googleapis.com'). credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none @@ -126,6 +126,10 @@ def __init__( host += ":443" self._host = host + @property + def host(self): + return self._host + def _prep_wrapped_messages(self, client_info): # Precompute the wrapped methods. self._wrapped_methods = { @@ -255,7 +259,7 @@ def _prep_wrapped_messages(self, client_info): default_retry=retries.Retry( initial=0.1, maximum=60.0, - multiplier=4.0, + multiplier=4, predicate=retries.if_exception_type( core_exceptions.Aborted, core_exceptions.DeadlineExceeded, diff --git a/google/pubsub_v1/services/subscriber/transports/grpc.py b/google/pubsub_v1/services/subscriber/transports/grpc.py index 0b1fcd3aa..85a1942af 100644 --- a/google/pubsub_v1/services/subscriber/transports/grpc.py +++ b/google/pubsub_v1/services/subscriber/transports/grpc.py @@ -70,7 +70,7 @@ def __init__( Args: host (Optional[str]): - The hostname to connect to. + The hostname to connect to (default: 'pubsub.googleapis.com'). credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none @@ -307,7 +307,8 @@ def update_subscription( ) -> Callable[[pubsub.UpdateSubscriptionRequest], pubsub.Subscription]: r"""Return a callable for the update subscription method over gRPC. - Updates an existing subscription. Note that certain + Updates an existing subscription by updating the + fields specified in the update mask. Note that certain properties of a subscription, such as its topic, are not modifiable. @@ -646,7 +647,8 @@ def update_snapshot( ) -> Callable[[pubsub.UpdateSnapshotRequest], pubsub.Snapshot]: r"""Return a callable for the update snapshot method over gRPC. - Updates an existing snapshot. Snapshots are used in + Updates an existing snapshot by updating the fields specified in + the update mask. Snapshots are used in `Seek `__ operations, which allow you to manage message acknowledgments in bulk. That is, you can set the acknowledgment state of messages diff --git a/google/pubsub_v1/services/subscriber/transports/grpc_asyncio.py b/google/pubsub_v1/services/subscriber/transports/grpc_asyncio.py index d32730c1e..b4a73baf4 100644 --- a/google/pubsub_v1/services/subscriber/transports/grpc_asyncio.py +++ b/google/pubsub_v1/services/subscriber/transports/grpc_asyncio.py @@ -115,7 +115,7 @@ def __init__( Args: host (Optional[str]): - The hostname to connect to. + The hostname to connect to (default: 'pubsub.googleapis.com'). credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none @@ -310,7 +310,8 @@ def update_subscription( ) -> Callable[[pubsub.UpdateSubscriptionRequest], Awaitable[pubsub.Subscription]]: r"""Return a callable for the update subscription method over gRPC. - Updates an existing subscription. Note that certain + Updates an existing subscription by updating the + fields specified in the update mask. Note that certain properties of a subscription, such as its topic, are not modifiable. @@ -659,7 +660,8 @@ def update_snapshot( ) -> Callable[[pubsub.UpdateSnapshotRequest], Awaitable[pubsub.Snapshot]]: r"""Return a callable for the update snapshot method over gRPC. - Updates an existing snapshot. Snapshots are used in + Updates an existing snapshot by updating the fields specified in + the update mask. Snapshots are used in `Seek `__ operations, which allow you to manage message acknowledgments in bulk. That is, you can set the acknowledgment state of messages diff --git a/google/pubsub_v1/services/subscriber/transports/rest.py b/google/pubsub_v1/services/subscriber/transports/rest.py index c633c4115..26dee31f2 100644 --- a/google/pubsub_v1/services/subscriber/transports/rest.py +++ b/google/pubsub_v1/services/subscriber/transports/rest.py @@ -36,9 +36,9 @@ import warnings try: - OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault] + OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] except AttributeError: # pragma: NO COVER - OptionalRetry = Union[retries.Retry, object] # type: ignore + OptionalRetry = Union[retries.Retry, object, None] # type: ignore from google.iam.v1 import iam_policy_pb2 # type: ignore @@ -547,7 +547,7 @@ def __init__( Args: host (Optional[str]): - The hostname to connect to. + The hostname to connect to (default: 'pubsub.googleapis.com'). credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none diff --git a/google/pubsub_v1/types/__init__.py b/google/pubsub_v1/types/__init__.py index c82fefd13..80135a019 100644 --- a/google/pubsub_v1/types/__init__.py +++ b/google/pubsub_v1/types/__init__.py @@ -30,6 +30,7 @@ GetSnapshotRequest, GetSubscriptionRequest, GetTopicRequest, + IngestionDataSourceSettings, ListSnapshotsRequest, ListSnapshotsResponse, ListSubscriptionsRequest, @@ -107,6 +108,7 @@ "GetSnapshotRequest", "GetSubscriptionRequest", "GetTopicRequest", + "IngestionDataSourceSettings", "ListSnapshotsRequest", "ListSnapshotsResponse", "ListSubscriptionsRequest", diff --git a/google/pubsub_v1/types/pubsub.py b/google/pubsub_v1/types/pubsub.py index ff7becd48..b04d5d745 100644 --- a/google/pubsub_v1/types/pubsub.py +++ b/google/pubsub_v1/types/pubsub.py @@ -30,6 +30,7 @@ manifest={ "MessageStoragePolicy", "SchemaSettings", + "IngestionDataSourceSettings", "Topic", "PubsubMessage", "GetTopicRequest", @@ -84,21 +85,32 @@ class MessageStoragePolicy(proto.Message): Attributes: allowed_persistence_regions (MutableSequence[str]): - A list of IDs of Google Cloud regions where - messages that are published to the topic may be - persisted in storage. Messages published by - publishers running in non-allowed Google Cloud - regions (or running outside of Google Cloud - altogether) are routed for storage in one of the - allowed regions. An empty list means that no - regions are allowed, and is not a valid - configuration. + Optional. A list of IDs of Google Cloud + regions where messages that are published to the + topic may be persisted in storage. Messages + published by publishers running in non-allowed + Google Cloud regions (or running outside of + Google Cloud altogether) are routed for storage + in one of the allowed regions. An empty list + means that no regions are allowed, and is not a + valid configuration. + enforce_in_transit (bool): + Optional. If true, ``allowed_persistence_regions`` is also + used to enforce in-transit guarantees for messages. That is, + Pub/Sub will fail Publish operations on this topic and + subscribe operations on any subscription attached to this + topic in any region that is not in + ``allowed_persistence_regions``. """ allowed_persistence_regions: MutableSequence[str] = proto.RepeatedField( proto.STRING, number=1, ) + enforce_in_transit: bool = proto.Field( + proto.BOOL, + number=2, + ) class SchemaSettings(proto.Message): @@ -112,17 +124,18 @@ class SchemaSettings(proto.Message): field will be ``_deleted-schema_`` if the schema has been deleted. encoding (google.pubsub_v1.types.Encoding): - The encoding of messages validated against ``schema``. + Optional. The encoding of messages validated against + ``schema``. first_revision_id (str): - The minimum (inclusive) revision allowed for validating - messages. If empty or not present, allow any revision to be - validated against last_revision or any revision created - before. + Optional. The minimum (inclusive) revision allowed for + validating messages. If empty or not present, allow any + revision to be validated against last_revision or any + revision created before. last_revision_id (str): - The maximum (inclusive) revision allowed for validating - messages. If empty or not present, allow any revision to be - validated against first_revision or any revision created - after. + Optional. The maximum (inclusive) revision allowed for + validating messages. If empty or not present, allow any + revision to be validated against first_revision or any + revision created after. """ schema: str = proto.Field( @@ -144,6 +157,116 @@ class SchemaSettings(proto.Message): ) +class IngestionDataSourceSettings(proto.Message): + r"""Settings for an ingestion data source on a topic. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + aws_kinesis (google.pubsub_v1.types.IngestionDataSourceSettings.AwsKinesis): + Optional. Amazon Kinesis Data Streams. + + This field is a member of `oneof`_ ``source``. + """ + + class AwsKinesis(proto.Message): + r"""Ingestion settings for Amazon Kinesis Data Streams. + + Attributes: + state (google.pubsub_v1.types.IngestionDataSourceSettings.AwsKinesis.State): + Output only. An output-only field that + indicates the state of the Kinesis ingestion + source. + stream_arn (str): + Required. The Kinesis stream ARN to ingest + data from. + consumer_arn (str): + Required. The Kinesis consumer ARN to used + for ingestion in Enhanced Fan-Out mode. The + consumer must be already created and ready to be + used. + aws_role_arn (str): + Required. AWS role ARN to be used for + Federated Identity authentication with Kinesis. + Check the Pub/Sub docs for how to set up this + role and the required permissions that need to + be attached to it. + gcp_service_account (str): + Required. The GCP service account to be used for Federated + Identity authentication with Kinesis (via a + ``AssumeRoleWithWebIdentity`` call for the provided role). + The ``aws_role_arn`` must be set up with + ``accounts.google.com:sub`` equals to this service account + number. + """ + + class State(proto.Enum): + r"""Possible states for managed ingestion from Amazon Kinesis + Data Streams. + + Values: + STATE_UNSPECIFIED (0): + Default value. This value is unused. + ACTIVE (1): + Ingestion is active. + KINESIS_PERMISSION_DENIED (2): + Permission denied encountered while consuming data from + Kinesis. This can happen if: + + - The provided ``aws_role_arn`` does not exist or does not + have the appropriate permissions attached. + - The provided ``aws_role_arn`` is not set up properly for + Identity Federation using ``gcp_service_account``. + - The Pub/Sub SA is not granted the + ``iam.serviceAccounts.getOpenIdToken`` permission on + ``gcp_service_account``. + PUBLISH_PERMISSION_DENIED (3): + Permission denied encountered while publishing to the topic. + This can happen due to Pub/Sub SA has not been granted the + `appropriate publish + permissions `__ + STREAM_NOT_FOUND (4): + The Kinesis stream does not exist. + CONSUMER_NOT_FOUND (5): + The Kinesis consumer does not exist. + """ + STATE_UNSPECIFIED = 0 + ACTIVE = 1 + KINESIS_PERMISSION_DENIED = 2 + PUBLISH_PERMISSION_DENIED = 3 + STREAM_NOT_FOUND = 4 + CONSUMER_NOT_FOUND = 5 + + state: "IngestionDataSourceSettings.AwsKinesis.State" = proto.Field( + proto.ENUM, + number=1, + enum="IngestionDataSourceSettings.AwsKinesis.State", + ) + stream_arn: str = proto.Field( + proto.STRING, + number=2, + ) + consumer_arn: str = proto.Field( + proto.STRING, + number=3, + ) + aws_role_arn: str = proto.Field( + proto.STRING, + number=4, + ) + gcp_service_account: str = proto.Field( + proto.STRING, + number=5, + ) + + aws_kinesis: AwsKinesis = proto.Field( + proto.MESSAGE, + number=1, + oneof="source", + message=AwsKinesis, + ) + + class Topic(proto.Message): r"""A topic resource. @@ -158,30 +281,30 @@ class Topic(proto.Message): 255 characters in length, and it must not start with ``"goog"``. labels (MutableMapping[str, str]): - See [Creating and managing labels] + Optional. See [Creating and managing labels] (https://cloud.google.com/pubsub/docs/labels). message_storage_policy (google.pubsub_v1.types.MessageStoragePolicy): - Policy constraining the set of Google Cloud - Platform regions where messages published to the - topic may be stored. If not present, then no - constraints are in effect. + Optional. Policy constraining the set of + Google Cloud Platform regions where messages + published to the topic may be stored. If not + present, then no constraints are in effect. kms_key_name (str): - The resource name of the Cloud KMS CryptoKey to be used to - protect access to messages published on this topic. + Optional. The resource name of the Cloud KMS CryptoKey to be + used to protect access to messages published on this topic. The expected format is ``projects/*/locations/*/keyRings/*/cryptoKeys/*``. schema_settings (google.pubsub_v1.types.SchemaSettings): - Settings for validating messages published - against a schema. + Optional. Settings for validating messages + published against a schema. satisfies_pzs (bool): - Reserved for future use. This field is set - only in responses from the server; it is ignored - if it is set in any requests. + Optional. Reserved for future use. This field + is set only in responses from the server; it is + ignored if it is set in any requests. message_retention_duration (google.protobuf.duration_pb2.Duration): - Indicates the minimum duration to retain a message after it - is published to the topic. If this field is set, messages - published to the topic in the last + Optional. Indicates the minimum duration to retain a message + after it is published to the topic. If this field is set, + messages published to the topic in the last ``message_retention_duration`` are always available to subscribers. For instance, it allows any attached subscription to `seek to a @@ -190,8 +313,33 @@ class Topic(proto.Message): this field is not set, message retention is controlled by settings on individual subscriptions. Cannot be more than 31 days or less than 10 minutes. + state (google.pubsub_v1.types.Topic.State): + Output only. An output-only field indicating + the state of the topic. + ingestion_data_source_settings (google.pubsub_v1.types.IngestionDataSourceSettings): + Optional. Settings for managed ingestion from + a data source into this topic. """ + class State(proto.Enum): + r"""The state of the topic. + + Values: + STATE_UNSPECIFIED (0): + Default value. This value is unused. + ACTIVE (1): + The topic does not have any persistent + errors. + INGESTION_RESOURCE_ERROR (2): + Ingestion from the data source has + encountered a permanent error. See the more + detailed error state in the corresponding + ingestion source configuration. + """ + STATE_UNSPECIFIED = 0 + ACTIVE = 1 + INGESTION_RESOURCE_ERROR = 2 + name: str = proto.Field( proto.STRING, number=1, @@ -224,6 +372,16 @@ class Topic(proto.Message): number=8, message=duration_pb2.Duration, ) + state: State = proto.Field( + proto.ENUM, + number=9, + enum=State, + ) + ingestion_data_source_settings: "IngestionDataSourceSettings" = proto.Field( + proto.MESSAGE, + number=10, + message="IngestionDataSourceSettings", + ) class PubsubMessage(proto.Message): @@ -239,14 +397,14 @@ class PubsubMessage(proto.Message): Attributes: data (bytes): - The message data field. If this field is - empty, the message must contain at least one - attribute. + Optional. The message data field. If this + field is empty, the message must contain at + least one attribute. attributes (MutableMapping[str, str]): - Attributes for this message. If this field is - empty, the message must contain non-empty data. - This can be used to filter messages on the - subscription. + Optional. Attributes for this message. If + this field is empty, the message must contain + non-empty data. This can be used to filter + messages on the subscription. message_id (str): ID of this message, assigned by the server when the message is published. Guaranteed to be unique within the topic. This @@ -259,15 +417,15 @@ class PubsubMessage(proto.Message): the server when it receives the ``Publish`` call. It must not be populated by the publisher in a ``Publish`` call. ordering_key (str): - If non-empty, identifies related messages for which publish - order should be respected. If a ``Subscription`` has - ``enable_message_ordering`` set to ``true``, messages - published with the same non-empty ``ordering_key`` value - will be delivered to subscribers in the order in which they - are received by the Pub/Sub system. All ``PubsubMessage``\ s - published in a given ``PublishRequest`` must specify the - same ``ordering_key`` value. For more information, see - `ordering + Optional. If non-empty, identifies related messages for + which publish order should be respected. If a + ``Subscription`` has ``enable_message_ordering`` set to + ``true``, messages published with the same non-empty + ``ordering_key`` value will be delivered to subscribers in + the order in which they are received by the Pub/Sub system. + All ``PubsubMessage``\ s published in a given + ``PublishRequest`` must specify the same ``ordering_key`` + value. For more information, see `ordering messages `__. """ @@ -364,10 +522,10 @@ class PublishResponse(proto.Message): Attributes: message_ids (MutableSequence[str]): - The server-assigned ID of each published - message, in the same order as the messages in - the request. IDs are guaranteed to be unique - within the topic. + Optional. The server-assigned ID of each + published message, in the same order as the + messages in the request. IDs are guaranteed to + be unique within the topic. """ message_ids: MutableSequence[str] = proto.RepeatedField( @@ -384,12 +542,12 @@ class ListTopicsRequest(proto.Message): Required. The name of the project in which to list topics. Format is ``projects/{project-id}``. page_size (int): - Maximum number of topics to return. + Optional. Maximum number of topics to return. page_token (str): - The value returned by the last ``ListTopicsResponse``; - indicates that this is a continuation of a prior - ``ListTopics`` call, and that the system should return the - next page of data. + Optional. The value returned by the last + ``ListTopicsResponse``; indicates that this is a + continuation of a prior ``ListTopics`` call, and that the + system should return the next page of data. """ project: str = proto.Field( @@ -411,11 +569,11 @@ class ListTopicsResponse(proto.Message): Attributes: topics (MutableSequence[google.pubsub_v1.types.Topic]): - The resulting topics. + Optional. The resulting topics. next_page_token (str): - If not empty, indicates that there may be more topics that - match the request; this value should be passed in a new - ``ListTopicsRequest``. + Optional. If not empty, indicates that there may be more + topics that match the request; this value should be passed + in a new ``ListTopicsRequest``. """ @property @@ -442,10 +600,10 @@ class ListTopicSubscriptionsRequest(proto.Message): attached to. Format is ``projects/{project}/topics/{topic}``. page_size (int): - Maximum number of subscription names to - return. + Optional. Maximum number of subscription + names to return. page_token (str): - The value returned by the last + Optional. The value returned by the last ``ListTopicSubscriptionsResponse``; indicates that this is a continuation of a prior ``ListTopicSubscriptions`` call, and that the system should return the next page of data. @@ -470,12 +628,13 @@ class ListTopicSubscriptionsResponse(proto.Message): Attributes: subscriptions (MutableSequence[str]): - The names of subscriptions attached to the - topic specified in the request. + Optional. The names of subscriptions attached + to the topic specified in the request. next_page_token (str): - If not empty, indicates that there may be more subscriptions - that match the request; this value should be passed in a new - ``ListTopicSubscriptionsRequest`` to get more subscriptions. + Optional. If not empty, indicates that there may be more + subscriptions that match the request; this value should be + passed in a new ``ListTopicSubscriptionsRequest`` to get + more subscriptions. """ @property @@ -500,9 +659,10 @@ class ListTopicSnapshotsRequest(proto.Message): Required. The name of the topic that snapshots are attached to. Format is ``projects/{project}/topics/{topic}``. page_size (int): - Maximum number of snapshot names to return. + Optional. Maximum number of snapshot names to + return. page_token (str): - The value returned by the last + Optional. The value returned by the last ``ListTopicSnapshotsResponse``; indicates that this is a continuation of a prior ``ListTopicSnapshots`` call, and that the system should return the next page of data. @@ -527,12 +687,13 @@ class ListTopicSnapshotsResponse(proto.Message): Attributes: snapshots (MutableSequence[str]): - The names of the snapshots that match the - request. + Optional. The names of the snapshots that + match the request. next_page_token (str): - If not empty, indicates that there may be more snapshots - that match the request; this value should be passed in a new - ``ListTopicSnapshotsRequest`` to get more snapshots. + Optional. If not empty, indicates that there may be more + snapshots that match the request; this value should be + passed in a new ``ListTopicSnapshotsRequest`` to get more + snapshots. """ @property @@ -610,24 +771,25 @@ class Subscription(proto.Message): field will be ``_deleted-topic_`` if the topic has been deleted. push_config (google.pubsub_v1.types.PushConfig): - If push delivery is used with this + Optional. If push delivery is used with this subscription, this field is used to configure it. bigquery_config (google.pubsub_v1.types.BigQueryConfig): - If delivery to BigQuery is used with this - subscription, this field is used to configure - it. - cloud_storage_config (google.pubsub_v1.types.CloudStorageConfig): - If delivery to Google Cloud Storage is used + Optional. If delivery to BigQuery is used with this subscription, this field is used to configure it. + cloud_storage_config (google.pubsub_v1.types.CloudStorageConfig): + Optional. If delivery to Google Cloud Storage + is used with this subscription, this field is + used to configure it. ack_deadline_seconds (int): - The approximate amount of time (on a best-effort basis) - Pub/Sub waits for the subscriber to acknowledge receipt - before resending the message. In the interval after the - message is delivered and before it is acknowledged, it is - considered to be *outstanding*. During that time period, the - message will not be redelivered (on a best-effort basis). + Optional. The approximate amount of time (on a best-effort + basis) Pub/Sub waits for the subscriber to acknowledge + receipt before resending the message. In the interval after + the message is delivered and before it is acknowledged, it + is considered to be *outstanding*. During that time period, + the message will not be redelivered (on a best-effort + basis). For pull subscriptions, this value is used as the initial value for the ack deadline. To override this value for a @@ -645,15 +807,16 @@ class Subscription(proto.Message): If the subscriber never acknowledges the message, the Pub/Sub system will eventually redeliver the message. retain_acked_messages (bool): - Indicates whether to retain acknowledged messages. If true, - then messages are not expunged from the subscription's - backlog, even if they are acknowledged, until they fall out - of the ``message_retention_duration`` window. This must be - true if you would like to [``Seek`` to a timestamp] + Optional. Indicates whether to retain acknowledged messages. + If true, then messages are not expunged from the + subscription's backlog, even if they are acknowledged, until + they fall out of the ``message_retention_duration`` window. + This must be true if you would like to [``Seek`` to a + timestamp] (https://cloud.google.com/pubsub/docs/replay-overview#seek_to_a_time) in the past to replay previously-acknowledged messages. message_retention_duration (google.protobuf.duration_pb2.Duration): - How long to retain unacknowledged messages in the + Optional. How long to retain unacknowledged messages in the subscription's backlog, from the moment a message is published. If ``retain_acked_messages`` is true, then this also configures the retention of acknowledged messages, and @@ -661,15 +824,16 @@ class Subscription(proto.Message): Defaults to 7 days. Cannot be more than 7 days or less than 10 minutes. labels (MutableMapping[str, str]): - See `Creating and managing + Optional. See `Creating and managing labels `__. enable_message_ordering (bool): - If true, messages published with the same ``ordering_key`` - in ``PubsubMessage`` will be delivered to the subscribers in - the order in which they are received by the Pub/Sub system. - Otherwise, they may be delivered in any order. + Optional. If true, messages published with the same + ``ordering_key`` in ``PubsubMessage`` will be delivered to + the subscribers in the order in which they are received by + the Pub/Sub system. Otherwise, they may be delivered in any + order. expiration_policy (google.pubsub_v1.types.ExpirationPolicy): - A policy that specifies the conditions for this + Optional. A policy that specifies the conditions for this subscription's expiration. A subscription is considered active as long as any connected subscriber is successfully consuming messages from the subscription or is issuing @@ -680,25 +844,25 @@ class Subscription(proto.Message): is set, but ``expiration_policy.ttl`` is not set, the subscription never expires. filter (str): - An expression written in the Pub/Sub `filter + Optional. An expression written in the Pub/Sub `filter language `__. If non-empty, then only ``PubsubMessage``\ s whose ``attributes`` field matches the filter are delivered on this subscription. If empty, then no messages are filtered out. dead_letter_policy (google.pubsub_v1.types.DeadLetterPolicy): - A policy that specifies the conditions for dead lettering - messages in this subscription. If dead_letter_policy is not - set, dead lettering is disabled. + Optional. A policy that specifies the conditions for dead + lettering messages in this subscription. If + dead_letter_policy is not set, dead lettering is disabled. - The Cloud Pub/Sub service account associated with this + The Pub/Sub service account associated with this subscriptions's parent project (i.e., service-{project_number}@gcp-sa-pubsub.iam.gserviceaccount.com) must have permission to Acknowledge() messages on this subscription. retry_policy (google.pubsub_v1.types.RetryPolicy): - A policy that specifies how Pub/Sub retries - message delivery for this subscription. + Optional. A policy that specifies how Pub/Sub + retries message delivery for this subscription. If not set, the default retry policy is applied. This generally implies that messages will be @@ -707,16 +871,16 @@ class Subscription(proto.Message): NACKs or acknowledgement deadline exceeded events for a given message. detached (bool): - Indicates whether the subscription is detached from its - topic. Detached subscriptions don't receive messages from - their topic and don't retain any backlog. ``Pull`` and - ``StreamingPull`` requests will return FAILED_PRECONDITION. - If the subscription is a push subscription, pushes to the - endpoint will not be made. + Optional. Indicates whether the subscription is detached + from its topic. Detached subscriptions don't receive + messages from their topic and don't retain any backlog. + ``Pull`` and ``StreamingPull`` requests will return + FAILED_PRECONDITION. If the subscription is a push + subscription, pushes to the endpoint will not be made. enable_exactly_once_delivery (bool): - If true, Pub/Sub provides the following guarantees for the - delivery of a message with a given value of ``message_id`` - on this subscription: + Optional. If true, Pub/Sub provides the following guarantees + for the delivery of a message with a given value of + ``message_id`` on this subscription: - The message sent to a subscriber is guaranteed not to be resent before the message's acknowledgement deadline @@ -848,7 +1012,7 @@ class State(proto.Enum): class RetryPolicy(proto.Message): - r"""A policy that specifies how Cloud Pub/Sub retries message delivery. + r"""A policy that specifies how Pub/Sub retries message delivery. Retry delay will be exponential based on provided minimum and maximum backoffs. https://en.wikipedia.org/wiki/Exponential_backoff. @@ -863,15 +1027,15 @@ class RetryPolicy(proto.Message): Attributes: minimum_backoff (google.protobuf.duration_pb2.Duration): - The minimum delay between consecutive - deliveries of a given message. Value should be - between 0 and 600 seconds. Defaults to 10 - seconds. + Optional. The minimum delay between + consecutive deliveries of a given message. Value + should be between 0 and 600 seconds. Defaults to + 10 seconds. maximum_backoff (google.protobuf.duration_pb2.Duration): - The maximum delay between consecutive - deliveries of a given message. Value should be - between 0 and 600 seconds. Defaults to 600 - seconds. + Optional. The maximum delay between + consecutive deliveries of a given message. Value + should be between 0 and 600 seconds. Defaults to + 600 seconds. """ minimum_backoff: duration_pb2.Duration = proto.Field( @@ -896,11 +1060,11 @@ class DeadLetterPolicy(proto.Message): Attributes: dead_letter_topic (str): - The name of the topic to which dead letter messages should - be published. Format is - ``projects/{project}/topics/{topic}``.The Cloud Pub/Sub - service account associated with the enclosing subscription's - parent project (i.e., + Optional. The name of the topic to which dead letter + messages should be published. Format is + ``projects/{project}/topics/{topic}``.The Pub/Sub service + account associated with the enclosing subscription's parent + project (i.e., service-{project_number}@gcp-sa-pubsub.iam.gserviceaccount.com) must have permission to Publish() to this topic. @@ -909,8 +1073,8 @@ class DeadLetterPolicy(proto.Message): topic since messages published to a topic with no subscriptions are lost. max_delivery_attempts (int): - The maximum number of delivery attempts for any message. The - value must be between 5 and 100. + Optional. The maximum number of delivery attempts for any + message. The value must be between 5 and 100. The number of delivery attempts is defined as 1 + (the sum of number of NACKs and number of times the acknowledgement @@ -941,12 +1105,12 @@ class ExpirationPolicy(proto.Message): Attributes: ttl (google.protobuf.duration_pb2.Duration): - Specifies the "time-to-live" duration for an associated - resource. The resource expires if it is not active for a - period of ``ttl``. The definition of "activity" depends on - the type of the associated resource. The minimum and maximum - allowed values for ``ttl`` depend on the type of the - associated resource, as well. If ``ttl`` is not set, the + Optional. Specifies the "time-to-live" duration for an + associated resource. The resource expires if it is not + active for a period of ``ttl``. The definition of "activity" + depends on the type of the associated resource. The minimum + and maximum allowed values for ``ttl`` depend on the type of + the associated resource, as well. If ``ttl`` is not set, the associated resource never expires. """ @@ -969,12 +1133,12 @@ class PushConfig(proto.Message): Attributes: push_endpoint (str): - A URL locating the endpoint to which messages should be - pushed. For example, a Webhook endpoint might use + Optional. A URL locating the endpoint to which messages + should be pushed. For example, a Webhook endpoint might use ``https://example.com/push``. attributes (MutableMapping[str, str]): - Endpoint configuration attributes that can be used to - control different aspects of the message delivery. + Optional. Endpoint configuration attributes that can be used + to control different aspects of the message delivery. The only currently supported attribute is ``x-goog-version``, which you can use to change the format @@ -999,21 +1163,21 @@ class PushConfig(proto.Message): For example: ``attributes { "x-goog-version": "v1" }`` oidc_token (google.pubsub_v1.types.PushConfig.OidcToken): - If specified, Pub/Sub will generate and attach an OIDC JWT - token as an ``Authorization`` header in the HTTP request for - every pushed message. + Optional. If specified, Pub/Sub will generate and attach an + OIDC JWT token as an ``Authorization`` header in the HTTP + request for every pushed message. This field is a member of `oneof`_ ``authentication_method``. pubsub_wrapper (google.pubsub_v1.types.PushConfig.PubsubWrapper): - When set, the payload to the push endpoint is - in the form of the JSON representation of a - PubsubMessage + Optional. When set, the payload to the push + endpoint is in the form of the JSON + representation of a PubsubMessage (https://cloud.google.com/pubsub/docs/reference/rpc/google.pubsub.v1#pubsubmessage). This field is a member of `oneof`_ ``wrapper``. no_wrapper (google.pubsub_v1.types.PushConfig.NoWrapper): - When set, the payload to the push endpoint is - not wrapped. + Optional. When set, the payload to the push + endpoint is not wrapped. This field is a member of `oneof`_ ``wrapper``. """ @@ -1024,19 +1188,20 @@ class OidcToken(proto.Message): Attributes: service_account_email (str): - `Service account + Optional. `Service account email `__ used for generating the OIDC token. For more information on setting up authentication, see `Push subscriptions `__. audience (str): - Audience to be used when generating OIDC - token. The audience claim identifies the + Optional. Audience to be used when generating + OIDC token. The audience claim identifies the recipients that the JWT is intended for. The audience value is a single case-sensitive string. Having multiple values (array) for the audience field is not supported. More info about the OIDC JWT token audience here: + https://tools.ietf.org/html/rfc7519#section-4.1.3 Note: if not specified, the Push endpoint URL will be used. @@ -1063,7 +1228,7 @@ class NoWrapper(proto.Message): Attributes: write_metadata (bool): - When true, writes the Pub/Sub message metadata to + Optional. When true, writes the Pub/Sub message metadata to ``x-goog-pubsub-:`` headers of the HTTP request. Writes the Pub/Sub message attributes to ``:`` headers of the HTTP request. @@ -1108,23 +1273,24 @@ class BigQueryConfig(proto.Message): Attributes: table (str): - The name of the table to which to write data, - of the form {projectId}.{datasetId}.{tableId} + Optional. The name of the table to which to + write data, of the form + {projectId}.{datasetId}.{tableId} use_topic_schema (bool): Optional. When true, use the topic's schema as the columns to write to in BigQuery, if it exists. ``use_topic_schema`` and ``use_table_schema`` cannot be enabled at the same time. write_metadata (bool): - When true, write the subscription name, message_id, - publish_time, attributes, and ordering_key to additional - columns in the table. The subscription name, message_id, and - publish_time fields are put in their own columns while all - other message properties (other than data) are written to a - JSON object in the attributes column. + Optional. When true, write the subscription name, + message_id, publish_time, attributes, and ordering_key to + additional columns in the table. The subscription name, + message_id, and publish_time fields are put in their own + columns while all other message properties (other than data) + are written to a JSON object in the attributes column. drop_unknown_fields (bool): - When true and use_topic_schema is true, any fields that are - a part of the topic schema that are not part of the BigQuery - table schema are dropped when writing to BigQuery. + Optional. When true and use_topic_schema is true, any fields + that are a part of the topic schema that are not part of the + BigQuery table schema are dropped when writing to BigQuery. Otherwise, the schemas must be kept in sync and any messages with extra fields are not written and remain in the subscription's backlog. @@ -1163,12 +1329,17 @@ class State(proto.Enum): SCHEMA_MISMATCH (4): Cannot write to the BigQuery table due to a schema mismatch. + IN_TRANSIT_LOCATION_RESTRICTION (5): + Cannot write to the destination because enforce_in_transit + is set to true and the destination locations are not in the + allowed regions. """ STATE_UNSPECIFIED = 0 ACTIVE = 1 PERMISSION_DENIED = 2 NOT_FOUND = 3 SCHEMA_MISMATCH = 4 + IN_TRANSIT_LOCATION_RESTRICTION = 5 table: str = proto.Field( proto.STRING, @@ -1215,35 +1386,35 @@ class CloudStorageConfig(proto.Message): requirements] (https://cloud.google.com/storage/docs/buckets#naming). filename_prefix (str): - User-provided prefix for Cloud Storage filename. See the - `object naming + Optional. User-provided prefix for Cloud Storage filename. + See the `object naming requirements `__. filename_suffix (str): - User-provided suffix for Cloud Storage filename. See the - `object naming + Optional. User-provided suffix for Cloud Storage filename. + See the `object naming requirements `__. Must not end in "/". text_config (google.pubsub_v1.types.CloudStorageConfig.TextConfig): - If set, message data will be written to Cloud - Storage in text format. + Optional. If set, message data will be + written to Cloud Storage in text format. This field is a member of `oneof`_ ``output_format``. avro_config (google.pubsub_v1.types.CloudStorageConfig.AvroConfig): - If set, message data will be written to Cloud - Storage in Avro format. + Optional. If set, message data will be + written to Cloud Storage in Avro format. This field is a member of `oneof`_ ``output_format``. max_duration (google.protobuf.duration_pb2.Duration): - The maximum duration that can elapse before a - new Cloud Storage file is created. Min 1 minute, - max 10 minutes, default 5 minutes. May not - exceed the subscription's acknowledgement - deadline. + Optional. The maximum duration that can + elapse before a new Cloud Storage file is + created. Min 1 minute, max 10 minutes, default 5 + minutes. May not exceed the subscription's + acknowledgement deadline. max_bytes (int): - The maximum bytes that can be written to a Cloud Storage - file before a new file is created. Min 1 KB, max 10 GiB. The - max_bytes limit may be exceeded in cases where messages are - larger than the limit. + Optional. The maximum bytes that can be written to a Cloud + Storage file before a new file is created. Min 1 KB, max 10 + GiB. The max_bytes limit may be exceeded in cases where + messages are larger than the limit. state (google.pubsub_v1.types.CloudStorageConfig.State): Output only. An output-only field that indicates whether or not the subscription can @@ -1265,11 +1436,16 @@ class State(proto.Enum): NOT_FOUND (3): Cannot write to the Cloud Storage bucket because it does not exist. + IN_TRANSIT_LOCATION_RESTRICTION (4): + Cannot write to the destination because enforce_in_transit + is set to true and the destination locations are not in the + allowed regions. """ STATE_UNSPECIFIED = 0 ACTIVE = 1 PERMISSION_DENIED = 2 NOT_FOUND = 3 + IN_TRANSIT_LOCATION_RESTRICTION = 4 class TextConfig(proto.Message): r"""Configuration for writing message data in text format. @@ -1285,13 +1461,13 @@ class AvroConfig(proto.Message): Attributes: write_metadata (bool): - When true, write the subscription name, message_id, - publish_time, attributes, and ordering_key as additional - fields in the output. The subscription name, message_id, and - publish_time fields are put in their own fields while all - other message properties other than data (for example, an - ordering_key, if present) are added as entries in the - attributes map. + Optional. When true, write the subscription name, + message_id, publish_time, attributes, and ordering_key as + additional fields in the output. The subscription name, + message_id, and publish_time fields are put in their own + fields while all other message properties other than data + (for example, an ordering_key, if present) are added as + entries in the attributes map. """ write_metadata: bool = proto.Field( @@ -1344,12 +1520,12 @@ class ReceivedMessage(proto.Message): Attributes: ack_id (str): - This ID can be used to acknowledge the - received message. + Optional. This ID can be used to acknowledge + the received message. message (google.pubsub_v1.types.PubsubMessage): - The message. + Optional. The message. delivery_attempt (int): - The approximate number of times that Cloud Pub/Sub has + Optional. The approximate number of times that Pub/Sub has attempted to deliver the associated message to a subscriber. More precisely, this is 1 + (number of NACKs) + (number of @@ -1431,9 +1607,10 @@ class ListSubscriptionsRequest(proto.Message): Required. The name of the project in which to list subscriptions. Format is ``projects/{project-id}``. page_size (int): - Maximum number of subscriptions to return. + Optional. Maximum number of subscriptions to + return. page_token (str): - The value returned by the last + Optional. The value returned by the last ``ListSubscriptionsResponse``; indicates that this is a continuation of a prior ``ListSubscriptions`` call, and that the system should return the next page of data. @@ -1458,11 +1635,13 @@ class ListSubscriptionsResponse(proto.Message): Attributes: subscriptions (MutableSequence[google.pubsub_v1.types.Subscription]): - The subscriptions that match the request. + Optional. The subscriptions that match the + request. next_page_token (str): - If not empty, indicates that there may be more subscriptions - that match the request; this value should be passed in a new - ``ListSubscriptionsRequest`` to get more subscriptions. + Optional. If not empty, indicates that there may be more + subscriptions that match the request; this value should be + passed in a new ``ListSubscriptionsRequest`` to get more + subscriptions. """ @property @@ -1566,12 +1745,12 @@ class PullResponse(proto.Message): Attributes: received_messages (MutableSequence[google.pubsub_v1.types.ReceivedMessage]): - Received Pub/Sub messages. The list will be empty if there - are no more messages available in the backlog, or if no - messages could be returned before the request timeout. For - JSON, the response can be entirely empty. The Pub/Sub system - may return fewer than the ``maxMessages`` requested even if - there are more messages available in the backlog. + Optional. Received Pub/Sub messages. The list will be empty + if there are no more messages available in the backlog, or + if no messages could be returned before the request timeout. + For JSON, the response can be entirely empty. The Pub/Sub + system may return fewer than the ``maxMessages`` requested + even if there are more messages available in the backlog. """ received_messages: MutableSequence["ReceivedMessage"] = proto.RepeatedField( @@ -1599,8 +1778,8 @@ class ModifyAckDeadlineRequest(proto.Message): delivery to another subscriber client. This typically results in an increase in the rate of message redeliveries (that is, duplicates). The minimum deadline you can specify - is 0 seconds. The maximum deadline you can specify is 600 - seconds (10 minutes). + is 0 seconds. The maximum deadline you can specify in a + single request is 600 seconds (10 minutes). """ subscription: str = proto.Field( @@ -1655,18 +1834,18 @@ class StreamingPullRequest(proto.Message): client to server. Format is ``projects/{project}/subscriptions/{sub}``. ack_ids (MutableSequence[str]): - List of acknowledgement IDs for acknowledging previously - received messages (received on this stream or a different - stream). If an ack ID has expired, the corresponding message - may be redelivered later. Acknowledging a message more than - once will not result in an error. If the acknowledgement ID - is malformed, the stream will be aborted with status - ``INVALID_ARGUMENT``. + Optional. List of acknowledgement IDs for acknowledging + previously received messages (received on this stream or a + different stream). If an ack ID has expired, the + corresponding message may be redelivered later. + Acknowledging a message more than once will not result in an + error. If the acknowledgement ID is malformed, the stream + will be aborted with status ``INVALID_ARGUMENT``. modify_deadline_seconds (MutableSequence[int]): - The list of new ack deadlines for the IDs listed in - ``modify_deadline_ack_ids``. The size of this list must be - the same as the size of ``modify_deadline_ack_ids``. If it - differs the stream will be aborted with + Optional. The list of new ack deadlines for the IDs listed + in ``modify_deadline_ack_ids``. The size of this list must + be the same as the size of ``modify_deadline_ack_ids``. If + it differs the stream will be aborted with ``INVALID_ARGUMENT``. Each element in this list is applied to the element in the same position in ``modify_deadline_ack_ids``. The new ack deadline is with @@ -1678,8 +1857,8 @@ class StreamingPullRequest(proto.Message): request. If the value is < 0 (an error), the stream will be aborted with status ``INVALID_ARGUMENT``. modify_deadline_ack_ids (MutableSequence[str]): - List of acknowledgement IDs whose deadline will be modified - based on the corresponding element in + Optional. List of acknowledgement IDs whose deadline will be + modified based on the corresponding element in ``modify_deadline_seconds``. This field can be used to indicate that more time is needed to process a message by the subscriber, or to make the message available for @@ -1693,36 +1872,37 @@ class StreamingPullRequest(proto.Message): 10 seconds. The maximum deadline you can specify is 600 seconds (10 minutes). client_id (str): - A unique identifier that is used to distinguish client - instances from each other. Only needs to be provided on the - initial request. When a stream disconnects and reconnects - for the same stream, the client_id should be set to the same - value so that state associated with the old stream can be - transferred to the new stream. The same client_id should not - be used for different client instances. + Optional. A unique identifier that is used to distinguish + client instances from each other. Only needs to be provided + on the initial request. When a stream disconnects and + reconnects for the same stream, the client_id should be set + to the same value so that state associated with the old + stream can be transferred to the new stream. The same + client_id should not be used for different client instances. max_outstanding_messages (int): - Flow control settings for the maximum number of outstanding - messages. When there are ``max_outstanding_messages`` or - more currently sent to the streaming pull client that have - not yet been acked or nacked, the server stops sending more - messages. The sending of messages resumes once the number of - outstanding messages is less than this value. If the value - is <= 0, there is no limit to the number of outstanding - messages. This property can only be set on the initial - StreamingPullRequest. If it is set on a subsequent request, - the stream will be aborted with status ``INVALID_ARGUMENT``. - max_outstanding_bytes (int): - Flow control settings for the maximum number of outstanding - bytes. When there are ``max_outstanding_bytes`` or more - worth of messages currently sent to the streaming pull - client that have not yet been acked or nacked, the server - will stop sending more messages. The sending of messages - resumes once the number of outstanding bytes is less than + Optional. Flow control settings for the maximum number of + outstanding messages. When there are + ``max_outstanding_messages`` currently sent to the streaming + pull client that have not yet been acked or nacked, the + server stops sending more messages. The sending of messages + resumes once the number of outstanding messages is less than this value. If the value is <= 0, there is no limit to the - number of outstanding bytes. This property can only be set - on the initial StreamingPullRequest. If it is set on a + number of outstanding messages. This property can only be + set on the initial StreamingPullRequest. If it is set on a subsequent request, the stream will be aborted with status ``INVALID_ARGUMENT``. + max_outstanding_bytes (int): + Optional. Flow control settings for the maximum number of + outstanding bytes. When there are ``max_outstanding_bytes`` + or more worth of messages currently sent to the streaming + pull client that have not yet been acked or nacked, the + server will stop sending more messages. The sending of + messages resumes once the number of outstanding bytes is + less than this value. If the value is <= 0, there is no + limit to the number of outstanding bytes. This property can + only be set on the initial StreamingPullRequest. If it is + set on a subsequent request, the stream will be aborted with + status ``INVALID_ARGUMENT``. """ subscription: str = proto.Field( @@ -1765,16 +1945,17 @@ class StreamingPullResponse(proto.Message): Attributes: received_messages (MutableSequence[google.pubsub_v1.types.ReceivedMessage]): - Received Pub/Sub messages. This will not be - empty. + Optional. Received Pub/Sub messages. This + will not be empty. acknowledge_confirmation (google.pubsub_v1.types.StreamingPullResponse.AcknowledgeConfirmation): - This field will only be set if + Optional. This field will only be set if ``enable_exactly_once_delivery`` is set to ``true``. modify_ack_deadline_confirmation (google.pubsub_v1.types.StreamingPullResponse.ModifyAckDeadlineConfirmation): - This field will only be set if + Optional. This field will only be set if ``enable_exactly_once_delivery`` is set to ``true``. subscription_properties (google.pubsub_v1.types.StreamingPullResponse.SubscriptionProperties): - Properties associated with this subscription. + Optional. Properties associated with this + subscription. """ class AcknowledgeConfirmation(proto.Message): @@ -1783,17 +1964,18 @@ class AcknowledgeConfirmation(proto.Message): Attributes: ack_ids (MutableSequence[str]): - Successfully processed acknowledgement IDs. + Optional. Successfully processed + acknowledgement IDs. invalid_ack_ids (MutableSequence[str]): - List of acknowledgement IDs that were - malformed or whose acknowledgement deadline has - expired. + Optional. List of acknowledgement IDs that + were malformed or whose acknowledgement deadline + has expired. unordered_ack_ids (MutableSequence[str]): - List of acknowledgement IDs that were out of - order. + Optional. List of acknowledgement IDs that + were out of order. temporary_failed_ack_ids (MutableSequence[str]): - List of acknowledgement IDs that failed - processing with temporary issues. + Optional. List of acknowledgement IDs that + failed processing with temporary issues. """ ack_ids: MutableSequence[str] = proto.RepeatedField( @@ -1819,14 +2001,15 @@ class ModifyAckDeadlineConfirmation(proto.Message): Attributes: ack_ids (MutableSequence[str]): - Successfully processed acknowledgement IDs. + Optional. Successfully processed + acknowledgement IDs. invalid_ack_ids (MutableSequence[str]): - List of acknowledgement IDs that were - malformed or whose acknowledgement deadline has - expired. + Optional. List of acknowledgement IDs that + were malformed or whose acknowledgement deadline + has expired. temporary_failed_ack_ids (MutableSequence[str]): - List of acknowledgement IDs that failed - processing with temporary issues. + Optional. List of acknowledgement IDs that + failed processing with temporary issues. """ ack_ids: MutableSequence[str] = proto.RepeatedField( @@ -1847,11 +2030,11 @@ class SubscriptionProperties(proto.Message): Attributes: exactly_once_delivery_enabled (bool): - True iff exactly once delivery is enabled for - this subscription. + Optional. True iff exactly once delivery is + enabled for this subscription. message_ordering_enabled (bool): - True iff message ordering is enabled for this - subscription. + Optional. True iff message ordering is + enabled for this subscription. """ exactly_once_delivery_enabled: bool = proto.Field( @@ -1909,7 +2092,7 @@ class CreateSnapshotRequest(proto.Message): CreateSnapshot request. Format is ``projects/{project}/subscriptions/{sub}``. labels (MutableMapping[str, str]): - See `Creating and managing + Optional. See `Creating and managing labels `__. """ @@ -1961,16 +2144,16 @@ class Snapshot(proto.Message): Attributes: name (str): - The name of the snapshot. + Optional. The name of the snapshot. topic (str): - The name of the topic from which this - snapshot is retaining messages. + Optional. The name of the topic from which + this snapshot is retaining messages. expire_time (google.protobuf.timestamp_pb2.Timestamp): - The snapshot is guaranteed to exist up until this time. A - newly-created snapshot expires no later than 7 days from the - time of its creation. Its exact lifetime is determined at - creation by the existing backlog in the source subscription. - Specifically, the lifetime of the snapshot is + Optional. The snapshot is guaranteed to exist up until this + time. A newly-created snapshot expires no later than 7 days + from the time of its creation. Its exact lifetime is + determined at creation by the existing backlog in the source + subscription. Specifically, the lifetime of the snapshot is ``7 days - (age of oldest unacked message in the subscription)``. For example, consider a subscription whose oldest unacked message is 3 days old. If a snapshot is created from this @@ -1980,7 +2163,7 @@ class Snapshot(proto.Message): snapshot that would expire in less than 1 hour after creation. labels (MutableMapping[str, str]): - See [Creating and managing labels] + Optional. See [Creating and managing labels] (https://cloud.google.com/pubsub/docs/labels). """ @@ -2027,12 +2210,13 @@ class ListSnapshotsRequest(proto.Message): Required. The name of the project in which to list snapshots. Format is ``projects/{project-id}``. page_size (int): - Maximum number of snapshots to return. + Optional. Maximum number of snapshots to + return. page_token (str): - The value returned by the last ``ListSnapshotsResponse``; - indicates that this is a continuation of a prior - ``ListSnapshots`` call, and that the system should return - the next page of data. + Optional. The value returned by the last + ``ListSnapshotsResponse``; indicates that this is a + continuation of a prior ``ListSnapshots`` call, and that the + system should return the next page of data. """ project: str = proto.Field( @@ -2054,11 +2238,11 @@ class ListSnapshotsResponse(proto.Message): Attributes: snapshots (MutableSequence[google.pubsub_v1.types.Snapshot]): - The resulting snapshots. + Optional. The resulting snapshots. next_page_token (str): - If not empty, indicates that there may be more snapshot that - match the request; this value should be passed in a new - ``ListSnapshotsRequest``. + Optional. If not empty, indicates that there may be more + snapshot that match the request; this value should be passed + in a new ``ListSnapshotsRequest``. """ @property @@ -2105,13 +2289,13 @@ class SeekRequest(proto.Message): subscription (str): Required. The subscription to affect. time (google.protobuf.timestamp_pb2.Timestamp): - The time to seek to. Messages retained in the subscription - that were published before this time are marked as - acknowledged, and messages retained in the subscription that - were published after this time are marked as unacknowledged. - Note that this operation affects only those messages - retained in the subscription (configured by the combination - of ``message_retention_duration`` and + Optional. The time to seek to. Messages retained in the + subscription that were published before this time are marked + as acknowledged, and messages retained in the subscription + that were published after this time are marked as + unacknowledged. Note that this operation affects only those + messages retained in the subscription (configured by the + combination of ``message_retention_duration`` and ``retain_acked_messages``). For example, if ``time`` corresponds to a point before the message retention window (or to a point before the system's notion of the @@ -2121,8 +2305,8 @@ class SeekRequest(proto.Message): This field is a member of `oneof`_ ``target``. snapshot (str): - The snapshot to seek to. The snapshot's topic must be the - same as that of the provided subscription. Format is + Optional. The snapshot to seek to. The snapshot's topic must + be the same as that of the provided subscription. Format is ``projects/{project}/snapshots/{snap}``. This field is a member of `oneof`_ ``target``. diff --git a/google/pubsub_v1/types/schema.py b/google/pubsub_v1/types/schema.py index ff1d22770..11853406d 100644 --- a/google/pubsub_v1/types/schema.py +++ b/google/pubsub_v1/types/schema.py @@ -161,7 +161,7 @@ class CreateSchemaRequest(proto.Message): component of the schema's resource name. See - https://cloud.google.com/pubsub/docs/admin#resource_names + https://cloud.google.com/pubsub/docs/pubsub-basics#resource_names for resource name constraints. """ diff --git a/owlbot.py b/owlbot.py index 0483b04df..c156909ff 100644 --- a/owlbot.py +++ b/owlbot.py @@ -101,7 +101,7 @@ count = s.replace( clients_to_patch, - r"Transport = type\(self\)\.get_transport_class\(transport\)", + r"Transport = type\(self\)\.get_transport_class\(cast\(str, transport\)\)", """\g<0> emulator_host = os.environ.get("PUBSUB_EMULATOR_HOST") @@ -324,7 +324,7 @@ if count < 1: raise Exception(".coveragerc replacement failed.") - s.move([library], excludes=["**/gapic_version.py", "README.rst", "docs/**/*", "setup.py", "testing/constraints-3.7.txt"]) + s.move([library], excludes=["**/gapic_version.py", "README.rst", "docs/**/*", "setup.py", "testing/constraints-3.7.txt", "testing/constraints-3.8.txt"]) s.remove_staging_dirs() # ---------------------------------------------------------------------------- diff --git a/scripts/fixup_pubsub_v1_keywords.py b/scripts/fixup_pubsub_v1_keywords.py index 8a294ba36..d8379e9bb 100644 --- a/scripts/fixup_pubsub_v1_keywords.py +++ b/scripts/fixup_pubsub_v1_keywords.py @@ -44,7 +44,7 @@ class pubsubCallTransformer(cst.CSTTransformer): 'create_schema': ('parent', 'schema', 'schema_id', ), 'create_snapshot': ('name', 'subscription', 'labels', ), 'create_subscription': ('name', 'topic', 'push_config', 'bigquery_config', 'cloud_storage_config', 'ack_deadline_seconds', 'retain_acked_messages', 'message_retention_duration', 'labels', 'enable_message_ordering', 'expiration_policy', 'filter', 'dead_letter_policy', 'retry_policy', 'detached', 'enable_exactly_once_delivery', 'topic_message_retention_duration', 'state', ), - 'create_topic': ('name', 'labels', 'message_storage_policy', 'kms_key_name', 'schema_settings', 'satisfies_pzs', 'message_retention_duration', ), + 'create_topic': ('name', 'labels', 'message_storage_policy', 'kms_key_name', 'schema_settings', 'satisfies_pzs', 'message_retention_duration', 'state', 'ingestion_data_source_settings', ), 'delete_schema': ('name', ), 'delete_schema_revision': ('name', 'revision_id', ), 'delete_snapshot': ('snapshot', ), diff --git a/testing/constraints-3.8.txt b/testing/constraints-3.8.txt index ad3f0fa58..30520e2d0 100644 --- a/testing/constraints-3.8.txt +++ b/testing/constraints-3.8.txt @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # This constraints file is required for unit tests. # List all library dependencies and extras in this file. -google-api-core +google-api-core==1.34.0 proto-plus protobuf grpc-google-iam-v1 diff --git a/tests/unit/gapic/pubsub_v1/test_publisher.py b/tests/unit/gapic/pubsub_v1/test_publisher.py index 6511e2f8b..aca4c59dd 100644 --- a/tests/unit/gapic/pubsub_v1/test_publisher.py +++ b/tests/unit/gapic/pubsub_v1/test_publisher.py @@ -29,6 +29,7 @@ import json import math import pytest +from google.api_core import api_core_version from proto.marshal.rules.dates import DurationRule, TimestampRule from proto.marshal.rules import wrappers from requests import Response @@ -75,6 +76,17 @@ def modify_default_endpoint(client): ) +# If default endpoint template is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint template so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint_template(client): + return ( + "test.{UNIVERSE_DOMAIN}" + if ("localhost" in client._DEFAULT_ENDPOINT_TEMPLATE) + else client._DEFAULT_ENDPOINT_TEMPLATE + ) + + def test__get_default_mtls_endpoint(): api_endpoint = "example.googleapis.com" api_mtls_endpoint = "example.mtls.googleapis.com" @@ -99,6 +111,254 @@ def test__get_default_mtls_endpoint(): assert PublisherClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi +def test__read_environment_variables(): + assert PublisherClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + assert PublisherClient._read_environment_variables() == (True, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"}): + assert PublisherClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} + ): + with pytest.raises(ValueError) as excinfo: + PublisherClient._read_environment_variables() + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + assert PublisherClient._read_environment_variables() == (False, "never", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + assert PublisherClient._read_environment_variables() == (False, "always", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}): + assert PublisherClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + PublisherClient._read_environment_variables() + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + ) + + with mock.patch.dict(os.environ, {"GOOGLE_CLOUD_UNIVERSE_DOMAIN": "foo.com"}): + assert PublisherClient._read_environment_variables() == ( + False, + "auto", + "foo.com", + ) + + +def test__get_client_cert_source(): + mock_provided_cert_source = mock.Mock() + mock_default_cert_source = mock.Mock() + + assert PublisherClient._get_client_cert_source(None, False) is None + assert ( + PublisherClient._get_client_cert_source(mock_provided_cert_source, False) + is None + ) + assert ( + PublisherClient._get_client_cert_source(mock_provided_cert_source, True) + == mock_provided_cert_source + ) + + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", return_value=True + ): + with mock.patch( + "google.auth.transport.mtls.default_client_cert_source", + return_value=mock_default_cert_source, + ): + assert ( + PublisherClient._get_client_cert_source(None, True) + is mock_default_cert_source + ) + assert ( + PublisherClient._get_client_cert_source( + mock_provided_cert_source, "true" + ) + is mock_provided_cert_source + ) + + +@mock.patch.object( + PublisherClient, + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(PublisherClient), +) +@mock.patch.object( + PublisherAsyncClient, + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(PublisherAsyncClient), +) +def test__get_api_endpoint(): + api_override = "foo.com" + mock_client_cert_source = mock.Mock() + default_universe = PublisherClient._DEFAULT_UNIVERSE + default_endpoint = PublisherClient._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=default_universe + ) + mock_universe = "bar.com" + mock_endpoint = PublisherClient._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=mock_universe + ) + + assert ( + PublisherClient._get_api_endpoint( + api_override, mock_client_cert_source, default_universe, "always" + ) + == api_override + ) + assert ( + PublisherClient._get_api_endpoint( + None, mock_client_cert_source, default_universe, "auto" + ) + == PublisherClient.DEFAULT_MTLS_ENDPOINT + ) + assert ( + PublisherClient._get_api_endpoint(None, None, default_universe, "auto") + == default_endpoint + ) + assert ( + PublisherClient._get_api_endpoint(None, None, default_universe, "always") + == PublisherClient.DEFAULT_MTLS_ENDPOINT + ) + assert ( + PublisherClient._get_api_endpoint( + None, mock_client_cert_source, default_universe, "always" + ) + == PublisherClient.DEFAULT_MTLS_ENDPOINT + ) + assert ( + PublisherClient._get_api_endpoint(None, None, mock_universe, "never") + == mock_endpoint + ) + assert ( + PublisherClient._get_api_endpoint(None, None, default_universe, "never") + == default_endpoint + ) + + with pytest.raises(MutualTLSChannelError) as excinfo: + PublisherClient._get_api_endpoint( + None, mock_client_cert_source, mock_universe, "auto" + ) + assert ( + str(excinfo.value) + == "mTLS is not supported in any universe other than googleapis.com." + ) + + +def test__get_universe_domain(): + client_universe_domain = "foo.com" + universe_domain_env = "bar.com" + + assert ( + PublisherClient._get_universe_domain( + client_universe_domain, universe_domain_env + ) + == client_universe_domain + ) + assert ( + PublisherClient._get_universe_domain(None, universe_domain_env) + == universe_domain_env + ) + assert ( + PublisherClient._get_universe_domain(None, None) + == PublisherClient._DEFAULT_UNIVERSE + ) + + with pytest.raises(ValueError) as excinfo: + PublisherClient._get_universe_domain("", None) + assert str(excinfo.value) == "Universe Domain cannot be an empty string." + + +@pytest.mark.parametrize( + "client_class,transport_class,transport_name", + [ + (PublisherClient, transports.PublisherGrpcTransport, "grpc"), + (PublisherClient, transports.PublisherRestTransport, "rest"), + ], +) +def test__validate_universe_domain(client_class, transport_class, transport_name): + client = client_class( + transport=transport_class(credentials=ga_credentials.AnonymousCredentials()) + ) + assert client._validate_universe_domain() == True + + # Test the case when universe is already validated. + assert client._validate_universe_domain() == True + + if transport_name == "grpc": + # Test the case where credentials are provided by the + # `local_channel_credentials`. The default universes in both match. + channel = grpc.secure_channel( + "http://localhost/", grpc.local_channel_credentials() + ) + client = client_class(transport=transport_class(channel=channel)) + assert client._validate_universe_domain() == True + + # Test the case where credentials do not exist: e.g. a transport is provided + # with no credentials. Validation should still succeed because there is no + # mismatch with non-existent credentials. + channel = grpc.secure_channel( + "http://localhost/", grpc.local_channel_credentials() + ) + transport = transport_class(channel=channel) + transport._credentials = None + client = client_class(transport=transport) + assert client._validate_universe_domain() == True + + # TODO: This is needed to cater for older versions of google-auth + # Make this test unconditional once the minimum supported version of + # google-auth becomes 2.23.0 or higher. + google_auth_major, google_auth_minor = [ + int(part) for part in google.auth.__version__.split(".")[0:2] + ] + if google_auth_major > 2 or (google_auth_major == 2 and google_auth_minor >= 23): + credentials = ga_credentials.AnonymousCredentials() + credentials._universe_domain = "foo.com" + # Test the case when there is a universe mismatch from the credentials. + client = client_class(transport=transport_class(credentials=credentials)) + with pytest.raises(ValueError) as excinfo: + client._validate_universe_domain() + assert ( + str(excinfo.value) + == "The configured universe domain (googleapis.com) does not match the universe domain found in the credentials (foo.com). If you haven't configured the universe domain explicitly, `googleapis.com` is the default." + ) + + # Test the case when there is a universe mismatch from the client. + # + # TODO: Make this test unconditional once the minimum supported version of + # google-api-core becomes 2.15.0 or higher. + api_core_major, api_core_minor = [ + int(part) for part in api_core_version.__version__.split(".")[0:2] + ] + if api_core_major > 2 or (api_core_major == 2 and api_core_minor >= 15): + client = client_class( + client_options={"universe_domain": "bar.com"}, + transport=transport_class( + credentials=ga_credentials.AnonymousCredentials(), + ), + ) + with pytest.raises(ValueError) as excinfo: + client._validate_universe_domain() + assert ( + str(excinfo.value) + == "The configured universe domain (bar.com) does not match the universe domain found in the credentials (googleapis.com). If you haven't configured the universe domain explicitly, `googleapis.com` is the default." + ) + + # Test that ValueError is raised if universe_domain is provided via client options and credentials is None + with pytest.raises(ValueError): + client._compare_universes("foo.bar", None) + + @pytest.mark.parametrize( "client_class,transport_name", [ @@ -209,12 +469,14 @@ def test_publisher_client_get_transport_class(): ], ) @mock.patch.object( - PublisherClient, "DEFAULT_ENDPOINT", modify_default_endpoint(PublisherClient) + PublisherClient, + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(PublisherClient), ) @mock.patch.object( PublisherAsyncClient, - "DEFAULT_ENDPOINT", - modify_default_endpoint(PublisherAsyncClient), + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(PublisherAsyncClient), ) def test_publisher_client_client_options(client_class, transport_class, transport_name): # Check that if channel is provided we won't create a new one. @@ -254,7 +516,9 @@ def test_publisher_client_client_options(client_class, transport_class, transpor patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, @@ -284,15 +548,23 @@ def test_publisher_client_client_options(client_class, transport_class, transpor # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has # unsupported value. with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): - with pytest.raises(MutualTLSChannelError): + with pytest.raises(MutualTLSChannelError) as excinfo: client = client_class(transport=transport_name) + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + ) # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. with mock.patch.dict( os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} ): - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: client = client_class(transport=transport_name) + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) # Check the case quota_project_id is provided options = client_options.ClientOptions(quota_project_id="octopus") @@ -302,7 +574,9 @@ def test_publisher_client_client_options(client_class, transport_class, transpor patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id="octopus", @@ -320,7 +594,9 @@ def test_publisher_client_client_options(client_class, transport_class, transpor patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, @@ -352,12 +628,14 @@ def test_publisher_client_client_options(client_class, transport_class, transpor ], ) @mock.patch.object( - PublisherClient, "DEFAULT_ENDPOINT", modify_default_endpoint(PublisherClient) + PublisherClient, + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(PublisherClient), ) @mock.patch.object( PublisherAsyncClient, - "DEFAULT_ENDPOINT", - modify_default_endpoint(PublisherAsyncClient), + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(PublisherAsyncClient), ) @mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) def test_publisher_client_mtls_env_auto( @@ -380,7 +658,9 @@ def test_publisher_client_mtls_env_auto( if use_client_cert_env == "false": expected_client_cert_source = None - expected_host = client.DEFAULT_ENDPOINT + expected_host = client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ) else: expected_client_cert_source = client_cert_source_callback expected_host = client.DEFAULT_MTLS_ENDPOINT @@ -412,7 +692,9 @@ def test_publisher_client_mtls_env_auto( return_value=client_cert_source_callback, ): if use_client_cert_env == "false": - expected_host = client.DEFAULT_ENDPOINT + expected_host = client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ) expected_client_cert_source = None else: expected_host = client.DEFAULT_MTLS_ENDPOINT @@ -446,7 +728,9 @@ def test_publisher_client_mtls_env_auto( patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, @@ -532,6 +816,113 @@ def test_publisher_client_get_mtls_endpoint_and_cert_source(client_class): assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT assert cert_source == mock_client_cert_source + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + client_class.get_mtls_endpoint_and_cert_source() + + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + ) + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} + ): + with pytest.raises(ValueError) as excinfo: + client_class.get_mtls_endpoint_and_cert_source() + + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) + + +@pytest.mark.parametrize("client_class", [PublisherClient, PublisherAsyncClient]) +@mock.patch.object( + PublisherClient, + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(PublisherClient), +) +@mock.patch.object( + PublisherAsyncClient, + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(PublisherAsyncClient), +) +def test_publisher_client_client_api_endpoint(client_class): + mock_client_cert_source = client_cert_source_callback + api_override = "foo.com" + default_universe = PublisherClient._DEFAULT_UNIVERSE + default_endpoint = PublisherClient._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=default_universe + ) + mock_universe = "bar.com" + mock_endpoint = PublisherClient._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=mock_universe + ) + + # If ClientOptions.api_endpoint is set and GOOGLE_API_USE_CLIENT_CERTIFICATE="true", + # use ClientOptions.api_endpoint as the api endpoint regardless. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch( + "google.auth.transport.requests.AuthorizedSession.configure_mtls_channel" + ): + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, api_endpoint=api_override + ) + client = client_class( + client_options=options, + credentials=ga_credentials.AnonymousCredentials(), + ) + assert client.api_endpoint == api_override + + # If ClientOptions.api_endpoint is not set and GOOGLE_API_USE_MTLS_ENDPOINT="never", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with GDU as the api endpoint. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + client = client_class(credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == default_endpoint + + # If ClientOptions.api_endpoint is not set and GOOGLE_API_USE_MTLS_ENDPOINT="always", + # use the DEFAULT_MTLS_ENDPOINT as the api endpoint. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + client = client_class(credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + + # If ClientOptions.api_endpoint is not set, GOOGLE_API_USE_MTLS_ENDPOINT="auto" (default), + # GOOGLE_API_USE_CLIENT_CERTIFICATE="false" (default), default cert source doesn't exist, + # and ClientOptions.universe_domain="bar.com", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with universe domain as the api endpoint. + options = client_options.ClientOptions() + universe_exists = hasattr(options, "universe_domain") + if universe_exists: + options = client_options.ClientOptions(universe_domain=mock_universe) + client = client_class( + client_options=options, credentials=ga_credentials.AnonymousCredentials() + ) + else: + client = client_class( + client_options=options, credentials=ga_credentials.AnonymousCredentials() + ) + assert client.api_endpoint == ( + mock_endpoint if universe_exists else default_endpoint + ) + assert client.universe_domain == ( + mock_universe if universe_exists else default_universe + ) + + # If ClientOptions does not have a universe domain attribute and GOOGLE_API_USE_MTLS_ENDPOINT="never", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with GDU as the api endpoint. + options = client_options.ClientOptions() + if hasattr(options, "universe_domain"): + delattr(options, "universe_domain") + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + client = client_class( + client_options=options, credentials=ga_credentials.AnonymousCredentials() + ) + assert client.api_endpoint == default_endpoint + @pytest.mark.parametrize( "client_class,transport_class,transport_name", @@ -558,7 +949,9 @@ def test_publisher_client_client_options_scopes( patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=["1", "2"], client_cert_source_for_mtls=None, quota_project_id=None, @@ -593,7 +986,9 @@ def test_publisher_client_client_options_credentials_file( patched.assert_called_once_with( credentials=None, credentials_file="credentials.json", - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, @@ -646,7 +1041,9 @@ def test_publisher_client_create_channel_credentials_file( patched.assert_called_once_with( credentials=None, credentials_file="credentials.json", - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, @@ -713,6 +1110,7 @@ def test_create_topic(request_type, transport: str = "grpc"): name="name_value", kms_key_name="kms_key_name_value", satisfies_pzs=True, + state=pubsub.Topic.State.ACTIVE, ) response = client.create_topic(request) @@ -726,6 +1124,7 @@ def test_create_topic(request_type, transport: str = "grpc"): assert response.name == "name_value" assert response.kms_key_name == "kms_key_name_value" assert response.satisfies_pzs is True + assert response.state == pubsub.Topic.State.ACTIVE def test_create_topic_empty_call(): @@ -765,6 +1164,7 @@ async def test_create_topic_async( name="name_value", kms_key_name="kms_key_name_value", satisfies_pzs=True, + state=pubsub.Topic.State.ACTIVE, ) ) response = await client.create_topic(request) @@ -779,6 +1179,7 @@ async def test_create_topic_async( assert response.name == "name_value" assert response.kms_key_name == "kms_key_name_value" assert response.satisfies_pzs is True + assert response.state == pubsub.Topic.State.ACTIVE @pytest.mark.asyncio @@ -949,6 +1350,7 @@ def test_update_topic(request_type, transport: str = "grpc"): name="name_value", kms_key_name="kms_key_name_value", satisfies_pzs=True, + state=pubsub.Topic.State.ACTIVE, ) response = client.update_topic(request) @@ -962,6 +1364,7 @@ def test_update_topic(request_type, transport: str = "grpc"): assert response.name == "name_value" assert response.kms_key_name == "kms_key_name_value" assert response.satisfies_pzs is True + assert response.state == pubsub.Topic.State.ACTIVE def test_update_topic_empty_call(): @@ -1001,6 +1404,7 @@ async def test_update_topic_async( name="name_value", kms_key_name="kms_key_name_value", satisfies_pzs=True, + state=pubsub.Topic.State.ACTIVE, ) ) response = await client.update_topic(request) @@ -1015,6 +1419,7 @@ async def test_update_topic_async( assert response.name == "name_value" assert response.kms_key_name == "kms_key_name_value" assert response.satisfies_pzs is True + assert response.state == pubsub.Topic.State.ACTIVE @pytest.mark.asyncio @@ -1333,8 +1738,7 @@ def test_publish_flattened(): # Call the method with a truthy value for each flattened field, # using the keyword arguments to the method. client.publish( - topic="topic_value", - messages=[pubsub.PubsubMessage(data=b"data_blob")], + topic="topic_value", messages=[pubsub.PubsubMessage(data=b"data_blob")] ) # Establish that the underlying call was made with the expected @@ -1437,6 +1841,7 @@ def test_get_topic(request_type, transport: str = "grpc"): name="name_value", kms_key_name="kms_key_name_value", satisfies_pzs=True, + state=pubsub.Topic.State.ACTIVE, ) response = client.get_topic(request) @@ -1450,6 +1855,7 @@ def test_get_topic(request_type, transport: str = "grpc"): assert response.name == "name_value" assert response.kms_key_name == "kms_key_name_value" assert response.satisfies_pzs is True + assert response.state == pubsub.Topic.State.ACTIVE def test_get_topic_empty_call(): @@ -1489,6 +1895,7 @@ async def test_get_topic_async( name="name_value", kms_key_name="kms_key_name_value", satisfies_pzs=True, + state=pubsub.Topic.State.ACTIVE, ) ) response = await client.get_topic(request) @@ -1503,6 +1910,7 @@ async def test_get_topic_async( assert response.name == "name_value" assert response.kms_key_name == "kms_key_name_value" assert response.satisfies_pzs is True + assert response.state == pubsub.Topic.State.ACTIVE @pytest.mark.asyncio @@ -1883,7 +2291,7 @@ async def test_list_topics_flattened_error_async(): def test_list_topics_pager(transport_name: str = "grpc"): client = PublisherClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), transport=transport_name, ) @@ -1933,7 +2341,7 @@ def test_list_topics_pager(transport_name: str = "grpc"): def test_list_topics_pages(transport_name: str = "grpc"): client = PublisherClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), transport=transport_name, ) @@ -1975,7 +2383,7 @@ def test_list_topics_pages(transport_name: str = "grpc"): @pytest.mark.asyncio async def test_list_topics_async_pager(): client = PublisherAsyncClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), ) # Mock the actual call within the gRPC stub, and fake the request. @@ -2025,7 +2433,7 @@ async def test_list_topics_async_pager(): @pytest.mark.asyncio async def test_list_topics_async_pages(): client = PublisherAsyncClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), ) # Mock the actual call within the gRPC stub, and fake the request. @@ -2323,7 +2731,7 @@ async def test_list_topic_subscriptions_flattened_error_async(): def test_list_topic_subscriptions_pager(transport_name: str = "grpc"): client = PublisherClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), transport=transport_name, ) @@ -2375,7 +2783,7 @@ def test_list_topic_subscriptions_pager(transport_name: str = "grpc"): def test_list_topic_subscriptions_pages(transport_name: str = "grpc"): client = PublisherClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), transport=transport_name, ) @@ -2419,7 +2827,7 @@ def test_list_topic_subscriptions_pages(transport_name: str = "grpc"): @pytest.mark.asyncio async def test_list_topic_subscriptions_async_pager(): client = PublisherAsyncClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), ) # Mock the actual call within the gRPC stub, and fake the request. @@ -2471,7 +2879,7 @@ async def test_list_topic_subscriptions_async_pager(): @pytest.mark.asyncio async def test_list_topic_subscriptions_async_pages(): client = PublisherAsyncClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), ) # Mock the actual call within the gRPC stub, and fake the request. @@ -2771,7 +3179,7 @@ async def test_list_topic_snapshots_flattened_error_async(): def test_list_topic_snapshots_pager(transport_name: str = "grpc"): client = PublisherClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), transport=transport_name, ) @@ -2823,7 +3231,7 @@ def test_list_topic_snapshots_pager(transport_name: str = "grpc"): def test_list_topic_snapshots_pages(transport_name: str = "grpc"): client = PublisherClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), transport=transport_name, ) @@ -2867,7 +3275,7 @@ def test_list_topic_snapshots_pages(transport_name: str = "grpc"): @pytest.mark.asyncio async def test_list_topic_snapshots_async_pager(): client = PublisherAsyncClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), ) # Mock the actual call within the gRPC stub, and fake the request. @@ -2919,7 +3327,7 @@ async def test_list_topic_snapshots_async_pager(): @pytest.mark.asyncio async def test_list_topic_snapshots_async_pages(): client = PublisherAsyncClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), ) # Mock the actual call within the gRPC stub, and fake the request. @@ -3365,6 +3773,7 @@ def test_create_topic_rest(request_type): name="name_value", kms_key_name="kms_key_name_value", satisfies_pzs=True, + state=pubsub.Topic.State.ACTIVE, ) # Wrap the value into a proper Response obj @@ -3383,6 +3792,7 @@ def test_create_topic_rest(request_type): assert response.name == "name_value" assert response.kms_key_name == "kms_key_name_value" assert response.satisfies_pzs is True + assert response.state == pubsub.Topic.State.ACTIVE def test_create_topic_rest_required_fields(request_type=pubsub.Topic): @@ -3631,6 +4041,7 @@ def test_update_topic_rest(request_type): name="name_value", kms_key_name="kms_key_name_value", satisfies_pzs=True, + state=pubsub.Topic.State.ACTIVE, ) # Wrap the value into a proper Response obj @@ -3649,6 +4060,7 @@ def test_update_topic_rest(request_type): assert response.name == "name_value" assert response.kms_key_name == "kms_key_name_value" assert response.satisfies_pzs is True + assert response.state == pubsub.Topic.State.ACTIVE def test_update_topic_rest_required_fields(request_type=pubsub.UpdateTopicRequest): @@ -4177,6 +4589,7 @@ def test_get_topic_rest(request_type): name="name_value", kms_key_name="kms_key_name_value", satisfies_pzs=True, + state=pubsub.Topic.State.ACTIVE, ) # Wrap the value into a proper Response obj @@ -4195,6 +4608,7 @@ def test_get_topic_rest(request_type): assert response.name == "name_value" assert response.kms_key_name == "kms_key_name_value" assert response.satisfies_pzs is True + assert response.state == pubsub.Topic.State.ACTIVE def test_get_topic_rest_required_fields(request_type=pubsub.GetTopicRequest): @@ -5921,7 +6335,7 @@ def test_credentials_transport_error(): ) # It is an error to provide an api_key and a credential. - options = mock.Mock() + options = client_options.ClientOptions() options.api_key = "api_key" with pytest.raises(ValueError): client = PublisherClient( @@ -7407,7 +7821,9 @@ def test_api_key_credentials(client_class, transport_class): patched.assert_called_once_with( credentials=mock_cred, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, diff --git a/tests/unit/gapic/pubsub_v1/test_schema_service.py b/tests/unit/gapic/pubsub_v1/test_schema_service.py index c3585ae3b..e73ef9114 100644 --- a/tests/unit/gapic/pubsub_v1/test_schema_service.py +++ b/tests/unit/gapic/pubsub_v1/test_schema_service.py @@ -29,6 +29,7 @@ import json import math import pytest +from google.api_core import api_core_version from proto.marshal.rules.dates import DurationRule, TimestampRule from proto.marshal.rules import wrappers from requests import Response @@ -73,6 +74,17 @@ def modify_default_endpoint(client): ) +# If default endpoint template is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint template so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint_template(client): + return ( + "test.{UNIVERSE_DOMAIN}" + if ("localhost" in client._DEFAULT_ENDPOINT_TEMPLATE) + else client._DEFAULT_ENDPOINT_TEMPLATE + ) + + def test__get_default_mtls_endpoint(): api_endpoint = "example.googleapis.com" api_mtls_endpoint = "example.mtls.googleapis.com" @@ -102,6 +114,270 @@ def test__get_default_mtls_endpoint(): ) +def test__read_environment_variables(): + assert SchemaServiceClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + assert SchemaServiceClient._read_environment_variables() == (True, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"}): + assert SchemaServiceClient._read_environment_variables() == ( + False, + "auto", + None, + ) + + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} + ): + with pytest.raises(ValueError) as excinfo: + SchemaServiceClient._read_environment_variables() + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + assert SchemaServiceClient._read_environment_variables() == ( + False, + "never", + None, + ) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + assert SchemaServiceClient._read_environment_variables() == ( + False, + "always", + None, + ) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}): + assert SchemaServiceClient._read_environment_variables() == ( + False, + "auto", + None, + ) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + SchemaServiceClient._read_environment_variables() + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + ) + + with mock.patch.dict(os.environ, {"GOOGLE_CLOUD_UNIVERSE_DOMAIN": "foo.com"}): + assert SchemaServiceClient._read_environment_variables() == ( + False, + "auto", + "foo.com", + ) + + +def test__get_client_cert_source(): + mock_provided_cert_source = mock.Mock() + mock_default_cert_source = mock.Mock() + + assert SchemaServiceClient._get_client_cert_source(None, False) is None + assert ( + SchemaServiceClient._get_client_cert_source(mock_provided_cert_source, False) + is None + ) + assert ( + SchemaServiceClient._get_client_cert_source(mock_provided_cert_source, True) + == mock_provided_cert_source + ) + + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", return_value=True + ): + with mock.patch( + "google.auth.transport.mtls.default_client_cert_source", + return_value=mock_default_cert_source, + ): + assert ( + SchemaServiceClient._get_client_cert_source(None, True) + is mock_default_cert_source + ) + assert ( + SchemaServiceClient._get_client_cert_source( + mock_provided_cert_source, "true" + ) + is mock_provided_cert_source + ) + + +@mock.patch.object( + SchemaServiceClient, + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(SchemaServiceClient), +) +@mock.patch.object( + SchemaServiceAsyncClient, + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(SchemaServiceAsyncClient), +) +def test__get_api_endpoint(): + api_override = "foo.com" + mock_client_cert_source = mock.Mock() + default_universe = SchemaServiceClient._DEFAULT_UNIVERSE + default_endpoint = SchemaServiceClient._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=default_universe + ) + mock_universe = "bar.com" + mock_endpoint = SchemaServiceClient._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=mock_universe + ) + + assert ( + SchemaServiceClient._get_api_endpoint( + api_override, mock_client_cert_source, default_universe, "always" + ) + == api_override + ) + assert ( + SchemaServiceClient._get_api_endpoint( + None, mock_client_cert_source, default_universe, "auto" + ) + == SchemaServiceClient.DEFAULT_MTLS_ENDPOINT + ) + assert ( + SchemaServiceClient._get_api_endpoint(None, None, default_universe, "auto") + == default_endpoint + ) + assert ( + SchemaServiceClient._get_api_endpoint(None, None, default_universe, "always") + == SchemaServiceClient.DEFAULT_MTLS_ENDPOINT + ) + assert ( + SchemaServiceClient._get_api_endpoint( + None, mock_client_cert_source, default_universe, "always" + ) + == SchemaServiceClient.DEFAULT_MTLS_ENDPOINT + ) + assert ( + SchemaServiceClient._get_api_endpoint(None, None, mock_universe, "never") + == mock_endpoint + ) + assert ( + SchemaServiceClient._get_api_endpoint(None, None, default_universe, "never") + == default_endpoint + ) + + with pytest.raises(MutualTLSChannelError) as excinfo: + SchemaServiceClient._get_api_endpoint( + None, mock_client_cert_source, mock_universe, "auto" + ) + assert ( + str(excinfo.value) + == "mTLS is not supported in any universe other than googleapis.com." + ) + + +def test__get_universe_domain(): + client_universe_domain = "foo.com" + universe_domain_env = "bar.com" + + assert ( + SchemaServiceClient._get_universe_domain( + client_universe_domain, universe_domain_env + ) + == client_universe_domain + ) + assert ( + SchemaServiceClient._get_universe_domain(None, universe_domain_env) + == universe_domain_env + ) + assert ( + SchemaServiceClient._get_universe_domain(None, None) + == SchemaServiceClient._DEFAULT_UNIVERSE + ) + + with pytest.raises(ValueError) as excinfo: + SchemaServiceClient._get_universe_domain("", None) + assert str(excinfo.value) == "Universe Domain cannot be an empty string." + + +@pytest.mark.parametrize( + "client_class,transport_class,transport_name", + [ + (SchemaServiceClient, transports.SchemaServiceGrpcTransport, "grpc"), + (SchemaServiceClient, transports.SchemaServiceRestTransport, "rest"), + ], +) +def test__validate_universe_domain(client_class, transport_class, transport_name): + client = client_class( + transport=transport_class(credentials=ga_credentials.AnonymousCredentials()) + ) + assert client._validate_universe_domain() == True + + # Test the case when universe is already validated. + assert client._validate_universe_domain() == True + + if transport_name == "grpc": + # Test the case where credentials are provided by the + # `local_channel_credentials`. The default universes in both match. + channel = grpc.secure_channel( + "http://localhost/", grpc.local_channel_credentials() + ) + client = client_class(transport=transport_class(channel=channel)) + assert client._validate_universe_domain() == True + + # Test the case where credentials do not exist: e.g. a transport is provided + # with no credentials. Validation should still succeed because there is no + # mismatch with non-existent credentials. + channel = grpc.secure_channel( + "http://localhost/", grpc.local_channel_credentials() + ) + transport = transport_class(channel=channel) + transport._credentials = None + client = client_class(transport=transport) + assert client._validate_universe_domain() == True + + # TODO: This is needed to cater for older versions of google-auth + # Make this test unconditional once the minimum supported version of + # google-auth becomes 2.23.0 or higher. + google_auth_major, google_auth_minor = [ + int(part) for part in google.auth.__version__.split(".")[0:2] + ] + if google_auth_major > 2 or (google_auth_major == 2 and google_auth_minor >= 23): + credentials = ga_credentials.AnonymousCredentials() + credentials._universe_domain = "foo.com" + # Test the case when there is a universe mismatch from the credentials. + client = client_class(transport=transport_class(credentials=credentials)) + with pytest.raises(ValueError) as excinfo: + client._validate_universe_domain() + assert ( + str(excinfo.value) + == "The configured universe domain (googleapis.com) does not match the universe domain found in the credentials (foo.com). If you haven't configured the universe domain explicitly, `googleapis.com` is the default." + ) + + # Test the case when there is a universe mismatch from the client. + # + # TODO: Make this test unconditional once the minimum supported version of + # google-api-core becomes 2.15.0 or higher. + api_core_major, api_core_minor = [ + int(part) for part in api_core_version.__version__.split(".")[0:2] + ] + if api_core_major > 2 or (api_core_major == 2 and api_core_minor >= 15): + client = client_class( + client_options={"universe_domain": "bar.com"}, + transport=transport_class( + credentials=ga_credentials.AnonymousCredentials(), + ), + ) + with pytest.raises(ValueError) as excinfo: + client._validate_universe_domain() + assert ( + str(excinfo.value) + == "The configured universe domain (bar.com) does not match the universe domain found in the credentials (googleapis.com). If you haven't configured the universe domain explicitly, `googleapis.com` is the default." + ) + + # Test that ValueError is raised if universe_domain is provided via client options and credentials is None + with pytest.raises(ValueError): + client._compare_universes("foo.bar", None) + + @pytest.mark.parametrize( "client_class,transport_name", [ @@ -213,13 +489,13 @@ def test_schema_service_client_get_transport_class(): ) @mock.patch.object( SchemaServiceClient, - "DEFAULT_ENDPOINT", - modify_default_endpoint(SchemaServiceClient), + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(SchemaServiceClient), ) @mock.patch.object( SchemaServiceAsyncClient, - "DEFAULT_ENDPOINT", - modify_default_endpoint(SchemaServiceAsyncClient), + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(SchemaServiceAsyncClient), ) def test_schema_service_client_client_options( client_class, transport_class, transport_name @@ -261,7 +537,9 @@ def test_schema_service_client_client_options( patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, @@ -291,15 +569,23 @@ def test_schema_service_client_client_options( # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has # unsupported value. with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): - with pytest.raises(MutualTLSChannelError): + with pytest.raises(MutualTLSChannelError) as excinfo: client = client_class(transport=transport_name) + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + ) # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. with mock.patch.dict( os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} ): - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: client = client_class(transport=transport_name) + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) # Check the case quota_project_id is provided options = client_options.ClientOptions(quota_project_id="octopus") @@ -309,7 +595,9 @@ def test_schema_service_client_client_options( patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id="octopus", @@ -327,7 +615,9 @@ def test_schema_service_client_client_options( patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, @@ -360,13 +650,13 @@ def test_schema_service_client_client_options( ) @mock.patch.object( SchemaServiceClient, - "DEFAULT_ENDPOINT", - modify_default_endpoint(SchemaServiceClient), + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(SchemaServiceClient), ) @mock.patch.object( SchemaServiceAsyncClient, - "DEFAULT_ENDPOINT", - modify_default_endpoint(SchemaServiceAsyncClient), + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(SchemaServiceAsyncClient), ) @mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) def test_schema_service_client_mtls_env_auto( @@ -389,7 +679,9 @@ def test_schema_service_client_mtls_env_auto( if use_client_cert_env == "false": expected_client_cert_source = None - expected_host = client.DEFAULT_ENDPOINT + expected_host = client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ) else: expected_client_cert_source = client_cert_source_callback expected_host = client.DEFAULT_MTLS_ENDPOINT @@ -421,7 +713,9 @@ def test_schema_service_client_mtls_env_auto( return_value=client_cert_source_callback, ): if use_client_cert_env == "false": - expected_host = client.DEFAULT_ENDPOINT + expected_host = client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ) expected_client_cert_source = None else: expected_host = client.DEFAULT_MTLS_ENDPOINT @@ -455,7 +749,9 @@ def test_schema_service_client_mtls_env_auto( patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, @@ -545,6 +841,115 @@ def test_schema_service_client_get_mtls_endpoint_and_cert_source(client_class): assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT assert cert_source == mock_client_cert_source + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + client_class.get_mtls_endpoint_and_cert_source() + + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + ) + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} + ): + with pytest.raises(ValueError) as excinfo: + client_class.get_mtls_endpoint_and_cert_source() + + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) + + +@pytest.mark.parametrize( + "client_class", [SchemaServiceClient, SchemaServiceAsyncClient] +) +@mock.patch.object( + SchemaServiceClient, + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(SchemaServiceClient), +) +@mock.patch.object( + SchemaServiceAsyncClient, + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(SchemaServiceAsyncClient), +) +def test_schema_service_client_client_api_endpoint(client_class): + mock_client_cert_source = client_cert_source_callback + api_override = "foo.com" + default_universe = SchemaServiceClient._DEFAULT_UNIVERSE + default_endpoint = SchemaServiceClient._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=default_universe + ) + mock_universe = "bar.com" + mock_endpoint = SchemaServiceClient._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=mock_universe + ) + + # If ClientOptions.api_endpoint is set and GOOGLE_API_USE_CLIENT_CERTIFICATE="true", + # use ClientOptions.api_endpoint as the api endpoint regardless. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch( + "google.auth.transport.requests.AuthorizedSession.configure_mtls_channel" + ): + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, api_endpoint=api_override + ) + client = client_class( + client_options=options, + credentials=ga_credentials.AnonymousCredentials(), + ) + assert client.api_endpoint == api_override + + # If ClientOptions.api_endpoint is not set and GOOGLE_API_USE_MTLS_ENDPOINT="never", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with GDU as the api endpoint. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + client = client_class(credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == default_endpoint + + # If ClientOptions.api_endpoint is not set and GOOGLE_API_USE_MTLS_ENDPOINT="always", + # use the DEFAULT_MTLS_ENDPOINT as the api endpoint. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + client = client_class(credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + + # If ClientOptions.api_endpoint is not set, GOOGLE_API_USE_MTLS_ENDPOINT="auto" (default), + # GOOGLE_API_USE_CLIENT_CERTIFICATE="false" (default), default cert source doesn't exist, + # and ClientOptions.universe_domain="bar.com", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with universe domain as the api endpoint. + options = client_options.ClientOptions() + universe_exists = hasattr(options, "universe_domain") + if universe_exists: + options = client_options.ClientOptions(universe_domain=mock_universe) + client = client_class( + client_options=options, credentials=ga_credentials.AnonymousCredentials() + ) + else: + client = client_class( + client_options=options, credentials=ga_credentials.AnonymousCredentials() + ) + assert client.api_endpoint == ( + mock_endpoint if universe_exists else default_endpoint + ) + assert client.universe_domain == ( + mock_universe if universe_exists else default_universe + ) + + # If ClientOptions does not have a universe domain attribute and GOOGLE_API_USE_MTLS_ENDPOINT="never", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with GDU as the api endpoint. + options = client_options.ClientOptions() + if hasattr(options, "universe_domain"): + delattr(options, "universe_domain") + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + client = client_class( + client_options=options, credentials=ga_credentials.AnonymousCredentials() + ) + assert client.api_endpoint == default_endpoint + @pytest.mark.parametrize( "client_class,transport_class,transport_name", @@ -571,7 +976,9 @@ def test_schema_service_client_client_options_scopes( patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=["1", "2"], client_cert_source_for_mtls=None, quota_project_id=None, @@ -611,7 +1018,9 @@ def test_schema_service_client_client_options_credentials_file( patched.assert_called_once_with( credentials=None, credentials_file="credentials.json", - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, @@ -671,7 +1080,9 @@ def test_schema_service_client_create_channel_credentials_file( patched.assert_called_once_with( credentials=None, credentials_file="credentials.json", - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, @@ -1448,7 +1859,7 @@ async def test_list_schemas_flattened_error_async(): def test_list_schemas_pager(transport_name: str = "grpc"): client = SchemaServiceClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), transport=transport_name, ) @@ -1498,7 +1909,7 @@ def test_list_schemas_pager(transport_name: str = "grpc"): def test_list_schemas_pages(transport_name: str = "grpc"): client = SchemaServiceClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), transport=transport_name, ) @@ -1540,7 +1951,7 @@ def test_list_schemas_pages(transport_name: str = "grpc"): @pytest.mark.asyncio async def test_list_schemas_async_pager(): client = SchemaServiceAsyncClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), ) # Mock the actual call within the gRPC stub, and fake the request. @@ -1590,7 +2001,7 @@ async def test_list_schemas_async_pager(): @pytest.mark.asyncio async def test_list_schemas_async_pages(): client = SchemaServiceAsyncClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), ) # Mock the actual call within the gRPC stub, and fake the request. @@ -1884,7 +2295,7 @@ async def test_list_schema_revisions_flattened_error_async(): def test_list_schema_revisions_pager(transport_name: str = "grpc"): client = SchemaServiceClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), transport=transport_name, ) @@ -1936,7 +2347,7 @@ def test_list_schema_revisions_pager(transport_name: str = "grpc"): def test_list_schema_revisions_pages(transport_name: str = "grpc"): client = SchemaServiceClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), transport=transport_name, ) @@ -1980,7 +2391,7 @@ def test_list_schema_revisions_pages(transport_name: str = "grpc"): @pytest.mark.asyncio async def test_list_schema_revisions_async_pager(): client = SchemaServiceAsyncClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), ) # Mock the actual call within the gRPC stub, and fake the request. @@ -2032,7 +2443,7 @@ async def test_list_schema_revisions_async_pager(): @pytest.mark.asyncio async def test_list_schema_revisions_async_pages(): client = SchemaServiceAsyncClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), ) # Mock the actual call within the gRPC stub, and fake the request. @@ -6368,7 +6779,7 @@ def test_credentials_transport_error(): ) # It is an error to provide an api_key and a credential. - options = mock.Mock() + options = client_options.ClientOptions() options.api_key = "api_key" with pytest.raises(ValueError): client = SchemaServiceClient( @@ -7821,7 +8232,9 @@ def test_api_key_credentials(client_class, transport_class): patched.assert_called_once_with( credentials=mock_cred, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, diff --git a/tests/unit/gapic/pubsub_v1/test_subscriber.py b/tests/unit/gapic/pubsub_v1/test_subscriber.py index 94f7e301e..4ff6a3f27 100644 --- a/tests/unit/gapic/pubsub_v1/test_subscriber.py +++ b/tests/unit/gapic/pubsub_v1/test_subscriber.py @@ -30,6 +30,7 @@ import json import math import pytest +from google.api_core import api_core_version from proto.marshal.rules.dates import DurationRule, TimestampRule from proto.marshal.rules import wrappers from requests import Response @@ -75,6 +76,17 @@ def modify_default_endpoint(client): ) +# If default endpoint template is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint template so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint_template(client): + return ( + "test.{UNIVERSE_DOMAIN}" + if ("localhost" in client._DEFAULT_ENDPOINT_TEMPLATE) + else client._DEFAULT_ENDPOINT_TEMPLATE + ) + + def test__get_default_mtls_endpoint(): api_endpoint = "example.googleapis.com" api_mtls_endpoint = "example.mtls.googleapis.com" @@ -101,6 +113,254 @@ def test__get_default_mtls_endpoint(): assert SubscriberClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi +def test__read_environment_variables(): + assert SubscriberClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + assert SubscriberClient._read_environment_variables() == (True, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"}): + assert SubscriberClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} + ): + with pytest.raises(ValueError) as excinfo: + SubscriberClient._read_environment_variables() + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + assert SubscriberClient._read_environment_variables() == (False, "never", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + assert SubscriberClient._read_environment_variables() == (False, "always", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}): + assert SubscriberClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + SubscriberClient._read_environment_variables() + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + ) + + with mock.patch.dict(os.environ, {"GOOGLE_CLOUD_UNIVERSE_DOMAIN": "foo.com"}): + assert SubscriberClient._read_environment_variables() == ( + False, + "auto", + "foo.com", + ) + + +def test__get_client_cert_source(): + mock_provided_cert_source = mock.Mock() + mock_default_cert_source = mock.Mock() + + assert SubscriberClient._get_client_cert_source(None, False) is None + assert ( + SubscriberClient._get_client_cert_source(mock_provided_cert_source, False) + is None + ) + assert ( + SubscriberClient._get_client_cert_source(mock_provided_cert_source, True) + == mock_provided_cert_source + ) + + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", return_value=True + ): + with mock.patch( + "google.auth.transport.mtls.default_client_cert_source", + return_value=mock_default_cert_source, + ): + assert ( + SubscriberClient._get_client_cert_source(None, True) + is mock_default_cert_source + ) + assert ( + SubscriberClient._get_client_cert_source( + mock_provided_cert_source, "true" + ) + is mock_provided_cert_source + ) + + +@mock.patch.object( + SubscriberClient, + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(SubscriberClient), +) +@mock.patch.object( + SubscriberAsyncClient, + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(SubscriberAsyncClient), +) +def test__get_api_endpoint(): + api_override = "foo.com" + mock_client_cert_source = mock.Mock() + default_universe = SubscriberClient._DEFAULT_UNIVERSE + default_endpoint = SubscriberClient._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=default_universe + ) + mock_universe = "bar.com" + mock_endpoint = SubscriberClient._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=mock_universe + ) + + assert ( + SubscriberClient._get_api_endpoint( + api_override, mock_client_cert_source, default_universe, "always" + ) + == api_override + ) + assert ( + SubscriberClient._get_api_endpoint( + None, mock_client_cert_source, default_universe, "auto" + ) + == SubscriberClient.DEFAULT_MTLS_ENDPOINT + ) + assert ( + SubscriberClient._get_api_endpoint(None, None, default_universe, "auto") + == default_endpoint + ) + assert ( + SubscriberClient._get_api_endpoint(None, None, default_universe, "always") + == SubscriberClient.DEFAULT_MTLS_ENDPOINT + ) + assert ( + SubscriberClient._get_api_endpoint( + None, mock_client_cert_source, default_universe, "always" + ) + == SubscriberClient.DEFAULT_MTLS_ENDPOINT + ) + assert ( + SubscriberClient._get_api_endpoint(None, None, mock_universe, "never") + == mock_endpoint + ) + assert ( + SubscriberClient._get_api_endpoint(None, None, default_universe, "never") + == default_endpoint + ) + + with pytest.raises(MutualTLSChannelError) as excinfo: + SubscriberClient._get_api_endpoint( + None, mock_client_cert_source, mock_universe, "auto" + ) + assert ( + str(excinfo.value) + == "mTLS is not supported in any universe other than googleapis.com." + ) + + +def test__get_universe_domain(): + client_universe_domain = "foo.com" + universe_domain_env = "bar.com" + + assert ( + SubscriberClient._get_universe_domain( + client_universe_domain, universe_domain_env + ) + == client_universe_domain + ) + assert ( + SubscriberClient._get_universe_domain(None, universe_domain_env) + == universe_domain_env + ) + assert ( + SubscriberClient._get_universe_domain(None, None) + == SubscriberClient._DEFAULT_UNIVERSE + ) + + with pytest.raises(ValueError) as excinfo: + SubscriberClient._get_universe_domain("", None) + assert str(excinfo.value) == "Universe Domain cannot be an empty string." + + +@pytest.mark.parametrize( + "client_class,transport_class,transport_name", + [ + (SubscriberClient, transports.SubscriberGrpcTransport, "grpc"), + (SubscriberClient, transports.SubscriberRestTransport, "rest"), + ], +) +def test__validate_universe_domain(client_class, transport_class, transport_name): + client = client_class( + transport=transport_class(credentials=ga_credentials.AnonymousCredentials()) + ) + assert client._validate_universe_domain() == True + + # Test the case when universe is already validated. + assert client._validate_universe_domain() == True + + if transport_name == "grpc": + # Test the case where credentials are provided by the + # `local_channel_credentials`. The default universes in both match. + channel = grpc.secure_channel( + "http://localhost/", grpc.local_channel_credentials() + ) + client = client_class(transport=transport_class(channel=channel)) + assert client._validate_universe_domain() == True + + # Test the case where credentials do not exist: e.g. a transport is provided + # with no credentials. Validation should still succeed because there is no + # mismatch with non-existent credentials. + channel = grpc.secure_channel( + "http://localhost/", grpc.local_channel_credentials() + ) + transport = transport_class(channel=channel) + transport._credentials = None + client = client_class(transport=transport) + assert client._validate_universe_domain() == True + + # TODO: This is needed to cater for older versions of google-auth + # Make this test unconditional once the minimum supported version of + # google-auth becomes 2.23.0 or higher. + google_auth_major, google_auth_minor = [ + int(part) for part in google.auth.__version__.split(".")[0:2] + ] + if google_auth_major > 2 or (google_auth_major == 2 and google_auth_minor >= 23): + credentials = ga_credentials.AnonymousCredentials() + credentials._universe_domain = "foo.com" + # Test the case when there is a universe mismatch from the credentials. + client = client_class(transport=transport_class(credentials=credentials)) + with pytest.raises(ValueError) as excinfo: + client._validate_universe_domain() + assert ( + str(excinfo.value) + == "The configured universe domain (googleapis.com) does not match the universe domain found in the credentials (foo.com). If you haven't configured the universe domain explicitly, `googleapis.com` is the default." + ) + + # Test the case when there is a universe mismatch from the client. + # + # TODO: Make this test unconditional once the minimum supported version of + # google-api-core becomes 2.15.0 or higher. + api_core_major, api_core_minor = [ + int(part) for part in api_core_version.__version__.split(".")[0:2] + ] + if api_core_major > 2 or (api_core_major == 2 and api_core_minor >= 15): + client = client_class( + client_options={"universe_domain": "bar.com"}, + transport=transport_class( + credentials=ga_credentials.AnonymousCredentials(), + ), + ) + with pytest.raises(ValueError) as excinfo: + client._validate_universe_domain() + assert ( + str(excinfo.value) + == "The configured universe domain (bar.com) does not match the universe domain found in the credentials (googleapis.com). If you haven't configured the universe domain explicitly, `googleapis.com` is the default." + ) + + # Test that ValueError is raised if universe_domain is provided via client options and credentials is None + with pytest.raises(ValueError): + client._compare_universes("foo.bar", None) + + @pytest.mark.parametrize( "client_class,transport_name", [ @@ -211,12 +471,14 @@ def test_subscriber_client_get_transport_class(): ], ) @mock.patch.object( - SubscriberClient, "DEFAULT_ENDPOINT", modify_default_endpoint(SubscriberClient) + SubscriberClient, + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(SubscriberClient), ) @mock.patch.object( SubscriberAsyncClient, - "DEFAULT_ENDPOINT", - modify_default_endpoint(SubscriberAsyncClient), + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(SubscriberAsyncClient), ) def test_subscriber_client_client_options( client_class, transport_class, transport_name @@ -258,7 +520,9 @@ def test_subscriber_client_client_options( patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, @@ -288,15 +552,23 @@ def test_subscriber_client_client_options( # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has # unsupported value. with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): - with pytest.raises(MutualTLSChannelError): + with pytest.raises(MutualTLSChannelError) as excinfo: client = client_class(transport=transport_name) + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + ) # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. with mock.patch.dict( os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} ): - with pytest.raises(ValueError): + with pytest.raises(ValueError) as excinfo: client = client_class(transport=transport_name) + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) # Check the case quota_project_id is provided options = client_options.ClientOptions(quota_project_id="octopus") @@ -306,7 +578,9 @@ def test_subscriber_client_client_options( patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id="octopus", @@ -324,7 +598,9 @@ def test_subscriber_client_client_options( patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, @@ -356,12 +632,14 @@ def test_subscriber_client_client_options( ], ) @mock.patch.object( - SubscriberClient, "DEFAULT_ENDPOINT", modify_default_endpoint(SubscriberClient) + SubscriberClient, + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(SubscriberClient), ) @mock.patch.object( SubscriberAsyncClient, - "DEFAULT_ENDPOINT", - modify_default_endpoint(SubscriberAsyncClient), + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(SubscriberAsyncClient), ) @mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) def test_subscriber_client_mtls_env_auto( @@ -384,7 +662,9 @@ def test_subscriber_client_mtls_env_auto( if use_client_cert_env == "false": expected_client_cert_source = None - expected_host = client.DEFAULT_ENDPOINT + expected_host = client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ) else: expected_client_cert_source = client_cert_source_callback expected_host = client.DEFAULT_MTLS_ENDPOINT @@ -416,7 +696,9 @@ def test_subscriber_client_mtls_env_auto( return_value=client_cert_source_callback, ): if use_client_cert_env == "false": - expected_host = client.DEFAULT_ENDPOINT + expected_host = client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ) expected_client_cert_source = None else: expected_host = client.DEFAULT_MTLS_ENDPOINT @@ -450,7 +732,9 @@ def test_subscriber_client_mtls_env_auto( patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, @@ -536,6 +820,113 @@ def test_subscriber_client_get_mtls_endpoint_and_cert_source(client_class): assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT assert cert_source == mock_client_cert_source + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + client_class.get_mtls_endpoint_and_cert_source() + + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + ) + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} + ): + with pytest.raises(ValueError) as excinfo: + client_class.get_mtls_endpoint_and_cert_source() + + assert ( + str(excinfo.value) + == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + ) + + +@pytest.mark.parametrize("client_class", [SubscriberClient, SubscriberAsyncClient]) +@mock.patch.object( + SubscriberClient, + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(SubscriberClient), +) +@mock.patch.object( + SubscriberAsyncClient, + "_DEFAULT_ENDPOINT_TEMPLATE", + modify_default_endpoint_template(SubscriberAsyncClient), +) +def test_subscriber_client_client_api_endpoint(client_class): + mock_client_cert_source = client_cert_source_callback + api_override = "foo.com" + default_universe = SubscriberClient._DEFAULT_UNIVERSE + default_endpoint = SubscriberClient._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=default_universe + ) + mock_universe = "bar.com" + mock_endpoint = SubscriberClient._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=mock_universe + ) + + # If ClientOptions.api_endpoint is set and GOOGLE_API_USE_CLIENT_CERTIFICATE="true", + # use ClientOptions.api_endpoint as the api endpoint regardless. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch( + "google.auth.transport.requests.AuthorizedSession.configure_mtls_channel" + ): + options = client_options.ClientOptions( + client_cert_source=mock_client_cert_source, api_endpoint=api_override + ) + client = client_class( + client_options=options, + credentials=ga_credentials.AnonymousCredentials(), + ) + assert client.api_endpoint == api_override + + # If ClientOptions.api_endpoint is not set and GOOGLE_API_USE_MTLS_ENDPOINT="never", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with GDU as the api endpoint. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + client = client_class(credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == default_endpoint + + # If ClientOptions.api_endpoint is not set and GOOGLE_API_USE_MTLS_ENDPOINT="always", + # use the DEFAULT_MTLS_ENDPOINT as the api endpoint. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + client = client_class(credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + + # If ClientOptions.api_endpoint is not set, GOOGLE_API_USE_MTLS_ENDPOINT="auto" (default), + # GOOGLE_API_USE_CLIENT_CERTIFICATE="false" (default), default cert source doesn't exist, + # and ClientOptions.universe_domain="bar.com", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with universe domain as the api endpoint. + options = client_options.ClientOptions() + universe_exists = hasattr(options, "universe_domain") + if universe_exists: + options = client_options.ClientOptions(universe_domain=mock_universe) + client = client_class( + client_options=options, credentials=ga_credentials.AnonymousCredentials() + ) + else: + client = client_class( + client_options=options, credentials=ga_credentials.AnonymousCredentials() + ) + assert client.api_endpoint == ( + mock_endpoint if universe_exists else default_endpoint + ) + assert client.universe_domain == ( + mock_universe if universe_exists else default_universe + ) + + # If ClientOptions does not have a universe domain attribute and GOOGLE_API_USE_MTLS_ENDPOINT="never", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with GDU as the api endpoint. + options = client_options.ClientOptions() + if hasattr(options, "universe_domain"): + delattr(options, "universe_domain") + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + client = client_class( + client_options=options, credentials=ga_credentials.AnonymousCredentials() + ) + assert client.api_endpoint == default_endpoint + @pytest.mark.parametrize( "client_class,transport_class,transport_name", @@ -562,7 +953,9 @@ def test_subscriber_client_client_options_scopes( patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=["1", "2"], client_cert_source_for_mtls=None, quota_project_id=None, @@ -597,7 +990,9 @@ def test_subscriber_client_client_options_credentials_file( patched.assert_called_once_with( credentials=None, credentials_file="credentials.json", - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, @@ -650,7 +1045,9 @@ def test_subscriber_client_create_channel_credentials_file( patched.assert_called_once_with( credentials=None, credentials_file="credentials.json", - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, @@ -1789,7 +2186,7 @@ async def test_list_subscriptions_flattened_error_async(): def test_list_subscriptions_pager(transport_name: str = "grpc"): client = SubscriberClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), transport=transport_name, ) @@ -1841,7 +2238,7 @@ def test_list_subscriptions_pager(transport_name: str = "grpc"): def test_list_subscriptions_pages(transport_name: str = "grpc"): client = SubscriberClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), transport=transport_name, ) @@ -1885,7 +2282,7 @@ def test_list_subscriptions_pages(transport_name: str = "grpc"): @pytest.mark.asyncio async def test_list_subscriptions_async_pager(): client = SubscriberAsyncClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), ) # Mock the actual call within the gRPC stub, and fake the request. @@ -1937,7 +2334,7 @@ async def test_list_subscriptions_async_pager(): @pytest.mark.asyncio async def test_list_subscriptions_async_pages(): client = SubscriberAsyncClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), ) # Mock the actual call within the gRPC stub, and fake the request. @@ -2888,6 +3285,7 @@ def test_pull_flattened_error(): client.pull( pubsub.PullRequest(), subscription="subscription_value", + return_immediately=True, max_messages=1277, ) @@ -2941,6 +3339,7 @@ async def test_pull_flattened_error_async(): await client.pull( pubsub.PullRequest(), subscription="subscription_value", + return_immediately=True, max_messages=1277, ) @@ -3727,7 +4126,7 @@ async def test_list_snapshots_flattened_error_async(): def test_list_snapshots_pager(transport_name: str = "grpc"): client = SubscriberClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), transport=transport_name, ) @@ -3777,7 +4176,7 @@ def test_list_snapshots_pager(transport_name: str = "grpc"): def test_list_snapshots_pages(transport_name: str = "grpc"): client = SubscriberClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), transport=transport_name, ) @@ -3819,7 +4218,7 @@ def test_list_snapshots_pages(transport_name: str = "grpc"): @pytest.mark.asyncio async def test_list_snapshots_async_pager(): client = SubscriberAsyncClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), ) # Mock the actual call within the gRPC stub, and fake the request. @@ -3869,7 +4268,7 @@ async def test_list_snapshots_async_pager(): @pytest.mark.asyncio async def test_list_snapshots_async_pages(): client = SubscriberAsyncClient( - credentials=ga_credentials.AnonymousCredentials, + credentials=ga_credentials.AnonymousCredentials(), ) # Mock the actual call within the gRPC stub, and fake the request. @@ -6991,6 +7390,7 @@ def test_pull_rest_flattened(): # get truthy value for each flattened field mock_args = dict( subscription="subscription_value", + return_immediately=True, max_messages=1277, ) mock_args.update(sample_request) @@ -7029,6 +7429,7 @@ def test_pull_rest_flattened_error(transport: str = "rest"): client.pull( pubsub.PullRequest(), subscription="subscription_value", + return_immediately=True, max_messages=1277, ) @@ -8970,7 +9371,7 @@ def test_credentials_transport_error(): ) # It is an error to provide an api_key and a credential. - options = mock.Mock() + options = client_options.ClientOptions() options.api_key = "api_key" with pytest.raises(ValueError): client = SubscriberClient( @@ -10484,7 +10885,9 @@ def test_api_key_credentials(client_class, transport_class): patched.assert_called_once_with( credentials=mock_cred, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format( + UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE + ), scopes=None, client_cert_source_for_mtls=None, quota_project_id=None, diff --git a/tests/unit/pubsub_v1/publisher/test_publisher_client.py b/tests/unit/pubsub_v1/publisher/test_publisher_client.py index 5d6013654..cc8eda56c 100644 --- a/tests/unit/pubsub_v1/publisher/test_publisher_client.py +++ b/tests/unit/pubsub_v1/publisher/test_publisher_client.py @@ -317,6 +317,32 @@ def test_publish_with_ordering_key_uses_extended_retry_deadline(creds): _assert_retries_equal(batch_commit_retry, expected_retry) +def test_publish_with_ordering_key_with_no_retry(creds): + client = publisher.Client( + credentials=creds, + publisher_options=types.PublisherOptions(enable_message_ordering=True), + ) + + # Use mocks in lieu of the actual batch class. + batch = mock.Mock(spec=client._batch_class) + future = mock.sentinel.future + future.add_done_callback = mock.Mock(spec=["__call__"]) + batch.publish.return_value = future + + topic = "topic/path" + client._set_batch(topic, batch) + + # Actually mock the batch class now. + batch_class = mock.Mock(spec=(), return_value=batch) + client._set_batch_class(batch_class) + + future = client.publish(topic, b"foo", ordering_key="first", retry=None) + assert future is mock.sentinel.future + + # Check the retry settings used for the batch. + batch_class.assert_called_once() + + def test_publish_attrs_bytestring(creds): client = publisher.Client(credentials=creds) @@ -447,20 +473,12 @@ def test_stop(creds): def test_gapic_instance_method(creds): client = publisher.Client(credentials=creds) - transport_mock = mock.Mock(create_topic=mock.sentinel) - fake_create_topic_rpc = mock.Mock() - transport_mock._wrapped_methods = { - transport_mock.create_topic: fake_create_topic_rpc - } - patcher = mock.patch.object(client, "_transport", new=transport_mock) - topic = gapic_types.Topic(name="projects/foo/topics/bar") - - with patcher: + with mock.patch.object(client, "create_topic") as patched: client.create_topic(topic) - assert fake_create_topic_rpc.call_count == 1 - _, args, _ = fake_create_topic_rpc.mock_calls[0] + assert patched.call_count == 1 + _, args, _ = patched.mock_calls[0] assert args[0] == gapic_types.Topic(name="projects/foo/topics/bar")