From 8dbad869521924fc3f7d7dc2d4f5d7e9100874b3 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Tue, 29 Dec 2020 08:03:51 -0800 Subject: [PATCH] fix: update retry and timeout settings (#38) --- .flake8 | 1 + .pre-commit-config.yaml | 17 ++ CONTRIBUTING.rst | 21 +- docs/conf.py | 6 +- google/cloud/billing_v1/__init__.py | 4 +- .../services/cloud_billing/async_client.py | 92 ++++++++- .../services/cloud_billing/transports/base.py | 98 ++++++++- .../services/cloud_billing/transports/grpc.py | 8 + .../cloud_billing/transports/grpc_asyncio.py | 8 + .../services/cloud_catalog/async_client.py | 4 +- .../services/cloud_catalog/transports/base.py | 4 +- .../services/cloud_catalog/transports/grpc.py | 8 + .../cloud_catalog/transports/grpc_asyncio.py | 8 + google/cloud/billing_v1/types/__init__.py | 44 ++-- noxfile.py | 5 +- scripts/fixup_billing_v1_keywords.py | 190 ++++++++++++++++++ synth.metadata | 124 +++++++++++- .../gapic/billing_v1/test_cloud_billing.py | 8 + .../gapic/billing_v1/test_cloud_catalog.py | 8 + 19 files changed, 594 insertions(+), 64 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 scripts/fixup_billing_v1_keywords.py diff --git a/.flake8 b/.flake8 index ed93163..29227d4 100644 --- a/.flake8 +++ b/.flake8 @@ -26,6 +26,7 @@ exclude = *_pb2.py # Standard linting exemptions. + **/.nox/** __pycache__, .git, *.pyc, diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..a9024b1 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml +- repo: https://github.com/psf/black + rev: 19.10b0 + hooks: + - id: black +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.4 + hooks: + - id: flake8 diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index bdde88f..bef5946 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -21,8 +21,8 @@ In order to add a feature: - The feature must be documented in both the API and narrative documentation. -- The feature must work fully on the following CPython versions: 2.7, - 3.5, 3.6, 3.7 and 3.8 on both UNIX and Windows. +- The feature must work fully on the following CPython versions: + 3.6, 3.7, 3.8 and 3.9 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -111,6 +111,16 @@ Coding Style should point to the official ``googleapis`` checkout and the the branch should be the main branch on that remote (``master``). +- This repository contains configuration for the + `pre-commit `__ tool, which automates checking + our linters during a commit. If you have it installed on your ``$PATH``, + you can enable enforcing those checks via: + +.. code-block:: bash + + $ pre-commit install + pre-commit installed at .git/hooks/pre-commit + Exceptions to PEP8: - Many unit tests use a helper method, ``_call_fut`` ("FUT" is short for @@ -192,25 +202,24 @@ Supported Python Versions We support: -- `Python 3.5`_ - `Python 3.6`_ - `Python 3.7`_ - `Python 3.8`_ +- `Python 3.9`_ -.. _Python 3.5: https://docs.python.org/3.5/ .. _Python 3.6: https://docs.python.org/3.6/ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ +.. _Python 3.9: https://docs.python.org/3.9/ Supported versions can be found in our ``noxfile.py`` `config`_. .. _config: https://github.com/googleapis/python-billing/blob/master/noxfile.py -Python 2.7 support is deprecated. All code changes should maintain Python 2.7 compatibility until January 1, 2020. We also explicitly decided to support Python 3 beginning with version -3.5. Reasons for this include: +3.6. Reasons for this include: - Encouraging use of newest versions of Python 3 - Taking the lead of `prominent`_ open-source `projects`_ diff --git a/docs/conf.py b/docs/conf.py index 798112f..28af727 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -345,10 +345,10 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - "python": ("http://python.readthedocs.org/en/latest/", None), - "google-auth": ("https://google-auth.readthedocs.io/en/stable", None), + "python": ("https://python.readthedocs.org/en/latest/", None), + "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None), "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,), - "grpc": ("https://grpc.io/grpc/python/", None), + "grpc": ("https://grpc.github.io/grpc/python/", None), "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), } diff --git a/google/cloud/billing_v1/__init__.py b/google/cloud/billing_v1/__init__.py index b010229..1ca4234 100644 --- a/google/cloud/billing_v1/__init__.py +++ b/google/cloud/billing_v1/__init__.py @@ -44,7 +44,7 @@ "AggregationInfo", "BillingAccount", "Category", - "CloudCatalogClient", + "CloudBillingClient", "CreateBillingAccountRequest", "GetBillingAccountRequest", "GetProjectBillingInfoRequest", @@ -63,5 +63,5 @@ "Sku", "UpdateBillingAccountRequest", "UpdateProjectBillingInfoRequest", - "CloudBillingClient", + "CloudCatalogClient", ) diff --git a/google/cloud/billing_v1/services/cloud_billing/async_client.py b/google/cloud/billing_v1/services/cloud_billing/async_client.py index d04579c..ceab5c4 100644 --- a/google/cloud/billing_v1/services/cloud_billing/async_client.py +++ b/google/cloud/billing_v1/services/cloud_billing/async_client.py @@ -197,7 +197,15 @@ async def get_billing_account( # and friendly error handling. rpc = gapic_v1.method_async.wrap_method( self._client._transport.get_billing_account, - default_timeout=None, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, ) @@ -252,7 +260,15 @@ async def list_billing_accounts( # and friendly error handling. rpc = gapic_v1.method_async.wrap_method( self._client._transport.list_billing_accounts, - default_timeout=None, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, ) @@ -340,7 +356,15 @@ async def update_billing_account( # and friendly error handling. rpc = gapic_v1.method_async.wrap_method( self._client._transport.update_billing_account, - default_timeout=None, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, ) @@ -426,7 +450,7 @@ async def create_billing_account( # and friendly error handling. rpc = gapic_v1.method_async.wrap_method( self._client._transport.create_billing_account, - default_timeout=None, + default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, ) @@ -499,7 +523,15 @@ async def list_project_billing_info( # and friendly error handling. rpc = gapic_v1.method_async.wrap_method( self._client._transport.list_project_billing_info, - default_timeout=None, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, ) @@ -583,7 +615,15 @@ async def get_project_billing_info( # and friendly error handling. rpc = gapic_v1.method_async.wrap_method( self._client._transport.get_project_billing_info, - default_timeout=None, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, ) @@ -704,7 +744,15 @@ async def update_project_billing_info( # and friendly error handling. rpc = gapic_v1.method_async.wrap_method( self._client._transport.update_project_billing_info, - default_timeout=None, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, ) @@ -844,7 +892,15 @@ async def get_iam_policy( # and friendly error handling. rpc = gapic_v1.method_async.wrap_method( self._client._transport.get_iam_policy, - default_timeout=None, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, ) @@ -985,7 +1041,15 @@ async def set_iam_policy( # and friendly error handling. rpc = gapic_v1.method_async.wrap_method( self._client._transport.set_iam_policy, - default_timeout=None, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, ) @@ -1072,7 +1136,15 @@ async def test_iam_permissions( # and friendly error handling. rpc = gapic_v1.method_async.wrap_method( self._client._transport.test_iam_permissions, - default_timeout=None, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, ) diff --git a/google/cloud/billing_v1/services/cloud_billing/transports/base.py b/google/cloud/billing_v1/services/cloud_billing/transports/base.py index ee8d2d1..eaf061f 100644 --- a/google/cloud/billing_v1/services/cloud_billing/transports/base.py +++ b/google/cloud/billing_v1/services/cloud_billing/transports/base.py @@ -107,47 +107,125 @@ def _prep_wrapped_messages(self, client_info): # Precompute the wrapped methods. self._wrapped_methods = { self.get_billing_account: gapic_v1.method.wrap_method( - self.get_billing_account, default_timeout=None, client_info=client_info, + self.get_billing_account, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, + client_info=client_info, ), self.list_billing_accounts: gapic_v1.method.wrap_method( self.list_billing_accounts, - default_timeout=None, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, client_info=client_info, ), self.update_billing_account: gapic_v1.method.wrap_method( self.update_billing_account, - default_timeout=None, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, client_info=client_info, ), self.create_billing_account: gapic_v1.method.wrap_method( self.create_billing_account, - default_timeout=None, + default_timeout=60.0, client_info=client_info, ), self.list_project_billing_info: gapic_v1.method.wrap_method( self.list_project_billing_info, - default_timeout=None, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, client_info=client_info, ), self.get_project_billing_info: gapic_v1.method.wrap_method( self.get_project_billing_info, - default_timeout=None, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, client_info=client_info, ), self.update_project_billing_info: gapic_v1.method.wrap_method( self.update_project_billing_info, - default_timeout=None, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, client_info=client_info, ), self.get_iam_policy: gapic_v1.method.wrap_method( - self.get_iam_policy, default_timeout=None, client_info=client_info, + self.get_iam_policy, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, + client_info=client_info, ), self.set_iam_policy: gapic_v1.method.wrap_method( - self.set_iam_policy, default_timeout=None, client_info=client_info, + self.set_iam_policy, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, + client_info=client_info, ), self.test_iam_permissions: gapic_v1.method.wrap_method( self.test_iam_permissions, - default_timeout=None, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + ), + ), + default_timeout=60.0, client_info=client_info, ), } diff --git a/google/cloud/billing_v1/services/cloud_billing/transports/grpc.py b/google/cloud/billing_v1/services/cloud_billing/transports/grpc.py index 44511c8..3be41ab 100644 --- a/google/cloud/billing_v1/services/cloud_billing/transports/grpc.py +++ b/google/cloud/billing_v1/services/cloud_billing/transports/grpc.py @@ -149,6 +149,10 @@ def __init__( ssl_credentials=ssl_credentials, scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) self._ssl_channel_credentials = ssl_credentials else: @@ -167,6 +171,10 @@ def __init__( ssl_credentials=ssl_channel_credentials, scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) self._stubs = {} # type: Dict[str, Callable] diff --git a/google/cloud/billing_v1/services/cloud_billing/transports/grpc_asyncio.py b/google/cloud/billing_v1/services/cloud_billing/transports/grpc_asyncio.py index d4e337c..acadeb9 100644 --- a/google/cloud/billing_v1/services/cloud_billing/transports/grpc_asyncio.py +++ b/google/cloud/billing_v1/services/cloud_billing/transports/grpc_asyncio.py @@ -194,6 +194,10 @@ def __init__( ssl_credentials=ssl_credentials, scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) self._ssl_channel_credentials = ssl_credentials else: @@ -212,6 +216,10 @@ def __init__( ssl_credentials=ssl_channel_credentials, scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) # Run the base constructor. diff --git a/google/cloud/billing_v1/services/cloud_catalog/async_client.py b/google/cloud/billing_v1/services/cloud_catalog/async_client.py index 49b0d07..4b95b43 100644 --- a/google/cloud/billing_v1/services/cloud_catalog/async_client.py +++ b/google/cloud/billing_v1/services/cloud_catalog/async_client.py @@ -177,7 +177,7 @@ async def list_services( # and friendly error handling. rpc = gapic_v1.method_async.wrap_method( self._client._transport.list_services, - default_timeout=None, + default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, ) @@ -251,7 +251,7 @@ async def list_skus( # and friendly error handling. rpc = gapic_v1.method_async.wrap_method( self._client._transport.list_skus, - default_timeout=None, + default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, ) diff --git a/google/cloud/billing_v1/services/cloud_catalog/transports/base.py b/google/cloud/billing_v1/services/cloud_catalog/transports/base.py index 64179c6..88276fa 100644 --- a/google/cloud/billing_v1/services/cloud_catalog/transports/base.py +++ b/google/cloud/billing_v1/services/cloud_catalog/transports/base.py @@ -105,10 +105,10 @@ def _prep_wrapped_messages(self, client_info): # Precompute the wrapped methods. self._wrapped_methods = { self.list_services: gapic_v1.method.wrap_method( - self.list_services, default_timeout=None, client_info=client_info, + self.list_services, default_timeout=60.0, client_info=client_info, ), self.list_skus: gapic_v1.method.wrap_method( - self.list_skus, default_timeout=None, client_info=client_info, + self.list_skus, default_timeout=60.0, client_info=client_info, ), } diff --git a/google/cloud/billing_v1/services/cloud_catalog/transports/grpc.py b/google/cloud/billing_v1/services/cloud_catalog/transports/grpc.py index e41b101..f3e47c8 100644 --- a/google/cloud/billing_v1/services/cloud_catalog/transports/grpc.py +++ b/google/cloud/billing_v1/services/cloud_catalog/transports/grpc.py @@ -148,6 +148,10 @@ def __init__( ssl_credentials=ssl_credentials, scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) self._ssl_channel_credentials = ssl_credentials else: @@ -166,6 +170,10 @@ def __init__( ssl_credentials=ssl_channel_credentials, scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) self._stubs = {} # type: Dict[str, Callable] diff --git a/google/cloud/billing_v1/services/cloud_catalog/transports/grpc_asyncio.py b/google/cloud/billing_v1/services/cloud_catalog/transports/grpc_asyncio.py index a2b28ef..9d0e6a4 100644 --- a/google/cloud/billing_v1/services/cloud_catalog/transports/grpc_asyncio.py +++ b/google/cloud/billing_v1/services/cloud_catalog/transports/grpc_asyncio.py @@ -193,6 +193,10 @@ def __init__( ssl_credentials=ssl_credentials, scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) self._ssl_channel_credentials = ssl_credentials else: @@ -211,6 +215,10 @@ def __init__( ssl_credentials=ssl_channel_credentials, scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) # Run the base constructor. diff --git a/google/cloud/billing_v1/types/__init__.py b/google/cloud/billing_v1/types/__init__.py index 5287065..a0dbe47 100644 --- a/google/cloud/billing_v1/types/__init__.py +++ b/google/cloud/billing_v1/types/__init__.py @@ -15,18 +15,6 @@ # limitations under the License. # -from .cloud_catalog import ( - Service, - Sku, - Category, - PricingInfo, - PricingExpression, - AggregationInfo, - ListServicesRequest, - ListServicesResponse, - ListSkusRequest, - ListSkusResponse, -) from .cloud_billing import ( BillingAccount, ProjectBillingInfo, @@ -40,18 +28,20 @@ GetProjectBillingInfoRequest, UpdateProjectBillingInfoRequest, ) +from .cloud_catalog import ( + Service, + Sku, + Category, + PricingInfo, + PricingExpression, + AggregationInfo, + ListServicesRequest, + ListServicesResponse, + ListSkusRequest, + ListSkusResponse, +) __all__ = ( - "Service", - "Sku", - "Category", - "PricingInfo", - "PricingExpression", - "AggregationInfo", - "ListServicesRequest", - "ListServicesResponse", - "ListSkusRequest", - "ListSkusResponse", "BillingAccount", "ProjectBillingInfo", "GetBillingAccountRequest", @@ -63,4 +53,14 @@ "ListProjectBillingInfoResponse", "GetProjectBillingInfoRequest", "UpdateProjectBillingInfoRequest", + "Service", + "Sku", + "Category", + "PricingInfo", + "PricingExpression", + "AggregationInfo", + "ListServicesRequest", + "ListServicesResponse", + "ListSkusRequest", + "ListSkusResponse", ) diff --git a/noxfile.py b/noxfile.py index 3884b3b..a57e24b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -81,9 +81,8 @@ def default(session): session.run( "py.test", "--quiet", - "--cov=google.cloud.cloudbilling", - "--cov=google.cloud", - "--cov=tests.unit", + "--cov=google/cloud", + "--cov=tests/unit", "--cov-append", "--cov-config=.coveragerc", "--cov-report=", diff --git a/scripts/fixup_billing_v1_keywords.py b/scripts/fixup_billing_v1_keywords.py new file mode 100644 index 0000000..dcee79c --- /dev/null +++ b/scripts/fixup_billing_v1_keywords.py @@ -0,0 +1,190 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import os +import libcst as cst +import pathlib +import sys +from typing import (Any, Callable, Dict, List, Sequence, Tuple) + + +def partition( + predicate: Callable[[Any], bool], + iterator: Sequence[Any] +) -> Tuple[List[Any], List[Any]]: + """A stable, out-of-place partition.""" + results = ([], []) + + for i in iterator: + results[int(predicate(i))].append(i) + + # Returns trueList, falseList + return results[1], results[0] + + +class billingCallTransformer(cst.CSTTransformer): + CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') + METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { + 'create_billing_account': ('billing_account', ), + 'get_billing_account': ('name', ), + 'get_iam_policy': ('resource', 'options', ), + 'get_project_billing_info': ('name', ), + 'list_billing_accounts': ('page_size', 'page_token', 'filter', ), + 'list_project_billing_info': ('name', 'page_size', 'page_token', ), + 'list_services': ('page_size', 'page_token', ), + 'list_skus': ('parent', 'start_time', 'end_time', 'currency_code', 'page_size', 'page_token', ), + 'set_iam_policy': ('resource', 'policy', ), + 'test_iam_permissions': ('resource', 'permissions', ), + 'update_billing_account': ('name', 'account', 'update_mask', ), + 'update_project_billing_info': ('name', 'project_billing_info', ), + + } + + def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: + try: + key = original.func.attr.value + kword_params = self.METHOD_TO_PARAMS[key] + except (AttributeError, KeyError): + # Either not a method from the API or too convoluted to be sure. + return updated + + # If the existing code is valid, keyword args come after positional args. + # Therefore, all positional args must map to the first parameters. + args, kwargs = partition(lambda a: not bool(a.keyword), updated.args) + if any(k.keyword.value == "request" for k in kwargs): + # We've already fixed this file, don't fix it again. + return updated + + kwargs, ctrl_kwargs = partition( + lambda a: not a.keyword.value in self.CTRL_PARAMS, + kwargs + ) + + args, ctrl_args = args[:len(kword_params)], args[len(kword_params):] + ctrl_kwargs.extend(cst.Arg(value=a.value, keyword=cst.Name(value=ctrl)) + for a, ctrl in zip(ctrl_args, self.CTRL_PARAMS)) + + request_arg = cst.Arg( + value=cst.Dict([ + cst.DictElement( + cst.SimpleString("'{}'".format(name)), + cst.Element(value=arg.value) + ) + # Note: the args + kwargs looks silly, but keep in mind that + # the control parameters had to be stripped out, and that + # those could have been passed positionally or by keyword. + for name, arg in zip(kword_params, args + kwargs)]), + keyword=cst.Name("request") + ) + + return updated.with_changes( + args=[request_arg] + ctrl_kwargs + ) + + +def fix_files( + in_dir: pathlib.Path, + out_dir: pathlib.Path, + *, + transformer=billingCallTransformer(), +): + """Duplicate the input dir to the output dir, fixing file method calls. + + Preconditions: + * in_dir is a real directory + * out_dir is a real, empty directory + """ + pyfile_gen = ( + pathlib.Path(os.path.join(root, f)) + for root, _, files in os.walk(in_dir) + for f in files if os.path.splitext(f)[1] == ".py" + ) + + for fpath in pyfile_gen: + with open(fpath, 'r') as f: + src = f.read() + + # Parse the code and insert method call fixes. + tree = cst.parse_module(src) + updated = tree.visit(transformer) + + # Create the path and directory structure for the new file. + updated_path = out_dir.joinpath(fpath.relative_to(in_dir)) + updated_path.parent.mkdir(parents=True, exist_ok=True) + + # Generate the updated source file at the corresponding path. + with open(updated_path, 'w') as f: + f.write(updated.code) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description="""Fix up source that uses the billing client library. + +The existing sources are NOT overwritten but are copied to output_dir with changes made. + +Note: This tool operates at a best-effort level at converting positional + parameters in client method calls to keyword based parameters. + Cases where it WILL FAIL include + A) * or ** expansion in a method call. + B) Calls via function or method alias (includes free function calls) + C) Indirect or dispatched calls (e.g. the method is looked up dynamically) + + These all constitute false negatives. The tool will also detect false + positives when an API method shares a name with another method. +""") + parser.add_argument( + '-d', + '--input-directory', + required=True, + dest='input_dir', + help='the input directory to walk for python files to fix up', + ) + parser.add_argument( + '-o', + '--output-directory', + required=True, + dest='output_dir', + help='the directory to output files fixed via un-flattening', + ) + args = parser.parse_args() + input_dir = pathlib.Path(args.input_dir) + output_dir = pathlib.Path(args.output_dir) + if not input_dir.is_dir(): + print( + f"input directory '{input_dir}' does not exist or is not a directory", + file=sys.stderr, + ) + sys.exit(-1) + + if not output_dir.is_dir(): + print( + f"output directory '{output_dir}' does not exist or is not a directory", + file=sys.stderr, + ) + sys.exit(-1) + + if os.listdir(output_dir): + print( + f"output directory '{output_dir}' is not empty", + file=sys.stderr, + ) + sys.exit(-1) + + fix_files(input_dir, output_dir) diff --git a/synth.metadata b/synth.metadata index 59b0ee0..bd872ed 100644 --- a/synth.metadata +++ b/synth.metadata @@ -3,23 +3,139 @@ { "git": { "name": ".", - "remote": "git@github.com:googleapis/python-billing", - "sha": "a9da72a796559ba13b95104cb719ba04300375ac" + "remote": "https://github.com/googleapis/python-billing.git", + "sha": "e7f139f44b6d5fc8f77b8a321bea39261edeb7f3" + } + }, + { + "git": { + "name": "googleapis", + "remote": "https://github.com/googleapis/googleapis.git", + "sha": "69697504d9eba1d064820c3085b4750767be6d08", + "internalRef": "348952930" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "d5fc0bcf9ea9789c5b0e3154a9e3b29e5cea6116" + "sha": "373861061648b5fe5e0ac4f8a38b32d639ee93e4" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "d5fc0bcf9ea9789c5b0e3154a9e3b29e5cea6116" + "sha": "373861061648b5fe5e0ac4f8a38b32d639ee93e4" + } + } + ], + "destinations": [ + { + "client": { + "source": "googleapis", + "apiName": "billing", + "apiVersion": "v1", + "language": "python", + "generator": "bazel" } } + ], + "generatedFiles": [ + ".flake8", + ".github/CONTRIBUTING.md", + ".github/ISSUE_TEMPLATE/bug_report.md", + ".github/ISSUE_TEMPLATE/feature_request.md", + ".github/ISSUE_TEMPLATE/support_request.md", + ".github/PULL_REQUEST_TEMPLATE.md", + ".github/release-please.yml", + ".github/snippet-bot.yml", + ".gitignore", + ".kokoro/build.sh", + ".kokoro/continuous/common.cfg", + ".kokoro/continuous/continuous.cfg", + ".kokoro/docker/docs/Dockerfile", + ".kokoro/docker/docs/fetch_gpg_keys.sh", + ".kokoro/docs/common.cfg", + ".kokoro/docs/docs-presubmit.cfg", + ".kokoro/docs/docs.cfg", + ".kokoro/populate-secrets.sh", + ".kokoro/presubmit/common.cfg", + ".kokoro/presubmit/presubmit.cfg", + ".kokoro/publish-docs.sh", + ".kokoro/release.sh", + ".kokoro/release/common.cfg", + ".kokoro/release/release.cfg", + ".kokoro/samples/lint/common.cfg", + ".kokoro/samples/lint/continuous.cfg", + ".kokoro/samples/lint/periodic.cfg", + ".kokoro/samples/lint/presubmit.cfg", + ".kokoro/samples/python3.6/common.cfg", + ".kokoro/samples/python3.6/continuous.cfg", + ".kokoro/samples/python3.6/periodic.cfg", + ".kokoro/samples/python3.6/presubmit.cfg", + ".kokoro/samples/python3.7/common.cfg", + ".kokoro/samples/python3.7/continuous.cfg", + ".kokoro/samples/python3.7/periodic.cfg", + ".kokoro/samples/python3.7/presubmit.cfg", + ".kokoro/samples/python3.8/common.cfg", + ".kokoro/samples/python3.8/continuous.cfg", + ".kokoro/samples/python3.8/periodic.cfg", + ".kokoro/samples/python3.8/presubmit.cfg", + ".kokoro/test-samples.sh", + ".kokoro/trampoline.sh", + ".kokoro/trampoline_v2.sh", + ".pre-commit-config.yaml", + ".trampolinerc", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.rst", + "LICENSE", + "MANIFEST.in", + "docs/_static/custom.css", + "docs/_templates/layout.html", + "docs/billing_v1/services.rst", + "docs/billing_v1/types.rst", + "docs/conf.py", + "docs/multiprocessing.rst", + "google/cloud/billing/__init__.py", + "google/cloud/billing/py.typed", + "google/cloud/billing_v1/__init__.py", + "google/cloud/billing_v1/py.typed", + "google/cloud/billing_v1/services/__init__.py", + "google/cloud/billing_v1/services/cloud_billing/__init__.py", + "google/cloud/billing_v1/services/cloud_billing/async_client.py", + "google/cloud/billing_v1/services/cloud_billing/client.py", + "google/cloud/billing_v1/services/cloud_billing/pagers.py", + "google/cloud/billing_v1/services/cloud_billing/transports/__init__.py", + "google/cloud/billing_v1/services/cloud_billing/transports/base.py", + "google/cloud/billing_v1/services/cloud_billing/transports/grpc.py", + "google/cloud/billing_v1/services/cloud_billing/transports/grpc_asyncio.py", + "google/cloud/billing_v1/services/cloud_catalog/__init__.py", + "google/cloud/billing_v1/services/cloud_catalog/async_client.py", + "google/cloud/billing_v1/services/cloud_catalog/client.py", + "google/cloud/billing_v1/services/cloud_catalog/pagers.py", + "google/cloud/billing_v1/services/cloud_catalog/transports/__init__.py", + "google/cloud/billing_v1/services/cloud_catalog/transports/base.py", + "google/cloud/billing_v1/services/cloud_catalog/transports/grpc.py", + "google/cloud/billing_v1/services/cloud_catalog/transports/grpc_asyncio.py", + "google/cloud/billing_v1/types/__init__.py", + "google/cloud/billing_v1/types/cloud_billing.py", + "google/cloud/billing_v1/types/cloud_catalog.py", + "mypy.ini", + "noxfile.py", + "renovate.json", + "scripts/decrypt-secrets.sh", + "scripts/fixup_billing_v1_keywords.py", + "scripts/readme-gen/readme_gen.py", + "scripts/readme-gen/templates/README.tmpl.rst", + "scripts/readme-gen/templates/auth.tmpl.rst", + "scripts/readme-gen/templates/auth_api_key.tmpl.rst", + "scripts/readme-gen/templates/install_deps.tmpl.rst", + "scripts/readme-gen/templates/install_portaudio.tmpl.rst", + "setup.cfg", + "testing/.gitignore", + "tests/unit/gapic/billing_v1/__init__.py", + "tests/unit/gapic/billing_v1/test_cloud_billing.py", + "tests/unit/gapic/billing_v1/test_cloud_catalog.py" ] } \ No newline at end of file diff --git a/tests/unit/gapic/billing_v1/test_cloud_billing.py b/tests/unit/gapic/billing_v1/test_cloud_billing.py index 9543ca6..6ee2c15 100644 --- a/tests/unit/gapic/billing_v1/test_cloud_billing.py +++ b/tests/unit/gapic/billing_v1/test_cloud_billing.py @@ -3068,6 +3068,10 @@ def test_cloud_billing_transport_channel_mtls_with_client_cert_source(transport_ scopes=("https://www.googleapis.com/auth/cloud-platform",), ssl_credentials=mock_ssl_cred, quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) assert transport.grpc_channel == mock_grpc_channel assert transport._ssl_channel_credentials == mock_ssl_cred @@ -3106,6 +3110,10 @@ def test_cloud_billing_transport_channel_mtls_with_adc(transport_class): scopes=("https://www.googleapis.com/auth/cloud-platform",), ssl_credentials=mock_ssl_cred, quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) assert transport.grpc_channel == mock_grpc_channel diff --git a/tests/unit/gapic/billing_v1/test_cloud_catalog.py b/tests/unit/gapic/billing_v1/test_cloud_catalog.py index cc9acb1..0ac2a31 100644 --- a/tests/unit/gapic/billing_v1/test_cloud_catalog.py +++ b/tests/unit/gapic/billing_v1/test_cloud_catalog.py @@ -1189,6 +1189,10 @@ def test_cloud_catalog_transport_channel_mtls_with_client_cert_source(transport_ scopes=("https://www.googleapis.com/auth/cloud-platform",), ssl_credentials=mock_ssl_cred, quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) assert transport.grpc_channel == mock_grpc_channel assert transport._ssl_channel_credentials == mock_ssl_cred @@ -1227,6 +1231,10 @@ def test_cloud_catalog_transport_channel_mtls_with_adc(transport_class): scopes=("https://www.googleapis.com/auth/cloud-platform",), ssl_credentials=mock_ssl_cred, quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) assert transport.grpc_channel == mock_grpc_channel