From 6327330a2021e2e9dae7dccfbd08a4a42a843e9f Mon Sep 17 00:00:00 2001 From: rahul yadav Date: Mon, 27 Oct 2025 11:45:01 +0530 Subject: [PATCH 1/9] fix: configure keepAlive time for gRPC TCP connections --- .../services/database_admin/transports/grpc.py | 1 + .../services/database_admin/transports/grpc_asyncio.py | 1 + .../services/instance_admin/transports/grpc.py | 1 + .../services/instance_admin/transports/grpc_asyncio.py | 1 + google/cloud/spanner_v1/services/spanner/transports/grpc.py | 1 + .../cloud/spanner_v1/services/spanner/transports/grpc_asyncio.py | 1 + 6 files changed, 6 insertions(+) diff --git a/google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc.py b/google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc.py index 8f31a1fb98..77aeafe15f 100644 --- a/google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc.py +++ b/google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc.py @@ -276,6 +276,7 @@ def __init__( options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) diff --git a/google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc_asyncio.py b/google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc_asyncio.py index 5171d84d40..b30f123623 100644 --- a/google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc_asyncio.py +++ b/google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc_asyncio.py @@ -325,6 +325,7 @@ def __init__( options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) diff --git a/google/cloud/spanner_admin_instance_v1/services/instance_admin/transports/grpc.py b/google/cloud/spanner_admin_instance_v1/services/instance_admin/transports/grpc.py index ee5b765210..d9971a4ad2 100644 --- a/google/cloud/spanner_admin_instance_v1/services/instance_admin/transports/grpc.py +++ b/google/cloud/spanner_admin_instance_v1/services/instance_admin/transports/grpc.py @@ -285,6 +285,7 @@ def __init__( options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) diff --git a/google/cloud/spanner_admin_instance_v1/services/instance_admin/transports/grpc_asyncio.py b/google/cloud/spanner_admin_instance_v1/services/instance_admin/transports/grpc_asyncio.py index f2df40d1f2..8d9adff29b 100644 --- a/google/cloud/spanner_admin_instance_v1/services/instance_admin/transports/grpc_asyncio.py +++ b/google/cloud/spanner_admin_instance_v1/services/instance_admin/transports/grpc_asyncio.py @@ -334,6 +334,7 @@ def __init__( options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) diff --git a/google/cloud/spanner_v1/services/spanner/transports/grpc.py b/google/cloud/spanner_v1/services/spanner/transports/grpc.py index 8b377d7725..f0bfd18586 100644 --- a/google/cloud/spanner_v1/services/spanner/transports/grpc.py +++ b/google/cloud/spanner_v1/services/spanner/transports/grpc.py @@ -267,6 +267,7 @@ def __init__( options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) diff --git a/google/cloud/spanner_v1/services/spanner/transports/grpc_asyncio.py b/google/cloud/spanner_v1/services/spanner/transports/grpc_asyncio.py index 2c6cec52a9..be00f6d305 100644 --- a/google/cloud/spanner_v1/services/spanner/transports/grpc_asyncio.py +++ b/google/cloud/spanner_v1/services/spanner/transports/grpc_asyncio.py @@ -315,6 +315,7 @@ def __init__( options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) From 5c84c7366a4f1695268828f0028589538b41259a Mon Sep 17 00:00:00 2001 From: rahul yadav Date: Mon, 27 Oct 2025 12:00:54 +0530 Subject: [PATCH 2/9] fix tests --- tests/unit/gapic/spanner_v1/test_spanner.py | 25 ++++++++++++--------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/unit/gapic/spanner_v1/test_spanner.py b/tests/unit/gapic/spanner_v1/test_spanner.py index 83d9d72f7f..46a2aaed56 100644 --- a/tests/unit/gapic/spanner_v1/test_spanner.py +++ b/tests/unit/gapic/spanner_v1/test_spanner.py @@ -1066,6 +1066,7 @@ def test_spanner_client_create_channel_credentials_file( options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) @@ -12180,6 +12181,7 @@ def test_spanner_transport_create_channel(transport_class, grpc_helpers): options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) @@ -12209,6 +12211,7 @@ def test_spanner_grpc_transport_client_cert_source_for_mtls(transport_class): options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) @@ -12416,11 +12419,12 @@ def test_spanner_transport_channel_mtls_with_client_cert_source(transport_class) scopes=None, ssl_credentials=mock_ssl_cred, quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), + ], + ) assert transport.grpc_channel == mock_grpc_channel assert transport._ssl_channel_credentials == mock_ssl_cred @@ -12460,11 +12464,12 @@ def test_spanner_transport_channel_mtls_with_adc(transport_class): scopes=None, ssl_credentials=mock_ssl_cred, quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), + ], + ) assert transport.grpc_channel == mock_grpc_channel From 4f0e6f516ff05d9cf6abee151889b1a38e6749e2 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 27 Oct 2025 06:33:21 +0000 Subject: [PATCH 3/9] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- tests/unit/gapic/spanner_v1/test_spanner.py | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/unit/gapic/spanner_v1/test_spanner.py b/tests/unit/gapic/spanner_v1/test_spanner.py index 46a2aaed56..80cb748024 100644 --- a/tests/unit/gapic/spanner_v1/test_spanner.py +++ b/tests/unit/gapic/spanner_v1/test_spanner.py @@ -12419,12 +12419,12 @@ def test_spanner_transport_channel_mtls_with_client_cert_source(transport_class) scopes=None, ssl_credentials=mock_ssl_cred, quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ("grpc.keepalive_time_ms", 120000), - ], - ) + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), + ], + ) assert transport.grpc_channel == mock_grpc_channel assert transport._ssl_channel_credentials == mock_ssl_cred @@ -12464,12 +12464,12 @@ def test_spanner_transport_channel_mtls_with_adc(transport_class): scopes=None, ssl_credentials=mock_ssl_cred, quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ("grpc.keepalive_time_ms", 120000), - ], - ) + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), + ], + ) assert transport.grpc_channel == mock_grpc_channel From 8d070a16b781c87f5dc4c354b527385e87b99f60 Mon Sep 17 00:00:00 2001 From: rahul yadav Date: Mon, 27 Oct 2025 12:29:53 +0530 Subject: [PATCH 4/9] fix tests --- .../test_database_admin.py | 25 +++++++++++-------- .../test_instance_admin.py | 5 ++++ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py b/tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py index f62b95c85d..3e60c11e46 100644 --- a/tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py +++ b/tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py @@ -1136,6 +1136,7 @@ def test_database_admin_client_create_channel_credentials_file( options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) @@ -23315,6 +23316,7 @@ def test_database_admin_transport_create_channel(transport_class, grpc_helpers): options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) @@ -23347,6 +23349,7 @@ def test_database_admin_grpc_transport_client_cert_source_for_mtls(transport_cla options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) @@ -23590,11 +23593,12 @@ def test_database_admin_transport_channel_mtls_with_client_cert_source(transport scopes=None, ssl_credentials=mock_ssl_cred, quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), + ], + ) assert transport.grpc_channel == mock_grpc_channel assert transport._ssl_channel_credentials == mock_ssl_cred @@ -23637,11 +23641,12 @@ def test_database_admin_transport_channel_mtls_with_adc(transport_class): scopes=None, ssl_credentials=mock_ssl_cred, quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), + ], + ) assert transport.grpc_channel == mock_grpc_channel diff --git a/tests/unit/gapic/spanner_admin_instance_v1/test_instance_admin.py b/tests/unit/gapic/spanner_admin_instance_v1/test_instance_admin.py index 52424e65d3..532014af96 100644 --- a/tests/unit/gapic/spanner_admin_instance_v1/test_instance_admin.py +++ b/tests/unit/gapic/spanner_admin_instance_v1/test_instance_admin.py @@ -1125,6 +1125,7 @@ def test_instance_admin_client_create_channel_credentials_file( options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) @@ -18621,6 +18622,7 @@ def test_instance_admin_transport_create_channel(transport_class, grpc_helpers): options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) @@ -18653,6 +18655,7 @@ def test_instance_admin_grpc_transport_client_cert_source_for_mtls(transport_cla options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) @@ -18881,6 +18884,7 @@ def test_instance_admin_transport_channel_mtls_with_client_cert_source(transport options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) assert transport.grpc_channel == mock_grpc_channel @@ -18928,6 +18932,7 @@ def test_instance_admin_transport_channel_mtls_with_adc(transport_class): options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), ], ) assert transport.grpc_channel == mock_grpc_channel From e909d222696a6b63014ffff5e0bda8794a313e7c Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 27 Oct 2025 07:02:32 +0000 Subject: [PATCH 5/9] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../test_database_admin.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py b/tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py index 3e60c11e46..e210da1d37 100644 --- a/tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py +++ b/tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py @@ -23593,12 +23593,12 @@ def test_database_admin_transport_channel_mtls_with_client_cert_source(transport scopes=None, ssl_credentials=mock_ssl_cred, quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ("grpc.keepalive_time_ms", 120000), - ], - ) + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), + ], + ) assert transport.grpc_channel == mock_grpc_channel assert transport._ssl_channel_credentials == mock_ssl_cred @@ -23641,12 +23641,12 @@ def test_database_admin_transport_channel_mtls_with_adc(transport_class): scopes=None, ssl_credentials=mock_ssl_cred, quota_project_id=None, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ("grpc.keepalive_time_ms", 120000), - ], - ) + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), + ], + ) assert transport.grpc_channel == mock_grpc_channel From 94d8f2a4d64505c68eb155bbfdfd36d3b7c4e6b0 Mon Sep 17 00:00:00 2001 From: Rahul Yadav Date: Tue, 28 Oct 2025 11:16:41 +0530 Subject: [PATCH 6/9] fix flaky abort retry error --- tests/unit/test_session.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 3b08cc5c65..862e0a1226 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -1702,8 +1702,9 @@ def unit_of_work(txn, *args, **kw): def _time(_results=[1, 2, 4, 8]): return _results.pop(0) - with mock.patch("time.time", _time): - with mock.patch("time.sleep") as sleep_mock: + with mock.patch("time.time", _time), \ + mock.patch("google.cloud.spanner_v1._helpers.random.random", return_value=0), \ + mock.patch("time.sleep") as sleep_mock: with self.assertRaises(Aborted): session.run_in_transaction(unit_of_work, timeout_secs=8) From 8d5866b0509b759479cd893a38aec62d1833ae36 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 28 Oct 2025 05:48:56 +0000 Subject: [PATCH 7/9] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- tests/unit/test_session.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 862e0a1226..7ea0458711 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -1702,11 +1702,11 @@ def unit_of_work(txn, *args, **kw): def _time(_results=[1, 2, 4, 8]): return _results.pop(0) - with mock.patch("time.time", _time), \ - mock.patch("google.cloud.spanner_v1._helpers.random.random", return_value=0), \ - mock.patch("time.sleep") as sleep_mock: - with self.assertRaises(Aborted): - session.run_in_transaction(unit_of_work, timeout_secs=8) + with mock.patch("time.time", _time), mock.patch( + "google.cloud.spanner_v1._helpers.random.random", return_value=0 + ), mock.patch("time.sleep") as sleep_mock: + with self.assertRaises(Aborted): + session.run_in_transaction(unit_of_work, timeout_secs=8) # unpacking call args into list call_args = [call_[0][0] for call_ in sleep_mock.call_args_list] From 5cf7e16fd09fcd33dfd41a746318a9c60fb18f4b Mon Sep 17 00:00:00 2001 From: Rahul Yadav Date: Tue, 28 Oct 2025 20:30:07 +0530 Subject: [PATCH 8/9] update owlbot to persist the changes --- owlbot.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/owlbot.py b/owlbot.py index cf460877a3..fb6c94ae79 100644 --- a/owlbot.py +++ b/owlbot.py @@ -17,6 +17,7 @@ from pathlib import Path import shutil from typing import List, Optional +import re import synthtool as s from synthtool import gcp @@ -185,6 +186,22 @@ def get_staging_dirs( )""" ) + count = s.replace( + [ + library / "google/cloud/spanner_v1/services/*/transports/grpc*", + library / "tests/unit/gapic/spanner_v1/*", + ], + "options=\\[.*?\\]", + """options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), + ]""", + flags=re.MULTILINE | re.DOTALL, + ) + if count < 1: + raise Exception("Expected replacements for gRPC channel options not made.") + s.move( library, excludes=[ @@ -201,6 +218,21 @@ def get_staging_dirs( for library in get_staging_dirs( spanner_admin_instance_default_version, "spanner_admin_instance" ): + count = s.replace( + [ + library / "google/cloud/spanner_admin_instance_v1/services/*/transports/grpc*", + library / "tests/unit/gapic/spanner_admin_instance_v1/*", + ], + "options=\\[.*?\\]", + """options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), + ]""", + flags=re.MULTILINE | re.DOTALL, + ) + if count < 1: + raise Exception("Expected replacements for gRPC channel options not made.") s.move( library, excludes=["google/cloud/spanner_admin_instance/**", "*.*", "docs/index.rst", "noxfile.py", "**/gapic_version.py", "testing/constraints-3.7.txt",], @@ -209,6 +241,21 @@ def get_staging_dirs( for library in get_staging_dirs( spanner_admin_database_default_version, "spanner_admin_database" ): + count = s.replace( + [ + library / "google/cloud/spanner_admin_database_v1/services/*/transports/grpc*", + library / "tests/unit/gapic/spanner_admin_database_v1/*", + ], + "options=\\[.*?\\]", + """options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ("grpc.keepalive_time_ms", 120000), + ]""", + flags=re.MULTILINE | re.DOTALL, + ) + if count < 1: + raise Exception("Expected replacements for gRPC channel options not made.") s.move( library, excludes=["google/cloud/spanner_admin_database/**", "*.*", "docs/index.rst", "noxfile.py", "**/gapic_version.py", "testing/constraints-3.7.txt",], From a09adb52686802a87e49cb8a835eb42fc7f49a06 Mon Sep 17 00:00:00 2001 From: Rahul Yadav Date: Tue, 28 Oct 2025 21:08:05 +0530 Subject: [PATCH 9/9] fix owlbot --- owlbot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/owlbot.py b/owlbot.py index fb6c94ae79..4547c4d2d0 100644 --- a/owlbot.py +++ b/owlbot.py @@ -191,7 +191,7 @@ def get_staging_dirs( library / "google/cloud/spanner_v1/services/*/transports/grpc*", library / "tests/unit/gapic/spanner_v1/*", ], - "options=\\[.*?\\]", + "^\s+options=\\[.*?\\]", """options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), @@ -223,7 +223,7 @@ def get_staging_dirs( library / "google/cloud/spanner_admin_instance_v1/services/*/transports/grpc*", library / "tests/unit/gapic/spanner_admin_instance_v1/*", ], - "options=\\[.*?\\]", + "^\s+options=\\[.*?\\]", """options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1), @@ -246,7 +246,7 @@ def get_staging_dirs( library / "google/cloud/spanner_admin_database_v1/services/*/transports/grpc*", library / "tests/unit/gapic/spanner_admin_database_v1/*", ], - "options=\\[.*?\\]", + "^\s+options=\\[.*?\\]", """options=[ ("grpc.max_send_message_length", -1), ("grpc.max_receive_message_length", -1),