fix: configure keepAlive time for gRPC TCP connections

google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc.py

@@ -276,6 +276,7 @@ def __init__(
                 options=[
                     ("grpc.max_send_message_length", -1),
                     ("grpc.max_receive_message_length", -1),
+                    ("grpc.keepalive_time_ms", 120000),
                 ],
             )
 

google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc_asyncio.py

@@ -325,6 +325,7 @@ def __init__(
                 options=[
                     ("grpc.max_send_message_length", -1),
                     ("grpc.max_receive_message_length", -1),
+                    ("grpc.keepalive_time_ms", 120000),
                 ],
             )
 

google/cloud/spanner_admin_instance_v1/services/instance_admin/transports/grpc.py

@@ -285,6 +285,7 @@ def __init__(
                 options=[
                     ("grpc.max_send_message_length", -1),
                     ("grpc.max_receive_message_length", -1),
+                    ("grpc.keepalive_time_ms", 120000),
                 ],
             )
 

google/cloud/spanner_admin_instance_v1/services/instance_admin/transports/grpc_asyncio.py

@@ -334,6 +334,7 @@ def __init__(
                 options=[
                     ("grpc.max_send_message_length", -1),
                     ("grpc.max_receive_message_length", -1),
+                    ("grpc.keepalive_time_ms", 120000),
                 ],
             )
 

google/cloud/spanner_v1/services/spanner/transports/grpc.py

@@ -267,6 +267,7 @@ def __init__(
                 options=[
                     ("grpc.max_send_message_length", -1),
                     ("grpc.max_receive_message_length", -1),
+                    ("grpc.keepalive_time_ms", 120000),
                 ],
             )
 

google/cloud/spanner_v1/services/spanner/transports/grpc_asyncio.py

@@ -315,6 +315,7 @@ def __init__(
                 options=[
                     ("grpc.max_send_message_length", -1),
                     ("grpc.max_receive_message_length", -1),
+                    ("grpc.keepalive_time_ms", 120000),
                 ],
             )
 

owlbot.py

@@ -17,6 +17,7 @@
 from pathlib import Path
 import shutil
 from typing import List, Optional
+import re
 
 import synthtool as s
 from synthtool import gcp
@@ -185,6 +186,22 @@ def get_staging_dirs(
         )"""
     )
 
+    count = s.replace(
+        [
+            library / "google/cloud/spanner_v1/services/*/transports/grpc*",
+            library / "tests/unit/gapic/spanner_v1/*",
+        ],
+        "^\s+options=\\[.*?\\]",
+        """options=[
+                    ("grpc.max_send_message_length", -1),
+                    ("grpc.max_receive_message_length", -1),
+                    ("grpc.keepalive_time_ms", 120000),
+                ]""",
+        flags=re.MULTILINE | re.DOTALL,
+    )
+    if count < 1:
+        raise Exception("Expected replacements for gRPC channel options not made.")
+
     s.move(
         library,
         excludes=[
@@ -201,6 +218,21 @@ def get_staging_dirs(
 for library in get_staging_dirs(
     spanner_admin_instance_default_version, "spanner_admin_instance"
 ):
+    count = s.replace(
+        [
+            library / "google/cloud/spanner_admin_instance_v1/services/*/transports/grpc*",
+            library / "tests/unit/gapic/spanner_admin_instance_v1/*",
+        ],
+        "^\s+options=\\[.*?\\]",
+        """options=[
+                    ("grpc.max_send_message_length", -1),
+                    ("grpc.max_receive_message_length", -1),
+                    ("grpc.keepalive_time_ms", 120000),
+                ]""",
+        flags=re.MULTILINE | re.DOTALL,
+    )
+    if count < 1:
+        raise Exception("Expected replacements for gRPC channel options not made.")
     s.move(
         library,
         excludes=["google/cloud/spanner_admin_instance/**", "*.*", "docs/index.rst", "noxfile.py", "**/gapic_version.py", "testing/constraints-3.7.txt",],
@@ -209,6 +241,21 @@ def get_staging_dirs(
 for library in get_staging_dirs(
     spanner_admin_database_default_version, "spanner_admin_database"
 ):
+    count = s.replace(
+        [
+            library / "google/cloud/spanner_admin_database_v1/services/*/transports/grpc*",
+            library / "tests/unit/gapic/spanner_admin_database_v1/*",
+        ],
+        "^\s+options=\\[.*?\\]",
+        """options=[
+                    ("grpc.max_send_message_length", -1),
+                    ("grpc.max_receive_message_length", -1),
+                    ("grpc.keepalive_time_ms", 120000),
+                ]""",
+        flags=re.MULTILINE | re.DOTALL,
+    )
+    if count < 1:
+        raise Exception("Expected replacements for gRPC channel options not made.")
     s.move(
         library,
         excludes=["google/cloud/spanner_admin_database/**", "*.*", "docs/index.rst", "noxfile.py", "**/gapic_version.py", "testing/constraints-3.7.txt",],

tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py

@@ -1136,6 +1136,7 @@ def test_database_admin_client_create_channel_credentials_file(
             options=[
                 ("grpc.max_send_message_length", -1),
                 ("grpc.max_receive_message_length", -1),
+                ("grpc.keepalive_time_ms", 120000),
             ],
         )
 
@@ -23315,6 +23316,7 @@ def test_database_admin_transport_create_channel(transport_class, grpc_helpers):
             options=[
                 ("grpc.max_send_message_length", -1),
                 ("grpc.max_receive_message_length", -1),
+                ("grpc.keepalive_time_ms", 120000),
             ],
         )
 
@@ -23347,6 +23349,7 @@ def test_database_admin_grpc_transport_client_cert_source_for_mtls(transport_cla
             options=[
                 ("grpc.max_send_message_length", -1),
                 ("grpc.max_receive_message_length", -1),
+                ("grpc.keepalive_time_ms", 120000),
             ],
         )
 
@@ -23593,6 +23596,7 @@ def test_database_admin_transport_channel_mtls_with_client_cert_source(transport
                 options=[
                     ("grpc.max_send_message_length", -1),
                     ("grpc.max_receive_message_length", -1),
+                    ("grpc.keepalive_time_ms", 120000),
                 ],
             )
             assert transport.grpc_channel == mock_grpc_channel
@@ -23640,6 +23644,7 @@ def test_database_admin_transport_channel_mtls_with_adc(transport_class):
                 options=[
                     ("grpc.max_send_message_length", -1),
                     ("grpc.max_receive_message_length", -1),
+                    ("grpc.keepalive_time_ms", 120000),
                 ],
             )
             assert transport.grpc_channel == mock_grpc_channel

tests/unit/gapic/spanner_admin_instance_v1/test_instance_admin.py

@@ -1125,6 +1125,7 @@ def test_instance_admin_client_create_channel_credentials_file(
             options=[
                 ("grpc.max_send_message_length", -1),
                 ("grpc.max_receive_message_length", -1),
+                ("grpc.keepalive_time_ms", 120000),
             ],
         )
 
@@ -18621,6 +18622,7 @@ def test_instance_admin_transport_create_channel(transport_class, grpc_helpers):
             options=[
                 ("grpc.max_send_message_length", -1),
                 ("grpc.max_receive_message_length", -1),
+                ("grpc.keepalive_time_ms", 120000),
             ],
         )
 
@@ -18653,6 +18655,7 @@ def test_instance_admin_grpc_transport_client_cert_source_for_mtls(transport_cla
             options=[
                 ("grpc.max_send_message_length", -1),
                 ("grpc.max_receive_message_length", -1),
+                ("grpc.keepalive_time_ms", 120000),
             ],
         )
 
@@ -18881,6 +18884,7 @@ def test_instance_admin_transport_channel_mtls_with_client_cert_source(transport
                 options=[
                     ("grpc.max_send_message_length", -1),
                     ("grpc.max_receive_message_length", -1),
+                    ("grpc.keepalive_time_ms", 120000),
                 ],
             )
             assert transport.grpc_channel == mock_grpc_channel
@@ -18928,6 +18932,7 @@ def test_instance_admin_transport_channel_mtls_with_adc(transport_class):
                 options=[
                     ("grpc.max_send_message_length", -1),
                     ("grpc.max_receive_message_length", -1),
+                    ("grpc.keepalive_time_ms", 120000),
                 ],
             )
             assert transport.grpc_channel == mock_grpc_channel

tests/unit/gapic/spanner_v1/test_spanner.py

@@ -1066,6 +1066,7 @@ def test_spanner_client_create_channel_credentials_file(
             options=[
                 ("grpc.max_send_message_length", -1),
                 ("grpc.max_receive_message_length", -1),
+                ("grpc.keepalive_time_ms", 120000),
             ],
         )
 
@@ -12180,6 +12181,7 @@ def test_spanner_transport_create_channel(transport_class, grpc_helpers):
             options=[
                 ("grpc.max_send_message_length", -1),
                 ("grpc.max_receive_message_length", -1),
+                ("grpc.keepalive_time_ms", 120000),
             ],
         )
 
@@ -12209,6 +12211,7 @@ def test_spanner_grpc_transport_client_cert_source_for_mtls(transport_class):
             options=[
                 ("grpc.max_send_message_length", -1),
                 ("grpc.max_receive_message_length", -1),
+                ("grpc.keepalive_time_ms", 120000),
             ],
         )
 
@@ -12419,6 +12422,7 @@ def test_spanner_transport_channel_mtls_with_client_cert_source(transport_class)
                 options=[
                     ("grpc.max_send_message_length", -1),
                     ("grpc.max_receive_message_length", -1),
+                    ("grpc.keepalive_time_ms", 120000),
                 ],
             )
             assert transport.grpc_channel == mock_grpc_channel
@@ -12463,6 +12467,7 @@ def test_spanner_transport_channel_mtls_with_adc(transport_class):
                 options=[
                     ("grpc.max_send_message_length", -1),
                     ("grpc.max_receive_message_length", -1),
+                    ("grpc.keepalive_time_ms", 120000),
                 ],
             )
             assert transport.grpc_channel == mock_grpc_channel

tests/unit/test_session.py

@@ -1702,10 +1702,11 @@ def unit_of_work(txn, *args, **kw):
         def _time(_results=[1, 2, 4, 8]):
             return _results.pop(0)
 
-        with mock.patch("time.time", _time):
-            with mock.patch("time.sleep") as sleep_mock:
-                with self.assertRaises(Aborted):
-                    session.run_in_transaction(unit_of_work, timeout_secs=8)
+        with mock.patch("time.time", _time), mock.patch(
+            "google.cloud.spanner_v1._helpers.random.random", return_value=0
+        ), mock.patch("time.sleep") as sleep_mock:
+            with self.assertRaises(Aborted):
+                session.run_in_transaction(unit_of_work, timeout_secs=8)
 
         # unpacking call args into list
         call_args = [call_[0][0] for call_ in sleep_mock.call_args_list]