fix: Update etcd-client-py to 0.3.0

changes/2014.feature.md

@@ -0,0 +1 @@
+Update `etcd-client-py` to 0.3.0

python.lock

@@ -44,7 +44,7 @@
 //     "coloredlogs~=15.0",
 //     "cryptography>=2.8",
 //     "dataclasses-json~=0.5.7",
-//     "etcd-client-py==0.2.4",
+//     "etcd-client-py==0.3.0",
 //     "faker~=13.12.0",
 //     "graphene~=3.3.0",
 //     "graypy==2.1.0",
@@ -935,36 +935,36 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "7abd327980258ec2ae980d2ff7fc32ede7448146b14d34c56bf0be074e2a149b",
-              "url": "https://files.pythonhosted.org/packages/29/e9/c5b62bb12c74d00d7bf07bfac9a5589ff4f2a8d2204290259e6a217cb037/boto3-1.34.77-py3-none-any.whl"
+              "hash": "bb8f433c04dcdffbd4a802df56c1c30f2be23b1161fd8fb45e4b76c1487ec122",
+              "url": "https://files.pythonhosted.org/packages/d1/6d/524832dcddf931a5c46d02349325bec061b807f779497247823fb3aeb106/boto3-1.34.80-py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "8ebed4fa5a3b84dd4037f28226985af00e00fb860d739fc8b1ed6381caa4b330",
-              "url": "https://files.pythonhosted.org/packages/b3/3b/eae53213118b7a8186325380a999bd10a39796798841d291128268835082/boto3-1.34.77.tar.gz"
+              "hash": "5627f6ecadb46fc7c9f8c368baf948f1b00a3fd2f8eb1275c254469853ad8fdb",
+              "url": "https://files.pythonhosted.org/packages/21/7b/7e5ef1c794f75f0255e4103f656d1d78d687862e69764d1ee51f5a958339/boto3-1.34.80.tar.gz"
             }
           ],
           "project_name": "boto3",
           "requires_dists": [
-            "botocore<1.35.0,>=1.34.77",
+            "botocore<1.35.0,>=1.34.80",
             "botocore[crt]<2.0a0,>=1.21.0; extra == \"crt\"",
             "jmespath<2.0.0,>=0.7.1",
             "s3transfer<0.11.0,>=0.10.0"
           ],
           "requires_python": ">=3.8",
-          "version": "1.34.77"
+          "version": "1.34.80"
         },
         {
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "6d6a402032ca0b89525212356a865397f8f2839683dd53d41b8cee1aa84b2b4b",
-              "url": "https://files.pythonhosted.org/packages/96/e0/cff4d2a817bb8fdab645e10ee6eaf66ccdf7ffda2de64b2b05bfbe20fe77/botocore-1.34.77-py3-none-any.whl"
+              "hash": "354a00f03faba52acc6f1a84fa4f035d48541633be98ccc24b59dc544f679f8b",
+              "url": "https://files.pythonhosted.org/packages/f8/32/908ce27648e93e3775b1aaf9c60d9f9155bc25711da6a78a36f58b4b54ce/botocore-1.34.80-py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "6dab60261cdbfb7d0059488ea39408d5522fad419c004ba5db3484e6df854ea8",
-              "url": "https://files.pythonhosted.org/packages/92/b8/8dcb4cd2ff47d1642eadba0781d0a64c488d80799e1dbb66f7bd224f0932/botocore-1.34.77.tar.gz"
+              "hash": "8402262e819f3d46df504bbd781e770858c0130b90f660699f75ef3a63abca5a",
+              "url": "https://files.pythonhosted.org/packages/b7/93/a41d69f847231dfc1e34402466ce6c615c4e0e98637e56aa72b099afd4bc/botocore-1.34.80.tar.gz"
             }
           ],
           "project_name": "botocore",
@@ -976,7 +976,7 @@
             "urllib3<1.27,>=1.25.4; python_version < \"3.10\""
           ],
           "requires_python": ">=3.8",
-          "version": "1.34.77"
+          "version": "1.34.80"
         },
         {
           "artifacts": [
@@ -1433,29 +1433,29 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "b0b93c4c83b27ca81152d12d2ba909d7c52f5487ec57108aaabfed42d9fc91ea",
-              "url": "https://files.pythonhosted.org/packages/38/52/910914c59d27b27ed928a25a26efb3b703f1af577781a0f6f2a608243518/etcd_client_py-0.2.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
+              "hash": "bdb463d17b6a2e8f437f5caff32eecc02f5b975df70a55bf11c9d8dc8c264b73",
+              "url": "https://files.pythonhosted.org/packages/48/b9/7d1898803f42db81f57f38d0ef6872a95c39b6ea3787c9b5b63d3d61ef45/etcd_client_py-0.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "e8333fe17ff949f1aa66456f4cac3f2495ae591ab88e20f0abb047cabecbbea4",
-              "url": "https://files.pythonhosted.org/packages/5e/6c/9725b1ec67f4c6169ee679300e2ec089c3d4a9114a14366db0ed3a69983d/etcd_client_py-0.2.4-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl"
+              "hash": "dd7b02d9f65360bd5256caa6a6af41335dd05231b987e3b051c7ba50c0edd921",
+              "url": "https://files.pythonhosted.org/packages/23/0f/1972ab4f96ab4330e6074a9e24d525ed8d45194d3baabdfd3d07e68719c7/etcd_client_py-0.3.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "738b0cabd1c22891a3889ed9c8120dcbfbe6284d3cd36dcd781cc9149a57cc81",
-              "url": "https://files.pythonhosted.org/packages/e0/12/2b96fc4d929741940de32a8b114f0412d56b4456485f1ed5516d649e900d/etcd_client_py-0.2.4.tar.gz"
+              "hash": "d2895945324a66a1c5f80cc28fd2daf87dc93c6b649cb82b9b28392ed190f0c7",
+              "url": "https://files.pythonhosted.org/packages/35/38/975c561fc973b053a8a4bed9fe9ba9def82fae14cd8e79a68088886a21e3/etcd_client_py-0.3.0.tar.gz"
             },
             {
               "algorithm": "sha256",
-              "hash": "ee83798b1f2cf916a47f9e194bdeeaff3ebfa87b6793b79a2ebbe1c847bd7a33",
-              "url": "https://files.pythonhosted.org/packages/e3/3f/1043da09fbfa364cb50f21a88f1b80bfc15e0713b4e0f4ca22f72a92140b/etcd_client_py-0.2.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"
+              "hash": "46bba57af0e2cb766375736caaebb9b84ca0e8d819144911de6add2bab28fdcc",
+              "url": "https://files.pythonhosted.org/packages/a8/bb/1c02626c7d2b6649653a1bc2aa8c4bfa39af210defd97d94d507e283049d/etcd_client_py-0.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"
             }
           ],
           "project_name": "etcd-client-py",
           "requires_dists": [],
           "requires_python": ">=3.10",
-          "version": "0.2.4"
+          "version": "0.3.0"
         },
         {
           "artifacts": [
@@ -4164,13 +4164,13 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "6b9d68a29aba1ee400c823d8e5fe88675282eb69d7211e72fe65dbe54b33daca",
-              "url": "https://files.pythonhosted.org/packages/ce/90/ef9e8bfa0ad79588b1bc2170cf01693c77570f715a4439631d19ac44647f/types_redis-4.6.0.20240311-py3-none-any.whl"
+              "hash": "a3b92760c49a034827a0c3825206728df4e61e981c1324099d4414335af4f52f",
+              "url": "https://files.pythonhosted.org/packages/a7/f9/9944b2480301c553c95dfd8b225c82ece5f3bf0756d9b97d678166c4491d/types_redis-4.6.0.20240409-py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "e049bbdff0e0a1f8e701b64636811291d21bff79bf1e7850850a44055224a85f",
-              "url": "https://files.pythonhosted.org/packages/e3/a4/80eb9b57eca7b189f7169d26a22ef82bf647802d3cb7d0353e244778756b/types-redis-4.6.0.20240311.tar.gz"
+              "hash": "ce217c279581d769df992c5b76d61c65425b0a679626048e633e643868eb881b",
+              "url": "https://files.pythonhosted.org/packages/6f/48/2f3ec8681c3ad9e9d21d4db16056caf9cda023f2df0b773f39f82d91efa0/types-redis-4.6.0.20240409.tar.gz"
             }
           ],
           "project_name": "types-redis",
@@ -4179,7 +4179,7 @@
             "types-pyOpenSSL"
           ],
           "requires_python": ">=3.8",
-          "version": "4.6.0.20240311"
+          "version": "4.6.0.20240409"
         },
         {
           "artifacts": [
@@ -4239,19 +4239,19 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475",
-              "url": "https://files.pythonhosted.org/packages/f9/de/dc04a3ea60b22624b51c703a84bbe0184abcd1d0b9bc8074b5d6b7ab90bb/typing_extensions-4.10.0-py3-none-any.whl"
+              "hash": "c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a",
+              "url": "https://files.pythonhosted.org/packages/01/f3/936e209267d6ef7510322191003885de524fc48d1b43269810cd589ceaf5/typing_extensions-4.11.0-py3-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb",
-              "url": "https://files.pythonhosted.org/packages/16/3a/0d26ce356c7465a19c9ea8814b960f8a36c3b0d07c323176620b7b483e44/typing_extensions-4.10.0.tar.gz"
+              "hash": "83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0",
+              "url": "https://files.pythonhosted.org/packages/f6/f3/b827b3ab53b4e3d8513914586dcca61c355fa2ce8252dea4da56e67bf8f2/typing_extensions-4.11.0.tar.gz"
             }
           ],
           "project_name": "typing-extensions",
           "requires_dists": [],
           "requires_python": ">=3.8",
-          "version": "4.10.0"
+          "version": "4.11.0"
         },
         {
           "artifacts": [
@@ -4571,7 +4571,7 @@
     "coloredlogs~=15.0",
     "cryptography>=2.8",
     "dataclasses-json~=0.5.7",
-    "etcd-client-py==0.2.4",
+    "etcd-client-py==0.3.0",
     "faker~=13.12.0",
     "graphene~=3.3.0",
     "graypy==2.1.0",

requirements.txt

@@ -93,4 +93,4 @@ types-tabulate
 backend.ai-krunner-alpine==5.2.0
 backend.ai-krunner-static-gnu==4.2.0
 
-etcd-client-py==0.2.4
+etcd-client-py==0.3.0

src/ai/backend/common/etcd.py

@@ -44,8 +44,8 @@
     CompareOp,
     CondVar,
     ConnectOptions,
-    GRpcStatusCode,
-    GRpcStatusError,
+    GRPCStatusCode,
+    GRPCStatusError,
     TxnOp,
     Watch,
 )
@@ -200,7 +200,9 @@ async def put(
         scope_prefix = self._merge_scope_prefix_map(scope_prefix_map)[scope]
         mangled_key = self._mangle_key(f"{_slash(scope_prefix)}{key}")
         async with self.etcd.connect() as communicator:
-            await communicator.put(mangled_key, str(val))
+            await communicator.put(
+                mangled_key.encode(self.encoding), str(val).encode(self.encoding)
+            )
 
     async def put_prefix(
         self,
@@ -238,7 +240,12 @@ def _flatten(prefix: str, inner_dict: NestedStrKeyedDict) -> None:
 
         actions = []
         for k, v in flattened_dict.items():
-            actions.append(TxnOp.put(self._mangle_key(f"{_slash(scope_prefix)}{k}"), str(v)))
+            actions.append(
+                TxnOp.put(
+                    self._mangle_key(f"{_slash(scope_prefix)}{k}").encode(self.encoding),
+                    str(v).encode(self.encoding),
+                )
+            )
 
         async with self.etcd.connect() as communicator:
             await communicator.txn(EtcdTransactionAction().and_then(actions).or_else([]))
@@ -264,7 +271,12 @@ async def put_dict(
 
         actions = []
         for k, v in flattened_dict_obj.items():
-            actions.append(TxnOp.put(self._mangle_key(f"{_slash(scope_prefix)}{k}"), str(v)))
+            actions.append(
+                TxnOp.put(
+                    self._mangle_key(f"{_slash(scope_prefix)}{k}").encode(self.encoding),
+                    str(v).encode(self.encoding),
+                )
+            )
 
         async with self.etcd.connect() as communicator:
             await communicator.txn(EtcdTransactionAction().and_then(actions).or_else([]))
@@ -308,9 +320,11 @@ async def get(
 
         async with self.etcd.connect() as communicator:
             for scope_prefix in scope_prefixes:
-                value = await communicator.get(self._mangle_key(f"{_slash(scope_prefix)}{key}"))
+                value = await communicator.get(
+                    self._mangle_key(f"{_slash(scope_prefix)}{key}").encode(self.encoding)
+                )
                 if value is not None:
-                    return value
+                    return bytes(value).decode(self.encoding)
         return None
 
     async def get_prefix(
@@ -377,8 +391,14 @@ async def get_prefix(
         async with self.etcd.connect() as communicator:
             for scope_prefix in scope_prefixes:
                 mangled_key_prefix = self._mangle_key(f"{_slash(scope_prefix)}{key_prefix}")
-                values = await communicator.get_prefix(mangled_key_prefix)
-                pair_sets.append([(self._demangle_key(k), v) for k, v in values.items()])
+                values = await communicator.get_prefix(mangled_key_prefix.encode(self.encoding))
+                pair_sets.append([
+                    (
+                        self._demangle_key(bytes(k).decode(self.encoding)),
+                        bytes(v).decode(self.encoding),
+                    )
+                    for k, v in values
+                ])
 
         pair_sets = [sorted(pairs, key=lambda x: x[0]) for pairs in pair_sets]
 
@@ -407,9 +427,15 @@ async def replace(
             result = await communicator.txn(
                 EtcdTransactionAction()
                 .when([
-                    Compare.value(mangled_key, CompareOp.EQUAL, initial_val),
+                    Compare.value(
+                        mangled_key.encode(self.encoding),
+                        CompareOp.EQUAL,
+                        initial_val.encode(self.encoding),
+                    ),
+                ])
+                .and_then([
+                    TxnOp.put(mangled_key.encode(self.encoding), new_val.encode(self.encoding))
                 ])
-                .and_then([TxnOp.put(mangled_key, new_val)])
                 .or_else([])
             )
 
@@ -425,7 +451,7 @@ async def delete(
         scope_prefix = self._merge_scope_prefix_map(scope_prefix_map)[scope]
         mangled_key = self._mangle_key(f"{_slash(scope_prefix)}{key}")
         async with self.etcd.connect() as communicator:
-            await communicator.delete(mangled_key)
+            await communicator.delete(mangled_key.encode(self.encoding))
 
     async def delete_multi(
         self,
@@ -438,7 +464,11 @@ async def delete_multi(
         async with self.etcd.connect() as communicator:
             actions = []
             for k in keys:
-                actions.append(TxnOp.delete(self._mangle_key(f"{_slash(scope_prefix)}{k}")))
+                actions.append(
+                    TxnOp.delete(
+                        self._mangle_key(f"{_slash(scope_prefix)}{k}").encode(self.encoding)
+                    )
+                )
             await communicator.txn(EtcdTransactionAction().and_then(actions).or_else([]))
 
     async def delete_prefix(
@@ -451,7 +481,7 @@ async def delete_prefix(
         scope_prefix = self._merge_scope_prefix_map(scope_prefix_map)[scope]
         mangled_key_prefix = self._mangle_key(f"{_slash(scope_prefix)}{key_prefix}")
         async with self.etcd.connect() as communicator:
-            await communicator.delete_prefix(mangled_key_prefix)
+            await communicator.delete_prefix(mangled_key_prefix.encode(self.encoding))
 
     async def _watch_impl(
         self,
@@ -471,7 +501,11 @@ async def _watch_impl(
                             ev = await asyncio.wait_for(iterator.__anext__(), wait_timeout)
                         except asyncio.TimeoutError:
                             pass
-                    yield Event(ev.key[scope_prefix_len:], ev.event, ev.value)
+                    yield Event(
+                        bytes(ev.key).decode(self.encoding)[scope_prefix_len:],
+                        ev.event,
+                        bytes(ev.value).decode(self.encoding),
+                    )
                     if once:
                         return
         finally:
@@ -498,7 +532,7 @@ async def watch(
             try:
                 async for ev in self._watch_impl(
                     lambda communicator: communicator.watch(
-                        mangled_key,
+                        mangled_key.encode(self.encoding),
                         ready_event=ready_event,
                     ),
                     scope_prefix_len,
@@ -508,10 +542,10 @@ async def watch(
                 ):
                     yield ev
                 ended_without_error = True
-            except GRpcStatusError as e:
+            except GRPCStatusError as e:
                 err_detail = e.args[0]
 
-                if err_detail["code"] == GRpcStatusCode.Unavailable:
+                if err_detail["code"] == GRPCStatusCode.Unavailable:
                     log.warning("watch(): error while connecting to Etcd server, retrying...")
                     await asyncio.sleep(self.watch_reconnect_intvl)
                     ended_without_error = False
@@ -538,7 +572,7 @@ async def watch_prefix(
             try:
                 async for ev in self._watch_impl(
                     lambda communicator: communicator.watch_prefix(
-                        mangled_key_prefix,
+                        mangled_key_prefix.encode(self.encoding),
                         ready_event=ready_event,
                     ),
                     scope_prefix_len,
@@ -548,10 +582,10 @@ async def watch_prefix(
                 ):
                     yield ev
                 ended_without_error = True
-            except GRpcStatusError as e:
+            except GRPCStatusError as e:
                 err_detail = e.args[0]
 
-                if err_detail["code"] == GRpcStatusCode.Unavailable:
+                if err_detail["code"] == GRPCStatusCode.Unavailable:
                     log.warning(
                         "watch_prefix(): error while connecting to Etcd server, retrying..."
                     )

src/ai/backend/common/lock.py

@@ -173,7 +173,7 @@ def __init__(
     async def __aenter__(self) -> EtcdCommunicator:
         self._etcd_client = self.etcd.etcd.with_lock(
             EtcdLockOption(
-                lock_name=self.lock_name,
+                lock_name=self.lock_name.encode("utf-8"),
                 timeout=self._timeout,
                 ttl=int(self._lifetime) if self._lifetime is not None else None,
             ),

src/ai/backend/manager/api/stream.py

@@ -40,7 +40,7 @@
 import zmq.asyncio
 from aiohttp import web
 from aiotools import adefer, apartial
-from etcd_client import GRpcStatusCode, GRpcStatusError
+from etcd_client import GRPCStatusCode, GRPCStatusError
 
 from ai.backend.common import redis_helper
 from ai.backend.common import validators as tx
@@ -700,10 +700,10 @@ async def stream_conn_tracker_gc(root_ctx: RootContext, app_ctx: PrivateContext)
                     await shared_config.etcd.get("config/idle/app-streaming-packet-timeout")
                     or "5m",
                 )
-            except GRpcStatusError as e:
+            except GRPCStatusError as e:
                 err_detail = e.args[0]
 
-                if err_detail["code"] == GRpcStatusCode.Unavailable:
+                if err_detail["code"] == GRPCStatusCode.Unavailable:
                     log.warning(
                         "stream_conn_tracker_gc(): error while connecting to Etcd server,"
                         " retrying..."