Jobergum/retry control plane (#608)

* Implement retry logic for control plane api requests * Fix import
vespa-engine · Oct 31, 2023 · 749b23a · 749b23a
1 parent 5ca79aa
commit 749b23a
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 17 deletions.
diff --git a/docs/sphinx/source/getting-started-pyvespa-cloud.ipynb b/docs/sphinx/source/getting-started-pyvespa-cloud.ipynb
@@ -97,8 +97,7 @@
     "res = requests.get(url=\"https://api.github.com/repos/vespa-engine/vespa/releases/latest\").json()\n",
     "os.environ[\"VERSION\"] = res[\"tag_name\"].replace(\"v\", \"\")\n",
     "!curl -fsSL https://github.com/vespa-engine/vespa/releases/download/v${VERSION}/vespa-cli_${VERSION}_linux_amd64.tar.gz | tar -zxf -\n",
-    "!ln -sf /content/vespa-cli_${VERSION}_linux_amd64/bin/vespa /bin/vespa\n",
-    "!which vespa"
+    "!ln -sf /content/vespa-cli_${VERSION}_linux_amd64/bin/vespa /bin/vespa"
    ]
   },
   {

diff --git a/vespa/deployment.py b/vespa/deployment.py
@@ -9,6 +9,7 @@
 from io import BytesIO
 from pathlib import Path
 from time import sleep, strftime, gmtime
+import random
 from typing import Tuple, Union, IO, Optional, List
 
 import docker
@@ -656,21 +657,34 @@ def _request(
         }
 
         body.seek(0)
-        self.connection.request(method, path, body, headers)
-        with self.connection.getresponse() as response:
-            parsed = json.load(response)
-            if response.status != 200:
-                raise RuntimeError(
-                    "Status code "
-                    + str(response.status)
-                    + " doing "
-                    + method
-                    + " at "
-                    + url
-                    + ":\n"
-                    + parsed["message"]
-                )
-            return parsed
+        retry_count = 0
+        max_retries = 3
+        while retry_count < max_retries:
+            try:
+                self.connection.request(method, path, body, headers)
+                with self.connection.getresponse() as response:
+                    parsed = json.load(response)
+                    if response.status != 200:
+                        raise RuntimeError(
+                            "Status code "
+                            + str(response.status)
+                            + " doing "
+                            + method
+                            + " at "
+                            + url
+                            + ":\n"
+                            + parsed["message"]
+                        )
+                    return parsed
+            except Exception as e:
+                retry_count += 1
+                if retry_count == max_retries:
+                    raise e
+                else:
+                    delay = min(2**retry_count, 1) + random.uniform(0, 1)
+                    sleep(delay)
+
+
 
     def get_mtls_endpoint(self, instance: Optional[str]="default", region: Optional[str]=None) -> str:
         if region is None: