Retry Pyxis requests on timeout

The gql RequestsHTTPTransport has retries enabled, but it doesn't retry
when Pyxis requests timeout, for some requests, Pyxis may take more time
than its gateway timeout value to process them, and it will return the
result with error in response. In such cases, retrying may work.

This is a workaround for JIRA: CLOUDWF-9342

freshmaker/pyxis_gql.py

@@ -22,6 +22,7 @@
 from functools import cached_property
 from typing import Optional
 
+import backoff
 import dogpile.cache
 from gql import Client, gql
 from gql.dsl import DSLQuery, DSLSchema, dsl_gql
@@ -45,6 +46,10 @@ def __init__(self, error: str | list[str], trace_id: Optional[str] = None):
         super().__init__(msg)
 
 
+class PyxisGQLRequestTimeout(PyxisGQLRequestError):
+    pass
+
+
 class PyxisGQL:
     region = dogpile.cache.make_region().configure(conf.dogpile_cache_backend)
 
@@ -73,6 +78,13 @@ def _get_graphql_schema(self):
         self._client.execute(query)
         return DSLSchema(self._client.schema)
 
+    @backoff.on_exception(
+        backoff.expo,
+        PyxisGQLRequestTimeout,
+        factor=30,
+        max_tries=3,
+        jitter=None,  # use deterministic backoff, do not apply random jitter
+    )
     def query(self, query_dsl):
         """Execute a GraphQL query with Domain Specific Language
 
@@ -85,6 +97,9 @@ def query(self, query_dsl):
         error = response[response_field_name]["error"]
         if error is not None:
             trace_id = self._client.transport.response_headers.get("trace_id", False)
+
+            if "Pyxis API was unable to fetch data from MongoDB in time" in error:
+                raise PyxisGQLRequestTimeout(error=error, trace_id=trace_id)
             raise PyxisGQLRequestError(error=error, trace_id=trace_id)
 
         return response

yum-packages.txt

@@ -2,6 +2,7 @@ git
 httpd
 mod_auth_gssapi
 mod_ssl
+python3-backoff
 python3-defusedxml
 python3-dogpile-cache
 python3-fedmsg