Add MaxErrorRetry to k8s config [ci fast]

Co-authored-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
Signed-off-by: naineshmp <pnainesh18@gmail.com>
Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>

Author: pditommaso

docs/config.rst

@@ -424,6 +424,7 @@ storageSubPath      The path in the persistent volume to be mounted (default: ro
 computeResourceType Define whether use Kubernetes ``Pod`` or ``Job`` resource type to carry out Nextflow tasks (default: ``Pod``).
 fetchNodeName       If you trace the hostname, activate this option (default: ``false``, requires version ``22.05.0-edge`` or later).
 volumeClaims        (deprecated)
+maxErrorRetry       Defines the Kubernetes API max request retries (default is set to 4)
 httpReadTimeout     Defines the Kubernetes client request HTTP connection read timeout e.g. ``'60s'`` (requires version ``22.10.0`` or later).
 httpConnectTimeout  Defines the Kubernetes client request HTTP connection timeout e.g. ``'60s'`` (requires version ``22.10.0`` or later).
 =================== ================

modules/nextflow/src/main/groovy/nextflow/k8s/K8sConfig.groovy

@@ -226,6 +226,9 @@ class K8sConfig implements Map<String,Object> {
         if( target.httpReadTimeout )
             result.httpReadTimeout = target.httpReadTimeout as Duration
 
+        if( target.maxErrorRetry )
+            result.maxErrorRetry = target.maxErrorRetry as Integer
+
         return result
     }
 

modules/nextflow/src/main/groovy/nextflow/k8s/client/ClientConfig.groovy

@@ -56,6 +56,8 @@ class ClientConfig {
 
     KeyManager[] keyManagers
 
+    Integer maxErrorRetry = 4
+
     /**
      * Timeout when reading from Input stream when a connection is established to a resource.
      * If the timeout expires before there is data available for read, a {@link java.net.SocketTimeoutException} is raised
@@ -80,7 +82,7 @@ class ClientConfig {
     }
 
     String toString() {
-        "${this.class.getSimpleName()}[ server=$server, namespace=$namespace, token=${cut(token)}, sslCert=${cut(sslCert)}, clientCert=${cut(clientCert)}, clientKey=${cut(clientKey)}, verifySsl=$verifySsl, fromFile=$isFromCluster, httpReadTimeout=$httpReadTimeout, httpConnectTimeout=$httpConnectTimeout ]"
+        "${this.class.getSimpleName()}[ server=$server, namespace=$namespace, token=${cut(token)}, sslCert=${cut(sslCert)}, clientCert=${cut(clientCert)}, clientKey=${cut(clientKey)}, verifySsl=$verifySsl, fromFile=$isFromCluster, httpReadTimeout=$httpReadTimeout, httpConnectTimeout=$httpConnectTimeout, maxErrorRetry=$maxErrorRetry ]"
     }
 
     private String cut(String str) {
@@ -128,6 +130,9 @@ class ClientConfig {
         else if( map.clientKeyFile )
             result.clientKey = Paths.get(map.clientKeyFile.toString()).bytes
 
+        if( map.maxErrorRetry )
+            result.maxErrorRetry = map.maxErrorRetry as Integer
+
         return result
     }
 

modules/nextflow/src/main/groovy/nextflow/k8s/client/K8sClient.groovy

@@ -618,18 +618,16 @@ class K8sClient {
      */
     protected K8sResponseApi makeRequest(String method, String path, String body=null) throws K8sResponseException {
 
-        final int maxAttempts = 5
+        final int maxRetries = config.maxErrorRetry
         int attempt = 0
 
-        while ( attempt < maxAttempts ) {
-            attempt++
-
+        while ( true ) {
             try {
                 return makeRequestCall( method, path, body )
             } catch ( K8sResponseException | SocketException | SocketTimeoutException e ) {
                 if ( e instanceof K8sResponseException && e.response.code != 500 )
                     throw e
-                if ( attempt >= maxAttempts )
+                if ( ++attempt > maxRetries )
                     throw e
                 log.debug "[K8s] API request threw socket exception: $e.message for $method $path - Retrying request (attempt=$attempt)"
                 final long delay = (Math.pow(3, attempt - 1) as long) * 250

modules/nextflow/src/test/groovy/nextflow/k8s/K8sConfigTest.groovy

@@ -143,9 +143,21 @@ class K8sConfigTest extends Specification {
         client.serviceAccount == 'that'
         client.httpConnectTimeout == null // testing default null
         client.httpReadTimeout == null // testing default null
+        client.maxErrorRetry == 4
 
     }
 
+    def 'should set maxErrorRetry' () {
+        given:
+        def CONFIG = [maxErrorRetry: 10, namespace: 'this', serviceAccount: 'that', client: [server: 'http://foo']]
+
+        when:
+        def config = new K8sConfig(CONFIG)
+        def client = config.getClient()
+        then:
+        client.maxErrorRetry == 10
+    }
+
     def 'should create client config with http request timeouts' () {
 
         given: