Refactor httpfs configuration

This commit refactor the httpfs configuration moving it into a indipendent class. This is needed to prevent a cyclic file system error when embedding nextflow library into another project. Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
nextflow-io · Feb 14, 2023 · 445218a · 445218a
1 parent a6b7fa3
commit 445218a
Show file tree

Hide file tree

Showing 4 changed files with 102 additions and 62 deletions.
diff --git a/modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemConfig.groovy b/modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemConfig.groovy
@@ -0,0 +1,54 @@
+package nextflow.file.http
+
+import groovy.transform.CompileStatic
+import nextflow.SysEnv
+
+/**
+ * Hold HTTP/FTP virtual file system configuration
+ *
+ * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
+ */
+@CompileStatic
+class XFileSystemConfig {
+
+    static XFileSystemConfig instance = new XFileSystemConfig()
+
+    static final public String  DEFAULT_RETRY_CODES = '404,410'
+
+    static final public int MAX_REDIRECT_HOPS = 5
+
+    static final public int DEFAULT_BACK_OFF_BASE = 3
+
+    static final public int DEFAULT_BACK_OFF_DELAY = 250
+
+    static final public int DEFAULT_MAX_ATTEMPTS = 3
+
+    private int maxAttempts = DEFAULT_MAX_ATTEMPTS
+
+    private int backOffBase = DEFAULT_BACK_OFF_BASE
+
+    private int backOffDelay = DEFAULT_BACK_OFF_DELAY
+
+    private List<Integer> retryCodes
+
+    {
+        maxAttempts = config('NXF_HTTPFS_MAX_ATTEMPTS', DEFAULT_MAX_ATTEMPTS) as Integer
+        backOffBase = config('NXF_HTTPFS_BACKOFF_BASE', DEFAULT_BACK_OFF_BASE) as Integer
+        backOffDelay = config('NXF_HTTPFS_DELAY', DEFAULT_BACK_OFF_DELAY) as Integer
+        retryCodes = config('NXF_HTTPFS_RETRY_CODES', DEFAULT_RETRY_CODES).tokenize(',').collect( val -> val as Integer )
+    }
+
+    static String config(String name, def defValue) {
+        return SysEnv.containsKey(name) ? SysEnv.get(name) : defValue.toString()
+    }
+
+    int maxAttempts() { maxAttempts }
+
+    int backOffBase() { backOffBase }
+
+    int backOffDelay() { backOffDelay }
+
+    List<Integer> retryCodes() { retryCodes }
+
+    static XFileSystemConfig config() { return instance }
+}
diff --git a/modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemProvider.groovy b/modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemProvider.groovy
@@ -46,6 +46,11 @@ import groovy.util.logging.Slf4j
 import nextflow.SysEnv
 import nextflow.extension.FilesEx
 import sun.net.www.protocol.ftp.FtpURLConnection
+
+import static XFileSystemConfig.*
+
+import static nextflow.file.http.XFileSystemConfig.config
+
 /**
  * Implements a read-only JSR-203 compliant file system provider for http/ftp protocols
  *
@@ -59,37 +64,6 @@ abstract class XFileSystemProvider extends FileSystemProvider {
 
     private Map<URI, FileSystem> fileSystemMap = new LinkedHashMap<>(20)
 
-    static final public String  DEFAULT_RETRY_CODES = '404,410'
-
-    static final public int MAX_REDIRECT_HOPS = 5
-
-    static final public int DEFAULT_BACK_OFF_BASE = 3
-
-    static final public int DEFAULT_BACK_OFF_DELAY = 250
-
-    static final public int DEFAULT_MAX_ATTEMPTS = 3
-
-    private int maxAttempts = DEFAULT_MAX_ATTEMPTS
-
-    private int backOffBase = DEFAULT_BACK_OFF_BASE
-
-    private int backOffDelay = DEFAULT_BACK_OFF_DELAY
-
-    private List<Integer> retryCodes
-
-    private Map<String,String> env = SysEnv.get();
-
-    {
-        maxAttempts = config('NXF_HTTPFS_MAX_ATTEMPTS', DEFAULT_MAX_ATTEMPTS) as Integer
-        backOffBase = config('NXF_HTTPFS_BACKOFF_BASE', DEFAULT_BACK_OFF_BASE) as Integer
-        backOffDelay = config('NXF_HTTPFS_DELAY', DEFAULT_BACK_OFF_DELAY) as Integer
-        retryCodes = config('NXF_HTTPFS_RETRY_CODES', DEFAULT_RETRY_CODES).tokenize(',').collect( val -> val as Integer )
-    }
-
-    protected int maxAttempts() { maxAttempts }
-    protected int backOffBase() { backOffBase }
-    protected int backOffDelay() { backOffDelay }
-    protected List<Integer> retryCodes() { retryCodes }
 
     protected static String config(String name, def defValue) {
         return SysEnv.containsKey(name) ? SysEnv.get(name) : defValue.toString()
@@ -224,8 +198,8 @@ abstract class XFileSystemProvider extends FileSystemProvider {
             log.debug "Remote redirect URL: $newPath"
             return toConnection0(newPath, attempt+1)
         }
-        else if( conn instanceof HttpURLConnection && conn.getResponseCode() in retryCodes && attempt < maxAttempts ) {
-            final delay = (Math.pow(backOffBase, attempt) as long) * backOffDelay
+        else if( conn instanceof HttpURLConnection && conn.getResponseCode() in config().retryCodes() && attempt < config().maxAttempts() ) {
+            final delay = (Math.pow(config().backOffBase(), attempt) as long) * config().backOffDelay()
             log.debug "Got HTTP error=${conn.getResponseCode()} waiting for ${delay}ms (attempt=${attempt+1})"
             Thread.sleep(delay)
             return toConnection0(url, attempt+1)

diff --git a/modules/nf-httpfs/src/test/nextflow/file/http/XFileSystemConfigTest.groovy b/modules/nf-httpfs/src/test/nextflow/file/http/XFileSystemConfigTest.groovy
@@ -0,0 +1,41 @@
+package nextflow.file.http
+
+import spock.lang.Specification
+
+import nextflow.SysEnv
+
+/**
+ *
+ * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
+ */
+class XFileSystemConfigTest extends Specification {
+
+    def 'should create with default config settings' () {
+        when:
+        def config = new XFileSystemConfig()
+        then:
+        config.retryCodes() == XFileSystemConfig.DEFAULT_RETRY_CODES.tokenize(',').collect( it -> it as int )
+        config.backOffDelay() == XFileSystemConfig.DEFAULT_BACK_OFF_DELAY
+        config.backOffBase() == XFileSystemConfig.DEFAULT_BACK_OFF_BASE
+        config.maxAttempts() == XFileSystemConfig.DEFAULT_MAX_ATTEMPTS
+    }
+
+    def 'should create with custom config settings' () {
+        given:
+        SysEnv.push([NXF_HTTPFS_MAX_ATTEMPTS: '10',
+                     NXF_HTTPFS_BACKOFF_BASE: '300',
+                     NXF_HTTPFS_DELAY       : '400',
+                     NXF_HTTPFS_RETRY_CODES : '1,2,3'])
+
+        when:
+        def config = new XFileSystemConfig()
+        then:
+        config.retryCodes() == [1,2,3]
+        config.backOffDelay() == 400
+        config.backOffBase() == 300
+        config.maxAttempts() == 10
+
+        cleanup:
+        SysEnv.pop()
+    }
+}
diff --git a/modules/nf-httpfs/src/test/nextflow/file/http/XFileSystemProviderTest.groovy b/modules/nf-httpfs/src/test/nextflow/file/http/XFileSystemProviderTest.groovy
@@ -31,35 +31,6 @@ import spock.lang.Unroll
  */
 class XFileSystemProviderTest extends Specification {
 
-    def 'should create with default config settings' () {
-        when:
-        def fs = new HttpFileSystemProvider()
-        then:
-        fs.retryCodes() == HttpFileSystemProvider.DEFAULT_RETRY_CODES.tokenize(',').collect( it -> it as int )
-        fs.backOffDelay() == HttpFileSystemProvider.DEFAULT_BACK_OFF_DELAY
-        fs.backOffBase() == HttpFileSystemProvider.DEFAULT_BACK_OFF_BASE
-        fs.maxAttempts() == HttpFileSystemProvider.DEFAULT_MAX_ATTEMPTS
-
-    }
-
-    def 'should create with custom config settings' () {
-        given:
-        SysEnv.push([NXF_HTTPFS_MAX_ATTEMPTS: '10',
-                     NXF_HTTPFS_BACKOFF_BASE: '300',
-                     NXF_HTTPFS_DELAY       : '400',
-                     NXF_HTTPFS_RETRY_CODES : '1,2,3'])
-
-        when:
-        def fs = new HttpFileSystemProvider()
-        then:
-        fs.retryCodes() == [1,2,3]
-        fs.backOffDelay() == 400
-        fs.backOffBase() == 300
-        fs.maxAttempts() == 10
-
-        cleanup:
-        SysEnv.pop()
-    }
 
     def "should return input stream"() {
         given: