Merge pull request #17 from mauro3/m3/remotes

RFC: Backend choice part of RemoteFile
helgee · Mar 7, 2021 · d838ae1 · d838ae1
2 parents 5fe6d31 + f8c9189
commit d838ae1
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 6 deletions.
diff --git a/src/RemoteFiles.jl b/src/RemoteFiles.jl
@@ -11,6 +11,7 @@ export DownloadError, RemoteFile, @RemoteFile, path, rm, isfile,
 
 include("backends.jl")
 
+"The list of supported backends on the current machine"
 const BACKENDS = AbstractBackend[Http()]
 
 _iscurl(curl) = occursin("libcurl", read(`$curl --version`, String))
@@ -34,6 +35,7 @@ struct RemoteFile
     updates::Symbol
     retries::Int
     try_backends::Bool
+    backends::Vector{AbstractBackend}
     wait::Int
     failed::Symbol
 end
@@ -76,6 +78,7 @@ function RemoteFile(uri::URI;
     updates::Symbol=:never,
     retries::Int=3,
     try_backends::Bool=true,
+    backends=BACKENDS,
     wait::Int=5,
     failed::Symbol=:error,
 )
@@ -89,7 +92,7 @@ function RemoteFile(uri::URI;
         end
     end
 
-    RemoteFile(uri, file, abspath(dir), updates, retries, try_backends, wait, failed)
+    RemoteFile(uri, file, abspath(dir), updates, retries, try_backends, backends, wait, failed)
 end
 RemoteFile(uri::String; kwargs...) = RemoteFile(URI(uri); kwargs...)
 
@@ -132,6 +135,7 @@ The following keyword arguments are available:
     - `:mondays`/`:weekly`, `:tuesdays`, etc.
 - `retries` (default: 3): How many retries should be attempted.
 - `try_backends` (default: `true`): Whether to retry with different backends.
+- `backends` (default `RemoteFiles.BACKENDS`): Which backends to try.
 - `wait` (default: 5): How many seconds to wait between retries.
 - `failed` (default: `:error`): What to do if the download fails. Either throw
     an exception (`:error`) or display a warning (`:warn`).

diff --git a/src/download.jl b/src/download.jl
@@ -16,7 +16,7 @@ Download `rf`.
 - `retries`: Override the number of retries in `rf` if `retries != 0`
 """
 function download(rf::RemoteFile; kwargs...)
-    for (i, backend) in enumerate(BACKENDS)
+    for (i, backend) in enumerate(rf.backends)
         try
             if i == 1
                 download(backend, rf; kwargs...)
@@ -99,4 +99,3 @@ function samecontent(file1, file2)
     h2 = hash(open(read, file2, "r"))
     h1 == h2
 end
-
diff --git a/test/runtests.jl b/test/runtests.jl
@@ -30,9 +30,9 @@ rm("tmp", force=true, recursive=true)
         @test r.file == "image.png"
 
 
-		@test_logs((:info, r"Downloading file 'image.png' from 'https://httpbin.org/image/png'."),
-			(:info, r"File 'image.png' was successfully downloaded."),
-			match_mode=:any, download(r, verbose=true))
+	@test_logs((:info, r"Downloading file 'image.png' from 'https://httpbin.org/image/png'."),
+		   (:info, r"File 'image.png' was successfully downloaded."),
+		   match_mode=:any, download(r, verbose=true))
         @test isfile(r)
         rm(r, force=true)
 
@@ -93,6 +93,31 @@ rm("tmp", force=true, recursive=true)
         @test isfile(r1)
         rm(dir, force=true, recursive=true)
     end
+    @testset "RemoteFile backends" begin
+        r = RemoteFile("https://httpbin.org/image/png", backends=[RemoteFiles.Http()], file="image.png")
+	@test_logs((:info, r"Downloading file 'image.png' from 'https://httpbin.org/image/png'."),
+		   (:info, r"File 'image.png' was successfully downloaded."),
+		   match_mode=:any, download(r, verbose=true))
+        @test isfile(r)
+        rm(r, force=true)
+
+        if RemoteFiles.CURL() in RemoteFiles.BACKENDS
+            r = RemoteFile("https://httpbin.org/image/png", backends=[RemoteFiles.CURL()], file="image.png")
+	    @test_logs((:info, r"Downloading file 'image.png' from 'https://httpbin.org/image/png'."),
+		       (:info, r"File 'image.png' was successfully downloaded."),
+		       match_mode=:any, download(r, verbose=true))
+            @test isfile(r)
+            rm(r, force=true)
+        end
+        if RemoteFiles.Wget() in RemoteFiles.BACKENDS
+            r = RemoteFile("https://httpbin.org/image/png", backends=[RemoteFiles.Wget()], file="image.png")
+	    @test_logs((:info, r"Downloading file 'image.png' from 'https://httpbin.org/image/png'."),
+		       (:info, r"File 'image.png' was successfully downloaded."),
+		       match_mode=:any, download(r, verbose=true))
+            @test isfile(r)
+            rm(r, force=true)
+        end
+    end
     @testset "RemoteFileSets" begin
         set = RemoteFileSet("Images",
             file1=RemoteFile("https://httpbin.org/image/png", file="image1.png"),