Merge #560

560: Support HTTP.jl 1.0 r=iamed2 a=quinnj



Co-authored-by: Jacob Quinn <quinn.jacobd@gmail.com>
Co-authored-by: mattBrzezinski <matt.brzezinski@invenia.ca>

.github/workflows/CI.yml

@@ -28,11 +28,7 @@ jobs:
         include:
           # Add a job using the earliest version of Julia supported by this package
           - os: ubuntu-latest
-            version: 1.3
-            arch: x64
-          # Add a 1.5 job because that's what Invenia actually uses
-          - os: ubuntu-latest
-            version: "1.5"
+            version: 1.6
             arch: x64
     env:
       MINIO_ACCESS_KEY: minio

Project.toml

@@ -24,7 +24,7 @@ XMLDict = "228000da-037f-5747-90a9-8195ccbf91a5"
 [compat]
 Compat = "3.29"
 GitHub = "5"
-HTTP = "0.9.17"
+HTTP = "1"
 IniFile = "0.5"
 JSON = "0.18, 0.19, 0.20, 0.21"
 MbedTLS = "0.6, 0.7, 1"
@@ -33,7 +33,7 @@ OrderedCollections = "1.3"
 StableRNGs = "1"
 URIs = "1"
 XMLDict = "0.3, 0.4"
-julia = "1.3"
+julia = "1.6"
 
 [extras]
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

bors.toml

@@ -1,6 +1,5 @@
 status = [
-    "Julia 1.3 - ubuntu-latest - x64",
-    "Julia 1.5 - ubuntu-latest - x64",
+    "Julia 1.6 - ubuntu-latest - x64",
     "Julia 1 - macOS-latest - x64",
     "Julia 1 - ubuntu-latest - x64",
 ]

src/AWSExceptions.jl

@@ -1,7 +1,6 @@
 module AWSExceptions
 
 using HTTP
-using HTTP.MessageRequest: body_was_streamed
 using JSON
 using XMLDict
 using XMLDict: XMLDictElement
@@ -120,7 +119,7 @@ function AWSException(e::HTTP.StatusError, body::AbstractString)
     message = get(info, "Message", message)
     message = get(info, "message", message)
 
-    streamed_body = e.response.body == body_was_streamed ? body : nothing
+    streamed_body = !HTTP.isbytes(e.response.body) ? body : nothing
 
     return AWSException(code, message, info, e, streamed_body)
 end

src/deprecated.jl

@@ -7,7 +7,11 @@ function legacy_response(
 
     # When a user defined I/O stream is passed in use the actual `HTTP.Response` body
     # instead of the `AWS.Response` body which requires the I/O stream to be seekable.
-    body = request.response_stream !== nothing ? response.response.body : response.body
+    body = if request.response_stream !== nothing
+        b"[Message Body was streamed]"
+    else
+        response.body
+    end
 
     # The stored service name is always lowercase and may not match the module name
     # specified by the user. We'll assume that the typical casing used is titlecase.

src/utilities/credentials.jl

@@ -158,7 +158,7 @@ function _ec2_metadata(metadata_endpoint::String)
 
         return request === nothing ? nothing : String(request.body)
     catch e
-        e isa HTTP.IOError || e isa HTTP.StatusError && e.status == 404 || rethrow(e)
+        e isa HTTP.RequestError || e isa HTTP.StatusError && e.status == 404 || rethrow(e)
     end
 
     return nothing

src/utilities/downloads_backend.jl

@@ -1,5 +1,3 @@
-using HTTP.MessageRequest: body_was_streamed
-
 """
     DownloadsBackend <: AWS.AbstractBackend
 
@@ -125,9 +123,7 @@ function _http_request(backend::DownloadsBackend, request::Request, response_str
 end
 
 function _http_response(req::Request, res::Downloads.Response; throw::Bool=true)
-    response = HTTP.Response(
-        res.status, res.headers; body=body_was_streamed, request=nothing
-    )
+    response = HTTP.Response(res.status, res.headers; body=IOBuffer(), request=nothing)
 
     if throw && HTTP.iserror(response)
         target = HTTP.resource(HTTP.URI(req.url))

src/utilities/request.jl

@@ -28,6 +28,10 @@ struct HTTPBackend <: AbstractBackend
     http_options::AbstractDict{Symbol,<:Any}
 end
 
+function statuserror(status, resp)
+    return HTTP.StatusError(status, resp.request.method, resp.request.target, resp)
+end
+
 function HTTPBackend(; kwargs...)
     return if isempty(kwargs)
         HTTPBackend(LittleDict{Symbol,Any}())
@@ -129,7 +133,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
             if HTTP.header(response, "Location") != ""
                 request.url = HTTP.header(response, "Location")
             else
-                e = HTTP.StatusError(response.status, response)
+                e = statuserror(response.status, response)
                 throw(AWSException(e, stream))
             end
         end
@@ -200,11 +204,6 @@ end
 function _http_request(http_backend::HTTPBackend, request::Request, response_stream::IO)
     http_options = merge(http_backend.http_options, request.http_options)
 
-    # HTTP options such as `status_exception` need to be used when creating the stack
-    http_stack = HTTP.stack(;
-        redirect=false, retry=false, aws_authorization=false, http_options...
-    )
-
     local buffer
     local response
 
@@ -219,12 +218,12 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str
         buffer = Base.BufferStream()
 
         response = @mock HTTP.request(
-            http_stack,
             request.request_method,
             HTTP.URI(request.url),
             HTTP.mkheaders(request.headers),
             request.content;
-            require_ssl_verification=false,
+            redirect=false,
+            retry=false,
             response_stream=buffer,
             http_options...,
         )
@@ -235,13 +234,8 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str
     end
 
     check = function (s, e)
-        # `Base.IOError` is needed because HTTP.jl can often have errors that aren't
-        # caught and wrapped in an `HTTP.IOError`
-        # https://github.com/JuliaWeb/HTTP.jl/issues/382
-        return isa(e, Sockets.DNSError) ||
-               isa(e, HTTP.ParseError) ||
-               isa(e, HTTP.IOError) ||
-               isa(e, Base.IOError) ||
+        return isa(e, HTTP.ConnectError) ||
+               isa(e, HTTP.RequestError) ||
                (isa(e, HTTP.StatusError) && _http_status(e) >= 500)
     end
 

test/AWSExceptions.jl

@@ -24,7 +24,7 @@
             "status_code" => 400,
         )
 
-        expected["body"] = HTTP.MessageRequest.body_was_streamed
+        expected["body"] = IOBuffer()
         expected["streamed_body"] = """
             <?xml version="1.0" encoding="UTF-8"?>
             <$err>
@@ -40,7 +40,7 @@
         resp = HTTP.Response(
             expected["status_code"], expected["headers"]; body=expected["body"], request=req
         )
-        status_error = HTTP.StatusError(expected["status_code"], resp)
+        status_error = AWS.statuserror(expected["status_code"], resp)
         ex = AWSException(status_error, expected["streamed_body"])
 
         _test_exception(ex, expected, msg)
@@ -50,7 +50,7 @@
 
     @testset "XMLRequest - Invalid XML" begin
         expected = Dict(
-            "body" => HTTP.MessageRequest.body_was_streamed,
+            "body" => IOBuffer(),
             "streamed_body" => """<?xml version="1.0" encoding="UTF-8"?>InvalidXML""",
             "headers" => ["Content-Type" => "application/xml"],
             "status_code" => 404,
@@ -59,7 +59,7 @@
         resp = HTTP.Response(
             expected["status_code"], expected["headers"]; body=expected["body"], request=req
         )
-        status_error = HTTP.StatusError(expected["status_code"], resp)
+        status_error = AWS.statuserror(expected["status_code"], resp)
         ex = @test_logs (:error,) AWSException(status_error, expected["streamed_body"])
 
         @test ex.code == "404"
@@ -74,7 +74,7 @@
             "status_code" => 400,
         )
 
-        expected["body"] = HTTP.MessageRequest.body_was_streamed
+        expected["body"] = IOBuffer()
         expected["streamed_body"] = """
             {
             "__type": "$(expected["code"])",
@@ -87,7 +87,7 @@
         resp = HTTP.Response(
             expected["status_code"], expected["headers"]; body=expected["body"], request=req
         )
-        status_error = HTTP.StatusError(expected["status_code"], resp)
+        status_error = AWS.statuserror(expected["status_code"], resp)
         ex = AWSException(status_error, expected["streamed_body"])
 
         _test_exception(ex, expected, "$msg")

test/issues.jl

@@ -171,7 +171,7 @@ try
                     attempt_num += 1
                     if attempt_num <= num_attempts_to_fail
                         write(response_stream, data[1:(n - 1)]) # an incomplete stream that shouldn't be retained
-                        throw(HTTP.IOError(EOFError(), "msg"))
+                        throw(HTTP.RequestError(HTTP.Request(), EOFError()))
                     else
                         write(response_stream, data)
                         return HTTP.Response(200, "{\"Location\": \"us-east-1\"}")
@@ -212,7 +212,7 @@ try
 
         @testset "Fail all 4 attempts then throw" begin
             err_t = if AWS.DEFAULT_BACKEND[] isa AWS.HTTPBackend
-                HTTP.IOError
+                HTTP.RequestError
             else
                 Downloads.RequestError
             end

test/patch.jl

@@ -139,7 +139,8 @@ end
 _http_options_patches = [
     @patch function HTTP.request(args...; kwargs...)
         options = Dict(kwargs)
-        delete!(options, :require_ssl_verification)
+        delete!(options, :redirect)
+        delete!(options, :retry)
         delete!(options, :response_stream)
         return options
     end
@@ -171,11 +172,11 @@ function gen_http_options_400_patches(message)
             if response_stream !== nothing
                 write(response_stream, body)
                 close(response_stream)  # Simulating current HTTP.jl 0.9.14 behaviour
-                body = HTTP.MessageRequest.body_was_streamed
+                body = IOBuffer()
             end
 
             response = HTTP.Response(400, headers; body=body, request=request)
-            exception = HTTP.StatusError(400, response)
+            exception = AWS.statuserror(400, response)
             return !status_exception ? response : throw(exception)
         end
         @patch function Downloads.request(args...; output=nothing, kwargs...)