From 3172add993a8758c7608b9a01165bf5765ebb8a2 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Mon, 20 Jun 2022 23:04:45 -0600 Subject: [PATCH 1/9] Support HTTP.jl 1.0 --- Project.toml | 2 +- src/AWSExceptions.jl | 3 +-- src/deprecated.jl | 2 +- src/utilities/credentials.jl | 2 +- src/utilities/downloads_backend.jl | 6 +----- src/utilities/request.jl | 18 ++++-------------- test/AWSExceptions.jl | 6 +++--- test/issues.jl | 4 ++-- test/patch.jl | 5 +++-- test/runtests.jl | 5 +++++ 10 files changed, 22 insertions(+), 31 deletions(-) diff --git a/Project.toml b/Project.toml index 8885953ef2..58976486cd 100644 --- a/Project.toml +++ b/Project.toml @@ -24,7 +24,7 @@ XMLDict = "228000da-037f-5747-90a9-8195ccbf91a5" [compat] Compat = "3.29" GitHub = "5" -HTTP = "0.9.17" +HTTP = "1" IniFile = "0.5" JSON = "0.18, 0.19, 0.20, 0.21" MbedTLS = "0.6, 0.7, 1" diff --git a/src/AWSExceptions.jl b/src/AWSExceptions.jl index 76ee1b201f..d464b94673 100644 --- a/src/AWSExceptions.jl +++ b/src/AWSExceptions.jl @@ -1,7 +1,6 @@ module AWSExceptions using HTTP -using HTTP.MessageRequest: body_was_streamed using JSON using XMLDict using XMLDict: XMLDictElement @@ -114,7 +113,7 @@ function AWSException(e::HTTP.StatusError, body::AbstractString) message = get(info, "Message", message) message = get(info, "message", message) - streamed_body = e.response.body == body_was_streamed ? body : nothing + streamed_body = !HTTP.isbytes(e.response.body) ? body : nothing return AWSException(code, message, info, e, streamed_body) end diff --git a/src/deprecated.jl b/src/deprecated.jl index ab092e2467..7ca2320460 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -7,7 +7,7 @@ function legacy_response( # When a user defined I/O stream is passed in use the actual `HTTP.Response` body # instead of the `AWS.Response` body which requires the I/O stream to be seekable. - body = request.response_stream !== nothing ? response.response.body : response.body + body = request.response_stream !== nothing ? bytes("[Message Body was streamed]") : response.body # The stored service name is always lowercase and may not match the module name # specified by the user. We'll assume that the typical casing used is titlecase. diff --git a/src/utilities/credentials.jl b/src/utilities/credentials.jl index ce80c1628b..82a1ffed6e 100644 --- a/src/utilities/credentials.jl +++ b/src/utilities/credentials.jl @@ -158,7 +158,7 @@ function _ec2_metadata(metadata_endpoint::String) return request === nothing ? nothing : String(request.body) catch e - e isa HTTP.IOError || e isa HTTP.StatusError && e.status == 404 || rethrow(e) + e isa HTTP.RequestError || e isa HTTP.StatusError && e.status == 404 || rethrow(e) end return nothing diff --git a/src/utilities/downloads_backend.jl b/src/utilities/downloads_backend.jl index 00c298ad06..9d6bccd494 100644 --- a/src/utilities/downloads_backend.jl +++ b/src/utilities/downloads_backend.jl @@ -1,5 +1,3 @@ -using HTTP.MessageRequest: body_was_streamed - """ DownloadsBackend <: AWS.AbstractBackend @@ -125,9 +123,7 @@ function _http_request(backend::DownloadsBackend, request::Request, response_str end function _http_response(req::Request, res::Downloads.Response; throw::Bool=true) - response = HTTP.Response( - res.status, res.headers; body=body_was_streamed, request=nothing - ) + response = HTTP.Response(res.status, res.headers; body=nothing, request=nothing) if throw && HTTP.iserror(response) target = HTTP.resource(HTTP.URI(req.url)) diff --git a/src/utilities/request.jl b/src/utilities/request.jl index 8bc9914532..ec3e1c1169 100644 --- a/src/utilities/request.jl +++ b/src/utilities/request.jl @@ -200,11 +200,6 @@ end function _http_request(http_backend::HTTPBackend, request::Request, response_stream::IO) http_options = merge(http_backend.http_options, request.http_options) - # HTTP options such as `status_exception` need to be used when creating the stack - http_stack = HTTP.stack(; - redirect=false, retry=false, aws_authorization=false, http_options... - ) - local buffer local response @@ -219,12 +214,12 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str buffer = Base.BufferStream() response = @mock HTTP.request( - http_stack, request.request_method, HTTP.URI(request.url), HTTP.mkheaders(request.headers), request.content; - require_ssl_verification=false, + redirect=false, + retry=false, response_stream=buffer, http_options..., ) @@ -235,13 +230,8 @@ function _http_request(http_backend::HTTPBackend, request::Request, response_str end check = function (s, e) - # `Base.IOError` is needed because HTTP.jl can often have errors that aren't - # caught and wrapped in an `HTTP.IOError` - # https://github.com/JuliaWeb/HTTP.jl/issues/382 - return isa(e, Sockets.DNSError) || - isa(e, HTTP.ParseError) || - isa(e, HTTP.IOError) || - isa(e, Base.IOError) || + return isa(e, HTTP.ConnectError) || + isa(e, HTTP.RequestError) || (isa(e, HTTP.StatusError) && _http_status(e) >= 500) end diff --git a/test/AWSExceptions.jl b/test/AWSExceptions.jl index f4d67a0bde..eaab301654 100644 --- a/test/AWSExceptions.jl +++ b/test/AWSExceptions.jl @@ -24,7 +24,7 @@ "status_code" => 400, ) - expected["body"] = HTTP.MessageRequest.body_was_streamed + expected["body"] = IOBuffer() expected["streamed_body"] = """ <$err> @@ -50,7 +50,7 @@ @testset "XMLRequest - Invalid XML" begin expected = Dict( - "body" => HTTP.MessageRequest.body_was_streamed, + "body" => IOBuffer(), "streamed_body" => """InvalidXML""", "headers" => ["Content-Type" => "application/xml"], "status_code" => 404, @@ -74,7 +74,7 @@ "status_code" => 400, ) - expected["body"] = HTTP.MessageRequest.body_was_streamed + expected["body"] = nothing expected["streamed_body"] = """ { "__type": "$(expected["code"])", diff --git a/test/issues.jl b/test/issues.jl index a5180b5bb1..9c67e7f437 100644 --- a/test/issues.jl +++ b/test/issues.jl @@ -171,7 +171,7 @@ try attempt_num += 1 if attempt_num <= num_attempts_to_fail write(response_stream, data[1:(n - 1)]) # an incomplete stream that shouldn't be retained - throw(HTTP.IOError(EOFError(), "msg")) + throw(HTTP.RequestError(HTTP.Request(), EOFError())) else write(response_stream, data) return HTTP.Response(200, "{\"Location\": \"us-east-1\"}") @@ -212,7 +212,7 @@ try @testset "Fail all 4 attempts then throw" begin err_t = if AWS.DEFAULT_BACKEND[] isa AWS.HTTPBackend - HTTP.IOError + HTTP.RequestError else Downloads.RequestError end diff --git a/test/patch.jl b/test/patch.jl index 27b586285c..71f39306d0 100644 --- a/test/patch.jl +++ b/test/patch.jl @@ -139,7 +139,8 @@ end _http_options_patches = [ @patch function HTTP.request(args...; kwargs...) options = Dict(kwargs) - delete!(options, :require_ssl_verification) + delete!(options, :redirect) + delete!(options, :retry) delete!(options, :response_stream) return options end @@ -171,7 +172,7 @@ function gen_http_options_400_patches(message) if response_stream !== nothing write(response_stream, body) close(response_stream) # Simulating current HTTP.jl 0.9.14 behaviour - body = HTTP.MessageRequest.body_was_streamed + body = nothing end response = HTTP.Response(400, headers; body=body, request=request) diff --git a/test/runtests.jl b/test/runtests.jl index 3df8afe121..e28a1b2e89 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -38,6 +38,11 @@ using StableRNGs Mocking.activate() +# backwards compat +function HTTP.StatusError(status, resp) + return HTTP.StatusError(status, resp.request.method, resp.request.target, resp) +end + include("patch.jl") const TEST_MINIO = begin From 18a34eef0376ca707113275b0ae79bbf8d4b3cbe Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Mon, 20 Jun 2022 23:07:16 -0600 Subject: [PATCH 2/9] Update src/deprecated.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/deprecated.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/deprecated.jl b/src/deprecated.jl index 7ca2320460..de85034ee2 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -7,7 +7,11 @@ function legacy_response( # When a user defined I/O stream is passed in use the actual `HTTP.Response` body # instead of the `AWS.Response` body which requires the I/O stream to be seekable. - body = request.response_stream !== nothing ? bytes("[Message Body was streamed]") : response.body + body = if request.response_stream !== nothing + bytes("[Message Body was streamed]") + else + response.body + end # The stored service name is always lowercase and may not match the module name # specified by the user. We'll assume that the typical casing used is titlecase. From 0ad1f5ce443f168c06cd1102cc6ff5f6c226414d Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Mon, 20 Jun 2022 23:16:45 -0600 Subject: [PATCH 3/9] update julia compat --- .github/workflows/CI.yml | 6 +----- Project.toml | 2 +- bors.toml | 3 +-- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 4ea9ca0314..97bafc5126 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -28,11 +28,7 @@ jobs: include: # Add a job using the earliest version of Julia supported by this package - os: ubuntu-latest - version: 1.3 - arch: x64 - # Add a 1.5 job because that's what Invenia actually uses - - os: ubuntu-latest - version: "1.5" + version: 1.6 arch: x64 env: MINIO_ACCESS_KEY: minio diff --git a/Project.toml b/Project.toml index 58976486cd..5dbd58f18a 100644 --- a/Project.toml +++ b/Project.toml @@ -33,7 +33,7 @@ OrderedCollections = "1.3" StableRNGs = "1" URIs = "1" XMLDict = "0.3, 0.4" -julia = "1.3" +julia = "1.6" [extras] Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" diff --git a/bors.toml b/bors.toml index 9fcf79fdab..247042f7ef 100644 --- a/bors.toml +++ b/bors.toml @@ -1,6 +1,5 @@ status = [ - "Julia 1.3 - ubuntu-latest - x64", - "Julia 1.5 - ubuntu-latest - x64", + "Julia 1.6 - ubuntu-latest - x64", "Julia 1 - macOS-latest - x64", "Julia 1 - ubuntu-latest - x64", ] From ed0f396cd8f7f873592eef5d87b449d37985e687 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Mon, 20 Jun 2022 23:29:09 -0600 Subject: [PATCH 4/9] fix --- src/deprecated.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/deprecated.jl b/src/deprecated.jl index de85034ee2..1072941af1 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -8,7 +8,7 @@ function legacy_response( # When a user defined I/O stream is passed in use the actual `HTTP.Response` body # instead of the `AWS.Response` body which requires the I/O stream to be seekable. body = if request.response_stream !== nothing - bytes("[Message Body was streamed]") + b"[Message Body was streamed]" else response.body end From 2f7bee3dd6519a8cc149be180957a6b2c2556451 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Tue, 21 Jun 2022 13:57:44 -0600 Subject: [PATCH 5/9] fix --- test/AWSExceptions.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/AWSExceptions.jl b/test/AWSExceptions.jl index eaab301654..b6e2954ff3 100644 --- a/test/AWSExceptions.jl +++ b/test/AWSExceptions.jl @@ -74,7 +74,7 @@ "status_code" => 400, ) - expected["body"] = nothing + expected["body"] = IOBuffer() expected["streamed_body"] = """ { "__type": "$(expected["code"])", From 21f027f3687b5deaf368852eeb4abe159b95515d Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Tue, 21 Jun 2022 14:15:29 -0600 Subject: [PATCH 6/9] fix test --- test/patch.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/patch.jl b/test/patch.jl index 71f39306d0..09fccc9438 100644 --- a/test/patch.jl +++ b/test/patch.jl @@ -172,7 +172,7 @@ function gen_http_options_400_patches(message) if response_stream !== nothing write(response_stream, body) close(response_stream) # Simulating current HTTP.jl 0.9.14 behaviour - body = nothing + body = IOBuffer() end response = HTTP.Response(400, headers; body=body, request=request) From 7ba8fb7c77195ee57f724c5f0bb3eb7fdcaf9d64 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Tue, 21 Jun 2022 14:30:52 -0600 Subject: [PATCH 7/9] fix downloads --- src/utilities/downloads_backend.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utilities/downloads_backend.jl b/src/utilities/downloads_backend.jl index 9d6bccd494..bbce7ebf92 100644 --- a/src/utilities/downloads_backend.jl +++ b/src/utilities/downloads_backend.jl @@ -123,7 +123,7 @@ function _http_request(backend::DownloadsBackend, request::Request, response_str end function _http_response(req::Request, res::Downloads.Response; throw::Bool=true) - response = HTTP.Response(res.status, res.headers; body=nothing, request=nothing) + response = HTTP.Response(res.status, res.headers; body=IOBuffer(), request=nothing) if throw && HTTP.iserror(response) target = HTTP.resource(HTTP.URI(req.url)) From 26f547eda8ddb3e7eef809c9ef92466eef20619f Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Tue, 28 Jun 2022 09:25:31 -0600 Subject: [PATCH 8/9] fix HTTP.StatusError convenience constructor --- src/utilities/request.jl | 4 +++- test/AWSExceptions.jl | 6 +++--- test/patch.jl | 2 +- test/runtests.jl | 5 ----- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/utilities/request.jl b/src/utilities/request.jl index ec3e1c1169..57b7cfd919 100644 --- a/src/utilities/request.jl +++ b/src/utilities/request.jl @@ -28,6 +28,8 @@ struct HTTPBackend <: AbstractBackend http_options::AbstractDict{Symbol,<:Any} end +statuserror(status, resp) = HTTP.StatusError(status, resp.request.method, resp.request.target, resp) + function HTTPBackend(; kwargs...) return if isempty(kwargs) HTTPBackend(LittleDict{Symbol,Any}()) @@ -129,7 +131,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers if HTTP.header(response, "Location") != "" request.url = HTTP.header(response, "Location") else - e = HTTP.StatusError(response.status, response) + e = statuserror(response.status, response) throw(AWSException(e, stream)) end end diff --git a/test/AWSExceptions.jl b/test/AWSExceptions.jl index b6e2954ff3..2ad9c8c25a 100644 --- a/test/AWSExceptions.jl +++ b/test/AWSExceptions.jl @@ -40,7 +40,7 @@ resp = HTTP.Response( expected["status_code"], expected["headers"]; body=expected["body"], request=req ) - status_error = HTTP.StatusError(expected["status_code"], resp) + status_error = AWS.statuserror(expected["status_code"], resp) ex = AWSException(status_error, expected["streamed_body"]) _test_exception(ex, expected, msg) @@ -59,7 +59,7 @@ resp = HTTP.Response( expected["status_code"], expected["headers"]; body=expected["body"], request=req ) - status_error = HTTP.StatusError(expected["status_code"], resp) + status_error = AWS.statuserror(expected["status_code"], resp) ex = @test_logs (:error,) AWSException(status_error, expected["streamed_body"]) @test ex.code == "404" @@ -87,7 +87,7 @@ resp = HTTP.Response( expected["status_code"], expected["headers"]; body=expected["body"], request=req ) - status_error = HTTP.StatusError(expected["status_code"], resp) + status_error = AWS.statuserror(expected["status_code"], resp) ex = AWSException(status_error, expected["streamed_body"]) _test_exception(ex, expected, "$msg") diff --git a/test/patch.jl b/test/patch.jl index 09fccc9438..bdf61f127d 100644 --- a/test/patch.jl +++ b/test/patch.jl @@ -176,7 +176,7 @@ function gen_http_options_400_patches(message) end response = HTTP.Response(400, headers; body=body, request=request) - exception = HTTP.StatusError(400, response) + exception = AWS.statuserror(400, response) return !status_exception ? response : throw(exception) end @patch function Downloads.request(args...; output=nothing, kwargs...) diff --git a/test/runtests.jl b/test/runtests.jl index e28a1b2e89..3df8afe121 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -38,11 +38,6 @@ using StableRNGs Mocking.activate() -# backwards compat -function HTTP.StatusError(status, resp) - return HTTP.StatusError(status, resp.request.method, resp.request.target, resp) -end - include("patch.jl") const TEST_MINIO = begin From 1a56edc64e3046ae12d6f5861db5fd28b5513ce0 Mon Sep 17 00:00:00 2001 From: mattBrzezinski Date: Tue, 28 Jun 2022 16:28:38 +0100 Subject: [PATCH 9/9] Update src/utilities/request.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/utilities/request.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utilities/request.jl b/src/utilities/request.jl index 57b7cfd919..d22198cd6b 100644 --- a/src/utilities/request.jl +++ b/src/utilities/request.jl @@ -28,7 +28,9 @@ struct HTTPBackend <: AbstractBackend http_options::AbstractDict{Symbol,<:Any} end -statuserror(status, resp) = HTTP.StatusError(status, resp.request.method, resp.request.target, resp) +function statuserror(status, resp) + return HTTP.StatusError(status, resp.request.method, resp.request.target, resp) +end function HTTPBackend(; kwargs...) return if isempty(kwargs)