diff --git a/docs/src/internal_architecture.md b/docs/src/internal_architecture.md index 17c754023..f3bfc200c 100644 --- a/docs/src/internal_architecture.md +++ b/docs/src/internal_architecture.md @@ -14,7 +14,6 @@ HTTP.BasicAuthLayer HTTP.CookieLayer HTTP.CanonicalizeLayer HTTP.MessageLayer -HTTP.AWS4AuthLayer HTTP.RetryLayer HTTP.ExceptionLayer HTTP.ConnectionPoolLayer diff --git a/src/AWS4AuthRequest.jl b/src/AWS4AuthRequest.jl deleted file mode 100644 index a5623966e..000000000 --- a/src/AWS4AuthRequest.jl +++ /dev/null @@ -1,197 +0,0 @@ -module AWS4AuthRequest - -using ..Base64 -using ..Dates -using MbedTLS: digest, MD_SHA256, MD_MD5 -import ..Layer, ..request, ..Headers -using URIs -using ..Pairs: getkv, setkv, rmkv -import ..@debug, ..DEBUG_LEVEL - -""" - request(AWS4AuthLayer, ::URI, ::Request, body) -> HTTP.Response - -Add a [AWS Signature Version 4](http://docs.aws.amazon.com/general/latest/gr/signature-version-4.html) -`Authorization` header to a `Request`. - - -Credentials are read from environment variables `AWS_ACCESS_KEY_ID`, -`AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN`. -""" -abstract type AWS4AuthLayer{Next <: Layer} <: Layer{Next} end -export AWS4AuthLayer - -function request(::Type{AWS4AuthLayer{Next}}, - url::URI, req, body; kw...) where Next - - if !haskey(kw, :aws_access_key_id) && - !haskey(ENV, "AWS_ACCESS_KEY_ID") - kw = merge(dot_aws_credentials(), kw) - end - - sign_aws4!(req.method, url, req.headers, req.body; kw...) - - return request(Next, url, req, body; kw...) -end - -# Normalize whitespace to the form required in the canonical headers. -# Note that the expected format for multiline headers seems not to be explicitly -# documented, but Amazon provides a test case for it, so we'll match that behavior. -# We replace each `\n` with a `,` and remove all whitespace around the newlines, -# then any remaining contiguous whitespace is replaced with a single space. -function _normalize_ws(s::AbstractString) - if any(isequal('\n'), s) - join(map(_normalize_ws, split(s, '\n')), ',') - else - replace(strip(s), r"\s+" => " ") - end -end - -function sign_aws4!(method::String, - url::URI, - headers::Headers, - body::Vector{UInt8}; - body_sha256::Vector{UInt8}=digest(MD_SHA256, body), - body_md5::Vector{UInt8}=digest(MD_MD5, body), - t::Union{DateTime,Nothing}=nothing, - timestamp::DateTime=now(Dates.UTC), - aws_service::String=String(split(url.host, ".")[1]), - aws_region::String=String(split(url.host, ".")[2]), - aws_access_key_id::String=ENV["AWS_ACCESS_KEY_ID"], - aws_secret_access_key::String=ENV["AWS_SECRET_ACCESS_KEY"], - aws_session_token::String=get(ENV, "AWS_SESSION_TOKEN", ""), - token_in_signature=true, - include_md5=true, - include_sha256=true, - kw...) - if t !== nothing - Base.depwarn("The `t` keyword argument to `sign_aws4!` is deprecated; use " * - "`timestamp` instead.", :sign_aws4!) - timestamp = t - end - - # ISO8601 date/time strings for time of request... - date = Dates.format(timestamp, dateformat"yyyymmdd") - datetime = Dates.format(timestamp, dateformat"yyyymmddTHHMMSS\Z") - - # Authentication scope... - scope = [date, aws_region, aws_service, "aws4_request"] - - # Signing key generated from today's scope string... - signing_key = string("AWS4", aws_secret_access_key) - for element in scope - signing_key = digest(MD_SHA256, element, signing_key) - end - - # Authentication scope string... - scope = join(scope, "/") - - # SHA256 hash of content... - content_hash = bytes2hex(body_sha256) - - # HTTP headers... - rmkv(headers, "Authorization") - setkv(headers, "host", url.host) - setkv(headers, "x-amz-date", datetime) - include_md5 && setkv(headers, "Content-MD5", base64encode(body_md5)) - if (aws_service == "s3" && method == "PUT") || include_sha256 - # This header is required for S3 PUT requests. See the documentation at - # https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html - setkv(headers, "x-amz-content-sha256", content_hash) - end - if aws_session_token != "" - setkv(headers, "x-amz-security-token", aws_session_token) - end - - # Sort and lowercase() Headers to produce canonical form... - unique_header_keys = Vector{String}() - normalized_headers = Dict{String,Vector{String}}() - for (k, v) in sort!([lowercase(k) => v for (k, v) in headers], by=first) - # Some services want the token included as part of the signature - if k == "x-amz-security-token" && !token_in_signature - continue - end - # In Amazon's examples, they exclude Content-Length from signing. This does not - # appear to be addressed in the documentation, so we'll just mimic the example. - if k == "content-length" - continue - end - if !haskey(normalized_headers, k) - normalized_headers[k] = Vector{String}() - push!(unique_header_keys, k) - end - push!(normalized_headers[k], _normalize_ws(v)) - end - canonical_headers = map(unique_header_keys) do k - string(k, ':', join(normalized_headers[k], ',')) - end - signed_headers = join(unique_header_keys, ';') - - # Sort Query String... - query = sort!(collect(queryparams(url.query)), by=first) - - # Paths for requests to S3 should be escaped but not normalized. See - # http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html#canonical-request - # Note that escapepath escapes ~ per RFC 1738, but Amazon includes an example in their - # signature v4 test suite where ~ remains unescaped. We follow the spec here and thus - # deviate from Amazon's example in this case. - path = escapepath(aws_service == "s3" ? url.path : URIs.normpath(url.path)) - - # Create hash of canonical request... - canonical_form = join([method, - path, - escapeuri(query), - join(canonical_headers, "\n"), - "", - signed_headers, - content_hash], "\n") - @debug 3 "AWS4 canonical_form: $canonical_form" - - canonical_hash = bytes2hex(digest(MD_SHA256, canonical_form)) - - # Create and sign "String to Sign"... - string_to_sign = "AWS4-HMAC-SHA256\n$datetime\n$scope\n$canonical_hash" - signature = bytes2hex(digest(MD_SHA256, string_to_sign, signing_key)) - - @debug 3 "AWS4 string_to_sign: $string_to_sign" - @debug 3 "AWS4 signature: $signature" - - # Append Authorization header... - setkv(headers, "Authorization", string( - "AWS4-HMAC-SHA256 ", - "Credential=$aws_access_key_id/$scope, ", - "SignedHeaders=$signed_headers, ", - "Signature=$signature" - )) -end - -using IniFile - -credentials = NamedTuple() - -""" -Load Credentials from [AWS CLI ~/.aws/credentials file] -(http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html). -""" -function dot_aws_credentials()::NamedTuple - - global credentials - if !isempty(credentials) - return credentials - end - - f = get(ENV, "AWS_CONFIG_FILE", joinpath(homedir(), ".aws", "credentials")) - p = get(ENV, "AWS_DEFAULT_PROFILE", get(ENV, "AWS_PROFILE", "default")) - - if !isfile(f) - return NamedTuple() - end - - ini = read(Inifile(), f) - - credentials = ( - aws_access_key_id = String(get(ini, p, "aws_access_key_id")), - aws_secret_access_key = String(get(ini, p, "aws_secret_access_key"))) -end - -end # module AWS4AuthRequest diff --git a/src/BasicAuthRequest.jl b/src/BasicAuthRequest.jl index 7b64b4152..f8c72cb47 100644 --- a/src/BasicAuthRequest.jl +++ b/src/BasicAuthRequest.jl @@ -1,32 +1,27 @@ module BasicAuthRequest using ..Base64 - -import ..Layer, ..request using URIs -using ..Pairs: getkv, setkv +import ..Messages: setheader, hasheader import ..@debug, ..DEBUG_LEVEL +export basicauthlayer """ - request(BasicAuthLayer, method, ::URI, headers, body) -> HTTP.Response + basicauthlayer(req) -> HTTP.Response Add `Authorization: Basic` header using credentials from url userinfo. """ -abstract type BasicAuthLayer{Next <: Layer} <: Layer{Next} end -export BasicAuthLayer - -function request(::Type{BasicAuthLayer{Next}}, - method::String, url::URI, headers, body; kw...) where Next - - userinfo = unescapeuri(url.userinfo) - - if !isempty(userinfo) && getkv(headers, "Authorization", "") == "" - @debug 1 "Adding Authorization: Basic header." - setkv(headers, "Authorization", "Basic $(base64encode(userinfo))") +function basicauthlayer(handler) + return function(req; basicauth::Bool=true, kw...) + if basicauth + userinfo = unescapeuri(req.url.userinfo) + if !isempty(userinfo) && !hasheader(req.headers, "Authorization") + @debug 1 "Adding Authorization: Basic header." + setheader(req.headers, "Authorization" => "Basic $(base64encode(userinfo))") + end + end + return handler(req; kw...) end - - return request(Next, method, url, headers, body; kw...) end - end # module BasicAuthRequest diff --git a/src/CanonicalizeRequest.jl b/src/CanonicalizeRequest.jl index e3c5b9f9c..4985dd7e6 100644 --- a/src/CanonicalizeRequest.jl +++ b/src/CanonicalizeRequest.jl @@ -1,27 +1,26 @@ module CanonicalizeRequest -import ..Layer, ..request using ..Messages using ..Strings: tocameldash +export canonicalizelayer + """ - request(CanonicalizeLayer, method, ::URI, headers, body) -> HTTP.Response + canonicalizelayer(req) -> HTTP.Response Rewrite request and response headers in Canonical-Camel-Dash-Format. """ -abstract type CanonicalizeLayer{Next <: Layer} <: Layer{Next} end -export CanonicalizeLayer - -function request(::Type{CanonicalizeLayer{Next}}, - method::String, url, headers, body; kw...) where Next - - headers = canonicalizeheaders(headers) - - res = request(Next, method, url, headers, body; kw...) - - res.headers = canonicalizeheaders(res.headers) - - return res +function canonicalizelayer(handler) + return function(req; canonicalize_headers::Bool=false, kw...) + if canonicalize_headers + req.headers = canonicalizeheaders(req.headers) + end + res = handler(req; kw...) + if canonicalize_headers + res.headers = canonicalizeheaders(res.headers) + end + return res + end end canonicalizeheaders(h::T) where {T} = T([tocameldash(k) => v for (k,v) in h]) diff --git a/src/ConnectionRequest.jl b/src/ConnectionRequest.jl index 790c72f11..e92693746 100644 --- a/src/ConnectionRequest.jl +++ b/src/ConnectionRequest.jl @@ -1,14 +1,13 @@ module ConnectionRequest -import ..Layer, ..request using URIs, ..Sockets using ..Messages using ..IOExtras using ..ConnectionPool -using MbedTLS: SSLContext -using ..Pairs: getkv, setkv +using MbedTLS: SSLContext, SSLConfig using Base64: base64encode import ..@debug, ..DEBUG_LEVEL +import ..Streams: Stream # hasdotsuffix reports whether s ends in "."+suffix. hasdotsuffix(s, suffix) = endswith(s, "." * suffix) @@ -46,8 +45,10 @@ function getproxy(scheme, host) return nothing end +export connectionlayer + """ - request(ConnectionPoolLayer, ::URI, ::Request, body) -> HTTP.Response + connectionlayer(req) -> HTTP.Response Retrieve an `IO` connection from the [`ConnectionPool`](@ref). @@ -57,61 +58,59 @@ Otherwise leave it open so that it can be reused. `IO` related exceptions from `Base` are wrapped in `HTTP.IOError`. See [`isioerror`](@ref). """ -abstract type ConnectionPoolLayer{Next <: Layer} <: Layer{Next} end -export ConnectionPoolLayer - -function request(::Type{ConnectionPoolLayer{Next}}, url::URI, req, body; - proxy=getproxy(url.scheme, url.host), - socket_type::Type=TCPSocket, kw...) where Next - - if proxy !== nothing - target_url = url - url = URI(proxy) - if target_url.scheme == "http" - req.target = string(target_url) +function connectionlayer(handler) + return function(req; proxy=getproxy(req.url.scheme, req.url.host), socket_type::Type=TCPSocket, kw...) + if proxy !== nothing + target_url = req.url + url = URI(proxy) + if target_url.scheme == "http" + req.target = string(target_url) + end + + userinfo = unescapeuri(url.userinfo) + if !isempty(userinfo) && !hasheader(req.headers, "Proxy-Authorization") + @debug 1 "Adding Proxy-Authorization: Basic header." + setheader(req.headers, "Proxy-Authorization" => "Basic $(base64encode(userinfo))") + end + else + url = req.url end - userinfo = unescapeuri(url.userinfo) - if !isempty(userinfo) && getkv(req.headers, "Proxy-Authorization", "") == "" - @debug 1 "Adding Proxy-Authorization: Basic header." - setkv(req.headers, "Proxy-Authorization", "Basic $(base64encode(userinfo))") + IOType = sockettype(url, socket_type) + local io + try + io = newconnection(IOType, url.host, url.port; kw...) + catch e + rethrow(isioerror(e) ? IOError(e, "during request($url)") : e) end - end - IOType = sockettype(url, socket_type) - local io - try - io = newconnection(IOType, url.host, url.port; kw...) - catch e - rethrow(isioerror(e) ? IOError(e, "during request($url)") : e) - end + try + if proxy !== nothing && target_url.scheme == "https" + # tunnel request + target_url = URI(target_url, port=443) + r = connect_tunnel(io, target_url, req) + if r.status != 200 + close(io) + return r + end + io = ConnectionPool.sslupgrade(io, target_url.host; kw...) + req.headers = filter(x->x.first != "Proxy-Authorization", req.headers) + end + + stream = Stream(req.response, io) + resp = handler(stream; kw...) - try - if proxy !== nothing && target_url.scheme == "https" - # tunnel request - target_url = URI(target_url, port=443) - r = connect_tunnel(io, target_url, req) - if r.status != 200 + if proxy !== nothing && target_url.scheme == "https" close(io) - return r end - io = ConnectionPool.sslupgrade(io, target_url.host; kw...) - req.headers = filter(x->x.first != "Proxy-Authorization", req.headers) - end - r = request(Next, io, req, body; kw...) - - if proxy !== nothing && target_url.scheme == "https" - close(io) + return resp + catch e + @debug 1 "❗️ ConnectionLayer $e. Closing: $io" + try; close(io); catch; end + rethrow(isioerror(e) ? IOError(e, "during request($url)") : e) end - - return r - catch e - @debug 1 "❗️ ConnectionLayer $e. Closing: $io" - try; close(io); catch; end - rethrow(isioerror(e) ? IOError(e, "during request($url)") : e) end - end sockettype(url::URI, default) = url.scheme in ("wss", "https") ? SSLContext : default diff --git a/src/ContentTypeRequest.jl b/src/ContentTypeRequest.jl index d2f00becc..64495902c 100644 --- a/src/ContentTypeRequest.jl +++ b/src/ContentTypeRequest.jl @@ -1,30 +1,26 @@ module ContentTypeDetection -import ..Layer, ..request using URIs -using ..Pairs: getkv, setkv import ..sniff import ..Form using ..Messages -import ..MessageRequest: bodylength, bodybytes +import ..IOExtras import ..@debug, ..DEBUG_LEVEL -abstract type ContentTypeDetectionLayer{Next <: Layer} <: Layer{Next} end -export ContentTypeDetectionLayer +export contenttypedetectionlayer -function request(::Type{ContentTypeDetectionLayer{Next}}, - method::String, url::URI, headers, body; kw...) where Next +function contenttypedetectionlayer(handler) + return function(req; detect_content_type::Bool=false, kw...) + if detect_content_type && (!hasheader(req.headers, "Content-Type") + && !isa(req.body, Form) + && isbytes(req.body)) - if (getkv(headers, "Content-Type", "") == "" - && !isa(body, Form) - && bodylength(body) != unknown_length - && bodylength(body) > 0) - - sn = sniff(bodybytes(body)) - setkv(headers, "Content-Type", sn) - @debug 1 "setting Content-Type header to: $sn" + sn = sniff(bytes(req.body)) + setheader(req.headers, "Content-Type" => sn) + @debug 1 "setting Content-Type header to: $sn" + end + return handler(req; kw...) end - return request(Next, method, url, headers, body; kw...) end end # module diff --git a/src/CookieRequest.jl b/src/CookieRequest.jl index 7ff0038c5..895653ac4 100644 --- a/src/CookieRequest.jl +++ b/src/CookieRequest.jl @@ -1,11 +1,9 @@ module CookieRequest import ..Dates -import ..Layer, ..request using URIs using ..Cookies -using ..Messages: ascii_lc_isequal -using ..Pairs: getkv, setkv +using ..Messages: Request, ascii_lc_isequal, header, setheader import ..@debug, ..DEBUG_LEVEL, ..access_threaded const default_cookiejar = Dict{String, Set{Cookie}}[] @@ -15,38 +13,47 @@ function __init__() return end +export cookielayer + """ - request(CookieLayer, method, ::URI, headers, body) -> HTTP.Response + cookielayer(req) -> HTTP.Response Add locally stored Cookies to the request headers. Store new Cookies found in the response headers. """ -abstract type CookieLayer{Next <: Layer} <: Layer{Next} end -export CookieLayer - -function request(::Type{CookieLayer{Next}}, - method::String, url::URI, headers, body; - cookies::Union{Bool, Dict{<:AbstractString, <:AbstractString}}=Dict{String, String}(), - cookiejar::Dict{String, Set{Cookie}}=access_threaded(Dict{String, Set{Cookie}}, default_cookiejar), - kw...) where {Next} - - hostcookies = get!(cookiejar, url.host, Set{Cookie}()) - - cookiestosend = getcookies(hostcookies, url) - if !(cookies isa Bool) - for (name, value) in cookies - push!(cookiestosend, Cookie(name, value)) +function cookielayer(handler) + return function(req::Request; cookies=true, cookiejar::Dict{String, Set{Cookie}}=access_threaded(Dict{String, Set{Cookie}}, default_cookiejar), kw...) + if cookies === true || (cookies isa AbstractDict && !isempty(cookies)) + url = req.url + hostcookies = get!(cookiejar, url.host, Set{Cookie}()) + cookiestosend = getcookies(hostcookies, url) + if !(cookies isa Bool) + for (name, value) in cookies + push!(cookiestosend, Cookie(name, value)) + end + end + if !isempty(cookiestosend) + existingcookie = header(req.headers, "Cookie") + if existingcookie != "" && get(req.context, :includedCookies, nothing) !== nothing + # this is a redirect where we previously included cookies + # we want to filter those out to avoid duplicate cookie sending + # and the case where a cookie was set to expire from the 1st request + previouscookies = Cookies.readcookies(req.headers, "") + previouslyincluded = req.context[:includedCookies] + filtered = filter(x -> !(x.name in previouslyincluded), previouscookies) + existingcookie = stringify("", filtered) + end + setheader(req.headers, "Cookie" => stringify(existingcookie, cookiestosend)) + req.context[:includedCookies] = map(x -> x.name, cookiestosend) + end + res = handler(req; kw...) + setcookies(hostcookies, url.host, res.headers) + return res + else + # skip + return handler(req; kw...) end end - if !isempty(cookiestosend) - setkv(headers, "Cookie", stringify(getkv(headers, "Cookie", ""), cookiestosend)) - end - - res = request(Next, method, url, headers, body; kw...) - - setcookies(hostcookies, url.host, res.headers) - - return res end function getcookies(cookies, url) diff --git a/src/DebugRequest.jl b/src/DebugRequest.jl index 405688c70..598332a27 100644 --- a/src/DebugRequest.jl +++ b/src/DebugRequest.jl @@ -1,31 +1,25 @@ module DebugRequest -import ..Layer, ..request +import ..DEBUG_LEVEL using ..IOExtras - -const live_mode = true +import ..Streams: Stream include("IODebug.jl") +export debuglayer + """ - request(DebugLayer, ::IO, ::Request, body) -> HTTP.Response + debuglayer(stream::Stream) -> HTTP.Response Wrap the `IO` stream in an `IODebug` stream and print Message data. """ -abstract type DebugLayer{Next <:Layer} <: Layer{Next} end -export DebugLayer - -function request(::Type{DebugLayer{Next}}, io::IO, req, body; kw...) where Next - - @static if live_mode - return request(Next, IODebug(io), req, body; kw...) - else - iod = IODebug(io) - try - return request(Next, iod, req, body; kw...) - finally - show_log(stdout, iod) +function debuglayer(handler) + return function(stream::Stream; verbose::Int=0, kw...) + # if debugging, wrap stream.stream in IODebug + if verbose >= 3 || DEBUG_LEVEL[] >= 3 + stream = Stream(stream.message, IODebug(stream.stream)) end + return handler(stream; verbose=verbose, kw...) end end diff --git a/src/DefaultHeadersRequest.jl b/src/DefaultHeadersRequest.jl new file mode 100644 index 000000000..fb7b0474c --- /dev/null +++ b/src/DefaultHeadersRequest.jl @@ -0,0 +1,62 @@ +module DefaultHeadersRequest + +export defaultheaderslayer, setuseragent! + +using ..Messages +using ..IOExtras +import ..Form, ..content_type + +""" + defaultheaderslayer(req) -> Response + +Sets default expected headers. +""" +function defaultheaderslayer(handler) + return function(req; iofunction=nothing, kw...) + headers = req.headers + if isempty(req.url.port) || + (req.url.scheme == "http" && req.url.port == "80") || + (req.url.scheme == "https" && req.url.port == "443") + hostheader = req.url.host + else + hostheader = req.url.host * ":" * req.url.port + end + defaultheader!(headers, "Host" => hostheader) + defaultheader!(headers, "Accept" => "*/*") + if USER_AGENT[] !== nothing + defaultheader!(headers, "User-Agent" => USER_AGENT[]) + end + + if !hasheader(headers, "Content-Length") && + !hasheader(headers, "Transfer-Encoding") && + !hasheader(headers, "Upgrade") + l = nbytes(req.body) + if l !== nothing + setheader(headers, "Content-Length" => string(l)) + elseif req.method == "GET" && iofunction isa Function + setheader(headers, "Content-Length" => "0") + end + end + if !hasheader(headers, "Content-Type") && req.body isa Form && req.method in ("POST", "PUT") + # "Content-Type" => "multipart/form-data; boundary=..." + setheader(headers, content_type(req.body)) + end + return handler(req; iofunction=iofunction, kw...) + end +end + +const USER_AGENT = Ref{Union{String, Nothing}}("HTTP.jl/$VERSION") + +""" + setuseragent!(x::Union{String, Nothing}) + +Set the default User-Agent string to be used in each HTTP request. +Can be manually overridden by passing an explicit `User-Agent` header. +Setting `nothing` will prevent the default `User-Agent` header from being passed. +""" +function setuseragent!(x::Union{String, Nothing}) + USER_AGENT[] = x + return +end + +end # module diff --git a/src/ExceptionRequest.jl b/src/ExceptionRequest.jl index 21241f5ab..749300e00 100644 --- a/src/ExceptionRequest.jl +++ b/src/ExceptionRequest.jl @@ -2,27 +2,25 @@ module ExceptionRequest export StatusError -import ..Layer, ..request import ..HTTP using ..Messages: iserror +export exceptionlayer + """ - request(ExceptionLayer, ::URI, ::Request, body) -> HTTP.Response + exceptionlayer(stream) -> HTTP.Response Throw a `StatusError` if the request returns an error response status. """ -abstract type ExceptionLayer{Next <: Layer} <: Layer{Next} end -export ExceptionLayer - -function request(::Type{ExceptionLayer{Next}}, a...; kw...) where Next - - res = request(Next, a...; kw...) - - if iserror(res) - throw(StatusError(res.status, res.request.method, res.request.target, res)) +function exceptionlayer(handler) + return function(stream; status_exception::Bool=true, kw...) + res = handler(stream; kw...) + if status_exception && iserror(res) + throw(StatusError(res.status, res.request.method, res.request.target, res)) + else + return res + end end - - return res end """ diff --git a/src/HTTP.jl b/src/HTTP.jl index 894bb2282..724222336 100644 --- a/src/HTTP.jl +++ b/src/HTTP.jl @@ -1,9 +1,6 @@ module HTTP -export startwrite, startread, closewrite, closeread, stack, insert, insert_default!, - remove_default!, AWS4AuthLayer, BasicAuthLayer, CanonicalizeLayer, ConnectionPoolLayer, - ContentTypeDetectionLayer, DebugLayer, ExceptionLayer, MessageLayer, RedirectLayer, - RetryLayer, StreamLayer, TimeoutLayer, TopLayer, +export startwrite, startread, closewrite, closeread, @logfmt_str, common_logfmt, combined_logfmt const DEBUG_LEVEL = Ref(0) @@ -40,7 +37,26 @@ include("ConnectionPool.jl") include("Messages.jl") ;using .Messages include("cookies.jl") ;using .Cookies include("Streams.jl") ;using .Streams -include("layers.jl") ;using .Layers +include("MessageRequest.jl"); using .MessageRequest +include("RedirectRequest.jl"); using .RedirectRequest +include("DefaultHeadersRequest.jl"); using .DefaultHeadersRequest +include("BasicAuthRequest.jl"); using .BasicAuthRequest +include("CookieRequest.jl"); using .CookieRequest +include("CanonicalizeRequest.jl"); using .CanonicalizeRequest +include("TimeoutRequest.jl"); using .TimeoutRequest +include("ExceptionRequest.jl"); using .ExceptionRequest + import .ExceptionRequest.StatusError +include("RetryRequest.jl"); using .RetryRequest +include("ConnectionRequest.jl"); using .ConnectionRequest +include("DebugRequest.jl"); using .DebugRequest +include("StreamRequest.jl"); using .StreamRequest +include("ContentTypeRequest.jl"); using .ContentTypeDetection + +include("download.jl") +include("Servers.jl") ;using .Servers; using .Servers: listen +include("Handlers.jl") ;using .Handlers; using .Handlers: serve +include("parsemultipart.jl") ;using .MultiPartParsing: parse_multipart_form +include("WebSockets.jl") ;using .WebSockets const nobody = UInt8[] @@ -86,7 +102,7 @@ e.g. ```julia HTTP.request("GET", "http://httpbin.org/ip"; retries=4, cookies=true) -HTTP.get("http://s3.us-east-1.amazonaws.com/"; aws_authorization=true) +HTTP.get("http://s3.us-east-1.amazonaws.com/") conf = (readtimeout = 10, retry = false, @@ -153,19 +169,7 @@ SSLContext options Basic Authentication options - Basic authentication is detected automatically from the provided url's `userinfo` (in the form `scheme://user:password@host`) - and adds the `Authorization: Basic` header - - -AWS Authentication options - - - `aws_authorization = false`, enable AWS4 Authentication. - - `aws_service = split(url.host, ".")[1]` - - `aws_region = split(url.host, ".")[2]` - - `aws_access_key_id = ENV["AWS_ACCESS_KEY_ID"]` - - `aws_secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"]` - - `aws_session_token = get(ENV, "AWS_SESSION_TOKEN", "")` - - `body_sha256 = digest(MD_SHA256, body)`, - - `body_md5 = digest(MD_MD5, body)`, + and adds the `Authorization: Basic` header; this can be disabled by passing `basicauth=false` Cookie options @@ -323,14 +327,90 @@ end """ function request(method, url, h=Header[], b=nobody; headers=h, body=b, query=nothing, kw...)::Response - return request(HTTP.stack(;kw...), string(method), request_uri(url, query), mkheaders(headers), body; kw...) + return request(HTTP.stack(), method, url, headers, body, query; kw...) end -function request(stack::Type{<:Layer}, method, url, h=Header[], b=nobody; - headers=h, body=b, query=nothing, kw...)::Response - return request(stack, string(method), request_uri(url, query), mkheaders(headers), body; kw...) + +const STREAM_LAYERS = [timeoutlayer, exceptionlayer, debuglayer] +const REQUEST_LAYERS = [messagelayer, redirectlayer, defaultheaderslayer, basicauthlayer, contenttypedetectionlayer, cookielayer, retrylayer, canonicalizelayer] + +pushlayer!(layer; request::Bool=true) = push!(request ? REQUEST_LAYERS : STREAM_LAYERS, layer) +pushfirstlayer!(layer; request::Bool=true) = pushfirst!(request ? REQUEST_LAYERS : STREAM_LAYERS, layer) +poplayer!(; request::Bool=true) = pop!(request ? REQUEST_LAYERS : STREAM_LAYERS) +popfirstlayer!(; request::Bool=true) = popfirst!(request ? REQUEST_LAYERS : STREAM_LAYERS) + +function stack( + # custom layers + requestlayers=(), + streamlayers=()) + + # stream layers + layers = foldr((x, y) -> x(y), streamlayers, init=streamlayer) + layers2 = foldr((x, y) -> x(y), STREAM_LAYERS, init=layers) + # request layers + # messagelayer must be the 1st/outermost layer to convert initial args to Request + layers3 = foldr((x, y) -> x(y), requestlayers; init=connectionlayer(layers2)) + return foldr((x, y) -> x(y), REQUEST_LAYERS; init=layers3) +end + +function request(stack::Base.Callable, method, url, h=Header[], b=nobody, q=nothing; + headers=h, body=b, query=q, kw...)::Response + return stack(string(method), request_uri(url, query), mkheaders(headers), body; kw...) end -request(::Type{Union{}}, resp::Response) = resp +macro client(requestlayers, streamlayers=[]) + esc(quote + get(a...; kw...) = request("GET", a...; kw...) + put(a...; kw...) = request("PUT", a...; kw...) + post(a...; kw...) = request("POST", a...; kw...) + patch(a...; kw...) = request("PATCH", a...; kw...) + head(u; kw...) = request("HEAD", u; kw...) + delete(a...; kw...) = request("DELETE", a...; kw...) + request(method, url, h=HTTP.Header[], b=HTTP.nobody; headers=h, body=b, query=nothing, kw...)::HTTP.Response = + HTTP.request(HTTP.stack($requestlayers, $streamlayers), method, url, headers, body, query; kw...) + end) +end + +""" + HTTP.get(url [, headers]; ) -> HTTP.Response + +Shorthand for `HTTP.request("GET", ...)`. See [`HTTP.request`](@ref). +""" +get(a...; kw...) = request("GET", a...; kw...) + +""" + HTTP.put(url, headers, body; ) -> HTTP.Response + +Shorthand for `HTTP.request("PUT", ...)`. See [`HTTP.request`](@ref). +""" +put(a...; kw...) = request("PUT", a...; kw...) + +""" + HTTP.post(url, headers, body; ) -> HTTP.Response + +Shorthand for `HTTP.request("POST", ...)`. See [`HTTP.request`](@ref). +""" +post(a...; kw...) = request("POST", a...; kw...) + +""" + HTTP.patch(url, headers, body; ) -> HTTP.Response + +Shorthand for `HTTP.request("PATCH", ...)`. See [`HTTP.request`](@ref). +""" +patch(a...; kw...) = request("PATCH", a...; kw...) + +""" + HTTP.head(url; ) -> HTTP.Response + +Shorthand for `HTTP.request("HEAD", ...)`. See [`HTTP.request`](@ref). +""" +head(u; kw...) = request("HEAD", u; kw...) + +""" + HTTP.delete(url [, headers]; ) -> HTTP.Response + +Shorthand for `HTTP.request("DELETE", ...)`. See [`HTTP.request`](@ref). +""" +delete(a...; kw...) = request("DELETE", a...; kw...) request_uri(url, query) = URI(URI(url); query=query) request_uri(url, ::Nothing) = URI(url) @@ -395,220 +475,6 @@ function openraw(method::Union{String,Symbol}, url, headers=Header[]; kw...)::Tu take!(socketready) end -""" - HTTP.get(url [, headers]; ) -> HTTP.Response - -Shorthand for `HTTP.request("GET", ...)`. See [`HTTP.request`](@ref). -""" -get(a...; kw...) = request("GET", a...; kw...) - -""" - HTTP.put(url, headers, body; ) -> HTTP.Response - -Shorthand for `HTTP.request("PUT", ...)`. See [`HTTP.request`](@ref). -""" -put(u, h=[], b=""; kw...) = request("PUT", u, h, b; kw...) - -""" - HTTP.post(url, headers, body; ) -> HTTP.Response - -Shorthand for `HTTP.request("POST", ...)`. See [`HTTP.request`](@ref). -""" -post(u, h=[], b=""; kw...) = request("POST", u, h, b; kw...) - -""" - HTTP.patch(url, headers, body; ) -> HTTP.Response - -Shorthand for `HTTP.request("PATCH", ...)`. See [`HTTP.request`](@ref). -""" -patch(u, h=[], b=""; kw...) = request("PATCH", u, h, b; kw...) - -""" - HTTP.head(url; ) -> HTTP.Response - -Shorthand for `HTTP.request("HEAD", ...)`. See [`HTTP.request`](@ref). -""" -head(u; kw...) = request("HEAD", u; kw...) - -""" - HTTP.delete(url [, headers]; ) -> HTTP.Response - -Shorthand for `HTTP.request("DELETE", ...)`. See [`HTTP.request`](@ref). -""" -delete(a...; kw...) = request("DELETE", a...; kw...) - -include("TopRequest.jl"); using .TopRequest -include("RedirectRequest.jl"); using .RedirectRequest -include("BasicAuthRequest.jl"); using .BasicAuthRequest -include("AWS4AuthRequest.jl"); using .AWS4AuthRequest -include("CookieRequest.jl"); using .CookieRequest -include("CanonicalizeRequest.jl"); using .CanonicalizeRequest -include("TimeoutRequest.jl"); using .TimeoutRequest -include("MessageRequest.jl"); using .MessageRequest -include("ExceptionRequest.jl"); using .ExceptionRequest - import .ExceptionRequest.StatusError -include("RetryRequest.jl"); using .RetryRequest -include("ConnectionRequest.jl"); using .ConnectionRequest -include("DebugRequest.jl"); using .DebugRequest -include("StreamRequest.jl"); using .StreamRequest -include("ContentTypeRequest.jl"); using .ContentTypeDetection -include("exceptions.jl") - -""" -The `stack()` function returns the default HTTP Layer-stack type. -This type is passed as the first parameter to the [`HTTP.request`](@ref) function. - -`stack()` accepts optional keyword arguments to enable/disable specific layers -in the stack: -`request(method, args...; kw...) request(stack(; kw...), args...; kw...)` - - -The minimal request execution stack is: - -```julia -stack = MessageLayer{ConnectionPoolLayer{StreamLayer}} -``` - -The figure below illustrates the full request execution stack and its -relationship with [`HTTP.Response`](@ref), [`HTTP.Parsers`](@ref), -[`HTTP.Stream`](@ref) and the [`HTTP.ConnectionPool`](@ref). - -``` - ┌────────────────────────────────────────────────────────────────────────────┐ - │ ┌───────────────────┐ │ - │ HTTP.jl Request Execution Stack │ HTTP.ParsingError ├ ─ ─ ─ ─ ┐ │ - │ └───────────────────┘ │ - │ ┌───────────────────┐ │ │ - │ │ HTTP.IOError ├ ─ ─ ─ │ - │ └───────────────────┘ │ │ │ - │ ┌───────────────────┐ │ - │ │ HTTP.StatusError │─ ─ │ │ │ - │ └───────────────────┘ │ │ - │ ┌───────────────────┐ │ │ │ - │ request(method, url, headers, body) -> │ HTTP.Response │ │ │ - │ ────────────────────────── └─────────▲─────────┘ │ │ │ - │ ║ ║ │ │ - │ ┌────────────────────────────────────────────────────────────┐ │ │ │ - │ │ request(TopLayer, method, ::URI, ::Headers, body) │ │ │ - │ ├────────────────────────────────────────────────────────────┤ │ │ │ - │ │ request(BasicAuthLayer, method, ::URI, ::Headers, body) │ │ │ - │ ├────────────────────────────────────────────────────────────┤ │ │ │ - │ │ request(BasicAuthLayer, method, ::URI, ::Headers, body) │ │ │ - │ ├────────────────────────────────────────────────────────────┤ │ │ │ - │ │ request(CookieLayer, method, ::URI, ::Headers, body) │ │ │ - │ ├────────────────────────────────────────────────────────────┤ │ │ │ - │ │ request(CanonicalizeLayer, method, ::URI, ::Headers, body) │ │ │ - │ ├────────────────────────────────────────────────────────────┤ │ │ │ - │ │ request(MessageLayer, method, ::URI, ::Headers, body) │ │ │ - │ ├────────────────────────────────────────────────────────────┤ │ │ │ - │ │ request(AWS4AuthLayer, ::URI, ::Request, body) │ │ │ - │ ├────────────────────────────────────────────────────────────┤ │ │ │ - │ │ request(RetryLayer, ::URI, ::Request, body) │ │ │ - │ ├────────────────────────────────────────────────────────────┤ │ │ │ - │ │ request(ExceptionLayer, ::URI, ::Request, body) ├ ─ ┘ │ - │ ├────────────────────────────────────────────────────────────┤ │ │ │ -┌┼───┤ request(ConnectionPoolLayer, ::URI, ::Request, body) ├ ─ ─ ─ │ -││ ├────────────────────────────────────────────────────────────┤ │ │ -││ │ request(DebugLayer, ::IO, ::Request, body) │ │ -││ ├────────────────────────────────────────────────────────────┤ │ │ -││ │ request(TimeoutLayer, ::IO, ::Request, body) │ │ -││ ├────────────────────────────────────────────────────────────┤ │ │ -││ │ request(StreamLayer, ::IO, ::Request, body) │ │ -││ └──────────────┬───────────────────┬─────────────────────────┘ │ │ -│└──────────────────┼────────║──────────┼───────────────║─────────────────────┘ -│ │ ║ │ ║ │ -│┌──────────────────▼───────────────┐ │ ┌──────────────────────────────────┐ -││ HTTP.Request │ │ │ HTTP.Response │ │ -││ │ │ │ │ -││ method::String ◀───┼──▶ status::Int │ │ -││ target::String │ │ │ headers::Vector{Pair} │ -││ headers::Vector{Pair} │ │ │ body::Vector{UInt8} │ │ -││ body::Vector{UInt8} │ │ │ │ -│└──────────────────▲───────────────┘ │ └───────────────▲────────────────┼─┘ -│┌──────────────────┴────────║──────────▼───────────────║──┴──────────────────┐ -││ HTTP.Stream <:IO ║ ╔══════╗ ║ │ │ -││ ┌───────────────────────────┐ ║ ┌──▼─────────────────────────┐ │ -││ │ startwrite(::Stream) │ ║ │ startread(::Stream) │ │ │ -││ │ write(::Stream, body) │ ║ │ read(::Stream) -> body │ │ -││ │ ... │ ║ │ ... │ │ │ -││ │ closewrite(::Stream) │ ║ │ closeread(::Stream) │ │ -││ └───────────────────────────┘ ║ └────────────────────────────┘ │ │ -│└───────────────────────────║────────┬──║──────║───────║──┬──────────────────┘ -│┌──────────────────────────────────┐ │ ║ ┌────▼───────║──▼────────────────┴─┐ -││ HTTP.Messages │ │ ║ │ HTTP.Parsers │ -││ │ │ ║ │ │ -││ writestartline(::IO, ::Request) │ │ ║ │ parse_status_line(bytes, ::Req') │ -││ writeheaders(::IO, ::Request) │ │ ║ │ parse_header_field(bytes, ::Req')│ -│└──────────────────────────────────┘ │ ║ └──────────────────────────────────┘ -│ ║ │ ║ -│┌───────────────────────────║────────┼──║────────────────────────────────────┐ -└▶ HTTP.ConnectionPool ║ │ ║ │ - │ ┌──────────────▼────────┐ ┌───────────────────────┐ │ - │ getconnection() -> │ HTTP.Connection <:IO │ │ HTTP.Connection <:IO │ │ - │ └───────────────────────┘ └───────────────────────┘ │ - │ ║ ╲│╱ ║ ╲│╱ │ - │ ║ │ ║ │ │ - │ ┌───────────▼───────────┐ ┌───────────▼───────────┐ │ - │ pool: [│ HTTP.Connection │,│ HTTP.Connection │...]│ - │ └───────────┬───────────┘ └───────────┬───────────┘ │ - │ ║ │ ║ │ │ - │ ┌───────────▼───────────┐ ┌───────────▼───────────┐ │ - │ │ Base.TCPSocket <:IO │ │MbedTLS.SSLContext <:IO│ │ - │ └───────────────────────┘ └───────────┬───────────┘ │ - │ ║ ║ │ │ - │ ║ ║ ┌───────────▼───────────┐ │ - │ ║ ║ │ Base.TCPSocket <:IO │ │ - │ ║ ║ └───────────────────────┘ │ - └───────────────────────────║───────────║────────────────────────────────────┘ - ║ ║ - ┌───────────────────────────║───────────║──────────────┐ ┏━━━━━━━━━━━━━━━━━━┓ - │ HTTP Server ▼ │ ┃ data flow: ════▶ ┃ - │ Request Response │ ┃ reference: ────▶ ┃ - └──────────────────────────────────────────────────────┘ ┗━━━━━━━━━━━━━━━━━━┛ -``` -*See `docs/src/layers`[`.monopic`](http://monodraw.helftone.com).* -""" -function stack(;redirect=true, - aws_authorization=false, - cookies=false, - canonicalize_headers=false, - retry=true, - status_exception=true, - readtimeout=0, - detect_content_type=false, - verbose=0, - kw...) - - NoLayer = Union - stack = TopLayer{ - (redirect ? RedirectLayer : NoLayer){ - BasicAuthLayer{ - (detect_content_type ? ContentTypeDetectionLayer : NoLayer){ - (cookies === true || (cookies isa AbstractDict && !isempty(cookies)) ? - CookieLayer : NoLayer){ - (canonicalize_headers ? CanonicalizeLayer : NoLayer){ - MessageLayer{ - (aws_authorization ? AWS4AuthLayer : NoLayer){ - (retry ? RetryLayer : NoLayer){ - (status_exception ? ExceptionLayer : NoLayer){ - ConnectionPoolLayer{ - (verbose >= 3 || - DEBUG_LEVEL[] >= 3 ? DebugLayer : NoLayer){ - (readtimeout > 0 ? TimeoutLayer : NoLayer){ - StreamLayer{Union{}} - }}}}}}}}}}}}} - - reduce(Layers.EXTRA_LAYERS; init=stack) do stack, (before, custom) - insert(stack, before, custom) - end -end - -include("download.jl") -include("Servers.jl") ;using .Servers; using .Servers: listen -include("Handlers.jl") ;using .Handlers; using .Handlers: serve -include("parsemultipart.jl") ;using .MultiPartParsing: parse_multipart_form -include("WebSockets.jl") ;using .WebSockets - import .ConnectionPool: Connection function Base.parse(::Type{T}, str::AbstractString)::T where T <: Message diff --git a/src/IOExtras.jl b/src/IOExtras.jl index 314292293..8e44a8f59 100644 --- a/src/IOExtras.jl +++ b/src/IOExtras.jl @@ -10,25 +10,37 @@ module IOExtras using ..Sockets using MbedTLS: MbedException -export bytes, ByteView, nobytes, CodeUnits, IOError, isioerror, +export bytes, isbytes, nbytes, ByteView, nobytes, IOError, isioerror, startwrite, closewrite, startread, closeread, tcpsocket, localport, safe_getpeername - """ - bytes(s::String) + bytes(x) -Get a `Vector{UInt8}`, a vector of bytes of a string. +If `x` is "castable" to an `AbstractVector{UInt8}`, then an +`AbstractVector{UInt8}` is returned; otherwise `x` is returned. """ function bytes end -bytes(s::SubArray{UInt8}) = unsafe_wrap(Array, pointer(s), length(s)) +bytes(s::AbstractVector{UInt8}) = s +bytes(s::AbstractString) = codeunits(s) +bytes(x) = x + +"""whether `x` is "castable" to an `AbstractVector{UInt8}`; i.e. you can call `bytes(x)` if `isbytes(x)` === true""" +isbytes(x) = x isa AbstractVector{UInt8} || x isa AbstractString -const CodeUnits = Union{Vector{UInt8}, Base.CodeUnits} -bytes(s::Base.CodeUnits) = bytes(String(s)) -bytes(s::String) = codeunits(s) -bytes(s::SubString{String}) = codeunits(s) +""" + nbytes(x) -> Int -bytes(s::Vector{UInt8}) = s +Length in bytes of `x` if `x` is `isbytes(x)`. +""" +function nbytes end +nbytes(x) = nothing +nbytes(x::AbstractVector{UInt8}) = length(x) +nbytes(x::AbstractString) = sizeof(x) +nbytes(x::Vector{T}) where T <: AbstractString = sum(sizeof, x) +nbytes(x::Vector{T}) where T <: AbstractVector{UInt8} = sum(length, x) +nbytes(x::IOBuffer) = bytesavailable(x) +nbytes(x::Vector{IOBuffer}) = sum(bytesavailable, x) """ isioerror(exception) diff --git a/src/MessageRequest.jl b/src/MessageRequest.jl index ec02c55c6..ff18ad69b 100644 --- a/src/MessageRequest.jl +++ b/src/MessageRequest.jl @@ -1,102 +1,21 @@ module MessageRequest -export body_is_a_stream, body_was_streamed, setuseragent!, resource - - -import ..Layer, ..request -using ..IOExtras using URIs -using ..Messages -import ..Messages: bodylength import ..Headers -import ..Form, ..content_type +using ..Messages -""" -"request-target" per https://tools.ietf.org/html/rfc7230#section-5.3 -""" -resource(uri::URI) = string( isempty(uri.path) ? "/" : uri.path, - !isempty(uri.query) ? "?" : "", uri.query, - !isempty(uri.fragment) ? "#" : "", uri.fragment) +export messagelayer """ - request(MessageLayer, method, ::URI, headers, body) -> HTTP.Response + messagelayer(method, ::URI, headers, body) -> HTTP.Response -Construct a [`Request`](@ref) object and set mandatory headers. +Construct a [`Request`](@ref) object. """ -struct MessageLayer{Next <: Layer} <: Layer{Next} end -export MessageLayer - -function request(::Type{MessageLayer{Next}}, - method::String, url::URI, headers::Headers, body; - http_version=v"1.1", - target=resource(url), - parent=nothing, iofunction=nothing, kw...) where Next - - if isempty(url.port) || - (url.scheme == "http" && url.port == "80") || - (url.scheme == "https" && url.port == "443") - hostheader = url.host - else - hostheader = url.host * ":" * url.port - end - defaultheader!(headers, "Host" => hostheader) - defaultheader!(headers, "Accept" => "*/*") - if USER_AGENT[] !== nothing - defaultheader!(headers, "User-Agent" => USER_AGENT[]) - end - - if !hasheader(headers, "Content-Length") && - !hasheader(headers, "Transfer-Encoding") && - !hasheader(headers, "Upgrade") - l = bodylength(body) - if l != unknown_length - setheader(headers, "Content-Length" => string(l)) - elseif method == "GET" && iofunction isa Function - setheader(headers, "Content-Length" => "0") - end +function messagelayer(handler) + return function(method::String, url::URI, headers::Headers, body; response_stream=nothing, http_version=v"1.1", kw...) + req = Request(method, resource(url), headers, body; url=url, version=http_version, responsebody=response_stream) + return handler(req; response_stream=response_stream, kw...) end - if !hasheader(headers, "Content-Type") && body isa Form && method in ("POST", "PUT") - # "Content-Type" => "multipart/form-data; boundary=..." - setheader(headers, content_type(body)) - end - - req = Request(method, target, headers, bodybytes(body); - parent=parent, version=http_version) - - return request(Next, url, req, body; iofunction=iofunction, kw...) end -const USER_AGENT = Ref{Union{String, Nothing}}("HTTP.jl/$VERSION") - -""" - setuseragent!(x::Union{String, Nothing}) - -Set the default User-Agent string to be used in each HTTP request. -Can be manually overridden by passing an explicit `User-Agent` header. -Setting `nothing` will prevent the default `User-Agent` header from being passed. -""" -function setuseragent!(x::Union{String, Nothing}) - USER_AGENT[] = x - return -end - -bodylength(body) = unknown_length -bodylength(body::AbstractVector{UInt8}) = length(body) -bodylength(body::AbstractString) = sizeof(body) -bodylength(body::Form) = length(body) -bodylength(body::Vector{T}) where T <: AbstractString = sum(sizeof, body) -bodylength(body::Vector{T}) where T <: AbstractArray{UInt8,1} = sum(length, body) -bodylength(body::IOBuffer) = bytesavailable(body) -bodylength(body::Vector{IOBuffer}) = sum(bytesavailable, body) - -const body_is_a_stream = UInt8[] -const body_was_streamed = bytes("[Message Body was streamed]") -bodybytes(body) = body_is_a_stream -bodybytes(body::Vector{UInt8}) = body -bodybytes(body::IOBuffer) = read(body) -bodybytes(body::AbstractVector{UInt8}) = Vector{UInt8}(body) -bodybytes(body::AbstractString) = bytes(body) -bodybytes(body::Vector) = length(body) == 1 ? bodybytes(body[1]) : - body_is_a_stream - end # module MessageRequest diff --git a/src/Messages.jl b/src/Messages.jl index c03acb1b6..b67b17671 100644 --- a/src/Messages.jl +++ b/src/Messages.jl @@ -55,7 +55,7 @@ Streaming of request and response bodies is handled by the module Messages export Message, Request, Response, - reset!, status, method, headers, uri, body, + reset!, status, method, headers, uri, body, resource, iserror, isredirect, ischunked, issafe, isidempotent, header, hasheader, headercontains, setheader, defaultheader!, appendheader, mkheaders, readheaders, headerscomplete, @@ -66,6 +66,7 @@ export Message, Request, Response, import ..HTTP +using ..URIs using ..Pairs using ..IOExtras using ..Parsers @@ -74,6 +75,7 @@ import ..bytes include("ascii.jl") +const nobody = UInt8[] const unknown_length = typemax(Int) abstract type Message end @@ -103,26 +105,25 @@ Represents a HTTP Response Message. You can get each data with [`HTTP.status`](@ref), [`HTTP.headers`](@ref), and [`HTTP.body`](@ref). """ -mutable struct Response <: Message +mutable struct Response{T} <: Message version::VersionNumber status::Int16 headers::Headers - body::Vector{UInt8} - request::Message + body::T # Vector{UInt8} or IO + request::Union{Message, Nothing} @doc """ Response(status::Int, headers=[]; body=UInt8[], request=nothing) -> HTTP.Response """ - function Response(status::Integer, headers=[]; body=UInt8[], request=nothing) - r = new() - r.version = v"1.1" - r.status = status - r.headers = mkheaders(headers) - r.body = bytes(body) - if request !== nothing - r.request = request - end - return r + function Response(status::Integer, headers=[]; body=nobody, request=nothing) + b = isbytes(body) ? bytes(body) : something(body, nobody) + return new{typeof(b)}( + v"1.1", + status, + mkheaders(headers), + b, + request + ) end end @@ -140,11 +141,11 @@ HTTP.Response(200, headers; body = "Hello") Response() = Request().response Response(s::Int, body::AbstractVector{UInt8}) = Response(s; body=body) -Response(s::Int, body::AbstractString) = Response(s, bytes(body)) +Response(s::Int, body::AbstractString) = Response(s; body=bytes(body)) -Response(body) = Response(200, body) +Response(body) = Response(200; body=body) -Base.convert(::Type{Response},s::AbstractString) = Response(s) +Base.convert(::Type{Response}, s::AbstractString) = Response(s) function reset!(r::Response) r.version = v"1.1" @@ -152,7 +153,7 @@ function reset!(r::Response) if !isempty(r.headers) empty!(r.headers) end - if !isempty(r.body) + if r.body isa Vector{UInt8} && !isempty(r.body) empty!(r.body) end end @@ -179,6 +180,7 @@ Get body from a response. body(r::Response) = getfield(r, :body) # HTTP Request +const Context = Dict{Symbol, Any} """ Request <: Message @@ -197,29 +199,30 @@ Represents a HTTP Request Message. - `headers::Vector{Pair{String,String}}` [RFC7230 3.2](https://tools.ietf.org/html/rfc7230#section-3.2) -- `body::Vector{UInt8}` +- `body::Union{Vector{UInt8}, IO}` [RFC7230 3.3](https://tools.ietf.org/html/rfc7230#section-3.3) - `response`, the `Response` to this `Request` -- `txcount`, number of times this `Request` has been sent (see RetryRequest.jl). - - `parent`, the `Response` (if any) that led to this request (e.g. in the case of a redirect). [RFC7230 6.4](https://tools.ietf.org/html/rfc7231#section-6.4) +- `context`, a `Dict{Symbol, Any}` store used by middleware to share state + You can get each data with [`HTTP.method`](@ref), [`HTTP.headers`](@ref), [`HTTP.uri`](@ref), and [`HTTP.body`](@ref). """ -mutable struct Request <: Message +mutable struct Request{T} <: Message method::String target::String version::VersionNumber headers::Headers - body::Vector{UInt8} + body::T # Vector{UInt8} or some kind of IO response::Response - txcount::Int - parent + url::URI + parent::Union{Response, Nothing} + context::Context end Request() = Request("", "") @@ -230,20 +233,29 @@ Request() = Request("", "") Constructor for `HTTP.Request`. For daily use, see [`HTTP.request`](@ref). """ -function Request(method::String, target, headers=[], body=UInt8[]; - version=v"1.1", parent=nothing) - r = Request(method, +function Request(method::String, target, headers=[], body=nobody; + version=v"1.1", url::URI=URI(), responsebody=nothing, parent=nothing, context=Context()) + b = isbytes(body) ? bytes(body) : body + r = Request{b === nothing ? Any : typeof(b)}(method, target == "" ? "/" : target, version, mkheaders(headers), - bytes(body), - Response(0), - 0, - parent) + b, + Response(0; body=responsebody), + url, + parent, + context) r.response.request = r return r end +""" +"request-target" per https://tools.ietf.org/html/rfc7230#section-5.3 +""" +resource(uri::URI) = string( isempty(uri.path) ? "/" : uri.path, + !isempty(uri.query) ? "?" : "", uri.query, + !isempty(uri.fragment) ? "#" : "", uri.fragment) + mkheaders(h::Headers) = h mkheaders(h)::Headers = Header[string(k) => string(v) for (k,v) in h] @@ -584,7 +596,7 @@ body_show_max = 1000 The first chunk of the Message Body (for display purposes). """ -bodysummary(bytes) = view(bytes, 1:min(length(bytes), body_show_max)) +bodysummary(body) = isbytes(body) ? view(bytes(body), 1:min(nbytes(body), body_show_max)) : "[Message Body was streamed]" function compactstartline(m::Message) b = IOBuffer() @@ -610,8 +622,8 @@ function Base.show(io::IO, m::Message) summary = bodysummary(m.body) validsummary = isvalidstr(summary) validsummary && write(io, summary) - if !validsummary || length(m.body) > length(summary) - println(io, "\n⋮\n$(length(m.body))-byte body") + if !validsummary || something(nbytes(m.body), 0) > length(summary) + println(io, "\n⋮\n$(nbytes(m.body))-byte body") end print(io, "\"\"\"") return diff --git a/src/RedirectRequest.jl b/src/RedirectRequest.jl index 9ee5c085c..50fb755cf 100644 --- a/src/RedirectRequest.jl +++ b/src/RedirectRequest.jl @@ -1,65 +1,65 @@ module RedirectRequest -import ..Layer, ..request using URIs using ..Messages using ..Pairs: setkv import ..Header import ..@debug, ..DEBUG_LEVEL +export redirectlayer, nredirects + """ - request(RedirectLayer, method, ::URI, headers, body) -> HTTP.Response + redirectlayer(req) -> HTTP.Response Redirects the request in the case of 3xx response status. """ -abstract type RedirectLayer{Next <: Layer} <: Layer{Next} end -export RedirectLayer - -function request(::Type{RedirectLayer{Next}}, - method::String, url::URI, headers, body; - redirect_limit=3, forwardheaders=true, kw...) where Next - count = 0 - while true - - # Verify the url before making the request. Verification is done in - # the redirect loop to also catch bad redirect URLs. - verify_url(url) - - res = request(Next, method, url, headers, body; reached_redirect_limit=(count == redirect_limit), kw...) - - if (count == redirect_limit - || !isredirect(res) - || (location = header(res, "Location")) == "") - return res +function redirectlayer(handler) + return function(req; redirect::Bool=true, redirect_limit::Int=3, forwardheaders::Bool=true, response_stream=nothing, kw...) + if !redirect || redirect_limit == 0 + # no redirecting + return handler(req; redirect_limit=redirect_limit, kw...) end + count = 0 + while true + # Verify the url before making the request. Verification is done in + # the redirect loop to also catch bad redirect URLs. + verify_url(req.url) + res = handler(req; redirect_limit=redirect_limit, kw...) - kw = merge(merge(NamedTuple(), kw), (parent = res,)) - oldurl = url - url = resolvereference(url, location) - if forwardheaders - headers = filter(headers) do h - # false return values are filtered out - header, value = h - if header == "Host" - return false - elseif (header in SENSITIVE_HEADERS - && !isdomainorsubdomain(url.host, oldurl.host)) - return false - else - return true + if (count == redirect_limit || !isredirect(res) + || (location = header(res, "Location")) == "") + return res + end + + # follow redirect + oldurl = req.url + url = resolvereference(req.url, location) + req = Request(req.method, resource(url), copy(req.headers), req.body; + url=url, version=req.version, responsebody=response_stream, parent=res, context=req.context) + if forwardheaders + req.headers = filter(req.headers) do (header, _) + # false return values are filtered out + if header == "Host" + return false + elseif (header in SENSITIVE_HEADERS && !isdomainorsubdomain(url.host, oldurl.host)) + return false + else + return true + end end + else + req.headers = Header[] end - else - headers = Header[] + @debug 1 "➡️ Redirect: $url" + count += 1 end - - @debug 1 "➡️ Redirect: $url" - - count += 1 + @assert false "Unreachable!" end +end - @assert false "Unreachable!" +function nredirects(req) + return req.parent === nothing ? 0 : (1 + nredirects(req.parent.request)) end const SENSITIVE_HEADERS = Set([ diff --git a/src/RetryRequest.jl b/src/RetryRequest.jl index 66aa7f08e..1c495e635 100644 --- a/src/RetryRequest.jl +++ b/src/RetryRequest.jl @@ -1,15 +1,16 @@ module RetryRequest import ..HTTP -import ..Layer, ..request using ..Sockets using ..IOExtras using ..MessageRequest using ..Messages import ..@debug, ..DEBUG_LEVEL, ..sprintcompact +export retrylayer + """ - request(RetryLayer, ::URI, ::Request, body) -> HTTP.Response + retrylayer(req) -> HTTP.Response Retry the request if it throws a recoverable exception. @@ -21,27 +22,27 @@ Methods of `isrecoverable(e)` define which exception types lead to a retry. e.g. `HTTP.IOError`, `Sockets.DNSError`, `Base.EOFError` and `HTTP.StatusError` (if status is ``5xx`). """ -abstract type RetryLayer{Next <: Layer} <: Layer{Next} end -export RetryLayer - -function request(::Type{RetryLayer{Next}}, url, req, body; - retries::Int=4, retry_non_idempotent::Bool=false, - kw...) where Next - - retry_request = Base.retry(request, - delays=ExponentialBackOff(n = retries), - check=(s,ex)->begin - retry = isrecoverable(ex, req, retry_non_idempotent) - if retry - @debug 1 "🔄 Retry $ex: $(sprintcompact(req))" - reset!(req.response) - else - @debug 1 "🚷 No Retry: $(no_retry_reason(ex, req))" - end - return s, retry - end) +function retrylayer(handler) + return function(req::Request; retry::Bool=true, retries::Int=4, retry_non_idempotent::Bool=false, kw...) + if !retry || retries == 0 + # no retry + return handler(req; kw...) + end + retry_request = Base.retry(handler, + delays=ExponentialBackOff(n = retries), + check=(s, ex)->begin + retry = isrecoverable(ex, req, retry_non_idempotent, get(req.context, :retrycount, 0)) + if retry + @debug 1 "🔄 Retry $ex: $(sprintcompact(req))" + reset!(req.response) + else + @debug 1 "🚷 No Retry: $(no_retry_reason(ex, req))" + end + return s, retry + end) - retry_request(Next, url, req, body; kw...) + return retry_request(req; kw...) + end end isrecoverable(e) = false @@ -51,11 +52,11 @@ isrecoverable(e::HTTP.StatusError) = e.status == 403 || # Forbidden e.status == 408 || # Timeout e.status >= 500 # Server Error -isrecoverable(e, req, retry_non_idempotent) = +isrecoverable(e, req, retry_non_idempotent, retrycount) = isrecoverable(e) && - !(req.body === body_was_streamed) && - !(req.response.body === body_was_streamed) && - (retry_non_idempotent || req.txcount == 0 || isidempotent(req)) + isbytes(req.body) && + isbytes(req.response.body) && + (retry_non_idempotent || retrycount == 0 || isidempotent(req)) # "MUST NOT automatically retry a request with a non-idempotent method" # https://tools.ietf.org/html/rfc7230#section-6.3.1 @@ -66,8 +67,8 @@ function no_retry_reason(ex, req) print(buf, ", ", ex isa HTTP.StatusError ? "HTTP $(ex.status): " : !isrecoverable(ex) ? "$ex not recoverable, " : "", - (req.body === body_was_streamed) ? "request streamed, " : "", - (req.response.body === body_was_streamed) ? "response streamed, " : "", + !isbytes(req.body) ? "request streamed, " : "", + !isbytes(req.response.body) ? "response streamed, " : "", !isidempotent(req) ? "$(req.method) non-idempotent" : "") return String(take!(buf)) end diff --git a/src/StreamRequest.jl b/src/StreamRequest.jl index 6a18961b7..6061a7be1 100644 --- a/src/StreamRequest.jl +++ b/src/StreamRequest.jl @@ -1,15 +1,17 @@ module StreamRequest -import ..Layer, ..request using ..IOExtras using ..Messages using ..Streams import ..ConnectionPool using ..MessageRequest +import ..RedirectRequest: nredirects import ..@debug, ..DEBUG_LEVEL, ..printlncompact, ..sprintcompact +export streamlayer + """ - request(StreamLayer, ::IO, ::Request, body) -> HTTP.Response + streamlayer(stream) -> HTTP.Response Create a [`Stream`](@ref) to send a `Request` and `body` to an `IO` stream and read the response. @@ -19,27 +21,18 @@ immediately so that the transmission can be aborted if the `Response` status indicates that the server does not wish to receive the message body. [RFC7230 6.5](https://tools.ietf.org/html/rfc7230#section-6.5). """ -abstract type StreamLayer{Next <: Layer} <: Layer{Next} end -export StreamLayer - -function request(::Type{StreamLayer{Next}}, io::IO, req::Request, body; - reached_redirect_limit=false, - response_stream=nothing, - iofunction=nothing, - verbose::Int=0, - kw...)::Response where Next - +function streamlayer(stream::Stream; iofunction=nothing, verbose=0, redirect_limit::Int=3, kw...)::Response + response = stream.message + req = response.request + io = stream.stream verbose == 1 && printlncompact(req) - - response = req.response - http = Stream(response, io) @debug 2 "client startwrite" - startwrite(http) + startwrite(stream) if verbose == 2 println(req) - if iofunction === nothing && req.body === body_is_a_stream - println("$(typeof(req)).body: $(sprintcompact(body))") + if iofunction === nothing && !isbytes(req.body) + println("$(typeof(req)).body: $(sprintcompact(req.body))") end end @@ -48,21 +41,20 @@ function request(::Type{StreamLayer{Next}}, io::IO, req::Request, body; @sync begin if iofunction === nothing @async try - writebody(http, req, body) + writebody(stream, req) @debug 2 "client closewrite" - closewrite(http) + closewrite(stream) catch e write_error = e isopen(io) && try; close(io); catch; end end @debug 2 "client startread" - startread(http) - readbody(http, response, response_stream, reached_redirect_limit) + startread(stream) + readbody(stream, response, redirect_limit == nredirects(req)) else - iofunction(http) + iofunction(stream) end - - if isaborted(http) + if isaborted(stream) # The server may have closed the connection. # Don't propagate such errors. try; close(io); catch; end @@ -77,50 +69,47 @@ function request(::Type{StreamLayer{Next}}, io::IO, req::Request, body; end @debug 2 "client closewrite" - closewrite(http) + closewrite(stream) @debug 2 "client closeread" - closeread(http) + closeread(stream) verbose == 1 && printlncompact(response) verbose == 2 && println(response) - return request(Next, response) + return response end -function writebody(http::Stream, req::Request, body) +function writebody(stream::Stream, req::Request) - if req.body === body_is_a_stream - writebodystream(http, req, body) - closebody(http) + if !isbytes(req.body) + writebodystream(stream, req.body) + closebody(stream) else - write(http, req.body) + write(stream, req.body) end - - req.txcount += 1 + req.context[:retrycount] = get(req.context, :retrycount, 0) + 1 return end -function writebodystream(http, req, body) +function writebodystream(stream, body) for chunk in body - writechunk(http, req, chunk) + writechunk(stream, chunk) end end -function writebodystream(http, req, body::IO) - req.body = body_was_streamed - write(http, body) +function writebodystream(stream, body::IO) + write(stream, body) end -writechunk(http, req, body::IO) = writebodystream(http, req, body) -writechunk(http, req, body) = write(http, body) +writechunk(stream, body::IO) = writebodystream(stream, body) +writechunk(stream, body) = write(stream, body) -function readbody(http::Stream, res::Response, response_stream, reached_redirect_limit) - if response_stream === nothing - res.body = read(http) +function readbody(stream::Stream, res::Response, redirectlimitreached) + if isbytes(res.body) + res.body = read(stream) else - if reached_redirect_limit || !isredirect(res) - res.body = body_was_streamed - write(response_stream, http) + if redirectlimitreached || !isredirect(res) + write(res.body, stream) end end end diff --git a/src/Streams.jl b/src/Streams.jl index e763a427d..a75c4a56c 100644 --- a/src/Streams.jl +++ b/src/Streams.jl @@ -45,7 +45,7 @@ Creates a `HTTP.Stream` that wraps an existing `IO` stream. for reuse. If a complete response has not been received, `closeread` throws `EOFError`. """ -Stream(r::M, io::S) where {M, S} = Stream{M,S}(r, io, false, false, true, 0, 0) +Stream(r::M, io::S) where {M, S} = Stream{M, S}(r, io, false, false, true, 0, 0) header(http::Stream, a...) = header(http.message, a...) setstatus(http::Stream, status) = (http.message.response.status = status) @@ -65,8 +65,8 @@ IOExtras.isopen(http::Stream) = isopen(http.stream) # Writing HTTP Messages -messagetowrite(http::Stream{Response}) = http.message.request -messagetowrite(http::Stream{Request}) = http.message.response +messagetowrite(http::Stream{<:Response}) = http.message.request +messagetowrite(http::Stream{<:Request}) = http.message.response IOExtras.iswritable(http::Stream) = iswritable(http.stream) @@ -124,7 +124,7 @@ function closebody(http::Stream) end end -function IOExtras.closewrite(http::Stream{Response}) +function IOExtras.closewrite(http::Stream{<:Response}) if !iswritable(http) return end @@ -132,7 +132,7 @@ function IOExtras.closewrite(http::Stream{Response}) closewrite(http.stream) end -function IOExtras.closewrite(http::Stream{Request}) +function IOExtras.closewrite(http::Stream{<:Request}) if iswritable(http) closebody(http) @@ -176,14 +176,14 @@ end https://tools.ietf.org/html/rfc7230#section-5.6 https://tools.ietf.org/html/rfc7231#section-6.2.1 """ -function handle_continue(http::Stream{Response}) +function handle_continue(http::Stream{<:Response}) if http.message.status == 100 @debug 1 "✅ Continue: $(http.stream)" readheaders(http.stream, http.message) end end -function handle_continue(http::Stream{Request}) +function handle_continue(http::Stream{<:Request}) if hasheader(http.message, "Expect", "100-continue") if !iswritable(http.stream) startwrite(http.stream) @@ -316,7 +316,7 @@ function Base.read(http::Stream) end """ - isaborted(::Stream{Response}) + isaborted(::Stream{<:Response}) Has the server signaled that it does not wish to receive the message body? @@ -325,7 +325,7 @@ Has the server signaled that it does not wish to receive the message body? immediately cease transmitting the body and close the connection." [RFC7230, 6.5](https://tools.ietf.org/html/rfc7230#section-6.5) """ -function isaborted(http::Stream{Response}) +function isaborted(http::Stream{<:Response}) if iswritable(http.stream) && iserror(http.message) && @@ -341,7 +341,7 @@ end incomplete(http::Stream) = http.ntoread > 0 && (http.readchunked || http.ntoread != unknown_length) -function IOExtras.closeread(http::Stream{Response}) +function IOExtras.closeread(http::Stream{<:Response}) if hasheader(http.message, "Connection", "close") # Close conncetion if server sent "Connection: close"... @@ -368,7 +368,7 @@ function IOExtras.closeread(http::Stream{Response}) return http.message end -function IOExtras.closeread(http::Stream{Request}) +function IOExtras.closeread(http::Stream{<:Request}) if incomplete(http) # Error if Message is not complete... close(http.stream) diff --git a/src/TimeoutRequest.jl b/src/TimeoutRequest.jl index 0651c8a9e..1dbb3226a 100644 --- a/src/TimeoutRequest.jl +++ b/src/TimeoutRequest.jl @@ -1,8 +1,8 @@ module TimeoutRequest -import ..Layer, ..request using ..ConnectionPool import ..@debug, ..DEBUG_LEVEL +import ..Streams: Stream struct ReadTimeoutError <:Exception readtimeout::Int @@ -12,39 +12,43 @@ function Base.showerror(io::IO, e::ReadTimeoutError) print(io, "ReadTimeoutError: Connection closed after $(e.readtimeout) seconds") end +export timeoutlayer + """ - request(TimeoutLayer, ::IO, ::Request, body) -> HTTP.Response + timeoutlayer(stream) -> HTTP.Response Close `IO` if no data has been received for `timeout` seconds. """ -abstract type TimeoutLayer{Next <: Layer} <: Layer{Next} end -export TimeoutLayer - -function request(::Type{TimeoutLayer{Next}}, io::IO, req, body; - readtimeout::Int=0, kw...) where Next - - wait_for_timeout = Ref{Bool}(true) - timedout = Ref{Bool}(false) - - @async while wait_for_timeout[] - if isreadable(io) && inactiveseconds(io) > readtimeout - timedout[] = true - close(io) - @debug 1 "💥 Read inactive > $(readtimeout)s: $io" - break +function timeoutlayer(handler) + return function(stream::Stream; readtimeout::Int=0, kw...) + if readtimeout <= 0 + # skip + return handler(stream; kw...) + end + io = stream.stream + wait_for_timeout = Ref{Bool}(true) + timedout = Ref{Bool}(false) + + @async while wait_for_timeout[] + if isreadable(io) && inactiveseconds(io) > readtimeout + timedout[] = true + close(io) + @debug 1 "💥 Read inactive > $(readtimeout)s: $io" + break + end + sleep(readtimeout / 10) end - sleep(readtimeout / 10) - end - try - return request(Next, io, req, body; kw...) - catch e - if timedout[] - throw(ReadTimeoutError(readtimeout)) + try + return handler(stream; kw...) + catch e + if timedout[] + throw(ReadTimeoutError(readtimeout)) + end + rethrow(e) + finally + wait_for_timeout[] = false end - rethrow(e) - finally - wait_for_timeout[] = false end end diff --git a/src/TopRequest.jl b/src/TopRequest.jl deleted file mode 100644 index 54510a74e..000000000 --- a/src/TopRequest.jl +++ /dev/null @@ -1,18 +0,0 @@ -module TopRequest - -import ..Layer, ..request - -export TopLayer - -""" - request(TopLayer, args...; kwargs...) - -This layer is at the top of every stack, and does nothing. -It's useful for inserting a custom layer at the top of the stack. -""" -abstract type TopLayer{Next <: Layer} <: Layer{Next} end - -request(::Type{TopLayer{Next}}, args...; kwargs...) where Next = - request(Next, args...; kwargs...) - -end diff --git a/src/exceptions.jl b/src/exceptions.jl deleted file mode 100644 index 6ddd09ee5..000000000 --- a/src/exceptions.jl +++ /dev/null @@ -1,7 +0,0 @@ -struct LayerNotFoundException <: Exception - var::String -end - -function Base.showerror(io::IO, e::LayerNotFoundException) - println(io, typeof(e), ": ", e.var) -end diff --git a/src/layers.jl b/src/layers.jl deleted file mode 100644 index bb58e53a7..000000000 --- a/src/layers.jl +++ /dev/null @@ -1,119 +0,0 @@ -module Layers -export Layer, next, top_layer, insert, insert_default!, remove_default! - -const EXTRA_LAYERS = Set{Tuple{Union{UnionAll, Type{Union{}}}, UnionAll}}() - -include("exceptions.jl") - -""" -## Request Execution Stack - -The Request Execution Stack is separated into composable layers. - -Each layer is defined by a nested type `Layer{Next}` where the `Next` -parameter defines the next layer in the stack. -The `request` method for each layer takes a `Layer{Next}` type as -its first argument and dispatches the request to the next layer -using `request(Next, ...)`. - -The example below defines three layers and three stacks each with -a different combination of layers. - - -```julia -abstract type Layer end -abstract type Layer1{Next <: Layer} <: Layer end -abstract type Layer2{Next <: Layer} <: Layer end -abstract type Layer3 <: Layer end - -request(::Type{Layer1{Next}}, data) where Next = "L1", request(Next, data) -request(::Type{Layer2{Next}}, data) where Next = "L2", request(Next, data) -request(::Type{Layer3}, data) = "L3", data - -const stack1 = Layer1{Layer2{Layer3}} -const stack2 = Layer2{Layer1{Layer3}} -const stack3 = Layer1{Layer3} -``` - -```julia -julia> request(stack1, "foo") -("L1", ("L2", ("L3", "foo"))) - -julia> request(stack2, "bar") -("L2", ("L1", ("L3", "bar"))) - -julia> request(stack3, "boo") -("L1", ("L3", "boo")) -``` - -This stack definition pattern gives the user flexibility in how layers are -combined but still allows Julia to do whole-stack compile time optimisations. - -e.g. the `request(stack1, "foo")` call above is optimised down to a single -function: -```julia -julia> code_typed(request, (Type{stack1}, String))[1].first -CodeInfo(:(begin - return (Core.tuple)("L1", (Core.tuple)("L2", (Core.tuple)("L3", data))) -end)) -``` -""" -abstract type Layer{Next} end - -""" - next(::Type{S}) where {T, S<:Layer{T}} - -Return the next `Layer` in the stack - -Example: -stack = MessageLayer{ConnectionPoolLayer{StreamLayer{Union{}}} -next(stack) # ConnectionPoolLayer{StreamLayer{Union{}}} -""" -next(::Type{S}) where {T, S<:Layer{T}} = T - -""" - top_layer(::Type{T}) where T <: Layer - -Return the parametric type of the top most `Layer` in the stack - -Example: -stack = MessageLayer{ConnectionPoolLayer{StreamLayer{Union{}}} -top_layer(stack) # MessageLayer -""" -top_layer(::Type{T}) where T <: Layer = T.name.wrapper -top_layer(::Type{Union{}}) = Union{} - -""" - insert(stack::Type{<:Layer}, layer_before::Type{<:Layer}, custom_layer::Type{<:Layer}) - -Insert your `custom_layer` in-front of the `layer_before` - -Example: -stack = MessageLayer{ConnectionPoolLayer{StreamLayer{Union{}}} -result = insert(stack, MessageLayer, TestLayer) # TestLayer{MessageLayer{ConnectionPoolLayer{StreamLayer{Union{}}}}} -""" -function insert(stack::Type{<:Layer}, layer_before::Type{<:Layer}, custom_layer::Type{<:Layer}) - new_stack = Union - head_layer = top_layer(stack) - rest_stack = stack - - while true - if head_layer === layer_before - return new_stack{custom_layer{rest_stack}} - else - head_layer === Union{} && break - new_stack = new_stack{head_layer{T}} where T - rest_stack = next(rest_stack) - head_layer = top_layer(rest_stack) - end - end - throw(LayerNotFoundException("$layer_before not found in $stack")) -end - -insert_default!(before::Type{<:Layer}, custom_layer::Type{<:Layer}) = - push!(EXTRA_LAYERS, (before, custom_layer)) - -remove_default!(before::Type{<:Layer}, custom_layer::Type{<:Layer}) = - delete!(EXTRA_LAYERS, (before, custom_layer)) - -end diff --git a/src/multipart.jl b/src/multipart.jl index a6071d65b..fe8671a47 100644 --- a/src/multipart.jl +++ b/src/multipart.jl @@ -10,6 +10,7 @@ Base.eof(f::Form) = f.index > length(f.data) Base.isopen(f::Form) = false Base.close(f::Form) = nothing Base.length(f::Form) = sum(x->isa(x, IOStream) ? filesize(x) - position(x) : bytesavailable(x), f.data) +IOExtras.nbytes(x::Form) = length(x) function Base.position(f::Form) index = f.index foreach(mark, f.data) diff --git a/src/sniff.jl b/src/sniff.jl index c0cc9abd8..62daea4a2 100644 --- a/src/sniff.jl +++ b/src/sniff.jl @@ -1,3 +1,5 @@ +const CodeUnits = Union{Vector{UInt8}, Base.CodeUnits} + # compression detection const ZIP = UInt8[0x50, 0x4b, 0x03, 0x04] const GZIP = UInt8[0x1f, 0x8b, 0x08] diff --git a/test/aws4.jl b/test/aws4.jl deleted file mode 100644 index 97cb3c67f..000000000 --- a/test/aws4.jl +++ /dev/null @@ -1,232 +0,0 @@ -using Dates -using Test -using HTTP -using HTTP: Headers, URI -using HTTP.AWS4AuthRequest: sign_aws4! - -const useragent = HTTP.MessageRequest.USER_AGENT[] -HTTP.setuseragent!(nothing) - -# Based on https://docs.aws.amazon.com/general/latest/gr/signature-v4-test-suite.html -# and https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html - -function test_sign!(method, headers, params, body=""; opts...) - sign_aws4!(method, - URI("https://example.amazonaws.com/" * params), - headers, - Vector{UInt8}(body); - timestamp=DateTime(2015, 8, 30, 12, 36), - aws_service="service", - aws_region="us-east-1", - # NOTE: These are the example credentials as specified in the AWS docs, - # they are not real - aws_access_key_id="AKIDEXAMPLE", - aws_secret_access_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", - include_md5=false, - include_sha256=false, - opts...) - headers -end - -function test_auth_string(headers, sig, key="AKIDEXAMPLE", date="20150830", service="service") - d = [ - "AWS4-HMAC-SHA256 Credential" => "$key/$date/us-east-1/$service/aws4_request", - "SignedHeaders" => headers, - "Signature" => sig, - ] - join(map(p->join(p, '='), d), ", ") -end - -header_keys(headers) = sort!(map(first, headers)) - -const required_headers = ["Authorization", "host", "x-amz-date"] - -@testset "AWS Signature Version 4" begin - # The signature for requests with no headers where the path ends up as simply / - slash_only_sig = "5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31" - noheaders = [ - ("get-vanilla", "", slash_only_sig), - ("get-vanilla-empty-query-key", "?Param1=value1", "a67d582fa61cc504c4bae71f336f98b97f1ea3c7a6bfe1b6e45aec72011b9aeb"), - ("get-utf8", "ሴ", "8318018e0b0f223aa2bbf98705b62bb787dc9c0e678f255a891fd03141be5d85"), - ("get-relative", "example/..", slash_only_sig), - ("get-relative-relative", "example1/example2/../..", slash_only_sig), - ("get-slash", "/", slash_only_sig), - ("get-slash-dot-slash", "./", slash_only_sig), - ("get-slashes", "example/", "9a624bd73a37c9a373b5312afbebe7a714a789de108f0bdfe846570885f57e84"), - ("get-slash-pointless-dot", "./example", "ef75d96142cf21edca26f06005da7988e4f8dc83a165a80865db7089db637ec5"), - ("get-space", "example space/", "652487583200325589f1fba4c7e578f72c47cb61beeca81406b39ddec1366741"), - ("post-vanilla", "", "5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b"), - ("post-vanilla-empty-query-value", "?Param1=value1", "28038455d6de14eafc1f9222cf5aa6f1a96197d7deb8263271d420d138af7f11"), - ] - @testset "$name" for (name, p, sig) in noheaders - m = startswith(name, "get") ? "GET" : "POST" - headers = test_sign!(m, Headers([]), p) - @test header_keys(headers) == required_headers - d = Dict(headers) - @test d["x-amz-date"] == "20150830T123600Z" - @test d["host"] == "example.amazonaws.com" - @test d["Authorization"] == test_auth_string("host;x-amz-date", sig) - end - - yesheaders = [ - ("get-header-key-duplicate", "", "", - Headers(["My-Header1" => "value2", - "My-Header1" => "value2", - "My-Header1" => "value1"]), - "host;my-header1;x-amz-date", - "c9d5ea9f3f72853aea855b47ea873832890dbdd183b4468f858259531a5138ea"), - ("get-header-value-multiline", "", "", - Headers(["My-Header1" => "value1\n value2\n value3"]), - "host;my-header1;x-amz-date", - "ba17b383a53190154eb5fa66a1b836cc297cc0a3d70a5d00705980573d8ff790"), - ("get-header-value-order", "", "", - Headers(["My-Header1" => "value4", - "My-Header1" => "value1", - "My-Header1" => "value3", - "My-Header1" => "value2"]), - "host;my-header1;x-amz-date", - "08c7e5a9acfcfeb3ab6b2185e75ce8b1deb5e634ec47601a50643f830c755c01"), - ("get-header-value-trim", "", "", - Headers(["My-Header1" => " value1", - "My-Header2" => " \"a b c\""]), - "host;my-header1;my-header2;x-amz-date", - "acc3ed3afb60bb290fc8d2dd0098b9911fcaa05412b367055dee359757a9c736"), - ("post-header-key-sort", "", "", - Headers(["My-Header1" => "value1"]), - "host;my-header1;x-amz-date", - "c5410059b04c1ee005303aed430f6e6645f61f4dc9e1461ec8f8916fdf18852c"), - ("post-header-value-case", "", "", - Headers(["My-Header1" => "VALUE1"]), - "host;my-header1;x-amz-date", - "cdbc9802e29d2942e5e10b5bccfdd67c5f22c7c4e8ae67b53629efa58b974b7d"), - ("post-x-www-form-urlencoded", "", "Param1=value1", - Headers(["Content-Type" => "application/x-www-form-urlencoded", - "Content-Length" => "13"]), - "content-type;host;x-amz-date", - "ff11897932ad3f4e8b18135d722051e5ac45fc38421b1da7b9d196a0fe09473a"), - ("post-x-www-form-urlencoded-parameters", "", "Param1=value1", - Headers(["Content-Type" => "application/x-www-form-urlencoded; charset=utf8", - "Content-Length" => "13"]), - "content-type;host;x-amz-date", - "1a72ec8f64bd914b0e42e42607c7fbce7fb2c7465f63e3092b3b0d39fa77a6fe"), - ] - @testset "$name" for (name, p, body, h, sh, sig) in yesheaders - hh = sort(map(first, h)) - m = startswith(name, "get") ? "GET" : "POST" - test_sign!(m, h, p, body) - @test header_keys(h) == sort(vcat(required_headers, hh)) - d = Dict(h) # collapses duplicates but we don't care here - @test d["x-amz-date"] == "20150830T123600Z" - @test d["host"] == "example.amazonaws.com" - @test d["Authorization"] == test_auth_string(sh, sig) - end - - @testset "AWS Security Token Service" begin - # Not a real security token, provided by AWS as an example - token = string("AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwd", - "QWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/k", - "McGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXD", - "vp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64", - "lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2I", - "CCR/oLxBA==") - @testset "Token included in signature" begin - sh = "host;x-amz-date;x-amz-security-token" - sig = "85d96828115b5dc0cfc3bd16ad9e210dd772bbebba041836c64533a82be05ead" - h = test_sign!("POST", Headers([]), "", aws_session_token=token) - d = Dict(h) - @test d["Authorization"] == test_auth_string(sh, sig) - @test haskey(d, "x-amz-security-token") - end - @testset "Token not included in signature" begin - sh = "host;x-amz-date" - sig = "5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b" - h = test_sign!("POST", Headers([]), "", aws_session_token=token, token_in_signature=false) - d = Dict(h) - @test d["Authorization"] == test_auth_string(sh, sig) - @test haskey(d, "x-amz-security-token") - end - end - - @testset "AWS Simple Storage Service" begin - s3url = "https://examplebucket.s3.amazonaws.com" - opts = (timestamp=DateTime(2013, 5, 24), - aws_service="s3", - aws_region="us-east-1", - # NOTE: These are the example credentials as specified in the AWS docs, - # they are not real - aws_access_key_id="AKIAIOSFODNN7EXAMPLE", - aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - include_md5=false) - - @testset "GET Object" begin - sh = "host;range;x-amz-content-sha256;x-amz-date" - sig = "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41" - h = Headers(["Range" => "bytes=0-9"]) - sign_aws4!("GET", URI(s3url * "/test.txt"), h, UInt8[]; opts...) - d = Dict(h) - @test d["Authorization"] == test_auth_string(sh, sig, opts.aws_access_key_id, "20130524", "s3") - @test haskey(d, "x-amz-content-sha256") # required for S3 requests - end - - @testset "PUT Object" begin - sh = "date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class" - sig = "98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd" - h = Headers(["Date" => "Fri, 24 May 2013 00:00:00 GMT", - "x-amz-storage-class" => "REDUCED_REDUNDANCY"]) - sign_aws4!("PUT", URI(s3url * "/test\$file.text"), h, UInt8[]; - # Override the SHA-256 of the request body, since the actual body is not provided - # for this example in the documentation, only the SHA - body_sha256=hex2bytes("44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072"), - opts...) - d = Dict(h) - @test d["Authorization"] == test_auth_string(sh, sig, opts.aws_access_key_id, "20130524", "s3") - @test haskey(d, "x-amz-content-sha256") - end - - @testset "GET Bucket Lifecycle" begin - sh = "host;x-amz-content-sha256;x-amz-date" - sig = "fea454ca298b7da1c68078a5d1bdbfbbe0d65c699e0f91ac7a200a0136783543" - h = Headers([]) - sign_aws4!("GET", URI(s3url * "/?lifecycle"), h, UInt8[]; opts...) - d = Dict(h) - @test d["Authorization"] == test_auth_string(sh, sig, opts.aws_access_key_id, "20130524", "s3") - @test haskey(d, "x-amz-content-sha256") - end - - @testset "GET Bucket (List Objects)" begin - sh = "host;x-amz-content-sha256;x-amz-date" - sig = "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7" - h = Headers([]) - sign_aws4!("GET", URI(s3url * "/?max-keys=2&prefix=J"), h, UInt8[]; opts...) - d = Dict(h) - @test d["Authorization"] == test_auth_string(sh, sig, opts.aws_access_key_id, "20130524", "s3") - @test haskey(d, "x-amz-content-sha256") - end - end - - @testset "HTTP.request with AWS authentication" begin - resp = HTTP.request("GET", - "https://httpbin.org/headers"; - aws_authorization=true, - timestamp=DateTime(2015, 8, 30, 12, 36), - aws_service="service", - aws_region="us-east-1", - # NOTE: These are the example credentials as specified in the AWS docs, - # they are not real - aws_access_key_id="AKIDEXAMPLE", - aws_secret_access_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", - include_md5=false, - include_sha256=false) - @test resp.status == 200 - headers = JSON.parse(String(resp.body))["headers"] - @test headers["Host"] == "httpbin.org" - @test headers["X-Amz-Date"] == "20150830T123600Z" - auth = "AWS4-HMAC-SHA256 " * - "Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, " * - "SignedHeaders=accept;host;x-amz-date, " * - "Signature=599128338d879e1e2aee2ce58222da02b47428696b7cf7c5d1b4a9cc75749ff9" - @test headers["Authorization"] == auth - end -end - -HTTP.setuseragent!(useragent) \ No newline at end of file diff --git a/test/chunking.jl b/test/chunking.jl index 08d027b53..ce0e12e0b 100644 --- a/test/chunking.jl +++ b/test/chunking.jl @@ -1,4 +1,6 @@ -using Test +module TestChunking + +using Test, Sockets using HTTP, HTTP.IOExtras using BufferedStreams @@ -15,8 +17,8 @@ using BufferedStreams "data: 3$(repeat("x", sz))\n\n" split1 = 106 split2 = 300 - - t = @async HTTP.listen("127.0.0.1", port) do http + server = Sockets.listen(ip"127.0.0.1", port) + t = @async HTTP.listen("127.0.0.1", port; server=server) do http startwrite(http) tcp = http.stream.io @@ -59,4 +61,7 @@ using BufferedStreams @test r == decoded_data end + close(server) end + +end # module diff --git a/test/client.jl b/test/client.jl index 1781214b5..0f120c0f9 100644 --- a/test/client.jl +++ b/test/client.jl @@ -1,5 +1,8 @@ -using ..TestRequest +module TestClient + using HTTP +include(joinpath(dirname(pathof(HTTP)), "../test/resources/TestRequest.jl")) +using .TestRequest using Sockets using JSON using Test @@ -8,10 +11,10 @@ using URIs status(r) = r.status @testset "Custom HTTP Stack" begin @testset "Low-level Request" begin - custom_stack = insert(stack(), StreamLayer, TestLayer) - result = request(custom_stack, "GET", "https://httpbin.org/ip") - + wasincluded = Ref(false) + result = TestRequest.get("https://httpbin.org/ip"; httptestlayer=wasincluded) @test status(result) == 200 + @test wasincluded[] end end @@ -51,7 +54,9 @@ end @test replace(replace(body, " "=>""), "\n"=>"") == "{\"cookies\":{\"foo\":\"bar\",\"hey\":\"sailor\"}}" r = HTTP.get("$sch://httpbin.org/cookies/delete?hey") - @test isempty(JSON.parse(String(r.body))["cookies"]) + cookies = JSON.parse(String(r.body))["cookies"] + @test length(cookies) == 1 + @test cookies["foo"] == "bar" end @testset "Client Streaming Test" begin @@ -345,7 +350,7 @@ end HTTP.startwrite(http) HTTP.write(http, sprint(JSON.print, data)) end - old_user_agent = HTTP.MessageRequest.USER_AGENT[] + old_user_agent = HTTP.DefaultHeadersRequest.USER_AGENT[] default_user_agent = "HTTP.jl/$VERSION" # Default values HTTP.setuseragent!(default_user_agent) @@ -377,7 +382,7 @@ import NetworkOptions, MbedTLS # Set up server with self-signed cert server = listen(IPv4(0), 8443) try - cert, key = joinpath.(@__DIR__, "resources", ("cert.pem", "key.pem")) + cert, key = joinpath.(dirname(pathof(HTTP)), "../test", "resources", ("cert.pem", "key.pem")) sslconfig = MbedTLS.SSLConfig(cert, key) tsk = @async HTTP.listen("0.0.0.0", 8443; server=server, sslconfig=sslconfig) do http HTTP.setstatus(http, 200) @@ -489,3 +494,5 @@ end end end end + +end # module diff --git a/test/cookies.jl b/test/cookies.jl index c8215d936..729c5af06 100644 --- a/test/cookies.jl +++ b/test/cookies.jl @@ -1,3 +1,8 @@ +module TestCookies + +using HTTP +using Sockets, Test + @testset "Cookies" begin c = HTTP.Cookies.Cookie() @test c.name == "" @@ -173,3 +178,5 @@ @test istaskdone(tsk) end end + +end # module diff --git a/test/insert_layers.jl b/test/insert_layers.jl deleted file mode 100644 index 27a63153f..000000000 --- a/test/insert_layers.jl +++ /dev/null @@ -1,55 +0,0 @@ -include("../src/exceptions.jl") - -using ..TestRequest - -@testset "HTTP Stack Inserting" begin - @testset "Insert - Beginning" begin - expected = TestLayer{TopLayer{RedirectLayer{BasicAuthLayer{MessageLayer{RetryLayer{ExceptionLayer{ConnectionPoolLayer{StreamLayer{Union{}}}}}}}}}} - result = insert(stack(), TopLayer, TestLayer) - - @test expected == result - end - - @testset "Insert - Middle" begin - expected = TopLayer{RedirectLayer{BasicAuthLayer{MessageLayer{RetryLayer{TestLayer{ExceptionLayer{ConnectionPoolLayer{StreamLayer{Union{}}}}}}}}}} - result = insert(stack(), ExceptionLayer, TestLayer) - - @test expected == result - end - - @testset "Insert - End" begin - expected = TopLayer{RedirectLayer{BasicAuthLayer{MessageLayer{RetryLayer{ExceptionLayer{ConnectionPoolLayer{StreamLayer{TestLayer{Union{}}}}}}}}}} - result = insert(stack(), Union{}, TestLayer) - - @test expected == result - end - - @testset "Insert - Non-existant layer" begin - @test_throws HTTP.Layers.LayerNotFoundException insert(stack(), AWS4AuthLayer, TestLayer) - end - - @testset "Insert - Multiple Same layer" begin - test_stack = insert(stack(), RetryLayer, ExceptionLayer) - - expected = TopLayer{RedirectLayer{BasicAuthLayer{MessageLayer{TestLayer{ExceptionLayer{RetryLayer{ExceptionLayer{ConnectionPoolLayer{StreamLayer{Union{}}}}}}}}}}} - result = insert(test_stack, ExceptionLayer, TestLayer) - - @test expected == result - end - - @testset "Inserted final layer runs handler" begin - TestRequest.FLAG[] = false - request(insert(stack(), Union{}, LastLayer), "GET", "https://httpbin.org/anything") - @test TestRequest.FLAG[] - end - - @testset "Insert/remove default layers" begin - top = HTTP.top_layer(stack()) - insert_default!(top, TestLayer) - @test HTTP.top_layer(stack()) <: TestLayer - remove_default!(top, TestLayer) - @test HTTP.top_layer(stack()) <: top - insert_default!(Union{}, TestLayer) - remove_default!(Union{}, TestLayer) - end -end diff --git a/test/loopback.jl b/test/loopback.jl index f4a1e5412..a7ef9822b 100644 --- a/test/loopback.jl +++ b/test/loopback.jl @@ -1,10 +1,11 @@ +module TestLoopback + using Test using HTTP using HTTP.IOExtras using HTTP.Parsers using HTTP.Messages using HTTP.Sockets -using HTTP.MessageRequest: bodylength mutable struct FunctionIO <: IO f::Function @@ -156,6 +157,13 @@ function Base.unsafe_write(lb::Loopback, p::Ptr{UInt8}, n::UInt) return n end +function HTTP.ConnectionPool.getconnection(::Type{Loopback}, + host::AbstractString, + port::AbstractString; + kw...)::Loopback + return Loopback() +end + function async_test(m=["GET","GET","GET","GET","GET"];kw...) r1 = r2 = r3 = r4 = r5 = nothing t1 = time() @@ -187,13 +195,6 @@ function async_test(m=["GET","GET","GET","GET","GET"];kw...) return t2 - t1 end -function HTTP.ConnectionPool.getconnection(::Type{Loopback}, - host::AbstractString, - port::AbstractString; - kw...)::Loopback - return Loopback() -end - @testset "loopback" begin global server_events @@ -232,21 +233,21 @@ end end @testset "lbopen - Body - Delay" begin - body = nothing - body_sent = false + body = Ref{Any}(nothing) + body_sent = Ref(false) r = lbopen("delay10", []) do http @sync begin @async begin write(http, "Hello World!") closewrite(http) - body_sent = true + body_sent[] = true end startread(http) - body = read(http) + body[] = read(http) closeread(http) end end - @test String(body) == "Hello World!" + @test String(body[]) == "Hello World!" end # "If [the response] indicates the server does not wish to receive the @@ -289,21 +290,20 @@ end FunctionIO(()->(sleep(0.1); " World!"))]) @test String(r.body) == "Hello World!" - hello_sent = false - world_sent = false + hello_sent = Ref(false) + world_sent = Ref(false) @test_throws HTTP.StatusError begin r = lbreq("abort", [], [ - FunctionIO(()->(hello_sent = true; sleep(0.1); "Hello")), - FunctionIO(()->(world_sent = true; " World!"))]) + FunctionIO(()->(hello_sent[] = true; sleep(0.5); "Hello")), + FunctionIO(()->(world_sent[] = true; " World!"))]) end - @test hello_sent - @test !world_sent + @test hello_sent[] + @test !world_sent[] end @testset "ASync - Pipeline limit = 0" begin server_events = [] t = async_test(;pipeline_limit=0) - @show t if haskey(ENV, "HTTP_JL_TEST_TIMING_SENSITIVE") @test server_events == [ "Request: GET /delay1 HTTP/1.1", @@ -322,7 +322,6 @@ end @testset "ASync - " begin server_events = [] t = async_test() - @show t if haskey(ENV, "HTTP_JL_TEST_TIMING_SENSITIVE") @test server_events == [ "Request: GET /delay1 HTTP/1.1", @@ -350,3 +349,5 @@ end "Response: HTTP/1.1 200 OK <= (POST /delay1 HTTP/1.1)"] end end + +end # module diff --git a/test/messages.jl b/test/messages.jl index e6d0cfa0e..4a01478ca 100644 --- a/test/messages.jl +++ b/test/messages.jl @@ -1,17 +1,12 @@ -using ..Test +using Test using HTTP.Messages import HTTP.Messages.appendheader import HTTP.URI import HTTP.request -import HTTP: bytes - +import HTTP: bytes, nbytes using HTTP: StatusError -using HTTP.MessageRequest: bodylength -using HTTP.MessageRequest: bodybytes -using HTTP.MessageRequest: unknown_length - using JSON @testset "HTTP.Messages" begin @@ -23,29 +18,29 @@ using JSON http_writes = ["POST", "PUT", "DELETE", "PATCH"] @testset "Body Length" begin - @test bodylength(7) == unknown_length - @test bodylength(UInt8[1,2,3]) == 3 - @test bodylength(view(UInt8[1,2,3], 1:2)) == 2 - @test bodylength("Hello") == 5 - @test bodylength(SubString("World!",1,5)) == 5 - @test bodylength(["Hello", " ", "World!"]) == 12 - @test bodylength(["Hello", " ", SubString("World!",1,5)]) == 11 - @test bodylength([SubString("Hello", 1,5), " ", SubString("World!",1,5)]) == 11 - @test bodylength([UInt8[1,2,3], UInt8[4,5,6]]) == 6 - @test bodylength([UInt8[1,2,3], view(UInt8[4,5,6],1:2)]) == 5 - @test bodylength([view(UInt8[1,2,3],1:2), view(UInt8[4,5,6],1:2)]) == 4 - @test bodylength(IOBuffer("foo")) == 3 - @test bodylength([IOBuffer("foo"), IOBuffer("bar")]) == 6 + @test nbytes(7) === nothing + @test nbytes(UInt8[1,2,3]) == 3 + @test nbytes(view(UInt8[1,2,3], 1:2)) == 2 + @test nbytes("Hello") == 5 + @test nbytes(SubString("World!",1,5)) == 5 + @test nbytes(["Hello", " ", "World!"]) == 12 + @test nbytes(["Hello", " ", SubString("World!",1,5)]) == 11 + @test nbytes([SubString("Hello", 1,5), " ", SubString("World!",1,5)]) == 11 + @test nbytes([UInt8[1,2,3], UInt8[4,5,6]]) == 6 + @test nbytes([UInt8[1,2,3], view(UInt8[4,5,6],1:2)]) == 5 + @test nbytes([view(UInt8[1,2,3],1:2), view(UInt8[4,5,6],1:2)]) == 4 + @test nbytes(IOBuffer("foo")) == 3 + @test nbytes([IOBuffer("foo"), IOBuffer("bar")]) == 6 end @testset "Body Bytes" begin - @test bodybytes(7) == UInt8[] - @test bodybytes(UInt8[1,2,3]) == UInt8[1,2,3] - @test bodybytes(view(UInt8[1,2,3], 1:2)) == UInt8[1,2] - @test bodybytes("Hello") == bytes("Hello") - @test bodybytes(SubString("World!",1,5)) == bytes("World") - @test bodybytes(["Hello", " ", "World!"]) == UInt8[] - @test bodybytes([UInt8[1,2,3], UInt8[4,5,6]]) == UInt8[] + @test bytes(7) == 7 + @test bytes(UInt8[1,2,3]) == UInt8[1,2,3] + @test bytes(view(UInt8[1,2,3], 1:2)) == UInt8[1,2] + @test bytes("Hello") == codeunits("Hello") + @test bytes(SubString("World!",1,5)) == codeunits("World") + @test bytes(["Hello", " ", "World!"]) == ["Hello", " ", "World!"] + @test bytes([UInt8[1,2,3], UInt8[4,5,6]]) == [UInt8[1,2,3], UInt8[4,5,6]] end @testset "Request" begin @@ -176,15 +171,15 @@ using JSON end @testset "Display" begin - @test repr(Response(200, []; body="Hello world.")) == "Response:\n\"\"\"\nHTTP/1.1 200 OK\r\n\r\nHello world.\"\"\"" + @test repr(Response(200, []; body="Hello world.")) == "Response{Base.CodeUnits{UInt8, String}}:\n\"\"\"\nHTTP/1.1 200 OK\r\n\r\nHello world.\"\"\"" # truncation of long bodies for body_show_max in (Messages.body_show_max, 100) Messages.set_show_max(body_show_max) - @test repr(Response(200, []; body="Hello world.\n"*'x'^10000)) == "Response:\n\"\"\"\nHTTP/1.1 200 OK\r\n\r\nHello world.\n"*'x'^(body_show_max-13)*"\n⋮\n10013-byte body\n\"\"\"" + @test repr(Response(200, []; body="Hello world.\n"*'x'^10000)) == "Response{Base.CodeUnits{UInt8, String}}:\n\"\"\"\nHTTP/1.1 200 OK\r\n\r\nHello world.\n"*'x'^(body_show_max-13)*"\n⋮\n10013-byte body\n\"\"\"" end # don't display raw binary (non-Unicode) data: - @test repr(Response(200, []; body=String([0xde,0xad,0xc1,0x71,0x1c]))) == "Response:\n\"\"\"\nHTTP/1.1 200 OK\r\n\r\n\n⋮\n5-byte body\n\"\"\"" + @test repr(Response(200, []; body=String([0xde,0xad,0xc1,0x71,0x1c]))) == "Response{Base.CodeUnits{UInt8, String}}:\n\"\"\"\nHTTP/1.1 200 OK\r\n\r\n\n⋮\n5-byte body\n\"\"\"" end end diff --git a/test/resources/TestRequest.jl b/test/resources/TestRequest.jl index 16d8ce1fe..beda0392f 100644 --- a/test/resources/TestRequest.jl +++ b/test/resources/TestRequest.jl @@ -1,18 +1,21 @@ module TestRequest -import HTTP: Layer, request, Response -abstract type TestLayer{Next <: Layer} <: Layer{Next} end -abstract type LastLayer{Next <: Layer} <: Layer{Next} end -export TestLayer, LastLayer, request +using HTTP -function request(::Type{TestLayer{Next}}, io::IO, req, body; kw...)::Response where Next - return request(Next, io, req, body; kw...) +function testrequestlayer(handler) + return function(req; httptestlayer=Ref(false), kw...) + httptestlayer[] = true + return handler(req; kw...) + end end -const FLAG = Ref(false) -function request(::Type{LastLayer{Next}}, resp)::Response where Next - FLAG[] = true - return request(Next, resp) +function teststreamlayer(handler) + return function(stream; httptestlayer=Ref(false), kw...) + httptestlayer[] = true + return handler(stream; kw...) + end end +HTTP.@client (testrequestlayer,) (teststreamlayer,) + end diff --git a/test/runtests.jl b/test/runtests.jl index 3e831cbcc..202239042 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -20,8 +20,6 @@ include(joinpath(dir, "resources/TestRequest.jl")) "handlers.jl", "server.jl", "async.jl", - "aws4.jl", - "insert_layers.jl", "mwe.jl", ] file = joinpath(dir, f) diff --git a/test/server.jl b/test/server.jl index 5c38f3e20..091d0b07a 100644 --- a/test/server.jl +++ b/test/server.jl @@ -27,7 +27,7 @@ end write(http, request.response.body) end - server = Sockets.listen(Sockets.InetAddr(parse(IPAddr, "127.0.0.1"), port)) + server = Sockets.listen(ip"127.0.0.1", port) tsk = @async HTTP.listen(handler, "127.0.0.1", port; server=server) sleep(3.0) @test !istaskdone(tsk) @@ -37,12 +37,12 @@ end sleep(0.5) @test istaskdone(tsk) - server = Sockets.listen(Sockets.InetAddr(parse(IPAddr, "127.0.0.1"), port)) + server = Sockets.listen(ip"127.0.0.1", port) tsk = @async HTTP.listen(handler, "127.0.0.1", port; server=server) handler2 = HTTP.Handlers.RequestHandlerFunction(req->HTTP.Response(200, req.body)) - server2 = Sockets.listen(Sockets.InetAddr(parse(IPAddr, "127.0.0.1"), port+100)) + server2 = Sockets.listen(ip"127.0.0.1", port+100) tsk2 = @async HTTP.serve(handler2, "127.0.0.1", port+100; server=server2) sleep(0.5) @test !istaskdone(tsk) @@ -66,7 +66,6 @@ end # large headers tcp = Sockets.connect(ip"127.0.0.1", port) x = "GET / HTTP/1.1\r\n$(repeat("Foo: Bar\r\n", 10000))\r\n"; - @show length(x) write(tcp, "GET / HTTP/1.1\r\n$(repeat("Foo: Bar\r\n", 10000))\r\n") sleep(0.1) try @@ -129,7 +128,7 @@ end # keep-alive vs. close: issue #81 port += 1 - server = Sockets.listen(Sockets.InetAddr(parse(IPAddr, "127.0.0.1"), port)) + server = Sockets.listen(ip"127.0.0.1", port) tsk = @async HTTP.listen(hello, "127.0.0.1", port; server=server, verbose=true) sleep(0.5) @test !istaskdone(tsk) @@ -267,7 +266,7 @@ end @testset "on_shutdown" begin @test HTTP.Servers.shutdown(nothing) === nothing - IOserver = Sockets.listen(Sockets.InetAddr(parse(IPAddr, "127.0.0.1"), 8052)) + IOserver = Sockets.listen(ip"127.0.0.1", 8052) # Shutdown adds 1 TEST_COUNT = Ref(0) @@ -349,7 +348,7 @@ end # @testset logs = with_testserver(combined_logfmt) do HTTP.get("http://localhost:32612", ["Referer" => "julialang.org"]) HTTP.get("http://localhost:32612/index.html") - useragent = HTTP.MessageRequest.USER_AGENT[] + useragent = HTTP.DefaultHeadersRequest.USER_AGENT[] HTTP.setuseragent!(nothing) HTTP.get("http://localhost:32612/index.html?a=b") HTTP.setuseragent!(useragent)