Crash due to Content-Length InexactError on large request

[Octogonapus]

We ran into a request error because of a large (2.28 GB) request body:

│    HTTP.RequestError:
│    HTTP.Request:
│    HTTP.Messages.Request:
│    """
│    POST /xxx HTTP/1.1
│    Authorization: Bearer xxx
│    Host: xxx
│    Accept: */*
│    User-Agent: HTTP.jl/1.9.4
│    Content-Length: 2281871749
│    Content-Type: multipart/form-data; boundary=fb808120ca037c02966686b858c2d6ee
│    Accept-Encoding: gzip
│    
│    [Message Body was streamed]"""Underlying error:
│    InexactError: trunc(Int32, 2281871749)
│    Stacktrace:
│      [1] (::HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{typeof(HTTP.StreamRequest.streamlayer)}}}}}})(req::HTTP.Messages.Request; proxy::Nothing, socket_type::Type, socket_type_tls::Type, readtimeout::Int64, connect_timeout::Int64, logerrors::Bool, logtag::Nothing, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :require_ssl_verification), Tuple{Nothing, Nothing, Int64, Bool}}})
│        @ HTTP.ConnectionRequest /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/ConnectionRequest.jl:143
│      [2] (::Base.var"#90#92"{Base.var"#90#91#93"{ExponentialBackOff, HTTP.RetryRequest.var"#2#5"{Int64, typeof(HTTP.RetryRequest.FALSE), HTTP.Messages.Request, Base.RefValue{Int64}}, HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}})(args::HTTP.Messages.Request; kwargs::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :require_ssl_verification), Tuple{Nothing, Nothing, Int64, Bool}}})
│        @ Base ./error.jl:296
│      [3] (::HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}})(req::HTTP.Messages.Request; retry::Bool, retries::Int64, retry_delays::ExponentialBackOff, retry_check::Function, retry_non_idempotent::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :require_ssl_verification), Tuple{Nothing, Nothing, Int64, Bool}}})
│        @ HTTP.RetryRequest /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/RetryRequest.jl:75
│      [4] manageretries
│        @ /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/RetryRequest.jl:30 [inlined]
│      [5] (::HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}})(req::HTTP.Messages.Request; cookies::Bool, cookiejar::HTTP.Cookies.CookieJar, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :require_ssl_verification), Tuple{Nothing, Nothing, Int64, Bool}}})
│        @ HTTP.CookieRequest /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/CookieRequest.jl:42
│      [6] managecookies
│        @ /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/CookieRequest.jl:19 [inlined]
│      [7] (::HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}})(req::HTTP.Messages.Request; iofunction::Nothing, decompress::Nothing, basicauth::Bool, detect_content_type::Bool, canonicalize_headers::Bool, kw::Base.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, NamedTuple{(:verbose, :require_ssl_verification), Tuple{Int64, Bool}}})
│        @ HTTP.HeadersRequest /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/HeadersRequest.jl:71
│      [8] defaultheaders
│        @ /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/HeadersRequest.jl:14 [inlined]
│      [9] (::HTTP.RedirectRequest.var"#redirects#3"{HTTP.RedirectRequest.var"#redirects#1#4"{HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}})(req::HTTP.Messages.Request; redirect::Bool, redirect_limit::Int64, redirect_method::Nothing, forwardheaders::Bool, response_stream::Nothing, kw::Base.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, NamedTuple{(:verbose, :require_ssl_verification), Tuple{Int64, Bool}}})
│        @ HTTP.RedirectRequest /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/RedirectRequest.jl:25
│     [10] redirects
│        @ /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/RedirectRequest.jl:14 [inlined]
│     [11] (::HTTP.MessageRequest.var"#makerequest#3"{HTTP.MessageRequest.var"#makerequest#1#4"{HTTP.RedirectRequest.var"#redirects#3"{HTTP.RedirectRequest.var"#redirects#1#4"{HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}})(method::String, url::URIs.URI, headers::Dict{String, String}, body::HTTP.Forms.Form; copyheaders::Bool, response_stream::Nothing, http_version::HTTP.Strings.HTTPVersion, verbose::Int64, kw::Base.Pairs{Symbol, Bool, Tuple{Symbol}, NamedTuple{(:require_ssl_verification,), Tuple{Bool}}})
│        @ HTTP.MessageRequest /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/MessageRequest.jl:35
│     [12] makerequest
│        @ /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/MessageRequest.jl:24 [inlined]
│     [13] request(stack::HTTP.MessageRequest.var"#makerequest#3"{HTTP.MessageRequest.var"#makerequest#1#4"{HTTP.RedirectRequest.var"#redirects#3"{HTTP.RedirectRequest.var"#redirects#1#4"{HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}, method::String, url::String, h::Dict{String, String}, b::HTTP.Forms.Form, q::Nothing; headers::Dict{String, String}, body::HTTP.Forms.Form, query::Nothing, kw::Base.Pairs{Symbol, Bool, Tuple{Symbol}, NamedTuple{(:require_ssl_verification,), Tuple{Bool}}})
│        @ HTTP /home/runner/.julia/packages/HTTP/SN7VW/src/HTTP.jl:457
│     [14] #request#20
│        @ /home/runner/.julia/packages/HTTP/SN7VW/src/HTTP.jl:315 [inlined]
│     [15] request
│        @ /home/runner/.julia/packages/HTTP/SN7VW/src/HTTP.jl:313 [inlined]

Note that the Content-Length 2281871749 is larger than typemax(Int32).

• Julia v1.9.4
• HTTP.jl v1.10.0
• MbelTLS.jl v1.1.8

[fredrikekre]

I can not reproduce this. I tried for example

server = HTTP.listen!() do http
    HTTP.setstatus(http, 200)
    bytes_total = Int(typemax(Int32) + 1)
    HTTP.setheader(http, "Content-Length" => string(bytes_total))
    HTTP.startwrite(http)
    io = open("/dev/random", "r")
    bytes_written = 0
    while bytes_written < bytes_total
        toread = min(bytes_total - bytes_written, 1024^2)
        chunk_size = write(http, read(io, toread))
        bytes_written += chunk_size
    end
end

HTTP.get("http://localhost:8081", response_stream = devnull)

Unfortunately HTTP.jl swallows the original stacktrace from the error e here

HTTP.jl/src/clientlayers/ConnectionRequest.jl

Line 121 in 57d57a5

catch e

: and we only see the trace for

HTTP.jl/src/clientlayers/ConnectionRequest.jl

Line 143 in 57d57a5

root_err isa HTTPError || throw(RequestError(req, root_err))

If you can still reproduce, perhaps try commenting out the catch block:

HTTP.jl/src/clientlayers/ConnectionRequest.jl

Lines 121 to 144 in 57d57a5

	catch e
	shouldreuse = false
	# manually unwrap CompositeException since it's not defined as a "wrapper" exception by ExceptionUnwrapping
	while e isa CompositeException
	e = e.exceptions[1]
	end
	root_err = ExceptionUnwrapping.unwrap_exception_to_root(e)
	# don't log if it's an HTTPError since we should have already logged it
	if logerrors && root_err isa StatusError
	msg = current_exceptions_to_string()
	@error msg type=Symbol("HTTP.StatusError") method=req.method url=req.url context=req.context logtag=logtag
	end
	if logerrors && !ExceptionUnwrapping.has_wrapped_exception(e, HTTPError)
	msg = current_exceptions_to_string()
	@error msg type=Symbol("HTTP.ConnectionRequest") method=req.method url=req.url context=req.context logtag=logtag
	end
	@debugv 1 "❗️ ConnectionLayer $root_err. Closing: $io"
	if @isdefined(stream) && stream.nwritten == -1
	# we didn't write anything, so don't need to worry about
	# idempotency of the request
	req.context[:nothingwritten] = true
	end
	root_err isa HTTPError || throw(RequestError(req, root_err))
	throw(root_err)

and see what the original error is

[Octogonapus]

huh I can't reproduce this either. not sure what is going on