From 2d0e66d9d41c813b6fb09179bdcd29910bf0d296 Mon Sep 17 00:00:00 2001 From: Filipe David Manana Date: Sun, 26 Sep 2010 11:58:45 +0100 Subject: [PATCH] New feature for the httpc module: allow streaming of PUT and POST request bodies. This is a must when uploading large bodies that are to large to store in a string or binary. Besides a string or binary, a body can now be a function and an accumulator. Example: -module(httpc_post_stream_test). -compile(export_all). -define(LEN, 1024 * 1024). prepare_data() -> {ok, Fd} = file:open("test_data.dat", [binary, write]), ok = file:write(Fd, lists:duplicate(?LEN, "1")), ok = file:close(Fd). test() -> inets:start(), ok = prepare_data(), {ok, Fd1} = file:open("test_data.dat", [binary, read]), BodyFun = fun(Fd) -> case file:read(Fd, 512) of eof -> eof; {ok, Data} -> {ok, Data, Fd} end end, {ok, {{_,200,_}, _, _}} = httpc:request(post, {"http://localhost:8888", [{"content-length", integer_to_list(?LEN)}], "text/plain", {BodyFun, Fd1}}, [], []), ok = file:close(Fd1). --- lib/inets/doc/src/httpc.xml | 4 +- lib/inets/src/http_client/httpc.erl | 4 +- lib/inets/src/http_client/httpc_request.erl | 64 ++++++++++++++++----- lib/inets/test/httpc_SUITE.erl | 40 +++++++++++++ 4 files changed, 96 insertions(+), 16 deletions(-) diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml index 9c8df28fec47..df333074cdbd 100644 --- a/lib/inets/doc/src/httpc.xml +++ b/lib/inets/doc/src/httpc.xml @@ -89,7 +89,9 @@ headers() = [header()] header() = {field(), value()} field() = string() value() = string() -body() = string() | binary() +body() = string() | binary() | {fun(acc()) -> send_fun_result(), acc()} +send_fun_result() = eof | {ok, iolist(), acc()} +acc() = term() filename() = string() ]]> diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index 851364001c56..b82a9db4c906 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -126,7 +126,9 @@ request(Url, Profile) -> %% Header = {Field, Value} %% Field = string() %% Value = string() -%% Body = string() | binary() - HTLM-code +%% Body = string() | binary() | {fun(SendAcc) -> SendFunResult, SendAcc} - HTLM-code +%% SendFunResult = eof | {ok, iolist(), NewSendAcc} +%% SendAcc = NewSendAcc = term() %% %% Description: Sends a HTTP-request. The function can be both %% syncronus and asynchronous in the later case the function will diff --git a/lib/inets/src/http_client/httpc_request.erl b/lib/inets/src/http_client/httpc_request.erl index d4df97ad40f5..5386d1eb4abc 100644 --- a/lib/inets/src/http_client/httpc_request.erl +++ b/lib/inets/src/http_client/httpc_request.erl @@ -101,15 +101,41 @@ send(SendAddr, Socket, SocketType, end, Version = HttpOptions#http_options.version, - Message = [method(Method), " ", Uri, " ", - version(Version), ?CRLF, - headers(FinalHeaders, Version), ?CRLF, Body], + do_send_body(SocketType, Socket, Method, Uri, Version, FinalHeaders, Body). + +do_send_body(SocketType, Socket, Method, Uri, Version, Headers, {DataFun, Acc}) + when is_function(DataFun, 1) -> + case do_send_body(SocketType, Socket, Method, Uri, Version, Headers, []) of + ok -> + data_fun_loop(SocketType, Socket, DataFun, Acc); + Error -> + Error + end; + +do_send_body(SocketType, Socket, Method, Uri, Version, Headers, Body) -> + Message = [method(Method), " ", Uri, " ", + version(Version), ?CRLF, + headers(Headers, Version), ?CRLF, Body], ?hcrd("send", [{message, Message}]), - http_transport:send(SocketType, Socket, lists:append(Message)). +data_fun_loop(SocketType, Socket, DataFun, Acc) -> + case DataFun(Acc) of + eof -> + ok; + {ok, Data, NewAcc} -> + DataBin = iolist_to_binary(Data), + ?hcrd("send", [{message, DataBin}]), + case http_transport:send(SocketType, Socket, DataBin) of + ok -> + data_fun_loop(SocketType, Socket, DataFun, NewAcc); + Error -> + Error + end + end. + %%------------------------------------------------------------------------- %% is_idempotent(Method) -> @@ -161,7 +187,6 @@ is_client_closing(Headers) -> %%%======================================================================== post_data(Method, Headers, {ContentType, Body}, HeadersAsIs) when (Method =:= post) orelse (Method =:= put) -> - ContentLength = body_length(Body), NewBody = case Headers#http_request_h.expect of "100-continue" -> ""; @@ -170,14 +195,22 @@ post_data(Method, Headers, {ContentType, Body}, HeadersAsIs) end, NewHeaders = case HeadersAsIs of - [] -> - Headers#http_request_h{'content-type' = - ContentType, - 'content-length' = - ContentLength}; - _ -> - HeadersAsIs - end, + [] -> + Headers#http_request_h{ + 'content-type' = ContentType, + 'content-length' = case body_length(Body) of + undefined -> + % on upload streaming the caller must give a + % value to the Content-Length header + % (or use chunked Transfer-Encoding) + Headers#http_request_h.'content-length'; + Len when is_list(Len) -> + Len + end + }; + _ -> + HeadersAsIs + end, {NewHeaders, NewBody}; @@ -190,7 +223,10 @@ body_length(Body) when is_binary(Body) -> integer_to_list(size(Body)); body_length(Body) when is_list(Body) -> - integer_to_list(length(Body)). + integer_to_list(length(Body)); + +body_length({DataFun, _Acc}) when is_function(DataFun, 1) -> + undefined. method(Method) -> http_util:to_upper(atom_to_list(Method)). diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 902e440c804c..6947f75b3d76 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -77,6 +77,7 @@ all(suite) -> http_head, http_get, http_post, + http_post_streaming, http_dummy_pipe, http_inets_pipe, http_trace, @@ -423,6 +424,45 @@ http_post(Config) when is_list(Config) -> {skip, "Failed to start local http-server"} end. +%%------------------------------------------------------------------------- +http_post_streaming(doc) -> + ["Test streaming http post request against local server. We" + " only care about the client side of the the post. The server" + " script will not actually use the post data."]; +http_post_streaming(suite) -> + []; +http_post_streaming(Config) when is_list(Config) -> + case ?config(local_server, Config) of + ok -> + Port = ?config(local_port, Config), + URL = case test_server:os_type() of + {win32, _} -> + ?URL_START ++ integer_to_list(Port) ++ + "/cgi-bin/cgi_echo.exe"; + _ -> + ?URL_START ++ integer_to_list(Port) ++ + "/cgi-bin/cgi_echo" + end, + %% Cgi-script expects the body length to be 100 + BodyFun = fun(0) -> + eof; + (LenLeft) -> + {ok, lists:duplicate(10, "1"), LenLeft - 10} + end, + + {ok, {{_,200,_}, [_ | _], [_ | _]}} = + httpc:request(post, {URL, + [{"expect", "100-continue"}, {"content-length", "100"}], + "text/plain", {BodyFun, 100}}, [], []), + + {ok, {{_,504,_}, [_ | _], []}} = + httpc:request(post, {URL, + [{"expect", "100-continue"}, {"content-length", "10"}], + "text/plain", {BodyFun, 10}}, [], []); + _ -> + {skip, "Failed to start local http-server"} + end. + %%------------------------------------------------------------------------- http_emulate_lower_versions(doc) -> ["Perform request as 0.9 and 1.0 clients."];