Permalink
Browse files

Merge branch 'master' of git://github.com/ferd/lhttpc

  • Loading branch information...
lpgauth committed Mar 17, 2013
2 parents 11fad66 + 73769cb commit 748781b634dcdcb7c4eaade50a4ea916947010ac
Showing with 391 additions and 363 deletions.
  1. +3 −0 README
  2. +1 −0 rebar.config
  3. +5 −5 src/lhttpc.app.src
  4. +36 −14 src/lhttpc.erl
  5. +40 −43 src/lhttpc_client.erl
  6. +225 −157 src/lhttpc_lb.erl
  7. +1 −2 src/lhttpc_lib.erl
  8. +0 −125 src/lhttpc_manager.erl
  9. +37 −6 src/lhttpc_sock.erl
  10. +18 −7 src/lhttpc_sup.erl
  11. +25 −4 test/lhttpc_tests.erl
View
3 README
@@ -10,3 +10,6 @@ Configuration: (environment variables)
kepp a HTTP/1.1 connection open. Changing this value
in runtime has no effect, this can however be done
through lhttpc_manager:update_connection_timeout/1.
+
+NOTE: THIS FORK OF LHTTPC IS ONLY RECOMMENDED IF YOU HAVE MANY REQUESTS TO DO TO A FEW RESTRICTED DOMAINS.
+It contains load-balancing mechanisms described in http://ferd.ca/rtb-where-erlang-blooms.html. It is not meant for general purpose use.
View
@@ -1 +1,2 @@
+{erl_opts, [debug_info]}.
{cover_enabled, true}.
View
@@ -29,11 +29,11 @@
%%% @end
{application, lhttpc,
[{description, "Lightweight HTTP Client"},
- {vsn, "1.2.6"},
- {modules, []},
- {registered, [lhttpc_manager]},
+ {vsn, "1.2.8"},
+ {modules, [lhttpc,lhttpc_sup,lhttpc_client,lhttpc_sock,lhttp_lb]},
+ {registered, [lhttpc_sup]},
{applications, [kernel, stdlib, ssl, crypto]},
- {mod, {lhttpc, nil}},
- {env, []}
+ {mod, {lhttpc, []}},
+ {env, [{connection_timeout, 300000}]}
]}.
View
@@ -52,7 +52,7 @@
%% @hidden
-spec start(normal | {takeover, node()} | {failover, node()}, any()) ->
{ok, pid()}.
-start(_, _) ->
+start(_, Opts) ->
case lists:member({seed,1}, ssl:module_info(exports)) of
true ->
% Make sure that the ssl random number generator is seeded
@@ -61,7 +61,9 @@ start(_, _) ->
false ->
ok
end,
- lhttpc_sup:start_link().
+ if is_list(Opts) -> lhttpc_sup:start_link(Opts);
+ true -> lhttpc_sup:start_link()
+ end.
%% @hidden
-spec stop(any()) -> ok.
@@ -320,12 +322,16 @@ request(URL, Method, Hdrs, Body, Timeout, Options) ->
headers(), iolist(), pos_integer(), [option()]) -> result().
request(Host, Port, Ssl, Path, Method, Hdrs, Body, Timeout, Options) ->
verify_options(Options, []),
- ReqId = erlang:now(),
+ ReqId = now(),
case proplists:is_defined(stream_to, Options) of
true ->
StreamTo = proplists:get_value(stream_to, Options),
Args = [ReqId, StreamTo, Host, Port, Ssl, Path, Method, Hdrs, Body, Options],
- Pid = spawn(lhttpc_client, request_with_timeout, [Timeout, Args]),
+ Pid = spawn(lhttpc_client, request, Args),
+ spawn(fun() ->
+ R = kill_client_after(Pid, Timeout),
+ StreamTo ! {response, ReqId, Pid, R}
+ end),
{ReqId, Pid};
false ->
Args = [ReqId, self(), Host, Port, Ssl, Path, Method, Hdrs, Body, Options],
@@ -338,7 +344,7 @@ request(Host, Port, Ssl, Path, Method, Hdrs, Body, Timeout, Options) ->
% linked client send us an exit signal, since this can be
% caught by the caller.
exit(Reason);
- {'EXIT', ReqId, Pid, Reason} ->
+ {'EXIT', Pid, Reason} ->
% This could happen if the process we're running in taps exits
% and the client process exits due to some exit signal being
% sent to it. Very unlikely though
@@ -397,7 +403,7 @@ send_body_part({Pid, 0}, IoList, Timeout) when is_pid(Pid) ->
send_body_part({Pid, 1}, IoList, Timeout);
{response, _ReqId, Pid, R} ->
R;
- {exit, Pid, Reason} ->
+ {exit, _ReqId, Pid, Reason} ->
exit(Reason);
{'EXIT', Pid, Reason} ->
exit(Reason)
@@ -410,9 +416,9 @@ send_body_part({Pid, Window}, IoList, _Timeout) when Window > 0, is_pid(Pid) ->
receive
{ack, Pid} ->
{ok, {Pid, Window}};
- {response, _ReqId, Pid, R} ->
+ {reponse, _ReqId, Pid, R} ->
R;
- {exit, Pid, Reason} ->
+ {exit, _ReqId, Pid, Reason} ->
exit(Reason);
{'EXIT', Pid, Reason} ->
exit(Reason)
@@ -515,7 +521,7 @@ read_response(Pid, Timeout) ->
read_response(Pid, Timeout);
{response, _ReqId, Pid, R} ->
R;
- {exit, Pid, Reason} ->
+ {exit, _ReqId, Pid, Reason} ->
exit(Reason);
{'EXIT', Pid, Reason} ->
exit(Reason)
@@ -537,6 +543,23 @@ kill_client(Pid) ->
erlang:error(Reason)
end.
+kill_client_after(Pid, Timeout) ->
+ erlang:monitor(process, Pid),
+ receive
+ {'DOWN', _, process, Pid, _Reason} -> exit(normal)
+ after Timeout ->
+ catch unlink(Pid), % or we'll kill ourself :O
+ exit(Pid, timeout),
+ receive
+ {'DOWN', _, process, Pid, timeout} ->
+ {error, timeout};
+ {'DOWN', _, process, Pid, Reason} ->
+ erlang:error(Reason)
+ after 1000 ->
+ exit(normal) % silent failure!
+ end
+ end.
+
-spec verify_options(options(), options()) -> ok.
verify_options([{send_retry, N} | Options], Errors)
when is_integer(N), N >= 0 ->
@@ -551,11 +574,8 @@ verify_options([{connection_timeout, infinity} | Options], Errors) ->
verify_options([{connection_timeout, MS} | Options], Errors)
when is_integer(MS), MS >= 0 ->
verify_options(Options, Errors);
-verify_options([{max_connections, MS} | Options], Errors)
- when is_integer(MS), MS >= 0 ->
- verify_options(Options, Errors);
-verify_options([{stream_to, Pid} | Options], Errors)
- when is_pid(Pid) ->
+verify_options([{max_connections, N} | Options], Errors)
+ when is_integer(N), N > 0 ->
verify_options(Options, Errors);
verify_options([{partial_upload, WindowSize} | Options], Errors)
when is_integer(WindowSize), WindowSize >= 0 ->
@@ -574,6 +594,8 @@ verify_options([{partial_download, DownloadOptions} | Options], Errors)
verify_options([{connect_options, List} | Options], Errors)
when is_list(List) ->
verify_options(Options, Errors);
+verify_options([{stream_to, Pid} | Options], Errors) when is_pid(Pid) ->
+ verify_options(Options, Errors);
verify_options([Option | Options], Errors) ->
verify_options(Options, [Option | Errors]);
verify_options([], []) ->
View
@@ -29,22 +29,20 @@
%%% @doc
%%% This module implements the HTTP request handling. This should normally
%%% not be called directly since it should be spawned by the lhttpc module.
-%%% @end
-module(lhttpc_client).
--export([request/10, request_with_timeout/2]).
+-export([request/10]).
-include("lhttpc_types.hrl").
-record(client_state, {
- req_id :: tuple(),
+ req_id :: term(),
host :: string(),
port = 80 :: integer(),
ssl = false :: true | false,
method :: string(),
request :: iolist(),
request_headers :: headers(),
- load_balancer:: pid(),
socket,
connect_timeout = infinity :: timeout(),
connect_options = [] :: [any()],
@@ -63,15 +61,10 @@
-define(CONNECTION_HDR(HDRS, DEFAULT),
string:to_lower(lhttpc_lib:header_value("connection", HDRS, DEFAULT))).
-request_with_timeout(Timeout, [ReqId, StreamTo, _Host, _Port, _Ssl, _Path, _Method, _Hdrs, _Body, _Options] = Args) ->
- TimerRef = erlang:send_after(Timeout, lhttpc_manager, {kill_client_after_timeout, ReqId, self(), StreamTo}),
- ok = apply(?MODULE, request, Args),
- erlang:cancel_timer(TimerRef).
-
--spec request(tuple(), pid(), string(), 1..65535, true | false, string(),
+-spec request(term(), pid(), string(), 1..65535, true | false, string(),
string() | atom(), headers(), iolist(), [option()]) -> no_return().
%% @spec (ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, RequestBody, Options) -> ok
-%% ReqId = tuple()
+%% ReqId = term()
%% From = pid()
%% Host = string()
%% Port = integer()
@@ -108,14 +101,16 @@ execute(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) ->
PartialUpload = proplists:is_defined(partial_upload, Options),
PartialDownload = proplists:is_defined(partial_download, Options),
PartialDownloadOptions = proplists:get_value(partial_download, Options, []),
- ConnectOptions = proplists:get_value(connect_options, Options, []),
NormalizedMethod = lhttpc_lib:normalize_method(Method),
MaxConnections = proplists:get_value(max_connections, Options, 10),
ConnectionTimeout = proplists:get_value(connection_timeout, Options, infinity),
{ChunkedUpload, Request} = lhttpc_lib:format_request(Path, NormalizedMethod,
Hdrs, Host, Port, Body, PartialUpload),
- LbRequest = {lb, Host, Port, Ssl, MaxConnections, ConnectionTimeout},
- {ok, Lb} = gen_server:call(lhttpc_manager, LbRequest, infinity),
+ Socket = case lhttpc_lb:checkout(Host, Port, Ssl, MaxConnections, ConnectionTimeout) of
+ {ok, S} -> S; % Re-using HTTP/1.1 connections
+ retry_later -> throw(retry_later);
+ no_socket -> undefined % Opening a new HTTP/1.1 connection
+ end,
State = #client_state{
req_id = ReqId,
host = Host,
@@ -125,10 +120,10 @@ execute(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) ->
request = Request,
requester = From,
request_headers = Hdrs,
- load_balancer = Lb,
+ socket = Socket,
connect_timeout = proplists:get_value(connect_timeout, Options,
infinity),
- connect_options = ConnectOptions,
+ connect_options = proplists:get_value(connect_options, Options, []),
attempts = 1 + proplists:get_value(send_retry, Options, 1),
partial_upload = PartialUpload,
upload_window = UploadWindowSize,
@@ -143,56 +138,59 @@ execute(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) ->
{R, undefined} ->
{ok, R};
{R, NewSocket} ->
- case lhttpc_sock:controlling_process(NewSocket, Lb, Ssl) of
- ok ->
- gen_server:cast(Lb, {store, NewSocket});
- _ ->
- ok
- end,
+ % The socket we ended up doing the request over is returned
+ % here, it might be the same as Socket, but we don't know.
+ lhttpc_lb:checkin(Host, Port, Ssl, NewSocket),
{ok, R}
end,
{response, ReqId, self(), Response}.
send_request(#client_state{attempts = 0}) ->
+ % Don't try again if the number of allowed attempts is 0.
throw(connection_closed);
send_request(#client_state{socket = undefined} = State) ->
+ Host = State#client_state.host,
+ Port = State#client_state.port,
+ Ssl = State#client_state.ssl,
+ Timeout = State#client_state.connect_timeout,
ConnectOptions = State#client_state.connect_options,
- ConnectTimeout = State#client_state.connect_timeout,
- Lb = State#client_state.load_balancer,
- SocketRequest = {socket, self(), ConnectOptions, ConnectTimeout},
- case gen_server:call(Lb, SocketRequest, infinity) of
+ SocketOptions = [binary, {packet, http}, {active, false} | ConnectOptions],
+ case lhttpc_sock:connect(Host, Port, SocketOptions, Timeout, Ssl) of
{ok, Socket} ->
- lhttpc_sock:setopts(Socket, [{active, false}], State#client_state.ssl),
send_request(State#client_state{socket = Socket});
+ {error, etimedout} ->
+ % TCP stack decided to give up
+ throw(connect_timeout);
+ {error, timeout} ->
+ throw(connect_timeout);
{error, Reason} ->
- throw(Reason)
+ erlang:error(Reason)
end;
send_request(State) ->
- Lb = State#client_state.load_balancer,
Socket = State#client_state.socket,
Ssl = State#client_state.ssl,
Request = State#client_state.request,
case lhttpc_sock:send(Socket, Request, Ssl) of
ok ->
if
- State#client_state.partial_upload -> partial_upload(State);
+ State#client_state.partial_upload -> partial_upload(State);
not State#client_state.partial_upload -> read_response(State)
end;
{error, closed} ->
- gen_server:cast(Lb, {remove, Socket}),
+ lhttpc_sock:close(Socket, Ssl),
NewState = State#client_state{
socket = undefined,
attempts = State#client_state.attempts - 1
},
send_request(NewState);
{error, Reason} ->
- gen_server:cast(Lb, {remove, Socket}),
+ lhttpc_sock:close(Socket, Ssl),
erlang:error(Reason)
end.
partial_upload(State) ->
Response = {ok, {self(), State#client_state.upload_window}},
- State#client_state.requester ! {response, State#client_state.req_id, self(), Response},
+ State#client_state.requester ! {response,State#client_state.req_id, self(), Response},
partial_upload_loop(State#client_state{attempts = 1, request = undefined}).
partial_upload_loop(State = #client_state{requester = Pid}) ->
@@ -233,16 +231,15 @@ encode_body_part(#client_state{chunked_upload = false}, Data) ->
check_send_result(_State, ok) ->
ok;
-check_send_result(#client_state{socket = Socket, load_balancer = Lb}, {error, Reason}) ->
- gen_server:cast(Lb, {remove, Socket}),
+check_send_result(#client_state{socket = Sock, ssl = Ssl}, {error, Reason}) ->
+ lhttpc_sock:close(Sock, Ssl),
throw(Reason).
read_response(#client_state{socket = Socket, ssl = Ssl} = State) ->
lhttpc_sock:setopts(Socket, [{packet, http}], Ssl),
read_response(State, nil, {nil, nil}, []).
read_response(State, Vsn, {StatusCode, _} = Status, Hdrs) ->
- Lb = State#client_state.load_balancer,
Socket = State#client_state.socket,
Ssl = State#client_state.ssl,
case lhttpc_sock:recv(Socket, Ssl) of
@@ -265,22 +262,22 @@ read_response(State, Vsn, {StatusCode, _} = Status, Hdrs) ->
Response = handle_response_body(State, Vsn, Status, Hdrs),
NewHdrs = element(2, Response),
ReqHdrs = State#client_state.request_headers,
- NewSocket = maybe_close_socket(Lb, Socket, Vsn, ReqHdrs, NewHdrs),
+ NewSocket = maybe_close_socket(Socket, Ssl, Vsn, ReqHdrs, NewHdrs),
{Response, NewSocket};
{error, closed} ->
% Either we only noticed that the socket was closed after we
% sent the request, the server closed it just after we put
% the request on the wire or the server has some issues and is
% closing connections without sending responses.
% If this the first attempt to send the request, we will try again.
- gen_server:cast(Lb, {remove, Socket}),
+ lhttpc_sock:close(Socket, Ssl),
NewState = State#client_state{
socket = undefined,
attempts = State#client_state.attempts - 1
},
send_request(NewState);
{error, timeout} ->
- gen_server:cast(Lb, {remove, Socket}),
+ lhttpc_sock:close(Socket, Ssl),
NewState = State#client_state{
socket = undefined,
attempts = 0
@@ -622,22 +619,22 @@ read_until_closed(Socket, Acc, Hdrs, Ssl) ->
erlang:error(Reason)
end.
-maybe_close_socket(Lb, Socket, {1, Minor}, ReqHdrs, RespHdrs) when Minor >= 1->
+maybe_close_socket(Socket, Ssl, {1, Minor}, ReqHdrs, RespHdrs) when Minor >= 1->
ClientConnection = ?CONNECTION_HDR(ReqHdrs, "keep-alive"),
ServerConnection = ?CONNECTION_HDR(RespHdrs, "keep-alive"),
if
ClientConnection =:= "close"; ServerConnection =:= "close" ->
- gen_server:cast(Lb, {remove, Socket}),
+ lhttpc_sock:close(Socket, Ssl),
undefined;
ClientConnection =/= "close", ServerConnection =/= "close" ->
Socket
end;
-maybe_close_socket(Lb, Socket, _, ReqHdrs, RespHdrs) ->
+maybe_close_socket(Socket, Ssl, _, ReqHdrs, RespHdrs) ->
ClientConnection = ?CONNECTION_HDR(ReqHdrs, "keep-alive"),
ServerConnection = ?CONNECTION_HDR(RespHdrs, "close"),
if
ClientConnection =:= "close"; ServerConnection =/= "keep-alive" ->
- gen_server:cast(Lb, {remove, Socket}),
+ lhttpc_sock:close(Socket, Ssl),
undefined;
ClientConnection =/= "close", ServerConnection =:= "keep-alive" ->
Socket
Oops, something went wrong.

0 comments on commit 748781b

Please sign in to comment.