Permalink
Browse files

Added option to allow caller to specify socket options

  • Loading branch information...
1 parent c75b467 commit 3e559c8d67e5b96b0c099f1ec2241a351178e62c chandrusf committed Sep 6, 2009
Showing with 94 additions and 17 deletions.
  1. +3 −1 lib/ibrowse/README
  2. +8 −2 lib/ibrowse/doc/ibrowse.html
  3. +48 −2 lib/ibrowse/src/ibrowse.erl
  4. +34 −11 lib/ibrowse/src/ibrowse_http_client.erl
  5. +1 −1 lib/ibrowse/vsn.mk
View
@@ -18,12 +18,14 @@ ibrowse is available under two different licenses. LGPL and the BSD license.
Comments to : Chandrashekhar.Mullaparthi@gmail.com
-Version : 1.5.2
+Version : 1.5.3
Latest version : git://github.com/cmullaparthi/ibrowse.git
CONTRIBUTIONS & CHANGE HISTORY
==============================
+05-09-2009 - * Introduced option to allow caller to set socket options.
+
29-07-2009 - * The ETS table created for load balancing of requests was not
being deleted which led to the node not being able to create
any more ETS tables if queries were made to many number of
@@ -203,12 +203,14 @@ <h3 class="function"><a name="send_req-5">send_req/5</a></h3>
<div class="spec">
<p><tt>send_req(Url::string(), Headers::<a href="#type-headerList">headerList()</a>, Method::<a href="#type-method">method()</a>, Body::<a href="#type-body">body()</a>, Options::<a href="#type-optionList">optionList()</a>) -&gt; <a href="#type-response">response()</a></tt>
<ul class="definitions"><li><tt><a name="type-optionList">optionList()</a> = [<a href="#type-option">option()</a>]</tt></li>
-<li><tt><a name="type-option">option()</a> = {max_sessions, integer()} | {response_format, <a href="#type-response_format">response_format()</a>} | {stream_chunk_size, integer()} | {max_pipeline_size, integer()} | {trace, <a href="#type-boolean">boolean()</a>} | {is_ssl, <a href="#type-boolean">boolean()</a>} | {ssl_options, [SSLOpt]} | {pool_name, atom()} | {proxy_host, string()} | {proxy_port, integer()} | {proxy_user, string()} | {proxy_password, string()} | {use_absolute_uri, <a href="#type-boolean">boolean()</a>} | {basic_auth, {<a href="#type-username">username()</a>, <a href="#type-password">password()</a>}} | {cookie, string()} | {content_length, integer()} | {content_type, string()} | {save_response_to_file, <a href="#type-srtf">srtf()</a>} | {stream_to, <a href="#type-stream_to">stream_to()</a>} | {http_vsn, {MajorVsn, MinorVsn}} | {host_header, string()} | {inactivity_timeout, integer()} | {connect_timeout, integer()} | {transfer_encoding, {chunked, ChunkSize}}</tt></li>
+<li><tt><a name="type-option">option()</a> = {max_sessions, integer()} | {response_format, <a href="#type-response_format">response_format()</a>} | {stream_chunk_size, integer()} | {max_pipeline_size, integer()} | {trace, <a href="#type-boolean">boolean()</a>} | {is_ssl, <a href="#type-boolean">boolean()</a>} | {ssl_options, [SSLOpt]} | {pool_name, atom()} | {proxy_host, string()} | {proxy_port, integer()} | {proxy_user, string()} | {proxy_password, string()} | {use_absolute_uri, <a href="#type-boolean">boolean()</a>} | {basic_auth, {<a href="#type-username">username()</a>, <a href="#type-password">password()</a>}} | {cookie, string()} | {content_length, integer()} | {content_type, string()} | {save_response_to_file, <a href="#type-srtf">srtf()</a>} | {stream_to, <a href="#type-stream_to">stream_to()</a>} | {http_vsn, {MajorVsn, MinorVsn}} | {host_header, string()} | {inactivity_timeout, integer()} | {connect_timeout, integer()} | {socket_options, Sock_opts} | {transfer_encoding, {chunked, ChunkSize}}</tt></li>
<li><tt><a name="type-stream_to">stream_to()</a> = <a href="#type-process">process()</a> | {<a href="#type-process">process()</a>, once}</tt></li>
<li><tt><a name="type-process">process()</a> = pid() | atom()</tt></li>
<li><tt><a name="type-username">username()</a> = string()</tt></li>
<li><tt><a name="type-password">password()</a> = string()</tt></li>
<li><tt>SSLOpt = term()</tt></li>
+<li><tt>Sock_opts = [Sock_opt]</tt></li>
+<li><tt>Sock_opt = term()</tt></li>
<li><tt>ChunkSize = integer()</tt></li>
<li><tt><a name="type-srtf">srtf()</a> = <a href="#type-boolean">boolean()</a> | <a href="#type-filename">filename()</a></tt></li>
<li><tt><a name="type-filename">filename()</a> = string()</tt></li>
@@ -271,6 +273,10 @@ <h3 class="function"><a name="send_req-5">send_req/5</a></h3>
for connection setup.
</li>
</ul>
+
+ <li> The <code>socket_options</code> option can be used to set
+ specific options on the socket. The <code>{active, true | false | once}</code>
+ and <code>{packet_type, Packet_type}</code> will be filtered out by ibrowse. </li>
</p>
<h3 class="function"><a name="send_req-6">send_req/6</a></h3>
@@ -417,6 +423,6 @@ <h3 class="function"><a name="trace_on-2">trace_on/2</a></h3>
<hr>
<div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
-<p><i>Generated by EDoc, Jul 29 2009, 18:43:30.</i></p>
+<p><i>Generated by EDoc, Sep 5 2009, 23:59:48.</i></p>
</body>
</html>
@@ -7,7 +7,7 @@
%%%-------------------------------------------------------------------
%% @author Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com>
%% @copyright 2005-2009 Chandrashekhar Mullaparthi
-%% @version 1.5.1
+%% @version 1.5.2
%% @doc The ibrowse application implements an HTTP 1.1 client. This
%% module implements the API of the HTTP client. There is one named
%% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is
@@ -57,7 +57,7 @@
%% driver isn't actually used.</p>
-module(ibrowse).
--vsn('$Id: ibrowse.erl,v 1.10 2009/07/08 11:05:45 chandrusf Exp $ ').
+-vsn('$Id: ibrowse.erl,v 1.11 2009/09/06 20:04:02 chandrusf Exp $ ').
-behaviour(gen_server).
%%--------------------------------------------------------------------
@@ -98,6 +98,7 @@
trace_on/2,
trace_off/2,
all_trace_off/0,
+ show_dest_status/0,
show_dest_status/2
]).
@@ -227,6 +228,10 @@ send_req(Url, Headers, Method, Body) ->
%% </li>
%% </ul>
%%
+%% <li> The <code>socket_options</code> option can be used to set
+%% specific options on the socket. The <code>{active, true | false | once}</code>
+%% and <code>{packet_type, Packet_type}</code> will be filtered out by ibrowse. </li>
+%%
%% @spec send_req(Url::string(), Headers::headerList(), Method::method(), Body::body(), Options::optionList()) -> response()
%% optionList() = [option()]
%% option() = {max_sessions, integer()} |
@@ -252,13 +257,16 @@ send_req(Url, Headers, Method, Body) ->
%% {host_header, string()} |
%% {inactivity_timeout, integer()} |
%% {connect_timeout, integer()} |
+%% {socket_options, Sock_opts} |
%% {transfer_encoding, {chunked, ChunkSize}}
%%
%% stream_to() = process() | {process(), once}
%% process() = pid() | atom()
%% username() = string()
%% password() = string()
%% SSLOpt = term()
+%% Sock_opts = [Sock_opt]
+%% Sock_opt = term()
%% ChunkSize = integer()
%% srtf() = boolean() | filename()
%% filename() = string()
@@ -480,6 +488,44 @@ all_trace_off() ->
ibrowse ! all_trace_off,
ok.
+show_dest_status() ->
+ Dests = lists:filter(fun({lb_pid, {Host, Port}, _}) when is_list(Host),
+ is_integer(Port) ->
+ true;
+ (_) ->
+ false
+ end, ets:tab2list(ibrowse_lb)),
+ All_ets = ets:all(),
+ io:format("~-40.40s | ~-5.5s | ~-10.10s | ~s~n",
+ ["Server:port", "ETS", "Num conns", "LB Pid"]),
+ io:format("~80.80.=s~n", [""]),
+ lists:foreach(fun({lb_pid, {Host, Port}, Lb_pid}) ->
+ case lists:dropwhile(
+ fun(Tid) ->
+ ets:info(Tid, owner) /= Lb_pid
+ end, All_ets) of
+ [] ->
+ io:format("~40.40s | ~-5.5s | ~-5.5s | ~s~n",
+ [Host ++ ":" ++ integer_to_list(Port),
+ "",
+ "",
+ io_lib:format("~p", [Lb_pid])]
+ );
+ [Tid | _] ->
+ catch (
+ begin
+ Size = ets:info(Tid, size),
+ io:format("~40.40s | ~-5.5s | ~-5.5s | ~s~n",
+ [Host ++ ":" ++ integer_to_list(Port),
+ integer_to_list(Tid),
+ integer_to_list(Size),
+ io_lib:format("~p", [Lb_pid])]
+ )
+ end
+ )
+ end
+ end, Dests).
+
%% @doc Shows some internal information about load balancing to a
%% specified Host:Port. Info about workers spawned using
%% spawn_worker_process/2 or spawn_link_worker_process/2 is not
@@ -6,7 +6,7 @@
%%% Created : 11 Oct 2003 by Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
%%%-------------------------------------------------------------------
-module(ibrowse_http_client).
--vsn('$Id: ibrowse_http_client.erl,v 1.21 2009/07/29 18:03:21 chandrusf Exp $ ').
+-vsn('$Id: ibrowse_http_client.erl,v 1.22 2009/09/06 20:04:02 chandrusf Exp $ ').
-behaviour(gen_server).
%%--------------------------------------------------------------------
@@ -52,6 +52,7 @@
-record(request, {url, method, options, from,
stream_to, caller_controls_socket = false,
+ caller_socket_options = [],
req_id,
stream_chunk_size,
save_response_to_file = false,
@@ -412,15 +413,31 @@ handle_sock_closed(#state{reply_buffer = Buf, reqs = Reqs, http_status_code = SC
State
end.
-do_connect(Host, Port, _Options, #state{is_ssl=true, ssl_options=SSLOptions}, Timeout) ->
+do_connect(Host, Port, Options, #state{is_ssl=true, ssl_options=SSLOptions}, Timeout) ->
+ Caller_socket_options = get_value(socket_options, Options, []),
+ Other_sock_options = filter_sock_options(SSLOptions ++ Caller_socket_options),
ssl:connect(Host, Port,
- [binary, {nodelay, true}, {active, false} | SSLOptions],
+ [binary, {nodelay, true}, {active, false} | Other_sock_options],
Timeout);
-do_connect(Host, Port, _Options, _State, Timeout) ->
+do_connect(Host, Port, Options, _State, Timeout) ->
+ Caller_socket_options = get_value(socket_options, Options, []),
+ Other_sock_options = filter_sock_options(Caller_socket_options),
gen_tcp:connect(Host, Port,
- [binary, {nodelay, true}, {active, false}],
+ [binary, {nodelay, true}, {active, false} | Other_sock_options],
Timeout).
+%% We don't want the caller to specify certain options
+filter_sock_options(Opts) ->
+ lists:filter(fun({active, _}) ->
+ false;
+ ({packet, _}) ->
+ false;
+ (list) ->
+ false;
+ (_) ->
+ true
+ end, Opts).
+
do_send(Req, #state{socket = Sock, is_ssl = true}) -> ssl:send(Sock, Req);
do_send(Req, #state{socket = Sock, is_ssl = false}) -> gen_tcp:send(Sock, Req).
@@ -461,6 +478,7 @@ active_once(#state{cur_req = #request{caller_controls_socket = true}}) ->
active_once(#state{socket = Socket, is_ssl = Is_ssl}) ->
do_setopts(Socket, [{active, once}], Is_ssl).
+do_setopts(_Sock, [], _) -> ok;
do_setopts(Sock, Opts, true) -> ssl:setopts(Sock, Opts);
do_setopts(Sock, Opts, false) -> inet:setopts(Sock, Opts).
@@ -517,9 +535,12 @@ send_req_1(From,
port = Port,
path = RelPath} = Url,
Headers, Method, Body, Options, Timeout,
- #state{status = Status} = State) ->
+ #state{status = Status,
+ socket = Socket,
+ is_ssl = Is_ssl} = State) ->
ReqId = make_req_id(),
Resp_format = get_value(response_format, Options, list),
+ Caller_socket_options = get_value(socket_options, Options, []),
{StreamTo, Caller_controls_socket} =
case get_value(stream_to, Options, undefined) of
{Caller, once} when is_pid(Caller) or
@@ -540,14 +561,15 @@ send_req_1(From,
method = Method,
stream_to = StreamTo,
caller_controls_socket = Caller_controls_socket,
+ caller_socket_options = Caller_socket_options,
options = Options,
req_id = ReqId,
save_response_to_file = SaveResponseToFile,
stream_chunk_size = get_stream_chunk_size(Options),
response_format = Resp_format,
from = From},
State_1 = State#state{reqs=queue:in(NewReq, State#state.reqs)},
- Headers_1 = add_auth_headers(Url, Options, Headers, State),
+ Headers_1 = add_auth_headers(Url, Options, Headers, State_1),
HostHeaderValue = case lists:keysearch(host_header, 1, Options) of
false ->
case Port of
@@ -559,7 +581,7 @@ send_req_1(From,
end,
{Req, Body_1} = make_request(Method,
[{"Host", HostHeaderValue} | Headers_1],
- AbsPath, RelPath, Body, Options, State#state.use_proxy),
+ AbsPath, RelPath, Body, Options, State_1#state.use_proxy),
case get(my_trace_flag) of
true ->
%%Avoid the binary operations if trace is not on...
@@ -569,12 +591,13 @@ send_req_1(From,
"--- Request End ---~n", [NReq]);
_ -> ok
end,
- case do_send(Req, State) of
+ do_setopts(Socket, Caller_socket_options, Is_ssl),
+ case do_send(Req, State_1) of
ok ->
- case do_send_body(Body_1, State) of
+ case do_send_body(Body_1, State_1) of
ok ->
State_2 = inc_pipeline_counter(State_1),
- active_once(State_1),
+ active_once(State_2),
Ref = case Timeout of
infinity ->
undefined;
View
@@ -1,2 +1,2 @@
-IBROWSE_VSN = 1.5.2
+IBROWSE_VSN = 1.5.3

0 comments on commit 3e559c8

Please sign in to comment.