Permalink
Browse files

update ibrowse to 4.0.1

  • Loading branch information...
1 parent a4eb1b3 commit fd4b78671c4367f4f6469156e428176648c07a76 @janl janl committed Nov 14, 2012
Showing with 566 additions and 265 deletions.
  1. +1 −1 NOTICE
  2. +4 −10 src/ibrowse/ibrowse.app.in
  3. +158 −92 src/ibrowse/ibrowse.erl
  4. +165 −103 src/ibrowse/ibrowse_http_client.erl
  5. +54 −37 src/ibrowse/ibrowse_lb.erl
  6. +62 −12 src/ibrowse/ibrowse_lib.erl
  7. +122 −10 src/ibrowse/ibrowse_test.erl
View
2 NOTICE
@@ -36,7 +36,7 @@ This product also includes the following third-party components:
* ibrowse (http://github.com/cmullaparthi/ibrowse/tree/master)
- Copyright 2009, Chandrashekhar Mullaparthi
+ Copyright 2005-2012, Chandrashekhar Mullaparthi
* Erlang OAuth (http://github.com/tim/erlang-oauth)
View
@@ -1,13 +1,7 @@
{application, ibrowse,
- [{description, "HTTP client application"},
- {vsn, "2.2.0"},
- {modules, [ ibrowse,
- ibrowse_http_client,
- ibrowse_app,
- ibrowse_sup,
- ibrowse_lib,
- ibrowse_lb ]},
- {registered, []},
- {applications, [kernel,stdlib,sasl]},
+ [{description, "Erlang HTTP client application"},
+ {vsn, "4.0.1"},
+ {registered, [ibrowse_sup, ibrowse]},
+ {applications, [kernel,stdlib]},
{env, []},
{mod, {ibrowse_app, []}}]}.
View
@@ -6,8 +6,7 @@
%%% Created : 11 Oct 2003 by Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
%%%-------------------------------------------------------------------
%% @author Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com>
-%% @copyright 2005-2011 Chandrashekhar Mullaparthi
-%% @version 2.1.3
+%% @copyright 2005-2012 Chandrashekhar Mullaparthi
%% @doc The ibrowse application implements an HTTP 1.1 client in erlang. This
%% module implements the API of the HTTP client. There is one named
%% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is
@@ -71,6 +70,7 @@
-export([
rescan_config/0,
rescan_config/1,
+ add_config/1,
get_config_value/1,
get_config_value/2,
spawn_worker_process/1,
@@ -97,7 +97,10 @@
trace_off/2,
all_trace_off/0,
show_dest_status/0,
- show_dest_status/2
+ show_dest_status/1,
+ show_dest_status/2,
+ get_metrics/0,
+ get_metrics/2
]).
-ifdef(debug).
@@ -136,7 +139,12 @@ start() ->
%% @doc Stop the ibrowse process. Useful when testing using the shell.
stop() ->
- catch gen_server:call(ibrowse, stop).
+ case catch gen_server:call(ibrowse, stop) of
+ {'EXIT',{noproc,_}} ->
+ ok;
+ Res ->
+ Res
+ end.
%% @doc This is the basic function to send a HTTP request.
%% The Status return value indicates the HTTP status code returned by the webserver
@@ -277,7 +285,8 @@ send_req(Url, Headers, Method, Body) ->
%% {transfer_encoding, {chunked, ChunkSize}} |
%% {headers_as_is, boolean()} |
%% {give_raw_headers, boolean()} |
-%% {preserve_chunked_encoding,boolean()}
+%% {preserve_chunked_encoding,boolean()} |
+%% {workaround, head_response_with_body}
%%
%% stream_to() = process() | {process(), once}
%% process() = pid() | atom()
@@ -287,7 +296,7 @@ send_req(Url, Headers, Method, Body) ->
%% Sock_opts = [Sock_opt]
%% Sock_opt = term()
%% ChunkSize = integer()
-%% srtf() = boolean() | filename()
+%% srtf() = boolean() | filename() | {append, filename()}
%% filename() = string()
%% response_format() = list | binary
send_req(Url, Headers, Method, Body, Options) ->
@@ -354,15 +363,16 @@ try_routing_request(_, _, _, _, _, _, _, _, _, _, _) ->
{error, retry_later}.
merge_options(Host, Port, Options) ->
- Config_options = get_config_value({options, Host, Port}, []),
+ Config_options = get_config_value({options, Host, Port}, []) ++
+ get_config_value({options, global}, []),
lists:foldl(
fun({Key, Val}, Acc) ->
- case lists:keysearch(Key, 1, Options) of
- false ->
- [{Key, Val} | Acc];
- _ ->
- Acc
- end
+ case lists:keysearch(Key, 1, Options) of
+ false ->
+ [{Key, Val} | Acc];
+ _ ->
+ Acc
+ end
end, Options, Config_options).
get_lb_pid(Url) ->
@@ -426,6 +436,8 @@ do_send_req(Conn_Pid, Parsed_url, Headers, Method, Body, Options, Timeout) ->
{error, req_timedout};
{'EXIT', {noproc, {gen_server, call, [Conn_Pid, _, _]}}} ->
{error, sel_conn_closed};
+ {'EXIT', {normal, _}} ->
+ {error, req_timedout};
{error, connection_closed} ->
{error, sel_conn_closed};
{'EXIT', Reason} ->
@@ -581,74 +593,98 @@ all_trace_off() ->
%% about workers spawned using spawn_worker_process/2 or
%% spawn_link_worker_process/2 is not included.
show_dest_status() ->
- Dests = lists:filter(fun({lb_pid, {Host, Port}, _}) when is_list(Host),
- is_integer(Port) ->
- true;
- (_) ->
- false
- end, ets:tab2list(ibrowse_lb)),
- All_ets = ets:all(),
io:format("~-40.40s | ~-5.5s | ~-10.10s | ~s~n",
["Server:port", "ETS", "Num conns", "LB Pid"]),
io:format("~80.80.=s~n", [""]),
- lists:foreach(fun({lb_pid, {Host, Port}, Lb_pid}) ->
- case lists:dropwhile(
- fun(Tid) ->
- ets:info(Tid, owner) /= Lb_pid
- end, All_ets) of
- [] ->
- io:format("~40.40s | ~-5.5s | ~-5.5s | ~s~n",
- [Host ++ ":" ++ integer_to_list(Port),
- "",
- "",
- io_lib:format("~p", [Lb_pid])]
- );
- [Tid | _] ->
- catch (
- begin
- Size = ets:info(Tid, size),
- io:format("~40.40s | ~-5.5s | ~-5.5s | ~s~n",
- [Host ++ ":" ++ integer_to_list(Port),
- io_lib:format("~p", [Tid]),
- integer_to_list(Size),
- io_lib:format("~p", [Lb_pid])]
- )
- end
- )
- end
- end, Dests).
-
+ Metrics = get_metrics(),
+ lists:foreach(
+ fun({Host, Port, Lb_pid, Tid, Size}) ->
+ io:format("~40.40s | ~-5.5s | ~-5.5s | ~p~n",
+ [Host ++ ":" ++ integer_to_list(Port),
+ integer_to_list(Tid),
+ integer_to_list(Size),
+ Lb_pid])
+ end, Metrics).
+
+show_dest_status(Url) ->
+ #url{host = Host, port = Port} = ibrowse_lib:parse_url(Url),
+ show_dest_status(Host, Port).
+
%% @doc Shows some internal information about load balancing to a
%% specified Host:Port. Info about workers spawned using
%% spawn_worker_process/2 or spawn_link_worker_process/2 is not
%% included.
show_dest_status(Host, Port) ->
+ case get_metrics(Host, Port) of
+ {Lb_pid, MsgQueueSize, Tid, Size,
+ {{First_p_sz, First_speculative_sz},
+ {Last_p_sz, Last_speculative_sz}}} ->
+ io:format("Load Balancer Pid : ~p~n"
+ "LB process msg q size : ~p~n"
+ "LB ETS table id : ~p~n"
+ "Num Connections : ~p~n"
+ "Smallest pipeline : ~p:~p~n"
+ "Largest pipeline : ~p:~p~n",
+ [Lb_pid, MsgQueueSize, Tid, Size,
+ First_p_sz, First_speculative_sz,
+ Last_p_sz, Last_speculative_sz]);
+ _Err ->
+ io:format("Metrics not available~n", [])
+ end.
+
+get_metrics() ->
+ Dests = lists:filter(fun({lb_pid, {Host, Port}, _}) when is_list(Host),
+ is_integer(Port) ->
+ true;
+ (_) ->
+ false
+ end, ets:tab2list(ibrowse_lb)),
+ All_ets = ets:all(),
+ lists:map(fun({lb_pid, {Host, Port}, Lb_pid}) ->
+ case lists:dropwhile(
+ fun(Tid) ->
+ ets:info(Tid, owner) /= Lb_pid
+ end, All_ets) of
+ [] ->
+ {Host, Port, Lb_pid, unknown, 0};
+ [Tid | _] ->
+ Size = case catch (ets:info(Tid, size)) of
+ N when is_integer(N) -> N;
+ _ -> 0
+ end,
+ {Host, Port, Lb_pid, Tid, Size}
+ end
+ end, Dests).
+
+get_metrics(Host, Port) ->
case ets:lookup(ibrowse_lb, {Host, Port}) of
[] ->
no_active_processes;
[#lb_pid{pid = Lb_pid}] ->
- io:format("Load Balancer Pid : ~p~n", [Lb_pid]),
- io:format("LB process msg q size : ~p~n", [(catch process_info(Lb_pid, message_queue_len))]),
+ MsgQueueSize = (catch process_info(Lb_pid, message_queue_len)),
+ %% {Lb_pid, MsgQueueSize,
case lists:dropwhile(
fun(Tid) ->
ets:info(Tid, owner) /= Lb_pid
end, ets:all()) of
[] ->
- io:format("Couldn't locate ETS table for ~p~n", [Lb_pid]);
+ {Lb_pid, MsgQueueSize, unknown, 0, unknown};
[Tid | _] ->
- First = ets:first(Tid),
- Last = ets:last(Tid),
- Size = ets:info(Tid, size),
- io:format("LB ETS table id : ~p~n", [Tid]),
- io:format("Num Connections : ~p~n", [Size]),
- case Size of
- 0 ->
- ok;
- _ ->
- {First_p_sz, _} = First,
- {Last_p_sz, _} = Last,
- io:format("Smallest pipeline : ~1000.p~n", [First_p_sz]),
- io:format("Largest pipeline : ~1000.p~n", [Last_p_sz])
+ try
+ Size = ets:info(Tid, size),
+ case Size of
+ 0 ->
+ ok;
+ _ ->
+ First = ets:first(Tid),
+ Last = ets:last(Tid),
+ [{_, First_p_sz, First_speculative_sz}] = ets:lookup(Tid, First),
+ [{_, Last_p_sz, Last_speculative_sz}] = ets:lookup(Tid, Last),
+ {Lb_pid, MsgQueueSize, Tid, Size,
+ {{First_p_sz, First_speculative_sz}, {Last_p_sz, Last_speculative_sz}}}
+ end
+ catch _:_ ->
+ not_available
end
end
end.
@@ -663,9 +699,15 @@ rescan_config() ->
%% Clear current configuration for ibrowse and load from the specified
%% file. Current configuration is cleared only if the specified
%% file is readable using file:consult/1
+rescan_config([{_,_}|_]=Terms) ->
+ gen_server:call(?MODULE, {rescan_config_terms, Terms});
rescan_config(File) when is_list(File) ->
gen_server:call(?MODULE, {rescan_config, File}).
+%% @doc Add additional configuration elements at runtime.
+add_config([{_,_}|_]=Terms) ->
+ gen_server:call(?MODULE, {add_config_terms, Terms}).
+
%%====================================================================
%% Server functions
%%====================================================================
@@ -701,44 +743,60 @@ import_config() ->
import_config(Filename) ->
case file:consult(Filename) of
{ok, Terms} ->
- ets:delete_all_objects(ibrowse_conf),
- Fun = fun({dest, Host, Port, MaxSess, MaxPipe, Options})
- when is_list(Host), is_integer(Port),
- is_integer(MaxSess), MaxSess > 0,
- is_integer(MaxPipe), MaxPipe > 0, is_list(Options) ->
- I = [{{max_sessions, Host, Port}, MaxSess},
- {{max_pipeline_size, Host, Port}, MaxPipe},
- {{options, Host, Port}, Options}],
- lists:foreach(
- fun({X, Y}) ->
- ets:insert(ibrowse_conf,
- #ibrowse_conf{key = X,
- value = Y})
- end, I);
- ({K, V}) ->
- ets:insert(ibrowse_conf,
- #ibrowse_conf{key = K,
- value = V});
- (X) ->
- io:format("Skipping unrecognised term: ~p~n", [X])
- end,
- lists:foreach(Fun, Terms);
+ apply_config(Terms);
_Err ->
ok
end.
+apply_config(Terms) ->
+ ets:delete_all_objects(ibrowse_conf),
+ insert_config(Terms).
+
+insert_config(Terms) ->
+ Fun = fun({dest, Host, Port, MaxSess, MaxPipe, Options})
+ when is_list(Host), is_integer(Port),
+ is_integer(MaxSess), MaxSess > 0,
+ is_integer(MaxPipe), MaxPipe > 0, is_list(Options) ->
+ I = [{{max_sessions, Host, Port}, MaxSess},
+ {{max_pipeline_size, Host, Port}, MaxPipe},
+ {{options, Host, Port}, Options}],
+ lists:foreach(
+ fun({X, Y}) ->
+ ets:insert(ibrowse_conf,
+ #ibrowse_conf{key = X,
+ value = Y})
+ end, I);
+ ({K, V}) ->
+ ets:insert(ibrowse_conf,
+ #ibrowse_conf{key = K,
+ value = V});
+ (X) ->
+ io:format("Skipping unrecognised term: ~p~n", [X])
+ end,
+ lists:foreach(Fun, Terms).
+
%% @doc Internal export
get_config_value(Key) ->
- [#ibrowse_conf{value = V}] = ets:lookup(ibrowse_conf, Key),
- V.
+ try
+ [#ibrowse_conf{value = V}] = ets:lookup(ibrowse_conf, Key),
+ V
+ catch
+ error:badarg ->
+ throw({error, ibrowse_not_running})
+ end.
%% @doc Internal export
get_config_value(Key, DefVal) ->
- case ets:lookup(ibrowse_conf, Key) of
- [] ->
- DefVal;
- [#ibrowse_conf{value = V}] ->
- V
+ try
+ case ets:lookup(ibrowse_conf, Key) of
+ [] ->
+ DefVal;
+ [#ibrowse_conf{value = V}] ->
+ V
+ end
+ catch
+ error:badarg ->
+ throw({error, ibrowse_not_running})
end.
set_config_value(Key, Val) ->
@@ -777,6 +835,14 @@ handle_call({rescan_config, File}, _From, State) ->
Ret = (catch import_config(File)),
{reply, Ret, State};
+handle_call({rescan_config_terms, Terms}, _From, State) ->
+ Ret = (catch apply_config(Terms)),
+ {reply, Ret, State};
+
+handle_call({add_config_terms, Terms}, _From, State) ->
+ Ret = (catch insert_config(Terms)),
+ {reply, Ret, State};
+
handle_call(Request, _From, State) ->
Reply = {unknown_request, Request},
{reply, Reply, State}.
Oops, something went wrong.

0 comments on commit fd4b786

Please sign in to comment.