Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

ibrowse was not reading all the options from its config file.

  • Loading branch information...
commit 9dddc77f05b8d91f8196e31500983f6b1e766251 1 parent c784ab8
chandrusf authored
View
6 lib/ibrowse/README
@@ -1,4 +1,4 @@
-$Id: README,v 1.15 2008/03/27 01:35:49 chandrusf Exp $
+$Id: README,v 1.16 2008/05/21 15:28:11 chandrusf Exp $
ibrowse is a HTTP client. The following are a list of features.
- RFC2616 compliant (AFAIK)
@@ -22,6 +22,10 @@ Comments to : Chandrashekhar.Mullaparthi@t-mobile.co.uk
CONTRIBUTIONS & CHANGE HISTORY
==============================
+21-05-2008 - * Fixed bug in reading some options from the ibrowse.conf file.
+ Reported by Erik Reitsma on the erlyaws mailing list
+ * Fixed bug when cleaning up closing connections
+
27-03-2008 - * Major rewrite of the load balancing feature. Additional module,
ibrowse_lb.erl, introduced to achieve this.
* Can now get a handle to a connection process which is not part of
View
9 lib/ibrowse/doc/ibrowse.html
@@ -181,10 +181,13 @@ <h3 class="function"><a name="send_req-3">send_req/3</a></h3>
<h3 class="function"><a name="send_req-4">send_req/4</a></h3>
<div class="spec">
<p><tt>send_req(Url, Headers, Method::<a href="#type-method">method()</a>, Body::<a href="#type-body">body()</a>) -&gt; <a href="#type-response">response()</a></tt>
-<ul class="definitions"><li><tt><a name="type-body">body()</a> = [] | string() | binary()</tt></li>
+<ul class="definitions"><li><tt><a name="type-body">body()</a> = [] | string() | binary() | <a href="#type-fun_arity_0">fun_arity_0()</a> | {<a href="#type-fun_arity_1">fun_arity_1()</a>, <a href="#type-initial_state">initial_state()</a>}</tt></li>
+<li><tt><a name="type-initial_state">initial_state()</a> = term()</tt></li>
</ul></p>
</div><p>Same as send_req/3.
- If a list is specified for the body it has to be a flat list.</p>
+ If a list is specified for the body it has to be a flat list. The body can also be a fun/0 or a fun/1. <br>
+ If fun/0, the connection handling process will repeatdely call the fun until it returns an error or eof. <pre>Fun() = {ok, Data} | eof</pre><br>
+ If fun/1, the connection handling process will repeatedly call the fun with the supplied state until it returns an error or eof. <pre>Fun(State) = {ok, Data} | {ok, Data, NewState} | eof</pre></p>
<h3 class="function"><a name="send_req-5">send_req/5</a></h3>
<div class="spec">
@@ -354,6 +357,6 @@ <h3 class="function"><a name="trace_on-2">trace_on/2</a></h3>
<hr>
<div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
-<p><i>Generated by EDoc, Mar 27 2008, 01:03:49.</i></p>
+<p><i>Generated by EDoc, Mar 27 2008, 01:20:55.</i></p>
</body>
</html>
View
1  lib/ibrowse/priv/ibrowse.conf
@@ -15,3 +15,4 @@
%% If SSL is to be used, both the options, is_ssl and ssl_options MUST be specified
%% where option() is all options supported in the ssl module
+{{options, "www.google.co.uk", 80}, [{proxy_host, "proxy"}, {proxy_port, 8080}, {proxy_user, "cmullaparthi"}, {proxy_password, "20nov99"}]}.
View
27 lib/ibrowse/src/ibrowse.erl
@@ -57,7 +57,7 @@
%% driver isn't actually used.</p>
-module(ibrowse).
--vsn('$Id: ibrowse.erl,v 1.6 2008/03/27 01:35:50 chandrusf Exp $ ').
+-vsn('$Id: ibrowse.erl,v 1.7 2008/05/21 15:28:11 chandrusf Exp $ ').
-behaviour(gen_server).
%%--------------------------------------------------------------------
@@ -239,10 +239,11 @@ send_req(Url, Headers, Method, Body, Options, Timeout) ->
end,
Max_sessions = get_max_sessions(Host, Port, Options),
Max_pipeline_size = get_max_pipeline_size(Host, Port, Options),
+ Options_1 = merge_options(Host, Port, Options),
{SSLOptions, IsSSL} =
- case get_value(is_ssl, Options, false) of
+ case get_value(is_ssl, Options_1, false) of
false -> {[], false};
- true -> {get_value(ssl_options, Options), true}
+ true -> {get_value(ssl_options, Options_1), true}
end,
case ibrowse_lb:spawn_connection(Lb_pid, Parsed_url,
Max_sessions,
@@ -250,7 +251,7 @@ send_req(Url, Headers, Method, Body, Options, Timeout) ->
{SSLOptions, IsSSL}) of
{ok, Conn_Pid} ->
do_send_req(Conn_Pid, Parsed_url, Headers,
- Method, Body, Options, Timeout);
+ Method, Body, Options_1, Timeout);
Err ->
Err
end;
@@ -258,6 +259,18 @@ send_req(Url, Headers, Method, Body, Options, Timeout) ->
{error, {url_parsing_failed, Err}}
end.
+merge_options(Host, Port, Options) ->
+ Config_options = get_config_value({options, Host, Port}, []),
+ lists:foldl(
+ fun({Key, Val}, Acc) ->
+ case lists:keysearch(Key, 1, Options) of
+ false ->
+ [{Key, Val} | Acc];
+ _ ->
+ Acc
+ end
+ end, Options, Config_options).
+
get_lb_pid(Url) ->
gen_server:call(?MODULE, {get_lb_pid, Url}).
@@ -354,8 +367,10 @@ send_req_direct(Conn_pid, Url, Headers, Method, Body, Options) ->
%% returned by spawn_worker_process/2 or spawn_link_worker_process/2
send_req_direct(Conn_pid, Url, Headers, Method, Body, Options, Timeout) ->
case catch parse_url(Url) of
- #url{} = Parsed_url ->
- case do_send_req(Conn_pid, Parsed_url, Headers, Method, Body, Options, Timeout) of
+ #url{host = Host,
+ port = Port} = Parsed_url ->
+ Options_1 = merge_options(Host, Port, Options),
+ case do_send_req(Conn_pid, Parsed_url, Headers, Method, Body, Options_1, Timeout) of
{error, {'EXIT', {noproc, _}}} ->
{error, worker_is_dead};
Ret ->
View
4 lib/ibrowse/src/ibrowse_http_client.erl
@@ -6,7 +6,7 @@
%%% Created : 11 Oct 2003 by Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
%%%-------------------------------------------------------------------
-module(ibrowse_http_client).
--vsn('$Id: ibrowse_http_client.erl,v 1.17 2008/03/27 01:35:50 chandrusf Exp $ ').
+-vsn('$Id: ibrowse_http_client.erl,v 1.18 2008/05/21 15:28:11 chandrusf Exp $ ').
-behaviour(gen_server).
%%--------------------------------------------------------------------
@@ -1294,7 +1294,7 @@ shutting_down(#state{lb_ets_tid = undefined}) ->
ok;
shutting_down(#state{lb_ets_tid = Tid,
cur_pipeline_size = Sz}) ->
- ets:delete(Tid, {Sz, self()}).
+ catch ets:delete(Tid, {Sz, self()}).
inc_pipeline_counter(#state{is_closing = true} = State) ->
State;
View
55 lib/ibrowse/src/ibrowse_test.erl
@@ -4,7 +4,7 @@
%%% Created : 14 Oct 2003 by Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
-module(ibrowse_test).
--vsn('$Id: ibrowse_test.erl,v 1.2 2008/03/27 01:35:50 chandrusf Exp $ ').
+-vsn('$Id: ibrowse_test.erl,v 1.3 2008/05/21 15:28:11 chandrusf Exp $ ').
-export([
load_test/3,
send_reqs_1/3,
@@ -32,6 +32,7 @@ send_reqs_1(Url, NumWorkers, NumReqsPerWorker) ->
Start_time = now(),
ets:new(pid_table, [named_table, public]),
ets:new(ibrowse_test_results, [named_table, public]),
+ ets:new(ibrowse_errors, [named_table, public, ordered_set]),
init_results(),
process_flag(trap_exit, true),
log_msg("Starting spawning of workers...~n", []),
@@ -45,15 +46,20 @@ send_reqs_1(Url, NumWorkers, NumReqsPerWorker) ->
log_msg("End time : ~1000.p~n", [calendar:now_to_local_time(End_time)]),
Elapsed_time_secs = trunc(timer:now_diff(End_time, Start_time) / 1000000),
log_msg("Elapsed : ~p~n", [Elapsed_time_secs]),
- log_msg("Reqs/sec : ~p~n", [(NumWorkers*NumReqsPerWorker) / Elapsed_time_secs]).
+ log_msg("Reqs/sec : ~p~n", [(NumWorkers*NumReqsPerWorker) / Elapsed_time_secs]),
+ dump_errors().
init_results() ->
ets:insert(ibrowse_test_results, {crash, 0}),
ets:insert(ibrowse_test_results, {send_failed, 0}),
ets:insert(ibrowse_test_results, {other_error, 0}),
ets:insert(ibrowse_test_results, {success, 0}),
+ ets:insert(ibrowse_test_results, {retry_later, 0}),
+ ets:insert(ibrowse_test_results, {trid_mismatch, 0}),
+ ets:insert(ibrowse_test_results, {success_no_trid, 0}),
ets:insert(ibrowse_test_results, {failed, 0}),
- ets:insert(ibrowse_test_results, {timeout, 0}).
+ ets:insert(ibrowse_test_results, {timeout, 0}),
+ ets:insert(ibrowse_test_results, {req_id, 0}).
spawn_workers(_Url, 0, _) ->
ok;
@@ -89,19 +95,54 @@ do_send_req(Url, NumReqs) ->
do_send_req_1(_Url, 0) ->
ets:delete(pid_table, self());
do_send_req_1(Url, NumReqs) ->
- case ibrowse:send_req(Url, [], get, [], [], 10000) of
- {ok, _Status, _Headers, _Body} ->
- ets:update_counter(ibrowse_test_results, success, 1);
+ Counter = integer_to_list(ets:update_counter(ibrowse_test_results, req_id, 1)),
+ case ibrowse:send_req(Url, [{"ib_req_id", Counter}], get, [], [], 10000) of
+ {ok, _Status, Headers, _Body} ->
+ case lists:keysearch("ib_req_id", 1, Headers) of
+ {value, {_, Counter}} ->
+ ets:update_counter(ibrowse_test_results, success, 1);
+ {value, _} ->
+ ets:update_counter(ibrowse_test_results, trid_mismatch, 1);
+ false ->
+ ets:update_counter(ibrowse_test_results, success_no_trid, 1)
+ end;
{error, req_timedout} ->
ets:update_counter(ibrowse_test_results, timeout, 1);
{error, send_failed} ->
ets:update_counter(ibrowse_test_results, send_failed, 1);
- _Err ->
+ {error, retry_later} ->
+ ets:update_counter(ibrowse_test_results, retry_later, 1);
+ Err ->
+ ets:insert(ibrowse_errors, {now(), Err}),
ets:update_counter(ibrowse_test_results, other_error, 1),
ok
end,
do_send_req_1(Url, NumReqs-1).
+dump_errors() ->
+ case ets:info(ibrowse_errors, size) of
+ 0 ->
+ ok;
+ _ ->
+ {A, B, C} = now(),
+ Filename = lists:flatten(
+ io_lib:format("ibrowse_errors_~p_~p_~p.txt" , [A, B, C])),
+ case file:open(Filename, [write, delayed_write, raw]) of
+ {ok, Iod} ->
+ dump_errors(ets:first(ibrowse_errors), Iod);
+ Err ->
+ io:format("failed to create file ~s. Reason: ~p~n", [Filename, Err]),
+ ok
+ end
+ end.
+
+dump_errors('$end_of_table', Iod) ->
+ file:close(Iod);
+dump_errors(Key, Iod) ->
+ [{_, Term}] = ets:lookup(ibrowse_errors, Key),
+ file:write(Iod, io_lib:format("~p~n", [Term])),
+ dump_errors(ets:next(ibrowse_errors, Key), Iod).
+
%%------------------------------------------------------------------------------
%% Unit Tests
%%------------------------------------------------------------------------------
View
2  lib/ibrowse/vsn.mk
@@ -1,2 +1,2 @@
-IBROWSE_VSN = 1.4
+IBROWSE_VSN = 1.4.1
Please sign in to comment.
Something went wrong with that request. Please try again.