ibrowse was not reading all the options from its config file.

lib/ibrowse/README

@@ -1,4 +1,4 @@
-$Id: README,v 1.15 2008/03/27 01:35:49 chandrusf Exp $
+$Id: README,v 1.16 2008/05/21 15:28:11 chandrusf Exp $
 
 ibrowse is a HTTP client. The following are a list of features.
 	- RFC2616 compliant (AFAIK) 
@@ -22,6 +22,10 @@ Comments to : Chandrashekhar.Mullaparthi@t-mobile.co.uk
 
 CONTRIBUTIONS & CHANGE HISTORY
 ==============================
+21-05-2008 - * Fixed bug in reading some options from the ibrowse.conf file.
+               Reported by Erik Reitsma on the erlyaws mailing list
+             * Fixed bug when cleaning up closing connections
+
 27-03-2008 - * Major rewrite of the load balancing feature. Additional module,
                ibrowse_lb.erl, introduced to achieve this.
              * Can now get a handle to a connection process which is not part of

lib/ibrowse/doc/ibrowse.html

@@ -181,10 +181,13 @@ <h3 class="function"><a name="send_req-3">send_req/3</a></h3>
 <h3 class="function"><a name="send_req-4">send_req/4</a></h3>
 <div class="spec">
 <p><tt>send_req(Url, Headers, Method::<a href="#type-method">method()</a>, Body::<a href="#type-body">body()</a>) -&gt; <a href="#type-response">response()</a></tt>
-<ul class="definitions"><li><tt><a name="type-body">body()</a> = [] | string() | binary()</tt></li>
+<ul class="definitions"><li><tt><a name="type-body">body()</a> = [] | string() | binary() | <a href="#type-fun_arity_0">fun_arity_0()</a> | {<a href="#type-fun_arity_1">fun_arity_1()</a>, <a href="#type-initial_state">initial_state()</a>}</tt></li>
+<li><tt><a name="type-initial_state">initial_state()</a> = term()</tt></li>
 </ul></p>
 </div><p>Same as send_req/3.
-  If a list is specified for the body it has to be a flat list.</p>
+  If a list is specified for the body it has to be a flat list. The body can also be a fun/0 or a fun/1. <br>
+  If fun/0, the connection handling process will repeatdely call the fun until it returns an error or eof. <pre>Fun() = {ok, Data} | eof</pre><br>
+  If fun/1, the connection handling process will repeatedly call the fun with the supplied state until it returns an error or eof. <pre>Fun(State) = {ok, Data} | {ok, Data, NewState} | eof</pre></p>
 
 <h3 class="function"><a name="send_req-5">send_req/5</a></h3>
 <div class="spec">
@@ -354,6 +357,6 @@ <h3 class="function"><a name="trace_on-2">trace_on/2</a></h3>
 <hr>
 
 <div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
-<p><i>Generated by EDoc, Mar 27 2008, 01:03:49.</i></p>
+<p><i>Generated by EDoc, Mar 27 2008, 01:20:55.</i></p>
 </body>
 </html>

lib/ibrowse/priv/ibrowse.conf

@@ -15,3 +15,4 @@
 %% If SSL is to be used, both the options, is_ssl and ssl_options MUST be specified
 %% where option() is all options supported in the ssl module
 
+{{options, "www.google.co.uk", 80}, [{proxy_host, "proxy"}, {proxy_port, 8080}, {proxy_user, "cmullaparthi"}, {proxy_password, "20nov99"}]}.

lib/ibrowse/src/ibrowse.erl

@@ -57,7 +57,7 @@
 %% driver isn't actually used.</p>
 
 -module(ibrowse).
--vsn('$Id: ibrowse.erl,v 1.6 2008/03/27 01:35:50 chandrusf Exp $ ').
+-vsn('$Id: ibrowse.erl,v 1.7 2008/05/21 15:28:11 chandrusf Exp $ ').
 
 -behaviour(gen_server).
 %%--------------------------------------------------------------------
@@ -239,25 +239,38 @@ send_req(Url, Headers, Method, Body, Options, Timeout) ->
 		     end,
 	    Max_sessions = get_max_sessions(Host, Port, Options),
 	    Max_pipeline_size = get_max_pipeline_size(Host, Port, Options),
+	    Options_1 = merge_options(Host, Port, Options),
 	    {SSLOptions, IsSSL} =
-		case get_value(is_ssl, Options, false) of
+		case get_value(is_ssl, Options_1, false) of
 		    false -> {[], false};
-		    true -> {get_value(ssl_options, Options), true}
+		    true -> {get_value(ssl_options, Options_1), true}
 		end,
 	    case ibrowse_lb:spawn_connection(Lb_pid, Parsed_url,
 					     Max_sessions, 
 					     Max_pipeline_size,
 					     {SSLOptions, IsSSL}) of
 		{ok, Conn_Pid} ->
 		    do_send_req(Conn_Pid, Parsed_url, Headers,
-				Method, Body, Options, Timeout);
+				Method, Body, Options_1, Timeout);
 		Err ->
 		    Err
 	    end;
 	Err ->
 	    {error, {url_parsing_failed, Err}}
     end.
 
+merge_options(Host, Port, Options) ->
+    Config_options = get_config_value({options, Host, Port}, []),
+    lists:foldl(
+      fun({Key, Val}, Acc) ->
+			case lists:keysearch(Key, 1, Options) of
+			    false ->
+				[{Key, Val} | Acc];
+			    _ ->
+				Acc
+			end
+      end, Options, Config_options).
+
 get_lb_pid(Url) ->
     gen_server:call(?MODULE, {get_lb_pid, Url}).
 
@@ -354,8 +367,10 @@ send_req_direct(Conn_pid, Url, Headers, Method, Body, Options) ->
 %% returned by spawn_worker_process/2 or spawn_link_worker_process/2
 send_req_direct(Conn_pid, Url, Headers, Method, Body, Options, Timeout) ->
     case catch parse_url(Url) of
-	#url{} = Parsed_url ->
-	    case do_send_req(Conn_pid, Parsed_url, Headers, Method, Body, Options, Timeout) of
+	#url{host = Host,
+	     port = Port} = Parsed_url ->
+	    Options_1 = merge_options(Host, Port, Options),
+	    case do_send_req(Conn_pid, Parsed_url, Headers, Method, Body, Options_1, Timeout) of
 		{error, {'EXIT', {noproc, _}}} ->
 		    {error, worker_is_dead};
 		Ret ->

lib/ibrowse/src/ibrowse_http_client.erl

@@ -6,7 +6,7 @@
 %%% Created : 11 Oct 2003 by Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
 %%%-------------------------------------------------------------------
 -module(ibrowse_http_client).
--vsn('$Id: ibrowse_http_client.erl,v 1.17 2008/03/27 01:35:50 chandrusf Exp $ ').
+-vsn('$Id: ibrowse_http_client.erl,v 1.18 2008/05/21 15:28:11 chandrusf Exp $ ').
 
 -behaviour(gen_server).
 %%--------------------------------------------------------------------
@@ -1294,7 +1294,7 @@ shutting_down(#state{lb_ets_tid = undefined}) ->
     ok;
 shutting_down(#state{lb_ets_tid = Tid,
 		     cur_pipeline_size = Sz}) ->
-    ets:delete(Tid, {Sz, self()}).
+    catch ets:delete(Tid, {Sz, self()}).
 
 inc_pipeline_counter(#state{is_closing = true} = State) ->
     State;

lib/ibrowse/src/ibrowse_test.erl

@@ -4,7 +4,7 @@
 %%% Created : 14 Oct 2003 by Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
 
 -module(ibrowse_test).
--vsn('$Id: ibrowse_test.erl,v 1.2 2008/03/27 01:35:50 chandrusf Exp $ ').
+-vsn('$Id: ibrowse_test.erl,v 1.3 2008/05/21 15:28:11 chandrusf Exp $ ').
 -export([
 	 load_test/3,
 	 send_reqs_1/3,
@@ -32,6 +32,7 @@ send_reqs_1(Url, NumWorkers, NumReqsPerWorker) ->
     Start_time = now(),
     ets:new(pid_table, [named_table, public]),
     ets:new(ibrowse_test_results, [named_table, public]),
+    ets:new(ibrowse_errors, [named_table, public, ordered_set]),
     init_results(),
     process_flag(trap_exit, true),
     log_msg("Starting spawning of workers...~n", []),
@@ -45,15 +46,20 @@ send_reqs_1(Url, NumWorkers, NumReqsPerWorker) ->
     log_msg("End time  : ~1000.p~n", [calendar:now_to_local_time(End_time)]),
     Elapsed_time_secs = trunc(timer:now_diff(End_time, Start_time) / 1000000),
     log_msg("Elapsed   : ~p~n", [Elapsed_time_secs]),
-    log_msg("Reqs/sec  : ~p~n", [(NumWorkers*NumReqsPerWorker) / Elapsed_time_secs]).
+    log_msg("Reqs/sec  : ~p~n", [(NumWorkers*NumReqsPerWorker) / Elapsed_time_secs]),
+    dump_errors().
 
 init_results() ->
     ets:insert(ibrowse_test_results, {crash, 0}),
     ets:insert(ibrowse_test_results, {send_failed, 0}),
     ets:insert(ibrowse_test_results, {other_error, 0}),
     ets:insert(ibrowse_test_results, {success, 0}),
+    ets:insert(ibrowse_test_results, {retry_later, 0}),
+    ets:insert(ibrowse_test_results, {trid_mismatch, 0}),
+    ets:insert(ibrowse_test_results, {success_no_trid, 0}),
     ets:insert(ibrowse_test_results, {failed, 0}),
-    ets:insert(ibrowse_test_results, {timeout, 0}).
+    ets:insert(ibrowse_test_results, {timeout, 0}),
+    ets:insert(ibrowse_test_results, {req_id, 0}).
 
 spawn_workers(_Url, 0, _) ->
     ok;
@@ -89,19 +95,54 @@ do_send_req(Url, NumReqs) ->
 do_send_req_1(_Url, 0) ->
     ets:delete(pid_table, self());
 do_send_req_1(Url, NumReqs) ->
-    case ibrowse:send_req(Url, [], get, [], [], 10000) of
-	{ok, _Status, _Headers, _Body} ->
-	    ets:update_counter(ibrowse_test_results, success, 1);
+    Counter = integer_to_list(ets:update_counter(ibrowse_test_results, req_id, 1)),
+    case ibrowse:send_req(Url, [{"ib_req_id", Counter}], get, [], [], 10000) of
+	{ok, _Status, Headers, _Body} ->
+	    case lists:keysearch("ib_req_id", 1, Headers) of
+		{value, {_, Counter}} ->
+		    ets:update_counter(ibrowse_test_results, success, 1);
+		{value, _} ->
+		    ets:update_counter(ibrowse_test_results, trid_mismatch, 1);
+		false ->
+		    ets:update_counter(ibrowse_test_results, success_no_trid, 1)
+	    end;
 	{error, req_timedout} ->
 	    ets:update_counter(ibrowse_test_results, timeout, 1);
 	{error, send_failed} ->
 	    ets:update_counter(ibrowse_test_results, send_failed, 1);
-	_Err ->
+	{error, retry_later} ->
+	    ets:update_counter(ibrowse_test_results, retry_later, 1);
+	Err ->
+	    ets:insert(ibrowse_errors, {now(), Err}),
 	    ets:update_counter(ibrowse_test_results, other_error, 1),
 	    ok
     end,
     do_send_req_1(Url, NumReqs-1).
 
+dump_errors() ->
+    case ets:info(ibrowse_errors, size) of
+	0 ->
+	    ok;
+	_ ->
+	    {A, B, C} = now(),
+	    Filename = lists:flatten(
+			 io_lib:format("ibrowse_errors_~p_~p_~p.txt" , [A, B, C])),
+	    case file:open(Filename, [write, delayed_write, raw]) of
+		{ok, Iod} ->
+		    dump_errors(ets:first(ibrowse_errors), Iod);
+		Err ->
+		    io:format("failed to create file ~s. Reason: ~p~n", [Filename, Err]),
+		    ok
+	    end
+    end.
+
+dump_errors('$end_of_table', Iod) ->
+    file:close(Iod);
+dump_errors(Key, Iod) ->
+    [{_, Term}] = ets:lookup(ibrowse_errors, Key),
+    file:write(Iod, io_lib:format("~p~n", [Term])),
+    dump_errors(ets:next(ibrowse_errors, Key), Iod).
+
 %%------------------------------------------------------------------------------
 %% Unit Tests
 %%------------------------------------------------------------------------------

lib/ibrowse/vsn.mk

@@ -1,2 +1,2 @@
-IBROWSE_VSN = 1.4
+IBROWSE_VSN = 1.4.1