Skip to content
Browse files

A mofidied version of a patch sent in by Ram Krishnan to enhance the …

…save_response_to_file option
  • Loading branch information...
1 parent 0ada9cb commit 5cc53c0e15d1ac75a3578daa68356f7f9cfefc61 chandrusf committed Feb 7, 2008
Showing with 271 additions and 139 deletions.
  1. +6 −0 lib/ibrowse/README
  2. +140 −75 lib/ibrowse/doc/ibrowse.html
  3. +16 −2 lib/ibrowse/src/ibrowse.erl
  4. +108 −61 lib/ibrowse/src/ibrowse_http_client.erl
  5. +1 −1 lib/ibrowse/vsn.mk
View
6 lib/ibrowse/README
@@ -22,6 +22,12 @@ Comments to : Chandrashekhar.Mullaparthi@t-mobile.co.uk
CONTRIBUTIONS & CHANGE HISTORY
==============================
+07-02-2008 - Ram Krishnan (kriyative _at_ gmail dot com) sent a simple patch to
+ enable specifying the filename in the save_response_to_file option.
+ When testing the patch, I realised that my original implementation
+ of this feature was quite flaky and a lot of corner cases were
+ not covered. Fixed all of them. Thanks Ram!
+
17-10-2007 - Matthew Reilly (matthew dot reilly _at_ sipphone dot com)
sent a bug report and a fix. If the chunk trailer spans two TCP
packets, then ibrowse fails to recognise that the chunked transfer
View
215 lib/ibrowse/doc/ibrowse.html
@@ -2,14 +2,16 @@
<html>
<head>
<title>Module ibrowse</title>
-<link rel="stylesheet" type="text/css" href="stylesheet.css">
+<link rel="stylesheet" type="text/css" href="stylesheet.css" title="EDoc">
</head>
<body bgcolor="white">
+<div class="navbar"><a name="#navbar_top"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<hr>
<h1>Module ibrowse</h1>
-The ibrowse application implements an HTTP 1.1 client.
+<ul class="index"><li><a href="#description">Description</a></li><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>The ibrowse application implements an HTTP 1.1 client.
<p>Copyright © 2005-2007 Chandrashekhar Mullaparthi</p>
-<ul><li><a href="#description">Description</a></li><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>
+
<p><b>Version:</b> 1.2.7</p>
<p><b>Behaviours:</b> <a href="gen_server.html"><tt>gen_server</tt></a>.</p>
<p><b>Authors:</b> Chandrashekhar Mullaparthi (<a href="mailto:chandrashekhar dot mullaparthi at gmail dot com"><tt>chandrashekhar dot mullaparthi at gmail dot com</tt></a>).</p>
@@ -66,23 +68,23 @@
speed achieved using only erlang has been good enough, so the
driver isn't actually used.</p>
<h2><a name="index">Function Index</a></h2>
-<table width="100%" border="1"><tr><td valign="top"><a href="#code_change-3">code_change/3</a></td><td/></tr>
+<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#code_change-3">code_change/3</a></td><td></td></tr>
<tr><td valign="top"><a href="#finished_async_request-0">finished_async_request/0</a></td><td>Internal export.</td></tr>
-<tr><td valign="top"><a href="#handle_call-3">handle_call/3</a></td><td/></tr>
-<tr><td valign="top"><a href="#handle_cast-2">handle_cast/2</a></td><td/></tr>
-<tr><td valign="top"><a href="#handle_info-2">handle_info/2</a></td><td/></tr>
-<tr><td valign="top"><a href="#init-1">init/1</a></td><td/></tr>
+<tr><td valign="top"><a href="#handle_call-3">handle_call/3</a></td><td></td></tr>
+<tr><td valign="top"><a href="#handle_cast-2">handle_cast/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#handle_info-2">handle_info/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#init-1">init/1</a></td><td></td></tr>
<tr><td valign="top"><a href="#reply-2">reply/2</a></td><td>Internal export.</td></tr>
<tr><td valign="top"><a href="#send_req-3">send_req/3</a></td><td>This is the basic function to send a HTTP request.</td></tr>
<tr><td valign="top"><a href="#send_req-4">send_req/4</a></td><td>Same as send_req/3.</td></tr>
<tr><td valign="top"><a href="#send_req-5">send_req/5</a></td><td>Same as send_req/4.</td></tr>
<tr><td valign="top"><a href="#send_req-6">send_req/6</a></td><td>Same as send_req/5.</td></tr>
<tr><td valign="top"><a href="#set_dest-3">set_dest/3</a></td><td>Sets options for a destination.</td></tr>
<tr><td valign="top"><a href="#shutting_down-0">shutting_down/0</a></td><td>Internal export.</td></tr>
-<tr><td valign="top"><a href="#start-0">start/0</a></td><td/></tr>
-<tr><td valign="top"><a href="#start_link-0">start_link/0</a></td><td/></tr>
-<tr><td valign="top"><a href="#stop-0">stop/0</a></td><td/></tr>
-<tr><td valign="top"><a href="#terminate-2">terminate/2</a></td><td/></tr>
+<tr><td valign="top"><a href="#start-0">start/0</a></td><td></td></tr>
+<tr><td valign="top"><a href="#start_link-0">start_link/0</a></td><td></td></tr>
+<tr><td valign="top"><a href="#stop-0">stop/0</a></td><td></td></tr>
+<tr><td valign="top"><a href="#terminate-2">terminate/2</a></td><td></td></tr>
<tr><td valign="top"><a href="#trace_off-0">trace_off/0</a></td><td>Turn tracing off for the ibrowse process.</td></tr>
<tr><td valign="top"><a href="#trace_off-2">trace_off/2</a></td><td>Turn tracing OFF for all connections to the specified HTTP
server.</td></tr>
@@ -93,54 +95,86 @@
<h2><a name="functions">Function Details</a></h2>
-<h3><a name="code_change-3">code_change/3</a></h3>
-<tt>code_change(OldVsn, State, Extra) -&gt; term()
-</tt>
+<h3 class="function"><a name="code_change-3">code_change/3</a></h3>
+<div class="spec">
+<p><tt>code_change() -&gt; term()</tt></p>
+</div>
-<h3><a name="finished_async_request-0">finished_async_request/0</a></h3>
-<tt>finished_async_request() -&gt; term()
-</tt><p>Internal export. Called by a HTTP connection process to
+<h3 class="function"><a name="finished_async_request-0">finished_async_request/0</a></h3>
+<div class="spec">
+<p><tt>finished_async_request() -&gt; term()</tt></p>
+</div><p>Internal export. Called by a HTTP connection process to
indicate to the load balancing process (ibrowse) that an
asynchronous request has finished processing.</p>
-<h3><a name="handle_call-3">handle_call/3</a></h3>
-<tt>handle_call(Req, From, State) -&gt; term()
-</tt>
+<h3 class="function"><a name="handle_call-3">handle_call/3</a></h3>
+<div class="spec">
+<p><tt>handle_call() -&gt; term()</tt></p>
+</div>
-<h3><a name="handle_cast-2">handle_cast/2</a></h3>
-<tt>handle_cast(Msg, State) -&gt; term()
-</tt>
+<h3 class="function"><a name="handle_cast-2">handle_cast/2</a></h3>
+<div class="spec">
+<p><tt>handle_cast() -&gt; term()</tt></p>
+</div>
-<h3><a name="handle_info-2">handle_info/2</a></h3>
-<tt>handle_info(Info, State) -&gt; term()
-</tt>
+<h3 class="function"><a name="handle_info-2">handle_info/2</a></h3>
+<div class="spec">
+<p><tt>handle_info() -&gt; term()</tt></p>
+</div>
-<h3><a name="init-1">init/1</a></h3>
-<tt>init(X1) -&gt; term()
-</tt>
+<h3 class="function"><a name="init-1">init/1</a></h3>
+<div class="spec">
+<p><tt>init() -&gt; term()</tt></p>
+</div>
-<h3><a name="reply-2">reply/2</a></h3>
-<tt>reply(OrigCaller, Reply) -&gt; term()
-</tt><p>Internal export. Called by a HTTP connection process to
+<h3 class="function"><a name="reply-2">reply/2</a></h3>
+<div class="spec">
+<p><tt>reply() -&gt; term()</tt></p>
+</div><p>Internal export. Called by a HTTP connection process to
indicate to the load balancing process (ibrowse) that a synchronous
request has finished processing.</p>
-<h3><a name="send_req-3">send_req/3</a></h3>
+<h3 class="function"><a name="send_req-3">send_req/3</a></h3>
+<div class="spec">
<p><tt>send_req(Url::string(), Headers::<a href="#type-headerList">headerList()</a>, Method::<a href="#type-method">method()</a>) -&gt; <a href="#type-response">response()</a></tt>
-<ul><li><tt><a name="type-headerList">headerList()</a> = [{<a href="#type-header">header()</a>, <a href="#type-value">value()</a>}]</tt></li><li><tt><a name="type-header">header()</a> = atom() | string()</tt></li><li><tt><a name="type-value">value()</a> = term()</tt></li><li><tt><a name="type-method">method()</a> = get | post | head | options | put | delete | trace | mkcol | propfind | proppatch | lock | unlock | move | copy</tt></li><li><tt>Status = string()</tt></li><li><tt>ResponseHeaders = [<a href="#type-respHeader">respHeader()</a>]</tt></li><li><tt><a name="type-respHeader">respHeader()</a> = {<a href="#type-headerName">headerName()</a>, <a href="#type-headerValue">headerValue()</a>}</tt></li><li><tt><a name="type-headerName">headerName()</a> = string()</tt></li><li><tt><a name="type-headerValue">headerValue()</a> = string()</tt></li><li><tt><a name="type-response">response()</a> = {ok, Status, ResponseHeaders, ResponseBody} | {error, Reason}</tt></li><li><tt>ResponseBody = string()</tt></li><li><tt>Reason = term()</tt></li></ul></p>
-<p>This is the basic function to send a HTTP request.
+<ul class="definitions"><li><tt><a name="type-headerList">headerList()</a> = [{<a href="#type-header">header()</a>, <a href="#type-value">value()</a>}]</tt></li>
+<li><tt><a name="type-header">header()</a> = atom() | string()</tt></li>
+<li><tt><a name="type-value">value()</a> = term()</tt></li>
+<li><tt><a name="type-method">method()</a> = get | post | head | options | put | delete | trace | mkcol | propfind | proppatch | lock | unlock | move | copy</tt></li>
+<li><tt>Status = string()</tt></li>
+<li><tt>ResponseHeaders = [<a href="#type-respHeader">respHeader()</a>]</tt></li>
+<li><tt><a name="type-respHeader">respHeader()</a> = {<a href="#type-headerName">headerName()</a>, <a href="#type-headerValue">headerValue()</a>}</tt></li>
+<li><tt><a name="type-headerName">headerName()</a> = string()</tt></li>
+<li><tt><a name="type-headerValue">headerValue()</a> = string()</tt></li>
+<li><tt><a name="type-response">response()</a> = {ok, Status, ResponseHeaders, ResponseBody} | {error, Reason}</tt></li>
+<li><tt>ResponseBody = string() | {file, Filename}</tt></li>
+<li><tt>Reason = term()</tt></li>
+</ul></p>
+</div><p>This is the basic function to send a HTTP request.
The Status return value indicates the HTTP status code returned by the webserver</p>
-<h3><a name="send_req-4">send_req/4</a></h3>
+<h3 class="function"><a name="send_req-4">send_req/4</a></h3>
+<div class="spec">
<p><tt>send_req(Url, Headers, Method::<a href="#type-method">method()</a>, Body::<a href="#type-body">body()</a>) -&gt; <a href="#type-response">response()</a></tt>
-<ul><li><tt><a name="type-body">body()</a> = [] | string() | binary()</tt></li></ul></p>
-<p>Same as send_req/3.
+<ul class="definitions"><li><tt><a name="type-body">body()</a> = [] | string() | binary()</tt></li>
+</ul></p>
+</div><p>Same as send_req/3.
If a list is specified for the body it has to be a flat list.</p>
-<h3><a name="send_req-5">send_req/5</a></h3>
+<h3 class="function"><a name="send_req-5">send_req/5</a></h3>
+<div class="spec">
<p><tt>send_req(Url::string(), Headers::<a href="#type-headerList">headerList()</a>, Method::<a href="#type-method">method()</a>, Body::<a href="#type-body">body()</a>, Options::<a href="#type-optionList">optionList()</a>) -&gt; <a href="#type-response">response()</a></tt>
-<ul><li><tt><a name="type-optionList">optionList()</a> = [<a href="#type-option">option()</a>]</tt></li><li><tt><a name="type-option">option()</a> = {max_sessions, integer()} | {max_pipeline_size, integer()} | {trace, <a href="#type-boolean">boolean()</a>} | {is_ssl, <a href="#type-boolean">boolean()</a>} | {ssl_options, [SSLOpt]} | {pool_name, atom()} | {proxy_host, string()} | {proxy_port, integer()} | {proxy_user, string()} | {proxy_password, string()} | {use_absolute_uri, <a href="#type-boolean">boolean()</a>} | {basic_auth, {<a href="#type-username">username()</a>, <a href="#type-password">password()</a>}} | {cookie, string()} | {content_length, integer()} | {content_type, string()} | {save_response_to_file, <a href="#type-boolean">boolean()</a>} | {stream_to, <a href="#type-process">process()</a>} | {http_vsn, {MajorVsn, MinorVsn}} | {host_header, string()} | {transfer_encoding, {chunked, ChunkSize}}</tt></li><li><tt><a name="type-process">process()</a> = pid() | atom()</tt></li><li><tt><a name="type-username">username()</a> = string()</tt></li><li><tt><a name="type-password">password()</a> = string()</tt></li><li><tt>SSLOpt = term()</tt></li><li><tt>ChunkSize = integer()</tt></li></ul></p>
-<p>Same as send_req/4.
+<ul class="definitions"><li><tt><a name="type-optionList">optionList()</a> = [<a href="#type-option">option()</a>]</tt></li>
+<li><tt><a name="type-option">option()</a> = {max_sessions, integer()} | {max_pipeline_size, integer()} | {trace, <a href="#type-boolean">boolean()</a>} | {is_ssl, <a href="#type-boolean">boolean()</a>} | {ssl_options, [SSLOpt]} | {pool_name, atom()} | {proxy_host, string()} | {proxy_port, integer()} | {proxy_user, string()} | {proxy_password, string()} | {use_absolute_uri, <a href="#type-boolean">boolean()</a>} | {basic_auth, {<a href="#type-username">username()</a>, <a href="#type-password">password()</a>}} | {cookie, string()} | {content_length, integer()} | {content_type, string()} | {save_response_to_file, <a href="#type-srtf">srtf()</a>} | {stream_to, <a href="#type-process">process()</a>} | {http_vsn, {MajorVsn, MinorVsn}} | {host_header, string()} | {transfer_encoding, {chunked, ChunkSize}}</tt></li>
+<li><tt><a name="type-process">process()</a> = pid() | atom()</tt></li>
+<li><tt><a name="type-username">username()</a> = string()</tt></li>
+<li><tt><a name="type-password">password()</a> = string()</tt></li>
+<li><tt>SSLOpt = term()</tt></li>
+<li><tt>ChunkSize = integer()</tt></li>
+<li><tt><a name="type-srtf">srtf()</a> = <a href="#type-boolean">boolean()</a> | <a href="#type-filename">filename()</a></tt></li>
+<li><tt><a name="type-filename">filename()</a> = string()</tt></li>
+</ul></p>
+</div><p>Same as send_req/4.
For a description of SSL Options, look in the ssl manpage. If the
HTTP Version to use is not specified, the default is 1.1.
<br>
@@ -150,18 +184,34 @@
ibrowse must have the stunnel host/port details, but that won't
make sense to the destination webserver. This option can then be
used to specify what should go in the <code>Host</code> header in
- the request.</p></p>
+ the request.</p>
+ <ul>
+ <li>When both the options <code>save_response_to_file</code> and <code>stream_to</code>
+ are specified, the former takes precedence.</li>
+
+ <li>For the <code>save_response_to_file</code> option, the response body is saved to
+ file only if the status code is in the 200-299 range. If not, the response body is returned
+ as a string.</li>
+ <li>Whenever an error occurs in the processing of a request, ibrowse will return as much
+ information as it has, such as HTTP Status Code and HTTP Headers. When this happens, the response
+ is of the form <code>{error, {Reason, {stat_code, StatusCode}, HTTP_headers}}</code></li>
+ </ul></p>
-<h3><a name="send_req-6">send_req/6</a></h3>
+<h3 class="function"><a name="send_req-6">send_req/6</a></h3>
+<div class="spec">
<p><tt>send_req(Url, Headers::<a href="#type-headerList">headerList()</a>, Method::<a href="#type-method">method()</a>, Body::<a href="#type-body">body()</a>, Options::<a href="#type-optionList">optionList()</a>, Timeout) -&gt; <a href="#type-response">response()</a></tt>
-<ul><li><tt>Timeout = integer() | infinity</tt></li></ul></p>
-<p>Same as send_req/5.
+<ul class="definitions"><li><tt>Timeout = integer() | infinity</tt></li>
+</ul></p>
+</div><p>Same as send_req/5.
All timeout values are in milliseconds.</p>
-<h3><a name="set_dest-3">set_dest/3</a></h3>
+<h3 class="function"><a name="set_dest-3">set_dest/3</a></h3>
+<div class="spec">
<p><tt>set_dest(Host::string(), Port::integer(), Opts::<a href="#type-opt_list">opt_list()</a>) -&gt; ok</tt>
-<ul><li><tt><a name="type-opt_list">opt_list()</a> = [opt]</tt></li><li><tt><a name="type-opt">opt()</a> = {max_sessions, integer()} | {max_pipeline_size, integer()} | {trace, <a href="#type-boolean">boolean()</a>}</tt></li></ul></p>
-<p>Sets options for a destination. If the options have not been
+<ul class="definitions"><li><tt><a name="type-opt_list">opt_list()</a> = [opt]</tt></li>
+<li><tt><a name="type-opt">opt()</a> = {max_sessions, integer()} | {max_pipeline_size, integer()} | {trace, <a href="#type-boolean">boolean()</a>}</tt></li>
+</ul></p>
+</div><p>Sets options for a destination. If the options have not been
set in the ibrowse.conf file, it can be set using this function
before sending the first request to the destination. If not,
defaults will be used. Entries in ibrowse.conf look like this.
@@ -175,45 +225,60 @@
Options = optionList() -- see options in send_req/5<br>
</code></p>
-<h3><a name="shutting_down-0">shutting_down/0</a></h3>
-<tt>shutting_down() -&gt; term()
-</tt><p>Internal export. Called by a HTTP connection process to
+<h3 class="function"><a name="shutting_down-0">shutting_down/0</a></h3>
+<div class="spec">
+<p><tt>shutting_down() -&gt; term()</tt></p>
+</div><p>Internal export. Called by a HTTP connection process to
indicate to ibrowse that it is shutting down and further requests
should not be sent it's way.</p>
-<h3><a name="start-0">start/0</a></h3>
-<tt>start() -&gt; term()
-</tt>
+<h3 class="function"><a name="start-0">start/0</a></h3>
+<div class="spec">
+<p><tt>start() -&gt; term()</tt></p>
+</div>
-<h3><a name="start_link-0">start_link/0</a></h3>
-<tt>start_link() -&gt; term()
-</tt>
+<h3 class="function"><a name="start_link-0">start_link/0</a></h3>
+<div class="spec">
+<p><tt>start_link() -&gt; term()</tt></p>
+</div>
-<h3><a name="stop-0">stop/0</a></h3>
-<tt>stop() -&gt; term()
-</tt>
+<h3 class="function"><a name="stop-0">stop/0</a></h3>
+<div class="spec">
+<p><tt>stop() -&gt; term()</tt></p>
+</div>
-<h3><a name="terminate-2">terminate/2</a></h3>
-<tt>terminate(Reason, State) -&gt; term()
-</tt>
+<h3 class="function"><a name="terminate-2">terminate/2</a></h3>
+<div class="spec">
+<p><tt>terminate() -&gt; term()</tt></p>
+</div>
-<h3><a name="trace_off-0">trace_off/0</a></h3>
-<tt>trace_off() -&gt; term()
-</tt><p>Turn tracing off for the ibrowse process</p>
+<h3 class="function"><a name="trace_off-0">trace_off/0</a></h3>
+<div class="spec">
+<p><tt>trace_off() -&gt; term()</tt></p>
+</div><p>Turn tracing off for the ibrowse process</p>
-<h3><a name="trace_off-2">trace_off/2</a></h3>
+<h3 class="function"><a name="trace_off-2">trace_off/2</a></h3>
+<div class="spec">
<p><tt>trace_off(Host, Port) -&gt; term()</tt></p>
-<p>Turn tracing OFF for all connections to the specified HTTP
+</div><p>Turn tracing OFF for all connections to the specified HTTP
server.</p>
-<h3><a name="trace_on-0">trace_on/0</a></h3>
-<tt>trace_on() -&gt; term()
-</tt><p>Turn tracing on for the ibrowse process</p>
+<h3 class="function"><a name="trace_on-0">trace_on/0</a></h3>
+<div class="spec">
+<p><tt>trace_on() -&gt; term()</tt></p>
+</div><p>Turn tracing on for the ibrowse process</p>
-<h3><a name="trace_on-2">trace_on/2</a></h3>
+<h3 class="function"><a name="trace_on-2">trace_on/2</a></h3>
+<div class="spec">
<p><tt>trace_on(Host, Port) -&gt; term()</tt>
-<ul><li><tt>Host = string()</tt></li><li><tt>Port = integer()</tt></li></ul></p>
-<p>Turn tracing on for all connections to the specified HTTP
+<ul class="definitions"><li><tt>Host = string()</tt></li>
+<li><tt>Port = integer()</tt></li>
+</ul></p>
+</div><p>Turn tracing on for all connections to the specified HTTP
server. Host is whatever is specified as the domain name in the URL</p>
+<hr>
+
+<div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<p><i>Generated by EDoc, Feb 7 2008, 11:49:30.</i></p>
</body>
</html>
View
18 lib/ibrowse/src/ibrowse.erl
@@ -167,7 +167,7 @@ set_dest(Host,Port,Opts) ->
%% headerName() = string()
%% headerValue() = string()
%% response() = {ok, Status, ResponseHeaders, ResponseBody} | {error, Reason}
-%% ResponseBody = string()
+%% ResponseBody = string() | {file, Filename}
%% Reason = term()
send_req(Url, Headers, Method) ->
send_req(Url, Headers, Method, [], []).
@@ -191,6 +191,17 @@ send_req(Url, Headers, Method, Body) ->
%% make sense to the destination webserver. This option can then be
%% used to specify what should go in the <code>Host</code> header in
%% the request.</p>
+%% <ul>
+%% <li>When both the options <code>save_response_to_file</code> and <code>stream_to</code>
+%% are specified, the former takes precedence.</li>
+%%
+%% <li>For the <code>save_response_to_file</code> option, the response body is saved to
+%% file only if the status code is in the 200-299 range. If not, the response body is returned
+%% as a string.</li>
+%% <li>Whenever an error occurs in the processing of a request, ibrowse will return as much
+%% information as it has, such as HTTP Status Code and HTTP Headers. When this happens, the response
+%% is of the form <code>{error, {Reason, {stat_code, StatusCode}, HTTP_headers}}</code></li>
+%% </ul>
%% @spec send_req(Url::string(), Headers::headerList(), Method::method(), Body::body(), Options::optionList()) -> response()
%% optionList() = [option()]
%% option() = {max_sessions, integer()} |
@@ -208,7 +219,7 @@ send_req(Url, Headers, Method, Body) ->
%% {cookie, string()} |
%% {content_length, integer()} |
%% {content_type, string()} |
-%% {save_response_to_file, boolean()} |
+%% {save_response_to_file, srtf()} |
%% {stream_to, process()} |
%% {http_vsn, {MajorVsn, MinorVsn}} |
%% {host_header, string()} |
@@ -219,6 +230,9 @@ send_req(Url, Headers, Method, Body) ->
%% password() = string()
%% SSLOpt = term()
%% ChunkSize = integer()
+%% srtf() = boolean() | filename()
+%% filename() = string()
+%%
send_req(Url, Headers, Method, Body, Options) ->
send_req(Url, Headers, Method, Body, Options, 30000).
View
169 lib/ibrowse/src/ibrowse_http_client.erl
@@ -35,11 +35,12 @@
reply_buffer=[], rep_buf_size=0, recvd_headers=[],
is_closing, send_timer, content_length,
deleted_crlf = false, transfer_encoding, chunk_size,
- chunks=[], save_response_to_file = false,
- tmp_file_name, tmp_file_fd}).
+ chunks=[]}).
-record(request, {url, method, options, from,
- stream_to, req_id}).
+ stream_to, req_id,
+ save_response_to_file = false,
+ tmp_file_name, tmp_file_fd}).
%%====================================================================
%% External functions
@@ -122,19 +123,19 @@ handle_info({{send_req, [Url, Headers, Method,
_PHost ->
ProxyUser = get_value(proxy_user, Options, []),
ProxyPassword = get_value(proxy_password, Options, []),
- SaveResponseToFile = get_value(save_response_to_file, Options, false),
Digest = http_auth_digest(ProxyUser, ProxyPassword),
State#state{use_proxy = true,
- save_response_to_file = SaveResponseToFile,
proxy_auth_digest = Digest}
end,
StreamTo = get_value(stream_to, Options, undefined),
ReqId = make_req_id(),
+ SaveResponseToFile = get_value(save_response_to_file, Options, false),
NewReq = #request{url=Url,
method=Method,
stream_to=StreamTo,
options=Options,
req_id=ReqId,
+ save_response_to_file = SaveResponseToFile,
from=From},
Reqs = queue:in(NewReq, State#state.reqs),
State_2 = check_ssl_options(Options, State_1#state{reqs = Reqs}),
@@ -185,12 +186,14 @@ handle_info({{send_req, [Url, Headers, Method,
#state{socket=Sock, status=Status, reqs=Reqs}=State) ->
do_trace("Recvd request in connected state. Status -> ~p NumPending: ~p~n", [Status, length(queue:to_list(Reqs))]),
StreamTo = get_value(stream_to, Options, undefined),
+ SaveResponseToFile = get_value(save_response_to_file, Options, false),
ReqId = make_req_id(),
NewReq = #request{url=Url,
stream_to=StreamTo,
method=Method,
options=Options,
req_id=ReqId,
+ save_response_to_file = SaveResponseToFile,
from=From},
State_1 = State#state{reqs=queue:in(NewReq, State#state.reqs)},
case send_req_1(Url, Headers, Method, Body, Options, Sock, State_1) of
@@ -309,27 +312,27 @@ handle_sock_data(Data, #state{status=get_header, socket=Sock}=State) ->
end;
handle_sock_data(Data, #state{status=get_body, content_length=CL,
- recvd_headers=Headers, cur_req=CurReq,
- chunk_size=CSz, reqs=Reqs, socket=Sock}=State) ->
+ http_status_code = StatCode,
+ recvd_headers=Headers,
+ chunk_size=CSz, socket=Sock}=State) ->
case (CL == undefined) and (CSz == undefined) of
true ->
case accumulate_response(Data, State) of
{error, Reason} ->
ibrowse:shutting_down(),
- {_, Reqs_1} = queue:out(Reqs),
- #request{from=From, stream_to=StreamTo, req_id=ReqId} = CurReq,
- do_reply(From, StreamTo, ReqId,
- {error, {file_open_error, Reason, Headers}}),
- do_error_reply(State#state{reqs=Reqs_1}, previous_request_failed),
+ fail_pipelined_requests(State,
+ {error, {Reason, {stat_code, StatCode}, Headers}}),
{stop, normal, State};
State_1 ->
do_setopts(Sock, [{active, true}], State#state.is_ssl),
{noreply, State_1}
end;
_ ->
case parse_11_response(Data, State) of
- {error, _Reason} ->
+ {error, Reason} ->
ibrowse:shutting_down(),
+ fail_pipelined_requests(State,
+ {error, {Reason, {stat_code, StatCode}, Headers}}),
{stop, normal, State};
stop ->
ibrowse:shutting_down(),
@@ -340,32 +343,41 @@ handle_sock_data(Data, #state{status=get_body, content_length=CL,
end
end.
-accumulate_response(Data, #state{save_response_to_file = true,
- tmp_file_fd = undefined,
- http_status_code=[$2 | _]}=State) ->
- TmpFilename = make_tmp_filename(),
+accumulate_response(Data,
+ #state{
+ cur_req = #request{save_response_to_file = SaveResponseToFile,
+ tmp_file_fd = undefined} = CurReq,
+ http_status_code=[$2 | _]}=State) when SaveResponseToFile /= false ->
+ TmpFilename = case SaveResponseToFile of
+ true -> make_tmp_filename();
+ F -> F
+ end,
case file:open(TmpFilename, [write, delayed_write, raw]) of
{ok, Fd} ->
- accumulate_response(Data, State#state{tmp_file_fd=Fd,
- tmp_file_name=TmpFilename});
+ accumulate_response(Data, State#state{
+ cur_req = CurReq#request{
+ tmp_file_fd = Fd,
+ tmp_file_name = TmpFilename}});
{error, Reason} ->
{error, {file_open_error, Reason}}
end;
-accumulate_response(Data, #state{save_response_to_file=true,
+accumulate_response(Data, #state{cur_req = #request{save_response_to_file = SaveResponseToFile,
+ tmp_file_fd = Fd},
transfer_encoding=chunked,
chunks = Chunks,
- http_status_code=[$2 | _],
- tmp_file_fd=Fd}=State) ->
+ http_status_code=[$2 | _]
+ } = State) when SaveResponseToFile /= false ->
case file:write(Fd, [Chunks | Data]) of
ok ->
State#state{chunks = []};
{error, Reason} ->
{error, {file_write_error, Reason}}
end;
-accumulate_response(Data, #state{save_response_to_file=true,
+accumulate_response(Data, #state{cur_req = #request{save_response_to_file = SaveResponseToFile,
+ tmp_file_fd = Fd},
reply_buffer = RepBuf,
- http_status_code=[$2 | _],
- tmp_file_fd=Fd}=State) ->
+ http_status_code=[$2 | _]
+ } = State) when SaveResponseToFile /= false ->
case file:write(Fd, [RepBuf | Data]) of
ok ->
State#state{reply_buffer = []};
@@ -374,8 +386,8 @@ accumulate_response(Data, #state{save_response_to_file=true,
end;
accumulate_response([], State) ->
State;
-accumulate_response(Data, #state{reply_buffer=RepBuf,
- cur_req=CurReq}=State) ->
+accumulate_response(Data, #state{reply_buffer = RepBuf,
+ cur_req = CurReq}=State) ->
#request{stream_to=StreamTo, req_id=ReqId} = CurReq,
case StreamTo of
undefined ->
@@ -409,8 +421,9 @@ handle_sock_closed(#state{cur_req=undefined}) ->
%% Connection-Close header and has closed the socket to indicate end
%% of response. There maybe requests pipelined which need a response.
handle_sock_closed(#state{reply_buffer=Buf, reqs=Reqs, http_status_code=SC,
- is_closing=IsClosing, cur_req=CurReq,
- tmp_file_name=TmpFilename, tmp_file_fd=Fd,
+ is_closing=IsClosing,
+ cur_req=#request{tmp_file_name=TmpFilename,
+ tmp_file_fd=Fd} = CurReq,
status=get_body, recvd_headers=Headers}=State) ->
#request{from=From, stream_to=StreamTo, req_id=ReqId} = CurReq,
case IsClosing of
@@ -635,7 +648,7 @@ parse_response(_Data, #state{cur_req = undefined}=State) ->
parse_response(Data, #state{reply_buffer=Acc, reqs=Reqs,
cur_req=CurReq}=State) ->
#request{from=From, stream_to=StreamTo, req_id=ReqId,
- options = CurReqOptions, method=Method} = CurReq,
+ method=Method} = CurReq,
MaxHeaderSize = safe_get_env(ibrowse, max_headers_size, infinity),
case scan_header(Data, Acc) of
{yes, Headers, Data_1} ->
@@ -652,10 +665,7 @@ parse_response(Data, #state{reply_buffer=Acc, reqs=Reqs,
false ->
ok
end,
- SaveResponseToFile = get_value(save_response_to_file, CurReqOptions, false),
State_1 = State#state{recvd_headers=Headers_1, status=get_body,
- save_response_to_file = SaveResponseToFile,
- tmp_file_fd = undefined, tmp_file_name = undefined,
http_status_code=StatCode, is_closing=IsClosing},
put(conn_close, ConnClose),
TransferEncoding = to_lower(get_value("transfer-encoding", LCHeaders, "false")),
@@ -690,33 +700,47 @@ parse_response(Data, #state{reply_buffer=Acc, reqs=Reqs,
_ when TransferEncoding == "chunked" ->
do_trace("Chunked encoding detected...~n",[]),
send_async_headers(ReqId, StreamTo, StatCode, Headers_1),
- parse_11_response(Data_1, State_1#state{transfer_encoding=chunked,
- chunk_size=chunk_start,
- reply_buffer=[], chunks=[]});
+ case parse_11_response(Data_1, State_1#state{transfer_encoding=chunked,
+ chunk_size=chunk_start,
+ reply_buffer=[], chunks=[]}) of
+ {error, Reason} ->
+ fail_pipelined_requests(State_1,
+ {error, {Reason,
+ {stat_code, StatCode}, Headers_1}}),
+ {error, Reason};
+ State_2 ->
+ State_2
+ end;
undefined when HttpVsn == "HTTP/1.0";
ConnClose == "close" ->
send_async_headers(ReqId, StreamTo, StatCode, Headers_1),
State_1#state{reply_buffer=[Data_1]};
undefined ->
- {_, Reqs_1} = queue:out(Reqs),
- do_reply(From, StreamTo, ReqId,
- {error, {content_length_undefined, Headers}}),
- do_error_reply(State_1#state{reqs=Reqs_1}, previous_request_failed),
+ fail_pipelined_requests(State_1,
+ {error, {content_length_undefined,
+ {stat_code, StatCode}, Headers}}),
{error, content_length_undefined};
V ->
case catch list_to_integer(V) of
V_1 when integer(V_1), V_1 >= 0 ->
send_async_headers(ReqId, StreamTo, StatCode, Headers_1),
do_trace("Recvd Content-Length of ~p~n", [V_1]),
- parse_11_response(Data_1,
- State_1#state{rep_buf_size=0,
- reply_buffer=[],
- content_length=V_1});
+ State_2 = State_1#state{rep_buf_size=0,
+ reply_buffer=[],
+ content_length=V_1},
+ case parse_11_response(Data_1, State_2) of
+ {error, Reason} ->
+ fail_pipelined_requests(State_1,
+ {error, {Reason,
+ {stat_code, StatCode}, Headers_1}}),
+ {error, Reason};
+ State_3 ->
+ State_3
+ end;
_ ->
- {_, Reqs_1} = queue:out(Reqs),
- do_reply(From, StreamTo, ReqId,
- {error, {content_length_undefined, Headers}}),
- do_error_reply(State_1#state{reqs=Reqs_1}, previous_request_failed),
+ fail_pipelined_requests(State_1,
+ {error, {content_length_undefined,
+ {stat_code, StatCode}, Headers}}),
{error, content_length_undefined}
end
end;
@@ -725,9 +749,7 @@ parse_response(Data, #state{reply_buffer=Acc, reqs=Reqs,
{no, Acc_1} when length(Acc_1) < MaxHeaderSize ->
State#state{reply_buffer=Acc_1};
{no, _Acc_1} ->
- do_reply(From, StreamTo, ReqId, {error, max_headers_size_exceeded}),
- {_, Reqs_1} = queue:out(Reqs),
- do_error_reply(State#state{reqs=Reqs_1}, previous_request_failed),
+ fail_pipelined_requests(State, {error, max_headers_size_exceeded}),
{error, max_headers_size_exceeded}
end.
@@ -835,6 +857,7 @@ parse_11_response(DataRecvd,
do_trace("RemData -> ~p~n", [RemData]),
case accumulate_response(RemChunk, State) of
{error, Reason} ->
+ do_trace("Error accumulating response --> ~p~n", [Reason]),
{error, Reason};
#state{reply_buffer = NewRepBuf,
chunks = NewChunks} = State_1 ->
@@ -867,18 +890,34 @@ parse_11_response(DataRecvd,
accumulate_response(DataRecvd, State#state{rep_buf_size=RepBufSz+DataLen})
end.
-handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId},
- #state{save_response_to_file = true,
- http_status_code=SCode,
- tmp_file_name=TmpFilename,
- tmp_file_fd=Fd,
- send_timer=ReqTimer,
- recvd_headers = RespHeaders}=State) ->
+handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
+ save_response_to_file = SaveResponseToFile,
+ tmp_file_name = TmpFilename,
+ tmp_file_fd = Fd
+ },
+ #state{http_status_code = SCode,
+ send_timer = ReqTimer,
+ reply_buffer = RepBuf,
+ transfer_encoding = TEnc,
+ chunks = Chunks,
+ recvd_headers = RespHeaders}=State) when SaveResponseToFile /= false ->
+ Body = case TEnc of
+ chunked ->
+ lists:flatten(lists:reverse(Chunks));
+ _ ->
+ lists:flatten(lists:reverse(RepBuf))
+ end,
State_1 = set_cur_request(State),
file:close(Fd),
- do_reply(From, StreamTo, ReqId, {ok, SCode, RespHeaders, {file, TmpFilename}}),
+ ResponseBody = case TmpFilename of
+ undefined ->
+ Body;
+ _ ->
+ {file, TmpFilename}
+ end,
+ do_reply(From, StreamTo, ReqId, {ok, SCode, RespHeaders, ResponseBody}),
cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}),
- State_1#state{tmp_file_name=undefined, tmp_file_fd=undefined};
+ State_1;
handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId},
#state{http_status_code=SCode, recvd_headers=RespHeaders,
reply_buffer=RepBuf, transfer_encoding=TEnc,
@@ -1193,7 +1232,14 @@ do_error_reply(#state{reqs = Reqs}, Err) ->
lists:foreach(fun(#request{from=From, stream_to=StreamTo, req_id=ReqId}) ->
do_reply(From, StreamTo, ReqId, {error, Err})
end, ReqList).
-
+
+fail_pipelined_requests(#state{reqs = Reqs, cur_req = CurReq} = State, Reply) ->
+ {_, Reqs_1} = queue:out(Reqs),
+ #request{from=From, stream_to=StreamTo, req_id=ReqId} = CurReq,
+ do_reply(From, StreamTo, ReqId, Reply),
+ do_error_reply(State#state{reqs = Reqs_1}, previous_request_failed).
+
+
split_list_at(List, N) ->
split_list_at(List, N, []).
split_list_at([], _, Acc) ->
@@ -1308,3 +1354,4 @@ to_lower([H|T], Acc) ->
to_lower(T, [H|Acc]);
to_lower([], Acc) ->
lists:reverse(Acc).
+
View
2 lib/ibrowse/vsn.mk
@@ -1,2 +1,2 @@
-IBROWSE_VSN = 1.2.9
+IBROWSE_VSN = 1.3.0

0 comments on commit 5cc53c0

Please sign in to comment.
Something went wrong with that request. Please try again.