diff --git a/src/yaws_revproxy.erl b/src/yaws_revproxy.erl
index 5ed69b58e..a6a10bb0a 100644
--- a/src/yaws_revproxy.erl
+++ b/src/yaws_revproxy.erl
@@ -38,7 +38,6 @@
%% TODO: Activate proxy keep-alive with a new option ?
-%% FIXME: Improve connection caching to enable this option
-define(proxy_keepalive, false).
@@ -203,7 +202,6 @@ out(Arg = #arg{state=RPState}) when RPState#revproxy.state == recvheaders ->
Resp#http_response.status =:= 205 orelse
Resp#http_response.status =:= 304 orelse
Resp#http_response.status =:= 406 ->
- %% FIXME: check all http codes
RPState2 = RPState1#revproxy{state=terminate},
out(Arg#arg{state=RPState2});
@@ -238,7 +236,7 @@ out(Arg = #arg{state=RPState}) when RPState#revproxy.state == recvheaders ->
%% The reponse content is not chunked.
-%% FIXME: use partial_post_size to split huge content and avoid memory
+%% TODO: use partial_post_size to split huge content and avoid memory
%% exhaustion.
out(Arg = #arg{state=RPState}) when RPState#revproxy.state == recvcontent ->
Len = list_to_integer((RPState#revproxy.headers)#headers.content_length),
@@ -377,7 +375,7 @@ recv_next_chunk(YawsPid, Arg = #arg{state=RPState}) ->
%%==========================================================================
-%% FIXME: find a better way to cache connections to backend servers. Here we can
+%% TODO: find a better way to cache connections to backend servers. Here we can
%% have 1 connection per gserv process for each backend server.
get_cached_connection(URL) ->
Key = lists:flatten(yaws_api:reformat_url(URL)),
@@ -446,7 +444,6 @@ do_connect(URL) ->
Err -> Err
end;
_ ->
- %% FIXME: catch this case in during config parsing
{error, unsupported_protocol}
end.
diff --git a/test/Makefile b/test/Makefile
index 13717a093..3cd8c2e25 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -1,6 +1,6 @@
include support/include.mk
-SUBDIRS = t1 t2 t3 eunit
+SUBDIRS = t1 t2 t3 t4 eunit
all: conf ibrowse
@cd src; $(MAKE) all
diff --git a/test/conf/revproxy.conf b/test/conf/revproxy.conf
new file mode 100644
index 000000000..4be843011
--- /dev/null
+++ b/test/conf/revproxy.conf
@@ -0,0 +1,143 @@
+
+
+logdir = ./logs
+
+# This the path to a directory where additional
+# beam code can be placed. The daemon will add this
+# directory to its search path
+
+ebin_dir = %YTOP%/test/ibrowse/ebin
+include_dir = %YTOP%/test/include
+
+
+
+# This is a debug variable, possible values are http | traffic | false
+# It is also possible to set the trace (possibly to the tty) while
+# invoking yaws from the shell as in
+# yaws -i -T -x (see man yaws)
+
+trace = false
+
+
+
+# it is possible to have yaws start additional
+# application specific code at startup
+#
+# runmod = mymodule
+
+
+# By default yaws will copy the erlang error_log and
+# end write it to a wrap log called report.log (in the logdir)
+# this feature can be turned off. This would typically
+# be the case when yaws runs within another larger app
+
+copy_error_log = true
+
+
+# Logs are wrap logs
+
+log_wrap_size = 1000000
+
+
+# Possibly resolve all hostnames in logfiles so webalizer
+# can produce the nice geography piechart
+
+log_resolve_hostname = false
+
+
+
+# fail completely or not if yaws fails
+# to bind a listen socket
+fail_on_bind_err = true
+
+
+
+# If yaws is started as root, it can, once it has opened
+# all relevant sockets for listening, change the uid to a
+# user with lower accessrights than root
+
+# username = nobody
+
+
+# If HTTP auth is used, it is possible to have a specific
+# auth log.
+# Deprecated and ignored. Now, this target must be set in server part
+#auth_log = true
+
+
+# When we're running multiple yaws systems on the same
+# host, we need to give each yaws system an individual
+# name. Yaws will write a number of runtime files under
+# /tmp/yaws/${id}
+# The default value is "default"
+
+
+# id = myname
+
+
+# earlier versions of Yaws picked the first virtual host
+# in a list of hosts with the same IP/PORT when the Host:
+# header doesn't match any name on any Host
+# This is often nice in testing environments but not
+# acceptable in real live hosting scenarios
+
+pick_first_virthost_on_nomatch = true
+
+
+# All unices are broken since it's not possible to bind to
+# a privileged port (< 1024) unless uid==0
+# There is a contrib in jungerl which makes it possible by means
+# of an external setuid root programm called fdsrv to listen to
+# to privileged port.
+# If we use this feature, it requires fdsrv to be properly installed.
+# Doesn't yet work with SSL.
+
+use_fdsrv = false
+
+
+
+
+# end then a set of virtual servers
+# First two virthosted servers on the same IP (0.0.0.0)
+# in this case, but an explicit IP can be given as well
+
+
+ port = 8000
+ listen = 0.0.0.0
+ listen_backlog = 512
+ deflate = true
+ docroot = %YTOP%/www
+ arg_rewrite_mod = rewritetest
+ revproxy = /revproxy1 http://localhost:8001
+ revproxy = /revproxy2 http://localhost:8002
+
+
+
+ port = 8001
+ listen = 0.0.0.0
+ listen_backlog = 512
+ deflate = true
+ partial_post_size = 2048000
+ appmods = posttest streamtest
+ docroot = %YTOP%/test/t4/www1
+
+
+
+ port = 8002
+ listen = 0.0.0.0
+ listen_backlog = 512
+ deflate = false
+ docroot = %YTOP%/test/t4/www2
+
+
+
+ port = 8003
+ listen = 0.0.0.0
+ listen_backlog = 512
+ deflate = false
+ fwdproxy = true
+ arg_rewrite_mod = rewritetest
+ docroot = %YTOP%/www
+
+
+
diff --git a/test/support/include.mk.in b/test/support/include.mk.in
index 070c85fdb..b7a4abd57 100644
--- a/test/support/include.mk.in
+++ b/test/support/include.mk.in
@@ -56,3 +56,7 @@ stdconf:
authconf:
cat ../conf/authconf.conf | \
../../scripts/Subst %YTOP% $(YTOP) > yaws.conf
+
+revproxyconf:
+ cat ../conf/revproxy.conf | \
+ ../../scripts/Subst %YTOP% $(YTOP) > yaws.conf
diff --git a/test/t4/Makefile b/test/t4/Makefile
new file mode 100644
index 000000000..49670e34d
--- /dev/null
+++ b/test/t4/Makefile
@@ -0,0 +1,38 @@
+include ../support/include.mk
+
+.PHONY: all test debug clean
+
+#
+all: conf setup app_test.beam rewritetest.beam posttest.beam streamtest.beam
+ @echo "all ok"
+
+
+# invoke as
+# TEST=test3 make test
+# or just make test to run all
+
+ULIMIT = 768
+
+test: conf start
+ dd if=/dev/zero of=../../www/1000.txt bs=1024 count=1000 >/dev/null 2>&1
+ dd if=/dev/zero of=../../www/2000.txt bs=1024 count=2000 >/dev/null 2>&1
+ dd if=/dev/zero of=../../www/3000.txt bs=1024 count=3000 >/dev/null 2>&1
+ dd if=/dev/zero of=../../www/10000.txt bs=1024 count=10000 >/dev/null 2>&1
+ ul=`ulimit -n` ; \
+ val=`expr $$ul '<' $(ULIMIT)` ; \
+ if [ $$val = 1 ] ; then \
+ echo trying to raise "ulimit -n" for the test... ; \
+ set -e ; \
+ ulimit -n $(ULIMIT) ; \
+ fi ; \
+ $(ERL) -noinput $(PA) -s tftest
+ $(MAKE) stop
+
+conf: revproxyconf
+
+debug:
+ $(ERL) $(PA)
+
+clean: tclean
+ -rm -f ../../www/1000.txt ../../www/2000.txt ../../www/3000.txt ../../www/10000.txt
+ -rm -rf localhost:8000 logs yaws.conf
diff --git a/test/t4/app_test.erl b/test/t4/app_test.erl
new file mode 100644
index 000000000..67f02df59
--- /dev/null
+++ b/test/t4/app_test.erl
@@ -0,0 +1,235 @@
+-module(app_test).
+-include("../include/tftest.hrl").
+-include("../ibrowse/src/ibrowse.hrl").
+-compile(export_all).
+
+
+%% Way to invoke just one test
+start([F]) ->
+ ?line {ok, _} = ibrowse:start_link(),
+ try
+ apply(app_test, F, []),
+ ok
+ catch
+ Error:Reason ->
+ throw({Error, Reason})
+ after
+ ibrowse:stop()
+ end.
+
+start() ->
+ ?line ok,
+ ?line {ok, _} = ibrowse:start_link(),
+ try
+ deflate_revproxy_test1(),
+ deflate_revproxy_test2(),
+ post_revproxy_test(),
+ streamcontent_revproxy_test(),
+ keepalive_revproxy_test(),
+ rewrite_revproxy_test(),
+ fwdproxy_test(),
+ ok
+ catch
+ Error:Reason ->
+ throw({Error, Reason})
+ after
+ ibrowse:stop()
+ end.
+
+
+deflate_revproxy_test1() ->
+ io:format("deflate_revproxy_test1\n", []),
+ Uri = "http://localhost:8000/revproxy1/hello.txt",
+ Res = "Hello, World!\n",
+
+ %% client: nodeflate - proxy: deflate - backend: deflate
+ %% ==> result: uncompressed
+ ?line {ok, "200", Hdrs1, Body1} = ibrowse:send_req(Uri, [], get),
+ ?line undefined = proplists:get_value("Content-Encoding", Hdrs1),
+ ?line Res = Body1,
+
+
+ %% client: deflate - proxy: deflate - backend: deflate
+ %% ==> result: compressed
+ ?line {ok, "200", Hdrs2, Body2} =
+ ibrowse:send_req(Uri, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs2),
+ ?line Res = binary_to_list(zlib:gunzip(Body2)),
+ ok.
+
+deflate_revproxy_test2() ->
+ io:format("deflate_revproxy_test2\n", []),
+ Uri = "http://localhost:8000/revproxy2/hello.txt",
+ Res = "Hello, World!\n",
+
+ %% client: nodeflate - proxy: deflate - backend: nodeflate
+ %% ==> result: uncompressed
+ ?line {ok, "200", Hdrs1, Body1} = ibrowse:send_req(Uri, [], get),
+ ?line undefined = proplists:get_value("Content-Encoding", Hdrs1),
+ ?line Res = Body1,
+
+ %% client: deflate - proxy: deflate - backend: nodeflate
+ %% ==> result: compressed
+ ?line {ok, "200", Hdrs2, Body2} =
+ ibrowse:send_req(Uri, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs2),
+ ?line Res = binary_to_list(zlib:gunzip(Body2)),
+ ok.
+
+
+
+post_revproxy_test() ->
+ io:format("post_revproxy_test\n",[]),
+ small_post(),
+ large_post(),
+ small_chunked_post(),
+ large_chunked_post(),
+ ok.
+
+small_post() ->
+ io:format(" small post\n",[]),
+ {ok, Bin} = file:read_file("../../www/1000.txt"),
+ Sz = integer_to_list(size(Bin)),
+ Uri = "http://localhost:8000/revproxy1/posttest/" ++ Sz,
+ Hdrs = [{content_length, Sz}, {content_type, "binary/octet-stream"}],
+ ?line {ok, "200", _, _} = ibrowse:send_req(Uri, Hdrs, post, Bin, []),
+ ok.
+
+large_post() ->
+ io:format(" large post\n",[]),
+ {ok, Bin} = file:read_file("../../www/10000.txt"),
+ Sz = integer_to_list(size(Bin)),
+ Uri = "http://localhost:8000/revproxy1/posttest/" ++ Sz,
+ Hdrs = [{content_length, Sz}, {content_type, "binary/octet-stream"}],
+ ?line {ok, "200", _, _} = ibrowse:send_req(Uri, Hdrs, post, Bin, []),
+ ok.
+
+small_chunked_post() ->
+ %% Chunk size is less than partial_post_size
+ io:format(" small chunked post\n",[]),
+ {ok, Bin} = file:read_file("../../www/3000.txt"),
+ Sz = integer_to_list(size(Bin)),
+ Uri = "http://localhost:8000/revproxy1/posttest/chunked/" ++ Sz,
+ Hdrs = [{content_type, "binary/octet-stream"}],
+ Opts = [{transfer_encoding, {chunked, 1000*1000}}],
+ ?line {ok, "200", _, _} = ibrowse:send_req(Uri, Hdrs, post, Bin, Opts),
+ ok.
+
+large_chunked_post() ->
+ %% Chunk size is greater than partial_post_size
+ io:format(" large chunked post\n",[]),
+ {ok, Bin} = file:read_file("../../www/10000.txt"),
+ Sz = integer_to_list(size(Bin)),
+ Uri = "http://localhost:8000/revproxy1/posttest/chunked/" ++ Sz,
+ Hdrs = [{content_type, "binary/octet-stream"}],
+ Opts = [{transfer_encoding, {chunked, 4000*1000}}],
+ ?line {ok, "200", _, _} = ibrowse:send_req(Uri, Hdrs, post, Bin, Opts),
+ ok.
+
+
+
+streamcontent_revproxy_test() ->
+ io:format("streamcontent_revproxy_test\n", []),
+ Uri = "http://localhost:8000/revproxy1/streamtest",
+ Res = "This is the data in the first chunk\n"
+ "and this is the second one\n"
+ "consequence",
+
+ ?line {ok, "200", Hdrs1, Body1} = ibrowse:send_req(Uri, [], get),
+ ?line "chunked" = proplists:get_value("Transfer-Encoding", Hdrs1),
+ ?line Res = Body1,
+
+ ?line {ok, "200", Hdrs2, Body2} =
+ ibrowse:send_req(Uri, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "chunked" = proplists:get_value("Transfer-Encoding", Hdrs2),
+ ?line Res = binary_to_list(zlib:gunzip(Body2)),
+ ok.
+
+
+keepalive_revproxy_test() ->
+ io:format("keepalive_revproxy_test\n", []),
+ Path1 = "/revproxy1/hello.txt",
+ Path2 = "/",
+ Path3 = "/revproxy2/hello.txt",
+ Res = "Hello, World!\n",
+
+ ?line {ok, Sock} = gen_tcp:connect("localhost", 8000,
+ [list, {active, false}]),
+
+ ?line ok = gen_tcp:send(Sock, "GET " ++ Path1 ++ " HTTP/1.1\r\n"
+ "Host: localhost:8000\r\n"
+ "Connection: Keep-Alive\r\n"
+ "\r\n"),
+ ?line {ok, Len1} = recv_hdrs(Sock),
+ ?line {ok, Res} = recv_body(Sock, Len1),
+
+ ?line ok = gen_tcp:send(Sock, "GET " ++ Path2 ++ " HTTP/1.1\r\n"
+ "Host: localhost:8000\r\n"
+ "Connection: Keep-Alive\r\n"
+ "\r\n"),
+ ?line {ok, Len2} = recv_hdrs(Sock),
+ ?line {ok, _} = recv_body(Sock, Len2),
+
+ ?line ok = gen_tcp:send(Sock, "GET " ++ Path3 ++ " HTTP/1.1\r\n"
+ "Host: localhost:8000\r\n"
+ "Connection: Keep-Alive\r\n"
+ "\r\n"),
+ ?line {ok, Len3} = recv_hdrs(Sock),
+ ?line {ok, Res} = recv_body(Sock, Len3),
+
+ gen_tcp:close(Sock),
+ ok.
+
+
+rewrite_revproxy_test() ->
+ io:format("rewrite_revproxy_test\n", []),
+ Uri1 = "http://localhost:8000/rewrite/revproxy1/hello.txt",
+ Uri2 = "http://localhost:8000/rewrite/revproxy2/hello.txt",
+ Res = "Hello, World!\n",
+
+ ?line {ok, "200", _, Body1} = ibrowse:send_req(Uri1, [], get),
+ ?line Res = Body1,
+
+ ?line {ok, "200", _, Body2} = ibrowse:send_req(Uri2, [], get),
+ ?line Res = Body2,
+ ok.
+
+
+fwdproxy_test() ->
+ io:format("fwdproxy_test\n", []),
+ Uri1 = "http://localhost:8001/rewrite/hello.txt",
+ Uri2 = "http://localhost:8002/rewrite/hello.txt",
+ Res = "Hello, World!\n",
+
+ Opts = [{proxy_host, "localhost"}, {proxy_port, 8003}],
+ ?line {ok, "200", _, Body1} = ibrowse:send_req(Uri1, [], get, [], Opts),
+ ?line Res = Body1,
+
+ ?line {ok, "200", _, Body2} = ibrowse:send_req(Uri2, [], get, [], Opts),
+ ?line Res = Body2,
+ ok.
+
+
+recv_hdrs(Sock) ->
+ inet:setopts(Sock, [{packet, http}]),
+ recv_hdrs(Sock, 0).
+recv_hdrs(Sock, Len) ->
+ inet:setopts(Sock, [{active, once}]),
+ receive
+ {http, Sock, http_eoh} ->
+ {ok, Len};
+ {http, Sock, {http_error, Error}} ->
+ {error, Error};
+ {http, Sock, {http_header, _, 'Content-Length', _, LenStr}} ->
+ recv_hdrs(Sock, list_to_integer(LenStr));
+ {http, Sock, {http_header, _, _, _, _}} ->
+ recv_hdrs(Sock, Len);
+ {http, Sock, {http_response, _, 200, "OK"}} ->
+ recv_hdrs(Sock, Len);
+ Other ->
+ {error, {"unexpected message", Other}}
+ end.
+
+recv_body(Sock, Len) ->
+ inet:setopts(Sock, [{packet, raw}, {active, false}]),
+ gen_tcp:recv(Sock, Len).
diff --git a/test/t4/posttest.erl b/test/t4/posttest.erl
new file mode 100644
index 000000000..d95a8e1ba
--- /dev/null
+++ b/test/t4/posttest.erl
@@ -0,0 +1,87 @@
+-module(posttest).
+-export([out/1]).
+
+-include("../../include/yaws.hrl").
+-include("../../include/yaws_api.hrl").
+
+out(Arg) ->
+ Url = yaws_api:request_url(Arg),
+ case Url#url.path of
+ "/posttest/chunked/" ++ ExpectedSize ->
+ if
+ (Arg#arg.headers)#headers.transfer_encoding =:= "chunked" ->
+ handle_post(list_to_integer(ExpectedSize), Arg);
+ true ->
+ Reason = io_lib:format("Expected a chunked transfer-encoding request\n~p",
+ [Arg#arg.headers]),
+ handle_post(0, Arg#arg{state={flush,500,Reason}})
+ end;
+ "/posttest/" ++ ExpectedSize ->
+ handle_post(list_to_integer(ExpectedSize), Arg);
+ _ ->
+ Reason = "unknown path: " ++ Url#url.path,
+ handle_post(0, Arg#arg{state={flush,500,Reason}})
+ end.
+
+
+handle_post(_, #arg{clidata=Data, state={flush, HttpCode, Reason}}) ->
+ %% Catch an error here but flush all remaining data.
+ case Data of
+ {partial, _} -> {get_more, undefined, {flush, HttpCode, Reason}};
+ _ -> [{status, HttpCode}, {html, Reason}]
+ end;
+
+handle_post(ExpectedSize, #arg{clidata=Data, cont=undefined}=Arg)
+ when is_binary(Data) ->
+ %% This is not a partial request
+ %% Expected: content_length =:= ExpectedSize AND
+ %% content_length < partial_post_size.
+
+ SC = get(sc),
+ if
+ size(Data) =:= ExpectedSize andalso
+ size(Data) < SC#sconf.partial_post_size ->
+ {status, 200};
+ true ->
+ Reason = io_lib:format("Post data too big. "
+ "Received: ~p bytes - Max: ~p bytes",
+ [size(Data), SC#sconf.partial_post_size]),
+ handle_post(ExpectedSize, Arg#arg{state={flush,500,Reason}})
+ end;
+handle_post(ExpectedSize, #arg{clidata={partial, Data}, cont=Cont}=Arg)
+ when is_binary(Data) ->
+ %% next Chunk of a partial request.
+ if
+ Cont =:= undefined ->
+ %% First chunk
+ {get_more, {cont, size(Data)}, undefined};
+ true ->
+ {cont, Sz0} = Cont,
+ Sz1 = Sz0 + size(Data),
+ if
+ Sz1 =< ExpectedSize ->
+ {get_more, {cont, Sz1}, undefined};
+ true ->
+ SC = get(sc),
+ Reason = io_lib:format("Chunk too big. "
+ "Received: ~p bytes - Max: ~p bytes",
+ [size(Data),
+ SC#sconf.partial_post_size]),
+ handle_post(ExpectedSize, Arg#arg{state={flush,500,Reason}})
+ end
+ end;
+handle_post(ExpectedSize, #arg{clidata=Data, cont=Cont}=Arg)
+ when is_binary(Data) ->
+ %% Last chunk of a partial request.
+ {cont, Sz0} = Cont,
+ Sz1 = Sz0 + size(Data),
+ if
+ Sz1 =:= ExpectedSize ->
+ {status, 200};
+ true ->
+ Reason = io_lib:format("Received data does not match "
+ "the expected size. "
+ "Received: ~p bytes - Expected: ~p bytes",
+ [Sz1, ExpectedSize]),
+ handle_post(ExpectedSize, Arg#arg{state={flush,500,Reason}})
+ end.
diff --git a/test/t4/rewritetest.erl b/test/t4/rewritetest.erl
new file mode 100644
index 000000000..103f204db
--- /dev/null
+++ b/test/t4/rewritetest.erl
@@ -0,0 +1,17 @@
+-module(rewritetest).
+-export([arg_rewrite/1]).
+
+-include("../../include/yaws.hrl").
+-include("../../include/yaws_api.hrl").
+
+
+arg_rewrite(Arg) ->
+ Url = yaws_api:request_url(Arg),
+ case Url#url.path of
+ "/rewrite" ++ Rest ->
+ Req0 = Arg#arg.req,
+ Req1 = Req0#http_request{path={abs_path,Rest}},
+ Arg#arg{req=Req1};
+ _ ->
+ Arg
+ end.
diff --git a/test/t4/streamtest.erl b/test/t4/streamtest.erl
new file mode 100644
index 000000000..a5319f612
--- /dev/null
+++ b/test/t4/streamtest.erl
@@ -0,0 +1,12 @@
+-module(streamtest).
+-export([out/1]).
+
+-include("../../include/yaws.hrl").
+-include("../../include/yaws_api.hrl").
+
+out(_Arg) ->
+ yaws_api:stream_chunk_deliver(self(), "and this is the second one\n"),
+ yaws_api:stream_chunk_deliver(self(), "con"),
+ yaws_api:stream_chunk_deliver(self(), "sequence"),
+ yaws_api:stream_chunk_end(self()),
+ {streamcontent, "text/plain", "This is the data in the first chunk\n"}.
diff --git a/test/t4/www1/hello.txt b/test/t4/www1/hello.txt
new file mode 100644
index 000000000..8ab686eaf
--- /dev/null
+++ b/test/t4/www1/hello.txt
@@ -0,0 +1 @@
+Hello, World!
diff --git a/test/t4/www2/hello.txt b/test/t4/www2/hello.txt
new file mode 100644
index 000000000..8ab686eaf
--- /dev/null
+++ b/test/t4/www2/hello.txt
@@ -0,0 +1 @@
+Hello, World!