diff --git a/src/yaws_revproxy.erl b/src/yaws_revproxy.erl index 5ed69b58e..a6a10bb0a 100644 --- a/src/yaws_revproxy.erl +++ b/src/yaws_revproxy.erl @@ -38,7 +38,6 @@ %% TODO: Activate proxy keep-alive with a new option ? -%% FIXME: Improve connection caching to enable this option -define(proxy_keepalive, false). @@ -203,7 +202,6 @@ out(Arg = #arg{state=RPState}) when RPState#revproxy.state == recvheaders -> Resp#http_response.status =:= 205 orelse Resp#http_response.status =:= 304 orelse Resp#http_response.status =:= 406 -> - %% FIXME: check all http codes RPState2 = RPState1#revproxy{state=terminate}, out(Arg#arg{state=RPState2}); @@ -238,7 +236,7 @@ out(Arg = #arg{state=RPState}) when RPState#revproxy.state == recvheaders -> %% The reponse content is not chunked. -%% FIXME: use partial_post_size to split huge content and avoid memory +%% TODO: use partial_post_size to split huge content and avoid memory %% exhaustion. out(Arg = #arg{state=RPState}) when RPState#revproxy.state == recvcontent -> Len = list_to_integer((RPState#revproxy.headers)#headers.content_length), @@ -377,7 +375,7 @@ recv_next_chunk(YawsPid, Arg = #arg{state=RPState}) -> %%========================================================================== -%% FIXME: find a better way to cache connections to backend servers. Here we can +%% TODO: find a better way to cache connections to backend servers. Here we can %% have 1 connection per gserv process for each backend server. get_cached_connection(URL) -> Key = lists:flatten(yaws_api:reformat_url(URL)), @@ -446,7 +444,6 @@ do_connect(URL) -> Err -> Err end; _ -> - %% FIXME: catch this case in during config parsing {error, unsupported_protocol} end. diff --git a/test/Makefile b/test/Makefile index 13717a093..3cd8c2e25 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,6 +1,6 @@ include support/include.mk -SUBDIRS = t1 t2 t3 eunit +SUBDIRS = t1 t2 t3 t4 eunit all: conf ibrowse @cd src; $(MAKE) all diff --git a/test/conf/revproxy.conf b/test/conf/revproxy.conf new file mode 100644 index 000000000..4be843011 --- /dev/null +++ b/test/conf/revproxy.conf @@ -0,0 +1,143 @@ + + +logdir = ./logs + +# This the path to a directory where additional +# beam code can be placed. The daemon will add this +# directory to its search path + +ebin_dir = %YTOP%/test/ibrowse/ebin +include_dir = %YTOP%/test/include + + + +# This is a debug variable, possible values are http | traffic | false +# It is also possible to set the trace (possibly to the tty) while +# invoking yaws from the shell as in +# yaws -i -T -x (see man yaws) + +trace = false + + + +# it is possible to have yaws start additional +# application specific code at startup +# +# runmod = mymodule + + +# By default yaws will copy the erlang error_log and +# end write it to a wrap log called report.log (in the logdir) +# this feature can be turned off. This would typically +# be the case when yaws runs within another larger app + +copy_error_log = true + + +# Logs are wrap logs + +log_wrap_size = 1000000 + + +# Possibly resolve all hostnames in logfiles so webalizer +# can produce the nice geography piechart + +log_resolve_hostname = false + + + +# fail completely or not if yaws fails +# to bind a listen socket +fail_on_bind_err = true + + + +# If yaws is started as root, it can, once it has opened +# all relevant sockets for listening, change the uid to a +# user with lower accessrights than root + +# username = nobody + + +# If HTTP auth is used, it is possible to have a specific +# auth log. +# Deprecated and ignored. Now, this target must be set in server part +#auth_log = true + + +# When we're running multiple yaws systems on the same +# host, we need to give each yaws system an individual +# name. Yaws will write a number of runtime files under +# /tmp/yaws/${id} +# The default value is "default" + + +# id = myname + + +# earlier versions of Yaws picked the first virtual host +# in a list of hosts with the same IP/PORT when the Host: +# header doesn't match any name on any Host +# This is often nice in testing environments but not +# acceptable in real live hosting scenarios + +pick_first_virthost_on_nomatch = true + + +# All unices are broken since it's not possible to bind to +# a privileged port (< 1024) unless uid==0 +# There is a contrib in jungerl which makes it possible by means +# of an external setuid root programm called fdsrv to listen to +# to privileged port. +# If we use this feature, it requires fdsrv to be properly installed. +# Doesn't yet work with SSL. + +use_fdsrv = false + + + + +# end then a set of virtual servers +# First two virthosted servers on the same IP (0.0.0.0) +# in this case, but an explicit IP can be given as well + + + port = 8000 + listen = 0.0.0.0 + listen_backlog = 512 + deflate = true + docroot = %YTOP%/www + arg_rewrite_mod = rewritetest + revproxy = /revproxy1 http://localhost:8001 + revproxy = /revproxy2 http://localhost:8002 + + + + port = 8001 + listen = 0.0.0.0 + listen_backlog = 512 + deflate = true + partial_post_size = 2048000 + appmods = posttest streamtest + docroot = %YTOP%/test/t4/www1 + + + + port = 8002 + listen = 0.0.0.0 + listen_backlog = 512 + deflate = false + docroot = %YTOP%/test/t4/www2 + + + + port = 8003 + listen = 0.0.0.0 + listen_backlog = 512 + deflate = false + fwdproxy = true + arg_rewrite_mod = rewritetest + docroot = %YTOP%/www + + + diff --git a/test/support/include.mk.in b/test/support/include.mk.in index 070c85fdb..b7a4abd57 100644 --- a/test/support/include.mk.in +++ b/test/support/include.mk.in @@ -56,3 +56,7 @@ stdconf: authconf: cat ../conf/authconf.conf | \ ../../scripts/Subst %YTOP% $(YTOP) > yaws.conf + +revproxyconf: + cat ../conf/revproxy.conf | \ + ../../scripts/Subst %YTOP% $(YTOP) > yaws.conf diff --git a/test/t4/Makefile b/test/t4/Makefile new file mode 100644 index 000000000..49670e34d --- /dev/null +++ b/test/t4/Makefile @@ -0,0 +1,38 @@ +include ../support/include.mk + +.PHONY: all test debug clean + +# +all: conf setup app_test.beam rewritetest.beam posttest.beam streamtest.beam + @echo "all ok" + + +# invoke as +# TEST=test3 make test +# or just make test to run all + +ULIMIT = 768 + +test: conf start + dd if=/dev/zero of=../../www/1000.txt bs=1024 count=1000 >/dev/null 2>&1 + dd if=/dev/zero of=../../www/2000.txt bs=1024 count=2000 >/dev/null 2>&1 + dd if=/dev/zero of=../../www/3000.txt bs=1024 count=3000 >/dev/null 2>&1 + dd if=/dev/zero of=../../www/10000.txt bs=1024 count=10000 >/dev/null 2>&1 + ul=`ulimit -n` ; \ + val=`expr $$ul '<' $(ULIMIT)` ; \ + if [ $$val = 1 ] ; then \ + echo trying to raise "ulimit -n" for the test... ; \ + set -e ; \ + ulimit -n $(ULIMIT) ; \ + fi ; \ + $(ERL) -noinput $(PA) -s tftest + $(MAKE) stop + +conf: revproxyconf + +debug: + $(ERL) $(PA) + +clean: tclean + -rm -f ../../www/1000.txt ../../www/2000.txt ../../www/3000.txt ../../www/10000.txt + -rm -rf localhost:8000 logs yaws.conf diff --git a/test/t4/app_test.erl b/test/t4/app_test.erl new file mode 100644 index 000000000..67f02df59 --- /dev/null +++ b/test/t4/app_test.erl @@ -0,0 +1,235 @@ +-module(app_test). +-include("../include/tftest.hrl"). +-include("../ibrowse/src/ibrowse.hrl"). +-compile(export_all). + + +%% Way to invoke just one test +start([F]) -> + ?line {ok, _} = ibrowse:start_link(), + try + apply(app_test, F, []), + ok + catch + Error:Reason -> + throw({Error, Reason}) + after + ibrowse:stop() + end. + +start() -> + ?line ok, + ?line {ok, _} = ibrowse:start_link(), + try + deflate_revproxy_test1(), + deflate_revproxy_test2(), + post_revproxy_test(), + streamcontent_revproxy_test(), + keepalive_revproxy_test(), + rewrite_revproxy_test(), + fwdproxy_test(), + ok + catch + Error:Reason -> + throw({Error, Reason}) + after + ibrowse:stop() + end. + + +deflate_revproxy_test1() -> + io:format("deflate_revproxy_test1\n", []), + Uri = "http://localhost:8000/revproxy1/hello.txt", + Res = "Hello, World!\n", + + %% client: nodeflate - proxy: deflate - backend: deflate + %% ==> result: uncompressed + ?line {ok, "200", Hdrs1, Body1} = ibrowse:send_req(Uri, [], get), + ?line undefined = proplists:get_value("Content-Encoding", Hdrs1), + ?line Res = Body1, + + + %% client: deflate - proxy: deflate - backend: deflate + %% ==> result: compressed + ?line {ok, "200", Hdrs2, Body2} = + ibrowse:send_req(Uri, [{"Accept-Encoding", "gzip, deflate"}], get), + ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs2), + ?line Res = binary_to_list(zlib:gunzip(Body2)), + ok. + +deflate_revproxy_test2() -> + io:format("deflate_revproxy_test2\n", []), + Uri = "http://localhost:8000/revproxy2/hello.txt", + Res = "Hello, World!\n", + + %% client: nodeflate - proxy: deflate - backend: nodeflate + %% ==> result: uncompressed + ?line {ok, "200", Hdrs1, Body1} = ibrowse:send_req(Uri, [], get), + ?line undefined = proplists:get_value("Content-Encoding", Hdrs1), + ?line Res = Body1, + + %% client: deflate - proxy: deflate - backend: nodeflate + %% ==> result: compressed + ?line {ok, "200", Hdrs2, Body2} = + ibrowse:send_req(Uri, [{"Accept-Encoding", "gzip, deflate"}], get), + ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs2), + ?line Res = binary_to_list(zlib:gunzip(Body2)), + ok. + + + +post_revproxy_test() -> + io:format("post_revproxy_test\n",[]), + small_post(), + large_post(), + small_chunked_post(), + large_chunked_post(), + ok. + +small_post() -> + io:format(" small post\n",[]), + {ok, Bin} = file:read_file("../../www/1000.txt"), + Sz = integer_to_list(size(Bin)), + Uri = "http://localhost:8000/revproxy1/posttest/" ++ Sz, + Hdrs = [{content_length, Sz}, {content_type, "binary/octet-stream"}], + ?line {ok, "200", _, _} = ibrowse:send_req(Uri, Hdrs, post, Bin, []), + ok. + +large_post() -> + io:format(" large post\n",[]), + {ok, Bin} = file:read_file("../../www/10000.txt"), + Sz = integer_to_list(size(Bin)), + Uri = "http://localhost:8000/revproxy1/posttest/" ++ Sz, + Hdrs = [{content_length, Sz}, {content_type, "binary/octet-stream"}], + ?line {ok, "200", _, _} = ibrowse:send_req(Uri, Hdrs, post, Bin, []), + ok. + +small_chunked_post() -> + %% Chunk size is less than partial_post_size + io:format(" small chunked post\n",[]), + {ok, Bin} = file:read_file("../../www/3000.txt"), + Sz = integer_to_list(size(Bin)), + Uri = "http://localhost:8000/revproxy1/posttest/chunked/" ++ Sz, + Hdrs = [{content_type, "binary/octet-stream"}], + Opts = [{transfer_encoding, {chunked, 1000*1000}}], + ?line {ok, "200", _, _} = ibrowse:send_req(Uri, Hdrs, post, Bin, Opts), + ok. + +large_chunked_post() -> + %% Chunk size is greater than partial_post_size + io:format(" large chunked post\n",[]), + {ok, Bin} = file:read_file("../../www/10000.txt"), + Sz = integer_to_list(size(Bin)), + Uri = "http://localhost:8000/revproxy1/posttest/chunked/" ++ Sz, + Hdrs = [{content_type, "binary/octet-stream"}], + Opts = [{transfer_encoding, {chunked, 4000*1000}}], + ?line {ok, "200", _, _} = ibrowse:send_req(Uri, Hdrs, post, Bin, Opts), + ok. + + + +streamcontent_revproxy_test() -> + io:format("streamcontent_revproxy_test\n", []), + Uri = "http://localhost:8000/revproxy1/streamtest", + Res = "This is the data in the first chunk\n" + "and this is the second one\n" + "consequence", + + ?line {ok, "200", Hdrs1, Body1} = ibrowse:send_req(Uri, [], get), + ?line "chunked" = proplists:get_value("Transfer-Encoding", Hdrs1), + ?line Res = Body1, + + ?line {ok, "200", Hdrs2, Body2} = + ibrowse:send_req(Uri, [{"Accept-Encoding", "gzip, deflate"}], get), + ?line "chunked" = proplists:get_value("Transfer-Encoding", Hdrs2), + ?line Res = binary_to_list(zlib:gunzip(Body2)), + ok. + + +keepalive_revproxy_test() -> + io:format("keepalive_revproxy_test\n", []), + Path1 = "/revproxy1/hello.txt", + Path2 = "/", + Path3 = "/revproxy2/hello.txt", + Res = "Hello, World!\n", + + ?line {ok, Sock} = gen_tcp:connect("localhost", 8000, + [list, {active, false}]), + + ?line ok = gen_tcp:send(Sock, "GET " ++ Path1 ++ " HTTP/1.1\r\n" + "Host: localhost:8000\r\n" + "Connection: Keep-Alive\r\n" + "\r\n"), + ?line {ok, Len1} = recv_hdrs(Sock), + ?line {ok, Res} = recv_body(Sock, Len1), + + ?line ok = gen_tcp:send(Sock, "GET " ++ Path2 ++ " HTTP/1.1\r\n" + "Host: localhost:8000\r\n" + "Connection: Keep-Alive\r\n" + "\r\n"), + ?line {ok, Len2} = recv_hdrs(Sock), + ?line {ok, _} = recv_body(Sock, Len2), + + ?line ok = gen_tcp:send(Sock, "GET " ++ Path3 ++ " HTTP/1.1\r\n" + "Host: localhost:8000\r\n" + "Connection: Keep-Alive\r\n" + "\r\n"), + ?line {ok, Len3} = recv_hdrs(Sock), + ?line {ok, Res} = recv_body(Sock, Len3), + + gen_tcp:close(Sock), + ok. + + +rewrite_revproxy_test() -> + io:format("rewrite_revproxy_test\n", []), + Uri1 = "http://localhost:8000/rewrite/revproxy1/hello.txt", + Uri2 = "http://localhost:8000/rewrite/revproxy2/hello.txt", + Res = "Hello, World!\n", + + ?line {ok, "200", _, Body1} = ibrowse:send_req(Uri1, [], get), + ?line Res = Body1, + + ?line {ok, "200", _, Body2} = ibrowse:send_req(Uri2, [], get), + ?line Res = Body2, + ok. + + +fwdproxy_test() -> + io:format("fwdproxy_test\n", []), + Uri1 = "http://localhost:8001/rewrite/hello.txt", + Uri2 = "http://localhost:8002/rewrite/hello.txt", + Res = "Hello, World!\n", + + Opts = [{proxy_host, "localhost"}, {proxy_port, 8003}], + ?line {ok, "200", _, Body1} = ibrowse:send_req(Uri1, [], get, [], Opts), + ?line Res = Body1, + + ?line {ok, "200", _, Body2} = ibrowse:send_req(Uri2, [], get, [], Opts), + ?line Res = Body2, + ok. + + +recv_hdrs(Sock) -> + inet:setopts(Sock, [{packet, http}]), + recv_hdrs(Sock, 0). +recv_hdrs(Sock, Len) -> + inet:setopts(Sock, [{active, once}]), + receive + {http, Sock, http_eoh} -> + {ok, Len}; + {http, Sock, {http_error, Error}} -> + {error, Error}; + {http, Sock, {http_header, _, 'Content-Length', _, LenStr}} -> + recv_hdrs(Sock, list_to_integer(LenStr)); + {http, Sock, {http_header, _, _, _, _}} -> + recv_hdrs(Sock, Len); + {http, Sock, {http_response, _, 200, "OK"}} -> + recv_hdrs(Sock, Len); + Other -> + {error, {"unexpected message", Other}} + end. + +recv_body(Sock, Len) -> + inet:setopts(Sock, [{packet, raw}, {active, false}]), + gen_tcp:recv(Sock, Len). diff --git a/test/t4/posttest.erl b/test/t4/posttest.erl new file mode 100644 index 000000000..d95a8e1ba --- /dev/null +++ b/test/t4/posttest.erl @@ -0,0 +1,87 @@ +-module(posttest). +-export([out/1]). + +-include("../../include/yaws.hrl"). +-include("../../include/yaws_api.hrl"). + +out(Arg) -> + Url = yaws_api:request_url(Arg), + case Url#url.path of + "/posttest/chunked/" ++ ExpectedSize -> + if + (Arg#arg.headers)#headers.transfer_encoding =:= "chunked" -> + handle_post(list_to_integer(ExpectedSize), Arg); + true -> + Reason = io_lib:format("Expected a chunked transfer-encoding request\n~p", + [Arg#arg.headers]), + handle_post(0, Arg#arg{state={flush,500,Reason}}) + end; + "/posttest/" ++ ExpectedSize -> + handle_post(list_to_integer(ExpectedSize), Arg); + _ -> + Reason = "unknown path: " ++ Url#url.path, + handle_post(0, Arg#arg{state={flush,500,Reason}}) + end. + + +handle_post(_, #arg{clidata=Data, state={flush, HttpCode, Reason}}) -> + %% Catch an error here but flush all remaining data. + case Data of + {partial, _} -> {get_more, undefined, {flush, HttpCode, Reason}}; + _ -> [{status, HttpCode}, {html, Reason}] + end; + +handle_post(ExpectedSize, #arg{clidata=Data, cont=undefined}=Arg) + when is_binary(Data) -> + %% This is not a partial request + %% Expected: content_length =:= ExpectedSize AND + %% content_length < partial_post_size. + + SC = get(sc), + if + size(Data) =:= ExpectedSize andalso + size(Data) < SC#sconf.partial_post_size -> + {status, 200}; + true -> + Reason = io_lib:format("Post data too big. " + "Received: ~p bytes - Max: ~p bytes", + [size(Data), SC#sconf.partial_post_size]), + handle_post(ExpectedSize, Arg#arg{state={flush,500,Reason}}) + end; +handle_post(ExpectedSize, #arg{clidata={partial, Data}, cont=Cont}=Arg) + when is_binary(Data) -> + %% next Chunk of a partial request. + if + Cont =:= undefined -> + %% First chunk + {get_more, {cont, size(Data)}, undefined}; + true -> + {cont, Sz0} = Cont, + Sz1 = Sz0 + size(Data), + if + Sz1 =< ExpectedSize -> + {get_more, {cont, Sz1}, undefined}; + true -> + SC = get(sc), + Reason = io_lib:format("Chunk too big. " + "Received: ~p bytes - Max: ~p bytes", + [size(Data), + SC#sconf.partial_post_size]), + handle_post(ExpectedSize, Arg#arg{state={flush,500,Reason}}) + end + end; +handle_post(ExpectedSize, #arg{clidata=Data, cont=Cont}=Arg) + when is_binary(Data) -> + %% Last chunk of a partial request. + {cont, Sz0} = Cont, + Sz1 = Sz0 + size(Data), + if + Sz1 =:= ExpectedSize -> + {status, 200}; + true -> + Reason = io_lib:format("Received data does not match " + "the expected size. " + "Received: ~p bytes - Expected: ~p bytes", + [Sz1, ExpectedSize]), + handle_post(ExpectedSize, Arg#arg{state={flush,500,Reason}}) + end. diff --git a/test/t4/rewritetest.erl b/test/t4/rewritetest.erl new file mode 100644 index 000000000..103f204db --- /dev/null +++ b/test/t4/rewritetest.erl @@ -0,0 +1,17 @@ +-module(rewritetest). +-export([arg_rewrite/1]). + +-include("../../include/yaws.hrl"). +-include("../../include/yaws_api.hrl"). + + +arg_rewrite(Arg) -> + Url = yaws_api:request_url(Arg), + case Url#url.path of + "/rewrite" ++ Rest -> + Req0 = Arg#arg.req, + Req1 = Req0#http_request{path={abs_path,Rest}}, + Arg#arg{req=Req1}; + _ -> + Arg + end. diff --git a/test/t4/streamtest.erl b/test/t4/streamtest.erl new file mode 100644 index 000000000..a5319f612 --- /dev/null +++ b/test/t4/streamtest.erl @@ -0,0 +1,12 @@ +-module(streamtest). +-export([out/1]). + +-include("../../include/yaws.hrl"). +-include("../../include/yaws_api.hrl"). + +out(_Arg) -> + yaws_api:stream_chunk_deliver(self(), "and this is the second one\n"), + yaws_api:stream_chunk_deliver(self(), "con"), + yaws_api:stream_chunk_deliver(self(), "sequence"), + yaws_api:stream_chunk_end(self()), + {streamcontent, "text/plain", "This is the data in the first chunk\n"}. diff --git a/test/t4/www1/hello.txt b/test/t4/www1/hello.txt new file mode 100644 index 000000000..8ab686eaf --- /dev/null +++ b/test/t4/www1/hello.txt @@ -0,0 +1 @@ +Hello, World! diff --git a/test/t4/www2/hello.txt b/test/t4/www2/hello.txt new file mode 100644 index 000000000..8ab686eaf --- /dev/null +++ b/test/t4/www2/hello.txt @@ -0,0 +1 @@ +Hello, World!