Permalink
Browse files

API plugin update: add decode_buffer/2 fun. Implemented in http for

chunk encoding and decompression (fix TSUN-129 and TSUN-106)
  • Loading branch information...
1 parent e943a5b commit 96dcaf989732ceb11f9753de4f8827cf459bc3cb @nniclausse nniclausse committed Sep 1, 2010
View
@@ -61,12 +61,13 @@
%% the parsing of the response
-record(http, {content_length= 0, % HTTP header: content length
body_size = 0, % current size of body,
- chunk_toread = -1, % chunk data to be read (-1 = not chunked)
+ chunk_toread = -1, % chunk data to be read (-1 = not chunked, -2 = not chunked, but last response was)
status = none, % HTTP resp. status :200, etc. 'none'
% if no current cnx.
close = false, % true if HTTP/1.0 or 'connection: close'
% has been received
partial=false, % true if headers are partially received
+ compressed={false,false}, % type of compression if body is compressed
cookie=[]
}).
@@ -97,3 +98,5 @@
-define(USER_AGENT, "Tsung").
-define(USER_AGENT_ERROR_MSG, "Total sum of user agents frequency is not equal to 100").
+-define(MAX_HEADER_SIZE, 65536). % used for http_chunk:decode
+
View
@@ -152,6 +152,34 @@ chunk_header_bad_test()->
Rep=ts_http_common:parse_line("transfer-encoding: cheddar\r\n",#http{},[]),
?assertMatch(#http{chunk_toread=-1}, Rep).
+split_body_test() ->
+ Data = << "HTTP header\r\nHeader: value\r\n\r\nbody\r\n" >>,
+ ?assertEqual({<< "HTTP header\r\nHeader: value" >>, << "body\r\n" >>}, ts_http:split_body(Data)).
+
+split_body2_test() ->
+ Data = << "HTTP header\r\nHeader: value\r\n\r\nbody\r\n\r\nnewline in body\r\n" >>,
+ ?assertEqual({<< "HTTP header\r\nHeader: value" >>, << "body\r\n\r\nnewline in body\r\n" >>}, ts_http:split_body(Data)).
+
+split_body3_test() ->
+ Data = << "HTTP header\r\nHeader: value\r\nTransfer-Encoding: chunked\r\n\r\n19\r\nbody\r\n\r\nnewline in body\r\n\r\n" >>,
+ ?assertEqual({<< "HTTP header\r\nHeader: value\r\nTransfer-Encoding: chunked" >>, << "19\r\nbody\r\n\r\nnewline in body\r\n\r\n" >>}, ts_http:split_body(Data)).
+
+decode_buffer_test() ->
+ Data = << "HTTP header\r\nHeader: value\r\nTransfer-Encoding: chunked\r\n\r\n19\r\nbody\r\n\r\nnewline in body\r\n0\r\n\r\n" >>,
+ ?assertEqual(<< "HTTP header\r\nHeader: value\r\nTransfer-Encoding: chunked\r\n\r\nbody\r\n\r\nnewline in body\r\n" >>, ts_http:decode_buffer(Data, #http{chunk_toread=-2})).
+
+decode_buffer2_test() ->
+ Data = << "HTTP header\r\nHeader: value\r\n\r\nbody\r\n\r\nnewline in body\r\n" >>,
+ ?assertEqual(<< "HTTP header\r\nHeader: value\r\n\r\nbody\r\n\r\nnewline in body\r\n" >>, ts_http:decode_buffer(Data, #http{chunk_toread=-1}) ).
+
+compress_chunk_test()->
+ <<A:10/binary, B/binary>> = zlib:gzip("sesame ouvre toi"),
+ Data1 = << "HTTP header\r\nHeader: value\r\nTransfer-Encoding: chunked\r\n\r\nA\r\n" >>,
+ Data2= <<"1A\r\n" >>,
+ Data3= <<"0\r\n\r\n" >>,
+ Data= <<Data1/binary, A/binary, Data2/binary, B/binary, Data3/binary>>,
+ ?assertEqual(<< "HTTP header\r\nHeader: value\r\nTransfer-Encoding: chunked\r\n\r\nsesame ouvre toi" >>, ts_http:decode_buffer(Data, #http{chunk_toread=-2, compressed={false,gzip}})).
+
myset_env()->
myset_env(0).
myset_env(N)->
@@ -362,7 +362,11 @@ extract_body_nohttp_test() ->
Data = << "random\r\nstuff" >>,
?assertEqual(Data, ts_search:extract_body(Data)).
-
+badarg_re_test() ->
+ Data = << "Below this line, is 1000 repeated lines">>,
+ Regexp = "is (\\d+) repeated lines",
+ {ok,Regexp2}=re:compile(Regexp),
+ ?assertEqual([{lines, "1000"}], ts_search:parse_dynvar([{re, 'lines', Regexp2 }],Data)).
myset_env()->
myset_env(0).
View
@@ -907,7 +907,7 @@ handle_data_msg(Data,State=#state_rcv{request=Req,clienttype=Type,maxcount=MaxCo
ts_mon:rcvmes({State#state_rcv.dump, self(), Data}),
{NewState, Opts, Close} = Type:parse(Data, State),
- NewBuffer=set_new_buffer(Req, NewState#state_rcv.buffer, Data),
+ NewBuffer=set_new_buffer(NewState, Data),
?DebugF("Dyndata is now ~p~n",[NewState#state_rcv.dyndata]),
case NewState#state_rcv.ack_done of
@@ -977,9 +977,9 @@ handle_data_msg(Data,State=#state_rcv{request=Req,datasize=OldSize})
handle_data_msg(<<32>>, State=#state_rcv{clienttype=ts_jabber}) ->
{State#state_rcv{ack_done = false},[]};
%% local ack, set ack_done to true
-handle_data_msg(Data, State=#state_rcv{request=Req, maxcount= MaxCount}) ->
+handle_data_msg(Data, State=#state_rcv{request=Req, clienttype=Type, maxcount= MaxCount}) ->
ts_mon:rcvmes({State#state_rcv.dump, self(), Data}),
- NewBuffer= set_new_buffer(Req, State#state_rcv.buffer, Data),
+ NewBuffer= set_new_buffer(State, Data),
DataSize = size(Data),
{PageTimeStamp, DynVars} = update_stats(State#state_rcv{datasize=DataSize,
buffer=NewBuffer}),
@@ -995,14 +995,17 @@ handle_data_msg(Data, State=#state_rcv{request=Req, maxcount= MaxCount}) ->
%%----------------------------------------------------------------------
%% Func: set_new_buffer/3
%%----------------------------------------------------------------------
-set_new_buffer(#ts_request{match=[], dynvar_specs=[]},_,_) ->
+set_new_buffer(#state_rcv{request = #ts_request{match=[], dynvar_specs=[]}} ,_) ->
<< >>;
-set_new_buffer(_, Buffer,closed) ->
- Buffer;
-set_new_buffer(_, OldBuffer, Data) when is_binary(OldBuffer) andalso is_binary(Data)->
+set_new_buffer(#state_rcv{clienttype=Type, buffer=Buffer, session=Session},closed) ->
+ Type:decode_buffer(Buffer,Session);
+set_new_buffer(#state_rcv{buffer=OldBuffer,ack_done=false},Data) ->
?Debug("Bufferize response~n"),
<< OldBuffer/binary, Data/binary >>;
-set_new_buffer(_, _, Data) -> % don't need buffer for non binary responses (erlang fun case)
+set_new_buffer(#state_rcv{clienttype=Type, buffer=OldBuffer, session=Session},Data) ->
+ ?Debug("decode response~n"),
+ Type:decode_buffer(<< OldBuffer/binary, Data/binary >>, Session);
+set_new_buffer(_State, Data) -> % don't need buffer for non binary responses (erlang fun case)
Data.
%%----------------------------------------------------------------------
View
@@ -34,6 +34,7 @@
parse/2,
parse_bidi/2,
parse_config/2,
+ decode_buffer/2,
new_session/0]).
@@ -87,6 +88,13 @@ session_defaults() ->
new_session() ->
#fs{}.
+%% @spec decode_buffer(Buffer::binary(),Session::record(fs)) -> NewBuffer::binary()
+%% @doc We need to decode buffer (remove chunks, decompress ...) for
+%% matching or dyn_variables
+%% @end
+decode_buffer(Buffer,#fs{}) ->
+ Buffer.
+
%% @spec init_dynparams() -> dyndata()
%% @doc Creates a new record/term for storing dynamic request data.
%% @end
View
@@ -35,6 +35,7 @@
session_defaults/0,
parse/2,
parse_config/2,
+ decode_buffer/2,
new_session/0]).
%%----------------------------------------------------------------------
@@ -55,6 +56,23 @@ session_defaults() ->
new_session() ->
#http{}.
+%% @spec decode_buffer(Buffer::binary(),Session::record(http)) -> NewBuffer::binary()
+%% @doc We need to decode buffer (remove chunks, decompress ...) for
+%% matching or dyn_variables
+%% @end
+decode_buffer(Buffer,#http{chunk_toread = -1, compressed={_,false}}) ->
+ Buffer;
+decode_buffer(Buffer,#http{chunk_toread = -1, compressed={_,Val}}) ->
+ {Headers, CompressedBody} = split_body(Buffer),
+ Body = decompress(CompressedBody, Val),
+ << Headers/binary, "\r\n\r\n", Body/binary >>;
+decode_buffer(Buffer,Http=#http{compressed={_,Comp}})->
+ {Headers, Body} = decode_chunk(Buffer),
+ ?DebugF("body is ~p~n",[Body]),
+ RealBody = decompress(Body, Comp),
+ ?DebugF("decoded buffer: ~p",[RealBody]),
+ <<Headers/binary, "\r\n\r\n", RealBody/binary >>.
+
%%----------------------------------------------------------------------
%% Function: get_message/21
%% Purpose: Build a message/request ,
@@ -179,3 +197,51 @@ subst(Req=#http_request{url=URL, body=Body, headers = Headers, userid=UserId, pa
%% currently, we only handle space conversion to %20
escape_url(URL)->
re:replace(URL," ","%20",[{return,list},global]).
+
+
+decompress(Buffer,gzip)->
+ zlib:gunzip(Buffer);
+decompress(Buffer,uncompress)->
+ zlib:uncompress(Buffer);
+decompress(Buffer,deflate)->
+ zlib:unzip(Buffer);
+decompress(Buffer,false)->
+ Buffer;
+decompress(Buffer,Else)->
+ ?LOGF("Unknown compression method, skip decompression ~p",[Else],?WARN),
+ Buffer.
+
+decode_chunk(Data)->
+ decode_chunk_header(Data,<<>>).
+
+decode_chunk_header(<<CRLF:4/binary, Data/binary >>,Headers) when CRLF == << "\r\n\r\n">> ->
+ decode_chunk_size(Data,Headers,<< >>, << >>);
+decode_chunk_header(<<CRLF:1/binary, Data/binary >>, Head) ->
+ decode_chunk_header(Data, <<Head/binary, CRLF/binary>> ).
+
+decode_chunk_size(<< >>, Headers, Body,Digits) ->
+ {Headers, Body};
+decode_chunk_size(<<Head:2/binary >>, Headers, Body, <<>>) when Head == << "\r\n" >> ->
+ %last CRLF, remove
+ {Headers, Body};
+decode_chunk_size(<<Head:2/binary, Data/binary >>, Headers, Body, <<>>) when Head == << "\r\n" >> ->
+ % CRLF but no digits, the CRLF is part of the body
+ decode_chunk_size(Data, Headers, <<Body/binary, Head/binary>>, <<>>);
+decode_chunk_size(<<Head:2/binary, Data/binary >>, Headers, Body,Digits) when Head == << "\r\n" >> ->
+ case httpd_util:hexlist_to_integer(binary_to_list(Digits)) of
+ 0 ->
+ decode_chunk_size(Data, Headers, Body ,<<>>);
+ Size ->
+ << Chunk:Size/binary, Tail/binary >> = Data,
+ decode_chunk_size(Tail, Headers, << Body/binary, Chunk/binary>> ,<<>>)
+ end;
+decode_chunk_size(<<Digit:1/binary, Data/binary >>, Headers, Body, PrevDigit) ->
+ decode_chunk_size(Data, Headers, Body, <<PrevDigit/binary, Digit/binary>>).
+
+split_body(Data) ->
+ case re:run(Data,"(.*)\r\n\r\n(.*)$",[{capture,all_but_first,binary},ungreedy,dotall]) of
+ nomatch -> Data;
+ {match, [Header,Body]} -> {Header,<< Body/binary,"\n" >>};
+ _ -> Data
+ end.
+
@@ -209,8 +209,8 @@ matchdomain_url(Cookie, Host, URL) ->
%% Purpose: parse the response from the server and keep information
%% about the response if State#state_rcv.session
%%----------------------------------------------------------------------
-parse(closed, State) ->
- {State#state_rcv{session= #http{}, ack_done = true}, [], true};
+parse(closed, State=#state_rcv{session=Http}) ->
+ {State#state_rcv{session=reset_session(Http), ack_done = true}, [], true};
parse(Data, State=#state_rcv{session=HTTP}) when HTTP#http.status == none;
HTTP#http.partial == true ->
@@ -239,9 +239,9 @@ parse(Data, State=#state_rcv{session=HTTP}) when HTTP#http.status == none;
{State#state_rcv{session= Http, ack_done = false,
datasize = TotalSize,
dyndata= DynData}, [], true};
- {ok, #http{status=100}, _} -> % Status 100 Continue, ignore.
+ {ok, Http=#http{status=100}, _} -> % Status 100 Continue, ignore.
%% FIXME: not tested
- {State#state_rcv{ack_done=false,session=#http{}},[],false};
+ {State#state_rcv{ack_done=false,session=reset_session(Http)},[],false};
{ok, Http, Tail} ->
DynData = concat_cookies(Http#http.cookie, State#state_rcv.dyndata),
check_resp_size(Http, length(Tail), DynData, State#state_rcv{acc=[]}, TotalSize)
@@ -266,7 +266,7 @@ parse(Data, State) ->
CLength = Http#http.content_length,
case Http#http.body_size + DataSize of
CLength -> % end of response
- {State#state_rcv{session=#http{}, acc=[], ack_done = true, datasize = CLength},
+ {State#state_rcv{session=reset_session(Http), acc=[], ack_done = true, datasize = CLength},
[], Http#http.close};
Size ->
NewHttp = (State#state_rcv.session)#http{body_size = Size},
@@ -279,18 +279,18 @@ parse(Data, State) ->
%% Purpose: Check response size
%% Returns: {NewState= record(state_rcv), SockOpts, Close}
%%----------------------------------------------------------------------
-check_resp_size(#http{content_length=CLength, close=Close}, CLength,
+check_resp_size(Http=#http{content_length=CLength, close=Close}, CLength,
DynData, State, DataSize) ->
%% end of response
- {State#state_rcv{session= #http{}, ack_done = true,
+ {State#state_rcv{session= reset_session(Http), ack_done = true,
datasize = DataSize,
dyndata= DynData}, [], Close};
-check_resp_size(#http{content_length=CLength, close = Close},
+check_resp_size(Http=#http{content_length=CLength, close = Close},
BodySize, DynData, State, DataSize) when BodySize > CLength ->
?LOGF("Error: HTTP Body (~p)> Content-Length (~p) !~n",
[BodySize, CLength], ?ERR),
ts_mon:add({ count, error_http_bad_content_length }),
- {State#state_rcv{session= #http{}, ack_done = true,
+ {State#state_rcv{session= reset_session(Http), ack_done = true,
datasize = DataSize,
dyndata= DynData}, [], Close};
check_resp_size(Http, BodySize, DynData, State, DataSize) ->
@@ -320,7 +320,7 @@ read_chunk(<<>>, State, Int, Acc) ->
{ State#state_rcv{acc = AccInt, ack_done = false }, [] }; % read more data
%% this code has been inspired by inets/http_lib.erl
%% Extensions not implemented
-read_chunk(<<Char:1/binary, Data/binary>>, State, Int, Acc) ->
+read_chunk(<<Char:1/binary, Data/binary>>, State=#state_rcv{session=Http}, Int, Acc) ->
case Char of
<<C>> when $0=<C,C=<$9 ->
read_chunk(Data, State, 16*Int+(C-$0), Acc+1);
@@ -332,7 +332,7 @@ read_chunk(<<Char:1/binary, Data/binary>>, State, Int, Acc) ->
read_chunk_data(Data, State, Int+3, Acc+1);
<<?CR>> when Int==0, size(Data) == 3 -> %% should be the end of transfer
?DebugF("Finish tranfer chunk ~p~n", [binary_to_list(Data)]),
- {State#state_rcv{session= #http{}, ack_done = true,
+ {State#state_rcv{session= reset_session(Http), ack_done = true,
datasize = Acc %% FIXME: is it the correct size?
}, []};
<<?CR>> when Int==0, size(Data) < 3 -> % lack ?CRLF, continue
@@ -343,7 +343,7 @@ read_chunk(<<Char:1/binary, Data/binary>>, State, Int, Acc) ->
_Other ->
?LOGF("Unexpected error while parsing chunk ~p~n", [_Other] ,?WARN),
ts_mon:add({count, error_http_unexpected_chunkdata}),
- {State#state_rcv{session= #http{}, ack_done = true}, []}
+ {State#state_rcv{session= reset_session(Http), ack_done = true}, []}
end.
%%----------------------------------------------------------------------
@@ -557,6 +557,9 @@ parse_line("content-length: "++Tail, Http, _Host)->
parse_line("connection: close"++_Tail, Http, _Host)->
?Debug("Connection Closed in Header ~n"),
Http#http{close=true};
+parse_line("content-encoding: "++Tail, Http=#http{compressed={Prev,_}}, _Host)->
+ ?DebugF("content encoding:~p ~n",[Tail]),
+ Http#http{compressed={list_to_atom(Tail),Prev}};
parse_line("transfer-encoding:"++[H|Tail], Http, _Host)->
?DebugF("~p transfer encoding~n",[[H]++Tail]),
case Tail of
@@ -607,3 +610,10 @@ get_line([H|T], true, Cur) ->
get_line(T, true, [H|Cur]);
get_line([], _, _) -> %% Headers are fragmented ... We need more data
{more}.
+
+%% we need to keep the compressed value of the current request
+reset_session(#http{compressed={Current,_},chunk_toread=Val}) when Val > -1 ->
+ #http{compressed={false,Current}, chunk_toread=-2} ;
+reset_session(#http{compressed={Current,_}} ) ->
+ #http{compressed={false,Current}}.
+
View
@@ -41,6 +41,7 @@
parse/2,
parse_bidi/2,
parse_config/2,
+ decode_buffer/2,
new_session/0]).
%%----------------------------------------------------------------------
@@ -51,6 +52,13 @@
session_defaults() ->
{ok, true, false}.
+%% @spec decode_buffer(Buffer::binary(),Session::record(jabber)) -> NewBuffer::binary()
+%% @doc We need to decode buffer (remove chunks, decompress ...) for
+%% matching or dyn_variables
+%% @end
+decode_buffer(Buffer,#jabber{}) ->
+ Buffer. % nothing to do for jabber
+
%%----------------------------------------------------------------------
%% Function: new_session/0
%% Purpose: initialize session information
View
@@ -31,6 +31,7 @@
session_defaults/0,
parse/2,
parse_config/2,
+ decode_buffer/2,
new_session/0
]).
@@ -52,6 +53,13 @@ parse_config(Element, Conf) ->
session_defaults() ->
{ok, true}.
+%% @spec decode_buffer(Buffer::binary(),Session::record(ldap)) -> NewBuffer::binary()
+%% @doc We need to decode buffer (remove chunks, decompress ...) for
+%% matching or dyn_variables
+%% @end
+decode_buffer(Buffer,_) ->
+ Buffer.
+
%%----------------------------------------------------------------------
%% Function: new_session/0
%% Purpose: initialize session information
View
@@ -40,6 +40,7 @@
session_defaults/0,
parse/2,
parse_config/2,
+ decode_buffer/2,
new_session/0]).
%%----------------------------------------------------------------------
@@ -50,6 +51,13 @@
session_defaults() ->
{ok, true}.
+%% @spec decode_buffer(Buffer::binary(),Session::record(mysql)) -> NewBuffer::binary()
+%% @doc We need to decode buffer (remove chunks, decompress ...) for
+%% matching or dyn_variables
+%% @end
+decode_buffer(Buffer,#mysql{}) ->
+ Buffer. % FIXME ?
+
%%----------------------------------------------------------------------
%% Function: new_session/0
%% Purpose: initialize session information
Oops, something went wrong.

0 comments on commit 96dcaf9

Please sign in to comment.