Skip to content

Commit

Permalink
add support for dynamic variables for the postgresql plugin (TSUN-141)
Browse files Browse the repository at this point in the history
  • Loading branch information
@nniclausse
nniclausse committed Apr 14, 2010
1 parent d27a392 commit 0a335d9
Show file tree
Hide file tree
Showing 7 changed files with 211 additions and 14 deletions.
61 changes: 59 additions & 2 deletions doc/user_manual.html
View file
Expand Up @@ -96,7 +96,7 @@
<!--CUT DEF section 1 --><H1 ALIGN=center>Tsung User’s manual</H1><DIV CLASS="center">

<TABLE BORDER=1 CELLSPACING=0 CELLPADDING=1><TR><TD ALIGN=left NOWRAP bgcolor="#F2F2F2"> Version:</TD><TD ALIGN=left NOWRAP>1.3.2a</TD></TR>
<TR><TD ALIGN=left NOWRAP bgcolor="#F2F2F2"> Date :</TD><TD ALIGN=left NOWRAP>April 7, 2010</TD></TR>
<TR><TD ALIGN=left NOWRAP bgcolor="#F2F2F2"> Date :</TD><TD ALIGN=left NOWRAP>April 14, 2010</TD></TR>
</TABLE>
</DIV><!--TOC section Contents-->
<H2 CLASS="section"><!--SEC ANCHOR -->Contents</H2><!--SEC END --><UL CLASS="toc"><LI CLASS="li-toc">
Expand Down Expand Up @@ -2463,7 +2463,64 @@ <H5 CLASS="paragraph"><!--SEC ANCHOR -->JSONPath</H5><!--SEC END --><P>Another w
</TD></TR>
</TABLE></TD></TR>
</TABLE></TD></TR>
</TABLE><!--TOC subsubsection Loops, If-->
</TABLE><!--TOC paragraph PostgreSQL-->
<H5 CLASS="paragraph"><!--SEC ANCHOR -->PostgreSQL</H5><!--SEC END --><P>
<B>New in 1.3.2!</B></P><P>Since the postgreSQL protocol is binary, regexp are not useful to
parse the output of the server. Instead, a specific parsing can be
done to extract content from the server’s response; to do this, use the
<TT>pgsql_expr</TT> attribute. Use <FONT COLOR=purple>data_row[L][C]</FONT> to
extract the C colum of the L line of the data output. You can also use
the litteral name of the column (<I>ie.</I> the field name of the
table). This example extract 3 dynamic variables from the server’s
response:</P><P>First one, extract the 3rd column of the fourth row, then the <TT>mtime</TT>
field from the second row, and then it extract some data of the
row_description.</P><TABLE BORDER=0 CELLSPACING=0 CELLPADDING=0><TR><TD><TABLE BORDER=0 CELLPADDING=0
CELLSPACING=0><TR><TD BGCOLOR=black COLSPAN="3"><TABLE CELLSPACING="1" CELLPADDING=0 BORDER=0><TR><TD>
</TD></TR>
</TABLE></TD></TR>
<TR><TD BGCOLOR=black COLSPAN="1"><TABLE CELLSPACING="1" CELLPADDING=0 BORDER=0><TR><TD>
</TD></TR>
</TABLE></TD><TD><TABLE BORDER=0 CELLPADDING="1" CELLSPACING=0><TR><TD><PRE CLASS="verbatim"> &lt;request&gt;
&lt;dyn_variable name="myvar" pgsql_expr="data_row[4][3]"/&gt;
&lt;dyn_variable name="mtime" pgsql_expr="data_row[2].mtime"/&gt;
&lt;dyn_variable name="row" pgsql_expr="row_description[1][3][1]"/&gt;
&lt;pgsql type="sql"&gt;SELECT * from pgbench_history LIMIT 20;&lt;/pgsql&gt;
&lt;/request&gt;
</PRE></TD></TR>
</TABLE></TD><TD BGCOLOR=black COLSPAN="1"><TABLE CELLSPACING="1" CELLPADDING=0 BORDER=0><TR><TD>
</TD></TR>
</TABLE></TD></TR>
<TR><TD BGCOLOR=black COLSPAN="3"><TABLE CELLSPACING="1" CELLPADDING=0 BORDER=0><TR><TD>
</TD></TR>
</TABLE></TD></TR>
</TABLE></TD></TR>
</TABLE><P>A row description looks like this:
</P><TABLE BORDER=0 CELLSPACING=0 CELLPADDING=0><TR><TD><TABLE BORDER=0 CELLPADDING=0
CELLSPACING=0><TR><TD BGCOLOR=black COLSPAN="3"><TABLE CELLSPACING="1" CELLPADDING=0 BORDER=0><TR><TD>
</TD></TR>
</TABLE></TD></TR>
<TR><TD BGCOLOR=black COLSPAN="1"><TABLE CELLSPACING="1" CELLPADDING=0 BORDER=0><TR><TD>
</TD></TR>
</TABLE></TD><TD><TABLE BORDER=0 CELLPADDING="1" CELLSPACING=0><TR><TD><PRE CLASS="verbatim">=INFO REPORT==== 14-Apr-2010::11:03:22 ===
ts_pgsql:(7:&lt;0.102.0&gt;) PGSQL: Pair={row_description,
[{"tid",text,1,23,4,-1,16395},
{"bid",text,2,23,4,-1,16395},
{"aid",text,3,23,4,-1,16395},
{"delta",text,4,23,4,-1,
16395},
{"mtime",text,5,1114,8,-1,
16395},
{"filler",text,6,1042,-1,26,
16395}]}
</PRE></TD></TR>
</TABLE></TD><TD BGCOLOR=black COLSPAN="1"><TABLE CELLSPACING="1" CELLPADDING=0 BORDER=0><TR><TD>
</TD></TR>
</TABLE></TD></TR>
<TR><TD BGCOLOR=black COLSPAN="3"><TABLE CELLSPACING="1" CELLPADDING=0 BORDER=0><TR><TD>
</TD></TR>
</TABLE></TD></TR>
</TABLE></TD></TR>
</TABLE><P>So in the example, the <TT>row</TT> variable equals "aid".</P><!--TOC subsubsection Loops, If-->
<H4 CLASS="subsubsection"><!--SEC ANCHOR --><A NAME="htoc63">6.7.5</A>  Loops, If</H4><!--SEC END --><P><B>Since 1.3.0</B>, it’s now possible to add conditional/unconditional loops in a session:</P><!--TOC paragraph &lt;for&gt;-->
<H5 CLASS="paragraph"><!--SEC ANCHOR -->&lt;for&gt;</H5><!--SEC END --><P>Repeat the enclosing actions a fixed number of times. A dynamic
variable is used as counter, so the current iteration could be used in
Expand Down
42 changes: 42 additions & 0 deletions doc/user_manual.tex
View file
Expand Up @@ -1959,6 +1959,48 @@ \subsubsection{Checking the server's response}
<dyn_variable name="array3_value" json="field.array[3].value"/>
\end{Verbatim}
\paragraph{PostgreSQL}
\strong{New in 1.3.2!}
Since the postgreSQL protocol is binary, regexp are not useful to
parse the output of the server. Instead, a specific parsing can be
done to extract content from the server's response; to do this, use the
\varname{pgsql\_expr} attribute. Use \userinput{data\_row[L][C]} to
extract the C colum of the L line of the data output. You can also use
the litteral name of the column (\ie the field name of the
table). This example extract 3 dynamic variables from the server's
response:
First one, extract the 3rd column of the fourth row, then the \varname{mtime}
field from the second row, and then it extract some data of the
row\_description.
\begin{Verbatim}
<request>
<dyn_variable name="myvar" pgsql_expr="data_row[4][3]"/>
<dyn_variable name="mtime" pgsql_expr="data_row[2].mtime"/>
<dyn_variable name="row" pgsql_expr="row_description[1][3][1]"/>
<pgsql type="sql">SELECT * from pgbench_history LIMIT 20;</pgsql>
</request>
\end{Verbatim}
A row description looks like this:
\begin{Verbatim}
=INFO REPORT==== 14-Apr-2010::11:03:22 ===
ts_pgsql:(7:<0.102.0>) PGSQL: Pair={row_description,
[{"tid",text,1,23,4,-1,16395},
{"bid",text,2,23,4,-1,16395},
{"aid",text,3,23,4,-1,16395},
{"delta",text,4,23,4,-1,
16395},
{"mtime",text,5,1114,8,-1,
16395},
{"filler",text,6,1042,-1,26,
16395}]}
\end{Verbatim}
So in the example, the \varname{row} variable equals "aid".
\subsubsection{Loops, If}
\strong{Since 1.3.0}, it's now possible to add conditional/unconditional loops in a session:
Expand Down
81 changes: 79 additions & 2 deletions src/tsung/ts_pgsql.erl
View file
Expand Up @@ -42,6 +42,8 @@
session_defaults/0,
parse/2,
parse_config/2,
to_pairs/1,
find_pair/2,
new_session/0]).


Expand Down Expand Up @@ -198,8 +200,10 @@ subst(Req=#pgsql_request{sql=SQL}, DynData) ->
%%% -- Internal funs --------------------

%%----------------------------------------------------------------------
%% Function: process_head/1
%%----------------------------------------------------------------------
%% @spec process_head(Bin::binary()) -> {ok, Pair::list(), Rest::binary()} |more
%% @doc parse postgresql binary, and return a tuple or more if the
%% response is not complete
%% ----------------------------------------------------------------------
process_head(<<Code:8/integer, Size:4/integer-unit:8, Tail/binary>>) ->
?DebugF("PGSQL: received [~p] size=~p Pckt size= ~p ~n",[Code, Size, size(Tail)]),
RealSize = Size-4,
Expand All @@ -212,3 +216,76 @@ process_head(<<Code:8/integer, Size:4/integer-unit:8, Tail/binary>>) ->
false -> more
end;
process_head(_) -> more.

%%% -- funs related to dyn_variables

%% @spec to_pairs(Bin::binary()) -> list()
%% @doc transform postgres binary into list of pairs
to_pairs(Bin) ->
to_pairs(Bin,[]).
%% internal fun, with accumulator
to_pairs(<< >>, Acc) -> lists:reverse(Acc);
to_pairs(<<Code:8/integer, Size:4/integer-unit:8, Tail/binary>>, Acc) ->
RealSize = Size-4,
case RealSize =< size(Tail) of
true ->
<< Packet:RealSize/binary, Data/binary >> = Tail,
{ok, Pair} = pgsql_proto:decode_packet(Code, Packet),
to_pairs(Data, [Pair| Acc] );
false ->
%% partial bin, should not happen; anyway send the current accumulated pairs
?LOGF("real size too small, abort ?!~p (Tail was~p)~n",[Acc,Tail], ?NOTICE),
lists:reverse(Acc) %
end.

%% @spec find_pairs(Expr::string(),Pairs::list()) -> term()
%% @doc Expr: expression like data_row[4][2], Pairs: list of pairs
%% extracted by pgsql_proto:decode_packet.
find_pair(Expr,Pairs)->
Fun= fun(A) ->
case catch list_to_integer(A) of
I when is_integer(I) ->
I;
_ ->
list_to_atom(A)
end
end,
Str=re:replace(Expr,"\\[(\\d+)\\]","\.\\1",[{return,list},global]),
Keys=lists:map(Fun, string:tokens(Str,".")),
find_pair_real(Keys,Pairs,1).

find_pair_real([Key,Row,ColName],Pairs,CurRow) when is_atom(ColName)->
case get_col_id(atom_to_list(ColName),Pairs) of
Col when is_integer(Col) ->
find_pair_real([Key,Row,Col],Pairs,CurRow);
_ ->
undefined
end;
find_pair_real([Key,SameRow,Y,Z],[{Key,Value}|_],SameRow) when is_atom(Key), is_list(Value) ->
case lists:nth(Y,Value) of
L when is_list(L) ->
lists:nth(Z,L);
T when is_tuple(T) ->
element(Z,T);
_ ->
undefined
end;
find_pair_real([Key,Row,Col],[{Key,Val}|_],Row) when is_atom(Key),is_list(Val),is_integer(Col)->
lists:nth(Col,Val);
find_pair_real([Key,Row,Col|_],[{Key,Val}|_],Row) when is_atom(Key),is_tuple(Val)->
element(Col,Val);
find_pair_real(A=[Key|_],[{Key,_Value}|Pairs],CurRow) -> %same key,different row
find_pair_real(A,Pairs,CurRow+1);
find_pair_real(Expr,[_|Pairs],Row) ->% not the same key
find_pair_real(Expr,Pairs,Row);
find_pair_real(_,_,_) ->
undefined.

get_col_id(ColName,Pairs) ->
Desc=proplists:get_value(row_description,Pairs),
case lists:keysearch(ColName,1,Desc) of
{value,T} ->
element(3,T); % column id is the third element of the tuple.
false ->
undefined
end.
21 changes: 19 additions & 2 deletions src/tsung/ts_search.erl
View file
Expand Up @@ -303,16 +303,25 @@ parse_dynvar(D=[{jsonpath,_VarName, _Expr}| _DynVarsSpecs],
parse_dynvar(D,Binary,String,json_error,DynVars)
end;

parse_dynvar(D=[{pgsql_expr,_VarName, _Expr}| _DynVarsSpecs],
Binary,String,undefined,DynVars) ->
Pairs=ts_pgsql:to_pairs(Binary),
parse_dynvar(D,Binary,String,Pairs,DynVars);

parse_dynvar([{xpath,VarName,_Expr}|DynVarsSpecs],Binary,String,xpath_error,DynVars)->
?LOGF("Couldn't execute XPath: page not parsed (varname=~p) ~n",
?LOGF("Couldn't execute XPath: page not parsed (varname=~p)~n",
[VarName],?ERR),
parse_dynvar(DynVarsSpecs, Binary,String,xpath_error,DynVars);

parse_dynvar([{jsonpath,VarName,_Expr}|DynVarsSpecs],Binary,String,json_error,DynVars)->
?LOGF("Couldn't execute JSONPath: page not parsed (varname=~p) ~n",
?LOGF("Couldn't execute JSONPath: page not parsed (varname=~p)~n",
[VarName],?ERR),
parse_dynvar(DynVarsSpecs, Binary,String,json_error,DynVars);

parse_dynvar([{pgsql_expr,VarName,_Expr}|DynVarsSpecs],Binary,String,pgsql_error,DynVars)->
?LOGF("Couldn't decode pgsql expr from PGSQL binary (varname=~p)~n", [VarName],?ERR),
parse_dynvar(DynVarsSpecs, Binary,String,json_error,DynVars);

parse_dynvar([{xpath,VarName, Expr}| DynVarsSpecs],Binary,String,Tree,DynVars)->
Result = mochiweb_xpath:execute(Expr,Tree),
Value = mochiweb_xpath_utils:string_value(Result),
Expand All @@ -331,6 +340,14 @@ parse_dynvar([{jsonpath,VarName, Expr}| DynVarsSpecs],Binary,String,JSON,DynVars
end,
parse_dynvar(DynVarsSpecs, Binary,String,JSON,ts_dynvars:set(VarName,Values,DynVars));

parse_dynvar([{pgsql_expr,VarName, Expr}| DynVarsSpecs],Binary,String,PGSQL,DynVars)->
Values = ts_pgsql:find_pair(Expr,PGSQL),
case Values of
undefined -> ?LOGF("Dyn Var: no Match (varname=~p), ~n",[VarName],?WARN);
_ -> ?LOGF("Dyn Var: Match (~p=~p), ~n",[VarName,Values],?DEB)
end,
parse_dynvar(DynVarsSpecs, Binary,String,PGSQL,ts_dynvars:set(VarName,Values,DynVars));

parse_dynvar(Args, _Binary,_String,_Tree, _DynVars) ->
?LOGF("Bad args while parsing Dyn Var (~p)~n", [Args], ?ERR),
[].
Expand Down
2 changes: 1 addition & 1 deletion src/tsung/ts_utils.erl
View file
Expand Up @@ -746,7 +746,7 @@ json_get_bin([],Val) ->
Val;
json_get_bin([Key|Keys],undefined) ->
undefined;
json_get_bin([N|Keys],L) when is_integer(N) andalso N =< length(L) ->
json_get_bin([N|Keys],L) when is_integer(N), N =< length(L) ->
Val = lists:nth(N,L),
json_get_bin(Keys,Val);
json_get_bin([Key|Keys],{struct,JSON}) when is_list(JSON) ->
Expand Down
17 changes: 10 additions & 7 deletions src/tsung_controller/ts_config.erl
View file
Expand Up @@ -429,17 +429,20 @@ parse(#xmlElement{name=dyn_variable, attributes=Attrs},
StrName = ts_utils:clean_str(getAttr(Attrs, name)),
{ok, [{atom,1,Name}],1} = erl_scan:string("'"++StrName++"'"),
{Type,Expr} = case {getAttr(string,Attrs,regexp,none),
getAttr(string,Attrs,pgsql_expr,none),
getAttr(string,Attrs,xpath,none),
getAttr(string,Attrs,jsonpath,none)} of
{none,none,none} ->
{none,none,none,none} ->
DefaultRegExp = ?DEF_REGEXP_DYNVAR_BEGIN ++ StrName
++?DEF_REGEXP_DYNVAR_END,
{regexp,DefaultRegExp};
{none,XPath,none} ->
{none,none,XPath,none} ->
{xpath,XPath};
{none,none,JSONPath} ->
{none,none,none,JSONPath} ->
{jsonpath,JSONPath};
{RegExp,_,_} ->
{none,PG,none,none} ->
{pgsql_expr,PG};
{RegExp,_,_,_} ->
{regexp,RegExp}
end,
FlattenExpr =lists:flatten(Expr),
Expand All @@ -453,9 +456,9 @@ parse(#xmlElement{name=dyn_variable, attributes=Attrs},
?LOGF("Add new xpath: ~s ~n", [Expr],?INFO),
CompiledXPathExp = mochiweb_xpath:compile_xpath(FlattenExpr),
{xpath,Name,CompiledXPathExp};
jsonpath ->
?LOGF("Add new jsonpath: ~s ~n", [Expr],?INFO),
{jsonpath,Name,Expr}
Other ->
?LOGF("Add ~s ~s ~p ~n", [Type,Name,Expr],?INFO),
{Type,Name,Expr}
end,
NewDynVar = [DynVar|DynVars],
?LOGF("Add new dyn variable=~p in session ~p~n",
Expand Down
1 change: 1 addition & 0 deletions tsung-1.0.dtd
View file
Expand Up @@ -163,6 +163,7 @@ repeat | if | change_type )*>
name CDATA #REQUIRED
xpath CDATA #IMPLIED
jsonpath CDATA #IMPLIED
pgsql_expr CDATA #IMPLIED
regexp CDATA #IMPLIED >

<!ELEMENT http_header EMPTY >
Expand Down

0 comments on commit 0a335d9

Please sign in to comment.