Permalink
Browse files

Add adhoc querying capabilities.

New module sqerl_adhoc generates SQL, with accompanying unit tests.
Updated sqerl with new entry points for adhoc_select, adhoc_insert
(with bulk support), and adhoc_delete.
  • Loading branch information...
langloisjp
langloisjp committed Oct 26, 2012
1 parent e514fb1 commit e37ed095a2fd078289f006c26d6459a4b879b773
Showing with 949 additions and 1 deletion.
  1. +176 −1 src/sqerl.erl
  2. +401 −0 src/sqerl_adhoc.erl
  3. +372 −0 test/sqerl_adhoc_tests.erl
View
@@ -32,7 +32,14 @@
statement/3,
statement/4,
execute/1,
- execute/2]).
+ execute/2,
+ adhoc_select/3,
+ adhoc_select/4,
+ adhoc_insert/2,
+ adhoc_insert/3,
+ adhoc_insert/4,
+ extract_insert_data/1,
+ adhoc_delete/2]).
-include_lib("eunit/include/eunit.hrl").
-include_lib("sqerl.hrl").
@@ -144,6 +151,172 @@ execute(QueryOrStatement, Parameters) ->
F = fun(Cn) -> sqerl_client:execute(Cn, QueryOrStatement, Parameters) end,
with_db(F).
+
+%% @doc Execute an adhoc query: adhoc_select(Columns, Table, Where)
+%% or adhoc_select(Columns, Table, Where, Clauses)
+%%
+%% Returns:
+%% - {ok, Rows}
+%% - {error, ErrorInfo}
+%%
+%% See execute/2 for more details on return data.
+%%
+%% Where Clause
+%% -------------
+%% Form: {where, Where}
+%%
+%% Where = all|undefined -- Does not generate a WHERE clause.
+%% Matches all records in table.
+%% Where = {Field, equals|nequals|gt|gte|lt|lte, Value}
+%% Where = {Field, in|notin, Values}
+%% Where = {'and'|'or', WhereList} -- Composes WhereList with AND or OR
+%%
+%% adhoc_select/4 takes an additional Clauses argument which
+%% is a list of additional clauses for the query.
+%%
+%% Order By Clause
+%% ---------------
+%% Form: {orderby, Fields | {Fields, asc|desc}}
+%%
+%% Limit/Offset Clause
+%% --------------------
+%% Form: {limit, Limit} | {limit, {Limit, offset, Offset}}
+%%
+%% See itest:adhoc_select_complex/0 for an example of a complex query
+%% that uses several clauses.
+%%
+%% -spec adhoc_select([binary() | string()], binary() | string(), sql_clause()) -> sql_result().
+adhoc_select(Columns, Table, Where) ->
+ adhoc_select(Columns, Table, Where, []).
+
+%% -spec adhoc_select([binary() | string()], binary() | string(), sql_clause(), [] | [sql_clause]) -> sql_result().
+adhoc_select(Columns, Table, Where, Clauses) ->
+ {SQL, Values} = sqerl_adhoc:select(Columns, Table,
+ [{where, Where}|Clauses], param_style()),
+ execute(SQL, Values).
+
+%% @doc Insert records.
+%%
+%% Prepares a statement and call it repeatedly.
+%% Inserts ?BULK_SIZE records at a time until
+%% there are fewer records and it inserts
+%% the rest at one time.
+%%
+%% - Columns, RowsValues
+%% e.g. {[<<"first_name">>, <<"last_name">>],
+%% [[<<"Joe">>, <<"Blow">>],
+%% [<<"John">>, <<"Doe">>]]}
+%%
+%% - Rows: list of proplists (such as returned by a select)
+%% e.g. [
+%% [{<<"id">>, 1},{<<"first_name">>, <<"Kevin">>}],
+%% [{<<"id">>, 2},{<<"first_name">>, <<"Mark">>}]
+%% ]
+%% Returns {ok, Count}
+%%
+%% 1> adhoc_insert(<<"users">>,
+%% {[<<"first_name">>, <<"last_name">>],
+%% [[<<"Joe">>, <<"Blow">>],
+%% [<<"John">>, <<"Doe">>]]}).
+%% {ok, 2}
+%%
+-define(DEFAULT_BATCH_SIZE, 100).
+-define(ADHOC_INSERT_STMT_ATOM, '__adhoc_insert').
+
+%% TODO: What if some inserts in a batch fail? Error out or continue?
+%% TODO: Transactionality? Retries?
+%% TODO: parallel inserts?
+
+adhoc_insert(Table, Rows) ->
+ adhoc_insert(Table, Rows, ?DEFAULT_BATCH_SIZE).
+
+adhoc_insert(Table, Rows, BatchSize) ->
+ %% reformat Rows to desired format
+ {Columns, RowsValues} = extract_insert_data(Rows),
+ adhoc_insert(Table, Columns, RowsValues, BatchSize).
+
+adhoc_insert(Table, Columns, RowsValues, BatchSize) when BatchSize > 0 ->
+ NumRows = length(RowsValues),
+ bulk_insert(Table, Columns, RowsValues, NumRows, BatchSize).
+
+bulk_insert(_Table, _Columns, _RowsValues, 0, _BatchSize) ->
+ {ok, 0};
+bulk_insert(Table, Columns, RowsValues, NumRows, BatchSize) when NumRows < BatchSize ->
+ %% Do one bulk insert since we have less than BULK_SIZE rows
+ SQL = sqerl_adhoc:insert(Table, Columns, length(RowsValues), param_style()),
+ execute(SQL, lists:flatten(RowsValues));
+bulk_insert(Table, Columns, RowsValues, NumRows, BatchSize) when NumRows >= BatchSize ->
+ %% Prepare a bulk insert statement and execute as many times as needed.
+ SQL = sqerl_adhoc:insert(Table, Columns, BatchSize, param_style()),
+ PrepInsertUnprepare = fun(Cn) ->
+ ok = sqerl_client:prepare(Cn, ?ADHOC_INSERT_STMT_ATOM, SQL),
+ try adhoc_prepared_insert(Cn, RowsValues, NumRows, BatchSize)
+ after sqerl_client:unprepare(Cn, ?ADHOC_INSERT_STMT_ATOM)
+ end
+ end,
+ {ok, Count, RemainingRowsValues} = with_db(PrepInsertUnprepare),
+ RemainingNumRows = NumRows - Count,
+ {ok, RemainingCount} = bulk_insert(Table, Columns, RemainingRowsValues, RemainingNumRows, BatchSize),
+ {ok, Count + RemainingCount}.
+
+%% @doc Insert data with insert statement already prepared
+adhoc_prepared_insert(Cn, RowsValues, NumRows, BatchSize) ->
+ adhoc_prepared_insert(Cn, RowsValues, NumRows, BatchSize, 0).
+
+adhoc_prepared_insert(Cn, RowsValues, NumRows, BatchSize, CountSoFar) when NumRows >= BatchSize ->
+ {RowsValuesToInsert, Rest} = lists:split(BatchSize, RowsValues),
+ {ok, Count} = sqerl_client:execute(Cn, ?ADHOC_INSERT_STMT_ATOM, lists:flatten(RowsValuesToInsert)),
+ adhoc_prepared_insert(Cn, Rest, NumRows - Count, BatchSize, CountSoFar + Count);
+adhoc_prepared_insert(_Cn, RowsValues, NumRows, BatchSize, CountSoFar) when NumRows < BatchSize ->
+ {ok, CountSoFar, RowsValues}.
+
+%% @doc Extract insert data from Rows (list of proplists).
+%% Assumes all rows have the same format.
+%% Returns {Columns, RowsValues}.
+%%
+%% 1> extract_insert_data([
+%% [{<<"id">>, 1}, {<<"name">>, <<"Joe">>}],
+%% [{<<"id">>, 2}, {<<"name">>, <<"Jeff">>}],
+%% ]).
+%% {[<<"id">>,<<"name">>],[[1,<<"Joe">>],[2,<<"Jeff">>]]}
+%%
+-spec extract_insert_data([[{binary(), any()}]]) -> {[binary()], [[any()]]}.
+extract_insert_data(Rows) ->
+ FirstRow = lists:nth(1, Rows),
+ Columns = [C || {C, _V} <- FirstRow],
+ RowsValues = [[V || {_C, V} <- Row] || Row <- Rows],
+ {Columns, RowsValues}.
+
+
+%% @doc Adhoc delete.
+%% Uses the same Where specifications as adhoc_select/3.
+%% Returns {ok, Count} or {error, ErrorInfo}.
+%%
+-spec adhoc_delete(binary(), term()) -> {ok, integer()} | {error, any()}.
+adhoc_delete(Table, Where) ->
+ {SQL, Values} = sqerl_adhoc:delete(Table, Where, param_style()),
+ execute(SQL, Values).
+
+%% The following illustrates how we could also implement adhoc update
+%% if ever desired.
+%%
+%% @doc Adhoc update.
+%% Updates records matching Where specifications with
+%% fields and values in given Row.
+%% Uses the same Where specifications as adhoc_select/3.
+%% Returns {ok, Count} or {error, ErrorInfo}.
+%%
+%%-spec adhoc_update(binary(), list(), term()) -> {ok, integer()} | {error, any()}.
+%%adhoc_update(Table, Row, Where) ->
+%% {SQL, Values} = sqerl_adhoc:update(Table, Row, Where, param_style()),
+%% execute(SQL, Values).
+
+
+%% @doc Shortcut for sqerl_client:parameter_style()
+-spec param_style() -> atom().
+param_style() -> sqerl_client:sql_parameter_style().
+
+
%% @doc Utility for generating specific message tuples from database-specific error
%% messages. The 1-argument form determines which database is being used by querying
%% Sqerl's configuration at runtime, while the 2-argument form takes the database type as a
@@ -179,3 +352,5 @@ do_parse_error({Code, Message}, CodeList) ->
false ->
{error, Message}
end.
+
+
Oops, something went wrong.

0 comments on commit e37ed09

Please sign in to comment.