Permalink
Browse files

Add a hybrid sliding window/uniform sample to bound size

  • Loading branch information...
1 parent fe57201 commit 7254f50ba9dd29bbf87163af259d9d338a41f73e @russelldb russelldb committed May 18, 2012
View
@@ -21,6 +21,14 @@
server
}).
+-record(slide_uniform, {
+ window = ?DEFAULT_SLIDING_WINDOW,
+ size = ?DEFAULT_SIZE,
+ reservoir = folsom_metrics_histogram_ets:new(folsom_slide_uniform,[ordered_set, {write_concurrency, true}, public]),
+ seed = now(),
+ server
+ }).
+
-record(uniform, {
size = ?DEFAULT_SIZE,
n = 1,
View
@@ -272,6 +272,12 @@ delete_histogram(Name, #histogram{type = slide, sample = #slide{reservoir = Rese
true = ets:delete(?HISTOGRAM_TABLE, Name),
true = ets:delete(?FOLSOM_TABLE, Name),
true = ets:delete(Reservoir),
+ ok;
+delete_histogram(Name, #histogram{type = slide_uniform, sample = #slide_uniform{reservoir = Reservoir, server=Pid}}) ->
+ folsom_sample_slide_server:stop(Pid),
+ true = ets:delete(?HISTOGRAM_TABLE, Name),
+ true = ets:delete(?FOLSOM_TABLE, Name),
+ true = ets:delete(Reservoir),
ok.
delete_history(Name, #history{tid = Tid}) ->
@@ -40,6 +40,8 @@ new(Name) ->
new(Name, slide) ->
new(Name, slide, ?DEFAULT_SLIDING_WINDOW);
+new(Name, slide_uniform) ->
+ new(Name, slide_uniform, {?DEFAULT_SLIDING_WINDOW, ?DEFAULT_SIZE});
new(Name, SampleType) ->
new(Name, SampleType, ?DEFAULT_SIZE).
View
@@ -36,6 +36,10 @@
%% API
+new(slide) ->
+ new(slide, ?DEFAULT_SLIDING_WINDOW);
+new(slide_uniform) ->
+ new(slide_uniform, {?DEFAULT_SLIDING_WINDOW, ?DEFAULT_SIZE});
new(Type) ->
new(Type, ?DEFAULT_SIZE, ?DEFAULT_ALPHA).
@@ -44,6 +48,8 @@ new(Type, Size) ->
new(slide, Size, _) ->
folsom_sample_slide:new(Size);
+new(slide_uniform, Sizes, _) ->
+ folsom_sample_slide_uniform:new(Sizes);
new(uniform, Size, _) ->
folsom_sample_uniform:new(Size);
new(none, Size, _) ->
@@ -58,7 +64,10 @@ update(none, Sample, Value) ->
update(exdec, Sample, Value) ->
folsom_sample_exdec:update(Sample, Value);
update(slide, Sample, Value) ->
- folsom_sample_slide:update(Sample, Value).
+ folsom_sample_slide:update(Sample, Value);
+update(slide_uniform, Sample, Value) ->
+ folsom_sample_slide_uniform:update(Sample, Value).
+
get_values(uniform, Sample) ->
folsom_sample_uniform:get_values(Sample);
@@ -67,4 +76,6 @@ get_values(none, Sample) ->
get_values(exdec, Sample) ->
folsom_sample_exdec:get_values(Sample);
get_values(slide, Sample) ->
- folsom_sample_slide:get_values(Sample).
+ folsom_sample_slide:get_values(Sample);
+get_values(slide_uniform, Sample) ->
+ folsom_sample_slide_uniform:get_values(Sample).
@@ -35,7 +35,7 @@
new(Size) ->
Sample = #slide{window = Size},
- Pid = folsom_sample_slide_sup:start_slide_server(Sample#slide.reservoir, Sample#slide.window),
+ Pid = folsom_sample_slide_sup:start_slide_server(?MODULE, Sample#slide.reservoir, Sample#slide.window),
Sample#slide{server=Pid}.
update(#slide{reservoir = Reservoir} = Sample, Value) ->
@@ -31,22 +31,22 @@
-behaviour(gen_server).
%% API
--export([start_link/2, stop/1]).
+-export([start_link/3, stop/1]).
--record(state, {reservoir, window}).
+-record(state, {sample_mod, reservoir, window}).
%% gen_server callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
terminate/2, code_change/3]).
-start_link(Reservoir, Window) ->
- gen_server:start_link(?MODULE, [Reservoir, Window], []).
+start_link(SampleMod, Reservoir, Window) ->
+ gen_server:start_link(?MODULE, [SampleMod, Reservoir, Window], []).
stop(Pid) ->
gen_server:cast(Pid, stop).
-init([Reservoir, Window]) ->
- {ok, #state{reservoir = Reservoir, window = Window}, timeout(Window)}.
+init([SampleMod, Reservoir, Window]) ->
+ {ok, #state{sample_mod = SampleMod, reservoir = Reservoir, window = Window}, timeout(Window)}.
handle_call(_Request, _From, State) ->
Reply = ok,
@@ -57,8 +57,8 @@ handle_cast(stop, State) ->
handle_cast(_Msg, State) ->
{noreply, State}.
-handle_info(timeout, State=#state{reservoir = Reservoir, window = Window}) ->
- folsom_sample_slide:trim(Reservoir, Window),
+handle_info(timeout, State=#state{sample_mod = SampleMod, reservoir = Reservoir, window = Window}) ->
+ SampleMod:trim(Reservoir, Window),
{noreply, State, timeout(Window)};
handle_info(_Info, State) ->
{noreply, State}.
@@ -33,13 +33,13 @@
]).
%% public functions
--export([start_slide_server/2]).
+-export([start_slide_server/3]).
start_link () ->
supervisor:start_link({local,?MODULE},?MODULE,[]).
-start_slide_server(Reservoir, Window) ->
- {ok, Pid} = supervisor:start_child(?MODULE, [Reservoir, Window]),
+start_slide_server(SampleMod, Reservoir, Window) ->
+ {ok, Pid} = supervisor:start_child(?MODULE, [SampleMod, Reservoir, Window]),
Pid.
%% @private
@@ -0,0 +1,72 @@
+%%%
+%%% Copyright 2012, Basho Technologies, Inc. All Rights Reserved.
+%%%
+%%% Licensed under the Apache License, Version 2.0 (the "License");
+%%% you may not use this file except in compliance with the License.
+%%% You may obtain a copy of the License at
+%%%
+%%% http://www.apache.org/licenses/LICENSE-2.0
+%%%
+%%% Unless required by applicable law or agreed to in writing, software
+%%% distributed under the License is distributed on an "AS IS" BASIS,
+%%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%%% See the License for the specific language governing permissions and
+%%% limitations under the License.
+%%%
+%%%-------------------------------------------------------------------
+%%% File: folsom_sample_slide.erl
+%%% @author Russell Brown <russelldb@basho.com>
+%%% @doc
+%%% Sliding window sample. Last Window seconds readings are recorded.
+%%% @end
+%%%-----------------------------------------------------------------
+
+-module(folsom_sample_slide_uniform).
+
+-export([
+ new/1,
+ update/2,
+ get_values/1,
+ moment/0,
+ trim/2
+ ]).
+
+-include("folsom.hrl").
+
+new({Window, SampleSize}) ->
+ Sample = #slide_uniform{window = Window, size = SampleSize},
+ Pid = folsom_sample_slide_sup:start_slide_server(?MODULE, Sample#slide_uniform.reservoir, Sample#slide_uniform.window),
+ Sample#slide_uniform{server=Pid}.
+
+update(#slide_uniform{reservoir = Reservoir, size = Size, seed = Seed} = Sample0, Value) ->
+ Moment = moment(),
+ ets:insert_new(Reservoir, {Moment, 0}),
+ MCnt = ets:update_counter(Reservoir, Moment, 1),
+ Sample = case MCnt > Size of
+ true ->
+ {Rnd, NewSeed} = random:uniform_s(Size, Seed),
+ maybe_update(Reservoir, {{Moment, Rnd}, Value}, Size),
+ Sample0#slide_uniform{seed = NewSeed};
+ false ->
+ ets:insert(Reservoir, {{Moment, MCnt}, Value}),
+ Sample0
+ end,
+ Sample.
+
+maybe_update(Reservoir, {{_Moment, Rnd}, _Value}=Obj, Size) when Rnd =< Size ->
+ ets:insert(Reservoir, Obj);
+maybe_update(_Reservoir, _Obj, _Size) ->
+ ok.
+
+get_values(#slide_uniform{window = Window, reservoir = Reservoir}) ->
+ Oldest = moment() - Window,
+ ets:select(Reservoir, [{{{'$1', '_'},'$2'},[{'>=', '$1', Oldest}],['$2']}]).
+
+moment() ->
+ folsom_utils:now_epoch().
+
+trim(Reservoir, Window) ->
+ Oldest = moment() - Window,
+ ets:select_delete(Reservoir, [{{{'$1', '_'},'_'},[{'<', '$1', Oldest}],['true']}]),
+ %% and trim the counters
+ ets:select_delete(Reservoir, [{{'$1','_'},[{is_integer, '$1'}, {'<', '$1', Oldest}],['true']}]).
View
@@ -50,4 +50,3 @@ now_epoch_micro() ->
get_ets_size(Tab) ->
ets:info(Tab, size).
-
View
@@ -23,13 +23,18 @@
-module(slide_statem_eqc).
+-compile(export_all).
+
+-ifdef(TEST).
+-ifdef(EQC).
+
-include("folsom.hrl").
-include_lib("eqc/include/eqc.hrl").
-include_lib("eqc/include/eqc_statem.hrl").
-include_lib("eunit/include/eunit.hrl").
--compile(export_all).
+
-define(NUMTESTS, 200).
-define(QC_OUT(P),
@@ -40,6 +45,7 @@
-record(state, {moment=1000,
sample,
+ name,
values=[]}).
initial_state() ->
@@ -57,14 +63,14 @@ command(S) ->
%% Next state transformation, S is the current state
next_state(S, V, {call, ?MODULE, new_histo, []}) ->
- S#state{sample=V};
+ S#state{name={call, erlang, element, [1, V]}, sample={call, erlang, element, [2, V]}};
next_state(S, V, {call, ?MODULE, tick, [_Moment]}) ->
S#state{moment=V};
next_state(#state{moment=Moment, values=Values0}=S, V, {call, ?MODULE, update, [_, _Val]}) ->
- S#state{values=Values0 ++[{Moment, V}]};
+ S#state{values=Values0 ++ [{Moment, V}]};
next_state(#state{values=Values, moment=Moment}=S, _V, {call, ?MODULE, trim, _}) ->
%% trim the model
- S#state{values=trim(Values, Moment, ?WINDOW)};
+ S#state{values = trim(Values, Moment, ?WINDOW)};
next_state(S,_V,{call, ?MODULE, _, _}) ->
S.
@@ -80,11 +86,23 @@ precondition(_S, {call, _, _, _}) ->
%% OBS: S is the state before next_state(S,_,<command>)
postcondition(#state{values=Values0, moment=Moment}, {call, ?MODULE, get_values, _}, Res) ->
Values = [V || {K, V} <- Values0, K >= Moment - ?WINDOW],
- lists:sort(Values) == lists:sort(Res);
-postcondition(#state{values=Values, sample={_Name, Sample}, moment=Moment}, {call, ?MODULE, trim, _}, _TrimCnt) ->
+ case lists:sort(Values) == lists:sort(Res) of
+ true ->
+ true;
+ _ ->
+ {"get values", {"model", lists:sort(Values)},
+ {"smaple", lists:sort(Res)}}
+ end;
+postcondition(#state{values=Values, sample=Sample, moment=Moment}, {call, ?MODULE, trim, _}, _TrimCnt) ->
%% check that values and the actual table contents are the same after a trim
Table = ets:tab2list(Sample#slide.reservoir),
- lists:sort(trim(Values, Moment, ?WINDOW)) == lists:sort(Table);
+ Model = lists:sort(trim(Values, Moment, ?WINDOW)),
+ case Model == lists:sort(Table) of
+ true ->
+ true;
+ _ ->
+ {"after trim", {"model", Model}, {"sample", lists:sort(Table)}}
+ end;
postcondition(_S, {call, ?MODULE, _, _}, _Res) ->
true.
@@ -103,10 +121,10 @@ prop_window() ->
{Actual, Expected} = case S#state.sample of
undefined ->
{S#state.values, []};
- {Name, _Sample} ->
- A = folsom_metrics:get_metric_value(Name),
+ Sample ->
+ A = folsom_metrics:get_metric_value(S#state.name),
E = [V || {K, V} <- S#state.values, K >= S#state.moment - ?WINDOW],
- folsom_metrics:delete_metric(Name),
+ folsom_metrics:delete_metric(S#state.name),
{A, E}
end,
?WHENFAIL(
@@ -128,16 +146,19 @@ tick(Moment) ->
meck:expect(folsom_utils, now_epoch, fun() -> Moment + IncrBy end),
Moment+IncrBy.
-update({_Name, Sample}, Val) ->
+update(Sample, Val) ->
Sample = folsom_sample_slide:update(Sample, Val),
Val.
-trim({_Name, Sample}, Window) ->
+trim(Sample, Window) ->
folsom_sample_slide:trim(Sample#slide.reservoir, Window).
-get_values({_Name, Sample}) ->
+get_values(Sample) ->
folsom_sample_slide:get_values(Sample).
%% private
trim(L, Moment, Window) ->
[{K, V} || {K, V} <- L, K >= Moment - Window].
+
+-endif.
+-endif.
Oops, something went wrong.

0 comments on commit 7254f50

Please sign in to comment.