Merge remote-tracking branch 'upstream/master'

README.md

@@ -73,6 +73,45 @@ Histograms are collections of values that have statistical analysis done to them
       > folsom_metrics:histogram_timed_update(Name, Fun).
       > folsom_metrics:notify({Name, Value}).
 
+###### Histogram sample types
+
+Each histogram draws its values from a `reservoir` of readings. You can select a `sample type` for a histogram by passing the name of the sample type as an atom when you create a new histogram.
+Some sample types have further arguments. The purpose of a sample type is to control the size and charecteristics of the reservoir of readings the histogram performs analysis upon.
+
+Folsom currently provides the following sample types:
+
+######  `uniform`
+
+This is a random uniform sample over the stream of readings. This is the default sample type, bounded in size to 1028 readings. When `size` readings have been taken, new readings replace older readings
+in the reservoir at random. You can set the sample size at creation time:
+
+      > folsom_metrics:new_histogram(Name, uniform, Size::integer()).
+
+Be sure you understand _why_ before you do this.
+
+###### `exdec`
+
+This is a  sample that exponentially decays less significant readings over time so as to give greater significance to newer readings. Read more here -
+[Forward Decay...](http://www.research.att.com/people/Cormode_Graham/library/publications/CormodeShkapenyukSrivastavaXu09.pdf).
+Again you can change defaults at creation time, if you think you need to:
+
+    > folsom_metrics:new_histogram(Name, exdec, Size::integer(), Alpha::float()).
+
+###### `slide`
+
+This is a sliding window in time over a stream of readings. The default window size is 60 seconds. Every reading that occurs in a sliding sixty second window is stored,
+with older readings being discarded. If you have a lot of readings per
+minute the `reservoir` may get pretty big and so it will take more time to calculate statistics. You can set the `window` size by providing a number of seconds.
+
+    > folsom_metrics:new_histogram(Name, slide, Seconds::integer()).
+
+###### `slide_uniform`
+
+This is a sliding window in time over a stream of readings with a random uniform sample per second, to bound the size of the total number of readings. The maximum size of the reservoir will be
+ `window size * sample size`. Default is a window of 60 seconds and a sample size of 1028. Again, you can change these at creation time:
+
+    > folsom_metrics:new_histogram(Name, slide_uniform, {Secs::interger(), Size::integer()).
+
 ##### Histories
 
 Histories are a collection of past events, such as errors or log messages.
@@ -88,6 +127,13 @@ Meters are increment only counters with mean rates and exponentially weighted mo
       > folsom_metrics:new_meter(Name).
       > folsom_metrics:notify({Name, Value}).
 
+###### `Spiral` meter
+
+A `spiral` is a type of meter that has a one minute sliding window count. The meter tracks an increment only counter and a total for the last minute. This is a sliding count with older readings dropping off per second.
+
+    > folsom_metrics:new_spiral(Name).
+    > folsom_metrics:notify({Name, Count}).
+
 ##### Meter Reader
 
 Meter readers are like a meter except that the values passed to it are monotonically increasing, e.g., reading from a water or gas meter, CPU jiffies, or I/O operation count.

include/folsom.hrl

@@ -6,6 +6,7 @@
 -define(METER_READER_TABLE, folsom_meter_readers).
 -define(HISTORY_TABLE, folsom_histories).
 -define(DURATION_TABLE, folsom_durations).
+-define(SPIRAL_TABLE, folsom_spirals).
 
 -define(DEFAULT_LIMIT, 5).
 -define(DEFAULT_SIZE, 1028). % mimic codahale's metrics
@@ -14,6 +15,14 @@
 -define(DEFAULT_INTERVAL, 5000).
 -define(DEFAULT_SAMPLE_TYPE, uniform).
 
+-record(spiral, {
+          tid = folsom_metrics_histogram_ets:new(folsom_spiral,
+                                                 [ordered_set,
+                                                  {write_concurrency, true},
+                                                  public]),
+          server
+         }).
+
 -record(slide, {
           window = ?DEFAULT_SLIDING_WINDOW,
           reservoir = folsom_metrics_histogram_ets:new(folsom_slide,
@@ -79,7 +88,7 @@
                       dist_buf_busy_limit,
                       %fullsweep_after, % included in garbage_collection
                       garbage_collection,
-                      global_heaps_size,
+                      %global_heaps_size, % deprecated
                       heap_sizes,
                       heap_type,
                       info,

rebar.config

@@ -1,7 +1,7 @@
 {sub_dirs, ["deps"]}.
 
 {deps, [
-    {'bear', ".*", {git, "git://github.com/boundary/bear.git", "master"}},
+    {'bear', ".*", {git, "git://github.com/boundary/bear.git", {tag, "0.1.1"}}},
     {meck, ".*", {git, "git://github.com/eproxus/meck", "master"}}
 ]}.
 

src/folsom.erl

@@ -36,24 +36,29 @@ stop() ->
 
 start(_Type, _Args) ->
     {ok, Pid} = folsom_sup:start_link(),
-    lists:foreach(
-      fun ({K, New}) ->
-              case application:get_env(?APP, K) of
-                  {ok, Name} when is_atom(Name) ->
-                      New(Name);
-                  {ok, Names} when is_list(Names) ->
-                      lists:foreach(New, Names);
-                  undefined ->
-                      ok
-              end
-      end,
-      [{counter, fun folsom_metrics:new_counter/1},
-       {gauge, fun folsom_metrics:new_gauge/1},
-       {histogram, fun folsom_metrics:new_histogram/1},
-       {history, fun folsom_metrics:new_history/1},
-       {meter, fun folsom_metrics:new_meter/1},
-       {meter_reader, fun folsom_metrics:new_meter_reader/1}]),
+    lists:foreach(fun configure/1,
+      [{counter, new_counter},
+       {gauge, new_gauge},
+       {histogram, new_histogram},
+       {history, new_history},
+       {meter, new_meter},
+       {meter_reader, new_meter_reader}]),
     {ok, Pid}.
 
 stop(_State) ->
     ok.
+
+%% internal
+configure({K, New}) ->
+    case application:get_env(?APP, K) of
+        {ok, Specs} when is_list(Specs) ->
+            [configure_metric(New, Spec) || Spec <- Specs];
+        {ok, Spec} ->
+            configure_metric(New, Spec);
+        undefined -> ok
+    end.
+
+configure_metric(New, Spec) when is_list(Spec) ->
+    apply(folsom_metrics, New, Spec);
+configure_metric(New, Name) ->
+    folsom_metrics:New(Name).

src/folsom_ets.erl

@@ -140,6 +140,8 @@ get_values(Name, meter_reader) ->
     folsom_metrics_meter_reader:get_values(Name);
 get_values(Name, duration) ->
     folsom_metrics_duration:get_values(Name);
+get_values(Name, spiral) ->
+    folsom_metrics_spiral:get_values(Name);
 get_values(_, Type) ->
     {error, Type, unsupported_metric_type}.
 
@@ -181,6 +183,10 @@ maybe_add_handler(meter_reader, Name, false) ->
     true = folsom_metrics_meter_reader:new(Name),
     true = ets:insert(?FOLSOM_TABLE, {Name, #metric{type = meter_reader}}),
     ok;
+maybe_add_handler(spiral, Name, false) ->
+    true = folsom_metrics_spiral:new(Name),
+    true = ets:insert(?FOLSOM_TABLE, {Name, #metric{type = spiral}}),
+    ok;
 maybe_add_handler(Type, _, false) ->
     {error, Type, unsupported_metric_type};
 maybe_add_handler(_, Name, true) ->
@@ -251,6 +257,13 @@ delete_metric(Name, meter) ->
 delete_metric(Name, meter_reader) ->
     true = ets:delete(?METER_READER_TABLE, Name),
     true = ets:delete(?FOLSOM_TABLE, Name),
+    ok;
+delete_metric(Name, spiral) ->
+    #spiral{tid=Tid, server=Pid} = folsom_metrics_spiral:get_value(Name),
+    folsom_sample_slide_server:stop(Pid),
+    true = ets:delete(Tid),
+    ets:delete(?SPIRAL_TABLE, Name),
+    ets:delete(?FOLSOM_TABLE, Name),
     ok.
 
 delete_histogram(Name, #histogram{type = uniform, sample = #uniform{reservoir = Reservoir}}) ->
@@ -344,6 +357,13 @@ notify(Name, Value, duration, false) ->
     add_handler(duration, Name),
     folsom_metrics_duration:update(Name, Value),
     ok;
+notify(Name, Value, spiral, true) ->
+    folsom_metrics_spiral:update(Name, Value),
+    ok;
+notify(Name, Value, spiral, false) ->
+    add_handler(spiral, Name),
+    folsom_metrics_spiral:update(Name, Value),
+    ok;
 notify(_, _, Type, _) ->
     {error, Type, unsupported_metric_type}.
 

src/folsom_metrics.erl

@@ -39,6 +39,7 @@
          new_duration/2,
          new_duration/3,
          new_duration/4,
+         new_spiral/1,
          delete_metric/1,
          notify/1,
          notify/2,
@@ -103,6 +104,9 @@ new_duration(Name, SampleType, SampleSize) ->
 new_duration(Name, SampleType, SampleSize, Alpha) ->
     folsom_ets:add_handler(duration, Name, SampleType, SampleSize, Alpha).
 
+new_spiral(Name) ->
+    folsom_ets:add_handler(spiral, Name).
+
 delete_metric(Name) ->
     folsom_ets:delete_handler(Name).
 

src/folsom_metrics_duration.erl

@@ -45,10 +45,10 @@ new(Name) ->
     ets:insert(?DURATION_TABLE, Dur).
 
 update(Name, timer_start) ->
-    StartTime = erlang:now(),
+    StartTime = os:timestamp(),
     ets:update_element(?DURATION_TABLE, Name, {3, StartTime});
 update(Name, timer_end) ->
-    EndTime = erlang:now(),
+    EndTime = os:timestamp(),
     case ets:lookup_element(?DURATION_TABLE, Name, 3) of
         undefined ->
             ok;

src/folsom_metrics_spiral.erl

@@ -0,0 +1,74 @@
+%%%
+%%% Copyright 2012, Basho Technologies, Inc.  All Rights Reserved.
+%%%
+%%% Licensed under the Apache License, Version 2.0 (the "License");
+%%% you may not use this file except in compliance with the License.
+%%% You may obtain a copy of the License at
+%%%
+%%%     http://www.apache.org/licenses/LICENSE-2.0
+%%%
+%%% Unless required by applicable law or agreed to in writing, software
+%%% distributed under the License is distributed on an "AS IS" BASIS,
+%%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%%% See the License for the specific language governing permissions and
+%%% limitations under the License.
+%%%
+
+
+%%%-------------------------------------------------------------------
+%%% File:      folsom_metrics_spiral.erl
+%%% @author    Russell Brown <russelldb@basho.com>
+%%% @doc A total count, and sliding window count of events over the last
+%%%      minute.
+%%% @end
+%%%------------------------------------------------------------------
+
+-module(folsom_metrics_spiral).
+
+-export([new/1,
+         update/2,
+         trim/2,
+         get_value/1,
+         get_values/1
+        ]).
+
+%% size of the window in seconds
+-define(WINDOW, 60).
+
+-include("folsom.hrl").
+
+new(Name) ->
+    Spiral = #spiral{},
+    Pid = folsom_sample_slide_sup:start_slide_server(?MODULE,
+                                                           Spiral#spiral.tid,
+                                                           ?WINDOW),
+    ets:insert_new(Spiral#spiral.tid, {count, 0}),
+    ets:insert(?SPIRAL_TABLE, {Name, Spiral#spiral{server=Pid}}).
+
+update(Name, Value) ->
+    #spiral{tid=Tid} = get_value(Name),
+    Moment = folsom_utils:now_epoch(),
+    ets:insert_new(Tid, {Moment, 0}),
+    ets:update_counter(Tid, Moment, Value),
+    ets:update_counter(Tid, count, Value).
+
+get_value(Name) ->
+    [{Name, Spiral}] =  ets:lookup(?SPIRAL_TABLE, Name),
+    Spiral.
+
+trim(Tid, _Window) ->
+    Oldest = oldest(),
+    ets:select_delete(Tid, [{{'$1','_'}, [{is_integer, '$1'}, {'<', '$1', Oldest}], ['true']}]).
+
+get_values(Name) ->
+    Oldest = oldest(),
+    #spiral{tid=Tid} = get_value(Name),
+    [{count, Count}] = ets:lookup(Tid, count),
+    One =lists:sum(ets:select(Tid, [{{'$1','$2'},[{is_integer, '$1'}, {'>=', '$1', Oldest}],['$2']}])),
+
+    [{count, Count}, {one, One}].
+
+oldest() ->
+    folsom_utils:now_epoch() - ?WINDOW.
+
+

src/folsom_sup.erl

@@ -107,7 +107,8 @@ create_tables() ->
               {?METER_TABLE, [set, named_table, public, {write_concurrency, true}]},
               {?METER_READER_TABLE, [set, named_table, public, {write_concurrency, true}]},
               {?HISTORY_TABLE, [set, named_table, public, {write_concurrency, true}]},
-              {?DURATION_TABLE, [ordered_set, named_table, public, {write_concurrency, true}]}
+              {?DURATION_TABLE, [ordered_set, named_table, public, {write_concurrency, true}]},
+              {?SPIRAL_TABLE, [set, named_table, public, {write_concurrency, true}]}
              ],
     [maybe_create_table(ets:info(Name), Name, Opts) || {Name, Opts} <- Tables],
     ok.

src/folsom_utils.erl

@@ -41,11 +41,11 @@ convert_tags(Tags) ->
     [to_atom(Tag) || Tag <- Tags].
 
 now_epoch() ->
-    {Mega, Sec, _} = erlang:now(),
+    {Mega, Sec, _} = os:timestamp(),
     (Mega * 1000000 + Sec).
 
 now_epoch_micro() ->
-    {Mega, Sec, Micro} = erlang:now(),
+    {Mega, Sec, Micro} = os:timestamp(),
     (Mega * 1000000 + Sec) * 1000000 + Micro.
 
 get_ets_size(Tab) ->

src/folsom_vm_metrics.erl

@@ -35,7 +35,7 @@
         ]).
 
 %% exported for eunit test
--export([convert_cpu_topology/2]).
+-export([convert_system_info/1]).
 
 -include("folsom.hrl").
 
@@ -98,8 +98,10 @@ convert_system_info({c_compiler_used, {Compiler, Version}}) ->
     [{compiler, Compiler}, {version, convert_c_compiler_version(Version)}];
 convert_system_info({cpu_topology, undefined}) ->
     undefined;
-convert_system_info({cpu_topology, List}) ->
+convert_system_info({cpu_topology, List}) when is_list(List) ->
     [{Type, convert_cpu_topology(Item, [])} || {Type, Item} <- List];
+convert_system_info({cpu_topology, {logical,Item}}) ->
+    convert_system_info({cpu_topology, [{processor,[{core,{logical,Item}}]}]});
 convert_system_info({dist_ctrl, List}) ->
     lists:map(fun({Node, Socket}) ->
                       {ok, Stats} = inet:getstat(Socket),
@@ -130,22 +132,24 @@ convert_allocated_areas({Key, Value}) ->
 convert_c_compiler_version({A, B, C}) ->
     list_to_binary(io_lib:format("~p.~p.~p", [A, B, C]));
 convert_c_compiler_version({A, B}) ->
-    list_to_binary(io_lib:format("~p.~p", [A, B])).
+    list_to_binary(io_lib:format("~p.~p", [A, B]));
+convert_c_compiler_version(A) ->
+    list_to_binary(io_lib:format("~p", [A])).
 
 convert_cpu_topology([{core, Value}| Tail], Acc) when is_tuple(Value) ->
-  convert_cpu_topology(Tail, lists:append(Acc, [{core, tuple_to_list(Value)}]));
+    convert_cpu_topology(Tail, lists:append(Acc, [{core, tuple_to_list(Value)}]));
 convert_cpu_topology([{core, Value}| Tail], Acc) when is_list(Value) ->
-  convert_cpu_topology(Tail, lists:append(Acc, [{core, convert_cpu_topology(Value, [])}]));
+    convert_cpu_topology(Tail, lists:append(Acc, [{core, convert_cpu_topology(Value, [])}]));
 convert_cpu_topology([{thread, Value}| Tail], Acc) ->
-  convert_cpu_topology(Tail, lists:append(Acc, [{thread, tuple_to_list(Value)}]));
-convert_cpu_topology({logical, Value}, Acc) ->
-  convert_cpu_topology([], lists:append(Acc, [logical, Value]));
+    convert_cpu_topology(Tail, lists:append(Acc, [{thread, tuple_to_list(Value)}]));
 convert_cpu_topology([{node, Value}| Tail], Acc) ->
-  convert_cpu_topology(Tail, lists:append(Acc, [{node, convert_cpu_topology(Value, [])}]));
+    convert_cpu_topology(Tail, lists:append(Acc, [{node, convert_cpu_topology(Value, [])}]));
 convert_cpu_topology([{processor, Value}| Tail], Acc) ->
-  convert_cpu_topology(Tail, lists:append(Acc, [{processor, convert_cpu_topology(Value, [])}]));
+    convert_cpu_topology(Tail, lists:append(Acc, [{processor, convert_cpu_topology(Value, [])}]));
+convert_cpu_topology({Key, Value}, _) ->
+    [{Key, Value}];
 convert_cpu_topology([], Acc) ->
-  Acc.
+    Acc.
 
 get_process_info(Pid) ->
     Info = [process_info(Pid, Key) || Key <- ?PROCESS_INFO],