Add token management to Background manager.

Makefile

@@ -7,7 +7,7 @@ PULSE_TESTS = worker_pool_pulse
 
 all: deps compile
 
-compile:
+compile:	
 	./rebar compile
 
 deps:
@@ -19,7 +19,7 @@ clean:
 distclean: clean
 	./rebar delete-deps
 
-test: all
+test: all	
 	./rebar skip_deps=true eunit
 
 # You should 'clean' before your first run of this target

ebin/riak_core.app

@@ -3,7 +3,7 @@
 {application, riak_core,
 [
   {description, "Riak Core"},
-  {vsn, "1.4.0"},
+  {vsn, "1.4.1"},
   {modules, [
              app_helper,
              bloom,
@@ -87,7 +87,10 @@
              riak_core_ssl_util,
              supervisor_pre_r14b04,
              vclock,
-             riak_core_bg_manager
+             riak_core_bg_manager,
+             riak_core_token_manager,
+             riak_core_table_manager,
+             riak_core_bg_manager_sup
             ]},
   {registered, []},
   {included_applications, [folsom]},

include/riak_core_bg_manager.hrl

@@ -0,0 +1,23 @@
+%% -------------------------------------------------------------------
+%%
+%% Copyright (c) 2013 Basho Technologies, Inc.  All Rights Reserved.
+%%
+%% This file is provided to you under the Apache License,
+%% Version 2.0 (the "License"); you may not use this file
+%% except in compliance with the License.  You may obtain
+%% a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+%%
+%% -------------------------------------------------------------------
+-define(LM_ETS_TABLE, lock_mgr_table).   %% name of private lock manager ETS table
+-define(LM_ETS_OPTS, [private, bag]).    %% creation time properties of lock manager ETS table
+
+-type concurrency_limit() :: non_neg_integer() | infinity.

include/riak_core_token_manager.hrl

@@ -0,0 +1,53 @@
+%% -------------------------------------------------------------------
+%%
+%% Copyright (c) 2013 Basho Technologies, Inc.  All Rights Reserved.
+%%
+%% This file is provided to you under the Apache License,
+%% Version 2.0 (the "License"); you may not use this file
+%% except in compliance with the License.  You may obtain
+%% a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+%%
+%% -------------------------------------------------------------------
+-type tm_token() :: any().
+-type tm_meta()  :: {atom(), any()}.                %% meta data to associate with a token
+-type tm_period() :: pos_integer().                 %% refill period in milliseconds
+-type tm_count() :: pos_integer().                  %% refill tokens to count at each refill period
+-type tm_rate() :: {tm_period(), tm_count()}.       %% token refresh rate
+-type tm_stat_event() :: refill_event | give_event. %% stat event type
+
+%% Results of a "ps" of live given or blocked tokens
+-record(tm_stat_live,
+        {
+          token      :: tm_token(),               %% token type
+          consumer   :: pid(),                    %% process asking for token
+          meta       :: [tm_meta()],              %% associated meta data
+          state      :: given | blocked | failed  %% result of last request
+        }).
+-type tm_stat_live() :: #tm_stat_live{}.
+
+%% Results of a "head" or "tail", per token
+-record(tm_stat_hist,
+        {
+          limit   :: tm_count(),       %% maximum available, defined by token rate during interval
+          refills :: tm_count(),       %% number of times this token was refilled during interval
+          given   :: tm_count(),       %% number of this token type given in interval
+          blocked :: tm_count()        %% number of blocked processes waiting for a token
+        }).
+-type tm_stat_hist() :: #tm_stat_hist{}.
+-define(DEFAULT_TM_STAT_HIST,
+        #tm_stat_hist{limit=0, refills=0, given=0, blocked=0}).
+
+-define(DEFAULT_TM_SAMPLE_WINDOW, 60).    %% in seconds
+-define(DEFAULT_TM_OUTPUT_SAMPLES, 20).   %% default number of sample windows displayed
+-define(DEFAULT_TM_KEPT_SAMPLES, 10000).  %% number of history samples to keep
+-define(TM_ETS_TABLE, token_mgr_table).   %% name of private token manager ETS table
+-define(TM_ETS_OPTS, [private, bag]).     %% creation time properties of token manager ETS table

rebar.config

@@ -6,14 +6,14 @@
 
 {deps, [
   {lager, "2.0.0", {git, "git://github.com/basho/lager", {tag, "2.0.0"}}},
-  {poolboy, ".*", {git, "git://github.com/basho/poolboy", {branch, "master"}}},
+  {poolboy, ".*", {git, "git://github.com/basho/poolboy", {branch, "develop"}}},
   {protobuffs, "0.8.*", {git, "git://github.com/basho/erlang_protobuffs",
-                              {branch, "master"}}},
-  {basho_stats, ".*", {git, "git://github.com/basho/basho_stats", "HEAD"}},
-  {riak_sysmon, ".*", {git, "git://github.com/basho/riak_sysmon", {branch, "master"}}},
+                              {tag, "0.8.1p1"}}},
+  {basho_stats, ".*", {git, "git://github.com/basho/basho_stats", {branch, "develop"}}},
+  {riak_sysmon, ".*", {git, "git://github.com/basho/riak_sysmon", {branch, "develop"}}},
   {webmachine, ".*", {git, "git://github.com/basho/webmachine",
-                                {tag, "64176ef9b"}}},
-  {folsom, ".*", {git, "git://github.com/basho/folsom.git", {tag, "0.7.4p1"}}},
+                                {tag, "1.10.4p1"}}},
+  {folsom, ".*", {git, "git://github.com/basho/folsom.git", {tag, "0.7.4p2"}}},
   {ranch, "0.4.0-p1", {git, "git://github.com/basho/ranch.git", {tag, "0.4.0-p1"}}}
 
        ]}.

rebar.config.script

@@ -0,0 +1,11 @@
+case erlang:system_info(otp_release) =< "R15B01" of
+    true  ->
+        HashDefine = [{d,old_hash}],
+        case lists:keysearch(erl_opts, 1, CONFIG) of
+            {value, {erl_opts, Opts}} ->
+                lists:keyreplace(erl_opts,1,CONFIG,{erl_opts,Opts++HashDefine});
+            false ->
+                CONFIG ++ [{erl_opts, HashDefine}]
+        end;
+    false -> CONFIG
+end.

src/chash.erl

@@ -97,12 +97,20 @@ lookup(IndexAsInt, CHash) ->
     {IndexAsInt, X} = proplists:lookup(IndexAsInt, Nodes),
     X.
 
+-ifndef(old_hash).
+sha(Bin) ->
+    crypto:hash(sha, Bin).
+-else.
+sha(Bin) ->
+    crypto:sha(Bin).
+-endif.
+
 %% @doc Given any term used to name an object, produce that object's key
 %%      into the ring.  Two names with the same SHA-1 hash value are
 %%      considered the same name.
 -spec key_of(ObjectName :: term()) -> index().
 key_of(ObjectName) ->    
-    crypto:sha(term_to_binary(ObjectName)).
+    sha(term_to_binary(ObjectName)).
 
 %% @doc Return all Nodes that own any partitions in the ring.
 -spec members(CHash :: chash()) -> [chash_node()].

src/merkerl.erl

@@ -323,8 +323,13 @@ contains_node(Tree,Node) ->
 getkids(Tree) ->
     [V || {_K,V} <- orddict:to_list(Tree#merk.children)].
 
+-ifndef(old_hash).
+sha(X) ->
+    crypto:hash(sha, term_to_binary(X)).
+-else.
 sha(X) ->
     crypto:sha(term_to_binary(X)).
+-endif.
 
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").

src/riak_core_apl.erl

@@ -325,7 +325,7 @@ six_node_test() ->
     ok.
 
 chbin_test_() ->
-    {timeout, 60, fun chbin_test_scenario/0}.
+    {timeout, 180, fun chbin_test_scenario/0}.
 
 chbin_test_scenario() ->
     [chbin_test_scenario(Size, NumNodes)