From 9d90058c28a249da81d2995be125477c971117b2 Mon Sep 17 00:00:00 2001 From: Anatolii Date: Fri, 25 Mar 2022 07:00:30 +0200 Subject: [PATCH 01/13] Fix comment --- test/depcache_tests.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/depcache_tests.erl b/test/depcache_tests.erl index 45824a8..c006383 100644 --- a/test/depcache_tests.erl +++ b/test/depcache_tests.erl @@ -79,7 +79,7 @@ get_set_depend_test() -> ?assertEqual(undefined, depcache:get(test_key, C)), - %% Set a key and hold it for one second. + %% Set a key and hold it for ten seconds. depcache:set(test_key, 123, 10, [test_key_dep], C), ?assertEqual({ok,123}, depcache:get(test_key, C)), From 82673402c98a2fa32d0e26d7679d532fa8676010 Mon Sep 17 00:00:00 2001 From: Anatolii Date: Sat, 26 Mar 2022 07:40:00 +0200 Subject: [PATCH 02/13] Add dot in an example --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a1fee60..315e9cc 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Usage Start a depcache server like this: - {ok, Server} = depcache:start_link([]) + {ok, Server} = depcache:start_link([]). Now you can get and set values using the returned `Server` pid. From 86b2b6852afbea103dc786f28fd3cc7336cd9883 Mon Sep 17 00:00:00 2001 From: Anatolii Date: Sat, 26 Mar 2022 15:54:44 +0200 Subject: [PATCH 03/13] Change a number to a constant Replace 3600 to the constant HOUR --- src/depcache.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/depcache.erl b/src/depcache.erl index d8008be..9235c1d 100644 --- a/src/depcache.erl +++ b/src/depcache.erl @@ -197,7 +197,7 @@ memo_send_errors(Key, Exception, Server) -> %% @doc Add the key to the depcache, hold it for 3600 seconds and no dependencies -spec set( key(), any(), depcache_server() ) -> ok. set(Key, Data, Server) -> - set(Key, Data, 3600, [], Server). + set(Key, Data, ?HOUR, [], Server). %% @doc Add the key to the depcache, hold it for MaxAge seconds and no dependencies -spec set( key(), any(), max_age_secs(), depcache_server() ) -> ok. From aaeeace8a10e5466bc52626aad2657ebc8b621ab Mon Sep 17 00:00:00 2001 From: Anatolii Kosorukov Date: Thu, 31 Mar 2022 21:16:16 +0300 Subject: [PATCH 04/13] Add documentation, option to rebar config Provide more information about function definitons of depcache API. Add EDoc options to API documentation (tune CSS styles, add overview file). Add new folder doc_src which contains additional files for documentation generation. Implement overload option of EDoc. Show hidden documetation by providing rebar3 option - add additional profile for EDoc private documentation. Separate some functions to make the code easier to read. Tunie Makefile: add command for edoc, clean; add new command edoc_private --- Makefile | 6 + doc_src/overview.edoc | 6 + doc_src/style.css | 71 +++ rebar.config | 10 +- src/depcache.erl | 1098 ++++++++++++++++++++++++++++++++------- test/depcache_tests.erl | 2 +- 6 files changed, 1017 insertions(+), 176 deletions(-) create mode 100644 doc_src/overview.edoc create mode 100644 doc_src/style.css diff --git a/Makefile b/Makefile index e011b49..b36d681 100644 --- a/Makefile +++ b/Makefile @@ -18,10 +18,16 @@ dialyzer: $(REBAR) $(REBAR) dialyzer edoc: + mkdir -p doc && cp -fR doc_src/* doc $(REBAR) edoc + +edoc_private: + mkdir -p doc && cp -fR doc_src/* doc + $(REBAR) as edoc_private edoc clean: $(REBAR) clean + rm -rf doc ./rebar3: erl -noshell -s inets start -s ssl start \ diff --git a/doc_src/overview.edoc b/doc_src/overview.edoc new file mode 100644 index 0000000..aed5ca2 --- /dev/null +++ b/doc_src/overview.edoc @@ -0,0 +1,6 @@ +@author Arjan Scherpenisse +@author Marc Worrell +@title depcache +@doc In-memory caching server with dependency checks and local in process memoization of lookups. +@copyright Apache-2.0 License 2022 by Marc Worrell, Arjan Scherpenisse +@reference See The Performance of Open Source Applications for more information. \ No newline at end of file diff --git a/doc_src/style.css b/doc_src/style.css new file mode 100644 index 0000000..cfdb6ba --- /dev/null +++ b/doc_src/style.css @@ -0,0 +1,71 @@ +/* standard EDoc style sheet */ +body { + font-family: Verdana, Arial, Helvetica, sans-serif; + margin-left: .25in; + margin-right: .2in; + margin-top: 0.2in; + margin-bottom: 0.2in; + color: #000000; + background-color: #ffffff; +} +h1,h2 { + margin-left: -0.2in; +} +div.navbar { + background-color: #add8e6; + padding: 0.2em; +} +h2.indextitle { + padding: 0.4em; + background-color: #add8e6; +} +h3.function,h3.typedecl { + background-color: #add8e6; + padding-left: 1em; +} +div.spec { + margin-left: 2em; + + background-color: #eeeeee; +} +a.module { + text-decoration:none +} +a.module:hover { + background-color: #eeeeee; +} +ul.definitions { + list-style-type: none; +} +ul.index { + list-style-type: none; + background-color: #eeeeee; +} + +/* + * Minor style tweaks + */ +ul { + list-style-type: square; +} +table { + border-collapse: collapse; +} +td { + padding: 3px; + vertical-align: middle; +} + +/* +Tune styles +*/ + +table[summary="navigation bar"] { + background-image: url('http://zotonic.com/lib/images/logo.png'); + background-repeat: no-repeat; + background-position: center; +} + +code, p>tt, a>tt { + font-size: 1.2em; +} diff --git a/rebar.config b/rebar.config index 4830c82..3a52947 100644 --- a/rebar.config +++ b/rebar.config @@ -8,5 +8,13 @@ deprecated_function_calls]}. {edoc_opts, [ - {preprocess, true} + {dir, "doc"}, {preprocess, true}, {stylesheet, "style.css"}, {title, "depcache"}, {application, depcache} ]}. + +{profiles, [ + {edoc_private, [ + {edoc_opts, [ + {private, true} + ]} + ]} +]}. \ No newline at end of file diff --git a/src/depcache.erl b/src/depcache.erl index 9235c1d..3901246 100644 --- a/src/depcache.erl +++ b/src/depcache.erl @@ -1,8 +1,19 @@ %% @author Arjan Scherpenisse %% @copyright 2009-2020 Marc Worrell, Arjan Scherpenisse +%% @doc %% -%% @doc In-memory caching server with dependency checks and local in process memoization of lookups. - +%% == depcache API == +%% {@link flush/2}, {@link flush/1}, {@link get/2}, {@link get/3}, {@link get_subkey/3}, {@link get_wait/2} +%% {@link set/4}, {@link set/5}, {@link size/1}, {@link set/3} +%%
+%% {@link memo/2}, {@link memo/3}, {@link memo/4}, {@link memo/5} +%%
+%% {@link flush_process_dict/0}, {@link in_process/1}, {@link in_process_server/1} +%% +%% === Internal === +%% {@link cleanup/1}, {@link cleanup/5} +%% +%% @end %% Copyright 2009-2020 Marc Worrell, Arjan Scherpenisse %% %% Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,6 +27,7 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. +%% -module(depcache). -author("Marc Worrell "). @@ -46,9 +58,11 @@ -type memo_fun() :: function() | mfa() | {module(), atom()}. -type depcache_server() :: pid() | atom(). --type max_age_secs() :: non_neg_integer(). +-type sec() :: non_neg_integer(). +-type max_age_secs() :: sec(). -type key() :: any(). -type dependencies() :: list( key() ). +-type proplist() :: proplists:proplist(). -export_type([ memo_fun/0, @@ -63,9 +77,9 @@ data_table :: ets:tab() }). --record(state, {now, serial, tables, wait_pids}). --record(meta, {key, expire, serial, depend}). --record(depend,{key, serial}). +-record(state, {now :: sec(), serial :: non_neg_integer(), tables :: tables(), wait_pids :: dict:dict()}). +-record(meta, {key :: key(), expire :: sec(), serial :: non_neg_integer(), depend :: dependencies()}). +-record(depend, {key :: key(), serial :: non_neg_integer()}). -record(cleanup_state, { pid :: pid(), @@ -75,31 +89,58 @@ callback :: mfa() | undefined }). +-type tables() :: #tables{meta_table :: ets:tab(), deps_table :: ets:tab(), data_table :: ets:tab()}. +-type state() :: #state{now :: sec(), serial :: non_neg_integer(), tables :: tables(), wait_pids :: dict:dict()}. +-type depend() :: #depend{key :: key(), serial :: non_neg_integer()}. +-type cleanup_state() :: #cleanup_state{pid :: pid(), tables :: tables(), name :: atom(), memory_max :: non_neg_integer(), callback :: mfa() | undefined}. +-type meta() :: #meta{key :: key(), expire :: sec(), serial :: non_neg_integer(), depend :: dependencies()}. + + %% @doc Start a depcache process. +%%
+%% See also: +%% [http://erlang.org/doc/man/gen_server.html#start_link-3 gen_server:start_link/3]. %% %% For Config, you can pass: -%% {callback, {Module, Function, Arguments}}: depcache event callback -%% {memory_max, MaxMemoryInMB}: number of MB to limit depcache size at. --spec start_link(Config :: proplists:proplist()) -> {ok, pid()} | ignore | {error, term()}. +%% `{callback, {Module, Function, Arguments}}': depcache event callback +%% `{memory_max, MaxMemoryInMB}': number of MB to limit depcache size at. +%% @param Config configuration options +%% @returns Result of starting gen_server item. + +-spec start_link(Config) -> Result when + Config :: proplist(), + Result :: {ok, pid()} | ignore | {error, term()}. start_link(Config) -> gen_server:start_link(?MODULE, Config, []). + %% @doc Start a named depcache process. +%%
+%% See also: +%% [http://erlang.org/doc/man/gen_server.html#start_link-4 gen_server:start_link/4]. %% %% For Config, you can pass: -%% {callback, {Module, Function, Arguments}}: depcache event callback -%% {memory_max, MaxMemoryInMB}: number of MB to limit depcache size at. --spec start_link(atom(), Config :: proplists:proplist()) -> {ok, pid()} | ignore | {error, term()}. +%% `{callback, {Module, Function, Arguments}}': depcache event callback +%% `{memory_max, MaxMemoryInMB}': number of MB to limit depcache size at. +%% @param Name of process +%% @param Config configuration options +%% @returns Result of starting gen_server item. + +-spec start_link(Name, Config) -> Result when + Name :: atom(), + Config :: proplist(), + Result :: {ok, pid()} | ignore | {error, term()}. start_link(Name, Config) -> gen_server:start_link({local, Name}, ?MODULE, [{name,Name}|Config], []). + -define(META_TABLE_PREFIX, $m). -define(DEPS_TABLE_PREFIX, $p). -define(DATA_TABLE_PREFIX, $d). -define(NAMED_TABLE(Name,Prefix), list_to_atom([Prefix,$:|atom_to_list(Name)])). -%% One hour, in seconds +%% One hour, in seconds. -define(HOUR, 3600). %% Default max size, in Mbs, of the stored data in the depcache before the gc kicks in. @@ -108,35 +149,75 @@ start_link(Name, Config) -> % Maximum time to wait for a get_wait/2 call before a timout failure (in secs). -define(MAX_GET_WAIT, 30). -% Number of slots visited for each gc iteration +% Number of slots visited for each gc iteration. -define(CLEANUP_BATCH, 100). -% Number of entries we keep in the local process dictionary for fast lookups +% Number of entries we keep in the local process dictionary for fast lookups. -define(PROCESS_DICT_THRESHOLD, 10000). %% @doc Cache the result of the function for an hour. --spec memo( memo_fun(), depcache_server() ) -> any(). +%% @param Fun a funciton for producing a value +%% @returns cached value + +-spec memo( Fun, Server ) -> Result when + Fun :: memo_fun(), + Server :: depcache_server(), + Result :: any(). memo(Fun, Server) -> memo(Fun, undefined, ?HOUR, [], Server). + %% @doc If Fun is a function then cache for an hour given the key. If %% Fun is a {M,F,A} tuple then derive the key from the tuple and -%% cache for MaxAge seconds. --spec memo( memo_fun(), max_age_secs() | key(), depcache_server() ) -> any(). +%% cache for `MaxAge' seconds. +%% +%% @param Fun a funciton for producing a value1 +%% @param MaxAge a caching time +%% @param Key a cache item key +%% @returns cached value + +-spec memo( Fun, MaxAge, Server ) -> Result when + Fun :: memo_fun(), + MaxAge :: max_age_secs(), + Server :: depcache_server(), + Result :: any(); +( Fun, Key, Server ) -> Result when + Fun :: memo_fun(), + Key :: key(), + Server :: depcache_server(), + Result :: any(). memo(Fun, MaxAge, Server) when is_tuple(Fun) -> memo(Fun, undefined, MaxAge, [], Server); memo(Fun, Key, Server) when is_function(Fun) -> memo(Fun, Key, ?HOUR, [], Server). -%% @doc Cache the result of the function as Key for MaxAge seconds. --spec memo( memo_fun(), key(), max_age_secs(), depcache_server() ) -> any(). + +%% @doc Cache the result of the function as Key for `MaxAge' seconds. +%% @returns cached value +%% @equiv memo(Fun, Key, MaxAge, [], Server) + +-spec memo( Fun, Key, MaxAge, Server ) -> Result when + Fun :: memo_fun(), + Key :: key(), + MaxAge :: max_age_secs(), + Server :: depcache_server(), + Result :: any(). memo(Fun, Key, MaxAge, Server) -> memo(Fun, Key, MaxAge, [], Server). -%% @doc Cache the result of the function as Key for MaxAge seconds, flush + +%% @doc Cache the result of the function as Key for `MaxAge' seconds, flush %% the cached result if any of the dependencies is changed. --spec memo( memo_fun(), key(), max_age_secs(), dependencies(), depcache_server() ) -> any(). +%% @returns cached value + +-spec memo( Fun, Key, MaxAge, Dep, Server ) -> Result when + Fun :: memo_fun(), + Key :: undefined | key(), + MaxAge :: max_age_secs(), + Dep :: dependencies(), + Server :: depcache_server(), + Result :: any(). memo(Fun, Key, MaxAge, Dep, Server) -> Key1 = case Key of undefined -> memo_key(Fun); @@ -148,64 +229,143 @@ memo(Fun, Key, MaxAge, Dep, Server) -> {throw, R} -> throw(R); undefined -> - try - Value = - case Fun of - {M,F,A} -> erlang:apply(M,F,A); - {M,F} -> M:F(); - _ when is_function(Fun) -> Fun() - end, - {Value1, MaxAge1, Dep1} = - case Value of - #memo{value=V, max_age=MA, deps=D} -> - MA1 = case is_integer(MA) of true -> MA; false -> MaxAge end, - {V, MA1, Dep++D}; - _ -> - {Value, MaxAge, Dep} - end, - case MaxAge of - 0 -> memo_send_replies(Key, Value1, Server); - _ -> set(Key, Value1, MaxAge1, Dep1, Server) - end, - Value1 - catch - ?WITH_STACKTRACE(Class, R, S) - memo_send_errors(Key, {throw, R}, Server), - erlang:raise(Class, R, S) - end + memo_key(Fun, Key, MaxAge, Dep, Server) end. +%% @private +%% @returns cached value +%% @see memo/5 + +-spec memo_key( Fun, Key, MaxAge, Dep, Server ) -> Result when + Fun :: memo_fun(), + Key :: key(), + MaxAge :: max_age_secs(), + Dep :: dependencies(), + Server :: depcache_server(), + Result :: any(). +memo_key(Fun, Key, MaxAge, Dep, Server) -> + try + Value = + case Fun of + {M,F,A} -> erlang:apply(M,F,A); + {M,F} -> M:F(); + _ when is_function(Fun) -> Fun() + end, + {Value1, MaxAge1, Dep1} = + case Value of + #memo{value=V, max_age=MA, deps=D} -> + MA1 = case is_integer(MA) of true -> MA; false -> MaxAge end, + {V, MA1, Dep++D}; + _ -> + {Value, MaxAge, Dep} + end, + case MaxAge of + 0 -> memo_send_replies(Key, Value1, Server); + _ -> set(Key, Value1, MaxAge1, Dep1, Server) + end, + Value1 + catch + ?WITH_STACKTRACE(Class, R, S) + memo_send_errors(Key, {throw, R}, Server), + erlang:raise(Class, R, S) + end. + + +%% @private %% @doc Calculate the key used for memo functions. +%% Returns cached value. + +-spec memo_key( MFA ) -> Result when + MFA :: {Module, Function, Arity}, + Module :: module(), + Function :: atom(), + Arity :: arity(), + Result :: tuple(); +( MF ) -> Result when + MF :: {Module, Function}, + Module :: module(), + Function :: atom(), + Result :: tuple(). memo_key({M,F,A}) -> WithoutContext = lists:filter(fun(T) when is_tuple(T) andalso element(1, T) =:= context -> false; (_) -> true end, A), {M,F,WithoutContext}; memo_key({M,F}) -> {M,F}. + +%% @private %% @doc Send the calculated value to the processes waiting for the result. +%%
+%% See also: +%% [http://erlang.org/doc/man/gen_server.html#reply-2 gen_server:reply/2]. +%% + +-spec memo_send_replies( Key, Value, Server ) -> Result when + Key :: key(), + Value :: any(), + Server :: depcache_server(), + Result :: ok. memo_send_replies(Key, Value, Server) -> Pids = get_waiting_pids(Key, Server), [ catch gen_server:reply(Pid, {ok, Value}) || Pid <- Pids ], ok. + +%% @private %% @doc Send an error to the processes waiting for the result. +%%
+%% See also: +%% [http://erlang.org/doc/man/gen_server.html#reply-2 gen_server:reply/2]. +%% + +-spec memo_send_errors( Key, Exception, Server ) -> Result when + Key :: key(), + Exception :: {throw, any()}, + Server :: depcache_server(), + Result :: list(). memo_send_errors(Key, Exception, Server) -> Pids = get_waiting_pids(Key, Server), [ catch gen_server:reply(Pid, Exception) || Pid <- Pids ]. -%% @doc Add the key to the depcache, hold it for 3600 seconds and no dependencies --spec set( key(), any(), depcache_server() ) -> ok. +%% @doc Add the key to the depcache, hold it for `3600' seconds and no dependencies. + +-spec set( Key, Data, Server ) -> Result when + Key :: key(), + Data :: any(), + Server :: depcache_server(), + Result :: ok. + set(Key, Data, Server) -> set(Key, Data, ?HOUR, [], Server). -%% @doc Add the key to the depcache, hold it for MaxAge seconds and no dependencies --spec set( key(), any(), max_age_secs(), depcache_server() ) -> ok. + +%% @doc Add the key to the depcache, hold it for `MaxAge' seconds and no dependencies. +%% @equiv set(Key, Data, MaxAge, [], Server) + +-spec set( Key, Data, MaxAge, Server ) -> Result when + Key :: key(), + Data :: any(), + MaxAge :: max_age_secs(), + Server :: depcache_server(), + Result :: ok. set(Key, Data, MaxAge, Server) -> set(Key, Data, MaxAge, [], Server). -%% @doc Add the key to the depcache, hold it for MaxAge seconds and check the dependencies --spec set( key(), any(), max_age_secs(), dependencies(), depcache_server() ) -> ok. + +%% @doc Add the key to the depcache, hold it for `MaxAge' seconds and check the dependencies. +%%
+%% See also: +%% [http://erlang.org/doc/man/gen_server.html#call-3 gen_server:call/3]. +%% + +-spec set( Key, Data, MaxAge, Depend, Server ) -> Result when + Key :: key(), + Data :: any(), + MaxAge :: max_age_secs(), + Depend :: dependencies(), + Server :: depcache_server(), + Result :: ok. set(Key, Data, MaxAge, Depend, Server) -> flush_process_dict(), gen_server:call(Server, {set, Key, Data, MaxAge, Depend}). @@ -214,7 +374,15 @@ set(Key, Data, MaxAge, Depend, Server) -> %% @doc Fetch the key from the cache, when the key does not exist then lock the entry and let %% the calling process insert the value. All other processes requesting the key will wait till %% the key is updated and receive the key's new value. --spec get_wait( key(), depcache_server() ) -> {ok, any()} | undefined | {throw, term()}. +%%
+%% See also: +%% [http://erlang.org/doc/man/gen_server.html#call-3 gen_server:call/3]. +%% + +-spec get_wait( Key, Server ) -> Result when + Key :: key(), + Server :: depcache_server(), + Result :: {ok, any()} | undefined | {throw, term()}. get_wait(Key, Server) -> case get_process_dict(Key, Server) of NoValue when NoValue =:= undefined orelse NoValue =:= depcache_disabled -> @@ -224,15 +392,35 @@ get_wait(Key, Server) -> end. -%% @doc Fetch the queue of pids that are waiting for a get_wait/1. This flushes the queue and +%% @private +%% @doc Fetch the queue of pids that are waiting for a `get_wait/2'. This flushes the queue and %% the key from the depcache. --spec get_waiting_pids( key(), depcache_server() ) -> list( {pid(), Tag :: term()} ). +%%
+%% See also: +%% [http://erlang.org/doc/man/gen_server.html#call-3 gen_server:call/3]. +%% +%% @see get_wait/2 + +-spec get_waiting_pids( Key, Server ) -> Result when + Key :: key(), + Server :: depcache_server(), + Result :: [{pid(), Tag}], + Tag :: atom(). + get_waiting_pids(Key, Server) -> gen_server:call(Server, {get_waiting_pids, Key}, ?MAX_GET_WAIT*1000). -%% @doc Fetch the key from the cache, return the data or an undefined if not found (or not valid) --spec get( key(), depcache_server() ) -> {ok, any()} | undefined. +%% @doc Fetch the key from the cache, return the data or an `undefined' if not found (or not valid). +%%
+%% See also: +%% [http://erlang.org/doc/man/gen_server.html#call-2 gen_server:call/2]. +%% + +-spec get( Key, Server ) -> Result when + Key :: key(), + Server :: depcache_server(), + Result :: {ok, any()} | undefined. get(Key, Server) -> case get_process_dict(Key, Server) of depcache_disabled -> gen_server:call(Server, {get, Key}); @@ -240,8 +428,17 @@ get(Key, Server) -> end. -%% @doc Fetch the key from the cache, return the data or an undefined if not found (or not valid) --spec get_subkey( key(), key(), depcache_server() ) -> {ok, any()} | undefined. +%% @doc Fetch the key from the cache, return the data or an `undefined' if not found (or not valid) +%%
+%% See also: +%% [http://erlang.org/doc/man/gen_server.html#call-2 gen_server:call/2]. +%% + +-spec get_subkey( Key, SubKey, Server ) -> Result when + Key :: key(), + SubKey :: key(), + Server :: depcache_server(), + Result :: {ok, any()} | undefined. get_subkey(Key, SubKey, Server) -> case in_process_server(Server) of true -> @@ -258,8 +455,17 @@ get_subkey(Key, SubKey, Server) -> end. -%% @doc Fetch the key from the cache, return the data or an undefined if not found (or not valid) --spec get( key(), key(), depcache_server() ) -> {ok, any()} | undefined. +%% @doc Fetch the key from the cache, return the data or an `undefined' if not found (or not valid) +%%
+%% See also: +%% [http://erlang.org/doc/man/gen_server.html#call-2 gen_server:call/2]. +%% + +-spec get( Key, SubKey, Server ) -> Result when + Key :: key(), + SubKey :: key(), + Server :: depcache_server(), + Result :: {ok, any()} | undefined. get(Key, SubKey, Server) -> case get_process_dict(Key, Server) of undefined -> @@ -271,28 +477,51 @@ get(Key, SubKey, Server) -> end. -%% @doc Flush the key and all keys depending on the key --spec flush( key(), depcache_server() ) -> ok. +%% @doc Flush the key and all keys depending on that key +%%
+%% See also: +%% [http://erlang.org/doc/man/gen_server.html#call-2 gen_server:call/2]. +%% + +-spec flush( Key, Server ) -> Result when + Key :: key(), + Server :: depcache_server(), + Result :: ok. flush(Key, Server) -> gen_server:call(Server, {flush, Key}), flush_process_dict(). %% @doc Flush all keys from the caches --spec flush( depcache_server() ) -> ok. +%%
+%% See also: +%% [http://erlang.org/doc/man/gen_server.html#call-2 gen_server:call/2]. +%% + +-spec flush( Server ) -> Result when + Server :: depcache_server(), + Result :: ok. flush(Server) -> gen_server:call(Server, flush), flush_process_dict(). %% @doc Return the total memory size of all stored terms --spec size( depcache_server() ) -> non_neg_integer(). + +-spec size( Server ) -> Result when + Server :: depcache_server(), + Result :: non_neg_integer(). size(Server) -> {_Meta, _Deps, Data} = get_tables(Server), ets:info(Data, memory). +%% @private %% @doc Fetch the depcache tables. + +-spec get_tables( Server ) -> Result when + Server :: depcache_server(), + Result :: tuple(). get_tables(Server) -> case erlang:get(depcache_tables) of {ok, Server, Tables} -> @@ -304,6 +533,13 @@ get_tables(Server) -> get_tables1(Server) end. +%% @private +%% @doc Get or init ets-tables. +%% @see get_tables/1 + +-spec get_tables1( Server ) -> Result when + Server :: depcache_server(), + Result :: tuple(). get_tables1(Server) when is_pid(Server) -> {ok, Tables} = gen_server:call(Server, get_tables), erlang:put(depcache_tables, {ok, Server, Tables}), @@ -317,7 +553,13 @@ get_tables1(Server) when is_atom(Server) -> erlang:put(depcache_tables, {ok, Server, Tables}), Tables. +%% @private %% @doc Fetch a value from the dependency cache, using the in-process cached tables. + +-spec get_process_dict(Key, Server ) -> Result when + Key :: key(), + Server :: depcache_server(), + Result :: tuple(). get_process_dict(Key, Server) -> case in_process_server(Server) of true -> @@ -343,7 +585,13 @@ get_process_dict(Key, Server) -> depcache_disabled end. +%% @private +%% @doc Get cached value by key. +-spec get_ets(Key, Server ) -> Result when + Key :: key(), + Server :: depcache_server(), + Result :: undefined | {ok, any()}. get_ets(Key, Server) -> {MetaTable, DepsTable, DataTable} = get_tables(Server), case get_concurrent(Key, get_now(), MetaTable, DepsTable, DataTable) of @@ -358,7 +606,10 @@ get_ets(Key, Server) -> %% @doc Check if we use a local process dict cache. --spec in_process_server( depcache_server() ) -> boolean(). + +-spec in_process_server( Server ) -> Result when + Server :: depcache_server(), + Result :: boolean(). in_process_server(Server) -> case erlang:get(depcache_in_process) of true -> @@ -370,7 +621,10 @@ in_process_server(Server) -> %% @doc Enable or disable the in-process caching using the process dictionary --spec in_process( undefined | boolean() ) -> undefined | boolean(). + +-spec in_process( IsChaching ) -> Result when + IsChaching :: undefined | boolean(), + Result :: undefined | boolean(). in_process(true) -> erlang:put(depcache_in_process, true); in_process(false) -> @@ -382,14 +636,20 @@ in_process(undefined) -> %% @doc Flush all items memoized in the process dictionary. + +-spec flush_process_dict() -> Result when + Result :: ok. flush_process_dict() -> [ erlang:erase({depcache, Key}) || {{depcache, Key},_Value} <- erlang:get() ], erlang:erase(depache_now), erlang:put(depcache_count, 0), ok. - +%% @private %% @doc Get the current system time in seconds + +-spec get_now() -> Result when + Result :: sec(). get_now() -> case erlang:get(depcache_now) of undefined -> @@ -405,8 +665,12 @@ get_now() -> %% gen_server callbacks -%% @spec init(Config) -> {ok, State} %% @doc Initialize the depcache. Creates ets tables for the deps, meta and data. Spawns garbage collector. + +-spec init(Config) -> Result when + Config :: proplist(), + State :: state(), + Result :: {ok, State}. init(Config) -> MemoryMaxMbs = case proplists:get_value(memory_max, Config) of undefined -> ?MEMORY_MAX; Mbs -> Mbs end, MemoryMaxWords = 1024 * 1024 * MemoryMaxMbs div erlang:system_info(wordsize), @@ -446,24 +710,194 @@ init(Config) -> }]), {ok, State}. +%% @doc Handling call messages + +-spec handle_call(Request, From, State) -> Result when + Request :: get_tables, + From :: {pid(), atom()}, + State :: state(), + Meta_table :: ets:tab(), + Deps_table :: ets:tab(), + Data_table :: ets:tab(), + Result :: {reply, {ok, {Meta_table, Deps_table, Data_table}}, State}; +(Request, From, State) -> Result when + Request :: get_wait, + From :: {pid(), atom()}, + State :: state(), + Result :: {reply, Reply, state()} | {noreply, state()}, + Reply :: undefined | {ok, term()}; +(Request, From, State) -> Result when + Request :: {get_waiting_pids, Key}, + Key :: key(), + From :: {pid(), atom()}, + State :: state(), + Result :: {reply, [{pid(), Tag}], state()}, + Tag :: atom(); +(Request, From, State) -> Result when + Request :: {get, Key}, + Key :: key(), + From :: {pid(), atom()}, + State :: state(), + Result :: {reply, Reply, State}, + Reply :: undefined | {ok, term()}; +(Request, From, State) -> Result when + Request :: {get, Key, SubKey}, + Key :: key(), + SubKey :: key(), + From :: {pid(), atom()}, + State :: state(), + Result :: {reply, Reply, State}, + Reply :: undefined | {ok, term()}; +(Request, From, State) -> Result when + Request :: {set, Key, Data, MaxAge, Depend}, + Key :: key(), + Data :: any(), + MaxAge :: max_age_secs(), + Depend :: dependencies(), + From :: {pid(), atom()}, + State :: state(), + Result :: {reply, Reply, State}, + Reply :: ok; +(Request, From, State) -> Result when + Request :: {flush, Key}, + Key :: key(), + From :: {pid(), atom()}, + State :: state(), + Result :: {reply, ok, State}; +(Request, From, State) -> Result when + Request :: flush, + From :: {pid(), atom()}, + State :: state(), + Result :: {reply, ok, State}. +handle_call(get_tables, _From, State) -> + handle_call_get_tables(State); + +handle_call({get_wait, Key}, From, State) -> + handle_call_get_wait(Key, From, State); + +handle_call({get_waiting_pids, Key}, _From, State) -> + handle_call_get_waiting_pids(Key, State); + +handle_call({get, Key}, _From, State) -> + handle_call_get(Key, State); + +handle_call({get, Key, SubKey}, _From, State) -> + handle_call_get_sub_key({Key, SubKey}, State); + +handle_call({set, Key, Data, MaxAge, Depend}, _From, State) -> + handle_call_set({Key, Data, MaxAge, Depend}, State); + +handle_call({flush, Key}, _From, State) -> + handle_call_flush(Key, State); + +handle_call(flush, _From, State) -> + handle_call_flush_all(State). + + +%% @doc Handling cast messages + +-spec handle_cast(Request, State) -> Result when + Request :: {flush, Key}, + Key :: key(), + State :: state(), + Result :: {noreply, State}; +(Request, State) -> Result when + Request :: flush, + State :: state(), + Result :: {noreply, State}; +(Request, State) -> Result when + Request :: any(), + Result :: {noreply, State}. + +handle_cast({flush, Key}, State) -> + flush_key(Key, State), + {noreply, State}; + +handle_cast(flush, #state{tables = Tables} = State) -> + ets:delete_all_objects(Tables#tables.data_table), + ets:delete_all_objects(Tables#tables.meta_table), + ets:delete_all_objects(Tables#tables.deps_table), + erase_process_dict(), + {noreply, State}; + +handle_cast(_Msg, State) -> + {noreply, State}. + + +%% @doc This function is called by a `gen_server' process when when it receives `tick' or +%% any other message than a synchronous or asynchronous request (or a system message). +%% @see gen_server:handle_info/2 + +-spec handle_info(Info, State) -> Result when + Info :: tick, + State :: state(), + Result :: {noreply, State}; +(Info, State) -> Result when + Info :: any(), + State :: state(), + Result :: {noreply, State}. + +handle_info(tick, State) -> + erase_process_dict(), + flush_message(tick), + {noreply, State#state{now=now_sec()}}; + +handle_info(_Msg, State) -> + {noreply, State}. + + +%% @doc This function is called by a `gen_server' process when it is about to terminate. +%% @see gen_server:terminate/2 + +-spec terminate(Reason, State) -> Result when + Reason :: normal | shutdown | {shutdown, term()} | term(), + State :: state(), + Result :: ok. + +terminate(_Reason, _State) -> ok. + -%%-------------------------------------------------------------------- -%% Function: handle_call(Request, From, State) -> {reply, Reply, State} | -%% {reply, Reply, State, Timeout} | -%% {noreply, State} | -%% {noreply, State, Timeout} | -%% {stop, Reason, Reply, State} | -%% {stop, Reason, State} -%% Description: Handling call messages -%%-------------------------------------------------------------------- +%% @doc This function is called by a gen_server process when it is to update +%% its internal state during a release upgrade/downgrade. +%% @see gen_server:code_change/3 -%% @doc Return the ets tables used by the cache -handle_call(get_tables, _From, #state{tables = Tables} = State) -> - {reply, {ok, {Tables#tables.meta_table, Tables#tables.deps_table, Tables#tables.data_table}}, State}; +-spec code_change(OldVersion, State, Extra) -> Result when + OldVersion :: (term() | {down, term()}), + State :: state(), + Extra :: term(), + Result :: {ok, State}. + +code_change(_OldVersion, State, _Extra) -> {ok, State}. + + +%% @private +%% @doc Return the ets-tables used by the cache +%% @see handle_call/3 +%% + +-spec handle_call_get_tables(State) -> Result when + State :: state(), + Meta_table :: ets:tab(), + Deps_table :: ets:tab(), + Data_table :: ets:tab(), + Result :: {reply, {ok, {Meta_table, Deps_table, Data_table}}, State}. +handle_call_get_tables(#state{tables = Tables} = State) -> + {reply, {ok, {Tables#tables.meta_table, Tables#tables.deps_table, Tables#tables.data_table}}, State}. + +%% @private %% @doc Fetch a key from the cache. When the key is not available then let processes wait till the %% key is available. -handle_call({get_wait, Key}, From, #state{tables = Tables} = State) -> +%% @see handle_call/3 +%% + +-spec handle_call_get_wait(Key, From, State) -> Result when + Key :: key(), + From :: {pid(), atom()}, + State :: state(), + Result :: {reply, Reply, state()} | {noreply, state()}, + Reply :: undefined | {ok, term()}. +handle_call_get_wait(Key, From, #state{tables = Tables} = State) -> case get_concurrent(Key, State#state.now, Tables#tables.meta_table, Tables#tables.deps_table, Tables#tables.data_table) of NotFound when NotFound =:= flush; NotFound =:= undefined -> case NotFound of @@ -482,10 +916,20 @@ handle_call({get_wait, Key}, From, #state{tables = Tables} = State) -> end; {ok, _Value} = Found -> {reply, Found, State} - end; + end. + +%% @private %% @doc Return the list of processes waiting for the rendering of a key. Flush the queue and the key. -handle_call({get_waiting_pids, Key}, _From, State) -> +%% @see handle_call/3 +%% + +-spec handle_call_get_waiting_pids(Key, State) -> Result when + Key :: key(), + State :: state(), + Result :: {reply, [{pid(), Tag}], state()}, + Tag :: atom(). +handle_call_get_waiting_pids(Key, State) -> {State1, Pids} = case dict:find(Key, State#state.wait_pids) of {ok, {_MaxAge, List}} -> WaitPids = dict:erase(Key, State#state.wait_pids), @@ -494,23 +938,58 @@ handle_call({get_waiting_pids, Key}, _From, State) -> {State, []} end, flush_key(Key, State1), - {reply, Pids, State1}; + {reply, Pids, State1}. -%% @doc Fetch a key from the cache, returns undefined when not found. -handle_call({get, Key}, _From, State) -> - {reply, get_in_depcache(Key, State), State}; +%% @private +%% @doc Fetch a key from the cache, returns `undefined' when not found. +%% @see handle_call/3 +%% + +-spec handle_call_get(Key, State) -> Result when + Key :: key(), + State :: state(), + Result :: {reply, Reply, State}, + Reply :: undefined | {ok, term()}. +handle_call_get(Key, State) -> + {reply, get_in_depcache(Key, State), State}. + -%% @doc Fetch a subkey from a key from the cache, returns undefined when not found. +%% @private +%% @doc Fetch a subkey from a key from the cache, returns `undefined' when not found. %% This is useful when the cached data is very large and the fetched data is small in comparison. -handle_call({get, Key, SubKey}, _From, State) -> +%% @see handle_call/3 +%% + +-spec handle_call_get_sub_key(Request, State) -> Result when + Request :: {Key, SubKey}, + Key :: key(), + SubKey :: key(), + State :: state(), + Result :: {reply, Reply, State}, + Reply :: undefined | {ok, term()}. +handle_call_get_sub_key({Key, SubKey}, State) -> case get_in_depcache(Key, State) of undefined -> {reply, undefined, State}; {ok, Value} -> {reply, {ok, find_value(SubKey, Value)}, State} - end; + end. -%% Add an entry to the cache table -handle_call({set, Key, Data, MaxAge, Depend}, _From, #state{tables = Tables} = State) -> + +%% @private +%% @doc Add an entry to the cache table. +%% @see handle_call/3 +%% + +-spec handle_call_set(Request, State) -> Result when + Request :: {Key, Data, MaxAge, Depend}, + Key :: key(), + Data :: any(), + MaxAge :: max_age_secs(), + Depend :: dependencies(), + State :: state(), + Result :: {reply, Reply, State}, + Reply :: ok. +handle_call_set({Key, Data, MaxAge, Depend}, #state{tables = Tables} = State) -> erase_process_dict(), State1 = State#state{serial=State#state.serial+1}, case MaxAge of @@ -542,56 +1021,46 @@ handle_call({set, Key, Data, MaxAge, Depend}, _From, #state{tables = Tables} = S {reply, ok, State1#state{wait_pids=WaitFroms}}; error -> {reply, ok, State1} - end; - -handle_call({flush, Key}, _From, State) -> - flush_key(Key, State), - {reply, ok, State}; + end. -handle_call(flush, _From, #state{tables = Tables} = State) -> - ets:delete_all_objects(Tables#tables.data_table), - ets:delete_all_objects(Tables#tables.meta_table), - ets:delete_all_objects(Tables#tables.deps_table), - erase_process_dict(), - {reply, ok, State}. +%% @private +%% @doc Flush cached data by key. +%% @see handle_call/3 +%% +-spec handle_call_flush(Key, State) -> Result when + Key :: key(), + State :: state(), + Result :: {reply, ok, State}. +handle_call_flush(Key, State) -> + flush_key(Key, State), + {reply, ok, State}. -%%-------------------------------------------------------------------- -%% Function: handle_cast(Msg, State) -> {noreply, State} | -%% {noreply, State, Timeout} | -%% {stop, Reason, State} -%% Description: Handling cast messages -%%-------------------------------------------------------------------- -handle_cast({flush, Key}, State) -> - flush_key(Key, State), - {noreply, State}; +%% @private +%% @doc Flush all cached data. +%% @see handle_call/3 +%% -handle_cast(flush, #state{tables = Tables} = State) -> +-spec handle_call_flush_all(State) -> Result when + State :: state(), + Result :: {reply, ok, State}. +handle_call_flush_all(#state{tables = Tables} = State) -> ets:delete_all_objects(Tables#tables.data_table), ets:delete_all_objects(Tables#tables.meta_table), ets:delete_all_objects(Tables#tables.deps_table), erase_process_dict(), - {noreply, State}; - -handle_cast(_Msg, State) -> - {noreply, State}. - - -handle_info(tick, State) -> - erase_process_dict(), - flush_message(tick), - {noreply, State#state{now=now_sec()}}; - -handle_info(_Msg, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> ok. -code_change(_OldVersion, State, _Extra) -> {ok, State}. - + {reply, ok, State}. + +%% @private %% @doc Fetch a value, from within the depcache process. Cache the value in the process dictionary of the depcache. + +-spec get_in_depcache(Key, State) -> Result when + Key :: key(), + State :: state(), + Result :: term(). get_in_depcache(Key, State) -> case erlang:get({depcache, Key}) of {memo, Value} -> @@ -607,10 +1076,29 @@ get_in_depcache(Key, State) -> erlang:put(depcache_count, incr(erlang:get(depcache_count))), Value end. - + + +%% @private +%% @doc Increment input value. + +-spec incr(Value) -> Result when + Value :: undefined, + Result :: 1; +(Value) -> Result when + Value :: integer(), + Result :: integer(). incr(undefined) -> 1; incr(N) -> N+1. + +%% @private +%% @doc Get decache value. + +-spec get_in_depcache_ets(Key, State) -> Result when + Key :: key(), + State :: state(), + Result :: undefined | {ok, Value}, + Value :: term(). get_in_depcache_ets(Key, #state{tables = Tables} = State) -> case get_concurrent(Key, State#state.now, Tables#tables.meta_table, Tables#tables.deps_table, Tables#tables.data_table) of flush -> @@ -623,34 +1111,72 @@ get_in_depcache_ets(Key, #state{tables = Tables} = State) -> end. +%% @private %% @doc Get a value from the depache. Called by the depcache and other processes. -%% @spec get_concurrent(term(), now:int(), tid(), tid(), tid()) -> {ok, term()} | undefined | flush + +-spec get_concurrent(Key, Now, MetaTable, DepsTable, DataTable) -> Result when + Key :: key(), + Now :: sec(), + MetaTable :: ets:tab(), + DepsTable :: ets:tab(), + DataTable :: ets:tab(), + Result :: flush | undefined | {ok, term()}. get_concurrent(Key, Now, MetaTable, DepsTable, DataTable) -> case ets:lookup(MetaTable, Key) of [] -> undefined; [#meta{serial=Serial, expire=Expire, depend=Depend}] -> - %% Check expiration - case Expire >= Now of - false -> - flush; - true -> - %% Check dependencies - case check_depend(Serial, Depend, DepsTable) of - true -> - case ets:lookup(DataTable, Key) of - [] -> undefined; - [{_Key,Data}] -> {ok, Data} - end; - false -> - flush - end - end + get_concurrent(Key, Now, DepsTable, DataTable, Serial, Expire, Depend) end. - - +%% @hidden + +-spec get_concurrent(Key, Now, DepsTable, DataTable, Serial, Expire, Depend) -> Result when + Key :: key(), + Now :: sec(), + DepsTable :: ets:tab(), + DataTable :: ets:tab(), + Serial :: non_neg_integer(), + Expire :: sec(), + Depend :: dependencies(), + Result :: flush | undefined | {ok, term()}. +get_concurrent(Key, Now, DepsTable, DataTable, Serial, Expire, Depend) -> + %% Check expiration + case Expire >= Now of + false -> + flush; + true -> + get_concurrent_check_depend(Key, DepsTable, DataTable, Serial, Depend) + end. + +%% @hidden + +-spec get_concurrent_check_depend(Key, DepsTable, DataTable, Serial, Depend) -> Result when + Key :: key(), + DepsTable :: ets:tab(), + DataTable :: ets:tab(), + Serial :: non_neg_integer(), + Depend :: dependencies(), + Result :: flush | undefined | {ok, term()}. +get_concurrent_check_depend(Key, DepsTable, DataTable, Serial, Depend) -> + %% Check dependencies + case check_depend(Serial, Depend, DepsTable) of + false -> + flush; + true -> + case ets:lookup(DataTable, Key) of + [] -> undefined; + [{_Key,Data}] -> {ok, Data} + end + end. + + +%% @private %% @doc Check if a key is usable as dependency key. That is a string, atom, integer etc, but not a list of lists. + +-spec is_simple_key(List) -> Result when + List :: list(), + Result :: true | false. is_simple_key([]) -> true; is_simple_key([H|_]) -> @@ -659,7 +1185,13 @@ is_simple_key(_Key) -> true. -%% @doc Flush a key from the cache, reset the in-process cache as well (we don't know if any cached value had a dependency) +%% @private +%% @doc Flush a key from the cache, reset the in-process cache as well (we don't know if any cached value had a dependency). + +-spec flush_key(Key, State) -> Result when + Key :: key(), + State :: state(), + Result :: ok. flush_key(Key, #state{tables = Tables}) -> ets:delete(Tables#tables.data_table, Key), ets:delete(Tables#tables.deps_table, Key), @@ -667,7 +1199,14 @@ flush_key(Key, #state{tables = Tables}) -> erase_process_dict(). -%% @doc Check if all dependencies are still valid, that is they have a serial before or equal to the serial of the entry +%% @private +%% @doc Check if all dependencies are still valid, that is they have a serial before or equal to the serial of the entry. + +-spec check_depend(Serial, Depend, DepsTable) -> Result when + Serial :: non_neg_integer(), + Depend :: [depend()], + DepsTable :: ets:tab(), + Result :: true | false. check_depend(_Serial, [], _DepsTable) -> true; check_depend(Serial, Depend, DepsTable) -> @@ -682,8 +1221,45 @@ check_depend(Serial, Depend, DepsTable) -> lists:foldl(CheckDepend, true, Depend). - -%% Map lookup +%% @private +%% @doc Search by value in some set of data. + +-spec find_value(Key, Map) -> Result when + Key :: key(), + Map :: map(), + Result :: undefined | any(); +(Key, List) -> Result when + Key :: integer(), + List :: list(), + Result :: undefined | any(); +(Key, Tree) -> Result when + Key :: key(), + Tree :: gb_trees:tree(), + Result :: undefined | any(); +(Key, {rsc_list, List}) -> Result when + Key :: integer(), + List :: proplist(), + Result :: any(); +(Key, {rsc_list, List}) -> Result when + Key :: integer(), + List :: list(), + Result :: undefined | any(); +(Key, {rsc_list, List}) -> Result when + Key :: key(), + List :: list(), + Result :: undefined | any(); +(Key, Tuple) -> Result when + Key :: integer(), + Tuple :: tuple(), + Result :: undefined | any(); +(Key, Tuple) -> Result when + Key :: key(), + Tuple :: tuple(), + Result :: undefined | any(); +(Key, Data) -> Result when + Key :: key(), + Data :: any(), + Result :: any(). find_value(Key, M) when is_map(M) -> maps:get(Key, M, undefined); find_value(Key, L) when is_integer(Key) andalso is_list(L) -> @@ -748,34 +1324,133 @@ find_value(_Key, _Data) -> %% Asks the depcache server to delete invalidated items. When the load of the data table is too high then %% This cleanup process starts to delete random entries. By using a random delete we don't need to keep %% a LRU list, which is a bit expensive. +%% @equiv cleanup(CleanUp_state, SlotNr, Now, Mode, Ct) +-spec cleanup(CleanUp_state) -> Result when + CleanUp_state :: cleanup_state(), + Result :: no_return(). cleanup(#cleanup_state{} = State) -> ?MODULE:cleanup(State, 0, now_sec(), normal, 0). -%% Wrap around the end of table +%% @doc Cleanup process for the depcache. Periodically checks a batch of depcache items for their validity. +%% Asks the depcache server to delete invalidated items. When the load of the data table is too high then +%% This cleanup process starts to delete random entries. By using a random delete we don't need to keep +%% a LRU list, which is a bit expensive. + +-spec cleanup(State, SlotNr, Now, Mode, Ct) -> Result when + State :: cleanup_state(), + SlotNr :: '$end_of_table' | non_neg_integer(), + Now :: sec(), + Mode :: normal | cache_full, + Ct :: integer(), + Result :: no_return(); +(State, SlotNr, Now, Mode, Ct) -> Result when + State :: cleanup_state(), + SlotNr :: '$end_of_table' | non_neg_integer(), + Now :: sec(), + Mode :: normal, + Ct :: 0, + Result :: no_return(); +(State, SlotNr, Now, Mode, Ct) -> Result when + State :: cleanup_state(), + SlotNr :: '$end_of_table' | non_neg_integer(), + Now :: sec(), + Mode :: cache_full, + Ct :: 0, + Result :: no_return(); +(State, SlotNr, Now, Mode, Ct) -> Result when + State :: cleanup_state(), + SlotNr :: '$end_of_table' | non_neg_integer(), + Now :: sec(), + Mode :: normal, + Ct :: integer(), + Result :: no_return(); +(State, SlotNr, Now, Mode, Ct) -> Result when + State :: cleanup_state(), + SlotNr :: '$end_of_table' | non_neg_integer(), + Now :: sec(), + Mode :: cache_full, + Ct :: integer(), + Result :: no_return(). cleanup(#cleanup_state{tables = #tables{meta_table = MetaTable}} = State, '$end_of_table', Now, _Mode, Ct) -> + cleanup_wrap_around_table(State, MetaTable, Now, Ct); + +cleanup(#cleanup_state{tables = #tables{meta_table = MetaTable}} = State, SlotNr, Now, normal, 0) -> + cleanup_normal(State, MetaTable, SlotNr, Now); + +cleanup(#cleanup_state{} = State, SlotNr, Now, cache_full, 0) -> + cleanup_is_cache_full(State, SlotNr, Now); + +cleanup(#cleanup_state{tables = #tables{meta_table = MetaTable}} = State, SlotNr, Now, normal, Ct) -> + cleanup_check_expire_stamp(State, MetaTable, SlotNr, Now, Ct); + +cleanup(#cleanup_state{pid = Pid, name = Name, callback = Callback, tables = #tables{meta_table = MetaTable}} = State, SlotNr, Now, cache_full, Ct) -> + cleanup_random(State, MetaTable, SlotNr, Now, Ct, Pid, Name, Callback). + + +%% @private +%% @doc Wrap around the end of table. +%% @see cleanup/5 + +-spec cleanup_wrap_around_table(State, MetaTable, Now, Ct) -> Result when + State :: cleanup_state(), + MetaTable :: ets:tab(), + Now :: sec(), + Ct :: integer(), + Result :: no_return(). +cleanup_wrap_around_table(State, MetaTable, Now, Ct) -> case ets:info(MetaTable, size) of 0 -> ?MODULE:cleanup(State, 0, Now, cleanup_mode(State), 0); _ -> ?MODULE:cleanup(State, 0, Now, cleanup_mode(State), Ct) - end; + end. -%% In normal cleanup, sleep a second between each batch before continuing our cleanup sweep -cleanup(#cleanup_state{tables = #tables{meta_table = MetaTable}} = State, SlotNr, Now, normal, 0) -> + +%% @private +%% @doc In `normal' cleanup, sleep a second between each batch before continuing our cleanup sweep. +%% @see cleanup/5 + +-spec cleanup_normal(State, MetaTable, SlotNr, Now) -> Result when + State :: cleanup_state(), + MetaTable :: ets:tab(), + SlotNr :: '$end_of_table' | non_neg_integer(), + Now :: sec(), + Result :: no_return(). +cleanup_normal(State, MetaTable, SlotNr, Now) -> timer:sleep(1000), case ets:info(MetaTable, size) of 0 -> ?MODULE:cleanup(State, SlotNr, Now, normal, 0); _ -> ?MODULE:cleanup(State, SlotNr, now_sec(), cleanup_mode(State), ?CLEANUP_BATCH) - end; + end. -%% After finishing a batch in cache_full mode, check if the cache is still full, if so keep deleting entries -cleanup(#cleanup_state{} = State, SlotNr, Now, cache_full, 0) -> + +%% @private +%% @doc After finishing a batch in `cache_full' mode, check if the cache is still full, if so keep deleting entries. +%% @see cleanup/5 + +-spec cleanup_is_cache_full(State, SlotNr, Now) -> Result when + State :: cleanup_state(), + SlotNr :: '$end_of_table' | non_neg_integer(), + Now :: sec(), + Result :: no_return(). +cleanup_is_cache_full(State, SlotNr, Now) -> case cleanup_mode(State) of normal -> ?MODULE:cleanup(State, SlotNr, Now, normal, 0); cache_full -> ?MODULE:cleanup(State, SlotNr, now_sec(), cache_full, ?CLEANUP_BATCH) - end; + end. -%% Normal cleanup behaviour - check expire stamp and dependencies -cleanup(#cleanup_state{tables = #tables{meta_table = MetaTable}} = State, SlotNr, Now, normal, Ct) -> + +%% @private +%% @doc Normal cleanup behaviour - check expire stamp and dependencies. +%% @see cleanup/5 + +-spec cleanup_check_expire_stamp(State, MetaTable, SlotNr, Now, Ct) -> Result when + State :: cleanup_state(), + MetaTable :: ets:tab(), + SlotNr :: '$end_of_table' | non_neg_integer(), + Now :: sec(), + Ct :: integer(), + Result :: no_return(). +cleanup_check_expire_stamp(State, MetaTable, SlotNr, Now, Ct) -> Slot = try ets:slot(MetaTable, SlotNr) catch @@ -787,10 +1462,24 @@ cleanup(#cleanup_state{tables = #tables{meta_table = MetaTable}} = State, SlotNr Entries -> lists:foreach(fun(Meta) -> flush_expired(Meta, Now, State) end, Entries), ?MODULE:cleanup(State, SlotNr + 1, Now, normal, Ct - 1) - end; + end. -%% Full cache cleanup mode - randomly delete every 10th entry -cleanup(#cleanup_state{pid = Pid, name = Name, callback = Callback, tables = #tables{meta_table = MetaTable}} = State, SlotNr, Now, cache_full, Ct) -> + +%% @private +%% @doc Full cache cleanup mode - randomly delete every 10th entry. +%% @see cleanup/5 + +-spec cleanup_random(State, MetaTable, SlotNr, Now, Ct, Pid, Name, Callback) -> Result when + State :: cleanup_state(), + MetaTable :: ets:tab(), + SlotNr :: '$end_of_table' | non_neg_integer(), + Now :: sec(), + Ct :: integer(), + Pid :: pid(), + Name :: atom(), + Callback :: undefined | mfa(), + Result :: no_return(). +cleanup_random(State, MetaTable, SlotNr, Now, Ct, Pid, Name, Callback) -> Slot = try ets:slot(MetaTable, SlotNr) catch @@ -821,6 +1510,7 @@ cleanup(#cleanup_state{pid = Pid, name = Name, callback = Callback, tables = #ta lists:foreach(RandomDelete, Entries1), ?MODULE:cleanup(State, SlotNr + 1, Now, cache_full, Ct - 1) end. + -ifdef(rand_only). rand_uniform(N) -> @@ -830,7 +1520,15 @@ rand_uniform(N) -> crypto:rand_uniform(1,N+1). -endif. -%% @doc Check if an entry is expired, if so delete it + +%% @private +%% @doc Check if an entry is expired, if so delete it. + +-spec flush_expired(Meta, Now, State) -> Result when + Meta :: meta(), + Now :: sec(), + State :: cleanup_state(), + Result :: flushed | ok. flush_expired( #meta{key=Key, serial=Serial, expire=Expire, depend=Depend}, Now, @@ -843,14 +1541,27 @@ flush_expired( end. +%% @private %% @doc When the data table is too large then we start to randomly delete keys. It also signals the cleanup process %% that it needs to be more aggressive, upping its batch size. -%% We use erts_debug:size() on the stored terms to calculate the total size of all terms stored. This -%% is better than counting the number of entries. Using the process_info(Pid,memory) is not very useful as the +%% We use `erts_debug:size()' on the stored terms to calculate the total size of all terms stored. This +%% is better than counting the number of entries. Using the `process_info(Pid,memory)' is not very useful as the %% garbage collection still needs to be done and then we delete too many entries. +%% @equiv cleanup_mode(DataTable, MemoryMax) + +-spec cleanup_mode(State) -> Result when + State :: cleanup_state(), + Result :: cache_full | normal. cleanup_mode(#cleanup_state{tables = #tables{data_table = DataTable}, memory_max = MemoryMax}) -> cleanup_mode(DataTable, MemoryMax). +%% @private +%% @doc Returns clear-up mode. + +-spec cleanup_mode(DataTable, MemoryMax) -> Result when + DataTable :: ets:tab(), + MemoryMax :: non_neg_integer(), + Result :: cache_full | normal. cleanup_mode(DataTable, MemoryMax) -> Memory = ets:info(DataTable, memory), if @@ -859,20 +1570,34 @@ cleanup_mode(DataTable, MemoryMax) -> end. +%% @private +%% @doc Returns the current tick count. -%% @doc Return the current tick count +-spec now_sec() -> Result when + Result :: sec(). now_sec() -> {M,S,_M} = os:timestamp(), M*1000000 + S. -%% @doc Safe erase of process dict, keeps some 'magical' proc_lib vars + +%% @private +%% @doc Safe erase of process dict, keeps some 'magical' `proc_lib' vars. + +-spec erase_process_dict() -> Result when + Result :: ok. erase_process_dict() -> Values = [ {K, erlang:get(K)} || K <- ['$initial_call', '$ancestors', '$erl_eval_max_line'] ], erlang:erase(), [ erlang:put(K,V) || {K,V} <- Values, V =/= undefined ], ok. + +%% @private %% @doc Flush all incoming messages, used when receiving timer ticks to prevent multiple ticks. + +-spec flush_message(Msg) -> Result when + Msg :: any(), + Result :: ok. flush_message(Msg) -> receive Msg -> flush_message(Msg) @@ -880,6 +1605,31 @@ flush_message(Msg) -> ok end. + +%% @private +%% @doc Returns the result of applying `Function' in `Module' to `Args'. + +-spec callback(Type, Name, MFA) -> Result when + Type :: eviction | atom(), + Name :: atom(), + MFA :: undefined, + Result :: nop | ok | term(); +(Type, Name, MFA) -> Result when + Type :: eviction | atom(), + Name :: atom(), + MFA :: {Module, Function, Arity}, + Module :: module(), + Function :: atom(), + Arity :: list(), + Result :: nop | ok | term(); +(Type, Name, MFA) -> Result when + Type :: eviction | atom(), + Name :: atom(), + MFA :: {Module, Function, Arity}, + Module :: module(), + Function :: atom(), + Arity :: arity(), + Result :: nop | ok | term(). callback(_Type, _Name, undefined) -> ok; callback(Type, Name, {M, F, A}) when is_list(A) -> diff --git a/test/depcache_tests.erl b/test/depcache_tests.erl index c006383..45824a8 100644 --- a/test/depcache_tests.erl +++ b/test/depcache_tests.erl @@ -79,7 +79,7 @@ get_set_depend_test() -> ?assertEqual(undefined, depcache:get(test_key, C)), - %% Set a key and hold it for ten seconds. + %% Set a key and hold it for one second. depcache:set(test_key, 123, 10, [test_key_dep], C), ?assertEqual({ok,123}, depcache:get(test_key, C)), From 161997d10ffbeae5bf2bbb12b9d8f07c42514ebe Mon Sep 17 00:00:00 2001 From: Anatolii Date: Thu, 31 Mar 2022 22:30:25 +0300 Subject: [PATCH 05/13] Update inner test comment --- test/depcache_tests.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/depcache_tests.erl b/test/depcache_tests.erl index 45824a8..c006383 100644 --- a/test/depcache_tests.erl +++ b/test/depcache_tests.erl @@ -79,7 +79,7 @@ get_set_depend_test() -> ?assertEqual(undefined, depcache:get(test_key, C)), - %% Set a key and hold it for one second. + %% Set a key and hold it for ten seconds. depcache:set(test_key, 123, 10, [test_key_dep], C), ?assertEqual({ok,123}, depcache:get(test_key, C)), From 0da7e2a8b46fa31834206b6ae31e80f82b438d71 Mon Sep 17 00:00:00 2001 From: Anatolii Kosorukov Date: Fri, 1 Apr 2022 14:04:51 +0300 Subject: [PATCH 06/13] Solve all dialyzer issues Refactor some -spec descriptions. Made some refactoring to explicely transform data. --- src/depcache.erl | 101 ++++++++--------------------------------------- 1 file changed, 16 insertions(+), 85 deletions(-) diff --git a/src/depcache.erl b/src/depcache.erl index 3901246..f429718 100644 --- a/src/depcache.erl +++ b/src/depcache.erl @@ -177,13 +177,10 @@ memo(Fun, Server) -> %% @param Key a cache item key %% @returns cached value --spec memo( Fun, MaxAge, Server ) -> Result when +-spec memo( Fun, MaxAge_Key, Server ) -> Result when Fun :: memo_fun(), + MaxAge_Key :: MaxAge | Key, MaxAge :: max_age_secs(), - Server :: depcache_server(), - Result :: any(); -( Fun, Key, Server ) -> Result when - Fun :: memo_fun(), Key :: key(), Server :: depcache_server(), Result :: any(). @@ -559,7 +556,7 @@ get_tables1(Server) when is_atom(Server) -> -spec get_process_dict(Key, Server ) -> Result when Key :: key(), Server :: depcache_server(), - Result :: tuple(). + Result :: tuple() | depcache_disabled | undefined. get_process_dict(Key, Server) -> case in_process_server(Server) of true -> @@ -797,16 +794,9 @@ handle_call(flush, _From, State) -> %% @doc Handling cast messages -spec handle_cast(Request, State) -> Result when - Request :: {flush, Key}, + Request :: {flush, Key} | flush | any(), Key :: key(), State :: state(), - Result :: {noreply, State}; -(Request, State) -> Result when - Request :: flush, - State :: state(), - Result :: {noreply, State}; -(Request, State) -> Result when - Request :: any(), Result :: {noreply, State}. handle_cast({flush, Key}, State) -> @@ -829,11 +819,7 @@ handle_cast(_Msg, State) -> %% @see gen_server:handle_info/2 -spec handle_info(Info, State) -> Result when - Info :: tick, - State :: state(), - Result :: {noreply, State}; -(Info, State) -> Result when - Info :: any(), + Info :: tick | any(), State :: state(), Result :: {noreply, State}. @@ -1224,42 +1210,12 @@ check_depend(Serial, Depend, DepsTable) -> %% @private %% @doc Search by value in some set of data. --spec find_value(Key, Map) -> Result when - Key :: key(), - Map :: map(), - Result :: undefined | any(); -(Key, List) -> Result when - Key :: integer(), - List :: list(), - Result :: undefined | any(); -(Key, Tree) -> Result when - Key :: key(), - Tree :: gb_trees:tree(), - Result :: undefined | any(); -(Key, {rsc_list, List}) -> Result when - Key :: integer(), - List :: proplist(), - Result :: any(); -(Key, {rsc_list, List}) -> Result when - Key :: integer(), - List :: list(), - Result :: undefined | any(); -(Key, {rsc_list, List}) -> Result when - Key :: key(), - List :: list(), - Result :: undefined | any(); -(Key, Tuple) -> Result when - Key :: integer(), - Tuple :: tuple(), - Result :: undefined | any(); -(Key, Tuple) -> Result when - Key :: key(), - Tuple :: tuple(), - Result :: undefined | any(); -(Key, Data) -> Result when - Key :: key(), - Data :: any(), - Result :: any(). +-spec find_value(Key, Data) -> Result when + Key :: key() | integer(), + Data :: map() | List | Rsc_list | tuple() | any(), + List :: list() | proplist(), + Rsc_list :: {rsc_list, List}, + Result :: undefined | any(). find_value(Key, M) when is_map(M) -> maps:get(Key, M, undefined); find_value(Key, L) when is_integer(Key) andalso is_list(L) -> @@ -1270,7 +1226,8 @@ find_value(Key, L) when is_integer(Key) andalso is_list(L) -> _:_ -> undefined end; find_value(Key, {GBSize, GBData}) when is_integer(GBSize) -> - case gb_trees:lookup(Key, {GBSize, GBData}) of + Tree = gb_trees:from_orddict([{GBSize, GBData}]), + case gb_trees:lookup(Key, Tree) of {value, Val} -> Val; _ -> @@ -1302,7 +1259,9 @@ find_value(Key, Tuple) when is_tuple(Tuple) -> Module = element(1, Tuple), case Module of dict -> - case dict:find(Key, Tuple) of + {Key, Value} = Tuple, + Dict = dict:append(Key, Value, dict:new()), + case dict:find(Key, Dict) of {ok, Val} -> Val; _ -> @@ -1343,34 +1302,6 @@ cleanup(#cleanup_state{} = State) -> Now :: sec(), Mode :: normal | cache_full, Ct :: integer(), - Result :: no_return(); -(State, SlotNr, Now, Mode, Ct) -> Result when - State :: cleanup_state(), - SlotNr :: '$end_of_table' | non_neg_integer(), - Now :: sec(), - Mode :: normal, - Ct :: 0, - Result :: no_return(); -(State, SlotNr, Now, Mode, Ct) -> Result when - State :: cleanup_state(), - SlotNr :: '$end_of_table' | non_neg_integer(), - Now :: sec(), - Mode :: cache_full, - Ct :: 0, - Result :: no_return(); -(State, SlotNr, Now, Mode, Ct) -> Result when - State :: cleanup_state(), - SlotNr :: '$end_of_table' | non_neg_integer(), - Now :: sec(), - Mode :: normal, - Ct :: integer(), - Result :: no_return(); -(State, SlotNr, Now, Mode, Ct) -> Result when - State :: cleanup_state(), - SlotNr :: '$end_of_table' | non_neg_integer(), - Now :: sec(), - Mode :: cache_full, - Ct :: integer(), Result :: no_return(). cleanup(#cleanup_state{tables = #tables{meta_table = MetaTable}} = State, '$end_of_table', Now, _Mode, Ct) -> cleanup_wrap_around_table(State, MetaTable, Now, Ct); From 6edd141bef66f70ae5bf0371f8ed38022fa50366 Mon Sep 17 00:00:00 2001 From: Anatolii Kosorukov Date: Fri, 1 Apr 2022 14:13:11 +0300 Subject: [PATCH 07/13] Fix possible input data inconsistency It is possible that input Key and dict Key is not the some. --- src/depcache.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/depcache.erl b/src/depcache.erl index f429718..03eeeba 100644 --- a/src/depcache.erl +++ b/src/depcache.erl @@ -1259,8 +1259,8 @@ find_value(Key, Tuple) when is_tuple(Tuple) -> Module = element(1, Tuple), case Module of dict -> - {Key, Value} = Tuple, - Dict = dict:append(Key, Value, dict:new()), + {Key1, Value} = Tuple, + Dict = dict:append(Key1, Value, dict:new()), case dict:find(Key, Dict) of {ok, Val} -> Val; From 77810c83d0d2bba52319b4694df84b9be62d02a7 Mon Sep 17 00:00:00 2001 From: Anatolii Kosorukov Date: Thu, 21 Apr 2022 08:54:16 +0300 Subject: [PATCH 08/13] Update documentation --- rebar.config | 7 ++++++- rebar.lock | 1 - src/depcache.erl | 17 +++++++++++------ 3 files changed, 17 insertions(+), 8 deletions(-) delete mode 100644 rebar.lock diff --git a/rebar.config b/rebar.config index 3a52947..b996dcc 100644 --- a/rebar.config +++ b/rebar.config @@ -3,6 +3,8 @@ {platform_define, "^(R|1|20)", fun_stacktrace} ]}. +{project_plugins, [rebar3_proper]}. + {xref_checks, [undefined_function_calls, locals_not_used, deprecated_function_calls]}. @@ -16,5 +18,8 @@ {edoc_opts, [ {private, true} ]} - ]} + ]}, + {test, [ + {deps, [{proper, "1.3.0"}]} + ]} ]}. \ No newline at end of file diff --git a/rebar.lock b/rebar.lock deleted file mode 100644 index 57afcca..0000000 --- a/rebar.lock +++ /dev/null @@ -1 +0,0 @@ -[]. diff --git a/src/depcache.erl b/src/depcache.erl index 03eeeba..b4d77ba 100644 --- a/src/depcache.erl +++ b/src/depcache.erl @@ -3,8 +3,8 @@ %% @doc %% %% == depcache API == -%% {@link flush/2}, {@link flush/1}, {@link get/2}, {@link get/3}, {@link get_subkey/3}, {@link get_wait/2} -%% {@link set/4}, {@link set/5}, {@link size/1}, {@link set/3} +%% {@link flush/1}, {@link flush/2}, {@link get/2}, {@link get/3}, {@link get_subkey/3}, +%% {@link get_wait/2}, {@link set/3}, {@link set/4}, {@link set/5}, {@link size/1} %%
%% {@link memo/2}, {@link memo/3}, {@link memo/4}, {@link memo/5} %%
@@ -230,6 +230,8 @@ memo(Fun, Key, MaxAge, Dep, Server) -> end. %% @private +%% @param MaxAge maximum lifetime of an element in the cache +%% @param Dep list of subkeys %% @returns cached value %% @see memo/5 @@ -326,6 +328,7 @@ memo_send_errors(Key, Exception, Server) -> %% @doc Add the key to the depcache, hold it for `3600' seconds and no dependencies. +%% @equiv set(Key, Data, 3600, [], Server) -spec set( Key, Data, Server ) -> Result when Key :: key(), @@ -351,9 +354,11 @@ set(Key, Data, MaxAge, Server) -> %% @doc Add the key to the depcache, hold it for `MaxAge' seconds and check the dependencies. +%% @param MaxAge maximum lifetime of an element in the cache +%% @param Depend list of subkeys %%
%% See also: -%% [http://erlang.org/doc/man/gen_server.html#call-3 gen_server:call/3]. +%% [http://erlang.org/doc/man/gen_server.html#call-2 gen_server:call/2]. %% -spec set( Key, Data, MaxAge, Depend, Server ) -> Result when @@ -364,7 +369,7 @@ set(Key, Data, MaxAge, Server) -> Server :: depcache_server(), Result :: ok. set(Key, Data, MaxAge, Depend, Server) -> - flush_process_dict(), + flush_process_dict(), gen_server:call(Server, {set, Key, Data, MaxAge, Depend}). @@ -507,7 +512,7 @@ flush(Server) -> -spec size( Server ) -> Result when Server :: depcache_server(), - Result :: non_neg_integer(). + Result :: non_neg_integer() | undefined. size(Server) -> {_Meta, _Deps, Data} = get_tables(Server), ets:info(Data, memory). @@ -984,7 +989,7 @@ handle_call_set({Key, Data, MaxAge, Depend}, #state{tables = Tables} = State) -> ets:delete(Tables#tables.data_table, Key); _ -> ets:insert(Tables#tables.data_table, {Key, Data}), - ets:insert(Tables#tables.meta_table, #meta{key=Key, expire=State1#state.now+MaxAge, serial=State1#state.serial, depend=Depend}) + ets:insert(Tables#tables.meta_table, #meta{key=Key, expire=State1#state.now+MaxAge, serial=State1#state.serial, depend=Depend}) end, %% Make sure all dependency keys are available in the deps table From f0fb0d6640aa9c0da1ad4e29b1857b9f878e3faf Mon Sep 17 00:00:00 2001 From: Anatolii Date: Thu, 21 Apr 2022 10:49:15 +0300 Subject: [PATCH 09/13] Update depcache.erl --- src/depcache.erl | 260 +++++++++++++++++++++++------------------------ 1 file changed, 130 insertions(+), 130 deletions(-) diff --git a/src/depcache.erl b/src/depcache.erl index b4d77ba..731330e 100644 --- a/src/depcache.erl +++ b/src/depcache.erl @@ -109,7 +109,7 @@ -spec start_link(Config) -> Result when Config :: proplist(), - Result :: {ok, pid()} | ignore | {error, term()}. + Result :: {ok, pid()} | ignore | {error, term()}. start_link(Config) -> gen_server:start_link(?MODULE, Config, []). @@ -127,9 +127,9 @@ start_link(Config) -> %% @returns Result of starting gen_server item. -spec start_link(Name, Config) -> Result when - Name :: atom(), - Config :: proplist(), - Result :: {ok, pid()} | ignore | {error, term()}. + Name :: atom(), + Config :: proplist(), + Result :: {ok, pid()} | ignore | {error, term()}. start_link(Name, Config) -> gen_server:start_link({local, Name}, ?MODULE, [{name,Name}|Config], []). @@ -162,8 +162,8 @@ start_link(Name, Config) -> -spec memo( Fun, Server ) -> Result when Fun :: memo_fun(), - Server :: depcache_server(), - Result :: any(). + Server :: depcache_server(), + Result :: any(). memo(Fun, Server) -> memo(Fun, undefined, ?HOUR, [], Server). @@ -179,11 +179,11 @@ memo(Fun, Server) -> -spec memo( Fun, MaxAge_Key, Server ) -> Result when Fun :: memo_fun(), - MaxAge_Key :: MaxAge | Key, - MaxAge :: max_age_secs(), - Key :: key(), - Server :: depcache_server(), - Result :: any(). + MaxAge_Key :: MaxAge | Key, + MaxAge :: max_age_secs(), + Key :: key(), + Server :: depcache_server(), + Result :: any(). memo(Fun, MaxAge, Server) when is_tuple(Fun) -> memo(Fun, undefined, MaxAge, [], Server); memo(Fun, Key, Server) when is_function(Fun) -> @@ -196,10 +196,10 @@ memo(Fun, Key, Server) when is_function(Fun) -> -spec memo( Fun, Key, MaxAge, Server ) -> Result when Fun :: memo_fun(), - Key :: key(), - MaxAge :: max_age_secs(), - Server :: depcache_server(), - Result :: any(). + Key :: key(), + MaxAge :: max_age_secs(), + Server :: depcache_server(), + Result :: any(). memo(Fun, Key, MaxAge, Server) -> memo(Fun, Key, MaxAge, [], Server). @@ -210,11 +210,11 @@ memo(Fun, Key, MaxAge, Server) -> -spec memo( Fun, Key, MaxAge, Dep, Server ) -> Result when Fun :: memo_fun(), - Key :: undefined | key(), - MaxAge :: max_age_secs(), - Dep :: dependencies(), - Server :: depcache_server(), - Result :: any(). + Key :: undefined | key(), + MaxAge :: max_age_secs(), + Dep :: dependencies(), + Server :: depcache_server(), + Result :: any(). memo(Fun, Key, MaxAge, Dep, Server) -> Key1 = case Key of undefined -> memo_key(Fun); @@ -237,11 +237,11 @@ memo(Fun, Key, MaxAge, Dep, Server) -> -spec memo_key( Fun, Key, MaxAge, Dep, Server ) -> Result when Fun :: memo_fun(), - Key :: key(), - MaxAge :: max_age_secs(), - Dep :: dependencies(), - Server :: depcache_server(), - Result :: any(). + Key :: key(), + MaxAge :: max_age_secs(), + Dep :: dependencies(), + Server :: depcache_server(), + Result :: any(). memo_key(Fun, Key, MaxAge, Dep, Server) -> try Value = @@ -319,9 +319,9 @@ memo_send_replies(Key, Value, Server) -> -spec memo_send_errors( Key, Exception, Server ) -> Result when Key :: key(), - Exception :: {throw, any()}, - Server :: depcache_server(), - Result :: list(). + Exception :: {throw, any()}, + Server :: depcache_server(), + Result :: list(). memo_send_errors(Key, Exception, Server) -> Pids = get_waiting_pids(Key, Server), [ catch gen_server:reply(Pid, Exception) || Pid <- Pids ]. @@ -332,9 +332,9 @@ memo_send_errors(Key, Exception, Server) -> -spec set( Key, Data, Server ) -> Result when Key :: key(), - Data :: any(), - Server :: depcache_server(), - Result :: ok. + Data :: any(), + Server :: depcache_server(), + Result :: ok. set(Key, Data, Server) -> set(Key, Data, ?HOUR, [], Server). @@ -345,10 +345,10 @@ set(Key, Data, Server) -> -spec set( Key, Data, MaxAge, Server ) -> Result when Key :: key(), - Data :: any(), - MaxAge :: max_age_secs(), - Server :: depcache_server(), - Result :: ok. + Data :: any(), + MaxAge :: max_age_secs(), + Server :: depcache_server(), + Result :: ok. set(Key, Data, MaxAge, Server) -> set(Key, Data, MaxAge, [], Server). @@ -363,11 +363,11 @@ set(Key, Data, MaxAge, Server) -> -spec set( Key, Data, MaxAge, Depend, Server ) -> Result when Key :: key(), - Data :: any(), - MaxAge :: max_age_secs(), - Depend :: dependencies(), - Server :: depcache_server(), - Result :: ok. + Data :: any(), + MaxAge :: max_age_secs(), + Depend :: dependencies(), + Server :: depcache_server(), + Result :: ok. set(Key, Data, MaxAge, Depend, Server) -> flush_process_dict(), gen_server:call(Server, {set, Key, Data, MaxAge, Depend}). @@ -383,8 +383,8 @@ set(Key, Data, MaxAge, Depend, Server) -> -spec get_wait( Key, Server ) -> Result when Key :: key(), - Server :: depcache_server(), - Result :: {ok, any()} | undefined | {throw, term()}. + Server :: depcache_server(), + Result :: {ok, any()} | undefined | {throw, term()}. get_wait(Key, Server) -> case get_process_dict(Key, Server) of NoValue when NoValue =:= undefined orelse NoValue =:= depcache_disabled -> @@ -405,9 +405,9 @@ get_wait(Key, Server) -> -spec get_waiting_pids( Key, Server ) -> Result when Key :: key(), - Server :: depcache_server(), - Result :: [{pid(), Tag}], - Tag :: atom(). + Server :: depcache_server(), + Result :: [{pid(), Tag}], + Tag :: atom(). get_waiting_pids(Key, Server) -> gen_server:call(Server, {get_waiting_pids, Key}, ?MAX_GET_WAIT*1000). @@ -421,8 +421,8 @@ get_waiting_pids(Key, Server) -> -spec get( Key, Server ) -> Result when Key :: key(), - Server :: depcache_server(), - Result :: {ok, any()} | undefined. + Server :: depcache_server(), + Result :: {ok, any()} | undefined. get(Key, Server) -> case get_process_dict(Key, Server) of depcache_disabled -> gen_server:call(Server, {get, Key}); @@ -438,9 +438,9 @@ get(Key, Server) -> -spec get_subkey( Key, SubKey, Server ) -> Result when Key :: key(), - SubKey :: key(), - Server :: depcache_server(), - Result :: {ok, any()} | undefined. + SubKey :: key(), + Server :: depcache_server(), + Result :: {ok, any()} | undefined. get_subkey(Key, SubKey, Server) -> case in_process_server(Server) of true -> @@ -465,9 +465,9 @@ get_subkey(Key, SubKey, Server) -> -spec get( Key, SubKey, Server ) -> Result when Key :: key(), - SubKey :: key(), - Server :: depcache_server(), - Result :: {ok, any()} | undefined. + SubKey :: key(), + Server :: depcache_server(), + Result :: {ok, any()} | undefined. get(Key, SubKey, Server) -> case get_process_dict(Key, Server) of undefined -> @@ -487,8 +487,8 @@ get(Key, SubKey, Server) -> -spec flush( Key, Server ) -> Result when Key :: key(), - Server :: depcache_server(), - Result :: ok. + Server :: depcache_server(), + Result :: ok. flush(Key, Server) -> gen_server:call(Server, {flush, Key}), flush_process_dict(). @@ -501,8 +501,8 @@ flush(Key, Server) -> %% -spec flush( Server ) -> Result when - Server :: depcache_server(), - Result :: ok. + Server :: depcache_server(), + Result :: ok. flush(Server) -> gen_server:call(Server, flush), flush_process_dict(). @@ -511,8 +511,8 @@ flush(Server) -> %% @doc Return the total memory size of all stored terms -spec size( Server ) -> Result when - Server :: depcache_server(), - Result :: non_neg_integer() | undefined. + Server :: depcache_server(), + Result :: non_neg_integer() | undefined. size(Server) -> {_Meta, _Deps, Data} = get_tables(Server), ets:info(Data, memory). @@ -522,8 +522,8 @@ size(Server) -> %% @doc Fetch the depcache tables. -spec get_tables( Server ) -> Result when - Server :: depcache_server(), - Result :: tuple(). + Server :: depcache_server(), + Result :: tuple(). get_tables(Server) -> case erlang:get(depcache_tables) of {ok, Server, Tables} -> @@ -559,9 +559,9 @@ get_tables1(Server) when is_atom(Server) -> %% @doc Fetch a value from the dependency cache, using the in-process cached tables. -spec get_process_dict(Key, Server ) -> Result when - Key :: key(), - Server :: depcache_server(), - Result :: tuple() | depcache_disabled | undefined. + Key :: key(), + Server :: depcache_server(), + Result :: tuple() | depcache_disabled | undefined. get_process_dict(Key, Server) -> case in_process_server(Server) of true -> @@ -591,9 +591,9 @@ get_process_dict(Key, Server) -> %% @doc Get cached value by key. -spec get_ets(Key, Server ) -> Result when - Key :: key(), - Server :: depcache_server(), - Result :: undefined | {ok, any()}. + Key :: key(), + Server :: depcache_server(), + Result :: undefined | {ok, any()}. get_ets(Key, Server) -> {MetaTable, DepsTable, DataTable} = get_tables(Server), case get_concurrent(Key, get_now(), MetaTable, DepsTable, DataTable) of @@ -610,8 +610,8 @@ get_ets(Key, Server) -> %% @doc Check if we use a local process dict cache. -spec in_process_server( Server ) -> Result when - Server :: depcache_server(), - Result :: boolean(). + Server :: depcache_server(), + Result :: boolean(). in_process_server(Server) -> case erlang:get(depcache_in_process) of true -> @@ -625,8 +625,8 @@ in_process_server(Server) -> %% @doc Enable or disable the in-process caching using the process dictionary -spec in_process( IsChaching ) -> Result when - IsChaching :: undefined | boolean(), - Result :: undefined | boolean(). + IsChaching :: undefined | boolean(), + Result :: undefined | boolean(). in_process(true) -> erlang:put(depcache_in_process, true); in_process(false) -> @@ -640,7 +640,7 @@ in_process(undefined) -> %% @doc Flush all items memoized in the process dictionary. -spec flush_process_dict() -> Result when - Result :: ok. + Result :: ok. flush_process_dict() -> [ erlang:erase({depcache, Key}) || {{depcache, Key},_Value} <- erlang:get() ], erlang:erase(depache_now), @@ -670,9 +670,9 @@ get_now() -> %% @doc Initialize the depcache. Creates ets tables for the deps, meta and data. Spawns garbage collector. -spec init(Config) -> Result when - Config :: proplist(), - State :: state(), - Result :: {ok, State}. + Config :: proplist(), + State :: state(), + Result :: {ok, State}. init(Config) -> MemoryMaxMbs = case proplists:get_value(memory_max, Config) of undefined -> ?MEMORY_MAX; Mbs -> Mbs end, MemoryMaxWords = 1024 * 1024 * MemoryMaxMbs div erlang:system_info(wordsize), @@ -715,57 +715,57 @@ init(Config) -> %% @doc Handling call messages -spec handle_call(Request, From, State) -> Result when - Request :: get_tables, - From :: {pid(), atom()}, - State :: state(), - Meta_table :: ets:tab(), - Deps_table :: ets:tab(), - Data_table :: ets:tab(), - Result :: {reply, {ok, {Meta_table, Deps_table, Data_table}}, State}; + Request :: get_tables, + From :: {pid(), atom()}, + State :: state(), + Meta_table :: ets:tab(), + Deps_table :: ets:tab(), + Data_table :: ets:tab(), + Result :: {reply, {ok, {Meta_table, Deps_table, Data_table}}, State}; (Request, From, State) -> Result when - Request :: get_wait, - From :: {pid(), atom()}, - State :: state(), - Result :: {reply, Reply, state()} | {noreply, state()}, - Reply :: undefined | {ok, term()}; + Request :: get_wait, + From :: {pid(), atom()}, + State :: state(), + Result :: {reply, Reply, state()} | {noreply, state()}, + Reply :: undefined | {ok, term()}; (Request, From, State) -> Result when - Request :: {get_waiting_pids, Key}, - Key :: key(), - From :: {pid(), atom()}, - State :: state(), - Result :: {reply, [{pid(), Tag}], state()}, - Tag :: atom(); + Request :: {get_waiting_pids, Key}, + Key :: key(), + From :: {pid(), atom()}, + State :: state(), + Result :: {reply, [{pid(), Tag}], state()}, + Tag :: atom(); (Request, From, State) -> Result when - Request :: {get, Key}, - Key :: key(), - From :: {pid(), atom()}, - State :: state(), - Result :: {reply, Reply, State}, - Reply :: undefined | {ok, term()}; + Request :: {get, Key}, + Key :: key(), + From :: {pid(), atom()}, + State :: state(), + Result :: {reply, Reply, State}, + Reply :: undefined | {ok, term()}; (Request, From, State) -> Result when - Request :: {get, Key, SubKey}, - Key :: key(), - SubKey :: key(), - From :: {pid(), atom()}, - State :: state(), - Result :: {reply, Reply, State}, - Reply :: undefined | {ok, term()}; + Request :: {get, Key, SubKey}, + Key :: key(), + SubKey :: key(), + From :: {pid(), atom()}, + State :: state(), + Result :: {reply, Reply, State}, + Reply :: undefined | {ok, term()}; (Request, From, State) -> Result when - Request :: {set, Key, Data, MaxAge, Depend}, - Key :: key(), - Data :: any(), - MaxAge :: max_age_secs(), - Depend :: dependencies(), - From :: {pid(), atom()}, - State :: state(), - Result :: {reply, Reply, State}, - Reply :: ok; + Request :: {set, Key, Data, MaxAge, Depend}, + Key :: key(), + Data :: any(), + MaxAge :: max_age_secs(), + Depend :: dependencies(), + From :: {pid(), atom()}, + State :: state(), + Result :: {reply, Reply, State}, + Reply :: ok; (Request, From, State) -> Result when - Request :: {flush, Key}, - Key :: key(), - From :: {pid(), atom()}, - State :: state(), - Result :: {reply, ok, State}; + Request :: {flush, Key}, + Key :: key(), + From :: {pid(), atom()}, + State :: state(), + Result :: {reply, ok, State}; (Request, From, State) -> Result when Request :: flush, From :: {pid(), atom()}, @@ -799,10 +799,10 @@ handle_call(flush, _From, State) -> %% @doc Handling cast messages -spec handle_cast(Request, State) -> Result when - Request :: {flush, Key} | flush | any(), - Key :: key(), - State :: state(), - Result :: {noreply, State}. + Request :: {flush, Key} | flush | any(), + Key :: key(), + State :: state(), + Result :: {noreply, State}. handle_cast({flush, Key}, State) -> flush_key(Key, State), @@ -824,9 +824,9 @@ handle_cast(_Msg, State) -> %% @see gen_server:handle_info/2 -spec handle_info(Info, State) -> Result when - Info :: tick | any(), - State :: state(), - Result :: {noreply, State}. + Info :: tick | any(), + State :: state(), + Result :: {noreply, State}. handle_info(tick, State) -> erase_process_dict(), @@ -841,9 +841,9 @@ handle_info(_Msg, State) -> %% @see gen_server:terminate/2 -spec terminate(Reason, State) -> Result when - Reason :: normal | shutdown | {shutdown, term()} | term(), - State :: state(), - Result :: ok. + Reason :: normal | shutdown | {shutdown, term()} | term(), + State :: state(), + Result :: ok. terminate(_Reason, _State) -> ok. From a44e3dd4742306a20d7af1958e9dd1789249b7d6 Mon Sep 17 00:00:00 2001 From: Anatolii Kosorukov Date: Sat, 23 Apr 2022 21:35:57 +0300 Subject: [PATCH 10/13] Add proper tests Add proper tests for depcache module. Add launching proper test by Makefile. Move dializer launching to rebar3 test profile. Add changes to rebar.config. --- Makefile | 3 +- rebar.config | 7 +- test/prop_depcache.erl | 164 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 test/prop_depcache.erl diff --git a/Makefile b/Makefile index b36d681..0b97e07 100644 --- a/Makefile +++ b/Makefile @@ -10,12 +10,13 @@ compile: $(REBAR) test: $(REBAR) $(REBAR) eunit + $(REBAR) as test proper -n 10 xref: $(REBAR) $(REBAR) xref dialyzer: $(REBAR) - $(REBAR) dialyzer + $(REBAR) as test dialyzer edoc: mkdir -p doc && cp -fR doc_src/* doc diff --git a/rebar.config b/rebar.config index b996dcc..3da72cc 100644 --- a/rebar.config +++ b/rebar.config @@ -20,6 +20,11 @@ ]} ]}, {test, [ - {deps, [{proper, "1.3.0"}]} + {dialyzer, [ + {warnings, [ + no_return + ]} + ]}, + {deps, [{proper, "1.4.0"}]} ]} ]}. \ No newline at end of file diff --git a/test/prop_depcache.erl b/test/prop_depcache.erl new file mode 100644 index 0000000..49cdb77 --- /dev/null +++ b/test/prop_depcache.erl @@ -0,0 +1,164 @@ +-module(prop_depcache). +-include_lib("proper/include/proper.hrl"). + +%%%%%%%%%%%%%%%%%% +%%% Properties %%% +%%%%%%%%%%%%%%%%%% +prop_set_get() -> + ?SETUP(setup(), + ?FORALL({Key,Value}, {key(),value()}, + begin + Server = whereis(dep), + undefined = depcache:get(Key, Server), + ok = depcache:set(Key, Value, Server), + {ok, Value} = depcache:get(Key, Server), + ok = depcache:flush(Key, Server), + undefined =:= depcache:get(Key, Server) + end + ) + ). + +prop_flush_all() -> + ?SETUP(setup(), + ?FORALL(List, ?SIZED(Size, resize(Size*3, list({key(), value()}))), + begin + Server = whereis(dep), + lists:foreach(fun({Key,Value}) -> + depcache:set(Key, Value, Server) + end, List), + ok = depcache:flush(Server), + lists:all(fun({Key, _Value}) -> + undefined =:= depcache:get(Key, Server) + end, List) + end + ) + ). + +prop_get_set_maxage() -> + ?SETUP(setup(), + ?FORALL({Key,Value}, {key(),value()}, + begin + Server = whereis(dep), + %% Set a key and hold it for one second. + TimeValueSec = 1, + ok = depcache:set(Key, Value, TimeValueSec, Server), + {ok, Value} = depcache:get(Key, Server), + %% Let the depcache time out. + timer:sleep(3000), + undefined =:= depcache:get(Key, Server) + end + ) + ). + +prop_get_set_maxage_0() -> + ?SETUP(setup(), + ?FORALL({Key,Value}, {key(),value()}, + begin + Server = whereis(dep), + %% Set a key and hold it for zero second. + TimeValueSec = 0, + ok = depcache:set(Key, Value, TimeValueSec, Server), + undefined =:= depcache:get(Key, Server) + end + ) + ). + +prop_get_set_depend() -> + ?SETUP(setup(), + ?FORALL({Key,{DepKey, DepValue}}, + ?SUCHTHAT({Key, {DepKey, _DepValue}}, {key(), {key(), value()}}, + Key =/= DepKey andalso not is_integer(DepKey) ), + begin + Server = whereis(dep), + TimeValueSec = 2, + ok = depcache:set(Key, [{DepKey, DepValue}], TimeValueSec, [DepKey], Server), + {ok, [{DepKey,DepValue}]} = depcache:get(Key, Server), + {ok, DepValue} = depcache:get(Key, DepKey, Server), + {ok, DepValue} = depcache:get_subkey(Key, DepKey, Server), + undefined == depcache:get(DepKey, Server) andalso + ok == depcache:flush(Key, Server) + end + ) + ). + +prop_get_set_depend_map() -> + ?SETUP(setup(), + ?FORALL({Key,{DepKey, DepValue}}, + ?SUCHTHAT({Key, {DepKey, _DepValue}}, {key(), {key(), value()}}, + Key =/= DepKey andalso not is_integer(DepKey) ), + begin + Server = whereis(dep), + ok = depcache:set(Key, #{ DepKey => DepValue }, Server), + {ok, #{DepKey := DepValue}} = depcache:get(Key, Server), + {ok, DepValue} = depcache:get(Key, DepKey, Server), + {ok, DepValue} = depcache:get_subkey(Key, DepKey, Server), + undefined == depcache:get(DepKey, Server) andalso + ok == depcache:flush(Key, Server) + end + ) + ). + +prop_memo() -> + ?SETUP(fun() -> + IncreaseFun = fun(X) -> + I = case erlang:get(X) of + undefined -> 1; + Num -> Num + 1 + end, + erlang:put(X, I), + I + end, + erlang:put("IncreaseFun", IncreaseFun), + erlang:put("MemoValue", 0), + fun() -> ok end + end, + ?SETUP(setup(), + ?FORALL(Key, key(), + begin + Server = whereis(dep), + IncreaseFun = erlang:get("IncreaseFun"), + MemoValue = erlang:get("MemoValue"), + IncreaserFunX = fun() -> IncreaseFun(ok) end, + + DepCacheMemo1 = depcache:memo(IncreaserFunX, Key, Server), + DepCacheMemo1 = MemoValue + 1, + ok = depcache:flush(Key, Server), + + DepCacheMemo2 = depcache:memo(IncreaserFunX, Key, Server), + DepCacheMemo2 = MemoValue + 2, + ok = depcache:flush(Key, Server), + + erlang:put("MemoValue", MemoValue + 2), + + true + end + ) + ) + ). + + +%%%%%%%%%%%%%%% +%%% Helpers %%% +%%%%%%%%%%%%%%% + +setup() -> + fun() -> + {ok, Server} = depcache:start_link([]), + register(dep, Server), + fun() -> + depcache:flush(Server), + case whereis(dep) of + undefined -> ok; + _Pid -> unregister(dep) + end, + ok + end + end. + + + +%%%%%%%%%%%%%%%%%% +%%% Generators %%% +%%%%%%%%%%%%%%%%%% +key() -> non_empty(any()). +value() -> non_empty(any()). From 4a3c8b674ea8c0a4feb5e0322f312aa7d1616275 Mon Sep 17 00:00:00 2001 From: Anatolii Kosorukov Date: Sat, 7 May 2022 08:54:14 +0300 Subject: [PATCH 11/13] Simplify documentation generation and testing --- .gitignore | 4 +++- Makefile | 2 -- {doc_src => doc}/overview.edoc | 0 {doc_src => doc}/style.css | 0 rebar.config | 4 +++- 5 files changed, 6 insertions(+), 4 deletions(-) rename {doc_src => doc}/overview.edoc (100%) rename {doc_src => doc}/style.css (100%) diff --git a/.gitignore b/.gitignore index df8bb39..feb44c2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ *.beam /ebin/depcache.app /.rebar/erlcinfo -doc/ _build rebar3 +doc/* +!doc/style.css +!doc/overview.edoc \ No newline at end of file diff --git a/Makefile b/Makefile index 0b97e07..bc88279 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,9 @@ dialyzer: $(REBAR) $(REBAR) as test dialyzer edoc: - mkdir -p doc && cp -fR doc_src/* doc $(REBAR) edoc edoc_private: - mkdir -p doc && cp -fR doc_src/* doc $(REBAR) as edoc_private edoc clean: diff --git a/doc_src/overview.edoc b/doc/overview.edoc similarity index 100% rename from doc_src/overview.edoc rename to doc/overview.edoc diff --git a/doc_src/style.css b/doc/style.css similarity index 100% rename from doc_src/style.css rename to doc/style.css diff --git a/rebar.config b/rebar.config index 3da72cc..afd771c 100644 --- a/rebar.config +++ b/rebar.config @@ -25,6 +25,8 @@ no_return ]} ]}, - {deps, [{proper, "1.4.0"}]} + {deps, [{proper, "1.4.0"}]}, + {erl_opts, [{platform_define, "^20", + 'OTP_GREATER_THAN_20'}]} ]} ]}. \ No newline at end of file From bdd34b1d83dcc4c1f7e2a4648e8fbe9d74431df5 Mon Sep 17 00:00:00 2001 From: Anatolii Kosorukov Date: Sat, 7 May 2022 09:07:37 +0300 Subject: [PATCH 12/13] Update rebar.config The setting of the configuration file that did not work during automatic testing was removed. --- rebar.config | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rebar.config b/rebar.config index afd771c..3da72cc 100644 --- a/rebar.config +++ b/rebar.config @@ -25,8 +25,6 @@ no_return ]} ]}, - {deps, [{proper, "1.4.0"}]}, - {erl_opts, [{platform_define, "^20", - 'OTP_GREATER_THAN_20'}]} + {deps, [{proper, "1.4.0"}]} ]} ]}. \ No newline at end of file From 1e387472205492549c047ff1c28cbb63a5f447ef Mon Sep 17 00:00:00 2001 From: Anatolii Kosorukov Date: Sat, 7 May 2022 11:46:28 +0300 Subject: [PATCH 13/13] Update OTP version list Update OTP version list which is supported. --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3b030bb..fb276e1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: - otp_version: [19.3,20.3,21,22,23] + otp_version: [20.3,21,22,23,24] os: [ubuntu-latest] container: