Permalink
Browse files

add an ets backend and a frontend to parameterize them.

  • Loading branch information...
1 parent 77c2281 commit 9218de6ff76322dcd06b76f6b1643ba63ea50da1 @etrepum committed May 17, 2012
Showing with 186 additions and 51 deletions.
  1. +21 −51 src/markov.erl
  2. +68 −0 src/markov_ets.erl
  3. +64 −0 src/markov_gb.erl
  4. +33 −0 src/markov_server.erl
View
72 src/markov.erl
@@ -1,20 +1,21 @@
-module(markov).
--export([new/0, input/2, output/1, output/2, from_list/1, to_list/1]).
+-export([new/0]).
+-export([new/1, input/2, output/1, output/2, from_list/2, to_list/1]).
-define(WORD_MAX, 100).
+-define(DEFAULT_STORAGE, markov_ets).
new() ->
- gb_trees:empty().
+ new(?DEFAULT_STORAGE).
-from_list(L) ->
- lists:foldl(fun ({K, V}, T) ->
- gb_trees:insert(K, subtree_from_list(V), T)
- end,
- gb_trees:empty(),
- L).
+new(M) ->
+ {M, M:new()}.
-to_list(T) ->
- to_orddict_iter(gb_trees:next(gb_trees:iterator(T))).
+from_list(L, {M, T}) ->
+ {M, M:from_list(L, T)}.
+
+to_list({M, T}) ->
+ M:to_list(T).
input(B, T) when is_binary(B) ->
input_tokens([Word || Word <- re:split(B, "\\s+", [{return, binary}]),
@@ -42,21 +43,8 @@ input([C | Rest], {A, B}, T) ->
input([], {A, B}, T) ->
append({A, B}, stop, T).
-append(K, V, T) ->
- case gb_trees:lookup(K, T) of
- none ->
- gb_trees:insert(K, {1, increment(V, gb_trees:empty())}, T);
- {value, {Count, SubT}} ->
- gb_trees:update(K, {1 + Count, increment(V, SubT)}, T)
- end.
-
-increment(K, SubT) ->
- case gb_trees:lookup(K, SubT) of
- none ->
- gb_trees:insert(K, 1, SubT);
- {value, N} ->
- gb_trees:update(K, 1 + N, SubT)
- end.
+append(K, V, {M, T}) ->
+ {M, M:append(K, V, T)}.
fetch(L, Max, T) ->
{Pair, Next} = case L of
@@ -71,40 +59,22 @@ fetch(L, Max, T) ->
Res
end.
-fetch(K={_A, B}, RState, Max, T) when Max > 0 ->
- case choose(gb_trees:lookup(K, T), RState) of
+fetch(K={_A, B}, RState, Max, T={M, MT}) when Max > 0 ->
+ case choose(M:lookup(K, MT), RState, M) of
{stop, _} ->
[];
{C, RState1} ->
[C | fetch({B, C}, RState1, Max - 1, T)]
- end.
+ end;
+fetch(_K, _RState, _Max, _T) ->
+ [].
-choose({value, {Count, SubT}}, RState) ->
+choose({value, {Count, SubT}}, RState, M) ->
{N, RState1} = random:uniform_s(Count, RState),
- {choose_nth(N, gb_trees:next(gb_trees:iterator(SubT))), RState1};
-choose(none, RState) ->
+ {M:choose_nth(N, SubT), RState1};
+choose(none, RState, _M) ->
{stop, RState}.
-choose_nth(N, {_K, V, Iter}) when N > V ->
- choose_nth(N - V, gb_trees:next(Iter));
-choose_nth(_N, {K, _V, _Iter}) ->
- K.
-
-subtree_from_list(L) ->
- lists:foldl(fun ({K, V}, {C, T}) ->
- {V + C, gb_trees:insert(K, V, T)}
- end,
- {0, gb_trees:empty()},
- L).
-
-to_orddict_iter(none) ->
- [];
-to_orddict_iter({K, V, Iter}) ->
- [{K, to_orddict_subtree(V)} | to_orddict_iter(gb_trees:next(Iter))].
-
-to_orddict_subtree({_Count, T}) ->
- gb_trees:to_list(T).
-
redact(S) ->
re:replace(
S,
View
68 src/markov_ets.erl
@@ -0,0 +1,68 @@
+-module(markov_ets).
+-export([new/0, from_list/1, from_list/2, to_list/1, append/3, lookup/2,
+ choose_nth/2]).
+
+-define(WORD_MAX, 100).
+
+new() ->
+ %% {{{token(), token()}, pair}, integer()}
+ %% {{{token(), token()}, token()}, integer()}
+ ets:new(?MODULE, [ordered_set]).
+
+from_list(L) ->
+ from_list(L, new()).
+
+from_list([{K, V} | Rest], T) ->
+ Total = lists:foldl(
+ fun ({Token, C}, Acc) ->
+ true = ets:insert_new(T, {{K, Token}, C}),
+ C + Acc
+ end,
+ 0,
+ V),
+ true = ets:insert_new(T, {{K, pair}, Total}),
+ from_list(Rest, T);
+from_list([], T) ->
+ T.
+
+to_list(T) ->
+ [{K, unroll(K, T)}
+ || K <- ets:select(T, [{{{'$1',pair},'_'},[],['$1']}])].
+
+unroll(K, T) ->
+ ets:select(
+ T,
+ [{{{K, '$1'}, '$2'}, [{'=/=', '$1', pair}], [{{'$1', '$2'}}]}]).
+
+inc(T, K) ->
+ try ets:update_counter(T, K, 1)
+ catch error:badarg ->
+ ets:insert_new(T, {K, 1}),
+ 1
+ end.
+
+append(K, V, T) ->
+ inc(T, {K, pair}),
+ inc(T, {K, V}),
+ T.
+
+lookup(K, T) ->
+ try ets:lookup_element(T, {K, pair}, 2)
+ of C ->
+ {value, {C, {K, T}}}
+ catch error:badarg ->
+ none
+ end.
+
+choose_nth(N, {K, T}) ->
+ %% NOTE: It's important that 'pair' sorts lower than
+ %% 'start', 'stop', and any binary!
+ choose_nth_iter(N, ets:next(T, {K, pair}), T).
+
+choose_nth_iter(N, K={_Prev, Next}, T) ->
+ case ets:lookup_element(T, K, 2) of
+ V when N > V ->
+ choose_nth_iter(N - V, ets:next(T, K), T);
+ _ ->
+ Next
+ end.
View
64 src/markov_gb.erl
@@ -0,0 +1,64 @@
+-module(markov_gb).
+-export([new/0, from_list/1, from_list/2, to_list/1, append/3, lookup/2,
+ choose_nth/2]).
+
+-define(WORD_MAX, 100).
+
+new() ->
+ gb_trees:empty().
+
+from_list(L) ->
+ from_list(L, new()).
+
+from_list(L, EmptyT) ->
+ lists:foldl(fun ({K, V}, T) ->
+ gb_trees:insert(K, subtree_from_list(V), T)
+ end,
+ EmptyT,
+ L).
+
+to_list(T) ->
+ to_orddict_iter(gb_trees:next(gb_trees:iterator(T))).
+
+append(K, V, T) ->
+ case gb_trees:lookup(K, T) of
+ none ->
+ gb_trees:insert(K, {1, increment(V, gb_trees:empty())}, T);
+ {value, {Count, SubT}} ->
+ gb_trees:update(K, {1 + Count, increment(V, SubT)}, T)
+ end.
+
+increment(K, SubT) ->
+ case gb_trees:lookup(K, SubT) of
+ none ->
+ gb_trees:insert(K, 1, SubT);
+ {value, N} ->
+ gb_trees:update(K, 1 + N, SubT)
+ end.
+
+lookup(K, T) ->
+ gb_trees:lookup(K, T).
+
+
+choose_nth(N, SubT) ->
+ choose_nth_iter(N, gb_trees:next(gb_trees:iterator(SubT))).
+
+choose_nth_iter(N, {_K, V, Iter}) when N > V ->
+ choose_nth_iter(N - V, gb_trees:next(Iter));
+choose_nth_iter(_N, {K, _V, _Iter}) ->
+ K.
+
+subtree_from_list(L) ->
+ lists:foldl(fun ({K, V}, {C, T}) ->
+ {V + C, gb_trees:insert(K, V, T)}
+ end,
+ {0, gb_trees:empty()},
+ L).
+
+to_orddict_iter(none) ->
+ [];
+to_orddict_iter({K, V, Iter}) ->
+ [{K, to_orddict_subtree(V)} | to_orddict_iter(gb_trees:next(Iter))].
+
+to_orddict_subtree({_Count, T}) ->
+ gb_trees:to_list(T).
View
33 src/markov_server.erl
@@ -0,0 +1,33 @@
+-module(markov_server).
+-behaviour(gen_server).
+
+-export([start_link/1, start/1]).
+
+-export([code_change/3, handle_call/3, handle_cast/2, handle_info/2,
+ init/1, terminate/2]).
+
+-record(state, {}).
+
+start_link(Args) ->
+ gen_server:start_link(?MODULE, Args, []).
+
+start(Args) ->
+ gen_server:start(?MODULE, Args, []).
+
+init([]) ->
+ {ok, #state{}}.
+
+handle_cast(_Req, State) ->
+ {noreply, State}.
+
+handle_info(_Req, State) ->
+ {noreply, State}.
+
+handle_call(_Req, _From, State) ->
+ {reply, ignored, State}.
+
+terminate(_Reason, _State) ->
+ ok.
+
+code_change(_OldVsn, State, _Extra) ->
+ {ok, State}.

0 comments on commit 9218de6

Please sign in to comment.