Skip to content
Newer
Older
100644 425 lines (366 sloc) 15 KB
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
1 %% -------------------------------------------------------------------
2 %%
3 %% This file is provided to you under the Apache License,
4 %% Version 2.0 (the "License"); you may not use this file
5 %% except in compliance with the License. You may obtain
6 %% a copy of the License at
7 %%
8 %% http://www.apache.org/licenses/LICENSE-2.0
9 %%
10 %% Unless required by applicable law or agreed to in writing,
11 %% software distributed under the License is distributed on an
12 %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
13 %% KIND, either express or implied. See the License for the
14 %% specific language governing permissions and limitations
15 %% under the License.
16 %%
17 %% -------------------------------------------------------------------
18
19 -module(riak_link_index).
20 -author("Kresten Krab Thorup <krab@trifork.com>").
21
22 -export([precommit/1,postcommit/1]).
23
e02bb69 @krestenkrab Allow a riak_link_set to be json encoded (configuration).
authored Apr 18, 2011
24 -define(CTYPE_ERLANG_BINARY,"application/x-erlang-binary").
25 -define(CTYPE_JSON,"application/json").
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
26 -define(MD_CTYPE,<<"content-type">>).
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
27 -define(MD_LINKS,<<"Links">>).
28 -define(MD_DELETED,<<"X-Riak-Deleted">>).
29 -define(IDX_PREFIX,"idx@").
30 -define(JSPOOL_HOOK, riak_kv_js_hook).
31
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
32 -ifdef(DEBUG).
33 -define(debug(A,B),error_logger:info_msg(A,B)).
34 -else.
35 -define(debug(A,B),ok).
36 -endif.
37
e02bb69 @krestenkrab Allow a riak_link_set to be json encoded (configuration).
authored Apr 18, 2011
38 -define(ENCODE_JSON,true).
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
39
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
40 precommit(Object) ->
41
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
42 ?debug("precommit in ~p", [Object]),
43
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
44 Bucket = riak_object:bucket(Object),
45 Key = riak_object:key(Object),
46
47 %% Indexing works in two phases: precommit will use a hook to add links as
48 %%
49 %% </riak/IBucket/IKey>; riaktag="idx@Tag"
50 %%
51 %% to the object being stored. Then postcommit creates empty-contents
52 %% objects named IBucket/IKey, with links to this object thus:
53 %%
54 %% <riak/Bucket/Key>; riaktag="Tag"
55 %%
56
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
57 case is_updated(Object) of
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
58 true ->
59 OldLinksToMe = get_index_links(riak_object:get_metadatas(Object)),
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
60 [{MD,_Value}] = index_contents(Bucket,
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
61 [{ riak_object:get_update_metadata(Object),
62 riak_object:get_update_value(Object) }]),
63 IndexedObject = riak_object:update_metadata(Object, MD);
64
65 false ->
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
66 {ok, StorageMod} = riak:local_client(),
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
67 case StorageMod:get(Bucket, Key) of
68 {ok, OldRO} ->
69 OldLinksToMe = get_index_links(riak_object:get_metadatas(OldRO));
70 _ ->
71 OldLinksToMe = []
72 end,
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
73 MDVs = index_contents(Bucket,
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
74 riak_object:get_contents(Object)),
75 IndexedObject = riak_object:set_contents(Object, MDVs)
76 end,
77
78 %% this only works in recent riak_kv master branch
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
79 put(?MODULE, {old_links, OldLinksToMe}),
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
80
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
81 ?debug("precommit out ~p", [IndexedObject]),
82
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
83 IndexedObject.
84
85 postcommit(Object) ->
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
86 try
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
87
88 case erlang:erase(?MODULE) of
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
89 {old_links, OldLinksToMe} ->
90 %% compute links to add/remove in postcommit
91 NewLinksToMe = get_index_links(Object),
92 LinksToRemove = ordsets:subtract(OldLinksToMe, NewLinksToMe),
93 LinksToAdd = ordsets:subtract(NewLinksToMe, OldLinksToMe),
94
95 ?debug("postcommit: old=~p, new=~p", [OldLinksToMe,NewLinksToMe]),
96
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
97 {ok, StorageMod} = riak:local_client(),
98 Bucket = riak_object:bucket(Object),
99 Key = riak_object:key(Object),
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
100 ClientID = StorageMod:get_client_id(),
101 add_links(StorageMod, LinksToAdd, Bucket, Key, ClientID),
102 remove_links(StorageMod, LinksToRemove, Bucket, Key, ClientID),
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
103 ok;
104 _ ->
105 error_logger:error_msg("error in pre/postcommit interaction", []),
106 ok
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
107 end
108
109 catch
110 Class:Reason ->
e02bb69 @krestenkrab Allow a riak_link_set to be json encoded (configuration).
authored Apr 18, 2011
111 error_logger:error_msg("error in postcommit ~p:~p ~p", [Class,Reason,erlang:get_stacktrace()]),
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
112 ok
113 end
114 .
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
115
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
116 add_links(StorageMod, Links, Bucket, Key, ClientID) ->
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
117 lists:foreach(fun({{IndexB,IndexK}, <<?IDX_PREFIX,Tag/binary>>}) ->
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
118 add_link(StorageMod, IndexB, IndexK, {{Bucket,Key},Tag}, ClientID)
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
119 end,
120 Links).
121
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
122
123 add_link(StorageMod, Bucket, Key, Link, ClientID) ->
124 update_links(
125 fun(VLinkSet) ->
126 ?debug("adding link ~p/~p -> ~p", [Bucket, Key, Link]),
e02bb69 @krestenkrab Allow a riak_link_set to be json encoded (configuration).
authored Apr 18, 2011
127 riak_link_set:add(Link, ClientID, VLinkSet)
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
128 end,
129 StorageMod, Bucket, Key).
130
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
131 remove_links(StorageMod, Links, Bucket, Key, ClientID) ->
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
132 lists:foreach(fun({{IndexB,IndexK}, <<?IDX_PREFIX,Tag/binary>>}) ->
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
133 remove_link(StorageMod, IndexB, IndexK, {{Bucket,Key},Tag}, ClientID)
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
134 end,
135 Links).
136
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
137 remove_link(StorageMod, Bucket, Key, Link, ClientID) ->
138 update_links(
139 fun(VLinkSet) ->
140 ?debug("removing link ~p/~p -> ~p", [Bucket, Key, Link]),
e02bb69 @krestenkrab Allow a riak_link_set to be json encoded (configuration).
authored Apr 18, 2011
141 riak_link_set:remove(Link, ClientID, VLinkSet)
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
142 end,
143 StorageMod, Bucket, Key).
144
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
145 update_links(Fun,StorageMod,Bucket,Key) ->
146 case StorageMod:get(Bucket,Key) of
147 {ok, Object} ->
148 ?debug("1", []),
149 VLinkSet = decode_merge_vsets(Object),
150 ?debug("decoded: ~p", [VLinkSet]),
151 VLinkSet2 = Fun(VLinkSet),
152 ?debug("transformed: ~p", [VLinkSet2]),
e02bb69 @krestenkrab Allow a riak_link_set to be json encoded (configuration).
authored Apr 18, 2011
153 Links = riak_link_set:values(VLinkSet2),
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
154 ?debug("new links: ~p", [Links]),
e02bb69 @krestenkrab Allow a riak_link_set to be json encoded (configuration).
authored Apr 18, 2011
155 case ?ENCODE_JSON of
156 true ->
157 Data = iolist_to_binary(mochijson2:encode(riak_link_set:to_json(VLinkSet2))),
158 CType = ?CTYPE_JSON;
159 false ->
160 Data = term_to_binary(VLinkSet2, [compressed]),
161 CType = ?CTYPE_ERLANG_BINARY
162 end,
163 IO1 = riak_object:update_value(Object, Data),
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
164 Updated = riak_object:update_metadata(IO1,
e02bb69 @krestenkrab Allow a riak_link_set to be json encoded (configuration).
authored Apr 18, 2011
165 dict:store(?MD_CTYPE, CType,
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
166 dict:store(?MD_LINKS, Links,
167 riak_object:get_update_metadata(IO1))));
168 _Got ->
169 ?debug("2: ~p from get(~p,~p)", [_Got, Bucket, Key]),
e02bb69 @krestenkrab Allow a riak_link_set to be json encoded (configuration).
authored Apr 18, 2011
170 VLinkSet2 = Fun(riak_link_set:new()),
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
171 ?debug("new set: ~p", [VLinkSet2]),
e02bb69 @krestenkrab Allow a riak_link_set to be json encoded (configuration).
authored Apr 18, 2011
172 case catch (riak_link_set:values(VLinkSet2)) of
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
173 Links -> ok
174 end,
175 ?debug("new links: ~p", [Links]),
e02bb69 @krestenkrab Allow a riak_link_set to be json encoded (configuration).
authored Apr 18, 2011
176 case ?ENCODE_JSON of
177 true ->
178 Data = iolist_to_binary(mochijson2:encode(riak_link_set:to_json(VLinkSet2))),
179 CType = ?CTYPE_JSON;
180 false ->
181 Data = term_to_binary(VLinkSet2, [compressed]),
182 CType = ?CTYPE_ERLANG_BINARY
183 end,
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
184 Updated = riak_object:new(Bucket,Key,
e02bb69 @krestenkrab Allow a riak_link_set to be json encoded (configuration).
authored Apr 18, 2011
185 Data,
186 dict:from_list([{?MD_CTYPE, CType},
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
187 {?MD_LINKS, Links}]))
188 end,
189
190 ?debug("storing ~p", [Updated]),
191 ok = StorageMod:put(Updated, 1).
192
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
193
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
194 decode_merge_vsets(Object) ->
195 lists:foldl(fun ({MD,V},Dict) ->
196 case dict:fetch(?MD_CTYPE, MD) of
e02bb69 @krestenkrab Allow a riak_link_set to be json encoded (configuration).
authored Apr 18, 2011
197 ?CTYPE_ERLANG_BINARY ->
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
198 Dict2 = binary_to_term(V),
e02bb69 @krestenkrab Allow a riak_link_set to be json encoded (configuration).
authored Apr 18, 2011
199 riak_link_set:merge(Dict,Dict2);
200 ?CTYPE_JSON ->
201 Dict2 = riak_link_set:from_json(mochijson2:decode(V)),
202 riak_link_set:merge(Dict,Dict2);
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
203 _ ->
204 Dict
205 end
206 end,
207 dict:new(),
208 riak_object:get_contents(Object)).
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
209
210
211 get_index_links(MDList) ->
212 ordsets:filter(fun({_, <<?IDX_PREFIX,_/binary>>}) ->
213 true;
214 (_) ->
215 false
216 end,
217 get_all_links(MDList)).
218
219 get_all_links(Object) when element(1,Object) =:= r_object ->
220 get_all_links
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
221 (case is_updated(Object) of
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
222 true ->
223 [riak_object:get_update_metadata(Object)]
224 ++ riak_object:get_metadatas(Object);
225 false ->
226 riak_object:get_metadatas(Object)
227 end);
228
229 get_all_links(MetaDatas) when is_list(MetaDatas) ->
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
230 Links = lists:foldl(fun(MetaData, Acc) ->
231 case dict:find(?MD_LINKS, MetaData) of
232 error ->
233 Acc;
234 {ok, LinksList} ->
235 LinksList ++ Acc
236 end
237 end,
238 [],
239 MetaDatas),
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
240
241 ordsets:from_list(Links).
242
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
243 index_contents(Bucket, Contents) ->
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
244
245 %% grab indexes from bucket properties
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
246 {ok, IndexHooks} = get_index_hooks(Bucket),
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
247
2e6386f @krestenkrab Make riak_link_index use vset module to manage conflits
authored Apr 18, 2011
248 ?debug("hooks are: ~p", [IndexHooks]),
249
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
250 lists:map
251 (fun({MD,Value}) ->
252 case dict:find(?MD_DELETED, MD) of
253 {ok, "true"} ->
254 {remove_idx_links(MD),Value};
255 _ ->
256 NewMD = compute_indexed_md(MD, Value, IndexHooks),
257 {NewMD, Value}
258 end
259 end,
260 Contents).
261
262 remove_idx_links(MD) ->
263 %% remove any "idx#..." links
264 case dict:find(?MD_LINKS, MD) of
265 error ->
266 MD;
267 {ok, Links} ->
268 dict:store
269 (?MD_LINKS,
270 lists:filter(fun({_,<<?IDX_PREFIX,_/binary>>}) ->
271 false;
272 (_) ->
273 true
274 end,
275 Links),
276 MD)
277 end.
278
279
280 compute_indexed_md(MD, Value, IndexHooks) ->
281 lists:foldl
282 (fun({struct, PropList}=IndexHook, MDAcc) ->
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
283 {<<"tag">>, Tag} = proplists:lookup(<<"tag">>, PropList),
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
284 Links = case dict:find(?MD_LINKS, MDAcc) of
285 error -> [];
286 {ok, MDLinks} -> MDLinks
287 end,
288 IdxTag = <<?IDX_PREFIX,Tag/binary>>,
289 KeepLinks =
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
290 lists:filter(fun({{_,_}, TagValue}) -> TagValue =/= IdxTag end,
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
291 Links),
292 NewLinksSansTag =
293 try apply_index_hook(IndexHook, MD, Value) of
294 {erlang, _, {ok, IL}} when is_list(IL) ->
295 IL;
296 {js, _, {ok, IL}} when is_list(IL) ->
297 IL;
298 _Val ->
299 error_logger:error_msg
300 ("indexing function returned ~p", [_Val]),
301 []
302 catch
303 _:_ ->
304 error_logger:error_msg
305 ("exception invoking indexing function", []),
306 []
307 end,
308
309 ResultLinks =
310 lists:map(fun({Bucket,Key}) when is_binary(Bucket), is_binary(Key) ->
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
311 {{Bucket, Key}, IdxTag};
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
312 ([Bucket, Key]) when is_binary(Bucket), is_binary(Key) ->
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
313 {{Bucket, Key}, IdxTag}
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
314 end,
315 NewLinksSansTag)
316 ++
317 KeepLinks,
318
319 dict:store(?MD_LINKS, ResultLinks, MDAcc)
320 end,
321 MD,
322 IndexHooks).
323
324
325 %%%%%% code from riak_kv_put_fsm %%%%%%
326
327
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
328 get_index_hooks(Bucket) ->
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
329
330 {ok,Ring} = riak_core_ring_manager:get_my_ring(),
331 BucketProps = riak_core_bucket:get_bucket(Bucket, Ring),
332
333 IndexHooks = proplists:get_value(link_index, BucketProps, []),
334 case IndexHooks of
335 <<"none">> ->
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
336 {ok, []};
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
337 {struct, Hook} ->
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
338 {ok, [{struct, Hook}]};
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
339 IndexHooks when is_list(IndexHooks) ->
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
340 {ok, IndexHooks};
341 V ->
342 error_logger:error_msg("bad value in bucket_prop ~p:link_index: ~p", [Bucket,V]),
343 {ok, []}
55daf57 @krestenkrab Initial commit
authored Apr 15, 2011
344 end.
345
346
347 apply_index_hook({struct, Hook}, MD, Value) ->
348 Mod = proplists:get_value(<<"mod">>, Hook),
349 Fun = proplists:get_value(<<"fun">>, Hook),
350 JSName = proplists:get_value(<<"name">>, Hook),
351 invoke_hook(Mod, Fun, JSName, MD, Value);
352 apply_index_hook(HookDef, _, _) ->
353 {error, {invalid_hook_def, HookDef}}.
354
355 invoke_hook(Mod0, Fun0, undefined, MD, Value) when Mod0 /= undefined, Fun0 /= undefined ->
356 Mod = binary_to_atom(Mod0, utf8),
357 Fun = binary_to_atom(Fun0, utf8),
358 try
359 {erlang, {Mod, Fun}, Mod:Fun(MD, Value)}
360 catch
361 Class:Exception ->
362 {erlang, {Mod, Fun}, {'EXIT', Mod, Fun, Class, Exception}}
363 end;
364 invoke_hook(undefined, undefined, JSName, MD, Value) when JSName /= undefined ->
365 {js, JSName, riak_kv_js_manager:blocking_dispatch
366 (?JSPOOL_HOOK, {{jsfun, JSName}, [jsonify_metadata(MD), Value]}, 5)};
367 invoke_hook(_, _, _, _, _) ->
368 {error, {invalid_hook_def, no_hook}}.
369
370
371
372
373 %%%%% code from riak_object %%%%%%
374
375 jsonify_metadata(MD) ->
376 MDJS = fun({LastMod, Now={_,_,_}}) ->
377 % convert Now to JS-readable time string
378 {LastMod, list_to_binary(
379 httpd_util:rfc1123_date(
380 calendar:now_to_local_time(Now)))};
381 ({<<"Links">>, Links}) ->
382 {<<"Links">>, [ [B, K, T] || {{B, K}, T} <- Links ]};
383 ({Name, List=[_|_]}) ->
384 {Name, jsonify_metadata_list(List)};
385 ({Name, Value}) ->
386 {Name, Value}
387 end,
388 {struct, lists:map(MDJS, dict:to_list(MD))}.
389
390 %% @doc convert strings to binaries, and proplists to JSON objects
391 jsonify_metadata_list([]) -> [];
392 jsonify_metadata_list(List) ->
393 Classifier = fun({Key,_}, Type) when (is_binary(Key) orelse is_list(Key)),
394 Type /= array, Type /= string ->
395 struct;
396 (C, Type) when is_integer(C), C >= 0, C =< 256,
397 Type /= array, Type /= struct ->
398 string;
399 (_, _) ->
400 array
401 end,
402 case lists:foldl(Classifier, undefined, List) of
403 struct -> {struct, [ {if is_list(Key) -> list_to_binary(Key);
404 true -> Key
405 end,
406 if is_list(Value) -> jsonify_metadata_list(Value);
407 true -> Value
408 end}
409 || {Key, Value} <- List]};
410 string -> list_to_binary(List);
411 array -> List
412 end.
ae23c07 @krestenkrab First version that actually works. Happy me.
authored Apr 15, 2011
413
414 is_updated(O) ->
415 M = riak_object:get_update_metadata(O),
416 V = riak_object:get_update_value(O),
417 case dict:find(clean, M) of
418 error -> true;
419 {ok,_} ->
420 case V of
421 undefined -> false;
422 _ -> true
423 end
424 end.
Something went wrong with that request. Please try again.