Permalink
Browse files

Merge pull request #8 from rzezeski/http-index

Index Creation via HTTP
  • Loading branch information...
2 parents 1a2bc1a + 1bfb2c1 commit d3c9002c7e4807e97f4c151f45bc2517a99bc59d @coderoshi coderoshi committed Oct 26, 2012
Showing with 358 additions and 11 deletions.
  1. +13 −0 README.md
  2. +1 −1 src/yokozuna_app.erl
  3. +15 −5 src/yz_events.erl
  4. +36 −3 src/yz_index.erl
  5. +20 −0 src/yz_kv.erl
  6. +27 −0 src/yz_misc.erl
  7. +9 −0 src/yz_schema.erl
  8. +4 −2 src/yz_solr.erl
  9. +233 −0 src/yz_wm_index.erl
View
@@ -83,6 +83,19 @@ build. The following instructions assume a devrel.
### Creating an Index ###
+An _index_ must be created in order for Yokozuna to index data.
+
+Currently the index name is a 1:1 mapping with the bucket name. This
+may eventually change to a 1:M mapping from index to bucket.
+
+You can create an index via the HTTP interface.
+
+ curl -XPUT -i -H 'content-type:application/json' http://localhost:8091/yz/index/name_of_index
+
+Optionally, you may create an index from the console.
+
+#### Index from Console
+
This command must be run from the Riak console--you must first attach
to it.
View
@@ -34,7 +34,7 @@ start(_StartType, _StartArgs) ->
case yokozuna_sup:start_link() of
{ok, Pid} ->
register_app(),
- Routes = yz_wm_search:routes() ++ yz_wm_extract:routes(),
+ Routes = yz_wm_search:routes() ++ yz_wm_extract:routes() ++ yz_wm_index:routes(),
yz_misc:add_routes(Routes),
{ok, Pid};
Error ->
View
@@ -65,6 +65,7 @@ init([]) ->
handle_cast({ring_event, Ring}=RE, S) ->
PrevRing = ?PREV_RING(S),
+ S2 = S#state{previous_ring=Ring},
Mapping = get_mapping(),
Mapping2 = new_mapping(RE, Mapping),
ok = set_mapping(Mapping2),
@@ -74,7 +75,7 @@ handle_cast({ring_event, Ring}=RE, S) ->
{Removed, Added, Same} = yz_misc:delta(Previous, Current),
ok = sync_indexes(Ring, Removed, Added, Same),
- {noreply, S}.
+ {noreply, S2}.
handle_info(tick, S) ->
ok = remove_non_owned_data(),
@@ -182,8 +183,17 @@ node_ops(Mapping, Nodes) ->
Added = sets:subtract(NodesSet, MappingNodesSet),
{sets:to_list(Removed), sets:to_list(Added)}.
-remove_indexes(_Names) ->
- throw(implement_remove_indexes).
+-spec remove_index(index_name()) -> ok.
+remove_index(Name) ->
+ case yz_index:exists(Name) of
+ true -> ok = yz_index:local_remove(Name);
+ false -> ok
+ end.
+
+-spec remove_indexes(index_set()) -> ok.
+remove_indexes(Names) ->
+ [ok = remove_index(N) || N <- Names],
+ ok.
-spec remove_node(node(), list()) -> list().
remove_node(Node, Mapping) ->
@@ -216,8 +226,8 @@ set_tick() ->
erlang:send_after(Interval, ?MODULE, tick),
ok.
-sync_indexes(Ring, _Removed, Added, Same) ->
- %% ok = remove_indexes(Removed),
+sync_indexes(Ring, Removed, Added, Same) ->
+ ok = remove_indexes(Removed),
ok = add_indexes(Ring, Added ++ Same).
watch_ring_events() ->
View
@@ -37,22 +37,34 @@ create(Name) ->
%% @doc Create the index `Name' across the entire cluster using
%% `SchemaName' as the schema.
--spec create(string(), schema_name()) -> ok.
+%% Returns 'ok' if created, or 'notfound' if the
+%% `SchemaName' is unknown
+-spec create(string(), schema_name()) -> ok | notfound.
create(Name, SchemaName) ->
- Info = make_info(Name, SchemaName),
- ok = add_to_ring(Name, Info).
+ case yz_schema:exists(SchemaName) of
+ false -> notfound;
+ true ->
+ Info = make_info(Name, SchemaName),
+ ok = add_to_ring(Name, Info)
+ end.
-spec exists(string()) -> boolean().
exists(Name) ->
true == yz_solr:ping(Name).
+%% @doc Removed the index `Name' from the entire cluster.
+-spec remove(string()) -> ok.
+remove(Name) ->
+ ok = remove_from_ring(Name).
+
-spec get_indexes_from_ring(ring()) -> indexes().
get_indexes_from_ring(Ring) ->
case riak_core_ring:get_meta(?YZ_META_INDEXES, Ring) of
{ok, Indexes} -> Indexes;
undefined -> []
end.
+-spec get_info_from_ring(ring(), index_name()) -> index_info().
get_info_from_ring(Ring, Name) ->
Indexes = get_indexes_from_ring(Ring),
orddict:fetch(Name, Indexes).
@@ -87,6 +99,16 @@ local_create(Ring, Name) ->
{ok, _, _} = yz_solr:core(create, CoreProps),
ok.
+%% @doc Remove the index `Name' locally.
+-spec local_remove(string()) -> ok.
+local_remove(Name) ->
+ CoreProps = [
+ {core, Name},
+ {delete_instance, "true"}
+ ],
+ {ok, _, _} = yz_solr:core(remove, CoreProps),
+ ok.
+
name(Info) ->
Info#index_info.name.
@@ -123,6 +145,17 @@ add_to_ring(Name, Info) ->
yz_misc:set_ring_meta(?YZ_META_INDEXES, Indexes2),
ok.
+-spec remove_index(indexes(), index_name()) -> indexes().
+remove_index(Indexes, Name) ->
+ orddict:erase(Name, Indexes).
+
+-spec remove_from_ring(index_name()) -> ok.
+remove_from_ring(Name) ->
+ Indexes = get_indexes_from_ring(yz_misc:get_ring(transformed)),
+ Indexes2 = remove_index(Indexes, Name),
+ yz_misc:set_ring_meta(?YZ_META_INDEXES, Indexes2),
+ ok.
+
index_dir(Name) ->
YZDir = app_helper:get_env(?YZ_APP_NAME, yz_dir, ?YZ_DEFAULT_DIR),
filename:absname(filename:join([YZDir, Name])).
View
@@ -116,6 +116,13 @@ install_hook(Bucket) when is_binary(Bucket) ->
Fun = index,
ok = riak_kv_vnode:add_obj_modified_hook(Bucket, Mod, Fun).
+%% @doc Uninstall the object modified hook on the given `Bucket'.
+-spec uninstall_hook(binary()) -> ok.
+uninstall_hook(Bucket) when is_binary(Bucket) ->
+ Mod = yz_kv,
+ Fun = index,
+ ok = remove_obj_modified_hook(Bucket, Mod, Fun).
+
%% @doc Write a value
-spec put(any(), binary(), binary(), binary(), string()) -> ok.
put(Client, Bucket, Key, Value, ContentType) ->
@@ -128,6 +135,19 @@ put(Client, Bucket, Key, Value, ContentType) ->
%% @private
%%
+%% @docs remove a hook from the bucket
+%%
+%% NOTE: Move this into riak_kv
+remove_obj_modified_hook(Bucket, Mod, Fun) ->
+ BProps = riak_core_bucket:get_bucket(Bucket),
+ Existing = proplists:get_value(obj_modified_hooks, BProps, []),
+ HookProp = {Mod, Fun},
+ Hooks = lists:delete(HookProp, Existing),
+ ok = riak_core_bucket:set_bucket(Bucket, [{obj_modified_hooks, Hooks}]).
+
+
+%% @private
+%%
%% @doc Determine whether process `Flag' is set.
-spec check_flag(term()) -> boolean().
check_flag(Flag) ->
View
@@ -96,6 +96,33 @@ make_dirs([Dir|Rest]) ->
ok = make_dir(Dir),
make_dirs(Rest).
+-spec delete_dir(string() | atom() | binary()) -> ok | {error, atom()}.
+delete_dir(Dir) ->
+ case filelib:is_file(Dir) of
+ true ->
+ case filelib:is_dir(Dir) of
+ true -> delete_dir([Dir], []);
+ false -> {error, enotdir}
+ end;
+ false -> {error, enoent}
+ end.
+
+delete_dir([], Acc) -> Acc;
+delete_dir([Path|Paths], Acc) ->
+ delete_dir(Paths,
+ case filelib:is_dir(Path) of
+ false ->
+ file:delete(Path);
+ % ok;
+ true ->
+ {ok, Listing} = file:list_dir(Path),
+ SubPaths = [filename:join(Path, Name) || Name <- Listing],
+ delete_dir(SubPaths, [Path | Acc]),
+ % dir should be clean by now
+ file:del_dir(Path)
+ % ok
+ end).
+
%% @doc Take a list `L' and pair adjacent elements wrapping around the
%% end by pairing the first with the last.
-spec make_pairs([T]) -> [{T,T}].
View
@@ -52,3 +52,12 @@ store(Name, RawSchema) when is_binary(RawSchema) ->
C = yz_kv:client(),
yz_kv:put(C, ?YZ_SCHEMA_BUCKET, Name, RawSchema, "text/xml").
+%% @doc Checks if the given `SchemaName' actually exists.
+-spec exists(schema_name()) -> true | false.
+exists(SchemaName) ->
+ case yz_schema:get(SchemaName) of
+ {notfound, _} -> false;
+ _ -> true
+ end.
+
+
View
@@ -24,7 +24,8 @@
-define(CORE_ALIASES, [{index_dir, instanceDir},
{cfg_file, config},
- {schema_file, schema}]).
+ {schema_file, schema},
+ {delete_instance, deleteInstanceDir}]).
-define(DEFAULT_URL, "http://localhost:8983/solr").
-define(DEFAULT_VCLOCK_N, 1000).
-define(QUERY(Str), {'query', [], [Str]}).
@@ -217,7 +218,8 @@ fpn_str(FPN) ->
?YZ_FPN_FIELD_S ++ ":" ++ integer_to_list(FPN).
convert_action(create) -> "CREATE";
-convert_action(status) -> "STATUS".
+convert_action(status) -> "STATUS";
+convert_action(remove) -> "UNLOAD".
%% TODO: Encoding functions copied from esolr, redo this.
encode_commit() ->
Oops, something went wrong.

0 comments on commit d3c9002

Please sign in to comment.