Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Remove JS extractor support #80

Merged
merged 1 commit into from

2 participants

@rzezeski

bz1144

It's been decided that JavaScript extractor support will be removed.
The main reasons for doing so are:

  1. This feature has never worked so no existing users are being broken.

  2. The documentation around this feature isn't entirely clear and the
    code to support it is ugly, IMO.

  3. Like any other JS support in Riak it has to pay the penalty of
    shipping bytes between the Erlang and JS VMs. There is also potential
    contention for JS VMs depending on the size of the pool and load on
    the system.

  4. Finally, the main reason I can think of for anyone to use the JS
    extractor support is to pull apart incoming JSON objects. This is
    already handled via an Erlang extractor that is invoked any time it
    sees an application/json content-type header. This will be much
    more efficient than using a JS extractor.

@rzezeski

While doing this I checked to make sure our documented ways of setting the extractor worked and it turns out that {qfun, Fun} causes the ring manager to crash. For now I simply removed that method from the documentation but I'd certainly be willing to talk about fixing it/removing it and adding another PR.

@kellymclaughlin
Collaborator

I get an error from make test:

src/riak_search_kv_hook.erl:326: Warning: function maybe_start_app/1 is unused
make: *** [test] Error 1

@kellymclaughlin
Collaborator

In regards to the {qfun, Fun} problem, since it's been previously documented and released, I'd say make a card to fix it unless you really feel strongly that it should just be removed. Nice catch btw.

@kellymclaughlin
Collaborator

+1 to merge.

@rzezeski rzezeski Remove JS extractor support
bz1144

It's been decided that JavaScript extractor support will be removed.
The main reasons for doing so are:

1. This feature has never worked so no existing users are being broken.

2. The documentation around this feature isn't entirely clear and the
code to support it is ugly, IMO.

3. Like any other JS support in Riak it has to pay the penalty of
shipping bytes between the Erlang and JS VMs.  There is also potential
contention for JS VMs depending on the size of the pool and load on
the system.

4. Finally, the main reason I can think of for anyone to use the JS
extractor support is to pull apart incoming JSON objects.  This is
already handled via an Erlang extractor that is invoked any time it
sees an `application/json` content-type header.  This will be much
more efficient than using a JS extractor.
6ce074c
@rzezeski rzezeski merged commit 9912613 into from
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Aug 19, 2011
  1. @rzezeski

    Remove JS extractor support

    rzezeski authored
    bz1144
    
    It's been decided that JavaScript extractor support will be removed.
    The main reasons for doing so are:
    
    1. This feature has never worked so no existing users are being broken.
    
    2. The documentation around this feature isn't entirely clear and the
    code to support it is ugly, IMO.
    
    3. Like any other JS support in Riak it has to pay the penalty of
    shipping bytes between the Erlang and JS VMs.  There is also potential
    contention for JS VMs depending on the size of the pool and load on
    the system.
    
    4. Finally, the main reason I can think of for anyone to use the JS
    extractor support is to pull apart incoming JSON objects.  This is
    already handled via an Erlang extractor that is invoked any time it
    sees an `application/json` content-type header.  This will be much
    more efficient than using a JS extractor.
This page is out of date. Refresh to see the latest.
Showing with 4 additions and 153 deletions.
  1. +0 −1  include/riak_search.hrl
  2. +4 −152 src/riak_search_kv_hook.erl
View
1  include/riak_search.hrl
@@ -8,7 +8,6 @@
-define(DEFAULT_INDEX, <<"search">>).
-define(RESULTVEC_SIZE, 1000).
-define(OPKEY(Tag, Op), {Tag, element(2, Op)}).
--define(JSPOOL_SEARCH_EXTRACT, riak_search_js_extract).
-record(search_state, {
parent=undefined,
View
156 src/riak_search_kv_hook.erl
@@ -21,21 +21,13 @@
-define(DEFAULT_ARGS, undefined).
-type user_funterm() :: {modfun, user_modname(), user_funname()} |
- {qfun, extract_qfun()} |
- {jsanon, user_strorbin()} |
- {jsanon, {user_strorbin(), user_strorbin()}} |
- {jsfun, user_strorbin()}.
+ {qfun, extract_qfun()}.
-type user_modname() :: string() | module().
-type user_funname() :: string() | atom().
--type user_strorbin() :: string() | binary().
-
-type extractdef() :: {funterm(), args()}.
-type funterm() :: {modfun, atom(), atom()} |
- {qfun, extract_qfun()} |
- {jsanon, binary()} |
- {jsanon, {binary(), binary()}} |
- {jsfun, binary()}.
+ {qfun, extract_qfun()}.
-type riak_object() :: tuple(). % no good way to define riak_object
@@ -130,12 +122,6 @@ validate_funterm({modfun, Mod, Fun}) ->
{modfun, to_modfun(Mod), to_modfun(Fun)};
validate_funterm({qfun, Fun}=FunTerm) when is_function(Fun) ->
FunTerm;
-validate_funterm({jsanon, {Bucket, Key}}) ->
- {jsanon, {to_binary(Bucket), to_binary(Key)}};
-validate_funterm({jsanon, Source}) ->
- {jsanon, to_binary(Source)};
-validate_funterm({jsfun, Name}) ->
- {jsfun, to_binary(Name)};
validate_funterm(FunTerm) ->
throw({"cannot parse funterm", FunTerm}).
@@ -145,33 +131,8 @@ erlify_json_funterm(<<"erlang">>, Props) ->
Fun = to_modfun(proplists:get_value(<<"function">>, Props, undefined)),
Arg = proplists:get_value(<<"arg">>, Props, undefined),
{{modfun, Mod, Fun}, Arg};
-erlify_json_funterm(<<"javascript">>, Props) ->
- Source = proplists:get_value(<<"source">>, Props, undefined),
- Name = proplists:get_value(<<"name">>, Props, undefined),
- Bucket = proplists:get_value(<<"bucket">>, Props, undefined),
- Key = proplists:get_value(<<"key">>, Props, undefined),
- Arg = proplists:get_value(<<"arg">>, Props, undefined),
- case Source of
- undefined ->
- case Name of
- undefined ->
- case (Bucket == undefined) orelse (Key == undefined) of
- true ->
- throw("javascript kv/search extractor must have"
- "name, source, or bucket and key");
- _ ->
- {{jsanon, {Bucket, Key}}, Arg}
- end;
- _ ->
- {{jsfun, Name}, Arg}
- end;
- _ ->
- {{jsanon, Source}, Arg}
- end;
-erlify_json_funterm(Lang, _Props) ->
- throw({"kv/search extractors must be written in erlang or javascript", Lang}).
-
-
+erlify_json_funterm(_Lang, _Props) ->
+ throw({"Malformed KV/Search extractor", _Props}).
-spec to_modfun(list() | atom()) -> atom().
to_modfun(List) when is_list(List) ->
@@ -184,16 +145,6 @@ to_modfun(Atom) when is_atom(Atom) ->
Atom;
to_modfun(Val) ->
throw({"cannot convert to module/function name", Val}).
-
--spec to_binary(atom() | string() | binary()) -> binary().
-to_binary(Atom) when is_atom(Atom) ->
- atom_to_binary(Atom, utf8);
-to_binary(List) when is_list(List) ->
- list_to_binary(List);
-to_binary(Bin) when is_binary(Bin) ->
- Bin;
-to_binary(Val) ->
- throw({"cannot convert to binary", Val}).
%%
%% Index the provided riak object and return ok on success.
@@ -252,51 +203,8 @@ run_extract(RiakObject, DefaultField, {{modfun, Mod, Fun}, Arg}) ->
Mod:Fun(RiakObject, DefaultField, Arg);
run_extract(RiakObject, DefaultField, {{qfun, Fun}, Arg}) ->
Fun(RiakObject, DefaultField, Arg);
-run_extract(RiakObject, DefaultField, {{Js, FunTerm}, Arg})
- when Js == jsanon; Js == jsfun ->
- Fun = if is_binary(FunTerm) -> FunTerm;
- is_tuple(FunTerm) ->
- {Bucket, Key} = FunTerm,
- {ok, Client} = riak:local_client(),
- try
- {ok, JSObj} = Client:get(Bucket, Key, 1),
- hd(riak_object:get_values(JSObj))
- catch
- error:{badmatch,{error,notfound}} ->
- throw({fail, {"Extractor not found", {Bucket, Key}}})
- end
- end,
- JsRObj = riak_object:to_json(RiakObject),
- case Arg of
- undefined ->
- JsArg = null;
- _ ->
- JsArg = Arg
- end,
- case riak_kv_js_manager:blocking_dispatch(?JSPOOL_SEARCH_EXTRACT, {{Js, Fun}, [JsRObj, DefaultField, JsArg]}, 5) of
- {ok, <<"fail">>} ->
- throw(fail);
- {ok, [{<<"fail">>, Message}]} ->
- throw({fail, Message});
- {ok, JsonFields} ->
- erlify_json_extract(JsonFields);
- {error, Error} ->
- lager:error("Error executing kv/search hook: ~s", [Error]),
- throw({fail, Error})
- end;
run_extract(_, _, ExtractDef) ->
throw({error, {not_implemented, ExtractDef}}).
-
-erlify_json_extract(R) ->
- erlify_json_extract(R, []).
-
-erlify_json_extract([], Acc) ->
- lists:reverse(Acc);
-erlify_json_extract([{FieldName, FieldData} | Rest], Acc) when is_binary(FieldName),
- is_binary(FieldData) ->
- erlify_json_extract(Rest, [{FieldName, FieldData} | Acc]);
-erlify_json_extract(R, _Acc) ->
- throw({fail, {bad_json_extractor, R}}).
%% Get the precommit hook from the bucket and strip any
%% existing index hooks.
@@ -396,67 +304,12 @@ run_qfun_extract_test() ->
Extractor = validate_extractor({{qfun, Fun1}, undefined}),
?assertEqual([{<<"data">>,<<"some data">>}],
run_extract(TestObj, <<"data">>, Extractor)).
-
-
-
-anon_js_extract_test() ->
- maybe_start_app(sasl),
- maybe_start_app(erlang_js),
- JsSup = maybe_start_link(riak_kv_js_sup:start_link()),
- JsMgr = maybe_start_link(riak_kv_js_manager:start_link(?JSPOOL_SEARCH_EXTRACT, 2)),
-
- %% Anonymous JSON function with default argument
- %% Join together all the values in a search field
- %% called "data" and the argument as "arg"
- JustObjectSource = "function(o, d) {
- var vals = [];
- for (var i = 0; i < o.values.length; i++) {
- vals.push(o.values[i].data);
- }
- data = vals.join(\" \");
- return {\"data\":data};
- }",
- ObjectArgSource = "function(o,d,a) {
- var vals = [];
- for (var i = 0; i < o.values.length; i++) {
- vals.push(o.values[i].data);
- }
- data = vals.join(\" \");
- return {\"data\":data, \"arg\":a.f};
- }",
-
- %% Try the anonymous function
- O = conflict_test_object(),
- Extractor1 = validate_extractor({{jsanon, JustObjectSource}, undefined}),
- ?assertEqual([{<<"data">>,<<"some data">>}],
- run_extract(O, <<"data">>, Extractor1)),
-
- %% Anonymous JSON function with provided argument
- %% Arg = {struct [{<<"f">>,<<"v">>}]},
- Arg = {struct, [{<<"f">>,<<"v">>}]},
- Extractor2 = validate_extractor({{jsanon, ObjectArgSource}, Arg}),
- ?assertEqual([{<<"data">>,<<"some data">>},
- {<<"arg">>, <<"v">>}],
- run_extract(O, <<"value">>, Extractor2)),
-
- stop_pid(JsMgr),
- stop_pid(JsSup),
- application:stop(erlang_js).
conflict_test_object() ->
O0 = riak_object:new(<<"b">>,<<"k">>,<<"v">>),
riak_object:set_contents(O0, [{dict:new(), <<"some">>},
{dict:new(), <<"data">>}]).
-
-maybe_start_app(App) ->
- case application:start(App) of
- {error, {already_started, _}} ->
- ok;
- ok ->
- ok
- end.
-
maybe_start_link({ok, Pid}) ->
Pid;
maybe_start_link({error, {already_started, _}}) ->
@@ -469,4 +322,3 @@ stop_pid(Pid) ->
exit(Pid, kill).
-endif. % TEST
-
Something went wrong with that request. Please try again.