Permalink
Browse files

Use new MapReduce component everywhere

This removes the need for CouchJS, and therefore to link against
Spidermonkey, to use plain old couch views and for rereducing
values when doing view merging.

Change-Id: I9d1f4d749d490ac3fd33db4d74b9e5e69b6908c1
Reviewed-on: http://review.couchbase.org/14112
Reviewed-by: Damien Katz <damien@couchbase.com>
Tested-by: Damien Katz <damien@couchbase.com>
  • Loading branch information...
fdmanana authored and Damien Katz committed Mar 20, 2012
1 parent 6f4d485 commit eaa3f3be6e5e3c580ccda3673121019f4f9cc6ae
Showing with 412 additions and 2,612 deletions.
  1. +4 −103 configure.ac
  2. +4 −0 src/couch_set_view/src/couch_set_view_mapreduce.erl
  3. +2 −0 src/couchdb/Makefile.am
  4. +2 −2 src/couchdb/couch_db.hrl
  5. +2 −1 src/couchdb/couch_index_merger.erl
  6. +11 −5 src/couchdb/couch_view.erl
  7. +3 −1 src/couchdb/couch_view_compactor.erl
  8. +7 −12 src/couchdb/couch_view_group.erl
  9. +301 −0 src/couchdb/couch_view_mapreduce.erl
  10. +27 −16 src/couchdb/couch_view_merger.erl
  11. +47 −31 src/couchdb/couch_view_updater.erl
  12. +1 −53 src/couchdb/priv/Makefile.am
  13. +0 −694 src/couchdb/priv/couch_js/http.c
  14. +0 −18 src/couchdb/priv/couch_js/http.h
  15. +0 −357 src/couchdb/priv/couch_js/main.c
  16. +0 −286 src/couchdb/priv/couch_js/utf8.c
  17. +0 −19 src/couchdb/priv/couch_js/utf8.h
  18. +0 −62 src/couchdb/priv/couch_js/win32/couchjs.vcxproj.tpl.in
  19. +0 −2 src/couchdb/priv/couch_js/win32/msbuild.bat.tpl.in
  20. +0 −20 src/couchdb/priv/spawnkillable/couchspawnkillable.sh
  21. +0 −145 src/couchdb/priv/spawnkillable/couchspawnkillable_win.c
  22. +0 −26 test/etap/170-os-daemons.es
  23. +0 −115 test/etap/170-os-daemons.t
  24. +0 −85 test/etap/171-os-daemons-config.es
  25. +0 −75 test/etap/171-os-daemons-config.t
  26. +0 −17 test/etap/172-os-daemon-errors.1.sh
  27. +0 −15 test/etap/172-os-daemon-errors.2.sh
  28. +0 −15 test/etap/172-os-daemon-errors.3.sh
  29. +0 −15 test/etap/172-os-daemon-errors.4.sh
  30. +0 −128 test/etap/172-os-daemon-errors.t
  31. +0 −117 test/etap/173-os-daemon-cfg-register.t
  32. +0 −165 test/etap/210-os-proc-pool.t
  33. +0 −11 test/etap/Makefile.am
  34. +1 −1 test/etap/run.tpl
View
@@ -140,31 +140,6 @@ AC_SUBST(ERL_INTERFACE_DIR_INCLUDE)
ERL_INTERFACE_DIR_LIB="-L`${ERL} -eval 'io:put_chars(code:lib_dir(erl_interface) ++ "/lib"), timer:sleep(10), erlang:halt().' -noshell -noinput`"
AC_SUBST(ERL_INTERFACE_DIR_LIB)
-AC_ARG_WITH([js-include], [AC_HELP_STRING([--with-js-include=PATH],
- [set PATH to the SpiderMonkey include directory])], [
- JS_INCLUDE="$withval"
- JS_FLAGS="-I$JS_INCLUDE"
-], [
- JS_FLAGS="-I/usr/include"
- JS_FLAGS="$JS_FLAGS -I/usr/include/js"
- JS_FLAGS="$JS_FLAGS -I/usr/include/mozjs"
- JS_FLAGS="$JS_FLAGS -I/usr/local/include"
- JS_FLAGS="$JS_FLAGS -I/opt/local/include"
- JS_FLAGS="$JS_FLAGS -I/usr/local/include/js"
- JS_FLAGS="$JS_FLAGS -I/opt/local/include/js"
-])
-AC_SUBST(JS_INCLUDE)
-
-AC_ARG_WITH([js-lib], [AC_HELP_STRING([--with-js-lib=PATH],
- [set PATH to the SpiderMonkey library directory])],
- [
- JS_LIB_DIR=$withval
- JS_LIB_FLAGS="-L$withval"
-], [
- JS_LIB_DIR=
-])
-AC_SUBST(JS_LIB_DIR)
-
AC_ARG_VAR([ERLC_FLAGS], [general flags to prepend to ERLC_FLAGS])
AC_ARG_VAR([FLAGS], [general flags to prepend to LDFLAGS and CPPFLAGS])
@@ -173,12 +148,12 @@ if test "$ac_test_ERLCFLAGS" != set; then
ERLC_FLAGS=+debug_info
fi
-LIB_FLAGS="$JS_LIB_FLAGS -L/usr/local/lib -L/opt/local/lib"
+LIB_FLAGS="-L/usr/local/lib -L/opt/local/lib"
LIBS="$LIB_FLAGS $LIBS"
case "$(uname -s)" in
CYGWIN*|MINGW*)
- FLAGS="$LIB_FLAGS $ERLANG_FLAGS $JS_FLAGS -DXP_WIN $FLAGS"
+ FLAGS="$LIB_FLAGS $ERLANG_FLAGS -DXP_WIN $FLAGS"
CPPFLAGS="$FLAGS $CPPFLAGS"
LDFLAGS="$FLAGS $LDFLAGS"
IS_WINDOWS="TRUE"
@@ -189,7 +164,7 @@ case "$(uname -s)" in
;;
*)
# XP_UNIX required for jsapi.h and has been tested to work on Linux and Darwin.
- FLAGS="$LIB_FLAGS $ERLANG_FLAGS $JS_FLAGS -DXP_UNIX $FLAGS"
+ FLAGS="$LIB_FLAGS $ERLANG_FLAGS -DXP_UNIX $FLAGS"
CPPFLAGS="$FLAGS $CPPFLAGS"
# manually linking libm is requred for FreeBSD 7.0
LDFLAGS="$FLAGS -lm $LDFLAGS"
@@ -198,25 +173,6 @@ esac
AM_CONDITIONAL([WINDOWS], [test x$IS_WINDOWS = xTRUE])
-AC_ARG_ENABLE([spidermonkey],
- [AS_HELP_STRING([--disable-spidermonkey],
- [Allow to build without spidermonkey (things may not work as you expect) @<:@default=off@:>@])],
- [ac_enable_spidermonkey="$enableval"],
- [ac_enable_spidermonkey="yes"])
-
-AM_CONDITIONAL([BUILD_COUCHJS], [ test "x${ac_enable_spidermonkey}" = "xyes" ])
-
-AS_IF([ test "x${ac_enable_spidermonkey}" = "xyes" ], [
- AC_CHECK_LIB([mozjs], [JS_NewContext], [JS_LIB_BASE=mozjs], [
- AC_CHECK_LIB([js], [JS_NewContext], [JS_LIB_BASE=js], [
- AC_CHECK_LIB([js3250], [JS_NewContext], [JS_LIB_BASE=js3250], [
- AC_CHECK_LIB([js32], [JS_NewContext], [JS_LIB_BASE=js32], [
- AC_MSG_ERROR([Could not find the js library.
-
-Is the Mozilla SpiderMonkey library installed?])])])])])
-
-AC_SUBST(JS_LIB_BASE)
-
AC_ARG_WITH([v8-lib], [AC_HELP_STRING([--with-v8-lib=PATH],
[set PATH to the V8 library directory])],
[
@@ -248,19 +204,6 @@ AC_CHECK_HEADER([v8.h], [], [AC_MSG_ERROR([Could not find the V8 JavaScript engi
# [AC_MSG_ERROR([Could not find the V8 JavaScript engine library.])])
if test x${IS_WINDOWS} = xTRUE; then
- if test -f "$JS_LIB_DIR/$JS_LIB_BASE.dll"; then
- # seamonkey 1.7- build layout on Windows
- JS_LIB_BINARY="$JS_LIB_DIR/$JS_LIB_BASE.dll"
- else
- # seamonkey 1.8+ build layout on Windows
- if test -f "$JS_LIB_DIR/../bin/$JS_LIB_BASE.dll"; then
- JS_LIB_BINARY="$JS_LIB_DIR/../bin/$JS_LIB_BASE.dll"
- else
- AC_MSG_ERROR([Could not find $JS_LIB_BASE.dll.])
- fi
- fi
- AC_SUBST(JS_LIB_BINARY)
-
# On windows we need to know the path to the openssl binaries.
AC_ARG_WITH([openssl-bin-dir], [AC_HELP_STRING([--with-openssl-bin-dir=PATH],
[path to the open ssl binaries for distribution on Windows])], [
@@ -317,37 +260,6 @@ if test x${IS_WINDOWS} = xTRUE; then
fi
fi
-JSLIB=-l$JS_LIB_BASE
-
-AC_CHECK_HEADER([jsapi.h], [], [
- AC_CHECK_HEADER([js/jsapi.h],
- [
- CPPFLAGS="$CPPFLAGS -I$JS_INCLUDE/js"
- ],
- [
- AC_MSG_ERROR([Could not find the jsapi header.
-
-Are the Mozilla SpiderMonkey headers installed?])
- ])])
-
-AC_SUBST(JSLIB)
-
-AC_LANG_PUSH(C)
-OLD_CFLAGS="$CFLAGS"
-CFLAGS="-Werror-implicit-function-declaration"
-AC_COMPILE_IFELSE(
- [AC_LANG_PROGRAM(
- [[#include <jsapi.h>]],
- [[JS_SetOperationCallback(0, 0);]]
- )],
- AC_DEFINE([USE_JS_SETOPCB], [], [Use new JS_SetOperationCallback])
-)
-CFLAGS="$OLD_CFLAGS"
-AC_LANG_POP(C)
-
-# END allow building without spidermonkey
-])
-
AC_ARG_WITH([win32-icu-binaries], [AC_HELP_STRING([--with-win32-icu-binaries=PATH],
[set PATH to the Win32 native ICU binaries directory])], [
ICU_CONFIG="" # supposed to be a command to query options...
@@ -507,7 +419,7 @@ AC_ARG_VAR([HELP2MAN_EXECUTABLE], [path to the `help2man' program])
if test -n "$HELP2MAN_EXECUTABLE"; then
help2man_enabled=true
else
- if test -f "$srcdir/bin/couchdb.1" -a -f "$srcdir/bin/couchjs.1"; then
+ if test -f "$srcdir/bin/couchdb.1"; then
help2man_enabled=true
else
help2man_enabled=false
@@ -601,8 +513,6 @@ AC_CONFIG_FILES([test/python/set_view/Makefile])
AC_CONFIG_FILES([utils/Makefile])
AC_CONFIG_FILES([var/Makefile])
if test x${IS_WINDOWS} = xTRUE; then
- AC_CONFIG_FILES([src/couchdb/priv/couch_js/win32/msbuild.bat.tpl])
- AC_CONFIG_FILES([src/couchdb/priv/couch_js/win32/couchjs.vcxproj.tpl])
AC_CONFIG_FILES([src/couchdb/priv/icu_driver/win32/msbuild.bat.tpl])
AC_CONFIG_FILES([src/couchdb/priv/icu_driver/win32/couch_icu_driver.vcxproj.tpl])
AC_CONFIG_FILES([src/couchdb/priv/couch_ejson_compare/win32/msbuild.bat.tpl])
@@ -631,15 +541,6 @@ if test x${IS_WINDOWS} = xTRUE; then
# probably would chmod +x if we weren't on windows...
fi
-AS_IF([ test "x${ac_enable_spidermonkey}" != "xyes" ], [
-echo
-echo You have chosen to build CouchDB _without_ spidermonkey. This means
-echo that a lot of things won't work for you, and you may encounter tons
-echo of trouble. You have been warned, so you're on your own if you choose
-echo to continue building.
-echo
-])
-
echo
echo "You have configured Apache CouchDB, time to relax."
echo
@@ -18,6 +18,7 @@
-export([start_map_context/1, start_reduce_context/1]).
-export([end_map_context/0, end_reduce_context/1]).
-export([map/1, reduce/2, reduce/3, rereduce/2, rereduce/3]).
+-export([builtin_reduce/3]).
start_map_context(#set_view_group{views = Views}) ->
@@ -213,6 +214,9 @@ group_reductions_results(List) ->
end.
+builtin_reduce(ReduceType, FunSrcs, Values) ->
+ builtin_reduce(ReduceType, FunSrcs, Values, []).
+
builtin_reduce(_Re, [], _KVs, Acc) ->
{ok, lists:reverse(Acc)};
builtin_reduce(Re, [<<"_sum", _/binary>> | BuiltinReds], KVs, Acc) ->
View
@@ -91,6 +91,7 @@ source_files = \
couch_view_compactor.erl \
couch_view_updater.erl \
couch_view_group.erl \
+ couch_view_mapreduce.erl \
couch_db_updater.erl \
couch_work_queue.erl \
json_stream_parse.erl \
@@ -170,6 +171,7 @@ compiled_files = \
couch_view_compactor.beam \
couch_view_updater.beam \
couch_view_group.beam \
+ couch_view_mapreduce.beam \
couch_db_updater.beam \
couch_work_queue.beam \
json_stream_parse.beam \
View
@@ -231,7 +231,6 @@
id_btree=nil,
current_seq=0,
purge_seq=0,
- query_server=nil,
waiting_delayed_commit=nil
}).
@@ -243,7 +242,8 @@
def,
btree=nil,
reduce_funs=[],
- options=[]
+ options=[],
+ ref
}).
-record(index_header,
@@ -296,7 +296,8 @@ get_ddoc(#httpdb{url = BaseUrl, headers = Headers} = HttpDb, Id) ->
case ibrowse:send_req(
Url, Headers, get, [], HttpDb#httpdb.ibrowse_options) of
{ok, "200", _RespHeaders, Body} ->
- {ok, couch_doc:from_json_obj(?JSON_DECODE(Body))};
+ Doc = couch_doc:from_json_obj(?JSON_DECODE(Body)),
+ {ok, couch_doc:with_ejson_body(Doc)};
{ok, _Code, _RespHeaders, Body} ->
{Props} = ?JSON_DECODE(Body),
case {get_value(<<"error">>, Props), get_value(<<"reason">>, Props)} of
View
@@ -192,24 +192,30 @@ fold_reduce({temp_reduce, #view{btree=Bt}}, Fun, Acc, Options) ->
end,
couch_btree:fold_reduce(Bt, WrapperFun, Acc, Options);
-fold_reduce({reduce, NthRed, Lang, #view{btree=Bt, reduce_funs=RedFuns}}, Fun, Acc, Options) ->
+fold_reduce({reduce, NthRed, _Lang, View}, Fun, Acc, Options) ->
+ #view{btree=Bt, reduce_funs=RedFuns} = View,
PreResultPadding = lists:duplicate(NthRed - 1, []),
PostResultPadding = lists:duplicate(length(RedFuns) - NthRed, []),
- {_Name, FunSrc} = lists:nth(NthRed,RedFuns),
+ couch_view_mapreduce:start_reduce_context(View),
ReduceFun =
fun(reduce, KVs) ->
- {ok, Reduced} = couch_query_servers:reduce(Lang, [FunSrc], detuple_kvs(expand_dups(KVs, []),[])),
+ KVs2 = expand_dups(KVs, []),
+ {ok, Reduced} = couch_view_mapreduce:reduce(View, NthRed, KVs2),
{0, PreResultPadding ++ Reduced ++ PostResultPadding};
(rereduce, Reds) ->
UserReds = [[lists:nth(NthRed, UserRedsList)] || {_, UserRedsList} <- Reds],
- {ok, Reduced} = couch_query_servers:rereduce(Lang, [FunSrc], UserReds),
+ {ok, Reduced} = couch_view_mapreduce:rereduce(View, NthRed, UserReds),
{0, PreResultPadding ++ Reduced ++ PostResultPadding}
end,
WrapperFun = fun({GroupedKey, _}, PartialReds, Acc0) ->
{_, Reds} = couch_btree:final_reduce(ReduceFun, PartialReds),
Fun(GroupedKey, lists:nth(NthRed, Reds), Acc0)
end,
- couch_btree:fold_reduce(Bt, WrapperFun, Acc, Options).
+ try
+ couch_btree:fold_reduce(Bt, WrapperFun, Acc, Options)
+ after
+ couch_view_mapreduce:end_reduce_context(View)
+ end.
get_key_pos(_Key, [], _N) ->
0;
@@ -131,9 +131,11 @@ compact_view(Fd, View, #view{btree=ViewBtree}=EmptyView, Acc0) ->
{Item, update_task(Acc, 1)}
end,
- % Copy each view btree.
+ couch_view_mapreduce:start_reduce_context(View),
{ok, NewBtreeRoot, Acc2} = couch_btree_copy:copy(View#view.btree, Fd,
[{before_kv_write, {BeforeKVWriteFun, Acc0}}]),
+ couch_view_mapreduce:end_reduce_context(View),
+
ViewBtree2 = ViewBtree#btree{root = NewBtreeRoot},
{EmptyView#view{btree = ViewBtree2}, Acc2}.
@@ -622,8 +622,7 @@ design_doc_to_view_group(#doc{id=Id,body={Fields}}) ->
reset_group(#group{views=Views}=Group) ->
Views2 = [View#view{btree=nil} || View <- Views],
- Group#group{fd=nil,query_server=nil,current_seq=0,
- id_btree=nil,views=Views2}.
+ Group#group{fd=nil, current_seq=0, id_btree=nil, views=Views2}.
reset_file(Db, Fd, DbName, #group{sig=Sig,name=Name} = Group) ->
?LOG_DEBUG("Resetting group index \"~s\" in db ~s", [Name, DbName]),
@@ -638,8 +637,8 @@ init_group(Db, Fd, #group{views=Views}=Group, nil) ->
init_group(Db, Fd, Group,
#index_header{seq=0, purge_seq=couch_db:get_purge_seq(Db),
id_btree_state=nil, view_states=[{nil, 0, 0} || _ <- Views]});
-init_group(_Db, Fd, #group{def_lang=Lang,views=Views}=
- Group, IndexHeader) ->
+init_group(_Db, Fd, #group{views=Views0} = Group, IndexHeader) ->
+ Views = [V#view{ref = make_ref()} || V <- Views0],
#index_header{seq=Seq, purge_seq=PurgeSeq,
id_btree_state=IdBtreeState, view_states=ViewStates} = IndexHeader,
StateUpdate = fun
@@ -650,20 +649,16 @@ init_group(_Db, Fd, #group{def_lang=Lang,views=Views}=
{ok, IdBtree} = couch_btree:open(
IdBtreeState, Fd, []),
Views2 = lists:zipwith(
- fun({BTState, USeq, PSeq}, #view{reduce_funs=RedFuns,options=Options}=View) ->
- FunSrcs = [FunSrc || {_Name, FunSrc} <- RedFuns],
+ fun({BTState, USeq, PSeq}, #view{options=Options} = View) ->
ReduceFun =
fun(reduce, KVs) ->
KVs2 = couch_view:expand_dups(KVs,[]),
- KVs3 = couch_view:detuple_kvs(KVs2,[]),
- {ok, Reduced} = couch_query_servers:reduce(Lang, FunSrcs,
- KVs3),
- {length(KVs3), Reduced};
+ {ok, Reduced} = couch_view_mapreduce:reduce(View, KVs2),
+ {length(KVs2), Reduced};
(rereduce, Reds) ->
Count = lists:sum([Count0 || {Count0, _} <- Reds]),
UserReds = [UserRedsList || {_, UserRedsList} <- Reds],
- {ok, Reduced} = couch_query_servers:rereduce(Lang, FunSrcs,
- UserReds),
+ {ok, Reduced} = couch_view_mapreduce:rereduce(View, UserReds),
{Count, Reduced}
end,
Oops, something went wrong.

0 comments on commit eaa3f3b

Please sign in to comment.