MB-7013: Make the new vtree work with Apache CouchDB

This commits makes the new vtree work with Apache CouchDB 1.3.x.
All GeoCouch JavaScript tests pass. You should be able to use it
as a replacement for your current installation.

Change-Id: I6e42fc8149c447a294a52955a8b576b80ba00c74
Reviewed-on: http://review.couchbase.org/22994
Tested-by: buildbot <build@couchbase.com>
Reviewed-by: Filipe David Borba Manana <fdmanana@gmail.com>

Makefile

@@ -3,18 +3,23 @@ VERSION=$(shell git describe)
 GEOCOUCH_PLT ?= ../geocouch.plt
 
 couchbase couchbase-check: REBAR_CONFIG := rebar_couchbase.config
+couchdb couchdb-check: REBAR_CONFIG := rebar_couchdb.config
 
 
 all:
-	@echo "Try \"make couchbase\" instead."
+	@echo "Try \"make couchbase\" or \"make couchdb\"instead."
 
 check:
-	@echo "Try \"make couchbase-check\" instead."
+	@echo "Try \"make couchbase-check\" \"make couchdb-check\"instead."
 
 .PHONY: couchbase
 couchbase: compile
 couchbase-check: do-check
 
+.PHONY: couchdb
+couchdb: compile
+couchdb-check: do-check
+
 compile:
 	./rebar -C $(REBAR_CONFIG) compile
 
@@ -28,6 +33,7 @@ do-check: clean compileforcheck dialyzer runtests clean-again
 
 clean clean-again:
 	./rebar -C rebar_couchbase.config clean
+	./rebar -C rebar_couchdb.config clean
 	rm -f *.tar.gz
 
 

README.md

@@ -36,3 +36,81 @@ After that you can compile GeoCouch from within the GeoCouch directory:
 After you've followed the build instructions you can run the tests with
 
     COUCH_SRC=<path-to-couchdb-source>/src/couchdb make couchbase-check
+
+
+For Apache CouchDB
+------------------
+
+This version of GeoCouch needs at least Apache CouchDB 1.3.x.
+
+
+### Checkout the code
+
+First checkout the source code for Apache CouchDB into a directory that will
+be referred to as <path-to-couchdb-source>.
+
+Then checkout the GeoCouch source:
+
+    git clone -b newvtree https://github.com/couchbase/geocouch.git
+
+There's a new directory called `geocouch` created. From now on this directory
+will be referred to as <path-to-geocouch-source>.
+
+
+### Build instructions
+
+Make sure you have built Apache CouchDB from source including `make dev`. So
+go to your <path-to-couchdb-source> and run:
+
+    ./bootstrap
+    ./configure
+    make dev
+
+After that you can compile GeoCouch from within the GeoCouch directory:
+
+    COUCH_SRC=<path-to-couchdb-source>/src/couchdb make couchdb
+
+Now copy the configuration file into your Apache CouchDB directory:
+
+    cp etc/couchdb/default.d/geocouch.ini <path-to-couchdb-source>/etc/couchdb/default.d/
+
+
+### Running tests
+
+After you've followed the build instructions you can run the tests with
+
+    COUCH_SRC=<path-to-couchdb-source>/src/couchdb make couchdb-check
+
+In order to run the JavaScript based tests, you need to start Apache CouchDB first:
+
+    cd <path-to-couchdb-source>
+    ERL_LIBS="<path-to-geocouch-source>" ./utils/run
+
+The tests can either be run from the command line or the browser.
+
+
+#### From command line
+
+From the command line the easiest way is to use the supplied runner script.
+From within the <path-to-geocouch-source>:
+
+    cd gc-couchdb
+    ./utils/runjstests.sh <path-to-couchdb-source>/test/javascript/run ./share/www/script/test
+
+
+#### From browser
+
+To run it from the browser first copy the JavaScript tests into the same directory as the other Apache CouchDB tests:
+
+   cp <path-to-geocouch-source>/gc-couchdb/share/www/script/test/* <path-to-couchdb-source>/share/www/script/test/
+
+Then add the tests to `<path-to-couchdb-source>/share/www/script/couch_tests.js`
+
+    loadTest("spatial.js");
+    loadTest("list_spatial.js");
+    loadTest("etags_spatial.js");
+    loadTest("multiple_spatial_rows.js");
+    loadTest("spatial_compaction.js");
+    loadTest("spatial_design_docs.js");
+    loadTest("spatial_bugfixes.js");
+    loadTest("spatial_offsets.js");

gc-couchdb/etc/couchdb/default.d/geocouch.ini

@@ -0,0 +1,10 @@
+[httpd_db_handlers]
+_spatial_cleanup = {couch_spatial_http, handle_cleanup_req}
+
+[httpd_design_handlers]
+_spatial = {couch_spatial_http, handle_spatial_req}
+_spatial/_list = {couch_spatial_list, handle_view_list_req}
+_spatial/_info = {couch_spatial_http, handle_info_req}
+_spatial/_compact = {couch_spatial_http, handle_compact_req}
+;deprecated API
+_spatiallist = {couch_spatial_list, handle_view_list_req_deprecated}

gc-couchdb/src/couch_spatial.hrl → gc-couchdb/include/couch_spatial.hrl

@@ -66,8 +66,7 @@
 -record(spatial, {
     root_dir=nil,
     seq=0,
-    treepos=nil,
-    treeheight=0, % height of the tree
+    vtree = nil,
     def=nil, % The function in the query/view server
     view_names=[],
     id_num=0, % comes from couch_spatial_group requirements
@@ -85,3 +84,13 @@
 %    start_response,
 %    send_row
 %}).
+
+% `#vtree_state` is a subset of the #vtree record that contains the
+% information that is stored in the header. It contains the information
+% that can't be retrieved from other sources (as e.g. the Design Document
+% or some configuration setting.
+-record(vtree_state, {
+          root = nil,
+          fill_min = 1,
+          fill_max = 5
+}).

gc-couchdb/rebar.config.script

@@ -0,0 +1,28 @@
+% Add the COUCH_SRC directory to include path
+IncludeDir = case os:getenv("COUCH_SRC") of
+    false -> []; % env var not defined
+    []    -> []; % env var set to empty string
+    Dir ->
+        % If a relative path is given, prepend "../" as we are one level under the root
+        case filename:absname(Dir) of
+            Dir ->
+                [{i, Dir}];
+            _ ->
+                [{i, filename:absname("../" ++ Dir)}]
+        end
+end,
+
+% Add "makecheck" setting for testing
+MakeCheck = case os:getenv("MAKECHECK") of
+    false -> []; % env var not defined
+    []    -> []; % env var set to empty string
+    _ -> [{d, makecheck}]
+end,
+
+% Add all variables to erl_opts
+case lists:keytake(erl_opts, 1, CONFIG) of
+    false ->
+        CONFIG ++ [{erl_opts, IncludeDir ++ MakeCheck}];
+    {value, {erl_opts, ErlOpts}, Config2} ->
+        Config2 ++ [{erl_opts, ErlOpts ++ IncludeDir ++ MakeCheck}]
+end.

gc-couchdb/src/couch_spatial.erl

@@ -19,6 +19,7 @@
 
 -include("couch_db.hrl").
 -include("couch_spatial.hrl").
+-include_lib("vtree/include/vtree.hrl").
 
 -record(acc, {
     meta_sent = false,
@@ -44,8 +45,12 @@ query_view(Db, DDoc, ViewName, Args, Callback, Acc0) ->
 
 query_view_count(Db, DDoc, ViewName, Args) ->
     {ok, View, _, _} = couch_spatial_util:get_view(Db, DDoc, ViewName, Args),
-    vtree:count_lookup(
-        View#spatial.fd, View#spatial.treepos, Args#spatial_args.bbox).
+    case Args#spatial_args.bbox of
+    nil ->
+        vtree_search:count_all(View#spatial.vtree);
+    Bbox ->
+        vtree_search:count_search(View#spatial.vtree, [Bbox])
+    end.
 
 
 get_info(Db, DDoc) ->
@@ -95,22 +100,43 @@ spatial_fold(View, Args, Callback, UserAcc) ->
         user_acc = UserAcc,
         update_seq = View#spatial.update_seq
     },
-    {ok, Acc2} = fold(View, fun do_fold/2, Acc, Bbox, Bounds),
+    Acc2 = fold(View, fun do_fold/2, Acc, Bbox, Bounds),
     finish_fold(Acc2, []).
 
 
 fold(Index, FoldFun, InitAcc, Bbox, Bounds) ->
     WrapperFun = fun(Node, Acc) ->
-        Expanded = couch_spatial_util:expand_dups([Node], []),
-        lists:foldl(fun(E, {ok, Acc2}) ->
-            FoldFun(E, Acc2)
-        end, {ok, Acc}, Expanded)
+        % NOTE vmx 2012-11-28: in Apache CouchDB the body is stored as
+        %     Erlang terms
+        Value = binary_to_term(Node#kv_node.body),
+        Expanded = couch_spatial_util:expand_dups(
+            [Node#kv_node{body=Value}], []),
+        fold_fun(FoldFun, Expanded, Acc)
     end,
-    {_State, Acc} = vtree:lookup(
-        Index#spatial.fd, Index#spatial.treepos, Bbox,
-        {WrapperFun, InitAcc}, Bounds),
-    {ok, Acc}.
-
+    case Bbox of
+    nil ->
+        vtree_search:all(Index#spatial.vtree, WrapperFun, InitAcc);
+    Bbox ->
+        Bboxes = case Bounds of
+        nil ->
+            [Bbox];
+        _ ->
+            couch_spatial_util:split_bbox_if_flipped(Bbox, Bounds)
+        end,
+        vtree_search:search(Index#spatial.vtree, Bboxes, WrapperFun, InitAcc)
+    end.
+
+
+% This is like a normal fold that can be interrupted in the middle
+fold_fun(_Fun, [], Acc) ->
+    {ok, Acc};
+fold_fun(Fun, [KV|Rest], Acc) ->
+    case Fun(KV, Acc) of
+    {ok, Acc2} ->
+        fold_fun(Fun, Rest, Acc2);
+    {stop, Acc2} ->
+        {stop, Acc2}
+    end.
 
 do_fold(_Kv, #acc{skip=N}=Acc) when N > 0 ->
     {ok, Acc#acc{skip=N-1, last_go=ok}};
@@ -129,12 +155,19 @@ do_fold(Kv, #acc{meta_sent=false}=Acc) ->
     end;
 do_fold(_Kv, #acc{limit=0}=Acc) ->
     {stop, Acc};
-do_fold({{_Bbox, _DocId}, {_Geom, _Value}}=Row, Acc) ->
+do_fold(#kv_node{}=Node, Acc) ->
+    #kv_node{
+        key = Mbb,
+        docid = DocId,
+        geometry = Geom,
+        body = Value
+    } = Node,
     #acc{
         limit = Limit,
         callback = Callback,
         user_acc = UserAcc
     } = Acc,
+    Row = {Mbb, DocId, Geom, Value},
     {Go, UserAcc2} = Callback({row, Row}, UserAcc),
     {Go, Acc#acc{
         limit = Limit-1,

gc-couchdb/src/couch_spatial_compactor.erl

@@ -26,8 +26,7 @@
 }).
 
 -record(spatial_acc, {
-    treepos = nil,
-    treeheight = 0,
+    vtree = nil,
     kvs = [],
     kvs_size = 0,
     changes = 0,
@@ -60,8 +59,7 @@ compact(State) ->
 
     #spatial_state{
         id_btree = EmptyIdBtree,
-        views = EmptyViews,
-        fd = EmptyFd
+        views = EmptyViews
     } = EmptyState,
 
     % XXX vmx 2012-10-22: Not sure why the [] case happens
@@ -128,14 +126,10 @@ compact(State) ->
     },
 
     {NewViews, _} = lists:mapfoldl(fun({View, EmptyView}, Acc) ->
-        case View#spatial.treepos of
-            % Tree is empty, just grab the the FD
-            nil -> {EmptyView#spatial{fd = EmptyFd}, Acc};
-            _ -> compact_spatial(View, EmptyView, BufferSize, Acc)
-        end
+         compact_spatial(View, EmptyView, BufferSize, Acc)
     end, SpatialAcc, lists:zip(Views, EmptyViews)),
 
-    unlink(EmptyFd),
+    unlink(EmptyState#spatial_state.fd),
     {ok, EmptyState#spatial_state{
         id_btree = NewIdBtree,
         views = NewViews,
@@ -159,32 +153,19 @@ recompact(State) ->
 
 
 compact_spatial(View, EmptyView, BufferSize, Acc0) ->
-    #spatial{
-        fd = OldFd,
-        treepos = OrigTreepos
-    } = View,
-    #spatial{
-        fd = NewFd,
-        treepos = EmptyTreepos,
-        treeheight = EmptyTreeheight
-    } = EmptyView,
-
     Fun = fun(Kv, Acc) ->
         #spatial_acc{
-            treepos = TreePos,
-            treeheight = TreeHeight,
+            vtree = Vt,
             kvs = Kvs,
             kvs_size = KvsSize0
         } = Acc,
         KvsSize = KvsSize0 + ?term_size(Kv),
-        case KvsSize >= BufferSize of
+        Acc3 = case KvsSize >= BufferSize of
         true ->
-            {ok, TreePos2, TreeHeight2} = vtree_bulk:bulk_load(
-                NewFd, TreePos, TreeHeight, [Kv|Kvs]),
+            Vt2 = vtree_insert:insert(Vt, [Kv|Kvs]),
             Acc2 = update_task(Acc, 1 + length(Kvs)),
             Acc2#spatial_acc{
-                treepos = TreePos2,
-                treeheight = TreeHeight2,
+                vtree = Vt2,
                 kvs = [],
                 kvs_size = 0
             };
@@ -193,30 +174,26 @@ compact_spatial(View, EmptyView, BufferSize, Acc0) ->
                 kvs = [Kv|Kvs],
                 kvs_size = KvsSize
             }
-        end
+        end,
+        {ok, Acc3}
     end,
 
     InitAcc = Acc0#spatial_acc{
         kvs = [],
         kvs_size = 0,
-        treepos = EmptyTreepos,
-        treeheight = EmptyTreeheight
+        vtree = EmptyView#spatial.vtree
     },
 
     %{TreePos3, TreeHeight3, Uncopied, _Total} = vtree:foldl(
-    FinalAcc = vtree:foldl(OldFd, OrigTreepos, Fun, InitAcc),
+    FinalAcc = vtree_search:all(View#spatial.vtree, Fun, InitAcc),
     #spatial_acc{
-        treepos = TreePos3,
-        treeheight = TreeHeight3,
+        vtree = Vt3,
         kvs = Uncopied
     } = FinalAcc,
-    {ok, NewTreePos, NewTreeHeight} = vtree_bulk:bulk_load(
-        NewFd, TreePos3, TreeHeight3, Uncopied),
+    NewVt = vtree_insert:insert(Vt3, Uncopied),
     FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
     NewView = EmptyView#spatial{
-        treepos = NewTreePos,
-        treeheight = NewTreeHeight,
-        fd = NewFd
+        vtree = NewVt
     },
     {NewView, FinalAcc2}.