Merge branch 'master' into bcrypt-hashing

Jenkinsfile

@@ -118,31 +118,6 @@ pipeline {
             deleteDir()
           } // node
         },
-        centos7erlangdefault: {
-          node(label: 'ubuntu') {
-            timeout(time: 45, unit: "MINUTES") {
-              sh 'docker pull couchdbdev/centos-7-erlang-default'
-              withDockerContainer(image: 'couchdbdev/centos-7-erlang-default', args: '-e LD_LIBRARY_PATH=/usr/local/bin') {
-                sh 'rm -f apache-couchdb-*.tar.gz'
-                unstash 'tarball'
-                sh '''
-                  cwd=$(pwd)
-                  mkdir -p $COUCHDB_IO_LOG_DIR
-
-                  # Build CouchDB from tarball
-                  builddir=$(mktemp -d)
-                  cd $builddir
-                  tar -xf $cwd/apache-couchdb-*.tar.gz
-                  cd apache-couchdb-*
-                  ./configure --with-curl
-                  make all
-                  make check || (build-aux/logfile-uploader.py && false)
-                '''
-              } // withDocker
-            } // timeout
-            deleteDir()
-          } // node
-        },
         centos7erlang183: {
           node(label: 'ubuntu') {
             timeout(time: 60, unit: "MINUTES") {
@@ -183,29 +158,6 @@ pipeline {
             deleteDir()
           } // node
         },
-        ubuntu1404erlangdefault: {
-          node(label: 'ubuntu') {
-            timeout(time: 45, unit: "MINUTES") {
-              sh 'docker pull couchdbdev/ubuntu-14.04-erlang-default'
-              withDockerContainer(image: 'couchdbdev/ubuntu-14.04-erlang-default') {
-                sh 'rm -f apache-couchdb-*.tar.gz'
-                unstash 'tarball'
-                sh '''
-                  cwd=$(pwd)
-                  mkdir -p $COUCHDB_IO_LOG_DIR
-                  builddir=$(mktemp -d)
-                  cd $builddir
-                  tar -xf $cwd/apache-couchdb-*.tar.gz
-                  cd apache-couchdb-*
-                  ./configure --with-curl
-                  make all
-                  make check || (build-aux/logfile-uploader.py && false)
-                '''
-              } // withDocker
-            } // timeout
-            deleteDir()
-          } // node
-        },
         ubuntu1404erlang183: {
           node(label: 'ubuntu') {
             timeout(time: 60, unit: "MINUTES") {
@@ -246,29 +198,6 @@ pipeline {
             deleteDir()
           } // node
         },
-        ubuntu1604erlangdefault: {
-          node(label: 'ubuntu') {
-            timeout(time: 45, unit: "MINUTES") {
-              sh 'docker pull couchdbdev/ubuntu-16.04-erlang-default'
-              withDockerContainer(image: 'couchdbdev/ubuntu-16.04-erlang-default') {
-                sh 'rm -f apache-couchdb-*.tar.gz'
-                unstash 'tarball'
-                sh '''
-                  cwd=$(pwd)
-                  mkdir -p $COUCHDB_IO_LOG_DIR
-                  builddir=$(mktemp -d)
-                  cd $builddir
-                  tar -xf $cwd/apache-couchdb-*.tar.gz
-                  cd apache-couchdb-*
-                  ./configure --with-curl
-                  make all
-                  make check || (build-aux/logfile-uploader.py && false)
-                '''
-              } // withDocker
-            } // timeout
-            deleteDir()
-          } // node
-        },
         ubuntu1604erlang183: {
           node(label: 'ubuntu') {
             timeout(time: 60, unit: "MINUTES") {
@@ -309,29 +238,6 @@ pipeline {
             deleteDir()
           } // node
         },
-        debian8erlangdefault: {
-          node(label: 'ubuntu') {
-            timeout(time: 45, unit: "MINUTES") {
-              sh 'docker pull couchdbdev/debian-8-erlang-default'
-              withDockerContainer(image: 'couchdbdev/debian-8-erlang-default') {
-                sh 'rm -f apache-couchdb-*.tar.gz'
-                unstash 'tarball'
-                sh '''
-                  cwd=$(pwd)
-                  mkdir -p $COUCHDB_IO_LOG_DIR
-                  builddir=$(mktemp -d)
-                  cd $builddir
-                  tar -xf $cwd/apache-couchdb-*.tar.gz
-                  cd apache-couchdb-*
-                  ./configure --with-curl
-                  make all
-                  make check || (build-aux/logfile-uploader.py && false)
-                '''
-              } // withDocker
-            } // timeout
-            deleteDir()
-          } // node
-        },
         debian8erlang183: {
           node(label: 'ubuntu') {
             timeout(time: 60, unit: "MINUTES") {
@@ -372,29 +278,6 @@ pipeline {
             deleteDir()
           } // node
         },
-        debian9erlangdefault: {
-          node(label: 'ubuntu') {
-            timeout(time: 45, unit: "MINUTES") {
-              sh 'docker pull couchdbdev/debian-9-erlang-default'
-              withDockerContainer(image: 'couchdbdev/debian-9-erlang-default') {
-                sh 'rm -f apache-couchdb-*.tar.gz'
-                unstash 'tarball'
-                sh '''
-                  cwd=$(pwd)
-                  mkdir -p $COUCHDB_IO_LOG_DIR
-                  builddir=$(mktemp -d)
-                  cd $builddir
-                  tar -xf $cwd/apache-couchdb-*.tar.gz
-                  cd apache-couchdb-*
-                  ./configure --with-curl
-                  make all
-                  make check || (build-aux/logfile-uploader.py && false)
-                '''
-              } // withDocker
-            } // timeout
-            deleteDir()
-          } // node
-        },
         debian9erlang183: {
           node(label: 'ubuntu') {
             timeout(time: 60, unit: "MINUTES") {

rel/overlay/etc/default.ini

@@ -431,14 +431,35 @@ ssl_certificate_max_depth = 3
 ; Re-check cluster state at least every cluster_quiet_period seconds
 ; cluster_quiet_period = 60
 
+; List of replicator client authentication plugins to try. Plugins will be
+; tried in order. The first to initialize successfully will be used for that
+; particular endpoint (source or target). Normally couch_replicator_auth_noop
+; would be used at the end of the list as a "catch-all". It doesn't do anything
+; and effectively implements the previous behavior of using basic auth.
+; There are currently two plugins available:
+;   couch_replicator_auth_session - use _session cookie authentication
+;   couch_replicator_auth_noop - use basic authentication (previous default)
+; Currently previous default behavior is still the default. To start using
+; session auth, use this as the list of plugins:
+; `couch_replicator_auth_session,couch_replicator_auth_noop`.
+; In a future release the session plugin might be used by default.
+;auth_plugins = couch_replicator_auth_noop
+
+
 [compaction_daemon]
 ; The delay, in seconds, between each check for which database and view indexes
 ; need to be compacted.
-check_interval = 300
+check_interval = 3600
 ; If a database or view index file is smaller then this value (in bytes),
 ; compaction will not happen. Very small files always have a very high
 ; fragmentation therefore it's not worth to compact them.
 min_file_size = 131072
+; With lots of databases and/or with lots of design docs in one or more
+; databases, the compaction_daemon can create significant CPU load when
+; checking whether databases and view indexes need compacting. The
+; snooze_period setting ensures a smoother CPU load. Defaults to
+; 3 seconds wait.
+; snooze_period = 3
 
 [compactions]
 ; List of compaction rules for the compaction daemon.

src/couch/test/couchdb_csp_tests.erl → src/chttpd/test/chttpd_csp_tests.erl

@@ -10,29 +10,28 @@
 % License for the specific language governing permissions and limitations under
 % the License.
 
--module(couchdb_csp_tests).
+-module(chttpd_csp_tests).
 
 -include_lib("couch/include/couch_eunit.hrl").
 
--define(TIMEOUT, 1000).
-
 
 setup() ->
     ok = config:set("csp", "enable", "true", false),
-    Addr = config:get("httpd", "bind_address", "127.0.0.1"),
-    Port = integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+    Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
+    Port = mochiweb_socket_server:get(chttpd, port),
     lists:concat(["http://", Addr, ":", Port, "/_utils/"]).
 
 teardown(_) ->
     ok.
 
 
+
 csp_test_() ->
     {
         "Content Security Policy tests",
         {
             setup,
-            fun test_util:start_couch/0, fun test_util:stop_couch/1,
+            fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1,
             {
                 foreach,
                 fun setup/0, fun teardown/1,

src/couch/src/couch_bt_engine.erl

@@ -464,7 +464,10 @@ fold_docs(St, UserFun, UserAcc, Options) ->
 
 
 fold_local_docs(St, UserFun, UserAcc, Options) ->
-    fold_docs_int(St, St#st.local_tree, UserFun, UserAcc, Options).
+    case fold_docs_int(St, St#st.local_tree, UserFun, UserAcc, Options) of
+        {ok, _Reds, FinalAcc} -> {ok, null, FinalAcc};
+        {ok, FinalAcc} -> {ok, FinalAcc}
+    end.
 
 
 fold_changes(St, SinceSeq, UserFun, UserAcc, Options) ->

src/couch/src/couch_compaction_daemon.erl

@@ -126,6 +126,7 @@ handle_config_terminate(_Server, _Reason, _State) ->
     erlang:send_after(?RELISTEN_DELAY, whereis(?MODULE), restart_config_listener).
 
 compact_loop(Parent) ->
+    SnoozePeriod = config:get_integer("compaction_daemon", "snooze_period", 3),
     {ok, _} = couch_server:all_databases(
         fun(DbName, Acc) ->
             case ets:info(?CONFIG_ETS, size) =:= 0 of
@@ -138,7 +139,8 @@ compact_loop(Parent) ->
                 {ok, Config} ->
                     case check_period(Config) of
                     true ->
-                        maybe_compact_db(Parent, DbName, Config);
+                        maybe_compact_db(Parent, DbName, Config),
+                        ok = timer:sleep(SnoozePeriod * 1000);
                     false ->
                         ok
                     end
@@ -150,8 +152,7 @@ compact_loop(Parent) ->
     true ->
         receive {Parent, have_config} -> ok end;
     false ->
-        PausePeriod = list_to_integer(
-            config:get("compaction_daemon", "check_interval", "300")),
+        PausePeriod = config:get_integer("compaction_daemon", "check_interval", 3600),
         ok = timer:sleep(PausePeriod * 1000)
     end,
     compact_loop(Parent).
@@ -229,7 +230,9 @@ maybe_compact_views(DbName, [DDocName | Rest], Config) ->
             maybe_compact_views(DbName, Rest, Config);
         timeout ->
             ok
-        end;
+        end,
+        SnoozePeriod = config:get_integer("compaction_daemon", "snooze_period", 3),
+        ok = timer:sleep(SnoozePeriod * 1000);
     false ->
         ok
     end.

src/couch/src/couch_db.erl

@@ -224,18 +224,7 @@ monitor(#db{main_pid=MainPid}) ->
     erlang:monitor(process, MainPid).
 
 start_compact(#db{} = Db) ->
-    start_compact(Db, []).
-
-start_compact(#db{} = Db, Opts) ->
-    case lists:keyfind(notify, 1, Opts) of
-        {notify, Pid, Term} ->
-            % We fake a gen_server call here which sends the
-            % response back to the specified pid.
-            Db#db.main_pid ! {'$gen_call', {Pid, Term}, start_compact},
-            ok;
-        _ ->
-            gen_server:call(Db#db.main_pid, start_compact)
-    end.
+    gen_server:call(Db#db.main_pid, start_compact).
 
 cancel_compact(#db{main_pid=Pid}) ->
     gen_server:call(Pid, cancel_compact).

src/couch/src/couch_httpd_misc_handlers.erl

@@ -61,30 +61,9 @@ handle_file_req(#httpd{method='GET'}=Req, Document) ->
 handle_file_req(Req, _) ->
     send_method_not_allowed(Req, "GET,HEAD").
 
-handle_utils_dir_req(#httpd{method='GET'}=Req, DocumentRoot) ->
-    "/" ++ UrlPath = couch_httpd:path(Req),
-    case couch_httpd:partition(UrlPath) of
-    {_ActionKey, "/", RelativePath} ->
-        % GET /_utils/path or GET /_utils/
-        CachingHeaders = [{"Cache-Control", "private, must-revalidate"}],
-        EnableCsp = config:get("csp", "enable", "false"),
-        Headers = maybe_add_csp_headers(CachingHeaders, EnableCsp),
-        couch_httpd:serve_file(Req, RelativePath, DocumentRoot, Headers);
-    {_ActionKey, "", _RelativePath} ->
-        % GET /_utils
-        RedirectPath = couch_httpd:path(Req) ++ "/",
-        couch_httpd:send_redirect(Req, RedirectPath)
-    end;
 handle_utils_dir_req(Req, _) ->
-    send_method_not_allowed(Req, "GET,HEAD").
-
-maybe_add_csp_headers(Headers, "true") ->
-    DefaultValues = "default-src 'self'; img-src 'self' data:; font-src 'self'; "
-                    "script-src 'self' 'unsafe-eval'; style-src 'self' 'unsafe-inline';",
-    Value = config:get("csp", "header_value", DefaultValues),
-    [{"Content-Security-Policy", Value} | Headers];
-maybe_add_csp_headers(Headers, _) ->
-    Headers.
+    send_error(Req, 410, <<"no_node_local_fauxton">>,
+        ?l2b("The web interface is no longer available on the node-local port.")).
 
 
 handle_all_dbs_req(#httpd{method='GET'}=Req) ->

src/couch/src/couch_key_tree.erl

@@ -73,7 +73,7 @@ stem/2
 
 %% @doc Merge a path into the given tree and then stem the result.
 %% Although Tree is of type tree(), it must not contain any branches.
--spec merge(revtree(), tree(), pos_integer()) ->
+-spec merge(revtree(), tree() | path(), pos_integer()) ->
                 {revtree(), new_leaf | new_branch | internal_node}.
 merge(RevTree, Tree, StemDepth) ->
     {Merged, Result} = merge(RevTree, Tree),
@@ -84,7 +84,7 @@ merge(RevTree, Tree, StemDepth) ->
 
 
 %% @doc Merge a path into a tree.
--spec merge(revtree(), tree()) ->
+-spec merge(revtree(), tree() | path()) ->
                 {revtree(), new_leaf | new_branch | internal_node}.
 merge(RevTree, Tree) ->
     {Merged, Result} = merge_tree(RevTree, Tree, []),
@@ -94,7 +94,7 @@ merge(RevTree, Tree) ->
 %% @doc Attempt to merge Tree into each branch of the RevTree.
 %% If it can't find a branch that the new tree merges into, add it as a
 %% new branch in the RevTree.
--spec merge_tree(revtree(), tree(), revtree()) ->
+-spec merge_tree(revtree(), tree() | path(), revtree()) ->
                 {revtree(), new_leaf | new_branch | internal_node}.
 merge_tree([], Tree, []) ->
     {[Tree], new_leaf};

src/couch/src/couch_query_servers.erl

@@ -324,7 +324,7 @@ validate_doc_update(DDoc, EditDoc, DiskDoc, Ctx, SecObj) ->
         couch_stats:increment_counter([couchdb, query_server, vdu_rejects], 1)
     end,
     case Resp of
-        1 ->
+        RespCode when RespCode =:= 1; RespCode =:= ok; RespCode =:= true ->
             ok;
         {[{<<"forbidden">>, Message}]} ->
             throw({forbidden, Message});

src/couch/src/test_engine_util.erl

@@ -58,7 +58,7 @@ gather(Module) ->
         case {atom_to_list(Fun), Arity} of
             {[$c, $e, $t, $_ | _], 0} ->
                 TestFun = make_test_fun(Module, Fun),
-                [{spawn, TestFun} | Acc];
+                [{timeout, 60, {spawn, TestFun}} | Acc];
             _ ->
                 Acc
         end

src/couch/test/couchdb_compaction_daemon_tests.erl

@@ -24,6 +24,7 @@
 start() ->
     Ctx = test_util:start_couch(),
     ok = config:set("compaction_daemon", "check_interval", "3", false),
+    ok = config:set("compaction_daemon", "snooze_period", "0", false),
     ok = config:set("compaction_daemon", "min_file_size", "100000", false),
     ok = config:delete("compactions", "_default", false),
     ok = meck:new(?MODS_TO_MOCK, [passthrough]),