Merge remote-tracking branch 'origin/prototype/fdb-layer' into protot…

…ype/fdb-layer-final-merge

.credo.exs

@@ -37,6 +37,7 @@
           ~r"/src/metrics",
           ~r"/src/minerl",
           ~r"/src/parse_trans",
+          ~r"/src/stream_data",
           ~r"/src/ssl_verify_fun",
           ~r"/test/elixir/deps/"
         ]

.gitignore

@@ -11,6 +11,8 @@
 .DS_Store
 .vscode
 .rebar/
+.rebar3/
+.erlfdb/
 .eunit/
 cover/
 core
@@ -28,6 +30,7 @@ ebin/
 erl_crash.dump
 erln8.config
 install.mk
+rebar.config
 rel/*.config
 rel/couchdb
 rel/dev*
@@ -48,6 +51,7 @@ src/couch/priv/icu_driver/couch_icu_driver.d
 src/mango/src/mango_cursor_text.nocompile
 src/docs/
 src/emilio/
+src/erlfdb/
 src/ets_lru/
 src/excoveralls/
 src/fauxton/
@@ -58,21 +62,27 @@ src/hyper/
 src/ibrowse/
 src/idna/
 src/ioq/
+src/hqueue/
+src/jaeger_passage/
 src/jiffy/
 src/ken/
 src/khash/
+src/local/
 src/meck/
 src/metrics/
 src/mimerl/
 src/mochiweb/
 src/oauth/
 src/parse_trans/
+src/passage/
 src/proper/
 src/rebar/
 src/recon/
 src/smoosh/
 src/snappy/
+src/stream_data/
 src/ssl_verify_fun/
+src/thrift_protocol/
 src/triq/
 src/unicode_util_compat/
 tmp/

FDB_NOTES.md

@@ -0,0 +1,57 @@
+Things of Note
+===
+
+
+1. If a replication sends us two revisions A and B where one is an
+   ancestor of the other, we likely have divergent behavior. However,
+   this should never happen In Theory.
+
+2. Multiple updates to the same document in a _bulk_docs (or if they
+   just happen to be in the same update batch in non-fdb CouchDB)
+   we likely have subtly different behavior.
+
+3. I'm relying on repeated reads in an fdb transaction to be "cheap"
+   in that the reads would be cached in the fdb_transaction object.
+   This needs to be checked for certainty but that appeared to
+   be how things behaved in testing.
+
+4. When attempting to create a doc from scratch in an interacitve_edit
+   update, with revisions specified *and* attachment stubs, the reported
+   error is now a conflict. Previously the missing_stubs error was
+   raised earlier.
+
+5. There may be a difference in behavior if a) there are no VDU functions
+   set on a db and no design documents in a batch. This is because in
+   this situation we don't run the prep_and_validate code on pre-fdb
+   CouchDB. The new code always checks stubs before merging revision trees.
+   I'm sure the old way would fail somehow, but it would fail further on
+   which means we may have failed with a different reason (conflict, etc)
+   before we got to the next place we check for missing stubs.
+
+6. For multi-doc updates we'll need to investigate user versions on
+   versionstamps within a transaction. Also this likely prevents the
+   ability to have multiple updates to the same doc in a single
+   _bulk_docs transaction
+
+7. Document body storage needs to be implemented beyond the single
+   key/value approach.
+
+8. We'll want to look at how we currently apply open options to individual
+    elements of an open_revs call. Might turn out that we have to grab a
+    full FDI even if we could look up a rev directly. (i.e., revs_info
+    would require us having the entire FDI, however it'd be wasteful to return
+    all of that in an open_revs call, but bug compatibility ftw!)
+
+9. Is it possible that a server_admin can delete a db without being able
+    to open it? If so that's probably changed behavior.
+
+10. All docs on large active databases might be a thing getting the doc
+    count. If we allow range requests up to 5s, and we continue to return
+    the doc count total we may have to play games with snapshot reads on
+    the doc count key or else it'll whack any _all_docs range requests
+
+11. Revision infos need to track their size f we want to maintain a database
+    size counter we'll want to store the size of a given doc body for each
+    revision so that we don't have to read the old body when updating the tree.
+
+12. Update sequences do not yet include an incarnation value.

Makefile

@@ -144,9 +144,11 @@ fauxton: share/www
 ################################################################################
 
 
-.PHONY: check
+# When we can run all the tests with FDB switch this back to be the default
+# "make check" command
+.PHONY: check-all-tests
 # target: check - Test everything
-check: all python-black
+check-all-tests: all python-black
 	@$(MAKE) emilio
 	@$(MAKE) eunit
 	@$(MAKE) javascript
@@ -159,6 +161,14 @@ else
 subdirs=$(shell ls src)
 endif
 
+.PHONY: check
+check:  all
+	@$(MAKE) emilio
+	make eunit apps=couch_eval,couch_expiring_cache,ctrace,couch_jobs,couch_views,fabric,mango,chttpd
+	make elixir tests=test/elixir/test/basics_test.exs,test/elixir/test/replication_test.exs,test/elixir/test/map_test.exs,test/elixir/test/all_docs_test.exs,test/elixir/test/bulk_docs_test.exs
+	make exunit apps=couch_rate,chttpd
+	make mango-test
+
 .PHONY: eunit
 # target: eunit - Run EUnit tests, use EUNIT_OPTS to provide custom options
 eunit: export BUILDDIR = $(shell pwd)
@@ -179,6 +189,7 @@ exunit: export MIX_ENV=test
 exunit: export ERL_LIBS = $(shell pwd)/src
 exunit: export ERL_AFLAGS = -config $(shell pwd)/rel/files/eunit.config
 exunit: export COUCHDB_QUERY_SERVER_JAVASCRIPT = $(shell pwd)/bin/couchjs $(shell pwd)/share/server/main.js
+exunit: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1
 exunit: couch elixir-init setup-eunit elixir-check-formatted elixir-credo
 	@mix test --cover --trace $(EXUNIT_OPTS)
 
@@ -200,7 +211,7 @@ soak-eunit: couch
 	while [ $$? -eq 0 ] ; do $(REBAR) -r eunit $(EUNIT_OPTS) ; done
 
 emilio:
-	@bin/emilio -c emilio.config src/ | bin/warnings_in_scope -s 3
+	@bin/emilio -c emilio.config src/ | bin/warnings_in_scope -s 3 || exit 0
 
 .venv/bin/black:
 	@python3 -m venv .venv
@@ -227,10 +238,25 @@ python-black-update: .venv/bin/black
 elixir: export MIX_ENV=integration
 elixir: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1
 elixir: elixir-init elixir-check-formatted elixir-credo devclean
-	@dev/run "$(TEST_OPTS)" -a adm:pass -n 1 \
+	@dev/run "$(TEST_OPTS)" \
+		-a adm:pass \
+		-n 1 \
+		--enable-erlang-views \
+		--locald-config test/elixir/test/config/test-config.ini \
+		--erlang-config rel/files/eunit.config \
+		--no-eval \
+		'mix test --trace --exclude without_quorum_test --exclude with_quorum_test $(EXUNIT_OPTS)'
+
+.PHONY: elixir-only
+elixir-only: devclean
+	@dev/run "$(TEST_OPTS)" \
+		-a adm:pass \
+		-n 1 \
 		--enable-erlang-views \
 		--locald-config test/elixir/test/config/test-config.ini \
-		--no-eval 'mix test --trace --exclude without_quorum_test --exclude with_quorum_test $(EXUNIT_OPTS)'
+		--erlang-config rel/files/eunit.config \
+		--no-eval \
+		'mix test --trace --exclude without_quorum_test --exclude with_quorum_test $(EXUNIT_OPTS)'
 
 .PHONY: elixir-init
 elixir-init: MIX_ENV=test
@@ -348,7 +374,7 @@ mango-test: devclean all
 	@cd src/mango && \
 		python3 -m venv .venv && \
 		.venv/bin/python3 -m pip install -r requirements.txt
-	@cd src/mango && ../../dev/run "$(TEST_OPTS)" -n 1 --admin=testuser:testpass '.venv/bin/python3 -m nose --with-xunit'
+	@cd src/mango && ../../dev/run "$(TEST_OPTS)" -n 1 --admin=adm:pass --erlang-config=rel/files/eunit.config '.venv/bin/python3 -m nose -v --with-xunit'
 
 ################################################################################
 # Developing

Makefile.win

@@ -163,6 +163,7 @@ exunit: export MIX_ENV=test
 exunit: export ERL_LIBS = $(shell echo %cd%)\src
 exunit: export ERL_AFLAGS = -config $(shell echo %cd%)/rel/files/eunit.config
 exunit: export COUCHDB_QUERY_SERVER_JAVASCRIPT = $(shell echo %cd%)/bin/couchjs $(shell echo %cd%)/share/server/main.js
+exunit: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1
 exunit: couch elixir-init setup-eunit elixir-check-formatted elixir-credo
 	@mix test --cover --trace $(EXUNIT_OPTS)
 
@@ -177,7 +178,7 @@ just-eunit:
 	@$(REBAR) -r eunit $(EUNIT_OPTS)
 
 emilio:
-	@bin\emilio -c emilio.config src\ | python.exe bin\warnings_in_scope -s 3
+	@bin\emilio -c emilio.config src\ | python.exe bin\warnings_in_scope -s 3 || exit 0
 
 .venv/bin/black:
 	@python.exe -m venv .venv

build-aux/Jenkinsfile.pr

@@ -21,7 +21,7 @@ cd build
 tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz
 cd apache-couchdb-*
 . /usr/local/kerl/${KERL_VER}/activate
-./configure --with-curl --spidermonkey-version 60
+./configure --with-curl
 make check || (make build-report && false)
 '''
 

configure

@@ -96,6 +96,24 @@ parse_opts() {
                 continue
                 ;;
 
+            --key-manager)
+                if [ -n "$2" ]; then
+                    eval AEGIS_KEY_MANAGER=$2
+                    shift 2
+                    continue
+                else
+                    printf 'ERROR: "--key-manager" requires a non-empty argument.\n' >&2
+                    exit 1
+                fi
+                ;;
+            --key-manager=?*)
+                eval AEGIS_KEY_MANAGER=${1#*=}
+                ;;
+            --key-manager=)
+                printf 'ERROR: "--key-manager" requires a non-empty argument.\n' >&2
+                exit 1
+                ;;
+
             --dev)
                 WITH_DOCS=0
                 WITH_FAUXTON=0
@@ -241,6 +259,7 @@ cat > $rootdir/config.erl << EOF
 {with_curl, $WITH_CURL}.
 {with_proper, $WITH_PROPER}.
 {erlang_md5, $ERLANG_MD5}.
+{aegis_key_manager, "$AEGIS_KEY_MANAGER"}.
 {spidermonkey_version, "$SM_VSN"}.
 EOF
 

dev/run

@@ -185,6 +185,12 @@ def get_args_parser():
         default=[],
         help="Optional key=val config overrides. Can be repeated",
     )
+    parser.add_option(
+        "--erlang-config",
+        dest="erlang_config",
+        default="rel/files/sys.config",
+        help="Specify an alternative Erlang application configuration"
+    )
     parser.add_option(
         "--degrade-cluster",
         dest="degrade_cluster",
@@ -241,6 +247,7 @@ def setup_context(opts, args):
         "haproxy": opts.haproxy,
         "haproxy_port": opts.haproxy_port,
         "config_overrides": opts.config_overrides,
+        "erlang_config": opts.erlang_config,
         "no_eval": opts.no_eval,
         "extra_args": opts.extra_args,
         "reset_logs": True,
@@ -605,7 +612,7 @@ def boot_node(ctx, node):
         "-args_file",
         os.path.join(node_etcdir, "vm.args"),
         "-config",
-        os.path.join(reldir, "files", "sys"),
+        os.path.join(ctx["rootdir"], ctx["erlang_config"]),
         "-couch_ini",
         os.path.join(node_etcdir, "default.ini"),
         os.path.join(node_etcdir, "local.ini"),

mix.exs

@@ -49,11 +49,14 @@ defmodule CouchDBTest.Mixfile do
   # Run "mix help compile.app" to learn about applications.
   def application do
     [
-      extra_applications: [:logger],
+      extra_applications: extra_applications(Mix.env()),
       applications: [:httpotion]
     ]
   end
 
+  defp extra_applications(:test), do: [:logger, :stream_data]
+  defp extra_applications(_), do: [:logger]
+
   # Specifies which paths to compile per environment.
   defp elixirc_paths(:test), do: ["test/elixir/lib", "test/elixir/test/support"]
   defp elixirc_paths(:integration), do: ["test/elixir/lib", "test/elixir/test/support"]
@@ -71,6 +74,7 @@ defmodule CouchDBTest.Mixfile do
       {:ibrowse,
        path: Path.expand("src/ibrowse", __DIR__), override: true, compile: false},
       {:credo, "~> 1.4.0", only: [:dev, :test, :integration], runtime: false}
+      {:stream_data, "~> 0.4.3", only: [:dev, :test, :integration], runtime: false}
     ]
   end
 

mix.lock

@@ -14,5 +14,6 @@
   "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
   "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm", "13104d7897e38ed7f044c4de953a6c28597d1c952075eb2e328bc6d6f2bfc496"},
+  "stream_data": {:hex, :stream_data, "0.4.3", "62aafd870caff0849a5057a7ec270fad0eb86889f4d433b937d996de99e3db25", [:mix], [], "hexpm", "7dafd5a801f0bc897f74fcd414651632b77ca367a7ae4568778191fc3bf3a19a"},
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm", "1d1848c40487cdb0b30e8ed975e34e025860c02e419cb615d255849f3427439d"},
 }

rebar.config.script

@@ -117,19 +117,27 @@ SubDirs = [
     "src/couch_log",
     "src/chttpd",
     "src/couch",
+    "src/couch_eval",
     "src/couch_event",
     "src/mem3",
     "src/couch_index",
     "src/couch_mrview",
+    "src/couch_js",
+    "src/couch_rate",
     "src/couch_replicator",
     "src/couch_plugins",
     "src/couch_pse_tests",
     "src/couch_stats",
     "src/couch_peruser",
     "src/couch_tests",
+    "src/couch_views",
+    "src/ctrace",
     "src/ddoc_cache",
     "src/dreyfus",
     "src/fabric",
+    "src/aegis",
+    "src/couch_jobs",
+    "src/couch_expiring_cache",
     "src/global_changes",
     "src/ioq",
     "src/jwtf",
@@ -138,13 +146,15 @@ SubDirs = [
     "src/rexi",
     "src/setup",
     "src/smoosh",
+    "src/ebtree",
     "rel"
 ].
 
 DepDescs = [
 %% Independent Apps
-{config,           "config",           {tag, "2.1.7"}},
+{config,           "config",           {tag, "2.1.8"}},
 {b64url,           "b64url",           {tag, "1.0.2"}},
+{erlfdb,           "erlfdb",           {tag, "v1.2.2"}},
 {ets_lru,          "ets-lru",          {tag, "1.1.0"}},
 {khash,            "khash",            {tag, "1.1.0"}},
 {snappy,           "snappy",           {tag, "CouchDB-1.0.4"}},
@@ -157,11 +167,15 @@ DepDescs = [
 %% Third party deps
 {folsom,           "folsom",           {tag, "CouchDB-0.8.3"}},
 {hyper,            "hyper",            {tag, "CouchDB-2.2.0-6"}},
-{ibrowse,          "ibrowse",          {tag, "CouchDB-4.0.1-1"}},
+{ibrowse,          "ibrowse",          {tag, "CouchDB-4.0.1-2"}},
+{jaeger_passage,   "jaeger-passage",   {tag, "CouchDB-0.1.14-1"}},
 {jiffy,            "jiffy",            {tag, "CouchDB-1.0.4-1"}},
+{local,            "local",            {tag, "0.2.1"}},
 {mochiweb,         "mochiweb",         {tag, "v2.20.0"}},
 {meck,             "meck",             {tag, "0.8.8"}},
-{recon,            "recon",            {tag, "2.5.0"}}
+{recon,            "recon",            {tag, "2.5.0"}},
+{passage,          "passage",          {tag, "0.2.6"}},
+{thrift_protocol,  "thrift-protocol",  {tag, "0.1.5"}}
 ].
 
 WithProper = lists:keyfind(with_proper, 1, CouchConfig) == {with_proper, true}.
@@ -212,6 +226,11 @@ AddConfig = [
     {post_hooks, [{compile, "escript support/build_js.escript"}]}
 ].
 
-C = lists:foldl(fun({K, V}, CfgAcc) ->
-    lists:keystore(K, 1, CfgAcc, {K, V})
+lists:foldl(fun({K, V}, CfgAcc) ->
+    case lists:keyfind(K, 1, CfgAcc) of
+        {K, Existent} when is_list(Existent) andalso is_list(V) ->
+            lists:keystore(K, 1, CfgAcc, {K, Existent ++ V});
+        false ->
+            lists:keystore(K, 1, CfgAcc, {K, V})
+    end
 end, CONFIG, AddConfig).