From 6adced02a672b065fb77432d58779b0ecf91359a Mon Sep 17 00:00:00 2001 From: ILYA Khlopotov Date: Thu, 27 Oct 2016 14:18:58 -0700 Subject: [PATCH 1/3] Format gen_server state to remove plain passwords Add format_status/2 to every gen_server to prune the state before passing it to SASL logger. There are two goals for this work: - eliminate plain text passwords in the logs - reduce the size of the terms we log COUCHDB-1606 --- src/couch_replicator.erl | 94 ++++---- src/couch_replicator_httpc_pool.erl | 25 +- src/couch_replicator_manager.erl | 25 +- src/couch_replicator_utils.erl | 34 +++ src/couch_replicator_worker.erl | 24 +- test/couch_replicator_state_format_tests.erl | 228 +++++++++++++++++++ 6 files changed, 386 insertions(+), 44 deletions(-) create mode 100644 test/couch_replicator_state_format_tests.erl diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl index 7f0c7ee..d245898 100644 --- a/src/couch_replicator.erl +++ b/src/couch_replicator.erl @@ -514,44 +514,6 @@ code_change(_OldVsn, #rep_state{}=State, _Extra) -> {ok, State}. -headers_strip_creds([], Acc) -> - lists:reverse(Acc); -headers_strip_creds([{Key, Value0} | Rest], Acc) -> - Value = case string:to_lower(Key) of - "authorization" -> - "****"; - _ -> - Value0 - end, - headers_strip_creds(Rest, [{Key, Value} | Acc]). - - -httpdb_strip_creds(#httpdb{url = Url, headers = Headers} = HttpDb) -> - HttpDb#httpdb{ - url = couch_util:url_strip_password(Url), - headers = headers_strip_creds(Headers, []) - }; -httpdb_strip_creds(LocalDb) -> - LocalDb. - - -rep_strip_creds(#rep{source = Source, target = Target} = Rep) -> - Rep#rep{ - source = httpdb_strip_creds(Source), - target = httpdb_strip_creds(Target) - }. - - -state_strip_creds(#rep_state{rep_details = Rep, source = Source, target = Target} = State) -> - % #rep_state contains the source and target at the top level and also - % in the nested #rep_details record - State#rep_state{ - rep_details = rep_strip_creds(Rep), - source = httpdb_strip_creds(Source), - target = httpdb_strip_creds(Target) - }. - - terminate(normal, #rep_state{rep_details = #rep{id = RepId} = Rep, checkpoint_history = CheckpointHistory} = State) -> terminate_cleanup(State), @@ -566,7 +528,7 @@ terminate(shutdown, #rep_state{rep_details = #rep{id = RepId}} = State) -> terminate(shutdown, {error, Class, Error, Stack, InitArgs}) -> #rep{id=RepId} = InitArgs, couch_stats:increment_counter([couch_replicator, failed_starts]), - CleanInitArgs = rep_strip_creds(InitArgs), + CleanInitArgs = couch_replicator_utils:format_rep_record(InitArgs), couch_log:error("~p:~p: Replication failed to start for args ~p: ~p", [Class, Error, CleanInitArgs, Stack]), case Error of @@ -600,8 +562,23 @@ terminate_cleanup(State) -> format_status(_Opt, [_PDict, State]) -> - [{data, [{"State", state_strip_creds(State)}]}]. - + Rep = couch_replicator_utils:format_rep_record(State#rep_state.rep_details), + [{data, [ + {"State", ?from_record(rep_state, State, [ + start_seq, + committed_seq, + current_through_seq, + %%seqs_in_progress = [], + highest_seq_done, + rep_starttime, + src_starttime, + tgt_starttime, + timer, % checkpoint timer + session_id, + source_seq, + use_checkpoints, + checkpoint_interval]) + ++ [{rep_details, Rep}]}]}]. do_last_checkpoint(#rep_state{seqs_in_progress = [], highest_seq_done = {_Ts, ?LOWEST_SEQ}} = State) -> @@ -1035,3 +1012,38 @@ rep_stats(State) -> {doc_write_failures, couch_replicator_stats:doc_write_failures(Stats)}, {checkpointed_source_seq, CommittedSeq} ]. + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +format_status_test() -> + S = "https://user_foo:top_secret@account_bar1.cloudant.com/database_baz/", + T = "https://user_foo:top_secret@account_bar2.cloudant.com/baz_backup/", + Rep0 = list_to_tuple([rep | record_info(fields, rep)]), + State0 = list_to_tuple([rep_state | record_info(fields, rep_state)]), + Rep = Rep0#rep{source = #httpdb{url = S}, target = #httpdb{url = T}}, + State = State0#rep_state{rep_details = Rep}, + ?assertEqual([{data, [{"State", [ + {start_seq,start_seq}, + {committed_seq,committed_seq}, + {current_through_seq,current_through_seq}, + {highest_seq_done,highest_seq_done}, + {rep_starttime,rep_starttime}, + {src_starttime,src_starttime}, + {tgt_starttime,tgt_starttime}, + {timer,timer}, + {session_id,session_id}, + {source_seq,source_seq}, + {use_checkpoints,use_checkpoints}, + {checkpoint_interval,checkpoint_interval}, + {rep_details,[{id,id}, + {options,options}, + {view,view}, + {doc_id,doc_id}, + {db_name,db_name}, + {source,"https://user_foo:*****@account_bar1.cloudant.com/database_baz/"}, + {target,"https://user_foo:*****@account_bar2.cloudant.com/baz_backup/"}]} + ]}]}], format_status(normal, [[], State])). + +-endif. diff --git a/src/couch_replicator_httpc_pool.erl b/src/couch_replicator_httpc_pool.erl index 09e3b23..300085e 100644 --- a/src/couch_replicator_httpc_pool.erl +++ b/src/couch_replicator_httpc_pool.erl @@ -20,7 +20,7 @@ % gen_server API -export([init/1, handle_call/3, handle_info/2, handle_cast/2]). --export([code_change/3, terminate/2]). +-export([code_change/3, terminate/2, format_status/2]). -include_lib("couch/include/couch_db.hrl"). @@ -151,6 +151,14 @@ terminate(_Reason, State) -> lists:foreach(fun ibrowse_http_client:stop/1, State#state.free), lists:foreach(fun ibrowse_http_client:stop/1, State#state.busy). +format_status(_Opt, [_PDict, #state{url = Url, limit = Limit}]) -> + [{data, [{"State", [ + {url, couch_util:url_strip_password(Url)}, + {limit, Limit} + ]}]}]; +format_status(_Opt, [_PDict, State]) -> + [{data, [{"State", State}]}]. + monitor_client(Callers, Worker, {ClientPid, _}) -> [{Worker, erlang:monitor(process, ClientPid)} | Callers]. @@ -190,3 +198,18 @@ release_worker_internal(Worker, State) -> false -> State#state{callers = NewCallers0} end. + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +format_status_test() -> + Url = "https://user_foo:top_secret@account_bar1.cloudant.com/database_baz/", + State0 = list_to_tuple([state | record_info(fields, state)]), + State = State0#state{url = Url}, + ?assertEqual([{data, [{"State", [ + {url, "https://user_foo:*****@account_bar1.cloudant.com/database_baz/"}, + {limit, limit} + ]}]}], format_status(normal, [[], State])). + +-endif. diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl index 953b1bf..a10a975 100644 --- a/src/couch_replicator_manager.erl +++ b/src/couch_replicator_manager.erl @@ -23,7 +23,7 @@ % gen_server callbacks -export([start_link/0, init/1, handle_call/3, handle_info/2, handle_cast/2]). --export([code_change/3, terminate/2]). +-export([code_change/3, terminate/2, format_status/2]). % changes callbacks -export([changes_reader/3, changes_reader_cb/3]). @@ -363,6 +363,14 @@ code_change(1, State, _Extra) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. +format_status(_Opt, [_PDict, #state{} = State]) -> + [{data, [{"State", ?from_record(state, State, [ + event_listener, + scan_pid, + max_retries, + epoch + ])}]}]. + start_changes_reader(DbName, Since, Epoch) -> spawn_link(?MODULE, changes_reader, [{self(), Epoch}, DbName, Since]). @@ -980,3 +988,18 @@ get_json_value(Key, Props, Default) when is_binary(Key) -> Else -> Else end. + +-ifdef(TEST). + +-include_lib("couch/include/couch_eunit.hrl"). + +format_status_test() -> + State = list_to_tuple([state | record_info(fields, state)]), + ?assertEqual([{data, [{"State", [ + {event_listener, event_listener}, + {scan_pid, scan_pid}, + {max_retries, max_retries}, + {epoch, epoch} + ]}]}], format_status(normal, [[], State])). + +-endif. diff --git a/src/couch_replicator_utils.erl b/src/couch_replicator_utils.erl index 223bac8..e9adf69 100644 --- a/src/couch_replicator_utils.erl +++ b/src/couch_replicator_utils.erl @@ -20,6 +20,7 @@ -export([mp_parse_doc/2]). -export([handle_db_event/3]). +-export([format_rep_record/1]). -include_lib("couch/include/couch_db.hrl"). -include_lib("ibrowse/include/ibrowse.hrl"). @@ -509,11 +510,44 @@ ejsort_array([], Acc)-> ejsort_array([V | R], Acc) -> ejsort_array(R, [ejsort(V) | Acc]). +format_rep_record(#rep{} = Rep) -> + ?from_record(rep, Rep, [ + id, + options, + view, + doc_id, + db_name + ]) + ++ [ + {source, format_httpdb(Rep#rep.source)}, + {target, format_httpdb(Rep#rep.target)} + ]. + +format_httpdb(#httpdb{url = Url}) -> + couch_util:url_strip_password(Url); +format_httpdb(LocalDb) -> + LocalDb. + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). +format_rep_record_test() -> + Rep0 = list_to_tuple([rep | record_info(fields, rep)]), + S = "https://user_foo:top_secret@account_bar1.cloudant.com/database_baz/", + T = "https://user_foo:top_secret@account_bar2.cloudant.com/baz_backup/", + Rep = Rep0#rep{source = #httpdb{url = S}, target = #httpdb{url = T}}, + ?assertEqual([ + {id,id}, + {options,options}, + {view,view}, + {doc_id,doc_id}, + {db_name,db_name}, + {source,"https://user_foo:*****@account_bar1.cloudant.com/database_baz/"}, + {target,"https://user_foo:*****@account_bar2.cloudant.com/baz_backup/"} + ], format_rep_record(Rep)). + ejsort_basic_values_test() -> ?assertEqual(ejsort(0), 0), ?assertEqual(ejsort(<<"a">>), <<"a">>), diff --git a/src/couch_replicator_worker.erl b/src/couch_replicator_worker.erl index ee0c455..f19b7fb 100644 --- a/src/couch_replicator_worker.erl +++ b/src/couch_replicator_worker.erl @@ -18,7 +18,7 @@ -export([start_link/5]). % gen_server callbacks --export([init/1, terminate/2, code_change/3]). +-export([init/1, terminate/2, code_change/3, format_status/2]). -export([handle_call/3, handle_cast/2, handle_info/2]). -include_lib("couch/include/couch_db.hrl"). @@ -210,6 +210,28 @@ terminate(_Reason, State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. +format_status(_Opt, [_PDict, #state{source = S, target = T} = State]) -> + [{data, [{"State", ?from_record(state, State, [ + cp, + loop, + max_parallel_conns, + writer, + pending_fetch, + flush_waiter, + source_db_compaction_notifier, + target_db_compaction_notifier, + batch, % = #batch{docs = [], size = 0} + max_retries + ]) ++ [ + {source, format_db(S)}, + {target, format_db(T)} + ]}]}]. + +format_db(#db{} = Db) -> + couch_db:format_db_record(Db); +format_db(Url) -> + couch_util:url_strip_password(Url). + queue_fetch_loop(Source, Target, Parent, Cp, ChangesManager) -> ChangesManager ! {get_changes, self()}, diff --git a/test/couch_replicator_state_format_tests.erl b/test/couch_replicator_state_format_tests.erl new file mode 100644 index 0000000..a65c5c3 --- /dev/null +++ b/test/couch_replicator_state_format_tests.erl @@ -0,0 +1,228 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_replicator_state_format_tests). + +-include_lib("couch/include/couch_eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + +%%-define(TIMEOUT_EUNIT, 30). +-define(TIMEOUT, 1000). + +-define(USER, "user_foo"). +-define(PASS, "top_secret"). + +setup() -> + DbName = ?tempdb(), + {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]), + ok = couch_db:close(Db), + DbName. + +setup(local) -> + setup(); +setup(remote) -> + {remote, setup()}; +setup({Service, {A, B}}) -> + Ctx = test_util:start_couch([couch_replicator]), + ok = config:set("admins", ?USER, ?PASS, _Persist=false), + Source = setup(A), + Target = setup(B), + {ok, Pid} = start(Service, {Source, Target}), + {Ctx, Pid, {Source, Target}}. + +teardown({remote, DbName}) -> + teardown(DbName); +teardown(DbName) -> + ok = couch_server:delete(DbName, [?ADMIN_CTX]), + ok. + +teardown({_, _}, {Ctx, _Pid, {Source, Target}}) -> + teardown(Source), + teardown(Target), + %%exit(Pid, kill), + ok = test_util:stop_couch(Ctx). + +start(replicator, {Source, Target}) -> + RepObject = {[ + {<<"source">>, db(Source)}, + {<<"target">>, db(Target)}, + {<<"use_checkpoints">>, false} + ]}, + {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_USER), + {ok, Pid} = gen_server:start_link(couch_replicator, Rep, []), + {ok, Pid}; +start(worker, {Source, Target}) -> + {ok, Pid} = couch_replicator_worker:start_link( + self(), db(Source), db(Target), self(), 1), + receive + {get_changes, _} -> + ok + after ?TIMEOUT -> + throw(timeout) + end, + {ok, Pid}; +start(httpc_pool, {Source, _Target}) -> + couch_replicator_httpc_pool:start_link(db(Source), []); +start(manager, {_Source, _Target}) -> + {ok, whereis(couch_replicator_manager)}. + + +state_format_test_() -> + Pairs = [{local, local}, {local, remote}, + {remote, local}, {remote, remote}], + Tests = [ + fun should_format_state/2, + fun should_strip_credentials/2 + ], + Targets = [replicator, worker, httpc_pool, manager], + { + "Make sure we format state of the process", + { + foreachx, + fun setup/1, fun teardown/2, + [{{Target, Pair}, Test} + || + Pair <- Pairs, + Test <- Tests, + Target <- Targets] + } + }. + +should_format_state({replicator, _} = Args, {_Ctx, Pid, _}) -> + {test_id(Args), ?_test(begin + State = get_state(Pid), + ExpectedStateKeys = [ + start_seq, + committed_seq, + current_through_seq, + highest_seq_done, + rep_starttime, + src_starttime, + tgt_starttime, + timer, + session_id, + source_seq, + use_checkpoints, + checkpoint_interval, + rep_details + ], + ?assertEqualLists(ExpectedStateKeys, proplists:get_keys(State)), + RepState = proplists:get_value(rep_details, State), + ExpectedRepKeys = [ + id, + options, + view, + doc_id, + db_name, + source, + target + ], + ?assertEqualLists(ExpectedRepKeys, proplists:get_keys(RepState)), + ok + end)}; +should_format_state({worker, _} = Args, {_Ctx, Pid, _}) -> + {test_id(Args), ?_test(begin + State = get_state(Pid), + ExpectedStateKeys = [ + cp, + loop, + max_parallel_conns, + writer, + pending_fetch, + flush_waiter, + source_db_compaction_notifier, + target_db_compaction_notifier, + batch, + source, + target + ], + ?assertEqualLists(ExpectedStateKeys, proplists:get_keys(State)), + ok + end)}; +should_format_state({httpc_pool, _} = Args, {_Ctx, Pid, _}) -> + {test_id(Args), ?_test(begin + State = get_state(Pid), + ExpectedStateKeys = [ + url, + limit + ], + ?assertEqualLists(ExpectedStateKeys, proplists:get_keys(State)), + ok + end)}; +should_format_state({manager, _} = Args, {_Ctx, Pid, _}) -> + {test_id(Args), ?_test(begin + State = get_state(Pid), + ExpectedStateKeys = [ + event_listener, + scan_pid, + max_retries, + epoch + ], + ?assertEqualLists(ExpectedStateKeys, proplists:get_keys(State)), + ok + end)}. + + +should_strip_credentials({replicator, _} = Args, {_Ctx, Pid, _}) -> + {test_id(Args), ?_test(begin + State = get_state(Pid), + RepState = proplists:get_value(rep_details, State), + Source = proplists:get_value(source, RepState), + Target = proplists:get_value(target, RepState), + ?assertNot(contains_password(Source)), + ?assertNot(contains_password(Target)), + ok + end)}; +should_strip_credentials({worker, _} = Args, {_Ctx, Pid, _}) -> + {test_id(Args), ?_test(begin + State = get_state(Pid), + Source = proplists:get_value(source, State), + Target = proplists:get_value(target, State), + ?assertNot(contains_password(Source)), + ?assertNot(contains_password(Target)), + ok + end)}; +should_strip_credentials({httpc_pool, _} = Args, {_Ctx, Pid, _}) -> + {test_id(Args), ?_test(begin + State = get_state(Pid), + URL = proplists:get_value(url, State), + ?assertNot(contains_password(URL)), + ok + end)}; +should_strip_credentials({manager, _} = Args, {_Ctx, _Pid, _}) -> + {test_id(Args), ?_test(ok)}. + +test_id({Target, {From, To}}) -> + lists:flatten(io_lib:format("~p: ~p -> ~p", [Target, From, To])). + +db({remote, DbName}) -> + iolist_to_binary([ + "http://", ?USER, ":", ?PASS, "@", + config:get("httpd", "bind_address", "127.0.0.1"), + ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)), + "/", DbName + ]); +db(DbName) -> + DbName. + +contains_password(Bin) when is_binary(Bin) -> + contains_password(?b2l(Bin)); +contains_password(String) -> + string:str(String, ?PASS) =/= 0. + +get_state(Pid) -> + {status, Pid, {module, _}, SItems} = sys:get_status(Pid), + Filtered = [Props || Props <- SItems, is_list(Props)], + Misc = lists:flatten(lists:foldl(fun(SItem, _) -> + proplists:get_all_values(data, SItem) + end, undefined, Filtered)), + proplists:get_value("State", Misc). From f5097fe24a2fdb9200d767234e1476f305a861ce Mon Sep 17 00:00:00 2001 From: ILYA Khlopotov Date: Fri, 28 Oct 2016 12:56:45 -0700 Subject: [PATCH 2/3] Remove catch all clause from format_state COUCHDB-1606 --- src/couch_replicator_httpc_pool.erl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/couch_replicator_httpc_pool.erl b/src/couch_replicator_httpc_pool.erl index 300085e..fed51db 100644 --- a/src/couch_replicator_httpc_pool.erl +++ b/src/couch_replicator_httpc_pool.erl @@ -155,9 +155,7 @@ format_status(_Opt, [_PDict, #state{url = Url, limit = Limit}]) -> [{data, [{"State", [ {url, couch_util:url_strip_password(Url)}, {limit, Limit} - ]}]}]; -format_status(_Opt, [_PDict, State]) -> - [{data, [{"State", State}]}]. + ]}]}]. monitor_client(Callers, Worker, {ClientPid, _}) -> [{Worker, erlang:monitor(process, ClientPid)} | Callers]. From c3107839d273f6d7c930280c8f1fce4ebdd42f7a Mon Sep 17 00:00:00 2001 From: ILYA Khlopotov Date: Fri, 28 Oct 2016 12:57:47 -0700 Subject: [PATCH 3/3] Reorder fields in debuggability order COUCHDB-1606 --- src/couch_replicator.erl | 25 +++++++++++++------------ src/couch_replicator_utils.erl | 27 +++++++++++++-------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl index d245898..3ac0968 100644 --- a/src/couch_replicator.erl +++ b/src/couch_replicator.erl @@ -564,7 +564,8 @@ terminate_cleanup(State) -> format_status(_Opt, [_PDict, State]) -> Rep = couch_replicator_utils:format_rep_record(State#rep_state.rep_details), [{data, [ - {"State", ?from_record(rep_state, State, [ + {"State", [{rep_details, Rep} | ?from_record(rep_state, State, [ + session_id, start_seq, committed_seq, current_through_seq, @@ -574,11 +575,10 @@ format_status(_Opt, [_PDict, State]) -> src_starttime, tgt_starttime, timer, % checkpoint timer - session_id, source_seq, use_checkpoints, checkpoint_interval]) - ++ [{rep_details, Rep}]}]}]. + ]}]}]. do_last_checkpoint(#rep_state{seqs_in_progress = [], highest_seq_done = {_Ts, ?LOWEST_SEQ}} = State) -> @@ -1025,6 +1025,15 @@ format_status_test() -> Rep = Rep0#rep{source = #httpdb{url = S}, target = #httpdb{url = T}}, State = State0#rep_state{rep_details = Rep}, ?assertEqual([{data, [{"State", [ + {rep_details,[ + {source,"https://user_foo:*****@account_bar1.cloudant.com/database_baz/"}, + {target,"https://user_foo:*****@account_bar2.cloudant.com/baz_backup/"}, + {id,id}, + {db_name,db_name}, + {doc_id,doc_id}, + {view,view}, + {options,options}]}, + {session_id,session_id}, {start_seq,start_seq}, {committed_seq,committed_seq}, {current_through_seq,current_through_seq}, @@ -1033,17 +1042,9 @@ format_status_test() -> {src_starttime,src_starttime}, {tgt_starttime,tgt_starttime}, {timer,timer}, - {session_id,session_id}, {source_seq,source_seq}, {use_checkpoints,use_checkpoints}, - {checkpoint_interval,checkpoint_interval}, - {rep_details,[{id,id}, - {options,options}, - {view,view}, - {doc_id,doc_id}, - {db_name,db_name}, - {source,"https://user_foo:*****@account_bar1.cloudant.com/database_baz/"}, - {target,"https://user_foo:*****@account_bar2.cloudant.com/baz_backup/"}]} + {checkpoint_interval,checkpoint_interval} ]}]}], format_status(normal, [[], State])). -endif. diff --git a/src/couch_replicator_utils.erl b/src/couch_replicator_utils.erl index e9adf69..4154126 100644 --- a/src/couch_replicator_utils.erl +++ b/src/couch_replicator_utils.erl @@ -511,16 +511,15 @@ ejsort_array([V | R], Acc) -> ejsort_array(R, [ejsort(V) | Acc]). format_rep_record(#rep{} = Rep) -> - ?from_record(rep, Rep, [ - id, - options, - view, - doc_id, - db_name - ]) - ++ [ + [ {source, format_httpdb(Rep#rep.source)}, {target, format_httpdb(Rep#rep.target)} + | ?from_record(rep, Rep, [ + id, + db_name, + doc_id, + view, + options]) ]. format_httpdb(#httpdb{url = Url}) -> @@ -539,13 +538,13 @@ format_rep_record_test() -> T = "https://user_foo:top_secret@account_bar2.cloudant.com/baz_backup/", Rep = Rep0#rep{source = #httpdb{url = S}, target = #httpdb{url = T}}, ?assertEqual([ - {id,id}, - {options,options}, - {view,view}, - {doc_id,doc_id}, - {db_name,db_name}, {source,"https://user_foo:*****@account_bar1.cloudant.com/database_baz/"}, - {target,"https://user_foo:*****@account_bar2.cloudant.com/baz_backup/"} + {target,"https://user_foo:*****@account_bar2.cloudant.com/baz_backup/"}, + {id,id}, + {db_name,db_name}, + {doc_id,doc_id}, + {view,view}, + {options,options} ], format_rep_record(Rep)). ejsort_basic_values_test() ->