Permalink
Browse files

Replicator now uses lhttpc instead of ibrowse

Change-Id: I9c920c8b350c8d2234253103696eb4b16b981527
Reviewed-on: http://review.couchbase.org/14577
Tested-by: buildbot <build@couchbase.com>
Reviewed-by: Damien Katz <damien@couchbase.com>
Tested-by: Damien Katz <damien@couchbase.com>
  • Loading branch information...
1 parent 5a3fdc9 commit 7a63d76fcab3dcb3a8bdd1f67684e86a11380b60 @fdmanana fdmanana committed with Damien Katz Apr 5, 2012
View
@@ -53,7 +53,6 @@ source_files = \
couch_external_server.erl \
couch_file.erl \
couch_db_frontend.erl \
- couch_httpc_pool.erl \
couch_httpd.erl \
couch_httpd_db.erl \
couch_httpd_auth.erl \
@@ -133,7 +132,6 @@ compiled_files = \
couch_file.beam \
couch_file_write_guard.beam \
couch_db_frontend.beam \
- couch_httpc_pool.beam \
couch_httpd.beam \
couch_httpd_db.beam \
couch_httpd_auth.beam \
@@ -69,9 +69,9 @@ db_open(#httpdb{} = Db1, _Options, Create) ->
false ->
ok;
true ->
- send_req(Db, [{method, put}], fun(_, _, _) -> ok end)
+ send_req(Db, [{method, "PUT"}], fun(_, _, _) -> ok end)
end,
- send_req(Db, [{method, head}],
+ send_req(Db, [{method, "HEAD"}],
fun(200, _, _) ->
{ok, Db};
(401, _, _) ->
@@ -100,9 +100,8 @@ db_open(DbName, Options, Create) ->
throw({unauthorized, DbName})
end.
-db_close(#httpdb{httpc_pool = Pool}) ->
- unlink(Pool),
- ok = couch_httpc_pool:stop(Pool);
+db_close(#httpdb{} = HttpDb) ->
+ ok = couch_api_wrap_httpc:tear_down(HttpDb);
db_close(DbName) ->
catch couch_db:close(DbName).
@@ -122,7 +121,7 @@ get_db_info(#db{name = DbName, user_ctx = UserCtx}) ->
ensure_full_commit(#httpdb{} = Db) ->
send_req(
Db,
- [{method, post}, {path, "_ensure_full_commit"},
+ [{method, "POST"}, {path, "_ensure_full_commit"},
{headers, [{"Content-Type", "application/json"}]}],
fun(201, _, {Props}) ->
{ok, get_value(<<"instance_start_time">>, Props)};
@@ -137,7 +136,7 @@ get_missing_revs(#httpdb{} = Db, IdRevList) ->
JsonBody = {[{Id, couch_doc:rev_to_str(Rev)} || {Id, Rev} <- IdRevList]},
send_req(
Db,
- [{method, post}, {path, "_revs_diff"}, {body, ?JSON_ENCODE(JsonBody)}],
+ [{method, "POST"}, {path, "_revs_diff"}, {body, ?JSON_ENCODE(JsonBody)}],
fun(200, _, {Props}) ->
ConvertToNativeFun = fun({Id, {Result}}) ->
MissingRev = couch_doc:parse_rev(
@@ -202,7 +201,6 @@ update_docs(Db, DocList, Options) ->
update_docs(_, [], _, _) ->
ok;
update_docs(#httpdb{} = HttpDb, DocList, Options, UpdateType) ->
- FullCommit = atom_to_list(not lists:member(delay_commit, Options)),
Prefix = case UpdateType of
replicated_changes ->
<<"{\"new_edits\":false,\"docs\":[">>;
@@ -222,31 +220,37 @@ update_docs(#httpdb{} = HttpDb, DocList, Options, UpdateType) ->
end,
byte_size(Prefix) + byte_size(Suffix) + length(DocList) - 1,
DocList),
- BodyFun = fun(eof) ->
- eof;
- ([]) ->
- {ok, Suffix, eof};
- ([prefix | Rest]) ->
- {ok, Prefix, Rest};
- ([Doc]) ->
- {ok, Doc, []};
- ([Doc | RestDocs]) ->
- {ok, [Doc, ","], RestDocs}
- end,
Headers = [
- {"Content-Length", Len},
- {"Content-Type", "application/json"},
- {"X-Couch-Full-Commit", FullCommit}
+ {"Content-Length", integer_to_list(Len)},
+ {"Content-Type", "application/json"}
],
- send_req(
- HttpDb,
- [{method, post}, {path, "_bulk_docs"},
- {body, {BodyFun, [prefix | Docs]}}, {headers, Headers}],
- fun(201, _, _) ->
- ok;
- (_, _, Error) ->
- {ok, Error}
- end);
+ ReqOptions = [
+ {method, "POST"},
+ {path, "_bulk_docs"},
+ {headers, maybe_add_delayed_commit(Headers, Options)},
+ {lhttpc_options, [{partial_upload, 3}]}
+ ],
+ SendDocsFun = fun(Data, {SendFun, N}) ->
+ {ok, SendFun2} = case N > 1 of
+ true ->
+ SendFun([Data, <<",">>]);
+ false ->
+ SendFun(Data)
+ end,
+ {SendFun2, N - 1}
+ end,
+ ReqCallback = fun(UploadFun) ->
+ {ok, UploadFun2} = UploadFun(Prefix),
+ {UploadFun3, 0} = lists:foldl(SendDocsFun, {UploadFun2, length(Docs)}, Docs),
+ {ok, UploadFun4} = UploadFun3(Suffix),
+ case UploadFun4(eof) of
+ {ok, 201, _Headers, _Body} ->
+ ok;
+ {ok, _Code, _Headers, Error} ->
+ {ok, Error}
+ end
+ end,
+ send_req(HttpDb, ReqOptions, ReqCallback);
update_docs(Db, DocList, Options, replicated_changes) ->
ok = couch_db:update_docs(Db, DocList, Options).
@@ -265,25 +269,36 @@ changes_since(#httpdb{headers = Headers1} = HttpDb, Style, StartSeq,
{QArgs, Method, Body, Headers} = case DocIds of
undefined ->
QArgs1 = maybe_add_changes_filter_q_args(BaseQArgs, Options),
- {QArgs1, get, [], Headers1};
+ {QArgs1, "GET", [], Headers1};
_ when is_list(DocIds) ->
Headers2 = [{"Content-Type", "application/json"} | Headers1],
JsonDocIds = ?JSON_ENCODE({[{<<"doc_ids">>, DocIds}]}),
- {[{"filter", "_doc_ids"} | BaseQArgs], post, JsonDocIds, Headers2}
+ {[{"filter", "_doc_ids"} | BaseQArgs], "POST", JsonDocIds, Headers2}
end,
+ ReqOptions = [
+ {method, Method},
+ {path, "_changes"},
+ {body, Body},
+ {headers, Headers},
+ {qs, QArgs},
+ {lhttpc_options, [{partial_download, [{window_size, 3}]}]}
+ ],
send_req(
HttpDb,
- [{method, Method}, {path, "_changes"}, {qs, QArgs},
- {headers, Headers}, {body, Body},
- {ibrowse_options, [{stream_to, {self(), once}}]}],
+ ReqOptions,
fun(200, _, DataStreamFun) ->
parse_changes_feed(Options, UserFun, DataStreamFun);
(405, _, _) when is_list(DocIds) ->
% CouchDB versions < 1.1.0 don't have the builtin _changes feed
% filter "_doc_ids" neither support POST
- send_req(HttpDb, [{method, get}, {path, "_changes"},
- {qs, BaseQArgs}, {headers, Headers1},
- {ibrowse_options, [{stream_to, {self(), once}}]}],
+ Req2Options = [
+ {method, "GET"},
+ {path, "_changes"},
+ {qs, BaseQArgs},
+ {headers, Headers1},
+ {lhttpc_options, [{partial_download, [{window_size, 3}]}]}
+ ],
+ send_req(HttpDb, Req2Options,
fun(200, _, DataStreamFun2) ->
UserFun2 = fun(#doc_info{id = Id} = DocInfo) ->
case lists:member(Id, DocIds) of
@@ -427,3 +442,11 @@ json_to_doc_info({Props}) ->
deleted = Del
}.
+maybe_add_delayed_commit(Headers, Options) ->
+ case lists:member(delay_commit, Options) of
+ true ->
+ [{"X-Couch-Full-Commit", "false"} | Headers];
+ false ->
+ Headers
+ end.
+
@@ -20,7 +20,7 @@
{"User-Agent", "CouchDB/" ++ couch_server:get_version()}
],
timeout, % milliseconds
- ibrowse_options = [],
+ lhttpc_options = [],
retries = 10,
wait = 250, % milliseconds
httpc_pool = nil,
Oops, something went wrong.

0 comments on commit 7a63d76

Please sign in to comment.