Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Replacement of inets with ibrowse. Fixes COUCHDB-179 and enhances rep…
…lication.

Thanks Jason Davies and Adam Kocoloski for the fix, Maximillian Dornseif for reporting.


git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@739047 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information
jchris committed Jan 29, 2009
1 parent 7cbee5a commit ace6dfe
Show file tree
Hide file tree
Showing 25 changed files with 3,073 additions and 42 deletions.
2 changes: 2 additions & 0 deletions CHANGES
Expand Up @@ -28,6 +28,8 @@ Database Core:
dramatically. The fix keeps only one document in the write queue at a time.
* Fix for databases sometimes incorrectly reporting that they contain 0
documents after compaction.
* CouchDB now uses ibrowse instead of inets for its internal HTTP client
implementation. This means better replication stability.

HTTP Interface:

Expand Down
2 changes: 1 addition & 1 deletion Makefile.am
Expand Up @@ -10,7 +10,7 @@
## License for the specific language governing permissions and limitations
## under the License.

SUBDIRS = bin etc src/couchdb src/mochiweb share test var utils
SUBDIRS = bin etc src/couchdb src/ibrowse src/mochiweb share test var utils

localdoc_DATA = AUTHORS.gz BUGS.gz CHANGES.gz NEWS.gz README.gz THANKS.gz

Expand Down
6 changes: 6 additions & 0 deletions NOTICE
Expand Up @@ -21,3 +21,9 @@ This product also includes the following third-party components:
* MochiWeb (http://code.google.com/p/mochiweb/)

Copyright 2007, Mochi Media Coporation

* ibrowse
(http://jungerl.cvs.sourceforge.net/viewvc/jungerl/jungerl/lib/ibrowse/)

Copyright 2008, Chandrashekhar Mullaparthi
This ASF redistribution is consistent with the terms of the BSD License.
2 changes: 2 additions & 0 deletions THANKS
Expand Up @@ -12,7 +12,9 @@ Some of these people are:
* Yoan Blanc <yoan.blanc@gmail.com>
* Paul Carey <paul.p.carey@gmail.com>
* Benoit Chesneau <bchesneau@gmail.com>
* Jason Davies <jason@jasondavies.com>
* Paul Joseph Davis <paul.joseph.davis@gmail.com>
* Maximillian Dornseif <md@hudora.de>
* Michael Gottesman <gottesmm@reed.edu>
* Michael Hendricks <michael@ndrix.org>
* Till Klampaeckel <till@klampaeckel.de>
Expand Down
5 changes: 3 additions & 2 deletions bin/Makefile.am
Expand Up @@ -28,8 +28,9 @@ couchdb: couchdb.tpl
-e "s|%ICU_CONFIG%|$(ICU_CONFIG)|g" \
-e "s|%bindir%|@bindir@|g" \
-e "s|%localerlanglibdir%|@localerlanglibdir@|g" \
-e "s|%mochiwebebindir%|couch-@version@/ebin|g" \
-e "s|%couchdbebindir%|mochiweb-r82/ebin|g" \
-e "s|%couchdbebindir%|couch-@version@/ebin|g" \
-e "s|%mochiwebebindir%|mochiweb-r82/ebin|g" \
-e "s|%ibrowseebindir%|ibrowse-1.4.1/ebin|g" \
-e "s|%defaultini%|default.ini|g" \
-e "s|%localini%|local.ini|g" \
-e "s|%localconfdir%|@localconfdir@|g" \
Expand Down
5 changes: 3 additions & 2 deletions bin/couchdb.tpl.in
Expand Up @@ -183,11 +183,12 @@ start_couchdb () {
%ERL% $interactive_option -smp auto -sasl errlog_type error +K true \
-pa %localerlanglibdir%/%couchdbebindir% \
%localerlanglibdir%/%mochiwebebindir% \
-eval \"application:load(inets)\" \
%localerlanglibdir%/%ibrowseebindir% \
-eval \"application:load(ibrowse)\" \
-eval \"application:load(crypto)\" \
-eval \"application:load(couch)\" \
-eval \"crypto:start()\" \
-eval \"inets:start()\" \
-eval \"ibrowse:start()\" \
-eval \"couch_server:start([$start_arguments]), receive done -> done end.\" "
if test "$BACKGROUND_BOOLEAN" = "true" \
-a "$RECURSED_BOOLEAN" = "false"; then
Expand Down
1 change: 1 addition & 0 deletions configure.ac
Expand Up @@ -267,6 +267,7 @@ AC_CONFIG_FILES([etc/Makefile])
AC_CONFIG_FILES([share/Makefile])
AC_CONFIG_FILES([src/couchdb/couch.app.tpl])
AC_CONFIG_FILES([src/couchdb/Makefile])
AC_CONFIG_FILES([src/ibrowse/Makefile])
AC_CONFIG_FILES([src/mochiweb/Makefile])
AC_CONFIG_FILES([test/Makefile])
AC_CONFIG_FILES([utils/Makefile])
Expand Down
13 changes: 13 additions & 0 deletions share/www/script/couch_tests.js
Expand Up @@ -2134,6 +2134,19 @@ var tests = {
T(docA._rev == docB._rev);
};
},

design_docs_test: new function() {
// make sure design docs replicate properly
this.init = function(dbA, dbB) {
dbA.save({ _id:"_design/test" });
};

this.afterAB1 = function() {
var docA = dbA.open("_design/test");
var docB = dbB.open("_design/test");
T(docA._rev == docB._rev);
};
},

attachments_test: new function () {
// Test attachments
Expand Down
2 changes: 1 addition & 1 deletion src/couchdb/couch.app.tpl.in
Expand Up @@ -24,4 +24,4 @@
couch_view,
couch_query_servers,
couch_db_update_notifier_sup]},
{applications,[kernel,stdlib,crypto,inets,mochiweb]}]}.
{applications,[kernel,stdlib,crypto,ibrowse,mochiweb]}]}.
11 changes: 10 additions & 1 deletion src/couchdb/couch_httpd.erl
Expand Up @@ -15,7 +15,7 @@

-export([start_link/0, stop/0, handle_request/3]).

-export([header_value/2,header_value/3,qs_value/2,qs_value/3,qs/1,path/1]).
-export([header_value/2,header_value/3,qs_value/2,qs_value/3,qs/1,path/1,absolute_uri/2]).
-export([verify_is_server_admin/1,unquote/1,quote/1,recv/2]).
-export([parse_form/1,json_body/1,body/1,doc_etag/1, make_etag/1, etag_respond/3]).
-export([primary_header_value/2,partition/1,serve_file/3]).
Expand Down Expand Up @@ -242,6 +242,15 @@ qs(#httpd{mochi_req=MochiReq}) ->
path(#httpd{mochi_req=MochiReq}) ->
MochiReq:get(path).

absolute_uri(#httpd{mochi_req=MochiReq}, Path) ->
Host = case MochiReq:get_header_value("Host") of
undefined ->
{ok, {Address, Port}} = inet:sockname(MochiReq:get(socket)),
inet_parse:ntoa(Address) ++ ":" ++ integer_to_list(Port);
Value -> Value
end,
"http://" ++ Host ++ Path.

unquote(UrlEncodedString) ->
mochiweb_util:unquote(UrlEncodedString).

Expand Down
3 changes: 2 additions & 1 deletion src/couchdb/couch_httpd_db.erl
Expand Up @@ -261,7 +261,8 @@ db_req(#httpd{method='GET',mochi_req=MochiReq, path_parts=[DbName,<<"_design/",_
PathFront = "/" ++ couch_httpd:quote(binary_to_list(DbName)) ++ "/",
RawSplit = regexp:split(MochiReq:get(raw_path),"_design%2F"),
{ok, [PathFront|PathTail]} = RawSplit,
RedirectTo = PathFront ++ "_design/" ++ mochiweb_util:join(PathTail, "%2F"),
RedirectTo = couch_httpd:absolute_uri(Req, PathFront ++ "_design/" ++
mochiweb_util:join(PathTail, "%2F")),
couch_httpd:send_response(Req, 301, [{"Location", RedirectTo}], <<>>);

db_req(#httpd{path_parts=[_DbName,<<"_design">>,Name]}=Req, Db) ->
Expand Down
3 changes: 2 additions & 1 deletion src/couchdb/couch_httpd_misc_handlers.erl
Expand Up @@ -50,7 +50,8 @@ handle_utils_dir_req(#httpd{method='GET'}=Req, DocumentRoot) ->
couch_httpd:serve_file(Req, RelativePath, DocumentRoot);
{_ActionKey, "", _RelativePath} ->
% GET /_utils
couch_httpd:send_response(Req, 301, [{"Location", "/_utils/"}], <<>>)
Headers = [{"Location", couch_httpd:absolute_uri(Req, "/_utils/")}],
couch_httpd:send_response(Req, 301, Headers, <<>>)
end;
handle_utils_dir_req(Req, _) ->
send_method_not_allowed(Req, "GET,HEAD").
Expand Down
96 changes: 64 additions & 32 deletions src/couchdb/couch_rep.erl
Expand Up @@ -157,30 +157,54 @@ replicate2(Source, DbSrc, Target, DbTgt, Options) ->
end.

pull_rep(DbTarget, DbSource, SourceSeqNum) ->
http:set_options([{max_pipeline_length, 101}, {pipeline_timeout, 5000}]),
{ok, {NewSeq, Stats}} =
enum_docs_since(DbSource, DbTarget, SourceSeqNum, {SourceSeqNum, []}),
http:set_options([{max_pipeline_length, 2}, {pipeline_timeout, 0}]),
{NewSeq, Stats}.

do_http_request(Url, Action, Headers) ->
do_http_request(Url, Action, Headers, []).

do_http_request(Url, Action, Headers, JsonBody) ->
?LOG_DEBUG("couch_rep HTTP client request:", []),
?LOG_DEBUG("\tAction: ~p", [Action]),
?LOG_DEBUG("\tUrl: ~p", [Url]),
Request =
do_http_request(Url, Action, Headers, JsonBody, 10).

do_http_request(Url, Action, _Headers, _JsonBody, 0) ->
?LOG_ERROR("couch_rep HTTP ~p request failed after 10 retries: ~p",
[Action, Url]);
do_http_request(Url, Action, Headers, JsonBody, Retries) ->
?LOG_DEBUG("couch_rep HTTP ~p request: ~p", [Action, Url]),
Body =
case JsonBody of
[] ->
{Url, Headers};
<<>>;
_ ->
{Url, Headers, "application/json; charset=utf-8", iolist_to_binary(?JSON_ENCODE(JsonBody))}
iolist_to_binary(?JSON_ENCODE(JsonBody))
end,
{ok, {{_, ResponseCode,_},_Headers, ResponseBody}} = http:request(Action, Request, [], []),
if
ResponseCode >= 200, ResponseCode < 500 ->
?JSON_DECODE(ResponseBody)
Options = [
{content_type, "application/json; charset=utf-8"},
{max_pipeline_size, 101},
{transfer_encoding, {chunked, 65535}}
],
case ibrowse:send_req(Url, Headers, Action, Body, Options) of
{ok, Status, ResponseHeaders, ResponseBody} ->
ResponseCode = list_to_integer(Status),
if
ResponseCode >= 200, ResponseCode < 300 ->
?JSON_DECODE(ResponseBody);
ResponseCode >= 300, ResponseCode < 400 ->
RedirectUrl = mochiweb_headers:get_value("Location",
mochiweb_headers:make(ResponseHeaders)),
do_http_request(RedirectUrl, Action, Headers, JsonBody, Retries-1);
ResponseCode >= 400, ResponseCode < 500 ->
?JSON_DECODE(ResponseBody);
ResponseCode == 500 ->
?LOG_INFO("retrying couch_rep HTTP ~p request due to 500 error: ~p",
[Action, Url]),
do_http_request(Url, Action, Headers, JsonBody, Retries - 1)
end;
{error, Reason} ->
?LOG_INFO("retrying couch_rep HTTP ~p request due to {error, ~p}: ~p",
[Action, Reason, Url]),
do_http_request(Url, Action, Headers, JsonBody, Retries - 1)
end.

save_docs_buffer(DbTarget, DocsBuffer, []) ->
Expand Down Expand Up @@ -223,20 +247,17 @@ wait_result({Pid,Ref}) ->
{'DOWN', Ref, _, _, Reason} -> exit(Reason)
end.

enum_docs_parallel(DbS, DbT, DocInfoList) ->
UpdateSeqs = [D#doc_info.update_seq || D <- DocInfoList],
enum_docs_parallel(DbS, DbT, InfoList) ->
UpdateSeqs = [Seq || {_, Seq, _, _} <- InfoList],
SaveDocsPid = spawn_link(fun() -> save_docs_buffer(DbT,[],UpdateSeqs) end),

Stats = pmap(fun(SrcDocInfo) ->
#doc_info{id=Id,
rev=Rev,
conflict_revs=Conflicts,
deleted_conflict_revs=DelConflicts,
update_seq=Seq} = SrcDocInfo,
SrcRevs = [Rev | Conflicts] ++ DelConflicts,

case get_missing_revs(DbT, [{Id, SrcRevs}]) of
{ok, [{Id, MissingRevs}]} ->
Stats = pmap(fun({Id, Seq, SrcRevs, MissingRevs}) ->
case MissingRevs of
[] ->
SaveDocsPid ! {self(), skip, Seq},
receive got_it -> ok end,
[{missing_checked, length(SrcRevs)}];
_ ->
{ok, DocResults} = open_doc_revs(DbS, Id, MissingRevs, [latest]),

% only save successful reads
Expand All @@ -247,13 +268,9 @@ enum_docs_parallel(DbS, DbT, DocInfoList) ->
receive got_it -> ok end,
[{missing_checked, length(SrcRevs)},
{missing_found, length(MissingRevs)},
{docs_read, length(Docs)}];
{ok, []} ->
SaveDocsPid ! {self(), skip, Seq},
receive got_it -> ok end,
[{missing_checked, length(SrcRevs)}]
end
end, DocInfoList),
{docs_read, length(Docs)}]
end
end, InfoList),

SaveDocsPid ! {self(), shutdown},

Expand Down Expand Up @@ -345,7 +362,22 @@ enum_docs_since(DbSource, DbTarget, StartSeq, InAcc) ->
[] ->
{ok, InAcc};
_ ->
Stats = enum_docs_parallel(DbSource, DbTarget, DocInfoList),
UpdateSeqs = [D#doc_info.update_seq || D <- DocInfoList],
SrcRevsList = lists:map(fun(SrcDocInfo) ->
#doc_info{id=Id,
rev=Rev,
conflict_revs=Conflicts,
deleted_conflict_revs=DelConflicts
} = SrcDocInfo,
SrcRevs = [Rev | Conflicts] ++ DelConflicts,
{Id, SrcRevs}
end, DocInfoList),
{ok, MissingRevsList} = get_missing_revs(DbTarget, SrcRevsList),
InfoList = lists:map(fun({{Id, SrcRevs}, Seq}) ->
MissingRevs = proplists:get_value(Id, MissingRevsList, []),
{Id, Seq, SrcRevs, MissingRevs}
end, lists:zip(SrcRevsList, UpdateSeqs)),
Stats = enum_docs_parallel(DbSource, DbTarget, InfoList),
OldStats = element(2, InAcc),
TotalStats = [
{<<"missing_checked">>,
Expand Down
2 changes: 1 addition & 1 deletion src/couchdb/couch_server_sup.erl
Expand Up @@ -102,7 +102,7 @@ start_server(IniFiles) ->


% ensure these applications are running
application:start(inets),
application:start(ibrowse),
application:start(crypto),

{ok, Pid} = supervisor:start_link(
Expand Down
47 changes: 47 additions & 0 deletions src/ibrowse/Makefile.am
@@ -0,0 +1,47 @@
## Licensed under the Apache License, Version 2.0 (the "License"); you may not
## use this file except in compliance with the License. You may obtain a copy
## of the License at
##
## http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
## License for the specific language governing permissions and limitations under
## the License.

ibrowseebindir = $(localerlanglibdir)/ibrowse-1.4.1/ebin

ibrowse_file_collection = \
ibrowse.erl \
ibrowse_app.erl \
ibrowse_http_client.erl \
ibrowse_lb.erl \
ibrowse_lib.erl \
ibrowse_sup.erl \
ibrowse_test.erl

ibrowseebin_static_file = ibrowse.app

ibrowseebin_make_generated_file_list = \
ibrowse.beam \
ibrowse_app.beam \
ibrowse_http_client.beam \
ibrowse_lb.beam \
ibrowse_lib.beam \
ibrowse_sup.beam \
ibrowse_test.beam

ibrowseebin_DATA = \
$(ibrowseebin_static_file) \
$(ibrowseebin_make_generated_file_list)

EXTRA_DIST = \
$(ibrowse_file_collection) \
$(ibrowseebin_static_file)

CLEANFILES = \
$(ibrowseebin_make_generated_file_list)

%.beam: %.erl
$(ERLC) $<
13 changes: 13 additions & 0 deletions src/ibrowse/ibrowse.app
@@ -0,0 +1,13 @@
{application, ibrowse,
[{description, "HTTP client application"},
{vsn, "1.4.1"},
{modules, [ ibrowse,
ibrowse_http_client,
ibrowse_app,
ibrowse_sup,
ibrowse_lib,
ibrowse_lb ]},
{registered, []},
{applications, [kernel,stdlib,sasl]},
{env, []},
{mod, {ibrowse_app, []}}]}.

0 comments on commit ace6dfe

Please sign in to comment.