src/ibrowse/Makefile.am src/ibrowse/ibrowse.app src/ibrowse/ibrowse.erl src/ibrowse/ibrowse.hrl src/ibrowse/ibrowse_app.erl src/ibrowse/ibrowse_http_client.erl src/ibrowse/ibrowse_lb.erl src/ibrowse/ibrowse_lib.erl src/ibrowse/ibrowse_sup.erl src/ibrowse/ibrowse_test.erl @@ -28,6 +28,8 @@ Database Core: dramatically. The fix keeps only one document in the write queue at a time. * Fix for databases sometimes incorrectly reporting that they contain 0 documents after compaction. + * CouchDB now uses ibrowse instead of inets for its internal HTTP client + implementation. This means better replication stability. HTTP Interface: CHANGES @@ -10,7 +10,7 @@ ## License for the specific language governing permissions and limitations ## under the License. -SUBDIRS = bin etc src/couchdb src/mochiweb share test var utils +SUBDIRS = bin etc src/couchdb src/ibrowse src/mochiweb share test var utils localdoc_DATA = AUTHORS.gz BUGS.gz CHANGES.gz NEWS.gz README.gz THANKS.gz Makefile.am @@ -21,3 +21,9 @@ This product also includes the following third-party components: * MochiWeb (http://code.google.com/p/mochiweb/) Copyright 2007, Mochi Media Coporation + + * ibrowse + (http://jungerl.cvs.sourceforge.net/viewvc/jungerl/jungerl/lib/ibrowse/) + + Copyright 2008, Chandrashekhar Mullaparthi + This ASF redistribution is consistent with the terms of the BSD License. \ No newline at end of file NOTICE @@ -12,7 +12,9 @@ Some of these people are: * Yoan Blanc <yoan.blanc@gmail.com> * Paul Carey <paul.p.carey@gmail.com> * Benoit Chesneau <bchesneau@gmail.com> + * Jason Davies <jason@jasondavies.com> * Paul Joseph Davis <paul.joseph.davis@gmail.com> + * Maximillian Dornseif <md@hudora.de> * Michael Gottesman <gottesmm@reed.edu> * Michael Hendricks <michael@ndrix.org> * Till Klampaeckel <till@klampaeckel.de> THANKS @@ -28,8 +28,9 @@ couchdb: couchdb.tpl -e "s|%ICU_CONFIG%|$(ICU_CONFIG)|g" \ -e "s|%bindir%|@bindir@|g" \ -e "s|%localerlanglibdir%|@localerlanglibdir@|g" \ - -e "s|%mochiwebebindir%|couch-@version@/ebin|g" \ - -e "s|%couchdbebindir%|mochiweb-r82/ebin|g" \ + -e "s|%couchdbebindir%|couch-@version@/ebin|g" \ + -e "s|%mochiwebebindir%|mochiweb-r82/ebin|g" \ + -e "s|%ibrowseebindir%|ibrowse-1.4.1/ebin|g" \ -e "s|%defaultini%|default.ini|g" \ -e "s|%localini%|local.ini|g" \ -e "s|%localconfdir%|@localconfdir@|g" \ bin/Makefile.am @@ -183,11 +183,12 @@ start_couchdb () { %ERL% $interactive_option -smp auto -sasl errlog_type error +K true \ -pa %localerlanglibdir%/%couchdbebindir% \ %localerlanglibdir%/%mochiwebebindir% \ - -eval \"application:load(inets)\" \ + %localerlanglibdir%/%ibrowseebindir% \ + -eval \"application:load(ibrowse)\" \ -eval \"application:load(crypto)\" \ -eval \"application:load(couch)\" \ -eval \"crypto:start()\" \ - -eval \"inets:start()\" \ + -eval \"ibrowse:start()\" \ -eval \"couch_server:start([$start_arguments]), receive done -> done end.\" " if test "$BACKGROUND_BOOLEAN" = "true" \ -a "$RECURSED_BOOLEAN" = "false"; then bin/couchdb.tpl.in @@ -267,6 +267,7 @@ AC_CONFIG_FILES([etc/Makefile]) AC_CONFIG_FILES([share/Makefile]) AC_CONFIG_FILES([src/couchdb/couch.app.tpl]) AC_CONFIG_FILES([src/couchdb/Makefile]) +AC_CONFIG_FILES([src/ibrowse/Makefile]) AC_CONFIG_FILES([src/mochiweb/Makefile]) AC_CONFIG_FILES([test/Makefile]) AC_CONFIG_FILES([utils/Makefile]) configure.ac @@ -2134,6 +2134,19 @@ var tests = { T(docA._rev == docB._rev); }; }, + + design_docs_test: new function() { + // make sure design docs replicate properly + this.init = function(dbA, dbB) { + dbA.save({ _id:"_design/test" }); + }; + + this.afterAB1 = function() { + var docA = dbA.open("_design/test"); + var docB = dbB.open("_design/test"); + T(docA._rev == docB._rev); + }; + }, attachments_test: new function () { // Test attachments share/www/script/couch_tests.js @@ -24,4 +24,4 @@ couch_view, couch_query_servers, couch_db_update_notifier_sup]}, - {applications,[kernel,stdlib,crypto,inets,mochiweb]}]}. + {applications,[kernel,stdlib,crypto,ibrowse,mochiweb]}]}. src/couchdb/couch.app.tpl.in @@ -15,7 +15,7 @@ -export([start_link/0, stop/0, handle_request/3]). --export([header_value/2,header_value/3,qs_value/2,qs_value/3,qs/1,path/1]). +-export([header_value/2,header_value/3,qs_value/2,qs_value/3,qs/1,path/1,absolute_uri/2]). -export([verify_is_server_admin/1,unquote/1,quote/1,recv/2]). -export([parse_form/1,json_body/1,body/1,doc_etag/1, make_etag/1, etag_respond/3]). -export([primary_header_value/2,partition/1,serve_file/3]). @@ -242,6 +242,15 @@ qs(#httpd{mochi_req=MochiReq}) -> path(#httpd{mochi_req=MochiReq}) -> MochiReq:get(path). +absolute_uri(#httpd{mochi_req=MochiReq}, Path) -> + Host = case MochiReq:get_header_value("Host") of + undefined -> + {ok, {Address, Port}} = inet:sockname(MochiReq:get(socket)), + inet_parse:ntoa(Address) ++ ":" ++ integer_to_list(Port); + Value -> Value + end, + "http://" ++ Host ++ Path. + unquote(UrlEncodedString) -> mochiweb_util:unquote(UrlEncodedString). src/couchdb/couch_httpd.erl @@ -261,7 +261,8 @@ db_req(#httpd{method='GET',mochi_req=MochiReq, path_parts=[DbName,<<"_design/",_ PathFront = "/" ++ couch_httpd:quote(binary_to_list(DbName)) ++ "/", RawSplit = regexp:split(MochiReq:get(raw_path),"_design%2F"), {ok, [PathFront|PathTail]} = RawSplit, - RedirectTo = PathFront ++ "_design/" ++ mochiweb_util:join(PathTail, "%2F"), + RedirectTo = couch_httpd:absolute_uri(Req, PathFront ++ "_design/" ++ + mochiweb_util:join(PathTail, "%2F")), couch_httpd:send_response(Req, 301, [{"Location", RedirectTo}], <<>>); db_req(#httpd{path_parts=[_DbName,<<"_design">>,Name]}=Req, Db) -> src/couchdb/couch_httpd_db.erl @@ -50,7 +50,8 @@ handle_utils_dir_req(#httpd{method='GET'}=Req, DocumentRoot) -> couch_httpd:serve_file(Req, RelativePath, DocumentRoot); {_ActionKey, "", _RelativePath} -> % GET /_utils - couch_httpd:send_response(Req, 301, [{"Location", "/_utils/"}], <<>>) + Headers = [{"Location", couch_httpd:absolute_uri(Req, "/_utils/")}], + couch_httpd:send_response(Req, 301, Headers, <<>>) end; handle_utils_dir_req(Req, _) -> send_method_not_allowed(Req, "GET,HEAD"). src/couchdb/couch_httpd_misc_handlers.erl @@ -157,30 +157,54 @@ replicate2(Source, DbSrc, Target, DbTgt, Options) -> end. pull_rep(DbTarget, DbSource, SourceSeqNum) -> - http:set_options([{max_pipeline_length, 101}, {pipeline_timeout, 5000}]), {ok, {NewSeq, Stats}} = enum_docs_since(DbSource, DbTarget, SourceSeqNum, {SourceSeqNum, []}), - http:set_options([{max_pipeline_length, 2}, {pipeline_timeout, 0}]), {NewSeq, Stats}. do_http_request(Url, Action, Headers) -> do_http_request(Url, Action, Headers, []). do_http_request(Url, Action, Headers, JsonBody) -> - ?LOG_DEBUG("couch_rep HTTP client request:", []), - ?LOG_DEBUG("\tAction: ~p", [Action]), - ?LOG_DEBUG("\tUrl: ~p", [Url]), - Request = + do_http_request(Url, Action, Headers, JsonBody, 10). + +do_http_request(Url, Action, _Headers, _JsonBody, 0) -> + ?LOG_ERROR("couch_rep HTTP ~p request failed after 10 retries: ~p", + [Action, Url]); +do_http_request(Url, Action, Headers, JsonBody, Retries) -> + ?LOG_DEBUG("couch_rep HTTP ~p request: ~p", [Action, Url]), + Body = case JsonBody of [] -> - {Url, Headers}; + <<>>; _ -> - {Url, Headers, "application/json; charset=utf-8", iolist_to_binary(?JSON_ENCODE(JsonBody))} + iolist_to_binary(?JSON_ENCODE(JsonBody)) end, - {ok, {{_, ResponseCode,_},_Headers, ResponseBody}} = http:request(Action, Request, [], []), - if - ResponseCode >= 200, ResponseCode < 500 -> - ?JSON_DECODE(ResponseBody) + Options = [ + {content_type, "application/json; charset=utf-8"}, + {max_pipeline_size, 101}, + {transfer_encoding, {chunked, 65535}} + ], + case ibrowse:send_req(Url, Headers, Action, Body, Options) of + {ok, Status, ResponseHeaders, ResponseBody} -> + ResponseCode = list_to_integer(Status), + if + ResponseCode >= 200, ResponseCode < 300 -> + ?JSON_DECODE(ResponseBody); + ResponseCode >= 300, ResponseCode < 400 -> + RedirectUrl = mochiweb_headers:get_value("Location", + mochiweb_headers:make(ResponseHeaders)), + do_http_request(RedirectUrl, Action, Headers, JsonBody, Retries-1); + ResponseCode >= 400, ResponseCode < 500 -> + ?JSON_DECODE(ResponseBody); + ResponseCode == 500 -> + ?LOG_INFO("retrying couch_rep HTTP ~p request due to 500 error: ~p", + [Action, Url]), + do_http_request(Url, Action, Headers, JsonBody, Retries - 1) + end; + {error, Reason} -> + ?LOG_INFO("retrying couch_rep HTTP ~p request due to {error, ~p}: ~p", + [Action, Reason, Url]), + do_http_request(Url, Action, Headers, JsonBody, Retries - 1) end. save_docs_buffer(DbTarget, DocsBuffer, []) -> @@ -223,20 +247,17 @@ wait_result({Pid,Ref}) -> {'DOWN', Ref, _, _, Reason} -> exit(Reason) end. -enum_docs_parallel(DbS, DbT, DocInfoList) -> - UpdateSeqs = [D#doc_info.update_seq || D <- DocInfoList], +enum_docs_parallel(DbS, DbT, InfoList) -> + UpdateSeqs = [Seq || {_, Seq, _, _} <- InfoList], SaveDocsPid = spawn_link(fun() -> save_docs_buffer(DbT,[],UpdateSeqs) end), - Stats = pmap(fun(SrcDocInfo) -> - #doc_info{id=Id, - rev=Rev, - conflict_revs=Conflicts, - deleted_conflict_revs=DelConflicts, - update_seq=Seq} = SrcDocInfo, - SrcRevs = [Rev | Conflicts] ++ DelConflicts, - - case get_missing_revs(DbT, [{Id, SrcRevs}]) of - {ok, [{Id, MissingRevs}]} -> + Stats = pmap(fun({Id, Seq, SrcRevs, MissingRevs}) -> + case MissingRevs of + [] -> + SaveDocsPid ! {self(), skip, Seq}, + receive got_it -> ok end, + [{missing_checked, length(SrcRevs)}]; + _ -> {ok, DocResults} = open_doc_revs(DbS, Id, MissingRevs, [latest]), % only save successful reads @@ -247,13 +268,9 @@ enum_docs_parallel(DbS, DbT, DocInfoList) -> receive got_it -> ok end, [{missing_checked, length(SrcRevs)}, {missing_found, length(MissingRevs)}, - {docs_read, length(Docs)}]; - {ok, []} -> - SaveDocsPid ! {self(), skip, Seq}, - receive got_it -> ok end, - [{missing_checked, length(SrcRevs)}] - end - end, DocInfoList), + {docs_read, length(Docs)}] + end + end, InfoList), SaveDocsPid ! {self(), shutdown}, @@ -345,7 +362,22 @@ enum_docs_since(DbSource, DbTarget, StartSeq, InAcc) -> [] -> {ok, InAcc}; _ -> - Stats = enum_docs_parallel(DbSource, DbTarget, DocInfoList), + UpdateSeqs = [D#doc_info.update_seq || D <- DocInfoList], + SrcRevsList = lists:map(fun(SrcDocInfo) -> + #doc_info{id=Id, + rev=Rev, + conflict_revs=Conflicts, + deleted_conflict_revs=DelConflicts + } = SrcDocInfo, + SrcRevs = [Rev | Conflicts] ++ DelConflicts, + {Id, SrcRevs} + end, DocInfoList), + {ok, MissingRevsList} = get_missing_revs(DbTarget, SrcRevsList), + InfoList = lists:map(fun({{Id, SrcRevs}, Seq}) -> + MissingRevs = proplists:get_value(Id, MissingRevsList, []), + {Id, Seq, SrcRevs, MissingRevs} + end, lists:zip(SrcRevsList, UpdateSeqs)), + Stats = enum_docs_parallel(DbSource, DbTarget, InfoList), OldStats = element(2, InAcc), TotalStats = [ {<<"missing_checked">>, src/couchdb/couch_rep.erl @@ -102,7 +102,7 @@ start_server(IniFiles) -> % ensure these applications are running - application:start(inets), + application:start(ibrowse), application:start(crypto), {ok, Pid} = supervisor:start_link( src/couchdb/couch_server_sup.erl @@ -24,6 +24,7 @@ run: ../bin/couchdb.tpl -e "s|%localerlanglibdir%|$(abs_top_srcdir)/src|g" \ -e "s|%mochiwebebindir%|mochiweb|g" \ -e "s|%couchdbebindir%|couchdb|g" \ + -e "s|%ibrowseebindir%|ibrowse|g" \ -e "s|%defaultini%|default_dev.ini|g" \ -e "s|%localini%|local_dev.ini|g" \ -e "s|%localerlanglibdir%|$(abs_top_srcdir)/src|g" \ utils/Makefile.am 28a4dab3fce36fe47b86e4b1e2efc7cb92b21c0b jchris jchris@13f79535-47bb-0310-9956-ffa450edef68 http://github.com/halorgium/couchdb/commit/3d89be3a57ee0e4951558f5816b25628bd5ed73e 3d89be3a57ee0e4951558f5816b25628bd5ed73e 2009-01-29T14:15:48-08:00 2009-01-29T14:15:48-08:00 Replacement of inets with ibrowse. Fixes COUCHDB-179 and enhances replication. Thanks Jason Davies and Adam Kocoloski for the fix, Maximillian Dornseif for reporting. git-svn-id: http://svn.apache.org/repos/asf/couchdb/trunk@739047 13f79535-47bb-0310-9956-ffa450edef68 bbcaedf64e20a580214f1b1f74cd40473073279c jchris jchris@13f79535-47bb-0310-9956-ffa450edef68