COUCHDB-430,514,764 Fix list HTTP header handling.

Currently calls to getRow() cause the HTTP headers to be sent immediately back
to the client. This happens even if an error is thrown after the getRow(), but
before any send(...) or start(...). Worse, if a list throws an exception an
extra, invalid header is sent to the client (resulting in various bad
behavior).

Erlang list handling will now wait until data has been sent BEFORE sending the
HTTP headers to the client. If an error is reported it will result in an HTTP
error code as expected. This does not change the behavior of errors thrown
AFTER data has been sent: They will still result in an HTTP 200 even if an
error is reported.

The line protocol between Erlang and os processes has been extended to support
an optional Header field on "chunks" and "end". The javascript list handling
has been updated to use this if a new header is set via start(...). This makes
it possible to begin processing with getRow(), but later reset the headers via
start(...). Again, if data has been sent(...) the new headers will NOT take
effect.

COUCHDB-430
COUCHDB-514
COUCHDB-764

CHANGES

@@ -35,6 +35,8 @@ Storage System:
 View Server:
 
  * Speedup in the communication with external view servers.
+ * Additional response headers may be varied prior to send().
+ * GetRow() is now side-effect free.
 
 Futon:
 

share/server/render.js

@@ -133,20 +133,23 @@ var Mime = (function() {
 ////
 
 var Render = (function() {
+  var new_header = false;
   var chunks = [];
 
 
   //  Start chunks
   var startResp = {};
   function start(resp) {
     startResp = resp || {};
+    new_header = true;
   };
 
   function sendStart() {
     startResp = applyContentType((startResp || {}), Mime.responseContentType);
     respond(["start", chunks, startResp]);
     chunks = [];
     startResp = {};
+    new_header = false;
   }
 
   function applyContentType(resp, responseContentType) {
@@ -162,7 +165,13 @@ var Render = (function() {
   };
 
   function blowChunks(label) {
-    respond([label||"chunks", chunks]);
+    if (new_header) {
+      respond([label||"chunks", chunks, startResp]);
+      new_header = false;
+    }
+    else {
+      respond([label||"chunks", chunks]);
+    }
     chunks = [];
   };
 
@@ -281,6 +290,7 @@ var Render = (function() {
     lastRow = false;
     chunks = [];
     startResp = {};
+    new_header = false;
   };
 
   function runList(listFun, ddoc, args) {

share/www/script/test/list_views.js

@@ -159,7 +159,17 @@ couchTests.list_views = function(debug) {
       }),
       secObj: stringFun(function(head, req) {
         return toJSON(req.secObj);
-      })
+      }),
+      setHeaderAfterGotRow: stringFun(function(head, req) {
+        getRow();
+        start({
+          code: 400,
+          headers: {
+            "X-My-Header": "MyHeader"
+          }
+        });
+        send("bad request");
+      }),
     }
   };
   var viewOnlyDesignDoc = {
@@ -476,4 +486,10 @@ couchTests.list_views = function(debug) {
     }
   });
   TEquals(200, resp.status, "should return a 200 response");
+
+  // TEST HTTP header response set after getRow() called in _list function.
+  var xhr = CouchDB.request("GET", "/test_suite_db/_design/lists/_list/setHeaderAfterGotRow/basicView");
+  T(xhr.status == 400);
+  T(xhr.getResponseHeader("X-My-Header") == "MyHeader");
+  T(xhr.responseText.match(/^bad request$/));
 };

src/couch_mrview/src/couch_mrview_show.erl

@@ -27,7 +27,9 @@
     resp,
     qserver,
     lname,
-    etag
+    etag,
+    code,
+    headers
 }).
 
 % /db/_design/foo/_show/bar/docid
@@ -213,7 +215,7 @@ handle_view_list(Req, Db, DDoc, LName, VDDoc, VName, Keys) ->
     end).
 
 
-list_cb({meta, Meta}, #lacc{resp=undefined} = Acc) ->
+list_cb({meta, Meta}, #lacc{code=undefined} = Acc) ->
     MetaProps = case couch_util:get_value(total, Meta) of
         undefined -> [];
         Total -> [{total_rows, Total}]
@@ -225,7 +227,7 @@ list_cb({meta, Meta}, #lacc{resp=undefined} = Acc) ->
         UpdateSeq -> [{update_seq, UpdateSeq}]
     end,
     start_list_resp({MetaProps}, Acc);
-list_cb({row, Row}, #lacc{resp=undefined} = Acc) ->
+list_cb({row, Row}, #lacc{code=undefined} = Acc) ->
     {ok, NewAcc} = start_list_resp({[]}, Acc),
     send_list_row(Row, NewAcc);
 list_cb({row, Row}, Acc) ->
@@ -237,27 +239,34 @@ list_cb(complete, Acc) ->
     true ->
         Resp = Resp0
     end,
-    [<<"end">>, Data] = couch_query_servers:proc_prompt(Proc, [<<"list_end">>]),
-    send_non_empty_chunk(Resp, Data),
-    couch_httpd:last_chunk(Resp),
-    {ok, Resp}.
+    case couch_query_servers:proc_prompt(Proc, [<<"list_end">>]) of
+        [<<"end">>, Data, Headers] ->
+            Acc2 = fixup_headers(Headers, Acc#lacc{resp=Resp}),
+            #lacc{resp = Resp2} = send_non_empty_chunk(Acc2, Data);
+        [<<"end">>, Data] ->
+            #lacc{resp = Resp2} = send_non_empty_chunk(Acc#lacc{resp=Resp}, Data)
+    end,
+    couch_httpd:last_chunk(Resp2),
+    {ok, Resp2}.
 
 start_list_resp(Head, Acc) ->
     #lacc{db=Db, req=Req, qserver=QServer, lname=LName, etag=ETag} = Acc,
     JsonReq = couch_httpd_external:json_req_obj(Req, Db),
 
     [<<"start">>,Chunk,JsonResp] = couch_query_servers:ddoc_proc_prompt(QServer,
         [<<"lists">>, LName], [Head, JsonReq]),
-    JsonResp2 = apply_etag(JsonResp, ETag),
+    Acc2 = send_non_empty_chunk(fixup_headers(JsonResp, Acc), Chunk),
+    {ok, Acc2}.
+
+fixup_headers(Headers, #lacc{etag=ETag} = Acc) ->
+    Headers2 = apply_etag(Headers, ETag),
     #extern_resp_args{
         code = Code,
         ctype = CType,
         headers = ExtHeaders
-    } = couch_httpd_external:parse_external_response(JsonResp2),
-    JsonHeaders = couch_httpd_external:default_or_content_type(CType, ExtHeaders),
-    {ok, Resp} = couch_httpd:start_chunked_response(Req, Code, JsonHeaders),
-    send_non_empty_chunk(Resp, Chunk),
-    {ok, Acc#lacc{resp=Resp}}.
+    } = couch_httpd_external:parse_external_response(Headers2),
+    Headers3 = couch_httpd_external:default_or_content_type(CType, ExtHeaders),
+    Acc#lacc{code=Code, headers=Headers3}.
 
 send_list_row(Row, #lacc{qserver = {Proc, _}, resp = Resp} = Acc) ->
     RowObj = case couch_util:get_value(id, Row) of
@@ -274,22 +283,44 @@ send_list_row(Row, #lacc{qserver = {Proc, _}, resp = Resp} = Acc) ->
         Doc -> [{doc, Doc}]
     end,
     try couch_query_servers:proc_prompt(Proc, [<<"list_row">>, {RowObj}]) of
+    [<<"chunks">>, Chunk, Headers] ->
+        Acc2 = send_non_empty_chunk(fixup_headers(Headers, Acc), Chunk),
+        {ok, Acc2};
     [<<"chunks">>, Chunk] ->
-        send_non_empty_chunk(Resp, Chunk),
-        {ok, Acc};
+        Acc2 = send_non_empty_chunk(Acc, Chunk),
+        {ok, Acc2};
+    [<<"end">>, Chunk, Headers] ->
+        Acc2 = send_non_empty_chunk(fixup_headers(Headers, Acc), Chunk),
+        #lacc{resp = Resp2} = Acc2,
+        couch_httpd:last_chunk(Resp2),
+        {stop, Acc2};
     [<<"end">>, Chunk] ->
-        send_non_empty_chunk(Resp, Chunk),
-        couch_httpd:last_chunk(Resp),
-        {stop, Acc}
+        Acc2 = send_non_empty_chunk(Acc, Chunk),
+        #lacc{resp = Resp2} = Acc2,
+        couch_httpd:last_chunk(Resp2),
+        {stop, Acc2}
     catch Error ->
-        couch_httpd:send_chunked_error(Resp, Error),
-        {stop, Acc}
+        case Resp of
+            undefined ->
+                {Code, _, _} = couch_httpd:error_info(Error),
+                #lacc{req=Req, headers=Headers} = Acc,
+                {ok, Resp2} = couch_httpd:start_chunked_response(Req, Code, Headers),
+                Acc2 = Acc#lacc{resp=Resp2, code=Code};
+            _ -> Resp2 = Resp, Acc2 = Acc
+        end,
+        couch_httpd:send_chunked_error(Resp2, Error),
+        {stop, Acc2}
     end.
 
-send_non_empty_chunk(_, []) ->
-    ok;
-send_non_empty_chunk(Resp, Chunk) ->
-    couch_httpd:send_chunk(Resp, Chunk).
+send_non_empty_chunk(Acc, []) ->
+    Acc;
+send_non_empty_chunk(#lacc{resp=undefined} = Acc, Chunk) ->
+    #lacc{req=Req, code=Code, headers=Headers} = Acc,
+    {ok, Resp} = couch_httpd:start_chunked_response(Req, Code, Headers),
+    send_non_empty_chunk(Acc#lacc{resp = Resp}, Chunk);
+send_non_empty_chunk(#lacc{resp=Resp} = Acc, Chunk) ->
+    couch_httpd:send_chunk(Resp, Chunk),
+    Acc.
 
 
 apply_etag({ExternalResponse}, CurrentEtag) ->