http: respond to errors immediately rather than scheduling

Responses are usually added to the response queue by worker threads. As an
optimization, and in order to close connections as quickly as possible, allow
for immediate replies (bypassing the queue) when there is no need for a worker
to service the request.

src/httpserver.cpp

@@ -240,30 +240,20 @@ static std::string RequestMethodString(HTTPRequest::RequestMethod m)
 /** HTTP request callback */
 static void http_request_cb(struct evhttp_request* req, void* arg)
 {
-    // Disable reading to work around a libevent bug, fixed in 2.2.0.
-    if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02020001) {
-        evhttp_connection* conn = evhttp_request_get_connection(req);
-        if (conn) {
-            bufferevent* bev = evhttp_connection_get_bufferevent(conn);
-            if (bev) {
-                bufferevent_disable(bev, EV_READ);
-            }
-        }
-    }
     std::unique_ptr<HTTPRequest> hreq(new HTTPRequest(req));
 
     LogPrint(BCLog::HTTP, "Received a %s request for %s from %s\n",
              RequestMethodString(hreq->GetRequestMethod()), hreq->GetURI(), hreq->GetPeer().ToString());
 
     // Early address-based allow check
     if (!ClientAllowed(hreq->GetPeer())) {
-        hreq->WriteReply(HTTP_FORBIDDEN);
+        hreq->WriteReplyImmediate(HTTP_FORBIDDEN);
         return;
     }
 
     // Early reject unknown HTTP methods
     if (hreq->GetRequestMethod() == HTTPRequest::UNKNOWN) {
-        hreq->WriteReply(HTTP_BADMETHOD);
+        hreq->WriteReplyImmediate(HTTP_BADMETHOD);
         return;
     }
 
@@ -288,14 +278,24 @@ static void http_request_cb(struct evhttp_request* req, void* arg)
     if (i != iend) {
         std::unique_ptr<HTTPWorkItem> item(new HTTPWorkItem(std::move(hreq), path, i->handler));
         assert(workQueue);
-        if (workQueue->Enqueue(item.get()))
+        if (workQueue->Enqueue(item.get())) {
+            // Disable reading to work around a libevent bug, fixed in 2.2.0.
+            if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02020001) {
+                evhttp_connection* conn = evhttp_request_get_connection(req);
+                if (conn) {
+                    bufferevent* bev = evhttp_connection_get_bufferevent(conn);
+                    if (bev) {
+                        bufferevent_disable(bev, EV_READ);
+                    }
+                }
+            }
             item.release(); /* if true, queue took ownership */
-        else {
+        } else {
             LogPrintf("WARNING: request rejected because http work queue depth exceeded, it can be increased with the -rpcworkqueue= setting\n");
-            item->req->WriteReply(HTTP_INTERNAL, "Work queue depth exceeded");
+            item->req->WriteReplyImmediate(HTTP_INTERNAL, "Work queue depth exceeded");
         }
     } else {
-        hreq->WriteReply(HTTP_NOTFOUND);
+        hreq->WriteReplyImmediate(HTTP_NOTFOUND);
     }
 }
 
@@ -632,6 +632,17 @@ void HTTPRequest::WriteReply(int nStatus, const std::string& strReply)
     req = nullptr; // transferred back to main thread
 }
 
+void HTTPRequest::WriteReplyImmediate(int nStatus, const std::string& strReply)
+{
+    assert(!replySent && req);
+    struct evbuffer* evb = evhttp_request_get_output_buffer(req);
+    assert(evb);
+    evbuffer_add(evb, strReply.data(), strReply.size());
+    evhttp_send_reply(req, nStatus, nullptr, nullptr);
+    replySent = true;
+    req = nullptr;
+}
+
 CService HTTPRequest::GetPeer()
 {
     evhttp_connection* con = evhttp_request_get_connection(req);

src/httpserver.h

@@ -114,6 +114,15 @@ class HTTPRequest
      * main thread, do not call any other HTTPRequest methods after calling this.
      */
     void WriteReply(int nStatus, const std::string& strReply = "");
+
+    /**
+     * Write HTTP reply from the callback thread
+     *
+     * @note Behavior is exactly the same as WriteReply, except that the send queue
+     * is bypassed. This should _only_ be called from inside the request
+     * callback, the from any other thread is undefined.
+     */
+    void WriteReplyImmediate(int nStatus, const std::string& strReply = "");
 };
 
 /** Event handler closure.