mpm_event,mod_http2: Keep compatibility with CONN_STATE_PROCESSING + OK

Before r1918022, returning OK with CONN_STATE_PROCESSING to mpm_event was
handled like/by CONN_STATE_LINGER "to not break old or third-party modules
which might return OK w/o touching the state and expect lingering close,
like with worker or prefork MPMs".

So we need a new return code to be allowed to apply the new POLLIN/POLLOUT
behaviour for CONN_STATE_PROCESSING, thus revive AGAIN as introduced by
Graham some times ago for a nonblocking WIP (moved to a branch/PR since then).

MPM event will advertise its ability to handle CONN_STATE_PROCESSING + AGAIN
with AP_MPMQ_CAN_AGAIN, and mod_http2 can use that to know how to return to
the MPM as expected. When !AP_MPMQ_CAN_AGAIN modules/mod_http2 can still use
CONN_STATE_WRITE_COMPLETION + CONN_SENSE_WANT_READ + c->clogging_input_filters
which will work in mpm_even-2.4.x still.

* include/ap_mmn.h:
  Bump MMN minor for AP_MPMQ_CAN_AGAIN and AGAIN.

* include/ap_mpm.h:
  Define AP_MPMQ_CAN_AGAIN.

* include/httpd.h:
  Define AGAIN.

* modules/http2/h2.h:
  No need for H2_USE_STATE_PROCESSING anymore with AP_MPMQ_CAN_AGAIN.

* modules/http2/h2_c1.c:
  For !keepalive case return to the MPM using CONN_STATE_PROCESSING + AGAIN
  or CONN_STATE_WRITE_COMPLETION + c->clogging_input_filters depending on
  AP_MPMQ_CAN_AGAIN only.

* modules/http2/h2_session.c:
  Can return to the MPM for h2_send_flow_blocked() provided it's async only.

* server/mpm/event/event.c:
  Rework process_socket()'s CONN_STATE_PROCESSING to handle AGAIN and preserve
  compatibility. Have a lingering_close label to goto there faster when
  process_lingering_close() is to be called. Improve relevant comments.



git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1918257 13f79535-47bb-0310-9956-ffa450edef68

include/ap_mmn.h

@@ -725,14 +725,15 @@
  * 20211221.19 (2.5.1-dev) Add AP_REG_NOTEMPTY_ATSTART
  * 20211221.20 (2.5.1-dev) Add CONN_STATE_KEEPALIVE and CONN_STATE_PROCESSING
  * 20211221.21 (2.5.1-dev) Add processing field struct process_score
+ * 20211221.22 (2.5.1-dev) Add AGAIN, AP_MPMQ_CAN_AGAIN.
  */
 
 #define MODULE_MAGIC_COOKIE 0x41503235UL /* "AP25" */
 
 #ifndef MODULE_MAGIC_NUMBER_MAJOR
 #define MODULE_MAGIC_NUMBER_MAJOR 20211221
 #endif
-#define MODULE_MAGIC_NUMBER_MINOR 21             /* 0...n */
+#define MODULE_MAGIC_NUMBER_MINOR 22             /* 0...n */
 
 /**
  * Determine if the server's current MODULE_MAGIC_NUMBER is at least a

include/ap_mpm.h

@@ -182,6 +182,8 @@ AP_DECLARE(apr_status_t) ap_os_create_privileged_process(
 #define AP_MPMQ_CAN_SUSPEND          17
 /** MPM supports additional pollfds */
 #define AP_MPMQ_CAN_POLL             18
+/** MPM reacts to AGAIN response */
+#define AP_MPMQ_CAN_AGAIN            19
 /** @} */
 
 /**

include/httpd.h

@@ -458,13 +458,16 @@ AP_DECLARE(const char *) ap_get_server_built(void);
 
 /* non-HTTP status codes returned by hooks */
 
-#define OK 0                    /**< Module has handled this stage. */
-#define DECLINED -1             /**< Module declines to handle */
-#define DONE -2                 /**< Module has served the response completely
-                                 *  - it's safe to die() with no more output
-                                 */
-#define SUSPENDED -3 /**< Module will handle the remainder of the request.
-                      * The core will never invoke the request again, */
+#define OK           0  /**< Module has handled this stage. */
+#define DECLINED    -1  /**< Module declines to handle */
+#define DONE        -2  /**< Module has served the response completely
+                         *   - it's safe to die() with no more output
+                         */
+#define SUSPENDED   -3  /**< Module will handle the remainder of the request.
+                         *   The core will never invoke the request again */
+#define AGAIN       -4  /**< Module wants to be called again when more
+                         *   data is available.
+                         */
 
 /** Returned by the bottom-most filter if no data was written.
  *  @see ap_pass_brigade(). */
@@ -1319,8 +1322,11 @@ struct conn_slave_rec {
  */
 typedef enum  {
     CONN_STATE_KEEPALIVE,           /* Kept alive in the MPM (using KeepAliveTimeout) */
-    CONN_STATE_PROCESSING,          /* Handled by process_connection() hooks, may be returned
-                                       to the MPM for POLLIN/POLLOUT (using Timeout) */
+    CONN_STATE_PROCESSING,          /* Processed by process_connection() hooks, returning
+                                     * AGAIN to the MPM in this state will make it wait for
+                                     * the connection to be readable or writable according to
+                                     * CONN_SENSE_WANT_READ or CONN_SENSE_WANT_WRITE respectively,
+                                     * where Timeout applies */
     CONN_STATE_HANDLER,             /* Processed by the modules handlers */
     CONN_STATE_WRITE_COMPLETION,    /* Flushed by the MPM before entering CONN_STATE_KEEPALIVE */
     CONN_STATE_SUSPENDED,           /* Suspended in the MPM until ap_run_resume_suspended() */

modules/http2/h2.h

@@ -49,12 +49,6 @@ struct h2_stream;
 #define H2_USE_WEBSOCKETS       0
 #endif
 
-#if AP_MODULE_MAGIC_AT_LEAST(20211221, 20)
-#define H2_USE_STATE_PROCESSING 1
-#else
-#define H2_USE_STATE_PROCESSING 0
-#endif
-
 /**
  * The magic PRIamble of RFC 7540 that is always sent when starting
  * a h2 communication.

modules/http2/h2_c1.c

@@ -47,23 +47,25 @@
 
 static struct h2_workers *workers;
 
-static int async_mpm;
+static int async_mpm, mpm_can_again;
 
 APR_OPTIONAL_FN_TYPE(ap_logio_add_bytes_in) *h2_c_logio_add_bytes_in;
 APR_OPTIONAL_FN_TYPE(ap_logio_add_bytes_out) *h2_c_logio_add_bytes_out;
 
 apr_status_t h2_c1_child_init(apr_pool_t *pool, server_rec *s)
 {
-    apr_status_t status = APR_SUCCESS;
     int minw, maxw;
     apr_time_t idle_limit;
 
-    status = ap_mpm_query(AP_MPMQ_IS_ASYNC, &async_mpm);
-    if (status != APR_SUCCESS) {
+    if (ap_mpm_query(AP_MPMQ_IS_ASYNC, &async_mpm)) {
         /* some MPMs do not implemnent this */
         async_mpm = 0;
-        status = APR_SUCCESS;
     }
+#ifdef AP_MPMQ_CAN_AGAIN
+    if (!async_mpm || ap_mpm_query(AP_MPMQ_CAN_AGAIN, &mpm_can_again)) {
+        mpm_can_again = 0;
+    }
+#endif
 
     h2_config_init(pool);
 
@@ -113,23 +115,23 @@ apr_status_t h2_c1_setup(conn_rec *c, request_rec *r, server_rec *s)
     return rv;
 }
 
-apr_status_t h2_c1_run(conn_rec *c)
+int h2_c1_run(conn_rec *c)
 {
+    int rc = OK;
     apr_status_t status;
     int mpm_state = 0, keepalive = 0;
     h2_conn_ctx_t *conn_ctx = h2_conn_ctx_get(c);
 
     ap_assert(conn_ctx);
     ap_assert(conn_ctx->session);
+    c->clogging_input_filters = 0;
     do {
         if (c->cs) {
-            c->cs->sense = CONN_SENSE_DEFAULT;
             c->cs->state = CONN_STATE_HANDLER;
         }
 
         status = h2_session_process(conn_ctx->session, async_mpm, &keepalive);
-
-        if (APR_STATUS_IS_EOF(status)) {
+        if (status != APR_SUCCESS) {
             ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, c, 
                           H2_SSSN_LOG(APLOGNO(03045), conn_ctx->session,
                           "process, closing conn"));
@@ -153,20 +155,39 @@ apr_status_t h2_c1_run(conn_rec *c)
             case H2_SESSION_ST_BUSY:
             case H2_SESSION_ST_WAIT:
                 if (keepalive) {
-                    /* flush then keep-alive */
+                    /* Flush then keep-alive */
+                    c->cs->sense = CONN_SENSE_DEFAULT;
                     c->cs->state = CONN_STATE_WRITE_COMPLETION;
                 }
                 else {
-                    /* let the MPM know that we are not done and want
-                     * the Timeout behaviour instead of a KeepAliveTimeout
+                    /* Let the MPM know that we are not done and want to wait
+                     * for read using Timeout instead of KeepAliveTimeout.
                      * See PR 63534. 
                      */
-#if H2_USE_STATE_PROCESSING
-                    c->cs->state = CONN_STATE_PROCESSING;
-#else
-                    c->cs->state = CONN_STATE_WRITE_COMPLETION;
-#endif
                     c->cs->sense = CONN_SENSE_WANT_READ;
+#ifdef AP_MPMQ_CAN_AGAIN
+                    if (mpm_can_again) {
+                        /* This tells the MPM to wait for the connection to be
+                         * readable (CONN_SENSE_WANT_READ) within the configured
+                         * Timeout and then come back to the process_connection()
+                         * hooks again when ready.
+                         */
+                        c->cs->state = CONN_STATE_PROCESSING;
+                        rc = AGAIN;
+                    }
+                    else
+#endif
+                    {
+                        /* This is a compat workaround to do the same using the
+                         * CONN_STATE_WRITE_COMPLETION state but with both
+                         * CONN_SENSE_WANT_READ to wait for readability rather
+                         * than writing and c->clogging_input_filters to force
+                         * reentering the process_connection() hooks from any
+                         * state when ready. This somehow will use Timeout too.
+                         */
+                        c->cs->state = CONN_STATE_WRITE_COMPLETION;
+                        c->clogging_input_filters = 1;
+                    }
                 }
                 break;
 
@@ -178,7 +199,7 @@ apr_status_t h2_c1_run(conn_rec *c)
         }
     }
 
-    return APR_SUCCESS;
+    return rc;
 }
 
 apr_status_t h2_c1_pre_close(struct h2_conn_ctx_t *ctx, conn_rec *c)
@@ -284,8 +305,7 @@ static int h2_c1_hook_process_connection(conn_rec* c)
             return !OK;
         }
     }
-    h2_c1_run(c);
-    return OK;
+    return h2_c1_run(c);
 
 declined:
     ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, c, "h2_h2, declined");

modules/http2/h2_session.c

@@ -1762,15 +1762,13 @@ static void unblock_c1_out(h2_session *session) {
     }
 }
 
-#if H2_USE_STATE_PROCESSING
 static int h2_send_flow_blocked(h2_session *session)
 {
     /* We are completely send blocked if either the connection window
      * is 0 or all stream flow windows are 0. */
     return ((nghttp2_session_get_remote_window_size(session->ngh2) <= 0) ||
              h2_mplx_c1_all_streams_send_win_exhausted(session->mplx));
 }
-#endif
 
 apr_status_t h2_session_process(h2_session *session, int async,
                                 int *pkeepalive)
@@ -1954,19 +1952,15 @@ apr_status_t h2_session_process(h2_session *session, int async,
                     break;
                 }
             }
-#if H2_USE_STATE_PROCESSING
             else if (async && h2_send_flow_blocked(session)) {
-                /* On a recent HTTPD, we can return to mpm c1 monitoring,
-                 * as it does not treat all connections as having KeepAlive
-                 * timing and being purgeable on load.
-                 * By returning to the MPM, we do not block a worker
+                /* By returning to the MPM, we do not block a worker
                  * and async wait for the client send window updates. */
                 ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, c,
                               H2_SSSN_LOG(APLOGNO(10502), session,
                               "BLOCKED, return to mpm c1 monitoring"));
                 goto leaving;
             }
-#endif
+
             /* No IO happening and input is exhausted. Wait with
              * the c1 connection timeout for sth to happen in our c1/c2 sockets/pipes */
             ap_log_cerror(APLOG_MARK, APLOG_TRACE2, status, c,