mk_server: scheduler: new interface to enqueue defer events object free

This patch implements a new scheduler interface that allows the plugins to
defer the release of some object, the context and requirement of implementation
is based on the following behavior:

 1. A connection is being handled by a plugin (e.g: CGI), the CGI plugin also
    registered it own 'event' into the Event Loop to read incoming data from the
    child process.

 2. Before to receive the response, the client for some reason closes
    the TCP connection.

 3. The Scheduler will trap the 'close event' on the connection and start
    notifying the protocol handler and the plugin associated about this.

 4. The plugin will catch the notification on it stage30_hangup() callback so it
    will release it resources (CGI context, child process,
    close file desriptors, etc).

 5. At this point in the same Event Loop round, it may contain many events notified
    in the array, e.g:

    [0]   TCP connection closed
    [N]   others...
    [16]  Read data from CGI child

    So when we reach the the 'TCP connection closed'[0], we perform a release procedure as
    stated on #3 and #4, but then we reach 'Read data from CGI child'[16] but our event
    context was already released due to 'TCP connection closed'[0]. For hence we face a
    memory corruption.

In order to avoid this kind of situations, the Scheduler and the plugins interface now
provides a new interface to request the Server to perform a 'memory release' of a linked
list of 'struct mk_event' after processing all the events in the round returned by mk_event_wait().

A plugin only requires to be aware of the broken connection stage30_hangup() and register it
event object mk_api->sched_event_free(struct mk_event *x).

Signed-off-by: Eduardo Silva <eduardo@monkey.io>

include/monkey/mk_plugin.h

@@ -202,6 +202,7 @@ struct plugin_api
     int (*sched_remove_client) (int);
     struct mk_sched_conn *(*sched_get_connection)(struct mk_sched_worker *,
                                                   int);
+    void (*sched_event_free) (struct mk_event *);
     struct mk_sched_worker *(*sched_worker_info)();
 
     /* worker's functions */

include/monkey/mk_scheduler.h

@@ -85,6 +85,9 @@ struct mk_sched_worker
 
     struct mk_http_session *request_handler;
 
+
+    struct mk_list event_free_queue;
+
     /*
      * This variable is used to signal the active workers,
      * just available because of ULONG_MAX bug described
@@ -231,6 +234,21 @@ int mk_sched_event_write(struct mk_sched_conn *conn,
 int mk_sched_event_close(struct mk_sched_conn *conn,
                          struct mk_sched_worker *sched,
                          int type);
+void mk_sched_event_free(struct mk_event *event);
+
+
+static inline void mk_sched_event_free_all(struct mk_sched_worker *sched)
+{
+    struct mk_list *tmp;
+    struct mk_list *head;
+    struct mk_event *event;
+
+    mk_list_foreach_safe(head, tmp, &sched->event_free_queue) {
+        event = mk_list_entry(head, struct mk_event, _head);
+        mk_list_del(&event->_head);
+        mk_mem_free(event);
+    }
+}
 
 static inline void mk_sched_conn_timeout_add(struct mk_sched_conn *conn,
                                              struct mk_sched_worker *sched)
@@ -249,6 +267,7 @@ static inline void mk_sched_conn_timeout_del(struct mk_sched_conn *conn)
     }
 }
 
+
 #define mk_sched_conn_read(conn, buf, s)        \
     conn->net->read(conn->event.fd, buf, s)
 #define mk_sched_conn_write(ch, buf, s)         \

mk_server/mk_plugin.c

@@ -316,6 +316,7 @@ void mk_plugin_api_init()
     /* Scheduler and Event callbacks */
     api->sched_loop           = mk_sched_loop;
     api->sched_get_connection = mk_sched_get_connection;
+    api->sched_event_free     = mk_sched_event_free;
     api->sched_remove_client  = mk_plugin_sched_remove_client;
     api->sched_worker_info    = mk_plugin_sched_get_thread_conf;
 

mk_server/mk_scheduler.c

@@ -349,6 +349,8 @@ void *mk_sched_launch_worker_loop(void *thread_conf)
         exit(EXIT_FAILURE);
     }
 
+    mk_list_init(&sched->event_free_queue);
+
     /*
      * ULONG_MAX BUG test only
      * =======================
@@ -664,3 +666,9 @@ int mk_sched_event_close(struct mk_sched_conn *conn,
     mk_sched_drop_connection(conn, sched);
     return 0;
 }
+
+void mk_sched_event_free(struct mk_event *event)
+{
+    struct mk_sched_worker *sched = mk_sched_get_thread_conf();
+    mk_list_add(&event->_head, &sched->event_free_queue);
+}

mk_server/mk_server.c

@@ -361,9 +361,6 @@ void mk_server_worker_loop()
                 }
             }
         }
-        else {
-            /* FIXME */
-        }
     }
 
     if (mk_config->scheduler_mode == MK_SCHEDULER_REUSEPORT) {
@@ -383,6 +380,7 @@ void mk_server_worker_loop()
     while (1) {
         mk_event_wait(evl);
         mk_event_foreach(event, evl) {
+            ret = 0;
             if (event->type == MK_EVENT_CONNECTION) {
                 conn = (struct mk_sched_conn *) event;
 
@@ -456,6 +454,7 @@ void mk_server_worker_loop()
                 continue;
             }
         }
+        mk_sched_event_free_all(sched);
     }
 }