Merge pull request #893 from pull/893

[RFC] Add extended stats

clang-format applied to the updated files

.travis.yml

@@ -23,6 +23,7 @@ before_script:
   - misc/install-perl-module.pl Net::EmptyPort
   - misc/install-perl-module.pl Scope::Guard
   - misc/install-perl-module.pl Starlet
+  - misc/install-perl-module.pl JSON
   # install the `ab` command (a.k.a. ApacheBench; optionally required for running some of the tests)
   - sudo apt-get install -qq apache2-utils
   # install nghttp2 with `--enable-app` (optionally required for running HTTP/2 tests)

CMakeLists.txt

@@ -240,6 +240,8 @@ SET(LIB_SOURCE_FILES
     lib/handler/reproxy.c
     lib/handler/throttle_resp.c
     lib/handler/status.c
+    lib/handler/status/events.c
+    lib/handler/status/requests.c
     lib/handler/configurator/access_log.c
     lib/handler/configurator/compress.c
     lib/handler/configurator/errordoc.c

include/h2o.h

@@ -275,6 +275,15 @@ typedef struct st_h2o_protocol_callbacks_t {
     int (*foreach_request)(h2o_context_t *ctx, int (*cb)(h2o_req_t *req, void *cbdata), void *cbdata);
 } h2o_protocol_callbacks_t;
 
+typedef h2o_iovec_t (*final_status_handler_cb)(void *ctx, h2o_globalconf_t *gconf, h2o_req_t *req);
+typedef struct st_h2o_status_handler_t {
+    h2o_iovec_t name;
+    void *(*init)(void); /* optional callback, allocates a context that will be passed to per_thread() */
+    void (*per_thread)(void *priv, h2o_context_t *ctx); /* optional callback, will be called for each thread */
+    h2o_iovec_t (*final)(void *ctx, h2o_globalconf_t *gconf, h2o_req_t *req); /* mandatory, will be passed the optional context */
+} h2o_status_handler_t;
+
+typedef H2O_VECTOR(h2o_status_handler_t) h2o_status_callbacks_t;
 struct st_h2o_globalconf_t {
     /**
      * a NULL-terminated list of host contexts (h2o_hostconf_t)
@@ -371,10 +380,7 @@ struct st_h2o_globalconf_t {
     } filecache;
 
     /* status */
-    struct {
-        /* optional callback set by the user of libh2o to feed additional json to the status handler */
-        h2o_iovec_t (*extra_status)(h2o_globalconf_t *conf, h2o_mem_pool_t *pool);
-    } status;
+    h2o_status_callbacks_t statuses;
 
     size_t _num_config_slots;
 };
@@ -411,6 +417,40 @@ typedef struct st_h2o_mimemap_type_t {
     } data;
 } h2o_mimemap_type_t;
 
+/* defined as negated form of the error codes defined in HTTP2-spec section 7 */
+#define H2O_HTTP2_ERROR_NONE 0
+#define H2O_HTTP2_ERROR_PROTOCOL -1
+#define H2O_HTTP2_ERROR_INTERNAL -2
+#define H2O_HTTP2_ERROR_FLOW_CONTROL -3
+#define H2O_HTTP2_ERROR_SETTINGS_TIMEOUT -4
+#define H2O_HTTP2_ERROR_STREAM_CLOSED -5
+#define H2O_HTTP2_ERROR_FRAME_SIZE -6
+#define H2O_HTTP2_ERROR_REFUSED_STREAM -7
+#define H2O_HTTP2_ERROR_CANCEL -8
+#define H2O_HTTP2_ERROR_COMPRESSION -9
+#define H2O_HTTP2_ERROR_CONNECT -10
+#define H2O_HTTP2_ERROR_ENHANCE_YOUR_CALM -11
+#define H2O_HTTP2_ERROR_INADEQUATE_SECURITY -12
+#define H2O_HTTP2_ERROR_MAX 13
+/* end of the HTT2-spec defined errors */
+#define H2O_HTTP2_ERROR_INCOMPLETE -255 /* an internal value indicating that all data is not ready */
+#define H2O_HTTP2_ERROR_PROTOCOL_CLOSE_IMMEDIATELY -256
+
+enum {
+    /* http1 protocol errors */
+    H2O_STATUS_ERROR_400 = 0,
+    H2O_STATUS_ERROR_403,
+    H2O_STATUS_ERROR_404,
+    H2O_STATUS_ERROR_405,
+    H2O_STATUS_ERROR_413,
+    H2O_STATUS_ERROR_416,
+    H2O_STATUS_ERROR_417,
+    H2O_STATUS_ERROR_500,
+    H2O_STATUS_ERROR_502,
+    H2O_STATUS_ERROR_503,
+    H2O_STATUS_ERROR_MAX,
+};
+
 /**
  * context of the http server.
  */
@@ -483,6 +523,20 @@ struct st_h2o_context_t {
          * timeout entry used for graceful shutdown
          */
         h2o_timeout_entry_t _graceful_shutdown_timeout;
+        struct {
+            /**
+             * counter for http2 errors internally emitted by h2o
+             */
+            uint64_t protocol_level_errors[H2O_HTTP2_ERROR_MAX];
+            /**
+             * premature close on read
+             */
+            uint64_t read_closed;
+            /**
+             * premature close on write
+             */
+            uint64_t write_closed;
+        } events;
     } http2;
 
     struct {
@@ -507,6 +561,11 @@ struct st_h2o_context_t {
         h2o_timestamp_string_t *value;
     } _timestamp_cache;
 
+    /**
+     * counter for http1 error status internally emitted by h2o
+     */
+    uint64_t emitted_error_status[H2O_STATUS_ERROR_MAX];
+
     H2O_VECTOR(h2o_pathconf_t *) _pathconfs_inited;
 };
 
@@ -1135,18 +1194,23 @@ h2o_hostconf_t *h2o_config_register_host(h2o_globalconf_t *config, h2o_iovec_t h
  * @param flags unused and must be set to zero
  *
  * Handling of the path argument has changed in version 2.0 (of the standard server).
- * 
+ *
  * Before 2.0, the function implicitely added a trailing `/` to the supplied path (if it did not end with a `/`), and when receiving
  * a HTTP request for a matching path without the trailing `/`, libh2o sent a 301 response redirecting the client to a URI with a
  * trailing `/`.
- * 
+ *
  * Since 2.0, the function retains the exact path given as the argument, and the handlers of the pathconf is invoked if one of the
  * following conditions are met:
  *
  * * request path is an exact match to the configuration path
  * * configuration path does not end with a `/`, and the request path begins with the configuration path followed by a `/`
  */
 h2o_pathconf_t *h2o_config_register_path(h2o_hostconf_t *hostconf, const char *path, int flags);
+/**
+ * registers an extra status handler
+ */
+void h2o_config_register_status_handler(h2o_globalconf_t *config, h2o_status_handler_t);
+void h2o_config_register_simple_status_handler(h2o_globalconf_t *config, h2o_iovec_t name, final_status_handler_cb status_handler);
 /**
  * disposes of the resources allocated for the global configuration
  */
@@ -1215,7 +1279,6 @@ static void h2o_context_set_filter_context(h2o_context_t *ctx, h2o_filter_t *fil
  * returns per-module context set by the on_context_init callback
  */
 static void *h2o_context_get_logger_context(h2o_context_t *ctx, h2o_logger_t *logger);
-
 /* built-in generators */
 
 enum {
@@ -1236,7 +1299,24 @@ void h2o_send_inline(h2o_req_t *req, const char *body, size_t len);
 /**
  * sends the given information as an error response to the client
  */
-void h2o_send_error(h2o_req_t *req, int status, const char *reason, const char *body, int flags);
+void h2o_send_error_generic(h2o_req_t *req, int status, const char *reason, const char *body, int flags);
+#define H2O_SEND_ERROR_XXX(status)                                                                                                 \
+    static inline void h2o_send_error_##status(h2o_req_t *req, const char *reason, const char *body, int flags)                    \
+    {                                                                                                                              \
+        req->conn->ctx->emitted_error_status[H2O_STATUS_ERROR_##status]++;                                                         \
+        h2o_send_error_generic(req, status, reason, body, flags);                                                                  \
+    }
+
+H2O_SEND_ERROR_XXX(400)
+H2O_SEND_ERROR_XXX(403)
+H2O_SEND_ERROR_XXX(404)
+H2O_SEND_ERROR_XXX(405)
+H2O_SEND_ERROR_XXX(416)
+H2O_SEND_ERROR_XXX(417)
+H2O_SEND_ERROR_XXX(500)
+H2O_SEND_ERROR_XXX(502)
+H2O_SEND_ERROR_XXX(503)
+
 /**
  * sends error response using zero timeout; can be called by output filters while processing the headers
  */

include/h2o/http2_internal.h

@@ -35,23 +35,6 @@ typedef struct st_h2o_http2_stream_t h2o_http2_stream_t;
 /* connection flow control window + alpha */
 #define H2O_HTTP2_DEFAULT_OUTBUF_SIZE 81920
 
-/* defined as negated form of the error codes defined in HTTP2-spec section 7 */
-#define H2O_HTTP2_ERROR_NONE 0
-#define H2O_HTTP2_ERROR_PROTOCOL -1
-#define H2O_HTTP2_ERROR_INTERNAL -2
-#define H2O_HTTP2_ERROR_FLOW_CONTROL -3
-#define H2O_HTTP2_ERROR_SETTINGS_TIMEOUT -4
-#define H2O_HTTP2_ERROR_STREAM_CLOSED -5
-#define H2O_HTTP2_ERROR_FRAME_SIZE -6
-#define H2O_HTTP2_ERROR_REFUSED_STREAM -7
-#define H2O_HTTP2_ERROR_CANCEL -8
-#define H2O_HTTP2_ERROR_COMPRESSION -9
-#define H2O_HTTP2_ERROR_CONNECT -10
-#define H2O_HTTP2_ERROR_ENHANCE_YOUR_CALM -11
-#define H2O_HTTP2_ERROR_INADEUATE_SECURITY -12
-#define H2O_HTTP2_ERROR_INCOMPLETE -255 /* an internal value indicating that all data is not ready */
-#define H2O_HTTP2_ERROR_PROTOCOL_CLOSE_IMMEDIATELY -256
-
 /* hpack */
 
 #define H2O_HTTP2_ENCODE_INT_MAX_LENGTH 5

lib/core/config.c

@@ -198,6 +198,23 @@ h2o_pathconf_t *h2o_config_register_path(h2o_hostconf_t *hostconf, const char *p
     return pathconf;
 }
 
+void h2o_config_register_status_handler(h2o_globalconf_t *config, h2o_status_handler_t status_handler)
+{
+    h2o_vector_reserve(NULL, &config->statuses, config->statuses.size + 1);
+    config->statuses.entries[config->statuses.size++] = status_handler;
+}
+
+void h2o_config_register_simple_status_handler(h2o_globalconf_t *config, h2o_iovec_t name, final_status_handler_cb status_handler)
+{
+    h2o_status_handler_t *sh;
+
+    h2o_vector_reserve(NULL, &config->statuses, config->statuses.size + 1);
+    sh = &config->statuses.entries[config->statuses.size++];
+    memset(sh, 0, sizeof(*sh));
+    sh->name = h2o_strdup(NULL, name.base, name.len);
+    sh->final = status_handler;
+}
+
 h2o_hostconf_t *h2o_config_register_host(h2o_globalconf_t *config, h2o_iovec_t host, uint16_t port)
 {
     h2o_hostconf_t *hostconf = NULL;

lib/core/proxy.c

@@ -336,7 +336,7 @@ static h2o_http1client_body_cb on_head(h2o_http1client_t *client, const char *er
     if (errstr != NULL && errstr != h2o_http1client_error_is_eos) {
         self->client = NULL;
         h2o_req_log_error(req, "lib/core/proxy.c", "%s", errstr);
-        h2o_send_error(req, 502, "Gateway Error", errstr, 0);
+        h2o_send_error_502(req, "Gateway Error", errstr, 0);
         return NULL;
     }
 
@@ -355,7 +355,7 @@ static h2o_http1client_body_cb on_head(h2o_http1client_t *client, const char *er
                     (req->res.content_length = h2o_strtosize(headers[i].value, headers[i].value_len)) == SIZE_MAX) {
                     self->client = NULL;
                     h2o_req_log_error(req, "lib/core/proxy.c", "%s", "invalid response from upstream (malformed content-length)");
-                    h2o_send_error(req, 502, "Gateway Error", "invalid response from upstream", 0);
+                    h2o_send_error_502(req, "Gateway Error", "invalid response from upstream", 0);
                     return NULL;
                 }
                 goto Skip;
@@ -433,7 +433,7 @@ static h2o_http1client_head_cb on_connect(h2o_http1client_t *client, const char
     if (errstr != NULL) {
         self->client = NULL;
         h2o_req_log_error(self->src_req, "lib/core/proxy.c", "%s", errstr);
-        h2o_send_error(self->src_req, 502, "Gateway Error", errstr, 0);
+        h2o_send_error_502(self->src_req, "Gateway Error", errstr, 0);
         return NULL;
     }
 
@@ -503,7 +503,7 @@ void h2o__proxy_process_request(h2o_req_t *req)
             h2o_req_log_error(req, "lib/core/proxy.c", "invalid URL supplied for internal redirection:%s://%.*s%.*s",
                               req->scheme->name.base, (int)req->authority.len, req->authority.base, (int)req->path.len,
                               req->path.base);
-            h2o_send_error(req, 502, "Gateway Error", "internal error", 0);
+            h2o_send_error_502(req, "Gateway Error", "internal error", 0);
             return;
         }
         if (port == 65535)

lib/core/request.c

@@ -128,7 +128,7 @@ static void call_handlers(h2o_req_t *req, h2o_handler_t **handler)
         if ((*handler)->on_req(*handler, req) == 0)
             return;
 
-    h2o_send_error(req, 404, "File Not Found", "not found", 0);
+    h2o_send_error_404(req, "File Not Found", "not found", 0);
 }
 
 static void process_hosted_request(h2o_req_t *req, h2o_hostconf_t *hostconf)
@@ -304,14 +304,14 @@ void h2o_reprocess_request(h2o_req_t *req, h2o_iovec_t method, const h2o_url_sch
     if (req->res_is_delegated) {
         if (req->num_delegated == req->conn->ctx->globalconf->max_delegations) {
             /* TODO log */
-            h2o_send_error(req, 502, "Gateway Error", "too many internal delegations", 0);
+            h2o_send_error_502(req, "Gateway Error", "too many internal delegations", 0);
             return;
         }
         ++req->num_delegated;
     } else {
         if (req->num_reprocessed >= 5) {
             /* TODO log */
-            h2o_send_error(req, 502, "Gateway Error", "too many internal reprocesses", 0);
+            h2o_send_error_502(req, "Gateway Error", "too many internal reprocesses", 0);
             return;
         }
         ++req->num_reprocessed;
@@ -456,7 +456,7 @@ void h2o_send_inline(h2o_req_t *req, const char *body, size_t len)
         h2o_send(req, &buf, 1, 1);
 }
 
-void h2o_send_error(h2o_req_t *req, int status, const char *reason, const char *body, int flags)
+void h2o_send_error_generic(h2o_req_t *req, int status, const char *reason, const char *body, int flags)
 {
     if (req->pathconf == NULL) {
         h2o_hostconf_t *hostconf = setup_before_processing(req);
@@ -478,20 +478,26 @@ void h2o_send_error(h2o_req_t *req, int status, const char *reason, const char *
     h2o_send_inline(req, body, SIZE_MAX);
 }
 
-static void send_error_deferred_cb(h2o_timeout_entry_t *entry)
-{
-    struct st_send_error_deferred_t *args = H2O_STRUCT_FROM_MEMBER(struct st_send_error_deferred_t, _timeout, entry);
-    reset_response(args->req);
-    h2o_send_error(args->req, args->status, args->reason, args->body, args->flags);
-}
+#define DECL_SEND_ERROR_DEFERRED(status_)                                                                                          \
+    static void send_error_deferred_cb_##status_(h2o_timeout_entry_t *entry)                                                       \
+    {                                                                                                                              \
+        struct st_send_error_deferred_t *args = H2O_STRUCT_FROM_MEMBER(struct st_send_error_deferred_t, _timeout, entry);          \
+        reset_response(args->req);                                                                                                 \
+        args->req->conn->ctx->emitted_error_status[H2O_STATUS_ERROR_##status_]++;                                                  \
+        h2o_send_error_generic(args->req, args->status, args->reason, args->body, args->flags);                                    \
+    }                                                                                                                              \
+                                                                                                                                   \
+    static void h2o_send_error_deferred_##status_(h2o_req_t *req, const char *reason, const char *body, int flags)                 \
+    {                                                                                                                              \
+        struct st_send_error_deferred_t *args = h2o_mem_alloc_pool(&req->pool, sizeof(*args));                                     \
+        *args = (struct st_send_error_deferred_t){req, status_, reason, body, flags};                                              \
+        args->_timeout.cb = send_error_deferred_cb_##status_;                                                                      \
+        h2o_timeout_link(req->conn->ctx->loop, &req->conn->ctx->zero_timeout, &args->_timeout);                                    \
+    }
 
-void h2o_send_error_deferred(h2o_req_t *req, int status, const char *reason, const char *body, int flags)
-{
-    struct st_send_error_deferred_t *args = h2o_mem_alloc_pool(&req->pool, sizeof(*args));
-    *args = (struct st_send_error_deferred_t){req, status, reason, body, flags};
-    args->_timeout.cb = send_error_deferred_cb;
-    h2o_timeout_link(req->conn->ctx->loop, &req->conn->ctx->zero_timeout, &args->_timeout);
-}
+DECL_SEND_ERROR_DEFERRED(502)
+
+#undef DECL_SEND_ERROR_DEFERRED
 
 void h2o_req_log_error(h2o_req_t *req, const char *module, const char *fmt, ...)
 {
@@ -559,7 +565,7 @@ void h2o_send_redirect_internal(h2o_req_t *req, h2o_iovec_t method, const char *
     /* parse the location URL */
     if (h2o_url_parse_relative(url_str, url_len, &url) != 0) {
         /* TODO log fprintf(stderr, "[proxy] cannot handle location header: %.*s\n", (int)url_len, url); */
-        h2o_send_error_deferred(req, 502, "Gateway Error", "internal error", 0);
+        h2o_send_error_deferred_502(req, "Gateway Error", "internal error", 0);
         return;
     }
     /* convert the location to absolute (while creating copies of the values passed to the deferred call) */

lib/handler/fastcgi.c

@@ -461,7 +461,7 @@ static void errorclose(struct st_fcgi_generator_t *generator)
     } else {
         h2o_req_t *req = generator->req;
         close_generator(generator);
-        h2o_send_error(req, 503, "Internal Server Error", "Internal Server Error", 0);
+        h2o_send_error_503(req, "Internal Server Error", "Internal Server Error", 0);
     }
 }