[RFC] Add extended stats

.travis.yml

@@ -23,6 +23,7 @@ before_script:
   - misc/install-perl-module.pl Net::EmptyPort
   - misc/install-perl-module.pl Scope::Guard
   - misc/install-perl-module.pl Starlet
+  - misc/install-perl-module.pl JSON
   # install the `ab` command (a.k.a. ApacheBench; optionally required for running some of the tests)
   - sudo apt-get install -qq apache2-utils
   # install nghttp2 with `--enable-app` (optionally required for running HTTP/2 tests)

CMakeLists.txt

@@ -239,6 +239,8 @@ SET(LIB_SOURCE_FILES
     lib/handler/redirect.c
     lib/handler/reproxy.c
     lib/handler/status.c
+    lib/handler/status/errors.c
+    lib/handler/status/requests.c
     lib/handler/configurator/access_log.c
     lib/handler/configurator/compress.c
     lib/handler/configurator/errordoc.c

include/h2o.h

@@ -275,6 +275,15 @@ typedef struct st_h2o_protocol_callbacks_t {
     int (*foreach_request)(h2o_context_t *ctx, int (*cb)(h2o_req_t *req, void *cbdata), void *cbdata);
 } h2o_protocol_callbacks_t;
 
+typedef h2o_iovec_t (*final_status_handler_cb)(void *ctx, h2o_globalconf_t *gconf, h2o_req_t *req);
+typedef struct st_h2o_status_handler_t {
+    h2o_iovec_t name;
+    void *(*init)(h2o_iovec_t *error); /* optional call back, allocates a context that will be passed to per_thread() */
+    void (*per_thread)(void *priv, h2o_context_t *ctx); /* optional callback, will be called for each thread */
+    h2o_iovec_t (*final)(void *ctx, h2o_globalconf_t *gconf, h2o_req_t *req); /* mandatory, will be passed the optional context */
+} h2o_status_handler_t;
+
+typedef H2O_VECTOR(h2o_status_handler_t) h2o_status_callbacks_t;
 struct st_h2o_globalconf_t {
     /**
      * a NULL-terminated list of host contexts (h2o_hostconf_t)
@@ -371,10 +380,7 @@ struct st_h2o_globalconf_t {
     } filecache;
 
     /* status */
-    struct {
-        /* optional callback set by the user of libh2o to feed additional json to the status handler */
-        h2o_iovec_t (*extra_status)(h2o_globalconf_t *conf, h2o_mem_pool_t *pool);
-    } status;
+    h2o_status_callbacks_t statuses;
 
     size_t _num_config_slots;
 };
@@ -411,6 +417,40 @@ typedef struct st_h2o_mimemap_type_t {
     } data;
 } h2o_mimemap_type_t;
 
+/* defined as negated form of the error codes defined in HTTP2-spec section 7 */
+#define H2O_HTTP2_ERROR_NONE 0
+#define H2O_HTTP2_ERROR_PROTOCOL -1
+#define H2O_HTTP2_ERROR_INTERNAL -2
+#define H2O_HTTP2_ERROR_FLOW_CONTROL -3
+#define H2O_HTTP2_ERROR_SETTINGS_TIMEOUT -4
+#define H2O_HTTP2_ERROR_STREAM_CLOSED -5
+#define H2O_HTTP2_ERROR_FRAME_SIZE -6
+#define H2O_HTTP2_ERROR_REFUSED_STREAM -7
+#define H2O_HTTP2_ERROR_CANCEL -8
+#define H2O_HTTP2_ERROR_COMPRESSION -9
+#define H2O_HTTP2_ERROR_CONNECT -10
+#define H2O_HTTP2_ERROR_ENHANCE_YOUR_CALM -11
+#define H2O_HTTP2_ERROR_INADEQUATE_SECURITY -12
+/* end of the HTT2-spec defined errors */
+#define H2O_HTTP2_ERROR_OTHER -13
+#define H2O_HTTP2_ERROR_MAX 14
+#define H2O_HTTP2_ERROR_INCOMPLETE -255 /* an internal value indicating that all data is not ready */
+#define H2O_HTTP2_ERROR_PROTOCOL_CLOSE_IMMEDIATELY -256
+
+enum {
+    /* http1 */
+    E_HTTP_400 = 0,
+    E_HTTP_403,
+    E_HTTP_404,
+    E_HTTP_405,
+    E_HTTP_416,
+    E_HTTP_417,
+    E_HTTP_500,
+    E_HTTP_502,
+    E_HTTP_503,
+    E_HTTP_MAX,
+};
+
 /**
  * context of the http server.
  */
@@ -479,6 +519,10 @@ struct st_h2o_context_t {
          * timeout entry used for graceful shutdown
          */
         h2o_timeout_entry_t _graceful_shutdown_timeout;
+        /**
+         * counter for http2 errors internally emitted by h2o
+         */
+        uint64_t emitted_errors[H2O_HTTP2_ERROR_MAX];
     } http2;
 
     struct {
@@ -503,6 +547,11 @@ struct st_h2o_context_t {
         h2o_timestamp_string_t *value;
     } _timestamp_cache;
 
+    /**
+     * counter for http1 errors internally emitted by h2o
+     */
+    unsigned long emitted_errors[E_HTTP_MAX];
+
     H2O_VECTOR(h2o_pathconf_t *) _pathconfs_inited;
 };
 
@@ -1143,6 +1192,11 @@ h2o_hostconf_t *h2o_config_register_host(h2o_globalconf_t *config, h2o_iovec_t h
  * * configuration path does not end with a `/`, and the request path begins with the configuration path followed by a `/`
  */
 h2o_pathconf_t *h2o_config_register_path(h2o_hostconf_t *hostconf, const char *path, int flags);
+/**
+ * registers an extra status handler
+ */
+void h2o_config_register_status_handler(h2o_globalconf_t *config, h2o_status_handler_t);
+void h2o_config_register_simple_status_handler(h2o_globalconf_t *config, h2o_iovec_t name, final_status_handler_cb status_handler);
 /**
  * disposes of the resources allocated for the global configuration
  */
@@ -1211,7 +1265,6 @@ static void h2o_context_set_filter_context(h2o_context_t *ctx, h2o_filter_t *fil
  * returns per-module context set by the on_context_init callback
  */
 static void *h2o_context_get_logger_context(h2o_context_t *ctx, h2o_logger_t *logger);
-
 /* built-in generators */
 
 enum {
@@ -1232,7 +1285,24 @@ void h2o_send_inline(h2o_req_t *req, const char *body, size_t len);
 /**
  * sends the given information as an error response to the client
  */
-void h2o_send_error(h2o_req_t *req, int status, const char *reason, const char *body, int flags);
+void h2o_send_error_generic(h2o_req_t *req, int status, const char *reason, const char *body, int flags);
+#define H2O_SEND_ERROR_XXX(status) \
+    static inline void h2o_send_error_ ## status(h2o_req_t *req, const char *reason, const char *body, int flags) \
+    { \
+        req->conn->ctx->emitted_errors[E_HTTP_ ## status]++; \
+        h2o_send_error_generic(req, status, reason, body, flags); \
+    }
+
+H2O_SEND_ERROR_XXX(400)
+H2O_SEND_ERROR_XXX(403)
+H2O_SEND_ERROR_XXX(404)
+H2O_SEND_ERROR_XXX(405)
+H2O_SEND_ERROR_XXX(416)
+H2O_SEND_ERROR_XXX(417)
+H2O_SEND_ERROR_XXX(500)
+H2O_SEND_ERROR_XXX(502)
+H2O_SEND_ERROR_XXX(503)
+
 /**
  * sends error response using zero timeout; can be called by output filters while processing the headers
  */

include/h2o/http2_internal.h

@@ -35,23 +35,6 @@ typedef struct st_h2o_http2_stream_t h2o_http2_stream_t;
 /* connection flow control window + alpha */
 #define H2O_HTTP2_DEFAULT_OUTBUF_SIZE 81920
 
-/* defined as negated form of the error codes defined in HTTP2-spec section 7 */
-#define H2O_HTTP2_ERROR_NONE 0
-#define H2O_HTTP2_ERROR_PROTOCOL -1
-#define H2O_HTTP2_ERROR_INTERNAL -2
-#define H2O_HTTP2_ERROR_FLOW_CONTROL -3
-#define H2O_HTTP2_ERROR_SETTINGS_TIMEOUT -4
-#define H2O_HTTP2_ERROR_STREAM_CLOSED -5
-#define H2O_HTTP2_ERROR_FRAME_SIZE -6
-#define H2O_HTTP2_ERROR_REFUSED_STREAM -7
-#define H2O_HTTP2_ERROR_CANCEL -8
-#define H2O_HTTP2_ERROR_COMPRESSION -9
-#define H2O_HTTP2_ERROR_CONNECT -10
-#define H2O_HTTP2_ERROR_ENHANCE_YOUR_CALM -11
-#define H2O_HTTP2_ERROR_INADEUATE_SECURITY -12
-#define H2O_HTTP2_ERROR_INCOMPLETE -255 /* an internal value indicating that all data is not ready */
-#define H2O_HTTP2_ERROR_PROTOCOL_CLOSE_IMMEDIATELY -256
-
 /* hpack */
 
 #define H2O_HTTP2_ENCODE_INT_MAX_LENGTH 5

lib/core/config.c

@@ -198,6 +198,23 @@ h2o_pathconf_t *h2o_config_register_path(h2o_hostconf_t *hostconf, const char *p
     return pathconf;
 }
 
+void h2o_config_register_status_handler(h2o_globalconf_t *config, h2o_status_handler_t status_handler)
+{
+    h2o_vector_reserve(NULL, &config->statuses, config->statuses.size + 1);
+    config->statuses.entries[config->statuses.size++] = status_handler;
+}
+
+void h2o_config_register_simple_status_handler(h2o_globalconf_t *config, h2o_iovec_t name, final_status_handler_cb status_handler)
+{
+    h2o_status_handler_t *sh;
+
+    h2o_vector_reserve(NULL, &config->statuses, config->statuses.size + 1);
+    sh = &config->statuses.entries[config->statuses.size++];
+    memset(sh, 0, sizeof(*sh));
+    sh->name = h2o_strdup(NULL, name.base, name.len);
+    sh->final = status_handler;
+}
+
 h2o_hostconf_t *h2o_config_register_host(h2o_globalconf_t *config, h2o_iovec_t host, uint16_t port)
 {
     h2o_hostconf_t *hostconf = NULL;

lib/core/proxy.c

@@ -336,7 +336,7 @@ static h2o_http1client_body_cb on_head(h2o_http1client_t *client, const char *er
     if (errstr != NULL && errstr != h2o_http1client_error_is_eos) {
         self->client = NULL;
         h2o_req_log_error(req, "lib/core/proxy.c", "%s", errstr);
-        h2o_send_error(req, 502, "Gateway Error", errstr, 0);
+        h2o_send_error_502(req, "Gateway Error", errstr, 0);
         return NULL;
     }
 
@@ -355,7 +355,7 @@ static h2o_http1client_body_cb on_head(h2o_http1client_t *client, const char *er
                     (req->res.content_length = h2o_strtosize(headers[i].value, headers[i].value_len)) == SIZE_MAX) {
                     self->client = NULL;
                     h2o_req_log_error(req, "lib/core/proxy.c", "%s", "invalid response from upstream (malformed content-length)");
-                    h2o_send_error(req, 502, "Gateway Error", "invalid response from upstream", 0);
+                    h2o_send_error_502(req, "Gateway Error", "invalid response from upstream", 0);
                     return NULL;
                 }
                 goto Skip;
@@ -419,7 +419,7 @@ static h2o_http1client_head_cb on_connect(h2o_http1client_t *client, const char
     if (errstr != NULL) {
         self->client = NULL;
         h2o_req_log_error(self->src_req, "lib/core/proxy.c", "%s", errstr);
-        h2o_send_error(self->src_req, 502, "Gateway Error", errstr, 0);
+        h2o_send_error_502(self->src_req, "Gateway Error", errstr, 0);
         return NULL;
     }
 
@@ -488,7 +488,7 @@ void h2o__proxy_process_request(h2o_req_t *req)
             h2o_req_log_error(req, "lib/core/proxy.c", "invalid URL supplied for internal redirection:%s://%.*s%.*s",
                               req->scheme->name.base, (int)req->authority.len, req->authority.base, (int)req->path.len,
                               req->path.base);
-            h2o_send_error(req, 502, "Gateway Error", "internal error", 0);
+            h2o_send_error_502(req, "Gateway Error", "internal error", 0);
             return;
         }
         if (port == 65535)

lib/core/request.c

@@ -128,7 +128,7 @@ static void call_handlers(h2o_req_t *req, h2o_handler_t **handler)
         if ((*handler)->on_req(*handler, req) == 0)
             return;
 
-    h2o_send_error(req, 404, "File Not Found", "not found", 0);
+    h2o_send_error_404(req, "File Not Found", "not found", 0);
 }
 
 static void process_hosted_request(h2o_req_t *req, h2o_hostconf_t *hostconf)
@@ -304,14 +304,14 @@ void h2o_reprocess_request(h2o_req_t *req, h2o_iovec_t method, const h2o_url_sch
     if (req->res_is_delegated) {
         if (req->num_delegated == req->conn->ctx->globalconf->max_delegations) {
             /* TODO log */
-            h2o_send_error(req, 502, "Gateway Error", "too many internal delegations", 0);
+            h2o_send_error_502(req, "Gateway Error", "too many internal delegations", 0);
             return;
         }
         ++req->num_delegated;
     } else {
         if (req->num_reprocessed >= 5) {
             /* TODO log */
-            h2o_send_error(req, 502, "Gateway Error", "too many internal reprocesses", 0);
+            h2o_send_error_502(req, "Gateway Error", "too many internal reprocesses", 0);
             return;
         }
         ++req->num_reprocessed;
@@ -456,7 +456,7 @@ void h2o_send_inline(h2o_req_t *req, const char *body, size_t len)
         h2o_send(req, &buf, 1, 1);
 }
 
-void h2o_send_error(h2o_req_t *req, int status, const char *reason, const char *body, int flags)
+void h2o_send_error_generic(h2o_req_t *req, int status, const char *reason, const char *body, int flags)
 {
     if (req->pathconf == NULL) {
         h2o_hostconf_t *hostconf = setup_before_processing(req);
@@ -478,20 +478,26 @@ void h2o_send_error(h2o_req_t *req, int status, const char *reason, const char *
     h2o_send_inline(req, body, SIZE_MAX);
 }
 
-static void send_error_deferred_cb(h2o_timeout_entry_t *entry)
-{
-    struct st_send_error_deferred_t *args = H2O_STRUCT_FROM_MEMBER(struct st_send_error_deferred_t, _timeout, entry);
-    reset_response(args->req);
-    h2o_send_error(args->req, args->status, args->reason, args->body, args->flags);
-}
+#define SEND_ERROR_DEFERRED(status_) \
+    static void send_error_deferred_cb_ ## status_ (h2o_timeout_entry_t *entry) \
+	{ \
+	    struct st_send_error_deferred_t *args = H2O_STRUCT_FROM_MEMBER(struct st_send_error_deferred_t, _timeout, entry); \
+	    reset_response(args->req); \
+	    args->req->conn->ctx->emitted_errors[E_HTTP_ ## status_]++; \
+	    h2o_send_error_generic(args->req, args->status, args->reason, args->body, args->flags); \
+	} \
+ \
+	static void h2o_send_error_deferred_ ## status_ (h2o_req_t *req, const char *reason, const char *body, int flags) \
+	{ \
+	    struct st_send_error_deferred_t *args = h2o_mem_alloc_pool(&req->pool, sizeof(*args)); \
+	    *args = (struct st_send_error_deferred_t){req, status_, reason, body, flags}; \
+	    args->_timeout.cb = send_error_deferred_cb_ ## status_; \
+	    h2o_timeout_link(req->conn->ctx->loop, &req->conn->ctx->zero_timeout, &args->_timeout); \
+	}
 
-void h2o_send_error_deferred(h2o_req_t *req, int status, const char *reason, const char *body, int flags)
-{
-    struct st_send_error_deferred_t *args = h2o_mem_alloc_pool(&req->pool, sizeof(*args));
-    *args = (struct st_send_error_deferred_t){req, status, reason, body, flags};
-    args->_timeout.cb = send_error_deferred_cb;
-    h2o_timeout_link(req->conn->ctx->loop, &req->conn->ctx->zero_timeout, &args->_timeout);
-}
+SEND_ERROR_DEFERRED(502)
+
+#undef SEND_ERROR_DEFERRED
 
 void h2o_req_log_error(h2o_req_t *req, const char *module, const char *fmt, ...)
 {
@@ -559,7 +565,7 @@ void h2o_send_redirect_internal(h2o_req_t *req, h2o_iovec_t method, const char *
     /* parse the location URL */
     if (h2o_url_parse_relative(url_str, url_len, &url) != 0) {
         /* TODO log fprintf(stderr, "[proxy] cannot handle location header: %.*s\n", (int)url_len, url); */
-        h2o_send_error_deferred(req, 502, "Gateway Error", "internal error", 0);
+        h2o_send_error_deferred_502(req, "Gateway Error", "internal error", 0);
         return;
     }
     /* convert the location to absolute (while creating copies of the values passed to the deferred call) */

lib/handler/fastcgi.c

@@ -461,7 +461,7 @@ static void errorclose(struct st_fcgi_generator_t *generator)
     } else {
         h2o_req_t *req = generator->req;
         close_generator(generator);
-        h2o_send_error(req, 503, "Internal Server Error", "Internal Server Error", 0);
+        h2o_send_error_503(req, "Internal Server Error", "Internal Server Error", 0);
     }
 }
 

lib/handler/file.c

@@ -562,7 +562,7 @@ static int try_dynamic_request(h2o_file_handler_t *self, h2o_req_t *req, char *r
 static void send_method_not_allowed(h2o_req_t *req)
 {
     h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_ALLOW, H2O_STRLIT("GET, HEAD"));
-    h2o_send_error(req, 405, "Method Not Allowed", "method not allowed", H2O_SEND_ERROR_KEEP_HEADERS);
+    h2o_send_error_405(req, "Method Not Allowed", "method not allowed", H2O_SEND_ERROR_KEEP_HEADERS);
 }
 
 static int serve_with_generator(struct st_h2o_sendfile_generator_t *generator, h2o_req_t *req, const char *rpath, size_t rpath_len,
@@ -622,7 +622,7 @@ static int serve_with_generator(struct st_h2o_sendfile_generator_t *generator, h
             content_range.base = h2o_mem_alloc_pool(&req->pool, 32);
             content_range.len = sprintf(content_range.base, "bytes */%zu", generator->bytesleft);
             h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_RANGE, content_range.base, content_range.len);
-            h2o_send_error(req, 416, "Request Range Not Satisfiable", "requested range not satisfiable",
+            h2o_send_error_416(req, "Request Range Not Satisfiable", "requested range not satisfiable",
                            H2O_SEND_ERROR_KEEP_HEADERS);
             goto Close;
         }
@@ -767,14 +767,14 @@ static int on_req(h2o_handler_t *_self, h2o_req_t *req)
     /* failed to open */
 
     if (errno == ENFILE || errno == EMFILE) {
-        h2o_send_error(req, 503, "Service Unavailable", "please try again later", 0);
+        h2o_send_error_503(req, "Service Unavailable", "please try again later", 0);
     } else {
         if (h2o_mimemap_has_dynamic_type(self->mimemap) && try_dynamic_request(self, req, rpath, rpath_len) == 0)
             return 0;
         if (errno == ENOENT || errno == ENOTDIR) {
             return -1;
         } else {
-            h2o_send_error(req, 403, "Access Forbidden", "access forbidden", 0);
+            h2o_send_error_403(req, "Access Forbidden", "access forbidden", 0);
         }
     }
     return 0;
@@ -888,13 +888,13 @@ static int specific_handler_on_req(h2o_handler_t *_self, h2o_req_t *req)
     /* open file (or send error or return -1) */
     if ((generator = create_generator(req, self->real_path.base, self->real_path.len, &is_dir, self->flags)) == NULL) {
         if (is_dir) {
-            h2o_send_error(req, 403, "Access Forbidden", "access forbidden", 0);
+            h2o_send_error_403(req, "Access Forbidden", "access forbidden", 0);
         } else if (errno == ENOENT) {
             return -1;
         } else if (errno == ENFILE || errno == EMFILE) {
-            h2o_send_error(req, 503, "Service Unavailable", "please try again later", 0);
+            h2o_send_error_503(req, "Service Unavailable", "please try again later", 0);
         } else {
-            h2o_send_error(req, 403, "Access Forbidden", "access forbidden", 0);
+            h2o_send_error_403(req, "Access Forbidden", "access forbidden", 0);
         }
         return 0;
     }