From c726913a8f5e8dff5df45e390bf3bc7bba30ec57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20R=C3=BChsen?= Date: Fri, 18 Sep 2015 17:04:16 +0200 Subject: [PATCH] More HTTP/2 related code, new option --(no-)http2 --- configure.ac | 2 +- docs/libmget/libmget-sections.txt | 1 + include/libmget.h | 6 ++ libmget/http.c | 104 ++++++++++++++++++------------ libmget/io.c | 64 +++++++++++------- libmget/net.c | 12 +++- libmget/ocsp.c | 2 +- libmget/ssl_gnutls.c | 26 ++++---- src/mget.c | 4 +- src/options.c | 14 +++- src/options.h | 1 + 11 files changed, 156 insertions(+), 80 deletions(-) diff --git a/configure.ac b/configure.ac index 84077b1..10ff921 100644 --- a/configure.ac +++ b/configure.ac @@ -357,7 +357,7 @@ AC_FUNC_ALLOCA AC_FUNC_FORK #AC_FUNC_MALLOC AC_FUNC_MMAP -AC_FUNC_REALLOC +#AC_FUNC_REALLOC AC_FUNC_FNMATCH AC_CHECK_FUNCS([\ clock_gettime dprintf dup2 futimens gettimeofday localtime_r memchr\ diff --git a/docs/libmget/libmget-sections.txt b/docs/libmget/libmget-sections.txt index c7f17fb..16682b0 100644 --- a/docs/libmget/libmget-sections.txt +++ b/docs/libmget/libmget-sections.txt @@ -41,5 +41,6 @@ mget_fdgetline mget_getline mget_ready_2_read mget_ready_2_write +mget_ready_2_transfer mget_read_file diff --git a/include/libmget.h b/include/libmget.h index fef2225..cf65700 100644 --- a/include/libmget.h +++ b/include/libmget.h @@ -263,6 +263,8 @@ int mget_ready_2_read(int fd, int timeout); int mget_ready_2_write(int fd, int timeout); +int + mget_ready_2_transfer(int fd, int timeout, short mode); int mget_strcmp(const char *s1, const char *s2) G_GNUC_MGET_PURE; int @@ -1144,6 +1146,8 @@ void mget_tcp_close(mget_tcp_t *tcp); void mget_tcp_set_timeout(mget_tcp_t *tcp, int timeout); +int + mget_tcp_get_timeout(mget_tcp_t *tcp) G_GNUC_MGET_PURE; void mget_tcp_set_connect_timeout(mget_tcp_t *tcp, int timeout); void @@ -1204,6 +1208,8 @@ ssize_t mget_tcp_write(mget_tcp_t *tcp, const char *buf, size_t count) G_GNUC_MGET_NONNULL_ALL; ssize_t mget_tcp_read(mget_tcp_t *tcp, char *buf, size_t count) G_GNUC_MGET_NONNULL_ALL; +int + mget_tcp_ready_2_transfer(mget_tcp_t *tcp, int flags) G_GNUC_MGET_NONNULL_ALL; /* * SSL routines diff --git a/libmget/http.c b/libmget/http.c index 7b55c25..0f1c3a3 100644 --- a/libmget/http.c +++ b/libmget/http.c @@ -1476,13 +1476,13 @@ static ssize_t _send_callback(nghttp2_session *session G_GNUC_MGET_UNUSED, mget_http_connection_t *conn = (mget_http_connection_t *)user_data; int rc; - debug_printf("writing... %zd\n", length); + // debug_printf("writing... %zd\n", length); if ((rc = mget_tcp_write(conn->tcp, (const char *)data, length)) <= 0) { // An error will be written by the mget_tcp_write function. - debug_printf("write rc %d, errno=%d\n", rc, errno); + // debug_printf("write rc %d, errno=%d\n", rc, errno); return rc ? NGHTTP2_ERR_CALLBACK_FAILURE : NGHTTP2_ERR_WOULDBLOCK; } - debug_printf("write rc %d\n",rc); + // debug_printf("write rc %d\n",rc); return rc; } @@ -1493,57 +1493,59 @@ static ssize_t _recv_callback(nghttp2_session *session G_GNUC_MGET_UNUSED, mget_http_connection_t *conn = (mget_http_connection_t *)user_data; int rc; - debug_printf("reading... %zd\n", length); + // debug_printf("reading... %zd\n", length); if ((rc = mget_tcp_read(conn->tcp, (char *)buf, length)) <= 0) { // 0 = timeout resp. blocking // -1 = failure - debug_printf("read rc %d, errno=%d\n", rc, errno); + // debug_printf("read rc %d, errno=%d\n", rc, errno); return rc ? NGHTTP2_ERR_CALLBACK_FAILURE : NGHTTP2_ERR_WOULDBLOCK; } - debug_printf("read rc %d\n",rc); + // debug_printf("read rc %d\n",rc); return rc; } -static int _on_frame_send_callback(nghttp2_session *session, +static void _print_frame_type(int type, const char tag) +{ + static const char *name[] = { + [NGHTTP2_DATA] = "DATA", + [NGHTTP2_HEADERS] = "HEADERS", + [NGHTTP2_PRIORITY] = "PRIORITY", + [NGHTTP2_RST_STREAM] = "RST_STREAM", + [NGHTTP2_SETTINGS] = "SETTINGS", + [NGHTTP2_PUSH_PROMISE] = "PUSH_PROMISE", + [NGHTTP2_PING] = "PING", + [NGHTTP2_GOAWAY] = "GOAWAY", + [NGHTTP2_WINDOW_UPDATE] = "WINDOW_UPDATE", + [NGHTTP2_CONTINUATION] = "CONTINUATION" + }; + + if ((unsigned) type < countof(name)) + debug_printf("[FRAME] %c %s\n", tag, name[type]); + else + debug_printf("[FRAME] %c Unknown type %d\n", tag, type); +} + +static int _on_frame_send_callback(nghttp2_session *session G_GNUC_MGET_UNUSED, const nghttp2_frame *frame, void *user_data G_GNUC_MGET_UNUSED) { - switch (frame->hd.type) { - case NGHTTP2_HEADERS: - if (nghttp2_session_get_stream_user_data(session, frame->hd.stream_id)) { - const nghttp2_nv *nva = frame->headers.nva; - debug_printf("[INFO] C ----------------------------> S (HEADERS)\n"); - for (unsigned i = 0; i < frame->headers.nvlen; i++) - debug_printf("%.*s: %.*s\n", (int)nva[i].namelen, nva[i].name, (int)nva[i].valuelen, nva[i].value); - } - break; - case NGHTTP2_RST_STREAM: - debug_printf("[INFO] C ----------------------------> S (RST_STREAM)\n"); - break; - case NGHTTP2_GOAWAY: - debug_printf("[INFO] C ----------------------------> S (GOAWAY)\n"); - break; + _print_frame_type(frame->hd.type, '>'); + + if (frame->hd.type == NGHTTP2_HEADERS) { + const nghttp2_nv *nva = frame->headers.nva; + + for (unsigned i = 0; i < frame->headers.nvlen; i++) + debug_printf("[FRAME] > %.*s: %.*s\n", (int)nva[i].namelen, nva[i].name, (int)nva[i].valuelen, nva[i].value); } + return 0; } static int _on_frame_recv_callback(nghttp2_session *session G_GNUC_MGET_UNUSED, const nghttp2_frame *frame, void *user_data G_GNUC_MGET_UNUSED) { - switch (frame->hd.type) { - case NGHTTP2_HEADERS: - debug_printf("[INFO] HEADERS cat %d\n", frame->headers.cat); - break; - case NGHTTP2_RST_STREAM: - debug_printf("[INFO] C <---------------------------- S (RST_STREAM)\n"); - break; - case NGHTTP2_GOAWAY: - debug_printf("[INFO] C <---------------------------- S (GOAWAY)\n"); - break; - default: - debug_printf("[INFO] C <---------------------------- S (hd.type %d)\n", frame->hd.type); - break; - } + _print_frame_type(frame->hd.type, '<'); + return 0; } @@ -2006,6 +2008,7 @@ mget_http_response_t *mget_http_get_response_cb( char *buf, *p = NULL; mget_http_response_t *resp = NULL; mget_decompressor_t *dc = NULL; + int ioflags; #ifdef WITH_LIBNGHTTP2 if (conn->protocol == MGET_PROTOCOL_HTTP_2_0) { @@ -2017,9 +2020,30 @@ mget_http_response_t *mget_http_get_response_cb( struct _body_callback_context ctx = { .resp = resp, .context = context, .body_callback = body_callback }; req->nghttp2_context = &ctx; - mget_tcp_set_timeout(conn->tcp, 1); // 1ms timeout + int timeout = mget_tcp_get_timeout(conn->tcp); + for (int rc = 0; rc == 0 && !ctx.done;) { - debug_printf("0 response status %d done %d\n", resp->code, ctx.done); + ioflags = 0; + if (nghttp2_session_want_write(conn->http2_session)) + ioflags |= MGET_IO_WRITABLE; + if (nghttp2_session_want_read(conn->http2_session)) + ioflags |= MGET_IO_READABLE; + + if (ioflags) + ioflags = mget_tcp_ready_2_transfer(conn->tcp, ioflags); + // debug_printf("ioflags=%d timeout=%d\n",ioflags,mget_tcp_get_timeout(conn->tcp)); + if (ioflags <= 0) break; // error or timeout + + mget_tcp_set_timeout(conn->tcp, 0); // 0 = immediate + rc = 0; + if (ioflags & MGET_IO_WRITABLE) { + rc = nghttp2_session_send(conn->http2_session); + } + if (!rc && (ioflags & MGET_IO_READABLE)) + rc = nghttp2_session_recv(conn->http2_session); + mget_tcp_set_timeout(conn->tcp, timeout); // restore old timeout + +/* while (nghttp2_session_want_write(conn->http2_session)) { rc = nghttp2_session_send(conn->http2_session); } @@ -2027,9 +2051,7 @@ mget_http_response_t *mget_http_get_response_cb( if (nghttp2_session_want_read(conn->http2_session)) { rc = nghttp2_session_recv(conn->http2_session); } - debug_printf("2 response status %d done %d\n", resp->code, ctx.done); - if (rc) - debug_printf("loop rc = %d\n", rc); +*/ } debug_printf("response status %d\n", resp->code); diff --git a/libmget/io.c b/libmget/io.c index 804f9f6..3a4d6c3 100644 --- a/libmget/io.c +++ b/libmget/io.c @@ -206,34 +206,51 @@ ssize_t mget_getline(char **buf, size_t *bufsize, FILE *fp) return -1; } +/** + * mget_ready_2_transfer: + * @fd: File descriptor to wait for. + * @timeout: Max. duration in milliseconds to wait. + * A value of 0 means the function returns immediately. + * A value of -1 means infinite timeout. + * + * Wait for a file descriptor to become ready to read or write. + * + * Returns: + * -1 on error. + * 0 on timeout. The file descriptor is not ready for reading nor writing. + * >0 The file descriptor is ready for reading or writing. Check for + * the bitwise or'ing of MGET_IO_WRITABLE and MGET_IO_WRITABLE. + * + */ #ifdef POLLIN -static int _ready_2_transfer(int fd, int timeout, short mode) +int mget_ready_2_transfer(int fd, int timeout, short mode) { - // 0: no timeout / immediate - // -1: INFINITE timeout - // >0: number of milliseconds to wait - if (timeout) { - int rc; + struct pollfd pollfd; + int rc; - if (mode == MGET_IO_READABLE) - mode = POLLIN; - else - mode = POLLOUT; + pollfd.fd = fd; + pollfd.events = 0; + pollfd.revents = 0; - // wait for socket to be ready to read - struct pollfd pollfd[1] = { { fd, mode, 0 } }; + if (mode & MGET_IO_READABLE) + pollfd.events |= POLLIN; + if (mode & MGET_IO_WRITABLE) + pollfd.events |= POLLOUT; - if ((rc = poll(pollfd, 1, timeout)) <= 0) - return rc < 0 ? -1 : 0; + // wait for socket to be ready to read or write + if ((rc = poll(&pollfd, 1, timeout)) <= 0) + return rc; - if (!(pollfd[0].revents & mode)) - return -1; - } + mode = 0; + if (pollfd.revents & POLLIN) + mode |= MGET_IO_READABLE; + if (pollfd.revents & POLLOUT) + mode |= MGET_IO_WRITABLE; - return 1; + return mode; } #else -static int _ready_2_transfer(int fd, int timeout, int mode) +int mget_ready_2_transfer(int fd, int timeout, int mode) { // 0: no timeout / immediate // -1: INFINITE timeout @@ -247,7 +264,10 @@ static int _ready_2_transfer(int fd, int timeout, int mode) FD_SET(fd, &fdset); if (mode == MGET_IO_READABLE) { - rc = select(fd + 1, &fdset, NULL, NULL, &tmo); + if (mode == MGET_IO_WRITABLE) + rc = select(fd + 1, &fdset, &fdset, NULL, &tmo); + else + rc = select(fd + 1, &fdset, NULL, NULL, &tmo); } else { rc = select(fd + 1, NULL, &fdset, NULL, &tmo); } @@ -276,7 +296,7 @@ static int _ready_2_transfer(int fd, int timeout, int mode) */ int mget_ready_2_read(int fd, int timeout) { - return _ready_2_transfer(fd, timeout, MGET_IO_READABLE); + return mget_ready_2_transfer(fd, timeout, MGET_IO_READABLE) > 0; } /** @@ -295,7 +315,7 @@ int mget_ready_2_read(int fd, int timeout) */ int mget_ready_2_write(int fd, int timeout) { - return _ready_2_transfer(fd, timeout, MGET_IO_WRITABLE); + return mget_ready_2_transfer(fd, timeout, MGET_IO_WRITABLE) > 0; } char *mget_read_file(const char *fname, size_t *size) diff --git a/libmget/net.c b/libmget/net.c index 0d18a08..e3c3d62 100644 --- a/libmget/net.c +++ b/libmget/net.c @@ -417,6 +417,11 @@ void mget_tcp_set_timeout(mget_tcp_t *tcp, int timeout) (tcp ? tcp : &_global_tcp)->timeout = timeout; } +int mget_tcp_get_timeout(mget_tcp_t *tcp) +{ + return (tcp ? tcp : &_global_tcp)->timeout; +} + void mget_tcp_set_bind_address(mget_tcp_t *tcp, const char *bind_address) { if (!tcp) @@ -539,6 +544,11 @@ static void _set_async(int fd) #endif } +int mget_tcp_ready_2_transfer(mget_tcp_t *tcp, int flags) +{ + return mget_ready_2_transfer(tcp->sockfd, tcp->timeout, flags); +} + int mget_tcp_connect(mget_tcp_t *tcp, const char *host, const char *port) { struct addrinfo *ai; @@ -619,7 +629,7 @@ int mget_tcp_connect(mget_tcp_t *tcp, const char *host, const char *port) mget_tcp_close(tcp); break; /* stop here - the server cert couldn't be validated */ } - + // do not free tcp->addrinfo when calling mget_tcp_close() struct addrinfo *ai_tmp = tcp->addrinfo; tcp->addrinfo = NULL; diff --git a/libmget/ocsp.c b/libmget/ocsp.c index 60b7a07..8ba663c 100644 --- a/libmget/ocsp.c +++ b/libmget/ocsp.c @@ -333,7 +333,7 @@ static int _ocsp_db_load(mget_ocsp_db_t *ocsp_db, const char *fname, int load_ho nentries = mget_hashmap_size(load_hosts ? ocsp_db->hosts : ocsp_db->fingerprints); - debug_printf(_("have %d OCSP %s%s in cache\n"), nentries, load_hosts ? "host" : "fingerprint", nentries !=1 ? "ies" : "y"); + debug_printf(_("have %d OCSP %s%s in cache\n"), nentries, load_hosts ? "host" : "fingerprint", nentries !=1 ? "s" : ""); } else if (errno != ENOENT) error_printf(_("Failed to open OCSP file '%s' (%d)\n"), fname, errno); diff --git a/libmget/ssl_gnutls.c b/libmget/ssl_gnutls.c index 6e097c8..348cc8a 100644 --- a/libmget/ssl_gnutls.c +++ b/libmget/ssl_gnutls.c @@ -1389,30 +1389,34 @@ void mget_ssl_server_close(void **session) ssize_t mget_ssl_read_timeout(void *session, char *buf, size_t count, int timeout) { ssize_t nbytes; - int rc; - - for (;;) { - if (gnutls_record_check_pending(session) <= 0 && - (rc = mget_ready_2_read((int)(ptrdiff_t)gnutls_transport_get_ptr(session), timeout)) <= 0) - return rc; - nbytes=gnutls_record_recv(session, buf, count); + gnutls_record_set_timeout(session, timeout); - if (nbytes >= 0 || nbytes != GNUTLS_E_AGAIN) - break; + if ((nbytes = gnutls_record_recv(session, buf, count)) < 0) { + if (nbytes == GNUTLS_E_AGAIN) + return 0; // indicate timeout + return -1; } - return nbytes < -1 ? -1 : nbytes; + return nbytes; } ssize_t mget_ssl_write_timeout(void *session, const char *buf, size_t count, int timeout) { + ssize_t nbytes; int rc; if ((rc = mget_ready_2_write((int)(ptrdiff_t)gnutls_transport_get_ptr(session), timeout)) <= 0) return rc; - return gnutls_record_send(session, buf, count); + if ((nbytes = gnutls_record_send(session, buf, count)) < 0) { + if (nbytes == GNUTLS_E_AGAIN) + return 0; // indicate timeout + + return -1; + } + + return nbytes; } #else // WITH_GNUTLS diff --git a/src/mget.c b/src/mget.c index 199199f..311f67d 100644 --- a/src/mget.c +++ b/src/mget.c @@ -1097,7 +1097,7 @@ void *downloader_thread(void *p) resp = http_get(job->iri, NULL, downloader, "HEAD"); if (resp) - print_status(downloader, "%d %s\n", resp->code, resp->reason); + print_status(downloader, "HTTP response %d %s\n", resp->code, resp->reason); else if (downloader->final_error) goto ready; } @@ -1189,7 +1189,7 @@ void *downloader_thread(void *p) resp = http_get(job->iri, NULL, downloader, NULL); if (resp) - print_status(downloader, "%d %s\n", resp->code, resp->reason); + print_status(downloader, "HTTP response %d %s\n", resp->code, resp->reason); else if (downloader->final_error) goto ready; } diff --git a/src/options.c b/src/options.c index eede39d..967782e 100644 --- a/src/options.c +++ b/src/options.c @@ -215,6 +215,7 @@ static int G_GNUC_MGET_NORETURN print_help(G_GNUC_MGET_UNUSED option_t opt, G_GN " --ocsp-stapling Use OCSP stapling to verify the server's certificate. (default: on)\n" " --ocsp Use OCSP server access to verify server's certificate. (default: on)\n" " --ocsp-file Set file for OCSP chaching. (default: .mget_ocsp)\n" + " --http2 Use HTTP/2 protocol if possible. (default: on)\n" "\n"); puts( "Directory options:\n" @@ -602,6 +603,9 @@ struct config config = { .tries = 20, .hsts = 1, .hsts_file = ".mget_hsts", +#if defined WITH_LIBNGHTTP2 + .http2 = 1, +#endif .ocsp = 1, .ocsp_stapling = 1, .ocsp_file = ".mget_ocsp" @@ -664,6 +668,7 @@ static const struct option options[] = { { "http-password", &config.http_password, parse_string, 1, 0 }, { "http-proxy", &config.http_proxy, parse_string, 1, 0 }, { "http-user", &config.http_username, parse_string, 1, 0 }, + { "http2", &config.http2, parse_bool, 0, 0 }, { "https-only", &config.https_only, parse_bool, 0, 0 }, { "https-proxy", &config.https_proxy, parse_string, 1, 0 }, { "ignore-case", &config.ignore_case, parse_bool, 0, 0 }, @@ -1246,6 +1251,12 @@ int init(int argc, const char *const *argv) " -bzip2" #endif +#if defined WITH_LIBNGHTTP2 + " +http2" +#else + " -http2" +#endif + "\n"); } @@ -1355,7 +1366,8 @@ int init(int argc, const char *const *argv) mget_ssl_set_config_string(MGET_SSL_KEY_FILE, config.private_key); mget_ssl_set_config_string(MGET_SSL_CRL_FILE, config.crl_file); mget_ssl_set_config_string(MGET_SSL_OCSP_CACHE, (const char *)config.ocsp_db); - mget_ssl_set_config_string(MGET_SSL_ALPN, "h2,h2-16,h2-14,spdy/3.1,http/1.1"); + if (config.http2) + mget_ssl_set_config_string(MGET_SSL_ALPN, "h2,h2-16,h2-14,spdy/3.1,http/1.1"); // convert host lists to lowercase for (int it = 0; it < mget_vector_size(config.domains); it++) { diff --git a/src/options.h b/src/options.h index d351440..f198999 100644 --- a/src/options.h +++ b/src/options.h @@ -112,6 +112,7 @@ struct config { struct mget_cookie_db_st *cookie_db; char + http2, ocsp_stapling, ocsp, mirror,