More HTTP/2 related code, new option --(no-)http2

configure.ac

@@ -357,7 +357,7 @@ AC_FUNC_ALLOCA
 AC_FUNC_FORK
 #AC_FUNC_MALLOC
 AC_FUNC_MMAP
-AC_FUNC_REALLOC
+#AC_FUNC_REALLOC
 AC_FUNC_FNMATCH
 AC_CHECK_FUNCS([\
  clock_gettime dprintf dup2 futimens gettimeofday localtime_r memchr\

docs/libmget/libmget-sections.txt

@@ -41,5 +41,6 @@ mget_fdgetline
 mget_getline
 mget_ready_2_read
 mget_ready_2_write
+mget_ready_2_transfer
 mget_read_file
 </SECTION>

include/libmget.h

@@ -263,6 +263,8 @@ int
 	mget_ready_2_read(int fd, int timeout);
 int
 	mget_ready_2_write(int fd, int timeout);
+int
+	mget_ready_2_transfer(int fd, int timeout, short mode);
 int
 	mget_strcmp(const char *s1, const char *s2) G_GNUC_MGET_PURE;
 int
@@ -1144,6 +1146,8 @@ void
 	mget_tcp_close(mget_tcp_t *tcp);
 void
 	mget_tcp_set_timeout(mget_tcp_t *tcp, int timeout);
+int
+	mget_tcp_get_timeout(mget_tcp_t *tcp) G_GNUC_MGET_PURE;
 void
 	mget_tcp_set_connect_timeout(mget_tcp_t *tcp, int timeout);
 void
@@ -1204,6 +1208,8 @@ ssize_t
 	mget_tcp_write(mget_tcp_t *tcp, const char *buf, size_t count) G_GNUC_MGET_NONNULL_ALL;
 ssize_t
 	mget_tcp_read(mget_tcp_t *tcp, char *buf, size_t count) G_GNUC_MGET_NONNULL_ALL;
+int
+	mget_tcp_ready_2_transfer(mget_tcp_t *tcp, int flags) G_GNUC_MGET_NONNULL_ALL;
 
 /*
  * SSL routines

libmget/http.c

@@ -1476,13 +1476,13 @@ static ssize_t _send_callback(nghttp2_session *session G_GNUC_MGET_UNUSED,
 	mget_http_connection_t *conn = (mget_http_connection_t *)user_data;
 	int rc;
 
-	debug_printf("writing... %zd\n", length);
+	// debug_printf("writing... %zd\n", length);
 	if ((rc = mget_tcp_write(conn->tcp, (const char *)data, length)) <= 0) {
 		// An error will be written by the mget_tcp_write function.
-		debug_printf("write rc %d, errno=%d\n", rc, errno);
+		// debug_printf("write rc %d, errno=%d\n", rc, errno);
 		return rc ? NGHTTP2_ERR_CALLBACK_FAILURE : NGHTTP2_ERR_WOULDBLOCK;
 	}
-	debug_printf("write rc %d\n",rc);
+	// debug_printf("write rc %d\n",rc);
 
 	return rc;
 }
@@ -1493,57 +1493,59 @@ static ssize_t _recv_callback(nghttp2_session *session G_GNUC_MGET_UNUSED,
 	mget_http_connection_t *conn = (mget_http_connection_t *)user_data;
 	int rc;
 
-	debug_printf("reading... %zd\n", length);
+	// debug_printf("reading... %zd\n", length);
 	if ((rc = mget_tcp_read(conn->tcp, (char *)buf, length)) <= 0) {
 		//  0 = timeout resp. blocking
 		// -1 = failure
-		debug_printf("read rc %d, errno=%d\n", rc, errno);
+		// debug_printf("read rc %d, errno=%d\n", rc, errno);
 		return rc ? NGHTTP2_ERR_CALLBACK_FAILURE : NGHTTP2_ERR_WOULDBLOCK;
 	}
-	debug_printf("read rc %d\n",rc);
+	// debug_printf("read rc %d\n",rc);
 
 	return rc;
 }
 
-static int _on_frame_send_callback(nghttp2_session *session,
+static void _print_frame_type(int type, const char tag)
+{
+	static const char *name[] = {
+		[NGHTTP2_DATA] = "DATA",
+		[NGHTTP2_HEADERS] = "HEADERS",
+		[NGHTTP2_PRIORITY] = "PRIORITY",
+		[NGHTTP2_RST_STREAM] = "RST_STREAM",
+		[NGHTTP2_SETTINGS] = "SETTINGS",
+		[NGHTTP2_PUSH_PROMISE] = "PUSH_PROMISE",
+		[NGHTTP2_PING] = "PING",
+		[NGHTTP2_GOAWAY] = "GOAWAY",
+		[NGHTTP2_WINDOW_UPDATE] = "WINDOW_UPDATE",
+		[NGHTTP2_CONTINUATION] = "CONTINUATION"
+	};
+
+	if ((unsigned) type < countof(name))
+		debug_printf("[FRAME] %c %s\n", tag, name[type]);
+	else
+		debug_printf("[FRAME] %c Unknown type %d\n", tag, type);
+}
+
+static int _on_frame_send_callback(nghttp2_session *session G_GNUC_MGET_UNUSED,
 	const nghttp2_frame *frame, void *user_data G_GNUC_MGET_UNUSED)
 {
-	switch (frame->hd.type) {
-	case NGHTTP2_HEADERS:
-		if (nghttp2_session_get_stream_user_data(session, frame->hd.stream_id)) {
-			const nghttp2_nv *nva = frame->headers.nva;
-			debug_printf("[INFO] C ----------------------------> S (HEADERS)\n");
-			for (unsigned i = 0; i < frame->headers.nvlen; i++)
-				debug_printf("%.*s: %.*s\n", (int)nva[i].namelen, nva[i].name, (int)nva[i].valuelen, nva[i].value);
-		}
-		break;
-	case NGHTTP2_RST_STREAM:
-		debug_printf("[INFO] C ----------------------------> S (RST_STREAM)\n");
-		break;
-	case NGHTTP2_GOAWAY:
-		debug_printf("[INFO] C ----------------------------> S (GOAWAY)\n");
-		break;
+	_print_frame_type(frame->hd.type, '>');
+
+	if (frame->hd.type == NGHTTP2_HEADERS) {
+		const nghttp2_nv *nva = frame->headers.nva;
+
+		for (unsigned i = 0; i < frame->headers.nvlen; i++)
+			debug_printf("[FRAME] > %.*s: %.*s\n", (int)nva[i].namelen, nva[i].name, (int)nva[i].valuelen, nva[i].value);
 	}
+
 	return 0;
 }
 
 static int _on_frame_recv_callback(nghttp2_session *session G_GNUC_MGET_UNUSED,
 	const nghttp2_frame *frame, void *user_data G_GNUC_MGET_UNUSED)
 {
-	switch (frame->hd.type) {
-	case NGHTTP2_HEADERS:
-		debug_printf("[INFO] HEADERS cat %d\n", frame->headers.cat);
-		break;
-	case NGHTTP2_RST_STREAM:
-		debug_printf("[INFO] C <---------------------------- S (RST_STREAM)\n");
-		break;
-	case NGHTTP2_GOAWAY:
-		debug_printf("[INFO] C <---------------------------- S (GOAWAY)\n");
-		break;
-	default:
-		debug_printf("[INFO] C <---------------------------- S (hd.type %d)\n", frame->hd.type);
-		break;
-	}
+	_print_frame_type(frame->hd.type, '<');
+
 	return 0;
 }
 
@@ -2006,6 +2008,7 @@ mget_http_response_t *mget_http_get_response_cb(
 	char *buf, *p = NULL;
 	mget_http_response_t *resp = NULL;
 	mget_decompressor_t *dc = NULL;
+	int ioflags;
 
 #ifdef WITH_LIBNGHTTP2
 	if (conn->protocol == MGET_PROTOCOL_HTTP_2_0) {
@@ -2017,19 +2020,38 @@ mget_http_response_t *mget_http_get_response_cb(
 		struct _body_callback_context ctx = { .resp = resp, .context = context, .body_callback = body_callback };
 		req->nghttp2_context = &ctx;
 
-		mget_tcp_set_timeout(conn->tcp, 1); // 1ms timeout
+		int timeout = mget_tcp_get_timeout(conn->tcp);
+
 		for (int rc = 0; rc == 0 && !ctx.done;) {
-			debug_printf("0 response status %d done %d\n", resp->code, ctx.done);
+			ioflags = 0;
+			if (nghttp2_session_want_write(conn->http2_session))
+				ioflags |= MGET_IO_WRITABLE;
+			if (nghttp2_session_want_read(conn->http2_session))
+				ioflags |= MGET_IO_READABLE;
+
+			if (ioflags)
+				ioflags = mget_tcp_ready_2_transfer(conn->tcp, ioflags);
+			// debug_printf("ioflags=%d timeout=%d\n",ioflags,mget_tcp_get_timeout(conn->tcp));
+			if (ioflags <= 0) break; // error or timeout
+
+			mget_tcp_set_timeout(conn->tcp, 0); // 0 = immediate
+			rc = 0;
+			if (ioflags & MGET_IO_WRITABLE) {
+				rc = nghttp2_session_send(conn->http2_session);
+			}
+			if (!rc && (ioflags & MGET_IO_READABLE))
+				rc = nghttp2_session_recv(conn->http2_session);
+			mget_tcp_set_timeout(conn->tcp, timeout); // restore old timeout
+
+/*
 			while (nghttp2_session_want_write(conn->http2_session)) {
 				rc = nghttp2_session_send(conn->http2_session);
 			}
 			debug_printf("1 response status %d done %d\n", resp->code, ctx.done);
 			if (nghttp2_session_want_read(conn->http2_session)) {
 				rc = nghttp2_session_recv(conn->http2_session);
 			}
-			debug_printf("2 response status %d done %d\n", resp->code, ctx.done);
-			if (rc)
-				debug_printf("loop rc = %d\n", rc);
+*/
 		}
 
 		debug_printf("response status %d\n", resp->code);

libmget/io.c

@@ -206,34 +206,51 @@ ssize_t mget_getline(char **buf, size_t *bufsize, FILE *fp)
 	return -1;
 }
 
+/**
+ * mget_ready_2_transfer:
+ * @fd: File descriptor to wait for.
+ * @timeout: Max. duration in milliseconds to wait.
+ * A value of 0 means the function returns immediately.
+ * A value of -1 means infinite timeout.
+ *
+ * Wait for a file descriptor to become ready to read or write.
+ *
+ * Returns:
+ * -1 on error.
+ * 0 on timeout. The file descriptor is not ready for reading nor writing.
+ * >0 The file descriptor is ready for reading or writing. Check for
+ * the bitwise or'ing of MGET_IO_WRITABLE and MGET_IO_WRITABLE.
+ *
+ */
 #ifdef POLLIN
-static int _ready_2_transfer(int fd, int timeout, short mode)
+int mget_ready_2_transfer(int fd, int timeout, short mode)
 {
-	// 0: no timeout / immediate
-	// -1: INFINITE timeout
-	// >0: number of milliseconds to wait
-	if (timeout) {
-		int rc;
+	struct pollfd pollfd;
+	int rc;
 
-		if (mode == MGET_IO_READABLE)
-			mode = POLLIN;
-		else
-			mode = POLLOUT;
+	pollfd.fd = fd;
+	pollfd.events = 0;
+	pollfd.revents = 0;
 
-		// wait for socket to be ready to read
-		struct pollfd pollfd[1] = { { fd, mode, 0 } };
+	if (mode & MGET_IO_READABLE)
+		pollfd.events |= POLLIN;
+	if (mode & MGET_IO_WRITABLE)
+		pollfd.events |= POLLOUT;
 
-		if ((rc = poll(pollfd, 1, timeout)) <= 0)
-			return rc < 0 ? -1 : 0;
+	// wait for socket to be ready to read or write
+	if ((rc = poll(&pollfd, 1, timeout)) <= 0)
+		return rc;
 
-		if (!(pollfd[0].revents & mode))
-			return -1;
-	}
+	mode = 0;
+	if (pollfd.revents & POLLIN)
+		mode |= MGET_IO_READABLE;
+	if (pollfd.revents & POLLOUT)
+		mode |= MGET_IO_WRITABLE;
 
-	return 1;
+	return mode;
 }
 #else
-static int _ready_2_transfer(int fd, int timeout, int mode)
+int mget_ready_2_transfer(int fd, int timeout, int mode)
 {
 	// 0: no timeout / immediate
 	// -1: INFINITE timeout
@@ -247,7 +264,10 @@ static int _ready_2_transfer(int fd, int timeout, int mode)
 		FD_SET(fd, &fdset);
 
 		if (mode == MGET_IO_READABLE) {
-			rc = select(fd + 1, &fdset, NULL, NULL, &tmo);
+			if (mode == MGET_IO_WRITABLE)
+				rc = select(fd + 1, &fdset, &fdset, NULL, &tmo);
+			else
+				rc = select(fd + 1, &fdset, NULL, NULL, &tmo);
 		} else {
 			rc = select(fd + 1, NULL, &fdset, NULL, &tmo);
 		}
@@ -276,7 +296,7 @@ static int _ready_2_transfer(int fd, int timeout, int mode)
  */
 int mget_ready_2_read(int fd, int timeout)
 {
-	return _ready_2_transfer(fd, timeout, MGET_IO_READABLE);
+	return mget_ready_2_transfer(fd, timeout, MGET_IO_READABLE) > 0;
 }
 
 /**
@@ -295,7 +315,7 @@ int mget_ready_2_read(int fd, int timeout)
  */
 int mget_ready_2_write(int fd, int timeout)
 {
-	return _ready_2_transfer(fd, timeout, MGET_IO_WRITABLE);
+	return mget_ready_2_transfer(fd, timeout, MGET_IO_WRITABLE) > 0;
 }
 
 char *mget_read_file(const char *fname, size_t *size)

libmget/net.c

@@ -417,6 +417,11 @@ void mget_tcp_set_timeout(mget_tcp_t *tcp, int timeout)
 	(tcp ? tcp : &_global_tcp)->timeout = timeout;
 }
 
+int mget_tcp_get_timeout(mget_tcp_t *tcp)
+{
+	return (tcp ? tcp : &_global_tcp)->timeout;
+}
+
 void mget_tcp_set_bind_address(mget_tcp_t *tcp, const char *bind_address)
 {
 	if (!tcp)
@@ -539,6 +544,11 @@ static void _set_async(int fd)
 #endif
 }
 
+int mget_tcp_ready_2_transfer(mget_tcp_t *tcp, int flags)
+{
+	return mget_ready_2_transfer(tcp->sockfd, tcp->timeout, flags);
+}
+
 int mget_tcp_connect(mget_tcp_t *tcp, const char *host, const char *port)
 {
 	struct addrinfo *ai;
@@ -619,7 +629,7 @@ int mget_tcp_connect(mget_tcp_t *tcp, const char *host, const char *port)
 							mget_tcp_close(tcp);
 							break; /* stop here - the server cert couldn't be validated */
 						}
-						
+
 						// do not free tcp->addrinfo when calling mget_tcp_close()
 						struct addrinfo *ai_tmp = tcp->addrinfo;
 						tcp->addrinfo = NULL;

libmget/ocsp.c

@@ -333,7 +333,7 @@ static int _ocsp_db_load(mget_ocsp_db_t *ocsp_db, const char *fname, int load_ho
 
 		nentries = mget_hashmap_size(load_hosts ? ocsp_db->hosts : ocsp_db->fingerprints);
 
-		debug_printf(_("have %d OCSP %s%s in cache\n"), nentries, load_hosts ? "host" : "fingerprint", nentries !=1 ? "ies" : "y");
+		debug_printf(_("have %d OCSP %s%s in cache\n"), nentries, load_hosts ? "host" : "fingerprint", nentries !=1 ? "s" : "");
 	} else if (errno != ENOENT)
 		error_printf(_("Failed to open OCSP file '%s' (%d)\n"), fname, errno);
 

libmget/ssl_gnutls.c

@@ -1389,30 +1389,34 @@ void mget_ssl_server_close(void **session)
 ssize_t mget_ssl_read_timeout(void *session, char *buf, size_t count, int timeout)
 {
 	ssize_t nbytes;
-	int rc;
-
-	for (;;) {
-		if (gnutls_record_check_pending(session) <= 0 &&
-			(rc = mget_ready_2_read((int)(ptrdiff_t)gnutls_transport_get_ptr(session), timeout)) <= 0)
-			return rc;
 
-		nbytes=gnutls_record_recv(session, buf, count);
+	gnutls_record_set_timeout(session, timeout);
 
-		if (nbytes >= 0 || nbytes != GNUTLS_E_AGAIN)
-			break;
+	if ((nbytes = gnutls_record_recv(session, buf, count)) < 0) {
+		if (nbytes == GNUTLS_E_AGAIN)
+			return 0; // indicate timeout
+		return -1;
 	}
 
-	return nbytes < -1 ? -1 : nbytes;
+	return nbytes;
 }
 
 ssize_t mget_ssl_write_timeout(void *session, const char *buf, size_t count, int timeout)
 {
+	ssize_t nbytes;
 	int rc;
 
 	if ((rc = mget_ready_2_write((int)(ptrdiff_t)gnutls_transport_get_ptr(session), timeout)) <= 0)
 		return rc;
 
-	return gnutls_record_send(session, buf, count);
+	if ((nbytes = gnutls_record_send(session, buf, count)) < 0) {
+		if (nbytes == GNUTLS_E_AGAIN)
+			return 0; // indicate timeout
+
+		return -1;
+	}
+
+	return nbytes;
 }
 
 #else // WITH_GNUTLS