sk-tcp: Improve TCP socket options handling

criu/include/sk-inet.h

@@ -87,6 +87,9 @@ extern void cpt_unlock_tcp_connections(void);
 extern int dump_one_tcp(int sk, struct inet_sk_desc *sd, SkOptsEntry *soe);
 extern int restore_one_tcp(int sk, struct inet_sk_info *si);
 
+extern int dump_tcp_opts(int sk, TcpOptsEntry *toe);
+extern int restore_tcp_opts(int sk, TcpOptsEntry *toe);
+
 #define SK_EST_PARAM	  "tcp-established"
 #define SK_INFLIGHT_PARAM "skip-in-flight"
 #define SK_CLOSE_PARAM	  "tcp-close"

criu/sk-inet.c

@@ -454,6 +454,7 @@ static int do_dump_one_inet_fd(int lfd, u32 id, const struct fd_parms *p, int fa
 	IpOptsEntry ipopts = IP_OPTS_ENTRY__INIT;
 	IpOptsRawEntry ipopts_raw = IP_OPTS_RAW_ENTRY__INIT;
 	SkOptsEntry skopts = SK_OPTS_ENTRY__INIT;
+	TcpOptsEntry tcpopts = TCP_OPTS_ENTRY__INIT;
 	int ret = -1, err = -1, proto, aux, type;
 
 	ret = do_dump_opt(lfd, SOL_SOCKET, SO_PROTOCOL, &proto, sizeof(proto));
@@ -521,6 +522,7 @@ static int do_dump_one_inet_fd(int lfd, u32 id, const struct fd_parms *p, int fa
 	ie.opts = &skopts;
 	ie.ip_opts = &ipopts;
 	ie.ip_opts->raw = &ipopts_raw;
+	ie.tcp_opts = &tcpopts;
 
 	ie.n_src_addr = PB_ALEN_INET;
 	ie.n_dst_addr = PB_ALEN_INET;
@@ -581,9 +583,20 @@ static int do_dump_one_inet_fd(int lfd, u32 id, const struct fd_parms *p, int fa
 
 	switch (proto) {
 	case IPPROTO_TCP:
-		err = (type != SOCK_RAW) ? dump_one_tcp(lfd, sk, &skopts) : 0;
 		if (sk->shutdown)
 			sk_encode_shutdown(&ie, sk->shutdown);
+
+		if (type == SOCK_RAW) {
+			err = 0;
+		} else {
+			err = dump_tcp_opts(lfd, &tcpopts);
+			if (err < 0)
+				goto err;
+
+			err = dump_one_tcp(lfd, sk, &skopts);
+			if (err < 0)
+				goto err;
+		}
 		break;
 	case IPPROTO_UDP:
 	case IPPROTO_UDPLITE:
@@ -939,6 +952,9 @@ static int open_inet_sk(struct file_desc *d, int *new_fd)
 	if (restore_socket_opts(sk, ie->opts))
 		goto err;
 
+	if (ie->proto == IPPROTO_TCP && restore_tcp_opts(sk, ie->tcp_opts))
+		goto err;
+
 	if (ie->has_shutdown &&
 	    (ie->proto == IPPROTO_UDP || ie->proto == IPPROTO_UDPLITE || ie->proto == IPPROTO_TCP)) {
 		if (shutdown(sk, sk_decode_shutdown(ie->shutdown))) {

criu/sk-tcp.c

@@ -136,7 +136,7 @@
 {
 	struct libsoccr_sk *socr = sk->priv;
 	int exit_code = -1;
-	int ret, aux;
+	int ret;
 	struct cr_img *img;
 	TcpStreamEntry tse = TCP_STREAM_ENTRY__INIT;
 	char *buf;
@@ -186,26 +186,6 @@
 		tse.rcv_wup = data.rcv_wup;
 	}
 
-	/*
-	 * TCP socket options
-	 */
-
-	if (dump_opt(sk->rfd, SOL_TCP, TCP_NODELAY, &aux))
-		goto err;
-
-	if (aux) {
-		tse.has_nodelay = true;
-		tse.nodelay = true;
-	}
-
-	if (dump_opt(sk->rfd, SOL_TCP, TCP_CORK, &aux))
-		goto err;
-
-	if (aux) {
-		tse.has_cork = true;
-		tse.cork = true;
-	}
-
 	/*
 	 * Push the stuff to image
 	 */
@@ -243,26 +223,27 @@
 	return exit_code;
 }
 
-int dump_one_tcp(int fd, struct inet_sk_desc *sk, SkOptsEntry *soe)
+int dump_tcp_opts(int fd, TcpOptsEntry *toe)
 {
-	soe->has_tcp_keepcnt = true;
-	if (dump_opt(fd, SOL_TCP, TCP_KEEPCNT, &soe->tcp_keepcnt)) {
-		pr_perror("Can't read TCP_KEEPCNT");
-		return -1;
-	}
+	int ret = 0;
 
-	soe->has_tcp_keepidle = true;
-	if (dump_opt(fd, SOL_TCP, TCP_KEEPIDLE, &soe->tcp_keepidle)) {
-		pr_perror("Can't read TCP_KEEPIDLE");
-		return -1;
-	}
+	ret |= dump_opt(fd, SOL_TCP, TCP_NODELAY, &toe->nodelay);
+	ret |= dump_opt(fd, SOL_TCP, TCP_CORK, &toe->cork);
+	ret |= dump_opt(fd, SOL_TCP, TCP_KEEPCNT, &toe->keepcnt);
+	ret |= dump_opt(fd, SOL_TCP, TCP_KEEPIDLE, &toe->keepidle);
+	ret |= dump_opt(fd, SOL_TCP, TCP_KEEPINTVL, &toe->keepintvl);
 
-	soe->has_tcp_keepintvl = true;
-	if (dump_opt(fd, SOL_TCP, TCP_KEEPINTVL, &soe->tcp_keepintvl)) {
-		pr_perror("Can't read TCP_KEEPINTVL");
-		return -1;
-	}
+	toe->has_nodelay = !!toe->nodelay;
+	toe->has_cork = !!toe->cork;
+	toe->has_keepcnt = !!toe->keepcnt;
+	toe->has_keepidle = !!toe->keepidle;
+	toe->has_keepintvl = !!toe->keepintvl;
+
+	return ret;
+}
 
+int dump_one_tcp(int fd, struct inet_sk_desc *sk, SkOptsEntry *soe)
+{
 	if (sk->dst_port == 0)
 		return 0;
 
@@ -396,6 +377,11 @@
 	if (libsoccr_restore(socr, &data, sizeof(data)))
 		goto err_c;
 
+	/*
+	 * Restoring TCP socket options in TcpStreamEntry is
+	 * for backward compatibility only, newer versions
+	 * of CRIU use TcpOptsEntry.
+	 */
 	if (tse->has_nodelay && tse->nodelay) {
 		aux = 1;
 		if (restore_opt(sk, SOL_TCP, TCP_NODELAY, &aux))
@@ -448,6 +434,27 @@
 	return 0;
 }
 
+int restore_tcp_opts(int sk, TcpOptsEntry *toe)
+{
+	int ret = 0;
+
+	if(!toe)
+		return ret;
+
+	if (toe->has_nodelay)
+		ret |= restore_opt(sk, SOL_TCP, TCP_NODELAY, &toe->nodelay);
+	if (toe->has_cork)
+		ret |= restore_opt(sk, SOL_TCP, TCP_CORK, &toe->cork);
+	if (toe->has_keepcnt)
+		ret |= restore_opt(sk, SOL_TCP, TCP_KEEPCNT, &toe->keepcnt);
+	if (toe->has_keepidle)
+		ret |= restore_opt(sk, SOL_TCP, TCP_KEEPIDLE, &toe->keepidle);
+	if (toe->has_keepintvl)
+		ret |= restore_opt(sk, SOL_TCP, TCP_KEEPINTVL, &toe->keepintvl);
+
+	return ret;
+}
+
 int restore_one_tcp(int fd, struct inet_sk_info *ii)
 {
 	struct libsoccr_sk *sk;

criu/sockets.c

@@ -585,6 +585,12 @@ int restore_socket_opts(int sk, SkOptsEntry *soe)
 		pr_debug("\tset keepalive for socket\n");
 		ret |= restore_opt(sk, SOL_SOCKET, SO_KEEPALIVE, &val);
 	}
+
+	/*
+	 * Restoring TCP socket options in SkOptsEntry is
+	 * for backward compatibility only, newer versions
+	 * of CRIU use TcpOptsEntry.
+	 */
 	if (soe->has_tcp_keepcnt) {
 		pr_debug("\tset keepcnt for socket\n");
 		ret |= restore_opt(sk, SOL_TCP, TCP_KEEPCNT, &soe->tcp_keepcnt);

images/sk-inet.proto

@@ -5,6 +5,7 @@ syntax = "proto2";
 import "opts.proto";
 import "fown.proto";
 import "sk-opts.proto";
+import "tcp-stream.proto";
 
 message ip_opts_raw_entry {
 	optional bool			hdrincl		= 1;
@@ -56,4 +57,5 @@ message inet_sk_entry {
 	optional string			ifname		= 17;
 	optional uint32			ns_id		= 18;
 	optional sk_shutdown		shutdown	= 19;
+	optional tcp_opts_entry		tcp_opts	= 20;
 }

images/sk-opts.proto

@@ -26,9 +26,12 @@ message sk_opts_entry {
 	optional bool		so_reuseport	= 17;
 	optional bool		so_broadcast	= 18;
 	optional bool		so_keepalive	= 19;
+
+	/* These three are deprecated, use tcp_opts_entry instead */
 	optional uint32		tcp_keepcnt	= 20;
 	optional uint32		tcp_keepidle	= 21;
 	optional uint32		tcp_keepintvl	= 22;
+
 	optional uint32		so_oobinline	= 23;
 	optional uint32		so_linger	= 24;
 

images/tcp-stream.proto

@@ -4,6 +4,14 @@ syntax = "proto2";
 
 import "opts.proto";
 
+message tcp_opts_entry {
+	optional bool		cork		= 1;
+	optional bool		nodelay		= 2;
+	optional uint32		keepcnt		= 3;
+	optional uint32		keepidle	= 4;
+	optional uint32		keepintvl	= 5;
+}
+
 message tcp_stream_entry {
 	required uint32		inq_len		= 1;
 	required uint32		inq_seq		= 2;
@@ -16,6 +24,7 @@ message tcp_stream_entry {
 	optional uint32		rcv_wscale	= 8;
 	optional uint32		timestamp	= 9;
 
+	/* These two are deprecated, use tcp_opts_entry instead */
 	optional bool		cork		= 10;
 	optional bool		nodelay		= 11;
 

test/zdtm/static/Makefile

@@ -127,6 +127,8 @@ TST_NOFILE	:=				\
 		sock_opts02			\
 		sock_ip_opts00			\
 		sock_ip_opts01			\
+		sock_tcp_opts00			\
+		sock_tcp_opts01			\
 		sk-unix-unconn			\
 		sk-unix-unconn-seqpacket	\
 		ipc_namespace			\
@@ -609,6 +611,7 @@ socket-tcp6-closed:	CFLAGS += -D ZDTM_IPV4V6
 socket-tcp-closed-last-ack:	CFLAGS += -D ZDTM_TCP_LAST_ACK
 socket-tcp-skip-in-flight:	CFLAGS += -D ZDTM_IPV4V6
 sock_ip_opts01:		CFLAGS += -DZDTM_VAL_ZERO
+sock_tcp_opts01:	CFLAGS += -DZDTM_VAL_ZERO
 tun_ns:			CFLAGS += -DTUN_NS
 mnt_ext_manual:		CFLAGS += -D ZDTM_EXTMAP_MANUAL
 mntns_pivot_root_ro:	CFLAGS += -DMNTNS_PIVOT_ROOT_RO

test/zdtm/static/sock_tcp_opts00.c

@@ -0,0 +1,96 @@
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+
+#include "zdtmtst.h"
+
+const char *test_doc = "Check that different tcp socket options are restored";
+const char *test_author = "Juntong Deng <juntong.deng@outlook.com>";
+
+#ifdef ZDTM_VAL_ZERO
+#define TCP_OPT_VAL 0
+#else
+#define TCP_OPT_VAL 1
+#endif
+
+#ifndef SOL_TCP
+#define SOL_TCP 6
+#endif
+
+struct sk_opt {
+	int level;
+	int opt;
+	int val;
+};
+
+struct sk_opt tcp_sk_opts[] = {
+	{ SOL_TCP, TCP_CORK, TCP_OPT_VAL },
+	{ SOL_TCP, TCP_NODELAY, TCP_OPT_VAL },
+};
+
+struct sk_conf {
+	int domain;
+	int type;
+	int protocol;
+	int sk;
+} sk_confs[] = {
+	{ AF_INET, SOCK_STREAM, IPPROTO_TCP },
+	{ AF_INET6, SOCK_STREAM, IPPROTO_TCP },
+};
+
+int main(int argc, char **argv)
+{
+	struct sk_opt *opts = tcp_sk_opts;
+	int n_opts = ARRAY_SIZE(tcp_sk_opts);
+	int exit_code = 1;
+	int i, j, val;
+	socklen_t len;
+
+	test_init(argc, argv);
+
+	for (i = 0; i < ARRAY_SIZE(sk_confs); i++) {
+		sk_confs[i].sk = socket(sk_confs[i].domain, sk_confs[i].type, sk_confs[i].protocol);
+		if (sk_confs[i].sk == -1) {
+			pr_perror("socket(%d,%d,%d) failed", sk_confs[i].domain, sk_confs[i].type,
+				  sk_confs[i].protocol);
+			goto close;
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(sk_confs); i++) {
+		for (j = 0; j < n_opts; j++) {
+			val = opts[j].val;
+			if (setsockopt(sk_confs[i].sk, opts[j].level, opts[j].opt, &val, sizeof(int)) == -1) {
+				pr_perror("setsockopt(%d, %d) failed", opts[j].level, opts[j].opt);
+				goto close;
+			}
+		}
+	}
+
+	test_daemon();
+	test_waitsig();
+
+	for (i = 0; i < ARRAY_SIZE(sk_confs); i++) {
+		for (j = 0; j < n_opts; j++) {
+			len = sizeof(int);
+			if (getsockopt(sk_confs[i].sk, opts[j].level, opts[j].opt, &val, &len) == -1) {
+				pr_perror("getsockopt(%d, %d) failed", opts[j].level, opts[j].opt);
+				goto close;
+			}
+
+			if (val != opts[j].val) {
+				fail("Unexpected value socket(%d,%d,%d) opts(%d,%d)", sk_confs[i].domain,
+				     sk_confs[i].type, sk_confs[i].protocol, opts[j].level, opts[j].opt);
+				goto close;
+			}
+		}
+	}
+
+	pass();
+	exit_code = 0;
+close:
+	for (i = 0; i < ARRAY_SIZE(sk_confs); i++)
+		close(sk_confs[i].sk);
+	return exit_code;
+}

test/zdtm/static/sock_tcp_opts00.desc

@@ -0,0 +1 @@
+{'flags': 'suid'}

test/zdtm/static/sock_tcp_opts01.c

@@ -0,0 +1 @@
+./sock_tcp_opts00.c

test/zdtm/static/sock_tcp_opts01.desc

@@ -0,0 +1 @@
+./sock_tcp_opts00.desc