OS-6538 lxbrand: most restartable syscalls not handled correctly

Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Approved by: Patrick Mooney <patrick.mooney@joyent.com>

usr/src/lib/brand/lx/lx_brand/Makefile.com

@@ -21,7 +21,7 @@
 #
 # Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
-# Copyright 2017 Joyent, Inc.
+# Copyright 2018 Joyent, Inc.
 #
 
 LX_CMN  =	$(SRC)/common/brand/lx
@@ -34,7 +34,6 @@ COBJS	=	capabilities.o		\
 		debug.o			\
 		dir.o			\
 		file.o			\
-		fcntl.o			\
 		fork.o			\
 		lx_brand.o		\
 		misc.o			\

usr/src/lib/brand/lx/lx_brand/common/lx_brand.c

@@ -25,7 +25,7 @@
  */
 
 /*
- * Copyright 2017 Joyent, Inc.
+ * Copyright 2018 Joyent, Inc.
  */
 
 #include <sys/types.h>
@@ -1088,7 +1088,7 @@ static lx_syscall_handler_t lx_handlers[] = {
 	lx_msgrcv,			/*  70: msgrcv */
 	lx_msgctl,			/*  71: msgctl */
 	NULL,				/*  72: fcntl */
-	lx_flock,			/*  73: flock */
+	NULL,				/*  73: flock */
 	lx_fsync,			/*  74: fsync */
 	lx_fdatasync,			/*  75: fdatasync */
 	lx_truncate,			/*  76: truncate */
@@ -1489,7 +1489,7 @@ static lx_syscall_handler_t lx_handlers[] = {
 	NULL,				/* 140: llseek */
 	NULL,				/* 141: getdents */
 	NULL,				/* 142: select */
-	lx_flock,			/* 143: flock */
+	NULL,				/* 143: flock */
 	NULL,				/* 144: msync */
 	NULL,				/* 145: readv */
 	NULL,				/* 146: writev */

usr/src/lib/brand/lx/lx_brand/sys/lx_syscall.h

@@ -25,7 +25,7 @@
  */
 
 /*
- * Copyright 2017 Joyent, Inc.
+ * Copyright 2018 Joyent, Inc.
  */
 
 #ifndef _SYS_LX_SYSCALL_H
@@ -57,7 +57,6 @@ extern long lx_fstat64(uintptr_t, uintptr_t);
 extern long lx_lstat64(uintptr_t, uintptr_t);
 extern long lx_fcntl(uintptr_t, uintptr_t, uintptr_t);
 extern long lx_fcntl64(uintptr_t, uintptr_t, uintptr_t);
-extern long lx_flock(uintptr_t, uintptr_t);
 extern long lx_readdir(uintptr_t, uintptr_t, uintptr_t);
 extern long lx_execve(uintptr_t, uintptr_t, uintptr_t);
 extern long lx_ioctl(uintptr_t, uintptr_t, uintptr_t);

usr/src/uts/common/brand/lx/os/lx_syscall.c

@@ -22,7 +22,7 @@
 /*
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
- * Copyright 2017 Joyent, Inc.
+ * Copyright 2018 Joyent, Inc.
  */
 
 #include <sys/kmem.h>
@@ -662,7 +662,7 @@ lx_sysent_t lx_sysent32[] = {
 	{"llseek",	lx_llseek,		0,		5}, /* 140 */
 	{"getdents",	lx_getdents_32,		0,		3}, /* 141 */
 	{"select",	lx_select,		0,		5}, /* 142 */
-	{"flock",	NULL,			0,		2}, /* 143 */
+	{"flock",	lx_flock,		0,		2}, /* 143 */
 	{"msync",	lx_msync,		0,		3}, /* 144 */
 	{"readv",	lx_readv,		0,		3}, /* 145 */
 	{"writev",	lx_writev,		0,		3}, /* 146 */
@@ -963,7 +963,7 @@ lx_sysent_t lx_sysent64[] = {
 	{"msgrcv",	NULL,			0,		5}, /* 70 */
 	{"msgctl",	NULL,			0,		3}, /* 71 */
 	{"fcntl",	lx_fcntl64,		0,		3}, /* 72 */
-	{"flock",	NULL,			0,		2}, /* 73 */
+	{"flock",	lx_flock,		0,		2}, /* 73 */
 	{"fsync",	NULL,			0,		1}, /* 74 */
 	{"fdatasync",	NULL,			0,		1}, /* 75 */
 	{"truncate",	NULL,			0,		2}, /* 76 */

usr/src/uts/common/brand/lx/sys/lx_syscalls.h

@@ -22,7 +22,7 @@
 /*
  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
- * Copyright 2017 Joyent, Inc.
+ * Copyright 2018 Joyent, Inc.
  */
 
 #ifndef _SYS_LINUX_SYSCALLS_H
@@ -79,6 +79,7 @@ extern long lx_fcntl();
 extern long lx_fcntl64();
 extern long lx_fgetxattr();
 extern long lx_flistxattr();
+extern long lx_flock();
 extern long lx_fremovexattr();
 extern long lx_fsetxattr();
 extern long lx_fstat32();

usr/src/uts/common/brand/lx/syscall/lx_fcntl.c

@@ -10,7 +10,7 @@
  */
 
 /*
- * Copyright 2017 Joyent, Inc.
+ * Copyright 2018 Joyent, Inc.
  */
 
 #include <sys/systm.h>
@@ -27,9 +27,11 @@
 #include <sys/lx_fcntl.h>
 #include <sys/lx_misc.h>
 #include <sys/lx_socket.h>
+#include <sys/brand.h>
 #include <sys/fs/fifonode.h>
 #include <sys/strsubr.h>
 #include <sys/stream.h>
+#include <sys/flock.h>
 
 extern int fcntl(int, int, intptr_t);
 extern int flock_check(vnode_t *, flock64_t *, offset_t, offset_t);
@@ -464,8 +466,13 @@ lx_fcntl_lock(int fd, int lx_cmd, void *arg)
 			break;
 
 		if ((error = VOP_FRLOCK(vp, cmd, &bf, flag, offset, NULL,
-		    fp->f_cred, NULL)) != 0)
+		    fp->f_cred, NULL)) != 0) {
+			if (cmd == F_SETLKW && error == EINTR) {
+				ttolxlwp(curthread)->br_syscall_restart =
+				    B_TRUE;
+			}
 			break;
+		}
 
 		if (cmd != F_GETLK)
 			break;
@@ -627,3 +634,68 @@ lx_fcntl64(int fd, int cmd, intptr_t arg)
 		return (lx_fcntl_common(fd, cmd, (ulong_t)arg));
 	}
 }
+
+/*
+ * Apply or remove an advisory lock on the entire file. F_FLOCK and F_FLOCKW
+ * are OFD-style locks. For more information, see the comment on ofdlock().
+ */
+long
+lx_flock(int fd, int op)
+{
+	int cmd;
+	int error;
+	flock64_t bf;
+	file_t *fp;
+
+	if (op & LX_LOCK_NB) {
+		cmd = F_FLOCK;
+		op &= ~LX_LOCK_NB;
+	} else {
+		cmd = F_FLOCKW;
+	}
+
+	switch (op) {
+	case LX_LOCK_UN:
+		bf.l_type = F_UNLCK;
+		break;
+	case LX_LOCK_SH:
+		bf.l_type = F_RDLCK;
+		break;
+	case LX_LOCK_EX:
+		bf.l_type = F_WRLCK;
+		break;
+	default:
+		return (set_errno(EINVAL));
+	}
+
+	bf.l_whence = 0;
+	bf.l_start = 0;
+	bf.l_len = 0;
+	bf.l_sysid = 0;
+	bf.l_pid = 0;
+
+	if ((fp = getf(fd)) == NULL)
+		return (set_errno(EBADF));
+
+	/*
+	 * See the locking comment in fcntl.c. In summary, the *_frlock
+	 * functions in the various file systems basically do some validation,
+	 * then funnel everything through the fs_frlock function. For OFD-style
+	 * locks, fs_frlock will do nothing. Once control returns here, we call
+	 * the ofdlock function to do the actual locking.
+	 */
+	error = VOP_FRLOCK(fp->f_vnode, cmd, &bf, fp->f_flag, fp->f_offset,
+	    NULL, fp->f_cred, NULL);
+	if (error != 0) {
+		releasef(fd);
+		return (set_errno(error));
+	}
+	error = ofdlock(fp, cmd, &bf, fp->f_flag, fp->f_offset);
+	if (error != 0) {
+		if (cmd == F_FLOCKW && error == EINTR)
+			ttolxlwp(curthread)->br_syscall_restart = B_TRUE;
+		(void) set_errno(error);
+	}
+	releasef(fd);
+	return (error);
+}

usr/src/uts/common/brand/lx/syscall/lx_ioctl.c

@@ -10,7 +10,7 @@
  */
 
 /*
- * Copyright 2017 Joyent, Inc.
+ * Copyright 2018 Joyent, Inc.
  */
 
 #include <sys/errno.h>
@@ -32,6 +32,7 @@
 #include <sys/sysmacros.h>
 #include <sys/lx_misc.h>
 #include <sys/lx_ptm.h>
+#include <sys/brand.h>
 #include <sys/sunddi.h>
 #include <sys/thread.h>
 #include <sys/proc.h>
@@ -1749,6 +1750,29 @@ static lx_ioc_cmd_translator_t lx_ioc_xlate_blk[] = {
 	LX_IOC_CMD_TRANSLATOR_END
 };
 
+/*
+ * Linux only restarts ioctls for "slow" devices. This includes terminals,
+ * pipes, and sockets. If additional "slow" devices are discovered in the
+ * future, they can be added here as well.
+ */
+static boolean_t
+lx_ioctl_is_slow_dev(file_t *fp)
+{
+	int rv;
+	struct termio s_tio;
+	vtype_t vt = fp->f_vnode->v_type;
+
+	if (vt == VFIFO || vt == VSOCK)
+		return (B_TRUE);
+
+	/* Check if it's a terminal using the isatty() approach. */
+	if (vt == VCHR && VOP_IOCTL(fp->f_vnode, TCGETA, (intptr_t)&s_tio,
+	    FLFAKE(fp), fp->f_cred, &rv, NULL) == 0)
+		return (B_TRUE);
+
+	return (B_FALSE);
+}
+
 static void
 lx_ioctl_vsd_free(void *data)
 {
@@ -1832,6 +1856,10 @@ lx_ioctl(int fdes, int cmd, intptr_t arg)
 	}
 
 	res = ict->lict_func(fp, ict->lict_cmd, arg, ict->lict_lxcmd);
+
+	if (ttolwp(curthread)->lwp_errno == EINTR && lx_ioctl_is_slow_dev(fp))
+		ttolxlwp(curthread)->br_syscall_restart = B_TRUE;
+
 	releasef(fdes);
 	return (res);
 }

usr/src/uts/common/brand/lx/syscall/lx_open.c

@@ -22,7 +22,7 @@
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
- * Copyright 2016 Joyent, Inc.
+ * Copyright 2018 Joyent, Inc.
  */
 
 #include <sys/systm.h>
@@ -37,6 +37,7 @@
 #include <sys/lx_types.h>
 #include <sys/lx_fcntl.h>
 #include <sys/lx_misc.h>
+#include <sys/brand.h>
 
 extern int fcntl(int, int, intptr_t);
 extern int openat(int, char *, int, int);
@@ -244,6 +245,10 @@ lx_openat(int atfd, char *path, int fmode, int cmode)
 
 			(void) set_errno(oerror);
 		}
+
+		if (ttolwp(curthread)->lwp_errno == EINTR)
+			ttolxlwp(curthread)->br_syscall_restart = B_TRUE;
+
 		return (ttolwp(curthread)->lwp_errno);
 	}