implement O_DIRECTORY as standardized in POSIX-2008,

for both native and linux emulations.
this fixes the rest of PR 43695.

lib/libc/sys/open.2

@@ -1,4 +1,4 @@
-.\"	$NetBSD: open.2,v 1.45 2010/09/06 19:48:38 wiz Exp $
+.\"	$NetBSD: open.2,v 1.46 2010/09/21 19:26:18 chs Exp $
 .\"
 .\" Copyright (c) 1980, 1991, 1993
 .\"	The Regents of the University of California.  All rights reserved.
@@ -145,6 +145,8 @@ be multiples of
 If the I/O request is made
 using an interface that supports scatter/gather via struct iovec, each
 element of the request must meet the above alignment constraints.
+.It Dv O_DIRECTORY
+Fail if the file is not a directory.
 .El
 .Pp
 Opening a file with
@@ -307,6 +309,8 @@ and there are no free inodes on the file system on which the
 file is being created.
 .It Bq Er ENOTDIR
 A component of the path prefix is not a directory.
+.It Bq Er ENOTDIR
+O_DIRECTORY is specified and the last path component is not a directory.
 .It Bq Er ENXIO
 The named file is a character special or block
 special file, and the device associated with this special file

sys/compat/linux/arch/alpha/linux_fcntl.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: linux_fcntl.h,v 1.2 2008/04/28 20:23:42 martin Exp $	*/
+/*	$NetBSD: linux_fcntl.h,v 1.3 2010/09/21 19:26:19 chs Exp $	*/
 
 /*-
  * Copyright (c) 1998 The NetBSD Foundation, Inc.
@@ -49,6 +49,7 @@
 #define LINUX_O_APPEND		0x0008
 #define LINUX_O_NDELAY		LINUX_O_NONBLOCK
 #define LINUX_O_SYNC		0x4000
+#define LINUX_O_DIRECTORY	0x8000
 
 #define LINUX_FASYNC		0x2000
 

sys/compat/linux/arch/amd64/linux_fcntl.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: linux_fcntl.h,v 1.2 2005/12/11 12:20:14 christos Exp $ */
+/*	$NetBSD: linux_fcntl.h,v 1.3 2010/09/21 19:26:19 chs Exp $ */
 
 /*-
  * Copyright (c) 2005 Emmanuel Dreyfus, all rights reserved.
@@ -42,6 +42,7 @@
 #define LINUX_O_NDELAY		04000
 #define LINUX_O_SYNC		010000
 #define LINUX_FASYNC		020000
+#define LINUX_O_DIRECTORY	0x10000
 
 #define LINUX_F_RDLCK		0
 #define LINUX_F_WRLCK		1	

sys/compat/linux/arch/arm/linux_fcntl.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: linux_fcntl.h,v 1.2 2008/04/28 20:23:42 martin Exp $	*/
+/*	$NetBSD: linux_fcntl.h,v 1.3 2010/09/21 19:26:19 chs Exp $	*/
 
 /*-
  * Copyright (c) 1995, 1998 The NetBSD Foundation, Inc.
@@ -46,8 +46,8 @@
 #define LINUX_O_APPEND		0x0400
 #define LINUX_O_NDELAY		0x0800
 #define LINUX_O_SYNC		0x1000
-
 #define LINUX_FASYNC		0x2000
+#define LINUX_O_DIRECTORY	0x4000
 
 /* fcntl(2) operations */
 #define LINUX_F_DUPFD		0

sys/compat/linux/arch/i386/linux_fcntl.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: linux_fcntl.h,v 1.5 2008/04/28 20:23:42 martin Exp $	*/
+/*	$NetBSD: linux_fcntl.h,v 1.6 2010/09/21 19:26:19 chs Exp $	*/
 
 /*-
  * Copyright (c) 1995, 1998 The NetBSD Foundation, Inc.
@@ -46,8 +46,8 @@
 #define LINUX_O_APPEND		0x0400
 #define LINUX_O_NDELAY		0x0800
 #define LINUX_O_SYNC		0x1000
-
 #define LINUX_FASYNC		0x2000
+#define LINUX_O_DIRECTORY	0x10000
 
 /* fcntl(2) operations */
 #define LINUX_F_DUPFD		0

sys/compat/linux/arch/mips/linux_fcntl.h

@@ -1,4 +1,4 @@
-/* $NetBSD: linux_fcntl.h,v 1.5 2008/04/28 20:23:43 martin Exp $ */
+/* $NetBSD: linux_fcntl.h,v 1.6 2010/09/21 19:26:20 chs Exp $ */
 
 /*-
  * Copyright (c) 1995, 1998, 2001 The NetBSD Foundation, Inc.
@@ -52,8 +52,8 @@
 #define LINUX_O_NONBLOCK	0x0080
 #define LINUX_O_NDELAY		LINUX_O_NONBLOCK
 #define LINUX_O_SYNC		0x0010
-
 #define LINUX_FASYNC		0x1000
+#define LINUX_O_DIRECTORY	0x10000
 
 /*
  * fcntl(2) operations

sys/compat/linux/arch/powerpc/linux_fcntl.h

@@ -1,4 +1,4 @@
-/* $NetBSD: linux_fcntl.h,v 1.7 2008/04/28 20:23:43 martin Exp $ */
+/* $NetBSD: linux_fcntl.h,v 1.8 2010/09/21 19:26:20 chs Exp $ */
 
 /*-
  * Copyright (c) 1995, 1998, 2001 The NetBSD Foundation, Inc.
@@ -53,8 +53,8 @@
 #define LINUX_O_NONBLOCK	04000
 #define LINUX_O_NDELAY		LINUX_O_NONBLOCK
 #define LINUX_O_SYNC		010000
-
 #define LINUX_FASYNC		020000
+#define LINUX_O_DIRECTORY	0x4000
 
 /*
  * fcntl(2) operations

sys/compat/linux/common/linux_file.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: linux_file.c,v 1.99 2010/07/01 02:38:28 rmind Exp $	*/
+/*	$NetBSD: linux_file.c,v 1.100 2010/09/21 19:26:19 chs Exp $	*/
 
 /*-
  * Copyright (c) 1995, 1998, 2008 The NetBSD Foundation, Inc.
@@ -35,7 +35,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: linux_file.c,v 1.99 2010/07/01 02:38:28 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: linux_file.c,v 1.100 2010/09/21 19:26:19 chs Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -101,6 +101,7 @@ linux_to_bsd_ioflags(int lflags)
 	res |= cvtto_bsd_mask(lflags, LINUX_O_SYNC, O_FSYNC);
 	res |= cvtto_bsd_mask(lflags, LINUX_FASYNC, O_ASYNC);
 	res |= cvtto_bsd_mask(lflags, LINUX_O_APPEND, O_APPEND);
+	res |= cvtto_bsd_mask(lflags, LINUX_O_DIRECTORY, O_DIRECTORY);
 
 	return res;
 }
@@ -121,6 +122,7 @@ bsd_to_linux_ioflags(int bflags)
 	res |= cvtto_linux_mask(bflags, O_FSYNC, LINUX_O_SYNC);
 	res |= cvtto_linux_mask(bflags, O_ASYNC, LINUX_FASYNC);
 	res |= cvtto_linux_mask(bflags, O_APPEND, LINUX_O_APPEND);
+	res |= cvtto_linux_mask(bflags, O_DIRECTORY, LINUX_O_DIRECTORY);
 
 	return res;
 }
@@ -632,4 +634,3 @@ LINUX_NOT_SUPPORTED(linux_sys_flistxattr)
 LINUX_NOT_SUPPORTED(linux_sys_removexattr)
 LINUX_NOT_SUPPORTED(linux_sys_lremovexattr)
 LINUX_NOT_SUPPORTED(linux_sys_fremovexattr)
-

sys/kern/vfs_vnops.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: vfs_vnops.c,v 1.177 2010/08/25 13:51:50 pooka Exp $	*/
+/*	$NetBSD: vfs_vnops.c,v 1.178 2010/09/21 19:26:19 chs Exp $	*/
 
 /*-
  * Copyright (c) 2009 The NetBSD Foundation, Inc.
@@ -66,7 +66,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vfs_vnops.c,v 1.177 2010/08/25 13:51:50 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_vnops.c,v 1.178 2010/09/21 19:26:19 chs Exp $");
 
 #include "veriexec.h"
 
@@ -140,6 +140,9 @@ vn_open(struct nameidata *ndp, int fmode, int cmode)
 	int error;
 	char *path;
 
+	if ((fmode & (O_CREAT | O_DIRECTORY)) == (O_CREAT | O_DIRECTORY))
+		return EINVAL;
+
 	ndp->ni_cnd.cn_flags &= TRYEMULROOT | NOCHROOT;
 
 	if (fmode & O_CREAT) {
@@ -260,6 +263,9 @@ vn_openchk(struct vnode *vp, kauth_cred_t cred, int fflags)
 	int permbits = 0;
 	int error;
 
+	if ((fflags & O_DIRECTORY) != 0 && vp->v_type != VDIR)
+		return ENOTDIR;
+
 	if ((fflags & FREAD) != 0) {
 		permbits = VREAD;
 	}

sys/sys/fcntl.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: fcntl.h,v 1.35 2009/03/11 06:05:29 mrg Exp $	*/
+/*	$NetBSD: fcntl.h,v 1.36 2010/09/21 19:26:18 chs Exp $	*/
 
 /*-
  * Copyright (c) 1983, 1990, 1993
@@ -98,6 +98,9 @@
 #define	O_TRUNC		0x00000400	/* truncate to zero length */
 #define	O_EXCL		0x00000800	/* error if already exists */
 
+/* defined by POSIX 1003.1; BSD default, but required to be bitwise distinct */
+#define	O_NOCTTY	0x00008000	/* don't assign controlling terminal */
+
 #if (_POSIX_C_SOURCE - 0) >= 199309L || (_XOPEN_SOURCE - 0) >= 500 || \
     defined(_NETBSD_SOURCE)
 #define	O_DSYNC		0x00010000	/* write: I/O data completion */
@@ -109,8 +112,7 @@
 #define	O_DIRECT	0x00080000	/* direct I/O hint */
 #endif
 
-/* defined by POSIX 1003.1; BSD default, but required to be bitwise distinct */
-#define	O_NOCTTY	0x00008000	/* don't assign controlling terminal */
+#define	O_DIRECTORY	0x00200000	/* fail if not a directory */
 
 #ifdef _KERNEL
 /* convert from open() flags to/from fflags; convert O_RD/WR to FREAD/FWRITE */
@@ -120,7 +122,8 @@
 /* all bits settable during open(2) */
 #define	O_MASK		(O_ACCMODE|O_NONBLOCK|O_APPEND|O_SHLOCK|O_EXLOCK|\
 			 O_ASYNC|O_SYNC|O_CREAT|O_TRUNC|O_EXCL|O_DSYNC|\
-			 O_RSYNC|O_NOCTTY|O_ALT_IO|O_NOFOLLOW|O_DIRECT)
+			 O_RSYNC|O_NOCTTY|O_ALT_IO|O_NOFOLLOW|O_DIRECT|\
+			 O_DIRECTORY)
 
 #define	FMARK		0x00001000	/* mark during gc() */
 #define	FDEFER		0x00002000	/* defer for next gc pass */