Add support to msdosfs and makefs to generate correct Unicode (UCS-2)…

… directory

entries from UTF8 encoded file names.

sbin/mount_msdos/mount_msdos.8

@@ -1,4 +1,4 @@
-.\" $NetBSD: mount_msdos.8,v 1.36 2012/11/16 15:00:18 tsutsui Exp $
+.\" $NetBSD: mount_msdos.8,v 1.37 2016/01/30 09:59:27 mlelstv Exp $
 .\"
 .\" Copyright (c) 1993, 1994 Christopher G. Demetriou
 .\" All rights reserved.
@@ -40,7 +40,7 @@
 .Nd mount an MS-DOS file system
 .Sh SYNOPSIS
 .Nm
-.Op Fl 9Gls
+.Op Fl 9GlsU
 .Op Fl g Ar gid
 .Op Fl M Ar mask
 .Op Fl m Ar mask
@@ -111,6 +111,19 @@ is the default.
 Otherwise
 .Fl l
 is assumed.
+.It Fl U
+The MS-DOS file system stores filenames in a short
+version using 8-bit characters according to some
+character set and a long version with 16-bit unicode
+characters.
+The default method to store encoding-agnostic UNIX filenames
+is to copy them byte-wise into both fields. This is
+transparent but generates wrong unicode characters
+for anything that is not ASCII. Setting the
+.Fl U
+flag interprets UNIX filenames as UTF-8 and generates
+correctly encoded long filenames. This forces
+.Fl l .
 .It Fl M Ar mask
 Specify the maximum file permissions for directories
 in the file system.

sbin/mount_msdos/mount_msdos.c

@@ -1,4 +1,4 @@
-/* $NetBSD: mount_msdos.c,v 1.47 2009/10/07 20:34:02 pooka Exp $ */
+/* $NetBSD: mount_msdos.c,v 1.48 2016/01/30 09:59:27 mlelstv Exp $ */
 
 /*
  * Copyright (c) 1994 Christopher G. Demetriou
@@ -36,7 +36,7 @@
 
 #include <sys/cdefs.h>
 #ifndef lint
-__RCSID("$NetBSD: mount_msdos.c,v 1.47 2009/10/07 20:34:02 pooka Exp $");
+__RCSID("$NetBSD: mount_msdos.c,v 1.48 2016/01/30 09:59:27 mlelstv Exp $");
 #endif /* not lint */
 
 #include <sys/param.h>
@@ -94,7 +94,7 @@ mount_msdos_parseargs(int argc, char **argv,
 	*mntflags = set_gid = set_uid = set_mask = set_dirmask = set_gmtoff = 0;
 	(void)memset(args, '\0', sizeof(*args));
 
-	while ((c = getopt(argc, argv, "Gsl9u:g:m:M:o:t:")) != -1) {
+	while ((c = getopt(argc, argv, "Gsl9Uu:g:m:M:o:t:")) != -1) {
 		switch (c) {
 		case 'G':
 			args->flags |= MSDOSFSMNT_GEMDOSFS;
@@ -108,6 +108,9 @@ mount_msdos_parseargs(int argc, char **argv,
 		case '9':
 			args->flags |= MSDOSFSMNT_NOWIN95;
 			break;
+		case 'U':
+			args->flags |= MSDOSFSMNT_UTF8;
+			break;
 		case 'u':
 			args->uid = a_uid(optarg);
 			set_uid = 1;

sys/fs/msdosfs/direntry.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: direntry.h,v 1.9 2016/01/23 01:26:14 dholland Exp $	*/
+/*	$NetBSD: direntry.h,v 1.10 2016/01/30 09:59:27 mlelstv Exp $	*/
 
 /*-
  * Copyright (C) 1994, 1995, 1997 Wolfgang Solfrank.
@@ -136,11 +136,12 @@ int	dos2unixfn(unsigned char dn[11], unsigned char *un, int lower);
 int	unix2dosfn(const unsigned char *un, unsigned char dn[12], int unlen,
 	    unsigned int gen);
 int	unix2winfn(const unsigned char *un, int unlen, struct winentry *wep,
-	    int cnt, int chksum);
+	    int cnt, int chksum, int utf8);
 int	winChkName(const unsigned char *un, int unlen, struct winentry *wep,
-	    int chksum);
-int	win2unixfn(struct winentry *wep, struct dirent *dp, int chksum);
+	    int chksum, int utf8);
+int	win2unixfn(struct winentry *wep, struct dirent *dp, int chksum,	
+	    int utf8);
 uint8_t winChksum(uint8_t *name);
-int	winSlotCnt(const unsigned char *un, int unlen);
+int	winSlotCnt(const unsigned char *un, int unlen, int utf8);
 #endif /* _KERNEL || MAKEFS */
 #endif /* _MSDOSFS_DIRENTRY_H_ */

sys/fs/msdosfs/msdosfs_lookup.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: msdosfs_lookup.c,v 1.34 2015/03/28 19:24:05 maxv Exp $	*/
+/*	$NetBSD: msdosfs_lookup.c,v 1.35 2016/01/30 09:59:27 mlelstv Exp $	*/
 
 /*-
  * Copyright (C) 1994, 1995, 1997 Wolfgang Solfrank.
@@ -52,7 +52,7 @@
 #endif
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: msdosfs_lookup.c,v 1.34 2015/03/28 19:24:05 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: msdosfs_lookup.c,v 1.35 2016/01/30 09:59:27 mlelstv Exp $");
 
 #include <sys/param.h>
 
@@ -187,12 +187,12 @@ msdosfs_lookup(void *v)
 		break;
 	case 2:
 		wincnt = winSlotCnt((const u_char *)cnp->cn_nameptr,
-		    cnp->cn_namelen) + 1;
+		    cnp->cn_namelen, pmp->pm_flags & MSDOSFSMNT_UTF8) + 1;
 		break;
 	case 3:
 		olddos = 0;
 		wincnt = winSlotCnt((const u_char *)cnp->cn_nameptr,
-		    cnp->cn_namelen) + 1;
+		    cnp->cn_namelen, pmp->pm_flags & MSDOSFSMNT_UTF8) + 1;
 		break;
 	}
 	if (pmp->pm_flags & MSDOSFSMNT_SHORTNAME)
@@ -282,7 +282,8 @@ msdosfs_lookup(void *v)
 					chksum = winChkName((const u_char *)cnp->cn_nameptr,
 							    cnp->cn_namelen,
 							    (struct winentry *)dep,
-							    chksum);
+							    chksum,
+							    pmp->pm_flags & MSDOSFSMNT_UTF8);
 					continue;
 				}
 
@@ -652,7 +653,8 @@ createde(struct denode *dep, struct denode *ddep, struct denode **depp, struct c
 				fndoffset -= sizeof(struct direntry);
 			}
 			if (!unix2winfn(un, unlen, (struct winentry *)ndep,
-						wcnt, chksum))
+					wcnt, chksum,
+					ddep->de_pmp->pm_flags & MSDOSFSMNT_UTF8))
 				break;
 		}
 	}

sys/fs/msdosfs/msdosfs_vnops.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: msdosfs_vnops.c,v 1.93 2015/04/04 12:34:44 riastradh Exp $	*/
+/*	$NetBSD: msdosfs_vnops.c,v 1.94 2016/01/30 09:59:27 mlelstv Exp $	*/
 
 /*-
  * Copyright (C) 1994, 1995, 1997 Wolfgang Solfrank.
@@ -48,7 +48,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: msdosfs_vnops.c,v 1.93 2015/04/04 12:34:44 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: msdosfs_vnops.c,v 1.94 2016/01/30 09:59:27 mlelstv Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -1541,7 +1541,7 @@ msdosfs_readdir(void *v)
 				if (pmp->pm_flags & MSDOSFSMNT_SHORTNAME)
 					continue;
 				chksum = win2unixfn((struct winentry *)dentp,
-				    dirbuf, chksum);
+				    dirbuf, chksum, pmp->pm_flags & MSDOSFSMNT_UTF8);
 				continue;
 			}
 

sys/fs/msdosfs/msdosfsmount.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: msdosfsmount.h,v 1.20 2014/07/08 09:21:52 hannken Exp $	*/
+/*	$NetBSD: msdosfsmount.h,v 1.21 2016/01/30 09:59:27 mlelstv Exp $	*/
 
 /*-
  * Copyright (C) 1994, 1995, 1997 Wolfgang Solfrank.
@@ -78,19 +78,20 @@ struct msdosfs_args {
 #define	MSDOSFSMNT_NOWIN95	4	/* Completely ignore Win95 entries */
 #define	MSDOSFSMNT_GEMDOSFS	8	/* This is a GEMDOS-flavour */
 #define MSDOSFSMNT_VERSIONED	16	/* Struct is versioned */
+#define MSDOSFSMNT_UTF8		32	/* Use UTF8 filenames */
 
 /* All flags above: */
 #define	MSDOSFSMNT_MNTOPT \
 	(MSDOSFSMNT_SHORTNAME|MSDOSFSMNT_LONGNAME|MSDOSFSMNT_NOWIN95 \
-	 |MSDOSFSMNT_GEMDOSFS|MSDOSFSMNT_VERSIONED)
+	 |MSDOSFSMNT_GEMDOSFS|MSDOSFSMNT_VERSIONED|MSDOSFSMNT_UTF8)
 
 #define	MSDOSFSMNT_RONLY	0x80000000	/* mounted read-only	*/
 #define	MSDOSFSMNT_WAITONFAT	0x40000000	/* mounted synchronous	*/
 #define	MSDOSFS_FATMIRROR	0x20000000	/* FAT is mirrored */
 
 #define MSDOSFSMNT_BITS "\177\20" \
     "b\00shortname\0b\01longname\0b\02nowin95\0b\03gemdosfs\0b\04mntversioned\0" \
-    "b\037ronly\0b\036waitonfat\0b\035fatmirror\0"
+    "b\05utf8\0b\037ronly\0b\036waitonfat\0b\035fatmirror\0"
 
 #ifdef _KERNEL
 #include <sys/mallocvar.h>

usr.sbin/makefs/msdos.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: msdos.c,v 1.15 2015/10/16 16:40:02 christos Exp $	*/
+/*	$NetBSD: msdos.c,v 1.16 2016/01/30 09:59:27 mlelstv Exp $	*/
 
 /*-
  * Copyright (c) 2013 The NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
 
 #include <sys/cdefs.h>
 #if defined(__RCSID) && !defined(__lint)
-__RCSID("$NetBSD: msdos.c,v 1.15 2015/10/16 16:40:02 christos Exp $");
+__RCSID("$NetBSD: msdos.c,v 1.16 2016/01/30 09:59:27 mlelstv Exp $");
 #endif	/* !__lint */
 
 #include <sys/param.h>
@@ -55,18 +55,25 @@ __RCSID("$NetBSD: msdos.c,v 1.15 2015/10/16 16:40:02 christos Exp $");
 #include <util.h>
 
 #include <ffs/buf.h>
+#include <fs/msdosfs/bpb.h>
 #include <fs/msdosfs/denode.h>
+#include <fs/msdosfs/msdosfsmount.h>
 #include "makefs.h"
 #include "msdos.h"
 #include "mkfs_msdos.h"
 
 static int msdos_populate_dir(const char *, struct denode *, fsnode *,
     fsnode *, fsinfo_t *);
 
+struct msdos_options_ex {
+	struct msdos_options options;
+	bool utf8;
+};
+
 void
 msdos_prep_opts(fsinfo_t *fsopts)
 {
-	struct msdos_options *msdos_opt = ecalloc(1, sizeof(*msdos_opt));
+	struct msdos_options_ex *msdos_opt = ecalloc(1, sizeof(*msdos_opt));
 	const option_t msdos_options[] = {
 #define AOPT(_opt, _type, _name, _min, _desc) { 			\
 	.letter = _opt,							\
@@ -76,7 +83,7 @@ msdos_prep_opts(fsinfo_t *fsopts)
 	    (sizeof(_type) == 1 ? OPT_INT8 :				\
 	    (sizeof(_type) == 2 ? OPT_INT16 :				\
 	    (sizeof(_type) == 4 ? OPT_INT32 : OPT_INT64)))),		\
-	.value = &msdos_opt->_name,					\
+	.value = &msdos_opt->options._name,				\
 	.minimum = _min,						\
 	.maximum = sizeof(_type) == 1 ? 0xff :				\
 	    (sizeof(_type) == 2 ? 0xffff :				\
@@ -85,6 +92,8 @@ msdos_prep_opts(fsinfo_t *fsopts)
 },
 ALLOPTS
 #undef AOPT	
+		{ 'U', "utf8", &msdos_opt->utf8, OPT_BOOL,
+		  0, 1, "Use UTF8 names" },
 		{ .name = NULL }
 	};
 
@@ -131,10 +140,11 @@ msdos_parse_opts(const char *option, fsinfo_t *fsopts)
 void
 msdos_makefs(const char *image, const char *dir, fsnode *root, fsinfo_t *fsopts)
 {
-	struct msdos_options *msdos_opt = fsopts->fs_specific;
+	struct msdos_options_ex *msdos_opt = fsopts->fs_specific;
 	struct vnode vp, rootvp;
 	struct timeval	start;
 	struct msdosfsmount *pmp;
+	uint32_t flags;
 
 	assert(image != NULL);
 	assert(dir != NULL);
@@ -145,31 +155,36 @@ msdos_makefs(const char *image, const char *dir, fsnode *root, fsinfo_t *fsopts)
 	 * XXX: pick up other options from the msdos specific ones?
 	 * Is minsize right here?
 	 */
-	msdos_opt->create_size = MAX(msdos_opt->create_size, fsopts->minsize);
-	msdos_opt->offset = fsopts->offset;
-	if (msdos_opt->bytes_per_sector == 0) {
+	msdos_opt->options.create_size = MAX(msdos_opt->options.create_size,
+		fsopts->minsize);
+	msdos_opt->options.offset = fsopts->offset;
+	if (msdos_opt->options.bytes_per_sector == 0) {
 		if (fsopts->sectorsize == -1)
 			fsopts->sectorsize = 512;
-		msdos_opt->bytes_per_sector = fsopts->sectorsize;
+		msdos_opt->options.bytes_per_sector = fsopts->sectorsize;
 	} else if (fsopts->sectorsize == -1) {
-		fsopts->sectorsize = msdos_opt->bytes_per_sector;
-	} else if (fsopts->sectorsize != msdos_opt->bytes_per_sector) {
+		fsopts->sectorsize = msdos_opt->options.bytes_per_sector;
+	} else if (fsopts->sectorsize != msdos_opt->options.bytes_per_sector) {
 		err(1, "inconsistent sectorsize -S %u"
 		    "!= -o bytes_per_sector %u", 
-		    fsopts->sectorsize, msdos_opt->bytes_per_sector);
+		    fsopts->sectorsize, msdos_opt->options.bytes_per_sector);
 	}
 
 		/* create image */
 	printf("Creating `%s'\n", image);
 	TIMER_START(start);
-	if (mkfs_msdos(image, NULL, msdos_opt) == -1)
+	if (mkfs_msdos(image, NULL, &msdos_opt->options) == -1)
 		return;
 	TIMER_RESULTS(start, "mkfs_msdos");
 
 	fsopts->fd = open(image, O_RDWR);
 	vp.fs = fsopts;
 
-	if ((pmp = msdosfs_mount(&vp, 0)) == NULL)
+	flags = 0;
+	if (msdos_opt->utf8)
+		flags |= MSDOSFSMNT_UTF8;
+
+	if ((pmp = msdosfs_mount(&vp, flags)) == NULL)
 		err(1, "msdosfs_mount");
 
 	if (msdosfs_root(pmp, &rootvp) != 0)

usr.sbin/makefs/msdos/msdosfs_vfsops.c

@@ -50,7 +50,7 @@
 #endif
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: msdosfs_vfsops.c,v 1.9 2015/03/29 05:52:59 agc Exp $");
+__KERNEL_RCSID(0, "$NetBSD: msdosfs_vfsops.c,v 1.10 2016/01/30 09:59:27 mlelstv Exp $");
 
 #include <sys/param.h>
 
@@ -152,6 +152,12 @@ msdosfs_mount(struct vnode *devvp, int flags)
 		}
 	}
 
+	pmp->pm_flags = flags & MSDOSFSMNT_MNTOPT;
+	if (pmp->pm_flags & MSDOSFSMNT_GEMDOSFS)
+		pmp->pm_flags |= MSDOSFSMNT_NOWIN95;
+	if (pmp->pm_flags & MSDOSFSMNT_NOWIN95)
+		pmp->pm_flags |= MSDOSFSMNT_SHORTNAME;
+
 	if (pmp->pm_Sectors == 0) {
 		pmp->pm_HiddenSects = getulong(b50->bpbHiddenSecs);
 		pmp->pm_HugeSectors = getulong(b50->bpbHugeSectors);

usr.sbin/makefs/msdos/msdosfs_vnops.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: msdosfs_vnops.c,v 1.16 2015/03/29 05:52:59 agc Exp $ */
+/*	$NetBSD: msdosfs_vnops.c,v 1.17 2016/01/30 09:59:27 mlelstv Exp $ */
 
 /*-
  * Copyright (C) 1994, 1995, 1997 Wolfgang Solfrank.
@@ -51,7 +51,7 @@
 #endif
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: msdosfs_vnops.c,v 1.16 2015/03/29 05:52:59 agc Exp $");
+__KERNEL_RCSID(0, "$NetBSD: msdosfs_vnops.c,v 1.17 2016/01/30 09:59:27 mlelstv Exp $");
 
 #include <sys/param.h>
 #include <sys/mman.h>
@@ -154,12 +154,12 @@ msdosfs_findslot(struct denode *dp, struct componentname *cnp)
 		break;
 	case 2:
 		wincnt = winSlotCnt((const u_char *)cnp->cn_nameptr,
-		    cnp->cn_namelen) + 1;
+		    cnp->cn_namelen, pmp->pm_flags & MSDOSFSMNT_UTF8) + 1;
 		break;
 	case 3:
 		olddos = 0;
 		wincnt = winSlotCnt((const u_char *)cnp->cn_nameptr,
-		    cnp->cn_namelen) + 1;
+		    cnp->cn_namelen, pmp->pm_flags & MSDOSFSMNT_UTF8) + 1;
 		break;
 	}
 
@@ -243,7 +243,8 @@ msdosfs_findslot(struct denode *dp, struct componentname *cnp)
 					chksum = winChkName((const u_char *)cnp->cn_nameptr,
 							    cnp->cn_namelen,
 							    (struct winentry *)dep,
-							    chksum);
+							    chksum,
+							    pmp->pm_flags & MSDOSFSMNT_UTF8);
 					continue;
 				}