FUSE: The FUSE design expects writethrough caching

At least prior to 7.23 (which adds FUSE_WRITEBACK_CACHE), the FUSE protocol
specifies only clean data to be cached.

Prior to this change, we implement and default to writeback caching.  This
is ok enough for local only filesystems without hardlinks, but violates the
general design contract with FUSE and breaks distributed filesystems or
concurrent access to hardlinks of the same inode.

In this change, add cache mode as an extension of cache enable/disable.  The
new modes are UC (was: cache disabled), WT (default), and WB (was: cache
enabled).

For now, WT caching is implemented as write-around, which meets the goal of
only caching clean data.  WT can be better than WA for workloads that
frequently read data that was recently written, but WA is trivial to
implement.  Note that this has no effect on O_WRONLY-opened files, which
were already coerced to write-around.

Refs:
  * https://sourceforge.net/p/fuse/mailman/message/8902254/
  * vgough/encfs#315

PR:		230258 (inspired by)


git-svn-id: svn+ssh://svn.freebsd.org/base/head@344186 ccf9f872-aa2e-dd11-9fc8-001c23d0bc1f

sys/fs/fuse/fuse_io.c

@@ -155,7 +155,13 @@ fuse_io_dispatch(struct vnode *vp, struct uio *uio, int ioflag,
 		}
 		break;
 	case UIO_WRITE:
-		if (directio) {
+		/*
+		 * Kludge: simulate write-through caching via write-around
+		 * caching.  Same effect, as far as never caching dirty data,
+		 * but slightly pessimal in that newly written data is not
+		 * cached.
+		 */
+		if (directio || fuse_data_cache_mode == FUSE_CACHE_WT) {
 			FS_DEBUG("direct write of vnode %ju via file handle %ju\n",
 			    (uintmax_t)VTOILLU(vp), (uintmax_t)fufh->fh_id);
 			err = fuse_write_directbackend(vp, uio, cred, fufh, ioflag);
@@ -363,7 +369,7 @@ fuse_write_directbackend(struct vnode *vp, struct uio *uio,
 		uio->uio_resid += diff;
 		uio->uio_offset -= diff;
 		if (uio->uio_offset > fvdat->filesize &&
-		    fuse_data_cache_enable) {
+		    fuse_data_cache_mode != FUSE_CACHE_UC) {
 			fuse_vnode_setsize(vp, cred, uio->uio_offset);
 			fvdat->flag &= ~FN_SIZECHANGE;
 		}

sys/fs/fuse/fuse_ipc.h

@@ -214,7 +214,13 @@ struct fuse_data {
 #define FSESS_NO_MMAP             0x0800 /* disable mmap */
 #define FSESS_BROKENIO            0x1000 /* fix broken io */
 
-extern int fuse_data_cache_enable;
+enum fuse_data_cache_mode {
+	FUSE_CACHE_UC,
+	FUSE_CACHE_WT,
+	FUSE_CACHE_WB,
+};
+
+extern int fuse_data_cache_mode;
 extern int fuse_data_cache_invalidate;
 extern int fuse_mmap_enable;
 extern int fuse_sync_resize;
@@ -248,7 +254,7 @@ fsess_opt_datacache(struct mount *mp)
 {
     struct fuse_data *data = fuse_get_mpdata(mp);
 
-    return (fuse_data_cache_enable ||
+    return (fuse_data_cache_mode != FUSE_CACHE_UC &&
         (data->dataflags & FSESS_NO_DATACACHE) == 0);
 }
 
@@ -257,7 +263,7 @@ fsess_opt_mmap(struct mount *mp)
 {
     struct fuse_data *data = fuse_get_mpdata(mp);
 
-    if (!(fuse_mmap_enable && fuse_data_cache_enable))
+    if (!fuse_mmap_enable || fuse_data_cache_mode == FUSE_CACHE_UC)
         return 0;
     return ((data->dataflags & (FSESS_NO_DATACACHE | FSESS_NO_MMAP)) == 0);
 }

sys/fs/fuse/fuse_node.c

@@ -94,16 +94,19 @@ __FBSDID("$FreeBSD$");
 
 MALLOC_DEFINE(M_FUSEVN, "fuse_vnode", "fuse vnode private data");
 
+static int sysctl_fuse_cache_mode(SYSCTL_HANDLER_ARGS);
+
 static int fuse_node_count = 0;
 
 SYSCTL_INT(_vfs_fuse, OID_AUTO, node_count, CTLFLAG_RD,
     &fuse_node_count, 0, "Count of FUSE vnodes");
 
-int	fuse_data_cache_enable = 1;
+int	fuse_data_cache_mode = FUSE_CACHE_WT;
 
-SYSCTL_INT(_vfs_fuse, OID_AUTO, data_cache_enable, CTLFLAG_RW,
-    &fuse_data_cache_enable, 0,
-    "enable caching of FUSE file data (including dirty data)");
+SYSCTL_PROC(_vfs_fuse, OID_AUTO, data_cache_mode, CTLTYPE_INT|CTLFLAG_RW,
+    &fuse_data_cache_mode, 0, sysctl_fuse_cache_mode, "I",
+    "Zero: disable caching of FUSE file data; One: write-through caching "
+    "(default); Two: write-back caching (generally unsafe)");
 
 int	fuse_data_cache_invalidate = 0;
 
@@ -116,7 +119,7 @@ int	fuse_mmap_enable = 1;
 
 SYSCTL_INT(_vfs_fuse, OID_AUTO, mmap_enable, CTLFLAG_RW,
     &fuse_mmap_enable, 0,
-    "If non-zero, and data_cache_enable is also non-zero, enable mmap(2) of "
+    "If non-zero, and data_cache_mode is also non-zero, enable mmap(2) of "
     "FUSE files");
 
 int	fuse_refresh_size = 0;
@@ -140,6 +143,28 @@ SYSCTL_INT(_vfs_fuse, OID_AUTO, fix_broken_io, CTLFLAG_RW,
     "If non-zero, print a diagnostic warning if a userspace filesystem returns"
     " EIO on reads of recently extended portions of files");
 
+static int
+sysctl_fuse_cache_mode(SYSCTL_HANDLER_ARGS)
+{
+	int val, error;
+
+	val = *(int *)arg1;
+	error = sysctl_handle_int(oidp, &val, 0, req);
+	if (error || !req->newptr)
+		return (error);
+
+	switch (val) {
+	case FUSE_CACHE_UC:
+	case FUSE_CACHE_WT:
+	case FUSE_CACHE_WB:
+		*(int *)arg1 = val;
+		break;
+	default:
+		return (EDOM);
+	}
+	return (0);
+}
+
 static void
 fuse_vnode_init(struct vnode *vp, struct fuse_vnode_data *fvdat,
     uint64_t nodeid, enum vtype vtyp)
@@ -375,7 +400,7 @@ fuse_vnode_refreshsize(struct vnode *vp, struct ucred *cred)
 	struct vattr va;
 
 	if ((fvdat->flag & FN_SIZECHANGE) != 0 ||
-	    fuse_data_cache_enable == 0 ||
+	    fuse_data_cache_mode == FUSE_CACHE_UC ||
 	    (fuse_refresh_size == 0 && fvdat->filesize != 0))
 		return;