ceph-fuse: add option to disable kernel pagecache

When multiple clients read/write to a file at the same time, we need
to disable cache and let reads/writes go to OSDs. In ceph-fuse case,
there are two caches, one is in ceph-fuse, another one is in kernel.
We can't disable/enable the kernel cache dynamically. So adding an
config option to completely disable kernel cache. Disabling kernel
cache may reduce performance, so option is disabled by default.

Fixes: 12653
Signed-off-by: Yan, Zheng <zyan@redhat.com>
(cherry picked from commit 0f11ec2)

Conflicts:
	src/common/config_opts.h
           follow the content in patchset, set fuse_use_invalidate_cb
           to True. Which is also the backport
           for http://tracker.ceph.com/issues/15634

src/client/fuse_ll.cc

@@ -471,7 +471,9 @@ static void fuse_ll_open(fuse_req_t req, fuse_ino_t ino,
   if (r == 0) {
     fi->fh = (long)fh;
 #if FUSE_VERSION >= FUSE_MAKE_VERSION(2, 8)
-    if (cfuse->client->cct->_conf->fuse_use_invalidate_cb)
+    if (cfuse->client->cct->_conf->fuse_disable_pagecache)
+      fi->direct_io = 1;
+    else if (cfuse->client->cct->_conf->fuse_use_invalidate_cb)
       fi->keep_cache = 1;
 #endif
     fuse_reply_open(req, fi);
@@ -673,6 +675,12 @@ static void fuse_ll_create(fuse_req_t req, fuse_ino_t parent, const char *name,
   if (r == 0) {
     fi->fh = (long)fh;
     fe.ino = cfuse->make_fake_ino(fe.attr.st_ino, fe.attr.st_dev);
+#if FUSE_VERSION >= FUSE_MAKE_VERSION(2, 8)
+    if (cfuse->client->cct->_conf->fuse_disable_pagecache)
+      fi->direct_io = 1;
+    else if (cfuse->client->cct->_conf->fuse_use_invalidate_cb)
+      fi->keep_cache = 1;
+#endif
     fuse_reply_create(req, &fe, fi);
   } else
     fuse_reply_err(req, -r);

src/common/config_opts.h

@@ -401,7 +401,8 @@ OPTION(client_permissions, OPT_BOOL, true)
 OPTION(client_dirsize_rbytes, OPT_BOOL, true)
 
 // note: the max amount of "in flight" dirty data is roughly (max - target)
-OPTION(fuse_use_invalidate_cb, OPT_BOOL, false) // use fuse 2.8+ invalidate callback to keep page cache consistent
+OPTION(fuse_use_invalidate_cb, OPT_BOOL, true) // use fuse 2.8+ invalidate callback to keep page cache consistent
+OPTION(fuse_disable_pagecache, OPT_BOOL, false)
 OPTION(fuse_allow_other, OPT_BOOL, true)
 OPTION(fuse_default_permissions, OPT_BOOL, true)
 OPTION(fuse_big_writes, OPT_BOOL, true)