Add support for libfuse3

### What changes are proposed in this pull request?

Add support for libfuse3 in jnifuse module. This PR adds

- Actual implementation code
- A configuration key `alluxio.fuse.jnifuse.libfuse.version` to set the
libfuse to use
- set 3 to use 3, 2 to use 2, and other value to load 2 first, if failed
load 3
- In macOS, always use 2
- Installed `libfuse3=3.2.6-r1` in the docker image. The version is the
latest version in the repo of base docker image
- Separate github actions to test jnifuse module using libfuse2 and
libfuse3 on java 8 and 11

### Why are the changes needed?

libfuse3 provides more features than libfuse2.

### Does this PR introduce any user facing changes?

Added a configuration key `alluxio.fuse.jnifuse.libfuse.version` to set
the libfuse version to use.

Set 3 to use 3, 2 to use 2, and other value to load 2 first, if failed
load 3.

In macOS, always use 2

pr-link: #15002
change-id: cid-9aaaff52dfc1052992697af2e6b4798857a598e8

.github/workflows/fuse_integration_tests.yml

@@ -0,0 +1,55 @@
+name: Fuse Integration Tests
+
+on: [pull_request]
+
+jobs:
+  build:
+    name: "modules: fuse"
+
+    strategy:
+      fail-fast: false
+      matrix:
+        java: ["8", "11"]
+        version: ["2", "3"]
+
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.pull_request.title, 'DOCFIX') &&
+        !contains(github.event.pull_request.title, 'SKIPCI')"
+
+    steps:
+      - name: checkout repo
+        uses: actions/checkout@v2
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: '10.11.0'
+
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-java${{ matrix.java }}-${{ hashFiles('**/pom.xml') }}
+
+      - name: Run tests
+        id: test0
+        run: |
+          mkdir -p ~/.m2
+          ALLUXIO_DOCKER_NO_TTY=true \
+          ALLUXIO_DOCKER_GIT_CLEAN=true \
+          ALLUXIO_DOCKER_MVN_RUNTOEND=true \
+          ALLUXIO_DOCKER_MVN_PROJECT_LIST=!webui,!shaded/client,!integration/tools/hms,!integration/yarn,!assembly/client,!assembly/server,!table/server/underdb/glue,!underfs/hdfs,!underfs/ozone,!underfs/adl,!underfs/abfs,!underfs/cosn,!underfs/wasb,!underfs/cos,!underfs/kodo,!underfs/oss,!underfs/swift \
+          dev/github/run_docker.sh "\"-Dtest=alluxio.client.fuse.**\""  "\"-Dalluxio.fuse.jnifuse.libfuse.version=${{ matrix.version }}\"" -pl tests
+        timeout-minutes: 60
+
+      - name: Archive artifacts
+        continue-on-error: true
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: artifact
+          path: |
+            **/target/surefire-reports/*
+            **/target/artifacts/*
+            **/target/logs/*
+          retention-days: 7

.github/workflows/java11_integration_tests.yml

@@ -19,7 +19,7 @@ jobs:
           - >-
             alluxio.client.fs.concurrent.**,alluxio.client.fs.io.**
           - >-
-            alluxio.client.**,!alluxio.client.fs.**,!alluxio.client.cli.**
+            alluxio.client.**,!alluxio.client.fs.**,!alluxio.client.cli.**,!alluxio.client.fuse.**
           - >-
             alluxio.job.**,alluxio.master.**,alluxio.stress.**
           - >-

.github/workflows/java8_integration_tests.yml

@@ -22,7 +22,7 @@ jobs:
           - >-
             alluxio.client.fs.concurrent.**,alluxio.client.fs.io.**
           - >-
-            alluxio.client.**,!alluxio.client.fs.**,!alluxio.client.cli.**
+            alluxio.client.**,!alluxio.client.fs.**,!alluxio.client.cli.**,!alluxio.client.fuse.**
           - >-
             alluxio.job.**,alluxio.master.**,alluxio.stress.**
           - >-

core/common/src/main/java/alluxio/conf/PropertyKey.java

@@ -5385,6 +5385,14 @@ public String toString() {
           .setConsistencyCheckLevel(ConsistencyCheckLevel.IGNORE)
           .setScope(Scope.CLIENT)
           .build();
+  public static final PropertyKey FUSE_JNIFUSE_LIBFUSE_VERSION =
+      new Builder(Name.FUSE_JNIFUSE_LIBFUSE_VERSION)
+          .setDefaultValue(0)
+          .setDescription("The version of libfuse used by libjnifuse. "
+              + "Set 2 to force use libfuse2, 3 to libfuse3, and "
+              + "other value to use libfuse2 first, libfuse3 if libfuse2 failed")
+          .setScope(Scope.ALL)
+          .build();
   public static final PropertyKey FUSE_PERMISSION_CHECK_ENABLED =
       new Builder(Name.FUSE_PERMISSION_CHECK_ENABLED)
           .setDefaultValue(true)
@@ -7195,6 +7203,8 @@ public static final class Name {
     public static final String FUSE_WEB_BIND_HOST = "alluxio.fuse.web.bind.host";
     public static final String FUSE_WEB_HOSTNAME = "alluxio.fuse.web.hostname";
     public static final String FUSE_WEB_PORT = "alluxio.fuse.web.port";
+    public static final String FUSE_JNIFUSE_LIBFUSE_VERSION =
+        "alluxio.fuse.jnifuse.libfuse.version";
 
     //
     // Security related properties

integration/docker/dockerfile-common.sh

@@ -13,11 +13,12 @@
 # Shared chunk of commands used both in Dockerfile and Dockerfile-dev.
 
 #######################################
-# Install libfuse with MAX_IDLE_THREAD able to be configured.
+# Install libfuse with MAX_IDLE_THREAD able to be configured and libfuse3.
 # Arguments:
 #   None
 #######################################
 function installLibfuse {
+  # libfuse2
   git clone https://github.com/Alluxio/libfuse.git
   cd libfuse
   git checkout fuse_2_9_5_customize_multi_threads
@@ -26,6 +27,10 @@ function installLibfuse {
   make -j8
   make install
   cd ..
+
+  # libfuse3
+  apk add fuse3=3.2.6-r1
+
 }
 
 #######################################

integration/fuse/src/main/java/alluxio/fuse/AlluxioFuse.java

@@ -20,6 +20,8 @@
 import alluxio.conf.PropertyKey;
 import alluxio.conf.ServerConfiguration;
 import alluxio.jnifuse.FuseException;
+import alluxio.jnifuse.LibFuse;
+import alluxio.jnifuse.utils.NativeLibraryLoader;
 import alluxio.metrics.MetricKey;
 import alluxio.metrics.MetricsSystem;
 import alluxio.retry.RetryUtils;
@@ -102,6 +104,10 @@ private AlluxioFuse() {}
   public static void main(String[] args) {
     LOG.info("Alluxio version: {}-{}", RuntimeConstants.VERSION, ProjectConstants.REVISION);
     AlluxioConfiguration conf = InstancedConfiguration.defaults();
+
+    // Parsing options needs to know which version is being used.
+    LibFuse.loadLibrary(AlluxioFuseUtils.getVersionPreference(conf));
+
     FileSystemContext fsContext = FileSystemContext.create(conf);
     try {
       InetSocketAddress confMasterAddress =
@@ -117,6 +123,7 @@ public static void main(String[] args) {
           + "Proceed with local configuration for FUSE: {}", e.toString());
     }
     conf = fsContext.getClusterConf();
+
     final FuseMountOptions opts = parseOptions(args, conf);
     if (opts == null) {
       System.exit(1);
@@ -153,6 +160,11 @@ public static FuseUmountable launchFuse(FileSystem fs, AlluxioConfiguration conf
       FuseMountOptions opts, boolean blocking) throws IOException {
     Preconditions.checkNotNull(opts,
         "Fuse mount options should not be null to launch a Fuse application");
+
+    // There are other entries to this method other than the main function above
+    // It is ok to call this function multiple times.
+    LibFuse.loadLibrary(AlluxioFuseUtils.getVersionPreference(conf));
+
     try {
       String mountPoint = opts.getMountPoint();
       if (!FileUtils.exists(mountPoint)) {
@@ -254,13 +266,24 @@ private static FuseMountOptions parseOptions(String[] args, AlluxioConfiguration
    */
   public static List<String> parseFuseOptions(String[] fuseOptions,
       AlluxioConfiguration alluxioConf) {
+
+    boolean using3 =
+        NativeLibraryLoader.getLoadState().equals(NativeLibraryLoader.LoadState.LOADED_3);
+
     List<String> res = new ArrayList<>();
     boolean noUserMaxWrite = true;
     for (final String opt : fuseOptions) {
       if (opt.isEmpty()) {
         continue;
       }
+
+      // libfuse3 has dropped big_writes
+      if (using3 && opt.equals("big_writes")) {
+        continue;
+      }
+
       res.add("-o" + opt);
+
       if (noUserMaxWrite && opt.startsWith("max_write")) {
         noUserMaxWrite = false;
       }

integration/fuse/src/main/java/alluxio/fuse/AlluxioFuseUtils.java

@@ -13,6 +13,7 @@
 
 import alluxio.AlluxioURI;
 import alluxio.client.file.FileSystem;
+import alluxio.conf.AlluxioConfiguration;
 import alluxio.conf.InstancedConfiguration;
 import alluxio.conf.PropertyKey;
 import alluxio.exception.AccessControlException;
@@ -24,6 +25,8 @@
 import alluxio.exception.FileAlreadyExistsException;
 import alluxio.exception.FileDoesNotExistException;
 import alluxio.exception.InvalidPathException;
+import alluxio.jnifuse.utils.Environment;
+import alluxio.jnifuse.utils.VersionPreference;
 import alluxio.metrics.MetricKey;
 import alluxio.metrics.MetricsSystem;
 import alluxio.util.CommonUtils;
@@ -63,6 +66,28 @@ public final class AlluxioFuseUtils {
 
   private AlluxioFuseUtils() {}
 
+  /**
+   * Gets the libjnifuse version preference set by user.
+   *
+   * @param conf the configuration object
+   * @return the version preference
+   */
+  public static VersionPreference getVersionPreference(AlluxioConfiguration conf) {
+    if (Environment.isMac()) {
+      LOG.info("osxfuse doesn't support libfuse3 api. Using libfuse version 2.");
+      return VersionPreference.VERSION_2;
+    }
+
+    final int val = conf.getInt(PropertyKey.FUSE_JNIFUSE_LIBFUSE_VERSION);
+    if (val == 2) {
+      return VersionPreference.VERSION_2;
+    } else if (val == 3) {
+      return VersionPreference.VERSION_3;
+    } else {
+      return VersionPreference.NO;
+    }
+  }
+
   /**
    * Retrieves the uid of the given user.
    *

integration/fuse/src/main/java/alluxio/fuse/StackMain.java

@@ -14,6 +14,7 @@
 import alluxio.conf.AlluxioConfiguration;
 import alluxio.conf.InstancedConfiguration;
 import alluxio.conf.PropertyKey;
+import alluxio.jnifuse.LibFuse;
 import alluxio.metrics.MetricsSystem;
 import alluxio.util.CommonUtils;
 import alluxio.util.ConfigurationUtils;
@@ -36,12 +37,13 @@ public static void main(String[] args) {
     }
     Path root = Paths.get(args[1]);
     Path mountPoint = Paths.get(args[0]);
+    AlluxioConfiguration conf = new InstancedConfiguration(
+        ConfigurationUtils.defaults());
+    LibFuse.loadLibrary(AlluxioFuseUtils.getVersionPreference(conf));
     StackFS fs = new StackFS(root, mountPoint);
     String[] fuseOpts = new String[args.length - 2];
     System.arraycopy(args, 2, fuseOpts, 0, args.length - 2);
     try {
-      AlluxioConfiguration conf = new InstancedConfiguration(
-          ConfigurationUtils.defaults());
       CommonUtils.PROCESS_TYPE.set(CommonUtils.ProcessType.CLIENT);
       MetricsSystem.startSinks(conf.get(PropertyKey.METRICS_CONF_FILE));
       fs.mount(true, false, fuseOpts);

integration/fuse/src/test/java/alluxio/jnifuse/struct/FuseFileInfoTest.java

@@ -13,6 +13,11 @@
 
 import static org.junit.Assert.assertEquals;
 
+import alluxio.conf.InstancedConfiguration;
+import alluxio.fuse.AlluxioFuseUtils;
+import alluxio.jnifuse.LibFuse;
+import alluxio.jnifuse.utils.VersionPreference;
+
 import jnr.ffi.Pointer;
 import jnr.ffi.Runtime;
 import org.junit.Test;
@@ -22,6 +27,8 @@
 public class FuseFileInfoTest {
   @Test
   public void offset() {
+    VersionPreference versionPreference = AlluxioFuseUtils.getVersionPreference(InstancedConfiguration.defaults());
+    LibFuse.loadLibrary(versionPreference);
     FuseFileInfo jnifi = FuseFileInfo.of(ByteBuffer.allocate(256));
     ru.serce.jnrfuse.struct.FuseFileInfo jnrfi =
         ru.serce.jnrfuse.struct.FuseFileInfo.of(Pointer.wrap(Runtime.getSystemRuntime(), 0x0));

integration/jnifuse/fs/src/main/java/alluxio/jnifuse/AbstractFuseFileSystem.java

@@ -16,6 +16,7 @@
 import alluxio.jnifuse.struct.FuseFileInfo;
 import alluxio.jnifuse.struct.Statvfs;
 import alluxio.jnifuse.utils.SecurityUtils;
+import alluxio.jnifuse.utils.VersionPreference;
 
 import org.apache.commons.lang3.SystemUtils;
 import org.slf4j.Logger;
@@ -35,12 +36,12 @@
  */
 public abstract class AbstractFuseFileSystem implements FuseFileSystem {
 
-  private static final Logger LOG = LoggerFactory.getLogger(AbstractFuseFileSystem.class);
-
   static {
-    LibFuse.loadLibrary();
+    LibFuse.loadLibrary(VersionPreference.NO);
   }
 
+  private static final Logger LOG = LoggerFactory.getLogger(AbstractFuseFileSystem.class);
+
   // timeout to mount a JNI fuse file system in ms
   private static final int MOUNT_TIMEOUT_MS = 2000;
 

integration/jnifuse/fs/src/main/java/alluxio/jnifuse/FuseFillDir.java

@@ -25,8 +25,4 @@ public static int apply(long fillerAddr, long bufaddr, String name, FileStat stb
       return fill(fillerAddr, bufaddr, name, null, off);
     }
   }
-
-  static {
-    LibFuse.loadLibrary();
-  }
 }

integration/jnifuse/fs/src/main/java/alluxio/jnifuse/LibFuse.java

@@ -12,6 +12,7 @@
 package alluxio.jnifuse;
 
 import alluxio.jnifuse.utils.NativeLibraryLoader;
+import alluxio.jnifuse.utils.VersionPreference;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -28,15 +29,11 @@ private enum LibraryState {
   private static AtomicReference<LibraryState> libraryLoaded =
       new AtomicReference<>(LibraryState.NOT_LOADED);
 
-  static {
-    LibFuse.loadLibrary();
-  }
-
   public native int fuse_main_real(AbstractFuseFileSystem fs, int argc, String[] argv);
 
   public native ByteBuffer fuse_get_context();
 
-  public static void loadLibrary() {
+  public static void loadLibrary(VersionPreference preference) {
     if (libraryLoaded.get() == LibraryState.LOADED) {
       return;
     }
@@ -45,7 +42,7 @@ public static void loadLibrary() {
         LibraryState.LOADING)) {
       String tmpDir = System.getenv("JNIFUSE_SHAREDLIB_DIR");
       try {
-        NativeLibraryLoader.getInstance().loadLibrary(tmpDir);
+        NativeLibraryLoader.getInstance().loadLibrary(preference, tmpDir);
       } catch (IOException e) {
         libraryLoaded.set(LibraryState.NOT_LOADED);
         throw new RuntimeException("Unable to load the jni-fuse shared library"

integration/jnifuse/fs/src/main/java/alluxio/jnifuse/struct/Fuse2FuseFileInfo.java

@@ -0,0 +1,45 @@
+/*
+ * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+ * (the "License"). You may not use this work except in compliance with the License, which is
+ * available at www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied, as more fully set forth in the License.
+ *
+ * See the NOTICE file distributed with this work for information regarding copyright ownership.
+ */
+
+package alluxio.jnifuse.struct;
+
+import jnr.ffi.NativeType;
+import jnr.ffi.Runtime;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Maps to struct fuse_file_info in /usr/include/fuse/fuse_common.h
+ */
+public class Fuse2FuseFileInfo extends FuseFileInfo {
+
+  // unused fields are omitted
+
+  /**
+   * Creates a FuseFileInfo class matching the struct fuse_file_info in libfuse2.
+   *
+   * This struct is not meant to be used directly.
+   * You should use {@link alluxio.jnifuse.struct.FuseFileInfo#of(ByteBuffer)}
+   * to create a FuseFileIfo that matches currently used libfuse.
+   *
+   * @param runtime the JNR runtime
+   * @param buffer the ByteBuffer containing struct fuse_file_info from JNR
+   */
+  public Fuse2FuseFileInfo(Runtime runtime, ByteBuffer buffer) {
+    super(runtime, buffer);
+
+    this.flags = new Signed32();
+    new UnsignedLong(); // fh_old
+    new Padding(NativeType.UCHAR, 4); // unused flags and paddings
+    this.fh = new u_int64_t();
+    new u_int64_t(); // lock_owner
+  }
+}