Skip to content

Commit

Permalink
Experiment with FUSE 3
Browse files Browse the repository at this point in the history 
This increases the maximum write size from 128 KB to 1 MB and supports
some interesting FUSE_READDIR_PLUS flags.  Remove stale
FUSE_CAP_BIG_WRITES and flag_utime_omit_ok options.  References s3fs-fuse#1159.
  • Loading branch information
@gaul
gaul committed May 28, 2023
1 parent 0d6b020 commit 62146a1
Show file tree
Hide file tree
Showing 6 changed files with 53 additions and 55 deletions.
27 changes: 14 additions & 13 deletions .github/workflows/linux-ci-helper.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ if [ "${CONTAINER_FULLNAME}" = "ubuntu:22.04" ]; then
PACKAGE_UPDATE_OPTIONS="update -y -qq"
PACKAGE_INSTALL_OPTIONS="install -y"

INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse3 libfuse3-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip"
INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""

Expand All @@ -94,7 +94,7 @@ elif [ "${CONTAINER_FULLNAME}" = "ubuntu:20.04" ]; then
PACKAGE_UPDATE_OPTIONS="update -y -qq"
PACKAGE_INSTALL_OPTIONS="install -y"

INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse3 libfuse3-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip"
INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""

Expand All @@ -103,7 +103,8 @@ elif [ "${CONTAINER_FULLNAME}" = "ubuntu:18.04" ]; then
PACKAGE_UPDATE_OPTIONS="update -y -qq"
PACKAGE_INSTALL_OPTIONS="install -y"

INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip"
# TODO: does not support FUSE3
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse3 libfuse3-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip"
INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""

Expand All @@ -112,7 +113,7 @@ elif [ "${CONTAINER_FULLNAME}" = "ubuntu:16.04" ]; then
PACKAGE_UPDATE_OPTIONS="update -y -qq"
PACKAGE_INSTALL_OPTIONS="install -y"

INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse3 libfuse3-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip"
INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""

Expand All @@ -121,7 +122,7 @@ elif [ "${CONTAINER_FULLNAME}" = "debian:bullseye" ]; then
PACKAGE_UPDATE_OPTIONS="update -y -qq"
PACKAGE_INSTALL_OPTIONS="install -y"

INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip unzip"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse3 libfuse3-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip unzip"
INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""

Expand All @@ -130,7 +131,7 @@ elif [ "${CONTAINER_FULLNAME}" = "debian:buster" ]; then
PACKAGE_UPDATE_OPTIONS="update -y -qq"
PACKAGE_INSTALL_OPTIONS="install -y"

INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip unzip"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse3 libfuse3-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip unzip"
INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""

Expand All @@ -146,7 +147,7 @@ elif [ "${CONTAINER_FULLNAME}" = "rockylinux:9" ]; then
#
PACKAGE_INSTALL_ADDITIONAL_OPTIONS="--allowerasing"

INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr diffutils curl python3 procps unzip xz https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm"
INSTALL_PACKAGES="curl-devel fuse3 fuse3-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr diffutils curl python3 procps unzip xz https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm"
INSTALL_CHECKER_PKGS="cppcheck"
INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=epel"

Expand All @@ -160,7 +161,7 @@ elif [ "${CONTAINER_FULLNAME}" = "rockylinux:8" ]; then
PACKAGE_UPDATE_OPTIONS="update -y -qq"
PACKAGE_INSTALL_OPTIONS="install -y"

INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr diffutils curl python3 unzip"
INSTALL_PACKAGES="curl-devel fuse3 fuse3-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr diffutils curl python3 unzip"
INSTALL_CHECKER_PKGS="cppcheck"
INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=powertools"

Expand All @@ -179,7 +180,7 @@ elif [ "${CONTAINER_FULLNAME}" = "centos:centos7" ]; then
# And in this version, it cannot be passed due to following error.
# "shellcheck: ./test/integration-test-main.sh: hGetContents: invalid argument (invalid byte sequence)"
#
INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr curl python3 epel-release unzip"
INSTALL_PACKAGES="curl-devel fuse3 fuse3-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr curl python3 epel-release unzip"
INSTALL_CHECKER_PKGS="cppcheck"
INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=epel"

Expand All @@ -189,7 +190,7 @@ elif [ "${CONTAINER_FULLNAME}" = "fedora:38" ]; then
PACKAGE_INSTALL_OPTIONS="install -y"

# TODO: Cannot use java-latest-openjdk (17) due to modules issue in S3Proxy/jclouds/Guice
INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel curl attr diffutils procps python3-pip unzip"
INSTALL_PACKAGES="curl-devel fuse3 fuse3-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel curl attr diffutils procps python3-pip unzip"
INSTALL_CHECKER_PKGS="cppcheck ShellCheck"
INSTALL_CHECKER_PKG_OPTIONS=""

Expand All @@ -199,7 +200,7 @@ elif [ "${CONTAINER_FULLNAME}" = "fedora:37" ]; then
PACKAGE_INSTALL_OPTIONS="install -y"

# TODO: Cannot use java-latest-openjdk (17) due to modules issue in S3Proxy/jclouds/Guice
INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel curl attr diffutils procps python3-pip unzip"
INSTALL_PACKAGES="curl-devel fuse3 fuse3-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel curl attr diffutils procps python3-pip unzip"
INSTALL_CHECKER_PKGS="cppcheck ShellCheck"
INSTALL_CHECKER_PKG_OPTIONS=""

Expand All @@ -208,7 +209,7 @@ elif [ "${CONTAINER_FULLNAME}" = "opensuse/leap:15" ]; then
PACKAGE_UPDATE_OPTIONS="refresh"
PACKAGE_INSTALL_OPTIONS="install -y"

INSTALL_PACKAGES="automake curl-devel fuse fuse-devel gcc-c++ java-11-openjdk-headless libxml2-devel make openssl-devel python3-pip curl attr ShellCheck unzip"
INSTALL_PACKAGES="automake curl-devel fuse3 fuse3-devel gcc-c++ java-11-openjdk-headless libxml2-devel make openssl-devel python3-pip curl attr ShellCheck unzip"
INSTALL_CHECKER_PKGS="cppcheck ShellCheck"
INSTALL_CHECKER_PKG_OPTIONS=""

Expand All @@ -217,7 +218,7 @@ elif [ "${CONTAINER_FULLNAME}" = "alpine:3.17" ]; then
PACKAGE_UPDATE_OPTIONS="update --no-progress"
PACKAGE_INSTALL_OPTIONS="add --no-progress --no-cache"

INSTALL_PACKAGES="bash curl g++ make automake autoconf libtool git curl-dev fuse-dev libxml2-dev coreutils procps attr sed mailcap openjdk11 aws-cli"
INSTALL_PACKAGES="bash curl g++ make automake autoconf libtool git curl-dev fuse3 fuse3-dev libxml2-dev coreutils procps attr sed mailcap openjdk11 aws-cli"
INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""

Expand Down
12 changes: 6 additions & 6 deletions configure.ac
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ case "$target" in
*)
# Default Case
# assume other supported linux system
min_fuse_version=2.8.4
min_fuse_version=3.0.0
;;
esac

Expand Down Expand Up @@ -183,13 +183,13 @@ dnl
dnl For PKG_CONFIG before checking nss/gnutls.
dnl this is redundant checking, but we need checking before following.
dnl
PKG_CHECK_MODULES([common_lib_checking], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 ])
PKG_CHECK_MODULES([common_lib_checking], [fuse3 >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 ])

AC_MSG_CHECKING([compile s3fs with])
case "${auth_lib}" in
openssl)
AC_MSG_RESULT(OpenSSL)
PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9 ])
PKG_CHECK_MODULES([DEPS], [fuse3 >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9 ])
AC_MSG_CHECKING([openssl 3.0 or later])
AC_COMPILE_IFELSE(
[AC_LANG_PROGRAM([[#include <openssl/opensslv.h>
Expand All @@ -206,7 +206,7 @@ gnutls)
AS_IF([test "$gnutls_nettle" = ""], [AC_CHECK_LIB(gcrypt, gcry_control, [gnutls_nettle=0])])
AS_IF([test $gnutls_nettle = 0],
[
PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 gnutls >= 2.12.0 ])
PKG_CHECK_MODULES([DEPS], [fuse3 >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 gnutls >= 2.12.0 ])
LIBS="-lgnutls -lgcrypt $LIBS"
AC_MSG_CHECKING([gnutls is build with])
AC_MSG_RESULT(gcrypt)
Expand All @@ -220,7 +220,7 @@ nettle)
AS_IF([test "$gnutls_nettle" = ""], [AC_CHECK_LIB(nettle, nettle_MD5Init, [gnutls_nettle=1])])
AS_IF([test $gnutls_nettle = 1],
[
PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nettle >= 2.7.1 ])
PKG_CHECK_MODULES([DEPS], [fuse3 >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nettle >= 2.7.1 ])
LIBS="-lgnutls -lnettle $LIBS"
AC_MSG_CHECKING([gnutls is build with])
AC_MSG_RESULT(nettle)
Expand All @@ -229,7 +229,7 @@ nettle)
;;
nss)
AC_MSG_RESULT(NSS)
PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nss >= 3.15.0 ])
PKG_CHECK_MODULES([DEPS], [fuse3 >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nss >= 3.15.0 ])
;;
*)
AC_MSG_ERROR([unknown ssl library type.])
Expand Down
2 changes: 2 additions & 0 deletions src/fdcache_entity.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
#include <unistd.h>
#include <limits.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <utime.h>

#include "common.h"
#include "fdcache_entity.h"
Expand Down

0 comments on commit 62146a1

Please sign in to comment.