Skip to content

Commit

Permalink
[ALLUXIO-3361] add fuse integration test (#8130)
Browse files Browse the repository at this point in the history 
* enable fuse integration test

* remove unnecessary files

* fix bugs

* improve coding styles

* add chown and chgrp

* fix chown without group

* change to @BeforeTest

* fix errors in linux cluster

* change to constants

* fix checkstyle

* small fix

* change to @BeforeClass

* improve fuse unit test

* fix @after bug

* change dd command to shorten the test time

* fix fuseMounted
  • Loading branch information
@LuQQiu @calvinjia
LuQQiu authored and calvinjia committed Nov 26, 2018
1 parent 7fcf84d commit 818a23e
Show file tree
Hide file tree
Showing 6 changed files with 461 additions and 25 deletions.
67 changes: 44 additions & 23 deletions integration/fuse/src/main/java/alluxio/fuse/AlluxioFuseFileSystem.java
View file
Expand Up @@ -26,6 +26,7 @@
import alluxio.util.CommonUtils;
import alluxio.util.WaitForOptions;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
Expand Down Expand Up @@ -61,16 +62,25 @@
* Implements the FUSE callbacks defined by jnr-fuse.
*/
@ThreadSafe
final class AlluxioFuseFileSystem extends FuseStubFS {
public final class AlluxioFuseFileSystem extends FuseStubFS {
private static final Logger LOG = LoggerFactory.getLogger(AlluxioFuseFileSystem.class);

private static final int MAX_OPEN_FILES = Integer.MAX_VALUE;
private static final int MAX_OPEN_WAITTIME_MS = 5000;

/**
* 4294967295 is unsigned long -1, -1 means that uid or gid is not set.
* 4294967295 or -1 occurs when chown without user name or group name.
* Please view https://github.com/SerCeMan/jnr-fuse/issues/67 for more details.
*/
@VisibleForTesting
public static final long ID_NOT_SET_VALUE = -1;
@VisibleForTesting
public static final long ID_NOT_SET_VALUE_UNSIGNED = 4294967295L;

private static final long UID = AlluxioFuseUtils.getUid(System.getProperty("user.name"));
private static final long GID = AlluxioFuseUtils.getGid(System.getProperty("user.name"));
private final boolean mIsUserGroupTranslation;

private final boolean mIsUserGroupTranslation;
private final FileSystem mFileSystem;
// base path within Alluxio namespace that is used for FUSE operations
// For example, if alluxio-fuse is mounted in /mnt/alluxio and mAlluxioRootPath
Expand All @@ -79,9 +89,9 @@ final class AlluxioFuseFileSystem extends FuseStubFS {
private final Path mAlluxioRootPath;
// Keeps a cache of the most recently translated paths from String to Alluxio URI
private final LoadingCache<String, AlluxioURI> mPathResolverCache;

// Table of open files with corresponding InputStreams and OutputStreams
private final Map<Long, OpenFileEntry> mOpenFiles;

private long mNextOpenFileId;

/**
Expand All @@ -90,7 +100,7 @@ final class AlluxioFuseFileSystem extends FuseStubFS {
* @param fs Alluxio file system
* @param opts options
*/
AlluxioFuseFileSystem(FileSystem fs, AlluxioFuseOptions opts) {
public AlluxioFuseFileSystem(FileSystem fs, AlluxioFuseOptions opts) {
super();
mFileSystem = fs;
mAlluxioRootPath = Paths.get(opts.getAlluxioRoot());
Expand Down Expand Up @@ -149,34 +159,45 @@ public int chown(String path, @uid_t long uid, @gid_t long gid) {
}

try {
String groupName = AlluxioFuseUtils.getGroupName(gid);
if (groupName.isEmpty()) {
// This should never be reached since input gid is always valid
// If user chown without group name, the primary group gid of the user name will be provided
LOG.error("Failed to get group name from gid {}.", gid);
return -ErrorCodes.EFAULT();
}

SetAttributeOptions options = SetAttributeOptions.defaults().setGroup(groupName);
SetAttributeOptions options = SetAttributeOptions.defaults();
final AlluxioURI uri = mPathResolverCache.getUnchecked(path);

if (uid != -1 && uid != 4294967295L) {
// 4294967295 is just unsigned long -1, -1 means that uid is not set
// 4294967295 or -1 occurs when chown without user name or chgrp
// Please view https://github.com/SerCeMan/jnr-fuse/issues/67 for more details
String userName = AlluxioFuseUtils.getUserName(uid);
String userName = "";
if (uid != ID_NOT_SET_VALUE && uid != ID_NOT_SET_VALUE_UNSIGNED) {
userName = AlluxioFuseUtils.getUserName(uid);
if (userName.isEmpty()) {
// This should never be reached
LOG.error("Failed to get user name from uid {}", uid);
return -ErrorCodes.EFAULT();
}
options.setOwner(userName);
LOG.info("Change owner and group of file {} to {}:{}", path, userName, groupName);
} else {
LOG.info("Change group of file {} to {}", path, groupName);
}

mFileSystem.setAttribute(uri, options);
String groupName = "";
if (gid != ID_NOT_SET_VALUE && gid != ID_NOT_SET_VALUE_UNSIGNED) {
groupName = AlluxioFuseUtils.getGroupName(gid);
if (groupName.isEmpty()) {
// This should never be reached
LOG.error("Failed to get group name from gid {}", gid);
return -ErrorCodes.EFAULT();
}
options.setGroup(groupName);
} else if (!userName.isEmpty()) {
groupName = AlluxioFuseUtils.getGroupName(userName);
options.setGroup(groupName);
}

if (userName.isEmpty() && groupName.isEmpty()) {
// This should never be reached
LOG.info("Unable to change owner and group of file {} when uid is {} and gid is {}",
path, userName, groupName);
} else if (userName.isEmpty()) {
LOG.info("Change group of file {} to {}", path, groupName);
mFileSystem.setAttribute(uri, options);
} else {
LOG.info("Change owner of file {} to {}", path, groupName);
mFileSystem.setAttribute(uri, options);
}
} catch (IOException | AlluxioException e) {
LOG.error("Exception on {}", path, e);
return -ErrorCodes.EIO();
Expand Down
11 changes: 9 additions & 2 deletions integration/fuse/src/main/java/alluxio/fuse/AlluxioFuseOptions.java
View file
Expand Up @@ -19,13 +19,20 @@
* Convenience class to pass around Alluxio-FUSE options.
*/
@ThreadSafe
final class AlluxioFuseOptions {
public final class AlluxioFuseOptions {
private final String mMountPoint;
private final String mAlluxioRoot;
private final boolean mDebug;
private final List<String> mFuseOpts;

AlluxioFuseOptions(String mountPoint, String alluxioRoot, boolean debug, List<String> fuseOpts) {
/**
* @param mountPoint the path to where the FS should be mounted
* @param alluxioRoot the path within alluxio that will be used as the mounted FS root
* @param debug whether the file system should be mounted in debug mode
* @param fuseOpts extra options to pass to the FUSE mount command
*/
public AlluxioFuseOptions(String mountPoint, String alluxioRoot,
boolean debug, List<String> fuseOpts) {
mMountPoint = mountPoint;
mAlluxioRoot = alluxioRoot;
mDebug = debug;
Expand Down
21 changes: 21 additions & 0 deletions integration/fuse/src/main/java/alluxio/fuse/AlluxioFuseUtils.java
View file
Expand Up @@ -112,6 +112,27 @@ public static String getGroupName(long gid) throws IOException {
return "";
}

/**
* Checks whether fuse is installed in local file system.
* Alluxio-Fuse only support mac and linux.
*
* @return true if fuse is installed, false otherwise
*/
public static boolean isFuseInstalled() {
try {
if (OSUtils.isLinux()) {
String result = ShellUtils.execCommand("fusermount", "-V");
return !result.isEmpty();
} else if (OSUtils.isMacOS()) {
String result = ShellUtils.execCommand("bash", "-c", "mount | grep FUSE");
return !result.isEmpty();
}
} catch (Exception e) {
return false;
}
return false;
}

/**
* Runs the "id" command with the given options on the passed username.
*
Expand Down
75 changes: 75 additions & 0 deletions integration/fuse/src/test/java/alluxio/fuse/AlluxioFuseFileSystemTest.java
View file
Expand Up @@ -18,6 +18,7 @@
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Mockito.atLeast;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
Expand Down Expand Up @@ -106,6 +107,54 @@ public void chown() throws Exception {
verify(mFileSystem).setAttribute(expectedPath, options);
}

@Test
public void chownWithoutValidGid() throws Exception {
long uid = AlluxioFuseUtils.getUid(System.getProperty("user.name"));
long gid = AlluxioFuseFileSystem.ID_NOT_SET_VALUE;
mFuseFs.chown("/foo/bar", uid, gid);
String userName = System.getProperty("user.name");
String groupName = AlluxioFuseUtils.getGroupName(userName);
AlluxioURI expectedPath = BASE_EXPECTED_URI.join("/foo/bar");
SetAttributeOptions options =
SetAttributeOptions.defaults().setGroup(groupName).setOwner(userName);
verify(mFileSystem).setAttribute(expectedPath, options);

gid = AlluxioFuseFileSystem.ID_NOT_SET_VALUE_UNSIGNED;
mFuseFs.chown("/foo/bar", uid, gid);
verify(mFileSystem, times(2)).setAttribute(expectedPath, options);
}

@Test
public void chownWithoutValidUid() throws Exception {
String userName = System.getProperty("user.name");
long uid = AlluxioFuseFileSystem.ID_NOT_SET_VALUE;
long gid = AlluxioFuseUtils.getGid(userName);
mFuseFs.chown("/foo/bar", uid, gid);

String groupName = AlluxioFuseUtils.getGroupName(userName);
AlluxioURI expectedPath = BASE_EXPECTED_URI.join("/foo/bar");
SetAttributeOptions options =
SetAttributeOptions.defaults().setGroup(groupName);
verify(mFileSystem).setAttribute(expectedPath, options);

uid = AlluxioFuseFileSystem.ID_NOT_SET_VALUE_UNSIGNED;
mFuseFs.chown("/foo/bar", uid, gid);
verify(mFileSystem, times(2)).setAttribute(expectedPath, options);
}

@Test
public void chownWithoutValidUidAndGid() throws Exception {
long uid = AlluxioFuseFileSystem.ID_NOT_SET_VALUE;
long gid = AlluxioFuseFileSystem.ID_NOT_SET_VALUE;
mFuseFs.chown("/foo/bar", uid, gid);
verify(mFileSystem, never()).setAttribute(any());

uid = AlluxioFuseFileSystem.ID_NOT_SET_VALUE_UNSIGNED;
gid = AlluxioFuseFileSystem.ID_NOT_SET_VALUE_UNSIGNED;
mFuseFs.chown("/foo/bar", uid, gid);
verify(mFileSystem, never()).setAttribute(any());
}

@Test
public void create() throws Exception {
mFileInfo.flags.set(O_WRONLY.intValue());
Expand Down Expand Up @@ -271,6 +320,19 @@ public void renameNewExist() throws Exception {
assertEquals(-ErrorCodes.EEXIST(), mFuseFs.rename("/old", "/new"));
}

@Test
public void rmdir() throws Exception {
AlluxioURI expectedPath = BASE_EXPECTED_URI.join("/foo/bar");
FileInfo info = new FileInfo();
info.setFolder(true);
URIStatus status = new URIStatus(info);
when(mFileSystem.getStatus(expectedPath)).thenReturn(status);
when(mFileSystem.exists(expectedPath)).thenReturn(true);
doNothing().when(mFileSystem).delete(expectedPath);
mFuseFs.rmdir("/foo/bar");
verify(mFileSystem).delete(expectedPath);
}

@Test
public void write() throws Exception {
FileOutStream fos = mock(FileOutStream.class);
Expand All @@ -295,6 +357,19 @@ public void write() throws Exception {
verify(fos, times(1)).write(expected);
}

@Test
public void unlink() throws Exception {
AlluxioURI expectedPath = BASE_EXPECTED_URI.join("/foo/bar");
FileInfo info = new FileInfo();
info.setFolder(false);
URIStatus status = new URIStatus(info);
when(mFileSystem.getStatus(expectedPath)).thenReturn(status);
when(mFileSystem.exists(expectedPath)).thenReturn(true);
doNothing().when(mFileSystem).delete(expectedPath);
mFuseFs.unlink("/foo/bar");
verify(mFileSystem).delete(expectedPath);
}

@Test
public void pathTranslation() throws Exception {
final LoadingCache<String, AlluxioURI> resolver = mFuseFs.getPathResolverCache();
Expand Down
5 changes: 5 additions & 0 deletions tests/pom.xml
View file
Expand Up @@ -215,6 +215,11 @@
<artifactId>alluxio-underfs-local</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.alluxio</groupId>
<artifactId>alluxio-integration-fuse</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>

<profiles>
Expand Down

0 comments on commit 818a23e

Please sign in to comment.