Skip to content

Commit

Permalink
HDDS-4077. Incomplete OzoneFileSystem statistics (#1329)
Browse files Browse the repository at this point in the history
  • Loading branch information
@adoroszlai
adoroszlai committed Aug 31, 2020
1 parent d34ab29 commit 78ca8bf
Show file tree
Hide file tree
Showing 3 changed files with 153 additions and 4 deletions.
15 changes: 11 additions & 4 deletions ...ne/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
View file
Expand Up @@ -357,15 +357,18 @@ public void testListStatus() throws IOException {
String dirPath = RandomStringUtils.randomAlphanumeric(5);
Path path = createPath("/" + dirPath);
paths.add(path);

long mkdirs = statistics.getLong(
StorageStatistics.CommonStatisticNames.OP_MKDIRS);
assertTrue("Makedirs returned with false for the path " + path,
fs.mkdirs(path));
assertCounter(++mkdirs, StorageStatistics.CommonStatisticNames.OP_MKDIRS);

long listObjects = statistics.getLong(Statistic.OBJECTS_LIST.getSymbol());
long omListStatus = omMetrics.getNumListStatus();
FileStatus[] statusList = fs.listStatus(createPath("/"));
assertEquals(1, statusList.length);
assertEquals(++listObjects,
statistics.getLong(Statistic.OBJECTS_LIST.getSymbol()).longValue());
assertCounter(++listObjects, Statistic.OBJECTS_LIST.getSymbol());
assertEquals(++omListStatus, omMetrics.getNumListStatus());
assertEquals(fs.getFileStatus(path), statusList[0]);

Expand All @@ -374,11 +377,11 @@ public void testListStatus() throws IOException {
paths.add(path);
assertTrue("Makedirs returned with false for the path " + path,
fs.mkdirs(path));
assertCounter(++mkdirs, StorageStatistics.CommonStatisticNames.OP_MKDIRS);

statusList = fs.listStatus(createPath("/"));
assertEquals(2, statusList.length);
assertEquals(++listObjects,
statistics.getLong(Statistic.OBJECTS_LIST.getSymbol()).longValue());
assertCounter(++listObjects, Statistic.OBJECTS_LIST.getSymbol());
assertEquals(++omListStatus, omMetrics.getNumListStatus());
for (Path p : paths) {
assertTrue(Arrays.asList(statusList).contains(fs.getFileStatus(p)));
Expand Down Expand Up @@ -528,4 +531,8 @@ private FileStatus getDirectoryStat(Path path) throws IOException {

return status;
}

private void assertCounter(long value, String key) {
assertEquals(value, statistics.getLong(key).longValue());
}
}
71 changes: 71 additions & 0 deletions ...p-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
View file
Expand Up @@ -26,13 +26,16 @@
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
Expand Down Expand Up @@ -692,6 +695,7 @@ private boolean mkdir(Path path) throws IOException {

@Override
public boolean mkdirs(Path f, FsPermission permission) throws IOException {
incrementCounter(Statistic.INVOCATION_MKDIRS);
LOG.trace("mkdir() path:{} ", f);
String key = pathToKey(f);
if (isEmpty(key)) {
Expand Down Expand Up @@ -735,6 +739,73 @@ public short getDefaultReplication() {
return adapter.getDefaultReplication();
}

@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path[] srcs,
Path dst) throws IOException {
incrementCounter(Statistic.INVOCATION_COPY_FROM_LOCAL_FILE);
super.copyFromLocalFile(delSrc, overwrite, srcs, dst);
}

@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
Path dst) throws IOException {
incrementCounter(Statistic.INVOCATION_COPY_FROM_LOCAL_FILE);
super.copyFromLocalFile(delSrc, overwrite, src, dst);
}

@Override
public boolean exists(Path f) throws IOException {
incrementCounter(Statistic.INVOCATION_EXISTS);
return super.exists(f);
}

@Override
public FileChecksum getFileChecksum(Path f, long length) throws IOException {
incrementCounter(Statistic.INVOCATION_GET_FILE_CHECKSUM);
return super.getFileChecksum(f, length);
}

@Override
public FileStatus[] globStatus(Path pathPattern) throws IOException {
incrementCounter(Statistic.INVOCATION_GLOB_STATUS);
return super.globStatus(pathPattern);
}

@Override
public FileStatus[] globStatus(Path pathPattern, PathFilter filter)
throws IOException {
incrementCounter(Statistic.INVOCATION_GLOB_STATUS);
return super.globStatus(pathPattern, filter);
}

@Override
@SuppressWarnings("deprecation")
public boolean isDirectory(Path f) throws IOException {
incrementCounter(Statistic.INVOCATION_IS_DIRECTORY);
return super.isDirectory(f);
}

@Override
@SuppressWarnings("deprecation")
public boolean isFile(Path f) throws IOException {
incrementCounter(Statistic.INVOCATION_IS_FILE);
return super.isFile(f);
}

@Override
public RemoteIterator<LocatedFileStatus> listFiles(Path f, boolean recursive)
throws IOException {
incrementCounter(Statistic.INVOCATION_LIST_FILES);
return super.listFiles(f, recursive);
}

@Override
public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f)
throws IOException {
incrementCounter(Statistic.INVOCATION_LIST_LOCATED_STATUS);
return super.listLocatedStatus(f);
}

/**
* Turn a path (relative or otherwise) into an Ozone key.
*
Expand Down
71 changes: 71 additions & 0 deletions ...e/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
View file
Expand Up @@ -24,12 +24,15 @@
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.annotation.InterfaceStability;
Expand Down Expand Up @@ -715,6 +718,7 @@ private boolean mkdir(Path path) throws IOException {

@Override
public boolean mkdirs(Path f, FsPermission permission) throws IOException {
incrementCounter(Statistic.INVOCATION_MKDIRS);
LOG.trace("mkdir() path:{} ", f);
String key = pathToKey(f);
if (isEmpty(key)) {
Expand Down Expand Up @@ -764,6 +768,73 @@ public short getDefaultReplication() {
return adapter.getDefaultReplication();
}

@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path[] srcs,
Path dst) throws IOException {
incrementCounter(Statistic.INVOCATION_COPY_FROM_LOCAL_FILE);
super.copyFromLocalFile(delSrc, overwrite, srcs, dst);
}

@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
Path dst) throws IOException {
incrementCounter(Statistic.INVOCATION_COPY_FROM_LOCAL_FILE);
super.copyFromLocalFile(delSrc, overwrite, src, dst);
}

@Override
public boolean exists(Path f) throws IOException {
incrementCounter(Statistic.INVOCATION_EXISTS);
return super.exists(f);
}

@Override
public FileChecksum getFileChecksum(Path f, long length) throws IOException {
incrementCounter(Statistic.INVOCATION_GET_FILE_CHECKSUM);
return super.getFileChecksum(f, length);
}

@Override
public FileStatus[] globStatus(Path pathPattern) throws IOException {
incrementCounter(Statistic.INVOCATION_GLOB_STATUS);
return super.globStatus(pathPattern);
}

@Override
public FileStatus[] globStatus(Path pathPattern, PathFilter filter)
throws IOException {
incrementCounter(Statistic.INVOCATION_GLOB_STATUS);
return super.globStatus(pathPattern, filter);
}

@Override
@SuppressWarnings("deprecation")
public boolean isDirectory(Path f) throws IOException {
incrementCounter(Statistic.INVOCATION_IS_DIRECTORY);
return super.isDirectory(f);
}

@Override
@SuppressWarnings("deprecation")
public boolean isFile(Path f) throws IOException {
incrementCounter(Statistic.INVOCATION_IS_FILE);
return super.isFile(f);
}

@Override
public RemoteIterator<LocatedFileStatus> listFiles(Path f, boolean recursive)
throws IOException {
incrementCounter(Statistic.INVOCATION_LIST_FILES);
return super.listFiles(f, recursive);
}

@Override
public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f)
throws IOException {
incrementCounter(Statistic.INVOCATION_LIST_LOCATED_STATUS);
return super.listLocatedStatus(f);
}

/**
* Turn a path (relative or otherwise) into an Ozone key.
*
Expand Down

0 comments on commit 78ca8bf

Please sign in to comment.