Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Prevent NPE on getPathSize for non-existant file #29

Merged
merged 1 commit into from

2 participants

Gabriel Reid Josh Wills
Gabriel Reid

Some FileSystem implementations (i.e. DistributedFileSystem) return
null (instead of throwing a FileNotFoundException) when listStatus
is called with a non-existant path. The null situation is now handled
instead of allowing an NPE to be thrown, and a warning is logged.

Gabriel Reid gabrielreid Prevent NPE on getPathSize for non-existant file
Some FileSystem implementations (i.e. DistributedFileSystem) return
null (instead of throwing a FileNotFoundException) when listStatus
is called with a non-existant path. The null situation is now handled
instead of allowing an NPE to be thrown, and a warning is logged.
0bde57f
Josh Wills jwills merged commit 9397eda into from
Josh Wills
Owner

Awesome-- thanks man.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on May 24, 2012
  1. Gabriel Reid

    Prevent NPE on getPathSize for non-existant file

    gabrielreid authored
    Some FileSystem implementations (i.e. DistributedFileSystem) return
    null (instead of throwing a FileNotFoundException) when listStatus
    is called with a non-existant path. The null situation is now handled
    instead of allowing an NPE to be thrown, and a warning is logged.
This page is out of date. Refresh to see the latest.
53 src/main/java/com/cloudera/crunch/io/SourceTargetHelper.java
View
@@ -17,6 +17,8 @@
import java.io.FileNotFoundException;
import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -24,28 +26,35 @@
/**
* Functions for configuring the inputs/outputs of MapReduce jobs.
- *
+ *
*/
public class SourceTargetHelper {
- public static long getPathSize(Configuration conf, Path path) throws IOException {
- return getPathSize(FileSystem.get(conf), path);
- }
-
- public static long getPathSize(FileSystem fs, Path path) throws IOException {
- FileStatus[] stati = null;
- try {
- stati = fs.listStatus(path);
- } catch (FileNotFoundException e) {
- return 0L;
- }
-
- if (stati.length == 0) {
- return 0L;
- }
- long size = 0;
- for (FileStatus status : stati) {
- size += status.getLen();
- }
- return size;
- }
+
+ private static final Log LOG = LogFactory.getLog(SourceTargetHelper.class);
+
+ public static long getPathSize(Configuration conf, Path path) throws IOException {
+ return getPathSize(FileSystem.get(conf), path);
+ }
+
+ public static long getPathSize(FileSystem fs, Path path) throws IOException {
+ FileStatus[] stati = null;
+ try {
+ stati = fs.listStatus(path);
+ if (stati == null) {
+ throw new FileNotFoundException(path + " doesn't exist");
+ }
+ } catch (FileNotFoundException e) {
+ LOG.warn("Returning 0 for getPathSize on non-existant path '" + path + "'");
+ return 0L;
+ }
+
+ if (stati.length == 0) {
+ return 0L;
+ }
+ long size = 0;
+ for (FileStatus status : stati) {
+ size += status.getLen();
+ }
+ return size;
+ }
}
36 src/test/java/com/cloudera/crunch/io/SourceTargetHelperTest.java
View
@@ -3,19 +3,39 @@
import static org.junit.Assert.assertEquals;
import java.io.File;
+import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.Test;
public class SourceTargetHelperTest {
- @Test
- public void testGetNonexistentPathSize() throws Exception {
- File tmp = File.createTempFile("pathsize", "");
- Path tmpPath = new Path(tmp.getAbsolutePath());
- tmp.delete();
- FileSystem fs = FileSystem.getLocal(new Configuration());
- assertEquals(0L, SourceTargetHelper.getPathSize(fs, tmpPath));
- }
+ @Test
+ public void testGetNonexistentPathSize() throws Exception {
+ File tmp = File.createTempFile("pathsize", "");
+ Path tmpPath = new Path(tmp.getAbsolutePath());
+ tmp.delete();
+ FileSystem fs = FileSystem.getLocal(new Configuration());
+ assertEquals(0L, SourceTargetHelper.getPathSize(fs, tmpPath));
+ }
+
+ @Test
+ public void testGetNonExistentPathSize_NonExistantPath() throws IOException {
+ FileSystem mockFs = new MockFileSystem();
+ assertEquals(0L, SourceTargetHelper.getPathSize(mockFs, new Path("does/not/exist")));
+ }
+
+ /**
+ * Mock FileSystem that returns null for {@link FileSystem#listStatus(Path)}.
+ */
+ static class MockFileSystem extends LocalFileSystem {
+
+ @Override
+ public FileStatus[] listStatus(Path f) throws IOException {
+ return null;
+ }
+ }
}
Something went wrong with that request. Please try again.