Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

Prevent NPE on getPathSize for non-existant file #29

Merged
merged 1 commit into from

2 participants

@gabrielreid

Some FileSystem implementations (i.e. DistributedFileSystem) return
null (instead of throwing a FileNotFoundException) when listStatus
is called with a non-existant path. The null situation is now handled
instead of allowing an NPE to be thrown, and a warning is logged.

@gabrielreid gabrielreid Prevent NPE on getPathSize for non-existant file
Some FileSystem implementations (i.e. DistributedFileSystem) return
null (instead of throwing a FileNotFoundException) when listStatus
is called with a non-existant path. The null situation is now handled
instead of allowing an NPE to be thrown, and a warning is logged.
0bde57f
@jwills jwills merged commit 9397eda into cloudera:master
@jwills
Owner

Awesome-- thanks man.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on May 24, 2012
  1. @gabrielreid

    Prevent NPE on getPathSize for non-existant file

    gabrielreid authored
    Some FileSystem implementations (i.e. DistributedFileSystem) return
    null (instead of throwing a FileNotFoundException) when listStatus
    is called with a non-existant path. The null situation is now handled
    instead of allowing an NPE to be thrown, and a warning is logged.
This page is out of date. Refresh to see the latest.
View
53 src/main/java/com/cloudera/crunch/io/SourceTargetHelper.java
@@ -17,6 +17,8 @@
import java.io.FileNotFoundException;
import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -24,28 +26,35 @@
/**
* Functions for configuring the inputs/outputs of MapReduce jobs.
- *
+ *
*/
public class SourceTargetHelper {
- public static long getPathSize(Configuration conf, Path path) throws IOException {
- return getPathSize(FileSystem.get(conf), path);
- }
-
- public static long getPathSize(FileSystem fs, Path path) throws IOException {
- FileStatus[] stati = null;
- try {
- stati = fs.listStatus(path);
- } catch (FileNotFoundException e) {
- return 0L;
- }
-
- if (stati.length == 0) {
- return 0L;
- }
- long size = 0;
- for (FileStatus status : stati) {
- size += status.getLen();
- }
- return size;
- }
+
+ private static final Log LOG = LogFactory.getLog(SourceTargetHelper.class);
+
+ public static long getPathSize(Configuration conf, Path path) throws IOException {
+ return getPathSize(FileSystem.get(conf), path);
+ }
+
+ public static long getPathSize(FileSystem fs, Path path) throws IOException {
+ FileStatus[] stati = null;
+ try {
+ stati = fs.listStatus(path);
+ if (stati == null) {
+ throw new FileNotFoundException(path + " doesn't exist");
+ }
+ } catch (FileNotFoundException e) {
+ LOG.warn("Returning 0 for getPathSize on non-existant path '" + path + "'");
+ return 0L;
+ }
+
+ if (stati.length == 0) {
+ return 0L;
+ }
+ long size = 0;
+ for (FileStatus status : stati) {
+ size += status.getLen();
+ }
+ return size;
+ }
}
View
36 src/test/java/com/cloudera/crunch/io/SourceTargetHelperTest.java
@@ -3,19 +3,39 @@
import static org.junit.Assert.assertEquals;
import java.io.File;
+import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.Test;
public class SourceTargetHelperTest {
- @Test
- public void testGetNonexistentPathSize() throws Exception {
- File tmp = File.createTempFile("pathsize", "");
- Path tmpPath = new Path(tmp.getAbsolutePath());
- tmp.delete();
- FileSystem fs = FileSystem.getLocal(new Configuration());
- assertEquals(0L, SourceTargetHelper.getPathSize(fs, tmpPath));
- }
+ @Test
+ public void testGetNonexistentPathSize() throws Exception {
+ File tmp = File.createTempFile("pathsize", "");
+ Path tmpPath = new Path(tmp.getAbsolutePath());
+ tmp.delete();
+ FileSystem fs = FileSystem.getLocal(new Configuration());
+ assertEquals(0L, SourceTargetHelper.getPathSize(fs, tmpPath));
+ }
+
+ @Test
+ public void testGetNonExistentPathSize_NonExistantPath() throws IOException {
+ FileSystem mockFs = new MockFileSystem();
+ assertEquals(0L, SourceTargetHelper.getPathSize(mockFs, new Path("does/not/exist")));
+ }
+
+ /**
+ * Mock FileSystem that returns null for {@link FileSystem#listStatus(Path)}.
+ */
+ static class MockFileSystem extends LocalFileSystem {
+
+ @Override
+ public FileStatus[] listStatus(Path f) throws IOException {
+ return null;
+ }
+ }
}
Something went wrong with that request. Please try again.