Skip to content

Commit

Permalink
Fix error reporting in AvroUtils.getSchemaFromPath()
Browse files Browse the repository at this point in the history
- report errors with an exception
- report errors exactly once
- provide the failing pathname
- don't generate spurious cascading NPE failures
  • Loading branch information
gnb authored and FelixGV committed Jun 9, 2015
1 parent 037a0dc commit e13f6a2
Showing 1 changed file with 11 additions and 15 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -54,31 +54,30 @@ private static Schema getSchemaFromPath(FileSystem fs, Path path, boolean checkS
try {
inStream = new BufferedInputStream(fs.open(path));
} catch(IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
throw new RuntimeException("Unable to open " + path, e1);
}
GenericDatumReader datum = new GenericDatumReader();

DataFileStream reader = null;
try {
reader = new DataFileStream(inStream, datum);
} catch(IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
throw new RuntimeException("Invalid avro format, path " + path, e);
}
return reader.getSchema();
} else {
FileStatus[] statuses = null;
if(fs.isDirectory(path)) {
// this is a directory, get schemas from all subfiles
statuses = fs.listStatus(path);
if(statuses == null || statuses.length == 0)
throw new IllegalArgumentException("No files in directory " + path);
} else {
// this is wildcard path, get schemas from all matched files
statuses = fs.globStatus(path);
if(statuses == null || statuses.length == 0)
throw new IllegalArgumentException("No matches for path pattern " + path);
}
if(statuses == null || statuses.length == 0)
throw new IllegalArgumentException("No files found in path pattern "
+ path.toUri().getPath());
List<Schema> schemas = new ArrayList<Schema>();
for(FileStatus status: statuses) {
if(!HadoopUtils.shouldPathBeIgnored(status.getPath())) {
Expand All @@ -96,22 +95,19 @@ private static Schema getSchemaFromPath(FileSystem fs, Path path, boolean checkS
for(int i = 1; i < schemas.size(); i++)
if(!schema.equals(schemas.get(i)))
throw new IllegalArgumentException("The directory "
+ path.toString()
+ path
+ " contains heterogenous schemas: found both '"
+ schema.toString() + "' and '"
+ schemas.get(i).toString() + "'.");
+ schema + "' and '"
+ schemas.get(i) + "'.");

return schema;
} else {
throw new IllegalArgumentException("No Valid metadata file found for Path:"
+ path.toString());
throw new IllegalArgumentException("No valid metadata file found for path " + path);
}
}
} catch(Exception e) {
// logger.error("failed to get metadata from path:" + path);
throw new RuntimeException(e);
throw new RuntimeException("Error getting schema for path " + path, e);
}

}

public static Schema getAvroSchemaFromPath(Path path) throws IOException {
Expand Down

0 comments on commit e13f6a2

Please sign in to comment.