Permalink
Browse files

Merge pull request #243 from rangadi/remote_sequence_file_loader

fix how Filesystem is initialized. It should be based on both path and conf.
  • Loading branch information...
2 parents 8dd1811 + 236f0da commit 00b6e7089271be34edb77a06ee85bc65d4c65de2 @rangadi rangadi committed Sep 11, 2012
@@ -8,7 +8,6 @@
import com.twitter.elephantbird.util.W3CLogParser;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.MapWritable;
@@ -58,8 +57,8 @@ protected void createInputReader(InputStream input, Configuration conf) throws I
in_ = new LineReader(input, conf);
String fileURI = getFieldDefinitionFile();
- FileSystem fs = FileSystem.get(URI.create(fileURI), conf);
- InputStream is = fs.open(new Path(fileURI));
+ Path path = new Path(fileURI);
+ InputStream is = path.getFileSystem(conf).open(path);
w3cLogParser_ = new W3CLogParser(is);
is.close();
}
@@ -5,7 +5,6 @@
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.DataInputBuffer;
@@ -44,7 +43,7 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws
FileSplit fileSplit = (FileSplit) inputSplit;
Path path = fileSplit.getPath();
// inhibit class loading during SequenceFile.Reader initialization
- reader = new SequenceFile.Reader(FileSystem.get(conf), path, conf) {
+ reader = new SequenceFile.Reader(path.getFileSystem(conf), path, conf) {
@Override
public synchronized Class<?> getKeyClass() {
return BytesWritable.class;
@@ -22,7 +22,7 @@
* use the following code:
* <code>
* Configuration conf = new Configuration();
- * FileSystem fs = FileSystem.get(conf);
+ * FileSystem fs = lzoPath.getFileSystem(conf);
* FSDataOutputStream outputStream = fs.create(lzoPath, true);
* LzopCodec codec = new LzopCodec();
* codec.setConf(conf);
@@ -76,7 +76,7 @@ public void setUp() throws Exception {
tempFilename = tempFile.getAbsolutePath();
Path path = new Path("file:///" + tempFilename);
Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(path.toUri(), conf);
+ FileSystem fs = path.getFileSystem(conf);
IntWritable key = new IntWritable();
Text value = new Text();
SequenceFile.Writer writer = null;
@@ -107,7 +107,7 @@ public void setup() throws IOException {
tempFilename = tempFile.getAbsolutePath();
final Path path = new Path("file:///" + tempFilename);
final Configuration conf = new Configuration();
- final FileSystem fs = FileSystem.get(path.toUri(), conf);
+ final FileSystem fs = path.getFileSystem(conf);
final IntWritable key = new IntWritable();
SequenceFile.Writer writer = null;
try {

0 comments on commit 00b6e70

Please sign in to comment.