Added modifications so that output to S3 works. Also some cleanups

saptarshiguha · Apr 21, 2015 · 698bef1 · 698bef1
1 parent 15e501f
commit 698bef1
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 6 deletions.
diff --git a/src/main/java/org/godhuli/rhipe/FileUtils.java b/src/main/java/org/godhuli/rhipe/FileUtils.java
@@ -149,15 +149,15 @@ public void copyToLocal(final FileSystem srcFS, final Path src, final File dst)
 
     public void makeFolderToDelete(final String s) throws IOException {
         final Path p = new Path(s);
-        final FileSystem fs = FileSystem.get(cfg);
+        final FileSystem fs = p.getFileSystem(getConf());
         fs.mkdirs(p);
         fs.deleteOnExit(p);
     }
 
     public void copyMain(final String src, final String dest) throws IOException {
         final File dst = new File(dest);
         final Path srcpath = new Path(src);
-        final FileSystem srcFS = FileSystem.get(cfg);
+        final FileSystem srcFS = srcpath.getFileSystem(getConf());
         final FileStatus[] srcs = srcFS.globStatus(srcpath);
         final boolean dstIsDir = dst.isDirectory();
         if (srcs.length > 1 && !dstIsDir) {
@@ -184,7 +184,7 @@ private void ls__(final String path, final ArrayList<String> lsco, final boolean
         final FileSystem srcFS = spath.getFileSystem(getConf());
         FileStatus[] srcs;
 
-        final URI fsUri = new URI(cfg.get("fs.default.name"));
+        final URI fsUri = new URI(getConf().get("fs.default.name"));
         final String fsUriScheme = fsUri.getScheme();
 
         srcs = srcFS.globStatus(spath);
@@ -270,7 +270,7 @@ public REXP readInfo(final String file) throws IOException {
     }
 
     public REXP mapRedOpts() {
-        final Iterator<Map.Entry<String, String>> iter = cfg.iterator();
+        final Iterator<Map.Entry<String, String>> iter = getConf().iterator();
         final List<REXP> ent = new ArrayList<REXP>();
         final List<String> str = new ArrayList<String>();
         while (iter.hasNext()) {

diff --git a/src/main/java/org/godhuli/rhipe/RHMR.java b/src/main/java/org/godhuli/rhipe/RHMR.java
@@ -296,7 +296,7 @@ public void setJob() throws ClassNotFoundException, IOException, URISyntaxExcept
 
         if (!output_folder[0].equals("")) {
             final Path ofp = new Path(output_folder[0]);
-            final FileSystem srcFs = FileSystem.get(job_.getConfiguration());
+            final FileSystem srcFs = path.getFileSystem(job_.getConfiguration());
             srcFs.delete(ofp, true);
             if (rhoptions_.get("rhipe_outputformat_class").equals("org.apache.hadoop.mapreduce.lib.output.NullOutputFormat")) {
                 srcFs.mkdirs(ofp);

diff --git a/src/main/java/org/godhuli/rhipe/SequenceFileIterator.java b/src/main/java/org/godhuli/rhipe/SequenceFileIterator.java
@@ -44,7 +44,6 @@ public void init(final String[] filenames, final int chunksize, final int maxn,
         notcomplete = true;
         mnum = maxn;
         sqr = new SequenceFile.Reader(cfg, SequenceFile.Reader.file(new Path(files[current])));
-//        sqr = new SequenceFile.Reader(fs, new Path(files[current]), cfg);
         k = new RHBytesWritable();
         v = new RHBytesWritable();
         kt = new Text();