#32 -- add 'fs' from Hadoop so we can add filesystem operations (part…

…icularly deletes) to flows
paulhoule · Oct 12, 2013 · 388b46c · 388b46c
1 parent 010fb62
commit 388b46c
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 1 deletion.
diff --git a/bakemono/src/main/java/com/ontology2/bakemono/Main.java b/bakemono/src/main/java/com/ontology2/bakemono/Main.java
@@ -9,6 +9,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.compress.GzipCodec;
@@ -56,6 +57,7 @@ public IncorrectUsageException(String message) {
         put("pse3",PSE3Tool.class);
         put("sieve3",Sieve3Tool.class);
         put("ranSample",RanSampleTool.class);
+        put("fs",FsShell.class);                    // from Hadoop
     }};
 
     Map<String, TopLevelCommand> myCommands=new HashMap<String,TopLevelCommand>() {{

diff --git a/haruhi/src/main/resources/com/ontology2/haruhi/shell/applicationContext.xml b/haruhi/src/main/resources/com/ontology2/haruhi/shell/applicationContext.xml
@@ -81,6 +81,17 @@
               </list>
             </constructor-arg>
         </bean>
+        <bean class="com.ontology2.haruhi.flows.JobStep">
+            <constructor-arg>
+              <list>
+                <value>'run'</value>
+                <value>'fs'</value>
+                <value>'-rmr'</value>
+                <value>pos[2]+pos[1]+'/accepted/'</value>
+                <value>tmpDir+'preprocessed/'+pos[1]+'/'</value>
+              </list>
+            </constructor-arg>
+        </bean>
       </list>
     </constructor-arg>
   </bean>

diff --git a/haruhi/src/test/java/com/ontology2/haruhi/flows/TestFlowBeans.java b/haruhi/src/test/java/com/ontology2/haruhi/flows/TestFlowBeans.java
@@ -26,7 +26,7 @@ public class TestFlowBeans {
         List<String> flowArgs=Lists.newArrayList("s3n://freebase-dumps/","1942-12-07-00-00","s3n://basekb-now/");
         List<FlowStep> steps=basekbNowFlow.generateSteps(flowArgs);
         assertNotNull(steps);
-        assertEquals(3,steps.size());
+        assertEquals(4,steps.size());
 
         {
             assertTrue(steps.get(0) instanceof SpringStep);
@@ -69,5 +69,20 @@ public class TestFlowBeans {
             assertEquals("s3n://basekb-now/1942-12-07-00-00/accepted/",args.get(i++));
             assertEquals("s3n://basekb-now/1942-12-07-00-00/sieved/",args.get(i++));
         }
+
+        {
+            assertTrue(steps.get(3) instanceof SpringStep);
+            SpringStep step2=(SpringStep) steps.get(3);
+            List<String> args=step2.getStepArgs(flowArgs);
+
+            assertEquals(5,args.size());
+
+            int i=0;
+            assertEquals("run",args.get(i++));
+            assertEquals("fs",args.get(i++));
+            assertEquals("-rmr",args.get(i++));
+            assertEquals("s3n://basekb-now/1942-12-07-00-00/accepted/",args.get(i++));
+            assertEquals("/preprocessed/1942-12-07-00-00/",args.get(i++));
+        }
     }
 }