Permalink
Browse files

output management for emr

  • Loading branch information...
1 parent 99d6bfd commit 087253be5766e74881b7964a726257f407b340cd @erh erh committed May 25, 2012
Showing with 266 additions and 18 deletions.
  1. +32 −0 buildscripts/emr/IOUtil.java
  2. +70 −8 buildscripts/emr/emr.java
  3. +164 −10 buildscripts/emr/emr.py
@@ -2,6 +2,7 @@
import java.io.*;
import java.net.*;
+import java.util.*;
public class IOUtil {
@@ -68,6 +69,37 @@ public static String readStringFully( InputStream in )
}
+ public static Map<String,Object> readPythonSettings( File file )
+ throws IOException {
+
+ String all = readStringFully( new FileInputStream( file ) );
+
+ Map<String,Object> map = new TreeMap<String,Object>();
+
+ for ( String line : all.split( "\n" ) ) {
+ line = line.trim();
+ if ( line.length() == 0 )
+ continue;
+
+ String[] pcs = line.split( "=" );
+ if ( pcs.length != 2 )
+ continue;
+
+ String name = pcs[0].trim();
+ String value = pcs[1].trim();
+
+ if ( value.startsWith( "\"" ) ) {
+ map.put( name , value.substring( 1 , value.length() - 1 ) );
+ }
+ else {
+ map.put( name , Long.parseLong( value ) );
+ }
+
+ }
+
+ return map;
+ }
+
public static String[] runCommand( String cmd , File dir )
throws IOException {
View
@@ -23,7 +23,9 @@ void copy( MongoSuite c ) {
mongo = c.mongo;
code = c.code;
workingDir = c.workingDir;
+
suite = c.suite;
+
}
void downloadTo( File localDir )
@@ -40,10 +42,12 @@ boolean runTest()
dir.mkdirs();
// download
+ System.out.println( "going to download" );
downloadTo( dir );
-
+
// explode
+ System.out.println( "going to explode" );
IOUtil.runCommand( "tar zxvf " + IOUtil.urlFileName( code ) , dir );
String[] res = IOUtil.runCommand( "tar zxvf " + IOUtil.urlFileName( mongo ) , dir );
for ( String x : res[0].split( "\n" ) ) {
@@ -54,31 +58,69 @@ boolean runTest()
throw new RuntimeException( "rename failed" );
}
+ List<String> cmd = new ArrayList<String>();
+ cmd.add( "/usr/bin/python" );
+ cmd.add( "buildscripts/smoke.py" );
+
+ File log_config = new File( dir , "log_config.py" );
+ System.out.println( "log_config: " + log_config.exists() );
+ if ( log_config.exists() ) {
+
+ java.util.Map<String,Object> properties = IOUtil.readPythonSettings( log_config );
+
+ cmd.add( "--buildlogger-builder" );
+ cmd.add( properties.get( "name" ).toString() );
+
+ cmd.add( "--buildlogger-buildnum" );
+ cmd.add( properties.get( "number" ).toString() );
+
+ cmd.add( "--buildlogger-credentials" );
+ cmd.add( "log_config.py" );
+
+ cmd.add( "--buildlogger-phase" );
+ {
+ int idx = suite.lastIndexOf( "/" );
+ if ( idx < 0 )
+ cmd.add( suite );
+ else
+ cmd.add( suite.substring( 0 , idx ) );
+ }
+
+ }
+
+ cmd.add( suite );
+
+ System.out.println( cmd );
+
+ Process p = Runtime.getRuntime().exec( cmd.toArray( new String[cmd.size()] ) , new String[]{} , dir );
- Process p = Runtime.getRuntime().exec( new String[]{ "/usr/bin/python" , "buildscripts/smoke.py" , suite } , new String[]{} , dir );
List<Thread> threads = new ArrayList<Thread>();
threads.add( new IOUtil.PipingThread( p.getInputStream() , System.out ) );
threads.add( new IOUtil.PipingThread( p.getErrorStream() , System.out ) );
-
- for ( Thread t : threads ) t.start();
+
+ for ( Thread t : threads )
+ t.start();
+
try {
- for ( Thread t : threads ) t.join();
+ for ( Thread t : threads ) {
+ t.join();
+ }
int rc = p.waitFor();
- System.out.println( "\n\nResult: " + rc );
-
return rc == 0;
}
catch ( InterruptedException ie ) {
+ ie.printStackTrace();
throw new RuntimeException( "sad" , ie );
}
-
+
}
public void readFields( DataInput in )
throws IOException {
mongo = in.readUTF();
code = in.readUTF();
workingDir = in.readUTF();
+
suite = in.readUTF();
}
@@ -87,6 +129,7 @@ public void write( final DataOutput out )
out.writeUTF( mongo );
out.writeUTF( code );
out.writeUTF( workingDir );
+
out.writeUTF( suite );
}
@@ -119,6 +162,14 @@ public void map( Text key, MongoSuite value, OutputCollector<Text,IntWritable> o
ip = ip.substring( ip.indexOf( ":" ) + 1 ).trim();
output.collect( new Text( ip ) , new IntWritable(1) );
}
+ catch ( RuntimeException re ) {
+ re.printStackTrace();
+ throw re;
+ }
+ catch ( IOException ioe ) {
+ ioe.printStackTrace();
+ throw ioe;
+ }
finally {
lock.unlock();
}
@@ -254,6 +305,7 @@ public static void main( String[] args ) throws Exception{
String workingDir = "/data/db/emr/";
+
// parse args
int pos = 0;
@@ -291,6 +343,16 @@ public static void main( String[] args ) throws Exception{
System.out.println( "output\t: " + output );
System.out.println( "suites\t: " + suites );
+ if ( false ) {
+ MongoSuite s = new MongoSuite();
+ s.mongo = mongo;
+ s.code = code;
+ s.workingDir = workingDir;
+ s.suite = suites;
+ s.runTest();
+ return;
+ }
+
// main hadoop set
conf.set( "mongo" , mongo );
conf.set( "code" , code );
Oops, something went wrong.

0 comments on commit 087253b

Please sign in to comment.