Skip to content
Browse files

Added an interface for exporting the measurements and a JSON implemen…

…tation. It can write to both stdout and to a file
  • Loading branch information...
1 parent 7234895 commit f56ec1a8558d9b705622ade5e973ed7cb28bdad8 @johanoskarsson johanoskarsson committed May 22, 2010
View
9 build.xml
@@ -2,12 +2,19 @@
<project name="ycsb" default="compile" basedir=".">
<property name="src.dir" value="src"/>
+ <property name="lib.dir" value="lib"/>
<property name="doc.dir" value="doc"/>
<property name="build.dir" value="build"/>
<property name="classes.dir" value="${build.dir}/classes"/>
<property name="jar.dir" value="${build.dir}/jar"/>
+ <path id="build.classpath">
+ <fileset dir="${lib.dir}">
+ <include name="**/*.jar" />
+ </fileset>
+ </path>
+
<target name="dbcompile-cassandra-0.5" depends="compile">
<property name="db.dir" value="db/cassandra-0.5"/>
<antcall target="dbcompile"/>
@@ -30,7 +37,7 @@
<target name="compile">
<mkdir dir="${classes.dir}"/>
- <javac srcdir="${src.dir}" destdir="${classes.dir}" excludes="com/yahoo/ycsb/db/**" deprecation="on">
+ <javac srcdir="${src.dir}" destdir="${classes.dir}" classpathref="build.classpath" deprecation="on">
<compilerarg value="-Xlint:unchecked"/>
</javac>
<antcall target="makejar"/>
View
BIN lib/jackson-core-asl-1.5.2.jar
Binary file not shown.
View
BIN lib/jackson-mapper-asl-1.5.2.jar
Binary file not shown.
View
67 src/com/yahoo/ycsb/Client.java
@@ -23,6 +23,8 @@
import java.util.*;
import com.yahoo.ycsb.measurements.Measurements;
+import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter;
+import com.yahoo.ycsb.measurements.exporter.TextMeasurementsExporter;
//import org.apache.log4j.BasicConfigurator;
@@ -373,6 +375,56 @@ public static boolean checkRequiredProperties(Properties props)
return true;
}
+
+ /**
+ * Exports the measurements to either sysout or a file using the exporter
+ * loaded from conf.
+ * @throws IOException Either failed to write to output stream or failed to close it.
+ */
+ private static void exportMeasurements(Properties props, int opcount, long runtime)
+ throws IOException
+ {
+ MeasurementsExporter exporter = null;
+ try
+ {
+ // if no destination file is provided the results will be written to stdout
+ OutputStream out;
+ String exportFile = props.getProperty("exportfile");
+ if (exportFile == null)
+ {
+ out = System.out;
+ } else
+ {
+ out = new FileOutputStream(exportFile);
+ }
+
+ // if no exporter is provided the default text one will be used
+ String exporterStr = props.getProperty("exporter");
+ try
+ {
+ exporter = (MeasurementsExporter) Class.forName(exporterStr).getConstructor(OutputStream.class).newInstance(out);
+ } catch (Exception e)
+ {
+ System.err.println("Could not find exporter " + exporterStr
+ + ", will use default text reporter.");
+ e.printStackTrace();
+ exporter = new TextMeasurementsExporter(out);
+ }
+
+ exporter.write("OVERALL", "RunTime(ms)", runtime);
+ double throughput = 1000.0 * ((double) opcount) / ((double) runtime);
+ exporter.write("OVERALL", "Throughput(ops/sec)", throughput);
+
+ Measurements.getMeasurements().exportMeasurements(exporter);
+ } finally
+ {
+ if (exporter != null)
+ {
+ exporter.close();
+ }
+ }
+ }
+
@SuppressWarnings("unchecked")
public static void main(String[] args)
{
@@ -382,7 +434,6 @@ public static void main(String[] args)
boolean dotransactions=true;
int threadcount=1;
int target=0;
- PrintStream out=System.out;
boolean status=false;
String label="";
@@ -717,11 +768,15 @@ public void run()
System.exit(0);
}
- out.println("[OVERALL],RunTime(ms), "+(en-st));
- double throughput=1000.0*((double)opcount)/((double)(en-st));
- out.println("[OVERALL],Throughput(ops/sec), "+throughput);
-
- Measurements.getMeasurements().printReport(out);
+ try
+ {
+ exportMeasurements(props, opcount, en - st);
+ } catch (IOException e)
+ {
+ System.err.println("Could not export measurements, error: " + e.getMessage());
+ e.printStackTrace();
+ System.exit(-1);
+ }
System.exit(0);
}
View
24 src/com/yahoo/ycsb/measurements/Measurements.java
@@ -17,10 +17,12 @@
package com.yahoo.ycsb.measurements;
-import java.io.PrintStream;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Properties;
+import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter;
+
/**
* Collects latency measurements, and reports them when requested.
*
@@ -134,16 +136,18 @@ public void reportReturnCode(String operation, int code)
}
data.get(operation).reportReturnCode(code);
}
-
- /**
- * Print the full report to the listed PrintStream.
- */
- public void printReport(PrintStream out)
+
+ /**
+ * Export the current measurements to a suitable format.
+ * @param exporter Exporter representing the type of format to write to.
+ * @throws IOException Thrown if the export failed.
+ */
+ public void exportMeasurements(MeasurementsExporter exporter) throws IOException
{
- for (OneMeasurement m : data.values())
- {
- m.printReport(out);
- }
+ for (OneMeasurement measurement : data.values())
+ {
+ measurement.exportMeasurements(exporter);
+ }
}
/**
View
14 src/com/yahoo/ycsb/measurements/OneMeasurement.java
@@ -17,7 +17,9 @@
package com.yahoo.ycsb.measurements;
-import java.io.PrintStream;
+import java.io.IOException;
+
+import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter;
/**
* A single measured metric (e.g. READ LATENCY)
@@ -40,8 +42,14 @@ public OneMeasurement(String _name) {
public abstract void reportReturnCode(int code);
public abstract void measure(int latency);
-
- public abstract void printReport(PrintStream out);
public abstract String getSummary();
+
+ /**
+ * Export the current measurements to a suitable format.
+ *
+ * @param exporter Exporter representing the type of format to write to.
+ * @throws IOException Thrown if the export failed.
+ */
+ public abstract void exportMeasurements(MeasurementsExporter exporter) throws IOException;
}
View
79 src/com/yahoo/ycsb/measurements/OneMeasurementHistogram.java
@@ -17,11 +17,13 @@
package com.yahoo.ycsb.measurements;
-import java.io.PrintStream;
+import java.io.IOException;
import java.text.DecimalFormat;
import java.util.HashMap;
import java.util.Properties;
+import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter;
+
/**
* Take measurements and maintain a histogram of a given metric, such as READ LATENCY.
@@ -108,45 +110,44 @@ public synchronized void measure(int latency)
}
}
- /* (non-Javadoc)
- * @see com.yahoo.ycsb.OneMeasurement#printReport(java.io.PrintStream)
- */
- public void printReport(PrintStream out)
- {
- out.println("["+getName()+"], Operations, "+operations);
- out.println("["+getName()+"], AverageLatency(ms), "+(((double)totallatency)/((double)operations)));
- out.println("["+getName()+"], MinLatency(ms), "+min);
- out.println("["+getName()+"], MaxLatency(ms), "+max);
-
- int opcounter=0;
- boolean done95th=false;
- for (int i=0; i<_buckets; i++)
- {
- opcounter+=histogram[i];
- if ( (!done95th) && (((double)opcounter)/((double)operations)>=0.95) )
- {
- out.println("["+getName()+"], 95thPercentileLatency(ms), "+i);
- done95th=true;
- }
- if (((double)opcounter)/((double)operations)>=0.99)
- {
- out.println("["+getName()+"], 99thPercentileLatency(ms), "+i);
- break;
- }
- }
- for (Integer I : returncodes.keySet())
- {
- int[] val=returncodes.get(I);
- out.println("["+getName()+"], Return="+I+", "+val[0]);
- }
-
- for (int i=0; i<_buckets; i++)
- {
- out.println("["+getName()+"], "+i+", "+histogram[i]);
- }
- out.println("["+getName()+"], >"+_buckets+", "+histogramoverflow);
- }
+ @Override
+ public void exportMeasurements(MeasurementsExporter exporter) throws IOException
+ {
+ exporter.write(getName(), "Operations", operations);
+ exporter.write(getName(), "AverageLatency(ms)", (((double)totallatency)/((double)operations)));
+ exporter.write(getName(), "MinLatency(ms)", min);
+ exporter.write(getName(), "MaxLatency(ms)", max);
+
+ int opcounter=0;
+ boolean done95th=false;
+ for (int i=0; i<_buckets; i++)
+ {
+ opcounter+=histogram[i];
+ if ( (!done95th) && (((double)opcounter)/((double)operations)>=0.95) )
+ {
+ exporter.write(getName(), "95thPercentileLatency(ms)", i);
+ done95th=true;
+ }
+ if (((double)opcounter)/((double)operations)>=0.99)
+ {
+ exporter.write(getName(), "99thPercentileLatency(ms)", i);
+ break;
+ }
+ }
+
+ for (Integer I : returncodes.keySet())
+ {
+ int[] val=returncodes.get(I);
+ exporter.write(getName(), "Return="+I, val[0]);
+ }
+
+ for (int i=0; i<_buckets; i++)
+ {
+ exporter.write(getName(), Integer.toString(i), histogram[i]);
+ }
+ exporter.write(getName(), ">"+_buckets, histogramoverflow);
+ }
@Override
public String getSummary() {
View
42 src/com/yahoo/ycsb/measurements/OneMeasurementTimeSeries.java
@@ -17,12 +17,14 @@
package com.yahoo.ycsb.measurements;
-import java.io.PrintStream;
+import java.io.IOException;
import java.text.DecimalFormat;
import java.util.HashMap;
import java.util.Properties;
import java.util.Vector;
+import com.yahoo.ycsb.measurements.exporter.MeasurementsExporter;
+
class SeriesUnit
{
/**
@@ -123,29 +125,31 @@ public void measure(int latency)
}
}
- @Override
- public void printReport(PrintStream out) {
- checkEndOfUnit(true);
- out.println("["+getName()+"], Operations, "+operations);
- out.println("["+getName()+"], AverageLatency(ms), "+(((double)totallatency)/((double)operations)));
- out.println("["+getName()+"], MinLatency(ms), "+min);
- out.println("["+getName()+"], MaxLatency(ms), "+max);
+ @Override
+ public void exportMeasurements(MeasurementsExporter exporter) throws IOException
+ {
+ checkEndOfUnit(true);
- //TODO: 95th and 99th percentile latency
+ exporter.write(getName(), "Operations", operations);
+ exporter.write(getName(), "AverageLatency(ms)", (((double)totallatency)/((double)operations)));
+ exporter.write(getName(), "MinLatency(ms)", min);
+ exporter.write(getName(), "MaxLatency(ms)", max);
- for (Integer I : returncodes.keySet())
- {
- int[] val=returncodes.get(I);
- out.println("["+getName()+"], Return="+I+", "+val[0]);
- }
+ //TODO: 95th and 99th percentile latency
- for (SeriesUnit unit : _measurements)
- {
- out.println("["+getName()+"], "+unit.time+", "+unit.average);
- }
- }
+ for (Integer I : returncodes.keySet())
+ {
+ int[] val=returncodes.get(I);
+ exporter.write(getName(), "Return="+I, val[0]);
+ }
+ for (SeriesUnit unit : _measurements)
+ {
+ exporter.write(getName(), Long.toString(unit.time), unit.average);
+ }
+ }
+
@Override
public void reportReturnCode(int code) {
Integer Icode=code;
View
74 src/com/yahoo/ycsb/measurements/exporter/JSONMeasurementsExporter.java
@@ -0,0 +1,74 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package com.yahoo.ycsb.measurements.exporter;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
+import org.codehaus.jackson.impl.DefaultPrettyPrinter;
+
+/**
+ * Export measurements into a machine readable JSON file.
+ */
+public class JSONMeasurementsExporter implements MeasurementsExporter
+{
+
+ private JsonFactory factory = new JsonFactory();
+ private JsonGenerator g;
+
+ public JSONMeasurementsExporter(OutputStream os) throws IOException
+ {
+
+ BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os));
+ g = factory.createJsonGenerator(bw);
+ g.setPrettyPrinter(new DefaultPrettyPrinter());
+ }
+
+ @Override
+ public void write(String metric, String measurement, int i) throws IOException
+ {
+ g.writeStartObject();
+ g.writeStringField("metric", metric);
+ g.writeStringField("measurement", measurement);
+ g.writeNumberField("value", i);
+ g.writeEndObject();
+ }
+
+ @Override
+ public void write(String metric, String measurement, double d) throws IOException
+ {
+ g.writeStartObject();
+ g.writeStringField("metric", metric);
+ g.writeStringField("measurement", measurement);
+ g.writeNumberField("value", d);
+ g.writeEndObject();
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ if (g != null)
+ {
+ g.close();
+ }
+ }
+
+}
View
57 src/com/yahoo/ycsb/measurements/exporter/MeasurementsExporter.java
@@ -0,0 +1,57 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package com.yahoo.ycsb.measurements.exporter;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * Used to export the collected measurements into a useful format, for example
+ * human readable text or machine readable JSON.
+ */
+public interface MeasurementsExporter extends Closeable
+{
+
+ /**
+ * Write a measurement to the exported format.
+ *
+ * @param metric
+ * Metric name, for example "READ LATENCY".
+ * @param measurement
+ * Measurement name, for example "Average latency".
+ * @param i
+ * Measurement to write.
+ * @throws IOException
+ * if writing failed
+ */
+ public void write(String metric, String measurement, int i) throws IOException;
+
+ /**
+ * Write a measurement to the exported format.
+ *
+ * @param metric
+ * Metric name, for example "READ LATENCY".
+ * @param measurement
+ * Measurement name, for example "Average latency".
+ * @param d
+ * Measurement to write.
+ * @throws IOException
+ * if writing failed
+ */
+ public void write(String metric, String measurement, double d) throws IOException;
+
+}
View
57 src/com/yahoo/ycsb/measurements/exporter/TextMeasurementsExporter.java
@@ -0,0 +1,57 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+package com.yahoo.ycsb.measurements.exporter;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+/**
+ * Write human readable text. Tries to emulate the previous print report method.
+ */
+public class TextMeasurementsExporter implements MeasurementsExporter
+{
+
+ private BufferedWriter bw;
+
+ public TextMeasurementsExporter(OutputStream os)
+ {
+ this.bw = new BufferedWriter(new OutputStreamWriter(os));
+ }
+
+ @Override
+ public void write(String metric, String measurement, int i) throws IOException
+ {
+ bw.write("[" + metric + "], " + measurement + ", " + i);
+ bw.newLine();
+ }
+
+ @Override
+ public void write(String metric, String measurement, double d) throws IOException
+ {
+ bw.write("[" + metric + "], " + measurement + ", " + d);
+ bw.newLine();
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ this.bw.close();
+ }
+
+}

0 comments on commit f56ec1a

Please sign in to comment.
Something went wrong with that request. Please try again.