Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Started implementation of collated results

  • Loading branch information...
commit 36bed99ed461eb5564d00a47cda98457d1d1260e 1 parent 15933d6
pbloem authored
111 Lilian/src/main/java/org/lilian/experiment/AbstractExperiment.java
View
@@ -12,9 +12,12 @@
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.PrintStream;
+import java.io.StringWriter;
import java.io.Writer;
import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
@@ -24,6 +27,7 @@
import java.util.logging.Logger;
import org.lilian.Global;
+import org.lilian.util.Series;
import freemarker.template.Configuration;
import freemarker.template.DefaultObjectWrapper;
@@ -35,11 +39,14 @@
private static final String STATE_FILE = "state.lilian";
// * FreeMarker config
- private static Configuration fmConfig = new Configuration();
+ protected static Configuration fmConfig = new Configuration();
private long t0;
+ private long t1;
private long t;
+ private String description = "";
+
protected File dir;
protected Logger logger;
@@ -73,7 +80,8 @@ public final void run()
new File(dir, STATE_FILE).delete();
}
- t = System.currentTimeMillis() - t0;
+ t1 = System.currentTimeMillis();
+ t = t1 - t0;
writeReport();
}
@@ -186,22 +194,22 @@ public void load()
}
}
- @Result(name = "Total running time of a run of the experiment in seconds.")
+ @Result(name = "run time", description = "Total running time of this run of the experiment in seconds.")
public double runtime()
{
- return 0;
+ return t/1000.0;
}
@Reportable(description = "The date and time at which the run of the experiment was started.")
public Date startTime()
{
- return null;
+ return new Date(t0);
}
@Reportable(description = "The date and time at which the run of the experiment was finished.")
public Date finishTime()
{
- return null;
+ return new Date(t1);
}
public void writeReport()
@@ -210,8 +218,24 @@ public void writeReport()
Map<String, Object> results = new HashMap<String, Object>();
// * Fill data model
- results.put("name", "Peter");
- results.put("result", "Hello world");
+ results.put("short_name", this.getClass().getName());
+ results.put("name", this.getClass().toString());
+ results.put("description", description());
+ results.put("start_date_time", new Date(this.t0).toString());
+ results.put("start_millis", t0);
+ results.put("end_date_time", new Date(this.t).toString());
+ results.put("end_millis", t);
+
+ // * Run through all methods tagged 'result'
+ List<Map<String, Object>> rs = new ArrayList<Map<String, Object>>();
+ for(Method method : Tools.allMethods(this.getClass()))
+ for(Annotation anno : method.getAnnotations())
+ if(anno instanceof Result)
+ processResult(rs, invoke(method), (Result) anno);
+
+ logger.info("Found " + rs.size() + " results");
+ results.put("results", rs);
+
Template tpl = null;
try
@@ -247,6 +271,67 @@ public void writeReport()
}
}
+ private void processResult(List<Map<String, Object>> rs, Object value, Result anno)
+ {
+ // * The method returns multiple results in a Results object
+ if(value instanceof Results)
+ {
+ Results results = (Results) value;
+
+ for(int i : Series.series(results.size()))
+ processResult(rs, results.value(i), results.annotation(i));
+
+ return;
+ }
+
+ Map<String, Object> resMap = new HashMap<String, Object>();
+
+
+ if(value instanceof Reporting)
+ {
+ Reporting reporting = (Reporting) value;
+
+ resMap.put("name", reporting.name());
+ resMap.put("description", reporting.description());
+
+ Template tpl = reporting.template();
+ Object dataModel = reporting.data();
+
+ resMap.put("value", "Failed to invoke result method. See the log file for details.");
+
+ try
+ {
+ StringWriter out = new StringWriter();
+ tpl.process(dataModel, out);
+ out.flush();
+
+ resMap.put("value", out.toString());
+ }
+ catch (TemplateException e) { Global.log().warning("Failed to process result-specific template " + tpl + " " + this + ". Exception: " + e.getMessage() + " - " + Arrays.toString(e.getStackTrace())); }
+ catch (IOException e) { throw new IllegalStateException("IOException on StringWriter", e); }
+ } else
+ {
+ resMap.put("value", value.toString());
+
+ resMap.put("name", anno.name());
+ resMap.put("description", anno.description());
+ }
+
+ rs.add(resMap);
+ }
+
+ private Object invoke(Method method)
+ {
+ try{
+ return method.invoke(this);
+ }
+ catch (InvocationTargetException e) { Global.log().warning("Failed to invoke result method " + method + " on experiment " + this + ". Exception: " + e.getMessage() + " - " + Arrays.toString(e.getStackTrace())); }
+ catch (IllegalArgumentException e) { Global.log().warning("Failed to invoke result method " + method + " on experiment " + this + ". Exception: " + e.getMessage() + " - " + Arrays.toString(e.getStackTrace())); }
+ catch (IllegalAccessException e) { Global.log().warning("Failed to invoke result method " + method + " on experiment " + this + ". Exception: " + e.getMessage() + " - " + Arrays.toString(e.getStackTrace())); }
+
+ return null;
+ }
+
public Experiment clone()
{
try
@@ -258,4 +343,14 @@ public Experiment clone()
throw new IllegalStateException(e);
}
}
+
+ public String description()
+ {
+ return description;
+ }
+
+ public void setDescription(String description)
+ {
+ this.description = description;
+ }
}
40 Lilian/src/main/java/org/lilian/experiment/BasicResults.java
View
@@ -0,0 +1,40 @@
+package org.lilian.experiment;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A collection of multiple experiment results
+ * @author Peter
+ *
+ */
+public class BasicResults implements Results
+{
+ private List<Object> values = new ArrayList<Object>();
+ private List<Result> annotations = new ArrayList<Result>();
+
+ @Override
+ public int size()
+ {
+ return values.size();
+ }
+
+ @Override
+ public Object value(int i)
+ {
+ return values.get(i);
+ }
+
+ @Override
+ public Result annotation(int i)
+ {
+ return annotations.get(i);
+ }
+
+ public void add(Object value, Result annotation)
+ {
+ values.add(value);
+ annotations.add(annotation);
+ }
+
+}
4 Lilian/src/main/java/org/lilian/experiment/Experiment.java
View
@@ -17,5 +17,9 @@
public void run();
public Experiment clone();
+
+ public String description();
+
+ public void setDescription(String description);
}
112 Lilian/src/main/java/org/lilian/experiment/MultiExperiment.java
View
@@ -24,31 +24,17 @@
import au.com.bytecode.opencsv.CSVWriter;
/**
- * Runs several independent experiments sequentially, each in its own
- * environment and directory inside the directory of the MultiExperiment.
- *
- * The MultiExperiment will detect which experiments inherit from the same class
- * and collate their results if they do so.
- *
- * For numerical results, it will compute the average, median, mode, variance, etc
- *
- * For each parameter varying between experiments, the results will be plotted
- * as appropriate.
- *
- *
+ * Runs multiple copies of the same experiment (possibly with different parameters)
* @author Peter
*
*/
-public class MultiExperiment extends AbstractExperiment
-{
- private Class<? extends Experiment> experiment;
- private Constructor<Experiment> structor;
-
+public abstract class MultiExperiment extends AbstractExperiment
+{
+ protected Class<? extends Experiment> experiment;
+
// * The experiments to perform
- private List<Experiment> experiments = new ArrayList<Experiment>();
- // * The parameters for which multiple values where defined
- private List<Integer> multiParameterIndices = new ArrayList<Integer>();
- private List<Parameter> multiParameter = new ArrayList<Parameter>();
+ protected List<Experiment> experiments = new ArrayList<Experiment>();
+
// * The methods for getting results
private List<Method> resultMethods = new ArrayList<Method>();
@@ -60,55 +46,13 @@
public @State int lastFinished;
public @State List<List<Object>> results;
public @State boolean sameSeed = true;
-
- /**
- * Repeat a single experiment a given number of times
- *
- * @param exp
- * @param sameSeed use same seed every time (perhaps to test whether code is
- * properly deterministic, would normally be false)
- * @param repeats
- */
- public MultiExperiment(Experiment exp, boolean sameSeed, int repeats)
- {
- experiment = exp.getClass();
-
- findResultMethods();
-
- for(int i : series(repeats))
- experiments.add(exp.clone());
-
- this.sameSeed = sameSeed;
- }
- /**
- * Create different experiments for different inputs
- *
- * @param ctr
- * @param sameSeed Whether to start each experiment with the same seed, or to give ach a new seed
- * @param inputs
- */
- public MultiExperiment(Constructor<Experiment> ctr, boolean sameSeed, int repeats, Object... inputs)
+ public MultiExperiment(Class<? extends Experiment> experiment, boolean sameSeed)
{
- experiment = ctr.getDeclaringClass();
- structor = ctr;
-
- for(int i : series(inputs.length))
- {
- if(inputs[i] instanceof Run.Multi<?>)
- {
- multiParameterIndices.add(i);
- for(Annotation annotation : ctr.getParameterAnnotations()[i])
- if(annotation instanceof Parameter)
- multiParameter.add((Parameter) annotation);
- }
- }
+ this.experiment = experiment;
+ this.sameSeed = sameSeed;
findResultMethods();
-
- createExperiments(inputs, new Object[inputs.length], 0, repeats);
-
- this.sameSeed = sameSeed;
}
private void findResultMethods()
@@ -124,34 +68,6 @@ private void findResultMethods()
}
}
- private void createExperiments(Object[] master, Object[] current, int i, int repeats)
- {
- if(master.length == i)
- {
- try
- {
- if(repeats == 1)
- experiments.add(structor.newInstance(current));
- else
- experiments.add(new MultiExperiment(structor.newInstance(current), false, repeats));
- } catch (Exception e)
- {
- throw new RuntimeException("Failed to create experiment", e);
- }
- } else if(! multiParameterIndices.contains(i))
- {
- current[i] = master[i];
- createExperiments(master, current, i + 1, repeats);
- } else
- {
- for(Object value : ((Run.Multi<?>) master[i]))
- {
- Object[] newCurrent = Arrays.copyOf(current, current.length);
- newCurrent[i] = value;
- createExperiments(master, newCurrent, i+1, repeats);
- }
- }
- }
@Override
protected void body()
@@ -230,5 +146,11 @@ public int size()
return experiments.size();
}
-
+ public void setDescription(String description)
+ {
+ for(Experiment experiment: experiments)
+ experiment.setDescription(description);
+
+ super.setDescription(description);
+ }
}
96 Lilian/src/main/java/org/lilian/experiment/MultiValueExperiment.java
View
@@ -0,0 +1,96 @@
+package org.lilian.experiment;
+
+import static org.lilian.util.Series.series;
+
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Runs several independent experiments sequentially, each in its own
+ * environment and directory inside the directory of the MultiExperiment.
+ *
+ * The MultiExperiment will detect which experiments inherit from the same class
+ * and collate their results if they do so.
+ *
+ * For numerical results, it will compute the average, median, mode, variance, etc
+ *
+ * For each parameter varying between experiments, the results will be plotted
+ * as appropriate.
+ *
+ *
+ * @author Peter
+ *
+ */
+public class MultiValueExperiment extends MultiExperiment
+{
+
+ private Constructor<Experiment> structor;
+
+ // * The parameters for which multiple values where defined
+ private List<Integer> multiParameterIndices = new ArrayList<Integer>();
+ private List<Parameter> multiParameter = new ArrayList<Parameter>();
+
+ public @State int repeats = 1;
+ /**
+ * Create different experiments for different inputs
+ *
+ * @param ctr
+ * @param sameSeed Whether to start each experiment with the same seed, or to give ach a new seed
+ * @param inputs
+ */
+ public MultiValueExperiment(Constructor<Experiment> ctr, boolean sameSeed, int repeats, Object... inputs)
+ {
+
+ super(ctr.getDeclaringClass(), sameSeed);
+ structor = ctr;
+
+ for(int i : series(inputs.length))
+ {
+ if(inputs[i] instanceof Run.Multi<?>)
+ {
+ multiParameterIndices.add(i);
+ for(Annotation annotation : ctr.getParameterAnnotations()[i])
+ if(annotation instanceof Parameter)
+ multiParameter.add((Parameter) annotation);
+ }
+ }
+
+ createExperiments(inputs, new Object[inputs.length], 0, repeats);
+
+ this.repeats = repeats;
+ this.sameSeed = sameSeed;
+ }
+
+ private void createExperiments(Object[] master, Object[] current, int i, int repeats)
+ {
+ if(master.length == i)
+ {
+ try
+ {
+ if(repeats == 1)
+ experiments.add(structor.newInstance(current));
+ else
+ experiments.add(new RepeatExperiment(structor.newInstance(current), repeats));
+ } catch (Exception e)
+ {
+ throw new RuntimeException("Failed to create experiment", e);
+ }
+ } else if(! multiParameterIndices.contains(i))
+ {
+ current[i] = master[i];
+ createExperiments(master, current, i + 1, repeats);
+ } else
+ {
+ for(Object value : ((Run.Multi<?>) master[i]))
+ {
+ Object[] newCurrent = Arrays.copyOf(current, current.length);
+ newCurrent[i] = value;
+ createExperiments(master, newCurrent, i+1, repeats);
+ }
+ }
+ }
+
+}
181 Lilian/src/main/java/org/lilian/experiment/RepeatExperiment.java
View
@@ -0,0 +1,181 @@
+package org.lilian.experiment;
+
+import static org.lilian.util.Series.series;
+
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.lilian.models.BasicFrequencyModel;
+
+import freemarker.template.Template;
+
+public class RepeatExperiment extends MultiExperiment
+{
+ private int repeats;
+
+ /**
+ * Repeat a single experiment a given number of times
+ *
+ * @param exp
+ * @param sameSeed use same seed every time (perhaps to test whether code is
+ * properly deterministic, would normally be false)
+ * @param repeats
+ */
+ public RepeatExperiment(Experiment exp, int repeats)
+ {
+ super(exp.getClass(), false);
+ this.repeats = repeats;
+
+ for(int i : series(repeats))
+ experiments.add(exp.clone());
+ }
+
+ @Result(name = "Repeat results")
+ public Results results()
+ {
+ BasicResults results = new BasicResults();
+
+ for(Method method : Tools.allMethods(experiment, Result.class))
+ {
+ Result anno = method.getAnnotation(Result.class);
+ CollatedResult cres = new CollatedResult(anno);
+
+ try
+ {
+ for(Experiment experiment : experiments)
+ cres.add(method.invoke(experiment));
+ } catch(Exception e) {
+ throw new RuntimeException("Error invoking.", e); // TODO Make nicer
+ }
+
+ results.add(cres, anno);
+
+ }
+
+ return results;
+ }
+
+ public int repeats()
+ {
+ return repeats;
+ }
+
+ private class CollatedResult implements Reporting
+ {
+ List<Object> values = new ArrayList<Object>();
+ Result annotation;
+
+ public CollatedResult(Result annotation)
+ {
+ this.annotation = annotation;
+ }
+
+ public void add(Object value)
+ {
+ values.add(value);
+ }
+
+ @Override
+ public String name()
+ {
+ return annotation.name() + "(over "+repeats()+" trials)";
+ }
+
+ @Override
+ public String description()
+ {
+ return annotation.description();
+ }
+
+ @Override
+ public Template template()
+ {
+ try
+ {
+ return fmConfig.getTemplate("repeat.ftl");
+ } catch (IOException e)
+ {
+ throw new RuntimeException("Could not load repeat.tpl template", e);
+ }
+ }
+
+ @Override
+ public Object data()
+ {
+ Map<String, Object> data = new HashMap<String, Object>();
+
+
+ boolean num = isNumeric();
+ data.put("mean", num ? mean() : "Data is not numeric");
+ data.put("std_dev", num ? standardDeviation() : "Data is not numeric");
+ data.put("median", num ? median() : "Data is not numeric");
+ data.put("mode", mode());
+ data.put("raw", values.toString());
+
+ return data;
+ }
+
+ /**
+ * Whether all result values represent numbers
+ * @return
+ */
+ public boolean isNumeric()
+ {
+ for(Object value : values)
+ if(! (value instanceof Number))
+ return false;
+ return true;
+ }
+
+ public double mean()
+ {
+ double sum = 0.0;
+ for(Object value : values)
+ sum += ((Number) value).doubleValue();
+
+ return sum/values.size();
+ }
+
+ public double standardDeviation()
+ {
+ double mean = mean();
+
+ double varSum = 0.0;
+ for(Object value : values)
+ {
+ double diff = mean - ((Number) value).doubleValue();
+ varSum += diff * diff;
+ }
+
+ double variance = varSum/(values.size() - 1);
+ return Math.sqrt(variance);
+ }
+
+ public double median()
+ {
+ List<Double> v = new ArrayList<Double>(values.size());
+
+ for(Object value : values)
+ v.add(((Number) value).doubleValue());
+
+ Collections.sort(v);
+
+ if(v.size() % 2 == 1)
+ return v.get(v.size()/2); // element s/2+1, but index s/2
+ return (v.get(v.size()/2 - 1) + v.get(v.size()/2)) / 2.0;
+ }
+
+ public Object mode()
+ {
+ BasicFrequencyModel<Object> model = new BasicFrequencyModel<Object>(values);
+
+ return model.sorted().get(0);
+ }
+ }
+}
19 Lilian/src/main/java/org/lilian/experiment/Reporting.java
View
@@ -0,0 +1,19 @@
+package org.lilian.experiment;
+
+import freemarker.template.Template;
+
+/**
+ * An object that can output its own complex report, in the form of a data model
+ * and a template
+ *
+ * @author Peter
+ *
+ */
+public interface Reporting
+{
+ public String name();
+ public String description();
+
+ public Template template();
+ public Object data();
+}
16 Lilian/src/main/java/org/lilian/experiment/Results.java
View
@@ -0,0 +1,16 @@
+package org.lilian.experiment;
+
+/**
+ * An object that contains multiple results.
+ *
+ * @author Peter
+ */
+public interface Results
+{
+
+ public int size();
+
+ public Object value(int i);
+ public Result annotation(int i);
+
+}
10 Lilian/src/main/java/org/lilian/experiment/Run.java
View
@@ -144,6 +144,10 @@ public static Experiment parseExperiment(Map<String, ?> in)
System.out.println("Found 'repeat' key. Each single experiment will be repeated " + repeats +" times.");
}
+ String description = "";
+ if(in.containsKey("description"))
+ description = in.get("description") + "";
+
List<String> parameters = new ArrayList<String>(in.keySet());
// * These are standard key. The rest are passed to the experiment
@@ -219,7 +223,7 @@ public static Experiment parseExperiment(Map<String, ?> in)
Experiment exp = null;
if(runMulti)
{
- MultiExperiment mexp = new MultiExperiment((Constructor<Experiment>)match, true, repeats, inputs);
+ MultiExperiment mexp = new MultiValueExperiment((Constructor<Experiment>)match, true, repeats, inputs);
numExperiments += mexp.size();
exp = mexp;
} else
@@ -232,7 +236,7 @@ public static Experiment parseExperiment(Map<String, ?> in)
numExperiments ++;
if(repeats > 1)
{
- MultiExperiment mexp = new MultiExperiment(exp, false, repeats);
+ MultiExperiment mexp = new RepeatExperiment(exp, repeats);
numExperiments += mexp.size();
exp = mexp;
}
@@ -240,6 +244,8 @@ public static Experiment parseExperiment(Map<String, ?> in)
throw new RuntimeException("Error instantiating experiment", e);
}
}
+
+ exp.setDescription(description);
return exp;
}
59 Lilian/src/main/java/org/lilian/experiment/Tools.java
View
@@ -0,0 +1,59 @@
+package org.lilian.experiment;
+
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Utility functions, mostly for reflection
+ * @author Peter
+ *
+ */
+public class Tools
+{
+
+ /**
+ * Returns all declared methods, including inherited.
+ */
+ public static List<Method> allMethods(Class<?> clss)
+ {
+ List<Method> list = new ArrayList<Method>();
+ allMethods(clss, list, null);
+ return list;
+ }
+
+ /**
+ * Returns all declared methods, including inherited, with the given annotation.
+ *
+ * @param clss
+ * @param annotationType
+ * @return
+ */
+ public static List<Method> allMethods(Class<?> clss, Class<? extends Annotation> annotationType)
+ {
+ List<Method> list = new ArrayList<Method>();
+ allMethods(clss, list, annotationType);
+ return list;
+ }
+
+ private static void allMethods(Class<?> clss, List<Method> list, Class<? extends Annotation> annotationType)
+ {
+ if(clss == null)
+ return;
+
+ for(Method method : clss.getDeclaredMethods())
+ if(annotationType == null)
+ list.add(method);
+ else
+ for(Annotation ann : method.getAnnotations())
+ if(annotationType.isAssignableFrom(ann.getClass()))
+ {
+ list.add(method);
+ break;
+ }
+
+ allMethods(clss.getSuperclass(), list, annotationType);
+ }
+
+}
57 Lilian/src/main/resources/templates/index.ftl
View
@@ -2,7 +2,60 @@
<title>Welcome!</title>
</head>
<body>
- <h1>Welcome ${name}</h1>
- <em>${result}</em>
+ <h1>Report: ${short_name}</h1>
+ <h2>Run information</h2>
+ <p class="explanation">
+ Basic information about this run of the experiment.
+ </p>
+
+ <dl>
+ <dt>
+ Experiment name
+ </dt>
+ <dd>
+ ${name}
+ </dd>
+
+ <dt>
+ description
+ </dt>
+ <dd>
+ ${description}
+ </dd>
+ <dt>
+ Start date/time
+ </dt>
+ <dd>
+ ${start_date_time} (${start_millis})
+ </dd>
+ <dt>
+ End date/time
+ </dt>
+ <dd>
+ ${end_date_time} (${end_millis})
+ </dd>
+ </dl>
+
+ <h2>Results</h2>
+ <p class="explanation">
+ The relevant results of the run, collated and analysed.
+ </p>
+
+ <#list results as result>
+ <h3>Result: ${result.name}</h3>
+ <p class="description">
+ ${result.description}
+ </p>
+ <p class="value">
+ ${result.value}
+ </p>
+ </#list>
+
+ <h2>Reportables</h2>
+ <p class="explanation">
+ Extensive information about the state of all software in the classpath at
+ the time of the run
+ </p>
+
</body>
</html>
36 Lilian/src/main/resources/templates/repeat.ftl
View
@@ -0,0 +1,36 @@
+<dl>
+ <dt>
+ Mean
+ </dt>
+ <dd>
+ ${mean}
+ </dd>
+
+ <dt>
+ (sample) standard deviation
+ </dt>
+ <dd>
+ ${std_dev}
+ </dd>
+
+ <dt>
+ Median
+ </dt>
+ <dd>
+ ${median}
+ </dd>
+
+ <dt>
+ Mode
+ </dt>
+ <dd>
+ ${mode}
+ </dd>
+
+ <dt>
+ raw values
+ </dt>
+ <dd>
+ ${raw}
+ </dd>
+</dl>
Please sign in to comment.
Something went wrong with that request. Please try again.