fixed bug hat CRASHED runs are capped for solution quality

automl · Mar 16, 2015 · 07391cd · 07391cd
1 parent 6a21382
commit 07391cd
Show file tree

Hide file tree

Showing 56 changed files with 40 additions and 87 deletions.
diff --git a/fanova/lib/DomainInter.jar b/fanova/lib/DomainInter.jar
diff --git a/fanova/lib/Jama-1.0.2.jar b/fanova/lib/Jama-1.0.2.jar
diff --git a/fanova/lib/commons-collections-3.2.1.jar b/fanova/lib/commons-collections-3.2.1.jar
diff --git a/fanova/lib/commons-io-2.1.jar b/fanova/lib/commons-io-2.1.jar
diff --git a/fanova/lib/commons-math-2.2.jar b/fanova/lib/commons-math-2.2.jar
diff --git a/fanova/lib/commons-math3-3.2.jar b/fanova/lib/commons-math3-3.2.jar
diff --git a/fanova/lib/exp4j-0.3.10.jar b/fanova/lib/exp4j-0.3.10.jar
diff --git a/fanova/lib/fastrf-src.jar b/fanova/lib/fastrf-src.jar
diff --git a/fanova/lib/fastrf.jar b/fanova/lib/fastrf.jar
diff --git a/fanova/lib/guava-14.0.1.jar b/fanova/lib/guava-14.0.1.jar
diff --git a/fanova/lib/jcip-annotations-src.jar b/fanova/lib/jcip-annotations-src.jar
diff --git a/fanova/lib/jcip-annotations.jar b/fanova/lib/jcip-annotations.jar
diff --git a/fanova/lib/jcommander.jar b/fanova/lib/jcommander.jar
diff --git a/fanova/lib/logback-access-1.0.13.jar b/fanova/lib/logback-access-1.0.13.jar
diff --git a/fanova/lib/logback-classic-1.0.13.jar b/fanova/lib/logback-classic-1.0.13.jar
diff --git a/fanova/lib/logback-core-1.0.13.jar b/fanova/lib/logback-core-1.0.13.jar
diff --git a/fanova/lib/numerics4j-1.3.jar b/fanova/lib/numerics4j-1.3.jar
diff --git a/fanova/lib/opencsv-2.3.jar b/fanova/lib/opencsv-2.3.jar
diff --git a/fanova/lib/slf4j-api-1.7.5.jar b/fanova/lib/slf4j-api-1.7.5.jar
diff --git a/fanova/lib/spi-0.2.4.jar b/fanova/lib/spi-0.2.4.jar
diff --git a/.../src/net/aclib/fanova/FAnovaExecutor.java → .../src/net/aeatk/fanova/FAnovaExecutor.java b/.../src/net/aclib/fanova/FAnovaExecutor.java → .../src/net/aeatk/fanova/FAnovaExecutor.java
@@ -1,11 +1,9 @@
-package net.aclib.fanova;
+package net.aeatk.fanova;
 
 import java.io.File;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.Vector;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -28,12 +26,12 @@
 import ca.ubc.cs.beta.aeatk.state.StateFactory;
 import ca.ubc.cs.beta.aeatk.state.StateFactoryOptions;
 import ca.ubc.cs.beta.aeatk.state.legacy.LegacyStateFactory;
-import ca.ubc.cs.beta.models.fastrf.RandomForest;
-import net.aclib.fanova.model.FunctionalANOVARunner;
-import net.aclib.fanova.model.FunctionalANOVAModelBuilder;
-import net.aclib.fanova.model.FunctionalANOVAVarianceDecompose;
-import net.aclib.fanova.options.FAnovaOptions;
-import net.aclib.fanova.options.FAnovaOptions.Improvements;
+
+import net.aeatk.fanova.model.FunctionalANOVAModelBuilder;
+import net.aeatk.fanova.model.FunctionalANOVARunner;
+import net.aeatk.fanova.model.FunctionalANOVAVarianceDecompose;
+import net.aeatk.fanova.options.FAnovaOptions;
+import net.aeatk.fanova.options.FAnovaOptions.Improvements;
 
 public class FAnovaExecutor {
 	private static Logger log;
@@ -47,7 +45,6 @@ public static void main(String[] args) {
 			try {
 				FAnovaOptions fanovaOpts = new FAnovaOptions();
 				JCommander jcom;
-				;
 
 				//Manhandle the options to support --restoreScenario
 				args = StateFactoryOptions.processScenarioStateRestore(args);
@@ -109,78 +106,26 @@ public static void main(String[] args) {
 				    }
 				    rh = subsampledRH;
 				}
-
-//				ReadCVS obj = new ReadCVS();
-//				obj.run();
-//
-//				FunctionalANOVAModelBuilderFromCSV famb = new FunctionalANOVAModelBuilderFromCSV();
-//				famb.learnModel(obj.getX(), obj.getY(), fanovaOpts.rfOptions, pool);
-
 				FunctionalANOVAModelBuilder famb = new FunctionalANOVAModelBuilder();
 				famb.learnModel(instances, rh, configSpace, fanovaOpts.rfOptions, fanovaOpts.mbOptions, scenarioOptions, true, pool);
-				//RandomForest rf = famb. 
+
 
 				//=== Handle fANOVA options.
 				boolean compareToDef = fanovaOpts.compare.equals(Improvements.DEFAULT) ? true : false;
 				double quantile = fanovaOpts.compare.equals(Improvements.QUANTILE) ? fanovaOpts.quantileToCompare : -1;
 
-
-				/*
-				 * Old code loading forest from a file.
-
-		//String prefix = "/ubc/cs/home/h/hutter/orcinus/home/hutter/experiments/surrogates/otherData/";
-		//String surrogateZipFilename = prefix + name;
-
-		//PrintWriter writer = null;
-		//			extension = "_1tree_" + extension;
-
-		//QuickZip surrogateZip = new QuickZip(surrogateZipFilename);
-
-		
-		//ParamConfigurationSpace configSpace = (ParamConfigurationSpace) surrogateZip.getObject(ZipFileName.CONFIG_SPACE_FILE);
-		//LinkedHashSet<ParamConfiguration> paramList = (LinkedHashSet<ParamConfiguration>) surrogateZip.getObject(ZipFileName.PARAMS_OBJECTS);
-		//for (ParamConfiguration paramConfiguration : paramList) {
-		//	paramConfiguration.toValueArray();
-		//}
-		
-		
-		
-		//=== Get the number of instance features.
-		Iterator iterator = instanceList.iterator(); 
-		ProblemInstance problemInstance = (ProblemInstance) iterator.next();
-		int numFeatures = problemInstance.getFeaturesDouble().length;
-
-		//=== Get the features X and preprocess the random forest with them.
-		double[][] X = new double[instanceList.size()][numFeatures];
-		int count = 0;	
-		for (ProblemInstance instance: instanceList){
-			X[count++] = instance.getFeaturesDouble();
-		}
-		forest = RandomForest.preprocessForest(forest, X);
-				 * 
-				 */
+
 				if(fanovaOpts.mode.equals("ipc")) {
 					FunctionalANOVAVarianceDecompose favd = new FunctionalANOVAVarianceDecompose(famb.getRandomForest(), rh.getAlgorithmRunsExcludingRedundant(), configSpace,pool.getRandom("FANOVA_BUILDER"), compareToDef, quantile, fanovaOpts.rfOptions.logModel);
 
 					FanovaRemote remote = new FanovaRemote(favd, new IPCMechanism("localhost", fanovaOpts.port), configSpace);
 					remote.run();
 				} else {
-					//run command line version of fanova
+					//=== Run command line version of fanova
 					FunctionalANOVARunner.decomposeVariance(famb.getRandomForest(), rh.getAlgorithmRunsExcludingRedundant(), configSpace,pool.getRandom("FANOVA_BUILDER"), compareToDef, quantile, fanovaOpts.computePairwiseInteration, outputDir,fanovaOpts.rfOptions.logModel, fanovaOpts.plotMarginals);
 				}
 
 
-//				List<AlgorithmRun> algorithmRuns = new ArrayList<AlgorithmRun>();
-//				List<String> names = new ArrayList<String>();
-//				names.add("X0");
-//				names.add("X1");
-//				
-//				RandomForest rf = famb.getRandomForest();
-//				double[][] X = new double[1][0];
-//				rf = RandomForest.preprocessForest(rf, X);
-//				
-				//FunctionalANOVARunnerFromCSV.decomposeVariance(rf, names, pool.getRandom("FANOVA_BUILDER"), false, 0, fanovaOpts.computePairwiseInteration, outputDir,fanovaOpts.rfOptions.logModel, fanovaOpts.plotMarginals);
-
 			} finally {
 				if(pool != null) {
 					pool.logUsage();

diff --git a/...va/src/net/aclib/fanova/FanovaRemote.java → ...va/src/net/aeatk/fanova/FanovaRemote.java b/...va/src/net/aclib/fanova/FanovaRemote.java → ...va/src/net/aeatk/fanova/FanovaRemote.java
@@ -1,17 +1,15 @@
-package net.aclib.fanova;
+package net.aeatk.fanova;
 
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
-
 import java.util.Vector;
 
-
 import ca.ubc.cs.beta.aeatk.parameterconfigurationspace.NormalizedRange;
 import ca.ubc.cs.beta.aeatk.parameterconfigurationspace.ParameterConfigurationSpace;
-import net.aclib.fanova.model.FunctionalANOVAVarianceDecompose;
+import net.aeatk.fanova.model.FunctionalANOVAVarianceDecompose;
 
 public class FanovaRemote {
 

diff --git a/...va/src/net/aclib/fanova/IPCMechanism.java → ...va/src/net/aeatk/fanova/IPCMechanism.java b/...va/src/net/aclib/fanova/IPCMechanism.java → ...va/src/net/aeatk/fanova/IPCMechanism.java
@@ -1,4 +1,4 @@
-package net.aclib.fanova;
+package net.aeatk.fanova;
 
 import java.io.BufferedReader;
 import java.io.IOException;

diff --git a/...et/aclib/fanova/eval/ModelEvaluation.java → ...et/aeatk/fanova/eval/ModelEvaluation.java b/...et/aclib/fanova/eval/ModelEvaluation.java → ...et/aeatk/fanova/eval/ModelEvaluation.java
@@ -1,4 +1,4 @@
-package net.aclib.fanova.eval;
+package net.aeatk.fanova.eval;
 
 import java.util.HashSet;
 import java.util.List;

diff --git a/...va/model/FunctionalANOVAModelBuilder.java → ...va/model/FunctionalANOVAModelBuilder.java b/...va/model/FunctionalANOVAModelBuilder.java → ...va/model/FunctionalANOVAModelBuilder.java
@@ -1,4 +1,4 @@
-package net.aclib.fanova.model;
+package net.aeatk.fanova.model;
 
 import java.util.ArrayList;
 import java.util.LinkedHashSet;
@@ -145,11 +145,21 @@ public void learnModel(List<ProblemInstance> instances, RunHistory runHistory, P
 		for(int j=0; j < runResponseValues.length; j++)
 		{ //=== Not sure if I Should be penalizing runs prior to the model
 			// but matlab sure does
-			if(runResponseValues[j] >= scenarioOptions.algoExecOptions.cutoffTime)
-			{	
-				runResponseValues[j] = scenarioOptions.algoExecOptions.cutoffTime * scenarioOptions.intraInstanceObj.getPenaltyFactor();
+
+			switch(scenarioOptions.getRunObjective())
+			{
+			case RUNTIME:
+				if(runResponseValues[j] >= scenarioOptions.algoExecOptions.cutoffTime)
+				{	
+					runResponseValues[j] = scenarioOptions.algoExecOptions.cutoffTime * scenarioOptions.getIntraInstanceObjective().getPenaltyFactor();
+				}
+				break;
+			case QUALITY:
+
+				break;
+			default:
+				throw new IllegalArgumentException("Not sure what objective this is: " + scenarioOptions.getRunObjective());
 			}
-
 		}
 
 		//=== Sanitize the data.

diff --git a/...b/fanova/model/FunctionalANOVARunner.java → ...k/fanova/model/FunctionalANOVARunner.java b/...b/fanova/model/FunctionalANOVARunner.java → ...k/fanova/model/FunctionalANOVARunner.java
@@ -1,4 +1,4 @@
-package net.aclib.fanova.model;
+package net.aeatk.fanova.model;
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintWriter;
@@ -16,7 +16,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import net.aclib.fanova.eval.ModelEvaluation;
+import net.aeatk.fanova.eval.ModelEvaluation;
 import ca.ubc.cs.beta.aeatk.algorithmrunresult.AlgorithmRunResult;
 import ca.ubc.cs.beta.aeatk.parameterconfigurationspace.NormalizedRange;
 import ca.ubc.cs.beta.aeatk.parameterconfigurationspace.ParameterConfiguration;

diff --git a/...del/FunctionalANOVAVarianceDecompose.java → ...del/FunctionalANOVAVarianceDecompose.java b/...del/FunctionalANOVAVarianceDecompose.java → ...del/FunctionalANOVAVarianceDecompose.java
@@ -1,4 +1,4 @@
-package net.aclib.fanova.model;
+package net.aeatk.fanova.model;
 
 import java.io.IOException;
 import java.text.DecimalFormat;
@@ -13,7 +13,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import net.aclib.fanova.eval.ModelEvaluation;
+import net.aeatk.fanova.eval.ModelEvaluation;
 import ca.ubc.cs.beta.aeatk.algorithmrunresult.AlgorithmRunResult;
 import ca.ubc.cs.beta.aeatk.parameterconfigurationspace.ParameterConfigurationSpace;
 import ca.ubc.cs.beta.models.fastrf.RandomForest;

diff --git a/...anova/model/RandomForestPreprocessor.java → ...anova/model/RandomForestPreprocessor.java b/...anova/model/RandomForestPreprocessor.java → ...anova/model/RandomForestPreprocessor.java
@@ -1,4 +1,4 @@
-package net.aclib.fanova.model;
+package net.aeatk.fanova.model;
 
 import java.util.ArrayList;
 import java.util.HashMap;

diff --git a/...nova/options/FAnovaNoArgumentHandler.java → ...nova/options/FAnovaNoArgumentHandler.java b/...nova/options/FAnovaNoArgumentHandler.java → ...nova/options/FAnovaNoArgumentHandler.java
@@ -1,4 +1,4 @@
-package net.aclib.fanova.options;
+package net.aeatk.fanova.options;
 
 import ca.ubc.cs.beta.aeatk.misc.options.NoArgumentHandler;
 

diff --git a/...t/aclib/fanova/options/FAnovaOptions.java → ...t/aeatk/fanova/options/FAnovaOptions.java b/...t/aclib/fanova/options/FAnovaOptions.java → ...t/aeatk/fanova/options/FAnovaOptions.java
@@ -1,4 +1,4 @@
-package net.aclib.fanova.options;
+package net.aeatk.fanova.options;
 
 import java.io.File;
 import java.util.Collection;

diff --git a/...lib/fanova/version/FAnovaVersionInfo.java → ...atk/fanova/version/FAnovaVersionInfo.java b/...lib/fanova/version/FAnovaVersionInfo.java → ...atk/fanova/version/FAnovaVersionInfo.java
@@ -1,11 +1,11 @@
-package net.aclib.fanova.version;
+package net.aeatk.fanova.version;
 
-import org.mangosdk.spi.ProviderFor;
+//import org.mangosdk.spi.ProviderFor;
 
 import ca.ubc.cs.beta.aeatk.misc.version.AbstractVersionInfo;
 import ca.ubc.cs.beta.aeatk.misc.version.VersionInfo;
 
-@ProviderFor(VersionInfo.class)
+//@ProviderFor(VersionInfo.class)
 public class FAnovaVersionInfo extends AbstractVersionInfo {
 
 	public FAnovaVersionInfo()

diff --git a/pyfanova/fanova.py b/pyfanova/fanova.py
@@ -289,7 +289,7 @@ def _start_fanova(self):
             self._heap_size,
             "-cp",
             ":".join(self._fanova_classpath()),
-            "net.aclib.fanova.FAnovaExecutor",
+            "net.aeatk.fanova.FAnovaExecutor",
             "--restoreScenario", self._scenario_dir,
             "--seed", str(self._seed),
             "--rf-num-trees", str(self._num_trees),

diff --git a/pyfanova/fanova/DomainInter.jar b/pyfanova/fanova/DomainInter.jar
diff --git a/pyfanova/fanova/Jama-1.0.2.jar b/pyfanova/fanova/Jama-1.0.2.jar
diff --git a/pyfanova/fanova/aclib-src.jar b/pyfanova/fanova/aclib-src.jar
diff --git a/pyfanova/fanova/aclib.jar b/pyfanova/fanova/aclib.jar
diff --git a/pyfanova/fanova/aeatk.jar b/pyfanova/fanova/aeatk.jar
diff --git a/pyfanova/fanova/commons-collections-3.2.1.jar b/pyfanova/fanova/commons-collections-3.2.1.jar
diff --git a/pyfanova/fanova/commons-io-2.1.jar b/pyfanova/fanova/commons-io-2.1.jar
diff --git a/pyfanova/fanova/commons-math-2.2.jar b/pyfanova/fanova/commons-math-2.2.jar
diff --git a/pyfanova/fanova/commons-math3-3.2.jar b/pyfanova/fanova/commons-math3-3.2.jar
diff --git a/pyfanova/fanova/exp4j-0.3.10.jar b/pyfanova/fanova/exp4j-0.3.10.jar
diff --git a/pyfanova/fanova/fanova.jar b/pyfanova/fanova/fanova.jar
diff --git a/pyfanova/fanova/fastrf-src.jar b/pyfanova/fanova/fastrf-src.jar
diff --git a/pyfanova/fanova/fastrf.jar b/pyfanova/fanova/fastrf.jar
diff --git a/pyfanova/fanova/guava-14.0.1.jar b/pyfanova/fanova/guava-14.0.1.jar
diff --git a/pyfanova/fanova/jcip-annotations-src.jar b/pyfanova/fanova/jcip-annotations-src.jar
diff --git a/pyfanova/fanova/jcip-annotations.jar b/pyfanova/fanova/jcip-annotations.jar
diff --git a/pyfanova/fanova/jcommander.jar b/pyfanova/fanova/jcommander.jar
diff --git a/pyfanova/fanova/logback-access-1.0.13.jar b/pyfanova/fanova/logback-access-1.0.13.jar
diff --git a/pyfanova/fanova/logback-classic-1.0.13.jar b/pyfanova/fanova/logback-classic-1.0.13.jar
diff --git a/pyfanova/fanova/logback-core-1.0.13.jar b/pyfanova/fanova/logback-core-1.0.13.jar
diff --git a/pyfanova/fanova/numerics4j-1.3.jar b/pyfanova/fanova/numerics4j-1.3.jar
diff --git a/pyfanova/fanova/opencsv-2.3.jar b/pyfanova/fanova/opencsv-2.3.jar
diff --git a/pyfanova/fanova/slf4j-api-1.7.5.jar b/pyfanova/fanova/slf4j-api-1.7.5.jar
diff --git a/pyfanova/fanova/spi-0.2.4.jar b/pyfanova/fanova/spi-0.2.4.jar