5k in 2 minutes

client/src/main/resources/log4j.properties

@@ -30,7 +30,7 @@ log4j.logger.akka=WARN, FILE
 log4j.logger.breeze=WARN, FILE
 log4j.logger.zingg=INFO
 log4j.logger.zingg.hash=INFO
-log4j.logger.zingg.distBlock=INFO
+log4j.logger.zingg.distBlock=WARN
 log4j.logger.zingg.block=INFO
 log4j.logger.zingg.Matcher=INFO
 log4j.logger.zingg.Model=INFO

core/src/main/java/zingg/distBlock/BFn.java

@@ -86,8 +86,8 @@ else if (hash1 != null && hash2 != null && hash1.equals(hash2)) {
     }
 
     public void estimateChildren(Context ctx) {
-		//result.approxChildren = ctx.getDataSample().select(ColName.HASH_COL + index).distinct().count();
-        result.approxChildren = 100;
+		result.approxChildren = ctx.getDataSample().select(ColName.HASH_COL + index).distinct().count();
+        //result.approxChildren = 100;
 	}
 
 

core/src/main/java/zingg/distBlock/BTreeBuilder.java

@@ -126,24 +126,24 @@ public BFn getBestNode(Tree<BFn> tree, BFn parent, BFn node,
 		for (Integer j : fnsToTry.keySet()) {
 			Fn fn = fnsToTry.get(j);
 			if (!isFunctionUsed(tree, node, fn)) {
-						BFn bFnToTry = new BFn(fn, new FnResult());
-						bFnToTry.estimateElimCount(context);
+				BFn bFnToTry = new BFn(fn, new FnResult());
+				bFnToTry.estimateElimCount(context);
+				if (bFnToTry.getResult().compareTo(least) > 0){		
+						//LOG.debug(" new is better");	
 						bFnToTry.estimateChildren(context);
 						//LOG.debug("Comparing " + bFnToTry.result  + " with  " + least);	
-						if (bFnToTry.getResult().approxChildren >= 1) {
-							if (bFnToTry.getResult().compareTo(least) > 0){		
-								//LOG.debug(" new is better");	
-								best = bFnToTry; 
-								least = bFnToTry.getResult();
-								//greedy, how much better can it get
-								if (bFnToTry.getResult().getElimCount() == 0 ) return bFnToTry;
-							}
-							/*else {
-								LOG.debug(" old is better ");
-							}*/
+						if (bFnToTry.getResult().approxChildren > 1) {
+							best = bFnToTry; 
+							least = bFnToTry.getResult();
+							//greedy, how much better can it get
+							if (bFnToTry.getResult().getElimCount() == 0 ) return bFnToTry;
 						}//childess is of no use
-
+				}
+					/*else {
+						LOG.debug(" old is better ");
+					}*/
 			}
+
 		}
 		return best;
 	}

models/100/model/classifier/best.model/bestModel/stages/0_vecAssembler_86a5aa12df5d/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.VectorAssembler","timestamp":1654787316522,"sparkVersion":"3.1.2","uid":"vecAssembler_86a5aa12df5d","paramMap":{"inputCols":["z_sim0","z_sim1","z_sim2","z_sim3","z_sim4","z_sim5","z_sim6","z_sim7","z_sim8","z_sim9","z_sim10","z_sim11","z_sim12","z_sim13","z_sim14","z_sim15","z_sim16","z_sim17","z_sim18","z_sim19"],"outputCol":"z_featurevector"},"defaultParamMap":{"handleInvalid":"error","outputCol":"vecAssembler_86a5aa12df5d__output"}}

models/100/model/classifier/best.model/bestModel/stages/1_poly_958661eebf7b/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.PolynomialExpansion","timestamp":1654787316588,"sparkVersion":"3.1.2","uid":"poly_958661eebf7b","paramMap":{"outputCol":"z_feature","inputCol":"z_featurevector","degree":3},"defaultParamMap":{"outputCol":"poly_958661eebf7b__output","degree":2}}

models/100/model/classifier/best.model/bestModel/stages/2_logreg_70037b80b4c6/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.LogisticRegressionModel","timestamp":1654787316644,"sparkVersion":"3.1.2","uid":"logreg_70037b80b4c6","paramMap":{"threshold":0.4,"regParam":1.0E-4,"predictionCol":"z_prediction","maxIter":100,"featuresCol":"z_feature","fitIntercept":true,"probabilityCol":"z_probability","labelCol":"z_isMatch"},"defaultParamMap":{"threshold":0.5,"regParam":0.0,"predictionCol":"prediction","elasticNetParam":0.0,"maxIter":100,"featuresCol":"features","rawPredictionCol":"rawPrediction","maxBlockSizeInMB":0.0,"family":"auto","fitIntercept":true,"aggregationDepth":2,"probabilityCol":"probability","tol":1.0E-6,"labelCol":"label","standardization":true}}

models/100/model/classifier/best.model/estimator/stages/0_vecAssembler_86a5aa12df5d/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.VectorAssembler","timestamp":1654787316259,"sparkVersion":"3.1.2","uid":"vecAssembler_86a5aa12df5d","paramMap":{"inputCols":["z_sim0","z_sim1","z_sim2","z_sim3","z_sim4","z_sim5","z_sim6","z_sim7","z_sim8","z_sim9","z_sim10","z_sim11","z_sim12","z_sim13","z_sim14","z_sim15","z_sim16","z_sim17","z_sim18","z_sim19"],"outputCol":"z_featurevector"},"defaultParamMap":{"handleInvalid":"error","outputCol":"vecAssembler_86a5aa12df5d__output"}}

models/100/model/classifier/best.model/estimator/stages/1_poly_958661eebf7b/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.PolynomialExpansion","timestamp":1654787316323,"sparkVersion":"3.1.2","uid":"poly_958661eebf7b","paramMap":{"outputCol":"z_feature","inputCol":"z_featurevector","degree":3},"defaultParamMap":{"outputCol":"poly_958661eebf7b__output","degree":2}}

models/100/model/classifier/best.model/estimator/stages/2_logreg_70037b80b4c6/metadata/part-00000

@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.LogisticRegression","timestamp":1654787316382,"sparkVersion":"3.1.2","uid":"logreg_70037b80b4c6","paramMap":{"predictionCol":"z_prediction","maxIter":100,"featuresCol":"z_feature","fitIntercept":true,"probabilityCol":"z_probability","labelCol":"z_isMatch"},"defaultParamMap":{"threshold":0.5,"regParam":0.0,"predictionCol":"prediction","elasticNetParam":0.0,"maxIter":100,"featuresCol":"features","rawPredictionCol":"rawPrediction","maxBlockSizeInMB":0.0,"family":"auto","fitIntercept":true,"aggregationDepth":2,"probabilityCol":"probability","tol":1.0E-6,"labelCol":"label","standardization":true}}