Bitmap Ratio Threshold now a configurable parameter

stanford-futuredata · May 7, 2018 · 18354d6 · 18354d6
1 parent 648c77e
commit 18354d6
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 4 deletions.
diff --git a/core/src/main/java/edu/stanford/futuredata/macrobase/pipeline/BasicBatchPipeline.java b/core/src/main/java/edu/stanford/futuredata/macrobase/pipeline/BasicBatchPipeline.java
@@ -33,6 +33,7 @@ public class BasicBatchPipeline implements Pipeline {
     private boolean pctileLow;
     private String predicateStr;
     private int numThreads;
+    private int bitmapRatioThreshold;
 
     private String summarizerType;
     private List<String> attributes;
@@ -73,6 +74,8 @@ public BasicBatchPipeline (PipelineConfig conf) {
         minRiskRatio = conf.get("minRatioMetric", 3.0);
         minSupport = conf.get("minSupport", 0.01);
         numThreads = conf.get("numThreads", Runtime.getRuntime().availableProcessors());
+        bitmapRatioThreshold = conf.get("bitmapRatioThreshold", 256);
+
 
         //if FDs are behind used, parse them into bitmaps. For now, all FDs must be in the first 31 attributes
         useFDs = conf.get("useFDs", false);
@@ -130,6 +133,7 @@ public BatchSummarizer getSummarizer(String outlierColumnName) throws MacroBaseE
                 summarizer.setAttributes(attributes);
                 summarizer.setMinSupport(minSupport);
                 summarizer.setMinRatioMetric(minRiskRatio);
+                summarizer.setBitmapRatioThreshold(bitmapRatioThreshold);
                 summarizer.setNumThreads(numThreads);
                 summarizer.setFDUsage(useFDs);
                 summarizer.setFDValues(functionalDependencies);

diff --git a/.../main/java/edu/stanford/futuredata/macrobase/analysis/summary/aplinear/APLSummarizer.java b/.../main/java/edu/stanford/futuredata/macrobase/analysis/summary/aplinear/APLSummarizer.java
@@ -25,6 +25,7 @@ public abstract class APLSummarizer extends BatchSummarizer {
 
     protected long numEvents = 0;
     protected long numOutliers = 0;
+    protected int bitmapRatioThreshold = 256;
 
     public abstract List<String> getAggregateNames();
     public abstract AggregationOp[] getAggregationOps();
@@ -81,7 +82,8 @@ public void process(DataFrame input) throws Exception {
                 encoder.getOutlierList(),
                 encoder.getColCardinalities(),
                 useFDs,
-                functionalDependencies
+                functionalDependencies,
+                bitmapRatioThreshold
         );
         log.info("Number of results: {}", aplResults.size());
         numOutliers = (long)getNumberOutliers(aggregateColumns);
@@ -100,4 +102,8 @@ public APLExplanation getResults() {
         return explanation;
     }
 
+    public void setBitmapRatioThreshold(int bitmapRatioThreshold) {
+        this.bitmapRatioThreshold = bitmapRatioThreshold;
+    }
+
 }
diff --git a/.../main/java/edu/stanford/futuredata/macrobase/analysis/summary/aplinear/APrioriLinear.java b/.../main/java/edu/stanford/futuredata/macrobase/analysis/summary/aplinear/APrioriLinear.java
@@ -57,7 +57,8 @@ public List<APLExplanationResult> explain(
             ArrayList<Integer>[] outlierList,
             int[] colCardinalities,
             boolean useFDs,
-            int[] functionalDependencies
+            int[] functionalDependencies,
+            int bitmapRatioThreshold
     ) {
         final int numAggregates = aggregateColumns.length;
         final int numRows = aggregateColumns[0].length;
@@ -198,7 +199,7 @@ public List<APLExplanationResult> explain(
                                 int[] curColumnTwoAttributes = byThreadAttributesTranspose[curThreadNum][colNumTwo];
                                 if (colCardinalities[colNumOne] < AttributeEncoder.cardinalityThreshold &&
                                         colCardinalities[colNumOne] < AttributeEncoder.cardinalityThreshold &&
-                                        colCardinalities[colNumOne] * colCardinalities[colNumTwo] < 256) {
+                                        colCardinalities[colNumOne] * colCardinalities[colNumTwo] < bitmapRatioThreshold) {
                                     // Bitmap-Bitmap
                                     allTwoBitmap(thisThreadSetAggregates, outlierList, aggregationOps, singleNextArray,
                                             byThreadBitmap[curThreadNum], colNumOne, colNumTwo, useIntSetAsArray,
@@ -232,7 +233,8 @@ public List<APLExplanationResult> explain(
                                     if (colCardinalities[colNumOne] < AttributeEncoder.cardinalityThreshold &&
                                             colCardinalities[colNumOne] < AttributeEncoder.cardinalityThreshold &&
                                             colCardinalities[colNumThree] < AttributeEncoder.cardinalityThreshold &&
-                                            colCardinalities[colNumOne] * colCardinalities[colNumTwo] * colCardinalities[colNumThree] < 256) {
+                                            colCardinalities[colNumOne] * colCardinalities[colNumTwo] *
+                                                    colCardinalities[colNumThree] < bitmapRatioThreshold) {
                                         // all 3 cols are bitmaps
                                         allThreeBitmap(thisThreadSetAggregates, outlierList, aggregationOps,
                                                 singleNextArray, byThreadBitmap[curThreadNum],