point arcsinh solver works with mb

stanford-futuredata · Apr 25, 2018 · b48723f · b48723f
1 parent d1ce6c5
commit b48723f
Show file tree

Hide file tree

Showing 24 changed files with 884 additions and 15 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,7 @@
 .project
 .settings/*
 *.log
+core/testconfig
 
 # Mobile Tools for Java (J2ME)
 .mtj.tmp/

diff --git a/core/demo/app_latency_arc_cube.csv b/core/demo/app_latency_arc_cube.csv
@@ -0,0 +1,49 @@
+app_version,country,device,latency:min,latency:max,latency:p05,latency:p10,latency:p50,latency:p90,latency:p95,asinh:0:sum,asinh:1:sum,asinh:2:sum,asinh:3:sum,asinh:4:sum,asinh:5:sum,asinh:6:sum,asinh:7:sum,asinh:8:sum
+0,0,0,0.004111863473803075,8.18093930868276,0.059341559870643414,0.12418329048104235,0.689949263616894,2.2664057971768905,3.265606999055996,201.0,151.1084037057634,175.2020069363426,256.5631096285591,438.1055265318691,832.8463352140877,1711.8029664531969,3733.482653588216,8534.30660545585
+0,0,1,0.0004266448080102065,6.355655090134978,0.0708343787003634,0.15320548441862958,0.7839532410242137,2.4308549276583697,3.541945688324472,243.0,194.12627414963183,236.1535229860248,358.0357896350832,622.6402965727309,1184.253629327964,2391.3101852819227,5031.277620871592,10899.978198084451
+0,0,2,0.0035731666454098593,6.219669175435734,0.060332124017079075,0.10457868941728232,0.7019034857826473,2.5951258490235793,3.3008129421037076,199.0,154.9676579125263,189.57760367067138,286.24186604239435,487.7513135740718,898.6635605563358,1750.6612563737858,3559.7910184744383,7493.2603804452965
+0,1,0,0.00027003408376986187,7.5914606236891355,0.09562833878171703,0.17303727001226976,0.7722762272345238,2.452986515667442,2.963794439416184,221.0,182.41065217765052,220.6304188323694,324.0480735410198,540.4529787811555,992.8170228119712,1971.8567380678085,4173.0777941384,9292.721507062795
+0,1,1,0.004712342795458156,5.785146106010697,0.04151752426835156,0.0792944269180359,0.7806961329185631,2.319150857109753,3.2421080983257187,195.0,152.68965828903146,183.1083252702816,268.1280805224043,447.2689253963264,816.2097498798414,1586.685362594818,3228.8935722169276,6799.249185908615
+0,1,2,0.00016239676113615254,5.8877856365467816,0.03781815306948392,0.07901775061934499,0.7239779139182773,2.1967062418350918,3.042528616028931,199.0,152.63271731835857,184.6301075467458,270.3322972098348,445.3781806964751,799.035683918864,1529.7127506865108,3078.767600004522,6439.188032813282
+0,2,0,0.0011734533839574508,4.448774553415015,0.03731440397618575,0.09614510974148384,0.7633987879335152,2.201317235798919,2.9934226443645797,209.0,162.59595310914065,192.22794094013634,273.09866432420756,433.80361578786017,743.1129084580951,1343.1804035601551,2525.33493607542,4890.2076658039405
+0,2,1,0.005946182801248096,5.02458458759357,0.058590650206898166,0.08977754868630539,0.7574226344899697,2.1054551690760754,2.891607014617969,218.0,166.16275163151693,189.56115652718668,261.01284834934734,405.61711232253765,686.0177242934121,1233.0948118481265,2317.719023021659,4504.929483189683
+0,2,2,0.009703912027257991,8.471223259352637,0.03988534386435691,0.09382769249380785,0.7648548261923434,2.382953681650195,3.0194329546102057,180.0,140.73635350099713,174.55277446822862,270.7807921162055,488.77552483657763,989.0458891215713,2183.31663715089,5142.423962364092,12696.60701892388
+0,3,0,0.006588975391639786,6.048863659265784,0.03886992416280064,0.08143578164048486,0.5973578489205636,2.229522158373864,2.9981675449821115,206.0,143.83340545443914,164.00592915033067,235.32659447538265,388.4531079821071,704.8211036461629,1369.137294801468,2800.198177542384,5960.430356814858
+0,3,1,0.002865285424857939,5.805290334664041,0.049599476856026864,0.1352697416913178,0.7590473271396065,2.388230057532723,2.95244345213644,193.0,153.11960313152488,180.43101520714234,258.3739810122978,417.7344615634208,733.3284743599053,1366.282137669789,2663.594288237731,5382.4697737651595
+0,3,2,0.004946847822630456,4.265021128043082,0.04226361798954095,0.11412722668261668,0.7041035877549661,2.1680474770228795,3.142909984503046,199.0,148.99955708898077,169.86897109384924,236.4012216090983,369.97774126615997,624.3350198951456,1108.4898105744512,2039.2677987163177,3848.900987388289
+1,0,0,0.005273790375943843,6.148202199692858,0.060816820574595,0.11391030346459043,0.8253575797475785,2.2463983103487766,2.8483508634458907,213.0,168.17153324198648,195.45574009049582,273.8099540743497,433.4912678144497,751.2404985710331,1397.2344760981894,2750.4859115686477,5669.8159334471575
+1,0,1,4.915336443307979,9.972200072578836,5.416021789515467,5.713464146007159,6.932963496663598,8.170018819489043,8.475293586417218,210.0,551.1483642279604,1450.544822068197,3828.189810190235,10130.719897292813,26881.610184545487,71518.90644159376,190773.74938382604,510186.89516451024
+1,0,2,0.0082208104648946,4.386771881466748,0.0811431203263088,0.1604249316596199,0.7041804216148777,2.1434092697251073,2.9702498657624092,207.0,156.11144183427095,175.22383801746523,239.96907200203262,370.4602429337031,619.0657461121153,1092.6017957981965,2004.1458888844704,3779.8084001153106
+1,1,0,0.007664333756228527,7.433415339989509,0.04607407531627443,0.08308704719829099,0.6033895225210639,2.1115733374909857,3.1935688127914634,208.0,145.34551659057055,170.6318311030542,252.7604022826725,429.98669077064085,803.7102124108019,1609.192009673653,3396.5669140621376,7477.83990669079
+1,1,1,4.9134942818527785,9.92552791032464,5.6469098972152905,5.854348456702195,7.016423036780294,8.39889747363235,8.740715479449234,220.0,581.0187966120117,1538.5115733289642,4084.581736331912,10872.341027741322,29014.754495429584,77629.38218563692,208226.09541883273,559932.6244943492
+1,1,2,0.00732891965603722,6.609532249080808,0.056088119370344316,0.10748098382730532,0.6990164498378735,2.094232322103351,2.588111379225547,195.0,146.662769969459,166.27417418391832,228.23135887726377,354.4113886287399,605.0117599574531,1117.213905833842,2204.5910611762656,4598.36628394069
+1,2,0,0.003295421476647752,5.0787308138592735,0.057030717919678155,0.07752980076109678,0.5770726752581055,2.0863918281394844,2.878039442929624,222.0,153.18412406493573,172.02502214417598,241.89761657444691,387.7016479049618,676.8637417817874,1254.8812017203556,2431.7875691172208,4873.3965459755245
+1,2,1,4.247411371171444,9.524646888501557,5.333677819691137,5.756190127850365,6.964025678902216,8.048467140736863,8.866957562602261,225.0,593.6336337193741,1570.8339069387396,4168.517731328618,11092.733131007515,29598.447713646365,79184.94990854879,212388.71352309277,571095.5518298195
+1,2,2,0.008766105927473775,4.8286677491633565,0.05614305867406352,0.10890593488930622,0.6805511763061003,2.2365253836165473,2.9197067349528627,193.0,140.88235885331534,159.45725418938662,223.30994600968597,355.71627945910336,616.323207950137,1130.975872749739,2161.915979454201,4259.18872838519
+1,3,0,0.0035314610921782954,4.324408559845507,0.05298716335554335,0.09389607800456097,0.6356696229173123,2.2228577365445488,2.7313177870066143,211.0,154.2154824296106,174.7580103023283,239.99326161975415,368.7669680695105,611.0201567453329,1068.8707916164196,1947.0444184884434,3657.7411208471267
+1,3,1,4.759656489212661,10.662544794820056,5.422290197469941,5.798676930261012,6.928219823982738,8.252897972641357,8.674810702966635,196.0,515.4609365717017,1359.6094930818927,3596.6881178099798,9542.281069198427,25389.402121194795,67747.4626751474,181285.41077862986,486463.78842310776
+1,3,2,0.0005565830698941332,6.202180124984274,0.04088033756679703,0.10507812548035037,0.8102158985540076,2.139632318992093,2.7471788785932936,225.0,174.7874730426657,200.75205972786992,275.87052980998385,428.075141788737,728.4168764815411,1332.393258408841,2580.477993485039,5229.872446445919
+2,0,0,0.014659111141172144,5.529580461068399,0.055914298238278565,0.10012909311287786,0.6539575613573129,2.396711200228377,2.744284550995906,189.0,144.21445864581307,167.35312140000627,231.9369177007944,354.5538013265251,578.7436401897787,994.1907796732959,1785.0013531550555,3337.515234027376
+2,0,1,0.007991039160661054,6.5969658781993905,0.10048438132634044,0.15431920370253055,0.7857351866472314,2.4383673352369533,3.391741096898691,197.0,163.57876849971453,199.62757926337483,296.41095701485693,497.9367127703489,911.5445001294537,1777.3258445838158,3634.9596043948573,7714.761747374742
+2,0,2,0.0038985590430934414,5.475326210511428,0.06550921117825287,0.10933931111674294,0.8337335263589745,2.2758224079769946,3.227210537235954,183.0,139.9854012540583,166.27233855881073,241.14180845669918,397.003917642471,714.5315694607932,1369.5547684638723,2743.4361832952345,5668.978608423776
+2,1,0,0.026850493819486638,7.631661167977855,0.08133506845815879,0.1387380647956311,0.7991512256843513,2.5096201804536893,3.3232357529167773,199.0,166.33582474134153,205.65084690668237,308.66081978348444,523.9442828120788,971.2833899642933,1926.833587184622,4036.5533880254284,8844.177175506144
+2,1,1,0.0029088940578763482,6.4499598699613525,0.06603385846604218,0.1357321362550738,0.8152696838056663,2.378959172422318,2.9159095439963707,195.0,160.81953814161946,194.1889004241512,279.2849121619921,447.7923137562371,777.1597013266035,1436.5882899353842,2798.8391599500983,5701.978701998766
+2,1,2,0.003354035853989559,4.635187266883492,0.05252750083345385,0.10873120676663009,0.5783963748950247,1.8607250907083168,3.0284641367602605,215.0,145.91066582656444,158.3318670421822,215.0744294592271,333.546464769788,563.8047853177437,1010.3662777680637,1886.0077703555583,3625.312647657386
+2,2,0,0.0001863425112293392,7.005251433275174,0.07576099489413995,0.1453039082745175,0.6654503794773633,2.301463349603971,2.6611304346743214,225.0,166.51592137402315,188.40501345744994,269.49502424846594,448.0708622814982,825.3843256090776,1635.2945689339126,3418.4125377769265,7443.289857838371
+2,2,1,0.0012769831149404508,5.463591821713238,0.05121665552147244,0.0987539333583908,0.6580231188517348,2.290499937781767,3.3901473659647605,213.0,158.76978807146904,189.92301978026381,286.1210059420229,494.59429079676147,932.013169121053,1857.6830722745192,3845.9247577422616,8177.005513290028
+2,2,2,0.0005771872800048348,6.508898633910245,0.03829108791927643,0.1041489338707916,0.7141620326128317,2.3426557901579845,2.8786292197974603,203.0,152.18314636205196,177.49048449893172,252.68761939337685,405.7418626740778,708.9102671214474,1321.4437744538102,2595.812895562298,5328.12558412054
+2,3,0,0.003840882058551833,5.15802248185584,0.057660415936300516,0.10319288212105698,0.6770903342151174,2.139748026758093,2.534258916730874,191.0,140.8013818077857,158.1870988512463,212.85698620631632,317.55442634715257,508.5979915406717,860.8223926647329,1526.2023921475384,2818.2269448165343
+2,3,1,0.003434625432122852,6.0658384605798625,0.044644143899943756,0.08953052396282594,0.7239661038477309,2.505729762372651,3.1370643743391433,227.0,176.04937446817425,217.93974954556305,321.83372173983906,524.0037648083277,911.8839626864035,1669.0126883587427,3181.8096049419128,6277.875826942221
+2,3,2,0.03863213740412976,5.309880999142209,0.07618283014068783,0.13389297491516686,0.5970515152743392,2.1156653617482606,2.5984254349011517,219.0,155.54787716603005,171.7900878685446,235.83165481743214,366.8882874102804,619.6971390875242,1111.0028617593555,2085.565337797765,4061.5597030449394
+3,0,0,0.004045468133890418,11.080746252112679,0.04830025479644624,0.13291121246772308,0.7924300393837787,2.332223021025655,3.2360974758396086,194.0,159.88305360575447,198.89165158982837,304.4731086599948,537.7806999380877,1066.4194394453177,2335.3540057574705,5562.641632189724,14186.587171407087
+3,0,1,0.018238581066285574,4.983313559301618,0.06651444909185215,0.11781700804780762,0.7610142343559447,2.4601899267468985,3.435730036560795,199.0,157.66546792628262,190.89333936107005,278.95708842513085,454.63660616687747,796.0536000229848,1465.3041071930772,2796.5827461399435,5483.8022047470795
+3,0,2,0.013546166722106612,4.479825657053874,0.0698187656407587,0.14636946210098029,0.7935846533549463,1.9122615155074016,2.632256460491121,207.0,161.5316472445962,174.33410767912045,222.8617017243533,320.0140943054242,500.9006544562416,837.4969112248895,1474.072591137135,2701.568338225125
+3,1,0,0.0003382939061853576,6.4182351847223185,0.05803206575658581,0.10215141776836491,0.6502678935606029,2.318723288147152,3.139661534821511,222.0,163.69827778491543,191.7406145962964,278.1825225483149,457.72745692613205,819.5366973323092,1560.5937783377856,3116.197944605873,6463.164355305533
+3,1,1,0.0021203394320872667,7.479085485521392,0.03140119890107057,0.05989610584328241,0.621247265337659,2.273413309531192,2.9332393925090963,188.0,132.9689654468992,158.6484328378953,235.79456781671797,401.0611161118638,753.4685812973837,1531.2361743437175,3313.7918622656066,7536.961504522647
+3,1,2,0.011325288677143743,4.507173717617364,0.07512707908890286,0.12043922748385916,0.6548055991967763,1.9403540804973178,2.5954468703345825,236.0,163.70430132476656,174.4584635831556,229.6332046349024,343.63635623647724,560.2744073861954,969.6466206019143,1751.226047713775,3263.32390460694
+3,2,0,0.00028733298625629994,7.085244557074901,0.03384460828008231,0.09333567912085461,0.5901737917820125,2.062688125309328,2.532193373431884,240.0,164.41488858903804,182.47378356172555,249.17567909235817,386.0129613930707,657.5351025129114,1213.6707371551424,2404.6891151406294,5070.026222395645
+3,2,1,0.0007125646307133928,5.79739388490157,0.07274398718222376,0.10232452344383824,0.8762898074754852,2.300940795225813,2.712014032012683,207.0,170.8573266374816,203.2540238450782,289.4989741904177,465.5362630888378,821.1624696458006,1559.6640802362817,3143.9139214442057,6642.85180012411
+3,2,2,0.0017989088083670886,7.0475998185017374,0.043886866955400665,0.08057508542846326,0.6426424150280567,2.8888064411642187,3.6037755181123936,221.0,177.50695907026815,228.32730134175193,359.48317855764407,632.6989785480171,1195.1096706692726,2373.7103456104196,4902.0474239115365,10454.714693981232
+3,3,0,0.001782396359889836,5.362226055657152,0.02978435916275491,0.06268994705706547,0.6096832824001557,2.267694389648853,2.799498403226423,223.0,161.19548861964424,188.52442152799506,271.0859000543323,439.7881443706011,773.7526736299453,1443.860605761354,2816.608526523331,5685.142938117595
+3,3,1,0.0016039687748705526,4.712990817192963,0.05496236183314528,0.11625502492057103,0.6520974936890809,2.315486929503886,3.0515535059765657,217.0,162.15231241609865,192.6310907978403,276.30167257445464,438.9272304201389,744.9432146667455,1325.6310099083921,2445.2850635189475,4639.956733344633
+3,3,2,0.01993955853355517,5.324024458682013,0.05046056837943474,0.11574555603288703,0.6410417982762897,2.229724447694185,2.8669597915563987,202.0,143.32868985472544,158.60634060144432,217.9307787371135,340.62245993572816,579.9091951531657,1049.8823650777026,1992.218688652472,3924.980322268166
diff --git a/core/demo/cube_arc_power.json b/core/demo/cube_arc_power.json
@@ -0,0 +1,25 @@
+{
+  "pipeline": "ArcPowerCubePipeline",
+  "inputURI": "csv://core/demo/app_latency_arc_cube.csv",
+  "cutoff": 5.0,
+  "minColumn": "latency:min",
+  "maxColumn": "latency:max",
+  "powerSumColumns": [
+    "asinh:0:sum",
+    "asinh:1:sum",
+    "asinh:2:sum",
+    "asinh:3:sum",
+    "asinh:4:sum",
+    "asinh:5:sum",
+    "asinh:6:sum",
+    "asinh:7:sum",
+    "asinh:8:sum"
+  ],
+  "attributes": [
+    "app_version",
+    "country",
+    "device"
+  ],
+  "minRatioMetric": 8.0,
+  "minSupport": 0.05
+}
diff --git a/core/src/main/java/edu/stanford/futuredata/macrobase/pipeline/ArcPowerCubePipeline.java b/core/src/main/java/edu/stanford/futuredata/macrobase/pipeline/ArcPowerCubePipeline.java
@@ -0,0 +1,119 @@
+package edu.stanford.futuredata.macrobase.pipeline;
+
+import edu.stanford.futuredata.macrobase.analysis.summary.aplinear.APLArcMomentSummarizer;
+import edu.stanford.futuredata.macrobase.analysis.summary.aplinear.APLExplanation;
+import edu.stanford.futuredata.macrobase.analysis.summary.aplinear.APLMomentSummarizer;
+import edu.stanford.futuredata.macrobase.analysis.summary.aplinear.APLSummarizer;
+import edu.stanford.futuredata.macrobase.datamodel.DataFrame;
+import edu.stanford.futuredata.macrobase.datamodel.Schema;
+import edu.stanford.futuredata.macrobase.util.MacrobaseException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+/**
+ * Pipeline for cubed data that has min, max and moment aggregates
+ */
+public class ArcPowerCubePipeline implements Pipeline {
+    Logger log = LoggerFactory.getLogger("ArcPowerCubePipeline");
+
+    // Ingest
+    private String inputURI;
+    private Map<String, String> restHeader;
+    private Map<String, Object> jsonBody;
+    private boolean usePost;
+
+    // Classifiers
+    private double cutoff;
+//    private boolean includeHi;
+//    private boolean includeLo;
+    private int k;
+    private Optional<String> minColumn;
+    private Optional<String> maxColumn;
+    private List<String> powerSumColumns;
+
+    // Explanation
+    private List<String> attributes;
+    private double minSupport;
+    private double minRatioMetric;
+
+    public ArcPowerCubePipeline(PipelineConfig conf) {
+        inputURI = conf.get("inputURI");
+        restHeader = conf.get("restHeader", null);
+        jsonBody = conf.get("jsonBody", null);
+        usePost = conf.get("usePost", true);
+
+        cutoff = conf.get("cutoff", 1.0);
+
+        minColumn = Optional.ofNullable(conf.get("minColumn"));
+        maxColumn = Optional.ofNullable(conf.get("maxColumn"));
+        powerSumColumns = conf.get("powerSumColumns", new ArrayList<String>());
+        k = powerSumColumns.size();
+
+        attributes = conf.get("attributes");
+        minSupport = conf.get("minSupport", 3.0);
+        minRatioMetric = conf.get("minRatioMetric", 0.01);
+    }
+
+    public APLExplanation results() throws Exception {
+        Map<String, Schema.ColType> colTypes = getColTypes();
+        long startTime = System.currentTimeMillis();
+        final List<String> requiredColumns = new ArrayList<>(attributes);
+        requiredColumns.addAll(colTypes.keySet());
+        DataFrame df = PipelineUtils.loadDataFrame(
+                inputURI,
+                colTypes,
+                restHeader,
+                jsonBody,
+                usePost,
+                requiredColumns
+        );
+        long elapsed = System.currentTimeMillis() - startTime;
+        log.info("Loading time: {}", elapsed);
+        log.info("{} rows", df.getNumRows());
+        log.info("Attributes: {}", attributes);
+//        CSVDataFrameWriter writer = new CSVDataFrameWriter();
+//        PrintWriter out = new PrintWriter("df_power.csv");
+//        writer.writeToStream(df, out);
+
+        startTime = System.currentTimeMillis();
+        APLSummarizer summarizer = getSummarizer();
+        summarizer.process(df);
+        APLExplanation explanation = summarizer.getResults();
+        elapsed = System.currentTimeMillis() - startTime;
+        log.info("Summarization time: {}", elapsed);
+
+        return explanation;
+    }
+
+    private Map<String, Schema.ColType> getColTypes() throws MacrobaseException {
+        Map<String, Schema.ColType> colTypes = new HashMap<>();
+        colTypes.put(
+                minColumn.orElseThrow(() -> new MacrobaseException("min column not present in config")),
+                Schema.ColType.DOUBLE);
+        colTypes.put(
+                maxColumn.orElseThrow(() -> new MacrobaseException("max column not present in config")),
+                Schema.ColType.DOUBLE);
+        for (String col : powerSumColumns) {
+            colTypes.put(col, Schema.ColType.DOUBLE);
+        }
+        return colTypes;
+    }
+
+    private APLSummarizer getSummarizer() throws Exception {
+        APLArcMomentSummarizer summarizer = new APLArcMomentSummarizer();
+        summarizer.setK(k);
+        summarizer.setMinColumn(minColumn.orElseThrow(
+                () -> new MacrobaseException("min column not present in config")));
+        summarizer.setMaxColumn(maxColumn.orElseThrow(
+                () -> new MacrobaseException("max column not present in config")));
+        summarizer.setPowerSumColumns(powerSumColumns);
+
+        summarizer.setAttributes(attributes);
+        summarizer.setMinSupport(minSupport);
+        summarizer.setMinRatioMetric(minRatioMetric);
+        summarizer.setQuantileCutoff(1.0 - cutoff/100.0);
+        return summarizer;
+    }
+}
diff --git a/core/src/main/java/edu/stanford/futuredata/macrobase/pipeline/PipelineUtils.java b/core/src/main/java/edu/stanford/futuredata/macrobase/pipeline/PipelineUtils.java
@@ -69,6 +69,9 @@ public static Pipeline createPipeline(
             case "PowerCubePipeline": {
                 return new PowerCubePipeline(conf);
             }
+            case "ArcPowerCubePipeline": {
+                return new ArcPowerCubePipeline(conf);
+            }
             default: {
                 throw new MacrobaseException("Bad Pipeline");
             }