diff --git a/Makefile b/Makefile index e32e5399..340661c4 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ include config.mk +TEST_BINARY_INPUT_PATH = test + BUILDDIR := build TARGETDIR := lib @@ -10,7 +12,7 @@ TARGET := $(TARGETDIR)/$(LIBRARY) INC := -I /usr/local/include LIB := -L /usr/local/lib -lcppunit -L lib -l$(LIB_BASE_NAME) -MODULES := hll cpc kll fi +MODULES := hll cpc kll fi theta .PHONY: all all: $(MODULES) $(LIBRARY) diff --git a/theta/include/binomial_bounds.hpp b/theta/include/binomial_bounds.hpp new file mode 100644 index 00000000..58de181e --- /dev/null +++ b/theta/include/binomial_bounds.hpp @@ -0,0 +1,444 @@ +/* + * Copyright 2019, Verizon Media. + * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms. + */ + +#ifndef BINOMIAL_BOUNDS_HPP_ +#define BINOMIAL_BOUNDS_HPP_ + +#include +#include + +/* + * This class enables the estimation of error bounds given a sample set size, the sampling + * probability theta, the number of standard deviations and a simple noDataSeen flag. This can + * be used to estimate error bounds for fixed threshold sampling as well as the error bounds + * calculations for sketches. + * + * author Alexander Saydakov + * author Lee Rhodes + * author Kevin Lang + */ + +namespace datasketches { + +static constexpr double delta_of_num_std_devs[] = { + 0.5000000000000000000, // not actually using this value + 0.1586553191586026479, + 0.0227502618904135701, + 0.0013498126861731796 +}; + +static constexpr double lb_equiv_table[] = { + 1.0, 2.0, 3.0, // fake values for k = 0 + 0.78733703534118149, 3.14426768537558132, 13.56789685109913535, // k = 1 + 0.94091379266077979, 2.64699271711145911, 6.29302733018320737, // k = 2 + 0.96869128474958188, 2.46531676590527127, 4.97375283467403051, // k = 3 + 0.97933572521046131, 2.37418810664669877, 4.44899975481712318, // k = 4 + 0.98479165917274258, 2.31863116255024693, 4.16712379778553554, // k = 5 + 0.98806033915698777, 2.28075536565225434, 3.99010556144099837, // k = 6 + 0.99021896790580399, 2.25302005857281529, 3.86784477136922078, // k = 7 + 0.99174267079089873, 2.23168103978522936, 3.77784896945266269, // k = 8 + 0.99287147837287648, 2.21465899260871879, 3.70851932988722410, // k = 9 + 0.99373900046805375, 2.20070155496262032, 3.65326029076638292, // k = 10 + 0.99442519013851438, 2.18900651202670815, 3.60803817612955413, // k = 11 + 0.99498066823221620, 2.17903457780744247, 3.57024330407946877, // k = 12 + 0.99543899410224412, 2.17040883161922693, 3.53810982030634591, // k = 13 + 0.99582322541263579, 2.16285726913676513, 3.51039837124298515, // k = 14 + 0.99614973311747690, 2.15617827879603396, 3.48621230377099778, // k = 15 + 0.99643042892560629, 2.15021897666090922, 3.46488605693562590, // k = 16 + 0.99667418783778317, 2.14486114872480016, 3.44591466064832730, // k = 17 + 0.99688774875812669, 2.14001181420209718, 3.42890765690452781, // k = 18 + 0.99707632299691795, 2.13559675336844634, 3.41355809420343803, // k = 19 + 0.99724399084971083, 2.13155592217421486, 3.39962113251016262, // k = 20 + 0.99739400151915447, 2.12784018863251845, 3.38689892877548004, // k = 21 + 0.99752896842633731, 2.12440890875851096, 3.37522975271599535, // k = 22 + 0.99765101725122918, 2.12122815311133195, 3.36448003577621080, // k = 23 + 0.99776189496810730, 2.11826934724291505, 3.35453840911279144, // k = 24 + 0.99786304821586214, 2.11550823850916458, 3.34531123809287578, // k = 25 + 0.99795568665180667, 2.11292409529477254, 3.33671916527694634, // k = 26 + 0.99804083063483517, 2.11049908609763293, 3.32869446834217797, // k = 27 + 0.99811933910984862, 2.10821776918189130, 3.32117898316676019, // k = 28 + 0.99819195457286014, 2.10606671027090897, 3.31412243534683171, // k = 29 + 0.99825930555178388, 2.10403415237001923, 3.30748113008135647, // k = 30 + 0.99832193858154028, 2.10210975877822648, 3.30121691946897045, // k = 31 + 0.99838032666573895, 2.10028440670842542, 3.29529629751144171, // k = 32 + 0.99843488390555990, 2.09855000145353188, 3.28968974413223236, // k = 33 + 0.99848596721417948, 2.09689934193824001, 3.28437111460505093, // k = 34 + 0.99853390005924325, 2.09532599155502908, 3.27931717312372939, // k = 35 + 0.99857895741078551, 2.09382418262592296, 3.27450718840060517, // k = 36 + 0.99862138880970974, 2.09238872751677718, 3.26992261182860489, // k = 37 + 0.99866141580770318, 2.09101494715108061, 3.26554677962434425, // k = 38 + 0.99869923565267982, 2.08969860402822860, 3.26136468165239535, // k = 39 + 0.99873502010169091, 2.08843585627218431, 3.25736275677081721, // k = 40 + 0.99876893292508839, 2.08722321436752623, 3.25352872241415980, // k = 41 + 0.99880111078502409, 2.08605749165553789, 3.24985141664350863, // k = 42 + 0.99883168573342118, 2.08493577529222307, 3.24632068399498053, // k = 43 + 0.99886077231613513, 2.08385540129560809, 3.24292724848112357, // k = 44 + 0.99888847451828155, 2.08281392374021834, 3.23966263299664092, // k = 45 + 0.99891488795844907, 2.08180908991394631, 3.23651906111521726, // k = 46 + 0.99894010085196783, 2.08083882998420222, 3.23348939240611344, // k = 47 + 0.99896419358239541, 2.07990122528650545, 3.23056705515594444, // k = 48 + 0.99898723510594323, 2.07899450946285924, 3.22774598963252402, // k = 49 + 0.99900929266780736, 2.07811704477046533, 3.22502059972006805, // k = 50 + 0.99903043086155208, 2.07726730587160091, 3.22238570890294795, // k = 51 + 0.99905070073845081, 2.07644388314946582, 3.21983651940365689, // k = 52 + 0.99907015770423868, 2.07564546080757850, 3.21736857351049821, // k = 53 + 0.99908884779227947, 2.07487081196367740, 3.21497773796417619, // k = 54 + 0.99910681586905525, 2.07411879634256024, 3.21266015316183484, // k = 55 + 0.99912410177549305, 2.07338834403498140, 3.21041222805715165, // k = 56 + 0.99914074347179849, 2.07267845454973099, 3.20823061166797174, // k = 57 + 0.99915677607464204, 2.07198819052374006, 3.20611216970604573, // k = 58 + 0.99917223149395795, 2.07131667846186929, 3.20405396962596001, // k = 59 + 0.99918714153457699, 2.07066309019154460, 3.20205326110445299, // k = 60 + 0.99920153247185794, 2.07002665203046377, 3.20010746990493544, // k = 61 + 0.99921543193525508, 2.06940663431663552, 3.19821417453343315, // k = 62 + 0.99922886570365677, 2.06880235245998279, 3.19637109973109546, // k = 63 + 0.99924185357357942, 2.06821315729285971, 3.19457610621114441, // k = 64 + 0.99925441845175555, 2.06763843812092318, 3.19282717869864996, // k = 65 + 0.99926658263325407, 2.06707761824370095, 3.19112241228646099, // k = 66 + 0.99927836173816331, 2.06653015295219689, 3.18946001739936946, // k = 67 + 0.99928977431994781, 2.06599552505539918, 3.18783829446098821, // k = 68 + 0.99930083753795884, 2.06547324585920933, 3.18625564538041317, // k = 69 + 0.99931156864562354, 2.06496285191821016, 3.18471055124089730, // k = 70 + 0.99932197985521043, 2.06446390392778767, 3.18320157510865442, // k = 71 + 0.99933208559809827, 2.06397598606787369, 3.18172735837393361, // k = 72 + 0.99934190032416836, 2.06349869971447220, 3.18028661102792398, // k = 73 + 0.99935143390791836, 2.06303166975550312, 3.17887810481605015, // k = 74 + 0.99936070171270330, 2.06257453607466346, 3.17750067581857820, // k = 75 + 0.99936971103502970, 2.06212696042919674, 3.17615321728274580, // k = 76 + 0.99937847392385493, 2.06168861430600714, 3.17483467831510779, // k = 77 + 0.99938700168914352, 2.06125918927764928, 3.17354405480557489, // k = 78 + 0.99939530099953799, 2.06083838987589729, 3.17228039269048168, // k = 79 + 0.99940338278830154, 2.06042593411496000, 3.17104278166036124, // k = 80 + 0.99941125463777780, 2.06002155276328835, 3.16983035274597569, // k = 81 + 0.99941892470027938, 2.05962498741951094, 3.16864227952240185, // k = 82 + 0.99942640059737187, 2.05923599161263837, 3.16747776846497686, // k = 83 + 0.99943368842187397, 2.05885433061945378, 3.16633606416374391, // k = 84 + 0.99944079790603269, 2.05847977868873500, 3.16521644518826406, // k = 85 + 0.99944773295734990, 2.05811212058944193, 3.16411821883858124, // k = 86 + 0.99945450059186669, 2.05775114781260982, 3.16304072400711789, // k = 87 + 0.99946110646314423, 2.05739666442039493, 3.16198332650733960, // k = 88 + 0.99946755770463369, 2.05704847678819647, 3.16094541781455973, // k = 89 + 0.99947385746861528, 2.05670640500335367, 3.15992641851471490, // k = 90 + 0.99948001256305474, 2.05637027420314666, 3.15892576988736096, // k = 91 + 0.99948602689656241, 2.05603991286400856, 3.15794293484717059, // k = 92 + 0.99949190674294641, 2.05571516158917689, 3.15697740043813724, // k = 93 + 0.99949765436329585, 2.05539586490317561, 3.15602867309343083, // k = 94 + 0.99950327557880314, 2.05508187237845164, 3.15509627710042651, // k = 95 + 0.99950877461972709, 2.05477304104951486, 3.15417975753007340, // k = 96 + 0.99951415481862682, 2.05446923022574879, 3.15327867462917766, // k = 97 + 0.99951942042375208, 2.05417030908833453, 3.15239260700215596, // k = 98 + 0.99952457390890004, 2.05387614661762541, 3.15152114915238712, // k = 99 + 0.99952962005008317, 2.05358662050909402, 3.15066390921020911, // k = 100 + 0.99953456216121594, 2.05330161104427589, 3.14982051097524618, // k = 101 + 0.99953940176368405, 2.05302100378725072, 3.14899059183684926, // k = 102 + 0.99954414373920031, 2.05274468493067275, 3.14817379948561893, // k = 103 + 0.99954879047621148, 2.05247255013657082, 3.14736979964868624, // k = 104 + 0.99955334485656522, 2.05220449388099269, 3.14657826610371671, // k = 105 + 0.99955780993869325, 2.05194041831310869, 3.14579888316276879, // k = 106 + 0.99956218652590678, 2.05168022402710903, 3.14503134811607765, // k = 107 + 0.99956647932785359, 2.05142381889103831, 3.14427536967733090, // k = 108 + 0.99957069025060719, 2.05117111251445294, 3.14353066260227365, // k = 109 + 0.99957482032178291, 2.05092201793428330, 3.14279695558593630, // k = 110 + 0.99957887261450651, 2.05067645094720774, 3.14207398336887422, // k = 111 + 0.99958284988383639, 2.05043432833224415, 3.14136149076028914, // k = 112 + 0.99958675435604505, 2.05019557189746138, 3.14065923143530767, // k = 113 + 0.99959058650074439, 2.04996010556124020, 3.13996696426707445, // k = 114 + 0.99959434898201494, 2.04972785368377686, 3.13928445867830419, // k = 115 + 0.99959804437042976, 2.04949874512311681, 3.13861149103462367, // k = 116 + 0.99960167394553423, 2.04927271043337100, 3.13794784369528656, // k = 117 + 0.99960523957651048, 2.04904968140490951, 3.13729330661277572, // k = 118 + 0.99960874253329735, 2.04882959397491504, 3.13664767767019725, // k = 119 + 0.99961218434327748, 2.04861238220240693, 3.13601075688413289 // k = 120 +}; + +static constexpr double ub_equiv_table[] = { + 1.0, 2.0, 3.0, // fake values for k = 0 + 0.99067760836669549, 1.75460517119302040, 2.48055626001627161, // k = 1 + 0.99270518097577565, 1.78855957509907171, 2.53863835259832626, // k = 2 + 0.99402032633599902, 1.81047286499563143, 2.57811676180597260, // k = 3 + 0.99492607629539975, 1.82625928017762362, 2.60759550546498531, // k = 4 + 0.99558653966013821, 1.83839160339161367, 2.63086812358551470, // k = 5 + 0.99608981951632813, 1.84812399034444752, 2.64993712523727254, // k = 6 + 0.99648648035983456, 1.85617372053235385, 2.66598485907860550, // k = 7 + 0.99680750790483330, 1.86298655802610824, 2.67976541374471822, // k = 8 + 0.99707292880049181, 1.86885682585270274, 2.69178781407745760, // k = 9 + 0.99729614928489241, 1.87398826101983218, 2.70241106542158604, // k = 10 + 0.99748667952445658, 1.87852708449801753, 2.71189717290596377, // k = 11 + 0.99765127712748836, 1.88258159501103250, 2.72044290303773550, // k = 12 + 0.99779498340305395, 1.88623391878036273, 2.72819957382063194, // k = 13 + 0.99792160418357412, 1.88954778748873764, 2.73528576807902368, // k = 14 + 0.99803398604944960, 1.89257337682371940, 2.74179612106766513, // k = 15 + 0.99813449883217231, 1.89535099316557876, 2.74780718300419835, // k = 16 + 0.99822494122659577, 1.89791339232732525, 2.75338173141955167, // k = 17 + 0.99830679915913834, 1.90028752122407241, 2.75857186416826039, // k = 18 + 0.99838117410831728, 1.90249575897183831, 2.76342117562634826, // k = 19 + 0.99844913407071090, 1.90455689090418900, 2.76796659454200267, // k = 20 + 0.99851147736424650, 1.90648682834171268, 2.77223944710058845, // k = 21 + 0.99856879856019987, 1.90829917277082473, 2.77626682032629901, // k = 22 + 0.99862183849734265, 1.91000561415842185, 2.78007199816156003, // k = 23 + 0.99867096266018507, 1.91161621560812023, 2.78367524259661536, // k = 24 + 0.99871656986212543, 1.91313978579765376, 2.78709435016625662, // k = 25 + 0.99875907577771272, 1.91458400425526065, 2.79034488416175463, // k = 26 + 0.99879885565047744, 1.91595563175945927, 2.79344064132371273, // k = 27 + 0.99883610756373287, 1.91726064301425936, 2.79639384757751941, // k = 28 + 0.99887095169674467, 1.91850441099725799, 2.79921543574803877, // k = 29 + 0.99890379414739527, 1.91969155477030995, 2.80191513182441554, // k = 30 + 0.99893466279047516, 1.92082633358913313, 2.80450167352080371, // k = 31 + 0.99896392088177777, 1.92191254955568525, 2.80698295731653502, // k = 32 + 0.99899147889385631, 1.92295362479495680, 2.80936614404217266, // k = 33 + 0.99901764688726757, 1.92395267400968351, 2.81165765979318394, // k = 34 + 0.99904238606342233, 1.92491244978191389, 2.81386337393604435, // k = 35 + 0.99906590152386343, 1.92583552644848055, 2.81598868034527072, // k = 36 + 0.99908829040739988, 1.92672418013918900, 2.81803841726804194, // k = 37 + 0.99910959420023460, 1.92758051694144683, 2.82001709302821268, // k = 38 + 0.99912996403594434, 1.92840654943159961, 2.82192875763732332, // k = 39 + 0.99914930224576892, 1.92920397044028391, 2.82377730628954282, // k = 40 + 0.99916781270195543, 1.92997447498220254, 2.82556612075063640, // k = 41 + 0.99918553179077207, 1.93071949211818605, 2.82729843191989971, // k = 42 + 0.99920250730914972, 1.93144048613876862, 2.82897728689417249, // k = 43 + 0.99921873345181211, 1.93213870990595638, 2.83060537017752267, // k = 44 + 0.99923435180002684, 1.93281536508689555, 2.83218527795750674, // k = 45 + 0.99924930425362390, 1.93347145882316340, 2.83371938965598247, // k = 46 + 0.99926370394567243, 1.93410820221384938, 2.83520990872793277, // k = 47 + 0.99927750755296074, 1.93472643138986200, 2.83665891945119597, // k = 48 + 0.99929082941537217, 1.93532697329771963, 2.83806833931606661, // k = 49 + 0.99930366295501472, 1.93591074716263734, 2.83943997143404658, // k = 50 + 0.99931598804721489, 1.93647857274021362, 2.84077557836653227, // k = 51 + 0.99932789059798210, 1.93703110239354714, 2.84207662106302905, // k = 52 + 0.99933946180485123, 1.93756904936378760, 2.84334468086129277, // k = 53 + 0.99935053819703512, 1.93809302131219852, 2.84458116874117195, // k = 54 + 0.99936126637970801, 1.93860365411038060, 2.84578731838604426, // k = 55 + 0.99937166229284458, 1.93910149816429112, 2.84696443486512862, // k = 56 + 0.99938169190727422, 1.93958709548454067, 2.84811369085281285, // k = 57 + 0.99939136927613959, 1.94006085573701625, 2.84923617230361970, // k = 58 + 0.99940074328745254, 1.94052339623206649, 2.85033291216254270, // k = 59 + 0.99940993070470086, 1.94097508636855309, 2.85140492437699322, // k = 60 + 0.99941868577388959, 1.94141633372043998, 2.85245314430358121, // k = 61 + 0.99942734443487780, 1.94184757038001976, 2.85347839582286156, // k = 62 + 0.99943556385736088, 1.94226915100517772, 2.85448160365493209, // k = 63 + 0.99944374522542034, 1.94268143723749631, 2.85546346373061510, // k = 64 + 0.99945159955424856, 1.94308482059116727, 2.85642486111805738, // k = 65 + 0.99945915301904620, 1.94347956957849988, 2.85736639994965458, // k = 66 + 0.99946660663832176, 1.94386600964031686, 2.85828887832701639, // k = 67 + 0.99947383703224091, 1.94424436597356021, 2.85919278275500233, // k = 68 + 0.99948075442870277, 1.94461502153473020, 2.86007887186090670, // k = 69 + 0.99948766082269458, 1.94497821937304138, 2.86094774077355396, // k = 70 + 0.99949422748713346, 1.94533411296001191, 2.86179981848076181, // k = 71 + 0.99950070756119658, 1.94568300035135167, 2.86263579405672886, // k = 72 + 0.99950704321753392, 1.94602523449961495, 2.86345610449197352, // k = 73 + 0.99951320334216121, 1.94636083782822311, 2.86426125541271404, // k = 74 + 0.99951920293474927, 1.94669011080745236, 2.86505169255406145, // k = 75 + 0.99952501670378524, 1.94701327348536779, 2.86582788270862920, // k = 76 + 0.99953071209267819, 1.94733044372333097, 2.86659027602854621, // k = 77 + 0.99953632734991515, 1.94764180764266825, 2.86733927778843167, // k = 78 + 0.99954171164873173, 1.94794766430732125, 2.86807526143834934, // k = 79 + 0.99954699274462655, 1.94824807472994621, 2.86879864789403882, // k = 80 + 0.99955216611081710, 1.94854317889829076, 2.86950970901679625, // k = 81 + 0.99955730019613043, 1.94883320227168610, 2.87020887436986527, // k = 82 + 0.99956213770650493, 1.94911826561721568, 2.87089648477021342, // k = 83 + 0.99956704264963037, 1.94939848545763539, 2.87157281693902178, // k = 84 + 0.99957166306481327, 1.94967401618316671, 2.87223821840905202, // k = 85 + 0.99957632713136491, 1.94994497791333288, 2.87289293193450135, // k = 86 + 0.99958087233392234, 1.95021155752212394, 2.87353731228213860, // k = 87 + 0.99958532555996271, 1.95047376805584349, 2.87417154907075201, // k = 88 + 0.99958956246481989, 1.95073180380688882, 2.87479599765507032, // k = 89 + 0.99959389351869277, 1.95098572880579013, 2.87541081987382086, // k = 90 + 0.99959807862052230, 1.95123574036898617, 2.87601637401948551, // k = 91 + 0.99960214057801977, 1.95148186921983324, 2.87661283691068093, // k = 92 + 0.99960607527256684, 1.95172415829728152, 2.87720042968334155, // k = 93 + 0.99960996433179616, 1.95196280898670693, 2.87777936649376898, // k = 94 + 0.99961379137860717, 1.95219787713926962, 2.87834989933620022, // k = 95 + 0.99961756088146103, 1.95242944583677058, 2.87891216133900230, // k = 96 + 0.99962125605327401, 1.95265762420910960, 2.87946647367488140, // k = 97 + 0.99962486179100551, 1.95288245314810638, 2.88001290210658567, // k = 98 + 0.99962843240297161, 1.95310404286672679, 2.88055166523392359, // k = 99 + 0.99963187276145504, 1.95332251980147475, 2.88108300006589957, // k = 100 + 0.99963525453173929, 1.95353785898848287, 2.88160703591438505, // k = 101 + 0.99963855412988778, 1.95375019354571577, 2.88212393551896184, // k = 102 + 0.99964190254169694, 1.95395953472205974, 2.88263389761985422, // k = 103 + 0.99964506565942202, 1.95416607430155409, 2.88313700661564098, // k = 104 + 0.99964834424233118, 1.95436972855640079, 2.88363350163803034, // k = 105 + 0.99965136548857458, 1.95457068540693513, 2.88412349413960101, // k = 106 + 0.99965436594726498, 1.95476896383092935, 2.88460710620208260, // k = 107 + 0.99965736463468602, 1.95496457504532373, 2.88508450078833789, // k = 108 + 0.99966034130443404, 1.95515761150707590, 2.88555580586194083, // k = 109 + 0.99966326130828520, 1.95534810382198998, 2.88602118761679094, // k = 110 + 0.99966601446035952, 1.95553622237747504, 2.88648066384146773, // k = 111 + 0.99966887679593697, 1.95572186728168163, 2.88693444915907094, // k = 112 + 0.99967161286551232, 1.95590523410490391, 2.88738271495714116, // k = 113 + 0.99967435412270333, 1.95608626483223702, 2.88782540459769166, // k = 114 + 0.99967701261934394, 1.95626497627117146, 2.88826277189363623, // k = 115 + 0.99967963265157778, 1.95644153684824573, 2.88869486674335008, // k = 116 + 0.99968216317182623, 1.95661589936000269, 2.88912184353694101, // k = 117 + 0.99968479674396349, 1.95678821614791332, 2.88954376359643561, // k = 118 + 0.99968729031337489, 1.95695842061650183, 2.88996069422501023, // k = 119 + 0.99968963358631413, 1.95712651709766305, 2.89037285320668502 // k = 120 +}; + +class binomial_bounds { + +public: + static double get_lower_bound(unsigned long long num_samples, double theta, unsigned num_std_devs) { + check_theta(theta); + check_num_std_devs(num_std_devs); + const double estimate = num_samples / theta; + const double lb = compute_approx_binomial_lower_bound(num_samples, theta, num_std_devs); + return std::min(estimate, std::max(static_cast(num_samples), lb)); + } + + static double get_upper_bound(unsigned long long num_samples, double theta, unsigned num_std_devs) { + check_theta(theta); + check_num_std_devs(num_std_devs); + const double estimate = num_samples / theta; + const double ub = compute_approx_binomial_upper_bound(num_samples, theta, num_std_devs); + return std::max(estimate, ub); + } + +private: + // our "classic" bounds, but now with continuity correction + static double cont_classic_lb(unsigned long long num_samples, double theta, double num_std_devs) { + const double n_hat = (num_samples - 0.5) / theta; + const double b = num_std_devs * std::sqrt((1.0 - theta) / theta); + const double d = 0.5 * b * std::sqrt((b * b) + (4.0 * n_hat)); + const double center = n_hat + (0.5 * (b * b)); + return (center - d); + } + + // our "classic" bounds, but now with continuity correction + static double cont_classic_ub(unsigned long long num_samples, double theta, double num_std_devs) { + const double n_hat = (num_samples + 0.5) / theta; + const double b = num_std_devs * std::sqrt((1.0 - theta) / theta); + const double d = 0.5 * b * std::sqrt((b * b) + (4.0 * n_hat)); + const double center = n_hat + (0.5 * (b * b)); + return (center + d); + } + + // This is a special purpose calculator for NStar, using a computational + // strategy inspired by its Bayesian definition. It is only appropriate + // for a very limited set of inputs. However, the procedure compute_approx_binomial_lower_bound() + // below does in fact only call it for suitably limited inputs. + // Outside of this limited range, two different bad things will happen. + // First, because we are not using logarithms, the values of intermediate + // quantities will exceed the dynamic range of doubles. Second, even if that + // problem were fixed, the running time of this procedure is essentially linear + // in est = (numSamples / p), and that can be Very, Very Big. + static unsigned long long special_n_star(unsigned long long num_samples, double p, double delta) { + const double q = 1.0 - p; + // Use a different algorithm if the following is true; this one will be too slow, or worse. + if ((num_samples / p) >= 500.0) throw std::invalid_argument("out of range"); + double cur_term = std::pow(p, num_samples); // curTerm = posteriorProbability (k, k, p) + if (cur_term <= 1e-100) throw std::logic_error("out of range"); // sanity check for non-use of logarithms + double tot = cur_term; + unsigned long long m = num_samples; + while (tot <= delta) { // this test can fail even the first time + cur_term = (cur_term * q * (m)) / ((m + 1) - num_samples); + tot += cur_term; + m += 1; + } + // we have reached a state where tot > delta, so back up one + return (m - 1); + } + + // The following procedure has very limited applicability. + // The above remarks about special_n_star() also apply here. + static unsigned long long special_n_prime_b(unsigned long long num_samples, double p, double delta) { + const double q = 1.0 - p; + const double one_minus_delta = 1.0 - delta; + double cur_term = std::pow(p, num_samples); // curTerm = posteriorProbability (k, k, p) + if (cur_term <= 1e-100) throw std::logic_error("out of range"); // sanity check for non-use of logarithms + double tot = cur_term; + unsigned long long m = num_samples; + while (tot < one_minus_delta) { + cur_term = (cur_term * q * (m)) / ((m + 1) - num_samples); + tot += cur_term; + m += 1; + } + return m; // no need to back up + } + + static unsigned long long special_n_prime_f(unsigned long long num_samples, double p, double delta) { + // Use a different algorithm if the following is true; this one will be too slow, or worse. + if ((num_samples / p) >= 500.0) throw std::invalid_argument("out of range"); //A super-small delta could also make it slow. + return special_n_prime_b(num_samples + 1, p, delta); + } + + // The following computes an approximation to the lower bound of a Frequentist + // confidence interval based on the tails of the Binomial distribution. + static double compute_approx_binomial_lower_bound(unsigned long long num_samples, double theta, unsigned num_std_devs) { + if (theta == 1) return num_samples; + if (num_samples == 0) return 0; + if (num_samples == 1) { + const double delta = delta_of_num_std_devs[num_std_devs]; + const double raw_lb = std::log(1 - delta) / std::log(1 - theta); + return std::floor(raw_lb); // round down + } + if (num_samples > 120) { + // plenty of samples, so gaussian approximation to binomial distribution isn't too bad + const double raw_lb = cont_classic_lb(num_samples, theta, num_std_devs); + return (raw_lb - 0.5); // fake round down + } + // at this point we know 2 <= num_samples <= 120 + if (theta > (1 - 1e-5)) { // empirically-determined threshold + return num_samples; + } + if (theta < (num_samples / 360.0)) { // empirically-determined threshold + // here we use the Gaussian approximation, but with a modified num_std_devs + const unsigned index = 3 * num_samples + (num_std_devs - 1); + const double raw_lb = cont_classic_lb(num_samples, theta, lb_equiv_table[index]); + return raw_lb - 0.5; // fake round down + } + // This is the most difficult range to approximate; we will compute an "exact" LB. + // We know that est <= 360, so specialNStar() shouldn't be ridiculously slow. + const double delta = delta_of_num_std_devs[num_std_devs]; + return special_n_star(num_samples, theta, delta); // no need to round + } + + // The following computes an approximation to the upper bound of a Frequentist + // confidence interval based on the tails of the Binomial distribution. + static double compute_approx_binomial_upper_bound(unsigned long long num_samples, double theta, unsigned num_std_devs) { + if (theta == 1) return num_samples; + if (num_samples == 0) { + const double delta = delta_of_num_std_devs[num_std_devs]; + const double raw_ub = std::log(delta) / std::log(1 - theta); + return std::ceil(raw_ub); // round up + } + if (num_samples > 120) { + // plenty of samples, so gaussian approximation to binomial distribution isn't too bad + const double raw_ub = cont_classic_ub(num_samples, theta, num_std_devs); + return (raw_ub + 0.5); // fake round up + } + // at this point we know 2 <= num_samples <= 120 + if (theta > (1 - 1e-5)) { // empirically-determined threshold + return num_samples + 1; + } + if (theta < (num_samples / 360.0)) { // empirically-determined threshold + // here we use the Gaussian approximation, but with a modified num_std_devs + const unsigned index = 3 * num_samples + (num_std_devs - 1); + const double raw_ub = cont_classic_ub(num_samples, theta, ub_equiv_table[index]); + return raw_ub + 0.5; // fake round up + } + // This is the most difficult range to approximate; we will compute an "exact" UB. + // We know that est <= 360, so specialNPrimeF() shouldn't be ridiculously slow. + const double delta = delta_of_num_std_devs[num_std_devs]; + return special_n_prime_f(num_samples, theta, delta); // no need to round + } + + static void check_theta(double theta) { + if (theta < 0 or theta > 1) { + throw std::invalid_argument("theta must be in [0, 1]"); + } + } + + static void check_num_std_devs(unsigned num_std_devs) { + if (num_std_devs < 1 or num_std_devs > 3) { + throw std::invalid_argument("num_std_devs must be 1, 2 or 3"); + } + } + +}; + +} /* namespace datasketches */ + +# endif diff --git a/theta/include/theta_a_not_b.hpp b/theta/include/theta_a_not_b.hpp new file mode 100644 index 00000000..7a32adf5 --- /dev/null +++ b/theta/include/theta_a_not_b.hpp @@ -0,0 +1,43 @@ +/* + * Copyright 2019, Verizon Media. + * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms. + */ + +#ifndef THETA_A_NOT_B_HPP_ +#define THETA_A_NOT_B_HPP_ + +#include +#include +#include + +#include + +namespace datasketches { + +/* + * author Alexander Saydakov + * author Lee Rhodes + * author Kevin Lang + */ + +template +class theta_a_not_b_alloc { +public: + explicit theta_a_not_b_alloc(uint64_t seed = update_theta_sketch_alloc::builder::DEFAULT_SEED); + + compact_theta_sketch_alloc compute(const theta_sketch_alloc& a, const theta_sketch_alloc& b, bool ordered = true) const; + +private: + typedef typename std::allocator_traits::template rebind_alloc AllocU64; + uint16_t seed_hash_; + +}; + +// alias with default allocator for convenience +typedef theta_a_not_b_alloc> theta_a_not_b; + +} /* namespace datasketches */ + +#include "theta_a_not_b_impl.hpp" + +# endif diff --git a/theta/include/theta_a_not_b_impl.hpp b/theta/include/theta_a_not_b_impl.hpp new file mode 100644 index 00000000..a794b63e --- /dev/null +++ b/theta/include/theta_a_not_b_impl.hpp @@ -0,0 +1,104 @@ +/* + * Copyright 2019, Verizon Media. + * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms. + */ + +#ifndef THETA_A_NOT_B_IMPL_HPP_ +#define THETA_A_NOT_B_IMPL_HPP_ + +#include + +namespace datasketches { + +/* + * author Alexander Saydakov + * author Lee Rhodes + * author Kevin Lang + */ + +template +theta_a_not_b_alloc::theta_a_not_b_alloc(uint64_t seed): +seed_hash_(theta_sketch_alloc::get_seed_hash(seed)) +{} + +constexpr uint8_t log2(uint32_t n) { + return (n > 1) ? 1 + log2(n >> 1) : 0; +} + +constexpr uint8_t lg_size_from_count(uint32_t n, double load_factor) { + uint8_t lg = log2(n) + 1; + if (n > (1 << lg) * load_factor) lg++; + return lg; +} + +template +compact_theta_sketch_alloc theta_a_not_b_alloc::compute(const theta_sketch_alloc& a, const theta_sketch_alloc& b, bool ordered) const { + if (a.is_empty()) return compact_theta_sketch_alloc(a, ordered); + if (a.get_seed_hash() != seed_hash_) throw std::invalid_argument("A seed hash mismatch"); + if (b.get_seed_hash() != seed_hash_) throw std::invalid_argument("B seed hash mismatch"); + if (a.get_num_retained() == 0 or b.is_empty()) return compact_theta_sketch_alloc(a, ordered); + + const uint64_t theta = std::min(a.get_theta64(), b.get_theta64()); + uint64_t* keys = nullptr; + uint32_t keys_size = 0; + uint32_t count = 0; + bool is_empty = a.is_empty(); + + if (b.get_num_retained() == 0) { + for (auto key: a) if (key < theta) ++count; + keys_size = count; + keys = AllocU64().allocate(keys_size); + std::copy_if(a.begin(), a.end(), keys, [theta](uint64_t key) { return key < theta; }); + if (ordered and !a.is_ordered()) std::sort(keys, &keys[keys_size]); + if (count == 0 and theta == theta_sketch_alloc::MAX_THETA) is_empty = true; + return compact_theta_sketch_alloc(is_empty, theta, keys, count, seed_hash_, a.is_ordered() or ordered); + } + + keys_size = a.get_num_retained(); + keys = AllocU64().allocate(keys_size); + + if (a.is_ordered() and b.is_ordered()) { // sort-based + const auto end = std::set_difference(a.begin(), a.end(), b.begin(), b.end(), keys); + count = end - keys; + } else { // hash-based + const uint8_t lg_size = lg_size_from_count(b.get_num_retained(), update_theta_sketch_alloc::REBUILD_THRESHOLD); + uint64_t* b_hash_table = AllocU64().allocate(1 << lg_size); + std::fill(b_hash_table, &b_hash_table[1 << lg_size], 0); + for (auto key: b) { + if (key < theta) { + update_theta_sketch_alloc::hash_search_or_insert(key, b_hash_table, lg_size); + } else if (b.is_ordered()) { + break; // early stop + } + } + + // scan A lookup B + for (auto key: a) { + if (key < theta) { + if (!update_theta_sketch_alloc::hash_search(key, b_hash_table, lg_size)) keys[count++] = key; + } else if (a.is_ordered()) { + break; // early stop + } + } + + AllocU64().deallocate(b_hash_table, 1 << lg_size); + } + + if (count == 0) { + AllocU64().deallocate(keys, keys_size); + keys = nullptr; + if (theta == theta_sketch_alloc::MAX_THETA) is_empty = true; + } else if (count < keys_size) { + uint64_t* keys_copy = AllocU64().allocate(count); + std::copy(keys, &keys[count], keys_copy); + AllocU64().deallocate(keys, keys_size); + keys = keys_copy; + if (ordered and !a.is_ordered()) std::sort(keys, &keys[count]); + } + + return compact_theta_sketch_alloc(is_empty, theta, keys, count, seed_hash_, a.is_ordered() or ordered); +} + +} /* namespace datasketches */ + +# endif diff --git a/theta/include/theta_intersection.hpp b/theta/include/theta_intersection.hpp new file mode 100644 index 00000000..3273af46 --- /dev/null +++ b/theta/include/theta_intersection.hpp @@ -0,0 +1,56 @@ +/* + * Copyright 2019, Verizon Media. + * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms. + */ + +#ifndef THETA_INTERSECTION_HPP_ +#define THETA_INTERSECTION_HPP_ + +#include +#include +#include + +#include + +namespace datasketches { + +/* + * author Alexander Saydakov + * author Lee Rhodes + * author Kevin Lang + */ + +template +class theta_intersection_alloc { +public: + explicit theta_intersection_alloc(uint64_t seed = update_theta_sketch_alloc::builder::DEFAULT_SEED); + theta_intersection_alloc(const theta_intersection_alloc& other); + theta_intersection_alloc(theta_intersection_alloc&& other) noexcept; + ~theta_intersection_alloc(); + + theta_intersection_alloc& operator=(theta_intersection_alloc other); + theta_intersection_alloc& operator=(theta_intersection_alloc&& other); + + void update(const theta_sketch_alloc& sketch); + compact_theta_sketch_alloc get_result(bool ordered = true) const; + bool has_result() const; + +private: + typedef typename std::allocator_traits::template rebind_alloc AllocU64; + bool is_valid_; + bool is_empty_; + uint64_t theta_; + uint8_t lg_size_; + uint64_t* keys_; + uint32_t num_keys_; + uint16_t seed_hash_; +}; + +// alias with default allocator for convenience +typedef theta_intersection_alloc> theta_intersection; + +} /* namespace datasketches */ + +#include "theta_intersection_impl.hpp" + +# endif diff --git a/theta/include/theta_intersection_impl.hpp b/theta/include/theta_intersection_impl.hpp new file mode 100644 index 00000000..9174e65f --- /dev/null +++ b/theta/include/theta_intersection_impl.hpp @@ -0,0 +1,177 @@ +/* + * Copyright 2019, Verizon Media. + * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms. + */ + +#ifndef THETA_INTERSECTION_IMPL_HPP_ +#define THETA_INTERSECTION_IMPL_HPP_ + +#include + +namespace datasketches { + +/* + * author Alexander Saydakov + * author Lee Rhodes + * author Kevin Lang + */ + +template +theta_intersection_alloc::theta_intersection_alloc(uint64_t seed): +is_valid_(false), +is_empty_(false), +theta_(theta_sketch_alloc::MAX_THETA), +lg_size_(0), +keys_(nullptr), +num_keys_(0), +seed_hash_(theta_sketch_alloc::get_seed_hash(seed)) +{} + +template +theta_intersection_alloc::theta_intersection_alloc(const theta_intersection_alloc& other): +is_valid_(other.is_valid_), +is_empty_(other.is_empty_), +theta_(other.theta_), +lg_size_(other.lg_size_), +keys_(other.keys_ == nullptr ? nullptr : AllocU64().allocate(1 << lg_size_)), +num_keys_(other.num_keys_), +seed_hash_(other.seed_hash_) +{ + if (keys_ != nullptr) std::copy(other.keys_, &other.keys_[1 << lg_size_], keys_); +} + +template +theta_intersection_alloc::theta_intersection_alloc(theta_intersection_alloc&& other) noexcept: +is_valid_(false), +is_empty_(false), +theta_(theta_sketch_alloc::MAX_THETA), +lg_size_(0), +keys_(nullptr), +num_keys_(0), +seed_hash_(other.seed_hash_) +{ + std::swap(is_valid_, other.is_valid_); + std::swap(is_empty_, other.is_empty_); + std::swap(theta_, other.theta_); + std::swap(lg_size_, other.lg_size_); + std::swap(keys_, other.keys_); + std::swap(num_keys_, other.num_keys_); +} + +template +theta_intersection_alloc::~theta_intersection_alloc() { + if (keys_ != nullptr) { + AllocU64().deallocate(keys_, 1 << lg_size_); + } +} + +template +theta_intersection_alloc& theta_intersection_alloc::operator=(theta_intersection_alloc other) { + std::swap(is_valid_, other.is_valid_); + std::swap(is_empty_, other.is_empty_); + std::swap(theta_, other.theta_); + std::swap(lg_size_, other.lg_size_); + std::swap(keys_, other.keys_); + std::swap(num_keys_, other.num_keys_); + std::swap(seed_hash_, other.seed_hash_); + return *this; +} + +template +theta_intersection_alloc& theta_intersection_alloc::operator=(theta_intersection_alloc&& other) { + std::swap(is_valid_, other.is_valid_); + std::swap(is_empty_, other.is_empty_); + std::swap(theta_, other.theta_); + std::swap(lg_size_, other.lg_size_); + std::swap(keys_, other.keys_); + std::swap(num_keys_, other.num_keys_); + std::swap(seed_hash_, other.seed_hash_); + return *this; +} + +constexpr uint8_t log2(uint32_t n) { + return (n > 1) ? 1 + log2(n >> 1) : 0; +} + +constexpr uint8_t lg_size_from_count(uint32_t n, double load_factor) { + uint8_t lg = log2(n) + 1; + if (n > (1 << lg) * load_factor) lg++; + return lg; +} + +template +void theta_intersection_alloc::update(const theta_sketch_alloc& sketch) { + if (is_empty_) return; + if (sketch.get_seed_hash() != seed_hash_) throw std::invalid_argument("seed hash mismatch"); + is_empty_ |= sketch.is_empty(); + theta_ = std::min(theta_, sketch.get_theta64()); + if (is_valid_ and num_keys_ == 0) return; + if (sketch.get_num_retained() == 0) { + is_valid_ = true; + if (keys_ != nullptr) { + AllocU64().deallocate(keys_, 1 << lg_size_); + keys_ = nullptr; + lg_size_ = 0; + num_keys_ = 0; + } + return; + } + if (!is_valid_) { // first update, clone incoming sketch + is_valid_ = true; + lg_size_ = lg_size_from_count(sketch.get_num_retained(), update_theta_sketch_alloc::REBUILD_THRESHOLD); + keys_ = AllocU64().allocate(1 << lg_size_); + std::fill(keys_, &keys_[1 << lg_size_], 0); + num_keys_ = sketch.get_num_retained(); + for (auto key: sketch) update_theta_sketch_alloc::hash_search_or_insert(key, keys_, lg_size_); + } else { // intersection + const uint32_t max_matches = std::min(num_keys_, sketch.get_num_retained()); + uint64_t* matched_keys = AllocU64().allocate(max_matches); + uint32_t match_count = 0; + for (auto key: sketch) { + if (key < theta_) { + if (update_theta_sketch_alloc::hash_search(key, keys_, lg_size_)) matched_keys[match_count++] = key; + } else if (sketch.is_ordered()) { + break; // early stop + } + } + if (match_count == 0) { + AllocU64().deallocate(keys_, 1 << lg_size_); + keys_ = nullptr; + lg_size_ = 0; + num_keys_ = 0; + if (theta_ == theta_sketch_alloc::MAX_THETA) is_empty_ = true; + } else { + const uint8_t lg_size = lg_size_from_count(match_count, update_theta_sketch_alloc::REBUILD_THRESHOLD); + if (lg_size != lg_size_) { + AllocU64().deallocate(keys_, 1 << lg_size_); + lg_size_ = lg_size; + keys_ = AllocU64().allocate(1 << lg_size_); + std::fill(keys_, &keys_[1 << lg_size_], 0); + } + for (uint32_t i = 0; i < match_count; i++) { + update_theta_sketch_alloc::hash_search_or_insert(matched_keys[i], keys_, lg_size_); + } + num_keys_ = match_count; + } + AllocU64().deallocate(matched_keys, max_matches); + } +} + +template +compact_theta_sketch_alloc theta_intersection_alloc::get_result(bool ordered) const { + if (!is_valid_) throw std::invalid_argument("calling get_result() before calling update() is undefined"); + if (num_keys_ == 0) return compact_theta_sketch_alloc(is_empty_, theta_, nullptr, 0, seed_hash_, ordered); + uint64_t* keys = AllocU64().allocate(num_keys_); + std::copy_if(keys_, &keys_[1 << lg_size_], keys, [](uint64_t key) { return key != 0; }); + if (ordered) std::sort(keys, &keys[num_keys_]); + return compact_theta_sketch_alloc(false, this->theta_, keys, num_keys_, seed_hash_, ordered); +} + +template +bool theta_intersection_alloc::has_result() const { + return is_valid_; +} + +} /* namespace datasketches */ + +# endif diff --git a/theta/include/theta_sketch.hpp b/theta/include/theta_sketch.hpp new file mode 100644 index 00000000..3eab4961 --- /dev/null +++ b/theta/include/theta_sketch.hpp @@ -0,0 +1,281 @@ +/* + * Copyright 2019, Verizon Media. + * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms. + */ + +#ifndef THETA_SKETCH_HPP_ +#define THETA_SKETCH_HPP_ + +#include +#include +#include + +namespace datasketches { + +/* + * author Alexander Saydakov + * author Lee Rhodes + * author Kevin Lang + */ + +// forward-declarations +template class theta_sketch_alloc; +template class update_theta_sketch_alloc; +template class compact_theta_sketch_alloc; +template class theta_union_alloc; +template class theta_intersection_alloc; +template class theta_a_not_b_alloc; + +// for serialization as raw bytes +typedef std::unique_ptr> void_ptr_with_deleter; + +template +class theta_sketch_alloc { +public: + static const uint64_t MAX_THETA = LLONG_MAX; // signed max for compatibility with Java + static const uint8_t SERIAL_VERSION = 3; + + theta_sketch_alloc(bool is_empty, uint64_t theta); + theta_sketch_alloc(const theta_sketch_alloc& other); + theta_sketch_alloc(theta_sketch_alloc&& other) noexcept; + virtual ~theta_sketch_alloc(); + + theta_sketch_alloc& operator=(const theta_sketch_alloc& other); + theta_sketch_alloc& operator=(theta_sketch_alloc&& other); + + bool is_empty() const; + double get_estimate() const; + double get_lower_bound(uint8_t num_std_devs) const; + double get_upper_bound(uint8_t num_std_devs) const; + bool is_estimation_mode() const; + double get_theta() const; + uint64_t get_theta64() const; + + virtual uint32_t get_num_retained() const = 0; + virtual uint16_t get_seed_hash() const = 0; + virtual bool is_ordered() const = 0; + virtual void to_stream(std::ostream& os, bool print_items = false) const = 0; + virtual void serialize(std::ostream& os) const = 0; + virtual std::pair serialize(unsigned header_size_bytes = 0) const = 0; + + typedef std::unique_ptr, std::function*)>> unique_ptr; + static unique_ptr deserialize(std::istream& is, uint64_t seed = update_theta_sketch_alloc::builder::DEFAULT_SEED); + static unique_ptr deserialize(const void* bytes, size_t size, uint64_t seed = update_theta_sketch_alloc::builder::DEFAULT_SEED); + + class const_iterator; + virtual const_iterator begin() const = 0; + virtual const_iterator end() const = 0; + +protected: + enum flags { IS_BIG_ENDIAN, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED }; + + bool is_empty_; + uint64_t theta_; + + static uint16_t get_seed_hash(uint64_t seed); + + static void check_sketch_type(uint8_t actual, uint8_t expected); + static void check_serial_version(uint8_t actual, uint8_t expected); + static void check_seed_hash(uint16_t actual, uint16_t expected); + static void check_size(size_t actual, size_t expected); +}; + +// update sketch + +template +class update_theta_sketch_alloc: public theta_sketch_alloc { +public: + class builder; + enum resize_factor { X1, X2, X4, X8 }; + static const uint8_t SKETCH_TYPE = 2; + + update_theta_sketch_alloc(const update_theta_sketch_alloc& other); + update_theta_sketch_alloc(update_theta_sketch_alloc&& other) noexcept; + virtual ~update_theta_sketch_alloc(); + + update_theta_sketch_alloc& operator=(const update_theta_sketch_alloc& other); + update_theta_sketch_alloc& operator=(update_theta_sketch_alloc&& other); + + virtual uint32_t get_num_retained() const; + virtual uint16_t get_seed_hash() const; + virtual bool is_ordered() const; + virtual void to_stream(std::ostream& os, bool print_items = false) const; + virtual void serialize(std::ostream& os) const; + // header space is reserved, but not initialized + virtual std::pair serialize(unsigned header_size_bytes = 0) const; + + //void update(const std::string& value); + void update(uint64_t value); + void update(int64_t value); + + // for compatibility with Java implementation + void update(uint32_t value); + void update(int32_t value); + void update(uint16_t value); + void update(int16_t value); + void update(uint8_t value); + void update(int8_t value); + void update(double value); + void update(float value); + + // Be very careful to hash input values consistently using the same approach + // either over time or on different platforms + // or while passing sketches from Java environment or to Java environment + // Otherwise two sketches that should represent overlapping sets will be disjoint + // For instance, for signed 32-bit values call update(int32_t) method above, + // which does widening conversion to int64_t, if compatibility with Java is expected + void update(const void* data, unsigned length); + + compact_theta_sketch_alloc compact(bool ordered = true) const; + + virtual typename theta_sketch_alloc::const_iterator begin() const; + virtual typename theta_sketch_alloc::const_iterator end() const; + + static update_theta_sketch_alloc deserialize(std::istream& is, uint64_t seed = builder::DEFAULT_SEED); + static update_theta_sketch_alloc deserialize(const void* bytes, size_t size, uint64_t seed = update_theta_sketch_alloc::builder::DEFAULT_SEED); + +private: + // resize threshold = 0.5 tuned for speed + static constexpr double RESIZE_THRESHOLD = 0.5; + // hash table rebuild threshold = 15/16 + static constexpr double REBUILD_THRESHOLD = 15.0 / 16.0; + + static constexpr uint8_t STRIDE_HASH_BITS = 7; + static constexpr uint32_t STRIDE_MASK = (1 << STRIDE_HASH_BITS) - 1; + + uint8_t lg_cur_size_; + uint8_t lg_nom_size_; + uint64_t* keys_; + uint32_t num_keys_; + resize_factor rf_; + float p_; + uint64_t seed_; + uint32_t capacity_; + + typedef typename std::allocator_traits::template rebind_alloc AllocU64; + + // for builder + update_theta_sketch_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, float p, uint64_t seed); + // for deserialize + update_theta_sketch_alloc(bool is_empty, uint64_t theta, uint8_t lg_cur_size, uint8_t lg_nom_size, uint64_t* keys, uint32_t num_keys, resize_factor rf, float p, uint64_t seed); + + void resize(); + void rebuild(); + + friend theta_union_alloc; + void internal_update(uint64_t hash); + + friend theta_intersection_alloc; + friend theta_a_not_b_alloc; + static inline uint32_t get_capacity(uint8_t lg_cur_size, uint8_t lg_nom_size); + static inline uint32_t get_stride(uint64_t hash, uint8_t lg_size); + static bool hash_search_or_insert(uint64_t hash, uint64_t* table, uint8_t lg_size); + static bool hash_search(uint64_t hash, const uint64_t* table, uint8_t lg_size); + + friend theta_sketch_alloc; + static update_theta_sketch_alloc internal_deserialize(std::istream& is, resize_factor rf, uint8_t lg_nom_size, uint8_t lg_cur_size, uint8_t flags_byte, uint64_t seed); + static update_theta_sketch_alloc internal_deserialize(const void* bytes, size_t size, resize_factor rf, uint8_t lg_nom_size, uint8_t lg_cur_size, uint8_t flags_byte, uint64_t seed); +}; + +// compact sketch + +template +class compact_theta_sketch_alloc: public theta_sketch_alloc { +public: + static const uint8_t SKETCH_TYPE = 3; + + compact_theta_sketch_alloc(const compact_theta_sketch_alloc& other); + compact_theta_sketch_alloc(const theta_sketch_alloc& other, bool ordered); + compact_theta_sketch_alloc(compact_theta_sketch_alloc&& other) noexcept; + virtual ~compact_theta_sketch_alloc(); + + compact_theta_sketch_alloc& operator=(const compact_theta_sketch_alloc& other); + compact_theta_sketch_alloc& operator=(compact_theta_sketch_alloc&& other); + + virtual uint32_t get_num_retained() const; + virtual uint16_t get_seed_hash() const; + virtual bool is_ordered() const; + virtual void to_stream(std::ostream& os, bool print_items = false) const; + virtual void serialize(std::ostream& os) const; + // header space is reserved, but not initialized + virtual std::pair serialize(unsigned header_size_bytes = 0) const; + + virtual typename theta_sketch_alloc::const_iterator begin() const; + virtual typename theta_sketch_alloc::const_iterator end() const; + + static compact_theta_sketch_alloc deserialize(std::istream& is, uint64_t seed = update_theta_sketch_alloc::builder::DEFAULT_SEED); + static compact_theta_sketch_alloc deserialize(const void* bytes, size_t size, uint64_t seed = update_theta_sketch_alloc::builder::DEFAULT_SEED); + +private: + typedef typename std::allocator_traits::template rebind_alloc AllocU64; + + uint64_t* keys_; + uint32_t num_keys_; + uint16_t seed_hash_; + bool is_ordered_; + + friend theta_sketch_alloc; + friend update_theta_sketch_alloc; + friend theta_union_alloc; + friend theta_intersection_alloc; + friend theta_a_not_b_alloc; + compact_theta_sketch_alloc(bool is_empty, uint64_t theta, uint64_t* keys, uint32_t num_keys, uint16_t seed_hash, bool is_ordered); + static compact_theta_sketch_alloc internal_deserialize(std::istream& is, uint8_t preamble_longs, uint8_t flags_byte, uint16_t seed_hash); + static compact_theta_sketch_alloc internal_deserialize(const void* bytes, size_t size, uint8_t preamble_longs, uint8_t flags_byte, uint16_t seed_hash); +}; + +// builder + +template +class update_theta_sketch_alloc::builder { +public: + static const uint8_t MIN_LG_K = 5; + static const uint8_t DEFAULT_LG_K = 12; + static const resize_factor DEFAULT_RESIZE_FACTOR = X8; + static const uint64_t DEFAULT_SEED = 9001; + + builder(); + builder& set_lg_k(uint8_t lg_k); + builder& set_resize_factor(resize_factor rf); + builder& set_p(float p); + builder& set_seed(uint64_t seed); + update_theta_sketch_alloc build() const; +private: + uint8_t lg_k_; + resize_factor rf_; + float p_; + uint64_t seed_; + + static uint8_t starting_sub_multiple(uint8_t lg_tgt, uint8_t lg_min, uint8_t lg_rf); +}; + +// iterator +template +class theta_sketch_alloc::const_iterator: public std::iterator { +public: + const_iterator& operator++(); + const_iterator operator++(int); + bool operator==(const const_iterator& other) const; + bool operator!=(const const_iterator& other) const; + uint64_t operator*() const; + +private: + const uint64_t* keys_; + uint32_t size_; + uint32_t index_; + const_iterator(const uint64_t* keys, uint32_t size, uint32_t index); + friend class update_theta_sketch_alloc; + friend class compact_theta_sketch_alloc; +}; + + +// aliases with default allocator for convenience +typedef theta_sketch_alloc> theta_sketch; +typedef update_theta_sketch_alloc> update_theta_sketch; +typedef compact_theta_sketch_alloc> compact_theta_sketch; + +} /* namespace datasketches */ + +#include "theta_sketch_impl.hpp" + +# endif diff --git a/theta/include/theta_sketch_impl.hpp b/theta/include/theta_sketch_impl.hpp new file mode 100644 index 00000000..3a45e1d6 --- /dev/null +++ b/theta/include/theta_sketch_impl.hpp @@ -0,0 +1,1072 @@ +/* + * Copyright 2019, Verizon Media. + * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms. + */ + +#ifndef THETA_SKETCH_IMPL_HPP_ +#define THETA_SKETCH_IMPL_HPP_ + +#include +#include +#include +#include + +#include "MurmurHash3.h" +#include "serde.hpp" +#include "binomial_bounds.hpp" + +namespace datasketches { + +/* + * author Alexander Saydakov + * author Lee Rhodes + * author Kevin Lang + */ + +template +theta_sketch_alloc::theta_sketch_alloc(bool is_empty, uint64_t theta): +is_empty_(is_empty), theta_(theta) +{} + +template +theta_sketch_alloc::theta_sketch_alloc(const theta_sketch_alloc& other): +is_empty_(other.is_empty_), theta_(other.theta_) +{} + +template +theta_sketch_alloc::theta_sketch_alloc(theta_sketch_alloc&& other) noexcept: +is_empty_(other.is_empty_), theta_(other.theta_) +{} + +template +theta_sketch_alloc& theta_sketch_alloc::operator=(const theta_sketch_alloc& other) { + is_empty_ = other.is_empty_; + theta_ = other.theta_; + return *this; +} + +template +theta_sketch_alloc& theta_sketch_alloc::operator=(theta_sketch_alloc&& other) { + std::swap(is_empty_, other.is_empty_); + std::swap(theta_, other.theta_); + return *this; +} + +template +theta_sketch_alloc::~theta_sketch_alloc() { +} + +template +bool theta_sketch_alloc::is_empty() const { + return is_empty_; +} + +template +double theta_sketch_alloc::get_estimate() const { + return get_num_retained() / get_theta(); +} + +template +double theta_sketch_alloc::get_lower_bound(uint8_t num_std_devs) const { + if (!is_estimation_mode()) return get_num_retained(); + return binomial_bounds::get_lower_bound(get_num_retained(), get_theta(), num_std_devs); +} + +template +double theta_sketch_alloc::get_upper_bound(uint8_t num_std_devs) const { + if (!is_estimation_mode()) return get_num_retained(); + return binomial_bounds::get_upper_bound(get_num_retained(), get_theta(), num_std_devs); +} + +template +bool theta_sketch_alloc::is_estimation_mode() const { + return theta_ < MAX_THETA and !is_empty_; +} + +template +double theta_sketch_alloc::get_theta() const { + return (double) theta_ / MAX_THETA; +} + +template +uint64_t theta_sketch_alloc::get_theta64() const { + return theta_; +} + +template +typename theta_sketch_alloc::unique_ptr theta_sketch_alloc::deserialize(std::istream& is, uint64_t seed) { + uint8_t preamble_longs; + is.read((char*)&preamble_longs, sizeof(preamble_longs)); + uint8_t serial_version; + is.read((char*)&serial_version, sizeof(serial_version)); + uint8_t type; + is.read((char*)&type, sizeof(type)); + uint8_t lg_nom_size; + is.read((char*)&lg_nom_size, sizeof(lg_nom_size)); + uint8_t lg_cur_size; + is.read((char*)&lg_cur_size, sizeof(lg_cur_size)); + uint8_t flags_byte; + is.read((char*)&flags_byte, sizeof(flags_byte)); + uint16_t seed_hash; + is.read((char*)&seed_hash, sizeof(seed_hash)); + + check_serial_version(serial_version, SERIAL_VERSION); + check_seed_hash(seed_hash, get_seed_hash(seed)); + + if (type == update_theta_sketch_alloc::SKETCH_TYPE) { + typename update_theta_sketch_alloc::resize_factor rf = static_cast::resize_factor>(preamble_longs >> 6); + typedef typename std::allocator_traits::template rebind_alloc> AU; + return unique_ptr( + static_cast*>(new (AU().allocate(1)) update_theta_sketch_alloc(update_theta_sketch_alloc::internal_deserialize(is, rf, lg_nom_size, lg_cur_size, flags_byte, seed))), + [](theta_sketch_alloc* ptr) { + ptr->~theta_sketch_alloc(); + AU().deallocate(static_cast*>(ptr), 1); + } + ); + } else if (type == compact_theta_sketch_alloc::SKETCH_TYPE) { + typedef typename std::allocator_traits::template rebind_alloc> AC; + return unique_ptr( + static_cast*>(new (AC().allocate(1)) compact_theta_sketch_alloc(compact_theta_sketch_alloc::internal_deserialize(is, preamble_longs, flags_byte, seed_hash))), + [](theta_sketch_alloc* ptr) { + ptr->~theta_sketch_alloc(); + AC().deallocate(static_cast*>(ptr), 1); + } + ); + } + throw std::invalid_argument("unsupported sketch type " + std::to_string((int) type)); +} + +template +typename theta_sketch_alloc::unique_ptr theta_sketch_alloc::deserialize(const void* bytes, size_t size, uint64_t seed) { + check_size(size, static_cast(8)); + const char* ptr = static_cast(bytes); + uint8_t preamble_longs; + copy_from_mem(&ptr, &preamble_longs, sizeof(preamble_longs)); + uint8_t serial_version; + copy_from_mem(&ptr, &serial_version, sizeof(serial_version)); + uint8_t type; + copy_from_mem(&ptr, &type, sizeof(type)); + uint8_t lg_nom_size; + copy_from_mem(&ptr, &lg_nom_size, sizeof(lg_nom_size)); + uint8_t lg_cur_size; + copy_from_mem(&ptr, &lg_cur_size, sizeof(lg_cur_size)); + uint8_t flags_byte; + copy_from_mem(&ptr, &flags_byte, sizeof(flags_byte)); + uint16_t seed_hash; + copy_from_mem(&ptr, &seed_hash, sizeof(seed_hash)); + + check_serial_version(serial_version, SERIAL_VERSION); + check_seed_hash(seed_hash, get_seed_hash(seed)); + + if (type == update_theta_sketch_alloc::SKETCH_TYPE) { + typename update_theta_sketch_alloc::resize_factor rf = static_cast::resize_factor>(preamble_longs >> 6); + typedef typename std::allocator_traits::template rebind_alloc> AU; + return unique_ptr( + static_cast*>(new (AU().allocate(1)) update_theta_sketch_alloc( + update_theta_sketch_alloc::internal_deserialize(ptr, size - (ptr - static_cast(bytes)), rf, lg_nom_size, lg_cur_size, flags_byte, seed)) + ), + [](theta_sketch_alloc* ptr) { + ptr->~theta_sketch_alloc(); + AU().deallocate(static_cast*>(ptr), 1); + } + ); + } else if (type == compact_theta_sketch_alloc::SKETCH_TYPE) { + typedef typename std::allocator_traits::template rebind_alloc> AC; + return unique_ptr( + static_cast*>(new (AC().allocate(1)) compact_theta_sketch_alloc( + compact_theta_sketch_alloc::internal_deserialize(ptr, size - (ptr - static_cast(bytes)), preamble_longs, flags_byte, seed_hash)) + ), + [](theta_sketch_alloc* ptr) { + ptr->~theta_sketch_alloc(); + AC().deallocate(static_cast*>(ptr), 1); + } + ); + } + throw std::invalid_argument("unsupported sketch type " + std::to_string((int) type)); +} + +template +uint16_t theta_sketch_alloc::get_seed_hash(uint64_t seed) { + HashState hashes; + MurmurHash3_x64_128(&seed, sizeof(seed), 0, hashes); + return hashes.h1; +} + +template +void theta_sketch_alloc::check_sketch_type(uint8_t actual, uint8_t expected) { + if (actual != expected) { + throw std::invalid_argument("Sketch type mismatch: expected " + std::to_string((int)expected) + ", actual " + std::to_string((int)actual)); + } +} + +template +void theta_sketch_alloc::check_serial_version(uint8_t actual, uint8_t expected) { + if (actual != expected) { + throw std::invalid_argument("Sketch serial version mismatch: expected " + std::to_string((int)expected) + ", actual " + std::to_string((int)actual)); + } +} + +template +void theta_sketch_alloc::check_seed_hash(uint16_t actual, uint16_t expected) { + if (actual != expected) { + throw std::invalid_argument("Sketch seed hash mismatch: expected " + std::to_string(expected) + ", actual " + std::to_string(actual)); + } +} + +template +void theta_sketch_alloc::check_size(size_t actual, size_t expected) { + if (actual < expected) { + throw std::invalid_argument("Given memory is smaller than expected: expected " + std::to_string((int)expected) + ", actual " + std::to_string((int) actual)); + } +} + +// update sketch + +template +update_theta_sketch_alloc::update_theta_sketch_alloc(uint8_t lg_cur_size, uint8_t lg_nom_size, resize_factor rf, float p, uint64_t seed): +theta_sketch_alloc(true, theta_sketch_alloc::MAX_THETA), +lg_cur_size_(lg_cur_size), +lg_nom_size_(lg_nom_size), +keys_(AllocU64().allocate(1 << lg_cur_size_)), +num_keys_(0), +rf_(rf), +p_(p), +seed_(seed), +capacity_(get_capacity(lg_cur_size, lg_nom_size)) +{ + if (p < 1) this->theta_ *= p; + std::fill(keys_, &keys_[1 << lg_cur_size_], 0); +} + +template +update_theta_sketch_alloc::update_theta_sketch_alloc(bool is_empty, uint64_t theta, uint8_t lg_cur_size, uint8_t lg_nom_size, uint64_t* keys, uint32_t num_keys, resize_factor rf, float p, uint64_t seed): +theta_sketch_alloc(is_empty, theta), +lg_cur_size_(lg_cur_size), +lg_nom_size_(lg_nom_size), +keys_(keys), +num_keys_(num_keys), +rf_(rf), +p_(p), +seed_(seed), +capacity_(get_capacity(lg_cur_size, lg_nom_size)) +{} + +template +update_theta_sketch_alloc::update_theta_sketch_alloc(const update_theta_sketch_alloc& other): +theta_sketch_alloc(other), +lg_cur_size_(other.lg_cur_size_), +lg_nom_size_(other.lg_nom_size_), +keys_(AllocU64().allocate(1 << lg_cur_size_)), +num_keys_(other.num_keys_), +rf_(other.rf_), +p_(other.p_), +seed_(other.seed_), +capacity_(other.capacity_) +{ + std::copy(other.keys_, &other.keys_[1 << lg_cur_size_], keys_); +} + +template +update_theta_sketch_alloc::update_theta_sketch_alloc(update_theta_sketch_alloc&& other) noexcept: +theta_sketch_alloc(std::move(other)), +lg_cur_size_(other.lg_cur_size_), +lg_nom_size_(other.lg_nom_size_), +keys_(nullptr), +num_keys_(other.num_keys_), +rf_(other.rf_), +p_(other.p_), +seed_(other.seed_), +capacity_(other.capacity_) +{ + std::swap(keys_, other.keys_); +} + +template +update_theta_sketch_alloc::~update_theta_sketch_alloc() { + AllocU64().deallocate(keys_, 1 << lg_cur_size_); +} + +template +update_theta_sketch_alloc& update_theta_sketch_alloc::operator=(const update_theta_sketch_alloc& other) { + theta_sketch_alloc::operator=(other); + if (lg_cur_size_ != other.lg_cur_size_) { + AllocU64().deallocate(keys_, 1 << lg_cur_size_); + lg_cur_size_ = other.lg_cur_size_; + keys_ = AllocU64().allocate(1 << lg_cur_size_); + } + lg_nom_size_ = other.lg_nom_size_; + std::copy(other.keys_, &other.keys_[1 << lg_cur_size_], keys_); + num_keys_ = other.num_keys_; + rf_ = other.rf_; + p_ = other.p_; + seed_ = other.seed_; + capacity_ = other.capacity_; + return *this; +} + +template +update_theta_sketch_alloc& update_theta_sketch_alloc::operator=(update_theta_sketch_alloc&& other) { + theta_sketch_alloc::operator=(std::move(other)); + std::swap(lg_cur_size_, other.lg_cur_size_); + lg_nom_size_ = other.lg_nom_size_; + std::swap(keys_, other.keys_); + num_keys_ = other.num_keys_; + rf_ = other.rf_; + p_ = other.p_; + seed_ = other.seed_; + capacity_ = other.capacity_; + return *this; +} + +template +uint32_t update_theta_sketch_alloc::get_num_retained() const { + return num_keys_; +} + +template +uint16_t update_theta_sketch_alloc::get_seed_hash() const { + return theta_sketch_alloc::get_seed_hash(seed_); +} + +template +bool update_theta_sketch_alloc::is_ordered() const { + return false; +} + +template +void update_theta_sketch_alloc::to_stream(std::ostream& os, bool print_items) const { + os << "### Update Theta sketch summary:" << std::endl; + os << " lg nominal size : " << (int) lg_nom_size_ << std::endl; + os << " lg current size : " << (int) lg_cur_size_ << std::endl; + os << " num retained keys : " << num_keys_ << std::endl; + os << " resize factor : " << (1 << rf_) << std::endl; + os << " sampling probability : " << p_ << std::endl; + os << " seed hash : " << this->get_seed_hash() << std::endl; + os << " ordered? : " << (this->is_ordered() ? "true" : "false") << std::endl; + os << " theta (fraction) : " << this->get_theta() << std::endl; + os << " theta (raw 64-bit) : " << this->theta_ << std::endl; + os << " estimation mode? : " << (this->is_estimation_mode() ? "true" : "false") << std::endl; + os << " estimate : " << this->get_estimate() << std::endl; + os << " lower bound 95% conf : " << this->get_lower_bound(2) << std::endl; + os << " upper bound 95% conf : " << this->get_upper_bound(2) << std::endl; + os << "### End sketch summary" << std::endl; + if (print_items) { + os << "### Retained keys" << std::endl; + for (auto key: *this) os << " " << key << std::endl; + os << "### End retained keys" << std::endl; + } +} + +template +void update_theta_sketch_alloc::serialize(std::ostream& os) const { + const uint8_t preamble_longs_and_rf = 3 | (rf_ << 6); + os.write((char*)&preamble_longs_and_rf, sizeof(preamble_longs_and_rf)); + const uint8_t serial_version = theta_sketch_alloc::SERIAL_VERSION; + os.write((char*)&serial_version, sizeof(serial_version)); + const uint8_t type = SKETCH_TYPE; + os.write((char*)&type, sizeof(type)); + os.write((char*)&lg_nom_size_, sizeof(lg_nom_size_)); + os.write((char*)&lg_cur_size_, sizeof(lg_cur_size_)); + const uint8_t flags_byte( + (this->is_empty() ? 1 << theta_sketch_alloc::flags::IS_EMPTY : 0) + ); + os.write((char*)&flags_byte, sizeof(flags_byte)); + const uint16_t seed_hash = get_seed_hash(); + os.write((char*)&seed_hash, sizeof(seed_hash)); + os.write((char*)&num_keys_, sizeof(num_keys_)); + os.write((char*)&p_, sizeof(p_)); + os.write((char*)&(this->theta_), sizeof(uint64_t)); + os.write((char*)keys_, sizeof(uint64_t) * (1 << lg_cur_size_)); +} + +template +std::pair update_theta_sketch_alloc::serialize(unsigned header_size_bytes) const { + const uint8_t preamble_longs = 3; + const size_t size = header_size_bytes + sizeof(uint64_t) * preamble_longs + sizeof(uint64_t) * (1 << lg_cur_size_); + typedef typename std::allocator_traits::template rebind_alloc AllocChar; + void_ptr_with_deleter data_ptr( + static_cast(AllocChar().allocate(size)), + [size](void* ptr) { AllocChar().deallocate(static_cast(ptr), size); } + ); + char* ptr = static_cast(data_ptr.get()) + header_size_bytes; + + const uint8_t preamble_longs_and_rf = preamble_longs | (rf_ << 6); + copy_to_mem(&preamble_longs_and_rf, &ptr, sizeof(preamble_longs_and_rf)); + const uint8_t serial_version = theta_sketch_alloc::SERIAL_VERSION; + copy_to_mem(&serial_version, &ptr, sizeof(serial_version)); + const uint8_t type = SKETCH_TYPE; + copy_to_mem(&type, &ptr, sizeof(type)); + copy_to_mem(&lg_nom_size_, &ptr, sizeof(lg_nom_size_)); + copy_to_mem(&lg_cur_size_, &ptr, sizeof(lg_cur_size_)); + const uint8_t flags_byte( + (this->is_empty() ? 1 << theta_sketch_alloc::flags::IS_EMPTY : 0) + ); + copy_to_mem(&flags_byte, &ptr, sizeof(flags_byte)); + const uint16_t seed_hash = get_seed_hash(); + copy_to_mem(&seed_hash, &ptr, sizeof(seed_hash)); + copy_to_mem(&num_keys_, &ptr, sizeof(num_keys_)); + copy_to_mem(&p_, &ptr, sizeof(p_)); + copy_to_mem(&(this->theta_), &ptr, sizeof(uint64_t)); + copy_to_mem(keys_, &ptr, sizeof(uint64_t) * (1 << lg_cur_size_)); + + return std::make_pair(std::move(data_ptr), size);; +} + +template +update_theta_sketch_alloc update_theta_sketch_alloc::deserialize(std::istream& is, uint64_t seed) { + uint8_t preamble_longs; + is.read((char*)&preamble_longs, sizeof(preamble_longs)); + resize_factor rf = static_cast(preamble_longs >> 6); + preamble_longs &= 0x3f; // remove resize factor + uint8_t serial_version; + is.read((char*)&serial_version, sizeof(serial_version)); + uint8_t type; + is.read((char*)&type, sizeof(type)); + uint8_t lg_nom_size; + is.read((char*)&lg_nom_size, sizeof(lg_nom_size)); + uint8_t lg_cur_size; + is.read((char*)&lg_cur_size, sizeof(lg_cur_size)); + uint8_t flags_byte; + is.read((char*)&flags_byte, sizeof(flags_byte)); + uint16_t seed_hash; + is.read((char*)&seed_hash, sizeof(seed_hash)); + theta_sketch_alloc::check_sketch_type(type, SKETCH_TYPE); + theta_sketch_alloc::check_serial_version(serial_version, theta_sketch_alloc::SERIAL_VERSION); + theta_sketch_alloc::check_seed_hash(seed_hash, theta_sketch_alloc::get_seed_hash(seed)); + return internal_deserialize(is, rf, lg_nom_size, lg_cur_size, flags_byte, seed); +} + +template +update_theta_sketch_alloc update_theta_sketch_alloc::internal_deserialize(std::istream& is, resize_factor rf, uint8_t lg_nom_size, uint8_t lg_cur_size, uint8_t flags_byte, uint64_t seed) { + uint32_t num_keys; + is.read((char*)&num_keys, sizeof(num_keys)); + float p; + is.read((char*)&p, sizeof(p)); + uint64_t theta; + is.read((char*)&theta, sizeof(theta)); + uint64_t* keys = AllocU64().allocate(1 << lg_cur_size); + is.read((char*)keys, sizeof(uint64_t) * (1 << lg_cur_size)); + const bool is_empty = flags_byte & (1 << theta_sketch_alloc::flags::IS_EMPTY); + return update_theta_sketch_alloc(is_empty, theta, lg_nom_size, lg_cur_size, keys, num_keys, rf, p, seed); +} + +template +update_theta_sketch_alloc update_theta_sketch_alloc::deserialize(const void* bytes, size_t size, uint64_t seed) { + theta_sketch_alloc::check_size(size, 8); + const char* ptr = static_cast(bytes); + uint8_t preamble_longs; + copy_from_mem(&ptr, &preamble_longs, sizeof(preamble_longs)); + resize_factor rf = static_cast(preamble_longs >> 6); + preamble_longs &= 0x3f; // remove resize factor + uint8_t serial_version; + copy_from_mem(&ptr, &serial_version, sizeof(serial_version)); + uint8_t type; + copy_from_mem(&ptr, &type, sizeof(type)); + uint8_t lg_nom_size; + copy_from_mem(&ptr, &lg_nom_size, sizeof(lg_nom_size)); + uint8_t lg_cur_size; + copy_from_mem(&ptr, &lg_cur_size, sizeof(lg_cur_size)); + uint8_t flags_byte; + copy_from_mem(&ptr, &flags_byte, sizeof(flags_byte)); + uint16_t seed_hash; + copy_from_mem(&ptr, &seed_hash, sizeof(seed_hash)); + theta_sketch_alloc::check_sketch_type(type, SKETCH_TYPE); + theta_sketch_alloc::check_serial_version(serial_version, theta_sketch_alloc::SERIAL_VERSION); + theta_sketch_alloc::check_seed_hash(seed_hash, theta_sketch_alloc::get_seed_hash(seed)); + return internal_deserialize(ptr, size - (ptr - static_cast(bytes)), rf, lg_nom_size, lg_cur_size, flags_byte, seed); +} + +template +update_theta_sketch_alloc update_theta_sketch_alloc::internal_deserialize(const void* bytes, size_t size, resize_factor rf, uint8_t lg_nom_size, uint8_t lg_cur_size, uint8_t flags_byte, uint64_t seed) { + const uint32_t table_size = 1 << lg_cur_size; + theta_sketch_alloc::check_size(size, 16 + sizeof(uint64_t) * table_size); + const char* ptr = static_cast(bytes); + uint32_t num_keys; + copy_from_mem(&ptr, &num_keys, sizeof(num_keys)); + float p; + copy_from_mem(&ptr, &p, sizeof(p)); + uint64_t theta; + copy_from_mem(&ptr, &theta, sizeof(theta)); + uint64_t* keys = AllocU64().allocate(table_size); + copy_from_mem(&ptr, keys, sizeof(uint64_t) * table_size); + const bool is_empty = flags_byte & (1 << theta_sketch_alloc::flags::IS_EMPTY); + return update_theta_sketch_alloc(is_empty, theta, lg_nom_size, lg_cur_size, keys, num_keys, rf, p, seed); +} + +template +void update_theta_sketch_alloc::update(uint64_t value) { + update(&value, sizeof(value)); +} + +template +void update_theta_sketch_alloc::update(int64_t value) { + update(&value, sizeof(value)); +} + +template +void update_theta_sketch_alloc::update(uint32_t value) { + update(static_cast(value)); +} + +template +void update_theta_sketch_alloc::update(int32_t value) { + update(static_cast(value)); +} + +template +void update_theta_sketch_alloc::update(uint16_t value) { + update(static_cast(value)); +} + +template +void update_theta_sketch_alloc::update(int16_t value) { + update(static_cast(value)); +} + +template +void update_theta_sketch_alloc::update(uint8_t value) { + update(static_cast(value)); +} + +template +void update_theta_sketch_alloc::update(int8_t value) { + update(static_cast(value)); +} + +template +void update_theta_sketch_alloc::update(double value) { + union { + int64_t long_value; + double double_value; + } long_double_union; + + if (value == 0.0) { + long_double_union.double_value = 0.0; // canonicalize -0.0 to 0.0 + } else if (std::isnan(value)) { + long_double_union.long_value = 0x7ff8000000000000L; // canonicalize NaN using value from Java's Double.doubleToLongBits() + } else { + long_double_union.double_value = value; + } + update(&long_double_union, sizeof(long_double_union)); +} + +template +void update_theta_sketch_alloc::update(float value) { + update(static_cast(value)); +} + +template +void update_theta_sketch_alloc::update(const void* data, unsigned length) { + HashState hashes; + MurmurHash3_x64_128(data, length, seed_, hashes); + const uint64_t hash = hashes.h1 >> 1; // Java implementation does logical shift >>> to make values positive + internal_update(hash); +} + +template +compact_theta_sketch_alloc update_theta_sketch_alloc::compact(bool ordered) const { + return compact_theta_sketch_alloc(*this, ordered); +} + +template +void update_theta_sketch_alloc::internal_update(uint64_t hash) { + this->is_empty_ = false; + if (hash >= this->theta_ or hash == 0) return; // hash == 0 is reserved to mark empty slots in the table + if (hash_search_or_insert(hash, keys_, lg_cur_size_)) { + num_keys_++; + if (num_keys_ > capacity_) { + if (lg_cur_size_ <= lg_nom_size_) { + resize(); + } else { + rebuild(); + } + } + } +} + +template +void update_theta_sketch_alloc::resize() { + const uint32_t cur_size = 1 << lg_cur_size_; + const uint8_t lg_tgt_size = lg_nom_size_ + 1; + const uint8_t factor = std::max(1, std::min(static_cast(rf_), lg_tgt_size - lg_cur_size_)); + const uint8_t lg_new_size = lg_cur_size_ + factor; + const uint32_t new_size = 1 << lg_new_size; + uint64_t* new_keys = AllocU64().allocate(new_size); + std::fill(new_keys, &new_keys[new_size], 0); + for (uint32_t i = 0; i < cur_size; i++) { + if (keys_[i] != 0) { + hash_search_or_insert(keys_[i], new_keys, lg_new_size); // TODO hash_insert + } + } + AllocU64().deallocate(keys_, cur_size); + keys_ = new_keys; + lg_cur_size_ += factor; + capacity_ = get_capacity(lg_cur_size_, lg_nom_size_); +} + +template +void update_theta_sketch_alloc::rebuild() { + const uint32_t cur_size = 1 << lg_cur_size_; + const uint32_t pivot = (1 << lg_nom_size_) + cur_size - num_keys_; + std::nth_element(&keys_[0], &keys_[pivot], &keys_[cur_size]); + this->theta_ = keys_[pivot]; + uint64_t* new_keys = AllocU64().allocate(cur_size); + std::fill(new_keys, &new_keys[cur_size], 0); + num_keys_ = 0; + for (uint32_t i = 0; i < cur_size; i++) { + if (keys_[i] != 0 and keys_[i] < this->theta_) { + hash_search_or_insert(keys_[i], new_keys, lg_cur_size_); // TODO hash_insert + num_keys_++; + } + } + AllocU64().deallocate(keys_, cur_size); + keys_ = new_keys; +} + +template +uint32_t update_theta_sketch_alloc::get_capacity(uint8_t lg_cur_size, uint8_t lg_nom_size) { + const double fraction = (lg_cur_size <= lg_nom_size) ? RESIZE_THRESHOLD : REBUILD_THRESHOLD; + return std::floor(fraction * (1 << lg_cur_size)); +} + +template +uint32_t update_theta_sketch_alloc::get_stride(uint64_t hash, uint8_t lg_size) { + // odd and independent of index assuming lg_size lowest bits of the hash were used for the index + return (2 * static_cast((hash >> lg_size) & STRIDE_MASK)) + 1; +} + +template +bool update_theta_sketch_alloc::hash_search_or_insert(uint64_t hash, uint64_t* table, uint8_t lg_size) { + const uint32_t mask = (1 << lg_size) - 1; + const uint32_t stride = get_stride(hash, lg_size); + uint32_t cur_probe = static_cast(hash) & mask; + + // search for duplicate or zero + const uint32_t loop_index = cur_probe; + do { + const uint64_t value = table[cur_probe]; + if (value == 0) { + table[cur_probe] = hash; // insert value + return true; + } else if (value == hash) { + return false; // found a duplicate + } + cur_probe = (cur_probe + stride) & mask; + } while (cur_probe != loop_index); + //std::cerr << "hash_search_or_insert: lg=" << (int)lg_size << ", hash=" << hash << std::endl; + //std::cerr << "cur_probe=" << cur_probe << ", stride=" << stride << std::endl; + throw std::logic_error("key not found and no empty slots!"); +} + +template +bool update_theta_sketch_alloc::hash_search(uint64_t hash, const uint64_t* table, uint8_t lg_size) { + const uint32_t mask = (1 << lg_size) - 1; + const uint32_t stride = update_theta_sketch_alloc::get_stride(hash, lg_size); + uint32_t cur_probe = static_cast(hash) & mask; + const uint32_t loop_index = cur_probe; + do { + const uint64_t value = table[cur_probe]; + if (value == 0) { + return false; + } else if (value == hash) { + return true; + } + cur_probe = (cur_probe + stride) & mask; + } while (cur_probe != loop_index); + //std::cerr << "hash_search: lg=" << (int)lg_size << ", hash=" << hash << std::endl; + //std::cerr << "cur_probe=" << cur_probe << ", stride=" << stride << std::endl; + throw std::logic_error("key not found and search wrapped"); +} + +template +typename theta_sketch_alloc::const_iterator update_theta_sketch_alloc::begin() const { + return typename theta_sketch_alloc::const_iterator(keys_, 1 << lg_cur_size_, 0); +} + +template +typename theta_sketch_alloc::const_iterator update_theta_sketch_alloc::end() const { + return typename theta_sketch_alloc::const_iterator(keys_, 1 << lg_cur_size_, 1 << lg_cur_size_); +} + +// compact sketch + +template +compact_theta_sketch_alloc::compact_theta_sketch_alloc(bool is_empty, uint64_t theta, uint64_t* keys, uint32_t num_keys, uint16_t seed_hash, bool is_ordered): +theta_sketch_alloc(is_empty, theta), +keys_(keys), +num_keys_(num_keys), +seed_hash_(seed_hash), +is_ordered_(is_ordered) +{} + +template +compact_theta_sketch_alloc::compact_theta_sketch_alloc(const compact_theta_sketch_alloc& other): +theta_sketch_alloc(other), +keys_(AllocU64().allocate(other.num_keys_)), +num_keys_(other.num_keys_), +seed_hash_(other.seed_hash_), +is_ordered_(other.is_ordered_) +{ + std::copy(other.keys_, &other.keys_[num_keys_], keys_); +} + +template +compact_theta_sketch_alloc::compact_theta_sketch_alloc(const theta_sketch_alloc& other, bool ordered): +theta_sketch_alloc(other), +keys_(AllocU64().allocate(other.get_num_retained())), +num_keys_(other.get_num_retained()), +seed_hash_(other.get_seed_hash()), +is_ordered_(other.is_ordered() or ordered) +{ + std::copy(other.begin(), other.end(), keys_); + if (ordered and !other.is_ordered()) std::sort(keys_, &keys_[num_keys_]); +} + +template +compact_theta_sketch_alloc::compact_theta_sketch_alloc(compact_theta_sketch_alloc&& other) noexcept: +theta_sketch_alloc(std::move(other)), +keys_(nullptr), +num_keys_(other.num_keys_), +seed_hash_(other.seed_hash_), +is_ordered_(other.is_ordered_) +{ + std::swap(keys_, other.keys_); +} + +template +compact_theta_sketch_alloc::~compact_theta_sketch_alloc() { + typedef typename std::allocator_traits::template rebind_alloc AllocU64; + AllocU64().deallocate(keys_, num_keys_); +} + +template +compact_theta_sketch_alloc& compact_theta_sketch_alloc::operator=(const compact_theta_sketch_alloc& other) { + theta_sketch_alloc::operator=(other); + if (num_keys_ != other.num_keys_) { + AllocU64().deallocate(keys_, num_keys_); + num_keys_ = other.num_keys_; + keys_ = AllocU64().allocate(num_keys_); + } + seed_hash_ = other.seed_hash_; + is_ordered_ = other.is_ordered_; + std::copy(other.keys_, &other.keys_[num_keys_], keys_); + return *this; +} + +template +compact_theta_sketch_alloc& compact_theta_sketch_alloc::operator=(compact_theta_sketch_alloc&& other) { + theta_sketch_alloc::operator=(std::move(other)); + std::swap(keys_, other.keys_); + std::swap(num_keys_, other.num_keys_); + std::swap(seed_hash_, other.seed_hash_); + std::swap(is_ordered_, other.is_ordered_); + return *this; +} + +template +uint32_t compact_theta_sketch_alloc::get_num_retained() const { + return num_keys_; +} + +template +uint16_t compact_theta_sketch_alloc::get_seed_hash() const { + return seed_hash_; +} + +template +bool compact_theta_sketch_alloc::is_ordered() const { + return is_ordered_; +} + +template +void compact_theta_sketch_alloc::to_stream(std::ostream& os, bool print_items) const { + os << "### Compact Theta sketch summary:" << std::endl; + os << " num retained keys : " << num_keys_ << std::endl; + os << " seed hash : " << this->get_seed_hash() << std::endl; + os << " ordered? : " << (this->is_ordered() ? "true" : "false") << std::endl; + os << " theta (fraction) : " << this->get_theta() << std::endl; + os << " theta (raw 64-bit) : " << this->theta_ << std::endl; + os << " estimation mode? : " << (this->is_estimation_mode() ? "true" : "false") << std::endl; + os << " estimate : " << this->get_estimate() << std::endl; + os << " lower bound 95% conf : " << this->get_lower_bound(2) << std::endl; + os << " upper bound 95% conf : " << this->get_upper_bound(2) << std::endl; + os << "### End sketch summary" << std::endl; + if (print_items) { + os << "### Retained keys" << std::endl; + for (auto key: *this) os << " " << key << std::endl; + os << "### End retained keys" << std::endl; + } +} + +template +void compact_theta_sketch_alloc::serialize(std::ostream& os) const { + const bool is_single_item = num_keys_ == 1 and !this->is_estimation_mode(); + const uint8_t preamble_longs = this->is_empty() or is_single_item ? 1 : this->is_estimation_mode() ? 3 : 2; + os.write((char*)&preamble_longs, sizeof(preamble_longs)); + const uint8_t serial_version = theta_sketch_alloc::SERIAL_VERSION; + os.write((char*)&serial_version, sizeof(serial_version)); + const uint8_t type = SKETCH_TYPE; + os.write((char*)&type, sizeof(type)); + const uint16_t unused16 = 0; + os.write((char*)&unused16, sizeof(unused16)); + const uint8_t flags_byte( + (1 << theta_sketch_alloc::flags::IS_COMPACT) | + (1 << theta_sketch_alloc::flags::IS_READ_ONLY) | + (this->is_empty() ? 1 << theta_sketch_alloc::flags::IS_EMPTY : 0) | + (this->is_ordered() ? 1 << theta_sketch_alloc::flags::IS_ORDERED : 0) + ); + os.write((char*)&flags_byte, sizeof(flags_byte)); + const uint16_t seed_hash = get_seed_hash(); + os.write((char*)&seed_hash, sizeof(seed_hash)); + if (!this->is_empty()) { + if (!is_single_item) { + os.write((char*)&num_keys_, sizeof(num_keys_)); + const uint32_t unused32 = 0; + os.write((char*)&unused32, sizeof(unused32)); + if (this->is_estimation_mode()) { + os.write((char*)&(this->theta_), sizeof(uint64_t)); + } + } + os.write((char*)keys_, sizeof(uint64_t) * num_keys_); + } +} + +template +std::pair compact_theta_sketch_alloc::serialize(unsigned header_size_bytes) const { + const bool is_single_item = num_keys_ == 1 and !this->is_estimation_mode(); + const uint8_t preamble_longs = this->is_empty() or is_single_item ? 1 : this->is_estimation_mode() ? 3 : 2; + const size_t size = header_size_bytes + sizeof(uint64_t) * preamble_longs + sizeof(uint64_t) * num_keys_; + typedef typename std::allocator_traits::template rebind_alloc AllocChar; + void_ptr_with_deleter data_ptr( + static_cast(AllocChar().allocate(size)), + [size](void* ptr) { AllocChar().deallocate(static_cast(ptr), size); } + ); + char* ptr = static_cast(data_ptr.get()) + header_size_bytes; + + copy_to_mem(&preamble_longs, &ptr, sizeof(preamble_longs)); + const uint8_t serial_version = theta_sketch_alloc::SERIAL_VERSION; + copy_to_mem(&serial_version, &ptr, sizeof(serial_version)); + const uint8_t type = SKETCH_TYPE; + copy_to_mem(&type, &ptr, sizeof(type)); + const uint16_t unused16 = 0; + copy_to_mem(&unused16, &ptr, sizeof(unused16)); + const uint8_t flags_byte( + (1 << theta_sketch_alloc::flags::IS_COMPACT) | + (1 << theta_sketch_alloc::flags::IS_READ_ONLY) | + (this->is_empty() ? 1 << theta_sketch_alloc::flags::IS_EMPTY : 0) | + (this->is_ordered() ? 1 << theta_sketch_alloc::flags::IS_ORDERED : 0) + ); + copy_to_mem(&flags_byte, &ptr, sizeof(flags_byte)); + const uint16_t seed_hash = get_seed_hash(); + copy_to_mem(&seed_hash, &ptr, sizeof(seed_hash)); + if (!this->is_empty()) { + if (!is_single_item) { + copy_to_mem(&num_keys_, &ptr, sizeof(num_keys_)); + const uint32_t unused32 = 0; + copy_to_mem(&unused32, &ptr, sizeof(unused32)); + if (this->is_estimation_mode()) { + copy_to_mem(&(this->theta_), &ptr, sizeof(uint64_t)); + } + } + copy_to_mem(keys_, &ptr, sizeof(uint64_t) * num_keys_); + } + + return std::make_pair(std::move(data_ptr), size);; +} + +template +compact_theta_sketch_alloc compact_theta_sketch_alloc::deserialize(std::istream& is, uint64_t seed) { + uint8_t preamble_longs; + is.read((char*)&preamble_longs, sizeof(preamble_longs)); + uint8_t serial_version; + is.read((char*)&serial_version, sizeof(serial_version)); + uint8_t type; + is.read((char*)&type, sizeof(type)); + uint16_t unused16; + is.read((char*)&unused16, sizeof(unused16)); + uint8_t flags_byte; + is.read((char*)&flags_byte, sizeof(flags_byte)); + uint16_t seed_hash; + is.read((char*)&seed_hash, sizeof(seed_hash)); + theta_sketch_alloc::check_sketch_type(type, SKETCH_TYPE); + theta_sketch_alloc::check_serial_version(serial_version, theta_sketch_alloc::SERIAL_VERSION); + theta_sketch_alloc::check_seed_hash(seed_hash, theta_sketch_alloc::get_seed_hash(seed)); + return internal_deserialize(is, preamble_longs, flags_byte, seed_hash); +} + +template +compact_theta_sketch_alloc compact_theta_sketch_alloc::internal_deserialize(std::istream& is, uint8_t preamble_longs, uint8_t flags_byte, uint16_t seed_hash) { + uint64_t theta = theta_sketch_alloc::MAX_THETA; + uint64_t* keys = nullptr; + uint32_t num_keys = 0; + + const bool is_empty = flags_byte & (1 << theta_sketch_alloc::flags::IS_EMPTY); + if (!is_empty) { + if (preamble_longs == 1) { + num_keys = 1; + } else { + is.read((char*)&num_keys, sizeof(num_keys)); + uint32_t unused32; + is.read((char*)&unused32, sizeof(unused32)); + if (preamble_longs > 2) { + is.read((char*)&theta, sizeof(theta)); + } + } + typedef typename std::allocator_traits::template rebind_alloc AllocU64; + keys = AllocU64().allocate(num_keys); + is.read((char*)keys, sizeof(uint64_t) * num_keys); + } + + const bool is_ordered = flags_byte & (1 << theta_sketch_alloc::flags::IS_ORDERED); + return compact_theta_sketch_alloc(is_empty, theta, keys, num_keys, seed_hash, is_ordered); +} + +template +compact_theta_sketch_alloc compact_theta_sketch_alloc::deserialize(const void* bytes, size_t size, uint64_t seed) { + theta_sketch_alloc::check_size(size, 8); + const char* ptr = static_cast(bytes); + uint8_t preamble_longs; + copy_from_mem(&ptr, &preamble_longs, sizeof(preamble_longs)); + uint8_t serial_version; + copy_from_mem(&ptr, &serial_version, sizeof(serial_version)); + uint8_t type; + copy_from_mem(&ptr, &type, sizeof(type)); + uint16_t unused16; + copy_from_mem(&ptr, &unused16, sizeof(unused16)); + uint8_t flags_byte; + copy_from_mem(&ptr, &flags_byte, sizeof(flags_byte)); + uint16_t seed_hash; + copy_from_mem(&ptr, &seed_hash, sizeof(seed_hash)); + theta_sketch_alloc::check_sketch_type(type, SKETCH_TYPE); + theta_sketch_alloc::check_serial_version(serial_version, theta_sketch_alloc::SERIAL_VERSION); + theta_sketch_alloc::check_seed_hash(seed_hash, theta_sketch_alloc::get_seed_hash(seed)); + return internal_deserialize(ptr, size - (ptr - static_cast(bytes)), preamble_longs, flags_byte, seed_hash); +} + +template +compact_theta_sketch_alloc compact_theta_sketch_alloc::internal_deserialize(const void* bytes, size_t size, uint8_t preamble_longs, uint8_t flags_byte, uint16_t seed_hash) { + const char* ptr = static_cast(bytes); + + uint64_t theta = theta_sketch_alloc::MAX_THETA; + uint64_t* keys = nullptr; + uint32_t num_keys = 0; + + const bool is_empty = flags_byte & (1 << theta_sketch_alloc::flags::IS_EMPTY); + if (!is_empty) { + if (preamble_longs == 1) { + num_keys = 1; + } else { + theta_sketch_alloc::check_size(size, 8); + copy_from_mem(&ptr, &num_keys, sizeof(num_keys)); + uint32_t unused32; + copy_from_mem(&ptr, &unused32, sizeof(unused32)); + if (preamble_longs > 2) { + theta_sketch_alloc::check_size(size - (ptr - static_cast(bytes)), 8); + copy_from_mem(&ptr, &theta, sizeof(theta)); + } + } + const size_t keys_size_bytes = sizeof(uint64_t) * num_keys; + theta_sketch_alloc::check_size(size - (ptr - static_cast(bytes)), keys_size_bytes); + typedef typename std::allocator_traits::template rebind_alloc AllocU64; + keys = AllocU64().allocate(num_keys); + copy_from_mem(&ptr, keys, keys_size_bytes); + } + + const bool is_ordered = flags_byte & (1 << theta_sketch_alloc::flags::IS_ORDERED); + return compact_theta_sketch_alloc(is_empty, theta, keys, num_keys, seed_hash, is_ordered); +} + +template +typename theta_sketch_alloc::const_iterator compact_theta_sketch_alloc::begin() const { + return typename theta_sketch_alloc::const_iterator(keys_, num_keys_, 0); +} + +template +typename theta_sketch_alloc::const_iterator compact_theta_sketch_alloc::end() const { + return typename theta_sketch_alloc::const_iterator(keys_, num_keys_, num_keys_); +} + +// builder + +template +update_theta_sketch_alloc::builder::builder(): +lg_k_(DEFAULT_LG_K), rf_(DEFAULT_RESIZE_FACTOR), p_(1), seed_(DEFAULT_SEED) {} + +template +typename update_theta_sketch_alloc::builder& update_theta_sketch_alloc::builder::set_lg_k(uint8_t lg_k) { + if (lg_k < MIN_LG_K) { + throw std::invalid_argument("lg_k must not be less than " + std::to_string(MIN_LG_K) + ": " + std::to_string(lg_k)); + } + lg_k_ = lg_k; + return *this; +} + +template +typename update_theta_sketch_alloc::builder& update_theta_sketch_alloc::builder::set_resize_factor(resize_factor rf) { + rf_ = rf; + return *this; +} + +template +typename update_theta_sketch_alloc::builder& update_theta_sketch_alloc::builder::set_p(float p) { + p_ = p; + return *this; +} + +template +typename update_theta_sketch_alloc::builder& update_theta_sketch_alloc::builder::set_seed(uint64_t seed) { + seed_ = seed; + return *this; +} + +template +uint8_t update_theta_sketch_alloc::builder::starting_sub_multiple(uint8_t lg_tgt, uint8_t lg_min, uint8_t lg_rf) { + return (lg_tgt <= lg_min) ? lg_min : (lg_rf == 0) ? lg_tgt : ((lg_tgt - lg_min) % lg_rf) + lg_min; +} + +template +update_theta_sketch_alloc update_theta_sketch_alloc::builder::build() const { + return update_theta_sketch_alloc(starting_sub_multiple(lg_k_ + 1, MIN_LG_K, static_cast(rf_)), lg_k_, rf_, p_, seed_); +} + +// iterator + +template +theta_sketch_alloc::const_iterator::const_iterator(const uint64_t* keys, uint32_t size, uint32_t index): +keys_(keys), size_(size), index_(index) { + while (index_ < size_ and keys_[index_] == 0) ++index_; +} + +template +typename theta_sketch_alloc::const_iterator& theta_sketch_alloc::const_iterator::operator++() { + do { + ++index_; + } while (index_ < size_ and keys_[index_] == 0); + return *this; +} + +template +typename theta_sketch_alloc::const_iterator theta_sketch_alloc::const_iterator::operator++(int) { + const_iterator tmp(*this); + operator++(); + return tmp; +} + +template +bool theta_sketch_alloc::const_iterator::operator==(const const_iterator& other) const { + return index_ == other.index_; +} + +template +bool theta_sketch_alloc::const_iterator::operator!=(const const_iterator& other) const { + return index_ != other.index_; +} + +template +uint64_t theta_sketch_alloc::const_iterator::operator*() const { + return keys_[index_]; +} + +} /* namespace datasketches */ + +# endif diff --git a/theta/include/theta_union.hpp b/theta/include/theta_union.hpp new file mode 100644 index 00000000..993204a9 --- /dev/null +++ b/theta/include/theta_union.hpp @@ -0,0 +1,61 @@ +/* + * Copyright 2019, Verizon Media. + * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms. + */ + +#ifndef THETA_UNION_HPP_ +#define THETA_UNION_HPP_ + +#include +#include +#include + +#include + +namespace datasketches { + +/* + * author Alexander Saydakov + * author Lee Rhodes + * author Kevin Lang + */ + +template +class theta_union_alloc { +public: + class builder; + void update(const theta_sketch_alloc& sketch); + compact_theta_sketch_alloc get_result(bool ordered = true) const; + +private: + bool is_empty_; + uint64_t theta_; + update_theta_sketch_alloc state_; + + // for builder + theta_union_alloc(uint64_t theta, update_theta_sketch_alloc&& state); +}; + +// builder + +template +class theta_union_alloc::builder { +public: + typedef typename update_theta_sketch_alloc::resize_factor resize_factor; + builder& set_lg_k(uint8_t lg_k); + builder& set_resize_factor(resize_factor rf); + builder& set_p(float p); + builder& set_seed(uint64_t seed); + theta_union_alloc build() const; +private: + typename update_theta_sketch_alloc::builder sketch_builder; +}; + +// alias with default allocator for convenience +typedef theta_union_alloc> theta_union; + +} /* namespace datasketches */ + +#include "theta_union_impl.hpp" + +# endif diff --git a/theta/include/theta_union_impl.hpp b/theta/include/theta_union_impl.hpp new file mode 100644 index 00000000..f276f642 --- /dev/null +++ b/theta/include/theta_union_impl.hpp @@ -0,0 +1,100 @@ +/* + * Copyright 2019, Verizon Media. + * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms. + */ + +#ifndef THETA_UNION_IMPL_HPP_ +#define THETA_UNION_IMPL_HPP_ + +namespace datasketches { + +/* + * author Alexander Saydakov + * author Lee Rhodes + * author Kevin Lang + */ + +template +theta_union_alloc::theta_union_alloc(uint64_t theta, update_theta_sketch_alloc&& state): +is_empty_(true), theta_(theta), state_(std::move(state)) {} + +template +void theta_union_alloc::update(const theta_sketch_alloc& sketch) { + if (sketch.is_empty()) return; + if (sketch.get_seed_hash() != state_.get_seed_hash()) throw std::invalid_argument("seed hash mismatch"); + is_empty_ = false; + if (sketch.get_theta64() < theta_) theta_ = sketch.get_theta64(); + if (sketch.is_ordered()) { + for (auto hash: sketch) { + if (hash >= theta_) break; // early stop + state_.internal_update(hash); + } + } else { + for (auto hash: sketch) if (hash < theta_) state_.internal_update(hash); + } + if (state_.get_theta64() < theta_) theta_ = state_.get_theta64(); +} + +template +compact_theta_sketch_alloc theta_union_alloc::get_result(bool ordered) const { + if (is_empty_) return state_.compact(ordered); + const uint32_t nom_num_keys = 1 << state_.lg_nom_size_; + if (theta_ >= state_.theta_ and state_.get_num_retained() <= nom_num_keys) return state_.compact(ordered); + uint64_t theta = std::min(theta_, state_.get_theta64()); + typedef typename std::allocator_traits::template rebind_alloc AllocU64; + uint64_t* keys = AllocU64().allocate(state_.get_num_retained()); + uint32_t num_keys = 0; + for (auto key: state_) { + if (key < theta) keys[num_keys++] = key; + } + if (num_keys == 0) return compact_theta_sketch_alloc(is_empty_, theta, nullptr, 0, state_.get_seed_hash(), ordered); + if (num_keys > nom_num_keys) { + std::nth_element(keys, &keys[nom_num_keys], &keys[num_keys]); + theta = keys[nom_num_keys]; + num_keys = nom_num_keys; + } + if (num_keys != state_.get_num_retained()) { + uint64_t* new_keys = AllocU64().allocate(num_keys); + std::copy(keys, &keys[num_keys], new_keys); + AllocU64().deallocate(keys, state_.get_num_retained()); + keys = new_keys; + } + if (ordered) std::sort(keys, &keys[num_keys]); + return compact_theta_sketch_alloc(false, theta, keys, num_keys, state_.get_seed_hash(), ordered); +} + +// builder + +template +typename theta_union_alloc::builder& theta_union_alloc::builder::set_lg_k(uint8_t lg_k) { + sketch_builder.set_lg_k(lg_k); + return *this; +} + +template +typename theta_union_alloc::builder& theta_union_alloc::builder::set_resize_factor(resize_factor rf) { + sketch_builder.set_resize_factor(rf); + return *this; +} + +template +typename theta_union_alloc::builder& theta_union_alloc::builder::set_p(float p) { + sketch_builder.set_p(p); + return *this; +} + +template +typename theta_union_alloc::builder& theta_union_alloc::builder::set_seed(uint64_t seed) { + sketch_builder.set_seed(seed); + return *this; +} + +template +theta_union_alloc theta_union_alloc::builder::build() const { + update_theta_sketch_alloc sketch = sketch_builder.build(); + return theta_union_alloc(sketch.get_theta64(), std::move(sketch)); +} + +} /* namespace datasketches */ + +# endif diff --git a/theta/test/theta_a_not_b_test.cpp b/theta/test/theta_a_not_b_test.cpp new file mode 100644 index 00000000..6ec90504 --- /dev/null +++ b/theta/test/theta_a_not_b_test.cpp @@ -0,0 +1,225 @@ +/* + * Copyright 2019, Verizon Media. + * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms. + */ + +#include +#include + +#include + +namespace datasketches { + +class theta_a_not_b_test: public CppUnit::TestFixture { + + CPPUNIT_TEST_SUITE(theta_a_not_b_test); + CPPUNIT_TEST(empty); + CPPUNIT_TEST(non_empty_no_retained_keys); + CPPUNIT_TEST(exact_mode_half_overlap); + CPPUNIT_TEST(exact_mode_disjoint); + CPPUNIT_TEST(exact_mode_full_overlap); + CPPUNIT_TEST(estimation_mode_half_overlap); + CPPUNIT_TEST(estimation_mode_disjoint); + CPPUNIT_TEST(estimation_mode_full_overlap); + CPPUNIT_TEST(seed_mismatch); + CPPUNIT_TEST_SUITE_END(); + + void empty() { + theta_a_not_b a_not_b; + update_theta_sketch a = update_theta_sketch::builder().build(); + update_theta_sketch b = update_theta_sketch::builder().build(); + compact_theta_sketch result = a_not_b.compute(a, b); + CPPUNIT_ASSERT_EQUAL(0U, result.get_num_retained()); + CPPUNIT_ASSERT(result.is_empty()); + CPPUNIT_ASSERT(!result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0.0, result.get_estimate()); + } + + void non_empty_no_retained_keys() { + update_theta_sketch a = update_theta_sketch::builder().build(); + a.update(1); + update_theta_sketch b = update_theta_sketch::builder().set_p(0.001).build(); + theta_a_not_b a_not_b; + + // B is still empty + compact_theta_sketch result = a_not_b.compute(a, b); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(!result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(1U, result.get_num_retained()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(1, result.get_theta(), 1e-10); + CPPUNIT_ASSERT_EQUAL(1.0, result.get_estimate()); + + // B is not empty in estimation mode and no entries + b.update(1); + CPPUNIT_ASSERT_EQUAL(0U, b.get_num_retained()); + + result = a_not_b.compute(a, b); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0U, result.get_num_retained()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(0.001, result.get_theta(), 1e-10); + CPPUNIT_ASSERT_EQUAL(0.0, result.get_estimate()); + } + + void exact_mode_half_overlap() { + update_theta_sketch a = update_theta_sketch::builder().build(); + int value = 0; + for (int i = 0; i < 1000; i++) a.update(value++); + + update_theta_sketch b = update_theta_sketch::builder().build(); + value = 500; + for (int i = 0; i < 1000; i++) b.update(value++); + + theta_a_not_b a_not_b; + + // unordered inputs, ordered result + compact_theta_sketch result = a_not_b.compute(a, b); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(!result.is_estimation_mode()); + CPPUNIT_ASSERT(result.is_ordered()); + CPPUNIT_ASSERT_EQUAL(500.0, result.get_estimate()); + + // unordered inputs, unordered result + result = a_not_b.compute(a, b, false); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(!result.is_estimation_mode()); + CPPUNIT_ASSERT(!result.is_ordered()); + CPPUNIT_ASSERT_EQUAL(500.0, result.get_estimate()); + + // ordered inputs + result = a_not_b.compute(a.compact(), b.compact()); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(!result.is_estimation_mode()); + CPPUNIT_ASSERT(result.is_ordered()); + CPPUNIT_ASSERT_EQUAL(500.0, result.get_estimate()); + + // A is ordered, so the result is ordered regardless + result = a_not_b.compute(a.compact(), b, false); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(!result.is_estimation_mode()); + CPPUNIT_ASSERT(result.is_ordered()); + CPPUNIT_ASSERT_EQUAL(500.0, result.get_estimate()); +} + + void exact_mode_disjoint() { + update_theta_sketch a = update_theta_sketch::builder().build(); + int value = 0; + for (int i = 0; i < 1000; i++) a.update(value++); + + update_theta_sketch b = update_theta_sketch::builder().build(); + for (int i = 0; i < 1000; i++) b.update(value++); + + theta_a_not_b a_not_b; + + // unordered inputs + compact_theta_sketch result = a_not_b.compute(a, b); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(!result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(1000.0, result.get_estimate()); + + // ordered inputs + result = a_not_b.compute(a.compact(), b.compact()); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(!result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(1000.0, result.get_estimate()); + } + + void exact_mode_full_overlap() { + update_theta_sketch sketch = update_theta_sketch::builder().build(); + int value = 0; + for (int i = 0; i < 1000; i++) sketch.update(value++); + + theta_a_not_b a_not_b; + + // unordered inputs + compact_theta_sketch result = a_not_b.compute(sketch, sketch); + CPPUNIT_ASSERT(result.is_empty()); + CPPUNIT_ASSERT(!result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0.0, result.get_estimate()); + + // ordered inputs + result = a_not_b.compute(sketch.compact(), sketch.compact()); + CPPUNIT_ASSERT(result.is_empty()); + CPPUNIT_ASSERT(!result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0.0, result.get_estimate()); + } + + void estimation_mode_half_overlap() { + update_theta_sketch a = update_theta_sketch::builder().build(); + int value = 0; + for (int i = 0; i < 10000; i++) a.update(value++); + + update_theta_sketch b = update_theta_sketch::builder().build(); + value = 5000; + for (int i = 0; i < 10000; i++) b.update(value++); + + theta_a_not_b a_not_b; + + // unordered inputs + compact_theta_sketch result = a_not_b.compute(a, b); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(result.is_estimation_mode()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(5000, result.get_estimate(), 5000 * 0.02); + + // ordered inputs + result = a_not_b.compute(a.compact(), b.compact()); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(result.is_estimation_mode()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(5000, result.get_estimate(), 5000 * 0.02); + } + + void estimation_mode_disjoint() { + update_theta_sketch a = update_theta_sketch::builder().build(); + int value = 0; + for (int i = 0; i < 10000; i++) a.update(value++); + + update_theta_sketch b = update_theta_sketch::builder().build(); + for (int i = 0; i < 10000; i++) b.update(value++); + + theta_a_not_b a_not_b; + + // unordered inputs + compact_theta_sketch result = a_not_b.compute(a, b); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(result.is_estimation_mode()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(10000, result.get_estimate(), 10000 * 0.02); + + // ordered inputs + result = a_not_b.compute(a.compact(), b.compact()); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(result.is_estimation_mode()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(10000, result.get_estimate(), 10000 * 0.02); + } + + void estimation_mode_full_overlap() { + update_theta_sketch sketch = update_theta_sketch::builder().build(); + int value = 0; + for (int i = 0; i < 10000; i++) sketch.update(value++); + + theta_a_not_b a_not_b; + + // unordered inputs + compact_theta_sketch result = a_not_b.compute(sketch, sketch); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0.0, result.get_estimate()); + + // ordered inputs + result = a_not_b.compute(sketch.compact(), sketch.compact()); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0.0, result.get_estimate()); + } + + void seed_mismatch() { + update_theta_sketch sketch = update_theta_sketch::builder().build(); + sketch.update(1); // non-empty should not be ignored + theta_a_not_b a_not_b(123); + CPPUNIT_ASSERT_THROW(a_not_b.compute(sketch, sketch), std::invalid_argument); + } + +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(theta_a_not_b_test); + +} /* namespace datasketches */ diff --git a/theta/test/theta_compact_empty_from_java.bin b/theta/test/theta_compact_empty_from_java.bin new file mode 100644 index 00000000..44730d3f Binary files /dev/null and b/theta/test/theta_compact_empty_from_java.bin differ diff --git a/theta/test/theta_compact_estimation_from_java.bin b/theta/test/theta_compact_estimation_from_java.bin new file mode 100644 index 00000000..7c6babf9 Binary files /dev/null and b/theta/test/theta_compact_estimation_from_java.bin differ diff --git a/theta/test/theta_compact_single_item_from_java.bin b/theta/test/theta_compact_single_item_from_java.bin new file mode 100644 index 00000000..be5ee687 Binary files /dev/null and b/theta/test/theta_compact_single_item_from_java.bin differ diff --git a/theta/test/theta_intersection_test.cpp b/theta/test/theta_intersection_test.cpp new file mode 100644 index 00000000..44ab400e --- /dev/null +++ b/theta/test/theta_intersection_test.cpp @@ -0,0 +1,226 @@ +/* + * Copyright 2019, Verizon Media. + * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms. + */ + +#include +#include + +#include + +namespace datasketches { + +class theta_intersection_test: public CppUnit::TestFixture { + + CPPUNIT_TEST_SUITE(theta_intersection_test); + CPPUNIT_TEST(invalid); + CPPUNIT_TEST(empty); + CPPUNIT_TEST(non_empty_no_retained_keys); + CPPUNIT_TEST(exact_mode_half_overlap_unordered); + CPPUNIT_TEST(exact_mode_half_overlap_ordered); + CPPUNIT_TEST(exact_mode_disjoint_unordered); + CPPUNIT_TEST(exact_mode_disjoint_ordered); + CPPUNIT_TEST(estimation_mode_half_overlap_unordered); + CPPUNIT_TEST(estimation_mode_half_overlap_ordered); + CPPUNIT_TEST(estimation_mode_disjoint_unordered); + CPPUNIT_TEST(estimation_mode_disjoint_ordered); + CPPUNIT_TEST(seed_mismatch); + CPPUNIT_TEST_SUITE_END(); + + void invalid() { + theta_intersection intersection; + CPPUNIT_ASSERT(!intersection.has_result()); + CPPUNIT_ASSERT_THROW(intersection.get_result(), std::invalid_argument); + } + + void empty() { + theta_intersection intersection; + update_theta_sketch sketch = update_theta_sketch::builder().build(); + intersection.update(sketch); + compact_theta_sketch result = intersection.get_result(); + CPPUNIT_ASSERT_EQUAL(0U, result.get_num_retained()); + CPPUNIT_ASSERT(result.is_empty()); + CPPUNIT_ASSERT(!result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0.0, result.get_estimate()); + + intersection.update(sketch); + result = intersection.get_result(); + CPPUNIT_ASSERT_EQUAL(0U, result.get_num_retained()); + CPPUNIT_ASSERT(result.is_empty()); + CPPUNIT_ASSERT(!result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0.0, result.get_estimate()); + } + + void non_empty_no_retained_keys() { + update_theta_sketch sketch = update_theta_sketch::builder().set_p(0.001).build(); + sketch.update(1); + theta_intersection intersection; + intersection.update(sketch); + compact_theta_sketch result = intersection.get_result(); + CPPUNIT_ASSERT_EQUAL(0U, result.get_num_retained()); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(result.is_estimation_mode()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(0.001, result.get_theta(), 1e-10); + CPPUNIT_ASSERT_EQUAL(0.0, result.get_estimate()); + + intersection.update(sketch); + result = intersection.get_result(); + CPPUNIT_ASSERT_EQUAL(0U, result.get_num_retained()); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(result.is_estimation_mode()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(0.001, result.get_theta(), 1e-10); + CPPUNIT_ASSERT_EQUAL(0.0, result.get_estimate()); + } + + void exact_mode_half_overlap_unordered() { + update_theta_sketch sketch1 = update_theta_sketch::builder().build(); + int value = 0; + for (int i = 0; i < 1000; i++) sketch1.update(value++); + + update_theta_sketch sketch2 = update_theta_sketch::builder().build(); + value = 500; + for (int i = 0; i < 1000; i++) sketch2.update(value++); + + theta_intersection intersection; + intersection.update(sketch1); + intersection.update(sketch2); + compact_theta_sketch result = intersection.get_result(); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(!result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(500.0, result.get_estimate()); + } + + void exact_mode_half_overlap_ordered() { + update_theta_sketch sketch1 = update_theta_sketch::builder().build(); + int value = 0; + for (int i = 0; i < 1000; i++) sketch1.update(value++); + + update_theta_sketch sketch2 = update_theta_sketch::builder().build(); + value = 500; + for (int i = 0; i < 1000; i++) sketch2.update(value++); + + theta_intersection intersection; + intersection.update(sketch1.compact()); + intersection.update(sketch2.compact()); + compact_theta_sketch result = intersection.get_result(); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(!result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(500.0, result.get_estimate()); + } + + void exact_mode_disjoint_unordered() { + update_theta_sketch sketch1 = update_theta_sketch::builder().build(); + int value = 0; + for (int i = 0; i < 1000; i++) sketch1.update(value++); + + update_theta_sketch sketch2 = update_theta_sketch::builder().build(); + for (int i = 0; i < 1000; i++) sketch2.update(value++); + + theta_intersection intersection; + intersection.update(sketch1); + intersection.update(sketch2); + compact_theta_sketch result = intersection.get_result(); + CPPUNIT_ASSERT(result.is_empty()); + CPPUNIT_ASSERT(!result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0.0, result.get_estimate()); + } + + void exact_mode_disjoint_ordered() { + update_theta_sketch sketch1 = update_theta_sketch::builder().build(); + int value = 0; + for (int i = 0; i < 1000; i++) sketch1.update(value++); + + update_theta_sketch sketch2 = update_theta_sketch::builder().build(); + for (int i = 0; i < 1000; i++) sketch2.update(value++); + + theta_intersection intersection; + intersection.update(sketch1.compact()); + intersection.update(sketch2.compact()); + compact_theta_sketch result = intersection.get_result(); + CPPUNIT_ASSERT(result.is_empty()); + CPPUNIT_ASSERT(!result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0.0, result.get_estimate()); + } + + void estimation_mode_half_overlap_unordered() { + update_theta_sketch sketch1 = update_theta_sketch::builder().build(); + int value = 0; + for (int i = 0; i < 10000; i++) sketch1.update(value++); + + update_theta_sketch sketch2 = update_theta_sketch::builder().build(); + value = 5000; + for (int i = 0; i < 10000; i++) sketch2.update(value++); + + theta_intersection intersection; + intersection.update(sketch1); + intersection.update(sketch2); + compact_theta_sketch result = intersection.get_result(); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(result.is_estimation_mode()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(5000, result.get_estimate(), 5000 * 0.02); + } + + void estimation_mode_half_overlap_ordered() { + update_theta_sketch sketch1 = update_theta_sketch::builder().build(); + int value = 0; + for (int i = 0; i < 10000; i++) sketch1.update(value++); + + update_theta_sketch sketch2 = update_theta_sketch::builder().build(); + value = 5000; + for (int i = 0; i < 10000; i++) sketch2.update(value++); + + theta_intersection intersection; + intersection.update(sketch1.compact()); + intersection.update(sketch2.compact()); + compact_theta_sketch result = intersection.get_result(); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(result.is_estimation_mode()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(5000, result.get_estimate(), 5000 * 0.02); + } + + void estimation_mode_disjoint_unordered() { + update_theta_sketch sketch1 = update_theta_sketch::builder().build(); + int value = 0; + for (int i = 0; i < 10000; i++) sketch1.update(value++); + + update_theta_sketch sketch2 = update_theta_sketch::builder().build(); + for (int i = 0; i < 10000; i++) sketch2.update(value++); + + theta_intersection intersection; + intersection.update(sketch1); + intersection.update(sketch2); + compact_theta_sketch result = intersection.get_result(); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0.0, result.get_estimate()); + } + + void estimation_mode_disjoint_ordered() { + update_theta_sketch sketch1 = update_theta_sketch::builder().build(); + int value = 0; + for (int i = 0; i < 10000; i++) sketch1.update(value++); + + update_theta_sketch sketch2 = update_theta_sketch::builder().build(); + for (int i = 0; i < 10000; i++) sketch2.update(value++); + + theta_intersection intersection; + intersection.update(sketch1.compact()); + intersection.update(sketch2.compact()); + compact_theta_sketch result = intersection.get_result(); + CPPUNIT_ASSERT(!result.is_empty()); + CPPUNIT_ASSERT(result.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0.0, result.get_estimate()); + } + + void seed_mismatch() { + update_theta_sketch sketch = update_theta_sketch::builder().build(); + sketch.update(1); // non-empty should not be ignored + theta_intersection intersection(123); + CPPUNIT_ASSERT_THROW(intersection.update(sketch), std::invalid_argument); + } + +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(theta_intersection_test); + +} /* namespace datasketches */ diff --git a/theta/test/theta_sketch_test.cpp b/theta/test/theta_sketch_test.cpp new file mode 100644 index 00000000..62f61921 --- /dev/null +++ b/theta/test/theta_sketch_test.cpp @@ -0,0 +1,420 @@ +/* + * Copyright 2019, Verizon Media. + * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms. + */ + +#include +#include + +#include + +namespace datasketches { + +class theta_sketch_test: public CppUnit::TestFixture { + + CPPUNIT_TEST_SUITE(theta_sketch_test); + CPPUNIT_TEST(empty); + CPPUNIT_TEST(non_empty_no_retained_keys); + CPPUNIT_TEST(single_item); + CPPUNIT_TEST(resize_exact); + CPPUNIT_TEST(estimation); + CPPUNIT_TEST(deserialize_update_empty_from_java_as_base); + CPPUNIT_TEST(deserialize_update_empty_from_java_as_subclass); + CPPUNIT_TEST(deserialize_update_estimation_from_java_as_base); + CPPUNIT_TEST(deserialize_update_estimation_from_java_as_subclass); + CPPUNIT_TEST(deserialize_compact_empty_from_java_as_base); + CPPUNIT_TEST(deserialize_compact_empty_from_java_as_subclass); + CPPUNIT_TEST(deserialize_single_item_from_java_as_base); + CPPUNIT_TEST(deserialize_single_item_from_java_as_subclass); + CPPUNIT_TEST(deserialize_compact_estimation_from_java_as_base); + CPPUNIT_TEST(deserialize_compact_estimation_from_java_as_subclass); + CPPUNIT_TEST(serialize_deserialize_stream_and_bytes_equivalency); + CPPUNIT_TEST_SUITE_END(); + + void empty() { + update_theta_sketch update_sketch = update_theta_sketch::builder().build(); + CPPUNIT_ASSERT(update_sketch.is_empty()); + CPPUNIT_ASSERT(!update_sketch.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(1.0, update_sketch.get_theta()); + CPPUNIT_ASSERT_EQUAL(0.0, update_sketch.get_estimate()); + CPPUNIT_ASSERT_EQUAL(0.0, update_sketch.get_lower_bound(1)); + CPPUNIT_ASSERT_EQUAL(0.0, update_sketch.get_upper_bound(1)); + + compact_theta_sketch compact_sketch = update_sketch.compact(); + CPPUNIT_ASSERT(compact_sketch.is_empty()); + CPPUNIT_ASSERT(!compact_sketch.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(1.0, compact_sketch.get_theta()); + CPPUNIT_ASSERT_EQUAL(0.0, compact_sketch.get_estimate()); + CPPUNIT_ASSERT_EQUAL(0.0, compact_sketch.get_lower_bound(1)); + CPPUNIT_ASSERT_EQUAL(0.0, compact_sketch.get_upper_bound(1)); + } + + void non_empty_no_retained_keys() { + update_theta_sketch update_sketch = update_theta_sketch::builder().set_p(0.001).build(); + update_sketch.update(1); + //update_sketch.to_stream(std::cerr); + CPPUNIT_ASSERT_EQUAL(0U, update_sketch.get_num_retained()); + CPPUNIT_ASSERT(!update_sketch.is_empty()); + CPPUNIT_ASSERT(update_sketch.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0.0, update_sketch.get_estimate()); + CPPUNIT_ASSERT_EQUAL(0.0, update_sketch.get_lower_bound(1)); + CPPUNIT_ASSERT(update_sketch.get_upper_bound(1) > 0); + + compact_theta_sketch compact_sketch = update_sketch.compact(); + CPPUNIT_ASSERT_EQUAL(0U, compact_sketch.get_num_retained()); + CPPUNIT_ASSERT(!compact_sketch.is_empty()); + CPPUNIT_ASSERT(compact_sketch.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0.0, compact_sketch.get_estimate()); + CPPUNIT_ASSERT_EQUAL(0.0, compact_sketch.get_lower_bound(1)); + CPPUNIT_ASSERT(compact_sketch.get_upper_bound(1) > 0); +} + + void single_item() { + update_theta_sketch update_sketch = update_theta_sketch::builder().build(); + update_sketch.update(1); + CPPUNIT_ASSERT(!update_sketch.is_empty()); + CPPUNIT_ASSERT(!update_sketch.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(1.0, update_sketch.get_theta()); + CPPUNIT_ASSERT_EQUAL(1.0, update_sketch.get_estimate()); + CPPUNIT_ASSERT_EQUAL(1.0, update_sketch.get_lower_bound(1)); + CPPUNIT_ASSERT_EQUAL(1.0, update_sketch.get_upper_bound(1)); + + compact_theta_sketch compact_sketch = update_sketch.compact(); + CPPUNIT_ASSERT(!compact_sketch.is_empty()); + CPPUNIT_ASSERT(!compact_sketch.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(1.0, compact_sketch.get_theta()); + CPPUNIT_ASSERT_EQUAL(1.0, compact_sketch.get_estimate()); + CPPUNIT_ASSERT_EQUAL(1.0, compact_sketch.get_lower_bound(1)); + CPPUNIT_ASSERT_EQUAL(1.0, compact_sketch.get_upper_bound(1)); + } + + void resize_exact() { + update_theta_sketch update_sketch = update_theta_sketch::builder().build(); + for (int i = 0; i < 2000; i++) update_sketch.update(i); + CPPUNIT_ASSERT(!update_sketch.is_empty()); + CPPUNIT_ASSERT(!update_sketch.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(1.0, update_sketch.get_theta()); + CPPUNIT_ASSERT_EQUAL(2000.0, update_sketch.get_estimate()); + CPPUNIT_ASSERT_EQUAL(2000.0, update_sketch.get_lower_bound(1)); + CPPUNIT_ASSERT_EQUAL(2000.0, update_sketch.get_upper_bound(1)); + + compact_theta_sketch compact_sketch = update_sketch.compact(); + CPPUNIT_ASSERT(!compact_sketch.is_empty()); + CPPUNIT_ASSERT(!compact_sketch.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(1.0, compact_sketch.get_theta()); + CPPUNIT_ASSERT_EQUAL(2000.0, compact_sketch.get_estimate()); + CPPUNIT_ASSERT_EQUAL(2000.0, compact_sketch.get_lower_bound(1)); + CPPUNIT_ASSERT_EQUAL(2000.0, compact_sketch.get_upper_bound(1)); +} + + void estimation() { + update_theta_sketch update_sketch = update_theta_sketch::builder().set_resize_factor(update_theta_sketch::resize_factor::X1).build(); + const int n = 8000; + for (int i = 0; i < n; i++) update_sketch.update(i); + //update_sketch.to_stream(std::cerr); + CPPUNIT_ASSERT(!update_sketch.is_empty()); + CPPUNIT_ASSERT(update_sketch.is_estimation_mode()); + CPPUNIT_ASSERT(update_sketch.get_theta() < 1.0); + CPPUNIT_ASSERT_DOUBLES_EQUAL((double) n, update_sketch.get_estimate(), n * 0.01); + CPPUNIT_ASSERT(update_sketch.get_lower_bound(1) < n); + CPPUNIT_ASSERT(update_sketch.get_upper_bound(1) > n); + + compact_theta_sketch compact_sketch = update_sketch.compact(); + CPPUNIT_ASSERT(!compact_sketch.is_empty()); + CPPUNIT_ASSERT(compact_sketch.is_estimation_mode()); + CPPUNIT_ASSERT(compact_sketch.get_theta() < 1.0); + CPPUNIT_ASSERT_DOUBLES_EQUAL((double) n, compact_sketch.get_estimate(), n * 0.01); + CPPUNIT_ASSERT(compact_sketch.get_lower_bound(1) < n); + CPPUNIT_ASSERT(compact_sketch.get_upper_bound(1) > n); +} + + void deserialize_update_empty_from_java_as_base() { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open("test/theta_update_empty_from_java.bin", std::ios::binary); + auto sketchptr = theta_sketch::deserialize(is); + CPPUNIT_ASSERT(sketchptr->is_empty()); + CPPUNIT_ASSERT(!sketchptr->is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0U, sketchptr->get_num_retained()); + CPPUNIT_ASSERT_EQUAL(1.0, sketchptr->get_theta()); + CPPUNIT_ASSERT_EQUAL(0.0, sketchptr->get_estimate()); + CPPUNIT_ASSERT_EQUAL(0.0, sketchptr->get_lower_bound(1)); + CPPUNIT_ASSERT_EQUAL(0.0, sketchptr->get_upper_bound(1)); + } + + void deserialize_update_empty_from_java_as_subclass() { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open("test/theta_update_empty_from_java.bin", std::ios::binary); + auto sketch = update_theta_sketch::deserialize(is); + CPPUNIT_ASSERT(sketch.is_empty()); + CPPUNIT_ASSERT(!sketch.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0U, sketch.get_num_retained()); + CPPUNIT_ASSERT_EQUAL(1.0, sketch.get_theta()); + CPPUNIT_ASSERT_EQUAL(0.0, sketch.get_estimate()); + CPPUNIT_ASSERT_EQUAL(0.0, sketch.get_lower_bound(1)); + CPPUNIT_ASSERT_EQUAL(0.0, sketch.get_upper_bound(1)); + } + + void deserialize_update_estimation_from_java_as_base() { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open("test/theta_update_estimation_from_java.bin", std::ios::binary); + auto sketchptr = theta_sketch::deserialize(is); + CPPUNIT_ASSERT(!sketchptr->is_empty()); + CPPUNIT_ASSERT(sketchptr->is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(5324U, sketchptr->get_num_retained()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(10000.0, sketchptr->get_estimate(), 10000 * 0.01); + CPPUNIT_ASSERT(sketchptr->get_lower_bound(1) < 10000); + CPPUNIT_ASSERT(sketchptr->get_upper_bound(1) > 10000); + } + + void deserialize_update_estimation_from_java_as_subclass() { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open("test/theta_update_estimation_from_java.bin", std::ios::binary); + auto sketch = update_theta_sketch::deserialize(is); + CPPUNIT_ASSERT(!sketch.is_empty()); + CPPUNIT_ASSERT(sketch.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(5324U, sketch.get_num_retained()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(10000.0, sketch.get_estimate(), 10000 * 0.01); + CPPUNIT_ASSERT(sketch.get_lower_bound(1) < 10000); + CPPUNIT_ASSERT(sketch.get_upper_bound(1) > 10000); + } + + void deserialize_compact_empty_from_java_as_base() { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open("test/theta_compact_empty_from_java.bin", std::ios::binary); + auto sketchptr = theta_sketch::deserialize(is); + CPPUNIT_ASSERT(sketchptr->is_empty()); + CPPUNIT_ASSERT(!sketchptr->is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0U, sketchptr->get_num_retained()); + CPPUNIT_ASSERT_EQUAL(1.0, sketchptr->get_theta()); + CPPUNIT_ASSERT_EQUAL(0.0, sketchptr->get_estimate()); + CPPUNIT_ASSERT_EQUAL(0.0, sketchptr->get_lower_bound(1)); + CPPUNIT_ASSERT_EQUAL(0.0, sketchptr->get_upper_bound(1)); + } + + void deserialize_compact_empty_from_java_as_subclass() { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open("test/theta_compact_empty_from_java.bin", std::ios::binary); + auto sketch = compact_theta_sketch::deserialize(is); + CPPUNIT_ASSERT(sketch.is_empty()); + CPPUNIT_ASSERT(!sketch.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(0U, sketch.get_num_retained()); + CPPUNIT_ASSERT_EQUAL(1.0, sketch.get_theta()); + CPPUNIT_ASSERT_EQUAL(0.0, sketch.get_estimate()); + CPPUNIT_ASSERT_EQUAL(0.0, sketch.get_lower_bound(1)); + CPPUNIT_ASSERT_EQUAL(0.0, sketch.get_upper_bound(1)); + } + + void deserialize_single_item_from_java_as_base() { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open("test/theta_compact_single_item_from_java.bin", std::ios::binary); + auto sketchptr = theta_sketch::deserialize(is); + CPPUNIT_ASSERT(!sketchptr->is_empty()); + CPPUNIT_ASSERT(!sketchptr->is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(1U, sketchptr->get_num_retained()); + CPPUNIT_ASSERT_EQUAL(1.0, sketchptr->get_theta()); + CPPUNIT_ASSERT_EQUAL(1.0, sketchptr->get_estimate()); + CPPUNIT_ASSERT_EQUAL(1.0, sketchptr->get_lower_bound(1)); + CPPUNIT_ASSERT_EQUAL(1.0, sketchptr->get_upper_bound(1)); + } + + void deserialize_single_item_from_java_as_subclass() { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open("test/theta_compact_single_item_from_java.bin", std::ios::binary); + auto sketch = compact_theta_sketch::deserialize(is); + CPPUNIT_ASSERT(!sketch.is_empty()); + CPPUNIT_ASSERT(!sketch.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(1U, sketch.get_num_retained()); + CPPUNIT_ASSERT_EQUAL(1.0, sketch.get_theta()); + CPPUNIT_ASSERT_EQUAL(1.0, sketch.get_estimate()); + CPPUNIT_ASSERT_EQUAL(1.0, sketch.get_lower_bound(1)); + CPPUNIT_ASSERT_EQUAL(1.0, sketch.get_upper_bound(1)); + } + + void deserialize_compact_estimation_from_java_as_base() { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open("test/theta_compact_estimation_from_java.bin", std::ios::binary); + auto sketchptr = theta_sketch::deserialize(is); + CPPUNIT_ASSERT(!sketchptr->is_empty()); + CPPUNIT_ASSERT(sketchptr->is_estimation_mode()); + CPPUNIT_ASSERT(sketchptr->is_ordered()); + CPPUNIT_ASSERT_EQUAL(4342U, sketchptr->get_num_retained()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(0.531700444213199, sketchptr->get_theta(), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(8166.25234614053, sketchptr->get_estimate(), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(7996.956955317471, sketchptr->get_lower_bound(2), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(8339.090301078124, sketchptr->get_upper_bound(2), 1e-10); + + // the same construction process in Java must have produced exactly the same sketch + update_theta_sketch update_sketch = update_theta_sketch::builder().build(); + const int n = 8192; + for (int i = 0; i < n; i++) update_sketch.update(i); + CPPUNIT_ASSERT_EQUAL(update_sketch.get_num_retained(), sketchptr->get_num_retained()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(update_sketch.get_theta(), sketchptr->get_theta(), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(update_sketch.get_estimate(), sketchptr->get_estimate(), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(update_sketch.get_lower_bound(1), sketchptr->get_lower_bound(1), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(update_sketch.get_upper_bound(1), sketchptr->get_upper_bound(1), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(update_sketch.get_lower_bound(2), sketchptr->get_lower_bound(2), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(update_sketch.get_upper_bound(2), sketchptr->get_upper_bound(2), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(update_sketch.get_lower_bound(3), sketchptr->get_lower_bound(3), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(update_sketch.get_upper_bound(3), sketchptr->get_upper_bound(3), 1e-10); + compact_theta_sketch compact_sketch = update_sketch.compact(); + // the sketches are ordered, so the iteration sequence must match exactly + auto iter = sketchptr->begin(); + for (auto key: compact_sketch) { + CPPUNIT_ASSERT_EQUAL(key, *iter); + ++iter; + } + } + + void deserialize_compact_estimation_from_java_as_subclass() { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open("test/theta_compact_estimation_from_java.bin", std::ios::binary); + auto sketch = compact_theta_sketch::deserialize(is); + CPPUNIT_ASSERT(!sketch.is_empty()); + CPPUNIT_ASSERT(sketch.is_estimation_mode()); + CPPUNIT_ASSERT_EQUAL(4342U, sketch.get_num_retained()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(0.531700444213199, sketch.get_theta(), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(8166.25234614053, sketch.get_estimate(), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(7996.956955317471, sketch.get_lower_bound(2), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(8339.090301078124, sketch.get_upper_bound(2), 1e-10); + + update_theta_sketch update_sketch = update_theta_sketch::builder().build(); + const int n = 8192; + for (int i = 0; i < n; i++) update_sketch.update(i); + CPPUNIT_ASSERT_EQUAL(update_sketch.get_num_retained(), sketch.get_num_retained()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(update_sketch.get_theta(), sketch.get_theta(), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(update_sketch.get_estimate(), sketch.get_estimate(), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(update_sketch.get_lower_bound(1), sketch.get_lower_bound(1), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(update_sketch.get_upper_bound(1), sketch.get_upper_bound(1), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(update_sketch.get_lower_bound(2), sketch.get_lower_bound(2), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(update_sketch.get_upper_bound(2), sketch.get_upper_bound(2), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(update_sketch.get_lower_bound(3), sketch.get_lower_bound(3), 1e-10); + CPPUNIT_ASSERT_DOUBLES_EQUAL(update_sketch.get_upper_bound(3), sketch.get_upper_bound(3), 1e-10); + } + + void serialize_deserialize_stream_and_bytes_equivalency() { + update_theta_sketch update_sketch = update_theta_sketch::builder().build(); + const int n = 8192; + for (int i = 0; i < n; i++) update_sketch.update(i); + + // update sketch stream and bytes comparison + { + std::stringstream s(std::ios::in | std::ios::out | std::ios::binary); + update_sketch.serialize(s); + auto data = update_sketch.serialize(); + CPPUNIT_ASSERT_EQUAL((size_t) s.tellp(), data.second); + for (size_t i = 0; i < data.second; ++i) { + CPPUNIT_ASSERT_EQUAL((char)s.get(), ((char*)data.first.get())[i]); + } + + // deserialize as base class + { + s.seekg(0); // rewind + auto deserialized_sketch_ptr1 = theta_sketch::deserialize(s); + auto deserialized_sketch_ptr2 = theta_sketch::deserialize(data.first.get(), data.second); + CPPUNIT_ASSERT_EQUAL(data.second, (size_t) s.tellg()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch_ptr1->is_empty(), deserialized_sketch_ptr2->is_empty()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch_ptr1->is_ordered(), deserialized_sketch_ptr2->is_ordered()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch_ptr1->get_num_retained(), deserialized_sketch_ptr2->get_num_retained()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch_ptr1->get_theta(), deserialized_sketch_ptr2->get_theta()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch_ptr1->get_estimate(), deserialized_sketch_ptr2->get_estimate()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch_ptr1->get_lower_bound(1), deserialized_sketch_ptr2->get_lower_bound(1)); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch_ptr1->get_upper_bound(1), deserialized_sketch_ptr2->get_upper_bound(1)); + // hash tables must be identical since they are restored from dumps, and iteration is deterministic + auto iter = deserialized_sketch_ptr1->begin(); + for (auto key: *deserialized_sketch_ptr2) { + CPPUNIT_ASSERT_EQUAL(key, *iter); + ++iter; + } + } + + // deserialize as subclass + { + s.seekg(0); // rewind + update_theta_sketch deserialized_sketch1 = update_theta_sketch::deserialize(s); + update_theta_sketch deserialized_sketch2 = update_theta_sketch::deserialize(data.first.get(), data.second); + CPPUNIT_ASSERT_EQUAL(data.second, (size_t) s.tellg()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch1.is_empty(), deserialized_sketch2.is_empty()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch1.is_ordered(), deserialized_sketch2.is_ordered()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch1.get_num_retained(), deserialized_sketch2.get_num_retained()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch1.get_theta(), deserialized_sketch2.get_theta()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch1.get_estimate(), deserialized_sketch2.get_estimate()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch1.get_lower_bound(1), deserialized_sketch2.get_lower_bound(1)); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch1.get_upper_bound(1), deserialized_sketch2.get_upper_bound(1)); + // hash tables must be identical since they are restored from dumps, and iteration is deterministic + auto iter = deserialized_sketch1.begin(); + for (auto key: deserialized_sketch2) { + CPPUNIT_ASSERT_EQUAL(key, *iter); + ++iter; + } + } + } + + // compact sketch stream and bytes comparison + { + std::stringstream s(std::ios::in | std::ios::out | std::ios::binary); + update_sketch.compact().serialize(s); + auto data = update_sketch.compact().serialize(); + CPPUNIT_ASSERT_EQUAL((size_t) s.tellp(), data.second); + for (size_t i = 0; i < data.second; ++i) { + CPPUNIT_ASSERT_EQUAL((char)s.get(), ((char*)data.first.get())[i]); + } + + // deserialize as base class + { + s.seekg(0); // rewind + auto deserialized_sketch_ptr1 = theta_sketch::deserialize(s); + auto deserialized_sketch_ptr2 = theta_sketch::deserialize(data.first.get(), data.second); + CPPUNIT_ASSERT_EQUAL(data.second, (size_t) s.tellg()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch_ptr1->is_empty(), deserialized_sketch_ptr2->is_empty()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch_ptr1->is_ordered(), deserialized_sketch_ptr2->is_ordered()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch_ptr1->get_num_retained(), deserialized_sketch_ptr2->get_num_retained()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch_ptr1->get_theta(), deserialized_sketch_ptr2->get_theta()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch_ptr1->get_estimate(), deserialized_sketch_ptr2->get_estimate()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch_ptr1->get_lower_bound(1), deserialized_sketch_ptr2->get_lower_bound(1)); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch_ptr1->get_upper_bound(1), deserialized_sketch_ptr2->get_upper_bound(1)); + // the sketches are ordered, so the iteration sequence must match exactly + auto iter = deserialized_sketch_ptr1->begin(); + for (auto key: *deserialized_sketch_ptr2) { + CPPUNIT_ASSERT_EQUAL(key, *iter); + ++iter; + } + } + + // deserialize as subclass + { + s.seekg(0); // rewind + compact_theta_sketch deserialized_sketch1 = compact_theta_sketch::deserialize(s); + compact_theta_sketch deserialized_sketch2 = compact_theta_sketch::deserialize(data.first.get(), data.second); + CPPUNIT_ASSERT_EQUAL(data.second, (size_t) s.tellg()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch1.is_empty(), deserialized_sketch2.is_empty()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch1.is_ordered(), deserialized_sketch2.is_ordered()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch1.get_num_retained(), deserialized_sketch2.get_num_retained()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch1.get_theta(), deserialized_sketch2.get_theta()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch1.get_estimate(), deserialized_sketch2.get_estimate()); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch1.get_lower_bound(1), deserialized_sketch2.get_lower_bound(1)); + CPPUNIT_ASSERT_EQUAL(deserialized_sketch1.get_upper_bound(1), deserialized_sketch2.get_upper_bound(1)); + // the sketches are ordered, so the iteration sequence must match exactly + auto iter = deserialized_sketch1.begin(); + for (auto key: deserialized_sketch2) { + CPPUNIT_ASSERT_EQUAL(key, *iter); + ++iter; + } + } + } + } + +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(theta_sketch_test); + +} /* namespace datasketches */ diff --git a/theta/test/theta_union_test.cpp b/theta/test/theta_union_test.cpp new file mode 100644 index 00000000..11edde61 --- /dev/null +++ b/theta/test/theta_union_test.cpp @@ -0,0 +1,98 @@ +/* + * Copyright 2019, Verizon Media. + * Licensed under the terms of the Apache License 2.0. See LICENSE file at the project root for terms. + */ + +#include +#include + +#include + +namespace datasketches { + +class theta_union_test: public CppUnit::TestFixture { + + CPPUNIT_TEST_SUITE(theta_union_test); + CPPUNIT_TEST(empty); + CPPUNIT_TEST(non_empty_no_retained_keys); + CPPUNIT_TEST(exact_mode_half_overlap); + CPPUNIT_TEST(estimation_mode_half_overlap); + CPPUNIT_TEST(seed_mismatch); + CPPUNIT_TEST_SUITE_END(); + + void empty() { + update_theta_sketch sketch1 = update_theta_sketch::builder().build(); + theta_union u = theta_union::builder().build(); + compact_theta_sketch sketch2 = u.get_result(); + CPPUNIT_ASSERT_EQUAL(0U, sketch2.get_num_retained()); + CPPUNIT_ASSERT(sketch2.is_empty()); + CPPUNIT_ASSERT(!sketch2.is_estimation_mode()); + + u.update(sketch1); + sketch2 = u.get_result(); + CPPUNIT_ASSERT_EQUAL(0U, sketch2.get_num_retained()); + CPPUNIT_ASSERT(sketch2.is_empty()); + CPPUNIT_ASSERT(!sketch2.is_estimation_mode()); + } + + void non_empty_no_retained_keys() { + update_theta_sketch update_sketch = update_theta_sketch::builder().set_p(0.001).build(); + update_sketch.update(1); + theta_union u = theta_union::builder().build(); + u.update(update_sketch); + compact_theta_sketch sketch = u.get_result(); + CPPUNIT_ASSERT_EQUAL(0U, sketch.get_num_retained()); + CPPUNIT_ASSERT(!sketch.is_empty()); + CPPUNIT_ASSERT(sketch.is_estimation_mode()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(0.001, sketch.get_theta(), 1e-10); + } + + void exact_mode_half_overlap() { + update_theta_sketch sketch1 = update_theta_sketch::builder().build(); + int value = 0; + for (int i = 0; i < 1000; i++) sketch1.update(value++); + + update_theta_sketch sketch2 = update_theta_sketch::builder().build(); + value = 500; + for (int i = 0; i < 1000; i++) sketch2.update(value++); + + theta_union u = theta_union::builder().build(); + u.update(sketch1); + u.update(sketch2); + compact_theta_sketch sketch3 = u.get_result(); + CPPUNIT_ASSERT(!sketch3.is_empty()); + CPPUNIT_ASSERT(!sketch3.is_estimation_mode()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(1500, sketch3.get_estimate(), 1500 * 0.01); + } + + void estimation_mode_half_overlap() { + update_theta_sketch sketch1 = update_theta_sketch::builder().build(); + int value = 0; + for (int i = 0; i < 10000; i++) sketch1.update(value++); + + update_theta_sketch sketch2 = update_theta_sketch::builder().build(); + value = 5000; + for (int i = 0; i < 10000; i++) sketch2.update(value++); + + theta_union u = theta_union::builder().build(); + u.update(sketch1); + u.update(sketch2); + compact_theta_sketch sketch3 = u.get_result(); + CPPUNIT_ASSERT(!sketch3.is_empty()); + CPPUNIT_ASSERT(sketch3.is_estimation_mode()); + CPPUNIT_ASSERT_DOUBLES_EQUAL(15000, sketch3.get_estimate(), 15000 * 0.01); + //sketch3.to_stream(std::cerr, true); + } + + void seed_mismatch() { + update_theta_sketch sketch = update_theta_sketch::builder().build(); + sketch.update(1); // non-empty should not be ignored + theta_union u = theta_union::builder().set_seed(123).build(); + CPPUNIT_ASSERT_THROW(u.update(sketch), std::invalid_argument); + } + +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(theta_union_test); + +} /* namespace datasketches */ diff --git a/theta/test/theta_update_empty_from_java.bin b/theta/test/theta_update_empty_from_java.bin new file mode 100644 index 00000000..7456dfb2 Binary files /dev/null and b/theta/test/theta_update_empty_from_java.bin differ diff --git a/theta/test/theta_update_estimation_from_java.bin b/theta/test/theta_update_estimation_from_java.bin new file mode 100644 index 00000000..43b11b0d Binary files /dev/null and b/theta/test/theta_update_estimation_from_java.bin differ diff --git a/theta/theta.mk b/theta/theta.mk new file mode 100644 index 00000000..bae06009 --- /dev/null +++ b/theta/theta.mk @@ -0,0 +1,31 @@ +THETA_INCDIR := theta/include +THETA_BUILDDIR := build/theta + +THETA_TSTDIR := theta/test +THETA_TSTBUILD := theta/build + +THETA_TEST_BIN := theta_test +THETA_TARGET := theta/$(THETA_TEST_BIN) + +THETA_TSTSOURCES := $(shell find $(THETA_TSTDIR) -type f -name "*.cpp") +THETA_TSTOBJS := $(patsubst $(THETA_TSTDIR)/%,$(THETA_TSTBUILD)/%,$(THETA_TSTSOURCES:.cpp=.o)) + +THETA_INCLIST := $(COM_INCLIST) -I $(THETA_INCDIR) + +$(THETA_TSTBUILD)/%.o: $(THETA_TSTDIR)/%.cpp + @mkdir -p $(THETA_TSTBUILD) + @echo "Compiling $<..."; + @$(CC) $(CPPFLAGS) $(INC) $(THETA_INCLIST) -c -o $@ $< + +.PHONY: theta_test theta_clean +theta_exec: $(COM_TSTOBJS) $(THETA_TSTOBJS) + @echo "Linking $(THETA_TARGET)..." + @$(CC) $^ -o $(THETA_TARGET) $(TSTLNKFLAGS) $(LIB) + +theta_test: theta_exec + @cd theta; DYLD_LIBRARY_PATH=../$(TARGETDIR) ./$(THETA_TEST_BIN) + +.PHONY: theta_clean +theta_clean: + @echo "Cleaning theta..."; + @$(RM) -r $(THETA_TSTBUILD) $(THETA_TARGET) $(THETA_TARGET)