diff --git a/Campaigns/payday.json b/Campaigns/payday.json
index 7b966e98..0083a086 100644
--- a/Campaigns/payday.json
+++ b/Campaigns/payday.json
@@ -21,8 +21,8 @@
"adminSSL": false,
"password": "startrekisbetterthanstarwars",
"verbosity": {
- "level": -5,
- "nobid-reason": true
+ "level": -3,
+ "nobid-reason": false
},
"geotags": {
"states": "",
@@ -49,10 +49,6 @@
"status": "file://logs/status&time=30",
"reasons": "file://logs/reasons&time=30"
},
- "redis": {
- "host": "localhost",
- "port": 6379
- },
"template": {
"default": "{creative_forward_url}",
"exchange": {
diff --git a/data/c1x_cookies.csv b/data/c1x_cookies.csv
new file mode 100644
index 00000000..bcc017c8
--- /dev/null
+++ b/data/c1x_cookies.csv
@@ -0,0 +1,803 @@
+842AAB10FBA04247B3A9CE00C9172350
+1DF805E5B2014E6680DBBE93193A3B8C
+4F5776B2B902411A9F06899372143724
+51F9D985BAF9445690AF22939DCC72A6
+5AC65210F9DB451F9CFC4F595DD12983
+AB54CF3C8FE64A0789ADC6232BCDB76E
+D12902D79A2645B58D16FAC78AEEC869
+32EA1930897D460DAB8D45411216FDFB
+84ECB0A561554E2DB8730F8C5750FA1A
+A2FA458A74B24941982386FF64667687
+52B673E309B848A2882749162FA6284A
+DEFB7051823643D78D2CC793CD050C97
+75E22B95ACA74867959F8B94BDE45C17
+872D788E135049A38CE04AD62CAE9CC6
+6F5F4AFBC88946DA8E9D0E1D64886B64
+12CA079080574B2C862F1C2B6D496853
+6C5369675F114546BA8744F6E8CC9D8E
+66638B9625DD464F97776D601E9FE23B
+A8C4D86036EA44A7B0F6F4BB872C82AF
+E61B0858CA244384B840B604D1994CE8
+267E226E19ED42ACA02A5B4CC688E1C3
+E6160AC9DD624530BE333FC9D2EF48C4
+7AF13101A2F14F56B2117A692380E8D1
+272BFBE0BCE5451984237DD86D766475
+9EFDC5D94BD94AAA93AD808016EAA73B
+3F192DCCA1024DFABB84F1892ED34783
+4F54CAD482DD44C2A594246BF36B4FF0
+5E1CDAC97E8042EAA8516D60C0B5CB66
+A743A8AF968D4F269123065C3A5DA276
+34ED788FD12D42AE89F50AC4DA33CBB3
+8FAF58F5C29F4A41B48C71E4DE047FA1
+57550C4DE37043FCABE121215C74D67B
+BBD2D370A43841698EC67D4E8AF67A8F
+F93122D04A274D6E82A4F8304B054D7D
+02881A7C806644599838EA17C9214BF6
+B20A16CF1C674E4F992531788E4AD431
+11667CC9B8E14D9F832F98FFD4CB9B7B
+374AF88B3BC549F29017A30AD4D4CBA1
+5364FC5B8BD84F90BDC5247822B2D98F
+9A8F29F4A5354B519C83A5DB52EC2A92
+$C1XUID
+81BDAF46B69940198D8E91202EB931AF
+5C99146BFED34D889ADA3EDFB10B2CF6
+CCDD531FDC644CF192A653F56D274709
+B2BE03DA47D948449C3174DF4DE7693B
+D8B1F784BD4343D0B769655E9D489EEC
+07E97D97470749F39B16BF59107C97A3
+36790BCD284C4C31A06465A7A0ECF37B
+47DA9A66499F4DB79DEF67A840965097
+7FC4A297389B4375A8B688D6BF0B76F5
+E89023C0D32840FCB657D47F2C0F1BE4
+1AB17288ABEE4359AECF955669DF532A
+CACA2DD5CC1A44D881E421A903ACB52E
+9CA05DF240FD424F8E622DE00E108941
+D26233DD681F456192C68FA39CE27E21
+FA7E506D7741431B90F01DB80870CA88
+2C6877FB35B04DE18BD03D03DC8818FD
+322D826F57514F07B98CE26D72688036
+990A278D7E4D413FB8443CDD778EC2B0
+D9F6E256ECF84E6988C4BAD367B9FF6B
+8B6CA16297B04C588ED59B90365F3388
+32344EE2A12B4887B914A2ED11E68A31
+6A5891EE1EA541BBB50ECECE5D6B434B
+33EC00B5AAF942DFBA894C9F51F9233F
+78998726D091469BBE081FA7E27885ED
+955ACE8711D54DACABCAA3018E4752BD
+01D06F28E18D4F4C8205A5A8964AFF3A
+BDA84C97A9894C14869B5DCAA967B238
+F30F3C4BA86C46BB93867857FFC3D37A
+F6FD4CAB0F8C4D5E8ED0D5C21B98FA51
+AF625D5808D34C6C891DD7DCDC26257C
+706AD983286347CF8AC03E0AB9650956
+EE2A63AA196B490DA787C326B3EE1FCC
+EEC517397D154500810F8608C77F7489
+F5FC8170CDEB40E9AEBCEC48C4C71529
+853865FA5EE1489780A457AEB9144A1B
+D8D95DDC4A114AA3B6F410D09595BB71
+77CEC1BD91284DC9AD6F339C58F457A2
+809082E044464F17A2C40C428C732DE1
+B9039564450E48099A0C54A12E663768
+49E42C7CC88F494AB04954184BD08D05
+B2CA7313AF5B446AA2D87DDB74B7F3EF
+92D6601CC90C419E8D65198796D88EA3
+8D6CE3736ECA41D488CD716C9FF9FDD9
+FFFC279CA849442AA1012AC4FF6218C9
+478B84AA0CC1421C9FA5AFD37053F447
+4CDD0785099243248C26D3E2F29C6F90
+64EE223350844877B1C2E9BBEDA9A59B
+CA2AB38776E343C1BE48EFCF9AD61766
+D9266CD552344CB69E1931696083F51A
+51EA32E255A64F8EA822025118AE9C42
+95C05AA1496F48478915462BB43A6199
+6629CCA37A004554B36CB891F2A3DEAF
+84137B33B9A0446DA4674FD541C35BC3
+FE16592488D64933AE453288AD37395F
+12E884D01F494F48B974E4502A6B2ADF
+7BD442FB23804390B31252C8AEB17E1E
+75F4B58C49254262920975A77721865D
+78A9E3668EBC4155B5D853569F33C7B0
+CA603D9D204342A0AE05638901017087
+D087E06D5A9F4E06BAE2F82C1AC2F6A3
+FA9F4968FA8E4953A6F97811FE3FD5B6
+1E714E3A22E54CA29575326789FD1DBB
+9786B01215534DEB9AAC2D5FEE23A497
+CF97A4C7070F4DB28463F17C56CFFE8C
+DAC84E61002B49D6BD9C7E8E7990A323
+9247D211715B445887C749081D0259E7
+F8FB05150D574CA3A8159ADE29B8BFA7
+35F47CA308C04819AA94C31548B35761
+36D31B977B324F0E9A87A2B660759809
+280223D72C1A4ABFA44A1DC46240C0D8
+D85E0656E67D42C99260CBE5660F6372
+9631DA92A22043E0B953AC9D49A6C803
+1556D5725F064FAD9FEA2CCDBB46178E
+3D9500FCC3824A239782805AF864A4F3
+942F7851B9914E0C9BD803E41A5A2113
+F044E6B403F245E1880A3931D5250218
+930354A257A641A8B7BCF8563C7A844C
+A62D7F22D54C4165A99A52E9C547A7F1
+4A31CE0D8E264848B91004E3E28F99B1
+5EE8624391CB4ACE9001123DF2B6217D
+997FF26D3FC24030A9EA76446B1473C0
+18CF4AF1CE1B4D7AA035EA4109432063
+24C04D2884ED4EBCAAEEF4A2138F4EFF
+7AD61431CDFD45CEADEA4065D4F5156A
+B970CA47167E46EB9AEC87427534D0BE
+E0B5BAA87EA842EC9F5A991118ACEA39
+D392F08C69E44A1891AE6B570D177485
+B5311E83832D4C4F9041C2182A72DC5D
+E67168633FC349A1886A1383FB721158
+A0F6BCDA1DE94AFEA1AB0002D773952F
+D964ADBEC5944140B12A483A3DE79256
+FE8513F6E790443FAEA4A42452262E3B
+2ADA72ED2BA242E28C838390EC9608F0
+3F2629708BFC4D4B8EBEE87266E9DF6F
+11A1A8B2B3684BC38820D94A44EACF7D
+2169B464664D4195B3CB5F57440E69F4
+2761811180D6485987790527D9EC452C
+51329589A0494A399B130B1BF8502FEB
+5894FA110646462097B09708453B7418
+B057739728D646B9AE53F076CBA6A6BE
+CC5D271FEF194D7E835FBADCF76587DE
+E6EE48C4F58C4C2DA3FD1C1702AB356D
+622FAADC08EC4B7480B99562F6406C7B
+CFC4F35F5E024D5683F7A7600EAFD085
+0EC230B76D4F41878F95D14B2784BB22
+9F6A40F51E744246B8787D69763597DB
+6A455FF3DD884EAC9D6AEBE8E3A818A6
+479BC24A4FBA407AB8826A4F406BE2AB
+97CBAB51C3194F3FB9C917F0A20E6BCA
+2009248A165241FAA2D65F185E1312F4
+AF3F38B64D274350AFC42710DBD07FD3
+2D6F22655E7A4D7085013687131DD5BC
+DE29077AEE5843E482CBC9833A405FA5
+EAA8C294DDFA41CDB144210FC02FB63F
+4E28D90BEA3E47F48E71F096BBE0483B
+1F4EF005155B4444A2037CC656BB8F95
+7F4E4DE2FFEC493FA5EC822AC4950D4A
+555D58DD44BA46559A344BC2B09DEE63
+7D4DEE41A148461F9672403443B33DE9
+D3F10A3223E84CBD973EF181826786A3
+6BC22F79E6734A6B87E66AC188DC050A
+DFB32C60EEA047309FF217FBB3C82C07
+9A24DA29E7004C419DD09CFDD7ACF970
+B88315B322184AF98777A60BBE9D3376
+D9447626366546868285DD7084A9C8A9
+822D08243117484989AEA6A9290E2A73
+6823BABBE69D4092BD456EE0DA10072F
+672E1FAD47D74B07B6E86A84C102A3FC
+02CD7832A1D14A478671ACC22964C8BC
+D2995F61F6EA444CA9FAC735EBD128E1
+E9A870E86EFC45DE85EA86C6099D54D6
+466D0EF38712486DAD600C2D28623364
+BD9DC1FDB06243DDA2327A1BC10D915F
+A575458EB6CB499CBB1C5CD6676CA225
+DAE57DB2318049B2AA44A00D4F7C5BE1
+7DE931B4459F4460AEA4DB305C4BD739
+997638C631B44974A9FDC4E6E18AF865
+3E28E8B0FB7046788AF65D8544AE82E7
+80AF1AAB04F34EE2A035D5091AF03C82
+C85F83AFB3744E95AA4AECBEDC9B1BA9
+F35AB5C9266246B396404877B9EE8BE2
+354340C3F26B426B97D1D8B922B7BA6E
+7A8DBF93402F45D7B48F7A25E0511366
+CE440A6E8AF94594925CF09D08C889FA
+3FF573A6E21F40B0AFE592DEF7A7FF6D
+A3116404F87F41CC88AB205C3B9482DE
+B1C6BE8BED4C438691A72B1CBB8B52B7
+9D70D006FB604597905767D4533E66E2
+A2DE2A03D00D423A8757454443D99DB9
+F38D50D677164C53B224BF5533AFECAE
+BC460173D1DD4B92ACA3F36F1CEBEDF1
+F4252457604244F59970FF09E6BD7E24
+B98E7B11432C46C68221823D04EB9A75
+0E5442023C15497DB7B32606FE272CA2
+1A9115AA921F4E19985A4E2697B4BE84
+1C098AD2CD6946B9A905546C7ECA8D9B
+59A27CFCA0844FF2808F411A5BE4CB43
+A0CD1C2D8D354015A3E243F795A04013
+9D67FF1100DC459F8C1D09285E282553
+17C5F6F82D9142D480CAC034942A1535
+2D09E6C3D2A04DBBB6B966E87CED76F5
+EA29BF048DD5427AB27D16FAC3150B7A
+3A85B67F035445E2BC3E1B1963925C6C
+5BC64300C8C24502809021940F146602
+73C871FB212B4F2698B1F07244F669BC
+91DE8FB4567B410C81E2CC5336503BCC
+FE2E00860D7243749061DB63060DAA1A
+11E7BC505F24467CA9A72E873BB22F4D
+A952845778BD4FB1A4A9D39536B4B03F
+5F35D658218A437A88B22AFD55FA1932
+B5D7BEA1BCF84B439ECAE42C602ECC95
+527A3B9EE28C4054BF6EBB9E885FA044
+A55D2DB2D52D4700A2F7336CF1519578
+1F698ECD0D504A45BA28EE130A19170E
+40A2007E841A40E9A3690341C51504AF
+C885E11504084CFAB60E811A0D4152B8
+E18C00B39EC541C28B10436A0141A3C0
+032149DFDDC14C7A963EA40265E6AFA2
+217D2ED421104C53AD77990BE5A78B22
+4BCF1344AECC4E67B59A88CCE7C137AE
+63D07F91ADC949F0860F87C48A4AD557
+31AAA33BAD1A4C10B56E279CB2C0147B
+4A927410D29A402C801B859DE98AE98F
+66DF0284B4F64A51BC3579A4E25ACEC9
+E319B529F0014F5790E21C81C9C0D0F9
+C207D8F9D84C4CDBB4A2A51C0BCA167F
+58414D142E284160BBEF8D8CEBC5E0A2
+67F9FAA217A54A2B8AAB0186D529666F
+A0B6E3A74E3B40879AB43AD613332C17
+4D2D6FC024A94AAEB7EE8D609977F30F
+E270722379D7436EBC88B33B9172C41D
+DA827D8914D24EFEBFB446E7257F1D1F
+0FC96D3567304FF5892D31F8A2B1B42A
+AE7BB359FC2F4CAE8677709BA489B78A
+94AB6223FDDA49C3929B3A09F9EDC74B
+F8590202B9FF43A4B9C013602C270702
+519BDC3C0F3A4D369F43F86737F3C550
+A4BC42688674400784DA9BD4B1E528D7
+9EEA781952754F4A9C6D4BF5F6B69165
+DDF87E447095424FAA884056CC19689A
+697B7F6DE7CF4146899A0FCFA3534836
+96C86362A61A4101B4012D0A24E9D3FC
+C9EE72367B854551970B797D480C970C
+B62AAE2023F84AE7A013533A2CF5EBB9
+158618F8BB6A43EB8C070274E0E32270
+25D3C5EAD5D64CA29A24E445740CE12B
+006E7CB502D64BBE92EF9DB273BE0132
+C5F43B00BD134F96AA494FAC8A230CB2
+DFDD9B077C9A41F7814C289F40244384
+0A843618E3D74724AC88487BECBA809E
+70F666C9FE7848E1A9DA0A4907B3B505
+941F0AE66FB742939733A29E26486373
+B40D58DD0FD345E28454DBF89F7F0C22
+3B2931A2380A4B8DB36CFA3B1C7FCA54
+91D312B59EC74B4BAE6E2160751AE0F8
+A1FB59CA083044BABEDE767508215C23
+55F5D9A2DC28465081A717622373DFD5
+C46F2FCB48534D4D8FAE5BB428EDF3F6
+37D4DD351CC24E179E46BEA451053818
+51457A4556BD4E319AE22A0EABE0EBD9
+8744231BBE84441888194CF510F93A50
+0172C25317F6420CAF304C0C7FA8EE0A
+9CBF7FDE2EA6435A8110590D7EB2BFD9
+FE6E2476F3804E0FAEAB8BD017399F37
+46E61691C004436E9D2C30EFEBFC659B
+EA82900B45324177AD3FE983AC5A3531
+4490C876694943B3A33E679420ED1E4C
+88168F2D42FA4E999FD26B69836255DB
+3893C462B872461C9A637B45C6D1D1EF
+E17D548871324CCB83016A25A7C73417
+8DD37E33706643A3AF5A6A0263BCAB85
+EA8AD61250E044849637C1A6366EEFAB
+3C1A45D3ADAE426B94F1987F83FA4DD4
+B7E8099E423C4E669FD947C581969A14
+B180AFC265E64A9A918E8D2289E17BE8
+D90211ADD7DE47FC8356D4D0A966F0EE
+DF08AD5CC5BE4725A04BBC439CF18B0F
+1F6FC80A9BD643C7B562C913CA4CA49C
+94ECB6C509E3469FA0EDBE93D4376AB7
+C9BBFC44F6324E56A00FE5E5CFD75E0B
+D5F8257009F84268BED1125ED453F421
+4381BEE727DB426FA9953C54F0E12B7E
+99007B05B81B40F087496E198E6BAF76
+A6D3A993763A4FD791909029A95C7C98
+5700AFDC479B4252A2B98C3B11F08440
+55842040C0A74EEA8C0849C670D7B2CC
+ED49081D2B114DFAB4BDF87FD60F914B
+B7F0554A44174C9D8AFB2FAEF7CCF3FF
+E6FCBC267780497BAA0BBBCB92ACEE22
+91E4A742234648AC8D44A84DD4480962
+A99972814B3746FCAEB32D96F6902B2D
+4EDF2337C9B54FECBC852B0E301AEFD1
+8D1A503BBDA64649BFDD4FC88F466FB6
+A4F2C0C6B2E34B7CB1DB97C0A0E3E3C1
+16C8146B8F4A46FBB2A755B3330ECD2B
+5A735360E99C40D3BD9D61E920923BFA
+75030A0CF2EA46B7AB1462BBF80FFB01
+5B28BDB7E0D84626897476178C2BB062
+5FB642BBE7194BC0B67584AB8353AEC5
+9143A09DF2084098BA784F11AE1818B5
+E89380D4E4584B5AB115E3D0996B06D1
+6BF296A238014DD8BC8D273522E48F1E
+2BCF4D527254441BBE47701046B8A2C0
+9E88C5EE50A4462E9EAD7738F08F2DD7
+51FB087AE37C43EF8C7C914889D221C4
+DB4606DC6EBA416181E184826F00B891
+377974DCC2A54317B3F3BA57761047FD
+1E92B5DCC80E45CABC935853091C1181
+4163EB7777CB4BBA8CA5311C33D1BFC3
+51FDAE6D77C44F18AD5BA4F959555AF3
+62DB79A1FA004E4FB753D4116656F73F
+363276A36D264ADCACC00D9747184BE5
+4D16566C4F9342C1A6038A21BCEA5775
+7869162E62014DAF988D0D9DBA7995D7
+80119704B7AF43CD8DB7B11447364DF7
+B249B1BA6F2E4E5C8572F8030ECEADAC
+E07B9607755242D782A68E5D7D0EAEDE
+FFFE7B254D594E16AC6E4181BC04336E
+F6EBF82A000F41AD8F402A59803E62E7
+399F46F952334487B8A28ACCBBC9B838
+7CB082B554F54EB683070331D788FD9E
+2D140447429340ADA267B9950636A4ED
+87077D28A31440028F4437CEDC14DE57
+057B46E153CA4C2DB7D3AF9E7FB92450
+FDADF2765F074208B0383EC8253953CB
+6FF98DA9215D4D9FB6030EB4B5681206
+3FBC630D2B844D89967F733F03B58B0E
+67D9C73375134505B9C072CFAF1ED4D9
+E3C66C179B664E74ABE0F62E2C45CE12
+7DDE4BAA4961497FB85874366C93BE79
+0A43A328F49F4288B07F46EDEB8F843D
+1FB48A8492E446F897158DFF765A1615
+BA31CB4DA94A4B238FFFC7489C721F97
+9105EE22FE8C4E1AB13BEF994CB02E3C
+4FC21A4E929246C68ACF5B6D8637C05A
+B440978817934369AB89F76DC96DE2EC
+9D3F7BC112404E849A056CA9D0A3CEC6
+7CC9CDE2E3914216B6B0C5050099AA01
+966A2DD98B8945EB97A145DA01109F98
+B5C05A607352438BBD78AE692C604B55
+5662095AA00B4FC3B6799F24E6A025A5
+12FF4AC48565445CBB62ADA56B8FCE73
+325E801295B74F628F27D8CF6F38FB88
+5A2F9E9839414215957EE7C96AFED912
+0FA970744EB94BE5A5FC246536AD0E83
+B2AAD3B703804E399D06F8B5CAA8133D
+BDD8BCF1B00346AB8FF9B10035DCE660
+DD0955108E1E4D5685A85B6C71D8FFEB
+DED74B2326C24186AEBD92B012F2DA2F
+79DC61E84BAB4FAEBF5055C93CA2902A
+CE4DC71D4CEC4F1A80C782254EF4F609
+B2943723A63341BFB5709159C7C1D4D5
+01CDD30C270A4E158795E0B295CE541B
+B79255D458B94A248F31AA6209357878
+C7118F45149E451590007E73194B3F69
+203C6F50CA5D40EA80E74E257D93D8AA
+3D99B9985B5641B199C07B631C296687
+D83F71569E1B4254BA296FDD0DB59D70
+9F5C0B9518214FE7853D16E08C01A761
+2A0F89C981894FF6A2CD3A31247350E3
+52D61324C769473389B07055FD997838
+3E8E9D374CAD4809B9D3EF7D33054324
+7F0BDF85D7F64F068113C563FB6ECD9A
+BD84D2BBD6544539B9FB3878C2F5B9E7
+689322F9AC7B483C9B735A54D9F0553F
+7F8F8F13A2074925B241890FABBA0762
+B21617C2AC0A4FBEB494C371B9FDEF58
+211F777958584F58B3138C9AAB310663
+FF34CE28C0A24C32A7342B3D9D3911D7
+0CD3775E0C9E4A15A1669556A8DCCEC5
+3752F6EBACB44863A40246BF883B50EA
+75034BDAC26D44A59DF1E2397785FC25
+15E8BE139AF04D94B0EFE5E49AC5C9BE
+AC09BADF9BD147EFA40EC72FDD3DF167
+7F6CC9C24C934986A2B5AB8BA6EC4AA5
+F43D5662B0CD4415B893812C38D6C531
+24B9DDA40F9D45858E2D9B1450B1628E
+09812CF21BA9486CA21B8F24A4C90E20
+1751ADEA5B814FD18D309CBDF7E0F196
+D2BA7402D12742F487EB4806696DB450
+ECC8BCE37ADE46BFBF72E947458B5C95
+C60B1BFA66EA4FED981DDC6B6531B8AD
+D3ADE02FD4614D59898610DB783B9E76
+08A90C61C01240C08E12F208E2314EC8
+46A76AAD77D146CE9A6B98925E6D3EFA
+43EB6DEED3FB4DD8AE630F755E8C637D
+FCC539ECB8FC4529ACB426CE60A7CCD8
+4C513C8CDED54F1094BBC5D31D45BF53
+F39AA91E43694B8392FE60418990C121
+12003DB59EBE4686BAAF63CD957AC793
+D17D25E6CD854C16AEF0D749C2F7A9BC
+15944D4877E848FFA98B6CD206D3E988
+5BD1228F98C44A6EB2B024F57482A6BA
+1C890FEC6E55440A8E842256607B16CA
+37FAD167E3FC41D88F8E22E8DB52437A
+6197A72FD7894623BFF0606BB576C267
+85A21D1E99764774ACF511A03D4C901B
+489511DBA17D444999DF970ADAC10049
+15760FF3AB224305ACB4BBBF6C024DB7
+8A44890C038A4414861B07444FCAB2FE
+C6E96E72A0DB402687F2CFEB92395F93
+3D9F6F4B140B47058926EB3FE9B5B219
+EB11582143DF4DC1B5C7A9DEFF594612
+FCA986ABA5BC43E6B826BDC674E92F6B
+5E0BA5BF05B34D15AEE2CEF53D90052D
+5778A5444E5244BFB71790BFC90912A4
+3F4123085A974013B479754D45E81D18
+96AD730E8076406384C46DF87D3FD872
+044E15B16452489A8755E4E5233A94D1
+73A435FD647847AFBB1F3B12CA067B90
+AD2A0A73153B4BA4BF7501BDCC086B6B
+E79B2A9BEA884537A350C5A3A2905B04
+11A8CF31CC5E4E38A3B845A8E6BC4151
+21897DD54247470FBE817BE320EE1E07
+C0BE371345B84BB0B8748F59A3BBFA16
+F5357E0DDD1C492890586D204EAFBF1A
+A8FDC32D4C554EFB89EDB7A59818DDDD
+6EF757C3EBB948519F160B30F3B60BFF
+C0DA486BE19F4AB796D0876258AE6B4B
+E4E1ED47D8EA497EA284699E459CBE12
+203DB049741541B4A35C0885F0159354
+7105241AB64E48198708D7F30ABF8683
+CBFF03D1E78D4E35A304827A3D4EE8C9
+EA4103901F4546E88675582D2871C5EC
+30D991129B6A45058C6CEE1490736513
+88503640FDAF4B17B46CA94AEEFA9128
+79B534050115480182F268B0ED064F55
+F0D76144672C42B79E7C20EAC5CE818C
+45D0D43753DC4CB49F55799F93A9575B
+A38979A144E84BC99A2DF7D96517C300
+51A8F8AFAF0140519C351E6ED53A812E
+C757941D43504E35A6AA843DBD691DF7
+362BF9873C634B248CD0CC6A2B072CB2
+E2D9EDF8C1E3407690CEAD0015180802
+2D4F84353DF14368B0EE0E360B9F1D6C
+8FBF287512704C3797AF250EDC7E00E5
+D477F3B107664AFFA0954B328116E821
+4CD60D7FD5AE48C3BA6FFBBD967A17AF
+6DF6566AF4B0488CA1243C98D6CB9EDC
+7824D7B4A5CA4B78ABBEB8AE2EE68256
+97BC533801C84E5EB39E7856353EDA84
+D4B92FB162CA4E78AE6E018EC8B7F5F4
+A074ADEBEAB1404FB5C4304978E814BC
+D6412E872CF145ECA4C22275075B9071
+911FB29353BB4AF7A50D4284AC93A68D
+257C6611858B4FEC89C458B5EA422B93
+370656C5C1AA4F49A63DF1E8B5922406
+060D57C61BF94DDAA197C908F238A8D2
+5722AC2189734D8CA87C30A8BC7C84E7
+E9AE4C0FF2FF41FCB3CBDF3F8B975D4D
+32E80E60F8534B728499F2ED0EAB217B
+9A430142B93C4C74A026B1882283540A
+15F7F77D9CD943BDACCFF32F3668E4E3
+BDAD39CC4BA048878F80F9777607BFFC
+9ABF8F5EC94B4EAFBCEAC3D905CFA88E
+6049796687754516AB4E55212A08850A
+FACCBC2B2B5E46D1AE1F5A4A969F130C
+901C145BFC7D43F693BA4876AF2AE2FA
+AF9694DBB22341429A676583846BB77C
+D77CA8143D744F62B220EF06D14746F2
+FEE5764AD7684962855C53B63E125BEA
+401A712D0D374400A8C21FFEDEC7E15D
+77B106B4049E4338BB8E9232A4B315A1
+6EB9F0337F344C509EB28814C7933B31
+4988F7316C5140388B9FFA0F6B242C91
+26736907436F42F6804D9E591C14EB14
+7204CC82D3184D48B1C7397158E92D4B
+C85C7EB34B234F19B63C482726F72234
+5868CBEDD9494575A96F4639B433DE7C
+A7E8FBA6AC654CFA9A11950A4D104914
+43D4DE527E7740E6B3E1EBDF9EF7AE69
+578AC598A6384F3CB82C50EE1F1382A3
+D0C5333F7CB645F1BCC03A15524E8D16
+F2FBF023F4AD4D4B966A373F0CF74CDC
+3E8B285B3A8D4854B0D7D30EBBD8C7A5
+C801B4CFCAED4BED9950145A5DAAEF87
+0B4A20E835C14638922A408CE1FB0EBD
+AAD2FC5B4C224C4D943615F986830D8C
+B73F449CE6664BCFAE5391535C2C66CB
+A44CED7C3FAC4FF2A93941CB9919FA47
+55A43194665141E9B8FC0470F49D058E
+E12760F54451480D94D3F28E6AA78576
+EC78F0A96B37416486901468EED3F6ED
+35DC5A270299486BACAD896A0771EFB3
+115603BD640B4952B8F89E63309D3746
+3D6073BBCF454B0BA1F90C45C0E8F9F2
+96D07C9588BA4086B11F2E6605C93DF9
+05BFA1F059FB450494D6D3622FBF6CDA
+BA0D899F17DF4BB7A19BF9B463E26A66
+D897C4EA40DB42A785B8C6DE95542B13
+9A5D686BB22A4D87919356E1D3F8444F
+B22047CD2D5B41548164892D1D74DEF5
+648B3A160767456DB3545578873AA472
+BA62F60C490C494186052714AD2259A1
+DC7DD510EA264CD4904816D0F36C6EAB
+E1A023F4F3814F4ABE672CFE5FE85B1E
+29FAC95B462944109CD73B89BA12A9EC
+A5164ED1A5EC4A83989EAF05972271C3
+1A1C01B32C0C4CF3B66B14D713D1CD4C
+320CB77FFAFC4373B67683DCE2CECA84
+56B8BB734B7B48E4B6123EF078F131F7
+D540F92FBE224D4AAF1E92729CFDD038
+2F853E9A4877461BB6DDFD83614F2783
+DD5507A6D7574D85A11AFA546D673234
+6FAA465FF8074D97BCD6CEEF8CBC6A2B
+A91A1DEF56DE4BDB9C796ADD929E2BDC
+D627149E889C400EB936F6E0BE917C2A
+F526BA622088473281FA2FB47ED622F5
+9CAE76B318694B549469FE17D88A9A8D
+7BCC91EFCD634588830AB81474A3DDE2
+9C7FA8A9A98E483D8F9D2AA9939E8F1A
+742C4612226A4545925F5A762E558FEB
+A347D76E1BD74FD1916E9599D63CC64B
+B9D0E6EF602D4A58A39CCF4F5170C2B2
+2BE6C9623875499EA3E08BB64707175E
+37749144603641808232CDE524D64616
+ECCFA0D461ED4A7DB75B2076283167AF
+FE74A70C9BF24C4D9E3B65BB518DC865
+1F9288401375457AB4113EAA6722A7B5
+6C883372DE7245CEBDCF0E4D8D508937
+3DD6A69FAB5649CA94D97631EFD66206
+F2D26BAD82854B07A78D7EE3AD27631D
+87083D3082B447B7932A682127F6C54F
+EF12159799984400AA32F27D01268E57
+69D6B72C5AF541FC9C2EE7AB7E604F06
+35ACEEDDA5484B0B9F8CA64C781FDDDD
+98D4EFFD552643CCAE4DC7E48D97FC74
+A31698B869284674BF3718CD8F5F3DC7
+99C9FABE278F4D8B9DA892B2F04B988E
+75F35BC3482C4067B06A53EF2F13C4C1
+BFE3E5A1F9AB43C2B269677D2451EA51
+F27EF325C30D4F94851701E3943025F6
+9E7A512871DC47D3B0A885D95FA68018
+E91C40FB931A48DA9A07B171B1B22BAD
+809EE1191423477BA1CFC856ADE0AD8F
+84450CCE2008432B8F3ACB08A4E6E595
+4E61508ECFBB4FD7B003FCAA5DFC81EE
+AEDA3B476B1C4ABD851799726CA6CBED
+F1D1C0732F5A45648A08EEDA613A8A93
+2D48299EEDB24E05A46D801A6E79D0D0
+7FB510E7133B4A20A85717194313D753
+091AD0876FEA4F01827E8040853C09F6
+7CAD012790CE47AE8B689CFE62F05433
+40D148370E9D4535944A4D687D16D1C8
+AC367D089D054CA8B357732478A08385
+E654371CB7414D3F846A455893332988
+9446721C6526463D865F49A11FD83E3C
+D56B696784A94413A4B07556F00DC3AA
+0C3B883793F84CF8BDC5931DE09D26A2
+3F648A03579242DC9E3497F69FF11CAC
+BEF66FC8D6B0435A9FF66DD6A4BDFC08
+3BE03E0C02FA4A7BB0C01065C062BE50
+C50D4ED469414E9DBEDEDC61979C8404
+553E861503994532A4E8E30F94AD4D34
+5DE0CCB75DA14479BB48F784AD82F17C
+151C5441385F43A394289BFFE7A38EF2
+44B21B2D22D34D7BBD307422F2159B31
+768A0500055540138F72F6D5923C213B
+E28C08674AA042AC9B82642067462F06
+81D57D9B616B47DB891E43CDEC995DA2
+2D08EA85933547F89052751A0211ECDF
+7863EBBEDA1340439ACFABEF19420EB0
+03168E41405F4996826BDE50A338984E
+A2E7BF86338C4462B0308BDE9362AEA9
+6358EAFE013142AA9D768D38DA56B582
+834919757208443698CF8B080F159896
+A96517EE47B74DF4BA45ED3D8CC17195
+7192BC3A56564FBDB4103E8AA973846C
+2963F8CB89CE4235AC3EA741B20A86D3
+32A1F87E39A34F5090CCCF5D61592D47
+ACF02F6E7E514BCFA32DB532386CBEFD
+BEA2CF8206FC4180B7AF420974D4646F
+B4AF53AA6D3C42149BC8E5E55608D9EC
+4391B76BC3ED4C17AC261C1421524CBB
+3DEECE574FD34A4A8CFB92014463B64A
+AD046A4D09AA4A57AF31EE7D33238777
+0B95A3BAA31E49B895652F64A41DDD60
+9A02C398E74C47E98D0DF4A5706720EC
+9E404818D34E4F0AAD463B45B30D4169
+029844ACA4EF42C4A212AC8F8C325935
+1AE42B48FC094A48891658AA482E72A1
+CE793045BCAB46DCA7B8A42D298590CF
+F99A2DBFDF004D34B0462E0EF786B603
+9F2DE0E62E23498FBAE3A6EFFF0175FF
+3FA2809B90584D5AB694BB9D1168543E
+C6EF272202584AF880A71FBF55768706
+F51CBDAFF6A9424DBD1E59BD56723B2F
+66147BA39D344A6FB859D91BF5192C1F
+829B5196D70C4C62875FD74C40339DC0
+26C3894F8118452BB28FC1822FD2F8CF
+612A44EA9AF94A77B05AB4C12E3D2AF9
+7CC616170D334697927F35A558851238
+7F44E2517A754C6F8AB7C5921EC24035
+47608AA6FD054CA592CAA36985BA8547
+F9541B0B0EB94795A20F797E65B72BDB
+25E634BA094F4F408906E1EFB81582A6
+D225B17DD0C14DB1A83633AF990297B4
+67D335C593B644DE879F9C524E0E8E36
+E78DF1725ED2404B9E339EEE9F955B13
+FFACEA276AA546048A6F308728368EBB
+261A136975274846BE791A60519449BF
+D1F59FAD20294EB69E594C35077F9152
+EAAF0C2E7C6045C3A2C20E0C6B910011
+CB2CDFACEBF841A58B5B5D2848A0DFA7
+98DB73B7FD284CD2832347DE8FBB2033
+EC5A11494013446291BC614FF138BA15
+03E7173ED58C4807B533FE7219A997DA
+B37D290718DB4BBAAF0B9FBE82CAD2CE
+EF1ADCAC0B97464BA3D3125C37C7454B
+0F8012796EA94478B6B275DD15EC0CB0
+22EC5D0FA5274A688B90C3BA7E0401A8
+4F899FDF667640B3956477DE74B99997
+3045BAE970C3457E9B0C526A5F2EF554
+A0FADA3614724C29944C48C44F866E47
+DE0A1C93AD8049C395E716A52A3F37F4
+4947BD9D258944BE85E96559BDCC6012
+E28C811054EE4A68BB089B35E6F5EEAD
+1D8FAE231F4D4C1F936A7D84BA53243D
+3B81CEC92A6C4767B72DA33243C10728
+199CA016CE83403DB1CE83007BD94304
+5C7066AE9D284FD4B28609B09557A2FE
+81DCA231C0984600883590EA1DCCEC7F
+A4B91232D2CB4A26AC9B34123B060323
+1738889144BC4FEE81E43412101DA966
+25B42838ADDA423C836FF80A4DFD997A
+7086E9D380304E128073D662544E37C3
+862E764B08454B4D86E3AE397DC2D672
+C1D84035380F4079895D46762BF3F2F8
+87815ABC6C804EB0A264E23CE3664F25
+A39AD9D65E654BDC874C276285EC1790
+FC55ECD4285A45D8A12A44BB9F7746C7
+022D9DF63AA94182B775A78EB37939F8
+3953D9D35CB94BE9B130A1353255A15F
+F0423688D55549548F82C99B079F9315
+7F168F9DF86741F3839C5BA8401B6B21
+92BFD3BDCBA542F1A5C0F4DA2EAAC680
+5F846D5F721943F39888E243E3EAD5E5
+349023242A07487DA625A6A17E6F66DF
+090A16C83D2D4627B326CEC54D3CA8BE
+31AB00ED6D214516BB0CC7EC2D2F3DD2
+CDB4D9B5120B49CEB417563830FC6B02
+D605C892734044EA86D5B8431FBAD939
+3BD5A50869114A77A59D8F58C185B571
+BD65590DFE1045F3BF6A14113FBB21E6
+6B517C314AB342E7B1618EC023A28F9F
+D2E93C8ED0A5477CB72776056E86D680
+F9CACF42C83245D78755CD7C7AE1EEC1
+429F3D67AE854E8F9FBB9974E2ED7A1B
+5879289D7FBE4AAEADFD6B305FF107CD
+FFABDEB75D6F4E999979E4555149816C
+12EE2329ADF345D9BC33AB5974C35084
+814EB12187AC49EB85B3BECE8BA9C504
+82C4DD5A2EC14396927B931CCE3057BB
+2B620E7D9E7043CFA8D5BABD1F2BDD2A
+B65F226D922B49EE9E1BA3219DEE392C
+76720EDA845E4219898ACDAC4E3F740A
+3EB89FA0522240EFBAE791AF29787E37
+67DB162F3B244B5581F5D6DE200D0509
+1C84986A23524D659683B061421EF221
+91C2584DF35D4D7482DFE349CAEDB82E
+C23F767DAEE04B0E86C0575151AB979A
+639FAB1ACFED4F5CA3257CFBA17A8090
+6F7963F99DD04541883E8DF6B5AB05A6
+78B62739452445ABBC41DFBCA857DDDE
+B375D270C532438FA0299F92648F6C92
+F8A80CF6E3F343989F7B6A3FD0257B5A
+D7429D5AFA844B74A52C6BED738BF193
+B5EF68A538084113B84CD2BAC65E98AF
+5682A3FEC1244CADAF47DF1425A0D251
+668D9A8BD8B84AFDA7974987CE91A8F6
+2694AF9642D4444DB19BEB6D076ED428
+C43CEDFA944846899999C01794FF349B
+77CEEFA2C0214ED1B27A2250A8608809
+0029F61461A54AF79418BF9A65452287
+F751109FD4314D489131722D8772C094
+02F9E56D3B9D4A069CA714F00829EDEC
+73CF76B307CF43B99F3E5CBFECE8C884
+7AAF0925DF124164BAABA9C6BAD481F6
+1CCF8C683B964E1EB6E48EBE4EE98C74
+4AEFB350A4FE4C8899B687E379DA823F
+E604EC91431441A68F75506E2CD20B9D
+7ADEBFE735274546AC12005A8E4FA611
+8856D48343E541CA80F2FAB70F7A8378
+2913E884114749F4A55C14DC516C526C
+E10DBCEE43864590A63464589DA53274
+E91CAE410BC04385B1ADAF2A697DC024
+C0CD84557B084195AF9BBAE19F62EBE8
+6AAACE3FB9074729B7AEDA54B63C8AE5
+98966BE94C34481385640BD08B7EED44
+C3E6DD19273849329DD2E890ECC58D94
+B06C6F86A0FB47E89198D36702996675
+BF3BD7965CDC49C08AA53DA13652AA9D
+4F70A9DE437549869668F3C2DE4FDA87
+DB92B9801AB646A9AE19DEB1C88A4BB6
+4165F617BD4348B985990CCDDFA0AB5C
+B26BF6FA710F4349BDEC204B47FB1BE6
+D689121E4AD64BA69A585C5E1F91BC7C
+DB392AF0850641A7ADB76FA67AB40637
+1017CCB8FB8D42DA8F08364227BD794E
+4946AD7CF2B5420A96E3A07B14AC0049
+77E553EFB4E148E498B1EA4AEF99D657
+5011D115C2B5408D8E4D5629D3EB9F71
+F11426D0534A4830BE3C8D4F4290FBD4
+4C944FF6B1124E9892D76E8967248618
+61315935BC344689BBD9D267DEBE9A9B
+6A01FCD050BB4F348785CC82A28D750D
+656A2B5A256F46E09F290AB4551E0E9E
+7B094A902BFF498CA2D6B2B053792232
+7DFD051DBCA443929994F2B17A360A3E
+4B50DFF8A99C4CB28F63AA31C42200D2
+1234462435804F9AA86653C9CC50B2B7
+29857BE77A67414A85D4F306706A877F
+CE18D3A8815F44E29AA00B15CFDDF20C
+F194296A247549CFA5DE61783BA5E770
+D4049EA4E58D46F6B4F9588BDBAA7883
+CAE5E3232D1E4AA4A3B34B22899E3F13
+E886E72F2FEA4094A6616A65F568F32C
+24B2F3B5EFDD4CE58E7C91B200515535
+92EB0798C7C94B89947DB2C5A6B178BA
+AD2EA4F62ADA4E4AA1AD7F8160846E91
+6014F57318E94E2F8A6A6DF0467007BA
+473B19F1ADEF47A285A290BC4F24C6E0
+61CDF6B343BF44B981F1ABE01AC2D3DE
+31D4D6961BC44356A9B334B3197E010D
+5A2B93C11326425BA6D77DC4545430F1
+15140A442E604344B6A4E525864C9122
+472F558070724284A7716F848D59CB2A
+929ED22134244C74829081C4C3CF1FEC
+92D360516BE7488C8D73D74A7EBC6568
+D902E79AEB9C444B90485EB2B2490724
+7FEA3B375E2C467BB6C82FC746FC0B2C
+533C3232F5624F3BA0DB6B3CFD710998
+83E263C202144F64AFAF3BA5A55EB92A
+7BB14935ED0B4CE98C1F6146D16BDDD5
+AB88B05A18674D45BC9CC286DB4E4696
+57A32563494F4E898AE60262660D8D51
+986F94C0C7B44089AE63ACE298AD6B89
+37C821F533A84121BCFDD53A7EA1AF5B
+A2AECED2EECA4A439425FF05DC33E8D3
+E9062372175343259EDB3EF3E60D294C
+CB567432597C45D8803B944949D3CE17
+899AC55A0D3C41E9825B2F1CCE4A37A7
+C00CE1D52E8B4D82807CE4B2CD9F171B
+6C0A0A0664924119B4D3799FE6508146
+83732EEDBD0041BBAA6C88FC252C71CF
+E73E0D1D91F8432D95BF54DED3943978
+86FD8CCFB99E48DEBB9A23B150CA2FEF
+AC17425723534FFFB60FF98795C36D1C
+EFC69B73AFE748E5B6C9A387FE444CDF
+E314F9F3274C48899C8BD76F79A0D2BB
+D80F3CD9E96D4B519C81AECF55DDFAC1
+D19109E8C3F440FDBBDF71DEFE6C0946
+0289C766C7E54F25BE7D4455D611FE4C
+BE85BDE248AF4A7AAFF9C660CC23BBF5
+E260BA4E91DA42E6AE270D32301A9C27
+23F3C1C94E8B44EF8CA6703363E9FA19
+A41AF3281CD3494984D107FBEE320F08
+F49CA66ED0174BCEB9F8CC33ED87B6C2
+09665147D3C14B13A54A0159D6892D76
+7DE107863677402E97DD97375653F865
+C3E2240529E94093BDBE81F0FF217092
+9F245D77073F40B1AD25EDED02C8CA82
+250B1502590C42309DE5DE84C6FB0798
+5B4073DEB12A445E9BFD7BFE8E1A1785
+0E04F858859446B6AD4439B7ED1A2169
+01D1D6B4AA8C4834BBE1712201D03561
+2340674A95594894BC8B6C5513896D07
+CF1F44EBC3804F1FB2BE60A28259A6EF
+3CAE03EEFB0643458DB4B6FC67588D4B
+7F48749BBE9F47F2A675BE31C68606D8
+B0AD4ACDFA3843F094429D88FF166DD2
+E6C94DCF367B4583B078F0F31FD1E2EB
+D9D3068F40164B75913AD48888F62E6F
+6554E969E43B487BB8BA58A0A9AD974B
+632C95E213844F48A2A4ED2C038154E0
+FB03A9C9CCBA4F9DA19F78A6AA18A56D
+C636A5F9704048CD94CE35F6381BAF57
+418A69BAB82A4861B3FC6B62988536DB
+259096E559164142A7D56925B0D186A2
+727F8063BC1C4A8880E76BCB95D0766A
+2A09469BDA7C4F40B48C7C7BF6059564
+6CC5F23F68894F5D82BF14E31E5063B4
+20F2135771DC4F2E991DFCC6C8F234C8
+530769B435674A1487FE80D1D17C548E
+DF0C4A8626DF46FF9A10B149865304C1
+4607509807344287B81D6DBE3D0DFEF0
+ED00F63A800C47DBB59B7FC47A06516D
+539EB6BD705443689BDDA7E8787581B5
+AC0E65F8900D4B41B973B3DCB210042F
+1DC5190F7FF248DEBEEC32F202EC32DE
+D8047B21B1CA42BEAC51CD15C6DAEAAD
+C3CB9745415C47948B7328AF4690759D
+3C516C6058614E5997330975A83BACA8
+C031F32EAA01416C832F6E0ABFC91637
+1087A5A69403492D91E7575B4CEE3166
+A62F4CC94C544AEB8AD2A8C7F9156507
+BCAC0F81EA2C48D8B59ABB8DDE37438B
+BF75C686AA8648F096B02E80201CEA87
+E594A6055123415CA7E6CA9FAAE3F187
+39EEA379FDB9477C8AFC5316AA262FE9
+CD1793217D954742A33C2CE0C21B1BE2
+871B5FDDE36C45969CEEEE93DD40FB97
diff --git a/libs/cuckoo4j.jar b/libs/cuckoo4j.jar
new file mode 100644
index 00000000..51050dbe
Binary files /dev/null and b/libs/cuckoo4j.jar differ
diff --git a/pom.xml b/pom.xml
index bcf8c0d9..4a26319f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -147,6 +147,20 @@
2.9.0
+
+
+ com.amazonaws
+ aws-java-sdk
+ 1.11.119
+
+
+
+ test
+ Cuckoo
+ 1.0
+ system
+ ${project.basedir}/libs/cuckoo4j.jar
+
diff --git a/src/com/xrtb/bidder/Controller.java b/src/com/xrtb/bidder/Controller.java
index 786a317b..c0d3c4f1 100644
--- a/src/com/xrtb/bidder/Controller.java
+++ b/src/com/xrtb/bidder/Controller.java
@@ -13,6 +13,7 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.xrtb.blocks.AwsCommander;
import com.xrtb.commands.BasicCommand;
import com.xrtb.commands.ClickLog;
import com.xrtb.commands.ConvertLog;
@@ -91,6 +92,8 @@ public enum Controller {
public static final int SET_PRICE = 13;
// Add a list of campaigns
public static final int ADD_CAMPAIGNS_LIST = 14;
+ // Add an aws object
+ public static final int CONFIGURE_AWS_OBJECT = 15;
/** The REDIS channel for sending commands to the bidders */
public static final String COMMANDS = "commands";
@@ -199,6 +202,27 @@ public void addCampaign(Campaign c) throws Exception {
Configuration.getInstance().deleteCampaign(c.owner, c.adId);
Configuration.getInstance().addCampaign(c);
}
+
+ public void configureAwsObject(BasicCommand c) {
+ System.out.println("ADDING AWS OBJECT " + c.owner + "/" + c.target);
+ BasicCommand m = new BasicCommand();
+ m.owner = c.owner;
+ m.to = c.from;
+ m.from = Configuration.instanceName;
+ m.id = c.id;
+ m.type = c.type;
+ AwsCommander aws = new AwsCommander(c.target);
+ if (aws.errored()) {
+ m.status = "Error";
+ m.msg = "AWS Object load failed: " + aws.getMessage();
+ responseQueue.add(m);
+ } else {
+ m.msg = "AWS Object " + c.target + " loaded ok";
+ m.name = "AWS Object Response";
+ sendLog(1, "ConfigureAws results:", m.msg);
+ responseQueue.add(m);
+ }
+ }
/**
* Add a campaign from REDIS
@@ -1181,6 +1205,21 @@ public void onMessage(String arg0, BasicCommand item) {
thread.start();
break;
+
+ case Controller.CONFIGURE_AWS_OBJECT:
+
+ task = () -> {
+ try {
+ Controller.getInstance().configureAwsObject(item);
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ };
+ thread = new Thread(task);
+ thread.start();
+
+ break;
case Controller.ADD_CAMPAIGNS_LIST:
diff --git a/src/com/xrtb/bidder/RTBServer.java b/src/com/xrtb/bidder/RTBServer.java
index 036c8736..d00ff6c9 100644
--- a/src/com/xrtb/bidder/RTBServer.java
+++ b/src/com/xrtb/bidder/RTBServer.java
@@ -420,6 +420,7 @@ public static String getSummary() throws Exception {
m.put("diskFree", Performance.getPercFreeDisk());
m.put("openfiles", Performance.getOpenFileDescriptorCount());
m.put("exchanges", BidRequest.getExchangeCounts());
+ m.put("lowonthreads", server.getThreadPool().isLowOnThreads());
return DbTools.mapper.writeValueAsString(m);
}
@@ -1006,20 +1007,21 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques
}
if (RTBServer.server.getThreadPool().isLowOnThreads()) {
- code = RTBServer.NOBID_CODE;
- json = "Server throttling";
if (br.id.equals("123")) {
- Controller.getInstance().sendLog(3,"RTBSerrver:handler", "Server throttled");
- }
- RTBServer.nobid++;
- response.setStatus(br.returnNoBidCode());
- response.setContentType(br.returnContentType());
- baseRequest.setHandled(true);
- br.writeNoBid(response, time);
+ Controller.getInstance().sendLog(3,"RTBSerrver:handler", "Server throttled, low on threads");
+ } else {
+ code = RTBServer.NOBID_CODE;
+ json = "Server throttling";
+ RTBServer.nobid++;
+ response.setStatus(br.returnNoBidCode());
+ response.setContentType(br.returnContentType());
+ baseRequest.setHandled(true);
+ br.writeNoBid(response, time);
- Controller.getInstance().sendRequest(br,false);
- return;
- }
+ Controller.getInstance().sendRequest(br,false);
+ return;
+ }
+ }
if (CampaignSelector.getInstance().size() == 0) {
if (br.id.equals("123")) {
diff --git a/src/com/xrtb/bidder/ZPublisher.java b/src/com/xrtb/bidder/ZPublisher.java
index d26ae34f..e2dbf29c 100644
--- a/src/com/xrtb/bidder/ZPublisher.java
+++ b/src/com/xrtb/bidder/ZPublisher.java
@@ -115,7 +115,7 @@ public ZPublisher(String address) throws Exception {
} else if (address.startsWith("redis")) {
String[] parts = address.split(":");
channel = parts[1];
- jedisPool = Configuration.getInstance().jedisPool;
+ // jedisPool = Configuration.getInstance().jedisPool; TND FIX THIS
} else if (address.startsWith("http")) {
http = new HttpPostGet();
int i = address.indexOf("&");
diff --git a/src/com/xrtb/blocks/AwsCommander.java b/src/com/xrtb/blocks/AwsCommander.java
new file mode 100644
index 00000000..bc13c8e8
--- /dev/null
+++ b/src/com/xrtb/blocks/AwsCommander.java
@@ -0,0 +1,101 @@
+package com.xrtb.blocks;
+
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.S3Object;
+import com.xrtb.common.Configuration;
+
+import redis.clients.jedis.Jedis;
+
+/**
+ * A class that handles the actual command received by the subscriber.
+ * @author Ben M. Faul
+ *
+ */
+public class AwsCommander {
+
+ String msg = null;
+ boolean err = false;
+ /**
+ * Constructor. Does the actual command.
+ * @param message String. The message sent in the command.
+ * @param port int. The port number to respond on.
+ */
+ public AwsCommander(String message) {
+ String[] tokens = message.split(",");
+ String[] parts = tokens[0].split(" ");
+ for (int i = 0; i < parts.length; i++) {
+ parts[i] = parts[i].trim();
+ }
+
+ try {
+ switch (parts[0]) {
+ case "load":
+ msg = load(parts);
+ break;
+ case "delete":
+ delete(parts[1]);
+ msg = "Symbol " + parts[1] + " removed";
+ break;
+ default:
+ }
+ } catch (Exception error) {
+ msg = error.toString();
+ err = true;
+ error.printStackTrace();
+ }
+ }
+
+ /**
+ * Return the message from the load
+ * @return String. The message to return to the caller.
+ */
+ public String getMessage() {
+ return msg;
+ }
+
+ /**
+ * Return whether the command errored
+ * @return boolean. Returns true if was an error.
+ */
+ public boolean errored() {
+ return err;
+ }
+
+ void delete(String key) {
+ LookingGlass.symbols.remove(key);
+ }
+
+ /**
+ * Load the file or s3 object.
+ * @param parts String[]. An array of tokens.
+ * @return String. The message returned from the load command.
+ * @throws Exception on I/O errirs.
+ */
+ String load(String[] parts) throws Exception {
+ String otype = null;
+ String symbolName = null;
+ String name;
+ String type = parts[1]; // file or S3
+ //for (int i=0;i bloomFilter;
+ int size;
+
+ /**
+ * Constructor for the File/S3 object to Bloom filter.
+ * @param name String. The name of the bloom filter.
+ * @param file String, the file name.
+ * @throws Exception on File Errors.
+ */
+ public Bloom(String name, String file) throws Exception {
+ File f = new File(file);
+ long size = f.length();
+ BufferedReader br = new BufferedReader(new FileReader(file));
+ makeFilter(br,size);
+
+ symbols.put(name, bloomFilter);
+ }
+
+ /**
+ * Constructor for the S3 version of the Bloom filter.
+ * @param name String. The name of the object.
+ * @param object S3Object. The object that contains the file.
+ * @throws Exception on S3 errors.
+ */
+ public Bloom(String name, S3Object object, long size) throws Exception {
+ InputStream objectData = object.getObjectContent();
+ BufferedReader br=new BufferedReader(new InputStreamReader(objectData));
+ makeFilter(br,size);
+
+ symbols.put(name, bloomFilter);
+ }
+
+ /**
+ * Reads a file or S3 object line by line and loads the filter.
+ * @param br BufferedReader. The line-by-line reader.
+ * @throws Exception on I/O errors.
+ */
+ void makeFilter(BufferedReader br, long size) throws Exception {
+ String[] parts;
+ int i;
+ long sz;
+
+ double fpp = 0.03; // desired false positive probability
+
+ String line = br.readLine();
+ sz = line.length() - 5;
+ sz = size / sz;
+ parts = eatquotedStrings(line);
+ this.size = 1;
+ for (i = 0; i < parts.length; i++) {
+ parts[i] = parts[i].replaceAll("\"", "");
+ }
+
+ bloomFilter = BloomFilter.create(Funnels.stringFunnel(Charset.forName("UTF-8")), sz,fpp);
+ bloomFilter.put(parts[0]);
+
+ while ((line = br.readLine()) != null) {
+ parts = eatquotedStrings(line);
+ for (i = 0; i < parts.length; i++) {
+ parts[i] = parts[i].replaceAll("\"", "");
+ }
+ bloomFilter.put(parts[0]);
+ this.size++;
+ }
+ br.close();
+ }
+
+ /**
+ * Returns the Bloom filter for your use.
+ * @return BloomFilter. The Guava bloom filter of the contents of this file.
+ */
+ public BloomFilter getBloom() {
+ return bloomFilter;
+ }
+
+ /**
+ * Check if this key is possibly in the bloom filter
+ * @param key String. The key to test for.
+ * @return boolean. Returns false if it is not in the filter. Returns true if it possibly is in there.
+ */
+ public boolean isMember(String key) {
+ return bloomFilter.mightContain(key);
+ }
+
+ /**
+ * Returns the number of elements.
+ * @return int. The number of elements in the filter.
+ */
+ public long getMembers() {
+ return size;
+ }
+}
diff --git a/src/com/xrtb/blocks/CIDRUtils.java b/src/com/xrtb/blocks/CIDRUtils.java
new file mode 100644
index 00000000..93fac193
--- /dev/null
+++ b/src/com/xrtb/blocks/CIDRUtils.java
@@ -0,0 +1,133 @@
+package com.xrtb.blocks;
+
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A class that enables to get an IP range from CIDR specification. It supports
+ * both IPv4 and IPv6.
+ */
+public class CIDRUtils {
+ private final String cidr;
+
+ private InetAddress inetAddress;
+ private InetAddress startAddress;
+ private InetAddress endAddress;
+ private final int prefixLength;
+
+
+ public CIDRUtils(String cidr) throws UnknownHostException {
+
+ this.cidr = cidr;
+
+ /* split CIDR to address and prefix part */
+ if (this.cidr.contains("/")) {
+ int index = this.cidr.indexOf("/");
+ String addressPart = this.cidr.substring(0, index);
+ String networkPart = this.cidr.substring(index + 1);
+
+ inetAddress = InetAddress.getByName(addressPart);
+ prefixLength = Integer.parseInt(networkPart);
+
+ calculate();
+ } else {
+ throw new IllegalArgumentException("not an valid CIDR format!");
+ }
+ }
+
+ public long getStartAddress() {
+ ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE);
+ buffer.put(startAddress.getAddress());
+ buffer.position(0);
+ Long longValue = buffer.getLong();
+ return longValue;
+ }
+
+ public long getEndAddress() {
+ ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE);
+ buffer.put(endAddress.getAddress());
+ buffer.position(0);
+ Long longValue = buffer.getLong();
+ return longValue;
+ }
+
+
+ private void calculate() throws UnknownHostException {
+
+ ByteBuffer maskBuffer;
+ int targetSize;
+ if (inetAddress.getAddress().length == 4) {
+ maskBuffer =
+ ByteBuffer
+ .allocate(4)
+ .putInt(-1);
+ targetSize = 4;
+ } else {
+ maskBuffer = ByteBuffer.allocate(16)
+ .putLong(-1L)
+ .putLong(-1L);
+ targetSize = 16;
+ }
+
+ BigInteger mask = (new BigInteger(1, maskBuffer.array())).not().shiftRight(prefixLength);
+
+ ByteBuffer buffer = ByteBuffer.wrap(inetAddress.getAddress());
+ BigInteger ipVal = new BigInteger(1, buffer.array());
+
+ BigInteger startIp = ipVal.and(mask);
+ BigInteger endIp = startIp.add(mask.not());
+
+ byte[] startIpArr = toBytes(startIp.toByteArray(), targetSize);
+ byte[] endIpArr = toBytes(endIp.toByteArray(), targetSize);
+
+ this.startAddress = InetAddress.getByAddress(startIpArr);
+ this.endAddress = InetAddress.getByAddress(endIpArr);
+
+ }
+
+ private byte[] toBytes(byte[] array, int targetSize) {
+ int counter = 0;
+ List newArr = new ArrayList();
+ while (counter < targetSize && (array.length - 1 - counter >= 0)) {
+ newArr.add(0, array[array.length - 1 - counter]);
+ counter++;
+ }
+
+ int size = newArr.size();
+ for (int i = 0; i < (targetSize - size); i++) {
+
+ newArr.add(0, (byte) 0);
+ }
+
+ byte[] ret = new byte[newArr.size()];
+ for (int i = 0; i < newArr.size(); i++) {
+ ret[i] = newArr.get(i);
+ }
+ return ret;
+ }
+
+ public String getNetworkAddress() {
+
+ return this.startAddress.getHostAddress();
+ }
+
+ public String getBroadcastAddress() {
+ return this.endAddress.getHostAddress();
+ }
+
+ public boolean isInRange(String ipAddress) throws UnknownHostException {
+ InetAddress address = InetAddress.getByName(ipAddress);
+ BigInteger start = new BigInteger(1, this.startAddress.getAddress());
+ BigInteger end = new BigInteger(1, this.endAddress.getAddress());
+ BigInteger target = new BigInteger(1, address.getAddress());
+
+ int st = start.compareTo(target);
+ int te = target.compareTo(end);
+
+ return (st == -1 || st == 0) && (te == -1 || te == 0);
+ }
+}
diff --git a/src/com/xrtb/blocks/Cuckoo.java b/src/com/xrtb/blocks/Cuckoo.java
new file mode 100644
index 00000000..2c2d75d5
--- /dev/null
+++ b/src/com/xrtb/blocks/Cuckoo.java
@@ -0,0 +1,103 @@
+package com.xrtb.blocks;
+
+import java.io.BufferedReader;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+
+
+import com.amazonaws.services.s3.model.S3Object;
+import com.github.mgunlogson.cuckoofilter4j.CuckooFilter;
+import com.google.common.hash.Funnels;
+
+
+/**
+ * Implements the BloomFilter for UTF-8 strings. Builds a Guava bloom filter from file or S3 object.
+ * @author Ben M. Faul
+ *
+ */
+public class Cuckoo extends LookingGlass {
+ CuckooFilter cuckooFilter;;
+
+ /**
+ * Constructor for the File/S3 object to Bloom filter.
+ * @param name String. The name of the bloom filter.
+ * @param file String, the file name.
+ * @throws Exception on File Errors.
+ */
+ public Cuckoo(String name, String file) throws Exception {
+ File f = new File(file);
+ long size = f.length();
+ BufferedReader br = new BufferedReader(new FileReader(file));
+ makeFilter(br,size);
+
+ symbols.put(name, cuckooFilter);
+ }
+
+ /**
+ * Constructor for the S3 version of the Bloom filter.
+ * @param name String. The name of the object.
+ * @param object S3Object. The object that contains the file.
+ * @throws Exception on S3 errors.
+ */
+ public Cuckoo(String name, S3Object object, long size) throws Exception {
+ InputStream objectData = object.getObjectContent();
+ BufferedReader br=new BufferedReader(new InputStreamReader(objectData));
+ makeFilter(br,size);
+ symbols.put(name, cuckooFilter);
+ }
+
+ /**
+ * Reads a file or S3 object line by line and loads the filter.
+ * @param br BufferedReader. The line-by-line reader.
+ * @throws Exception on I/O errors.
+ */
+ void makeFilter(BufferedReader br, long sz) throws Exception {
+ String[] parts;
+ int i;
+
+
+ String line = br.readLine();
+ line = line.trim();
+ i = 0;
+
+ parts = eatquotedStrings(line);
+ for (i = 0; i < parts.length; i++) {
+ parts[i] = parts[i].replaceAll("\"", "");
+ }
+ long size = parts[0].length() - 5;
+ size = sz / size;
+ double fpp = 0.03; // desired false positive probability
+ cuckooFilter = new CuckooFilter.Builder<>(Funnels.stringFunnel(Charset.forName("UTF-8")), size).build();
+ cuckooFilter.put(parts[0]);
+
+
+ while ((line = br.readLine()) != null) {
+ parts = eatquotedStrings(line);
+ for (i = 0; i < parts.length; i++) {
+ parts[i] = parts[i].replaceAll("\"", "");
+ }
+ cuckooFilter.put(parts[0]);
+ }
+ br.close();
+ }
+
+ /**
+ * Returns the Bloom filter for your use.
+ * @return BloomFilter. The Guava bloom filter of the contents of this file.
+ */
+ public CuckooFilter getCuckoo() {
+ return cuckooFilter;
+ }
+
+ /**
+ * Returns the number of elements.
+ * @return int. The number of elements in the filter.
+ */
+ public long getMembers() {
+ return cuckooFilter.getCount();
+ }
+}
diff --git a/src/com/xrtb/tools/LookingGlass.java b/src/com/xrtb/blocks/LookingGlass.java
similarity index 94%
rename from src/com/xrtb/tools/LookingGlass.java
rename to src/com/xrtb/blocks/LookingGlass.java
index de2f7d2c..1aecffd1 100644
--- a/src/com/xrtb/tools/LookingGlass.java
+++ b/src/com/xrtb/blocks/LookingGlass.java
@@ -1,9 +1,10 @@
-package com.xrtb.tools;
+package com.xrtb.blocks;
import java.io.BufferedReader;
import java.io.FileReader;
import java.util.Date;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import com.xrtb.bidder.Controller;
@@ -15,7 +16,7 @@ public class LookingGlass {
public static volatile Map symbols = new ConcurrentHashMap();
// My map
- Map myMap = new ConcurrentHashMap();
+ protected Map myMap = new ConcurrentHashMap();
/**
* Default constructor
@@ -89,4 +90,9 @@ public static String[] eatquotedStrings(String line) {
// }
return line.split(regex, -1);
}
+
+ public static Object get(String name) {
+ Object x = symbols.get(name);
+ return x;
+ }
}
diff --git a/src/com/xrtb/tools/Membership.java b/src/com/xrtb/blocks/Membership.java
similarity index 99%
rename from src/com/xrtb/tools/Membership.java
rename to src/com/xrtb/blocks/Membership.java
index 0c8837c0..17680c9f 100644
--- a/src/com/xrtb/tools/Membership.java
+++ b/src/com/xrtb/blocks/Membership.java
@@ -1,4 +1,4 @@
-package com.xrtb.tools;
+package com.xrtb.blocks;
import java.io.BufferedReader;
import java.io.FileReader;
diff --git a/src/com/xrtb/blocks/NavMap.java b/src/com/xrtb/blocks/NavMap.java
index 8a63b253..efb7e302 100644
--- a/src/com/xrtb/blocks/NavMap.java
+++ b/src/com/xrtb/blocks/NavMap.java
@@ -3,109 +3,182 @@
import java.io.BufferedReader;
import java.io.FileReader;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Iterator;
import java.util.Map;
-import java.util.NavigableMap;
+import java.util.Set;
import java.util.TreeMap;
-import org.jboss.netty.handler.ipfilter.CIDR;
-
-import com.xrtb.bidder.Controller;
-import com.xrtb.common.Configuration;
-import com.xrtb.tools.LookingGlass;
-
-public class NavMap extends LookingGlass {
-
- public static List in = new ArrayList();
- public static List out = new ArrayList();
-
- NavigableMap map = new TreeMap();
- public static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
-
- public static List inPoints = new ArrayList();
- public static List outPoints = new ArrayList();
- public static int testPoints = 1000;
- public static int current = 0;
-
- public static boolean searchTable(String key, String ip) {
- NavMap x = (NavMap) symbols.get(key);
- if (x == null)
- return false;
- return x.search(ip);
- }
- public static boolean searchTable(String key, long ip) {
- NavMap x = (NavMap) symbols.get(key);
- if (x == null)
- return false;
- return x.search(ip);
- }
-
- public static void main(String args[]) throws Exception {
- // NavMap sr = new NavMap("ISP", "/home/ben/Downloads/ISP.txt", false);
-
- NavMap sr = new NavMap("ISP", "data/METHBOT.txt", true);
+import com.amazonaws.services.s3.model.S3Object;
+import com.google.common.collect.Maps;
+
+public class NavMap extends LookingGlass implements Set {
+
+ // The backing TreeMap.
+ TreeMap tree = Maps.newTreeMap();
+
+ // The name of the object
+ String name;
+
+ /**
+ * Older form of constructor where you are told what kind it is cidr or not
+ *
+ * @param name
+ * String. The name of the key.
+ * @param file
+ * String. The file name.
+ * @param cidr
+ * boolean. Is a cidr is true. Else false is range
+ * @throws Exception
+ * on I/O errors.
+ */
+ public NavMap(String name, String file, boolean cidr) throws Exception {
+ this.name = name;
+ if (cidr) {
+ BufferedReader br = new BufferedReader(new FileReader(file));
+ doCidr(br, file);
+ } else {
+ BufferedReader br = new BufferedReader(new FileReader(file));
+ doRanges(br, file);
+ }
- /*for (int i = 0; i < 1; i++) {
- // String ip = "1.0.10.0";
- String ip = "223.255.192.255";
- long now = System.nanoTime();
- boolean x = sr.search(ip);
- now = System.nanoTime() - now;
- now /= 1000;
- System.out.println(ip + " " + x + " " + now + " micro seconds");
- } */
- // 203.180.58.167-203.180.58.187
- // 203.212.37.105-203.212.37.125
+ symbols.put(name, this);
}
- public NavMap(String name, String file, boolean isCidr) throws Exception {
- super();
-
- if (isCidr)
- doCidr(name, file);
- else
- doRanges(name, file);
+ /**
+ * A navigable hashmap for handling CIDR lists and range maps made from a
+ * file.
+ *
+ * @param name
+ * String. The name of the object.
+ * @param file
+ * String. The name of the file.
+ * @throws Exception
+ * on I/O errors.
+ */
+
+ public NavMap(String name, String file) throws Exception {
+ this.name = name;
+ if (file.endsWith("cidr")) {
+ BufferedReader br = new BufferedReader(new FileReader(file));
+ doCidr(br, file);
+ } else if (file.endsWith("range")) {
+ BufferedReader br = new BufferedReader(new FileReader(file));
+ doRanges(br, file);
+ } else
+ throw new Exception(file + " Not in range or CIDR form");
+ symbols.put(name, this);
}
- void doCidr(String name, String file) throws Exception {
- BufferedReader br = new BufferedReader(new FileReader(file));
- long start = 0;
- long end = 0;
- String message = null;
- XRange r = null;
- int k = 0;
+ /**
+ * A navigable hashmap for handling CIDR lists and range maps from an S3
+ * object.
+ *
+ * @param name
+ * String. The name of the object.
+ * @param object
+ * S3Object. The S3 object for this..
+ * @throws Exception
+ * on I/O errors.
+ */
+ public NavMap(String name, S3Object object) throws Exception {
+ this.name = name;
+ String file = object.getBucketName();
+ InputStream objectData = object.getObjectContent();
+ BufferedReader br = new BufferedReader(new InputStreamReader(objectData));
+ if (file.endsWith("cidr")) {
+ doCidr(br, file);
+ } else if (file.endsWith("range")) {
+ doRanges(br, file);
+ } else
+ throw new Exception(file + " Not in range or CIDR form");
+ }
- message = "Initialize CIDR navmap: " + file + " as " + name;
+ /**
+ * Process CIDR lists.
+ *
+ * @param br
+ * BufferedReader. The line-by-line reader.
+ * @param file
+ * String. The file name.
+ * @throws Exception
+ * on I/O errors.
+ */
+ void doCidr(BufferedReader br, String file) throws Exception {
+ long start;
+ long end;
+ String messagel;
+ XRange r;
+ int k;
+
+ k = 0;
for (String line; (line = br.readLine()) != null;) {
if (!(line.startsWith("#") || line.length() < 10)) {
- CIDR cidr = CIDR.newCIDR(line);
- start = ipToLong(cidr.getBaseAddress().toString().substring(1));
- end = ipToLong(cidr.getEndAddress().toString().substring(1));
+ CIDRUtils cidr = new CIDRUtils(line);
+ start = cidr.getStartAddress();
+ end = cidr.getEndAddress();
r = new XRange(end, k);
k++;
- map.put(start, r);
+ tree.put(start, r);
}
}
- message += ", total records = " + k;
+ }
- if (Configuration.isInitialized()) {
- System.out.format("[%s] - %d - %s - %s - %s\n", Controller.sdf.format(new Date()), 1,
- Configuration.instanceName, "NavMap", message);
+ /**
+ * Search the navmap using a long representation of the
+ *
+ * @param key
+ * @return
+ */
+ public Boolean search(long key) {
+ Map.Entry entry = tree.floorEntry(key);
+ if (entry == null)
+ return false;
+ else if (key <= entry.getValue().upper) {
+ return true; // return entry.getValue().value;
} else {
- System.out.format("[%s] - %d - %s - %s - %s\n", "", 1,
- "", "NavMap", message);
+ return false;
}
- symbols.put(name, this);
}
- void doRanges(String name, String file) throws Exception {
- BufferedReader br = new BufferedReader(new FileReader(file));
- long x = 0, k = 0;
+ public static long ipToLong(String ipAddress) {
+
+ String[] ipAddressInArray = ipAddress.split("\\.");
+
+ long result = 0;
+ for (int i = 0; i < ipAddressInArray.length; i++) {
+
+ int power = 3 - i;
+ int ip = Integer.parseInt(ipAddressInArray[i]);
+ result += ip * Math.pow(256, power);
+
+ }
+
+ return result;
+ }
+
+ public static String longToIp(long ip) {
+ if (ip > 4294967295L || ip < 0) {
+ throw new IllegalArgumentException("invalid ip");
+ }
+ StringBuilder ipAddress = new StringBuilder();
+ for (int i = 3; i >= 0; i--) {
+ int shift = i * 8;
+ ipAddress.append((ip & (0xff << shift)) >> shift);
+ if (i > 0) {
+ ipAddress.append(".");
+ }
+ }
+ return ipAddress.toString();
+ }
+
+ void doRanges(BufferedReader br, String file) throws Exception {
long oldstart = 0;
long oldend = 0;
long start = 0;
@@ -116,9 +189,9 @@ void doRanges(String name, String file) throws Exception {
XRange r = null;
long over = 0;
- int linek = 0;
+ int linek = 9;
+ int k = 0;
- message = "Initialize RANGE navmap: " + file + " as " + name;
for (String line; (line = br.readLine()) != null;) {
if (!(line.startsWith("#") || line.length() < 10)) {
parts = line.split("-");
@@ -126,7 +199,7 @@ void doRanges(String name, String file) throws Exception {
start = ipToLong(parts[0]);
end = ipToLong(parts[1]);
-
+
if (oldstart == 0) {
oldstart = start;
oldend = end;
@@ -137,79 +210,119 @@ void doRanges(String name, String file) throws Exception {
} else {
r = new XRange(oldend, k);
k++;
-
- map.put(oldstart, r);
-
- outPoints.add(oldend+1);
+
+ tree.put(oldstart, r);
+
oldstart = start;
oldend = end;
}
}
- if (k % 1000 == 0 && in.size() < 10) {
- in.add(end + 10);
- }
+
}
linek++;
}
}
r = new XRange(end, k);
- k++;
- map.put(start, r);
+ tree.put(start, r);
double d = (double) over / (double) linek;
- message += ", overlaps = " + over + ", total records = " + k + ", % overlap = " + d;
+ }
- System.out.format("[%s] - %d - %s - %s - %s\n", sdf.format(new Date()), 1,
- "localhost", "NavMap", message);
- symbols.put(name, this);
+ @Override
+ public int size() {
+ return tree.size();
+ }
+ @Override
+ public boolean isEmpty() {
+ if (tree.size() == 0)
+ return true;
+ return false;
}
- public boolean search(String ip) {
- long address = ipToLong(ip);
+ public boolean searchTable(Object key) {
+ return contains(key);
+ }
+
+ @Override
+ public boolean contains(Object key) {
+ // System.out.println("Looking for: " + key);
+ long address = 0;
+ if (key instanceof Long)
+ address = (long) key;
+ else
+ try {
+ InetAddress inetAddress = InetAddress.getByName((String) key);
+ ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE);
+ buffer.put(inetAddress.getAddress());
+ buffer.position(0);
+ address = buffer.getLong();
+ } catch (UnknownHostException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ // System.out.println("X = " + address);
return search(address);
}
- public boolean search(long key) {
- Map.Entry entry = map.floorEntry(key);
- if (entry == null)
- return false;
- else if (key <= entry.getValue().upper) {
- return true; // return entry.getValue().value;
- } else {
- return false;
- }
+ @Override
+ public Iterator iterator() {
+ // TODO Auto-generated method stub
+ return null;
}
- public static long ipToLong(String ipAddress) {
+ @Override
+ public Object[] toArray() {
+ // TODO Auto-generated method stub
+ return null;
+ }
- String[] ipAddressInArray = ipAddress.split("\\.");
+ @Override
+ public Object[] toArray(Object[] a) {
+ // TODO Auto-generated method stub
+ return null;
+ }
- long result = 0;
- for (int i = 0; i < ipAddressInArray.length; i++) {
+ @Override
+ public boolean add(Object e) {
+ // TODO Auto-generated method stub
+ return false;
+ }
- int power = 3 - i;
- int ip = Integer.parseInt(ipAddressInArray[i]);
- result += ip * Math.pow(256, power);
+ @Override
+ public boolean remove(Object o) {
+ // TODO Auto-generated method stub
+ return false;
+ }
- }
+ @Override
+ public boolean containsAll(Collection c) {
+ // TODO Auto-generated method stub
+ return false;
+ }
- return result;
+ @Override
+ public boolean addAll(Collection c) {
+ // TODO Auto-generated method stub
+ return false;
}
- public static String longToIp(long ip) {
- if (ip > 4294967295l || ip < 0) {
- throw new IllegalArgumentException("invalid ip");
- }
- StringBuilder ipAddress = new StringBuilder();
- for (int i = 3; i >= 0; i--) {
- int shift = i * 8;
- ipAddress.append((ip & (0xff << shift)) >> shift);
- if (i > 0) {
- ipAddress.append(".");
- }
- }
- return ipAddress.toString();
+ @Override
+ public boolean retainAll(Collection c) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public boolean removeAll(Collection c) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public void clear() {
+ // TODO Auto-generated method stub
+
}
}
diff --git a/src/com/xrtb/blocks/SimpleMultiset.java b/src/com/xrtb/blocks/SimpleMultiset.java
new file mode 100644
index 00000000..1f0f58b1
--- /dev/null
+++ b/src/com/xrtb/blocks/SimpleMultiset.java
@@ -0,0 +1,90 @@
+package com.xrtb.blocks;
+
+import java.io.BufferedReader;
+
+
+
+import java.io.FileReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+
+import com.amazonaws.services.s3.model.S3Object;
+import com.google.common.collect.HashMultiset;
+import com.google.common.collect.Multiset;
+
+
+/**
+ * Implements the Multiset counter for UTF-8 strings. Builds a Guava Multiset from file or S3 object.
+ * @author Ben M. Faul
+ *
+ */
+public class SimpleMultiset extends LookingGlass {
+ Multiset ms = HashMultiset.create();
+ int size = 0;
+
+ /**
+ * Constructor for the File/S3 object to Bloom filter.
+ * @param name String. The name of the bloom filter.
+ * @param file String, the file name.
+ * @throws Exception on File Errors.
+ */
+ public SimpleMultiset(String name, String file) throws Exception {
+ BufferedReader br = new BufferedReader(new FileReader(file));
+ System.out.print("Initialize Multiset: " + name + " from " + file + ", enttries = ");
+ while(br.readLine() != null) {
+ size++;
+ }
+ br.close();
+
+ br = new BufferedReader(new FileReader(file));
+ makeFilter(br);
+ System.out.println(ms.size() + " elements");
+
+ symbols.put(name, ms);
+ }
+
+ /**
+ * Constructor for the S3 version of the Multiset filter.
+ * @param name String. The name of the object.
+ * @param object S3Object. The object that contains the file.
+ * @throws Exception on S3 errors.
+ */
+ public SimpleMultiset(String name, S3Object object) throws Exception {
+ InputStream objectData = object.getObjectContent();
+ BufferedReader br=new BufferedReader(new InputStreamReader(objectData));
+ makeFilter(br);
+ System.out.println(size + " elements");
+
+ symbols.put(name, ms);
+ }
+
+ /**
+ * Reads a file or S3 object line by line and loads the filter.
+ * @param br BufferedReader. The line-by-line reader.
+ * @throws Exception on I/O errors.
+ */
+ void makeFilter(BufferedReader br) throws Exception {
+ for (String line; (line = br.readLine()) != null;) {
+ line = line.trim();
+ ms.add(line);
+ }
+ br.close();
+ }
+
+ /**
+ * Returns the Multiset filter for your use.
+ * @return Multiset. The Guava multiset of the contents of this file.
+ */
+ public Multiset getMultiset() {
+ return ms;
+ }
+
+ /**
+ * Returns the number of elements.
+ * @return int. The number of elements in the filter.
+ */
+ public int getMembers() {
+ return size;
+ }
+}
diff --git a/src/com/xrtb/blocks/SimpleSet.java b/src/com/xrtb/blocks/SimpleSet.java
new file mode 100644
index 00000000..5ecde5d1
--- /dev/null
+++ b/src/com/xrtb/blocks/SimpleSet.java
@@ -0,0 +1,84 @@
+package com.xrtb.blocks;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Set;
+
+import com.amazonaws.services.s3.model.S3Object;
+import com.google.common.collect.Sets;
+
+/**
+ * A guava set. Reads a file or S3 object and shoves the lines into a guava set.
+ * @author Ben M. Faul
+ *
+ */
+public class SimpleSet extends LookingGlass {
+ Set set = Sets.newHashSet();
+ String message;
+
+ /**
+ * A simple set. Reads from file, shoves into Set, line by line.
+ * @param name String. The name of the object.
+ * @param file String. The file name.
+ * @throws Exception on I/O errors.
+ */
+ public SimpleSet(String name, String file) throws Exception {
+ BufferedReader br = new BufferedReader(new FileReader(file));
+ message = "Initialize Simple Membership: " + file + " as " + name;
+
+ makeSet(br);
+
+ symbols.put(name,set);
+ }
+
+ /**
+ * A simple set. Reads S3 object and put into the Set.
+ * @param name String. The name of the object.
+ * @param object S3Object. The S3 object to read.
+ * @throws Exception on S3 or I/O options.
+ */
+ public SimpleSet(String name, S3Object object) throws Exception {
+ InputStream objectData = object.getObjectContent();
+ BufferedReader br=new BufferedReader(new InputStreamReader(objectData));
+ message = "Initialize Simple Membership: " + object.getBucketName() + " as " + name;
+ makeSet(br);
+
+ symbols.put(name, this);
+ }
+
+ /**
+ * Make the set from a buffered reader.
+ * @param br BufferedReader. Read line-by-line and place in the set.
+ * @throws Exception on I/O errors.
+ */
+ void makeSet(BufferedReader br) throws Exception {
+ String[] parts = null;
+ int i;
+ for (String line; (line = br.readLine()) != null;) {
+ parts = eatquotedStrings(line);
+ for (i = 0; i < parts.length; i++) {
+ parts[i] = parts[i].replaceAll("\"", "");
+ }
+ set.add(parts[0]);
+ }
+ br.close();
+ }
+
+ /**
+ * Return the simple set size
+ * @return int. Returns the size.
+ */
+ public int size() {
+ return set.size();
+ }
+
+ /**
+ * Return the Set.
+ * @return Set. The Guava set.
+ */
+ public Set getSet() {
+ return set;
+ }
+}
diff --git a/src/com/xrtb/commands/ConfigureAwsObject.java b/src/com/xrtb/commands/ConfigureAwsObject.java
new file mode 100644
index 00000000..74a485e5
--- /dev/null
+++ b/src/com/xrtb/commands/ConfigureAwsObject.java
@@ -0,0 +1,37 @@
+package com.xrtb.commands;
+
+import com.xrtb.bidder.Controller;
+import com.xrtb.common.Campaign;
+
+/**
+ * A class that is used to encapsulate a 0MQ command for adding a campaign to the bidder.
+ * Jackson will be used to create the structure.
+ * @author Ben M. Faul
+ *
+ */
+public class ConfigureAwsObject extends BasicCommand {
+
+ /**
+ * Empty constructor for Jackson
+ */
+ public ConfigureAwsObject() {
+ super();
+ cmd = Controller.CONFIGURE_AWS_OBJECT;
+ msg = "An AWS object is being configured in the system";
+ }
+
+ /**
+ * Configure an AWS Object.
+ * @param to String. The bidder that will execute the command.
+ * @param name String. The name of the owner of the campaign.
+ * @param target String. The command to execute.
+ */
+ public ConfigureAwsObject(String to, String name, String target) {
+ super(to);
+ cmd = Controller.CONFIGURE_AWS_OBJECT;
+ status = "ok";
+ this.owner = name;
+ this.target = target;
+ msg = "An AWS OBJECT is befing configured: " + name +"/" + target;
+ }
+}
diff --git a/src/com/xrtb/commands/PixelClickConvertLog.java b/src/com/xrtb/commands/PixelClickConvertLog.java
index 62bb7320..b173f207 100644
--- a/src/com/xrtb/commands/PixelClickConvertLog.java
+++ b/src/com/xrtb/commands/PixelClickConvertLog.java
@@ -1,12 +1,12 @@
package com.xrtb.commands;
-
/**
* Base class for logging pixel loads, clicks and conversions.
+ *
* @author Ben M. Faul
*
*/
-public class PixelClickConvertLog {
+public class PixelClickConvertLog {
public String instance;
public String payload;
public double lat;
@@ -21,65 +21,54 @@ public class PixelClickConvertLog {
public static final int PIXEL = 0;
public static final int CLICK = 1;
public static final int CONVERT = 2;
-
- public static void main(String [] args) {
+
+ public static void main(String[] args) {
PixelClickConvertLog x = new PixelClickConvertLog();
x.create("//pixel/citenko/4/3/25c40279-dd90-4caa-afc9-d0474705e0d1/0.0425/32.83/-83.65");
-
+
}
-
+
/**
* Default constructor
*/
public PixelClickConvertLog() {
-
+
}
-
+
/**
* Create the log from a chunk of text
- * @param data String. The data to use.
+ *
+ * @param data
+ * String. The data to use.
*/
public void create(String data) {
-
+ time = System.currentTimeMillis();
if (data.contains("redirect")) {
doClick(data);
-
return;
}
- String [] parts = data.split("/");
- payload = data;
- exchange = parts[3];
- ad_id = parts[4];
- creative_id = parts[5];
- bid_id = parts[6];
- try {
- price = Double.parseDouble(parts[7]);
- lat = Double.parseDouble(parts[8]);
- lon = Double.parseDouble(parts[9]);
- } catch (Exception error) {
- //error.printStackTrace();
- doClick(payload);
- }
+
+ doClick(payload);
type = PIXEL;
- time = System.currentTimeMillis();
-
}
-
+
/**
* Process a click
- * @param payload String. The data to turn into the log message.
+ *
+ * @param payload
+ * String. The data to turn into the log message.
*/
void doClick(String payload) {
this.payload = payload;
- String [] parts = payload.split("/");
-
- for (int i=0;i -1) {
- String [] items = parts[i].split("=");
- switch(items[0]) {
+ String[] items = parts[i].split("=");
+ switch (items[0]) {
case "lat":
try {
- lat = Double.parseDouble(items[1]);
+ lat = Double.parseDouble(items[1]);
} catch (Exception error) {
lat = 0;
}
@@ -103,10 +92,10 @@ void doClick(String payload) {
bid_id = items[1];
break;
case "ad_id":
- ad_id=items[1];
+ ad_id = items[1];
break;
case "creative_id":
- creative_id=items[1];
+ creative_id = items[1];
break;
case "exchange":
exchange = items[1];
diff --git a/src/com/xrtb/common/Configuration.java b/src/com/xrtb/common/Configuration.java
index c5e60266..8b7de40f 100644
--- a/src/com/xrtb/common/Configuration.java
+++ b/src/com/xrtb/common/Configuration.java
@@ -20,17 +20,32 @@
import java.util.Map;
import java.util.Set;
-import com.aerospike.client.AerospikeClient;
-import com.aerospike.client.async.AsyncClientPolicy;
-import com.aerospike.client.policy.ClientPolicy;
+
import com.aerospike.redisson.AerospikeHandler;
import com.aerospike.redisson.RedissonClient;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.GetObjectTaggingRequest;
+import com.amazonaws.services.s3.model.GetObjectTaggingResult;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.model.Tag;
+
import com.fasterxml.jackson.annotation.JsonIgnore;
+
import com.xrtb.bidder.Controller;
import com.xrtb.bidder.DeadmanSwitch;
import com.xrtb.bidder.RTBServer;
import com.xrtb.bidder.WebCampaign;
+import com.xrtb.blocks.Bloom;
+import com.xrtb.blocks.Cuckoo;
+import com.xrtb.blocks.LookingGlass;
import com.xrtb.blocks.NavMap;
+import com.xrtb.blocks.SimpleMultiset;
+import com.xrtb.blocks.SimpleSet;
import com.xrtb.db.DataBaseObject;
import com.xrtb.db.Database;
import com.xrtb.db.User;
@@ -42,13 +57,10 @@
import com.xrtb.geo.GeoTag;
import com.xrtb.pojo.BidRequest;
import com.xrtb.tools.DbTools;
-import com.xrtb.tools.LookingGlass;
import com.xrtb.tools.MacroProcessing;
import com.xrtb.tools.NashHorn;
import com.xrtb.tools.ZkConnect;
-import redis.clients.jedis.JedisPool;
-import redis.clients.jedis.JedisPoolConfig;
/**
* The singleton class that makes up the Configuration object. A configuration
@@ -131,7 +143,10 @@ public class Configuration {
public String password;
// The Jedis pool, if it is used
- public JedisPool jedisPool;
+ public MyJedisPool jedisPool;
+
+ public static AmazonS3Client s3;
+ public static String s3_bucket;
/**
* HTTP admin port, usually same as bidder, but set this for a different
@@ -336,6 +351,23 @@ public void initialize(String path, String shard, int port, int sslPort) throws
filesList = (List) m.get("lists");
initializeLookingGlass(filesList);
}
+
+ if (m.get("s3") != null) {
+ Map ms3 = (Map) m.get("s3");
+ String accessKey = ms3.get("access_key_id");
+ String secretAccessKey = ms3.get("secret_access_key");
+ String region = ms3.get("region");
+ s3_bucket = ms3.get("bucket");
+
+ s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretAccessKey));
+ ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(s3_bucket));
+
+ try {
+ processDirectory(s3, listing, s3_bucket);
+ } catch (Exception error) {
+ System.err.println("ERROR IN AWS LISTING: " + error.toString());
+ }
+ }
/**
* SSL
*/
@@ -470,7 +502,7 @@ public void initialize(String path, String shard, int port, int sslPort) throws
ForensiqClient.getInstance().connections = (int) (Integer) fraud.get("connections");
forensiq = fx;
}
- } else {
+ } else {
System.out.println("*** Fraud detection is set to MMDB");
String db = (String) fraud.get("db");
MMDBClient fy = MMDBClient.build(db);
@@ -478,7 +510,7 @@ public void initialize(String path, String shard, int port, int sslPort) throws
fy.bidOnError = (Boolean) fraud.get("bidOnError");
}
if (fraud.get("watchlist") != null) {
- fy.setWatchlist((List)fraud.get("watchlist"));
+ fy.setWatchlist((List) fraud.get("watchlist"));
}
forensiq = fy;
}
@@ -551,11 +583,25 @@ public void initialize(String path, String shard, int port, int sslPort) throws
Integer rport = (Integer) redis.get("port");
if (rport == null)
rport = 6379;
-
- jedisPool = new JedisPool(host,rport);
+
+ // JedisPoolConfig poolConfig = new JedisPoolConfig();;
+ // configJedis.setMaxTotal(rsize);
+ // configJedis.setMaxWaitMillis(10);
+
+ // poolConfig.setMaxIdle(4000);
+ // Tests whether connections are dead during idle periods
+ // poolConfig.setTestWhileIdle(true);
+ // poolConfig.setMaxTotal(4000);
+ // poolConfig.setMaxWaitMillis(30);
+
+ // jedisPool = new JedisPool(poolConfig,host,rport);
+
+ MyJedisPool.host = host;
+ MyJedisPool.port = rport;
+ jedisPool = new MyJedisPool(1000, 1000, 5);
System.out.println(
- "*** JEDISPOOL = " + jedisPool + ". host = " + host + ", port = " + port + ", size = " + rsize);
+ "*** JEDISPOOL = " + jedisPool + ". host = " + host + ", port = " + rport + ", size = " + rsize);
}
Map zeromq = (Map) m.get("zeromq");
@@ -570,13 +616,13 @@ public void initialize(String path, String shard, int port, int sslPort) throws
cacheHost = value;
if (r.get("port") != null)
cachePort = (Integer) r.get("port");
- if (r.get("maxconns") != null)
+ if (r.get("maxconns") != null)
maxconns = (Integer) r.get("maxconns");
- AerospikeHandler.getInstance(cacheHost,cachePort,maxconns);
+ AerospikeHandler.getInstance(cacheHost, cachePort, maxconns);
redisson = new RedissonClient(AerospikeHandler.getInstance());
Database.getInstance(redisson);
- System.out.println("*** Aerospike connection set to: " + cacheHost + ":" + cachePort + ", connections = " + maxconns + ", handlers: " +
- AerospikeHandler.getInstance().getCount() + " ***");
+ System.out.println("*** Aerospike connection set to: " + cacheHost + ":" + cachePort + ", connections = "
+ + maxconns + ", handlers: " + AerospikeHandler.getInstance().getCount() + " ***");
String key = (String) m.get("deadmanswitch");
if (key != null) {
@@ -689,6 +735,130 @@ public String requstLogStrategyAsString() {
return "all";
}
+ public void processDirectory(AmazonS3Client s3, ObjectListing listing, String bucket) throws Exception {
+ for (S3ObjectSummary objectSummary : listing.getObjectSummaries()) {
+ long size = objectSummary.getSize();
+ System.out.println("*** Processing S3 " + objectSummary.getKey() + ", size = " + size);
+ S3Object object = s3.getObject(new GetObjectRequest(bucket, objectSummary.getKey()));
+
+ String bucketName = object.getBucketName();
+ String keyName = object.getKey();
+
+ if (keyName.contains("Darren")) {
+ System.out.println("HERE");
+ }
+ GetObjectTaggingRequest request = new GetObjectTaggingRequest(bucketName, keyName);
+ GetObjectTaggingResult result = s3.getObjectTagging(request);
+ List tags = result.getTagSet();
+ String type = null;
+ String name = null;
+
+ if (tags.isEmpty()) {
+ System.err.println("Error: " + keyName + " has no tags");
+ } else {
+ for (Tag tag : tags) {
+ String key = tag.getKey();
+ String value = tag.getValue();
+
+ if (key.equals("type")) {
+ type = value;
+ }
+
+ if (key.equals("name")) {
+ name = value;
+ }
+ }
+
+ if (name == null)
+ throw new Exception("Error: " + keyName + " is missing a name tag");
+ if (name.contains(" "))
+ throw new Exception("Error: " + keyName + " has a name attribute with a space in it");
+ if (type == null)
+ throw new Exception("Error: " + keyName + " has no type tag");
+
+ if (!name.startsWith("$"))
+ name = "$" + name;
+
+ readData(type, name, object, size);
+ }
+ }
+ }
+
+ public static String readData(String fileName) throws Exception {
+ String message = "";
+ int i = fileName.indexOf(".");
+ if (i == -1)
+ throw new Exception("Filename is missing type field");
+ String type = fileName.substring(i);
+ NavMap map;
+ SimpleMultiset set;
+ SimpleSet sset;
+ Bloom b;
+ Cuckoo c;
+ switch(type) {
+ case "range":
+ map = new NavMap(fileName,fileName,false);
+ message = "Added NavMap " + fileName + ": from file, has " + map.size() + " members";
+ break;
+ case "cidr":
+ map = new NavMap(fileName,fileName,true);
+ message = "Added NavMap " + fileName + ": from file, has " + map.size() + " members";
+ break;
+ case "bloom":
+ b = new Bloom(fileName,fileName);
+ message = "Initialize Bloom Filter: " + fileName + " from file, members = " + b.getMembers();
+ break;
+ case "cuckoo":
+ c = new Cuckoo(fileName,fileName);
+ break;
+ case "multiset":
+ set = new SimpleMultiset(fileName, fileName);
+ message = "Initialize Multiset " + fileName + " from file, entries = " + set.getMembers();
+ break;
+ case "set":
+ sset = new SimpleSet(fileName, fileName);
+ message = "Initialize Multiset " + fileName + " from file, entries = " + sset.size();
+ break;
+
+ default:
+ message = "Unknown type: " + type;
+ }
+ System.out.println("*** " + message);
+ return message;
+ }
+
+ public static String readData(String type, String name, S3Object object, long size) throws Exception {
+ String message = "";
+ switch (type) {
+ case "range":
+ case "cidr":
+ NavMap map = new NavMap(name, object);
+ message = "Added NavMap " + name + ": has " + map.size() + " members";
+ break;
+ case "set":
+ SimpleSet set = new SimpleSet(name, object);
+ message = "Initialize Set: " + name + " from S3, entries = " + set.size();
+ break;
+ case "bloom":
+ Bloom b = new Bloom(name, object, size);
+ message = "Initialize Bloom Filter: " + name + " from S3, members = " + b.getMembers();
+ break;
+
+ case "cuckoo":
+ Cuckoo c = new Cuckoo(name, object, size);
+ message = "Initialize Cuckoo Filter: " + name + " from S3, entries = " + c.getMembers();
+ break;
+ case "multiset":
+ SimpleMultiset ms = new SimpleMultiset(name, object);
+ message = "Initialize Multiset " + name + " from S3, entries = " + ms.getMembers();
+ break;
+ default:
+ message = "Unknown type: " + type;
+ }
+ System.out.println("*** " + message);
+ return message;
+ }
+
public int requstLogStrategyAsInt(String x) {
switch (x) {
case "all":
@@ -839,8 +1009,8 @@ public static Configuration getInstance(String fileName, String shard, int port,
synchronized (Configuration.class) {
if (theInstance == null) {
theInstance = new Configuration();
- theInstance.initialize(fileName, shard, port, sslPort);
try {
+ theInstance.initialize(fileName, shard, port, sslPort);
theInstance.shell = new JJS();
} catch (Exception error) {
diff --git a/src/com/xrtb/common/Deals.java b/src/com/xrtb/common/Deals.java
index 3e733b7f..2b0859db 100644
--- a/src/com/xrtb/common/Deals.java
+++ b/src/com/xrtb/common/Deals.java
@@ -9,32 +9,124 @@
import com.xrtb.tools.XORShiftRandom;
+/**
+ * A class that chooses deals. For duplicates you can choose random or highest (and random of dup highest). It extends
+ * List simply as a convenience for configuration.
+ * @author Ben M. Faul
+ *
+ */
public class Deals extends ArrayList {
+ // Serialize id
+ private static final long serialVersionUID = 1L;
+
+ // Random number generator
private static XORShiftRandom rand = new XORShiftRandom();
- private Map map = new HashMap();
- private Set s2;
+ // A map of the deals
+ private Map map = new HashMap();
+ // Set of ids in the deals list
+ private Set s2 = new HashSet();
+ /**
+ * Default constructor for Jackson to use.
+ */
public Deals() {
}
- public Deal findDeal(List ids) {
+ /**
+ * Find a random deal in an intersection with a list of ids.
+ * @param ids List. A list of String ids.
+ * @return Deal. A random deal from the intersection of ids with this set.
+ */
+ public Deal findDealRandom(List ids) {
Set intersection = new HashSet(ids); // use the copy constructor
intersection.retainAll(s2);
if (intersection.size()==0)
return null;
+
int x = rand.random(intersection.size());
List nameList = new ArrayList(intersection);
String key = nameList.get(x);
return map.get(key);
}
+ /**
+ * Return the highest price deal in the list of ids that match. If duplicates are found, then choose one of those at random.
+ * @param ids List. A list of ids to match.
+ * @return Deal. The hoghest price deal that matches.
+ */
+ public Deal findDealHighest(List ids) {
+ Set intersection = new HashSet(ids); // use the copy constructor
+ intersection.retainAll(s2);
+ if (intersection.size()==0)
+ return null;
+
+ List nameList = new ArrayList(intersection);
+ List candidates = new ArrayList();
+ Deal x = null;
+ for (int i=0;i< nameList.size();i++) {
+ String key = nameList.get(i);
+ Deal test = map.get(key);
+ if (x == null || x.price < test.price) {
+ x = test;
+ candidates.clear();
+ candidates.add(x);
+ } else {
+ if (x.price == test.price)
+ candidates.add(test);
+ }
+ }
+ if (candidates.size()==1)
+ return x;
+ int r = rand.random(candidates.size());
+ return candidates.get(r);
+ }
+
@Override
public boolean add(Deal d) {
s2.add(d.id);
map.put(d.id,d);
- return add(d);
+ return super.add(d);
+ }
+
+ public static void main(String args[]) {
+
+ Deals deals = new Deals();
+ Deal a = new Deal("a",1);
+ Deal b = new Deal("b",2);
+ deals.add(a);
+ deals.add(b);
+
+ List test = new ArrayList();
+ test.add("c");
+ Deal x = deals.findDealRandom(test);
+ System.out.println(x);
+
+ test.add("a");
+ x = deals.findDealRandom(test);
+ System.out.println(x.id);
+
+ test.add("b");
+
+ for (int i=0;i<10;i++) {
+ x = deals.findDealRandom(test);
+ System.out.println(x.id);
+ }
+
+ for (int i=0;i<10;i++) {
+ x = deals.findDealHighest(test);
+ System.out.println(x.id);
+ }
+
+ Deal z = new Deal("z",2);
+ deals.add(z);
+ test.add("z");
+
+ for (int i=0;i<10;i++) {
+ x = deals.findDealHighest(test);
+ System.out.println(x.id);
+ }
}
}
diff --git a/src/com/xrtb/common/MyJedisPool.java b/src/com/xrtb/common/MyJedisPool.java
index 511924b9..1cbf5e99 100644
--- a/src/com/xrtb/common/MyJedisPool.java
+++ b/src/com/xrtb/common/MyJedisPool.java
@@ -1,165 +1,24 @@
package com.xrtb.common;
-
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-
-import edu.emory.mathcs.backport.java.util.concurrent.locks.ReentrantLock;
import redis.clients.jedis.Jedis;
-/**
- * A threas safe class that sanely handles broken resources in Jedis. Stupid jedis pool is brain dead.
- * @author Ben M. Faul
- *
- */
-public class MyJedisPool {
- private final BlockingQueue pool;
- private final ReentrantLock lock = new ReentrantLock();
- private int createdObjects = 0;
- private int size = 64;
-
- String host = "localhost";
- int port = 6379;
-
- /**
- * Create a default pool. We create 64 by default.
- */
- public MyJedisPool() {
- this(64,true);
-
- }
-
- /**
- * Create a jedis pool to the host and port, with default number of connections.
- * @param host String. The hostname.
- * @param port int. The port to connect to.
- */
- public MyJedisPool(String host, int port) {
- this(64, true);
- this.host = host;
- this.port = port;
- }
-
- /**
- * Create a jedis pool to the host and port, with specified number of connections.
- * @param host String. The hostname.
- * @param port int. The port to connect to.
- * @param size int. The number of JEDIS objects to keep around.
- */
- public MyJedisPool(String host, int port, int size) {
- this(size, true);
- this.host = host;
- this.port = port;
- }
-
- /**
- * Create the pool.
- * @param size int. The size of the pool at max.
- * @param dynamicCreation boolean. Allow dynamic creations
- */
- protected MyJedisPool(int size, Boolean dynamicCreation) {
- // Enable the fairness; otherwise, some threads
- // may wait forever.
- pool = new ArrayBlockingQueue(size,true);
- this.size = size;
- if (!dynamicCreation) {
- lock.lock();
- }
- }
-
- /**
- * Get a resource.
- * @return Jedis. The connection to JEDIS.
- * @throws Exception on connection errors.
- */
- public Jedis getResource() throws Exception {
- if (pool.size() == 0 && createdObjects < size) {
- ++createdObjects;
- System.out.println("MAKE OBJECT: " + createdObjects);
- return createObject();
- }
-
- return pool.take();
- }
-
- /**
- * Return a resource to the pool.
- * @param resource Jedis. The thing to return.
- * @throws Exception on addition errors.
- */
- public void returnResource(Jedis resource) throws Exception {
- // Will throws Exception when the queue is full,
- // but it should never happen.
- pool.add(resource);
- }
-
- /**
- * Create the pool, when dynamic. Will open the connections and stuff them all in.
- */
- public void createPool() {
- if (lock.isLocked()) {
- for (int i = 0; i < size; ++i) {
- pool.add(createObject());
- createdObjects++;
- }
- }
- }
-
- /**
- * Create an object for the pool.
- * @return Jedis. The new resource.
- */
- protected Jedis createObject() {
- Jedis jedis = new Jedis(host,port);
- jedis.connect();
- return jedis;
- }
-
- /**
- * Publish from the pool.
- * @param channel String. The channel to publish on.
- * @param message String. The message to publish.
- */
- public void publish(String channel, String message) {
- Jedis j = null;
- try {
- j = getResource();
- j.publish(channel, message);
- } catch (Exception error) {
- j.disconnect();
- j.connect();
- j.publish(channel, message);
- }
- try {
- returnResource(j);
- } catch (Exception e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
-
- /**
- * Do sismember from the pool.
- * @param key String. The set we are checking.
- * @param value. String. The value we are checking membership on.
- * @return boolean. Returns true if in the set, else false.
- */
- public boolean sismember(String key, String value) {
- boolean t = false;
- Jedis j = null;
- try {
- j = getResource();
- t = j.sismember(key, value);
- } catch (Exception error) {
- j.disconnect();
- j.connect();
- t = j.sismember(key, value);
- }
- try {
- returnResource(j);
- } catch (Exception e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- return t;
- }
-}
+public class MyJedisPool extends ObjectPool {
+
+ public static String host = "localhost";
+ public static int port = 6379;
+ public MyJedisPool(int minIdle) {
+ super(minIdle);
+ }
+
+ public MyJedisPool(final int minIdle, final int maxIdle, final long validationInterval) {
+ super(minIdle, maxIdle, validationInterval);
+ }
+
+ @Override
+ protected Jedis createObject() {
+ Jedis jedis = new Jedis(host,port);
+ jedis.connect();
+
+ return jedis;
+ }
+
+}
\ No newline at end of file
diff --git a/src/com/xrtb/common/Node.java b/src/com/xrtb/common/Node.java
index 9b56494b..09226098 100644
--- a/src/com/xrtb/common/Node.java
+++ b/src/com/xrtb/common/Node.java
@@ -22,6 +22,8 @@
import com.fasterxml.jackson.databind.node.MissingNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
+import com.google.common.hash.BloomFilter;
+import com.xrtb.blocks.LookingGlass;
import com.xrtb.blocks.NavMap;
import com.xrtb.pojo.BidRequest;
@@ -646,21 +648,26 @@ public boolean testInternal(Object value) throws Exception {
case MEMBER:
case NOT_MEMBER:
- if (sval != null && sval.startsWith("@")) {
- boolean t = NavMap.searchTable(sval,svalue);
- if (operator == NOT_MEMBER)
- return !t;
- else
- return t;
- }
- if (sval != null && sval.startsWith("$")) {
- boolean t = jedisIsMember(sval,svalue);
+ if (sval != null && (sval.startsWith("@") || sval.startsWith("$"))) {
+ Object x = LookingGlass.get(sval);
+ boolean t = false;
+ if (x instanceof NavMap) {
+ NavMap nm = (NavMap)x;
+ t = nm.contains(svalue);
+ } else if (x instanceof BloomFilter) {
+ BloomFilter b = (BloomFilter)x;
+ t = b.mightContain(svalue);
+ } else if (x instanceof Set) {
+ Set set = (Set)x;
+ t = set.contains(svalue);
+ }
+
if (operator == NOT_MEMBER)
return !t;
else
return t;
}
-
+
if (qvalue == null) {
if (lval != null)
qvalue = new HashSet(lval);
@@ -802,10 +809,23 @@ public boolean processLTE(int operator, Number ival, Number nvalue, String sval,
boolean jedisIsMember(String key, String member) {
boolean t = false;
+ if (Configuration.getInstance().jedisPool == null)
+ return false;
+
try {
- Jedis jedis = Configuration.getInstance().jedisPool.getResource();
+
+ if (member.equals("842AAB10FBA04247B3A9CE00C9172350")) {
+ System.out.println("$$$$$$$$$$$$$$$$$$$ KEYTEST on " + key);
+ }
+ //Jedis jedis = Configuration.getInstance().jedisPool.getResource();
+ Jedis jedis = Configuration.getInstance().jedisPool.borrowObject();
t = jedis.sismember(key,member);
- Configuration.getInstance().jedisPool.returnResource(jedis);
+
+ if (member.equals("842AAB10FBA04247B3A9CE00C9172350")) {
+ System.out.println("$$$$$$$$$$$$$$$$$$$ TEST RETURNS: " + t);
+ }
+ //Configuration.getInstance().jedisPool.returnResource(jedis);
+ Configuration.getInstance().jedisPool.returnObject(jedis);
//t = Configuration.getInstance().jedisPool.sismember(key, member);
} catch (Exception error) {
error.printStackTrace();
diff --git a/src/com/xrtb/common/ObjectPool.java b/src/com/xrtb/common/ObjectPool.java
new file mode 100644
index 00000000..f226b6c7
--- /dev/null
+++ b/src/com/xrtb/common/ObjectPool.java
@@ -0,0 +1,129 @@
+package com.xrtb.common;
+
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ * @see simple pool>
+ */
+abstract class ObjectPool
+{
+ private ConcurrentLinkedQueue pool;
+
+ private ScheduledExecutorService executorService;
+
+ /**
+ * Creates the pool.
+ *
+ * @param minIdle minimum number of objects residing in the pool
+ */
+ public ObjectPool(final int minIdle)
+ {
+ // initialize pool
+ initialize(minIdle);
+ }
+
+ /**
+ * Creates the pool.
+ *
+ * @param minIdle minimum number of objects residing in the pool
+ * @param maxIdle maximum number of objects residing in the pool
+ * @param validationInterval time in seconds for periodical checking of minIdle / maxIdle conditions in a separate thread.
+ * When the number of objects is less than minIdle, missing instances will be created.
+ * When the number of objects is greater than maxIdle, too many instances will be removed.
+ */
+ public ObjectPool(final int minIdle, final int maxIdle, final long validationInterval)
+ {
+ // initialize pool
+ initialize(minIdle);
+
+ // check pool conditions in a separate thread
+ executorService = Executors.newSingleThreadScheduledExecutor();
+ executorService.scheduleWithFixedDelay(new Runnable()
+ {
+ @Override
+ public void run()
+ {
+ int size = pool.size();
+ if (size < minIdle)
+ {
+ int sizeToBeAdded = minIdle - size;
+ for (int i = 0; i < sizeToBeAdded; i++)
+ {
+ pool.add(createObject());
+ }
+ } else if (size > maxIdle)
+ {
+ int sizeToBeRemoved = size - maxIdle;
+ for (int i = 0; i < sizeToBeRemoved; i++)
+ {
+ pool.poll();
+ }
+ }
+ }
+ }, validationInterval, validationInterval, TimeUnit.SECONDS);
+ }
+
+ /**
+ * Gets the next free object from the pool. If the pool doesn't contain any objects,
+ * a new object will be created and given to the caller of this method back.
+ *
+ * @return T borrowed object
+ */
+ public T borrowObject()
+ {
+ T object;
+ if ((object = pool.poll()) == null)
+ {
+ object = createObject();
+ }
+
+ return object;
+ }
+
+ /**
+ * Returns object back to the pool.
+ *
+ * @param object object to be returned
+ */
+ public void returnObject(T object)
+ {
+ if (object == null)
+ {
+ return;
+ }
+
+ this.pool.offer(object);
+ }
+
+ /**
+ * Shutdown this pool.
+ */
+ public void shutdown()
+ {
+ if (executorService != null)
+ {
+ executorService.shutdown();
+ }
+ }
+
+ /**
+ * Creates a new object.
+ *
+ * @return T new object
+ */
+ protected abstract T createObject();
+
+ private void initialize(final int minIdle)
+ {
+ pool = new ConcurrentLinkedQueue();
+
+ for (int i = 0; i < minIdle; i++)
+ {
+ pool.add(createObject());
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/com/xrtb/exchanges/C1X.java b/src/com/xrtb/exchanges/C1X.java
index af2f463d..c229ecc1 100644
--- a/src/com/xrtb/exchanges/C1X.java
+++ b/src/com/xrtb/exchanges/C1X.java
@@ -6,9 +6,9 @@
import com.fasterxml.jackson.databind.node.MissingNode;
import com.fasterxml.jackson.databind.node.TextNode;
+import com.xrtb.blocks.LookingGlass;
import com.xrtb.pojo.BidRequest;
import com.xrtb.tools.IsoTwo2Iso3;
-import com.xrtb.tools.LookingGlass;
/**
* A class to handle C1X ad exchange
diff --git a/src/com/xrtb/exchanges/adx/AdxBidRequest.java b/src/com/xrtb/exchanges/adx/AdxBidRequest.java
index 5f39b93c..68a191dd 100644
--- a/src/com/xrtb/exchanges/adx/AdxBidRequest.java
+++ b/src/com/xrtb/exchanges/adx/AdxBidRequest.java
@@ -18,6 +18,7 @@
import com.google.protobuf.ByteString;
import com.xrtb.bidder.Controller;
import com.xrtb.bidder.RTBServer;
+import com.xrtb.blocks.LookingGlass;
import com.xrtb.common.Campaign;
import com.xrtb.common.Configuration;
import com.xrtb.common.Creative;
@@ -37,7 +38,6 @@
import com.xrtb.pojo.BidResponse;
import com.xrtb.pojo.Impression;
import com.xrtb.pojo.Video;
-import com.xrtb.tools.LookingGlass;
interface Command {
void runCommand(BidRequest br, RealtimeBidding.BidRequest x, ObjectNode root, Map db, String key);
diff --git a/src/com/xrtb/exchanges/adx/AdxGeoCodes.java b/src/com/xrtb/exchanges/adx/AdxGeoCodes.java
index b47fe138..535d13ed 100644
--- a/src/com/xrtb/exchanges/adx/AdxGeoCodes.java
+++ b/src/com/xrtb/exchanges/adx/AdxGeoCodes.java
@@ -5,7 +5,7 @@
import java.util.HashMap;
import java.util.Map;
-import com.xrtb.tools.LookingGlass;
+import com.xrtb.blocks.LookingGlass;
public class AdxGeoCodes extends LookingGlass {
diff --git a/src/com/xrtb/exchanges/adx/MakeMasterGeoTable.java b/src/com/xrtb/exchanges/adx/MakeMasterGeoTable.java
index 699524a8..7e39d79e 100644
--- a/src/com/xrtb/exchanges/adx/MakeMasterGeoTable.java
+++ b/src/com/xrtb/exchanges/adx/MakeMasterGeoTable.java
@@ -8,7 +8,7 @@
import java.util.HashMap;
import java.util.Map;
-import com.xrtb.tools.LookingGlass;
+import com.xrtb.blocks.LookingGlass;
public class MakeMasterGeoTable {
diff --git a/src/com/xrtb/exchanges/google/GoogleBidResponse.java b/src/com/xrtb/exchanges/google/GoogleBidResponse.java
index ead6f894..ecfb1c50 100644
--- a/src/com/xrtb/exchanges/google/GoogleBidResponse.java
+++ b/src/com/xrtb/exchanges/google/GoogleBidResponse.java
@@ -224,20 +224,23 @@ public GoogleBidResponse(GoogleBidRequest br, Impression imp, Campaign camp, Cre
} else {
adm = substitute(creat.unencodedAdm);
}
- this.forwardUrl = adm;
+
} else if (this.creat.isNative()) {
if (br.usesEncodedAdm) {
adm = substitute(this.creat.getEncodedNativeAdm(br));
} else {
adm = substitute(this.creat.unencodedAdm);
}
- this.forwardUrl = adm;
} else {
adm = substitute(getTemplate());
}
//////////////////
+ this.forwardUrl = adm;
+
+ // System.out.println(adm);
+
Bid.Builder bb = Bid.newBuilder();
bb.addAdomain(camp.adomain);
@@ -278,6 +281,8 @@ public GoogleBidResponse(GoogleBidRequest br, Impression imp, Campaign camp, Cre
internal = builder.build();
+ // System.out.println(internal);
+
// add this to the log
byte[] bytes = internal.toByteArray();
protobuf = new String(Base64.encodeBase64(bytes));
diff --git a/src/com/xrtb/jmq/Publisher.java b/src/com/xrtb/jmq/Publisher.java
index 771a316a..41fde2ae 100644
--- a/src/com/xrtb/jmq/Publisher.java
+++ b/src/com/xrtb/jmq/Publisher.java
@@ -4,12 +4,18 @@
import org.zeromq.ZMQ.Context;
import org.zeromq.ZMQ.Socket;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
public class Publisher {
Socket publisher = null;
Context context = JMQContext.getInstance();
boolean running = false;
String topicName = null;
+
+ ObjectMapper mapper = new ObjectMapper();
public static void main(String[] args) throws Exception {
@@ -23,6 +29,9 @@ public static void main(String[] args) throws Exception {
public Publisher(String binding, String topicName) throws Exception {
+ mapper.setSerializationInclusion(Include.NON_NULL);
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
context = ZMQ.context(1);
publisher = context.socket(ZMQ.PUB);
publisher.bind(binding);
@@ -38,7 +47,7 @@ public Publisher(String binding, String topicName) throws Exception {
public void publish(Object message) {
publisher.sendMore(topicName);
- String msg = Tools.serialize(message);
+ String msg = Tools.serialize(mapper, message);
if (msg != null)
publisher.send(msg);
else
@@ -48,7 +57,7 @@ public void publish(Object message) {
public void publishAsync(Object message) {
Runnable u = () -> {
publisher.sendMore(topicName);
- String msg = Tools.serialize(message);
+ String msg = Tools.serialize(mapper,message);
publisher.send(msg);
};
Thread nthread = new Thread(u);
diff --git a/src/com/xrtb/jmq/RTopic.java b/src/com/xrtb/jmq/RTopic.java
index e77cc909..00949113 100644
--- a/src/com/xrtb/jmq/RTopic.java
+++ b/src/com/xrtb/jmq/RTopic.java
@@ -14,6 +14,10 @@
import org.zeromq.ZMQ.Context;
import org.zeromq.ZMQ.Socket;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
public class RTopic implements EventIF {
Socket publisher = null;
@@ -22,9 +26,15 @@ public class RTopic implements EventIF {
boolean running = false;
String topicName = null;
public Map m = new HashMap();
+ ObjectMapper mapper = new ObjectMapper();
+
public RTopic(String address) throws Exception {
context = ZMQ.context(1);
+
+ mapper.setSerializationInclusion(Include.NON_NULL);
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
if (address.contains("&")) {
String [] parts = address.split("&");
subscriber = new Subscriber(this,parts[0]);
@@ -39,6 +49,8 @@ public RTopic(String address) throws Exception {
* @throws Exception
*/
public RTopic(List addresses) throws Exception {
+ mapper.setSerializationInclusion(Include.NON_NULL);
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
context = ZMQ.context(1);
subscriber = new MSubscriber(this,addresses);
@@ -108,7 +120,7 @@ public void addListener(MessageListener> z) {
@Override
public void handleMessage(String id, String msg) {
- Object [] x = Tools.deSerialize(msg);
+ Object [] x = Tools.deSerialize(mapper,msg);
String name = (String)x[0];
Object o = m.get(name);
if (o != null) {
diff --git a/src/com/xrtb/jmq/Tools.java b/src/com/xrtb/jmq/Tools.java
index 9be181b1..6572c463 100644
--- a/src/com/xrtb/jmq/Tools.java
+++ b/src/com/xrtb/jmq/Tools.java
@@ -11,13 +11,7 @@
public class Tools {
- static final ObjectMapper mapper = new ObjectMapper();
- static {
- mapper.setSerializationInclusion(Include.NON_NULL);
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- public static String serialize(Object o) {
+ public static String serialize(ObjectMapper mapper, Object o) {
String contents = null;
StringBuilder sb = new StringBuilder();
try {
@@ -38,7 +32,7 @@ public static String serialize(Object o) {
return contents;
}
- public static Object[] deSerialize(String o) {
+ public static Object[] deSerialize(ObjectMapper mapper, String o) {
Object obj = null;
String name = null;
if (o.charAt(0) == '{') {
diff --git a/src/com/xrtb/tools/IsoTwo2Iso3.java b/src/com/xrtb/tools/IsoTwo2Iso3.java
index e20a7289..f76bf16a 100644
--- a/src/com/xrtb/tools/IsoTwo2Iso3.java
+++ b/src/com/xrtb/tools/IsoTwo2Iso3.java
@@ -5,7 +5,7 @@
import java.util.HashMap;
import java.util.Map;
-import com.xrtb.tools.LookingGlass;
+import com.xrtb.blocks.LookingGlass;
/**
* A class to create a map to look up 2 char ISO country codes and return 3 char country codes.
diff --git a/src/test/java/TestBloom.java b/src/test/java/TestBloom.java
new file mode 100644
index 00000000..032d6b38
--- /dev/null
+++ b/src/test/java/TestBloom.java
@@ -0,0 +1,63 @@
+package test.java;
+
+import static org.junit.Assert.*;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.hash.BloomFilter;
+import com.xrtb.bidder.Controller;
+import com.xrtb.blocks.Bloom;
+import com.xrtb.blocks.LookingGlass;
+import com.xrtb.blocks.NavMap;
+
+/**
+ * A class for testing that the bid has the right parameters
+ * @author Ben M. Faul
+ *
+ */
+public class TestBloom {
+ static Controller c;
+ public static String test = "";
+
+ @BeforeClass
+ public static void testSetup() {
+
+ }
+
+ @AfterClass
+ public static void testCleanup() {
+
+ }
+
+ /**
+ * Test a valid bid response.
+ * @throws Exception on networking errors.
+ */
+ @Test
+ public void testBloom() throws Exception {
+
+
+ new Bloom("$test","data/c1x_cookies.csv");
+ BloomFilter b = (BloomFilter)LookingGlass.get("$test");
+ assertNotNull(b);
+
+ boolean p = b.mightContain("842AAB10FBA04247B3A9CE00C9172350");
+
+ BufferedReader br = new BufferedReader(new FileReader("data/c1x_cookies.csv"));
+ String line = null;
+ int nP = 0;
+ int k = 0;
+ while((line = br.readLine()) != null) {
+ p = b.mightContain(line);
+ if (p)
+ nP++;
+ k++;
+ }
+ assertTrue(k == nP);
+ }
+}
\ No newline at end of file
diff --git a/src/test/java/TestCidr.java b/src/test/java/TestCidr.java
index 4f957b36..59544bce 100644
--- a/src/test/java/TestCidr.java
+++ b/src/test/java/TestCidr.java
@@ -8,6 +8,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
import com.xrtb.bidder.Controller;
+import com.xrtb.blocks.LookingGlass;
import com.xrtb.blocks.NavMap;
/**
@@ -38,29 +39,22 @@ public void testCidr() throws Exception {
NavMap sr = new NavMap("CIDR", "data/METHBOT.txt", true);
-
- long x = NavMap.ipToLong("45.33.224.0");
- boolean p = NavMap.searchTable("CIDR", x);
+ boolean p = sr.contains("45.33.224.0");
assertTrue(p);
- x = NavMap.ipToLong("45.33.239.255");
- p = NavMap.searchTable("CIDR", x);
+ p = sr.contains("45.33.239.255");
assertTrue(p);
- x = NavMap.ipToLong("44.33.224.0");
- p = NavMap.searchTable("CIDR", x);
+ p = sr.contains("44.33.224.0");
assertFalse(p);
- x = NavMap.ipToLong("165.52.0.0");
- p = NavMap.searchTable("CIDR", x);
+ p = sr.contains("165.52.0.0");
assertTrue(p);
- x = NavMap.ipToLong("165.55.255.255");
- p = NavMap.searchTable("CIDR", x);
+ p = sr.contains("165.55.255.255");
assertTrue(p);
- x = NavMap.ipToLong("166.55.255.255");
- p = NavMap.searchTable("CIDR", x);
+ p = sr.contains("166.55.255.255");
assertFalse(p);
}
diff --git a/src/test/java/TestDeals.java b/src/test/java/TestDeals.java
index ecf5eb0c..529b79a5 100644
--- a/src/test/java/TestDeals.java
+++ b/src/test/java/TestDeals.java
@@ -8,6 +8,7 @@
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
@@ -22,6 +23,8 @@
import com.xrtb.bidder.Controller;
import com.xrtb.common.Configuration;
+import com.xrtb.common.Deal;
+import com.xrtb.common.Deals;
import com.xrtb.common.HttpPostGet;
import com.xrtb.jmq.MessageListener;
import com.xrtb.jmq.RTopic;
@@ -134,4 +137,60 @@ public void testPrivateResponse() throws Exception {
int code = http.getResponseCode();
assertEquals(code,200);
}
+
+ /**
+ * Test the deal object
+ */
+ @Test
+ public void testDealsObject() {
+
+ Deals deals = new Deals();
+ Deal a = new Deal("a",1);
+ Deal b = new Deal("b",2);
+ deals.add(a);
+ deals.add(b);
+
+ List test = new ArrayList();
+ test.add("c");
+ Deal x = deals.findDealRandom(test);
+ assertNull(x);
+
+ // Must be a
+ test.add("a");
+ x = deals.findDealRandom(test);
+ assertTrue(x.id.equals("a"));
+
+ test.add("b");
+
+ // Can be a or B
+ int isA = 0;
+ for (int i=0;i<10;i++) {
+ x = deals.findDealRandom(test);
+ if (x.id.equals("a"))
+ isA++;
+ }
+ assertTrue(isA != 10);
+
+
+ // Must be b
+ int isB = 0;
+ for (int i=0;i<10;i++) {
+ x = deals.findDealHighest(test);
+ if (x.id.equals("b"))
+ isB++;
+ }
+ assertTrue(isB == 10);
+
+ Deal z = new Deal("z",2);
+ deals.add(z);
+ test.add("z");
+
+ // Can be b or z
+ isB = 0;
+ for (int i=0;i<10;i++) {
+ x = deals.findDealHighest(test);
+ if (x.id.equals("b"))
+ isB++;
+ }
+ }
}
diff --git a/src/test/java/TestNavMap.java b/src/test/java/TestNavMap.java
deleted file mode 100644
index 399989ce..00000000
--- a/src/test/java/TestNavMap.java
+++ /dev/null
@@ -1,55 +0,0 @@
-package test.java;
-
-import static org.junit.Assert.*;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import com.xrtb.bidder.Controller;
-import com.xrtb.blocks.NavMap;
-import com.xrtb.fraud.ForensiqClient;
-import com.xrtb.fraud.FraudLog;
-/**
- * A class for testing that the bid has the right parameters
- * @author Ben M. Faul
- *
- */
-public class TestNavMap {
- static Controller c;
- public static String test = "";
-
- @BeforeClass
- public static void testSetup() {
-
- }
-
- @AfterClass
- public static void testCleanup() {
-
- }
-
- /**
- * Test a valid bid response.
- * @throws Exception on networking errors.
- */
- @Test
- public void testNavMap() throws Exception {
- NavMap sr = new NavMap("CIDR", "data/TESTCIDR1.txt", false);
-
- long x = NavMap.ipToLong("192.168.0.12");
- boolean p = NavMap.searchTable("CIDR", x);
- assertTrue(p);
-
- x = NavMap.ipToLong("191.168.0.12");
- p = NavMap.searchTable("CIDR", x);
- assertFalse(p);
-
- x = NavMap.ipToLong("223.255.231.255");
- p = NavMap.searchTable("CIDR", x);
- assertFalse(p);
- }
-}
\ No newline at end of file