diff --git a/Campaigns/payday.json b/Campaigns/payday.json index 7b966e98..0083a086 100644 --- a/Campaigns/payday.json +++ b/Campaigns/payday.json @@ -21,8 +21,8 @@ "adminSSL": false, "password": "startrekisbetterthanstarwars", "verbosity": { - "level": -5, - "nobid-reason": true + "level": -3, + "nobid-reason": false }, "geotags": { "states": "", @@ -49,10 +49,6 @@ "status": "file://logs/status&time=30", "reasons": "file://logs/reasons&time=30" }, - "redis": { - "host": "localhost", - "port": 6379 - }, "template": { "default": "{creative_forward_url}", "exchange": { diff --git a/data/c1x_cookies.csv b/data/c1x_cookies.csv new file mode 100644 index 00000000..bcc017c8 --- /dev/null +++ b/data/c1x_cookies.csv @@ -0,0 +1,803 @@ +842AAB10FBA04247B3A9CE00C9172350 +1DF805E5B2014E6680DBBE93193A3B8C +4F5776B2B902411A9F06899372143724 +51F9D985BAF9445690AF22939DCC72A6 +5AC65210F9DB451F9CFC4F595DD12983 +AB54CF3C8FE64A0789ADC6232BCDB76E +D12902D79A2645B58D16FAC78AEEC869 +32EA1930897D460DAB8D45411216FDFB +84ECB0A561554E2DB8730F8C5750FA1A +A2FA458A74B24941982386FF64667687 +52B673E309B848A2882749162FA6284A +DEFB7051823643D78D2CC793CD050C97 +75E22B95ACA74867959F8B94BDE45C17 +872D788E135049A38CE04AD62CAE9CC6 +6F5F4AFBC88946DA8E9D0E1D64886B64 +12CA079080574B2C862F1C2B6D496853 +6C5369675F114546BA8744F6E8CC9D8E +66638B9625DD464F97776D601E9FE23B +A8C4D86036EA44A7B0F6F4BB872C82AF +E61B0858CA244384B840B604D1994CE8 +267E226E19ED42ACA02A5B4CC688E1C3 +E6160AC9DD624530BE333FC9D2EF48C4 +7AF13101A2F14F56B2117A692380E8D1 +272BFBE0BCE5451984237DD86D766475 +9EFDC5D94BD94AAA93AD808016EAA73B +3F192DCCA1024DFABB84F1892ED34783 +4F54CAD482DD44C2A594246BF36B4FF0 +5E1CDAC97E8042EAA8516D60C0B5CB66 +A743A8AF968D4F269123065C3A5DA276 +34ED788FD12D42AE89F50AC4DA33CBB3 +8FAF58F5C29F4A41B48C71E4DE047FA1 +57550C4DE37043FCABE121215C74D67B +BBD2D370A43841698EC67D4E8AF67A8F +F93122D04A274D6E82A4F8304B054D7D +02881A7C806644599838EA17C9214BF6 +B20A16CF1C674E4F992531788E4AD431 +11667CC9B8E14D9F832F98FFD4CB9B7B +374AF88B3BC549F29017A30AD4D4CBA1 +5364FC5B8BD84F90BDC5247822B2D98F +9A8F29F4A5354B519C83A5DB52EC2A92 +$C1XUID +81BDAF46B69940198D8E91202EB931AF +5C99146BFED34D889ADA3EDFB10B2CF6 +CCDD531FDC644CF192A653F56D274709 +B2BE03DA47D948449C3174DF4DE7693B +D8B1F784BD4343D0B769655E9D489EEC +07E97D97470749F39B16BF59107C97A3 +36790BCD284C4C31A06465A7A0ECF37B +47DA9A66499F4DB79DEF67A840965097 +7FC4A297389B4375A8B688D6BF0B76F5 +E89023C0D32840FCB657D47F2C0F1BE4 +1AB17288ABEE4359AECF955669DF532A +CACA2DD5CC1A44D881E421A903ACB52E +9CA05DF240FD424F8E622DE00E108941 +D26233DD681F456192C68FA39CE27E21 +FA7E506D7741431B90F01DB80870CA88 +2C6877FB35B04DE18BD03D03DC8818FD +322D826F57514F07B98CE26D72688036 +990A278D7E4D413FB8443CDD778EC2B0 +D9F6E256ECF84E6988C4BAD367B9FF6B +8B6CA16297B04C588ED59B90365F3388 +32344EE2A12B4887B914A2ED11E68A31 +6A5891EE1EA541BBB50ECECE5D6B434B +33EC00B5AAF942DFBA894C9F51F9233F +78998726D091469BBE081FA7E27885ED +955ACE8711D54DACABCAA3018E4752BD +01D06F28E18D4F4C8205A5A8964AFF3A +BDA84C97A9894C14869B5DCAA967B238 +F30F3C4BA86C46BB93867857FFC3D37A +F6FD4CAB0F8C4D5E8ED0D5C21B98FA51 +AF625D5808D34C6C891DD7DCDC26257C +706AD983286347CF8AC03E0AB9650956 +EE2A63AA196B490DA787C326B3EE1FCC +EEC517397D154500810F8608C77F7489 +F5FC8170CDEB40E9AEBCEC48C4C71529 +853865FA5EE1489780A457AEB9144A1B +D8D95DDC4A114AA3B6F410D09595BB71 +77CEC1BD91284DC9AD6F339C58F457A2 +809082E044464F17A2C40C428C732DE1 +B9039564450E48099A0C54A12E663768 +49E42C7CC88F494AB04954184BD08D05 +B2CA7313AF5B446AA2D87DDB74B7F3EF +92D6601CC90C419E8D65198796D88EA3 +8D6CE3736ECA41D488CD716C9FF9FDD9 +FFFC279CA849442AA1012AC4FF6218C9 +478B84AA0CC1421C9FA5AFD37053F447 +4CDD0785099243248C26D3E2F29C6F90 +64EE223350844877B1C2E9BBEDA9A59B +CA2AB38776E343C1BE48EFCF9AD61766 +D9266CD552344CB69E1931696083F51A +51EA32E255A64F8EA822025118AE9C42 +95C05AA1496F48478915462BB43A6199 +6629CCA37A004554B36CB891F2A3DEAF +84137B33B9A0446DA4674FD541C35BC3 +FE16592488D64933AE453288AD37395F +12E884D01F494F48B974E4502A6B2ADF +7BD442FB23804390B31252C8AEB17E1E +75F4B58C49254262920975A77721865D +78A9E3668EBC4155B5D853569F33C7B0 +CA603D9D204342A0AE05638901017087 +D087E06D5A9F4E06BAE2F82C1AC2F6A3 +FA9F4968FA8E4953A6F97811FE3FD5B6 +1E714E3A22E54CA29575326789FD1DBB +9786B01215534DEB9AAC2D5FEE23A497 +CF97A4C7070F4DB28463F17C56CFFE8C +DAC84E61002B49D6BD9C7E8E7990A323 +9247D211715B445887C749081D0259E7 +F8FB05150D574CA3A8159ADE29B8BFA7 +35F47CA308C04819AA94C31548B35761 +36D31B977B324F0E9A87A2B660759809 +280223D72C1A4ABFA44A1DC46240C0D8 +D85E0656E67D42C99260CBE5660F6372 +9631DA92A22043E0B953AC9D49A6C803 +1556D5725F064FAD9FEA2CCDBB46178E +3D9500FCC3824A239782805AF864A4F3 +942F7851B9914E0C9BD803E41A5A2113 +F044E6B403F245E1880A3931D5250218 +930354A257A641A8B7BCF8563C7A844C +A62D7F22D54C4165A99A52E9C547A7F1 +4A31CE0D8E264848B91004E3E28F99B1 +5EE8624391CB4ACE9001123DF2B6217D +997FF26D3FC24030A9EA76446B1473C0 +18CF4AF1CE1B4D7AA035EA4109432063 +24C04D2884ED4EBCAAEEF4A2138F4EFF +7AD61431CDFD45CEADEA4065D4F5156A +B970CA47167E46EB9AEC87427534D0BE +E0B5BAA87EA842EC9F5A991118ACEA39 +D392F08C69E44A1891AE6B570D177485 +B5311E83832D4C4F9041C2182A72DC5D +E67168633FC349A1886A1383FB721158 +A0F6BCDA1DE94AFEA1AB0002D773952F +D964ADBEC5944140B12A483A3DE79256 +FE8513F6E790443FAEA4A42452262E3B +2ADA72ED2BA242E28C838390EC9608F0 +3F2629708BFC4D4B8EBEE87266E9DF6F +11A1A8B2B3684BC38820D94A44EACF7D +2169B464664D4195B3CB5F57440E69F4 +2761811180D6485987790527D9EC452C +51329589A0494A399B130B1BF8502FEB +5894FA110646462097B09708453B7418 +B057739728D646B9AE53F076CBA6A6BE +CC5D271FEF194D7E835FBADCF76587DE +E6EE48C4F58C4C2DA3FD1C1702AB356D +622FAADC08EC4B7480B99562F6406C7B +CFC4F35F5E024D5683F7A7600EAFD085 +0EC230B76D4F41878F95D14B2784BB22 +9F6A40F51E744246B8787D69763597DB +6A455FF3DD884EAC9D6AEBE8E3A818A6 +479BC24A4FBA407AB8826A4F406BE2AB +97CBAB51C3194F3FB9C917F0A20E6BCA +2009248A165241FAA2D65F185E1312F4 +AF3F38B64D274350AFC42710DBD07FD3 +2D6F22655E7A4D7085013687131DD5BC +DE29077AEE5843E482CBC9833A405FA5 +EAA8C294DDFA41CDB144210FC02FB63F +4E28D90BEA3E47F48E71F096BBE0483B +1F4EF005155B4444A2037CC656BB8F95 +7F4E4DE2FFEC493FA5EC822AC4950D4A +555D58DD44BA46559A344BC2B09DEE63 +7D4DEE41A148461F9672403443B33DE9 +D3F10A3223E84CBD973EF181826786A3 +6BC22F79E6734A6B87E66AC188DC050A +DFB32C60EEA047309FF217FBB3C82C07 +9A24DA29E7004C419DD09CFDD7ACF970 +B88315B322184AF98777A60BBE9D3376 +D9447626366546868285DD7084A9C8A9 +822D08243117484989AEA6A9290E2A73 +6823BABBE69D4092BD456EE0DA10072F +672E1FAD47D74B07B6E86A84C102A3FC +02CD7832A1D14A478671ACC22964C8BC +D2995F61F6EA444CA9FAC735EBD128E1 +E9A870E86EFC45DE85EA86C6099D54D6 +466D0EF38712486DAD600C2D28623364 +BD9DC1FDB06243DDA2327A1BC10D915F +A575458EB6CB499CBB1C5CD6676CA225 +DAE57DB2318049B2AA44A00D4F7C5BE1 +7DE931B4459F4460AEA4DB305C4BD739 +997638C631B44974A9FDC4E6E18AF865 +3E28E8B0FB7046788AF65D8544AE82E7 +80AF1AAB04F34EE2A035D5091AF03C82 +C85F83AFB3744E95AA4AECBEDC9B1BA9 +F35AB5C9266246B396404877B9EE8BE2 +354340C3F26B426B97D1D8B922B7BA6E +7A8DBF93402F45D7B48F7A25E0511366 +CE440A6E8AF94594925CF09D08C889FA +3FF573A6E21F40B0AFE592DEF7A7FF6D +A3116404F87F41CC88AB205C3B9482DE +B1C6BE8BED4C438691A72B1CBB8B52B7 +9D70D006FB604597905767D4533E66E2 +A2DE2A03D00D423A8757454443D99DB9 +F38D50D677164C53B224BF5533AFECAE +BC460173D1DD4B92ACA3F36F1CEBEDF1 +F4252457604244F59970FF09E6BD7E24 +B98E7B11432C46C68221823D04EB9A75 +0E5442023C15497DB7B32606FE272CA2 +1A9115AA921F4E19985A4E2697B4BE84 +1C098AD2CD6946B9A905546C7ECA8D9B +59A27CFCA0844FF2808F411A5BE4CB43 +A0CD1C2D8D354015A3E243F795A04013 +9D67FF1100DC459F8C1D09285E282553 +17C5F6F82D9142D480CAC034942A1535 +2D09E6C3D2A04DBBB6B966E87CED76F5 +EA29BF048DD5427AB27D16FAC3150B7A +3A85B67F035445E2BC3E1B1963925C6C +5BC64300C8C24502809021940F146602 +73C871FB212B4F2698B1F07244F669BC +91DE8FB4567B410C81E2CC5336503BCC +FE2E00860D7243749061DB63060DAA1A +11E7BC505F24467CA9A72E873BB22F4D +A952845778BD4FB1A4A9D39536B4B03F +5F35D658218A437A88B22AFD55FA1932 +B5D7BEA1BCF84B439ECAE42C602ECC95 +527A3B9EE28C4054BF6EBB9E885FA044 +A55D2DB2D52D4700A2F7336CF1519578 +1F698ECD0D504A45BA28EE130A19170E +40A2007E841A40E9A3690341C51504AF +C885E11504084CFAB60E811A0D4152B8 +E18C00B39EC541C28B10436A0141A3C0 +032149DFDDC14C7A963EA40265E6AFA2 +217D2ED421104C53AD77990BE5A78B22 +4BCF1344AECC4E67B59A88CCE7C137AE +63D07F91ADC949F0860F87C48A4AD557 +31AAA33BAD1A4C10B56E279CB2C0147B +4A927410D29A402C801B859DE98AE98F +66DF0284B4F64A51BC3579A4E25ACEC9 +E319B529F0014F5790E21C81C9C0D0F9 +C207D8F9D84C4CDBB4A2A51C0BCA167F +58414D142E284160BBEF8D8CEBC5E0A2 +67F9FAA217A54A2B8AAB0186D529666F +A0B6E3A74E3B40879AB43AD613332C17 +4D2D6FC024A94AAEB7EE8D609977F30F +E270722379D7436EBC88B33B9172C41D +DA827D8914D24EFEBFB446E7257F1D1F +0FC96D3567304FF5892D31F8A2B1B42A +AE7BB359FC2F4CAE8677709BA489B78A +94AB6223FDDA49C3929B3A09F9EDC74B +F8590202B9FF43A4B9C013602C270702 +519BDC3C0F3A4D369F43F86737F3C550 +A4BC42688674400784DA9BD4B1E528D7 +9EEA781952754F4A9C6D4BF5F6B69165 +DDF87E447095424FAA884056CC19689A +697B7F6DE7CF4146899A0FCFA3534836 +96C86362A61A4101B4012D0A24E9D3FC +C9EE72367B854551970B797D480C970C +B62AAE2023F84AE7A013533A2CF5EBB9 +158618F8BB6A43EB8C070274E0E32270 +25D3C5EAD5D64CA29A24E445740CE12B +006E7CB502D64BBE92EF9DB273BE0132 +C5F43B00BD134F96AA494FAC8A230CB2 +DFDD9B077C9A41F7814C289F40244384 +0A843618E3D74724AC88487BECBA809E +70F666C9FE7848E1A9DA0A4907B3B505 +941F0AE66FB742939733A29E26486373 +B40D58DD0FD345E28454DBF89F7F0C22 +3B2931A2380A4B8DB36CFA3B1C7FCA54 +91D312B59EC74B4BAE6E2160751AE0F8 +A1FB59CA083044BABEDE767508215C23 +55F5D9A2DC28465081A717622373DFD5 +C46F2FCB48534D4D8FAE5BB428EDF3F6 +37D4DD351CC24E179E46BEA451053818 +51457A4556BD4E319AE22A0EABE0EBD9 +8744231BBE84441888194CF510F93A50 +0172C25317F6420CAF304C0C7FA8EE0A +9CBF7FDE2EA6435A8110590D7EB2BFD9 +FE6E2476F3804E0FAEAB8BD017399F37 +46E61691C004436E9D2C30EFEBFC659B +EA82900B45324177AD3FE983AC5A3531 +4490C876694943B3A33E679420ED1E4C +88168F2D42FA4E999FD26B69836255DB +3893C462B872461C9A637B45C6D1D1EF +E17D548871324CCB83016A25A7C73417 +8DD37E33706643A3AF5A6A0263BCAB85 +EA8AD61250E044849637C1A6366EEFAB +3C1A45D3ADAE426B94F1987F83FA4DD4 +B7E8099E423C4E669FD947C581969A14 +B180AFC265E64A9A918E8D2289E17BE8 +D90211ADD7DE47FC8356D4D0A966F0EE +DF08AD5CC5BE4725A04BBC439CF18B0F +1F6FC80A9BD643C7B562C913CA4CA49C +94ECB6C509E3469FA0EDBE93D4376AB7 +C9BBFC44F6324E56A00FE5E5CFD75E0B +D5F8257009F84268BED1125ED453F421 +4381BEE727DB426FA9953C54F0E12B7E +99007B05B81B40F087496E198E6BAF76 +A6D3A993763A4FD791909029A95C7C98 +5700AFDC479B4252A2B98C3B11F08440 +55842040C0A74EEA8C0849C670D7B2CC +ED49081D2B114DFAB4BDF87FD60F914B +B7F0554A44174C9D8AFB2FAEF7CCF3FF +E6FCBC267780497BAA0BBBCB92ACEE22 +91E4A742234648AC8D44A84DD4480962 +A99972814B3746FCAEB32D96F6902B2D +4EDF2337C9B54FECBC852B0E301AEFD1 +8D1A503BBDA64649BFDD4FC88F466FB6 +A4F2C0C6B2E34B7CB1DB97C0A0E3E3C1 +16C8146B8F4A46FBB2A755B3330ECD2B +5A735360E99C40D3BD9D61E920923BFA +75030A0CF2EA46B7AB1462BBF80FFB01 +5B28BDB7E0D84626897476178C2BB062 +5FB642BBE7194BC0B67584AB8353AEC5 +9143A09DF2084098BA784F11AE1818B5 +E89380D4E4584B5AB115E3D0996B06D1 +6BF296A238014DD8BC8D273522E48F1E +2BCF4D527254441BBE47701046B8A2C0 +9E88C5EE50A4462E9EAD7738F08F2DD7 +51FB087AE37C43EF8C7C914889D221C4 +DB4606DC6EBA416181E184826F00B891 +377974DCC2A54317B3F3BA57761047FD +1E92B5DCC80E45CABC935853091C1181 +4163EB7777CB4BBA8CA5311C33D1BFC3 +51FDAE6D77C44F18AD5BA4F959555AF3 +62DB79A1FA004E4FB753D4116656F73F +363276A36D264ADCACC00D9747184BE5 +4D16566C4F9342C1A6038A21BCEA5775 +7869162E62014DAF988D0D9DBA7995D7 +80119704B7AF43CD8DB7B11447364DF7 +B249B1BA6F2E4E5C8572F8030ECEADAC +E07B9607755242D782A68E5D7D0EAEDE +FFFE7B254D594E16AC6E4181BC04336E +F6EBF82A000F41AD8F402A59803E62E7 +399F46F952334487B8A28ACCBBC9B838 +7CB082B554F54EB683070331D788FD9E +2D140447429340ADA267B9950636A4ED +87077D28A31440028F4437CEDC14DE57 +057B46E153CA4C2DB7D3AF9E7FB92450 +FDADF2765F074208B0383EC8253953CB +6FF98DA9215D4D9FB6030EB4B5681206 +3FBC630D2B844D89967F733F03B58B0E +67D9C73375134505B9C072CFAF1ED4D9 +E3C66C179B664E74ABE0F62E2C45CE12 +7DDE4BAA4961497FB85874366C93BE79 +0A43A328F49F4288B07F46EDEB8F843D +1FB48A8492E446F897158DFF765A1615 +BA31CB4DA94A4B238FFFC7489C721F97 +9105EE22FE8C4E1AB13BEF994CB02E3C +4FC21A4E929246C68ACF5B6D8637C05A +B440978817934369AB89F76DC96DE2EC +9D3F7BC112404E849A056CA9D0A3CEC6 +7CC9CDE2E3914216B6B0C5050099AA01 +966A2DD98B8945EB97A145DA01109F98 +B5C05A607352438BBD78AE692C604B55 +5662095AA00B4FC3B6799F24E6A025A5 +12FF4AC48565445CBB62ADA56B8FCE73 +325E801295B74F628F27D8CF6F38FB88 +5A2F9E9839414215957EE7C96AFED912 +0FA970744EB94BE5A5FC246536AD0E83 +B2AAD3B703804E399D06F8B5CAA8133D +BDD8BCF1B00346AB8FF9B10035DCE660 +DD0955108E1E4D5685A85B6C71D8FFEB +DED74B2326C24186AEBD92B012F2DA2F +79DC61E84BAB4FAEBF5055C93CA2902A +CE4DC71D4CEC4F1A80C782254EF4F609 +B2943723A63341BFB5709159C7C1D4D5 +01CDD30C270A4E158795E0B295CE541B +B79255D458B94A248F31AA6209357878 +C7118F45149E451590007E73194B3F69 +203C6F50CA5D40EA80E74E257D93D8AA +3D99B9985B5641B199C07B631C296687 +D83F71569E1B4254BA296FDD0DB59D70 +9F5C0B9518214FE7853D16E08C01A761 +2A0F89C981894FF6A2CD3A31247350E3 +52D61324C769473389B07055FD997838 +3E8E9D374CAD4809B9D3EF7D33054324 +7F0BDF85D7F64F068113C563FB6ECD9A +BD84D2BBD6544539B9FB3878C2F5B9E7 +689322F9AC7B483C9B735A54D9F0553F +7F8F8F13A2074925B241890FABBA0762 +B21617C2AC0A4FBEB494C371B9FDEF58 +211F777958584F58B3138C9AAB310663 +FF34CE28C0A24C32A7342B3D9D3911D7 +0CD3775E0C9E4A15A1669556A8DCCEC5 +3752F6EBACB44863A40246BF883B50EA +75034BDAC26D44A59DF1E2397785FC25 +15E8BE139AF04D94B0EFE5E49AC5C9BE +AC09BADF9BD147EFA40EC72FDD3DF167 +7F6CC9C24C934986A2B5AB8BA6EC4AA5 +F43D5662B0CD4415B893812C38D6C531 +24B9DDA40F9D45858E2D9B1450B1628E +09812CF21BA9486CA21B8F24A4C90E20 +1751ADEA5B814FD18D309CBDF7E0F196 +D2BA7402D12742F487EB4806696DB450 +ECC8BCE37ADE46BFBF72E947458B5C95 +C60B1BFA66EA4FED981DDC6B6531B8AD +D3ADE02FD4614D59898610DB783B9E76 +08A90C61C01240C08E12F208E2314EC8 +46A76AAD77D146CE9A6B98925E6D3EFA +43EB6DEED3FB4DD8AE630F755E8C637D +FCC539ECB8FC4529ACB426CE60A7CCD8 +4C513C8CDED54F1094BBC5D31D45BF53 +F39AA91E43694B8392FE60418990C121 +12003DB59EBE4686BAAF63CD957AC793 +D17D25E6CD854C16AEF0D749C2F7A9BC +15944D4877E848FFA98B6CD206D3E988 +5BD1228F98C44A6EB2B024F57482A6BA +1C890FEC6E55440A8E842256607B16CA +37FAD167E3FC41D88F8E22E8DB52437A +6197A72FD7894623BFF0606BB576C267 +85A21D1E99764774ACF511A03D4C901B +489511DBA17D444999DF970ADAC10049 +15760FF3AB224305ACB4BBBF6C024DB7 +8A44890C038A4414861B07444FCAB2FE +C6E96E72A0DB402687F2CFEB92395F93 +3D9F6F4B140B47058926EB3FE9B5B219 +EB11582143DF4DC1B5C7A9DEFF594612 +FCA986ABA5BC43E6B826BDC674E92F6B +5E0BA5BF05B34D15AEE2CEF53D90052D +5778A5444E5244BFB71790BFC90912A4 +3F4123085A974013B479754D45E81D18 +96AD730E8076406384C46DF87D3FD872 +044E15B16452489A8755E4E5233A94D1 +73A435FD647847AFBB1F3B12CA067B90 +AD2A0A73153B4BA4BF7501BDCC086B6B +E79B2A9BEA884537A350C5A3A2905B04 +11A8CF31CC5E4E38A3B845A8E6BC4151 +21897DD54247470FBE817BE320EE1E07 +C0BE371345B84BB0B8748F59A3BBFA16 +F5357E0DDD1C492890586D204EAFBF1A +A8FDC32D4C554EFB89EDB7A59818DDDD +6EF757C3EBB948519F160B30F3B60BFF +C0DA486BE19F4AB796D0876258AE6B4B +E4E1ED47D8EA497EA284699E459CBE12 +203DB049741541B4A35C0885F0159354 +7105241AB64E48198708D7F30ABF8683 +CBFF03D1E78D4E35A304827A3D4EE8C9 +EA4103901F4546E88675582D2871C5EC +30D991129B6A45058C6CEE1490736513 +88503640FDAF4B17B46CA94AEEFA9128 +79B534050115480182F268B0ED064F55 +F0D76144672C42B79E7C20EAC5CE818C +45D0D43753DC4CB49F55799F93A9575B +A38979A144E84BC99A2DF7D96517C300 +51A8F8AFAF0140519C351E6ED53A812E +C757941D43504E35A6AA843DBD691DF7 +362BF9873C634B248CD0CC6A2B072CB2 +E2D9EDF8C1E3407690CEAD0015180802 +2D4F84353DF14368B0EE0E360B9F1D6C +8FBF287512704C3797AF250EDC7E00E5 +D477F3B107664AFFA0954B328116E821 +4CD60D7FD5AE48C3BA6FFBBD967A17AF +6DF6566AF4B0488CA1243C98D6CB9EDC +7824D7B4A5CA4B78ABBEB8AE2EE68256 +97BC533801C84E5EB39E7856353EDA84 +D4B92FB162CA4E78AE6E018EC8B7F5F4 +A074ADEBEAB1404FB5C4304978E814BC +D6412E872CF145ECA4C22275075B9071 +911FB29353BB4AF7A50D4284AC93A68D +257C6611858B4FEC89C458B5EA422B93 +370656C5C1AA4F49A63DF1E8B5922406 +060D57C61BF94DDAA197C908F238A8D2 +5722AC2189734D8CA87C30A8BC7C84E7 +E9AE4C0FF2FF41FCB3CBDF3F8B975D4D +32E80E60F8534B728499F2ED0EAB217B +9A430142B93C4C74A026B1882283540A +15F7F77D9CD943BDACCFF32F3668E4E3 +BDAD39CC4BA048878F80F9777607BFFC +9ABF8F5EC94B4EAFBCEAC3D905CFA88E +6049796687754516AB4E55212A08850A +FACCBC2B2B5E46D1AE1F5A4A969F130C +901C145BFC7D43F693BA4876AF2AE2FA +AF9694DBB22341429A676583846BB77C +D77CA8143D744F62B220EF06D14746F2 +FEE5764AD7684962855C53B63E125BEA +401A712D0D374400A8C21FFEDEC7E15D +77B106B4049E4338BB8E9232A4B315A1 +6EB9F0337F344C509EB28814C7933B31 +4988F7316C5140388B9FFA0F6B242C91 +26736907436F42F6804D9E591C14EB14 +7204CC82D3184D48B1C7397158E92D4B +C85C7EB34B234F19B63C482726F72234 +5868CBEDD9494575A96F4639B433DE7C +A7E8FBA6AC654CFA9A11950A4D104914 +43D4DE527E7740E6B3E1EBDF9EF7AE69 +578AC598A6384F3CB82C50EE1F1382A3 +D0C5333F7CB645F1BCC03A15524E8D16 +F2FBF023F4AD4D4B966A373F0CF74CDC +3E8B285B3A8D4854B0D7D30EBBD8C7A5 +C801B4CFCAED4BED9950145A5DAAEF87 +0B4A20E835C14638922A408CE1FB0EBD +AAD2FC5B4C224C4D943615F986830D8C +B73F449CE6664BCFAE5391535C2C66CB +A44CED7C3FAC4FF2A93941CB9919FA47 +55A43194665141E9B8FC0470F49D058E +E12760F54451480D94D3F28E6AA78576 +EC78F0A96B37416486901468EED3F6ED +35DC5A270299486BACAD896A0771EFB3 +115603BD640B4952B8F89E63309D3746 +3D6073BBCF454B0BA1F90C45C0E8F9F2 +96D07C9588BA4086B11F2E6605C93DF9 +05BFA1F059FB450494D6D3622FBF6CDA +BA0D899F17DF4BB7A19BF9B463E26A66 +D897C4EA40DB42A785B8C6DE95542B13 +9A5D686BB22A4D87919356E1D3F8444F +B22047CD2D5B41548164892D1D74DEF5 +648B3A160767456DB3545578873AA472 +BA62F60C490C494186052714AD2259A1 +DC7DD510EA264CD4904816D0F36C6EAB +E1A023F4F3814F4ABE672CFE5FE85B1E +29FAC95B462944109CD73B89BA12A9EC +A5164ED1A5EC4A83989EAF05972271C3 +1A1C01B32C0C4CF3B66B14D713D1CD4C +320CB77FFAFC4373B67683DCE2CECA84 +56B8BB734B7B48E4B6123EF078F131F7 +D540F92FBE224D4AAF1E92729CFDD038 +2F853E9A4877461BB6DDFD83614F2783 +DD5507A6D7574D85A11AFA546D673234 +6FAA465FF8074D97BCD6CEEF8CBC6A2B +A91A1DEF56DE4BDB9C796ADD929E2BDC +D627149E889C400EB936F6E0BE917C2A +F526BA622088473281FA2FB47ED622F5 +9CAE76B318694B549469FE17D88A9A8D +7BCC91EFCD634588830AB81474A3DDE2 +9C7FA8A9A98E483D8F9D2AA9939E8F1A +742C4612226A4545925F5A762E558FEB +A347D76E1BD74FD1916E9599D63CC64B +B9D0E6EF602D4A58A39CCF4F5170C2B2 +2BE6C9623875499EA3E08BB64707175E +37749144603641808232CDE524D64616 +ECCFA0D461ED4A7DB75B2076283167AF +FE74A70C9BF24C4D9E3B65BB518DC865 +1F9288401375457AB4113EAA6722A7B5 +6C883372DE7245CEBDCF0E4D8D508937 +3DD6A69FAB5649CA94D97631EFD66206 +F2D26BAD82854B07A78D7EE3AD27631D +87083D3082B447B7932A682127F6C54F +EF12159799984400AA32F27D01268E57 +69D6B72C5AF541FC9C2EE7AB7E604F06 +35ACEEDDA5484B0B9F8CA64C781FDDDD +98D4EFFD552643CCAE4DC7E48D97FC74 +A31698B869284674BF3718CD8F5F3DC7 +99C9FABE278F4D8B9DA892B2F04B988E +75F35BC3482C4067B06A53EF2F13C4C1 +BFE3E5A1F9AB43C2B269677D2451EA51 +F27EF325C30D4F94851701E3943025F6 +9E7A512871DC47D3B0A885D95FA68018 +E91C40FB931A48DA9A07B171B1B22BAD +809EE1191423477BA1CFC856ADE0AD8F +84450CCE2008432B8F3ACB08A4E6E595 +4E61508ECFBB4FD7B003FCAA5DFC81EE +AEDA3B476B1C4ABD851799726CA6CBED +F1D1C0732F5A45648A08EEDA613A8A93 +2D48299EEDB24E05A46D801A6E79D0D0 +7FB510E7133B4A20A85717194313D753 +091AD0876FEA4F01827E8040853C09F6 +7CAD012790CE47AE8B689CFE62F05433 +40D148370E9D4535944A4D687D16D1C8 +AC367D089D054CA8B357732478A08385 +E654371CB7414D3F846A455893332988 +9446721C6526463D865F49A11FD83E3C +D56B696784A94413A4B07556F00DC3AA +0C3B883793F84CF8BDC5931DE09D26A2 +3F648A03579242DC9E3497F69FF11CAC +BEF66FC8D6B0435A9FF66DD6A4BDFC08 +3BE03E0C02FA4A7BB0C01065C062BE50 +C50D4ED469414E9DBEDEDC61979C8404 +553E861503994532A4E8E30F94AD4D34 +5DE0CCB75DA14479BB48F784AD82F17C +151C5441385F43A394289BFFE7A38EF2 +44B21B2D22D34D7BBD307422F2159B31 +768A0500055540138F72F6D5923C213B +E28C08674AA042AC9B82642067462F06 +81D57D9B616B47DB891E43CDEC995DA2 +2D08EA85933547F89052751A0211ECDF +7863EBBEDA1340439ACFABEF19420EB0 +03168E41405F4996826BDE50A338984E +A2E7BF86338C4462B0308BDE9362AEA9 +6358EAFE013142AA9D768D38DA56B582 +834919757208443698CF8B080F159896 +A96517EE47B74DF4BA45ED3D8CC17195 +7192BC3A56564FBDB4103E8AA973846C +2963F8CB89CE4235AC3EA741B20A86D3 +32A1F87E39A34F5090CCCF5D61592D47 +ACF02F6E7E514BCFA32DB532386CBEFD +BEA2CF8206FC4180B7AF420974D4646F +B4AF53AA6D3C42149BC8E5E55608D9EC +4391B76BC3ED4C17AC261C1421524CBB +3DEECE574FD34A4A8CFB92014463B64A +AD046A4D09AA4A57AF31EE7D33238777 +0B95A3BAA31E49B895652F64A41DDD60 +9A02C398E74C47E98D0DF4A5706720EC +9E404818D34E4F0AAD463B45B30D4169 +029844ACA4EF42C4A212AC8F8C325935 +1AE42B48FC094A48891658AA482E72A1 +CE793045BCAB46DCA7B8A42D298590CF +F99A2DBFDF004D34B0462E0EF786B603 +9F2DE0E62E23498FBAE3A6EFFF0175FF +3FA2809B90584D5AB694BB9D1168543E +C6EF272202584AF880A71FBF55768706 +F51CBDAFF6A9424DBD1E59BD56723B2F +66147BA39D344A6FB859D91BF5192C1F +829B5196D70C4C62875FD74C40339DC0 +26C3894F8118452BB28FC1822FD2F8CF +612A44EA9AF94A77B05AB4C12E3D2AF9 +7CC616170D334697927F35A558851238 +7F44E2517A754C6F8AB7C5921EC24035 +47608AA6FD054CA592CAA36985BA8547 +F9541B0B0EB94795A20F797E65B72BDB +25E634BA094F4F408906E1EFB81582A6 +D225B17DD0C14DB1A83633AF990297B4 +67D335C593B644DE879F9C524E0E8E36 +E78DF1725ED2404B9E339EEE9F955B13 +FFACEA276AA546048A6F308728368EBB +261A136975274846BE791A60519449BF +D1F59FAD20294EB69E594C35077F9152 +EAAF0C2E7C6045C3A2C20E0C6B910011 +CB2CDFACEBF841A58B5B5D2848A0DFA7 +98DB73B7FD284CD2832347DE8FBB2033 +EC5A11494013446291BC614FF138BA15 +03E7173ED58C4807B533FE7219A997DA +B37D290718DB4BBAAF0B9FBE82CAD2CE +EF1ADCAC0B97464BA3D3125C37C7454B +0F8012796EA94478B6B275DD15EC0CB0 +22EC5D0FA5274A688B90C3BA7E0401A8 +4F899FDF667640B3956477DE74B99997 +3045BAE970C3457E9B0C526A5F2EF554 +A0FADA3614724C29944C48C44F866E47 +DE0A1C93AD8049C395E716A52A3F37F4 +4947BD9D258944BE85E96559BDCC6012 +E28C811054EE4A68BB089B35E6F5EEAD +1D8FAE231F4D4C1F936A7D84BA53243D +3B81CEC92A6C4767B72DA33243C10728 +199CA016CE83403DB1CE83007BD94304 +5C7066AE9D284FD4B28609B09557A2FE +81DCA231C0984600883590EA1DCCEC7F +A4B91232D2CB4A26AC9B34123B060323 +1738889144BC4FEE81E43412101DA966 +25B42838ADDA423C836FF80A4DFD997A +7086E9D380304E128073D662544E37C3 +862E764B08454B4D86E3AE397DC2D672 +C1D84035380F4079895D46762BF3F2F8 +87815ABC6C804EB0A264E23CE3664F25 +A39AD9D65E654BDC874C276285EC1790 +FC55ECD4285A45D8A12A44BB9F7746C7 +022D9DF63AA94182B775A78EB37939F8 +3953D9D35CB94BE9B130A1353255A15F +F0423688D55549548F82C99B079F9315 +7F168F9DF86741F3839C5BA8401B6B21 +92BFD3BDCBA542F1A5C0F4DA2EAAC680 +5F846D5F721943F39888E243E3EAD5E5 +349023242A07487DA625A6A17E6F66DF +090A16C83D2D4627B326CEC54D3CA8BE +31AB00ED6D214516BB0CC7EC2D2F3DD2 +CDB4D9B5120B49CEB417563830FC6B02 +D605C892734044EA86D5B8431FBAD939 +3BD5A50869114A77A59D8F58C185B571 +BD65590DFE1045F3BF6A14113FBB21E6 +6B517C314AB342E7B1618EC023A28F9F +D2E93C8ED0A5477CB72776056E86D680 +F9CACF42C83245D78755CD7C7AE1EEC1 +429F3D67AE854E8F9FBB9974E2ED7A1B +5879289D7FBE4AAEADFD6B305FF107CD +FFABDEB75D6F4E999979E4555149816C +12EE2329ADF345D9BC33AB5974C35084 +814EB12187AC49EB85B3BECE8BA9C504 +82C4DD5A2EC14396927B931CCE3057BB +2B620E7D9E7043CFA8D5BABD1F2BDD2A +B65F226D922B49EE9E1BA3219DEE392C +76720EDA845E4219898ACDAC4E3F740A +3EB89FA0522240EFBAE791AF29787E37 +67DB162F3B244B5581F5D6DE200D0509 +1C84986A23524D659683B061421EF221 +91C2584DF35D4D7482DFE349CAEDB82E +C23F767DAEE04B0E86C0575151AB979A +639FAB1ACFED4F5CA3257CFBA17A8090 +6F7963F99DD04541883E8DF6B5AB05A6 +78B62739452445ABBC41DFBCA857DDDE +B375D270C532438FA0299F92648F6C92 +F8A80CF6E3F343989F7B6A3FD0257B5A +D7429D5AFA844B74A52C6BED738BF193 +B5EF68A538084113B84CD2BAC65E98AF +5682A3FEC1244CADAF47DF1425A0D251 +668D9A8BD8B84AFDA7974987CE91A8F6 +2694AF9642D4444DB19BEB6D076ED428 +C43CEDFA944846899999C01794FF349B +77CEEFA2C0214ED1B27A2250A8608809 +0029F61461A54AF79418BF9A65452287 +F751109FD4314D489131722D8772C094 +02F9E56D3B9D4A069CA714F00829EDEC +73CF76B307CF43B99F3E5CBFECE8C884 +7AAF0925DF124164BAABA9C6BAD481F6 +1CCF8C683B964E1EB6E48EBE4EE98C74 +4AEFB350A4FE4C8899B687E379DA823F +E604EC91431441A68F75506E2CD20B9D +7ADEBFE735274546AC12005A8E4FA611 +8856D48343E541CA80F2FAB70F7A8378 +2913E884114749F4A55C14DC516C526C +E10DBCEE43864590A63464589DA53274 +E91CAE410BC04385B1ADAF2A697DC024 +C0CD84557B084195AF9BBAE19F62EBE8 +6AAACE3FB9074729B7AEDA54B63C8AE5 +98966BE94C34481385640BD08B7EED44 +C3E6DD19273849329DD2E890ECC58D94 +B06C6F86A0FB47E89198D36702996675 +BF3BD7965CDC49C08AA53DA13652AA9D +4F70A9DE437549869668F3C2DE4FDA87 +DB92B9801AB646A9AE19DEB1C88A4BB6 +4165F617BD4348B985990CCDDFA0AB5C +B26BF6FA710F4349BDEC204B47FB1BE6 +D689121E4AD64BA69A585C5E1F91BC7C +DB392AF0850641A7ADB76FA67AB40637 +1017CCB8FB8D42DA8F08364227BD794E +4946AD7CF2B5420A96E3A07B14AC0049 +77E553EFB4E148E498B1EA4AEF99D657 +5011D115C2B5408D8E4D5629D3EB9F71 +F11426D0534A4830BE3C8D4F4290FBD4 +4C944FF6B1124E9892D76E8967248618 +61315935BC344689BBD9D267DEBE9A9B +6A01FCD050BB4F348785CC82A28D750D +656A2B5A256F46E09F290AB4551E0E9E +7B094A902BFF498CA2D6B2B053792232 +7DFD051DBCA443929994F2B17A360A3E +4B50DFF8A99C4CB28F63AA31C42200D2 +1234462435804F9AA86653C9CC50B2B7 +29857BE77A67414A85D4F306706A877F +CE18D3A8815F44E29AA00B15CFDDF20C +F194296A247549CFA5DE61783BA5E770 +D4049EA4E58D46F6B4F9588BDBAA7883 +CAE5E3232D1E4AA4A3B34B22899E3F13 +E886E72F2FEA4094A6616A65F568F32C +24B2F3B5EFDD4CE58E7C91B200515535 +92EB0798C7C94B89947DB2C5A6B178BA +AD2EA4F62ADA4E4AA1AD7F8160846E91 +6014F57318E94E2F8A6A6DF0467007BA +473B19F1ADEF47A285A290BC4F24C6E0 +61CDF6B343BF44B981F1ABE01AC2D3DE +31D4D6961BC44356A9B334B3197E010D +5A2B93C11326425BA6D77DC4545430F1 +15140A442E604344B6A4E525864C9122 +472F558070724284A7716F848D59CB2A +929ED22134244C74829081C4C3CF1FEC +92D360516BE7488C8D73D74A7EBC6568 +D902E79AEB9C444B90485EB2B2490724 +7FEA3B375E2C467BB6C82FC746FC0B2C +533C3232F5624F3BA0DB6B3CFD710998 +83E263C202144F64AFAF3BA5A55EB92A +7BB14935ED0B4CE98C1F6146D16BDDD5 +AB88B05A18674D45BC9CC286DB4E4696 +57A32563494F4E898AE60262660D8D51 +986F94C0C7B44089AE63ACE298AD6B89 +37C821F533A84121BCFDD53A7EA1AF5B +A2AECED2EECA4A439425FF05DC33E8D3 +E9062372175343259EDB3EF3E60D294C +CB567432597C45D8803B944949D3CE17 +899AC55A0D3C41E9825B2F1CCE4A37A7 +C00CE1D52E8B4D82807CE4B2CD9F171B +6C0A0A0664924119B4D3799FE6508146 +83732EEDBD0041BBAA6C88FC252C71CF +E73E0D1D91F8432D95BF54DED3943978 +86FD8CCFB99E48DEBB9A23B150CA2FEF +AC17425723534FFFB60FF98795C36D1C +EFC69B73AFE748E5B6C9A387FE444CDF +E314F9F3274C48899C8BD76F79A0D2BB +D80F3CD9E96D4B519C81AECF55DDFAC1 +D19109E8C3F440FDBBDF71DEFE6C0946 +0289C766C7E54F25BE7D4455D611FE4C +BE85BDE248AF4A7AAFF9C660CC23BBF5 +E260BA4E91DA42E6AE270D32301A9C27 +23F3C1C94E8B44EF8CA6703363E9FA19 +A41AF3281CD3494984D107FBEE320F08 +F49CA66ED0174BCEB9F8CC33ED87B6C2 +09665147D3C14B13A54A0159D6892D76 +7DE107863677402E97DD97375653F865 +C3E2240529E94093BDBE81F0FF217092 +9F245D77073F40B1AD25EDED02C8CA82 +250B1502590C42309DE5DE84C6FB0798 +5B4073DEB12A445E9BFD7BFE8E1A1785 +0E04F858859446B6AD4439B7ED1A2169 +01D1D6B4AA8C4834BBE1712201D03561 +2340674A95594894BC8B6C5513896D07 +CF1F44EBC3804F1FB2BE60A28259A6EF +3CAE03EEFB0643458DB4B6FC67588D4B +7F48749BBE9F47F2A675BE31C68606D8 +B0AD4ACDFA3843F094429D88FF166DD2 +E6C94DCF367B4583B078F0F31FD1E2EB +D9D3068F40164B75913AD48888F62E6F +6554E969E43B487BB8BA58A0A9AD974B +632C95E213844F48A2A4ED2C038154E0 +FB03A9C9CCBA4F9DA19F78A6AA18A56D +C636A5F9704048CD94CE35F6381BAF57 +418A69BAB82A4861B3FC6B62988536DB +259096E559164142A7D56925B0D186A2 +727F8063BC1C4A8880E76BCB95D0766A +2A09469BDA7C4F40B48C7C7BF6059564 +6CC5F23F68894F5D82BF14E31E5063B4 +20F2135771DC4F2E991DFCC6C8F234C8 +530769B435674A1487FE80D1D17C548E +DF0C4A8626DF46FF9A10B149865304C1 +4607509807344287B81D6DBE3D0DFEF0 +ED00F63A800C47DBB59B7FC47A06516D +539EB6BD705443689BDDA7E8787581B5 +AC0E65F8900D4B41B973B3DCB210042F +1DC5190F7FF248DEBEEC32F202EC32DE +D8047B21B1CA42BEAC51CD15C6DAEAAD +C3CB9745415C47948B7328AF4690759D +3C516C6058614E5997330975A83BACA8 +C031F32EAA01416C832F6E0ABFC91637 +1087A5A69403492D91E7575B4CEE3166 +A62F4CC94C544AEB8AD2A8C7F9156507 +BCAC0F81EA2C48D8B59ABB8DDE37438B +BF75C686AA8648F096B02E80201CEA87 +E594A6055123415CA7E6CA9FAAE3F187 +39EEA379FDB9477C8AFC5316AA262FE9 +CD1793217D954742A33C2CE0C21B1BE2 +871B5FDDE36C45969CEEEE93DD40FB97 diff --git a/libs/cuckoo4j.jar b/libs/cuckoo4j.jar new file mode 100644 index 00000000..51050dbe Binary files /dev/null and b/libs/cuckoo4j.jar differ diff --git a/pom.xml b/pom.xml index bcf8c0d9..4a26319f 100644 --- a/pom.xml +++ b/pom.xml @@ -147,6 +147,20 @@ 2.9.0 + + + com.amazonaws + aws-java-sdk + 1.11.119 + + + + test + Cuckoo + 1.0 + system + ${project.basedir}/libs/cuckoo4j.jar + diff --git a/src/com/xrtb/bidder/Controller.java b/src/com/xrtb/bidder/Controller.java index 786a317b..c0d3c4f1 100644 --- a/src/com/xrtb/bidder/Controller.java +++ b/src/com/xrtb/bidder/Controller.java @@ -13,6 +13,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.xrtb.blocks.AwsCommander; import com.xrtb.commands.BasicCommand; import com.xrtb.commands.ClickLog; import com.xrtb.commands.ConvertLog; @@ -91,6 +92,8 @@ public enum Controller { public static final int SET_PRICE = 13; // Add a list of campaigns public static final int ADD_CAMPAIGNS_LIST = 14; + // Add an aws object + public static final int CONFIGURE_AWS_OBJECT = 15; /** The REDIS channel for sending commands to the bidders */ public static final String COMMANDS = "commands"; @@ -199,6 +202,27 @@ public void addCampaign(Campaign c) throws Exception { Configuration.getInstance().deleteCampaign(c.owner, c.adId); Configuration.getInstance().addCampaign(c); } + + public void configureAwsObject(BasicCommand c) { + System.out.println("ADDING AWS OBJECT " + c.owner + "/" + c.target); + BasicCommand m = new BasicCommand(); + m.owner = c.owner; + m.to = c.from; + m.from = Configuration.instanceName; + m.id = c.id; + m.type = c.type; + AwsCommander aws = new AwsCommander(c.target); + if (aws.errored()) { + m.status = "Error"; + m.msg = "AWS Object load failed: " + aws.getMessage(); + responseQueue.add(m); + } else { + m.msg = "AWS Object " + c.target + " loaded ok"; + m.name = "AWS Object Response"; + sendLog(1, "ConfigureAws results:", m.msg); + responseQueue.add(m); + } + } /** * Add a campaign from REDIS @@ -1181,6 +1205,21 @@ public void onMessage(String arg0, BasicCommand item) { thread.start(); break; + + case Controller.CONFIGURE_AWS_OBJECT: + + task = () -> { + try { + Controller.getInstance().configureAwsObject(item); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + }; + thread = new Thread(task); + thread.start(); + + break; case Controller.ADD_CAMPAIGNS_LIST: diff --git a/src/com/xrtb/bidder/RTBServer.java b/src/com/xrtb/bidder/RTBServer.java index 036c8736..d00ff6c9 100644 --- a/src/com/xrtb/bidder/RTBServer.java +++ b/src/com/xrtb/bidder/RTBServer.java @@ -420,6 +420,7 @@ public static String getSummary() throws Exception { m.put("diskFree", Performance.getPercFreeDisk()); m.put("openfiles", Performance.getOpenFileDescriptorCount()); m.put("exchanges", BidRequest.getExchangeCounts()); + m.put("lowonthreads", server.getThreadPool().isLowOnThreads()); return DbTools.mapper.writeValueAsString(m); } @@ -1006,20 +1007,21 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques } if (RTBServer.server.getThreadPool().isLowOnThreads()) { - code = RTBServer.NOBID_CODE; - json = "Server throttling"; if (br.id.equals("123")) { - Controller.getInstance().sendLog(3,"RTBSerrver:handler", "Server throttled"); - } - RTBServer.nobid++; - response.setStatus(br.returnNoBidCode()); - response.setContentType(br.returnContentType()); - baseRequest.setHandled(true); - br.writeNoBid(response, time); + Controller.getInstance().sendLog(3,"RTBSerrver:handler", "Server throttled, low on threads"); + } else { + code = RTBServer.NOBID_CODE; + json = "Server throttling"; + RTBServer.nobid++; + response.setStatus(br.returnNoBidCode()); + response.setContentType(br.returnContentType()); + baseRequest.setHandled(true); + br.writeNoBid(response, time); - Controller.getInstance().sendRequest(br,false); - return; - } + Controller.getInstance().sendRequest(br,false); + return; + } + } if (CampaignSelector.getInstance().size() == 0) { if (br.id.equals("123")) { diff --git a/src/com/xrtb/bidder/ZPublisher.java b/src/com/xrtb/bidder/ZPublisher.java index d26ae34f..e2dbf29c 100644 --- a/src/com/xrtb/bidder/ZPublisher.java +++ b/src/com/xrtb/bidder/ZPublisher.java @@ -115,7 +115,7 @@ public ZPublisher(String address) throws Exception { } else if (address.startsWith("redis")) { String[] parts = address.split(":"); channel = parts[1]; - jedisPool = Configuration.getInstance().jedisPool; + // jedisPool = Configuration.getInstance().jedisPool; TND FIX THIS } else if (address.startsWith("http")) { http = new HttpPostGet(); int i = address.indexOf("&"); diff --git a/src/com/xrtb/blocks/AwsCommander.java b/src/com/xrtb/blocks/AwsCommander.java new file mode 100644 index 00000000..bc13c8e8 --- /dev/null +++ b/src/com/xrtb/blocks/AwsCommander.java @@ -0,0 +1,101 @@ +package com.xrtb.blocks; + +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.S3Object; +import com.xrtb.common.Configuration; + +import redis.clients.jedis.Jedis; + +/** + * A class that handles the actual command received by the subscriber. + * @author Ben M. Faul + * + */ +public class AwsCommander { + + String msg = null; + boolean err = false; + /** + * Constructor. Does the actual command. + * @param message String. The message sent in the command. + * @param port int. The port number to respond on. + */ + public AwsCommander(String message) { + String[] tokens = message.split(","); + String[] parts = tokens[0].split(" "); + for (int i = 0; i < parts.length; i++) { + parts[i] = parts[i].trim(); + } + + try { + switch (parts[0]) { + case "load": + msg = load(parts); + break; + case "delete": + delete(parts[1]); + msg = "Symbol " + parts[1] + " removed"; + break; + default: + } + } catch (Exception error) { + msg = error.toString(); + err = true; + error.printStackTrace(); + } + } + + /** + * Return the message from the load + * @return String. The message to return to the caller. + */ + public String getMessage() { + return msg; + } + + /** + * Return whether the command errored + * @return boolean. Returns true if was an error. + */ + public boolean errored() { + return err; + } + + void delete(String key) { + LookingGlass.symbols.remove(key); + } + + /** + * Load the file or s3 object. + * @param parts String[]. An array of tokens. + * @return String. The message returned from the load command. + * @throws Exception on I/O errirs. + */ + String load(String[] parts) throws Exception { + String otype = null; + String symbolName = null; + String name; + String type = parts[1]; // file or S3 + //for (int i=0;i bloomFilter; + int size; + + /** + * Constructor for the File/S3 object to Bloom filter. + * @param name String. The name of the bloom filter. + * @param file String, the file name. + * @throws Exception on File Errors. + */ + public Bloom(String name, String file) throws Exception { + File f = new File(file); + long size = f.length(); + BufferedReader br = new BufferedReader(new FileReader(file)); + makeFilter(br,size); + + symbols.put(name, bloomFilter); + } + + /** + * Constructor for the S3 version of the Bloom filter. + * @param name String. The name of the object. + * @param object S3Object. The object that contains the file. + * @throws Exception on S3 errors. + */ + public Bloom(String name, S3Object object, long size) throws Exception { + InputStream objectData = object.getObjectContent(); + BufferedReader br=new BufferedReader(new InputStreamReader(objectData)); + makeFilter(br,size); + + symbols.put(name, bloomFilter); + } + + /** + * Reads a file or S3 object line by line and loads the filter. + * @param br BufferedReader. The line-by-line reader. + * @throws Exception on I/O errors. + */ + void makeFilter(BufferedReader br, long size) throws Exception { + String[] parts; + int i; + long sz; + + double fpp = 0.03; // desired false positive probability + + String line = br.readLine(); + sz = line.length() - 5; + sz = size / sz; + parts = eatquotedStrings(line); + this.size = 1; + for (i = 0; i < parts.length; i++) { + parts[i] = parts[i].replaceAll("\"", ""); + } + + bloomFilter = BloomFilter.create(Funnels.stringFunnel(Charset.forName("UTF-8")), sz,fpp); + bloomFilter.put(parts[0]); + + while ((line = br.readLine()) != null) { + parts = eatquotedStrings(line); + for (i = 0; i < parts.length; i++) { + parts[i] = parts[i].replaceAll("\"", ""); + } + bloomFilter.put(parts[0]); + this.size++; + } + br.close(); + } + + /** + * Returns the Bloom filter for your use. + * @return BloomFilter. The Guava bloom filter of the contents of this file. + */ + public BloomFilter getBloom() { + return bloomFilter; + } + + /** + * Check if this key is possibly in the bloom filter + * @param key String. The key to test for. + * @return boolean. Returns false if it is not in the filter. Returns true if it possibly is in there. + */ + public boolean isMember(String key) { + return bloomFilter.mightContain(key); + } + + /** + * Returns the number of elements. + * @return int. The number of elements in the filter. + */ + public long getMembers() { + return size; + } +} diff --git a/src/com/xrtb/blocks/CIDRUtils.java b/src/com/xrtb/blocks/CIDRUtils.java new file mode 100644 index 00000000..93fac193 --- /dev/null +++ b/src/com/xrtb/blocks/CIDRUtils.java @@ -0,0 +1,133 @@ +package com.xrtb.blocks; + +import java.math.BigInteger; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * A class that enables to get an IP range from CIDR specification. It supports + * both IPv4 and IPv6. + */ +public class CIDRUtils { + private final String cidr; + + private InetAddress inetAddress; + private InetAddress startAddress; + private InetAddress endAddress; + private final int prefixLength; + + + public CIDRUtils(String cidr) throws UnknownHostException { + + this.cidr = cidr; + + /* split CIDR to address and prefix part */ + if (this.cidr.contains("/")) { + int index = this.cidr.indexOf("/"); + String addressPart = this.cidr.substring(0, index); + String networkPart = this.cidr.substring(index + 1); + + inetAddress = InetAddress.getByName(addressPart); + prefixLength = Integer.parseInt(networkPart); + + calculate(); + } else { + throw new IllegalArgumentException("not an valid CIDR format!"); + } + } + + public long getStartAddress() { + ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE); + buffer.put(startAddress.getAddress()); + buffer.position(0); + Long longValue = buffer.getLong(); + return longValue; + } + + public long getEndAddress() { + ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE); + buffer.put(endAddress.getAddress()); + buffer.position(0); + Long longValue = buffer.getLong(); + return longValue; + } + + + private void calculate() throws UnknownHostException { + + ByteBuffer maskBuffer; + int targetSize; + if (inetAddress.getAddress().length == 4) { + maskBuffer = + ByteBuffer + .allocate(4) + .putInt(-1); + targetSize = 4; + } else { + maskBuffer = ByteBuffer.allocate(16) + .putLong(-1L) + .putLong(-1L); + targetSize = 16; + } + + BigInteger mask = (new BigInteger(1, maskBuffer.array())).not().shiftRight(prefixLength); + + ByteBuffer buffer = ByteBuffer.wrap(inetAddress.getAddress()); + BigInteger ipVal = new BigInteger(1, buffer.array()); + + BigInteger startIp = ipVal.and(mask); + BigInteger endIp = startIp.add(mask.not()); + + byte[] startIpArr = toBytes(startIp.toByteArray(), targetSize); + byte[] endIpArr = toBytes(endIp.toByteArray(), targetSize); + + this.startAddress = InetAddress.getByAddress(startIpArr); + this.endAddress = InetAddress.getByAddress(endIpArr); + + } + + private byte[] toBytes(byte[] array, int targetSize) { + int counter = 0; + List newArr = new ArrayList(); + while (counter < targetSize && (array.length - 1 - counter >= 0)) { + newArr.add(0, array[array.length - 1 - counter]); + counter++; + } + + int size = newArr.size(); + for (int i = 0; i < (targetSize - size); i++) { + + newArr.add(0, (byte) 0); + } + + byte[] ret = new byte[newArr.size()]; + for (int i = 0; i < newArr.size(); i++) { + ret[i] = newArr.get(i); + } + return ret; + } + + public String getNetworkAddress() { + + return this.startAddress.getHostAddress(); + } + + public String getBroadcastAddress() { + return this.endAddress.getHostAddress(); + } + + public boolean isInRange(String ipAddress) throws UnknownHostException { + InetAddress address = InetAddress.getByName(ipAddress); + BigInteger start = new BigInteger(1, this.startAddress.getAddress()); + BigInteger end = new BigInteger(1, this.endAddress.getAddress()); + BigInteger target = new BigInteger(1, address.getAddress()); + + int st = start.compareTo(target); + int te = target.compareTo(end); + + return (st == -1 || st == 0) && (te == -1 || te == 0); + } +} diff --git a/src/com/xrtb/blocks/Cuckoo.java b/src/com/xrtb/blocks/Cuckoo.java new file mode 100644 index 00000000..2c2d75d5 --- /dev/null +++ b/src/com/xrtb/blocks/Cuckoo.java @@ -0,0 +1,103 @@ +package com.xrtb.blocks; + +import java.io.BufferedReader; + +import java.io.File; +import java.io.FileReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.Charset; + + +import com.amazonaws.services.s3.model.S3Object; +import com.github.mgunlogson.cuckoofilter4j.CuckooFilter; +import com.google.common.hash.Funnels; + + +/** + * Implements the BloomFilter for UTF-8 strings. Builds a Guava bloom filter from file or S3 object. + * @author Ben M. Faul + * + */ +public class Cuckoo extends LookingGlass { + CuckooFilter cuckooFilter;; + + /** + * Constructor for the File/S3 object to Bloom filter. + * @param name String. The name of the bloom filter. + * @param file String, the file name. + * @throws Exception on File Errors. + */ + public Cuckoo(String name, String file) throws Exception { + File f = new File(file); + long size = f.length(); + BufferedReader br = new BufferedReader(new FileReader(file)); + makeFilter(br,size); + + symbols.put(name, cuckooFilter); + } + + /** + * Constructor for the S3 version of the Bloom filter. + * @param name String. The name of the object. + * @param object S3Object. The object that contains the file. + * @throws Exception on S3 errors. + */ + public Cuckoo(String name, S3Object object, long size) throws Exception { + InputStream objectData = object.getObjectContent(); + BufferedReader br=new BufferedReader(new InputStreamReader(objectData)); + makeFilter(br,size); + symbols.put(name, cuckooFilter); + } + + /** + * Reads a file or S3 object line by line and loads the filter. + * @param br BufferedReader. The line-by-line reader. + * @throws Exception on I/O errors. + */ + void makeFilter(BufferedReader br, long sz) throws Exception { + String[] parts; + int i; + + + String line = br.readLine(); + line = line.trim(); + i = 0; + + parts = eatquotedStrings(line); + for (i = 0; i < parts.length; i++) { + parts[i] = parts[i].replaceAll("\"", ""); + } + long size = parts[0].length() - 5; + size = sz / size; + double fpp = 0.03; // desired false positive probability + cuckooFilter = new CuckooFilter.Builder<>(Funnels.stringFunnel(Charset.forName("UTF-8")), size).build(); + cuckooFilter.put(parts[0]); + + + while ((line = br.readLine()) != null) { + parts = eatquotedStrings(line); + for (i = 0; i < parts.length; i++) { + parts[i] = parts[i].replaceAll("\"", ""); + } + cuckooFilter.put(parts[0]); + } + br.close(); + } + + /** + * Returns the Bloom filter for your use. + * @return BloomFilter. The Guava bloom filter of the contents of this file. + */ + public CuckooFilter getCuckoo() { + return cuckooFilter; + } + + /** + * Returns the number of elements. + * @return int. The number of elements in the filter. + */ + public long getMembers() { + return cuckooFilter.getCount(); + } +} diff --git a/src/com/xrtb/tools/LookingGlass.java b/src/com/xrtb/blocks/LookingGlass.java similarity index 94% rename from src/com/xrtb/tools/LookingGlass.java rename to src/com/xrtb/blocks/LookingGlass.java index de2f7d2c..1aecffd1 100644 --- a/src/com/xrtb/tools/LookingGlass.java +++ b/src/com/xrtb/blocks/LookingGlass.java @@ -1,9 +1,10 @@ -package com.xrtb.tools; +package com.xrtb.blocks; import java.io.BufferedReader; import java.io.FileReader; import java.util.Date; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import com.xrtb.bidder.Controller; @@ -15,7 +16,7 @@ public class LookingGlass { public static volatile Map symbols = new ConcurrentHashMap(); // My map - Map myMap = new ConcurrentHashMap(); + protected Map myMap = new ConcurrentHashMap(); /** * Default constructor @@ -89,4 +90,9 @@ public static String[] eatquotedStrings(String line) { // } return line.split(regex, -1); } + + public static Object get(String name) { + Object x = symbols.get(name); + return x; + } } diff --git a/src/com/xrtb/tools/Membership.java b/src/com/xrtb/blocks/Membership.java similarity index 99% rename from src/com/xrtb/tools/Membership.java rename to src/com/xrtb/blocks/Membership.java index 0c8837c0..17680c9f 100644 --- a/src/com/xrtb/tools/Membership.java +++ b/src/com/xrtb/blocks/Membership.java @@ -1,4 +1,4 @@ -package com.xrtb.tools; +package com.xrtb.blocks; import java.io.BufferedReader; import java.io.FileReader; diff --git a/src/com/xrtb/blocks/NavMap.java b/src/com/xrtb/blocks/NavMap.java index 8a63b253..efb7e302 100644 --- a/src/com/xrtb/blocks/NavMap.java +++ b/src/com/xrtb/blocks/NavMap.java @@ -3,109 +3,182 @@ import java.io.BufferedReader; import java.io.FileReader; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.Iterator; import java.util.Map; -import java.util.NavigableMap; +import java.util.Set; import java.util.TreeMap; -import org.jboss.netty.handler.ipfilter.CIDR; - -import com.xrtb.bidder.Controller; -import com.xrtb.common.Configuration; -import com.xrtb.tools.LookingGlass; - -public class NavMap extends LookingGlass { - - public static List in = new ArrayList(); - public static List out = new ArrayList(); - - NavigableMap map = new TreeMap(); - public static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); - - public static List inPoints = new ArrayList(); - public static List outPoints = new ArrayList(); - public static int testPoints = 1000; - public static int current = 0; - - public static boolean searchTable(String key, String ip) { - NavMap x = (NavMap) symbols.get(key); - if (x == null) - return false; - return x.search(ip); - } - public static boolean searchTable(String key, long ip) { - NavMap x = (NavMap) symbols.get(key); - if (x == null) - return false; - return x.search(ip); - } - - public static void main(String args[]) throws Exception { - // NavMap sr = new NavMap("ISP", "/home/ben/Downloads/ISP.txt", false); - - NavMap sr = new NavMap("ISP", "data/METHBOT.txt", true); +import com.amazonaws.services.s3.model.S3Object; +import com.google.common.collect.Maps; + +public class NavMap extends LookingGlass implements Set { + + // The backing TreeMap. + TreeMap tree = Maps.newTreeMap(); + + // The name of the object + String name; + + /** + * Older form of constructor where you are told what kind it is cidr or not + * + * @param name + * String. The name of the key. + * @param file + * String. The file name. + * @param cidr + * boolean. Is a cidr is true. Else false is range + * @throws Exception + * on I/O errors. + */ + public NavMap(String name, String file, boolean cidr) throws Exception { + this.name = name; + if (cidr) { + BufferedReader br = new BufferedReader(new FileReader(file)); + doCidr(br, file); + } else { + BufferedReader br = new BufferedReader(new FileReader(file)); + doRanges(br, file); + } - /*for (int i = 0; i < 1; i++) { - // String ip = "1.0.10.0"; - String ip = "223.255.192.255"; - long now = System.nanoTime(); - boolean x = sr.search(ip); - now = System.nanoTime() - now; - now /= 1000; - System.out.println(ip + " " + x + " " + now + " micro seconds"); - } */ - // 203.180.58.167-203.180.58.187 - // 203.212.37.105-203.212.37.125 + symbols.put(name, this); } - public NavMap(String name, String file, boolean isCidr) throws Exception { - super(); - - if (isCidr) - doCidr(name, file); - else - doRanges(name, file); + /** + * A navigable hashmap for handling CIDR lists and range maps made from a + * file. + * + * @param name + * String. The name of the object. + * @param file + * String. The name of the file. + * @throws Exception + * on I/O errors. + */ + + public NavMap(String name, String file) throws Exception { + this.name = name; + if (file.endsWith("cidr")) { + BufferedReader br = new BufferedReader(new FileReader(file)); + doCidr(br, file); + } else if (file.endsWith("range")) { + BufferedReader br = new BufferedReader(new FileReader(file)); + doRanges(br, file); + } else + throw new Exception(file + " Not in range or CIDR form"); + symbols.put(name, this); } - void doCidr(String name, String file) throws Exception { - BufferedReader br = new BufferedReader(new FileReader(file)); - long start = 0; - long end = 0; - String message = null; - XRange r = null; - int k = 0; + /** + * A navigable hashmap for handling CIDR lists and range maps from an S3 + * object. + * + * @param name + * String. The name of the object. + * @param object + * S3Object. The S3 object for this.. + * @throws Exception + * on I/O errors. + */ + public NavMap(String name, S3Object object) throws Exception { + this.name = name; + String file = object.getBucketName(); + InputStream objectData = object.getObjectContent(); + BufferedReader br = new BufferedReader(new InputStreamReader(objectData)); + if (file.endsWith("cidr")) { + doCidr(br, file); + } else if (file.endsWith("range")) { + doRanges(br, file); + } else + throw new Exception(file + " Not in range or CIDR form"); + } - message = "Initialize CIDR navmap: " + file + " as " + name; + /** + * Process CIDR lists. + * + * @param br + * BufferedReader. The line-by-line reader. + * @param file + * String. The file name. + * @throws Exception + * on I/O errors. + */ + void doCidr(BufferedReader br, String file) throws Exception { + long start; + long end; + String messagel; + XRange r; + int k; + + k = 0; for (String line; (line = br.readLine()) != null;) { if (!(line.startsWith("#") || line.length() < 10)) { - CIDR cidr = CIDR.newCIDR(line); - start = ipToLong(cidr.getBaseAddress().toString().substring(1)); - end = ipToLong(cidr.getEndAddress().toString().substring(1)); + CIDRUtils cidr = new CIDRUtils(line); + start = cidr.getStartAddress(); + end = cidr.getEndAddress(); r = new XRange(end, k); k++; - map.put(start, r); + tree.put(start, r); } } - message += ", total records = " + k; + } - if (Configuration.isInitialized()) { - System.out.format("[%s] - %d - %s - %s - %s\n", Controller.sdf.format(new Date()), 1, - Configuration.instanceName, "NavMap", message); + /** + * Search the navmap using a long representation of the + * + * @param key + * @return + */ + public Boolean search(long key) { + Map.Entry entry = tree.floorEntry(key); + if (entry == null) + return false; + else if (key <= entry.getValue().upper) { + return true; // return entry.getValue().value; } else { - System.out.format("[%s] - %d - %s - %s - %s\n", "", 1, - "", "NavMap", message); + return false; } - symbols.put(name, this); } - void doRanges(String name, String file) throws Exception { - BufferedReader br = new BufferedReader(new FileReader(file)); - long x = 0, k = 0; + public static long ipToLong(String ipAddress) { + + String[] ipAddressInArray = ipAddress.split("\\."); + + long result = 0; + for (int i = 0; i < ipAddressInArray.length; i++) { + + int power = 3 - i; + int ip = Integer.parseInt(ipAddressInArray[i]); + result += ip * Math.pow(256, power); + + } + + return result; + } + + public static String longToIp(long ip) { + if (ip > 4294967295L || ip < 0) { + throw new IllegalArgumentException("invalid ip"); + } + StringBuilder ipAddress = new StringBuilder(); + for (int i = 3; i >= 0; i--) { + int shift = i * 8; + ipAddress.append((ip & (0xff << shift)) >> shift); + if (i > 0) { + ipAddress.append("."); + } + } + return ipAddress.toString(); + } + + void doRanges(BufferedReader br, String file) throws Exception { long oldstart = 0; long oldend = 0; long start = 0; @@ -116,9 +189,9 @@ void doRanges(String name, String file) throws Exception { XRange r = null; long over = 0; - int linek = 0; + int linek = 9; + int k = 0; - message = "Initialize RANGE navmap: " + file + " as " + name; for (String line; (line = br.readLine()) != null;) { if (!(line.startsWith("#") || line.length() < 10)) { parts = line.split("-"); @@ -126,7 +199,7 @@ void doRanges(String name, String file) throws Exception { start = ipToLong(parts[0]); end = ipToLong(parts[1]); - + if (oldstart == 0) { oldstart = start; oldend = end; @@ -137,79 +210,119 @@ void doRanges(String name, String file) throws Exception { } else { r = new XRange(oldend, k); k++; - - map.put(oldstart, r); - - outPoints.add(oldend+1); + + tree.put(oldstart, r); + oldstart = start; oldend = end; } } - if (k % 1000 == 0 && in.size() < 10) { - in.add(end + 10); - } + } linek++; } } r = new XRange(end, k); - k++; - map.put(start, r); + tree.put(start, r); double d = (double) over / (double) linek; - message += ", overlaps = " + over + ", total records = " + k + ", % overlap = " + d; + } - System.out.format("[%s] - %d - %s - %s - %s\n", sdf.format(new Date()), 1, - "localhost", "NavMap", message); - symbols.put(name, this); + @Override + public int size() { + return tree.size(); + } + @Override + public boolean isEmpty() { + if (tree.size() == 0) + return true; + return false; } - public boolean search(String ip) { - long address = ipToLong(ip); + public boolean searchTable(Object key) { + return contains(key); + } + + @Override + public boolean contains(Object key) { + // System.out.println("Looking for: " + key); + long address = 0; + if (key instanceof Long) + address = (long) key; + else + try { + InetAddress inetAddress = InetAddress.getByName((String) key); + ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE); + buffer.put(inetAddress.getAddress()); + buffer.position(0); + address = buffer.getLong(); + } catch (UnknownHostException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + // System.out.println("X = " + address); return search(address); } - public boolean search(long key) { - Map.Entry entry = map.floorEntry(key); - if (entry == null) - return false; - else if (key <= entry.getValue().upper) { - return true; // return entry.getValue().value; - } else { - return false; - } + @Override + public Iterator iterator() { + // TODO Auto-generated method stub + return null; } - public static long ipToLong(String ipAddress) { + @Override + public Object[] toArray() { + // TODO Auto-generated method stub + return null; + } - String[] ipAddressInArray = ipAddress.split("\\."); + @Override + public Object[] toArray(Object[] a) { + // TODO Auto-generated method stub + return null; + } - long result = 0; - for (int i = 0; i < ipAddressInArray.length; i++) { + @Override + public boolean add(Object e) { + // TODO Auto-generated method stub + return false; + } - int power = 3 - i; - int ip = Integer.parseInt(ipAddressInArray[i]); - result += ip * Math.pow(256, power); + @Override + public boolean remove(Object o) { + // TODO Auto-generated method stub + return false; + } - } + @Override + public boolean containsAll(Collection c) { + // TODO Auto-generated method stub + return false; + } - return result; + @Override + public boolean addAll(Collection c) { + // TODO Auto-generated method stub + return false; } - public static String longToIp(long ip) { - if (ip > 4294967295l || ip < 0) { - throw new IllegalArgumentException("invalid ip"); - } - StringBuilder ipAddress = new StringBuilder(); - for (int i = 3; i >= 0; i--) { - int shift = i * 8; - ipAddress.append((ip & (0xff << shift)) >> shift); - if (i > 0) { - ipAddress.append("."); - } - } - return ipAddress.toString(); + @Override + public boolean retainAll(Collection c) { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean removeAll(Collection c) { + // TODO Auto-generated method stub + return false; + } + + @Override + public void clear() { + // TODO Auto-generated method stub + } } diff --git a/src/com/xrtb/blocks/SimpleMultiset.java b/src/com/xrtb/blocks/SimpleMultiset.java new file mode 100644 index 00000000..1f0f58b1 --- /dev/null +++ b/src/com/xrtb/blocks/SimpleMultiset.java @@ -0,0 +1,90 @@ +package com.xrtb.blocks; + +import java.io.BufferedReader; + + + +import java.io.FileReader; +import java.io.InputStream; +import java.io.InputStreamReader; + + +import com.amazonaws.services.s3.model.S3Object; +import com.google.common.collect.HashMultiset; +import com.google.common.collect.Multiset; + + +/** + * Implements the Multiset counter for UTF-8 strings. Builds a Guava Multiset from file or S3 object. + * @author Ben M. Faul + * + */ +public class SimpleMultiset extends LookingGlass { + Multiset ms = HashMultiset.create(); + int size = 0; + + /** + * Constructor for the File/S3 object to Bloom filter. + * @param name String. The name of the bloom filter. + * @param file String, the file name. + * @throws Exception on File Errors. + */ + public SimpleMultiset(String name, String file) throws Exception { + BufferedReader br = new BufferedReader(new FileReader(file)); + System.out.print("Initialize Multiset: " + name + " from " + file + ", enttries = "); + while(br.readLine() != null) { + size++; + } + br.close(); + + br = new BufferedReader(new FileReader(file)); + makeFilter(br); + System.out.println(ms.size() + " elements"); + + symbols.put(name, ms); + } + + /** + * Constructor for the S3 version of the Multiset filter. + * @param name String. The name of the object. + * @param object S3Object. The object that contains the file. + * @throws Exception on S3 errors. + */ + public SimpleMultiset(String name, S3Object object) throws Exception { + InputStream objectData = object.getObjectContent(); + BufferedReader br=new BufferedReader(new InputStreamReader(objectData)); + makeFilter(br); + System.out.println(size + " elements"); + + symbols.put(name, ms); + } + + /** + * Reads a file or S3 object line by line and loads the filter. + * @param br BufferedReader. The line-by-line reader. + * @throws Exception on I/O errors. + */ + void makeFilter(BufferedReader br) throws Exception { + for (String line; (line = br.readLine()) != null;) { + line = line.trim(); + ms.add(line); + } + br.close(); + } + + /** + * Returns the Multiset filter for your use. + * @return Multiset. The Guava multiset of the contents of this file. + */ + public Multiset getMultiset() { + return ms; + } + + /** + * Returns the number of elements. + * @return int. The number of elements in the filter. + */ + public int getMembers() { + return size; + } +} diff --git a/src/com/xrtb/blocks/SimpleSet.java b/src/com/xrtb/blocks/SimpleSet.java new file mode 100644 index 00000000..5ecde5d1 --- /dev/null +++ b/src/com/xrtb/blocks/SimpleSet.java @@ -0,0 +1,84 @@ +package com.xrtb.blocks; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Set; + +import com.amazonaws.services.s3.model.S3Object; +import com.google.common.collect.Sets; + +/** + * A guava set. Reads a file or S3 object and shoves the lines into a guava set. + * @author Ben M. Faul + * + */ +public class SimpleSet extends LookingGlass { + Set set = Sets.newHashSet(); + String message; + + /** + * A simple set. Reads from file, shoves into Set, line by line. + * @param name String. The name of the object. + * @param file String. The file name. + * @throws Exception on I/O errors. + */ + public SimpleSet(String name, String file) throws Exception { + BufferedReader br = new BufferedReader(new FileReader(file)); + message = "Initialize Simple Membership: " + file + " as " + name; + + makeSet(br); + + symbols.put(name,set); + } + + /** + * A simple set. Reads S3 object and put into the Set. + * @param name String. The name of the object. + * @param object S3Object. The S3 object to read. + * @throws Exception on S3 or I/O options. + */ + public SimpleSet(String name, S3Object object) throws Exception { + InputStream objectData = object.getObjectContent(); + BufferedReader br=new BufferedReader(new InputStreamReader(objectData)); + message = "Initialize Simple Membership: " + object.getBucketName() + " as " + name; + makeSet(br); + + symbols.put(name, this); + } + + /** + * Make the set from a buffered reader. + * @param br BufferedReader. Read line-by-line and place in the set. + * @throws Exception on I/O errors. + */ + void makeSet(BufferedReader br) throws Exception { + String[] parts = null; + int i; + for (String line; (line = br.readLine()) != null;) { + parts = eatquotedStrings(line); + for (i = 0; i < parts.length; i++) { + parts[i] = parts[i].replaceAll("\"", ""); + } + set.add(parts[0]); + } + br.close(); + } + + /** + * Return the simple set size + * @return int. Returns the size. + */ + public int size() { + return set.size(); + } + + /** + * Return the Set. + * @return Set. The Guava set. + */ + public Set getSet() { + return set; + } +} diff --git a/src/com/xrtb/commands/ConfigureAwsObject.java b/src/com/xrtb/commands/ConfigureAwsObject.java new file mode 100644 index 00000000..74a485e5 --- /dev/null +++ b/src/com/xrtb/commands/ConfigureAwsObject.java @@ -0,0 +1,37 @@ +package com.xrtb.commands; + +import com.xrtb.bidder.Controller; +import com.xrtb.common.Campaign; + +/** + * A class that is used to encapsulate a 0MQ command for adding a campaign to the bidder. + * Jackson will be used to create the structure. + * @author Ben M. Faul + * + */ +public class ConfigureAwsObject extends BasicCommand { + + /** + * Empty constructor for Jackson + */ + public ConfigureAwsObject() { + super(); + cmd = Controller.CONFIGURE_AWS_OBJECT; + msg = "An AWS object is being configured in the system"; + } + + /** + * Configure an AWS Object. + * @param to String. The bidder that will execute the command. + * @param name String. The name of the owner of the campaign. + * @param target String. The command to execute. + */ + public ConfigureAwsObject(String to, String name, String target) { + super(to); + cmd = Controller.CONFIGURE_AWS_OBJECT; + status = "ok"; + this.owner = name; + this.target = target; + msg = "An AWS OBJECT is befing configured: " + name +"/" + target; + } +} diff --git a/src/com/xrtb/commands/PixelClickConvertLog.java b/src/com/xrtb/commands/PixelClickConvertLog.java index 62bb7320..b173f207 100644 --- a/src/com/xrtb/commands/PixelClickConvertLog.java +++ b/src/com/xrtb/commands/PixelClickConvertLog.java @@ -1,12 +1,12 @@ package com.xrtb.commands; - /** * Base class for logging pixel loads, clicks and conversions. + * * @author Ben M. Faul * */ -public class PixelClickConvertLog { +public class PixelClickConvertLog { public String instance; public String payload; public double lat; @@ -21,65 +21,54 @@ public class PixelClickConvertLog { public static final int PIXEL = 0; public static final int CLICK = 1; public static final int CONVERT = 2; - - public static void main(String [] args) { + + public static void main(String[] args) { PixelClickConvertLog x = new PixelClickConvertLog(); x.create("//pixel/citenko/4/3/25c40279-dd90-4caa-afc9-d0474705e0d1/0.0425/32.83/-83.65"); - + } - + /** * Default constructor */ public PixelClickConvertLog() { - + } - + /** * Create the log from a chunk of text - * @param data String. The data to use. + * + * @param data + * String. The data to use. */ public void create(String data) { - + time = System.currentTimeMillis(); if (data.contains("redirect")) { doClick(data); - return; } - String [] parts = data.split("/"); - payload = data; - exchange = parts[3]; - ad_id = parts[4]; - creative_id = parts[5]; - bid_id = parts[6]; - try { - price = Double.parseDouble(parts[7]); - lat = Double.parseDouble(parts[8]); - lon = Double.parseDouble(parts[9]); - } catch (Exception error) { - //error.printStackTrace(); - doClick(payload); - } + + doClick(payload); type = PIXEL; - time = System.currentTimeMillis(); - } - + /** * Process a click - * @param payload String. The data to turn into the log message. + * + * @param payload + * String. The data to turn into the log message. */ void doClick(String payload) { this.payload = payload; - String [] parts = payload.split("/"); - - for (int i=0;i ms3 = (Map) m.get("s3"); + String accessKey = ms3.get("access_key_id"); + String secretAccessKey = ms3.get("secret_access_key"); + String region = ms3.get("region"); + s3_bucket = ms3.get("bucket"); + + s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretAccessKey)); + ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(s3_bucket)); + + try { + processDirectory(s3, listing, s3_bucket); + } catch (Exception error) { + System.err.println("ERROR IN AWS LISTING: " + error.toString()); + } + } /** * SSL */ @@ -470,7 +502,7 @@ public void initialize(String path, String shard, int port, int sslPort) throws ForensiqClient.getInstance().connections = (int) (Integer) fraud.get("connections"); forensiq = fx; } - } else { + } else { System.out.println("*** Fraud detection is set to MMDB"); String db = (String) fraud.get("db"); MMDBClient fy = MMDBClient.build(db); @@ -478,7 +510,7 @@ public void initialize(String path, String shard, int port, int sslPort) throws fy.bidOnError = (Boolean) fraud.get("bidOnError"); } if (fraud.get("watchlist") != null) { - fy.setWatchlist((List)fraud.get("watchlist")); + fy.setWatchlist((List) fraud.get("watchlist")); } forensiq = fy; } @@ -551,11 +583,25 @@ public void initialize(String path, String shard, int port, int sslPort) throws Integer rport = (Integer) redis.get("port"); if (rport == null) rport = 6379; - - jedisPool = new JedisPool(host,rport); + + // JedisPoolConfig poolConfig = new JedisPoolConfig();; + // configJedis.setMaxTotal(rsize); + // configJedis.setMaxWaitMillis(10); + + // poolConfig.setMaxIdle(4000); + // Tests whether connections are dead during idle periods + // poolConfig.setTestWhileIdle(true); + // poolConfig.setMaxTotal(4000); + // poolConfig.setMaxWaitMillis(30); + + // jedisPool = new JedisPool(poolConfig,host,rport); + + MyJedisPool.host = host; + MyJedisPool.port = rport; + jedisPool = new MyJedisPool(1000, 1000, 5); System.out.println( - "*** JEDISPOOL = " + jedisPool + ". host = " + host + ", port = " + port + ", size = " + rsize); + "*** JEDISPOOL = " + jedisPool + ". host = " + host + ", port = " + rport + ", size = " + rsize); } Map zeromq = (Map) m.get("zeromq"); @@ -570,13 +616,13 @@ public void initialize(String path, String shard, int port, int sslPort) throws cacheHost = value; if (r.get("port") != null) cachePort = (Integer) r.get("port"); - if (r.get("maxconns") != null) + if (r.get("maxconns") != null) maxconns = (Integer) r.get("maxconns"); - AerospikeHandler.getInstance(cacheHost,cachePort,maxconns); + AerospikeHandler.getInstance(cacheHost, cachePort, maxconns); redisson = new RedissonClient(AerospikeHandler.getInstance()); Database.getInstance(redisson); - System.out.println("*** Aerospike connection set to: " + cacheHost + ":" + cachePort + ", connections = " + maxconns + ", handlers: " + - AerospikeHandler.getInstance().getCount() + " ***"); + System.out.println("*** Aerospike connection set to: " + cacheHost + ":" + cachePort + ", connections = " + + maxconns + ", handlers: " + AerospikeHandler.getInstance().getCount() + " ***"); String key = (String) m.get("deadmanswitch"); if (key != null) { @@ -689,6 +735,130 @@ public String requstLogStrategyAsString() { return "all"; } + public void processDirectory(AmazonS3Client s3, ObjectListing listing, String bucket) throws Exception { + for (S3ObjectSummary objectSummary : listing.getObjectSummaries()) { + long size = objectSummary.getSize(); + System.out.println("*** Processing S3 " + objectSummary.getKey() + ", size = " + size); + S3Object object = s3.getObject(new GetObjectRequest(bucket, objectSummary.getKey())); + + String bucketName = object.getBucketName(); + String keyName = object.getKey(); + + if (keyName.contains("Darren")) { + System.out.println("HERE"); + } + GetObjectTaggingRequest request = new GetObjectTaggingRequest(bucketName, keyName); + GetObjectTaggingResult result = s3.getObjectTagging(request); + List tags = result.getTagSet(); + String type = null; + String name = null; + + if (tags.isEmpty()) { + System.err.println("Error: " + keyName + " has no tags"); + } else { + for (Tag tag : tags) { + String key = tag.getKey(); + String value = tag.getValue(); + + if (key.equals("type")) { + type = value; + } + + if (key.equals("name")) { + name = value; + } + } + + if (name == null) + throw new Exception("Error: " + keyName + " is missing a name tag"); + if (name.contains(" ")) + throw new Exception("Error: " + keyName + " has a name attribute with a space in it"); + if (type == null) + throw new Exception("Error: " + keyName + " has no type tag"); + + if (!name.startsWith("$")) + name = "$" + name; + + readData(type, name, object, size); + } + } + } + + public static String readData(String fileName) throws Exception { + String message = ""; + int i = fileName.indexOf("."); + if (i == -1) + throw new Exception("Filename is missing type field"); + String type = fileName.substring(i); + NavMap map; + SimpleMultiset set; + SimpleSet sset; + Bloom b; + Cuckoo c; + switch(type) { + case "range": + map = new NavMap(fileName,fileName,false); + message = "Added NavMap " + fileName + ": from file, has " + map.size() + " members"; + break; + case "cidr": + map = new NavMap(fileName,fileName,true); + message = "Added NavMap " + fileName + ": from file, has " + map.size() + " members"; + break; + case "bloom": + b = new Bloom(fileName,fileName); + message = "Initialize Bloom Filter: " + fileName + " from file, members = " + b.getMembers(); + break; + case "cuckoo": + c = new Cuckoo(fileName,fileName); + break; + case "multiset": + set = new SimpleMultiset(fileName, fileName); + message = "Initialize Multiset " + fileName + " from file, entries = " + set.getMembers(); + break; + case "set": + sset = new SimpleSet(fileName, fileName); + message = "Initialize Multiset " + fileName + " from file, entries = " + sset.size(); + break; + + default: + message = "Unknown type: " + type; + } + System.out.println("*** " + message); + return message; + } + + public static String readData(String type, String name, S3Object object, long size) throws Exception { + String message = ""; + switch (type) { + case "range": + case "cidr": + NavMap map = new NavMap(name, object); + message = "Added NavMap " + name + ": has " + map.size() + " members"; + break; + case "set": + SimpleSet set = new SimpleSet(name, object); + message = "Initialize Set: " + name + " from S3, entries = " + set.size(); + break; + case "bloom": + Bloom b = new Bloom(name, object, size); + message = "Initialize Bloom Filter: " + name + " from S3, members = " + b.getMembers(); + break; + + case "cuckoo": + Cuckoo c = new Cuckoo(name, object, size); + message = "Initialize Cuckoo Filter: " + name + " from S3, entries = " + c.getMembers(); + break; + case "multiset": + SimpleMultiset ms = new SimpleMultiset(name, object); + message = "Initialize Multiset " + name + " from S3, entries = " + ms.getMembers(); + break; + default: + message = "Unknown type: " + type; + } + System.out.println("*** " + message); + return message; + } + public int requstLogStrategyAsInt(String x) { switch (x) { case "all": @@ -839,8 +1009,8 @@ public static Configuration getInstance(String fileName, String shard, int port, synchronized (Configuration.class) { if (theInstance == null) { theInstance = new Configuration(); - theInstance.initialize(fileName, shard, port, sslPort); try { + theInstance.initialize(fileName, shard, port, sslPort); theInstance.shell = new JJS(); } catch (Exception error) { diff --git a/src/com/xrtb/common/Deals.java b/src/com/xrtb/common/Deals.java index 3e733b7f..2b0859db 100644 --- a/src/com/xrtb/common/Deals.java +++ b/src/com/xrtb/common/Deals.java @@ -9,32 +9,124 @@ import com.xrtb.tools.XORShiftRandom; +/** + * A class that chooses deals. For duplicates you can choose random or highest (and random of dup highest). It extends + * List simply as a convenience for configuration. + * @author Ben M. Faul + * + */ public class Deals extends ArrayList { + // Serialize id + private static final long serialVersionUID = 1L; + + // Random number generator private static XORShiftRandom rand = new XORShiftRandom(); - private Map map = new HashMap(); - private Set s2; + // A map of the deals + private Map map = new HashMap(); + // Set of ids in the deals list + private Set s2 = new HashSet(); + /** + * Default constructor for Jackson to use. + */ public Deals() { } - public Deal findDeal(List ids) { + /** + * Find a random deal in an intersection with a list of ids. + * @param ids List. A list of String ids. + * @return Deal. A random deal from the intersection of ids with this set. + */ + public Deal findDealRandom(List ids) { Set intersection = new HashSet(ids); // use the copy constructor intersection.retainAll(s2); if (intersection.size()==0) return null; + int x = rand.random(intersection.size()); List nameList = new ArrayList(intersection); String key = nameList.get(x); return map.get(key); } + /** + * Return the highest price deal in the list of ids that match. If duplicates are found, then choose one of those at random. + * @param ids List. A list of ids to match. + * @return Deal. The hoghest price deal that matches. + */ + public Deal findDealHighest(List ids) { + Set intersection = new HashSet(ids); // use the copy constructor + intersection.retainAll(s2); + if (intersection.size()==0) + return null; + + List nameList = new ArrayList(intersection); + List candidates = new ArrayList(); + Deal x = null; + for (int i=0;i< nameList.size();i++) { + String key = nameList.get(i); + Deal test = map.get(key); + if (x == null || x.price < test.price) { + x = test; + candidates.clear(); + candidates.add(x); + } else { + if (x.price == test.price) + candidates.add(test); + } + } + if (candidates.size()==1) + return x; + int r = rand.random(candidates.size()); + return candidates.get(r); + } + @Override public boolean add(Deal d) { s2.add(d.id); map.put(d.id,d); - return add(d); + return super.add(d); + } + + public static void main(String args[]) { + + Deals deals = new Deals(); + Deal a = new Deal("a",1); + Deal b = new Deal("b",2); + deals.add(a); + deals.add(b); + + List test = new ArrayList(); + test.add("c"); + Deal x = deals.findDealRandom(test); + System.out.println(x); + + test.add("a"); + x = deals.findDealRandom(test); + System.out.println(x.id); + + test.add("b"); + + for (int i=0;i<10;i++) { + x = deals.findDealRandom(test); + System.out.println(x.id); + } + + for (int i=0;i<10;i++) { + x = deals.findDealHighest(test); + System.out.println(x.id); + } + + Deal z = new Deal("z",2); + deals.add(z); + test.add("z"); + + for (int i=0;i<10;i++) { + x = deals.findDealHighest(test); + System.out.println(x.id); + } } } diff --git a/src/com/xrtb/common/MyJedisPool.java b/src/com/xrtb/common/MyJedisPool.java index 511924b9..1cbf5e99 100644 --- a/src/com/xrtb/common/MyJedisPool.java +++ b/src/com/xrtb/common/MyJedisPool.java @@ -1,165 +1,24 @@ package com.xrtb.common; - -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; - -import edu.emory.mathcs.backport.java.util.concurrent.locks.ReentrantLock; import redis.clients.jedis.Jedis; -/** - * A threas safe class that sanely handles broken resources in Jedis. Stupid jedis pool is brain dead. - * @author Ben M. Faul - * - */ -public class MyJedisPool { - private final BlockingQueue pool; - private final ReentrantLock lock = new ReentrantLock(); - private int createdObjects = 0; - private int size = 64; - - String host = "localhost"; - int port = 6379; - - /** - * Create a default pool. We create 64 by default. - */ - public MyJedisPool() { - this(64,true); - - } - - /** - * Create a jedis pool to the host and port, with default number of connections. - * @param host String. The hostname. - * @param port int. The port to connect to. - */ - public MyJedisPool(String host, int port) { - this(64, true); - this.host = host; - this.port = port; - } - - /** - * Create a jedis pool to the host and port, with specified number of connections. - * @param host String. The hostname. - * @param port int. The port to connect to. - * @param size int. The number of JEDIS objects to keep around. - */ - public MyJedisPool(String host, int port, int size) { - this(size, true); - this.host = host; - this.port = port; - } - - /** - * Create the pool. - * @param size int. The size of the pool at max. - * @param dynamicCreation boolean. Allow dynamic creations - */ - protected MyJedisPool(int size, Boolean dynamicCreation) { - // Enable the fairness; otherwise, some threads - // may wait forever. - pool = new ArrayBlockingQueue(size,true); - this.size = size; - if (!dynamicCreation) { - lock.lock(); - } - } - - /** - * Get a resource. - * @return Jedis. The connection to JEDIS. - * @throws Exception on connection errors. - */ - public Jedis getResource() throws Exception { - if (pool.size() == 0 && createdObjects < size) { - ++createdObjects; - System.out.println("MAKE OBJECT: " + createdObjects); - return createObject(); - } - - return pool.take(); - } - - /** - * Return a resource to the pool. - * @param resource Jedis. The thing to return. - * @throws Exception on addition errors. - */ - public void returnResource(Jedis resource) throws Exception { - // Will throws Exception when the queue is full, - // but it should never happen. - pool.add(resource); - } - - /** - * Create the pool, when dynamic. Will open the connections and stuff them all in. - */ - public void createPool() { - if (lock.isLocked()) { - for (int i = 0; i < size; ++i) { - pool.add(createObject()); - createdObjects++; - } - } - } - - /** - * Create an object for the pool. - * @return Jedis. The new resource. - */ - protected Jedis createObject() { - Jedis jedis = new Jedis(host,port); - jedis.connect(); - return jedis; - } - - /** - * Publish from the pool. - * @param channel String. The channel to publish on. - * @param message String. The message to publish. - */ - public void publish(String channel, String message) { - Jedis j = null; - try { - j = getResource(); - j.publish(channel, message); - } catch (Exception error) { - j.disconnect(); - j.connect(); - j.publish(channel, message); - } - try { - returnResource(j); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - /** - * Do sismember from the pool. - * @param key String. The set we are checking. - * @param value. String. The value we are checking membership on. - * @return boolean. Returns true if in the set, else false. - */ - public boolean sismember(String key, String value) { - boolean t = false; - Jedis j = null; - try { - j = getResource(); - t = j.sismember(key, value); - } catch (Exception error) { - j.disconnect(); - j.connect(); - t = j.sismember(key, value); - } - try { - returnResource(j); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - return t; - } -} +public class MyJedisPool extends ObjectPool { + + public static String host = "localhost"; + public static int port = 6379; + public MyJedisPool(int minIdle) { + super(minIdle); + } + + public MyJedisPool(final int minIdle, final int maxIdle, final long validationInterval) { + super(minIdle, maxIdle, validationInterval); + } + + @Override + protected Jedis createObject() { + Jedis jedis = new Jedis(host,port); + jedis.connect(); + + return jedis; + } + +} \ No newline at end of file diff --git a/src/com/xrtb/common/Node.java b/src/com/xrtb/common/Node.java index 9b56494b..09226098 100644 --- a/src/com/xrtb/common/Node.java +++ b/src/com/xrtb/common/Node.java @@ -22,6 +22,8 @@ import com.fasterxml.jackson.databind.node.MissingNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; +import com.google.common.hash.BloomFilter; +import com.xrtb.blocks.LookingGlass; import com.xrtb.blocks.NavMap; import com.xrtb.pojo.BidRequest; @@ -646,21 +648,26 @@ public boolean testInternal(Object value) throws Exception { case MEMBER: case NOT_MEMBER: - if (sval != null && sval.startsWith("@")) { - boolean t = NavMap.searchTable(sval,svalue); - if (operator == NOT_MEMBER) - return !t; - else - return t; - } - if (sval != null && sval.startsWith("$")) { - boolean t = jedisIsMember(sval,svalue); + if (sval != null && (sval.startsWith("@") || sval.startsWith("$"))) { + Object x = LookingGlass.get(sval); + boolean t = false; + if (x instanceof NavMap) { + NavMap nm = (NavMap)x; + t = nm.contains(svalue); + } else if (x instanceof BloomFilter) { + BloomFilter b = (BloomFilter)x; + t = b.mightContain(svalue); + } else if (x instanceof Set) { + Set set = (Set)x; + t = set.contains(svalue); + } + if (operator == NOT_MEMBER) return !t; else return t; } - + if (qvalue == null) { if (lval != null) qvalue = new HashSet(lval); @@ -802,10 +809,23 @@ public boolean processLTE(int operator, Number ival, Number nvalue, String sval, boolean jedisIsMember(String key, String member) { boolean t = false; + if (Configuration.getInstance().jedisPool == null) + return false; + try { - Jedis jedis = Configuration.getInstance().jedisPool.getResource(); + + if (member.equals("842AAB10FBA04247B3A9CE00C9172350")) { + System.out.println("$$$$$$$$$$$$$$$$$$$ KEYTEST on " + key); + } + //Jedis jedis = Configuration.getInstance().jedisPool.getResource(); + Jedis jedis = Configuration.getInstance().jedisPool.borrowObject(); t = jedis.sismember(key,member); - Configuration.getInstance().jedisPool.returnResource(jedis); + + if (member.equals("842AAB10FBA04247B3A9CE00C9172350")) { + System.out.println("$$$$$$$$$$$$$$$$$$$ TEST RETURNS: " + t); + } + //Configuration.getInstance().jedisPool.returnResource(jedis); + Configuration.getInstance().jedisPool.returnObject(jedis); //t = Configuration.getInstance().jedisPool.sismember(key, member); } catch (Exception error) { error.printStackTrace(); diff --git a/src/com/xrtb/common/ObjectPool.java b/src/com/xrtb/common/ObjectPool.java new file mode 100644 index 00000000..f226b6c7 --- /dev/null +++ b/src/com/xrtb/common/ObjectPool.java @@ -0,0 +1,129 @@ +package com.xrtb.common; + +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * + * @see simple pool + */ +abstract class ObjectPool +{ + private ConcurrentLinkedQueue pool; + + private ScheduledExecutorService executorService; + + /** + * Creates the pool. + * + * @param minIdle minimum number of objects residing in the pool + */ + public ObjectPool(final int minIdle) + { + // initialize pool + initialize(minIdle); + } + + /** + * Creates the pool. + * + * @param minIdle minimum number of objects residing in the pool + * @param maxIdle maximum number of objects residing in the pool + * @param validationInterval time in seconds for periodical checking of minIdle / maxIdle conditions in a separate thread. + * When the number of objects is less than minIdle, missing instances will be created. + * When the number of objects is greater than maxIdle, too many instances will be removed. + */ + public ObjectPool(final int minIdle, final int maxIdle, final long validationInterval) + { + // initialize pool + initialize(minIdle); + + // check pool conditions in a separate thread + executorService = Executors.newSingleThreadScheduledExecutor(); + executorService.scheduleWithFixedDelay(new Runnable() + { + @Override + public void run() + { + int size = pool.size(); + if (size < minIdle) + { + int sizeToBeAdded = minIdle - size; + for (int i = 0; i < sizeToBeAdded; i++) + { + pool.add(createObject()); + } + } else if (size > maxIdle) + { + int sizeToBeRemoved = size - maxIdle; + for (int i = 0; i < sizeToBeRemoved; i++) + { + pool.poll(); + } + } + } + }, validationInterval, validationInterval, TimeUnit.SECONDS); + } + + /** + * Gets the next free object from the pool. If the pool doesn't contain any objects, + * a new object will be created and given to the caller of this method back. + * + * @return T borrowed object + */ + public T borrowObject() + { + T object; + if ((object = pool.poll()) == null) + { + object = createObject(); + } + + return object; + } + + /** + * Returns object back to the pool. + * + * @param object object to be returned + */ + public void returnObject(T object) + { + if (object == null) + { + return; + } + + this.pool.offer(object); + } + + /** + * Shutdown this pool. + */ + public void shutdown() + { + if (executorService != null) + { + executorService.shutdown(); + } + } + + /** + * Creates a new object. + * + * @return T new object + */ + protected abstract T createObject(); + + private void initialize(final int minIdle) + { + pool = new ConcurrentLinkedQueue(); + + for (int i = 0; i < minIdle; i++) + { + pool.add(createObject()); + } + } +} \ No newline at end of file diff --git a/src/com/xrtb/exchanges/C1X.java b/src/com/xrtb/exchanges/C1X.java index af2f463d..c229ecc1 100644 --- a/src/com/xrtb/exchanges/C1X.java +++ b/src/com/xrtb/exchanges/C1X.java @@ -6,9 +6,9 @@ import com.fasterxml.jackson.databind.node.MissingNode; import com.fasterxml.jackson.databind.node.TextNode; +import com.xrtb.blocks.LookingGlass; import com.xrtb.pojo.BidRequest; import com.xrtb.tools.IsoTwo2Iso3; -import com.xrtb.tools.LookingGlass; /** * A class to handle C1X ad exchange diff --git a/src/com/xrtb/exchanges/adx/AdxBidRequest.java b/src/com/xrtb/exchanges/adx/AdxBidRequest.java index 5f39b93c..68a191dd 100644 --- a/src/com/xrtb/exchanges/adx/AdxBidRequest.java +++ b/src/com/xrtb/exchanges/adx/AdxBidRequest.java @@ -18,6 +18,7 @@ import com.google.protobuf.ByteString; import com.xrtb.bidder.Controller; import com.xrtb.bidder.RTBServer; +import com.xrtb.blocks.LookingGlass; import com.xrtb.common.Campaign; import com.xrtb.common.Configuration; import com.xrtb.common.Creative; @@ -37,7 +38,6 @@ import com.xrtb.pojo.BidResponse; import com.xrtb.pojo.Impression; import com.xrtb.pojo.Video; -import com.xrtb.tools.LookingGlass; interface Command { void runCommand(BidRequest br, RealtimeBidding.BidRequest x, ObjectNode root, Map db, String key); diff --git a/src/com/xrtb/exchanges/adx/AdxGeoCodes.java b/src/com/xrtb/exchanges/adx/AdxGeoCodes.java index b47fe138..535d13ed 100644 --- a/src/com/xrtb/exchanges/adx/AdxGeoCodes.java +++ b/src/com/xrtb/exchanges/adx/AdxGeoCodes.java @@ -5,7 +5,7 @@ import java.util.HashMap; import java.util.Map; -import com.xrtb.tools.LookingGlass; +import com.xrtb.blocks.LookingGlass; public class AdxGeoCodes extends LookingGlass { diff --git a/src/com/xrtb/exchanges/adx/MakeMasterGeoTable.java b/src/com/xrtb/exchanges/adx/MakeMasterGeoTable.java index 699524a8..7e39d79e 100644 --- a/src/com/xrtb/exchanges/adx/MakeMasterGeoTable.java +++ b/src/com/xrtb/exchanges/adx/MakeMasterGeoTable.java @@ -8,7 +8,7 @@ import java.util.HashMap; import java.util.Map; -import com.xrtb.tools.LookingGlass; +import com.xrtb.blocks.LookingGlass; public class MakeMasterGeoTable { diff --git a/src/com/xrtb/exchanges/google/GoogleBidResponse.java b/src/com/xrtb/exchanges/google/GoogleBidResponse.java index ead6f894..ecfb1c50 100644 --- a/src/com/xrtb/exchanges/google/GoogleBidResponse.java +++ b/src/com/xrtb/exchanges/google/GoogleBidResponse.java @@ -224,20 +224,23 @@ public GoogleBidResponse(GoogleBidRequest br, Impression imp, Campaign camp, Cre } else { adm = substitute(creat.unencodedAdm); } - this.forwardUrl = adm; + } else if (this.creat.isNative()) { if (br.usesEncodedAdm) { adm = substitute(this.creat.getEncodedNativeAdm(br)); } else { adm = substitute(this.creat.unencodedAdm); } - this.forwardUrl = adm; } else { adm = substitute(getTemplate()); } ////////////////// + this.forwardUrl = adm; + + // System.out.println(adm); + Bid.Builder bb = Bid.newBuilder(); bb.addAdomain(camp.adomain); @@ -278,6 +281,8 @@ public GoogleBidResponse(GoogleBidRequest br, Impression imp, Campaign camp, Cre internal = builder.build(); + // System.out.println(internal); + // add this to the log byte[] bytes = internal.toByteArray(); protobuf = new String(Base64.encodeBase64(bytes)); diff --git a/src/com/xrtb/jmq/Publisher.java b/src/com/xrtb/jmq/Publisher.java index 771a316a..41fde2ae 100644 --- a/src/com/xrtb/jmq/Publisher.java +++ b/src/com/xrtb/jmq/Publisher.java @@ -4,12 +4,18 @@ import org.zeromq.ZMQ.Context; import org.zeromq.ZMQ.Socket; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + public class Publisher { Socket publisher = null; Context context = JMQContext.getInstance(); boolean running = false; String topicName = null; + + ObjectMapper mapper = new ObjectMapper(); public static void main(String[] args) throws Exception { @@ -23,6 +29,9 @@ public static void main(String[] args) throws Exception { public Publisher(String binding, String topicName) throws Exception { + mapper.setSerializationInclusion(Include.NON_NULL); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + context = ZMQ.context(1); publisher = context.socket(ZMQ.PUB); publisher.bind(binding); @@ -38,7 +47,7 @@ public Publisher(String binding, String topicName) throws Exception { public void publish(Object message) { publisher.sendMore(topicName); - String msg = Tools.serialize(message); + String msg = Tools.serialize(mapper, message); if (msg != null) publisher.send(msg); else @@ -48,7 +57,7 @@ public void publish(Object message) { public void publishAsync(Object message) { Runnable u = () -> { publisher.sendMore(topicName); - String msg = Tools.serialize(message); + String msg = Tools.serialize(mapper,message); publisher.send(msg); }; Thread nthread = new Thread(u); diff --git a/src/com/xrtb/jmq/RTopic.java b/src/com/xrtb/jmq/RTopic.java index e77cc909..00949113 100644 --- a/src/com/xrtb/jmq/RTopic.java +++ b/src/com/xrtb/jmq/RTopic.java @@ -14,6 +14,10 @@ import org.zeromq.ZMQ.Context; import org.zeromq.ZMQ.Socket; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + public class RTopic implements EventIF { Socket publisher = null; @@ -22,9 +26,15 @@ public class RTopic implements EventIF { boolean running = false; String topicName = null; public Map m = new HashMap(); + ObjectMapper mapper = new ObjectMapper(); + public RTopic(String address) throws Exception { context = ZMQ.context(1); + + mapper.setSerializationInclusion(Include.NON_NULL); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + if (address.contains("&")) { String [] parts = address.split("&"); subscriber = new Subscriber(this,parts[0]); @@ -39,6 +49,8 @@ public RTopic(String address) throws Exception { * @throws Exception */ public RTopic(List addresses) throws Exception { + mapper.setSerializationInclusion(Include.NON_NULL); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); context = ZMQ.context(1); subscriber = new MSubscriber(this,addresses); @@ -108,7 +120,7 @@ public void addListener(MessageListener z) { @Override public void handleMessage(String id, String msg) { - Object [] x = Tools.deSerialize(msg); + Object [] x = Tools.deSerialize(mapper,msg); String name = (String)x[0]; Object o = m.get(name); if (o != null) { diff --git a/src/com/xrtb/jmq/Tools.java b/src/com/xrtb/jmq/Tools.java index 9be181b1..6572c463 100644 --- a/src/com/xrtb/jmq/Tools.java +++ b/src/com/xrtb/jmq/Tools.java @@ -11,13 +11,7 @@ public class Tools { - static final ObjectMapper mapper = new ObjectMapper(); - static { - mapper.setSerializationInclusion(Include.NON_NULL); - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - } - - public static String serialize(Object o) { + public static String serialize(ObjectMapper mapper, Object o) { String contents = null; StringBuilder sb = new StringBuilder(); try { @@ -38,7 +32,7 @@ public static String serialize(Object o) { return contents; } - public static Object[] deSerialize(String o) { + public static Object[] deSerialize(ObjectMapper mapper, String o) { Object obj = null; String name = null; if (o.charAt(0) == '{') { diff --git a/src/com/xrtb/tools/IsoTwo2Iso3.java b/src/com/xrtb/tools/IsoTwo2Iso3.java index e20a7289..f76bf16a 100644 --- a/src/com/xrtb/tools/IsoTwo2Iso3.java +++ b/src/com/xrtb/tools/IsoTwo2Iso3.java @@ -5,7 +5,7 @@ import java.util.HashMap; import java.util.Map; -import com.xrtb.tools.LookingGlass; +import com.xrtb.blocks.LookingGlass; /** * A class to create a map to look up 2 char ISO country codes and return 3 char country codes. diff --git a/src/test/java/TestBloom.java b/src/test/java/TestBloom.java new file mode 100644 index 00000000..032d6b38 --- /dev/null +++ b/src/test/java/TestBloom.java @@ -0,0 +1,63 @@ +package test.java; + +import static org.junit.Assert.*; + +import java.io.BufferedReader; +import java.io.FileReader; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.hash.BloomFilter; +import com.xrtb.bidder.Controller; +import com.xrtb.blocks.Bloom; +import com.xrtb.blocks.LookingGlass; +import com.xrtb.blocks.NavMap; + +/** + * A class for testing that the bid has the right parameters + * @author Ben M. Faul + * + */ +public class TestBloom { + static Controller c; + public static String test = ""; + + @BeforeClass + public static void testSetup() { + + } + + @AfterClass + public static void testCleanup() { + + } + + /** + * Test a valid bid response. + * @throws Exception on networking errors. + */ + @Test + public void testBloom() throws Exception { + + + new Bloom("$test","data/c1x_cookies.csv"); + BloomFilter b = (BloomFilter)LookingGlass.get("$test"); + assertNotNull(b); + + boolean p = b.mightContain("842AAB10FBA04247B3A9CE00C9172350"); + + BufferedReader br = new BufferedReader(new FileReader("data/c1x_cookies.csv")); + String line = null; + int nP = 0; + int k = 0; + while((line = br.readLine()) != null) { + p = b.mightContain(line); + if (p) + nP++; + k++; + } + assertTrue(k == nP); + } +} \ No newline at end of file diff --git a/src/test/java/TestCidr.java b/src/test/java/TestCidr.java index 4f957b36..59544bce 100644 --- a/src/test/java/TestCidr.java +++ b/src/test/java/TestCidr.java @@ -8,6 +8,7 @@ import org.junit.BeforeClass; import org.junit.Test; import com.xrtb.bidder.Controller; +import com.xrtb.blocks.LookingGlass; import com.xrtb.blocks.NavMap; /** @@ -38,29 +39,22 @@ public void testCidr() throws Exception { NavMap sr = new NavMap("CIDR", "data/METHBOT.txt", true); - - long x = NavMap.ipToLong("45.33.224.0"); - boolean p = NavMap.searchTable("CIDR", x); + boolean p = sr.contains("45.33.224.0"); assertTrue(p); - x = NavMap.ipToLong("45.33.239.255"); - p = NavMap.searchTable("CIDR", x); + p = sr.contains("45.33.239.255"); assertTrue(p); - x = NavMap.ipToLong("44.33.224.0"); - p = NavMap.searchTable("CIDR", x); + p = sr.contains("44.33.224.0"); assertFalse(p); - x = NavMap.ipToLong("165.52.0.0"); - p = NavMap.searchTable("CIDR", x); + p = sr.contains("165.52.0.0"); assertTrue(p); - x = NavMap.ipToLong("165.55.255.255"); - p = NavMap.searchTable("CIDR", x); + p = sr.contains("165.55.255.255"); assertTrue(p); - x = NavMap.ipToLong("166.55.255.255"); - p = NavMap.searchTable("CIDR", x); + p = sr.contains("166.55.255.255"); assertFalse(p); } diff --git a/src/test/java/TestDeals.java b/src/test/java/TestDeals.java index ecf5eb0c..529b79a5 100644 --- a/src/test/java/TestDeals.java +++ b/src/test/java/TestDeals.java @@ -8,6 +8,7 @@ import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; @@ -22,6 +23,8 @@ import com.xrtb.bidder.Controller; import com.xrtb.common.Configuration; +import com.xrtb.common.Deal; +import com.xrtb.common.Deals; import com.xrtb.common.HttpPostGet; import com.xrtb.jmq.MessageListener; import com.xrtb.jmq.RTopic; @@ -134,4 +137,60 @@ public void testPrivateResponse() throws Exception { int code = http.getResponseCode(); assertEquals(code,200); } + + /** + * Test the deal object + */ + @Test + public void testDealsObject() { + + Deals deals = new Deals(); + Deal a = new Deal("a",1); + Deal b = new Deal("b",2); + deals.add(a); + deals.add(b); + + List test = new ArrayList(); + test.add("c"); + Deal x = deals.findDealRandom(test); + assertNull(x); + + // Must be a + test.add("a"); + x = deals.findDealRandom(test); + assertTrue(x.id.equals("a")); + + test.add("b"); + + // Can be a or B + int isA = 0; + for (int i=0;i<10;i++) { + x = deals.findDealRandom(test); + if (x.id.equals("a")) + isA++; + } + assertTrue(isA != 10); + + + // Must be b + int isB = 0; + for (int i=0;i<10;i++) { + x = deals.findDealHighest(test); + if (x.id.equals("b")) + isB++; + } + assertTrue(isB == 10); + + Deal z = new Deal("z",2); + deals.add(z); + test.add("z"); + + // Can be b or z + isB = 0; + for (int i=0;i<10;i++) { + x = deals.findDealHighest(test); + if (x.id.equals("b")) + isB++; + } + } } diff --git a/src/test/java/TestNavMap.java b/src/test/java/TestNavMap.java deleted file mode 100644 index 399989ce..00000000 --- a/src/test/java/TestNavMap.java +++ /dev/null @@ -1,55 +0,0 @@ -package test.java; - -import static org.junit.Assert.*; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Paths; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import com.xrtb.bidder.Controller; -import com.xrtb.blocks.NavMap; -import com.xrtb.fraud.ForensiqClient; -import com.xrtb.fraud.FraudLog; -/** - * A class for testing that the bid has the right parameters - * @author Ben M. Faul - * - */ -public class TestNavMap { - static Controller c; - public static String test = ""; - - @BeforeClass - public static void testSetup() { - - } - - @AfterClass - public static void testCleanup() { - - } - - /** - * Test a valid bid response. - * @throws Exception on networking errors. - */ - @Test - public void testNavMap() throws Exception { - NavMap sr = new NavMap("CIDR", "data/TESTCIDR1.txt", false); - - long x = NavMap.ipToLong("192.168.0.12"); - boolean p = NavMap.searchTable("CIDR", x); - assertTrue(p); - - x = NavMap.ipToLong("191.168.0.12"); - p = NavMap.searchTable("CIDR", x); - assertFalse(p); - - x = NavMap.ipToLong("223.255.231.255"); - p = NavMap.searchTable("CIDR", x); - assertFalse(p); - } -} \ No newline at end of file