Update benchmark

.travis.yml

@@ -2,7 +2,7 @@ language: python
 
 env:
   global:
-    PIP_PREFER_BINARY=true
+    PIP_PREFER_BINARY=true # https://github.com/numba/llvmlite/issues/471
 
 python:
   - "2.7"

docs/benchmark.rst

@@ -39,7 +39,7 @@ P@N Performance
 Execution Time
 --------------
 
-.. csv-table:: Time Complexity in Seconds (average of 10 independent trials)
+.. csv-table:: Time Elapsed in Seconds (average of 10 independent trials)
    :file: tables/time.csv
    :header-rows: 1
 

docs/tables/prc.csv

@@ -1,17 +1,18 @@
-Data,#Samples,# Dimensions,Outlier Perc,ABOD,CBLOF,FB,HBOS,IForest,KNN,LOF,MCD,OCSVM,PCA,LSCP
-arrhythmia,452,274,14.6018,0.3808,0.4471,0.415,0.5111,0.5125,0.4464,0.4334,0.3995,0.4614,0.4613,0.4234
-cardio,1831,21,9.6122,0.2374,0.5747,0.1673,0.4476,0.509,0.3323,0.1541,0.4479,0.5011,0.609,0.176
-glass,214,9,4.2056,0.1702,0.0726,0.1059,0,0.0726,0.0726,0.1476,0,0.1726,0.0726,0.1226
-ionosphere,351,33,35.8974,0.8442,0.5815,0.696,0.3295,0.6477,0.8602,0.7063,0.881,0.7,0.5729,0.7184
-letter,1600,32,6.25,0.3801,0.0554,0.377,0.0715,0.092,0.3312,0.3641,0.1904,0.151,0.0875,0.3165
-lympho,148,18,4.0541,0.4483,0.7517,0.7517,0.8467,0.7517,0.7517,0.7517,0.435,0.7517,0.7517,0.7517
-mnist,7603,100,9.2069,0.3555,0.4271,0.3317,0.1188,0.2897,0.4204,0.3343,0.2485,0.3962,0.3846,0.3502
-musk,3062,166,3.1679,0.0507,0.8481,0.2,0.9783,0.9667,0.2733,0.1695,0.977,1,0.9799,0.1358
-optdigits,5216,64,2.8758,0.006,0,0.0248,0.2194,0.0217,0,0.0234,0,0,0,0.022
-pendigits,6870,16,2.2707,0.0812,0.2801,0.0615,0.2979,0.3394,0.0984,0.0653,0.0881,0.3287,0.3187,0.0613
-pima,768,8,34.8958,0.5193,0.5558,0.4476,0.5424,0.5031,0.5413,0.4555,0.4953,0.4704,0.4943,0.4754
-satellite,6435,36,31.6395,0.3902,0.3951,0.3917,0.569,0.5694,0.4994,0.3893,0.6848,0.5346,0.4784,0.4071
-satimage-2,5803,36,1.2235,0.213,0.7846,0.0666,0.6939,0.853,0.3809,0.0555,0.6481,0.9356,0.8041,0.0508
-vertebral,240,6,12.5,0.0601,0.0183,0.0496,0.0071,0.031,0.0238,0.0506,0.0071,0.0238,0.0226,0.0506
-vowels,1456,12,3.4341,0.571,0.0731,0.323,0.1297,0.1802,0.5093,0.3551,0.1715,0.2791,0.1364,0.335
-wbc,378,30,5.5556,0.306,0.4985,0.5188,0.5817,0.5132,0.4952,0.5188,0.4568,0.5125,0.4767,0.5188
+Data,#Samples,# Dimensions,Outlier Perc,ABOD,CBLOF,FB,HBOS,IForest,KNN,LOF,MCD,OCSVM,PCA
+arrhythmia,452,274,14.6018,0.3808,0.4539,0.4230,0.5111,0.4961,0.4464,0.4334,0.3995,0.4614,0.4613
+cardio,1831,21,9.6122,0.2374,0.5876,0.1690,0.4476,0.5041,0.3323,0.1541,0.4317,0.5011,0.6090
+glass,214,9,4.2056,0.1702,0.0726,0.1476,0.0000,0.0726,0.0726,0.1476,0.0000,0.1726,0.0726
+ionosphere,351,33,35.8974,0.8442,0.6088,0.7056,0.3295,0.6369,0.8602,0.7063,0.8806,0.7000,0.5729
+letter,1600,32,6.2500,0.3801,0.0749,0.3642,0.0715,0.1003,0.3312,0.3641,0.1933,0.1510,0.0875
+lympho,148,18,4.0541,0.4483,0.7517,0.7517,0.8467,0.9267,0.7517,0.7517,0.5183,0.7517,0.7517
+mnist,7603,100,9.2069,0.3555,0.3348,0.3299,0.1188,0.3135,0.4204,0.3343,0.3462,0.3962,0.3846
+musk,3062,166,3.1679,0.0507,0.7766,0.2230,0.9783,0.9680,0.2733,0.1695,0.9742,1.0000,0.9799
+optdigits,5216,64,2.8758,0.0060,0.0000,0.0244,0.2194,0.0301,0.0000,0.0234,0.0000,0.0000,0.0000
+pendigits,6870,16,2.2707,0.0812,0.2768,0.0658,0.2979,0.3422,0.0984,0.0653,0.0893,0.3287,0.3187
+pima,768,8,34.8958,0.5193,0.5413,0.4480,0.5424,0.5111,0.5413,0.4555,0.4962,0.4704,0.4943
+satellite,6435,36,31.6395,0.3902,0.4152,0.3902,0.5690,0.5676,0.4994,0.3893,0.6845,0.5346,0.4784
+satimage-2,5803,36,1.2235,0.2130,0.8846,0.0555,0.6939,0.8754,0.3809,0.0555,0.6481,0.9356,0.8041
+shuttle,49097,9,7.1511,0.1977,0.2943,0.0695,0.9551,0.9546,0.2184,0.1424,0.7506,0.9542,0.9501
+vertebral,240,6,12.5000,0.0601,0.0000,0.0644,0.0071,0.0343,0.0238,0.0506,0.0071,0.0238,0.0226
+vowels,1456,12,3.4341,0.5710,0.0831,0.3224,0.1297,0.1875,0.5093,0.3551,0.2186,0.2791,0.1364
+wbc,378,30,5.5556,0.3060,0.5055,0.5188,0.5817,0.5088,0.4952,0.5188,0.4577,0.5125,0.4767

docs/tables/roc.csv

@@ -1,17 +1,18 @@
-Data,#Samples,# Dimensions,Outlier Perc,ABOD,CBLOF,FB,HBOS,IForest,KNN,LOF,MCD,OCSVM,PCA,LSCP
-arrhythmia,452,274,14.6018,0.7688,0.7854,0.7768,0.8219,0.811,0.7861,0.7787,0.779,0.7812,0.7815,0.7825
-cardio,1831,21,9.6122,0.5692,0.9269,0.6034,0.8351,0.9257,0.7236,0.5736,0.8267,0.9348,0.9504,0.7078
-glass,214,9,4.2056,0.7951,0.7953,0.8379,0.7389,0.7258,0.8508,0.8644,0.7877,0.6324,0.6747,0.8389
-ionosphere,351,33,35.8974,0.9248,0.8122,0.8677,0.5614,0.8487,0.9267,0.8753,0.9553,0.8419,0.7962,0.882
-letter,1600,32,6.25,0.8783,0.5364,0.8654,0.5927,0.6209,0.8766,0.8594,0.8083,0.6118,0.5283,0.8363
-lympho,148,18,4.0541,0.911,0.9712,0.9768,0.9957,0.991,0.9745,0.9771,0.9159,0.9759,0.9847,0.9771
-mnist,7603,100,9.2069,0.7815,0.8757,0.7152,0.5742,0.8003,0.8481,0.7161,0.8461,0.8529,0.8527,0.7652
-musk,3062,166,3.1679,0.1844,0.9911,0.5279,1,0.9995,0.7986,0.5287,0.9999,1,1,0.5029
-optdigits,5216,64,2.8758,0.4667,0.5081,0.4492,0.8732,0.7097,0.3708,0.45,0.3824,0.4997,0.5086,0.419
-pendigits,6870,16,2.2707,0.6878,0.9425,0.4699,0.9238,0.9434,0.7486,0.4698,0.8341,0.9303,0.9352,0.5036
-pima,768,8,34.8958,0.6794,0.7285,0.6271,0.7,0.6731,0.7078,0.6271,0.6753,0.6215,0.6481,0.6501
-satellite,6435,36,31.6395,0.5714,0.619,0.5567,0.7581,0.7075,0.6836,0.5573,0.8031,0.6622,0.5988,0.5722
-satimage-2,5803,36,1.2235,0.819,0.9827,0.4586,0.9804,0.9951,0.9536,0.4577,0.996,0.9978,0.9822,0.6852
-vertebral,240,6,12.5,0.4262,0.3964,0.4101,0.3263,0.3916,0.3817,0.4081,0.3971,0.4431,0.4027,0.3806
-vowels,1456,12,3.4341,0.9606,0.5993,0.9388,0.6727,0.7421,0.968,0.941,0.7912,0.7802,0.6027,0.944
-wbc,378,30,5.5556,0.9047,0.9264,0.9335,0.9516,0.9247,0.9366,0.9349,0.9169,0.9319,0.9159,0.9315
+Data,#Samples,# Dimensions,Outlier Perc,ABOD,CBLOF,FB,HBOS,IForest,KNN,LOF,MCD,OCSVM,PCA
+arrhythmia,452,274,14.6018,0.7688,0.7835,0.7781,0.8219,0.8005,0.7861,0.7787,0.7790,0.7812,0.7815
+cardio,1831,21,9.6122,0.5692,0.9276,0.5867,0.8351,0.9213,0.7236,0.5736,0.8135,0.9348,0.9504
+glass,214,9,4.2056,0.7951,0.8504,0.8726,0.7389,0.7569,0.8508,0.8644,0.7901,0.6324,0.6747
+ionosphere,351,33,35.8974,0.9248,0.8134,0.8730,0.5614,0.8499,0.9267,0.8753,0.9557,0.8419,0.7962
+letter,1600,32,6.2500,0.8783,0.5070,0.8660,0.5927,0.6420,0.8766,0.8594,0.8074,0.6118,0.5283
+lympho,148,18,4.0541,0.9110,0.9728,0.9753,0.9957,0.9941,0.9745,0.9771,0.9000,0.9759,0.9847
+mnist,7603,100,9.2069,0.7815,0.8009,0.7205,0.5742,0.8159,0.8481,0.7161,0.8666,0.8529,0.8527
+musk,3062,166,3.1679,0.1844,0.9879,0.5263,1.0000,0.9999,0.7986,0.5287,0.9998,1.0000,1.0000
+optdigits,5216,64,2.8758,0.4667,0.5089,0.4434,0.8732,0.7253,0.3708,0.4500,0.3979,0.4997,0.5086
+pendigits,6870,16,2.2707,0.6878,0.9486,0.4595,0.9238,0.9435,0.7486,0.4698,0.8344,0.9303,0.9352
+pima,768,8,34.8958,0.6794,0.7348,0.6235,0.7000,0.6806,0.7078,0.6271,0.6753,0.6215,0.6481
+satellite,6435,36,31.6395,0.5714,0.6693,0.5572,0.7581,0.7022,0.6836,0.5573,0.8030,0.6622,0.5988
+satimage-2,5803,36,1.2235,0.8190,0.9917,0.4570,0.9804,0.9947,0.9536,0.4577,0.9959,0.9978,0.9822
+shuttle,49097,9,7.1511,0.6234,0.6272,0.4724,0.9855,0.9971,0.6537,0.5264,0.9903,0.9917,0.9898
+vertebral,240,6,12.5000,0.4262,0.3486,0.4166,0.3263,0.3905,0.3817,0.4081,0.3906,0.4431,0.4027
+vowels,1456,12,3.4341,0.9606,0.5856,0.9425,0.6727,0.7585,0.9680,0.9410,0.8076,0.7802,0.6027
+wbc,378,30,5.5556,0.9047,0.9227,0.9325,0.9516,0.9310,0.9366,0.9349,0.9210,0.9319,0.9159

docs/tables/time.csv

@@ -1,17 +1,18 @@
-Data,#Samples,# Dimensions,Outlier Perc,ABOD,CBLOF,FB,HBOS,IForest,KNN,LOF,MCD,OCSVM,PCA,LSCP
-arrhythmia,452,274,14.6018,0.374,0.1409,0.5566,0.1469,0.2572,0.0976,0.0723,0.6016,0.0476,0.0596,1.8041
-cardio,1831,21,9.6122,0.3637,0.0217,0.8016,0.0056,0.2867,0.2226,0.105,0.5143,0.0914,0.0038,3.3281
-glass,214,9,4.2056,0.0352,0.0128,0.0278,0.0016,0.1567,0.0127,0.0022,0.0383,0.0015,0.001,0.2012
-ionosphere,351,33,35.8974,0.0587,0.0169,0.066,0.0078,0.1849,0.0228,0.006,0.0578,0.0051,0.0022,0.3703
-letter,1600,32,6.25,0.3319,0.0221,0.7389,0.008,0.2572,0.1637,0.0924,0.9713,0.0873,0.0045,3.1145
-lympho,148,18,4.0541,0.0284,0.0171,0.027,0.0043,0.1667,0.0087,0.0021,0.0338,0.0012,0.001,0.1388
-mnist,7603,100,9.2069,7.0412,0.0475,45.7463,0.0478,1.9339,6.7763,6.4203,2.5654,4.7525,0.1513,146.2224
-musk,3062,166,3.1679,2.0537,0.0455,11.7174,0.0493,0.9907,1.7191,1.5493,9.68,1.212,0.1475,38.6129
-optdigits,5216,64,2.8758,2.1879,0.0318,11.8058,0.0255,0.6474,1.7063,1.4831,0.9539,1.3605,0.0445,38.288
-pendigits,6870,16,2.2707,1.2799,0.0326,3.6209,0.008,0.5069,0.6721,0.536,1.566,0.8901,0.0054,15.3573
-pima,768,8,34.8958,0.1135,0.0153,0.0837,0.0018,0.1811,0.0443,0.0096,0.0502,0.0104,0.001,0.6555
-satellite,6435,36,31.6395,1.6487,0.0333,7.0337,0.0163,0.6374,1.0924,0.9317,1.8058,1.288,0.0247,23.2239
-satimage-2,5803,36,1.2235,1.6925,0.0406,6.12,0.0174,0.6441,1.0753,0.8087,1.9241,1.1575,0.0169,23.5951
-vertebral,240,6,12.5,0.0448,0.0157,0.0324,0.0017,0.1779,0.0167,0.003,0.0496,0.0018,0.0011,0.2316
-vowels,1456,12,3.4341,0.268,0.0191,0.2979,0.0037,0.258,0.1093,0.0364,0.7747,0.0424,0.002,1.8441
-wbc,378,30,5.5556,0.0695,0.0216,0.0806,0.0079,0.2084,0.0275,0.0074,0.0639,0.0059,0.0022,0.4293
+Data,#Samples,# Dimensions,Outlier Perc,ABOD,CBLOF,FB,HBOS,IForest,KNN,LOF,MCD,OCSVM,PCA
+arrhythmia,452,274,14.6018,0.3667,0.2123,0.5651,0.1383,0.2669,0.1075,0.0743,1.4165,0.0473,0.0596
+cardio,1831,21,9.6122,0.3824,0.1255,0.7741,0.0053,0.2672,0.2249,0.0993,0.5418,0.0883,0.0035
+glass,214,9,4.2056,0.0352,0.0359,0.0317,0.0022,0.1724,0.0173,0.0025,0.0325,0.0010,0.0011
+ionosphere,351,33,35.8974,0.0645,0.0459,0.0728,0.0082,0.1864,0.0302,0.0070,0.0718,0.0048,0.0018
+letter,1600,32,6.2500,0.3435,0.1014,0.7361,0.0080,0.2617,0.1882,0.0935,1.1942,0.0888,0.0041
+lympho,148,18,4.0541,0.0277,0.0353,0.0266,0.0037,0.1712,0.0111,0.0021,0.0327,0.0014,0.0012
+mnist,7603,100,9.2069,7.4192,1.1339,48.2750,0.0480,1.9314,7.3431,6.7901,4.7448,5.0203,0.1569
+musk,3062,166,3.1679,2.3860,0.4134,13.8610,0.0587,1.2736,2.2057,1.9835,25.5501,1.3774,0.1637
+optdigits,5216,64,2.8758,2.7279,0.4977,14.2399,0.0303,0.7783,2.1205,1.7799,1.8599,1.5618,0.0519
+pendigits,6870,16,2.2707,1.4339,0.2847,3.8185,0.0090,0.5879,0.8659,0.5936,2.2209,0.9666,0.0062
+pima,768,8,34.8958,0.1357,0.0698,0.0908,0.0019,0.1923,0.0590,0.0102,0.0474,0.0087,0.0013
+satellite,6435,36,31.6395,1.7970,0.4269,7.5566,0.0161,0.6449,1.2578,0.9868,2.6916,1.3697,0.0245
+satimage-2,5803,36,1.2235,1.5209,0.3705,5.6561,0.0148,0.5529,1.0587,0.7525,2.3935,1.1114,0.0151
+shuttle,49097,9,7.1511,14.3611,1.2524,59.2131,0.0953,3.3906,9.4958,11.1500,12.1449,44.6830,0.0378
+vertebral,240,6,12.5000,0.0529,0.0444,0.0339,0.0014,0.1786,0.0161,0.0025,0.0446,0.0015,0.0010
+vowels,1456,12,3.4341,0.3380,0.0889,0.3125,0.0044,0.2751,0.1125,0.0367,0.9745,0.0469,0.0023
+wbc,378,30,5.5556,0.1014,0.0691,0.0771,0.0063,0.2030,0.0287,0.0078,0.0864,0.0062,0.0035