Skip to content

Commit

Permalink
removed results & changed random state
Browse files Browse the repository at this point in the history
  • Loading branch information
Ubuntu committed Apr 27, 2016
1 parent c42833f commit 1c8a4dd
Show file tree
Hide file tree
Showing 3 changed files with 2 additions and 163 deletions.
5 changes: 2 additions & 3 deletions ann_benchmarks/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,11 +354,10 @@ def get_dataset(which='glove', limit=-1):
X = numpy.vstack(X)
import sklearn.cross_validation

# Here Eric is most welcome to use any other random_state
# Last time, Leo was testing using random_state==1
# Here Erik is most welcome to use any other random_state
# However, it is best to use a new random seed for each major re-evaluation,
# so that we test on a trully bind data.
X_train, X_test = sklearn.cross_validation.train_test_split(X, test_size=10000, random_state=2016)
X_train, X_test = sklearn.cross_validation.train_test_split(X, test_size=10000, random_state=2)
print(X_train.shape, X_test.shape)
return X_train, X_test

Expand Down
115 changes: 0 additions & 115 deletions results/glove.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ rpforest RPForest(leaf_size=40, n_trees=5) 36.7686510086 9.87219810486e-05 0.021
bruteforce1(nmslib) Nmslib(method_name=seq_search, method_param=['copyMem=1']) 3.86724305153 0.0862678928375 1.0
rpforest RPForest(leaf_size=10, n_trees=10) 80.1546099186 9.33468341827e-05 0.0219
rpforest RPForest(leaf_size=20, n_trees=400) 2957.39264107 0.00393772816658 0.5019
SW-graph(nmslib) Nmslib(method_name=small_world_rand, method_param=['NN=5', 'initIndexAttempts=5', 'initSearchAttempts=2']) 295.622782946 0.000802574157715 0.4326
panns PANNS(n_trees=10, n_cand=10) 639.457413197 0.0040747730732 0.0271
rpforest RPForest(leaf_size=100, n_trees=40) 267.536391973 0.00112784314156 0.196
ball BallTree(leaf_size=200) 12.9479379654 0.343198341846 1.0
Expand All @@ -27,7 +26,6 @@ rpforest RPForest(leaf_size=20, n_trees=100) 722.858472109 0.00099794793129 0.21
BallTree(nmslib) Nmslib(method_name=vptree, method_param=['tuneK=10', 'desiredRecall=0.85']) 204.200890064 0.0310087051392 0.795
bruteforce BruteForce() 0.145766973495 1.40714517403 1.0
nearpy NearPy(n_bits=12, hash_counts=20) 354.039633989 0.109765942097 0.3515
SW-graph(nmslib) Nmslib(method_name=small_world_rand, method_param=['NN=20', 'initIndexAttempts=4', 'initSearchAttempts=2']) 835.058357954 0.00272746491432 0.8145
nearpy NearPy(n_bits=12, hash_counts=40) 668.519620895 0.200587756872 0.5188
nearpy NearPy(n_bits=16, hash_counts=10) 201.011022091 0.00371514201164 0.068
rpforest RPForest(leaf_size=5, n_trees=40) 335.109559059 0.000251724004745 0.052
Expand All @@ -43,13 +41,11 @@ BallTree(nmslib) Nmslib(method_name=vptree, method_param=['tuneK=10', 'desiredRe
nearpy NearPy(n_bits=16, hash_counts=40) 713.709898949 0.0192187888622 0.2212
rpforest RPForest(leaf_size=10, n_trees=40) 318.112294912 0.000313908100128 0.0721
flann FLANN(target_precision=0.800000) 1968.18843913 0.0028135330677 0.7513
SW-graph(nmslib) Nmslib(method_name=small_world_rand, method_param=['NN=20', 'initIndexAttempts=4', 'initSearchAttempts=32']) 845.318381786 0.0321285920143 0.9757
rpforest RPForest(leaf_size=3, n_trees=5) 44.7247550488 4.44700717926e-05 0.0044
BallTree(nmslib) Nmslib(method_name=vptree, method_param=['tuneK=10', 'desiredRecall=0.90']) 200.292738914 0.0342704021931 0.8466
flann FLANN(target_precision=0.500000) 647.046700954 0.00055372595787 0.3732
rpforest RPForest(leaf_size=5, n_trees=200) 1641.66669202 0.0011424779892 0.1821
nearpy NearPy(n_bits=12, hash_counts=10) 192.767081976 0.0415629379749 0.2024
SW-graph(nmslib) Nmslib(method_name=small_world_rand, method_param=['NN=17', 'initIndexAttempts=4', 'initSearchAttempts=2']) 715.145844936 0.00233191204071 0.7822
ball BallTree(leaf_size=100) 13.6964509487 0.343614621162 1.0
rpforest RPForest(leaf_size=400, n_trees=5) 30.6615920067 0.000497461080551 0.066
rpforest RPForest(leaf_size=10, n_trees=3) 24.2783131599 4.05759811401e-05 0.0071
Expand All @@ -63,7 +59,6 @@ rpforest RPForest(leaf_size=200, n_trees=5) 33.369093895 0.000270153999329 0.047
nearpy NearPy(n_bits=16, hash_counts=5) 117.37446785 0.00193560910225 0.0357
rpforest RPForest(leaf_size=20, n_trees=5) 38.8645989895 6.99541568756e-05 0.015
rpforest RPForest(leaf_size=40, n_trees=10) 72.9464428425 0.000179342031479 0.0427
SW-graph(nmslib) Nmslib(method_name=small_world_rand, method_param=['NN=20', 'initIndexAttempts=4', 'initSearchAttempts=48']) 847.949594021 0.0449159810543 0.9841
kd KDTree(leaf_size=10) 13.2552900314 0.509452296019 1.0
BallTree(nmslib) Nmslib(method_name=vptree, method_param=['tuneK=10', 'desiredRecall=0.1']) 76.2676689625 0.000629935026169 0.1082
rpforest RPForest(leaf_size=400, n_trees=20) 124.217647076 0.00197582411766 0.208
Expand All @@ -77,13 +72,11 @@ rpforest RPForest(leaf_size=40, n_trees=200) 1384.77476406 0.00313244986534 0.43
ball BallTree(leaf_size=40) 13.3588030338 0.342107228041 1.0
nearpy NearPy(n_bits=10, hash_counts=5) 107.293065071 0.0534425308704 0.1855
rpforest RPForest(leaf_size=10, n_trees=400) 3014.490309 0.00272457408905 0.397
SW-graph(nmslib) Nmslib(method_name=small_world_rand, method_param=['NN=20', 'initIndexAttempts=4', 'initSearchAttempts=16']) 812.161186934 0.0165439920425 0.9525
rpforest RPForest(leaf_size=3, n_trees=3) 26.209913969 2.80690193176e-05 0.0028
rpforest RPForest(leaf_size=40, n_trees=400) 2728.73781586 0.00623660111427 0.6207
nearpy NearPy(n_bits=14, hash_counts=20) 351.91968894 0.0312160151005 0.2124
lshf LSHF(n_est=5, n_cand=10) 20.3374519348 0.0810469880104 0.0103
BallTree(nmslib) Nmslib(method_name=vptree, method_param=['tuneK=10', 'desiredRecall=0.6']) 126.314934015 0.0118184540272 0.5728
SW-graph(nmslib) Nmslib(method_name=small_world_rand, method_param=['NN=20', 'initIndexAttempts=4', 'initSearchAttempts=4']) 830.099664927 0.00528782296181 0.8764
rpforest RPForest(leaf_size=5, n_trees=20) 165.413131952 0.000136290073395 0.028
ball BallTree(leaf_size=20) 14.098500967 0.351178482771 1.0
rpforest RPForest(leaf_size=200, n_trees=200) 1280.52543402 0.0103447229862 0.657
Expand All @@ -92,7 +85,6 @@ BallTree(nmslib) Nmslib(method_name=vptree, method_param=['tuneK=10', 'desiredRe
lshf LSHF(n_est=20, n_cand=100) 77.5567779541 0.507922759056 0.0951
flann FLANN(target_precision=0.200000) 494.667067051 0.000197943210602 0.1122
rpforest RPForest(leaf_size=10, n_trees=200) 1562.85028815 0.00138846802711 0.2557
SW-graph(nmslib) Nmslib(method_name=small_world_rand, method_param=['NN=3', 'initIndexAttempts=5', 'initSearchAttempts=2']) 211.680727959 0.00056936788559 0.2413
rpforest RPForest(leaf_size=100, n_trees=200) 1353.51925302 0.00567569804192 0.5496
rpforest RPForest(leaf_size=400, n_trees=10) 61.2972989082 0.000981947183609 0.12
kd KDTree(leaf_size=100) 11.8067228794 0.358493808031 1.0
Expand All @@ -101,8 +93,6 @@ rpforest RPForest(leaf_size=20, n_trees=3) 23.4474790096 4.8318862915e-05 0.0094
panns PANNS(n_trees=10, n_cand=100) 645.715856075 0.00406477403641 0.0271
rpforest RPForest(leaf_size=20, n_trees=20) 149.556242943 0.000220011949539 0.0568
rpforest RPForest(leaf_size=10, n_trees=5) 39.9351899624 5.64558506012e-05 0.0121
SW-graph(nmslib) Nmslib(method_name=small_world_rand, method_param=['NN=11', 'initIndexAttempts=5', 'initSearchAttempts=2']) 562.084706068 0.00160191702843 0.6911
SW-graph(nmslib) Nmslib(method_name=small_world_rand, method_param=['NN=14', 'initIndexAttempts=4', 'initSearchAttempts=2']) 573.340883017 0.00194364690781 0.7348
rpforest RPForest(leaf_size=200, n_trees=3) 20.1447839737 0.000169105052948 0.0297
flann FLANN(target_precision=0.950000) 7457.9571681 0.015343072176 0.9026
rpforest RPForest(leaf_size=100, n_trees=20) 139.668900013 0.000576925992966 0.1134
Expand All @@ -114,7 +104,6 @@ rpforest RPForest(leaf_size=100, n_trees=3) 21.6384329796 0.000101777791977 0.02
ball BallTree(leaf_size=1000) 12.5644199848 0.340618407011 1.0
kd KDTree(leaf_size=40) 12.7546768188 0.381024254084 1.0
panns PANNS(n_trees=10, n_cand=50) 657.446552992 0.00413969516754 0.0271
SW-graph(nmslib) Nmslib(method_name=small_world_rand, method_param=['NN=20', 'initIndexAttempts=4', 'initSearchAttempts=8']) 846.154056072 0.010144867897 0.9222
lshf LSHF(n_est=5, n_cand=20) 20.3808999062 0.0921115300655 0.0131
rpforest RPForest(leaf_size=200, n_trees=10) 65.8298540115 0.000524737119675 0.0852
rpforest RPForest(leaf_size=40, n_trees=40) 290.310339928 0.000657279968262 0.1462
Expand All @@ -128,7 +117,6 @@ rpforest RPForest(leaf_size=400, n_trees=200) 1211.91789412 0.0188235809803 0.78
bruteforce0(nmslib) Nmslib(method_name=seq_search, method_param=['copyMem=0']) 3.54172301292 0.163267335892 1.0
lshf LSHF(n_est=10, n_cand=20) 39.4296400547 0.200653875113 0.0265
flann FLANN(target_precision=0.980000) 3720.07483411 0.0980142829418 1.0
SW-graph(nmslib) Nmslib(method_name=small_world_rand, method_param=['NN=8', 'initIndexAttempts=5', 'initSearchAttempts=2']) 418.825798035 0.00116217398643 0.6006
rpforest RPForest(leaf_size=400, n_trees=3) 18.6479139328 0.000302742004395 0.0414
flann FLANN(target_precision=0.990000) 4266.82339311 0.0975360808372 1.0
rpforest RPForest(leaf_size=100, n_trees=400) 2609.17380309 0.010918585062 0.7321
Expand All @@ -140,106 +128,3 @@ ball BallTree(leaf_size=400) 12.0417780876 0.335959160089 1.0
rpforest RPForest(leaf_size=100, n_trees=100) 655.93829608 0.00277264118195 0.3749
flann FLANN(target_precision=0.700000) 1288.26174283 0.00151859402657 0.5877
rpforest RPForest(leaf_size=3, n_trees=100) 847.26354599 0.000533706903458 0.075
annoy Annoy(n_trees=400, search_k=400) 1307.56087017 0.000457568883896 0.3375
falconn FALCONN(K=16, L=46, T=46) 15.0794169903 0.000335726976395 0.3536
annoy Annoy(n_trees=200, search_k=200000) 669.735510111 0.0329408318996 0.9894
kgraph KGraph(P=500) 3346.25359106 0.389367910862 0.8971
falconn FALCONN(K=16, L=51, T=51) 16.9528150558 0.000360759973526 0.3762
falconn FALCONN(K=16, L=966, T=966) 284.075728893 0.00567807888985 0.9739
falconn FALCONN(K=16, L=170, T=170) 52.6578638554 0.0011066930294 0.6862
annoy Annoy(n_trees=200, search_k=40000) 634.109462023 0.00784254717827 0.9221
annoy Annoy(n_trees=400, search_k=200000) 1303.5594759 0.0315657000542 0.9912
annoy Annoy(n_trees=200, search_k=1000) 679.378338099 0.000471024036407 0.4462
kgraph KGraph(P=50) 3326.22845912 0.382956544161 0.6205
annoy Annoy(n_trees=200, search_k=400) 636.962444782 0.000298217058182 0.3088
falconn FALCONN(K=16, L=24, T=24) 7.92915606499 0.000184276103973 0.2323
falconn FALCONN(K=16, L=7, T=7) 2.95339488983 6.87639713287e-05 0.0938
falconn FALCONN(K=16, L=544, T=544) 162.198051929 0.00334365391731 0.917
annoy Annoy(n_trees=400, search_k=10000) 1300.88312411 0.00251197218895 0.8114
falconn FALCONN(K=16, L=30, T=30) 9.91768407822 0.000226016998291 0.2652
kgraph KGraph(P=1000) 3332.57121491 0.397260385036 0.9316
kgraph KGraph(P=4000) 3369.09592104 0.459113996029 0.9676
annoy Annoy(n_trees=100, search_k=20000) 364.613081932 0.00414688420296 0.8397
annoy Annoy(n_trees=100, search_k=10000) 366.026244164 0.00220604300499 0.7615
kgraph KGraph(P=2000) 3362.1876688 0.41406251812 0.953
annoy Annoy(n_trees=400, search_k=20000) 1414.4567821 0.00508588981628 0.8804
falconn FALCONN(K=16, L=878, T=878) 267.917775869 0.00571572303772 0.9679
annoy Annoy(n_trees=200, search_k=100000) 685.356554031 0.0191763088703 0.9729
falconn FALCONN(K=16, L=449, T=449) 133.880192041 0.00295110988617 0.8877
falconn FALCONN(K=16, L=9, T=9) 4.53184914589 8.56499671936e-05 0.1119
kgraph KGraph(P=20) 3361.23490906 0.383054105043 0.5279
falconn FALCONN(K=16, L=63, T=63) 19.3421931267 0.000471467971802 0.4272
annoy Annoy(n_trees=200, search_k=400000) 682.961287975 0.0591782901287 0.9967
falconn FALCONN(K=16, L=37, T=37) 12.970664978 0.000287461996078 0.3047
falconn FALCONN(K=16, L=408, T=408) 121.61743784 0.00269181108475 0.8727
falconn FALCONN(K=16, L=77, T=77) 24.4565889835 0.000550612926483 0.4766
falconn FALCONN(K=16, L=1, T=1) 2.10531997681 2.365899086e-05 0.016
annoy Annoy(n_trees=400, search_k=400000) 1416.90651703 0.0585625748634 0.9979
kgraph KGraph(P=10000) 3370.68318081 0.684432445049 0.98
falconn FALCONN(K=16, L=370, T=370) 112.198954821 0.00243748211861 0.8534
falconn FALCONN(K=16, L=4, T=4) 2.96774291992 4.67638969421e-05 0.0557
annoy Annoy(n_trees=100, search_k=400) 371.620517969 0.000240019083023 0.2928
falconn FALCONN(K=16, L=57, T=57) 18.4620769024 0.000421772956848 0.4027
falconn FALCONN(K=16, L=1416, T=1416) 422.325224876 0.0087351360321 0.99
falconn FALCONN(K=16, L=94, T=94) 29.6914720535 0.000665446996689 0.5285
falconn FALCONN(K=16, L=305, T=305) 92.2625789642 0.00203111600876 0.8165
falconn FALCONN(K=16, L=659, T=659) 197.697454929 0.00437748718262 0.9435
falconn FALCONN(K=16, L=251, T=251) 76.7947700024 0.00171723818779 0.7779
annoy Annoy(n_trees=100, search_k=400000) 368.788991928 0.0575219018459 0.9975
falconn FALCONN(K=16, L=8, T=8) 3.09926104546 7.86108970642e-05 0.1036
falconn FALCONN(K=16, L=21, T=21) 7.90961003304 0.000174631118774 0.2118
falconn FALCONN(K=16, L=336, T=336) 100.317620993 0.0022447719574 0.8366
annoy Annoy(n_trees=100, search_k=2000) 368.243417978 0.000634358882904 0.5235
falconn FALCONN(K=16, L=27, T=27) 9.57541894913 0.000216135978699 0.2483
falconn FALCONN(K=16, L=11, T=11) 4.54193997383 0.00010404086113 0.1302
annoy Annoy(n_trees=100, search_k=100000) 367.429883957 0.0177980670929 0.9656
kgraph KGraph(P=100) 3390.30672193 0.383763358116 0.7435
annoy Annoy(n_trees=200, search_k=20000) 641.810635805 0.00433827900887 0.8656
falconn FALCONN(K=16, L=188, T=188) 56.6029219627 0.00124499201775 0.7106
annoy Annoy(n_trees=200, search_k=10000) 635.55557704 0.00242992401123 0.7915
annoy Annoy(n_trees=200, search_k=200) 635.13813591 0.000229732036591 0.226
annoy Annoy(n_trees=400, search_k=2000) 1316.67872787 0.000937170028687 0.586
falconn FALCONN(K=16, L=70, T=70) 22.0877561569 0.000493844032288 0.4527
annoy Annoy(n_trees=100, search_k=4000) 339.618172884 0.00100338196754 0.6267
falconn FALCONN(K=16, L=17, T=17) 6.69212985039 0.000142482042313 0.1828
falconn FALCONN(K=16, L=10, T=10) 4.50494790077 9.38079357147e-05 0.1216
annoy Annoy(n_trees=200, search_k=4000) 634.886514187 0.00116959285736 0.6632
annoy Annoy(n_trees=400, search_k=40000) 1304.71075797 0.00821539402008 0.9351
annoy Annoy(n_trees=200, search_k=100) 669.872941017 0.00018730711937 0.1737
falconn FALCONN(K=16, L=85, T=85) 26.0982899666 0.000581559181213 0.5026
falconn FALCONN(K=16, L=1170, T=1170) 345.279306173 0.00682651185989 0.9839
annoy Annoy(n_trees=100, search_k=100) 334.928418159 0.000121731996536 0.155
falconn FALCONN(K=16, L=5, T=5) 3.06666398048 5.23960590363e-05 0.0691
falconn FALCONN(K=16, L=798, T=798) 235.728566885 0.0048431558609 0.9611
annoy Annoy(n_trees=100, search_k=200) 338.035387993 0.000153580188751 0.2155
falconn FALCONN(K=16, L=41, T=41) 13.7509088516 0.000302683830261 0.3297
falconn FALCONN(K=16, L=725, T=725) 213.792033195 0.00441681408882 0.9524
falconn FALCONN(K=16, L=599, T=599) 176.235280991 0.003695966959 0.9322
falconn FALCONN(K=16, L=1063, T=1063) 311.402531147 0.00628771901131 0.9792
annoy Annoy(n_trees=200, search_k=2000) 676.901097059 0.000721218109131 0.5526
falconn FALCONN(K=16, L=15, T=15) 5.46862578392 0.000126718044281 0.167
falconn FALCONN(K=16, L=19, T=19) 6.93683695793 0.000150528907776 0.1983
kgraph KGraph(P=200) 3353.35622287 0.385667653084 0.8272
falconn FALCONN(K=16, L=277, T=277) 85.1621961594 0.00175514817238 0.7975
annoy Annoy(n_trees=400, search_k=100) 1256.71473598 0.000308988809586 0.1927
falconn FALCONN(K=16, L=228, T=228) 70.2304759026 0.00147809791565 0.7578
falconn FALCONN(K=16, L=6, T=6) 2.89934206009 6.04231357574e-05 0.0814
annoy Annoy(n_trees=400, search_k=100000) 1253.449404 0.0169799571037 0.975
annoy Annoy(n_trees=100, search_k=200000) 334.437412024 0.0331848750114 0.9861
annoy Annoy(n_trees=400, search_k=4000) 1254.53909206 0.00130312585831 0.6963
falconn FALCONN(K=16, L=127, T=127) 38.5714218616 0.000849563121796 0.6049
annoy Annoy(n_trees=100, search_k=40000) 334.074970007 0.00746205806732 0.9031
annoy Annoy(n_trees=100, search_k=1000) 334.495769978 0.000370069980621 0.415
falconn FALCONN(K=16, L=207, T=207) 62.3584799767 0.00135173201561 0.7345
falconn FALCONN(K=16, L=1287, T=1287) 378.367150068 0.00746319413185 0.9874
falconn FALCONN(K=16, L=13, T=13) 5.35854697227 0.000111395835876 0.1498
falconn FALCONN(K=16, L=140, T=140) 43.0882668495 0.00092963886261 0.633
falconn FALCONN(K=16, L=33, T=33) 11.4742801189 0.000246792078018 0.2826
annoy Annoy(n_trees=400, search_k=1000) 1254.81966686 0.00061609005928 0.4808
falconn FALCONN(K=16, L=3, T=3) 2.30151295662 3.84030342102e-05 0.0437
falconn FALCONN(K=16, L=154, T=154) 47.0802681446 0.00103542995453 0.6594
annoy Annoy(n_trees=400, search_k=200) 1273.28712106 0.000361413955688 0.2485
falconn FALCONN(K=16, L=2, T=2) 2.1254491806 3.00879478455e-05 0.031
falconn FALCONN(K=16, L=494, T=494) 150.545552969 0.00303271198273 0.9025
falconn FALCONN(K=16, L=115, T=115) 36.0455739498 0.000773469924927 0.5788
falconn FALCONN(K=16, L=104, T=104) 31.7708568573 0.000711711168289 0.5527

0 comments on commit 1c8a4dd

Please sign in to comment.