Skip to content

Commit

Permalink
Update benchmark notebook
Browse files Browse the repository at this point in the history
  • Loading branch information
@dillondaudert
dillondaudert committed Nov 24, 2018
1 parent 2b50730 commit a571335
Showing 1 changed file with 79 additions and 94 deletions.
173 changes: 79 additions & 94 deletions benchmark/Benchmarks.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,175 +38,160 @@
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"train_x, train_y = FashionMNIST.traindata()\n",
"test_x, test_y = FashionMNIST.testdata()\n",
"train_x = reshape(train_x, size(train_x)[1]*size(train_x)[2], size(train_x)[3])\n",
"test_x = reshape(test_x, size(test_x)[1]*size(test_x)[2], size(test_x)[3])\n",
"\n",
"data = [convert.(Float32, train_x[:,i]) for i = 1:size(train_x, 2)]\n",
"data = data[1:5000]\n",
"queries = [convert.(Float32, test_x[:,i]) for i = 1:size(test_x, 2)]\n",
"queries = queries[1:500];"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"knn_graph = DescentGraph(data, 10)\n",
"nn = getindex.(knn_graph.graph, 1);"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"brute_graph = brute_knn(data, Euclidean(), 10)\n",
"true_nn = getindex.(brute_graph, 1);"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.9964999999999998"
"get_fmnist (generic function with 1 method)"
]
},
"execution_count": 6,
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"recall(nn, true_nn)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"true_idx, true_dist = brute_search(data, queries, 10, Euclidean());"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"idx, dist = search(knn_graph, queries, 10, 70);"
"function get_fmnist()\n",
" train_x, train_y = FashionMNIST.traindata()\n",
" test_x, test_y = FashionMNIST.testdata()\n",
" train_x = reshape(train_x, size(train_x)[1]*size(train_x)[2], size(train_x)[3])\n",
" test_x = reshape(test_x, size(test_x)[1]*size(test_x)[2], size(test_x)[3])\n",
"\n",
" data = [convert.(Float32, train_x[:,i]) for i = 1:size(train_x, 2)]\n",
" data = data[1:5000]\n",
" queries = [convert.(Float32, test_x[:,i]) for i = 1:size(test_x, 2)]\n",
" queries = queries[1:500]\n",
" return data, queries\n",
"end"
]
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.8865999999999992"
"bmark (generic function with 1 method)"
]
},
"execution_count": 9,
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"recall(idx, true_idx)"
"function bmark(name, data, queries, metric)\n",
" @show name\n",
" knn_graph = DescentGraph(data, 10, metric)\n",
" nn = getindex.(knn_graph.graph, 1)\n",
" brute_graph = brute_knn(data, metric, 10)\n",
" true_nn = getindex.(brute_graph, 1)\n",
" @show recall(nn, true_nn)\n",
" true_idx, true_dist = brute_search(data, queries, 10, metric)\n",
" idx, dist = search(knn_graph, queries, 10, 70)\n",
" @show recall(idx, true_idx)\n",
" q_per_sec = length(queries)/(@belapsed search($knn_graph, $queries, 10, 70))\n",
" @show q_per_sec\n",
"end"
]
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name = \"Fashion MNIST\"\n",
"recall(nn, true_nn) = 0.9968800000000001\n",
"recall(idx, true_idx) = 0.911199999999999\n",
"q_per_sec = 312.67200712455934\n"
]
},
{
"data": {
"text/plain": [
"330.94124478737336"
"312.67200712455934"
]
},
"execution_count": 10,
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"q_per_sec = length(queries)/(@belapsed search(knn_graph, queries, 10, 70))"
"fm_data, fm_queries = get_fmnist()\n",
"bmark(\"Fashion MNIST\", fm_data, fm_queries, Euclidean())"
]
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name = \"Cosine Random\"\n",
"recall(nn, true_nn) = 0.6250399999999964\n",
"recall(idx, true_idx) = 0.8749999999999994\n",
"q_per_sec = 188.72105219481088\n"
]
},
{
"data": {
"text/plain": [
"BenchmarkTools.Trial: \n",
" memory estimate: 505.40 MiB\n",
" allocs estimate: 10388471\n",
" --------------\n",
" minimum time: 989.240 ms (12.54% GC)\n",
" median time: 1.029 s (14.36% GC)\n",
" mean time: 1.034 s (15.30% GC)\n",
" maximum time: 1.101 s (18.45% GC)\n",
" --------------\n",
" samples: 5\n",
" evals/sample: 1"
"188.72105219481088"
]
},
"execution_count": 11,
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"@benchmark DescentGraph(data, 10)"
"# Cosine Tests \n",
"rn_data = [rand(800) for _ in 1:5000]\n",
"rn_queries = [rand(800) for _ in 1:500]\n",
"bmark(\"Cosine Random\", rn_data, rn_queries, CosineDist())"
]
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"name = \"Hamming Random\"\n",
"recall(nn, true_nn) = 0.12610000000000463\n",
"recall(idx, true_idx) = 0.8365999999999981\n",
"q_per_sec = 195.5048559294746\n"
]
},
{
"data": {
"text/plain": [
"BenchmarkTools.Trial: \n",
" memory estimate: 691.46 MiB\n",
" allocs estimate: 25057017\n",
" --------------\n",
" minimum time: 1.527 s (10.49% GC)\n",
" median time: 1.605 s (14.83% GC)\n",
" mean time: 1.618 s (14.63% GC)\n",
" maximum time: 1.734 s (17.92% GC)\n",
" --------------\n",
" samples: 4\n",
" evals/sample: 1"
"195.5048559294746"
]
},
"execution_count": 12,
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"@benchmark search(knn_graph, queries, 10, 70)"
"# Hamming Tests\n",
"ham_data = [rand([0, 1], 800) for _ in 1:5000]\n",
"ham_queries = [rand([0, 1], 800) for _ in 1:500]\n",
"bmark(\"Hamming Random\", ham_data, ham_queries, Hamming())"
]
},
{
Expand Down

0 comments on commit a571335

Please sign in to comment.