Skip to content

Commit

Permalink
Merge pull request #589 from bethgelab/performance
Browse files Browse the repository at this point in the history 
added a performance comparison with code for Foolbox 1.8, 2.4, and 3.1
  • Loading branch information
@jonasrauber
jonasrauber committed Sep 22, 2020
2 parents 3b3d6f3 + 09e5105 commit 5ae0f10
Show file tree
Hide file tree
Showing 5 changed files with 1,020 additions and 0 deletions.
350 changes: 350 additions & 0 deletions performance/foolbox_1.8.0.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,350 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"ExecuteTime": {
"end_time": "2020-09-22T11:35:07.525791Z",
"start_time": "2020-09-22T11:35:07.522612Z"
}
},
"outputs": [],
"source": [
"#!sudo pip3 install \"foolbox==1.8.0\""
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"ExecuteTime": {
"end_time": "2020-09-22T11:35:08.491505Z",
"start_time": "2020-09-22T11:35:07.529546Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1.4.0+cu100\n",
"0.5.0+cu100\n",
"1.8.0\n",
"1.19.0\n"
]
}
],
"source": [
"import time\n",
"import warnings\n",
"import torch\n",
"import torchvision\n",
"import foolbox\n",
"import numpy as np\n",
"print(torch.__version__)\n",
"print(torchvision.__version__)\n",
"print(foolbox.__version__)\n",
"print(np.__version__)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"ExecuteTime": {
"end_time": "2020-09-22T11:35:08.925632Z",
"start_time": "2020-09-22T11:35:08.494985Z"
}
},
"outputs": [],
"source": [
"model = torchvision.models.resnet18(pretrained=True).eval()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"ExecuteTime": {
"end_time": "2020-09-22T11:35:11.738032Z",
"start_time": "2020-09-22T11:35:08.927859Z"
}
},
"outputs": [],
"source": [
"preprocessing = (np.asarray([0.485, 0.456, 0.406]).reshape((3, 1, 1)), np.asarray([0.229, 0.224, 0.225]).reshape((3, 1, 1)))\n",
"fmodel = foolbox.models.PyTorchModel(model, bounds=(0, 1), num_classes=1000, preprocessing=preprocessing)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"ExecuteTime": {
"end_time": "2020-09-22T11:35:11.755412Z",
"start_time": "2020-09-22T11:35:11.741660Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(16, 3, 224, 224) (16,)\n"
]
}
],
"source": [
"images = np.load(\"images.npy\")\n",
"labels = np.load(\"labels.npy\")\n",
"print(images.shape, labels.shape)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"ExecuteTime": {
"end_time": "2020-09-22T11:35:11.763303Z",
"start_time": "2020-09-22T11:35:11.757842Z"
}
},
"outputs": [],
"source": [
"def accuracy(fmodel, images, labels):\n",
" assert len(images) == len(labels)\n",
" classes = np.asarray([fmodel.predictions(images[i]).argmax(axis=0) for i in range(len(images))])\n",
" return (classes == labels).mean()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"ExecuteTime": {
"end_time": "2020-09-22T11:35:11.861800Z",
"start_time": "2020-09-22T11:35:11.766791Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.9375\n"
]
}
],
"source": [
"print(accuracy(fmodel, images, labels))"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"ExecuteTime": {
"end_time": "2020-09-22T11:35:19.045631Z",
"start_time": "2020-09-22T11:35:11.863870Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"88.9 ms ± 8.24 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
]
}
],
"source": [
"%timeit accuracy(fmodel, images, labels)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"ExecuteTime": {
"end_time": "2020-09-22T11:35:23.168698Z",
"start_time": "2020-09-22T11:35:19.047958Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"5.02 ms ± 338 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
]
}
],
"source": [
"%timeit accuracy(fmodel, images[:1], labels[:1])"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"ExecuteTime": {
"end_time": "2020-09-22T11:35:23.177998Z",
"start_time": "2020-09-22T11:35:23.171378Z"
}
},
"outputs": [],
"source": [
"def run_attack(attack, fmodel, images, labels, epsilons):\n",
" assert len(images) == len(labels)\n",
" success = []\n",
" with warnings.catch_warnings():\n",
" warnings.simplefilter(\"ignore\")\n",
" for i in range(len(images)):\n",
" adv = attack(images[i], label=labels[i], unpack=False)\n",
" success.append([adv.distance.value <= eps for eps in epsilons])\n",
" success = np.asarray(success).T\n",
" return success"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"ExecuteTime": {
"end_time": "2020-09-22T11:38:04.962813Z",
"start_time": "2020-09-22T11:35:23.180342Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"attack took 161.8 seconds\n"
]
}
],
"source": [
"# apply the attack\n",
"start = time.time()\n",
"attack = foolbox.attacks.PGD(fmodel, distance=foolbox.distances.Linf)\n",
"epsilons = [0.002]\n",
"success = run_attack(attack, fmodel, images, labels, epsilons=epsilons)\n",
"print(f\"attack took {time.time() - start:.1f} seconds\")"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"ExecuteTime": {
"end_time": "2020-09-22T11:38:04.972084Z",
"start_time": "2020-09-22T11:38:04.965966Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.002 0.0625\n"
]
}
],
"source": [
"# calculate and report the robust accuracy\n",
"robust_accuracy = 1 - success.mean(axis=-1)\n",
"for eps, acc in zip(epsilons, robust_accuracy):\n",
" print(eps, acc.item())"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"ExecuteTime": {
"end_time": "2020-09-22T11:40:49.550952Z",
"start_time": "2020-09-22T11:38:04.974466Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"attack took 164.6 seconds\n"
]
}
],
"source": [
"# apply the attack\n",
"start = time.time()\n",
"attack = foolbox.attacks.PGD(fmodel, distance=foolbox.distances.Linf)\n",
"epsilons = [0.0, 0.001, 0.01, 0.03, 0.1, 0.3, 0.5, 1.0]\n",
"success = run_attack(attack, fmodel, images, labels, epsilons=epsilons)\n",
"print(f\"attack took {time.time() - start:.1f} seconds\")"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"ExecuteTime": {
"end_time": "2020-09-22T11:40:49.560523Z",
"start_time": "2020-09-22T11:40:49.553662Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.0 0.9375\n",
"0.001 0.375\n",
"0.01 0.0\n",
"0.03 0.0\n",
"0.1 0.0\n",
"0.3 0.0\n",
"0.5 0.0\n",
"1.0 0.0\n"
]
}
],
"source": [
"# calculate and report the robust accuracy\n",
"robust_accuracy = 1 - success.mean(axis=-1)\n",
"for eps, acc in zip(epsilons, robust_accuracy):\n",
" print(eps, acc.item())"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.9"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

0 comments on commit 5ae0f10

Please sign in to comment.