Fixing missing type in metric and adding demo to notebook

Signed-off-by: Kieran Fraser <Kieran.Fraser@ibm.com>
Trusted-AI · Oct 25, 2023 · 2b77357 · 2b77357
1 parent 2737a3f
commit 2b77357
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 11 deletions.
diff --git a/art/metrics/metrics.py b/art/metrics/metrics.py
@@ -23,7 +23,7 @@
 
 from functools import reduce
 import logging
-from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING
+from typing import Any, Dict, List, Tuple, Optional, Union, TYPE_CHECKING
 
 import numpy as np
 import numpy.linalg as la

diff --git a/notebooks/attack_parallel_auto_attack.ipynb b/notebooks/attack_parallel_auto_attack.ipynb
@@ -16,7 +16,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -56,7 +56,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
@@ -92,7 +92,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -158,7 +158,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -168,11 +168,7 @@
       "/Users/kieranfraser/git/personal/art/env/lib/python3.9/multiprocessing/resource_tracker.py:216: UserWarning: resource_tracker: There appear to be 1 leaked semaphore objects to clean up at shutdown\n",
       "  warnings.warn('resource_tracker: There appear to be %d '\n",
       "/Users/kieranfraser/git/personal/art/env/lib/python3.9/multiprocessing/resource_tracker.py:216: UserWarning: resource_tracker: There appear to be 1 leaked semaphore objects to clean up at shutdown\n",
-      "  warnings.warn('resource_tracker: There appear to be %d '\n",
-      "/Users/kieranfraser/git/personal/art/env/lib/python3.9/multiprocessing/resource_tracker.py:216: UserWarning: resource_tracker: There appear to be 1 leaked semaphore objects to clean up at shutdown\n",
-      "  warnings.warn('resource_tracker: There appear to be %d '\n",
-      "/Users/kieranfraser/git/personal/art/env/lib/python3.9/multiprocessing/resource_tracker.py:229: UserWarning: resource_tracker: '/loky-2595-nh_wgd4i': [Errno 2] No such file or directory\n",
-      "  warnings.warn('resource_tracker: %r: %s' % (name, e))\n"
+      "  warnings.warn('resource_tracker: There appear to be %d '\n"
      ]
     }
    ],
@@ -194,7 +190,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -226,6 +222,37 @@
     "- The fourth column images are adversarial images generated by AutoAttack in parallel mode. Note how parallel model achieves lower L2 distance between the original and adversarial image - as all jobs are run in parallel mode, more attacks can be evaluated and successful attacks with lower perturbations added can be selected."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Parallel\n",
+      "clean acc: 0.5\n",
+      "robust acc: 0.0\n",
+      "l2: 1.2837049961090088\n",
+      "----\n",
+      "Non-parallel\n",
+      "clean acc: 0.5\n",
+      "robust acc: 0.0\n",
+      "l2: 1.6767956018447876\n"
+     ]
+    }
+   ],
+   "source": [
+    "from art.metrics.metrics import adversarial_accuracy_and_perturbation\n",
+    "\n",
+    "clean_accuracy, robust_accuracy, perturbation = adversarial_accuracy_and_perturbation(classifier, x_train, parallel_adv, y_train)\n",
+    "print(f'Parallel\\nclean acc: {clean_accuracy}\\nrobust acc: {robust_accuracy}\\nl2: {perturbation}')\n",
+    "print('----')\n",
+    "clean_accuracy, robust_accuracy, perturbation = adversarial_accuracy_and_perturbation(classifier, x_train, nonparallel_adv, y_train)\n",
+    "print(f'Non-parallel\\nclean acc: {clean_accuracy}\\nrobust acc: {robust_accuracy}\\nl2: {perturbation}')"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 6,