From 2b77357f3c68158ff000221b01a812baf4f735c5 Mon Sep 17 00:00:00 2001
From: Kieran Fraser <Kieran.Fraser@ibm.com>
Date: Wed, 25 Oct 2023 16:42:00 +0100
Subject: [PATCH] Fixing missing type in metric and adding demo to notebook

Signed-off-by: Kieran Fraser <Kieran.Fraser@ibm.com>
---
 art/metrics/metrics.py                      |  2 +-
 notebooks/attack_parallel_auto_attack.ipynb | 47 ++++++++++++++++-----
 2 files changed, 38 insertions(+), 11 deletions(-)

diff --git a/art/metrics/metrics.py b/art/metrics/metrics.py
index 8a24ca14f2..ab0d3c36cb 100644
--- a/art/metrics/metrics.py
+++ b/art/metrics/metrics.py
@@ -23,7 +23,7 @@
 
 from functools import reduce
 import logging
-from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING
+from typing import Any, Dict, List, Tuple, Optional, Union, TYPE_CHECKING
 
 import numpy as np
 import numpy.linalg as la
diff --git a/notebooks/attack_parallel_auto_attack.ipynb b/notebooks/attack_parallel_auto_attack.ipynb
index e23485d463..79b615f888 100644
--- a/notebooks/attack_parallel_auto_attack.ipynb
+++ b/notebooks/attack_parallel_auto_attack.ipynb
@@ -16,7 +16,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -56,7 +56,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
@@ -92,7 +92,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -158,7 +158,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -168,11 +168,7 @@
       "/Users/kieranfraser/git/personal/art/env/lib/python3.9/multiprocessing/resource_tracker.py:216: UserWarning: resource_tracker: There appear to be 1 leaked semaphore objects to clean up at shutdown\n",
       "  warnings.warn('resource_tracker: There appear to be %d '\n",
       "/Users/kieranfraser/git/personal/art/env/lib/python3.9/multiprocessing/resource_tracker.py:216: UserWarning: resource_tracker: There appear to be 1 leaked semaphore objects to clean up at shutdown\n",
-      "  warnings.warn('resource_tracker: There appear to be %d '\n",
-      "/Users/kieranfraser/git/personal/art/env/lib/python3.9/multiprocessing/resource_tracker.py:216: UserWarning: resource_tracker: There appear to be 1 leaked semaphore objects to clean up at shutdown\n",
-      "  warnings.warn('resource_tracker: There appear to be %d '\n",
-      "/Users/kieranfraser/git/personal/art/env/lib/python3.9/multiprocessing/resource_tracker.py:229: UserWarning: resource_tracker: '/loky-2595-nh_wgd4i': [Errno 2] No such file or directory\n",
-      "  warnings.warn('resource_tracker: %r: %s' % (name, e))\n"
+      "  warnings.warn('resource_tracker: There appear to be %d '\n"
      ]
     }
    ],
@@ -194,7 +190,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -226,6 +222,37 @@
     "- The fourth column images are adversarial images generated by AutoAttack in parallel mode. Note how parallel model achieves lower L2 distance between the original and adversarial image - as all jobs are run in parallel mode, more attacks can be evaluated and successful attacks with lower perturbations added can be selected."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Parallel\n",
+      "clean acc: 0.5\n",
+      "robust acc: 0.0\n",
+      "l2: 1.2837049961090088\n",
+      "----\n",
+      "Non-parallel\n",
+      "clean acc: 0.5\n",
+      "robust acc: 0.0\n",
+      "l2: 1.6767956018447876\n"
+     ]
+    }
+   ],
+   "source": [
+    "from art.metrics.metrics import adversarial_accuracy_and_perturbation\n",
+    "\n",
+    "clean_accuracy, robust_accuracy, perturbation = adversarial_accuracy_and_perturbation(classifier, x_train, parallel_adv, y_train)\n",
+    "print(f'Parallel\\nclean acc: {clean_accuracy}\\nrobust acc: {robust_accuracy}\\nl2: {perturbation}')\n",
+    "print('----')\n",
+    "clean_accuracy, robust_accuracy, perturbation = adversarial_accuracy_and_perturbation(classifier, x_train, nonparallel_adv, y_train)\n",
+    "print(f'Non-parallel\\nclean acc: {clean_accuracy}\\nrobust acc: {robust_accuracy}\\nl2: {perturbation}')"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 6,