From 3fcbad03ab70abda3d40858bf37a149101a936a4 Mon Sep 17 00:00:00 2001 From: GiulioZizzo Date: Fri, 28 Jul 2023 16:55:53 +0100 Subject: [PATCH 1/3] adding label check to trades adversarial trainer Signed-off-by: GiulioZizzo --- .../adversarial_trainer_trades_pytorch.py | 9 +++- ...test_adversarial_trainer_trades_pytorch.py | 47 +++++++++++++++---- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/art/defences/trainer/adversarial_trainer_trades_pytorch.py b/art/defences/trainer/adversarial_trainer_trades_pytorch.py index 2b9bd24789..e896a66a75 100644 --- a/art/defences/trainer/adversarial_trainer_trades_pytorch.py +++ b/art/defences/trainer/adversarial_trainer_trades_pytorch.py @@ -126,8 +126,13 @@ def fit( # compute accuracy if validation_data is not None: (x_test, y_test) = validation_data + output = np.argmax(self.predict(x_test), axis=1) - nb_correct_pred = np.sum(output == np.argmax(y_test, axis=1)) + if y_test.ndim > 1: + nb_correct_pred = np.sum(output == np.argmax(y_test, axis=1)) + else: + nb_correct_pred = np.sum(output == y_test) + logger.info( "epoch: %s time(s): %.1f loss: %.4f acc(tr): %.4f acc(val): %.4f", i_epoch, @@ -240,7 +245,7 @@ def _batch_process(self, x_batch: np.ndarray, y_batch: np.ndarray) -> Tuple[floa ) # Check label shape - if self._classifier._reduce_labels: # pylint: disable=W0212 + if self._classifier._reduce_labels and y_preprocessed.ndim > 1: # pylint: disable=W0212 y_preprocessed = np.argmax(y_preprocessed, axis=1) i_batch = torch.from_numpy(x_preprocessed).to(self._classifier._device) # pylint: disable=W0212 diff --git a/tests/defences/trainer/test_adversarial_trainer_trades_pytorch.py b/tests/defences/trainer/test_adversarial_trainer_trades_pytorch.py index 98c32a7c01..a361655eb7 100644 --- a/tests/defences/trainer/test_adversarial_trainer_trades_pytorch.py +++ b/tests/defences/trainer/test_adversarial_trainer_trades_pytorch.py @@ -63,22 +63,38 @@ def fix_get_mnist_subset(get_mnist_dataset): yield x_train_mnist[:n_train], y_train_mnist[:n_train], x_test_mnist[:n_test], y_test_mnist[:n_test] -@pytest.mark.skip_framework("tensorflow", "keras", "scikitlearn", "mxnet", "kerastf") -def test_adversarial_trainer_trades_pytorch_fit_and_predict(get_adv_trainer, fix_get_mnist_subset): +@pytest.mark.only_with_platform("pytorch") +@pytest.mark.parametrize("label_format", ["one_hot", "numerical"]) +def test_adversarial_trainer_trades_pytorch_fit_and_predict(get_adv_trainer, fix_get_mnist_subset, label_format): (x_train_mnist, y_train_mnist, x_test_mnist, y_test_mnist) = fix_get_mnist_subset x_test_mnist_original = x_test_mnist.copy() + if label_format == "one_hot": + assert y_train_mnist.shape[-1] == 10 + assert y_test_mnist.shape[-1] == 10 + if label_format == "numerical": + y_test_mnist = np.argmax(y_test_mnist, axis=1) + y_train_mnist = np.argmax(y_train_mnist, axis=1) + trainer = get_adv_trainer() if trainer is None: logging.warning("Couldn't perform this test because no trainer is defined for this framework configuration") return predictions = np.argmax(trainer.predict(x_test_mnist), axis=1) - accuracy = np.sum(predictions == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0] + + if label_format == "one_hot": + accuracy = np.sum(predictions == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0] + else: + accuracy = np.sum(predictions == y_test_mnist) / x_test_mnist.shape[0] trainer.fit(x_train_mnist, y_train_mnist, nb_epochs=20) predictions_new = np.argmax(trainer.predict(x_test_mnist), axis=1) - accuracy_new = np.sum(predictions_new == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0] + + if label_format == "one_hot": + accuracy_new = np.sum(predictions_new == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0] + else: + accuracy_new = np.sum(predictions_new == y_test_mnist) / x_test_mnist.shape[0] np.testing.assert_array_almost_equal( float(np.mean(x_test_mnist_original - x_test_mnist)), @@ -92,13 +108,21 @@ def test_adversarial_trainer_trades_pytorch_fit_and_predict(get_adv_trainer, fix trainer.fit(x_train_mnist, y_train_mnist, nb_epochs=20, validation_data=(x_train_mnist, y_train_mnist)) -@pytest.mark.skip_framework("tensorflow", "keras", "scikitlearn", "mxnet", "kerastf") +@pytest.mark.only_with_platform("pytorch") +@pytest.mark.parametrize("label_format", ["one_hot", "numerical"]) def test_adversarial_trainer_trades_pytorch_fit_generator_and_predict( - get_adv_trainer, fix_get_mnist_subset, image_data_generator + get_adv_trainer, fix_get_mnist_subset, image_data_generator, label_format ): (x_train_mnist, y_train_mnist, x_test_mnist, y_test_mnist) = fix_get_mnist_subset x_test_mnist_original = x_test_mnist.copy() + if label_format == "one_hot": + assert y_train_mnist.shape[-1] == 10 + assert y_test_mnist.shape[-1] == 10 + if label_format == "numerical": + y_test_mnist = np.argmax(y_test_mnist, axis=1) + y_train_mnist = np.argmax(y_train_mnist, axis=1) + generator = image_data_generator() trainer = get_adv_trainer() @@ -107,11 +131,18 @@ def test_adversarial_trainer_trades_pytorch_fit_generator_and_predict( return predictions = np.argmax(trainer.predict(x_test_mnist), axis=1) - accuracy = np.sum(predictions == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0] + if label_format == "one_hot": + accuracy = np.sum(predictions == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0] + else: + accuracy = np.sum(predictions == y_test_mnist) / x_test_mnist.shape[0] trainer.fit_generator(generator=generator, nb_epochs=20) predictions_new = np.argmax(trainer.predict(x_test_mnist), axis=1) - accuracy_new = np.sum(predictions_new == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0] + + if label_format == "one_hot": + accuracy_new = np.sum(predictions_new == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0] + else: + accuracy_new = np.sum(predictions_new == y_test_mnist) / x_test_mnist.shape[0] np.testing.assert_array_almost_equal( float(np.mean(x_test_mnist_original - x_test_mnist)), From 5d0c872a249837bd7be86bc3aeead352526e1666 Mon Sep 17 00:00:00 2001 From: GiulioZizzo Date: Mon, 14 Aug 2023 09:08:35 +0100 Subject: [PATCH 2/3] add from_logits=True to begin addressing issue #2227 Signed-off-by: GiulioZizzo --- .../trainer/test_adversarial_trainer_trades_pytorch.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/defences/trainer/test_adversarial_trainer_trades_pytorch.py b/tests/defences/trainer/test_adversarial_trainer_trades_pytorch.py index a361655eb7..8d081f418c 100644 --- a/tests/defences/trainer/test_adversarial_trainer_trades_pytorch.py +++ b/tests/defences/trainer/test_adversarial_trainer_trades_pytorch.py @@ -34,7 +34,7 @@ def _get_adv_trainer(): if framework in ["tensorflow", "tensorflow2v1"]: trainer = None if framework == "pytorch": - classifier, _ = image_dl_estimator() + classifier, _ = image_dl_estimator(from_logits=True) attack = ProjectedGradientDescent( classifier, norm=np.inf, @@ -121,7 +121,6 @@ def test_adversarial_trainer_trades_pytorch_fit_generator_and_predict( assert y_test_mnist.shape[-1] == 10 if label_format == "numerical": y_test_mnist = np.argmax(y_test_mnist, axis=1) - y_train_mnist = np.argmax(y_train_mnist, axis=1) generator = image_data_generator() From 2338a0875f130a7af5f789ec8abd9590953ed987 Mon Sep 17 00:00:00 2001 From: GiulioZizzo Date: Wed, 16 Aug 2023 08:26:00 +0100 Subject: [PATCH 3/3] use check_and_transform_label_format to assure consistant label format Signed-off-by: GiulioZizzo --- .../adversarial_trainer_trades_pytorch.py | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/art/defences/trainer/adversarial_trainer_trades_pytorch.py b/art/defences/trainer/adversarial_trainer_trades_pytorch.py index e896a66a75..c965635419 100644 --- a/art/defences/trainer/adversarial_trainer_trades_pytorch.py +++ b/art/defences/trainer/adversarial_trainer_trades_pytorch.py @@ -33,6 +33,7 @@ from art.estimators.classification.pytorch import PyTorchClassifier from art.data_generators import DataGenerator from art.attacks.attack import EvasionAttack +from art.utils import check_and_transform_label_format if TYPE_CHECKING: import torch @@ -97,6 +98,15 @@ def fit( ind = np.arange(len(x)) logger.info("Adversarial Training TRADES") + y = check_and_transform_label_format(y, nb_classes=self.classifier.nb_classes) + + if validation_data is not None: + (x_test, y_test) = validation_data + y_test = check_and_transform_label_format(y_test, nb_classes=self.classifier.nb_classes) + + x_preprocessed_test, y_preprocessed_test = self._classifier._apply_preprocessing( # pylint: disable=W0212 + x_test, y_test, fit=True + ) for i_epoch in trange(nb_epochs, desc="Adversarial Training TRADES - Epochs"): # Shuffle the examples @@ -107,7 +117,6 @@ def fit( train_n = 0.0 for batch_id in range(nb_batches): - # Create batch data x_batch = x[ind[batch_id * batch_size : min((batch_id + 1) * batch_size, x.shape[0])]].copy() y_batch = y[ind[batch_id * batch_size : min((batch_id + 1) * batch_size, x.shape[0])]] @@ -125,13 +134,8 @@ def fit( # compute accuracy if validation_data is not None: - (x_test, y_test) = validation_data - - output = np.argmax(self.predict(x_test), axis=1) - if y_test.ndim > 1: - nb_correct_pred = np.sum(output == np.argmax(y_test, axis=1)) - else: - nb_correct_pred = np.sum(output == y_test) + output = np.argmax(self.predict(x_preprocessed_test), axis=1) + nb_correct_pred = np.sum(output == np.argmax(y_preprocessed_test, axis=1)) logger.info( "epoch: %s time(s): %.1f loss: %.4f acc(tr): %.4f acc(val): %.4f", @@ -193,7 +197,6 @@ def fit_generator( train_n = 0.0 for batch_id in range(nb_batches): # pylint: disable=W0612 - # Create batch data x_batch, y_batch = generator.get_batch() x_batch = x_batch.copy() @@ -237,6 +240,8 @@ def _batch_process(self, x_batch: np.ndarray, y_batch: np.ndarray) -> Tuple[floa x_batch_pert = self._attack.generate(x_batch, y=y_batch) # Apply preprocessing + y_batch = check_and_transform_label_format(y_batch, nb_classes=self.classifier.nb_classes) + x_preprocessed, y_preprocessed = self._classifier._apply_preprocessing( # pylint: disable=W0212 x_batch, y_batch, fit=True ) @@ -245,7 +250,7 @@ def _batch_process(self, x_batch: np.ndarray, y_batch: np.ndarray) -> Tuple[floa ) # Check label shape - if self._classifier._reduce_labels and y_preprocessed.ndim > 1: # pylint: disable=W0212 + if self._classifier._reduce_labels: # pylint: disable=W0212 y_preprocessed = np.argmax(y_preprocessed, axis=1) i_batch = torch.from_numpy(x_preprocessed).to(self._classifier._device) # pylint: disable=W0212