From c8bd56385af1a62a91929d3ac957ee9e074916cc Mon Sep 17 00:00:00 2001 From: klchen0112 Date: Fri, 15 Dec 2023 14:34:15 +0800 Subject: [PATCH 1/3] Perf attack save --- torchattacks/attack.py | 111 ++++++++++++++++++++++++++++------------- 1 file changed, 76 insertions(+), 35 deletions(-) diff --git a/torchattacks/attack.py b/torchattacks/attack.py index 4661272d..eaa3ca90 100644 --- a/torchattacks/attack.py +++ b/torchattacks/attack.py @@ -248,6 +248,7 @@ def save( self, data_loader, save_path=None, + save_every_iter=False, verbose=True, return_verbose=False, save_predictions=False, @@ -260,6 +261,7 @@ def save( Arguments: save_path (str): save_path. data_loader (torch.utils.data.DataLoader): data loader. + save_every_iter (bool): True for save every results every iter. (Default: False) verbose (bool): True for displaying detailed information. (Default: True) return_verbose (bool): True for returning detailed information. (Default: False) save_predictions (bool): True for saving predicted labels (Default: False) @@ -319,45 +321,33 @@ def save( if save_path is not None: adv_input_list.append(adv_inputs.detach().cpu()) label_list.append(labels.detach().cpu()) - - adv_input_list_cat = torch.cat(adv_input_list, 0) - label_list_cat = torch.cat(label_list, 0) - - save_dict = { - "adv_inputs": adv_input_list_cat, - "labels": label_list_cat, - } # nopep8 - if save_predictions: pred_list.append(pred.detach().cpu()) - pred_list_cat = torch.cat(pred_list, 0) - save_dict["preds"] = pred_list_cat - if save_clean_inputs: input_list.append(inputs.detach().cpu()) - input_list_cat = torch.cat(input_list, 0) - save_dict["clean_inputs"] = input_list_cat - - if self.normalization_used is not None: - save_dict["adv_inputs"] = self.inverse_normalize( - save_dict["adv_inputs"] - ) # nopep8 - if save_clean_inputs: - save_dict["clean_inputs"] = self.inverse_normalize( - save_dict["clean_inputs"] - ) # nopep8 - - if save_type == "int": - save_dict["adv_inputs"] = self.to_type( - save_dict["adv_inputs"], "int" - ) # nopep8 - if save_clean_inputs: - save_dict["clean_inputs"] = self.to_type( - save_dict["clean_inputs"], "int" - ) # nopep8 - - save_dict["save_type"] = save_type - torch.save(save_dict, save_path) + if save_every_iter: + self._save_adv_examples( + save_type, + save_path, + adv_input_list, + label_list, + save_predictions = save_predictions, + pred_list = pred_list if save_predictions else None, + save_clean_inputs = save_clean_inputs, + input_list = input_list if save_clean_inputs else None, + ) + + if save_path is not None and not save_every_iter: + self._save_adv_examples( + save_type, + save_path, + adv_input_list, + label_list, + save_predictions = save_predictions, + pred_list = pred_list if save_predictions else None, + save_clean_inputs = save_clean_inputs, + input_list = input_list if save_clean_inputs else None, + ) # To avoid erasing the printed information. if verbose: @@ -388,6 +378,57 @@ def to_type(inputs, type): raise ValueError(type + " is not a valid type. [Options: float, int]") return inputs + + def _save_adv_examples( + self, + save_type, + save_path, + adv_input_list, + label_list, + save_predictions = False, + pred_list = [], + save_clean_inputs = False, + input_list = [], + ): + + + adv_input_list_cat = torch.cat(adv_input_list, 0) + label_list_cat = torch.cat(label_list, 0) + + save_dict = { + "adv_inputs": adv_input_list_cat, + "labels": label_list_cat, + } + + if save_predictions: + pred_list_cat = torch.cat(pred_list, 0) + save_dict["preds"] = pred_list_cat + + if save_clean_inputs: + input_list_cat = torch.cat(input_list, 0) + save_dict["clean_inputs"] = input_list_cat + + if self.normalization_used is not None: + save_dict["adv_inputs"] = self.inverse_normalize( + save_dict["adv_inputs"] + ) # nopep8 + if save_clean_inputs: + save_dict["clean_inputs"] = self.inverse_normalize( + save_dict["clean_inputs"] + ) # nopep8 + + if save_type == "int": + save_dict["adv_inputs"] = self.to_type( + save_dict["adv_inputs"], "int" + ) # nopep8 + if save_clean_inputs: + save_dict["clean_inputs"] = self.to_type( + save_dict["clean_inputs"], "int" + ) # nopep8 + + save_dict["save_type"] = save_type + torch.save(save_dict, save_path) + @staticmethod def _save_print(progress, rob_acc, l2, elapsed_time, end): print( From 12819b5dd78fb50672a8e91176d9a045e9c8cfd2 Mon Sep 17 00:00:00 2001 From: klchen0112 Date: Wed, 22 May 2024 22:34:06 +0800 Subject: [PATCH 2/3] fix l2 inf --- torchattacks/attack.py | 75 +++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/torchattacks/attack.py b/torchattacks/attack.py index eaa3ca90..2396e9f8 100644 --- a/torchattacks/attack.py +++ b/torchattacks/attack.py @@ -253,6 +253,7 @@ def save( return_verbose=False, save_predictions=False, save_clean_inputs=False, + save_labels=False, save_type="float", ): r""" @@ -270,7 +271,8 @@ def save( """ if save_path is not None: adv_input_list = [] - label_list = [] + if save_labels: + label_list = [] if save_predictions: pred_list = [] if save_clean_inputs: @@ -278,7 +280,9 @@ def save( correct = 0 total = 0 - l2_distance = [] + l2_distance_total = 0 + linf_distance_total = 0 + wrong_label = 0 total_batch = len(data_loader) given_training = self.model.training @@ -299,15 +303,21 @@ def save( correct += right_idx.sum() rob_acc = 100 * float(correct) / total + inputs_inver = self.inverse_normalize(inputs) + adv_inptus_inver = self.inverse_normalize(adv_inputs) # Calculate l2 distance - delta = (adv_inputs - inputs.to(self.device)).view( + delta = (adv_inptus_inver - inputs_inver.to(self.device)).view( batch_size, -1 ) # nopep8 - l2_distance.append( - torch.norm(delta[~right_idx], p=2, dim=1) - ) # nopep8 - l2 = torch.cat(l2_distance).mean().item() - + l2_distance_total += torch.norm( + delta[~right_idx], p=2, dim=1 + ).sum().item() + linf_distance_total += torch.norm( + delta[~right_idx], p=float("inf"), dim=1 + ).sum().item() + + l2 = l2_distance_total / (total - correct) + linf = linf_distance_total / (total - correct) # Calculate time computation progress = (step + 1) / total_batch * 100 end = time.time() @@ -315,12 +325,13 @@ def save( if verbose: self._save_print( - progress, rob_acc, l2, elapsed_time, end="\r" + progress, rob_acc, l2, linf, elapsed_time, end="\r" ) # nopep8 if save_path is not None: adv_input_list.append(adv_inputs.detach().cpu()) - label_list.append(labels.detach().cpu()) + if save_labels: + label_list.append(labels.detach().cpu()) if save_predictions: pred_list.append(pred.detach().cpu()) if save_clean_inputs: @@ -331,23 +342,23 @@ def save( save_path, adv_input_list, label_list, - save_predictions = save_predictions, - pred_list = pred_list if save_predictions else None, - save_clean_inputs = save_clean_inputs, - input_list = input_list if save_clean_inputs else None, + save_predictions=save_predictions, + pred_list=pred_list if save_predictions else None, + save_clean_inputs=save_clean_inputs, + input_list=input_list if save_clean_inputs else None, ) if save_path is not None and not save_every_iter: self._save_adv_examples( - save_type, - save_path, - adv_input_list, - label_list, - save_predictions = save_predictions, - pred_list = pred_list if save_predictions else None, - save_clean_inputs = save_clean_inputs, - input_list = input_list if save_clean_inputs else None, - ) + save_type, + save_path, + adv_input_list, + label_list, + save_predictions=save_predictions, + pred_list=pred_list if save_predictions else None, + save_clean_inputs=save_clean_inputs, + input_list=input_list if save_clean_inputs else None, + ) # To avoid erasing the printed information. if verbose: @@ -378,19 +389,17 @@ def to_type(inputs, type): raise ValueError(type + " is not a valid type. [Options: float, int]") return inputs - def _save_adv_examples( self, save_type, save_path, adv_input_list, label_list, - save_predictions = False, - pred_list = [], - save_clean_inputs = False, - input_list = [], - ): - + save_predictions=False, + pred_list=[], + save_clean_inputs=False, + input_list=[], + ): adv_input_list_cat = torch.cat(adv_input_list, 0) label_list_cat = torch.cat(label_list, 0) @@ -430,10 +439,10 @@ def _save_adv_examples( torch.save(save_dict, save_path) @staticmethod - def _save_print(progress, rob_acc, l2, elapsed_time, end): + def _save_print(progress, rob_acc, l2, linf, elapsed_time, end): print( - "- Save progress: %2.2f %% / Robust accuracy: %2.2f %% / L2: %1.5f (%2.3f it/s) \t" - % (progress, rob_acc, l2, elapsed_time), + "- Save progress: %2.2f %% / Robust accuracy: %2.2f %% / L2: %1.5f Linf: %1.5f (%2.3f it/s) \t" + % (progress, rob_acc, l2, linf, elapsed_time), end=end, ) From b23644c37999a7381f4c817cf0f5ae477008484b Mon Sep 17 00:00:00 2001 From: klchen0112 Date: Wed, 29 May 2024 23:50:24 +0800 Subject: [PATCH 3/3] add attacks l0 linf --- torchattacks/attack.py | 51 ++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/torchattacks/attack.py b/torchattacks/attack.py index 2396e9f8..3b49e82b 100644 --- a/torchattacks/attack.py +++ b/torchattacks/attack.py @@ -280,9 +280,11 @@ def save( correct = 0 total = 0 + l0_distance_total = 0 + l1_distance_total = 0 l2_distance_total = 0 linf_distance_total = 0 - wrong_label = 0 + total_batch = len(data_loader) given_training = self.model.training @@ -302,22 +304,28 @@ def save( right_idx = pred == labels.to(self.device) correct += right_idx.sum() rob_acc = 100 * float(correct) / total - - inputs_inver = self.inverse_normalize(inputs) - adv_inptus_inver = self.inverse_normalize(adv_inputs) + if self._normalization_applied is True: + inputs_inver = self.inverse_normalize(inputs) + adv_inptus_inver = self.inverse_normalize(adv_inputs) + else: + inputs_inver = inputs + adv_inptus_inver = adv_inputs # Calculate l2 distance delta = (adv_inptus_inver - inputs_inver.to(self.device)).view( batch_size, -1 ) # nopep8 + l0_distance_total += torch.count_nonzero(delta).item() + l1_distance_total += torch.sum(torch.abs(delta)).item() l2_distance_total += torch.norm( delta[~right_idx], p=2, dim=1 ).sum().item() linf_distance_total += torch.norm( delta[~right_idx], p=float("inf"), dim=1 ).sum().item() - - l2 = l2_distance_total / (total - correct) - linf = linf_distance_total / (total - correct) + l0 = l0_distance_total / (total) + l1 = l1_distance_total / (total) + l2 = l2_distance_total / (total ) + linf = linf_distance_total / (total ) # Calculate time computation progress = (step + 1) / total_batch * 100 end = time.time() @@ -325,7 +333,8 @@ def save( if verbose: self._save_print( - progress, rob_acc, l2, linf, elapsed_time, end="\r" + type(self).__name__, + progress, rob_acc, l0,l1,l2, linf, elapsed_time, end="\r" ) # nopep8 if save_path is not None: @@ -341,7 +350,7 @@ def save( save_type, save_path, adv_input_list, - label_list, + label_list if save_labels else None, save_predictions=save_predictions, pred_list=pred_list if save_predictions else None, save_clean_inputs=save_clean_inputs, @@ -353,7 +362,7 @@ def save( save_type, save_path, adv_input_list, - label_list, + label_list if save_labels else None, save_predictions=save_predictions, pred_list=pred_list if save_predictions else None, save_clean_inputs=save_clean_inputs, @@ -362,13 +371,13 @@ def save( # To avoid erasing the printed information. if verbose: - self._save_print(progress, rob_acc, l2, elapsed_time, end="\n") + self._save_print(type(self).__name__,progress, rob_acc,l0,l1, l2,linf, elapsed_time, end="\n") if given_training: self.model.train() if return_verbose: - return rob_acc, l2, elapsed_time + return rob_acc, l0,l1,l2,linf, elapsed_time @staticmethod def to_type(inputs, type): @@ -400,15 +409,19 @@ def _save_adv_examples( save_clean_inputs=False, input_list=[], ): - adv_input_list_cat = torch.cat(adv_input_list, 0) - label_list_cat = torch.cat(label_list, 0) - save_dict = { "adv_inputs": adv_input_list_cat, - "labels": label_list_cat, + } + + if label_list: + label_list_cat = torch.cat(label_list, 0) + save_dict["labels"] = label_list_cat + + + if save_predictions: pred_list_cat = torch.cat(pred_list, 0) save_dict["preds"] = pred_list_cat @@ -439,10 +452,10 @@ def _save_adv_examples( torch.save(save_dict, save_path) @staticmethod - def _save_print(progress, rob_acc, l2, linf, elapsed_time, end): + def _save_print(atk_name,progress, rob_acc,l0,l1, l2, linf, elapsed_time, end): print( - "- Save progress: %2.2f %% / Robust accuracy: %2.2f %% / L2: %1.5f Linf: %1.5f (%2.3f it/s) \t" - % (progress, rob_acc, l2, linf, elapsed_time), + "- %s Save progress: %2.2f %% / Robust accuracy: %2.2f %% / L0: %1.5f L1: %1.5f L2: %1.5f Linf: %1.5f (%2.3f it/s) \t" + % (atk_name,progress, rob_acc,l0,l1, l2, linf, elapsed_time), end=end, )