We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Firstly, thank you for your amazing work and for sharing it with us.
I try to use your framework to run robustness tests on a very simple convnet for classification task. The implementation is as following:
import pickle def load_pickled_data(file, columns): """ Loads pickled training and test data. Parameters ---------- file : Name of the pickle file. columns : list of strings List of columns in pickled data we're interested in. Returns ------- A tuple of datasets for given columns. """ with open(file, mode='rb') as f: dataset = pickle.load(f) return tuple(map(lambda c: dataset[c], columns)) import torch import numpy as np from typing import Tuple class TSRDataset(torch.utils.data.Dataset): def __init__(self, pickled_X: np.array, pickled_y: np.array, transform=None) -> None: super(TSRDataset).__init__() self.X_ = pickled_X self.y_ = pickled_y # # 4. Make function to load images # def load_image(self, index: int) -> Image.Image: # "Opens an image via a path and returns it." # image_path = self.paths[index] # return Image.open(image_path) def __len__(self) -> int: "Returns the total number of samples." return len(self.X_) # Overwrite the __getitem__() method (required for subclasses of torch.utils.data.Dataset) def __getitem__(self, index: int) -> Tuple[torch.Tensor, int]: "Returns one sample of data, data and label (X, y)." img = self.X_[index] class_idx = self.y_[index] return img, class_idx # return data, label (X, y) import torch.nn as nn class CNN(nn.Module): def __init__(self, params): super(CNN, self).__init__() self.conv1 = nn.Conv2d(in_channels=1, out_channels=params.conv1_d, kernel_size=params.conv1_k, stride=1, padding=self.get_pad(params.conv1_k, 1)) self.conv2 = nn.Conv2d(in_channels=params.conv1_d, out_channels=params.conv2_d, kernel_size=params.conv2_k, stride=1, padding=self.get_pad(params.conv2_k, 1)) self.conv3 = nn.Conv2d(in_channels=params.conv2_d, out_channels=params.conv3_d, kernel_size=params.conv3_k, stride=1, padding=self.get_pad(params.conv3_k, 1)) self.fc4 = nn.Linear(in_features=3584, out_features=params.fc4_size) self.out = nn.Linear(in_features=params.fc4_size, out_features=params.num_classes) self.params = params def get_pad(self, ksize, stride): in_height = in_width = 32 stride_height = stride_width = stride strides = [stride_height, stride_width] filter_height = filter_width = ksize if (in_height % strides[0] == 0): pad_along_height = max(filter_height - stride_height, 0) else: pad_along_height = max(filter_height - (in_height % stride_height), 0) if (in_width % strides[1] == 0): pad_along_width = max(filter_width - stride_width, 0) else: pad_along_width = max(filter_width - (in_width % stride_width), 0) pad_top = pad_along_height // 2 pad_bottom = pad_along_height - pad_top pad_left = pad_along_width // 2 pad_right = pad_along_width - pad_left return [pad_left, pad_right] def forward(self, x, is_training=True): # Convolutions x = self.conv1(x) x = nn.functional.relu(x) x = nn.functional.max_pool2d(x, kernel_size=2, stride=2, padding=self.get_pad(2,2)) # x = nn.functional.dropout(x, p=self.params.conv1_p, training=is_training) x1 = x.clone() x = self.conv2(x) x = nn.functional.relu(x) x = nn.functional.max_pool2d(x, kernel_size=2, stride=2, padding=self.get_pad(2,2)) # x = nn.functional.dropout(x, p=self.params.conv2_p, training=is_training) x2 = x.clone() x = self.conv3(x) x = nn.functional.relu(x) x = nn.functional.max_pool2d(x, kernel_size=2, stride=2, padding=self.get_pad(2,2)) # x = nn.functional.dropout(x, p=self.params.conv3_p, training=is_training) x3 = x.clone() # Fully connected x1 = nn.functional.max_pool2d(x1, kernel_size=4, stride=4, padding=self.get_pad(4,4)) x1 = torch.flatten(x1, 1, -1) x2 = nn.functional.max_pool2d(x2, kernel_size=2, stride=2, padding=self.get_pad(2,2)) x2 = torch.flatten(x2, 1, -1) x3 = torch.flatten(x3, 1, -1) x = torch.cat((x1, x2, x3), dim=-1) x = self.fc4(x) x = nn.functional.relu(x) # x = nn.functional.dropout(x, p=self.params.fc4_p, training=is_training) x = self.out(x) return x def get_model(): from collections import namedtuple Parameters = namedtuple('Parameters', [ # Data parameters 'num_classes', 'image_size', # Training parameters 'batch_size', 'max_epochs', 'log_epoch', 'print_epoch', # Optimisations 'learning_rate_decay', 'learning_rate', 'l2_reg_enabled', 'l2_lambda', 'early_stopping_enabled', 'early_stopping_patience', 'resume_training', # Layers architecture 'conv1_k', 'conv1_d', 'conv1_p', 'conv2_k', 'conv2_d', 'conv2_p', 'conv3_k', 'conv3_d', 'conv3_p', 'fc4_size', 'fc4_p' ]) parameters = Parameters( # Data parameters num_classes = 43, image_size = (32, 32), # Training parameters batch_size = 256, max_epochs = 30, log_epoch = 1, print_epoch = 1, # Optimisations learning_rate_decay = True, learning_rate = 0.001, l2_reg_enabled = False, l2_lambda = 0.0001, early_stopping_enabled = True, early_stopping_patience = 25, resume_training = False, # Layers architecture conv1_k = 5, conv1_d = 32, conv1_p = 0.9, conv2_k = 5, conv2_d = 64, conv2_p = 0.8, conv3_k = 5, conv3_d = 128, conv3_p = 0.7, fc4_size = 1024, fc4_p = 0.5 ) return CNN(parameters) def tsr_data(eps): """Example dataloader. For MNIST and CIFAR you can actually use existing ones in utils.py.""" assert eps is not None data_path = '/kaggle/input/reqnew/train_extended_preprocessed.p' X_test, y_test = load_pickled_data(data_path, columns = ['features', 'labels']) # You can access the mean and std stored in config file. mean = torch.tensor(np.mean(X_test, axis=(0,1,2))) std = torch.tensor(np.std(X_test, axis=(0,1,2))) # normalize = transforms.Normalize(mean=mean, std=std) test_data = TSRDataset(X_test, y_test, transform=None) # Load entire dataset. testloader = torch.utils.data.DataLoader(test_data,\ batch_size=10000, shuffle=False, num_workers=0) X, labels = next(iter(testloader)) # Set data_max and data_min to be None if no clip. For CIFAR-10 we clip to [0,1]. data_max = torch.reshape((1. - mean) / std, (1, -1, 1, 1)) data_min = torch.reshape((0. - mean) / std, (1, -1, 1, 1)) if eps is None: raise ValueError('You must specify an epsilon') # Rescale epsilon. ret_eps = torch.reshape(eps / std, (1, -1, 1, 1)) return torch.tensor(X).permute((0,3,1,2)), torch.argmax(torch.tensor(labels), dim=-1), data_max, data_min, ret_eps
My loaded data is actually an np array of shape [N, H, W, C] where C is 1 since it is in grayscale. The labels are classes that range from 0 to 42.
When I run alpha-beta-CROWN with the attached configuration, I got the following output:
/kaggle/working Configurations: general: device: cuda seed: 100 conv_mode: patches deterministic: false double_fp: false loss_reduction_func: sum record_bounds: false sparse_alpha: true save_adv_example: false precompile_jit: false complete_verifier: bab enable_incomplete_verification: true csv_name: null results_file: out.txt root_path: '' model: name: 'Customized("custom_model_data", "get_model")' path: /kaggle/working/model_torch.pt onnx_path: null onnx_path_prefix: '' cache_onnx_conversion: false onnx_quirks: null input_shape: null onnx_loader: default_onnx_and_vnnlib_loader onnx_optimization_flags: none data: start: 0 end: 10 select_instance: null num_outputs: 43 mean: 0.0 std: 1.0 pkl_path: null dataset: 'Customized("custom_model_data", "tsr_data")' data_filter_path: null data_idx_file: null specification: type: lp robustness_type: verified-acc norm: .inf epsilon: 0.00784313725 vnnlib_path: null vnnlib_path_prefix: '' solver: batch_size: 256 min_batch_size_ratio: 0.1 use_float64_in_last_iteration: false early_stop_patience: 10 start_save_best: 0.5 bound_prop_method: alpha-crown prune_after_crown: false crown: batch_size: 1000000000 max_crown_size: 1000000000 alpha-crown: alpha: true lr_alpha: 0.1 iteration: 100 share_slopes: false no_joint_opt: false lr_decay: 0.98 full_conv_alpha: true beta-crown: lr_alpha: 0.01 lr_beta: 0.05 lr_decay: 0.98 optimizer: adam iteration: 20 beta: true beta_warmup: true enable_opt_interm_bounds: false all_node_split_LP: false forward: refine: false dynamic: false max_dim: 10000 multi_class: multi_class_method: allclass_domain label_batch_size: 32 skip_with_refined_bound: true mip: parallel_solvers: null solver_threads: 1 refine_neuron_timeout: 15 refine_neuron_time_percentage: 0.8 early_stop: true adv_warmup: true mip_solver: gurobi bab: initial_max_domains: 1 max_domains: .inf decision_thresh: 0 timeout: 300 timeout_scale: 1 override_timeout: null get_upper_bound: false dfs_percent: 0.0 pruning_in_iteration: true pruning_in_iteration_ratio: 0.2 sort_targets: false batched_domain_list: true optimized_intermediate_layers: '' interm_transfer: true cut: enabled: false bab_cut: false lp_cut: false method: null lr: 0.01 lr_decay: 1.0 iteration: 100 bab_iteration: -1 early_stop_patience: -1 lr_beta: 0.02 number_cuts: 50 topk_cuts_in_filter: 100 batch_size_primal: 100 max_num: 1000000000 patches_cut: false cplex_cuts: false cplex_cuts_wait: 0 cplex_cuts_revpickup: true cut_reference_bounds: true fix_intermediate_bounds: false branching: method: kfsb candidates: 3 reduceop: min sb_coeff_thresh: 0.001 input_split: enable: false enhanced_bound_prop_method: alpha-crown enhanced_branching_method: naive enhanced_bound_patience: 100000000.0 attack_patience: 100000000.0 adv_check: 0 sort_domain_interval: -1 attack: enabled: false beam_candidates: 8 beam_depth: 7 max_dive_fix_ratio: 0.8 min_local_free_ratio: 0.2 mip_start_iteration: 5 mip_timeout: 30.0 adv_pool_threshold: null refined_mip_attacker: false refined_batch_size: null attack: pgd_order: before pgd_steps: 100 pgd_restarts: 100 pgd_early_stop: true pgd_lr_decay: 0.99 pgd_alpha: auto pgd_loss_mode: null enable_mip_attack: false cex_path: ./test_cex.txt attack_mode: PGD gama_lambda: 10.0 gama_decay: 0.9 check_clean: false input_split: pgd_steps: 100 pgd_restarts: 30 pgd_alpha: auto input_split_enhanced: pgd_steps: 200 pgd_restarts: 5000000 pgd_alpha: auto input_split_check_adv: pgd_steps: 5 pgd_restarts: 5 pgd_alpha: auto debug: lp_test: null Experiments at Fri May 5 09:26:50 2023 on b2ccc2a2c33d CNN( (conv1): Conv2d(1, 32, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2)) (conv2): Conv2d(32, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2)) (conv3): Conv2d(64, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2)) (fc4): Linear(in_features=3584, out_features=1024, bias=True) (out): Linear(in_features=1024, out_features=43, bias=True) ) /kaggle/working/alpha-beta-CROWN/complete_verifier/custom_model_data.py:211: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor). return torch.tensor(X).permute((0,3,1,2)), torch.argmax(torch.tensor(labels), dim=-1), data_max, data_min, ret_eps Internal results will be saved to Verified_ret_[Customized_model]_start=0_end=10_iter=20_b=256_timeout=300_branching=kfsb-min-3_lra-init=0.1_lra=0.01_lrb=0.05_PGD=before_cplex_cuts=False_multiclass=allclass_domain.npy. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% idx: 0, vnnlib ID: 0 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% Attack parameters: initialization=uniform, steps=100, restarts=100, alpha=0.007461458444595337, initialization=uniform, GAMA=False Model output of first 5 examples: tensor([[-26.63533783, -5.86781883, -4.54763603, -21.69909477, -20.81317139, -0.61611140, -16.72828102, -13.13037777, -18.94099617, -26.45909882, -4.61056852, -12.54259109, -0.75518376, -16.09009552, -2.99634385, -11.60897160, -17.43187332, -9.52975750, -18.21758842, -29.91522217, -33.70221329, -23.20652962, -18.36230087, -25.41592407, -33.91773987, -21.16673660, -17.64099312, -39.80070496, -35.47123337, -31.05479431, -22.17935181, -18.36315536, -43.14264679, -13.38010883, 12.96510029, -20.21791458, -4.28863621, -6.38604784, 41.21236038, -6.59563351, 4.28117037, -27.70765305, -9.50008297]], device='cuda:0') Adv example prediction (first 2 examples and 2 restarts): tensor([[[-26.52582550, -3.93510056, -6.41698170, -20.35298538, -20.99528694, -6.70625925, -16.09720039, -15.72246552, -17.80824852, -23.99833107, -11.98600864, -10.19097519, 2.43953228, -9.03533268, -2.28550172, -9.02405930, -19.05975914, -7.56664610, -19.64624596, -25.21205711, -34.47130585, -25.68278313, -19.72745895, -25.75965881, -34.31901932, -21.08712769, -16.40643311, -36.51839066, -30.18853760, -27.88558769, -23.38009453, -19.63435936, -35.52644348, -10.87827301, 20.88450241, -9.49922180, -2.24221802, 1.64629138, 29.64516830, -7.80061817, 5.89254475, -24.37758255, -11.45957851], [-26.52582741, -3.93510199, -6.41698503, -20.35298538, -20.99528885, -6.70626211, -16.09720039, -15.72246647, -17.80824661, -23.99833107, -11.98601246, -10.19097233, 2.43953443, -9.03533077, -2.28550124, -9.02405930, -19.05976105, -7.56664419, -19.64624596, -25.21205330, -34.47130585, -25.68278503, -19.72745895, -25.75965881, -34.31901550, -21.08712578, -16.40643501, -36.51838684, -30.18853188, -27.88558388, -23.38009453, -19.63436317, -35.52643967, -10.87827301, 20.88450432, -9.49921989, -2.24221563, 1.64629233, 29.64516640, -7.80062008, 5.89254475, -24.37757874, -11.45957756]]], device='cuda:0') PGD attack margin (first 2 examles and 10 specs): tensor([[[56.17099380, 33.58026886, 36.06214905, 49.99815369, 50.64045715, 36.35142899, 45.74237061, 45.36763382, 47.45341492, 53.64349747]]], device='cuda:0') number of violation: 0 Attack finished in 16.8530 seconds. PGD attack failed /kaggle/working/alpha-beta-CROWN/auto_LiRPA/parse_graph.py:147: FutureWarning: 'torch.onnx.symbolic_helper._set_opset_version' is deprecated in version 1.13 and will be removed in version 1.14. Please remove its usage and avoid setting internal variables directly. _set_opset_version(12) /kaggle/working/alpha-beta-CROWN/auto_LiRPA/parse_graph.py:45: FutureWarning: 'torch.onnx._patch_torch._node_getitem' is deprecated in version 1.13 and will be removed in version 1.14. Please Internally use '_node_get' in symbolic_helper instead.. attrs = {k: n[k] for k in n.attributeNames()} Model prediction is: tensor([[-26.63533783, -5.86781883, -4.54763603, -21.69909477, -20.81317139, -0.61611140, -16.72828102, -13.13037777, -18.94099617, -26.45909882, -4.61056852, -12.54259109, -0.75518376, -16.09009552, -2.99634385, -11.60897160, -17.43187332, -9.52975750, -18.21758842, -29.91522217, -33.70221329, -23.20652962, -18.36230087, -25.41592407, -33.91773987, -21.16673660, -17.64099312, -39.80070496, -35.47123337, -31.05479431, -22.17935181, -18.36315536, -43.14264679, -13.38010883, 12.96510029, -20.21791458, -4.28863621, -6.38604784, 41.21236038, -6.59563351, 4.28117037, -27.70765305, -9.50008297]], device='cuda:0') /kaggle/working/alpha-beta-CROWN/auto_LiRPA/bound_general.py:946: UserWarning: Creating an identity matrix with size 8192x8192 for node BoundMaxPool(name="/input"). This may indicate poor performance for bound computation. If you see this message on a small network please submit a bug report. ref_intermediate_lb, ref_intermediate_ub) /kaggle/working/alpha-beta-CROWN/auto_LiRPA/bound_general.py:946: UserWarning: Creating an identity matrix with size 4096x4096 for node BoundMaxPool(name="/input.4"). This may indicate poor performance for bound computation. If you see this message on a small network please submit a bug report. ref_intermediate_lb, ref_intermediate_ub) layer /12 using sparse-features alpha with shape [11222]; unstable size 11222; total size 32768 (torch.Size([1, 32, 32, 32])) layer /12 start_node /input using full alpha with unstable size None total_size 8192 output_shape torch.Size([32, 16, 16]) layer /12 start_node /x.3 using full alpha with unstable size 64 total_size 64 output_shape 64 layer /12 start_node /input.4 using full alpha with unstable size None total_size 4096 output_shape torch.Size([64, 8, 8]) layer /12 start_node /x.7 using full alpha with unstable size 128 total_size 128 output_shape 128 layer /12 start_node /x.11 using full alpha with unstable size 1024 total_size 1024 output_shape torch.Size([1024]) layer /12 start_node /28 using full alpha with unstable size None total_size 42 output_shape 42 Traceback (most recent call last): File "/kaggle/working/alpha-beta-CROWN/complete_verifier/abcrown.py", line 647, in <module> main() File "/kaggle/working/alpha-beta-CROWN/complete_verifier/abcrown.py", line 535, in main incomplete_verifier(model_ori, x, data_ub=data_max, data_lb=data_min, vnnlib=vnnlib) File "/kaggle/working/alpha-beta-CROWN/complete_verifier/abcrown.py", line 69, in incomplete_verifier domain, x, data_lb, data_ub, vnnlib, stop_criterion_func=stop_criterion_min(arguments.Config["bab"]["decision_thresh"])) File "/kaggle/working/alpha-beta-CROWN/complete_verifier/beta_CROWN_solver.py", line 1069, in build_the_model (self.x,), share_slopes=share_slopes, c=self.c, bound_upper=False) File "/kaggle/working/alpha-beta-CROWN/auto_LiRPA/optimized_bounds.py", line 1050, in init_slope final_node_name=final_node_name) File "/kaggle/working/alpha-beta-CROWN/auto_LiRPA/backward_bound.py", line 760, in get_alpha_crown_start_nodes assert not sparse_intermediate_bounds or use_sparse_conv is False # Double check our assumption holds. If this fails, then we created wrong shapes for alpha. AssertionError
To solve this issue, I tried to change full_conv_alpha, conv_mode, or share_alpha parameters but I got another error with those:
/kaggle/working /opt/conda/lib/python3.7/site-packages/onnx/mapping.py:27: DeprecationWarning: `np.object` is a deprecated alias for the builtin `object`. To silence this warning, use `object` by itself. Doing this will not modify any behavior and is safe. Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations int(TensorProto.STRING): np.dtype(np.object) Configurations: general: device: cuda seed: 100 conv_mode: matrix deterministic: false double_fp: false loss_reduction_func: sum record_bounds: false sparse_alpha: false save_adv_example: false precompile_jit: false complete_verifier: bab enable_incomplete_verification: true csv_name: null results_file: out.txt root_path: '' model: name: 'Customized("custom_model_data", "get_model")' path: /kaggle/working/model_torch.pt onnx_path: null onnx_path_prefix: '' cache_onnx_conversion: false onnx_quirks: null input_shape: null onnx_loader: default_onnx_and_vnnlib_loader onnx_optimization_flags: none data: start: 0 end: 10 select_instance: null num_outputs: 43 mean: 0.0 std: 1.0 pkl_path: null dataset: 'Customized("custom_model_data", "tsr_data")' data_filter_path: null data_idx_file: null specification: type: lp robustness_type: verified-acc norm: .inf epsilon: 0.00784313725 vnnlib_path: null vnnlib_path_prefix: '' solver: batch_size: 256 min_batch_size_ratio: 0.1 use_float64_in_last_iteration: false early_stop_patience: 10 start_save_best: 0.5 bound_prop_method: alpha-crown prune_after_crown: false crown: batch_size: 1000000000 max_crown_size: 1000000000 alpha-crown: alpha: true lr_alpha: 0.1 iteration: 100 share_slopes: false no_joint_opt: false lr_decay: 0.98 full_conv_alpha: false beta-crown: lr_alpha: 0.01 lr_beta: 0.05 lr_decay: 0.98 optimizer: adam iteration: 20 beta: true beta_warmup: true enable_opt_interm_bounds: false all_node_split_LP: false forward: refine: false dynamic: false max_dim: 10000 multi_class: multi_class_method: allclass_domain label_batch_size: 32 skip_with_refined_bound: true mip: parallel_solvers: null solver_threads: 1 refine_neuron_timeout: 15 refine_neuron_time_percentage: 0.8 early_stop: true adv_warmup: true mip_solver: gurobi bab: initial_max_domains: 1 max_domains: .inf decision_thresh: 0 timeout: 300 timeout_scale: 1 override_timeout: null get_upper_bound: false dfs_percent: 0.0 pruning_in_iteration: true pruning_in_iteration_ratio: 0.2 sort_targets: false batched_domain_list: true optimized_intermediate_layers: '' interm_transfer: true cut: enabled: false bab_cut: false lp_cut: false method: null lr: 0.01 lr_decay: 1.0 iteration: 100 bab_iteration: -1 early_stop_patience: -1 lr_beta: 0.02 number_cuts: 50 topk_cuts_in_filter: 100 batch_size_primal: 100 max_num: 1000000000 patches_cut: false cplex_cuts: false cplex_cuts_wait: 0 cplex_cuts_revpickup: true cut_reference_bounds: true fix_intermediate_bounds: false branching: method: kfsb candidates: 3 reduceop: min sb_coeff_thresh: 0.001 input_split: enable: false enhanced_bound_prop_method: alpha-crown enhanced_branching_method: naive enhanced_bound_patience: 100000000.0 attack_patience: 100000000.0 adv_check: 0 sort_domain_interval: -1 attack: enabled: false beam_candidates: 8 beam_depth: 7 max_dive_fix_ratio: 0.8 min_local_free_ratio: 0.2 mip_start_iteration: 5 mip_timeout: 30.0 adv_pool_threshold: null refined_mip_attacker: false refined_batch_size: null attack: pgd_order: before pgd_steps: 100 pgd_restarts: 100 pgd_early_stop: true pgd_lr_decay: 0.99 pgd_alpha: auto pgd_loss_mode: null enable_mip_attack: false cex_path: ./test_cex.txt attack_mode: PGD gama_lambda: 10.0 gama_decay: 0.9 check_clean: false input_split: pgd_steps: 100 pgd_restarts: 30 pgd_alpha: auto input_split_enhanced: pgd_steps: 200 pgd_restarts: 5000000 pgd_alpha: auto input_split_check_adv: pgd_steps: 5 pgd_restarts: 5 pgd_alpha: auto debug: lp_test: null Experiments at Mon May 1 17:49:40 2023 on d2707ae0421e CNN( (conv1): Conv2d(1, 32, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2)) (conv2): Conv2d(32, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2)) (conv3): Conv2d(64, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2)) (fc4): Linear(in_features=3584, out_features=1024, bias=True) (out): Linear(in_features=1024, out_features=43, bias=True) ) /kaggle/working/alpha-beta-CROWN/complete_verifier/custom_model_data.py:211: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor). return torch.tensor(X).permute((0,3,1,2)), torch.argmax(torch.tensor(labels), dim=-1), data_max, data_min, ret_eps Internal results will be saved to Verified_ret_[Customized_model]_start=0_end=10_iter=20_b=256_timeout=300_branching=kfsb-min-3_lra-init=0.1_lra=0.01_lrb=0.05_PGD=before_cplex_cuts=False_multiclass=allclass_domain.npy. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% idx: 0, vnnlib ID: 0 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% Attack parameters: initialization=uniform, steps=100, restarts=100, alpha=0.007461458444595337, initialization=uniform, GAMA=False Model output of first 5 examples: tensor([[-26.63533783, -5.86781883, -4.54763603, -21.69909477, -20.81317139, -0.61611140, -16.72828102, -13.13037777, -18.94099617, -26.45909882, -4.61056852, -12.54259109, -0.75518376, -16.09009552, -2.99634385, -11.60897160, -17.43187332, -9.52975750, -18.21758842, -29.91522217, -33.70221329, -23.20652962, -18.36230087, -25.41592407, -33.91773987, -21.16673660, -17.64099312, -39.80070496, -35.47123337, -31.05479431, -22.17935181, -18.36315536, -43.14264679, -13.38010883, 12.96510029, -20.21791458, -4.28863621, -6.38604784, 41.21236038, -6.59563351, 4.28117037, -27.70765305, -9.50008297]], device='cuda:0') Adv example prediction (first 2 examples and 2 restarts): tensor([[[-26.52582550, -3.93510056, -6.41698170, -20.35298538, -20.99528694, -6.70625925, -16.09720039, -15.72246552, -17.80824852, -23.99833107, -11.98600864, -10.19097519, 2.43953228, -9.03533268, -2.28550172, -9.02405930, -19.05975914, -7.56664610, -19.64624596, -25.21205711, -34.47130585, -25.68278313, -19.72745895, -25.75965881, -34.31901932, -21.08712769, -16.40643311, -36.51839066, -30.18853760, -27.88558769, -23.38009453, -19.63435936, -35.52644348, -10.87827301, 20.88450241, -9.49922180, -2.24221802, 1.64629138, 29.64516830, -7.80061817, 5.89254475, -24.37758255, -11.45957851], [-26.52582741, -3.93510199, -6.41698503, -20.35298538, -20.99528885, -6.70626211, -16.09720039, -15.72246647, -17.80824661, -23.99833107, -11.98601246, -10.19097233, 2.43953443, -9.03533077, -2.28550124, -9.02405930, -19.05976105, -7.56664419, -19.64624596, -25.21205330, -34.47130585, -25.68278503, -19.72745895, -25.75965881, -34.31901550, -21.08712578, -16.40643501, -36.51838684, -30.18853188, -27.88558388, -23.38009453, -19.63436317, -35.52643967, -10.87827301, 20.88450432, -9.49921989, -2.24221563, 1.64629233, 29.64516640, -7.80062008, 5.89254475, -24.37757874, -11.45957756]]], device='cuda:0') PGD attack margin (first 2 examles and 10 specs): tensor([[[56.17099380, 33.58026886, 36.06214905, 49.99815369, 50.64045715, 36.35142899, 45.74237061, 45.36763382, 47.45341492, 53.64349747]]], device='cuda:0') number of violation: 0 Attack finished in 13.5393 seconds. PGD attack failed /kaggle/working/alpha-beta-CROWN/auto_LiRPA/parse_graph.py:147: FutureWarning: 'torch.onnx.symbolic_helper._set_opset_version' is deprecated in version 1.13 and will be removed in version 1.14. Please remove its usage and avoid setting internal variables directly. _set_opset_version(12) /kaggle/working/alpha-beta-CROWN/auto_LiRPA/parse_graph.py:45: FutureWarning: 'torch.onnx._patch_torch._node_getitem' is deprecated in version 1.13 and will be removed in version 1.14. Please Internally use '_node_get' in symbolic_helper instead.. attrs = {k: n[k] for k in n.attributeNames()} Model prediction is: tensor([[-26.63533783, -5.86781883, -4.54763603, -21.69909477, -20.81317139, -0.61611140, -16.72828102, -13.13037777, -18.94099617, -26.45909882, -4.61056852, -12.54259109, -0.75518376, -16.09009552, -2.99634385, -11.60897160, -17.43187332, -9.52975750, -18.21758842, -29.91522217, -33.70221329, -23.20652962, -18.36230087, -25.41592407, -33.91773987, -21.16673660, -17.64099312, -39.80070496, -35.47123337, -31.05479431, -22.17935181, -18.36315536, -43.14264679, -13.38010883, 12.96510029, -20.21791458, -4.28863621, -6.38604784, 41.21236038, -6.59563351, 4.28117037, -27.70765305, -9.50008297]], device='cuda:0') /kaggle/working/alpha-beta-CROWN/auto_LiRPA/bound_general.py:946: UserWarning: Creating an identity matrix with size 8192x8192 for node BoundMaxPool(name="/input"). This may indicate poor performance for bound computation. If you see this message on a small network please submit a bug report. ref_intermediate_lb, ref_intermediate_ub) /kaggle/working/alpha-beta-CROWN/auto_LiRPA/bound_general.py:946: UserWarning: Creating an identity matrix with size 16384x16384 for node BoundConv(name="/x.3"). This may indicate poor performance for bound computation. If you see this message on a small network please submit a bug report. ref_intermediate_lb, ref_intermediate_ub) /kaggle/working/alpha-beta-CROWN/auto_LiRPA/bound_general.py:946: UserWarning: Creating an identity matrix with size 4096x4096 for node BoundMaxPool(name="/input.4"). This may indicate poor performance for bound computation. If you see this message on a small network please submit a bug report. ref_intermediate_lb, ref_intermediate_ub) /kaggle/working/alpha-beta-CROWN/auto_LiRPA/bound_general.py:946: UserWarning: Creating an identity matrix with size 8192x8192 for node BoundConv(name="/x.7"). This may indicate poor performance for bound computation. If you see this message on a small network please submit a bug report. ref_intermediate_lb, ref_intermediate_ub) layer /12 start_node /input using full alpha with unstable size None total_size 8192 output_shape torch.Size([32, 16, 16]) layer /12 start_node /x.3 using full alpha with unstable size None total_size 16384 output_shape torch.Size([64, 16, 16]) layer /12 start_node /input.4 using full alpha with unstable size None total_size 4096 output_shape torch.Size([64, 8, 8]) layer /12 start_node /x.7 using full alpha with unstable size None total_size 8192 output_shape torch.Size([128, 8, 8]) layer /12 start_node /x.11 using full alpha with unstable size 1024 total_size 1024 output_shape torch.Size([1024]) layer /12 start_node /28 using full alpha with unstable size None total_size 42 output_shape 42 Traceback (most recent call last): File "/kaggle/working/alpha-beta-CROWN/complete_verifier/abcrown.py", line 647, in <module> main() File "/kaggle/working/alpha-beta-CROWN/complete_verifier/abcrown.py", line 535, in main incomplete_verifier(model_ori, x, data_ub=data_max, data_lb=data_min, vnnlib=vnnlib) File "/kaggle/working/alpha-beta-CROWN/complete_verifier/abcrown.py", line 69, in incomplete_verifier domain, x, data_lb, data_ub, vnnlib, stop_criterion_func=stop_criterion_min(arguments.Config["bab"]["decision_thresh"])) File "/kaggle/working/alpha-beta-CROWN/complete_verifier/beta_CROWN_solver.py", line 1069, in build_the_model (self.x,), share_slopes=share_slopes, c=self.c, bound_upper=False) File "/kaggle/working/alpha-beta-CROWN/auto_LiRPA/optimized_bounds.py", line 1054, in init_slope node.init_opt_parameters(start_nodes) File "/kaggle/working/alpha-beta-CROWN/auto_LiRPA/operators/pooling.py", line 54, in init_opt_parameters dtype=torch.float, device=ref.device, requires_grad=True) TypeError: empty(): argument 'size' must be tuple of SymInts, but found element of type torch.Size at pos 2
I could not solve the problem and I guess the second one is not the way to go. I am in need of your help. Sorry for the long text. Thanks in advance
env.txt
The text was updated successfully, but these errors were encountered:
Would be great to have an opinion on this..
Thanks again
Sorry, something went wrong.
Hi @yusiyoh , this issue should have been fixed in our internal development version which will be released soon.
No branches or pull requests
Firstly, thank you for your amazing work and for sharing it with us.
I try to use your framework to run robustness tests on a very simple convnet for classification task. The implementation is as following:
My loaded data is actually an np array of shape [N, H, W, C] where C is 1 since it is in grayscale. The labels are classes that range from 0 to 42.
When I run alpha-beta-CROWN with the attached configuration, I got the following output:
To solve this issue, I tried to change full_conv_alpha, conv_mode, or share_alpha parameters but I got another error with those:
I could not solve the problem and I guess the second one is not the way to go. I am in need of your help.
Sorry for the long text. Thanks in advance
env.txt
The text was updated successfully, but these errors were encountered: