Skip to content

Commit

Permalink
update inference utils
Browse files Browse the repository at this point in the history
  • Loading branch information
@lwsong
lwsong committed Nov 25, 2019
1 parent 9a33d3b commit 785432e
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 37 deletions.
2 changes: 1 addition & 1 deletion inference_utils.py
View file
Expand Up @@ -41,5 +41,5 @@ def inference_via_confidence(confidence_mtx1, confidence_mtx2, label_vec1, label
max_accuracy = accuracy_now
best_precision = ratio1/(ratio1+ratio2)
best_recall = ratio1
print('maximum inference accuracy is:', max_accuracy)
print('membership inference accuracy is:', max_accuracy)
return max_accuracy
72 changes: 36 additions & 36 deletions membership_inference_results.ipynb
View file
Expand Up @@ -27,150 +27,150 @@
"########### when naturally train the model ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (1.0, 0.9824902723735408)\n",
"maximum inference accuracy is: 0.5585306164243293\n",
"membership inference accuracy is: 0.5585306164243293\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.045263157894736845, 0.029182879377431907)\n",
"maximum inference accuracy is: 0.5427329510546794\n",
"membership inference accuracy is: 0.5427329510546794\n",
"\n",
"########### when PGD-based adversarial training is adopted ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (0.9989473684210526, 0.9669260700389105)\n",
"maximum inference accuracy is: 0.6168687282408356\n",
"membership inference accuracy is: 0.6168687282408356\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.99, 0.7762645914396887)\n",
"maximum inference accuracy is: 0.6882561949621135\n",
"membership inference accuracy is: 0.6882561949621135\n",
"\n",
"########### when distributional adversarial training is adopted ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (0.9957894736842106, 0.9377431906614786)\n",
"maximum inference accuracy is: 0.6223202949006759\n",
"membership inference accuracy is: 0.6223202949006759\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.8326315789473684, 0.5505836575875487)\n",
"maximum inference accuracy is: 0.6406952693016588\n",
"membership inference accuracy is: 0.6406952693016588\n",
"\n",
"########### when difference-based adversarial training is adopted ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (0.9952631578947368, 0.9377431906614786)\n",
"maximum inference accuracy is: 0.5805519148064715\n",
"membership inference accuracy is: 0.5805519148064715\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.9942105263157894, 0.8385214007782101)\n",
"maximum inference accuracy is: 0.6558775343026828\n",
"membership inference accuracy is: 0.6558775343026828\n",
"\n",
"########### when duality-based verification is adopted ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (0.9889473684210527, 0.9280155642023347)\n",
"maximum inference accuracy is: 0.5589913987302887\n",
"membership inference accuracy is: 0.5589913987302887\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.9852631578947368, 0.8365758754863813)\n",
"maximum inference accuracy is: 0.6039596559492116\n",
"membership inference accuracy is: 0.6039596559492116\n",
"\n",
"########### when abstract interpretation-based verification is adopted ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (0.9926315789473684, 0.8326848249027238)\n",
"maximum inference accuracy is: 0.6510567274216671\n",
"membership inference accuracy is: 0.6510567274216671\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.8568421052631578, 0.5038910505836576)\n",
"maximum inference accuracy is: 0.6563721073110793\n",
"membership inference accuracy is: 0.6563721073110793\n",
"\n",
"########### when interval bound propagation-based verification is adopted ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (0.991578947368421, 0.857976653696498)\n",
"maximum inference accuracy is: 0.6045054269916035\n",
"membership inference accuracy is: 0.6045054269916035\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.9442105263157895, 0.6867704280155642)\n",
"maximum inference accuracy is: 0.6627769813639157\n",
"membership inference accuracy is: 0.6627769813639157\n",
"\n",
"\n",
"************************* For FMNIST dataset **************************************\n",
"########### when naturally train the model ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (1.0, 0.9218)\n",
"maximum inference accuracy is: 0.5711666666666667\n",
"membership inference accuracy is: 0.5711666666666667\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.0435, 0.0414)\n",
"maximum inference accuracy is: 0.50945\n",
"membership inference accuracy is: 0.50945\n",
"\n",
"########### when PGD-based adversarial training is adopted ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (0.9993333333333333, 0.9088)\n",
"maximum inference accuracy is: 0.5832166666666667\n",
"membership inference accuracy is: 0.5832166666666667\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.9691, 0.6806)\n",
"maximum inference accuracy is: 0.6449083333333333\n",
"membership inference accuracy is: 0.6449083333333333\n",
"\n",
"########### when distributional adversarial training is adopted ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (0.9798333333333333, 0.9062)\n",
"maximum inference accuracy is: 0.5734750000000001\n",
"membership inference accuracy is: 0.5734750000000001\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.6762666666666667, 0.5161)\n",
"maximum inference accuracy is: 0.5948833333333332\n",
"membership inference accuracy is: 0.5948833333333332\n",
"\n",
"########### when difference-based adversarial training is adopted ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (0.9934833333333334, 0.9092)\n",
"maximum inference accuracy is: 0.5702083333333333\n",
"membership inference accuracy is: 0.5702083333333333\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.9013, 0.724)\n",
"maximum inference accuracy is: 0.5882666666666666\n",
"membership inference accuracy is: 0.5882666666666666\n",
"\n",
"########### when duality-based verification is adopted ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (0.7512833333333333, 0.7429)\n",
"maximum inference accuracy is: 0.5057750000000001\n",
"membership inference accuracy is: 0.5057750000000001\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.65765, 0.6536)\n",
"maximum inference accuracy is: 0.5042416666666667\n",
"membership inference accuracy is: 0.5042416666666667\n",
"\n",
"########### when abstract interpretation-based verification is adopted ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (0.8644, 0.8547)\n",
"maximum inference accuracy is: 0.5079083333333334\n",
"membership inference accuracy is: 0.5079083333333334\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.74115, 0.7328)\n",
"maximum inference accuracy is: 0.5068583333333334\n",
"membership inference accuracy is: 0.5068583333333334\n",
"\n",
"########### when interval bound propagation-based verification is adopted ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (0.8984833333333333, 0.8626)\n",
"maximum inference accuracy is: 0.5213166666666667\n",
"membership inference accuracy is: 0.5213166666666667\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.8259666666666666, 0.7844)\n",
"maximum inference accuracy is: 0.520575\n",
"membership inference accuracy is: 0.520575\n",
"\n",
"\n",
"************************* For CIFAR10 dataset **************************************\n",
"########### when naturally train the model ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (1.0, 0.9501)\n",
"maximum inference accuracy is: 0.57427\n",
"membership inference accuracy is: 0.57427\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.0, 0.0)\n",
"maximum inference accuracy is: 0.50864\n",
"membership inference accuracy is: 0.50864\n",
"\n",
"########### when PGD-based adversarial training is adopted ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (0.99998, 0.8725)\n",
"maximum inference accuracy is: 0.74892\n",
"membership inference accuracy is: 0.74892\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.96078, 0.4661)\n",
"maximum inference accuracy is: 0.75665\n",
"membership inference accuracy is: 0.75665\n",
"\n",
"########### when distributional adversarial training is adopted ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (1.0, 0.901)\n",
"maximum inference accuracy is: 0.67159\n",
"membership inference accuracy is: 0.67159\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.40564, 0.2592)\n",
"maximum inference accuracy is: 0.6423500000000001\n",
"membership inference accuracy is: 0.6423500000000001\n",
"\n",
"########### when difference-based adversarial training is adopted ##############\n",
"-------with benign inputs-------\n",
"model accuracy for training and test- (0.99504, 0.8799)\n",
"maximum inference accuracy is: 0.61177\n",
"membership inference accuracy is: 0.61177\n",
"-------with adversarial inputs-------\n",
"model accuracy for training and test- (0.76058, 0.465)\n",
"maximum inference accuracy is: 0.67082\n",
"membership inference accuracy is: 0.67082\n",
"\n",
"\n"
]
Expand Down

0 comments on commit 785432e

Please sign in to comment.