diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e52f7cb --- /dev/null +++ b/.gitignore @@ -0,0 +1,104 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# MacOS +*.DS_Store diff --git a/requirement.txt b/requirement.txt new file mode 100644 index 0000000..5863c73 --- /dev/null +++ b/requirement.txt @@ -0,0 +1,7 @@ +keras +tensorflow +numpy +scipy +matplotlib +future +h5py \ No newline at end of file diff --git a/setup_cifar.py b/setup_cifar.py new file mode 100644 index 0000000..c8a99e0 --- /dev/null +++ b/setup_cifar.py @@ -0,0 +1,123 @@ +## setup_cifar.py -- cifar data and model loading code +## +## Copyright (C) 2016, Nicholas Carlini . +## +## This program is licenced under the BSD 2-Clause licence, +## contained in the LICENCE file in this directory. + + +import tensorflow as tf +import numpy as np +import os +import pickle +import gzip +import pickle +import urllib.request + +from keras.models import Sequential +from keras.layers import Dense, Dropout, Activation, Flatten +from keras.layers import Conv2D, MaxPooling2D +from keras.utils import np_utils +from keras.models import load_model + +def load_batch(fpath, label_key='labels'): + f = open(fpath, 'rb') + d = pickle.load(f, encoding="bytes") + for k, v in d.items(): + del(d[k]) + d[k.decode("utf8")] = v + f.close() + data = d["data"] + labels = d[label_key] + + data = data.reshape(data.shape[0], 3, 32, 32) + final = np.zeros((data.shape[0], 32, 32, 3),dtype=np.float32) + final[:,:,:,0] = data[:,0,:,:] + final[:,:,:,1] = data[:,1,:,:] + final[:,:,:,2] = data[:,2,:,:] + + final /= 255 + labels2 = np.zeros((len(labels), 10)) + labels2[np.arange(len(labels2)), labels] = 1 + + return final, labels + +def load_batch(fpath): + f = open(fpath,"rb").read() + size = 32*32*3+1 + labels = [] + images = [] + for i in range(10000): + arr = np.fromstring(f[i*size:(i+1)*size],dtype=np.uint8) + lab = np.identity(10)[arr[0]] + img = arr[1:].reshape((3,32,32)).transpose((1,2,0)) + + labels.append(lab) + images.append(img/255) + return np.array(images),np.array(labels) + + +class CIFAR: + def __init__(self): + train_data = [] + train_labels = [] + + if not os.path.exists("cifar-10-batches-bin"): + urllib.request.urlretrieve("https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz", + "cifar-data.tar.gz") + os.popen("tar -xzf cifar-data.tar.gz").read() + + + for i in range(5): + r,s = load_batch("cifar-10-batches-bin/data_batch_"+str(i+1)+".bin") + train_data.extend(r) + train_labels.extend(s) + + train_data = np.array(train_data,dtype=np.float32) + train_labels = np.array(train_labels) + + self.test_data, self.test_labels = load_batch("cifar-10-batches-bin/test_batch.bin") + + VALIDATION_SIZE = 5000 + + self.validation_data = train_data[:VALIDATION_SIZE, :, :, :] + self.validation_labels = train_labels[:VALIDATION_SIZE] + self.train_data = train_data[VALIDATION_SIZE:, :, :, :] + self.train_labels = train_labels[VALIDATION_SIZE:] + +class CIFARModel: + def __init__(self, restore, session=None): + self.num_channels = 3 + self.image_size = 32 + self.num_labels = 10 + + model = Sequential() + + model.add(Conv2D(64, (3, 3), + input_shape=(32, 32, 3))) + model.add(Activation('relu')) + model.add(Conv2D(64, (3, 3))) + model.add(Activation('relu')) + model.add(MaxPooling2D(pool_size=(2, 2))) + + model.add(Conv2D(128, (3, 3))) + model.add(Activation('relu')) + model.add(Conv2D(128, (3, 3))) + model.add(Activation('relu')) + model.add(MaxPooling2D(pool_size=(2, 2))) + + model.add(Flatten()) + model.add(Dense(256)) + model.add(Activation('relu')) + model.add(Dense(256)) + model.add(Activation('relu')) + model.add(Dense(10)) + + model.load_weights(restore) + + self.model = model + + def predict(self, data): + return self.model(data) + + diff --git a/setup_mnist.py b/setup_mnist.py index 6bf96f8..8c16b8f 100644 --- a/setup_mnist.py +++ b/setup_mnist.py @@ -7,6 +7,10 @@ ## Modified for MagNet's use. +from __future__ import print_function +from future.standard_library import install_aliases +install_aliases() + import numpy as np import os import gzip diff --git a/worker.py b/worker.py index 97296fb..ad68690 100644 --- a/worker.py +++ b/worker.py @@ -5,6 +5,8 @@ ## This program is licenced under the BSD 2-Clause licence, ## contained in the LICENCE file in this directory. +from __future__ import print_function + import matplotlib matplotlib.use('Agg') from scipy.stats import entropy @@ -19,6 +21,7 @@ from utils import prepare_data import utils import matplotlib.pyplot as plt +import six class AEDetector: @@ -218,7 +221,8 @@ def __init__(self, examples, labels, name=""): examples: Path or object of input examples. labels: Ground truth labels. """ - if isinstance(examples, str): self.data = utils.load_obj(examples) + + if isinstance(examples, six.string_types): self.data = utils.load_obj(examples) else: self.data = examples self.labels = labels self.name = name @@ -332,7 +336,7 @@ def plot_various_confidences(self, graph_name, drop_rate, none.append(none_acc) size = 2.5 - plt.plot(confs, none, c="green", label="No fefense", marker="x", markersize=size) + plt.plot(confs, none, c="green", label="No defense", marker="x", markersize=size) plt.plot(confs, det_only, c="orange", label="With detector", marker="o", markersize=size) plt.plot(confs, ref_only, c="blue", label="With reformer", marker="^", markersize=size) plt.plot(confs, both, c="red", label="With detector & reformer", marker="s", markersize=size)