Drop older versions (#938)

* remove raw keras imports * keras is now installed as part of tf * remove raw keras import from Travis * update copyright year * remove raw keras imports * remove python 2.7 from travis * update README * remove keras install instructions * remove raw keras import * fix import order * remove unused import * fix imports * remove unused import * fix travis keras setup command * it's always OK to import keras now that it's part of tf * pylint * fix imports * fix imports * fix import for tf 1.8 * fix tf 1.8 imports * fix another import * fix imports
cleverhans-lab · Jan 8, 2019 · e1f6a36 · e1f6a36
1 parent a7ccd7c
commit e1f6a36
Show file tree

Hide file tree

Showing 9 changed files with 58 additions and 78 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -3,7 +3,6 @@ sudo: required
 dist: trusty
 language: python
 python:
-  - 2.7
   - 3.5
 env:
   - KERAS_BACKEND=tensorflow TENSORFLOW_V=1.8.0 PYTORCH=False
@@ -12,13 +11,7 @@ env:
 
 install:
   # code below is taken from http://conda.pydata.org/docs/travis.html
-  # We do this conditionally because it saves us some downloading if the
-  # version is the same.
-  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
-      wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh;
-    else
-      wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
-    fi
+  - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
   - bash miniconda.sh -b -p $HOME/miniconda
   - export PATH="$HOME/miniconda/bin:$PATH"
   - hash -r
@@ -31,11 +24,7 @@ install:
   - source activate test-environment
 
   # install TensorFlow
-  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" && "$TENSORFLOW_V" == "1.8.0" ]]; then
-      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.8.0-cp27-none-linux_x86_64.whl;
-    elif [[ "$TRAVIS_PYTHON_VERSION" == "2.7" && "$TENSORFLOW_V" == "1.12.0" ]]; then
-      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.12.0rc1-cp27-none-linux_x86_64.whl;
-    elif [[ "$TRAVIS_PYTHON_VERSION" == "3.5" && "$TENSORFLOW_V" == "1.8.0" ]]; then
+  - if [[ "$TRAVIS_PYTHON_VERSION" == "3.5" && "$TENSORFLOW_V" == "1.8.0" ]]; then
       pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.8.0-cp35-cp35m-linux_x86_64.whl;
     elif [[ "$TRAVIS_PYTHON_VERSION" == "3.5" && "$TENSORFLOW_V" == "1.12.0" ]]; then
       pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.12.0rc1-cp35-cp35m-linux_x86_64.whl;
@@ -66,7 +55,7 @@ script:
   # exit on first error
   - set -e
   # run keras backend init to initialize backend config
-  - python -c "import keras.backend"
+  - python -c "from tensorflow import keras"
   # create dataset directory to avoid concurrent directory creation at runtime
   - mkdir ~/.keras/datasets
   # set up keras backend

diff --git a/LICENSE b/LICENSE
@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 Google Inc., OpenAI and Pennsylvania State University
+Copyright (c) 2019 Google Inc., OpenAI and Pennsylvania State University
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

diff --git a/README.md b/README.md
@@ -77,9 +77,18 @@ pip install -e .
 
 Although CleverHans is likely to work on many other machine configurations, we
 currently [test it](https://travis-ci.org/tensorflow/cleverhans) it with Python
-{2.7, 3.5} and TensorFlow {1.8, 1.12} on Ubuntu 14.04.5 LTS (Trusty Tahr).
-Support for TensorFlow prior to 1.8 is deprecated.
-Backwards compatibility wrappers for these versions may be removed after
+3.5 and TensorFlow {1.8, 1.12} on Ubuntu 14.04.5 LTS (Trusty Tahr).
+Support for Python 2.7 is deprecated.
+CleverHans 3.0.1 supports Python 2.7 and the master branch is likely to
+continue to work in Python 2.7 for some time, but we no longer run the tests
+in Python 2.7 and we do not plan to fix bugs affecting only Python 2.7 after
+2019-07-04.
+Support for TensorFlow prior to 1.12 is deprecated.
+Backwards compatibility wrappers for these versions may be removed after 2019-07-07,
+and we will not fix bugs for those versions after that date.
+Support for TensorFlow prior to 1.8 is deprecated with an earlier deprecation
+date:
+backwards compatibility wrappers for these versions may be removed after
 2019-01-26, and we will not fix bugs for those versions after that date.
 Support for TensorFlow 1.3 and earlier is already deprecated: we do not fix
 bugs for those versions and any remaining wrapper code for those versions
@@ -145,14 +154,6 @@ carefully by the adversary. The adversary then uses the substitute
 model’s gradients to find adversarial examples that are misclassified by the
 black-box model as well.
 
-Some models used in the tutorials are defined using [Keras](https://keras.io),
-which should be installed before running these tutorials.
-Installation instructions for Keras can be found
-[here](https://keras.io/#installation).
-Note that you should configure Keras to use the TensorFlow backend. You
-can find instructions for
-setting the Keras backend [on this page](https://keras.io/backend/).
-
 NOTE: the tutorials are maintained carefully, in the sense that we use
 continuous integration to make sure they continue working. They are not
 considered part of the API and they can change at any time without warning.

diff --git a/cleverhans/dataset.py b/cleverhans/dataset.py
@@ -15,13 +15,25 @@
 import struct
 import tempfile
 import sys
+import warnings
+
 import numpy as np
+from tensorflow import keras
 import tensorflow as tf
-
+try:
+  from tensorflow.python.keras.utils import np_utils
+  from tensorflow.keras.datasets import cifar10
+except ImportError:
+  # In tf 1.8, np_utils doesn't seem to be publicly exposed.
+  # In later tf versions, it is, and in pre-tf keras it was too.
+  from tensorflow.python.keras import _impl
+  np_utils = _impl.keras.utils.np_utils
+  # In tf 1.8, "from tensorflow.keras.datasets import cifar10" doesn't work even though the module exists
+  cifar10 = keras.datasets.cifar10
+  warnings.warn("Support for TensorFlow versions prior to 1.12 is deprecated."
+                " CleverHans using earlier versions may quit working on or after 2019-07-07.")
 from cleverhans import utils
 
-# Only load keras if user tries to use a dataset that requires it
-keras_imported = False
 
 
 class Dataset(object):
@@ -260,11 +272,6 @@ def data_cifar10(train_start=0, train_end=50000, test_start=0, test_end=10000):
   :return:
   """
 
-  global keras_imported
-  if not keras_imported:
-    import keras
-    from keras.datasets import cifar10
-    from keras.utils import np_utils
 
   # These values are specific to CIFAR10
   img_rows = 32

diff --git a/cleverhans/devtools/mocks.py b/cleverhans/devtools/mocks.py
@@ -8,10 +8,9 @@
 
 import copy
 
-from keras.utils import np_utils
 import numpy as np
 
-from cleverhans.dataset import Dataset
+from cleverhans.dataset import Dataset, np_utils
 
 
 def random_feed_dict(rng, placeholders):

diff --git a/cleverhans/utils_keras.py b/cleverhans/utils_keras.py
@@ -1,18 +1,17 @@
 """
 Model construction utilities based on keras
 """
-from distutils.version import LooseVersion
-import keras
-from keras.models import Sequential
-from keras.layers import Dense, Activation, Flatten
+from tensorflow import keras
 
 from .model import Model, NoSuchLayerError
 
-if LooseVersion(keras.__version__) >= LooseVersion('2.0.0'):
-  from keras.layers import Conv2D
-else:
-  from keras.layers import Convolution2D
-
+# Assignment rather than import because direct import from within Keras doesn't work in tf 1.8
+Sequential = keras.models.Sequential
+Conv2D = keras.layers.Conv2D
+Dense = keras.layers.Dense
+Activation = keras.layers.Activation
+Flatten = keras.layers.Flatten
+KerasModel = keras.models.Model
 
 def conv_2d(filters, kernel_shape, strides, padding, input_shape=None):
   """
@@ -31,22 +30,13 @@ def conv_2d(filters, kernel_shape, strides, padding, input_shape=None):
                       layer of the model
   :return: the Keras layer
   """
-  if LooseVersion(keras.__version__) >= LooseVersion('2.0.0'):
-    if input_shape is not None:
-      return Conv2D(filters=filters, kernel_size=kernel_shape,
-                    strides=strides, padding=padding,
-                    input_shape=input_shape)
-    else:
-      return Conv2D(filters=filters, kernel_size=kernel_shape,
-                    strides=strides, padding=padding)
+  if input_shape is not None:
+    return Conv2D(filters=filters, kernel_size=kernel_shape,
+                  strides=strides, padding=padding,
+                  input_shape=input_shape)
   else:
-    if input_shape is not None:
-      return Convolution2D(filters, kernel_shape[0], kernel_shape[1],
-                           subsample=strides, border_mode=padding,
-                           input_shape=input_shape)
-    else:
-      return Convolution2D(filters, kernel_shape[0], kernel_shape[1],
-                           subsample=strides, border_mode=padding)
+    return Conv2D(filters=filters, kernel_size=kernel_shape,
+                  strides=strides, padding=padding)
 
 
 def cnn_model(logits=False, input_ph=None, img_rows=28, img_cols=28,
@@ -69,9 +59,10 @@ def cnn_model(logits=False, input_ph=None, img_rows=28, img_cols=28,
   model = Sequential()
 
   # Define the layers successively (convolution layers are version dependent)
-  if keras.backend.image_dim_ordering() == 'th':
+  if keras.backend.image_data_format() == 'channels_first':
     input_shape = (channels, img_rows, img_cols)
   else:
+    assert keras.backend.image_data_format() == 'channels_last'
     input_shape = (img_rows, img_cols, channels)
 
   layers = [conv_2d(nb_filters, (8, 8), (2, 2), "same",
@@ -192,7 +183,6 @@ def fprop(self, x):
     :return: A dictionary mapping layer names to the symbolic
              representation of their output.
     """
-    from keras.models import Model as KerasModel
 
     if self.keras_model is None:
       # Get the input layer

diff --git a/cleverhans_tutorials/mnist_tutorial_keras_tf.py b/cleverhans_tutorials/mnist_tutorial_keras_tf.py
@@ -14,10 +14,9 @@
 import os
 
 import tensorflow as tf
+from tensorflow import keras
 from tensorflow.python.platform import flags
 import numpy as np
-import keras
-from keras import backend
 
 from cleverhans.attacks import FastGradientMethod
 from cleverhans.dataset import MNIST
@@ -59,22 +58,16 @@ def mnist_tutorial(train_start=0, train_end=60000, test_start=0,
   :param label_smoothing: float, amount of label smoothing for cross entropy
   :return: an AccuracyReport object
   """
-  keras.layers.core.K.set_learning_phase(0)
+  tf.keras.backend.set_learning_phase(0)
 
   # Object used to keep track of (and return) key accuracies
   report = AccuracyReport()
 
   # Set TF random seed to improve reproducibility
   tf.set_random_seed(1234)
 
-  if not hasattr(backend, "tf"):
-    raise RuntimeError("This tutorial requires keras to be configured"
-                       " to use the TensorFlow backend.")
-
-  if keras.backend.image_dim_ordering() != 'tf':
-    keras.backend.set_image_dim_ordering('tf')
-    print("INFO: '~/.keras/keras.json' sets 'image_dim_ordering' to "
-          "'th', temporarily setting to 'tf'")
+  if keras.backend.image_data_format() != 'channels_last':
+    raise NotImplementedError("this tutorial requires keras to be configured to channels_last format")
 
   # Create TF session and set as Keras backend session
   sess = tf.Session()

diff --git a/setup.py b/setup.py
@@ -19,9 +19,6 @@
       extras_require={
           "tf": ["tensorflow>=1.0.0"],
           "tf_gpu": ["tensorflow-gpu>=1.0.0"],
-          "test": [
-              "keras == 2.1.5",  # Keras 2.1.6 is incompatible with TF 1.4
-          ],
           "pytorch": ["torch==0.4.0", "torchvision==0.2.1"],
       },
       packages=find_packages())
diff --git a/tests_tf/test_utils_keras.py b/tests_tf/test_utils_keras.py
@@ -8,11 +8,15 @@
 
 from cleverhans.utils_keras import KerasModelWrapper
 
+# Weird imports / assignment because the normal import syntax doesn't work for tf.keras in tf 1.8
+from tensorflow import keras
+Sequential = keras.models.Sequential
+Dense = keras.layers.Dense
+Activation = keras.layers.Activation
+
 
 class TestKerasModelWrapper(unittest.TestCase):
   def setUp(self):
-    from keras.models import Sequential
-    from keras.layers import Dense, Activation
     import tensorflow as tf
 
     def dummy_model():