Merge branch 'master' into 'stable'

Release v0.11

See merge request secml/secml!8

.gitlab-ci.yml

@@ -1,8 +1,9 @@
 variables:
     PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+    TORCH_HOME: "$CI_PROJECT_DIR/.cache/torch"
     SECML_HOME_DIR: "$CI_PROJECT_DIR/secml-data"
     TOX_WORKDIR: "$CI_PROJECT_DIR/.tox"
-    TOX_TESTENV_PASSENV: "PIP_CACHE_DIR SECML_HOME_DIR"
+    TOX_TESTENV_PASSENV: "PIP_CACHE_DIR TORCH_HOME SECML_HOME_DIR"
     TOX_USEDEVELOP: "False"
 
 stages:
@@ -15,13 +16,9 @@ stages:
 .test-cache:
   cache: &test-cache
     paths:
-      - "$CI_PROJECT_DIR/.cache/pip"
-      - "$CI_PROJECT_DIR/secml-data"
-
-.test-cache-py2:
-  cache: &test-cache-py2
-    key: "test-cache-py2"
-    <<: *test-cache
+      - $PIP_CACHE_DIR
+      - $TORCH_HOME
+      - $SECML_HOME_DIR
 
 .test-cache-py35:
   cache: &test-cache-py35
@@ -128,12 +125,6 @@ package:docs:
     variables:
       - $SKIP_TESTS
 
-test:py2:min:
-  extends: .test
-  image: python:2
-  script: tox -e py2-min
-  cache: *test-cache-py2
-
 test:py35:latest:
   extends: .test
   image: python:3.5
@@ -159,6 +150,25 @@ test:py37:min:
   cache: *test-cache-py37
 
 
+.test:notebooks:
+  extends: .test
+  needs: ["package:docs"]  # Use [] after gitlab 12.6
+  only:
+    changes:
+      - src/secml/**/*
+      - tutorials/**/*
+      - .gitlab-ci.yml
+      - requirements.txt
+      - setup.py
+      - tox.ini
+
+test:notebooks:py35:latest:
+  extends: .test:notebooks
+  image: python:3.5
+  script: tox -e latest-notebooks
+  cache: *test-cache-py35
+
+
 .test:install:
   stage: test
   needs: ["package"]
@@ -206,14 +216,7 @@ test:py37:min:
   after_script:
     - python -c "import secml"
     - python -c "from secml.ml.classifiers import CClassifierPyTorch"
-    - python -c "from secml.ml.classifiers import CModelCleverhans"
-
-test:install:whl:py2:
-  extends: .test:install:whl
-  image: python:2
-  cache:
-    <<: *test-cache-py2
-    policy: pull
+    - python -c "from secml.adv.attacks.evasion import CAttackEvasionCleverhans"
 
 test:install:whl:py35:
   extends: .test:install:whl
@@ -236,13 +239,6 @@ test:install:whl:py37:
     <<: *test-cache-py37
     policy: pull
 
-test:install:whl:extras:py2:
-  extends: .test:install:whl:extras
-  image: python:2
-  cache:
-    <<: *test-cache-py2
-    policy: pull
-
 test:install:whl:extras:py35:
   extends: .test:install:whl:extras
   image: python:3.5

CHANGELOG.md

@@ -1,6 +1,57 @@
+## v0.11 (02/12/2019)
+- #653 Added new `secml.ml.model_zoo` package, which provides a zoo of pre-trained SecML models. The list of available models will be greatly expanded in the future. See https://secml.gitlab.io/secml.ml.model_zoo.html for more details.
+- #629 Greatly improved the performance of the `grad_f_x` method for `CClassifier` and `CPreProcess` classes, especially when nested via `preprocess` attribute.
+- #613 Support for Python 2.7 is dropped. Python version 3.5, 3.6, or 3.7 is now required.
+
+### Requirements (2 changes)
+- #633 The following dependencies are now required: `numpy >= 1.17`, `scipy >= 1.3.1`, `scikit-learn >= 0.21` `matplotlib = 3`.
+- #622 Removed dependency on `six` library.
+
+### Added (5 changes)
+- #539 Added new core interface to get and set the state of an object instance: `set_state`, `get_state`, `save_state`, `load_state`. The state of an object is a simple human-readable Python dictionary object which stores the data necessary to restore an instance to a specific status. Please not that to guarantee the exact match between the original object instance and the restored one, the standard save/load interface should be used.
+- #647 Added new function `core.attr_utils.get_protected` which returns a protected attribute from a class (if exists).
+- #629 `CClassifier` and `CPreProcess` classes now provide a `gradient` method, which computes the gradient by doing a forward and a backward pass on the classifier or preprocessor function chain, accepting an optional pre-multiplier `w`.
+- #539 Added new accessible attributes to multiple classes: `CNormalizerMinMax .m .q`; `CReducerLDA .lda`; `CClassifierKNN .tr`; `CClassifierRidge .tr`; `CClassifierSGD .tr`; `CClassifierPyTorch .trained`.
+- #640 Added `random_state` parameter to `CClassifierDecisionTree`.
+
+### Improved (6 changes)
+- #631 Data objects are now stored using protocol 4 by `pickle_utils.save`. This protocol adds support for very large objects, pickling more kinds of objects, and some data format optimizations.
+- #639 Objective function parameter (`objective_function`) in `CAttackEvasionCleverhans` is now correctly populated for the following attacks: `CarliniWagnerL2`, `FastGradientMethod`, `ProjectedGradientDescent`, `LBFGS`, `MomentumIterativeMethod`, `MadryEtAl`, `BasicIterativeMethod`.
+- #638 The sequence of modifications to the attack point (`x_seq` parameter) is now correctly populated in `CAttackEvasionCleverhans`.
+- #595 A pre-trained classifier can now be passed to `CClassifierRejectThreshold` to avoid running fit twice.
+- #627 Slight improvement of `CKernel.gradient()` method performance by removing unnecessary calls.
+- #630 Sparse data can now be used in `CKernelHistIntersect`.
+
+### Changed (2 changes)
+- #616 Renamed `CModelCleverhans` to `_CModelCleverhans` as this class is not supposed to be explicitly used.
+- #111 Default value of the parameter `tol` changed from `-inf` to `None` in `CClassifierSGD`. This change should not alter the classifier behavior when using the default parameters.
+
+### Fixed (8 changes)
+- #611 Fixed `CDataloaderMNIST` crashing depending on the desired number of samples and digits to load.
+- #652 Number of gradient computations returned by `CAttackEvasionCleverhans.grad_eval` is now accurate.
+- #650 Fixed `CAttackEvasionCleverhans.f_eval` wrongly returns the number of gradient evaluations.
+- #637 Fixed checks on `y_taget` in `CAttackEvasionCleverhans` which compared the 0 label to untargeted case (`y_true = None`).
+- #648 Function `core.attr_utils.is_public` now correctly return False for properties.
+- #649 Fixed wrong use of `core.attr_utils.is_public` in `CCreator` and `CDatasetHeader`.
+- #655 Fixed `CClassifierRejectThreshold.n_classes` not taking into account the rejected class (label -1).
+- #636 Fixed a `TypeError` raised by `CFigure.clabel()` when using matplotlib 3.
+
+### Removed & Deprecated (4 changes)
+- #628 Method `is_linear` of `CClassifier` and `CNormalizer` subclasses is now deprecated.
+- #641 Parameter `random_seed` of `CClassifierLogistic` is now deprecated. Use `random_state` instead.
+- #603 Removed deprecated class `CNormalizerMeanSTD`.
+- #603 Removed deprecated parameter `batch_size` from `CKernel` and subclasses.
+
+### Documentation (4 changes)
+- #625 Reorganized notebooks tutorials into different categories: *Machine Learning*, *Adversarial Machine Learning*, and *Explainable Machine Learning*.
+- #615 Added a tutorial notebook on the use of Cleverhans library wrapper.
+- #607 Settings module `secml.settings` is now correctly displayed in the docs.
+- #626 Added missing reference to `CPlotMetric` class in docs.
+
+
 ## v0.10 (29/10/2019)
 - #535 Added new package `secml.explanation`, which provides different methods for explaining machine learning models. See documentation and examples for more information.
-- #584 **[beta]** Added `CModelCleverhans` and `CAttackEvasionCleverhans` to support adversarial attacks from [CleverHans](https://github.com/tensorflow/cleverhans), a Python library to benchmark vulnerability of machine learning systems to adversarial examples.
+- #584 **[beta]** Added `CAttackEvasionCleverhans` to support adversarial attacks from [CleverHans](https://github.com/tensorflow/cleverhans), a Python library to benchmark vulnerability of machine learning systems to adversarial examples.
 
 ### Requirements (1 change)
 - #580 PyTorch version `1.3` is now supported.
@@ -36,7 +87,7 @@
 - #575 Parameter `batch_size` of `CKernel` is now deprecated.
 - #597 Removed unused parameter `gamma` from `CKernelChebyshevDistance`.
 - #596 Removed `CKernelHamming`.
-- #602 Renamed `CNormalizerMeanSTD` to `CNormalizerMeanStd`. The old class has been deprecated and will be removed in a future vearsion.
+- #602 Renamed `CNormalizerMeanSTD` to `CNormalizerMeanStd`. The old class has been deprecated and will be removed in a future version.
 
 ### Documentation (5 changes)
 - #538 Added a notebook tutorial on the use of Explainable ML methods provided by the `secml.explanation` package.

MANIFEST.in

@@ -4,4 +4,5 @@ include update/*
 include requirements.txt
 include src/secml/VERSION
 include src/secml/VERSION_REV
-include src/secml/secml.conf
+include src/secml/secml.conf
+include src/secml/ml/model_zoo/models_dict.json

README.md

@@ -31,7 +31,7 @@ Please see our [ROADMAP](https://secml.gitlab.io/roadmap.html) for an overview
 of the future development directions.
 
 [![Status Alpha](https://img.shields.io/badge/status-alpha-yellow.svg)](.)
-[![Python 2.7 | 3.5 | 3.6 | 3.7](https://img.shields.io/badge/python-2.7%20%7C%203.5%20%7C%203.6%20%7C%203.7-brightgreen.svg)](.)
+[![Python 3.5 | 3.6 | 3.7](https://img.shields.io/badge/python-3.5%20%7C%203.6%20%7C%203.7-brightgreen.svg)](.)
 [![Platform Linux | MacOS ](https://img.shields.io/badge/platform-linux%20%7C%20macos-lightgrey.svg)](.)
 [![Apache License 2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0)
 
@@ -47,16 +47,15 @@ procedure.
 
 ### Operating System requirements
 
-SecML can run under Python 2.7 and Python >= 3.5 with no configuration steps 
-required, as all its dependencies are available as wheel packages for the main 
-macOS versions and Linux distributions.
+SecML can run under Python >= 3.5 with no additional configuration steps 
+required, as all its dependencies are available as wheel packages for 
+the primary macOS versions and Linux distributions.
 
 However, to support additional advanced features more packages can be necessary
 depending on the Operating System used:
 
  - Linux (Ubuntu >= 16.04 or equivalent dist)
-   - `python-tk` (Python 2.7), `python3-tk` (Python >= 3.5), for running
-     MatplotLib Tk-based backends;
+   - `python3-tk`, for running MatplotLib Tk-based backends;
    - NVIDIA<sup>®</sup> CUDA<sup>®</sup> Toolkit for running `tf-gpu`
      [extra component](#extra-components).
      See the [TensorFlow Guide](https://www.tensorflow.org/install/gpu).

docs/source/index.rst

@@ -3,11 +3,12 @@
 .. toctree::
    :hidden:
    :caption: User Guide
-   :titlesonly:
+   :maxdepth: 2
    :numbered:
-   :glob:
 
-   tutorials/*
+   tutorials.ml
+   tutorials.adv
+   tutorials/10-Explanation
 
 .. toctree::
    :hidden:

docs/source/pyplots/bar.py

@@ -13,12 +13,12 @@
 fig.sp.bar(X, -Y2, facecolor='#ff9999', edgecolor='white')
 
 for x, y in zip(X, Y1):
-    fig.sp.text(x + 0.4, y + 0.05, '%.2f' % y, ha='center', va='bottom')
+    fig.sp.text(x, y, '%.2f' % y, ha='center', va='bottom')
 
 for x, y in zip(X, Y2):
-    fig.sp.text(x + 0.4, -y - 0.05, '%.2f' % y, ha='center', va='top')
+    fig.sp.text(x, -y - 0.02, '%.2f' % y, ha='center', va='top')
 
-fig.sp.xlim(-.5, n)
+fig.sp.xlim(-.5, n-.5)
 fig.sp.xticks(())
 fig.sp.ylim(-1.25, 1.25)
 fig.sp.yticks(())

docs/source/pyplots/clabel.py

@@ -5,14 +5,15 @@
 def f(x, y):
     return (1 - x / 2 + x ** 5 + y ** 3) * (-x ** 2 - y ** 2).exp()
 
+
 fig = CFigure()
 
 x_linspace = CArray.linspace(-3, 3, 256)
 y_linspace = CArray.linspace(-3, 3, 256)
 
 X, Y = CArray.meshgrid((x_linspace, y_linspace))
 
-C = fig.sp.contour(X, Y, f(X, Y), linewidth=.5, cmap='hot')
+C = fig.sp.contour(X, Y, f(X, Y), linewidths=.5, cmap='hot')
 fig.sp.clabel(C, inline=1, fontsize=10)
 
 fig.sp.xticks(())

docs/source/pyplots/hist.py

@@ -9,7 +9,7 @@
 x = mu + sigma * CArray.randn((10000,))
 num_bins = 50
 # the histogram of the data
-n, bins, patches = fig.sp.hist(x, num_bins, normed=1, facecolor='green', alpha=0.5)
+n, bins, patches = fig.sp.hist(x, num_bins, density=1, facecolor='green', alpha=0.5)
 # add a 'best fit' line
 y = bins.normpdf(mu, sigma)
 fig.sp.plot(bins, y, 'r--')

docs/source/pyplots/tick_params.py

@@ -12,7 +12,7 @@
 
 fig.sp.xticks(CArray([-pi, -pi / 2, 0, pi / 2, pi]))
 fig.sp.xticklabels(CArray(["- pi", "-pi/2", "0", "pi/2", "pi"]))
-fig.sp.tick_params(direction='out', length=6, width=2, colors='r', right='off')
+fig.sp.tick_params(direction='out', length=6, width=2, colors='r', right=False)
 fig.sp.yticks(CArray([-1, 0, +1]))
 
 fig.show()

docs/source/secml.adv.attacks.evasion.rst

@@ -33,7 +33,7 @@ CAttackEvasionPGDLS
 CAttackEvasionCleverhans
 ------------------------
 
-.. automodule:: secml.adv.attacks.evasion.c_attack_evasion_cleverhans
+.. automodule:: secml.adv.attacks.evasion.cleverhans.c_attack_evasion_cleverhans
    :members:
    :undoc-members:
    :show-inheritance:

docs/source/secml.figure.rst

@@ -44,6 +44,10 @@ by calling the active subplot via :meth:`.CFigure.sp`
    :members:
    :undoc-members:
 
+.. autoclass:: secml.figure._plots.c_plot_metric.CPlotMetric
+   :members:
+   :undoc-members:
+
 .. autoclass:: secml.figure._plots.c_plot_sec_eval.CPlotSecEval
    :members:
    :undoc-members:

docs/source/secml.ml.classifiers.rst

@@ -109,15 +109,6 @@ CClassifierPyTorch
    :undoc-members:
    :show-inheritance:
 
-
-CModelCleverhans
-----------------
-
-.. automodule:: secml.ml.classifiers.tf.c_model_cleverhans
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 clf\_utils
 ----------
 

docs/source/secml.ml.model_zoo.rst

@@ -0,0 +1,15 @@
+secml.ml.model_zoo
+==================
+
+.. automodule:: secml.ml.model_zoo
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+load_model
+----------
+
+.. automodule:: secml.ml.model_zoo.load_model
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/secml.ml.rst

@@ -14,3 +14,4 @@ secml.ml
    secml.ml.kernel
    secml.ml.peval
    secml.ml.stats
+   secml.ml.model_zoo

docs/source/tutorials.adv.rst

@@ -0,0 +1,13 @@
+Adversarial Machine Learning
+============================
+
+.. toctree::
+   :titlesonly:
+
+   tutorials/03-Evasion
+   tutorials/04-Transferability
+   tutorials/05-Poisoning
+   tutorials/06-MNIST_dataset
+   tutorials/07-NeuralNetworks-MNIST
+   tutorials/08-ImageNet
+   tutorials/09-Cleverhans

docs/source/tutorials.ml.rst

@@ -0,0 +1,8 @@
+Machine Learning
+================
+
+.. toctree::
+   :titlesonly:
+
+   tutorials/01-Training
+   tutorials/02-NeuralNetworks

docs/source/tutorials/02-NeuralNetworks.nblink

@@ -0,0 +1,3 @@
+{
+    "path": "../../../tutorials/02-NeuralNetworks.ipynb"
+}

docs/source/tutorials/03-Evasion.nblink

@@ -0,0 +1,3 @@
+{
+    "path": "../../../tutorials/03-Evasion.ipynb"
+}

docs/source/tutorials/04-Transferability.nblink

@@ -0,0 +1,3 @@
+{
+    "path": "../../../tutorials/04-Transferability.ipynb"
+}

docs/source/tutorials/05-Poisoning.nblink

@@ -0,0 +1,3 @@
+{
+    "path": "../../../tutorials/05-Poisoning.ipynb"
+}

docs/source/tutorials/06-MNIST_dataset.nblink

@@ -0,0 +1,3 @@
+{
+    "path": "../../../tutorials/06-MNIST_dataset.ipynb"
+}