Release v0.10

pralab · Oct 29, 2019 · f99cf9b · f99cf9b
2 parents e2193fb + 26a0f3c
commit f99cf9b
Show file tree

Hide file tree

Showing 74 changed files with 3,018 additions and 1,370 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -42,6 +42,7 @@ stages:
 package:
   stage: package
   image: ${CI_REGISTRY}/pralab/docker-helper-images/python35-setuptools:latest
+  interruptible: true
   before_script:
     - echo $CI_PROJECT_DIR
   script:
@@ -70,6 +71,7 @@ package:
 package:docs:
   stage: package
   image: ${CI_REGISTRY}/pralab/docker-helper-images/python35-sphinx:latest
+  interruptible: true
   script:
     - echo $CI_PROJECT_DIR
     - python -V
@@ -105,6 +107,7 @@ package:docs:
 .test:
   stage: test
   needs: ["package"]
+  interruptible: true
   before_script:
     - echo $CI_PROJECT_DIR
     - python -V
@@ -119,6 +122,9 @@ package:docs:
   except:
     refs:
       - schedules
+      - tags
+    changes:
+      - src/secml/VERSION
     variables:
       - $SKIP_TESTS
 
@@ -156,6 +162,7 @@ test:py37:min:
 .test:install:
   stage: test
   needs: ["package"]
+  interruptible: true
   before_script:
     - echo $CI_PROJECT_DIR
     - python -V
@@ -195,10 +202,11 @@ test:py37:min:
   script:
     - DIST_WHL=$(find $CI_PROJECT_DIR/dist -iname \*.whl -exec basename {} \;)
     - echo $DIST_WHL
-    - pip install ${CI_PROJECT_DIR}/dist/${DIST_WHL}[pytorch]
+    - pip install ${CI_PROJECT_DIR}/dist/${DIST_WHL}[pytorch,cleverhans]
   after_script:
     - python -c "import secml"
     - python -c "from secml.ml.classifiers import CClassifierPyTorch"
+    - python -c "from secml.ml.classifiers import CModelCleverhans"
 
 test:install:whl:py2:
   extends: .test:install:whl
@@ -296,10 +304,10 @@ release:gitlab-releases:
   script:
     - DIST_ZIP=$(find $CI_PROJECT_DIR/dist -iname \*.zip -exec basename {} \;)
     - echo $DIST_ZIP
-    - 'curl -s --request POST --header "PRIVATE-TOKEN: $API_ACCESS_TOKEN" --data name="$DIST_ZIP" --data url="$CI_PROJECT_URL/-/jobs/artifacts/$CI_COMMIT_TAG/raw/dist/$DIST_ZIP?job=release" $CI_PROJECT_DOMAIN/api/v4/projects/$CI_PROJECT_ID/releases/$CI_COMMIT_TAG/assets/links'
+    - 'curl -s --request POST --header "PRIVATE-TOKEN: $API_ACCESS_TOKEN" --data name="$DIST_ZIP" --data url="$CI_PROJECT_URL/-/jobs/artifacts/$CI_COMMIT_TAG/raw/dist/$DIST_ZIP?job=release:gitlab-releases" $CI_PROJECT_DOMAIN/api/v4/projects/$CI_PROJECT_ID/releases/$CI_COMMIT_TAG/assets/links'
     - DIST_WHL=$(find $CI_PROJECT_DIR/dist -iname \*.whl -exec basename {} \;)
     - echo $DIST_WHL
-    - 'curl -s --request POST --header "PRIVATE-TOKEN: $API_ACCESS_TOKEN" --data name="$DIST_WHL" --data url="$CI_PROJECT_URL/-/jobs/artifacts/$CI_COMMIT_TAG/raw/dist/$DIST_WHL?job=release" $CI_PROJECT_DOMAIN/api/v4/projects/$CI_PROJECT_ID/releases/$CI_COMMIT_TAG/assets/links'
+    - 'curl -s --request POST --header "PRIVATE-TOKEN: $API_ACCESS_TOKEN" --data name="$DIST_WHL" --data url="$CI_PROJECT_URL/-/jobs/artifacts/$CI_COMMIT_TAG/raw/dist/$DIST_WHL?job=release:gitlab-releases" $CI_PROJECT_DOMAIN/api/v4/projects/$CI_PROJECT_ID/releases/$CI_COMMIT_TAG/assets/links'
   artifacts:  # Keep the distribution package (default expire 0)
     name: "${CI_PROJECT_NAME}_${CI_COMMIT_REF_NAME}"
     paths:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,51 @@
+## v0.10 (29/10/2019)
+- #535 Added new package `secml.explanation`, which provides different methods for explaining machine learning models. See documentation and examples for more information.
+- #584 **[beta]** Added `CModelCleverhans` and `CAttackEvasionCleverhans` to support adversarial attacks from [CleverHans](https://github.com/tensorflow/cleverhans), a Python library to benchmark vulnerability of machine learning systems to adversarial examples.
+
+### Requirements (1 change)
+- #580 PyTorch version `1.3` is now supported.
+
+### Added (4 changes)
+- #565 Added new abstract interface `CClassifierDNN` from which new classes implementing Deep Neural Networks can inherit.
+- #555 Added `CNormalizerDNN`, which allows using a `CClassifierDNN` as a preprocessor.
+- #593 Added `CDataLoaderTorchDataset`, which allows converting a `torchvision` dataset into a `CDataset`.
+- #598 Added gradient method for `CKernelHistIntersection`.
+
+### Improved (6 changes)
+- #562 Extended support of `CClassifierPyTorch` to nested PyTorch modules.
+- #594 `CClassifierPyTorch.load_model()` is now able to also load models trained with PyTorch (without using our wrapper). New parameter `classes` added to the method to match classes to indexes in the loaded model.
+- #579 Left side single row/column broadcast is now supported for sparse vs sparse `CArray` operations.
+- #582 Improved performance of `CNormalizerMeanStd` when multiple channels are defined.
+- #576 Vastly improved the performance of kernels by removing loops over samples in many classes and refactoring main routines.
+- #562 Improved `grad_f_x` computation at a specific layer in `CClassifierPyTorch`.
+
+### Changed (4 changes)
+- #578 `CClassifierPyTorch` now inherits from `CClassifierDNN`. The following changed accordingly: parameter `torch_model` renamed to `model`; property `layer_shapes` is now defined; method `save_checkpoint` removed.
+- #562 Parameter `layer` of `CClassifierPyTorch.get_layer_output()` is now renamed `layer_names` as a list of layers names is supported (a single layer name is still supported as input). A dictionary is returned if multiple layers are requested. See the documentation for more information.
+- #533 Double initialization in `CAttackEvasionPGDLS` will now be executed regardless of the classifier type (linear or nonlinear) if the `double_init` parameter of `.run()` method is set to `True`.
+- #591 It is now not required to call the `fit` method of `CNormalizerMeanSTD` if fixed mean/std values are used.
+
+### Fixed (4 changes)
+- #561 Fixed `CConstraintBox` not always applied correctly for float data.
+- #577 Fixed `CClassifierPyTorch.decision_function` applying preprocess twice.
+- #581 Fixed gradient computation of `CKernelChebyshevDistance`.
+- #599 Kernels using distances are now based on negative distances (to correctly represent similarity measures). Affected classes are: `CKernelChebyshevDistance`, `CKernelEuclidean`.
+
+### Removed & Deprecated (5 changes)
+- #561 Removed parameter `precision` from `CConstraint.is_violated()`.
+- #575 Parameter `batch_size` of `CKernel` is now deprecated.
+- #597 Removed unused parameter `gamma` from `CKernelChebyshevDistance`.
+- #596 Removed `CKernelHamming`.
+- #602 Renamed `CNormalizerMeanSTD` to `CNormalizerMeanStd`. The old class has been deprecated and will be removed in a future vearsion.
+
+### Documentation (5 changes)
+- #538 Added a notebook tutorial on the use of Explainable ML methods provided by the `secml.explanation` package.
+- #573 Improved visualization of attack results in `07-ImageNet` tutorial.
+- #610 Fixed spacing between parameter and parameter type in the docs.
+- #605 Fixed documentation of classes requiring extra components not being displayed.
+- #608 Added acknowledgments to `README`.
+
+
 ## v0.9 (11/10/2019)
 - #536 Added `CClassifierPytorch` to support Neural Networks (NNs) through [PyTorch](https://pytorch.org/) deep learning platform.
 

diff --git a/README.md b/README.md
@@ -11,11 +11,12 @@ It comes with a set of powerful features:
   supported by `scikit-learn` are available, as well as Neural Networks (NNs)
   through [PyTorch](https://pytorch.org/) deep learning platform.
 - **Built-in attack algorithms.** Evasion and poisoning attacks based on a
-  custom-developed fast solver.
+  custom-developed fast solver. In addition, we provide connectors to other 
+  third-party Adversarial Machine Learning libraries.
 - **Visualize your results.** We provide visualization and plotting framework
   based on the widely-known library [matplotlib](https://matplotlib.org/).
 - **Explain your results.** Explainable ML methods to interpret model decisions
-  via influential features and prototypes. _(coming soon)_  
+  via influential features and prototypes.
 - **Extensible.** Easily create new wrappers for ML models or attack algorithms
   extending our abstract interfaces.
 - **Multi-processing.** Do you want to save time further? We provide full
@@ -53,14 +54,15 @@ macOS versions and Linux distributions.
 However, to support additional advanced features more packages can be necessary
 depending on the Operating System used:
 
- - Linux (Ubuntu >= 16.04 or equivalent dist):
+ - Linux (Ubuntu >= 16.04 or equivalent dist)
    - `python-tk` (Python 2.7), `python3-tk` (Python >= 3.5), for running
      MatplotLib Tk-based backends;
    - NVIDIA<sup>®</sup> CUDA<sup>®</sup> Toolkit for running `tf-gpu`
      [extra component](#extra-components).
      See the [TensorFlow Guide](https://www.tensorflow.org/install/gpu).
 
  - macOS (macOS >= 10.12 Sierra)
+   - Nothing to note.
 
 
 ### Installation process
@@ -105,9 +107,7 @@ of the `[extras]` section.
 
 ### Available extra components
  - `pytorch` : Neural Networks (NNs) through [PyTorch](https://pytorch.org/) deep learning platform.  
-   Will install: `torch >= 1.1, < 1.3`, `torchvision >= 0.2.2`
-
-### _Coming soon_
+   Will install: `torch >= 1.1`, `torchvision >= 0.2.2`
  - `cleverhans` : Wrapper of [CleverHans](https://github.com/tensorflow/cleverhans), 
    a Python library to benchmark vulnerability of machine learning systems
    to adversarial examples. Will install: `tensorflow >= 1.14.*, < 2`, `cleverhans`
@@ -138,20 +138,13 @@ The `secml.adv` package contains evasion and poisoning attacks based on a
 custom-developed solver, along with classes to easily perform security
 evaluation of Machine Learning algorithms.
 
+The `secml.explanation` package contains different explainable 
+Machine Learning methods that allow interpreting classifiers decisions 
+by analyzing the relevant components such as features or training prototypes.
+
 The `secml.figure` package contains a visualization and plotting framework
 based on [matplotlib](https://matplotlib.org/).
 
-_(coming soon)_ The `secml.explanation` package contains few different
-explainable Machine Learning methods that allow interpreting classifiers
-decisions by analyzing the relevant components such as features or training
-prototypes.
-
-_(coming soon)_ The `secml.tf.clvhs` package contains support classes for the
-[CleverHans](https://github.com/tensorflow/cleverhans) library for benchmarking
-machine learning systems' vulnerability to adversarial examples. 
-This package will be available only if the extra component `cleverhans`
-has been specified during installation.
-
 
 ## Contributors
 
@@ -208,13 +201,13 @@ This library is maintained by
 [PRALab - Pattern Recognition and Applications Lab](https://pralab.diee.unica.it).
 
 List of contributors:
- - Marco Melis (maintainer) [1]_
- - Ambra Demontis [1]_
- - Maura Pintor [1]_ , [2]_
- - Battista Biggio [1]_ , [2]_
+ - Marco Melis (maintainer) [1]
+ - Ambra Demontis [1]
+ - Maura Pintor [1], [2]
+ - Battista Biggio [1], [2]
 
-.. [1] Department of Electrical and Electronic Engineering, University of Cagliari, Italy  
-.. [2] Pluribus One, Italy
+[1] Department of Electrical and Electronic Engineering, University of Cagliari, Italy  
+[2] Pluribus One, Italy
 
 
 ## Credits
@@ -227,6 +220,15 @@ List of contributors:
   Computing in Science & Engineering, vol. 9, no. 3, pp. 90-95, 2007.](
   https://doi.org/10.1109/MCSE.2007.55)
 - `pytorch` Paszke, Adam, et al. "Automatic differentiation in pytorch.", NIPS-W, 2017.
+- `cleverhans` [Papernot, Nicolas, et al. "Technical Report on the CleverHans v2.1.0 
+  Adversarial Examples Library." arXiv preprint arXiv:1610.00768 (2018).](
+  https://arxiv.org/abs/1610.00768)
+
+
+## Acknowledgements
+SecML has been partially developed with the support of European Union’s 
+[ALOHA project](https://www.aloha-h2020.eu/) Horizon 2020 Research and 
+Innovation programme, grant agreement No. 780788.
 
 
 ## Copyright

diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -54,6 +54,10 @@
     'exclude-members': ''
 }
 
+# The following modules should be faked by sphinx (e.g. extras)
+autodoc_mock_imports = [
+    "pytest", "torch", "torchvision", "cleverhans", "tensorflow"]
+
 # Autosummary pages will be generated by sphinx-autogen instead of sphinx-build
 # autosummary_generate = True
 
@@ -83,6 +87,7 @@
     'sklearn': ("https://scikit-learn.org/stable/", None),
     'matplotlib': ('https://matplotlib.org/', None),
     'pytorch': ('https://pytorch.org/docs/stable/', None),
+    'cleverhans': ('https://cleverhans.readthedocs.io/en/latest/', None),
 }
 
 # -- Options for HTML output -------------------------------------------------
@@ -120,3 +125,13 @@
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
+
+# Temporary work-around for spacing problem between parameter and parameter
+# type in the doc, see https://github.com/numpy/numpydoc/issues/215. The bug
+# has been fixed in sphinx (https://github.com/sphinx-doc/sphinx/pull/5976) but
+# through a change in sphinx basic.css except rtd_theme does not use basic.css.
+# In an ideal world, this would get fixed in this PR:
+# https://github.com/readthedocs/sphinx_rtd_theme/pull/747/files
+# Thanks to: https://github.com/dask/dask-jobqueue
+def setup(app):
+    app.add_stylesheet("basic.css")
diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -27,6 +27,13 @@
    secml.adv
    secml.optim
 
+.. toctree::
+   :hidden:
+   :caption: Explanation
+   :maxdepth: 2
+
+   secml.explanation
+
 .. toctree::
    :hidden:
    :caption: Visualization
@@ -42,12 +49,12 @@
    secml.parallel
    secml.utils
    secml.settings
+   secml.testing
 
 .. toctree::
    :caption: References
    :hidden:
 
-   modindex
    update
    changelog
    roadmap
diff --git a/docs/source/modules.rst b/docs/source/modules.rst
diff --git a/docs/source/roadmap.rst b/docs/source/roadmap.rst
@@ -3,11 +3,9 @@ ROADMAP
 
 **SecML is still in alpha stage and the roadmap is subject to change at any time.**
 
-1. (Q4 2019) `Cleverhans <https://github.com/tensorflow/cleverhans>`_ library wrapper
-2. (Q4 2019) Model Zoo
-3. (Q4 2019) `secml.explanations` package
-4. (Q1 2020) Improved support for `Tensorflow 2 <https://www.tensorflow.org/>`_ library
-5. (Q2 2020) `Foolbox <https://foolbox.readthedocs.io/>`_ library wrapper
-6. (Q2 2020) `Keras <https://keras.io/>`_ library wrapper
+1. (Q4 2019) Model Zoo
+2. (Q1 2020) Support for `Tensorflow 2 <https://www.tensorflow.org/>`_ library
+3. (Q2 2020) `Foolbox <https://foolbox.readthedocs.io/>`_ library wrapper
+4. (Q2 2020) `Keras <https://keras.io/>`_ library wrapper
 
 For further details and the most up-to-date roadmap see: https://gitlab.com/secml/secml/milestones
diff --git a/docs/source/secml.adv.attacks.evasion.rst b/docs/source/secml.adv.attacks.evasion.rst
@@ -30,3 +30,11 @@ CAttackEvasionPGDLS
    :undoc-members:
    :show-inheritance:
 
+CAttackEvasionCleverhans
+------------------------
+
+.. automodule:: secml.adv.attacks.evasion.c_attack_evasion_cleverhans
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
diff --git a/docs/source/secml.data.loader.rst b/docs/source/secml.data.loader.rst
@@ -87,6 +87,14 @@ CDataLoaderSvmLight
    :undoc-members:
    :show-inheritance:
 
+CDataLoaderTorchDataset
+-----------------------
+
+.. automodule:: secml.data.loader.c_dataloader_torchvision
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 loader\_utils
 -------------
 

diff --git a/docs/source/secml.explanation.rst b/docs/source/secml.explanation.rst
@@ -0,0 +1,50 @@
+secml.explanation
+=================
+
+.. automodule:: secml.explanation
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+
+CExplainer
+----------
+
+.. automodule:: secml.explanation.c_explainer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+CExplainerGradient
+------------------
+
+.. automodule:: secml.explanation.c_explainer_gradient
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+CExplainerGradientInput
+-----------------------
+
+.. automodule:: secml.explanation.c_explainer_gradient_input
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+CExplainerIntegratedGradients
+-----------------------------
+
+.. automodule:: secml.explanation.c_explainer_integrated_gradients
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+CExplainerInfluenceFunctions
+----------------------------
+
+.. automodule:: secml.explanation.c_explainer_influence_functions
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+