Merge pull request #2245 from GiulioZizzo/huggingface_integration

Implement estimator for Hugging Face models

.github/workflows/ci-huggingface.yml

@@ -0,0 +1,65 @@
+name: CI Huggingface
+on:
+  # Run on manual trigger
+  workflow_dispatch:
+
+  # Run on pull requests
+  pull_request:
+    paths-ignore:
+      - '*.md'
+
+  # Run on merge queue
+  merge_group:
+
+  # Run when pushing to main or dev branches
+  push:
+    branches:
+      - main
+      - dev*
+
+  # Run scheduled CI flow daily
+  schedule:
+    - cron: '0 8 * * 0'
+
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: Huggingface 4.30
+            framework: huggingface
+            python: 3.9
+            torch: 1.13.1+cpu
+            torchvision: 0.14.1+cpu
+            torchaudio: 0.13.1
+            transformers: 4.30.2
+
+    name: ${{ matrix.name }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get -y -q install ffmpeg libavcodec-extra
+          python -m pip install --upgrade pip setuptools wheel
+          pip3 install -r requirements_test.txt
+          pip install tensorflow==2.10.1
+          pip install keras==2.10.0
+          pip install torch==${{ matrix.torch }} -f https://download.pytorch.org/whl/cpu/torch_stable.html
+          pip install torchvision==${{ matrix.torchvision }} -f https://download.pytorch.org/whl/cpu/torch_stable.html
+          pip install torchaudio==${{ matrix.torchaudio }} -f https://download.pytorch.org/whl/cpu/torch_stable.html
+          pip install transformers==${{ matrix.transformers }}
+          pip list
+      - name: Run Tests
+        run: ./run_tests.sh ${{ matrix.framework }}
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          fail_ci_if_error: true

art/attacks/poisoning/hidden_trigger_backdoor/hidden_trigger_backdoor_pytorch.py

@@ -240,6 +240,8 @@ def poison(  # pylint: disable=W0221
 
                 for _ in range(feat2.size(0)):
                     dist_min_index = (dist == torch.min(dist)).nonzero().squeeze()
+                    if dist_min_index.dim() > 1:  # If multiple values in dist equal torch.min(dist), return the first.
+                        dist_min_index = dist_min_index[0]
                     feat1[dist_min_index[1]] = feat11[dist_min_index[0]]
                     dist[dist_min_index[0], dist_min_index[1]] = 1e5
 

art/attacks/poisoning/perturbations/image_perturbations.py

@@ -49,18 +49,29 @@ def add_single_bd(x: np.ndarray, distance: int = 2, pixel_value: int = 1) -> np.
     return x
 
 
-def add_pattern_bd(x: np.ndarray, distance: int = 2, pixel_value: int = 1) -> np.ndarray:
+def add_pattern_bd(x: np.ndarray, distance: int = 2, pixel_value: int = 1, channels_first: bool = False) -> np.ndarray:
     """
     Augments a matrix by setting a checkerboard-like pattern of values some `distance` away from the bottom-right
     edge to 1. Works for single images or a batch of images.
 
     :param x: A single image or batch of images of shape NWHC, NHW, or HC. Pixels will be added to all channels.
     :param distance: Distance from bottom-right walls.
     :param pixel_value: Value used to replace the entries of the image matrix.
+    :param channels_first: If the data is provided in channels first format we transpose to NWHC or HC depending on
+                           input shape
     :return: Backdoored image.
     """
     x = np.copy(x)
+    original_dtype = x.dtype
     shape = x.shape
+    if channels_first:
+        if len(shape) == 4:
+            # Transpose the image putting channels last
+            x = np.transpose(x, (0, 2, 3, 1))
+        if len(shape) == 2:
+            # HC to CH
+            x = np.transpose(x)
+
     if len(shape) == 4:
         height, width = x.shape[1:3]
         x[:, height - distance, width - distance, :] = pixel_value
@@ -81,7 +92,15 @@ def add_pattern_bd(x: np.ndarray, distance: int = 2, pixel_value: int = 1) -> np
         x[height - distance - 2, width - distance] = pixel_value
     else:
         raise ValueError(f"Invalid array shape: {shape}")
-    return x
+
+    if channels_first:
+        if len(shape) == 4:
+            # Putting channels first again
+            x = np.transpose(x, (0, 3, 1, 2))
+        if len(shape) == 2:
+            x = np.transpose(x)
+
+    return x.astype(original_dtype)
 
 
 def insert_image(

art/estimators/classification/__init__.py

@@ -17,6 +17,7 @@
 from art.estimators.classification.lightgbm import LightGBMClassifier
 from art.estimators.classification.mxnet import MXClassifier
 from art.estimators.classification.pytorch import PyTorchClassifier
+from art.estimators.classification.hugging_face import HuggingFaceClassifierPyTorch
 from art.estimators.classification.query_efficient_bb import QueryEfficientGradientEstimationClassifier
 from art.estimators.classification.scikitlearn import SklearnClassifier
 from art.estimators.classification.tensorflow import (