Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add drop_last option to method fit of PyTorchClassifier #1883

Merged
merged 7 commits into from
Nov 10, 2022
Merged

Add drop_last option to method fit of PyTorchClassifier #1883

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c00332d
Update method fit of PyTorchClassifier
Oct 19, 2022
58c0a82
Add drop_last option to method fit of PyTorchClassifier
Oct 19, 2022
de1e9d2
Fix line length
Oct 20, 2022
4a32eea
Add try-except
Oct 20, 2022
6e05685
Fix style checks
Oct 21, 2022
5aefcc7
Update fit methods
Oct 25, 2022
fc12c05
Update missing parameter application
Nov 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions art/estimators/certification/derandomized_smoothing/pytorch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,18 +155,21 @@ def fit( # pylint: disable=W0221
batch_size: int = 128,
nb_epochs: int = 10,
training_mode: bool = True,
drop_last: bool = False,
scheduler: Optional[Any] = None,
**kwargs,
) -> None:
"""
Fit the classifier on the training set `(x, y)`.

:param x: Training data.
:param y: Target values (class labels) one-hot-encoded of shape (nb_samples, nb_classes) or index labels of
shape (nb_samples,).
:param batch_size: Size of batches.
:param nb_epochs: Number of epochs to use for training.
:param training_mode: `True` for model set to training mode and `'False` for model set to evaluation mode.
:param drop_last: Set to ``True`` to drop the last incomplete batch, if the dataset size is not divisible by
the batch size. If ``False`` and the size of dataset is not divisible by the batch size, then
the last batch will be smaller. (default: ``False``)
:param scheduler: Learning rate scheduler to run at the start of every epoch.
:param kwargs: Dictionary of framework-specific arguments. This parameter is not currently supported for PyTorch
and providing it takes no effect.
Expand All @@ -187,7 +190,11 @@ def fit( # pylint: disable=W0221
# Check label shape
y_preprocessed = self.reduce_labels(y_preprocessed)

num_batch = int(np.ceil(len(x_preprocessed) / float(batch_size)))
num_batch = len(x_preprocessed) / float(batch_size)
if drop_last:
num_batch = int(np.floor(num_batch))
else:
num_batch = int(np.ceil(num_batch))
ind = np.arange(len(x_preprocessed))

# Start training
Expand All @@ -207,7 +214,15 @@ def fit( # pylint: disable=W0221
self._optimizer.zero_grad()

# Perform prediction
model_outputs = self._model(i_batch)
try:
model_outputs = self._model(i_batch)
except ValueError as err:
if "Expected more than 1 value per channel when training" in str(err):
logger.exception(
"Try dropping the last incomplete batch by setting drop_last=True in "
"method PyTorchClassifier.fit."
)
raise err

# Form the loss function
loss = self._loss(model_outputs[-1], o_batch) # lgtm [py/call-to-non-callable]
Expand Down
35 changes: 30 additions & 5 deletions art/estimators/certification/randomized_smoothing/pytorch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from __future__ import absolute_import, division, print_function, unicode_literals

import logging
from typing import List, Optional, Tuple, Union, TYPE_CHECKING
from typing import List, Optional, Tuple, Union, Any, TYPE_CHECKING

import warnings
import random
Expand Down Expand Up @@ -136,6 +136,8 @@ def fit( # pylint: disable=W0221
batch_size: int = 128,
nb_epochs: int = 10,
training_mode: bool = True,
drop_last: bool = False,
scheduler: Optional[Any] = None,
**kwargs,
) -> None:
"""
Expand All @@ -147,6 +149,10 @@ def fit( # pylint: disable=W0221
:param batch_size: Size of batches.
:param nb_epochs: Number of epochs to use for training.
:param training_mode: `True` for model set to training mode and `'False` for model set to evaluation mode.
:param drop_last: Set to ``True`` to drop the last incomplete batch, if the dataset size is not divisible by
the batch size. If ``False`` and the size of dataset is not divisible by the batch size, then
the last batch will be smaller. (default: ``False``)
:param scheduler: Learning rate scheduler to run at the start of every epoch.
:param kwargs: Dictionary of framework-specific arguments. This parameter is not currently supported for PyTorch
and providing it takes no effect.
"""
Expand All @@ -166,18 +172,26 @@ def fit( # pylint: disable=W0221
# Check label shape
y_preprocessed = self.reduce_labels(y_preprocessed)

num_batch = int(np.ceil(len(x_preprocessed) / float(batch_size)))
num_batch = len(x_preprocessed) / float(batch_size)
if drop_last:
num_batch = int(np.floor(num_batch))
else:
num_batch = int(np.ceil(num_batch))
ind = np.arange(len(x_preprocessed))
std = torch.tensor(self.scale).to(self._device)

x_preprocessed = torch.from_numpy(x_preprocessed).to(self._device)
y_preprocessed = torch.from_numpy(y_preprocessed).to(self._device)

# Start training
for _ in tqdm(range(nb_epochs)):
# Shuffle the examples
random.shuffle(ind)

# Train for one epoch
for m in range(num_batch):
i_batch = torch.from_numpy(x_preprocessed[ind[m * batch_size : (m + 1) * batch_size]]).to(self._device)
o_batch = torch.from_numpy(y_preprocessed[ind[m * batch_size : (m + 1) * batch_size]]).to(self._device)
i_batch = x_preprocessed[ind[m * batch_size : (m + 1) * batch_size]]
o_batch = y_preprocessed[ind[m * batch_size : (m + 1) * batch_size]]

# Add random noise for randomized smoothing
i_batch = i_batch + torch.randn_like(i_batch, device=self._device) * std
Expand All @@ -186,7 +200,15 @@ def fit( # pylint: disable=W0221
self._optimizer.zero_grad()

# Perform prediction
model_outputs = self._model(i_batch)
try:
model_outputs = self._model(i_batch)
except ValueError as err:
if "Expected more than 1 value per channel when training" in str(err):
logger.exception(
"Try dropping the last incomplete batch by setting drop_last=True in "
"method PyTorchClassifier.fit."
)
raise err

# Form the loss function
loss = self._loss(model_outputs[-1], o_batch) # lgtm [py/call-to-non-callable]
Expand All @@ -203,6 +225,9 @@ def fit( # pylint: disable=W0221

self._optimizer.step()

if scheduler is not None:
scheduler.step()

def predict(self, x: np.ndarray, batch_size: int = 128, **kwargs) -> np.ndarray: # type: ignore
"""
Perform prediction of the given classifier for a batch of inputs, taking an expectation over transformations.
Expand Down
34 changes: 29 additions & 5 deletions art/estimators/classification/pytorch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,8 @@ def fit( # pylint: disable=W0221
batch_size: int = 128,
nb_epochs: int = 10,
training_mode: bool = True,
drop_last: bool = False,
scheduler: Optional[Any] = None,
**kwargs,
) -> None:
"""
Expand All @@ -373,8 +375,12 @@ def fit( # pylint: disable=W0221
:param batch_size: Size of batches.
:param nb_epochs: Number of epochs to use for training.
:param training_mode: `True` for model set to training mode and `'False` for model set to evaluation mode.
:param drop_last: Set to ``True`` to drop the last incomplete batch, if the dataset size is not divisible by
the batch size. If ``False`` and the size of dataset is not divisible by the batch size, then
the last batch will be smaller. (default: ``False``)
:param scheduler: Learning rate scheduler to run at the start of every epoch.
:param kwargs: Dictionary of framework-specific arguments. This parameter is not currently supported for PyTorch
and providing it takes no effect.
and providing it takes no effect.
"""
import torch # lgtm [py/repeated-import]

Expand All @@ -392,24 +398,39 @@ def fit( # pylint: disable=W0221
# Check label shape
y_preprocessed = self.reduce_labels(y_preprocessed)

num_batch = int(np.ceil(len(x_preprocessed) / float(batch_size)))
num_batch = len(x_preprocessed) / float(batch_size)
if drop_last:
num_batch = int(np.floor(num_batch))
else:
num_batch = int(np.ceil(num_batch))
ind = np.arange(len(x_preprocessed))

x_preprocessed = torch.from_numpy(x_preprocessed).to(self._device)
y_preprocessed = torch.from_numpy(y_preprocessed).to(self._device)

# Start training
for _ in range(nb_epochs):
# Shuffle the examples
random.shuffle(ind)

# Train for one epoch
for m in range(num_batch):
i_batch = torch.from_numpy(x_preprocessed[ind[m * batch_size : (m + 1) * batch_size]]).to(self._device)
o_batch = torch.from_numpy(y_preprocessed[ind[m * batch_size : (m + 1) * batch_size]]).to(self._device)
i_batch = x_preprocessed[ind[m * batch_size : (m + 1) * batch_size]]
o_batch = y_preprocessed[ind[m * batch_size : (m + 1) * batch_size]]

# Zero the parameter gradients
self._optimizer.zero_grad()

# Perform prediction
model_outputs = self._model(i_batch)
try:
model_outputs = self._model(i_batch)
except ValueError as err:
if "Expected more than 1 value per channel when training" in str(err):
logger.exception(
"Try dropping the last incomplete batch by setting drop_last=True in "
"method PyTorchClassifier.fit."
)
raise err

# Form the loss function
loss = self._loss(model_outputs[-1], o_batch) # lgtm [py/call-to-non-callable]
Expand All @@ -426,6 +447,9 @@ def fit( # pylint: disable=W0221

self._optimizer.step()

if scheduler is not None:
scheduler.step()

def fit_generator(self, generator: "DataGenerator", nb_epochs: int = 20, **kwargs) -> None:
"""
Fit the classifier using the generator that yields batches as specified.
Expand Down