Merge 0099574 into a635d59

.travis.yml

@@ -21,27 +21,23 @@ env:
   - TESTCMD="--cov-append csrank/tests/test_choice_functions.py"
   - TESTCMD="--cov-append csrank/tests/test_discrete_choice.py"
   - TESTCMD="--cov-append csrank/tests/test_ranking.py"
-  - TESTCMD="--cov-append csrank/tests/test_fate.py csrank/tests/test_losses.py csrank/tests/test_metrics.py csrank/tests/test_tuning.py csrank/tests/test_util.py"
+  - TESTCMD="--cov-append csrank/tests/test_fate.py csrank/tests/test_losses.py csrank/tests/test_metrics.py csrank/tests/test_tuning.py csrank/tests/test_util.py csrank/tests/test_tuning.py csrank/tests/test_callback.py"
 
 script:
   - pytest -v --cov=csrank --ignore experiments $TESTCMD
   - travis-sphinx build -n --source=docs
 
 after_success:
   - coveralls
+  - travis-sphinx deploy
 
 deploy:
-  - provider: pypi
-    distributions: sdist bdist_wheel
-    user: __token__
-    password:
-      secure: INa6/h17ejVh3vxAGc4V7uLY30tJON2n/PWV3y0x0DnAR+pH5LiR3ZIE8/CNu1NpH+R7I1u2m0PHLYh5H8XpuewHaToer4p41jIiGF6WXnUg6PQN7zuBk9tqgmJElr+aYp/oK/B6VHTfFwEk2QlXmcqVofpKQNWrzm6+EMWXJjSs3Z9XyGIJvzJ2ihaHvr4URlNfjX98Ij0uIzzEBzN7RSJISQQo672F0b+JvgR3nQYfa5Sp4ijBnEuTtU2BYiXcojaT283O49DL0LGQHApIu0Blf79FXA2QJET65ujIQDxKBZ+WhPEHupwIiBsJ1JLl5C0wBj0UAXyogypXhm9tXfOwB12MLsbT8pbH7YMY4vS8lKtXltWTw66JbruNxECCcvHMw6SnCX0D1H40LKCvQLLJ4L2IOkVcpLI702Tlf13OF4kic+OGfSBv0nRDTLzyvMU5NACoMLnBipbpN4qXKeJdoHWOKECiRYFqZLacxPSGXx/RteiEpi25Ghgk2x53Y6pNCpzW118qW/Ij2LvPffB9PT/BPe0MIjveDLJ3PI93xevDljyZHE8v7JGOPo4gX3YA4hYwMB6gZtO+bdIemGQIP5ymrIpKqSDw/CpFLUCUVzL8sOu5AE3pbDFUGdn5njSex7Leafs+px7IH6dLE2MQqCU1o9Qr4V4FTuxta60=
-    on:
-      tags: true
-      repo: kiudee/cs-ranking
-  - provider: pages
-    skip_cleanup: true
-    github_token: $GH_TOKEN
-    on:
-      branch: master
-      repo: kiudee/cs-ranking
+  provider: pypi
+  distributions: sdist bdist_wheel
+  user: __token__
+  password:
+    secure: INa6/h17ejVh3vxAGc4V7uLY30tJON2n/PWV3y0x0DnAR+pH5LiR3ZIE8/CNu1NpH+R7I1u2m0PHLYh5H8XpuewHaToer4p41jIiGF6WXnUg6PQN7zuBk9tqgmJElr+aYp/oK/B6VHTfFwEk2QlXmcqVofpKQNWrzm6+EMWXJjSs3Z9XyGIJvzJ2ihaHvr4URlNfjX98Ij0uIzzEBzN7RSJISQQo672F0b+JvgR3nQYfa5Sp4ijBnEuTtU2BYiXcojaT283O49DL0LGQHApIu0Blf79FXA2QJET65ujIQDxKBZ+WhPEHupwIiBsJ1JLl5C0wBj0UAXyogypXhm9tXfOwB12MLsbT8pbH7YMY4vS8lKtXltWTw66JbruNxECCcvHMw6SnCX0D1H40LKCvQLLJ4L2IOkVcpLI702Tlf13OF4kic+OGfSBv0nRDTLzyvMU5NACoMLnBipbpN4qXKeJdoHWOKECiRYFqZLacxPSGXx/RteiEpi25Ghgk2x53Y6pNCpzW118qW/Ij2LvPffB9PT/BPe0MIjveDLJ3PI93xevDljyZHE8v7JGOPo4gX3YA4hYwMB6gZtO+bdIemGQIP5ymrIpKqSDw/CpFLUCUVzL8sOu5AE3pbDFUGdn5njSex7Leafs+px7IH6dLE2MQqCU1o9Qr4V4FTuxta60=
+  on:
+    tags: true
+    repo: kiudee/cs-ranking
+    python: 3.8

csrank/callbacks.py

@@ -3,50 +3,95 @@
 import warnings
 
 import numpy as np
-from keras.callbacks import Callback, LearningRateScheduler, EarlyStopping
-
 from csrank.tunable import Tunable
 from csrank.util import print_dictionary
+from keras import backend as K
+from keras.callbacks import Callback
+
 
+class EarlyStoppingWithWeights(Callback, Tunable):
 
-class EarlyStoppingWithWeights(EarlyStopping, Tunable):
-    """Stop training when a monitored quantity has stopped improving.
-    # Arguments
-        monitor: quantity to be monitored.
-        min_delta: minimum change in the monitored quantity
-            to qualify as an improvement, i.e. an absolute
-            change of less than min_delta, will count as no
-            improvement.
-        patience: number of epochs with no improvement
-            after which training will be stopped.
-        verbose: verbosity mode.
-        mode: one of {auto, min, max}. In `min` mode,
-            training will stop when the quantity
-            monitored has stopped decreasing; in `max`
-            mode it will stop when the quantity
-            monitored has stopped increasing; in `auto`
-            mode, the direction is automatically inferred
-            from the name of the monitored quantity.
-    """
-
-    def __init__(self, **kwargs):
-        super(EarlyStoppingWithWeights, self).__init__(**kwargs)
+    def __init__(self, monitor='val_loss', min_delta=0, patience=0, verbose=0, mode='auto', baseline=None,
+                 restore_best_weights=False, **kwargs):
+        """
+            Stop training when a monitored quantity has stopped improving.
+
+            Parameters
+            ----------
+            monitor: string
+                Quantity to be monitored, could be the loss or an accuracy value, which is monitored by the model.
+                In case of accuracy the we check the change in increase, while in case of loss we check the change in
+                decrease of the loss
+            min_delta: float
+                Minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less
+                than min_delta, will count as no improvement
+            patience: unsigned int
+                number of epochs with no improvement after which training will be stopped
+            verbose : bool
+                verbosity mode, 1: print, 0 to not
+            mode: one of {auto, min, max}.
+                In `min` mode, training will stop when the quantity monitored has stopped decreasing; in `max` mode
+                it will stop when the quantity monitored has stopped increasing; in `auto` mode, the direction is
+                automatically inferred from the name of the monitored quantity
+            baseline: float
+                Baseline value for the monitored quantity to reach. Training will stop if the model doesn't show
+                improvement over the baseline
+            restore_best_weights: bool
+                whether to restore model weights from the epoch with the best value of the monitored quantity.
+                If False, the model weights obtained at the last step of training are used
+            **kwargs
+                Keyword arguments for the callback
+        """
+        super(EarlyStoppingWithWeights, self).__init__()
+        self.monitor = monitor
+        self.baseline = baseline
+        self.patience = patience
+        self.verbose = verbose
+        self.min_delta = min_delta
+        self.wait = 0
+        self.stopped_epoch = 0
+        self.restore_best_weights = restore_best_weights
+        self.best_weights = None
+
+        if mode not in ['auto', 'min', 'max']:
+            warnings.warn('EarlyStopping mode %s is unknown, '
+                          'fallback to auto mode.' % mode,
+                          RuntimeWarning)
+            mode = 'auto'
+
+        if mode == 'min':
+            self.monitor_op = np.less
+        elif mode == 'max':
+            self.monitor_op = np.greater
+        else:
+            if 'acc' in self.monitor:
+                self.monitor_op = np.greater
+            else:
+                self.monitor_op = np.less
+
+        if self.monitor_op == np.greater:
+            self.min_delta *= 1
+        else:
+            self.min_delta *= -1
         self.logger = logging.getLogger(EarlyStoppingWithWeights.__name__)
 
     def on_train_begin(self, logs=None):
-        super(EarlyStoppingWithWeights, self).on_train_begin(logs=logs)
-        self.epoch = 0
+        # Allow instances to be re-used
+        self.wait = 0
+        self.stopped_epoch = 0
+        if self.baseline is not None:
+            self.best = self.baseline
+        else:
+            self.best = np.Inf if self.monitor_op == np.less else -np.Inf
 
     def on_epoch_end(self, epoch, logs=None):
-        self.epoch += 1
+        self.stopped_epoch += 1
         current = logs.get(self.monitor)
         self.best_weights = self.model.get_weights()
         if current is None:
-            warnings.warn(
-                'Early stopping conditioned on metric `%s` '
-                'which is not available. Available metrics are: %s' %
-                (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning
-            )
+            self.logger.warning(
+                'Early stopping conditioned on metric `%s` which is not available. Available metrics are: %s' % (
+                    self.monitor, ','.join(list(logs.keys()))), RuntimeWarning)
             return
         if self.monitor_op(current - self.min_delta, self.best):
             self.best = current
@@ -55,15 +100,27 @@ def on_epoch_end(self, epoch, logs=None):
         else:
             self.wait += 1
             if self.wait >= self.patience:
-                self.stopped_epoch = self.epoch
                 self.model.stop_training = True
 
     def on_train_end(self, logs=None):
         if self.stopped_epoch > 0:
-            self.logger.info("Setting best weights for final epoch {}".format(self.epoch))
+            self.logger.info("Setting best weights for final epoch {}".format(self.stopped_epoch))
             self.model.set_weights(self.best_weights)
 
     def set_tunable_parameters(self, patience=300, min_delta=2, **point):
+        """
+            Set tunable parameters of the EarlyStopping callback to the values provided.
+
+            Parameters
+            ----------
+            patience: unsigned int
+                Number of epochs with no improvement after which training will be stopped.
+            min_delta: float
+                Minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less
+                than min_delta, will count as no improvement.
+            point: dict
+                Dictionary containing parameter values which are not tuned for the network
+        """
         self.patience = patience
         self.min_delta = min_delta
         if len(point) > 0:
@@ -72,44 +129,81 @@ def set_tunable_parameters(self, patience=300, min_delta=2, **point):
                                 ' called: {}'.format(print_dictionary(point)))
 
 
-class weightHistory(Callback):
-    def on_train_begin(self, logs={}):
-        self.zero_weights = []
-        self.norm = []
-        self.hidden_units_used = []
+class LRScheduler(Callback, Tunable):
 
-    def on_batch_end(self, batch, logs={}):
-        hidden = [layer for layer in self.model.layers
-                  if layer.name == 'hidden_1']
-
-        y = np.array(hidden[0].get_weights()[0])
-        close = np.isclose(y, 0, atol=1e-3)
-        self.hidden_units_used.append(len(np.unique(np.where(np.logical_not(close))[1])))
-        self.norm.append(np.abs(y).sum())
-        self.zero_weights.append(close.sum())
+    def __init__(self, epochs_drop=300, drop=0.1, verbose=0, **kwargs):
+        """
+            Learning rate scheduler with step-decay function
 
+            Parameters
+            ----------
+            epochs_drop: unsigned int
+                The number of epochs after which the learning rate is reduced
+            drop: float [0,1):
+                The percentage of the learning rate which needs to be dropped
+            verbose: bool or int in {0,1}
+                int. 0: quiet, 1: update messages
+            **kwargs
+                Keyword arguments for the callback
 
-class LRScheduler(LearningRateScheduler, Tunable):
-    """Learning rate scheduler.
+        """
+        super(LRScheduler, self).__init__(**kwargs)
+        self.verbose = verbose
+        self.epochs_drop = epochs_drop
+        self.drop = drop
+        self.initial_lr = None
+        self.logger = logging.getLogger(LRScheduler.__name__)
 
-        # Arguments
-            epochs_drop: unsigned int
-            drop:
-            verbose: int. 0: quiet, 1: update messages.
+    def step_decay(self, epoch):
         """
+            The step-decay function, which takes the current epoch and update the learning rate according to the
+            formulae.
 
-    def __init__(self, epochs_drop=300, drop=0.1, **kwargs):
-        super(LRScheduler, self).__init__(self.step_decay, **kwargs)
+            .. math::
+                lr = lr_0 * d_r^{\\lfloor \\frac{e}{e_{\\text{drop}}}\\rfloor}
 
-        self.epochs_drop = epochs_drop
-        self.drop = drop
+            where :math:`lr_0` is the learning rate at the zeroth epoch and :math:`0 < d_r < 1` is the rate with which
+            the learning rate should be reduced, :math:`e` is the current epoch and :math:`e_{\\text{drop}}`
+            is the number of epochs after which the learning rate is decreased.
 
-    def step_decay(self, epoch, lr):
+            Parameters
+            ----------
+            epoch: unsigned int
+                Current epoch
+
+        """
         step = math.floor((1 + epoch) / self.epochs_drop)
-        lrate = lr * math.pow(self.drop, step)
-        return lrate
+        new_lr = self.initial_lr * math.pow(self.drop, step)
+        return new_lr
+
+    def on_epoch_begin(self, epoch, logs=None):
+        if not hasattr(self.model.optimizer, 'lr'):
+            raise ValueError('Optimizer must have a "lr" attribute.')
+        if epoch == 0:
+            self.initial_lr = float(K.get_value(self.model.optimizer.lr))
+        lr = self.step_decay(epoch)
+        K.set_value(self.model.optimizer.lr, lr)
+        if self.verbose > 0:
+            print('\nEpoch %05d: LearningRateScheduler setting learning '
+                  'rate to %s.' % (epoch + 1, lr))
+
+    def on_epoch_end(self, epoch, logs=None):
+        logs = logs or {}
+        logs['lr'] = K.get_value(self.model.optimizer.lr)
 
     def set_tunable_parameters(self, epochs_drop=300, drop=0.1, **point):
+        """
+            Set tunable parameters of the LRScheduler callback to the values provided.
+
+            Parameters
+            ----------
+            epochs_drop: unsigned int
+                The number of epochs after which the learning rate is reduced
+            drop: float [0,1):
+                The percentage of the learning rate which needs to be dropped
+            point: dict
+                Dictionary containing parameter values which are not tuned for the network
+        """
         self.epochs_drop = epochs_drop
         self.drop = drop
         if len(point) > 0:
@@ -121,15 +215,26 @@ def set_tunable_parameters(self, epochs_drop=300, drop=0.1, **point):
 class DebugOutput(Callback):
 
     def __init__(self, delta=100, **kwargs):
+        """ Logging the epochs when done.
+
+            Parameters
+            ----------
+            delta: unsigned int
+                The number of epochs after which the message is logged
+            kwargs:
+                Keyword arguments
+
+        """
         super(DebugOutput, self).__init__(**kwargs)
         self.delta = delta
+        self.epoch = 0
+        self.logger = logging.getLogger(DebugOutput.__name__)
 
     def on_train_end(self, logs=None):
         self.logger.debug('Total number of epochs: {}'.format(self.epoch))
 
     def on_train_begin(self, logs=None):
         self.epoch = 0
-        self.logger = logging.getLogger('DebugOutput')
 
     def on_epoch_end(self, epoch, logs=None):
         self.epoch += 1

csrank/objectranking/expected_rank_regression.py

@@ -129,7 +129,7 @@ def predict(self, X, **kwargs):
 
     def set_tunable_parameters(self, alpha=0.0, l1_ratio=0.5, tol=1e-4, **point):
         """
-            Set tunable parameters of the PairwiseSVM model to the values provided.
+            Set tunable parameters of the ExpectedRankRegression model to the values provided.
 
             Parameters
             ----------