Merge pull request #58 from markroxor/cupy

Use cupy to leverage cuda

fromscratchtoml/__init__.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2017 Mohit Rathore <mrmohitrathoremr@gmail.com>
+# Licensed under the GNU General Public License v3.0 - https://www.gnu.org/licenses/gpl-3.0.en.html
+
+import logging
+import numpy as np  # noqa:F401
+
+logging.basicConfig()
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+def use(backend):
+    global np
+    if backend == "numpy":
+        import numpy as np   # noqa:F401
+        logging.debug("Using numpy backend.")
+    elif backend == "cupy":
+        import cupy as np
+        logging.debug("Using cupy backend.")
+    else:
+        raise ImportError('Only available backends are cupy or numpy.')

fromscratchtoml/cluster/dbscan.py

@@ -4,7 +4,7 @@
 # Copyright (C) 2017 Mohit Rathore <mrmohitrathoremr@gmail.com>
 # Licensed under the GNU General Public License v3.0 - https://www.gnu.org/licenses/gpl-3.0.en.html
 
-import numpy as np
+from fromscratchtoml import np
 
 from ..base import BaseModel
 from ..toolbox.exceptions import InvalidArgumentError

fromscratchtoml/cluster/kmeans.py

@@ -4,7 +4,7 @@
 # Copyright (C) 2017 Mohit Rathore <mrmohitrathoremr@gmail.com>
 # Licensed under the GNU General Public License v3.0 - https://www.gnu.org/licenses/gpl-3.0.en.html
 
-import numpy as np
+from fromscratchtoml import np
 
 from ..base import BaseModel
 from ..toolbox.exceptions import InvalidArgumentError
@@ -21,7 +21,7 @@ class KMeans(BaseModel):
 
     Examples
     --------
-    >>> import numpy as np
+    >>> from fromscratchtoml import np
     >>> from fromscratchtoml.cluster import KMeans as KMeans
     >>> from fromscratchtoml.toolbox.random import Distribution
     >>> X1 = Distribution.linear(pts=500, covr=[[1.2, -1],[-1, 1]], mean=[0, 0])

fromscratchtoml/decomposition/decomposition.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 # Licensed under the GNU General Public License v3.0 - https://www.gnu.org/licenses/gpl-3.0.en.html
 
-import numpy as np
+from fromscratchtoml import np
 
 import logging
 

fromscratchtoml/neighbors/knn.py

@@ -4,7 +4,7 @@
 # Copyright (C) 2017 Mohit Rathore <mrmohitrathoremr@gmail.com>
 # Licensed under the GNU General Public License v3.0 - https://www.gnu.org/licenses/gpl-3.0.en.html
 
-import numpy as np
+from fromscratchtoml import np
 from collections import Counter
 
 from ..base import BaseModel

fromscratchtoml/neural_network/activations.py

@@ -4,7 +4,7 @@
 # Copyright (C) 2017 Mohit Rathore <mrmohitrathoremr@gmail.com>
 # Licensed under the GNU General Public License v3.0 - https://www.gnu.org/licenses/gpl-3.0.en.html
 
-import numpy as np
+from fromscratchtoml import np
 
 import logging
 
@@ -52,6 +52,7 @@ def sigmoid(x, return_deriv=False):
         -------
         numpy.ndarray : sigmoid of x
         """
+        x = np.clip(x, -100, 100)
         _sigmoid = 1.0 / (1.0 + np.exp(-x))
 
         if return_deriv:

fromscratchtoml/neural_network/layers/dense.py

@@ -4,7 +4,8 @@
 # Copyright (C) 2017 Mohit Rathore <mrmohitrathoremr@gmail.com>
 # Licensed under the GNU General Public License v3.0 - https://www.gnu.org/licenses/gpl-3.0.en.html
 
-import numpy as np
+# from fromscratchtoml import np
+from fromscratchtoml import np
 
 from fromscratchtoml.neural_network.layers import Layer
 import logging

fromscratchtoml/neural_network/losses.py

@@ -4,7 +4,7 @@
 # Copyright (C) 2017 Mohit Rathore <mrmohitrathoremr@gmail.com>
 # Licensed under the GNU General Public License v3.0 - https://www.gnu.org/licenses/gpl-3.0.en.html
 
-import numpy as np
+from fromscratchtoml import np
 
 import logging
 
@@ -33,6 +33,41 @@ def mean_squared_error(y_predicted, y_target, return_deriv=False):
     """
     if len(y_target.shape) == 1:
         y_target = np.expand_dims(y_target, axis=1)
+
     if return_deriv:
-        return np.mean(np.square(y_predicted - y_target)), y_predicted - y_target
+        return np.mean(np.square(y_predicted - y_target)), (y_predicted - y_target)
+
     return np.mean(np.square(y_predicted - y_target))
+
+
+def cross_entropy(y_predicted, y_target, return_deriv=False):
+    """
+    Calculates the cross entropy loss between the predicted and the target ouputs.
+    Parameters
+    ----------
+    y_predicted : numpy.ndarray
+        The ouput predicted by the model.
+    y_target : numpy.ndarray
+        The expected output.
+    return_deriv : bool, optional
+        If set to true, the function returns derivative of the error along with the error.
+    Returns
+    -------
+    numpy.array : The error.
+    numpy.array : The error's derivative, optional.
+    """
+    if len(y_target.shape) == 1:
+        y_target = np.expand_dims(y_target, axis=1)
+
+    eps = 1e-9
+
+    y_predicted[y_predicted < eps] = eps
+    y_predicted[y_predicted > 1 - eps] = 1 - eps
+
+    crl = -(y_target * np.log(y_predicted) + (1 - y_target) * np.log(1 - y_predicted))
+
+    if return_deriv:
+        deriv = -(y_target / y_predicted) + (1 - y_target) / (1 - y_predicted)
+        return crl / len(y_target), deriv / len(y_target)
+
+    return crl / len(y_target)

fromscratchtoml/neural_network/models/sequential.py

@@ -5,7 +5,7 @@
 # Licensed under the GNU General Public License v3.0 - https://www.gnu.org/licenses/gpl-3.0.en.html
 
 from __future__ import print_function
-import numpy as np
+from fromscratchtoml import np
 
 from fromscratchtoml.toolbox import progress, binary_visualize
 from .. import losses
@@ -95,7 +95,7 @@ def accuracy(self, X, y):
         total_samples = y.shape[0]
 
         errors = np.count_nonzero(diff_arr) / 2
-        return (100 - (errors / (total_samples * 0.01)))
+        return float(100 - (errors / (total_samples * 0.01)))
 
     def fit(self, X, y, epochs, batch_size=None):
         """
@@ -112,6 +112,9 @@ def fit(self, X, y, epochs, batch_size=None):
         batch_size : int, optional
             The number of data points to be processed in a single iteration.
         """
+        X = np.asarray(X, dtype=np.float64)
+        y = np.asarray(y, dtype=np.float64)
+
         if batch_size is None:
             batch_size = X.shape[0]
 
@@ -127,7 +130,7 @@ def fit(self, X, y, epochs, batch_size=None):
                 acc = self.accuracy(X, y)
                 print("\nepoch: {}/{} ".format(epoch + 1, epochs), end="")
                 print(" acc: {:0.2f} ".format(acc), end="")
-                print(" loss: {:0.3f} ".format(loss))
+                print(" loss: {:0.3f} ".format(float(np.sum(loss))))
                 if self.vis_each_epoch:
                     binary_visualize(X, clf=self, draw_contour=True)
 
@@ -204,6 +207,7 @@ def predict(self, X, prob=False):
         -------
         numpy.array : The prediction.
         """
+        X = np.asarray(X)
         predictions = self.forwardpass(X)
 
         if prob is False:

fromscratchtoml/svm/svc.py

@@ -6,7 +6,7 @@
 
 import cvxopt
 # import torch as ch
-import numpy as np
+from fromscratchtoml import np
 
 from functools import partial
 

fromscratchtoml/test/cluster/test_dbscan.py

@@ -5,7 +5,7 @@
 # Licensed under the GNU General Public License v3.0 - https://www.gnu.org/licenses/gpl-3.0.en.html
 
 import unittest
-import numpy as np
+from fromscratchtoml import np
 
 from sklearn.datasets import load_iris
 

fromscratchtoml/test/cluster/test_kmeans.py

@@ -5,7 +5,7 @@
 # Licensed under the GNU General Public License v3.0 - https://www.gnu.org/licenses/gpl-3.0.en.html
 
 import unittest
-import numpy as np
+from fromscratchtoml import np
 
 from sklearn.cluster import KMeans as skl_KMeans
 from fromscratchtoml.cluster import KMeans as fs2ml_KMeans

fromscratchtoml/test/neighbors/test_knn.py

@@ -5,7 +5,7 @@
 # Licensed under the GNU General Public License v3.0 - https://www.gnu.org/licenses/gpl-3.0.en.html
 
 import unittest
-import numpy as np
+from fromscratchtoml import np
 
 from sklearn.neighbors import KNeighborsClassifier as sk_KNeighborsClassifier
 from sklearn import datasets

fromscratchtoml/test/neural_network/test_neural_network.py

@@ -6,7 +6,7 @@
 
 import unittest
 
-import numpy as np
+from fromscratchtoml import np
 from fromscratchtoml.neural_network.models import Sequential
 from fromscratchtoml.neural_network.layers import Dense, Activation
 from fromscratchtoml.neural_network.optimizers import StochasticGradientDescent
@@ -43,7 +43,7 @@ def setUp(self):
 
         self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(X, y, test_size=50, random_state=42)
 
-    def test_dense_acts_sgd(self):
+    def test_dense_acts_sgd_mse(self):
         model = Sequential()
 
         model.add(Dense(10, input_dim=2, seed=1))
@@ -78,3 +78,39 @@ def test_dense_acts_sgd(self):
         predictions = model.predict(self.X_test)
 
         self.assertTrue(np.allclose((predictions), np.argmax(self.y_test, axis=1)))
+
+    def test_dense_acts_sgd_cross_entropy(self):
+        model = Sequential()
+
+        model.add(Dense(10, input_dim=2, seed=1))
+        model.add(Activation('sigmoid'))
+
+        model.add(Dense(2, seed=7))
+        model.add(Activation('tanh'))
+
+        model.add(Dense(2, seed=2))
+        model.add(Activation('relu'))
+
+        model.add(Dense(2, seed=3))
+        model.add(Activation('leaky_relu'))
+
+        model.add(Dense(2, seed=4))
+        model.add(Activation('linear'))
+
+        model.add(Dense(2, seed=6))
+        model.add(Activation('softmax'))
+
+        sgd = StochasticGradientDescent(learning_rate=0.05)
+        model.compile(optimizer=sgd, loss="cross_entropy")
+
+        model.fit(self.X_train, self.y_train, epochs=9, batch_size=2)
+
+        expected_biases = np.array([[1.38503523, -0.51962709]], dtype=np.float128)
+        self.assertTrue(np.allclose(expected_biases, model.layers[-2].biases))
+
+        expected_weights = np.array([[-1.15492119, 1.33037864], [0.04502013, -1.54036933]], dtype=np.float128)
+        self.assertTrue(np.allclose(expected_weights, model.layers[-2].weights))
+
+        predictions = model.predict(self.X_test)
+
+        self.assertTrue(np.allclose((predictions), np.argmax(self.y_test, axis=1)))

fromscratchtoml/test/test_decomposition.py

@@ -6,7 +6,7 @@
 import unittest
 
 from fromscratchtoml.decomposition import Decomposition
-import numpy as np
+from fromscratchtoml import np
 
 import logging
 

fromscratchtoml/test/test_support_vector_machines.py

@@ -6,7 +6,7 @@
 
 import unittest
 
-import numpy as np
+from fromscratchtoml import np
 
 from fromscratchtoml import svm
 from fromscratchtoml.test.toolbox import _tempfile, _test_data_path  # noqa:F401

fromscratchtoml/toolbox/__init__.py

@@ -6,6 +6,7 @@
 import sys
 import logging
 
+# from fromscratchtoml import np
 import numpy as np
 
 from .exceptions import ParameterRequiredException
@@ -71,6 +72,15 @@ def binary_visualize(X, y=None, clf=None, coarse=50, xlabel="x", ylabel="y",
         The seed value for randomising plot colors.
 
     """
+    try:
+        import cupy
+        if isinstance(y, cupy.core.core.ndarray):
+            y = cupy.asnumpy(y)
+        if isinstance(X, cupy.core.core.ndarray):
+            X = cupy.asnumpy(X)
+    except ImportError:
+        pass
+
     np.random.seed(color_seed)
 
     if len(X.shape) != 2 or X.shape[1] != 2:
@@ -79,6 +89,7 @@ def binary_visualize(X, y=None, clf=None, coarse=50, xlabel="x", ylabel="y",
 
     if clf:
         y = clf.predict(X)
+        y = cupy.asnumpy(y)
 
     # Also handles the cases when y is None
     unq, y = np.unique(y, return_inverse=True)
@@ -104,7 +115,7 @@ def binary_visualize(X, y=None, clf=None, coarse=50, xlabel="x", ylabel="y",
         _X, _Y = np.meshgrid(np.arange(x_min, x_max, 1 / (coarse * 1.0)), np.arange(y_min, y_max, 1 / (coarse * 1.0)))
 
         Z = np.c_[_X.ravel(), _Y.ravel()]
-        Z = clf.predict(Z)
+        Z = cupy.asnumpy(clf.predict(Z))
 
         unq, Z = np.unique(Z, return_inverse=True)
         Z = Z.reshape(_X.shape)

fromscratchtoml/toolbox/kernels.py

@@ -3,7 +3,7 @@
 #
 # Copyright (C) 2017 Mohit Rathore <mrmohitrathoremr@gmail.com>
 # Licensed under the GNU General Public License v3.0 - https://www.gnu.org/licenses/gpl-3.0.en.html
-import numpy as np
+from fromscratchtoml import np
 
 
 def linear(x, y, **kwargs):

fromscratchtoml/toolbox/preprocess.py

@@ -4,8 +4,8 @@
 # Copyright (C) 2017 Mohit Rathore <mrmohitrathoremr@gmail.com>
 # Licensed under the GNU General Public License v3.0 - https://www.gnu.org/licenses/gpl-3.0.en.html
 
-import numpy as np
-
+from fromscratchtoml import np
+import numpy
 
 import logging
 
@@ -15,12 +15,22 @@
 
 
 def to_onehot(y):
-    unq, y5 = np.unique(y, return_inverse=True)
 
-    a = []
+    try:
+        import cupy
+        if isinstance(y, cupy.core.core.ndarray):
+            y = np.asnumpy(y)
+    except ImportError:
+        pass
+
+    unq, _ = numpy.unique(y, return_inverse=True)
+
+    # a = []
+    a = np.zeros((len(y), len(unq)))
     for i in range(len(y)):
-        x = np.zeros(len(unq))
-        x[int(y[i])] = 1.
-        a.append(x)
+        # x = np.zeros(len(unq))
+        a[i][int(y[i])] = 1.
+        # a.append(x)
 
-    return np.array(a, dtype=np.float128)
+    # return np.array(a)
+    return a