SVM lecture

ML-course · Feb 20, 2017 · efbe4f4 · efbe4f4
1 parent 87df3fa
commit efbe4f4
Show file tree

Hide file tree

Showing 9 changed files with 1,712 additions and 159 deletions.
diff --git a/03 - Model Selection.ipynb b/03 - Model Selection.ipynb
diff --git a/05 - Support Vector Machines.ipynb b/05 - Support Vector Machines.ipynb
diff --git a/images/05_gaussian_curves.png b/images/05_gaussian_curves.png
diff --git a/images/05_hinge.png b/images/05_hinge.png
diff --git a/images/05_kernel_map.png b/images/05_kernel_map.png
diff --git a/images/05_loss.png b/images/05_loss.png
diff --git a/images/05_margin.png b/images/05_margin.png
diff --git a/mglearn/plot_svm.py b/mglearn/plot_svm.py
@@ -0,0 +1,260 @@
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn import svm
+
+# Set of examples originating from sci-kit learn documentation
+# Code source: Gaël Varoquaux
+# Adaptations by Joaquin Vanschoren
+
+def plot_svm_linear():
+    # we create 40 separable points
+    np.random.seed(0)
+    X = np.r_[np.random.randn(20, 2) - [2, 2], np.random.randn(20, 2) + [2, 2]]
+    Y = [0] * 20 + [1] * 20
+
+    # fit the model
+    clf = svm.SVC(kernel='linear')
+    clf.fit(X, Y)
+
+    # get the separating hyperplane
+    w = clf.coef_[0]
+    a = -w[0] / w[1]
+    xx = np.linspace(-5, 5)
+    yy = a * xx - (clf.intercept_[0]) / w[1]
+
+    # plot the parallels to the separating hyperplane that pass through the
+    # support vectors
+    b = clf.support_vectors_[0]
+    yy_down = a * xx + (b[1] - a * b[0])
+    b = clf.support_vectors_[-1]
+    yy_up = a * xx + (b[1] - a * b[0])
+
+    # plot the line, the points, and the nearest vectors to the plane
+    plt.plot(xx, yy, 'k-')
+    plt.plot(xx, yy_down, 'k--')
+    plt.plot(xx, yy_up, 'k--')
+
+    plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1],
+                s=85, edgecolors='k', c='w')
+    plt.scatter(X[:, 0], X[:, 1], c=Y, cmap=plt.cm.bwr)
+
+    # Add coefficients
+    for i, coef in enumerate(clf.dual_coef_[0]):
+        plt.annotate("%0.2f" % (coef), (clf.support_vectors_[i, 0]+0.15,clf.support_vectors_[i, 1]), fontsize=8)
+
+    plt.axis('tight')
+    plt.show()
+
+def plot_svm_kernels():
+    # Our dataset and targets
+    X = np.c_[(.4, -.7),
+              (-1.5, -1),
+              (-1.4, -.9),
+              (-1.3, -1.2),
+              (-1.1, -.2),
+              (-1.2, -.4),
+              (-.5, 1.2),
+              (-1.5, 2.1),
+              (1, 1),
+              # --
+              (1.3, .8),
+              (1.2, .5),
+              (.2, -2),
+              (.5, -2.4),
+              (.2, -2.3),
+              (0, -2.7),
+              (1.3, 2.1)].T
+    Y = [0] * 8 + [1] * 8
+
+    # figure number
+    fignum = 1
+
+    # fit the model
+    for kernel in ('linear', 'poly', 'rbf'):
+        clf = svm.SVC(kernel=kernel, gamma=2)
+        clf.fit(X, Y)
+
+        # plot the line, the points, and the nearest vectors to the plane
+        plt.figure(fignum, figsize=(4, 3))
+        plt.suptitle('kernel = %s' % kernel)
+
+        plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1],
+                    s=85, edgecolors='k', c='w', zorder=10)
+        plt.scatter(X[:, 0], X[:, 1], c=Y, zorder=10, cmap=plt.cm.bwr)
+
+        for i, coef in enumerate(clf.dual_coef_[0]):
+            plt.annotate("%0.2f" % (coef), (clf.support_vectors_[i, 0]+0.15,clf.support_vectors_[i, 1]), fontsize=8, zorder=11)
+
+        plt.axis('tight')
+        x_min = -3
+        x_max = 3
+        y_min = -3
+        y_max = 3
+
+        XX, YY = np.mgrid[x_min:x_max:200j, y_min:y_max:200j]
+        Z = clf.decision_function(np.c_[XX.ravel(), YY.ravel()])
+
+        # Put the result into a color plot
+        Z = Z.reshape(XX.shape)
+        plt.figure(fignum, figsize=(4, 3))
+        #plt.pcolormesh(XX, YY, Z > 0, cmap=plt.cm.bwr, alpha=0.1)
+        plt.contour(XX, YY, Z, colors=['k', 'k', 'k'], linestyles=['--', '-', '--'],
+                    levels=[-.5, 0, .5])
+
+        plt.xlim(x_min, x_max)
+        plt.ylim(y_min, y_max)
+
+        plt.xticks(())
+        plt.yticks(())
+        fignum = fignum + 1
+    plt.show()
+
+def plot_svm_margins():
+    # we create 40 separable points
+    np.random.seed(0)
+    X = np.r_[np.random.randn(20, 2) - [2, 2], np.random.randn(20, 2) + [2, 2]]
+    Y = [0] * 20 + [1] * 20
+
+    # figure number
+    fignum = 1
+
+    # fit the model
+    for name, penalty in (('unreg', 1), ('reg', 0.05)):
+
+        clf = svm.SVC(kernel='linear', C=penalty)
+        clf.fit(X, Y)
+
+        # get the separating hyperplane
+        w = clf.coef_[0]
+        a = -w[0] / w[1]
+        xx = np.linspace(-5, 5)
+        yy = a * xx - (clf.intercept_[0]) / w[1]
+
+        # plot the parallels to the separating hyperplane that pass through the
+        # support vectors
+        margin = 1 / np.sqrt(np.sum(clf.coef_ ** 2))
+        yy_down = yy + a * margin
+        yy_up = yy - a * margin
+
+        # plot the line, the points, and the nearest vectors to the plane
+        plt.figure(fignum, figsize=(4, 3))
+        plt.suptitle('C = %s' % penalty)
+        plt.plot(xx, yy, 'k-')
+        plt.plot(xx, yy_down, 'k--')
+        plt.plot(xx, yy_up, 'k--')
+
+        plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1],
+                    s=85, edgecolors='k', c='w', zorder=10)
+        plt.scatter(X[:, 0], X[:, 1], c=Y, zorder=10, cmap=plt.cm.bwr)
+
+        plt.axis('tight')
+        x_min = -4.8
+        x_max = 4.2
+        y_min = -6
+        y_max = 6
+
+        XX, YY = np.mgrid[x_min:x_max:200j, y_min:y_max:200j]
+        Z = clf.predict(np.c_[XX.ravel(), YY.ravel()])
+
+        # Put the result into a color plot
+        Z = Z.reshape(XX.shape)
+        plt.figure(fignum, figsize=(4, 3))
+        #plt.pcolormesh(XX, YY, Z > 0, cmap=plt.cm.bwr, alpha=0.1)
+        plt.contour(XX, YY, Z, colors=['k', 'k', 'k'], linestyles=['--', '-', '--'],
+                    levels=[-.5, 0, .5])
+
+        # Add coefficients
+        for i, coef in enumerate(clf.dual_coef_[0]):
+            plt.annotate("%0.2f" % (coef), (clf.support_vectors_[i, 0]+0.15,clf.support_vectors_[i, 1]), fontsize=8, zorder=11)
+
+        plt.xlim(x_min, x_max)
+        plt.ylim(y_min, y_max)
+
+        plt.xticks(())
+        plt.yticks(())
+        fignum = fignum + 1
+
+    plt.show()
+
+def plot_svm_margins_nonlin():
+    # we create 40 separable points
+    # Our dataset and targets
+    X = np.c_[(.4, -.7),
+              (-1.5, -1),
+              (-1.4, -.9),
+              (-1.3, -1.2),
+              (-1.1, -.2),
+              (-1.2, -.4),
+              (-.5, 1.2),
+              (-1.5, 2.1),
+              (1, 1),
+              # --
+              (1.3, .8),
+              (1.2, .5),
+              (.2, -2),
+              (.5, -2.4),
+              (.2, -2.3),
+              (0, -2.7),
+              (1.3, 2.1)].T
+    Y = [0] * 8 + [1] * 8
+
+    # figure number
+    fignum = 1
+
+    # fit the model
+    for name, penalty in (('unreg', 1), ('reg', 0.05)):
+
+        clf = svm.SVC(kernel='linear', C=penalty)
+        clf.fit(X, Y)
+
+        # get the separating hyperplane
+        w = clf.coef_[0]
+        a = -w[0] / w[1]
+        xx = np.linspace(-5, 5)
+        yy = a * xx - (clf.intercept_[0]) / w[1]
+
+        # plot the parallels to the separating hyperplane that pass through the
+        # support vectors
+        margin = 1 / np.sqrt(np.sum(clf.coef_ ** 2))
+        yy_down = yy + a * margin
+        yy_up = yy - a * margin
+
+        # plot the line, the points, and the nearest vectors to the plane
+        plt.figure(fignum, figsize=(4, 3))
+        plt.suptitle('C = %s' % penalty)
+        plt.plot(xx, yy, 'k-')
+        plt.plot(xx, yy_down, 'k--')
+        plt.plot(xx, yy_up, 'k--')
+
+        plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1],
+                    s=85, edgecolors='k', c='w', zorder=10)
+        plt.scatter(X[:, 0], X[:, 1], c=Y, zorder=10, cmap=plt.cm.bwr)
+
+        plt.axis('tight')
+        x_min = -4.8
+        x_max = 4.2
+        y_min = -6
+        y_max = 6
+
+        XX, YY = np.mgrid[x_min:x_max:200j, y_min:y_max:200j]
+        Z = clf.predict(np.c_[XX.ravel(), YY.ravel()])
+
+        # Put the result into a color plot
+        Z = Z.reshape(XX.shape)
+        plt.figure(fignum, figsize=(4, 3))
+        #plt.pcolormesh(XX, YY, Z > 0, cmap=plt.cm.bwr, alpha=0.1)
+        plt.contour(XX, YY, Z, colors=['k', 'k', 'k'], linestyles=['--', '-', '--'],
+                    levels=[-.5, 0, .5])
+
+        # Add coefficients
+        for i, coef in enumerate(clf.dual_coef_[0]):
+            plt.annotate("%0.2f" % (coef), (clf.support_vectors_[i, 0]+0.15,clf.support_vectors_[i, 1]), fontsize=8, zorder=11)
+
+        plt.xlim(x_min, x_max)
+        plt.ylim(y_min, y_max)
+
+        plt.xticks(())
+        plt.yticks(())
+        fignum = fignum + 1
+
+    plt.show()
diff --git a/mglearn/plots.py b/mglearn/plots.py
@@ -29,7 +29,7 @@
 from .plot_dbscan import plot_dbscan
 from .plot_ridge import plot_ridge_n_samples
 from .plot_overfitting import plot_overfitting
-
+from .plot_svm import plot_svm_linear, plot_svm_margins, plot_svm_kernels,  plot_svm_margins_nonlin
 
 __all__ = ['plot_linear_svc_regularization',
            "plot_animal_tree", "plot_tree_progressive",
@@ -68,5 +68,9 @@
            'plot_decision_threshold',
            'plot_dbscan',
            'plot_ridge_n_samples',
-           'plot_overfitting'
+           'plot_overfitting',
+           'plot_svm_linear',
+           'plot_svm_margins',
+           'plot_svm_kernels',
+           'plot_svm_margins_nonlin'
            ]