python3

Author: lazyprogrammer

logistic_regression_class/bad_xor.py

@@ -0,0 +1,78 @@
+# logisitc regression classifier for the XOR problem.
+#
+# the notes for this class can be found at: 
+# https://deeplearningcourses.com/c/data-science-logistic-regression-in-python
+# https://www.udemy.com/data-science-logistic-regression-in-python
+
+from __future__ import print_function, division
+from builtins import range
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+N = 4
+D = 2
+
+# XOR
+X = np.array([
+    [0, 0],
+    [0, 1],
+    [1, 0],
+    [1, 1],
+])
+T = np.array([0, 1, 1, 0])
+
+# add a column of ones
+ones = np.ones((N, 1))
+
+# add a column of xy = x*y
+Xb = np.concatenate((ones, X), axis=1)
+
+# randomly initialize the weights
+w = np.random.randn(D + 1)
+
+# calculate the model output
+z = Xb.dot(w)
+
+def sigmoid(z):
+    return 1/(1 + np.exp(-z))
+
+
+Y = sigmoid(z)
+
+# calculate the cross-entropy error
+def cross_entropy(T, Y):
+    return -(T*np.log(Y) + (1-T)*np.log(1-Y)).sum()
+
+
+# let's do gradient descent 100 times
+learning_rate = 0.001
+error = []
+w_mags = []
+for i in range(100000):
+    e = cross_entropy(T, Y)
+    error.append(e)
+    if i % 1000 == 0:
+        print(e)
+
+    # gradient descent weight udpate with regularization
+    w += learning_rate * Xb.T.dot(T - Y)
+
+    w_mags.append(w.dot(w))
+
+    # recalculate Y
+    Y = sigmoid(Xb.dot(w))
+
+plt.plot(error)
+plt.title("Cross-entropy per iteration")
+plt.show()
+
+plt.plot(w_mags)
+plt.title("w^2 magnitudes")
+plt.show()
+
+print("Final w:", w)
+print("Final classification rate:", 1 - np.abs(T - np.round(Y)).sum() / N)

logistic_regression_class/l1_regularization.py

@@ -2,6 +2,12 @@
 # https://deeplearningcourses.com/c/data-science-logistic-regression-in-python
 # https://www.udemy.com/data-science-logistic-regression-in-python
 
+from __future__ import print_function, division
+from builtins import range
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+
 import numpy as np
 import matplotlib.pyplot as plt
 
@@ -25,7 +31,7 @@ def sigmoid(z):
 w = np.random.randn(D) / np.sqrt(D) # randomly initialize w
 learning_rate = 0.001
 l1 = 3.0 # try different values - what effect does it have on w?
-for t in xrange(5000):
+for t in range(5000):
   # update w
   Yhat = sigmoid(X.dot(w))
   delta = Yhat - Y
@@ -39,7 +45,7 @@ def sigmoid(z):
 plt.plot(costs)
 plt.show()
 
-print "final w:", w
+print("final w:", w)
 
 # plot our w vs true w
 plt.plot(true_w, label='true w')

logistic_regression_class/logistic1.py

@@ -6,6 +6,12 @@
 # https://deeplearningcourses.com/c/data-science-logistic-regression-in-python
 # https://www.udemy.com/data-science-logistic-regression-in-python
 
+from __future__ import print_function, division
+from builtins import range
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+
 import numpy as np
 
 N = 100
@@ -24,4 +30,4 @@
 def sigmoid(z):
     return 1/(1 + np.exp(-z))
 
-print sigmoid(z)
+print(sigmoid(z))

logistic_regression_class/logistic2.py

@@ -5,6 +5,13 @@
 # https://deeplearningcourses.com/c/data-science-logistic-regression-in-python
 # https://www.udemy.com/data-science-logistic-regression-in-python
 
+from __future__ import print_function, division
+from builtins import range
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+
+
 import numpy as np
 
 N = 100
@@ -41,14 +48,14 @@ def sigmoid(z):
 # calculate the cross-entropy error
 def cross_entropy(T, Y):
     E = 0
-    for i in xrange(len(T)):
+    for i in range(len(T)):
         if T[i] == 1:
             E -= np.log(Y[i])
         else:
             E -= np.log(1 - Y[i])
     return E
 
-print cross_entropy(T, Y)
+print(cross_entropy(T, Y))
 
 # try it with our closed-form solution
 w = np.array([0, 4, 4])
@@ -58,5 +65,5 @@ def cross_entropy(T, Y):
 Y = sigmoid(z)
 
 # calculate the cross-entropy error
-print cross_entropy(T, Y)
+print(cross_entropy(T, Y))
 

logistic_regression_class/logistic3.py

@@ -4,6 +4,12 @@
 # https://deeplearningcourses.com/c/data-science-logistic-regression-in-python
 # https://www.udemy.com/data-science-logistic-regression-in-python
 
+from __future__ import print_function, division
+from builtins import range
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+
 import numpy as np
 import matplotlib.pyplot as plt
 
@@ -42,7 +48,7 @@ def sigmoid(z):
 # calculate the cross-entropy error
 def cross_entropy(T, Y):
     E = 0
-    for i in xrange(len(T)):
+    for i in range(len(T)):
         if T[i] == 1:
             E -= np.log(Y[i])
         else:
@@ -52,9 +58,9 @@ def cross_entropy(T, Y):
 
 # let's do gradient descent 100 times
 learning_rate = 0.1
-for i in xrange(100):
+for i in range(100):
     if i % 10 == 0:
-        print cross_entropy(T, Y)
+        print(cross_entropy(T, Y))
 
     # gradient descent weight udpate
     # w += learning_rate * np.dot((T - Y).T, Xb) # old
@@ -64,7 +70,7 @@ def cross_entropy(T, Y):
     Y = sigmoid(Xb.dot(w))
 
 
-print "Final w:", w
+print("Final w:", w)
 
 # plot the data and separating line
 plt.scatter(X[:,0], X[:,1], c=T, s=100, alpha=0.5)

logistic_regression_class/logistic4.py

@@ -5,6 +5,13 @@
 # https://deeplearningcourses.com/c/data-science-logistic-regression-in-python
 # https://www.udemy.com/data-science-logistic-regression-in-python
 
+from __future__ import print_function, division
+from builtins import range
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+
+
 import numpy as np
 
 N = 100
@@ -42,7 +49,7 @@ def sigmoid(z):
 # calculate the cross-entropy error
 def cross_entropy(T, Y):
     E = 0
-    for i in xrange(len(T)):
+    for i in range(len(T)):
         if T[i] == 1:
             E -= np.log(Y[i])
         else:
@@ -52,9 +59,9 @@ def cross_entropy(T, Y):
 
 # let's do gradient descent 100 times
 learning_rate = 0.1
-for i in xrange(100):
+for i in range(100):
     if i % 10 == 0:
-        print cross_entropy(T, Y)
+        print(cross_entropy(T, Y))
 
     # gradient descent weight udpate with regularization
     # w += learning_rate * ( np.dot((T - Y).T, Xb) - 0.1*w ) # old
@@ -64,6 +71,6 @@ def cross_entropy(T, Y):
     Y = sigmoid(Xb.dot(w))
 
 
-print "Final w:", w
+print("Final w:", w)
 
 

logistic_regression_class/logistic_donut.py

@@ -4,6 +4,13 @@
 # https://deeplearningcourses.com/c/data-science-logistic-regression-in-python
 # https://www.udemy.com/data-science-logistic-regression-in-python
 
+from __future__ import print_function, division
+from builtins import range
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+
+
 import numpy as np
 import matplotlib.pyplot as plt
 
@@ -15,16 +22,16 @@
 
 # distance from origin is radius + random normal
 # angle theta is uniformly distributed between (0, 2pi)
-R1 = np.random.randn(N/2) + R_inner
-theta = 2*np.pi*np.random.random(N/2)
+R1 = np.random.randn(N//2) + R_inner
+theta = 2*np.pi*np.random.random(N//2)
 X_inner = np.concatenate([[R1 * np.cos(theta)], [R1 * np.sin(theta)]]).T
 
-R2 = np.random.randn(N/2) + R_outer
-theta = 2*np.pi*np.random.random(N/2)
+R2 = np.random.randn(N//2) + R_outer
+theta = 2*np.pi*np.random.random(N//2)
 X_outer = np.concatenate([[R2 * np.cos(theta)], [R2 * np.sin(theta)]]).T
 
 X = np.concatenate([ X_inner, X_outer ])
-T = np.array([0]*(N/2) + [1]*(N/2)) # labels: first 50 are 0, last 50 are 1
+T = np.array([0]*(N//2) + [1]*(N//2)) # labels: first 50 are 0, last 50 are 1
 
 plt.scatter(X[:,0], X[:,1], c=T)
 plt.show()
@@ -53,24 +60,17 @@ def sigmoid(z):
 
 # calculate the cross-entropy error
 def cross_entropy(T, Y):
-    # E = 0
-    # for i in xrange(N):
-    #     if T[i] == 1:
-    #         E -= np.log(Y[i])
-    #     else:
-    #         E -= np.log(1 - Y[i])
-    # return E
     return -(T*np.log(Y) + (1-T)*np.log(1-Y)).sum()
 
 
 # let's do gradient descent 100 times
 learning_rate = 0.0001
 error = []
-for i in xrange(5000):
+for i in range(5000):
     e = cross_entropy(T, Y)
     error.append(e)
     if i % 500 == 0:
-        print e
+        print(e)
 
     # gradient descent weight udpate with regularization
     # w += learning_rate * ( np.dot((T - Y).T, Xb) - 0.01*w ) # old
@@ -83,5 +83,5 @@ def cross_entropy(T, Y):
 plt.title("Cross-entropy per iteration")
 plt.show()
 
-print "Final w:", w
-print "Final classification rate:", 1 - np.abs(T - np.round(Y)).sum() / N
+print("Final w:", w)
+print("Final classification rate:", 1 - np.abs(T - np.round(Y)).sum() / N)

logistic_regression_class/logistic_visualize.py

@@ -4,6 +4,13 @@
 # https://deeplearningcourses.com/c/data-science-logistic-regression-in-python
 # https://www.udemy.com/data-science-logistic-regression-in-python
 
+from __future__ import print_function, division
+from builtins import range
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+
+
 import numpy as np
 import matplotlib.pyplot as plt
 

logistic_regression_class/logistic_xor.py

@@ -4,6 +4,12 @@
 # https://deeplearningcourses.com/c/data-science-logistic-regression-in-python
 # https://www.udemy.com/data-science-logistic-regression-in-python
 
+from __future__ import print_function, division
+from builtins import range
+# Note: you may need to update your version of future
+# sudo pip install -U future
+
+
 import numpy as np
 import matplotlib.pyplot as plt
 
@@ -42,7 +48,7 @@ def sigmoid(z):
 # calculate the cross-entropy error
 def cross_entropy(T, Y):
     E = 0
-    for i in xrange(len(T)):
+    for i in range(len(T)):
         if T[i] == 1:
             E -= np.log(Y[i])
         else:
@@ -51,13 +57,13 @@ def cross_entropy(T, Y):
 
 
 # let's do gradient descent 100 times
-learning_rate = 0.001
+learning_rate = 0.01
 error = []
-for i in xrange(10000):
+for i in range(10000):
     e = cross_entropy(T, Y)
     error.append(e)
-    if i % 100 == 0:
-        print e
+    if i % 1000 == 0:
+        print(e)
 
     # gradient descent weight udpate with regularization
     w += learning_rate * ( Xb.T.dot(T - Y) - 0.01*w )
@@ -69,5 +75,5 @@ def cross_entropy(T, Y):
 plt.title("Cross-entropy per iteration")
 plt.show()
 
-print "Final w:", w
-print "Final classification rate:", 1 - np.abs(T - np.round(Y)).sum() / N
+print("Final w:", w)
+print("Final classification rate:", 1 - np.abs(T - np.round(Y)).sum() / N)