Changed the cost function to specify the sum of squared residuals con…

…cretely. Adjusted the formula to fully employ matrix operations.

Added ordinary least squares function.

docs/source/conf.py

@@ -16,7 +16,6 @@
 import sys
 import os
 
-#sys.path.insert(0,"D:/Work/misc/python/projects/machine-learning/")
 sys.path.insert(0, os.path.abspath('../../'))
 
 # If extensions (or modules to document with autodoc) are in another directory,
@@ -120,7 +119,7 @@
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'alabaster'
+# html_theme = 'alabaster'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

docs/source/linear_regression.rst

@@ -3,4 +3,5 @@ Linear Regression
 
 Documentation for the linear regression module.
 
-.. autofunction:: linear_regression.cost_function
+.. autofunction:: linear_regression.sum_of_squared_residuals
+.. autofunction:: linear_regression.ordinary_least_squares

linear_regression.py

@@ -2,43 +2,76 @@
 
 import numpy as np
 
-def cost_function(x, y, theta):
-    """Computes the cost for given matrix of x values (including :math:`x_0`)
-    and vector of y values for specified theta.
-
-    Cost is defined as the sum of the squared differences between predicted and
-    true values:
+def sum_of_squared_residuals(x, y, beta):
+    """
+    Provide a measure for the difference between observed :math:`y` and
+    predicted :math:`x^T*\\beta` data.
 
-    .. math:: J(\\theta) = \\frac{1}{2n} \\displaystyle\\sum_{i=1}^{n}( f_\\theta (x_i)-y_i)^2
-    .. math:: f_\\theta (x_i)= \\theta_0 + \\theta_1 x_1 + ... + \\theta_i x_i = X^T \\theta
+    This measure is also called the sum of the squared residuals (SSR).
 
     Parameters
     ----------
     x : np.array
-        Corresponds to matrix :math:`X`.
+        Corresponds to a matrix :math:`x` which contains all :math:`x_i` values
+        (including :math:`x_0=1`).
     y : np.array
-        Corresponds to vector :math:`y`.
-    theta : np.array
-        Corresponds to vector :math:`\\theta`.
+        Corresponds to vector :math:`y` which refers to the observed values.
+    beta : np.array
+        Corresponds to vector :math:`\\beta` which contains the parameters to
+        calculate the predicted values.
 
     Returns
     -------
-    j : float
+    ssr : numpy.array
+
+    References
+    --------
+    .. [1] https://en.wikipedia.org/wiki/Residual_sum_of_squares
+
+    Notes
+    -----
+    .. math:: SSR(\\beta) = \\displaystyle\\sum_{i=1}^{n}(x_i\\beta_i-y_i)^2
+    .. math:: SSR(\\beta) = (x\\beta-y)^T (x\\beta-y)
+
+    """
+
+    diff_score = np.dot(x, beta) - y
+    ssr = np.dot(diff_score.transpose(), diff_score)
+
+    return ssr.ravel()
 
+def ordinary_least_squares(x, y):
     """
+    Analytically calculate the unknown parameters for a linear regression model
+    which minimize the sum of the squared errors of observed and predicted data.
 
-    n = len(y)
+    Parameters
+    ----------
+    x : np.array
+        Corresponds to a matrix :math:`x` which contains all :math:`x_i` values
+        (including :math:`x_0=1`).
+    y : np.array
+        Corresponds to vector :math:`y` which refers to the observed values.
 
-    y_predicted = np.dot(x, theta)
-    y_diff = y_predicted - y
+    Returns
+    -------
+    beta : numpy.array
 
-    y_diff_squared = y_diff ** 2
-    sum_score = sum(y_diff_squared)
+    References
+    --------
+    .. [2] https://en.wikipedia.org/wiki/Ordinary_least_squares
 
-    j = sum_score / (2*n)
+    Notes
+    -----
+    .. math:: \\hat{\\beta} =  (x^Tx)^{-1}x^Ty
+
+    """
 
-    return j
+    x_t = x.transpose()
+    inverse = np.linalg.pinv(np.dot(x_t, x))
+    beta = np.dot(inverse, np.dot(x_t, y))
 
+    return beta
 
 if __name__ == "__main__":
     x = np.array([[1, 11, 104],
@@ -51,7 +84,8 @@ def cost_function(x, y, theta):
                   [19],
                   [22]])
 
-    theta = np.zeros((3,1))
+    beta = np.zeros((3,1))
 
-    print(cost_function(x, y, theta))
+    print(sum_of_squared_residuals(x, y, beta))
+    print(ordinary_least_squares(x, y))