Variance_Covariance.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GRADED FUNCTION: DO NOT EDIT THIS LINE\n",
    "def mean_naive(X):\n",
    "    \"Compute the mean for a dataset X nby iterating over the data points\"\n",
    "    # X is of size (D,N) where D is the dimensionality and N the number of data points\n",
    "    D, N = X.shape\n",
    "    mean = np.zeros((D,1))\n",
    "    ### Edit the code; iterate over the dataset and compute the mean vector.\n",
    "    for i in range(D):\n",
    "        val = 0\n",
    "        for n in range(N):\n",
    "            val = val + X[i,n]\n",
    "        val = val / N\n",
    "        mean[i, 0] = val\n",
    "    return mean\n",
    "\n",
    "def cov_naive(X):\n",
    "    \"\"\"Compute the covariance for a dataset of size (D,N) \n",
    "    where D is the dimension and N is the number of data points\"\"\"\n",
    "    D, N = X.shape\n",
    "    ### Edit the code below to compute the covariance matrix by iterating over the dataset.\n",
    "    covariance = np.zeros((D, D))\n",
    "    mean = np.zeros((D, 1))\n",
    "    ### Update covariance\n",
    "    for row in range(D):\n",
    "        val = 0\n",
    "        for col in range(N):\n",
    "            val += X[row, col]\n",
    "        val = val / N\n",
    "        mean[row, 0] = val\n",
    "    new_X = np.zeros((D, N))\n",
    "    for row in range(D):\n",
    "        for col in range(N):\n",
    "            new_X[row, col] = X[row, col] - mean[row, 0]\n",
    "#     covariance = new_X @ (new_X.T) # D * D mmatrix\n",
    "    covariance = np.matmul(new_X, new_X.T)\n",
    "    for i in range(D):\n",
    "        for j in range(D):\n",
    "            covariance[i, j] = covariance[i, j] / N\n",
    "    return covariance\n",
    "\n",
    "\n",
    "def mean(X):\n",
    "    \"Compute the mean for a dataset of size (D,N) where D is the dimension and N is the number of data points\"\n",
    "    # given a dataset of size (D, N), the mean should be an array of size (D,1)\n",
    "    # you can use np.mean, but pay close attention to the shape of the mean vector you are returning.\n",
    "    D, N = X.shape\n",
    "    ### Edit the code to compute a (D,1) array `mean` for the mean of dataset.\n",
    "    mean = np.zeros((D,1))\n",
    "    ### Update mean here\n",
    "    mean = np.mean(X, axis=1, keepdims=True)\n",
    "    ###\n",
    "    return mean\n",
    "\n",
    "def cov(X):\n",
    "    \"Compute the covariance for a dataset\"\n",
    "    # X is of size (D,N)\n",
    "    # It is possible to vectorize our code for computing the covariance with matrix multiplications,\n",
    "    # i.e., we do not need to explicitly\n",
    "    # iterate over the entire dataset as looping in Python tends to be slow\n",
    "    # We challenge you to give a vectorized implementation without using np.cov, but if you choose to use np.cov,\n",
    "    # be sure to pass in bias=True.\n",
    "    D, N = X.shape\n",
    "    ### Edit the code to compute the covariance matrix\n",
    "    covariance_matrix = np.zeros((D, D))\n",
    "    ### Update covariance_matrix here\n",
    "    covariance_matrix = np.cov(X, bias=True)\n",
    "    ###\n",
    "    return covariance_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = [[1,2,3], [4,5,6], [7,8,9], [10, 11, 12]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]]"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "np_array = np.array(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1,  2,  3],\n",
       "       [ 4,  5,  6],\n",
       "       [ 7,  8,  9],\n",
       "       [10, 11, 12]])"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 1,  4,  7, 10],\n",
       "       [ 2,  5,  8, 11],\n",
       "       [ 3,  6,  9, 12]])"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np_array.shape\n",
    "np_array = np_array.T\n",
    "print(np_array.shape)\n",
    "np_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5.5],\n",
       "       [6.5],\n",
       "       [7.5]])"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean_naive(np_array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5.5],\n",
       "       [6.5],\n",
       "       [7.5]])"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean(np_array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[11.25, 11.25, 11.25],\n",
       "       [11.25, 11.25, 11.25],\n",
       "       [11.25, 11.25, 11.25]])"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cov_naive(np_array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[11.25, 11.25, 11.25],\n",
       "       [11.25, 11.25, 11.25],\n",
       "       [11.25, 11.25, 11.25]])"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cov(np_array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = np.arange(6).reshape(2,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 1, 2],\n",
       "       [3, 4, 5]])"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 3)"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.],\n",
       "       [4.]])"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean_naive(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.],\n",
       "       [4.]])"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.66666667, 0.66666667],\n",
       "       [0.66666667, 0.66666667]])"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cov_naive(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.66666667, 0.66666667],\n",
       "       [0.66666667, 0.66666667]])"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cov(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0,  2,  4],\n",
       "       [ 6,  8, 10]])"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.multiply(X_test, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2, 3, 4],\n",
       "       [5, 6, 7]])"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.add(X_test, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.power(2,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GRADED FUNCTION: DO NOT EDIT THIS LINE\n",
    "def affine_mean(mean, A, b):\n",
    "    \"\"\"Compute the mean after affine transformation\n",
    "    Args:\n",
    "        x: ndarray, the mean vector\n",
    "        A, b: affine transformation applied to x\n",
    "    Returns:\n",
    "        mean vector after affine transformation\n",
    "    \"\"\"\n",
    "#     print(mean.shape, A.shape)\n",
    "    affine_multiplied = np.matmul(A, mean)\n",
    "    affine_mean = np.add(affine_multiplied, b)\n",
    "    return affine_mean\n",
    "\n",
    "def affine_covariance(S, A, b):\n",
    "    \"\"\"Compute the covariance matrix after affine transformation\n",
    "    Args:\n",
    "        S: ndarray, the covariance matrix\n",
    "        A, b: affine transformation applied to each element in X        \n",
    "    Returns:\n",
    "        covariance matrix after the transformation\n",
    "    \"\"\"\n",
    "    affine_cov = np.zeros(S.shape) # affine_cov has shape (D, D)\n",
    "#     print('A:', A.shape, 'b:', b.shape, 'S:', S.shape)\n",
    "    affine_cov = np.matmul(A, S)\n",
    "    affine_cov = np.matmul(affine_cov, A.T)\n",
    "    return affine_cov"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "random = np.random.RandomState(42)\n",
    "A = random.randn(4,4)\n",
    "b = random.randn(4,1)\n",
    "X = random.randn(4,100) # D = 4, N = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "X1 = (A @ X) + b  # applying affine transformation to each sample in X\n",
    "X2 = (A @ X1) + b # twice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.testing.assert_almost_equal(mean(X1), affine_mean(mean(X), A, b))\n",
    "np.testing.assert_almost_equal(cov(X1),  affine_covariance(cov(X), A, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.96587216]\n",
      " [ 0.3952622 ]\n",
      " [-0.8632775 ]\n",
      " [-1.70075537]]\n",
      "[[-0.96587216]\n",
      " [ 0.3952622 ]\n",
      " [-0.8632775 ]\n",
      " [-1.70075537]]\n"
     ]
    }
   ],
   "source": [
    "print(affine_mean(mean(X), A, b))\n",
    "print(mean(X1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 2.63643786  2.09602474 -1.30482866 -1.21104151]\n",
      " [ 2.09602474  3.00709703 -1.22107565 -2.4208805 ]\n",
      " [-1.30482866 -1.22107565  1.00076797 -0.08144533]\n",
      " [-1.21104151 -2.4208805  -0.08144533  6.35730278]]\n",
      "[[ 2.63643786  2.09602474 -1.30482866 -1.21104151]\n",
      " [ 2.09602474  3.00709703 -1.22107565 -2.4208805 ]\n",
      " [-1.30482866 -1.22107565  1.00076797 -0.08144533]\n",
      " [-1.21104151 -2.4208805  -0.08144533  6.35730278]]\n"
     ]
    }
   ],
   "source": [
    "print(affine_covariance(cov(X), A, b))\n",
    "print(cov(X1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.testing.assert_almost_equal(mean(X2), affine_mean(mean(X1), A, b))\n",
    "np.testing.assert_almost_equal(cov(X2),  affine_covariance(cov(X1), A, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "np_array = np.array([[5, 2, -1], [2, 2, 2], [-1, 2, 5]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, 3)"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np_array.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 5,  2, -1])"
      ]
     },
     "execution_count": 223,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np_array[0:, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 5,  2, -1])"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np_array[0, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.linalg.matrix_rank(np_array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = np.array([[ 1, 2, 3],\n",
    "              [-1, 1, 4]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.70710678,  0.89442719,  0.6       ],\n",
       "       [-0.70710678,  0.4472136 ,  0.8       ]])"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c/np.linalg.norm(c, axis=0, ord=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = np.array([[1,2,3]])\n",
    "e = np.array([1,2,3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1,  4,  9],\n",
       "       [-1,  2, 12]])"
      ]
     },
     "execution_count": 233,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c * d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1,  4,  9],\n",
       "       [-1,  2, 12]])"
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c * e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1. , 1. , 1.5])"
      ]
     },
     "execution_count": 239,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f = c * e\n",
    "np.std(f, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 3)"
      ]
     },
     "execution_count": 240,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1,  4,  9],\n",
       "       [-1,  2, 12]])"
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = np.array([1,2,3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  2.,  3.],\n",
       "       [-1.,  1.,  4.]])"
      ]
     },
     "execution_count": 244,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.divide(f, c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  2.,  3.],\n",
       "       [-1.,  1.,  4.]])"
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f/c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'numpy.ndarray' object has no attribute 'type'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m-------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-249-ae494cf3f58b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mnan\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'numpy.ndarray' object has no attribute 'type'"
     ]
    }
   ],
   "source": [
    "c.type != nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}