05 Statistics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ch 05. Statistics\n",
    "* Code from Data Science from Scratch by Joel Grus\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from linear_algebra import sum_of_squares, dot\n",
    "import math\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Number of friends of members"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_friends = [100,49,41,40,25,21,21,19,19,18,18,16,15,15,15,15,14,14,13,13,13,13,12,12,11,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,8,8,8,8,8,8,8,8,8,8,8,8,8,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "204"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(num_friends)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Describing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGLhJREFUeJzt3Xm4JHV97/H3R5aIgCIwIIsw4EVReQR0NCwuRFxQUTAXRQKGRS8mweBCNITrhrkmeuPFiF59BEFwAUVEFvGquUhUIiLDDoIXAyjgAAOyDS4IfO8fVadoDmfpMzPdfTjn/Xqefrqrurrq2zU159P1q6pfpaqQJAngcaMuQJI0exgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoaApJbkqyS6jrmOUkrw+yY1JliXZfgXn9aIkP19ZtY2b9wlJ/scg5q35w1CYx5LckORl48YdkOS8seGqenZV/fs081mYpJKsOqBSR+3jwNuraq2qumT8m+13v68NjWVJ7ppsRlX1o6p6xkCrnUSSjZIcl2RJknuTXJPkyCRrDni5H0ry5UEuQyuPoaBZbxaEzebAVdNMs20bGmtV1ToTTTDK75FkXeB8YA1gx6paG3g5sA7wtFHVpdnHUNCUevcmkrwgyeIk9yS5NclR7WQ/bJ/van8p75jkcUnel+SXSW5L8sUkT+qZ71+2792R5P3jlvOhJKcm+XKSe4AD2mWfn+Su9pfup5Os3jO/SvI3Sa5tfwX/Y5KntZ+5J8kpvdOP+44T1prkT5IsA1YBLkvynzNcd7skuSnJ3ye5BfjC2LieaTZO8o0kS5Ncn+TQnvc+1Nb9xfY7XZVkUc/72ye5uH3va8Djpyjn3cC9wH5VdQNAVd1YVe+oqsvb+e2U5MIkd7fPO/Us6xF7lb2//nv2FPdP8qsktyf57+17uwFHAHu328Zl7fgDklzX1n59kn1nsm41OIaCZuKTwCer6ok0vy5Pace/uH1ep/2lfD5wQPv4M2BLYC3g0wBJngV8BtgX2Ah4ErDJuGXtAZxK80v2K8CDwLuA9YEdgV2Bvxn3md2A5wE7AO8FjmmX8VRgG2CfSb7XhLVW1R+qaq12mm2ranl+UT8FWJdmb+Pg3jeSPA44C7iM5vvvCrwzySt7Jnsd8FWa9XAmD6/D1YHTgS+18/868F+nqONlwGlV9dBEb7Z7EmcDRwPrAUcBZydZbwbf9YXAM9rv8YEkz6yq7wD/BHyt3Ta2bZurjgZe1e6x7ARcOoPlaIAMBZ3e/vq+q20L/8wU0/4R+C9J1q+qZVX1kymm3Rc4qqquq6plwD8Ab2qbUPYCzqqq86rqfuADwPhOuM6vqtOr6qGq+l1VXVRVP6mqB9pfup8DXjLuMx+rqnuq6irgSuB77fLvBv4PMNlB4qlq7dfFPevx6J7xDwEfbAPmd+M+83xgQVV9uKrur6rrgGOBN/VMc15VfbuqHqQJgG3b8TsAqwH/WlV/rKpTgQunqG89YMkU778GuLaqvtSu45OBa4DXTvO9ex3Z/ltdRhN0204x7UPANknWqKol7b+ZZgFDQXtW1TpjDx7967vXW4CnA9e0zQu7TzHtxsAve4Z/CawKbNi+d+PYG1X1W+COcZ+/sXcgydOTfCvJLW2T0j/R7DX0urXn9e8mGF6LiU1Va7+e27MeD+0Zv7Sqfj/JZzYHNh4XykeMW+4tPa9/Czy+DauNgZvrkT1a9n6H8e6g2SubzPh1MDa/8XtwUxlf64Tru6ruA/YG/gpYkuTsJFvPYDkaIENBfauqa6tqH2AD4GPAqW1TwERd7f6a5o/emM2AB2j+UC8BNh17I8kaNL9kH7G4ccOfpfnlulXbfHUEkOX/Nn3XuqKm6ob4RuD63lCuqrWr6tV9zHcJsEmS3nWw2RTT/1/g9W2T1UTGr4Ox+d3cvr4PeELPe0/po8Yxj1oHVfXdqno5TVBdQ7OHpFnAUFDfkuyXZEHbLj122uWDwFKa5oAteyY/GXhXki2SrMXD7coP0BwreG17YHN14Eim/wO/NnAPsKz9VfnXK+2LTV3rIP0UuKc9EL1GklWSbJPk+X189nya4Do0yapJ/hx4wRTTHwU8ETgxyeYASTZJclSS5wDfBp6e5C/a+e0NPAv4Vvv5S2ma1FZrD3bvNYPveSuwcCyQkmyY5HXtD4o/AMtotiPNAoaCZmI34Kr2jJxPAm+qqt+3zT8fAf6jbQbZATiepg38h8D1wO+BvwVo24//luYA6hKas2Juo/kDMZm/A/6infZY4Gsr8XtNWusgtccJXgts1y73duDzNAfep/vs/cCf0xwgv5OmOea0Kab/Dc0B3T8CFyS5FzgHuBv4RVXdAewOHEbT1PReYPequr2dxftpTi64kybET5rBV/16+3xHkotp/u4cRrN38huaY0NTNVtqiOJNdjRq7a/zu2iahq4fdT3SfOaegkYiyWuTPKFtQvg4cAVww2irkjSwUEjy1CTnJrm6vejmHe34DyW5Ocml7aOfg2qae/agaT74NbAVTVOUu63SiA2s+SjJRsBGVXVxkrWBi4A9gTcCy6rq4wNZsCRpuQ2sL5aqWkJ7sUxV3ZvkamZ2zrMkaciGcqA5yUKaMzu2oemD5QCa0wsXA4dV1Z0TfOZg2m4B1lxzzedtvbXXtkjSTFx00UW3V9WCmXxm4KHQnlnyA+AjVXVakg1pTr0r4B9pmpgOmmoeixYtqsWLFw+0Tkmaa5JcVFWLpp/yYQM9+yjJasA3gK9U1WkAVXVrVT3YXgB1LFNfcCNJGqJBnn0U4Djg6qo6qmd8b/8rr6fpuEySNAsM8qYfOwNvBq5IMtYt7hHAPkm2o2k+ugF42wBrkCTNwCDPPjqPifuz+faglilJWjFe0SxJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqTOqqMuYCYWHn529/qGj75mheaxvJ+XpLnMPQVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1DAVJUsdQkCR1BhYKSZ6a5NwkVye5Ksk72vHrJvm3JNe2z08eVA2SpJkZ5J7CA8BhVfVMYAfgkCTPAg4HzqmqrYBz2mFJ0iwwsFCoqiVVdXH7+l7gamATYA/gxHayE4E9B1WDJGlmhnLntSQLge2BC4ANq2oJNMGRZINJPnMwcDDAZpttNu0yeu+otjLu0CZJ89HADzQnWQv4BvDOqrqn389V1TFVtaiqFi1YsGBwBUqSOgMNhSSr0QTCV6rqtHb0rUk2at/fCLhtkDVIkvo3yLOPAhwHXF1VR/W8dSawf/t6f+CMQdUgSZqZQR5T2Bl4M3BFkkvbcUcAHwVOSfIW4FfAGwZYgyRpBgYWClV1HpBJ3t51UMuVJC0/r2iWJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHWGcpOdxxpv0iNpvnJPQZLUMRQkSR1DQZLUMRQkSR1DQZLUMRQkSR1DQZLUMRQkSZ3H7MVr/V5g5oVoktQ/9xQkSR1DQZLUMRQkSR1DQZLUMRQkSR1DQZLUMRQkSR1DQZLUMRQkSR1DQZLUMRQkSZ2+QiHJ5kle1r5eI8nagy1LkjQK04ZCkv8GnAp8rh21KXD6IIuSJI1GP3sKhwA7A/cAVNW1wAbTfSjJ8UluS3Jlz7gPJbk5yaXt49XLW7gkaeXrJxT+UFX3jw0kWRWoPj53ArDbBOM/UVXbtY9v91emJGkY+gmFHyQ5AlgjycuBrwNnTfehqvoh8JsVrE+SNET9hMLhwFLgCuBtwLeB963AMt+e5PK2eenJk02U5OAki5MsXrp06QosbnILDz/7ETfhWdHpJOmxbtpQqKqHqurYqnpDVe3Vvu6n+WginwWeBmwHLAH+1xTLPaaqFlXVogULFizn4iRJMzHp7TiTXMEUxw6q6jkzXVhV3doz/2OBb810HpKkwZnqHs27r+yFJdmoqpa0g68HrpxqeknScE0aClX1y7HXSZ4CvIBmz+HCqrpluhknORnYBVg/yU3AB4FdkmzXzucGmmMUkqRZYqo9BQCSvBX4APB9IMCnkny4qo6f6nNVtc8Eo49briolSUMxbSgA7wG2r6o7AJKsB/wYmDIUJEmPPf2cknoTcG/P8L3AjYMpR5I0Sv3sKdwMXJDkDJpjAXsAP03yboCqOmqA9UmShqifUPjP9jHmjPbZnlIlaY6ZNhSq6kiAtrvsqqplA69KkjQS/XSdvU2SS2iuKbgqyUVJnj340iRJw9bPgeZjgHdX1eZVtTlwGHDsYMuSJI1CP6GwZlWdOzZQVf8OrDmwiiRJI9PPgebrkrwf+FI7vB9w/eBKkiSNSj97CgcBC4DT2sf6wIGDLEqSNBr9nH10J3BokrU880iS5rZ+zj7aKcnPgJ+1w9sm+czAK5MkDV0/xxQ+AbwSOBOgqi5L8uKBVjUC3llNkvo7pkBVje/r6MEB1CJJGrF+9hRuTLITUElWBw4Frh5sWZKkUehnT+GvgEOATWg6x9uuHZYkzTH9nH10O7DvEGqRJI1YP2cfbZnkrCRLk9yW5IwkWw6jOEnScPXTfHQScAqwEbAx8HXg5EEWJUkajX5CIVX1pap6oH18meZmO5KkOaafs4/OTXI48FWaMNgbODvJugBV9ZsB1idJGqJ+QmHv9vlt48YfRBMSHl+QpDmin7OPthhGIZKk0evrimZJ0vxgKEiSOpOGQpKd2+c/GV45kqRRmmpP4ej2+fxhFCJJGr2pDjT/MckXgE2SHD3+zao6dHBlSZJGYapQ2B14GfBS4KLhlCNJGqVJQ6HtCO+rSa6uqsuGWJMkaUT6OfvojiTfbDvDuzXJN5JsOvDKJElD108ofIHmVpwb09xT4ax2nCRpjuknFDaoqi/0dIh3ArBgwHVJkkagn1BYmmS/JKu0j/2AOwZdmCRp+PoJhYOANwK3AEuAvdpxkqQ5pp8O8X4FvG6mM05yPM1prbdV1TbtuHWBrwELgRuAN1bVnTOdtyRpMAbZ99EJwG7jxh0OnFNVWwHntMOSpFliYKFQVT8Ext+AZw/gxPb1icCeg1q+JGnmht1L6oZVtQSgfd5gsgmTHJxkcZLFS5cuHVqBkjSfTRsKSd7X83poPaZW1TFVtaiqFi1Y4BmwkjQMU3Wd/d4kO9KcbTRmRXtMvTXJRu38NwJuW8H5SZJWoqn2FH4OvAHYMsmPkhwDrJfkGSuwvDOB/dvX+wNnrMC8JEkr2VShcCdwBPALYBcevr/C4Ul+PN2Mk5xMs2fxjCQ3JXkL8FHg5UmuBV7eDkuSZomprlPYDfgg8DTgKOAy4L6qOrCfGVfVPpO8teuMKpQkDc2kewpVdURV7UpzkdmXaQJkQZLzkpw1pPokSUM07RXNwHer6kLgwiR/XVUvTLL+oAuTJA3ftKekVtV7ewYPaMfdPqiCJEmjM6OL17wDmyTNbcO+olmSNIsZCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkjqEgSeoYCpKkzqqjWGiSG4B7gQeBB6pq0SjqkCQ90khCofVnVXX7CJcvSRrH5iNJUmdUoVDA95JclOTgiSZIcnCSxUkWL126dMjlSdL8NKpQ2Lmqngu8CjgkyYvHT1BVx1TVoqpatGDBguFXKEnz0EhCoap+3T7fBnwTeMEo6pAkPdLQQyHJmknWHnsNvAK4cth1SJIebRRnH20IfDPJ2PJPqqrvjKAOSdI4Qw+FqroO2HbYy5UkTc9TUiVJHUNBktQxFCRJnVF2czHnLDz8bABu+OhrJhw/0XuSNJu4pyBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSOoSBJ6hgKkqSON9lZAVPdPKf3veWdnzftkTRs7ilIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSpYyhIkjqGgiSp48VrM7Q8F6VNNo9hXng2Vy546/cCv0EvV5ptVtY26p6CJKljKEiSOoaCJKljKEiSOoaCJKkzklBIsluSnyf5RZLDR1GDJOnRhh4KSVYB/jfwKuBZwD5JnjXsOiRJjzaKPYUXAL+oquuq6n7gq8AeI6hDkjROqmq4C0z2Anarqre2w28G/rSq3j5uuoOBg9vBbYArh1ro7LQ+cPuoi5glXBcN10PD9dAYvx42r6oFM5nBKK5ozgTjHpVMVXUMcAxAksVVtWjQhc12roeHuS4aroeG66GxMtbDKJqPbgKe2jO8KfDrEdQhSRpnFKFwIbBVki2SrA68CThzBHVIksYZevNRVT2Q5O3Ad4FVgOOr6qppPnbM4Ct7THA9PMx10XA9NFwPjRVeD0M/0CxJmr28olmS1DEUJEmdWR0K87k7jCRPTXJukquTXJXkHe34dZP8W5Jr2+cnj7rWYUiySpJLknyrHd4iyQXtevhae9LCnJZknSSnJrmm3S52nMfbw7va/xdXJjk5yePnwzaR5PgktyW5smfchNtAGke3fz8vT/LcfpYxa0PB7jB4ADisqp4J7AAc0n7/w4Fzqmor4Jx2eD54B3B1z/DHgE+06+FO4C0jqWq4Pgl8p6q2BralWR/zbntIsglwKLCoqrahOWHlTcyPbeIEYLdx4ybbBl4FbNU+DgY+288CZm0oMM+7w6iqJVV1cfv6Xpo/AJvQrIMT28lOBPYcTYXDk2RT4DXA59vhAC8FTm0nmfPrIckTgRcDxwFU1f1VdRfzcHtorQqskWRV4AnAEubBNlFVPwR+M270ZNvAHsAXq/ETYJ0kG023jNkcCpsAN/YM39SOm3eSLAS2By4ANqyqJdAEB7DB6Cobmn8F3gs81A6vB9xVVQ+0w/Nh29gSWAp8oW1G+3ySNZmH20NV3Qx8HPgVTRjcDVzE/Nsmxky2DSzX39DZHAp9dYcx1yVZC/gG8M6qumfU9Qxbkt2B26rqot7RE0w617eNVYHnAp+tqu2B+5gHTUUTadvM9wC2ADYG1qRpKhlvrm8T01mu/yezORTmfXcYSVajCYSvVNVp7ehbx3YB2+fbRlXfkOwMvC7JDTRNiC+l2XNYp206gPmxbdwE3FRVF7TDp9KExHzbHgBeBlxfVUur6o/AacBOzL9tYsxk28By/Q2dzaEwr7vDaNvNjwOurqqjet46E9i/fb0/cMawaxumqvqHqtq0qhbSbAPfr6p9gXOBvdrJ5sN6uAW4Mckz2lG7Aj9jnm0PrV8BOyR5Qvv/ZGxdzKttosdk28CZwF+2ZyHtANw91sw0lVl9RXOSV9P8KhzrDuMjIy5paJK8EPgRcAUPt6UfQXNc4RRgM5r/HG+oqvEHnuakJLsAf1dVuyfZkmbPYV3gEmC/qvrDKOsbtCTb0RxsXx24DjiQ5ofdvNsekhwJ7E1zlt4lwFtp2svn9DaR5GRgF5ousm8FPgiczgTbQBuYn6Y5W+m3wIFVtXjaZczmUJAkDddsbj6SJA2ZoSBJ6hgKkqSOoSBJ6hgKkqSOoaA5Kck/J9klyZ4z7WE3yYK2t81Lkrxo3HsvanvnvDTJGhN89scrWns7n4W9PWFKw2IoaK76U5prOl5Cc73HTOwKXFNV21fV+M/uC3y8qrarqt+NjWx79aWqdlqBmqWRMxQ0pyT5lySXA88Hzqe5qOmzST4wwbSbJzmn7Wv+nCSbtReI/U/g1eP3BpK8FXgj8IEkX2n3RM5NchLNRYYkWdYz/XuSXNjO/8h23ML2XgjHtnsc3xtbRpLnJbksyfnAIT3zeXaSn7b1XJ5kq5W/5qRWVfnwMaceNN2ufwpYDfiPKaY7C9i/fX0QcHr7+gDg05N85gRgr/b1LjQd023R8/6y9vkVNDdRD82Pr2/RdH29kOYq3O3a6U6hufIW4HLgJe3rfwGubF9/Cti3fb06sMao17GPuftwT0Fz0fbApcDWNH3iTGZH4KT29ZeAFy7Hsn5aVddPMP4V7eMS4OK2lrFf+NdX1aXt64uAhUmeBKxTVT/oqWfM+cARSf4e2Lx6mq2klW3V6SeRHhvapp8TaHqDvJ3m5itJcimwYx9/TJenz5f7JisH+Oeq+ty4GhcCvf3xPAis0U4/4fKr6qQkF9DcaOi7Sd5aVd9fjlqlabmnoDmjqi6tqu2A/0dzC9fvA6+scQeFe/yYpudVaA4gn7cSy/kucFB7PwySbJJk0hvgVHMXtbvbjhDH6qH97JbAdVV1NE3Pl89ZiXVKj+CeguaUJAuAO6vqoSRbV9VUzUeHAscneQ/NXc0OXFl1VNX3kjwTOL/prJJlwH40ewaTObCt57c0oTJmb2C/JH8EbgE+vLLqlMazl1RJUsfmI0lSx1CQJHUMBUlSx1CQJHUMBUlSx1CQJHUMBUlS5/8DofHIGFaf8b4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x111ec5940>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "friend_counts = Counter(num_friends)\n",
    "xs = range(101)\n",
    "ys = [friend_counts[x] for x in xs]\n",
    "plt.bar(xs, ys)\n",
    "plt.axis([0, 101, 0, 25])\n",
    "plt.title(\"Histogram of Friend Counts\")\n",
    "plt.xlabel(\"# of friends\")\n",
    "plt.ylabel(\"# of people\")\n",
    "# plt.rcParams['figure.figsize'] = [10, 10]\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simple statistics about data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "204"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_points = len(num_friends)\n",
    "num_points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "largest_value = max(num_friends) # 100\n",
    "largest_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smallest_value = min(num_friends) # 1\n",
    "smallest_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sort the list\n",
    "sorted_values = sorted(num_friends)\n",
    "sorted_values[:25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smallest_value = sorted_values[0]\n",
    "smallest_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "second_smallest_value = sorted_values[1]\n",
    "second_smallest_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "49"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "second_largest_value = sorted_values[-2]\n",
    "second_largest_value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Central Tendencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7.333333333333333"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def mean(x):\n",
    "    return sum(x) / len(x)\n",
    "mean(num_friends)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Median\n",
    "\n",
    "* in order to find the median, we have to sort our data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def median(v):\n",
    "    \"\"\"finds the 'middle-most' value of v\"\"\"\n",
    "    n = len(v)\n",
    "    sorted_v = sorted(v)\n",
    "    midpoint = n // 2\n",
    "\n",
    "    if n % 2 == 1:\n",
    "        # if odd, return the middle value\n",
    "        return sorted_v[midpoint]\n",
    "    else:\n",
    "        # if even, return the average of the middle values\n",
    "        lo = midpoint - 1\n",
    "        hi = midpoint\n",
    "        return (sorted_v[lo] + sorted_v[hi]) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "median(num_friends)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* the mean is very sensitive to outliers in our data\n",
    "* If our friendliest user had 200 friends (instead of 100), then the mean would rise to 7.82, while the median would stay the same. If outliers are likely to be bad data (or otherwise unrep‐ resentative of whatever phenomenon we’re trying to understand), then the mean can sometimes give us a misleading picture. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Quantile\n",
    "* A generalization of the median is the quantile, which represents the value less than which a certain percentile of the data lies. (The median represents the value less than which 50% of the data lies.)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def quantile(x, p):\n",
    "    \"\"\"returns the pth-percentile value in x\"\"\"\n",
    "    p_index = int(p * len(x))\n",
    "    return sorted(x)[p_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quantile(num_friends, 0.10) # 1\n",
    "quantile(num_friends, 0.25) # 3\n",
    "quantile(num_friends, 0.75) # 9\n",
    "quantile(num_friends, 0.90) # 13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mode(x):\n",
    "    \"\"\"returns a list, might be more than one mode\"\"\"\n",
    "    counts = Counter(x)\n",
    "    max_count = max(counts.values())\n",
    "    return [x_i for x_i, count in counts.items()\n",
    "            if count == max_count]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[6, 1]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mode(num_friends) # 1 and 6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* most frequently we’ll just use the mean."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dispersion\n",
    "* Dispersion refers to measures of how spread out our data is. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Range\n",
    "* the difference between the largest and smallest elements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \"range\" already means something in Python, so we'll use a different name\n",
    "def data_range(x):\n",
    "    return max(x) - min(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_range(num_friends) # 99"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Variance\n",
    "* A more complex measure of dispersion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def de_mean(x):\n",
    "    \"\"\"translate x by subtracting its mean (so the result has mean 0)\"\"\"\n",
    "    x_bar = mean(x)\n",
    "    return [x_i - x_bar for x_i in x]\n",
    "\n",
    "def variance(x):\n",
    "    \"\"\"assumes x has at least two elements\"\"\"\n",
    "    n = len(x)\n",
    "    deviations = de_mean(x)\n",
    "    return sum_of_squares(deviations) / (n - 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "81.54351395730716"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "variance(num_friends) # 81.54"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Standard deviation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def standard_deviation(x):\n",
    "    return math.sqrt(variance(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.03014473623248"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "standard_deviation(num_friends) # 9.03"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Both the range and the standard deviation have the same outlier problem\n",
    "* if our friendliest user had instead 200 friends, the standard deviation would be 14.89, more than 60% higher!\n",
    "* A more robust alternative computes the difference between the 75th percentile value and the 25th percentile value:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def interquartile_range(x):\n",
    "    return quantile(x, 0.75) - quantile(x, 0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "interquartile_range(num_friends) # 6"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Correlation\n",
    "* DataSciencester’s VP of Growth has a theory that the amount of time people spend on the site is related to the number of friends they have on the site (she’s not a VP for nothing), and she’s asked you to verify this.\n",
    "* investigate the relationship between daily_minutes that shows how many minutes per day each user spends on DataSciencester and num_friends list.\n",
    "* covariance, the paired analogue of variance; variance measures how a single variable deviates from its mean, covariance measures how two variables vary in tandem from their means:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "daily_minutes = [1,68.77,51.25,52.08,38.36,44.54,57.13,51.4,41.42,31.22,34.76,54.01,38.79,47.59,49.1,27.66,41.03,36.73,48.65,28.12,46.62,35.57,32.98,35,26.07,23.77,39.73,40.57,31.65,31.21,36.32,20.45,21.93,26.02,27.34,23.49,46.94,30.5,33.8,24.23,21.4,27.94,32.24,40.57,25.07,19.42,22.39,18.42,46.96,23.72,26.41,26.97,36.76,40.32,35.02,29.47,30.2,31,38.11,38.18,36.31,21.03,30.86,36.07,28.66,29.08,37.28,15.28,24.17,22.31,30.17,25.53,19.85,35.37,44.6,17.23,13.47,26.33,35.02,32.09,24.81,19.33,28.77,24.26,31.98,25.73,24.86,16.28,34.51,15.23,39.72,40.8,26.06,35.76,34.76,16.13,44.04,18.03,19.65,32.62,35.59,39.43,14.18,35.24,40.13,41.82,35.45,36.07,43.67,24.61,20.9,21.9,18.79,27.61,27.21,26.61,29.77,20.59,27.53,13.82,33.2,25,33.1,36.65,18.63,14.87,22.2,36.81,25.53,24.62,26.25,18.21,28.08,19.42,29.79,32.8,35.99,28.32,27.79,35.88,29.06,36.28,14.1,36.63,37.49,26.9,18.58,38.48,24.48,18.95,33.55,14.24,29.04,32.51,25.63,22.22,19,32.73,15.16,13.9,27.2,32.01,29.27,33,13.74,20.42,27.32,18.23,35.35,28.48,9.08,24.62,20.12,35.26,19.92,31.02,16.49,12.16,30.7,31.22,34.65,13.13,27.51,33.2,31.57,14.1,33.42,17.44,10.12,24.42,9.82,23.39,30.93,15.03,21.67,31.09,33.29,22.61,26.89,23.48,8.38,27.81,32.35,23.84]\n",
    "\n",
    "def covariance(x, y):\n",
    "    n = len(x)\n",
    "    return dot(de_mean(x), de_mean(y)) / (n - 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22.425435139573064"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "covariance(num_friends, daily_minutes) # 22.43"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Recall that dot sums up the products of corresponding pairs of elements. When corresponding elements of x and y are either both above their means or both below their means, a positive number enters the sum. When one is above its mean and the other below, a negative number enters the sum. Accordingly, a “large” positive covariance means that x tends to be large when y is large and small when y is small. A “large” negative covariance means the opposite—that x tends to be small when y is large and vice versa. A covariance close to zero means that no such relationship exists.\n",
    "* For this reason, it’s more common to look at the <b>correlation</b>, which divides out the standard deviations of both variables:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def correlation(x, y):\n",
    "    stdev_x = standard_deviation(x)\n",
    "    stdev_y = standard_deviation(y)\n",
    "    if stdev_x > 0 and stdev_y > 0:\n",
    "        return covariance(x, y) / stdev_x / stdev_y\n",
    "    else:\n",
    "        return 0 # if no variation, correlation is zero"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.24736957366478218"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correlation(num_friends, daily_minutes) # 0.25"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* The correlation is unitless and always lies between -1 (perfect anti-correlation) and 1 (perfect correlation). A number like 0.25 represents a relatively weak positive correlation."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Outliers\n",
    "* Number of friends vs number of minutes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x113cc99b0>"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3X+cXHV97/HXO5sBNlTcoIELCzHopdAiJZG10ksfyo8iVX7lIhS5atHyuGlvrRVsU5P+ENC2xKYI2vZqc0WNLYUgxIA/AzdEveVRkQ0JIEKKUlSWFKJmAckCm83n/nHOJLOT+XFmd87M7Mz7+XjMY2bOnh/fk8njfM75/vh8FRGYmVnvmtXuApiZWXs5EJiZ9TgHAjOzHudAYGbW4xwIzMx6nAOBmVmPcyAwM+txDgRmZj3OgcDMrMfNbncBsnjlK18ZCxYsaHcxzMxmlE2bNv0kIubVW29GBIIFCxYwPDzc7mKYmc0okn6YZT1XDZmZ9bjcAoGkYyRtKXk9K+kySQdLulPSo+n73LzKYGZm9eUWCCJia0QsjIiFwInATuCLwDJgQ0QcDWxIv5uZWZu0qmrodOAHEfFD4Dxgdbp8NbC4RWUwM7MKWhUI3g7cmH4+NCK2AaTvh7SoDGZmVkHuvYYk7QecCyxvcLslwBKA+fPn51Ays+zWbR5h5fqtPDk6xuED/Sw98xgWLxpsd7HMmqIVTwRvAe6LiKfS709JOgwgfX+60kYRsSoihiJiaN68ut1gzXKzbvMIy9c+yMjoGAGMjI6xfO2DrNs80u6imTVFKwLBxeytFgK4Hbgk/XwJcFsLymA2ZSvXb2VsfGLSsrHxCVau39qmEpk1V66BQNIc4AxgbcniFcAZkh5N/7YizzKYTdeTo2MNLTebaXJtI4iIncArypb9lKQXkdmMcPhAPyMVLvqHD/S3oTRmzeeRxWZ1LD3zGPoLfZOW9Rf6WHrmMW0qkVlzzYhcQ2btVOwd5F5D1q0cCMwyWLxo0Bd+61quGjIz63EOBGZmPc6BwMysxzkQmJn1OAcCM7Me50BgZtbjHAjMzHqcA4GZWY9zIDAz63EOBGZmPc6BwMysxzkQmJn1OAcCM7Me50BgZtbjHAjMzHqcA4GZWY9zIDAz63G5BgJJA5JukfSIpIcl/ZqkgyXdKenR9H1unmUwM7Pa8n4i+Djw9Yg4FjgBeBhYBmyIiKOBDel3MzNrk9wCgaSDgDcC1wNExEsRMQqcB6xOV1sNLM6rDGZmVl+eTwSvBrYDn5W0WdKnJR0IHBoR2wDS90NyLIOZmdWRZyCYDbwO+GRELAKep4FqIElLJA1LGt6+fXteZTQz63l5BoIngCci4p70+y0kgeEpSYcBpO9PV9o4IlZFxFBEDM2bNy/HYpqZ9bbcAkFE/CfwY0nHpItOB74H3A5cki67BLgtrzKYmVl9s3Pe//uAGyTtBzwGvIck+Nws6VLgR8CFOZfBzMxqyDUQRMQWYKjCn07P87hmZpZd3k8E1mLrNo+wcv1Wnhwd4/CBfpaeeQyLFw22u1hm1sEcCLrIus0jLF/7IGPjEwCMjI6xfO2DAA4GZlaVcw11kZXrt+4JAkVj4xOsXL+1TSUys5nAgaCLPDk61tByMzNwIOgqhw/0N7TczAwcCLrK0jOPob/QN2lZf6GPpWceU2ULMzM3FneVYoOwew2ZWSMcCLrM4kWDvvCbWUNcNWRm1uP8RNBhPCDMzFrNgaCDeECYmbWDq4Y6iAeEmVk7OBB0EA8IM7N2cCDoIB4QZmbt4EDQQTwgzMzawY3FHcQDwrJz7yqz5nEg6DAeEFafe1eZNZerhmagdZtHOHnFXRy17CucvOIu1m0eaXeRWqpa76rL1mzpyX8Ps+nyE8EM47vh2r2oevHfw2y6/EQww3isQf1eVL3272E2XbkGAkmPS3pQ0hZJw+mygyXdKenR9H1unmXoNh5rULl3Vble+vcwm65WPBGcGhELI2Io/b4M2BARRwMb0u+WkccaJFU+V59/PIM1zrmX/j3MpqtuIJD0fkkHKXG9pPskvXkaxzwPWJ1+Xg0snsa+eo7HGiQWLxrk7mWncd1FC/3vYTZNWZ4IficingXeDMwD3gOsyLj/AO6QtEnSknTZoRGxDSB9P6TBMve00rthAYMD/Vx9/vE92zDqfw+z6cvSa0jp+1uBz0bE/ZJUa4MSJ0fEk5IOAe6U9EjWgqWBYwnA/Pnzs242IzU6OMpjDSbzv4fZ9GQJBJsk3QEcBSyX9DJgd5adR8ST6fvTkr4I/CrwlKTDImKbpMOAp6tsuwpYBTA0NBRZjtfpKl3wgZ7vDmpm7aWI2tdYSbOAhcBjETEq6RXAYEQ8UGe7A4FZEfFc+vlO4MPA6cBPI2KFpGXAwRHxJ7X2NTQ0FMPDw9nPqgOV9/+HpC57/9mzGB0b32f9wYF+7l52WiuLaGZdRtKmko46VWV5Igjgl4GzSS7kBwIHZNjuUOCLaS3SbOBfIuLrku4FbpZ0KfAj4MIM+5rxqvX/L19W5O6PZtYqWQLB/yapCjqNJBA8B9wKvL7WRhHxGHBCheU/JXkq6CmNXtjd/dHMWiVLr6E3RMR7gRcAImIHsF+upepC1S7sB+5XeWDUqcfOy7M4ZmZ7ZHkiGJfUR1JFhKR5ZGws7naljb8DcwpEwDNj4xV7/iw985iKbQSFvlnAvtVDGx/Z3opTMDPL9ETwCeCLwCGS/gr4V+DqXEs1AxQbf0dGxwhgx85xRsfGCfb2/CnNgrl40SBvO3GQvrTnbZ/E204c5JkKDcXgNgIza526TwQRcYOkTST1+gIWR8TDuZesw1Vq/C1VTHxWfCpYt3mEWzeNMJH20pqI4NZNI7y8v1Cx15DbCMysVeoGAkn/FBHvAh6psKxnZbljL12nWq+hAwqz6C/07VNl5BQJZtYqWaqGjiv9krYXnJhPcWaOLHfspetUCxyjO8edIsHM2qrqE4Gk5cCfAv2SnmVvqomXSEf89rJKjb+lyu/qDx/oZ6RCMDh8oN8pEsysrao+EUTE1RHxMmBlRBwUES9LX6+IiOUtLGNHKk92NndOgYH+QtW7emcNNbNOlaX76NckvbF8YUR8K4fydKRqSeEauZMvrtdIcrlmltXMrJosgWBpyecDSBLHbSIZadyVyscH/PyFXYzvTnr7TCcpXN5VQJ7P2MymIkv30XNKv0s6Evib3ErUJsWL/8joGCIdPUcyPqBcedfQTlFrPuNOK6uZdY4sTwTlngBe2+yCtFP5nXSWnNedOODL8xmb2VRkGUfwd+y9NhZTUt+fZ6Fard7gsEo6ccBXrZ5JZmbVZBlHMEzSJrAJ+DfggxHxzlxL1WKN3jF3am8f90wys6nI0kawut46M121O+miQp84cL/ZVRPKdYpW9Uwys+6SpWroZOBK4FXp+gIiIl6db9Fap9LgsGKD8eAMu5h6cJqZNSpLY/H1wOUkVUONVaTPEOV30qUppc3Mul2WQPBMRHwt95K0WfFOeib0xfegMTNrpiyBYKOklcBa4MXiwoi4L7dStVGr+uJP9WI+EwKVmc0sWQLBG9L3oZJlQZeOLG5FX/zpXMw9aMzMmi1Lr6FTW1GQTjEwp1BxNPHAnMI+y/583YPceM+PmYhglmD/2bN4YXx33Tv86VzMPWjMzJqtVhrqd0bEP0v6QKW/R8THshwgnb9gGBiJiLMlHQXcBBwM3Ae8KyJearzo+Ygqw4rLl//5ugf552//aM/33QFj48lUzvXu8KdzMfegMTNrtloDyg5M319W5ZXV+4HSqS0/ClwbEUcDO4BLG9hX7ipNG1lp+Y33/Ljmfop3+JVUu2hnuZh70JiZNVvVJ4KI+Mf0/aqp7lzSEcBZwF8BH5AkkraF/5GusppkjMInp3qMZpul5O6+0vJSE9UeHUpUG6RWadxC1ou5B42ZWbPVqhr6RK0NI+IPM+z/OuBP2PsE8QpgNCJ2pd+fACpewSQtAZYAzJ8/P8OhmqNSEKi0vE+qGwz6pIrLp3sx96AxM2umWo3Fvwd8F7gZeJK9U1VmIuls4OmI2CTplOLiCqtWvJpGxCrSKTGHhoayJARtqYvfcOSkNoJKagWK0ot5sSvp5Wu2+A7fzFquViA4DLgQuAjYBawBbo2IHRn3fTJwrqS3kkxocxDJE8KApNnpU8ERJEGmZaY7GKt0+wP362PnSxNV01YPZqjzb/e4AA9OM7Nacxb/NCI+lXYffTcwADwk6V1ZdhwRyyPiiIhYALwduCsi3gFsBC5IV7sEuG0a5W9I8aI7MjpGsPeiu27zyJ515lboJlpcvm7zCEtvuX/P9s+/NMHsPvHOk+ZPuQG3VlfSvGX59zCz7lc3DbWk1wGXAe8EvkaSc2g6PkjScPx9kjaD66e5v8yqXXQvW7OFk1fcxbrNI1xxznEU+ibXYBX6xBXnHMdVX3qI8YnJ9//jE8FXHtg2aSL7SpPXV9POcQHtDEJm1jlqNRZfBZxN0vXzJmB5SSNvQyLiG8A30s+Pkcx73HK1Lq7Fu+Grzz+elRecULG65LI1Wypuu2Pn+JQbcNs5LsCD08wMarcR/AXwGHBC+vrrpPfnnjTUv5J/8Zqr3rwDxbvhu5ed1rJ68ul0JZ0uD04zM6hdNXQUcDrJU8HZwDnpq/h5xqk0GKvcyOjYnmqiclV6gwJU3aaadZtHOHnFXVy+ZgsHFGYx0F9ouFppujw4zcyg9oCyH7ayIK1Q2n+/1pNBtZ47tYYNjIyOcfmaLVy2ZkvdyWzKewrt2DlOf6GPay9a2NIeOx6cZmYAigwjZNttaGgohoeHm7rPBcu+kmm90ov6og/fUTEhXSX9hb6qd/Ynr7irYiAaHOjn7mVdmdTVzNpA0qaIGKq3XpY01D1tZHSMD9ycNBK/MJ59grZK2USLffarPY24kdbM2qFWr6ENEXG6pI9GxAdbWahmK00X3Sdx8RuObGj73UHVHkO1lF7Yy6uDKnEjrZm1Q82RxZLeRDI6+CbK0kPMlBnKytNFT0TUTQ3RLIcP9Nd9Ciiq10jrEcBmlpdageBDwDKSNBDlcw/MmBnK6qWLzkt/oY9Tj51X9ykAaLhx2dNTmlkz1eo1dAtwi6S/iIiPtLBMTZUlXXSzFS/slUbuVlq3XgOxp6c0szzVTTERER+RdK6kv01fZ7eiYK3wzpPm70kV3ackZ1Az1Wv8zdpn3yOAzSxPdXsNSbqaJCXEDemi90s6OSKW51qyaSrWqdey8ZHtXPNbJ0y6q87afiBVH1dQrLp5eX+h6oxn9aqDSnkEsJnlKUv30bOAhRGxG0DSamAz0LGBYN3mET5w85aqk8wUlda1D//wZ5nbEwp94qLXH8mtm0aqVv2MjU9wQGEW/YW+fdJHNDpyuJ1pKMys+2UdRzAA/Cz9/PKcytI0f7r2gbpBoKiYfbSWWYKDDijwzNj4pB47Q686uGaPoNGd47zjpPmTuq6+7cTGk9N5BLCZ5SlLILga2CxpI0kX0jfSwU8DADvHdzdtX8VxB3+5+Ph9/lbMOFptpPDAnAK3bhrZ02A9EcGtm0YYetXBUwoGvvCbWR7qBoKIuFHSN4DXkwSCD0bEf+ZdsE4xEcGae3/Ml+/fts8TQekYATF5zs3+Qh8RuLePmXW8TFVDEbENuD3nsnSs8YnY0+g7MjrG0i/cz/APfzapjSBgTzAoNgRfXqXKyb19zKyT1O0+avsa3x3c8O0f7XO3X94sUa1Xj3v7mFkn6cpAUG3e4Waq1RZdfGo49dh5Tcv3X5y/4KhlX2l47gMzs1pqBgJJsyR9t1WFaZZGsoQWDQ70c91FC7nuooV1J6/JYnx38OX7pz6XcSlPMm9mearZRhARuyXdL2l+RLQmU1sTjDXQa6jQJ1ZecMI+F+diV02JzF1Ry42OTX0u4/KyuNHZzPKSpbH4MOAhSd8Bni8ujIhza20k6QDgW8D+6XFuiYgrJB0F3AQcDNwHvCsiXppi+ael2uje0ov3us0jLL3lfsYn2jeBj1NMmFmesgSCq6a47xeB0yLi55IKwL9K+hrwAeDaiLhJ0qeAS4FPTvEY01JMDHf5mi0MzCkQwT5dRLNOb1nJrBpzHDfCKSbMLE9Zks59E3gcKKSf7yW5k6+3XUTEz9OvhfRVTF99S7p8NbC48WLXNtCfrbH48jVb9tS779g5zujYeMU6+MWLBrl72Wl7EtRlNdUqpXKeZN7M8lQ3EEj6nyQX7n9MFw0C67LsXFKfpC3A08CdwA+A0YjYla7yRLq/StsukTQsaXj79u1ZDrfHleceRyHD7Xit63SxDr5UoymtB5t0x7540WBTGp3NzCrJUjX0XpLso/cARMSjkg7JsvOImAAWShoAvgj8UqXVqmy7ClgFyeT1WY5XVJ6bp1jtUy0TaDXFOvgsmUzLNfuO3SkmzCwvWQLBixHxktJqEUmzqX0zvY+IGE3TVJwEDEianT4VHAE82ViRsylv8F25fmvDgaA41WSWWcZKNZJi2sys3bIEgm9K+lOgX9IZwO8DX6q3kaR5wHgaBPqB3wA+CmwELiDpOXQJcNtUC5/FVC7ksPeOPsssY6VqzTjmeYfNrBNlCQTLSHr2PAj8LvBV4NMZtjsMWC2pj6Qt4uaI+LKk7wE3SfpLknkNrp9SyevIOml8JaXpoqvlC6rm1GPnVS1PtXmHwSmmzax9FBkaQCXtBxxLUiW0tdX9/oeGhmJ4eDjz+s3o+z8L+NhFC6cUTCpVDVVLVT13ToEXxndPe/IaM7NykjZFxFC99bL0GjqLpLfPJ4C/B74v6S3TL2J+rvrSQ9MeALYbWL72gYpdN+uplAKi2uCvHTvHq44aNjNrhSxJ564BTo2IUyLiTcCpwLX5Fmt6duxsrFG4mrHx3ft03Zw7p5BpnEL5xbzRwV8eNWxmrZIlEDwdEd8v+f4YybiAnjRnv9lcee5xmTKcll7Mqw0KqxZUPGrYzFqlamOxpPPTjw9J+ipwM0kbwYUko4t7wqIP38HPX9jFeDpMuFjtk6UnUZC0DZS2F5Q3CgOemN7M2qpWr6FzSj4/Bbwp/bwdmJtbiZpg7pxC06qHKu2nke6kpb2Dag0Kc68hM2uXTL2G2q0dvYaardb4AjOzPGTtNVR3HEGaNvp9wILS9euloW6natUwUx1X0Axu/DWzTpVlQNk6kkFfXyLpVTkjVEox0a4gAG78NbPOlSUQvBARn8i9JE1WevEXDSZHajI3/ppZJ8sSCD4u6QrgDpLJZgCIiLpzErTLus0jLP3C/Xt6+rQzCDgBnZl1uiyB4HjgXSQTyhSrhooTzHSkK29/aE8QaKe5cwpuIDazjpclEPx34NXtmld4KhpNN52XZnVhNTPLU5aRxfcDA3kXpFM1Oj2lmdlMk+WJ4FDgEUn3MrmNoGO7j9Yyt2SS+oEqn0sHdU11PgMzs5kiSyC4IvdStNAV5xzHyvVbeaas+mjOfrMrNuqWj0mYs18fO1+ayNQAnSUfkZlZu3XlyOKjln2l6oW6v9BX9e6+2M20Wk+fRp4O+maJay48wb2FzKxtmjmy+Dn29sDcDygAz0fEQdMrYn5qhbZaF/HiduX5gYoambYyS+OLmVknqHu9ioiXRcRB6esA4G0kE9R0rGZUyVSaHKaRNBHju8OTy5jZjNDwjWtErKODxxAAVKvtarT/T/mFf6DBAOP8QmY2E2SpGjq/5OssYIj2Dtatq7whuCigoXQT5Rf+RptTnF/IzGaCLL2GSucl2AU8DpxXbyNJRwKfB/4LyYjkVRHxcUkHA2tIspk+DvxWROxoqNR1HD7QXzXBXOm1vNiVtNoAtBfK2gOqBZhqTj12XkPrm5m1Q91AEBHvmeK+dwF/FBH3SXoZsEnSncC7gQ0RsULSMmAZ8MEpHqOipWceU7d3z0B/gc0fejMAC5Z9peI6Y+O7OXnFXZx67Dw2PrK94cegjY9sb3ALM7PWqzVV5YdqbBcR8ZFaO46IbcC29PNzkh4GBkmeJk5JV1sNfIMmB4LSvv/VngxGx8b3ZCitZWR0jH/+9o+mVA63EZjZTFCrsfj5Ci+AS2nwwi1pAbAIuAc4NA0SxWBxSJVtlkgaljS8fXvjd9aLFw3WTfh2+Zot05qjoF7js9sIzGwmqPpEEBHXFD+nVTvvB94D3ARcU227cpJ+AbgVuCwinlXG3D0RsQpYBcmAsqzHa8R0d1pre89BYGYzRc3uo5IOlvSXwAMkQeN1EfHBiHg6y84lFUiCwA0RsTZd/JSkw9K/HwZk2tdUtSPNw+BAP1eff7xHFZvZjFA1EEhaCdwLPAccHxFXNtK7R8mt//XAwxHxsZI/3Q5ckn6+BLit4VI34IpzjqNvVvMziFbb40B/MgeBg4CZzRS1ngj+CDgc+HPgSUnPpq/nJD2bYd8nk05oI2lL+norsAI4Q9KjwBnp91w1O91DoU/0Fyrv1VmrzWymqdVGMK3rZ0T8K9VvnE+fzr6zaPaE9X0SuyP2pKi+fM2WiuuNejIaM5thujI3WjFLaLOCAMBE2bDiaj2C3FPIzGaargwEjWQJbUSwNzPpqcfOo7/QN+nv7ilkZjNRVwaCrAO5yuut+gt9vPOk+RT6alf0j41PsPGR7Vx9/vEMDvQj3FPIzGauLLmGZpxauYYgaexdecEJwN6Zx4p1/wD/kmEk8ZOjYyxeNOgLv5nNeF0ZCOrlGhqfSOYKqNTNc+FVd7A7wzHcFmBm3aIrq4YWLxrcU21TTbXqo2qZSEuJpK3g5BV3sW7zyFSLaWbWEbryiSCLABZ9+A4ikvTSpVVDtZTOZ1BtSkszs5mkK58IsnYf3bFznNGx8Um9gaqZpaRBuDy/UKUpLc3MZpKuDART7T5aa5vdUb06yemmzWwm68pAkMeFeaC/4EFkZtaVujIQNDrJfBZS0hvJg8jMrNt0ZSBodJL5LHbsHOfyNVs4oDCLgf6CB5GZWdfoyl5DjU4yn1WQBIT+Qh/XXrTQAcDMukJXPhHkXWfvnkJm1k26MhBUqstvVL1pBdxTyMy6RVcGgiwji8vNnVOYVPd/7UULeXzFWVX34Z5CZtYturKNANiTEO41y7+6z1wCRYPpaOJadf2V8ha5p5CZdZOuDQRF1YIAwN3LTqu7fTFIlGcpdUOxmXWLrg8Eg1VSUjdSbeR002bWzbqyjaCUB4GZmdWW2xOBpM8AZwNPR8Rr02UHA2uABcDjwG9FxI48jl+cvP7J0TEG5hTYf/asSVlGfYdvZpbI84ngc8Bvli1bBmyIiKOBDen3plu3eYSlX7ifkdGxPYPAnn9xF9detLDiZDRmZr0st0AQEd8Cfla2+Dxgdfp5NbA4j2NfeftDjO+e3Eg8vju48vaH8jicmdmM1uo2gkMjYhtA+n5IHgepNsvY6Ni4ZxUzMyvTsY3FkpZIGpY0vH379qbttzgBjYOBmVmi1YHgKUmHAaTvT1dbMSJWRcRQRAzNmzevoYPMrZOG2rmCzMz2anUguB24JP18CXBbHgc561cOq7uOcwWZmSVyCwSSbgT+DThG0hOSLgVWAGdIehQ4I/3edBsfqV+V5FxBZmaJ3MYRRMTFVf50el7HLKp3t+8BZWZme3VsY/F01Lrb96xiZmaTdWUgqJZW4joPKDMz20dXJp1zxlAzs+y6MhCAM4aamWXVlVVDZmaWnQOBmVmPcyAwM+txDgRmZj3OgcDMrMc5EJiZ9TgHAjOzHudAYGbW4xwIzMx6nAOBmVmPcyAwM+txDgRmZj3OgcDMrMc5EJiZ9biuTUNtZjZTrds80tL5VBwIzMw6yLrNIyxf+yBj4xMAjIyOsXztgwC5BYO2VA1J+k1JWyV9X9KydpTBzKwTrVy/dU8QKBobn2Dl+q25HbPlgUBSH/APwFuAXwYulvTLrS6HmVknenJ0rKHlzdCOJ4JfBb4fEY9FxEvATcB5bSiHmVnHOXygv6HlzdCOQDAI/Ljk+xPpMjOznrf0zGPoL/RNWtZf6GPpmcfkdsx2NBarwrLYZyVpCbAEYP78+XmXycysIxQbhLu919ATwJEl348AnixfKSJWAasAhoaG9gkUZmbdavGiwVwv/OXaUTV0L3C0pKMk7Qe8Hbi9DeUwMzPa8EQQEbsk/QGwHugDPhMRD7W6HGZmlmjLgLKI+Crw1XYc28zMJnOuITOzHudAYGbW4xTR+R1yJG0HftjAJq8EfpJTcTpVL54z9OZ59+I5Q2+e93TP+VURMa/eSjMiEDRK0nBEDLW7HK3Ui+cMvXnevXjO0Jvn3apzdtWQmVmPcyAwM+tx3RoIVrW7AG3Qi+cMvXnevXjO0Jvn3ZJz7so2AjMzy65bnwjMzCyjrgoEvTLzmaQjJW2U9LCkhyS9P11+sKQ7JT2avs9td1mbTVKfpM2Svpx+P0rSPek5r0nzV3UVSQOSbpH0SPqb/1q3/9aSLk//b39X0o2SDujG31rSZyQ9Lem7Jcsq/rZKfCK9vj0g6XXNKkfXBIIem/lsF/BHEfFLwEnAe9NzXQZsiIijgQ3p927zfuDhku8fBa5Nz3kHcGlbSpWvjwNfj4hjgRNIzr9rf2tJg8AfAkMR8VqSnGRvpzt/688Bv1m2rNpv+xbg6PS1BPhkswrRNYGAHpr5LCK2RcR96efnSC4MgyTnuzpdbTWwuD0lzIekI4CzgE+n3wWcBtySrtKN53wQ8EbgeoCIeCkiRuny35okD1q/pNnAHGAbXfhbR8S3gJ+VLa72254HfD4S3wYGJB3WjHJ0UyDoyZnPJC0AFgH3AIdGxDZIggVwSPtKlovrgD8BdqffXwGMRsSu9Hs3/uavBrYDn02rxD4t6UC6+LeOiBHgb4EfkQSAZ4BNdP9vXVTtt83tGtdNgSDTzGfdRNIvALcCl0XEs+0uT54knQ08HRGbShdXWLXbfvPZwOuAT0bEIuB5uqgaqJK0Tvw84CjgcOBAkmqRct32W9eT2//3bgoEmWY+6xaSCiRB4IaIWJsufqr4qJi+P92u8uXgZOBcSY+TVPudRvKEMJBWH0B3/uZPAE9ExD3p91tIAkM3/9a/AfxHRGyPiHFgLfDf6P7fuqjab5vbNa6bAkHPzHyW1o1fDzwcER8r+dPtwCXp50uA21oUN84xAAAEeUlEQVRdtrxExPKIOCIiFpD8tndFxDuAjcAF6Wpddc4AEfGfwI8lFWcuPx34Hl38W5NUCZ0kaU76f714zl39W5eo9tveDvx22nvoJOCZYhXStEVE17yAtwL/DvwA+LN2lyfH8/x1kkfCB4At6eutJHXmG4BH0/eD213WnM7/FODL6edXA98Bvg98Adi/3eXL4XwXAsPp770OmNvtvzVwFfAI8F3gn4D9u/G3Bm4kaQcZJ7njv7Tab0tSNfQP6fXtQZJeVU0ph0cWm5n1uG6qGjIzsylwIDAz63EOBGZmPc6BwMysxzkQmJn1OAcC6xiSQtI1Jd//WNKVTdr35yRdUH/NaR/nwjRD6May5QskjUnaUvLaJ3umpCFJn2hSWd4t6e+bsS/rbrPrr2LWMi8C50u6OiJ+0u7CFEnqi4iJjKtfCvx+RGys8LcfRMTCGseZHRHDJGMGzFrGTwTWSXaRTM13efkfyu/oJf08fT9F0jcl3Szp3yWtkPQOSd+R9KCk15Ts5jck/b90vbPT7fskrZR0b5rj/XdL9rtR0r+QDN4pL8/F6f6/K+mj6bIPkQz2+5SklVlOWNKVklZJugP4fHrc4lwLB6b56u9NE86dly5/t6S1kr6e5qz/m5L9vSc9v2+SpOUoLr8wLev9kr6VpWzWO/xEYJ3mH4AHSi9uGZwA/BJJOt/HgE9HxK8qmbDnfcBl6XoLgDcBrwE2SvqvwG+TDNV/vaT9gbvTizIkqc1fGxH/UXowSYeT5MY/kSQv/h2SFkfEhyWdBvxxemdf7jWStqSf746I96afTwR+PSLGJJ1Ssv6fkaTS+B1JA8B3JP3f9G8LSbLOvghslfR3JIH0qnR/z5CkZNicrv8h4MyIGEn3ZbaHA4F1lIh4VtLnSSYmGcu42b2R5lyR9AOgeCF/EDi1ZL2bI2I38Kikx4BjgTcDv1LytPFykok/XgK+Ux4EUq8HvhER29Nj3kAyZ8C6OuWsVjV0e0RUOtc3kyTa++P0+wHA/PTzhoh4Jj3+94BXAa8sK9ca4BfT9e8GPifpZpIkbmZ7OBBYJ7oOuA/4bMmyXaRVmWkistKG1hdLPu8u+b6byf/Hy/OpBEn+lvdFxPrSP6R35s9XKV+ldMDTUes4b4uIrZMWSm9g8jlPsPc8K+aMiYjfS7c7C9giaWFE/HR6xbZu4TYC6zgR8TPgZiZPRfg4SZUHJLnqC1PY9YWSZqXtBq8GtgLrgf+lJK03kn5RycQvtdwDvEnSK5VMkXox8M0plKee9cD70sCHpEUZynWKpFek53Nh8Q+SXhMR90TEh4CfMDmdsfU4PxFYp7oG+IOS7/8HuE3Sd0gyMla7i65lK8kF+1Dg9yLiBUmfJmk7uC+94G6nzhSIEbFN0nKSOngBX42IPFIif4Tk6eiBtGyPA2fXKdeVwL+RZLS8j2S+X4CVko5Oy7sBuD+H8toM5eyjZmY9zlVDZmY9zoHAzKzHORCYmfU4BwIzsx7nQGBm1uMcCMzMepwDgZlZj3MgMDPrcf8f6Ajsa9XYurEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x114305160>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.xlabel('Number of Friends')\n",
    "plt.ylabel('Number of Minutes')\n",
    "plt.scatter(x=num_friends, y=daily_minutes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* The person with 100 friends (who spends only one minute per day on the site) is huge outlier, and correlation can be very sensitive to outliers. What happens if we ignore him?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "outlier = num_friends.index(100) # index of outlier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outlier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_friends_good = [x\n",
    "                    for i, x in enumerate(num_friends)\n",
    "                    if i != outlier]\n",
    "\n",
    "daily_minutes_good = [x\n",
    "                      for i, x in enumerate(daily_minutes)\n",
    "                      if i != outlier]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5736792115665573"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correlation(num_friends_good, daily_minutes_good) # 0.57"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Without the outlier, there is a much stronger correlation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x22b9af62e10>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3W+MXNWZ5/Hv00U5tD2ZtE0MaxobMywyGdaDvfRgVl6NghlCJuFPb0Im45DIu4rkN6tVMn88sVdImJEjnLWUZF+MRmsl2fUqGUL4k8YM0TgIHGUHTZxpY4jjwRaBiTFtBztjNwlxExr72Rd1y+lun3OrbvlW1e1bv4+Euut2letcME+dfs5znmPujoiIzH593R6AiIjkQwFdRKQkFNBFREpCAV1EpCQU0EVESkIBXUSkJBTQRURKQgFdRKQkFNBFREriok6+2Xvf+15funRpJ99SRGTW27t378/dfWGj53U0oC9dupTR0dFOvqWIyKxnZoebeZ5SLiIiJdEwoJvZMjN7fso/vzCzz5rZAjN7ysxeSr7O78SARUQkrGFAd/dD7r7C3VcANwCngW8DG4Gn3f0a4OnksYiIdEnWlMstwMvufhi4C9iRXN8BDOc5MBERySZrQP8T4MHk+8vc/RhA8vXS0AvMbL2ZjZrZ6IkTJ1ofqYiIpGq6ysXM5gB3ApuyvIG7bwe2AwwNDek0DRHpGSP7xti26xBHxye4fKCfDbctY3jlYNveL0vZ4h8Bz7n768nj181skbsfM7NFwPH8hyciMjuN7Btj02P7mZg8A8DY+ASbHtsP0LagniXlspbfpFsAdgLrku/XAY/nNSgRkdlu265D54J53cTkGbbtOtS292wqoJvZXOBW4LEpl7cCt5rZS8nPtuY/PBGR2eno+ESm63loKuXi7qeBS2Zc+1dqVS8iIjLD5QP9jAWC9+UD/W17T+0UFRFpgw23LaO/Wpl2rb9aYcNty9r2nh3t5SIi0ivqC59FrXIREZEMhlcOtjWAz6SUi4hISSigi4iUhAK6iEhJKKCLiJSEArqISEkooIuIlIQCuohISSigi4iUhAK6iEhJKKCLiJSEArqISEkooIuIlIQCuohISSigi4iUhAK6iEhJKKCLiJSEArqISEkooIuIlIQCuohISSigi4iURFMB3cwGzOwRMztoZi+a2X8wswVm9pSZvZR8nd/uwYqISFyzM/T/Cfy9u18LXA+8CGwEnnb3a4Cnk8ciItIlDQO6mf028AfAVwHc/W13HwfuAnYkT9sBDLdrkCIi0lgzM/TfAU4A/9vM9pnZV8xsHnCZux8DSL5e2sZxiohIA80E9IuAfw/8jbuvBH5FhvSKma03s1EzGz1x4kSLwxQRkUaaCeivAa+5+57k8SPUAvzrZrYIIPl6PPRid9/u7kPuPrRw4cI8xiwiIgENA7q7/ww4YmbLkku3AP8M7ATWJdfWAY+3ZYQiItKUi5p83n8DvmFmc4BXgP9C7cPgW2b2aeBV4GPtGaKIiDSjqYDu7s8DQ4Ef3ZLvcEREpFXaKSoiUhLNplxEOm5k3xjbdh3i6PgElw/0s+G2ZQyvHOz2sEQKSwFdCmlk3xibHtvPxOQZAMbGJ9j02H4ABXWRCKVcpJC27Tp0LpjXTUyeYduuQ10akUjxKaBLIR0dn8h0XUQU0KWgLh/oz3RdRBTQpaA23LaM/mpl2rX+aoUNty2LvEJEtCgqhVRf+FSVi0jzFNClsIZXDiqAi2SglIuISEkooIuIlIQCuohISSiHLlHaei8yuyigS5C23ovMPkq5SJC23ovMPgroEqSt9yKzjwK6BGnrvcjso4AuQdp6LzL7aFFUgrT1XtpNVVT5U0CXKG29l3ZRFVV7KKBLR2g2JlOlVVHp70XrFNCl7TQbk5kaVVFpAtAaLYpK26mmXWZKq6KqTwDGxidwfjMBGNk31tlBzkIK6NJ2qmmXmdKqqDQBaF1TAd3Mfmpm+83seTMbTa4tMLOnzOyl5Ov89g5VZivVtMtMwysHeeAjyxkc6MeAwYF+HvjIcoZXDmoCcAGy5NBvdvefT3m8EXja3bea2cbk8edyHZ2Uwobblk3LoYNq2iVeRXX5QD9jgeCtCUBjF5JyuQvYkXy/Axi+8OFIGaXNxkRm0qa21jU7Q3fgu2bmwP9y9+3AZe5+DMDdj5nZpe0apMx+qmmXZmlTW+uaDeir3f1oErSfMrODzb6Bma0H1gMsWbKkhSFKu6lETIpGE4DWNBXQ3f1o8vW4mX0buBF43cwWJbPzRcDxyGu3A9sBhoaGPJ9hS0zW4KwacZHyaJhDN7N5Zvbu+vfAB4AfAzuBdcnT1gGPt2uQ0pxW6ndVIiZSHs0sil4G/IOZvQD8EHjS3f8e2ArcamYvAbcmj6WLWgnOKhETKY+GKRd3fwW4PnD9X4Fb2jEoaU0rwVklYiLloZ2iJdLKBp4Nty2jWrFp16oVU4mYyCykgF4iLdfvzlyq1tK1yKykbosdFKtAyatssJX63W27DjF5dnoEnzzramMqMgspoHdIrDxw9PBJHt071rWyQS2KipSHUi4dEqtAeXDPkdzKBlspW1TjLJHyUEDvkNiM94yHE9atzJBbKVtU3wyR8lBA75DYjLdiFrzeygy5lfSJGmeJlIdy6B0SayH70RsGp+XQ69dbmSG3WlOuvhki5aAZeofEZsJbhpfnNkNW+kSkt5lHcrjtMDQ05KOjox17v3YqaofCoo5LRFpnZnvdfajR85RyaUGROxSWKX2iDyeRbBTQGwgFlbRqEgWcfBT5Q1OkqBTQSd/BGQoqM4N5nTbj5EcfmiLZ9XxAT5sJxoJKxSxYP67NOPnRDlaR7Hq+yiVtJpi2GUjVJO2lHawi2fV8QE+bCcaCR720UJtx2kclmCLZ9XzKJW0zTmwzUD3HrgDePjr5XSS7ng/ojYI2KKh0iz40RbLp+YCuoC0iZdHzAR3iM0HVQovIbKKAnkK10PnRrk+R9lNAT1G2WuhuBVX9piPSGT1ftpimTLXQrZxmlJdWDt4Qkew0Q09x87UL+foPXg1eb8W9I/t5cM8RzrhTMWPtqsVsGV7ekZlzN9NHZftNR6Somg7oZlYBRoExd7/dzK4CvgksAJ4DPuXub7dnmN2x++CJTNfT3Duyf9qHwxl3vv6DV/mXE2/y3KtvtD0d0c2g2urBGyKSTZaUy2eAF6c8/gLwJXe/BjgFfDrPgRVBKAilXU/z4J4jwevPvnyyI+mIbqaPtOtTpDOaCuhmdgXwYeAryWMD1gCPJE/ZAQy3Y4Dd1Bc+7jN6PU3sMOiYVj400nQzqOrcUpHOaDbl8mXgL4F3J48vAcbd/Z3k8WtA8P9OM1sPrAdYsmRJ6yPtgrORGBy7nibWoTHt+Xnq9gYq7foUab+GAd3MbgeOu/teM3t//XLgqcFo5e7bge1QO4KuxXHOemtXLQ4usMZkndE3I20DlWrERWa/Zmboq4E7zexDwMXAb1ObsQ+Y2UXJLP0K4Gj7hplNEQPUluHlAOdVuew+eCKYXhns0IKhasRFyqNhQHf3TcAmgGSG/hfufo+ZPQzcTa3SZR3weBvH2bQ8A9T8uVVOnZ4MXk97/9iHyZbh5ecCe2y80NkFw17fDVvED3+RVl3IxqLPAX9mZj+hllP/aj5DujB5bmK5747rqFamZ5eqFeO+O64LPn9k3xgbHnlh2uadDY+8kLp5p9sLhr1cI97NzVYi7ZBpY5G7fw/4XvL9K8CN+Q/pwuQZoLIuJN7/xAEmz0zPfU+ece5/4kBqgO7mgmEv14j3+m8nUj6l2ymaFqBa+fU6S7ANpWfSrhdBWj/4suvl306knErXyyVWb33ztQv163VAt1M+3VSmXj0iUMIZeixN0ii3nsfCmBmEqg3rJeXdXoCLvX+v1oj38m8nUk6lC+gQTpN89qHng8+tz9TzqIqJlY67p1ffQPs3/Kg88Xzd3mwlkjfzNmxgiRkaGvLR0dGOvd9USzc+men5gwP9PLtxTaZZ9cq/+m60zHHunIuCuf35c6u8NXn2vFli3mmP1Vufida7P7txTW7vIyL5M7O97j7U6Hmly6Hn5ej4ROYyxLdmpHSmXo8ttJ06PdmR5lxaABQpPwX0iMsH+lPLEEMmJs9Gr2ddaGs10I7sG2P11me4auOTrN76zLkPHy0AipTfrM2hxw6LaEV/tRJcGIvl3U+dngymYtLEFuDedVEf4xPnp2laCbRpeXItAIqU36ycodcPi6g3sKofFnHvyP4GrwxrpWxvw8MzUjEPv8Dcavhf5/y5VYZXDvLRGwbPdVGsmPHRGwbZfOd1LbW1Dc3EG22U6dXyRJFeMSsXRa/e9J1gN8KKGS8/8KHga67a+GSwHaQB/7L1w8HXZF1InVvtY/KsT0vTVCvGtruvBwjOkB/4SO23iiyVFrH+LzODeV3aPeat26WZImXU7KLorEy5xFrLprWcveemJcH2tffclF+P9tOTZ/nyx1cEA9rqrc9EZ8/PblyTKejFZuKxnuudypOrNFKku2ZlQG9FrH1tq3n3mNgmnTyrTGKvOeMeXQ/oBPVGEemungnoEG5fm6e0trp59piJ/VmDU3bFdiPlodJIke4qfEAPVbMMpgQ0yC+PG0th9BlU+uy8XHmsrS7Eq0zqPWaypCnSKlbUuVGkdxW6yiVWzbL0kv5oZcjIvrFgBUpaE657R/Zz9abvsHTjk1y96TvnqmXWrlocfP4nVi1h293XT6sY2Xb39Q1b5IaqTHYfPJF5Y1GsYqbbaY1uHkQtIgWfof/tnvAZnP/4ykm++MfhxccV93+XyRmnOE+edTbvrG0Gmvma0cMnpy2W1j80oHHePWsADc2e/zRS656WphjZN8aje8emfdA9uneMoSsXdDWoqzeKSHcVOqCfjRStnPX44mNok079eii1Eduu/+CeI2wZXs7QlQvYffAER8cn+DfvuZihKxe0djMRraQpirz42KudG0WKoNABPW+hIBhzxj33Domh3H4rOzi1+CgiIYXOoVvG63mqmEVnwvc/cSBznj52fiVk36mqviwiElLogB7bJpTn3tZ5cyrB62tXLU7tkJiWpw/J8/BqLT6KSEihUy7z51aj/cVjpYnRU4OAiwObbj7/n5YzevhkcOFz98ETwfx2TCx/D/F0SH12X/+AqD+G+KJrkRcftfVfpHsK3ctlxf3fDQbJtJ4psQ6JQHRbfkzWnilp7xE7YCL2ATTQX+X5+z4QfZ8iiv37UhMwkQtTil4usRnv6UDf8Xqf8rRNR1krMGIz4fufOBD8zWHenEp0EfXmaxcGe8nEPk/TZvtFVeTqG5Fe0DCgm9nFwPeBdyXPf8Td7zOzq4BvAguA54BPufvb7RxsI6dOT3LfHdfl2vc79iGw4ZEXzvsNoVo5v7d5O04fKipV34h0VzOLor8G1rj79cAK4INmdhPwBeBL7n4NcAr4dPuG2bxO9P0eXjkY3Cn6RmRWfXR8IlMuHmrtBWYbVd+IdFfDGbrXkuxvJg+ryT8OrAE+kVzfAWwG/ibPwcV6qTQSm1XHFuxaWcgLvcfmnQeCqZL39Ff55VvvZLqX2KaqItOpSCLd1VQO3cwqwF7g3wJ/DbwMjLv7O8lTXgNyT5KuXbU4mHeu9kHo+M60boexTUKjh0/y6N6xXHp4W2RWbZbeqz1kcBbOaotcfSPSC5oK6O5+BlhhZgPAt4H3hZ4Weq2ZrQfWAyxZku0wiVgvlaErFwRz2PVuh6EOjbEmWPXnzbzeykLeeGChtH49VoI5b06Fs05pZrXa+i/SPZmqXNx93My+B9wEDJjZRcks/QrgaOQ124HtUCtbzDrAtB7moZlgvUNj3dRmWyGxmXOjhbxQmmYgErQH5laj1SzVSh+b77xOs1oRuWDNVLksBCaTYN4P/CG1BdHdwN3UKl3WAY+3c6DNenDPkVz+nLSFvHiPl3DUdie6YPrGxKRmtSKSi2Zm6IuAHUkevQ/4lrv/nZn9M/BNM9sC7AO+2sZxTpPWNKuVRdSsx7bF6q1jxicmo/XxeVeAaKemSO9qpsrlR8DKwPVXgBvbMaipQgEqbQNL1sqYeXNq2/+zBMGsddUVs1pnxUDeP89cuQ5pFulthW7ONbJvjA2PzOhqmDwOOTo+ET1lKOZXb59h9PBJfvbGWzjwszfeYvTwydTXZJ1Vn/uAmfk5k3NpYqsNwEb2jbF66zNctfFJVm99JrVrpIgUV6ED+v1PHJg2o4XaFv9YeeDlA/1sGV7O6qunH0Ix8/FMoWPu6sfQhcS6HQ70h8smBwf62bbrULBD47Zdh3ILqK3s1Iy19VVQF5l9Ch3QQxUjEO9/cvO1CxnZN8YPf3pq2vWZj5vxjcjxdxDfjbr5zuuibW3Tui3mFVBb2amZZ1tfEemuQjfnymr3wRM8+aNjwVl9Vo3S8GmVKaF8fGwXqRE+SamVOnidfiTS2wod0I1saeajySw3ps+mb6mf+TiLWDVJLNDH0kSxt28loLayU7OVM01FpJgKHdCzxtpYcKqrmHF2ytR75uOQUOAGMleTxNJHafeSV4+ZNOq/IlIehc6hx/qZzJ9bjeaqYwuTBsFFyTSxBcPNOw9kzjtXIlP0WFPFpZf0d2SxshPdKUWkMwo9Q4/NHus9W0Kz19HDJ8MHSWR873plSihwxzYRpf12EKuNj43rB6+cyq3HTCPaqSpSDoUO6I1ywqEgtPvgieCfFcuXz5tT4e13zk6brVf7aht+/jTlOLvYe8TEdorGtNpjRkR6V6EDOmSfPcYCXiy74u7n5z2Sx41y8s2+B8SPoOuv9jER6AUc2/GqxUoRiSl0Dh3iuxhj17MGvNOTZ4Nljtt2HeLmaxfmcxPEf3O4uFoJrgesXbU4uk4gIhJS6Bl6K4dSxGbCsUMxYsbGJ6JBOMaIlzPGfnMYPz3JPTctmda//aM3DLJleDlDVy5Qoy0RaZp5C90JWzU0NOSjo6NNP3/11meCKY9YOqJeFRN6Tdaa83pJY9Z/O6HOjQ98pNb8KzSu+XOrvDV5NvgaBW8RATCzve4+1Oh5hU65xPLXaQuGWXPoMWfcW8pXx8oZN9y2jOqMVdNqn+Eef42ISBaFDuix3ZUxlw/0R4NwrA48dn0wSXGE8thZnfuQCSy+htoBQHoJpIhISKEDelo2KLZgGAvCsUXGtMXH2Kab2Ian2IfD5fVuizn0mBERiSn0omiael46tmAY+llskTFt8TFWNhna8PTRGwanLdbWr7dS0y4iktWsDehp9emxn2W9nvbeEN/wNLNiZXjlYHRRNCY22xcRiSl0QI91W2wU6mKlg3metxn6EBjZN8aje8emHZbx6N4xhq5cEC2nvObSebx0/FfnXc968pKISKEDeizDnJZ5bqV2HbK1nI1p5bCI02+fZfXVC3j25d8ce7f66gVsGV6e+f1FpLcVOqDPn1sNtp2dPzfcURHiQbWeBpl5/f4nDkyrA7+Qg5VbOSxibHyCk796e9q15159g5F9Y6pDF5FMZmWVS1r1Syx4xmrXT52ezK0OPO0IuIHIh5CZ6tBFJB+FDuhvRGq0Y9chHlSzLjK20tUw1vvl5msXZv5wUldFEcmq0AE9bcYba86VtQ59bjX8ryA2o04T6/2y++CJ1A+hEHVVFJGsGgZ0M1tsZrvN7EUzO2Bmn0muLzCzp8zspeTr/LwHFwvON1+7kA0PvzDtNJ8ND79wLu8c2gy0ZXh58Hose/NW5BCLNGk59FiAnjcnvPM0z06PItIbmlkUfQf4c3d/zszeDew1s6eA/ww87e5bzWwjsBH4XJ6Di9V7b955IHic3OadB1IPag5d/2xkw0+oR/lUoRLItAOXY6cvVSt9wPkfHlk7PYqINAzo7n4MOJZ8/0szexEYBO4C3p88bQfwPXIO6JAtCMf6olyILIdEp+0UjX04xXaQKocuIlllKls0s6XASmAPcFkS7HH3Y2Z2ae6jy1koOKeVRsZq2t91UV+wMmX3wRMNWxLMlDarFxHJoumAbma/BTwKfNbdf2FNVo2Y2XpgPcCSJUtaGWNm947sn7b9fu2qxQxduSA6q37on45Ma5RVrRj33XFdS4dEx9I9sQ+HtFm9iEgWTVW5mFmVWjD/hrs/llx+3cwWJT9fBBwPvdbdt7v7kLsPLVyYz0JfbGPR/LlV7h3Zz9d/8Oq07fdf/8GrbHrsR9FZ9Y1Lp6/n3rh0fuopQzFppZGxD4f6rH7mYq02FYlIVg1n6Fabin8VeNHdvzjlRzuBdcDW5OvjbRlhwH13XMeGR14Izqr/7FvZFjnHxifOS3k8+/JJ7h3Zn/mQ6NjmJUivgMnaHExEJKSZGfpq4FPAGjN7PvnnQ9QC+a1m9hJwa/K4I4ZXDvLx3198bkZcMePjv7+Y4ZWDmU8minlwz5HaKUOVGacMVYyB/vBvCLE+6QDvibwmdl1EJKtmqlz+gXiDw1vyHU5z0roa5uXcbHvmB4TD7dcvypz3jmVj1CVXRPJS6OZcEK5MSetq2F/ta1hD3oyKWe2UoUC9eyvVLOOBSpq06yIiWRV663+9MmTqjtD645Cj4xM88JHfO++mWrnJtasWt9Q9MSatjYGISB4KHdBjM/G0szuHVw7yxY+vmFY1Un8cMjjQzydvWjItH//Jm5awZXh5NNgOzK0GP2jq/WRCYm0MVJ4oInkpdMolrRVutc+mpUOqfXYuOGY5B7SeKgkdKBHbru8eb3mbdiwe5HOQhohISKEDeqxscP7cKm/++p3pFxssLrYSUIdXDjJ6+OR5Z4R+I3CUHDROxag8UUTaqdABPW2GPLUGHWqP02bIkD2gjuwb46EfHplWTfPQD4/wnv5qsG+M8uEi0k2FzqHHWuHGeovn3dAq1tXx7XfOKB8uIoVT6Bk6hGfV23Yd6khDq1j3xtOTZ/nyx1cE0zehMkulWUSkEwof0EMBMpaKqc+QOxFUQx80sQZc9eeLiLRToVMusTp0INrQKvaakX1j0WPrYvoiC62x62kbnkRE2q3QM/S0APnsxjXBWW/sNfc/cYC3Js9mmj3H+sLErue5EUlEJKtCz9BbCZCxn506PZl59py2GSlEu0FFpJsKHdAHIn3PY9cb/SxkbHwimoqJHdQcu67doCLSTYVOucTai6e0HefNt7I1u+qz8PmgED+oeffBE6kLr6pyEZFuKHRAj9WbvzExGQ2oWRstnk3Zxh9L39SDfiwfrwAuIt1Q6JRLns2xsqp/UIRUzFTNIiKFU+iAHstJpzXHipUUWvLamX9W7PSh+qw/9JrYUXOqZhGRbip0QG9l6/8nVi0J/uyem5YE/6zNd14XXciMvX/WKhcRkU4odA4dsm/9r7fBndohce2qxeeux/LbsYXMrK14RUS6pfABPeTmaxfy9UAL23o54Zbh5cH+5jFZFzJVzSIiRTQrA3paOWGnqJpFRIqm0Dn0GG2xFxE536wM6NpiLyJyvlkZ0LXFXkTkfA1z6Gb2NeB24Li7/7vk2gLgIWAp8FPgj939VDsGqC32IiLNMU9rjAKY2R8AbwL/d0pA/x/ASXffamYbgfnu/rlGbzY0NOSjo6NND25k3xgbHn5h2jFw1T5j28euV/AWkZ5hZnvdfajR8xqmXNz9+8DJGZfvAnYk3+8AhjOPsAmxMz037zzQjrcTEZnVWs2hX+buxwCSr5fmN6TfiJ3pGbsuItLL2l6HbmbrgfUAS5aEt+W3Qocxi4hM1+oM/XUzWwSQfD0ee6K7b3f3IXcfWrgwfDBEzPzIYRXz5lTa3m1RRGS2aTWg7wTWJd+vAx7PZzjT3XfHded1T+wzqFb61L5WRGSGhgHdzB4E/hFYZmavmdmnga3ArWb2EnBr8rgtKjMieqXPojl07RQVkV7WMIfu7msjP7ol57GcZ9uuQ0yemVHlcqbWQTHUk1w7RUWklxV6p2hsxn3GXTtFRURmKHRAj824px40MfXgCVW5iEgvK3T73A23LYseJKH2tSIi0xU6oKtni4hI8wod0EEHSYiINKvQOXQREWmeArqISEkooIuIlIQCuohISSigi4iURMMTi3J9M7MTwOEGT3sv8PMODKeIevneobfvX/feu5q5/yvdvWG72o4G9GaY2WgzRy2VUS/fO/T2/evee/PeId/7V8pFRKQkFNBFREqiiAF9e7cH0EW9fO/Q2/eve+9dud1/4XLoIiLSmiLO0EVEpAWFCehm9kEzO2RmPzGzjd0eT7uZ2dfM7LiZ/XjKtQVm9pSZvZR8nd/NMbaLmS02s91m9qKZHTCzzyTXS3//Znaxmf3QzF5I7v3+5PpVZrYnufeHzGxOt8faLmZWMbN9ZvZ3yeNeuvefmtl+M3vezEaTa7n9vS9EQDezCvDXwB8BvwusNbPf7e6o2u7/AB+ccW0j8LS7XwM8nTwuo3eAP3f39wE3Af81+e/dC/f/a2CNu18PrAA+aGY3AV8AvpTc+yng010cY7t9BnhxyuNeuneAm919xZRSxdz+3hcioAM3Aj9x91fc/W3gm8BdXR5TW7n794GTMy7fBexIvt8BDHd0UB3i7sfc/bnk+19S+597kB64f695M3lYTf5xYA3wSHK9lPcOYGZXAB8GvpI8Nnrk3lPk9ve+KAF9EDgy5fFrybVec5m7H4Na0AMu7fJ42s7MlgIrgT30yP0nKYfngePAU8DLwLi7v5M8pcx//78M/CVwNnl8Cb1z71D78P6ume01s/XJtdz+3hflgAsLXFP5TcmZ2W8BjwKfdfdf1CZr5efuZ4AVZjYAfBt4X+hpnR1V+5nZ7cBxd99rZu+vXw48tXT3PsVqdz9qZpcCT5nZwTz/8KLM0F8DFk95fAVwtEtj6abXzWwRQPL1eJfH0zZmVqUWzL/h7o8ll3vm/gHcfRz4HrV1hAEzq0+wyvr3fzVwp5n9lFpadQ21GXsv3DsA7n40+Xqc2of5jeT4974oAf2fgGuS1e45wJ8AO7s8pm7YCaxLvl8HPN7FsbRNkjf9KvCiu39xyo9Kf/9mtjCZmWNm/cAfUltD2A3cnTytlPfu7pvc/Qp3X0rt//Fn3P0eeuDeAcxsnpm9u/498AHgx+T4974wG4vM7EPUPq0rwNfc/fNdHlLooDyWAAAAp0lEQVRbmdmDwPupdVp7HbgPGAG+BSwBXgU+5u4zF05nPTP7j8D/A/bzm1zqf6eWRy/1/ZvZ71Fb+KpQm1B9y93/ysx+h9qsdQGwD/iku/+6eyNtryTl8hfufnuv3Htyn99OHl4E/K27f97MLiGnv/eFCegiInJhipJyERGRC6SALiJSEgroIiIloYAuIlISCugiIiWhgC4iUhIK6CIiJaGALiJSEv8fTqC42poBVz8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x22b992a41d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(x=num_friends_good, y=daily_minutes_good)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* You investigate further and discover that the outlier was actually an internal test account that no one ever bothered to remove. So you feel pretty justified in excluding it."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simpson’s Paradox\n",
    "* One not uncommon surprise when analyzing data is Simpson’s Paradox, in which correlations can be misleading when confounding variables are ignored.\n",
    "* For example, imagine that you can identify all of your members as either East Coast data scientists or West Coast data scientists. You decide to examine which coast’s data scientists are friendlier:\n",
    "\n",
    "| coast      | # of members | avg. # of friends| \n",
    "|------------|--------------|------------------|\n",
    "| West Coast |          101 |               8.2| \n",
    "| East Coast |          103 |               6.5| "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* It certainly looks like the West Coast data scientists are friendlier than the East Coast data scientists. Your coworkers advance all sorts of theories as to why this might be: maybe it’s the sun, or the coffee, or the organic produce, or the laid-back Pacific vibe?\n",
    "* When playing with the data you discover something very strange. If you only look at people with PhDs, the East Coast data scientists have more friends on average. And if you only look at people without PhDs, the East Coast data scientists also have more friends on average!\n",
    "\n",
    "| coast | degree | # of members | avg. # of friends| \n",
    "| ----------------------------------------------| \n",
    "| West Coast | PhD | 35 | 3.1| \n",
    "| East Coast | PhD | 70 | 3.2| \n",
    "| West Coast | no PhD | 66 | 10.9| \n",
    "| East Coast | no PhD | 33 | 13.4| "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Once you account for the users’ degrees, the correlation goes in the opposite direction!\n",
    "Bucketing the data as East Coast/West Coast disguised the fact that the East\n",
    "Coast data scientists skew much more heavily toward PhD types.\n",
    "* This phenomenon crops up in the real world with some regularity. The key issue is\n",
    "that correlation is measuring the relationship between your two variables all else being\n",
    "equal. If your data classes are assigned at random, as they might be in a well-designed\n",
    "experiment, “all else being equal” might not be a terrible assumption. But when there\n",
    "is a deeper pattern to class assignments, “all else being equal” can be an awful assumption.\n",
    "* The only real way to avoid this is by knowing your data and by doing what you can to\n",
    "make sure you’ve checked for possible confounding factors. Obviously, this is not\n",
    "always possible. If you didn’t have the educational attainment of these 200 data scientists,\n",
    "you might simply conclude that there was something inherently more sociable\n",
    "about the West Coast."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}