DSFromScratch/Chap5/063-Chap5Statistics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "#from linear_algebra import sum_of_squares, dot\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Chap 5: Statistics\n",
    "\n",
    "#### Describing a Single Set of Data\n",
    "\n",
    "With a small enough data set, simply looking at the data itself might be best.\n",
    "\n",
    "But for larger datasets you need to distill and communicate relevant features of the data:\n",
    "\n",
    "As a first approach we put the friend counts into a histogram using `Counter` and `plt.bar()`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_friends = [100,49,41,40,25,21,21,19,19,18,18,16,15,15,15,15,14,14,13,13,13,13,12,12,11,10,10,10,10,10,10,10,\n",
    "               10,10,10,10,10,10,10,10,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,8,8,8,8,8,8,8,8,8,8,8,8,8,7,7,7,7,7,7,\n",
    "               7,7,7,7,7,7,7,7,7,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,4\n",
    "               ,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,2,2,2,2,2,2,\n",
    "               2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({1: 22, 6: 22, 3: 20, 4: 20, 9: 18, 2: 17, 5: 17, 7: 15, 10: 15, 8: 13, 13: 4, 15: 4, 12: 2, 14: 2, 18: 2, 19: 2, 21: 2, 11: 1, 16: 1, 25: 1, 100: 1, 40: 1, 41: 1, 49: 1})\n"
     ]
    }
   ],
   "source": [
    "friend_counts = Counter(num_friends)\n",
    "print (friend_counts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For each distinct value the number of times it appears in the array is put into a dictionary and sorted from high to low number of counts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "range(0, 101)\n"
     ]
    }
   ],
   "source": [
    "xs = range(101) # largest value is 100\n",
    "print(xs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 22, 17, 20, 20, 17, 22, 15, 13, 18, 15, 1, 2, 4, 2, 4, 1, 0, 2, 2, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]\n"
     ]
    }
   ],
   "source": [
    "# loops over all numbers 1 - 100 and uses that as a key to pick out the counts.\n",
    "ys = [friend_counts[x] for x in xs]\n",
    "print(ys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEZCAYAAAB1mUk3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGYtJREFUeJzt3XmYZXV95/H3B1pBVhGEEpBuUQdRWcSVZQIjCsxoxBij\ngCsYkzFGiUsikqV7MtGJzIhbNI8bBBkRBaPQRgM6pqIogoIssmlAGkS6QNYGfJDlO3+cU3Ipqm7d\nWu691VXv1/PUU/fs3/vr6vu55/e759xUFZKkpW2DYRcgSRo+w0CSZBhIkgwDSRKGgSQJw0CShGGg\nKST5SZLfGXYdw5Tk95Jcl+TOJHvMcV/7JblivmqbsO8Tk/xtP/atpcMwWIKS/DzJCyfMe0OS745P\nV9Uzq+o70+xneZIHkyzWv6P/DfxJVW1RVRdPXNg+93VtWKxLcutUO6qqc6pq175WO4UkI0k+k+SX\nSe5IcnmSlUke0+fjrkzyuX4eQ/Nnsf4n1uzM9ArEtNukD7WQZMN+7HcGlgOXd1lewO5tWGxeVY+b\nbKVhPo8kWwHnAhsBz6+qLYEXA1sCTx5WXVp4DANNqvPsIclzk/ywfVd5Y5L/06727+3v29t3x89P\n46+SXJtkbZJ/SrJFx35f3y67uV2v8zgrk5yW5OQktwNvaI/9/SS3JbkhyceSLOvY34NJ3pLkp219\nf5tk5yTfS3J7klM715/wHCerdfMkj06yjub/xyVJfjZVMzFJECbZP8n1Sf4iyY3ACePzOtZ5QpLT\nk9yU5Ookb+tYtjLJF5Oc1LbrpUn26lj+rCQXtM/3VGDjLv+U7wLurKrXVdX1AFV1Q1W9s6p+0u5v\nnyTnt218XpK9O471sLPItraT28fjZ4avT7KmfS7HtssOBo4FXt2eNf24nf/G9vne2f4+vEvtGiDD\nQOO6vbv/CPDh9l3lk4EvtfPHxxS2aN8dnwccCbwe2B/YGdgc+AeAJE8HPg4cDjyB5t3p9hOO9TLg\nS1X1WODzwP3AnwGPA/YGXgj8yYRtDgKeBbwA+Avgk8ARwBOB3drjTWayWj9eVb+pqs3bNtmtqp7a\npW2mMgI8FtgJ+KN2XkETQsBq4Mc07XAgcHSSF3ds/7vAKTRttJqm3UjyKOArwEk0bXIa8Ptd6jgQ\n+OepFrZnDl8DPgxsDXwI+Jd2/lQmnkHuCzwVeBHwN0l2qaqzgPcDX2zPmp6VZBOav6WDq2oLYB/g\noi7H0QAZBkvXV5PcOv5D+2Izhd8AT0mydVXdU1XnT1jeGSRHAMdX1Zqqugd4L827ww1oXrTOrKpz\nq+p+4G8mOda5VbUaoKruraofV9X51bgO+BTNi3enD1TV3VV1BfAT4Oz2+OuAb9AExWQmq/WwPHwM\nZLousAvbd9S3Jvlwx/wHgJVVdV9V3Tthm+cB21TV+6rqgaq6FvgMcFjHOudU1VnV3DzsZGD3dv7e\nwLKq+mi77ZeBH3apb2vgxi7LXwL8tKpOqaoHq+pU4EqaMOpFAavaAL0EuBjoNtj+ALBbko2raqz9\nN9MCYBgsXYdW1ePGf3jku+1ObwJ2Aa5suxFe0mXd7YE1HdNrgGXAdu2y33aVVNWvgVsmbH9950SS\npyZZ3XZP3Q68D9hmwjY3dTz+NTA2YXqzWdTaq2dV1VZtO/5Zx/ybq+q+KbbZCdihI4xvowmibTvW\nWdvx+B5g4zakngDcMGF/a5jaLe02U5nYBuP726HLNhN1tvc9TNHebeC+GngLcGP777rLDI6jPjIM\nlq6eB32r6uqqOqKqHg8cB5ye5pMokw04/5Jm4HXccpqunjGad6g7/raAZh9bTzzchOl/BK4Antx2\nHf3lTGqfxmS13sfDX9ymM1Ut3Qbjrweu6Qjjrapqy6rq5d34jTzyhXqnLut/C/i9Lst/CayYZH/j\ngXM3sEnHspHpS/ytR7RBVX2zqg5q93MV8OkZ7E99ZBhoWklek2T83fgdNP/JHwRubn93firlC8A7\nkqxIshnNO/lTq+pB4HTgd5O8oO37XtXD4TenGQC9J8nTaN5VzpdutfbT+cC6doB54yQbJnlGkud0\n2WY8dM4F7k/ytiTLkryCpttpKscDW7SD0TsBJNkhyQeTPBP4OvDUJIe1dbwa2JVmHAGaPv3D2mM9\nB3jlFHVNZgxY0Y6RkGTbJC9rxw7uA+6i6TbSAmAYLE29fIS0c51DgMuS3EkzwPjqtj//1zQvoN9r\nuzueB5xA08f9HeBqmm6DtwNU1eXA24Av0rwjvZOmi2din3qndwOvaY/9SeDUaZ7LTD4eO2WtPe5r\nVl8G0obNS4E9gZ/TtMGngS26bdZuex/wCprB71uAPwC+3OVYt9EM1N4HnJfkDuCbwO3Af1TVrW0t\n7wZ+1f5+STsf4K+BpwC3AitpBvUfUdcU06fRhMUtSX7UPn4nzVnHr2g+gDCf4a45SD+/3CbJjsDn\naPpgHwQ+VVUfS7ISeDMP9fUeW1X/2rdCtCAl2ZTmRekpVdWt31tSn/U7DEaAkaq6qD0NvwA4lGYQ\naV1VHd+3g2tBSvJS4P/RnJV+EHhuVT17uFVJ6ms3UVWtraqL2sd30QwEjg9+9eWqVS14h9J0Ef2C\nZqzhsO6rSxqEvp4ZPOxAyQpgFHgmzVWRb6QZjPwR8K6qumMghUiSHmEgA8htF9HpwNHtGcIngJ2r\nak+az1PbXSRJQ9T3M4M094X5GvCNqvrIJMuXA6uravdJlg3mtEWSFpmqmlFX/CDODE4ALu8MgnZg\nedwraG4hMKmqWvI/K1euHHoNC+XHtrAdbIfp22E2Jr2b43xJsi/wGuDS9q6FRXMnwyOS7EnzcdNr\ngT/uZx2SpO76GgZV9T1gsnu5e02BJC0gXoG8HjjggAOGXcKCYVs0bIeG7dCYj3YY2EdLZyNJLeT6\nJGkhSkItwAFkSdICZxhIkgwDSZJhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQ\nJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgSWI9CoORkRUkIQkjIyvmtI/Zbi9Ji1Wqatg1TClJ\njdeXBBivNcym7of2MbvtJWl9kISqyky2WW/ODCRJ/WMYSJIMA0mSYSBJwjCQJGEYSJIwDCRJGAaS\nJAwDSRKGgSQJw0CShGEgSaLPYZBkxyTfTnJZkkuTvL2dv1WSs5NcleSsJFv2sw5JUnd9vWtpkhFg\npKouSrIZcAFwKHAkcEtVHZfkPcBWVXXMJNt711JJmqEFd9fSqlpbVRe1j+8CrgB2pAmEk9rVTgJe\n3s86JEndDWzMIMkKYE/gB8B2VTUGTWAA2w6qDknSIy0bxEHaLqLTgaOr6q4kE/topuyzWbVqVcfU\nKHDAI9YZGVnB2NgaADbYYBMefPAeALbbbjlr1147+8IlaT0wOjrK6OjonPbR9286S7IM+Brwjar6\nSDvvCuCAqhprxxX+rap2nWTbnsYMJi6bfj3HDCQtXgtuzKB1AnD5eBC0zgTe2D5+A3DGAOqQJE2h\n358m2hf4DnApzVvyAo4Fzge+BDwRWAO8qqpun2R7zwwkaYZmc2bQ926iuTAMJGnmFmo3kSRpgTMM\nJEmGgSTJMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCY1MrKCJIyMrBh2\nKZI0EN7CevLj9rSeJC1E3sJakjQrhoEkyTCQJBkGkiQMA0kShoEkCcNAksR6HAa9XhjmBWSSNL31\n9qKzRk14PL5suvW86EzS4uVFZ5KkWTEMJEmGgSTJMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKE\nYSBJoscwSLI8yYvax49JsnmP2302yViSSzrmrUzyiyQXtj+HzK50SdJ8mTYMkrwZOB34ZDtrR+Cr\nPe7/RODgSeYfX1V7tT//2uO+JEl90suZwVuBfYE7AarqZ8C2vey8qs4Bbptk0YzupidJ6q9ewuDe\nqvrN+ESSZTx0j+jZ+tMkFyX5TJIt57gvSdIc9RIG/57kWOAxSV4MnAasnsMxPwHsXFV7AmuB4+ew\nL0nSPFjWwzrHAG8CLgX+GPg68JnZHrCqbu6Y/DTTBMuqVas6pkaBA2Z76IcZGVnB2NgaALbbbjlr\n1147p/UkaVhGR0cZHR2d0z76/k1nSVYAq6tqt3Z6pKrWto/fATy3qo6YYtu+fdPZxG0626Hzm866\nrSdJC9FsvulsyjODJJfSZWygqnbvoaBTaN7Kb53kOmAl8F+S7Ak8CFxLc7YhSRqiKc8MkizvtmFV\nrelLRQ+vwTMDSZqheT0z6HyxTzICPI/mVfGH4908kqTFoZeLzv4QOB94BfBK4AdJjup3YZKkwZl2\nADnJVcA+VXVLO7018P2q2qXvxdlNJEkzNptuol6uM7gFWNcxva6dJ0laJHq5zuA/gPOSnEHzFvlQ\n4JIk7wSoKi8ak6T1XC9hcHX7M+6M9ndPdy6VJC18PV90lmQzgKq6q68VPfyYjhlI0gz1ZcwgyTOT\n/Bi4DLgsyQVJnjHbIiVJC08vA8ifAt5ZVcurajnwLpp7CkmSFolewmDTqvq38YmqGgU27VtFkqSB\n62UA+Zokfw2c3E6/FrimfyVJkgatlzODo4DHA/8MfBnYpp0nSVokZvJpok2r6u4+1zPxmH6aSJJm\nqF+fJtonyeXAFe30Hkk+McsaJUkLUC/dRB8CDqa9BUVVXQz8Tj+LGryNSEISRkZWDLsYSRq4XgaQ\nqarrm66T33qgP+UMy72MdwWNjc3ozEqSFoVewuD6JPsAleRRwNG0XUaSpMWhl26i/w68FdgB+CWw\nZzstSVokev400TAM8tNEve57IbeXJEH/Pk20c5LVSW5OclOSM5LsPPsyJUkLTS/dRKcAXwKeAGwP\nnAZ8oZ9FSZIGq5cw2KSqTq6q+9uf/wts3O/CJEmD08unib6R5BjgVJrO81cDX0/yOICqurWP9UmS\nBmDaAeQkP++yuKqqb+MHDiBL0szNZgB52jODqnrS7EuSJK0PehkzkCQtcoaBJGnqMEiyb/t7o8GV\nI0kahm5nBh9tf587iEIkScPTbQD5viSfAnZI8tGJC6vq7f0rS5I0SN3C4KXAi2i+y+CCwZQjSRqG\nXq4z2KP9QpuB8zoDSZq5vtyoDrglyVfam9TdlOTLSXacZY2SpAWolzA4ETiT5iZ12wOr23mSpEWi\nl26ii6tqjwnzLqqqPftaGXYTSdJs9Kub6FdJXptkw/bntcAtsytRkrQQ9RIGRwGvAtYCNwKvBI7s\nZedJPptkLMklHfO2SnJ2kquSnJVky9kULkmaP3392ssk+wF3AZ+rqt3beR8Abqmq45K8B9iqqo6Z\nYnu7iSRphvrVTTRrVXUOcNuE2YcCJ7WPTwJe3s8aJEnTG8aN6ratqjGAqloLbDuEGiRJHRbCXUvt\nd5GkIZv2y22S/FVV/V37eKOquneOxxxLsl1VjSUZAW7qtvKqVas6pkaBA+Z4eElaXEZHRxkdHZ3T\nPqYcQG4Hd78D/OP4NQVJLqyqvWZ0gGQFsLqqdmunPwDcWlUfcABZkubffH/t5ZXAHwA7J/luO711\nkl2q6qoeCzqF5q381kmuA1YCfw+cluQoYA3Nx1YlSUPU7cxgf+A84PvAc4FdgX8Bvg3sUlX79L04\nzwwkacbm+8zgYOBvgCcDxwOXAHdXVU8XnEmS1h9Tfpqoqo6tqgOBa4GTgQ2Bxyc5J8nqAdUnSRqA\naT9NBJxVVT8CfpTkLVW1X5Jt+l2YJGlwZnQ7ikF/0Y1jBpI0c7MZM+jrvYnmyjCQpJlbcPcmkiSt\nHwwDSZJhIEkyDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNA\nkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKG\ngSQJWDasAye5FrgDeBC4r6qeN6xaJGmpG1oY0ITAAVV12xBrkCQx3G6iDPn4kqTWMF+MC/hmkh8m\nefMQ65CkJW+Y3UT7VtWNSR5PEwpXVNU5E1datWpVx9QocMBgqpOk9cTo6Cijo6Nz2keqan6qmUsR\nyUpgXVUdP2F+jdeXhOZkApoeJtrpzsfjy6ZbbzbbNMsWQntJUjdJqKpMv+ZDhtJNlGSTJJu1jzcF\nDgJ+MoxaJEnD6ybaDvhKkmpr+HxVnT2kWiRpyVsQ3URTsZtIkmZuvekmkiQtLIaBJMkwkCQZBpIk\nDIN5MzKygiQkYWRkxaTLJs6XpIXCTxP1vE2zbKr2mlhf53oPLfPTSJL6z08TSZJmxTCQJBkGkiTD\nQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGczL1N5ht\nNOW3nvW6P785TdIg+U1nPW/TLJvqG8x63V+39u62P785TVKv/KYzSdKsGAaSJMNAkmQYSJIwDCRJ\nGAaSJAwDSRKGwQzN7mKyTt0uJuunxXKhWq8X5vXjmIuh/bQ4zcffqBed9bxNf9br9SK2uV50tlgu\nVOu1jfpzzP4eR5qtyV4fvehMkjRjhoEkaXhhkOSQJFcm+WmS9wyrDknSkMIgyQbAPwAHA88ADk/y\ntGHUIq2PRkdHh13CgmA7zJ9hnRk8D/hZVa2pqvuAU4FDh1SLtN7xRbBhO8yfYYXBDsD1HdO/aOdJ\nkobAAWRJ0nCuM0jyAmBVVR3STh8DVFV9YMJ6fqBbkmZhptcZDCsMNgSuAg4EbgTOBw6vqisGXowk\niWXDOGhVPZDkT4GzabqqPmsQSNLwLOjbUUiSBmNBDiAv5QvSkuyY5NtJLktyaZK3t/O3SnJ2kquS\nnJVky2HXOghJNkhyYZIz2+kl1w5JtkxyWpIr2r+L5y/RdnhHkp8kuSTJ55M8eqm0Q5LPJhlLcknH\nvCmfe5L3JvlZ+zdzUC/HWHBh4AVp3A+8s6qeAewNvLV9/scA36qqXYBvA+8dYo2DdDRwecf0UmyH\njwBfr6pdgT2AK1li7ZBke+BtwF5VtTtNF/fhLJ12OJHmNbHTpM89ydOBVwG7Av8V+ESaO9l1teDC\ngCV+QVpVra2qi9rHdwFXADvStMFJ7WonAS8fToWDk2RH4L8Bn+mYvaTaIckWwH+uqhMBqur+qrqD\nJdYOrQ2BTZMsAx4D3MASaYeqOge4bcLsqZ77y4BT27+Va4Gf0byudrUQw8AL0lpJVgB7Aj8Atquq\nMWgCA9h2eJUNzIeAP+ehe/PC0muHJwG/SnJi2132qSSbsMTaoap+CXwQuI4mBO6oqm+xxNphgm2n\neO4TX0NvoIfX0IUYBgKSbAacDhzdniFMHOlf1CP/SV4CjLVnSd1OcRd1O9B0h+wFfLyq9gLupuke\nWGp/D4+leSe8HNie5gzhNSyxdpjGnJ77QgyDG4CdOqZ3bOctGe1p8OnAyVV1Rjt7LMl27fIR4KZh\n1Tcg+wIvS3IN8AXghUlOBtYusXb4BXB9Vf2onf4yTTgstb+HFwHXVNWtVfUA8BVgH5ZeO3Sa6rnf\nADyxY72eXkMXYhj8EHhKkuVJHg0cBpw55JoG7QTg8qr6SMe8M4E3to/fAJwxcaPFpKqOraqdqmpn\nmr+Bb1fV64DVLK12GAOuT/Kf2lkHApexxP4eaLqHXpBk43Yw9ECaDxYspXYIDz9Lnuq5nwkc1n7a\n6knAU2gu7O2+84V4nUGSQ2g+QTF+QdrfD7mkgUmyL/Ad4FKa074CjqX5x/wSTeKvAV5VVbcPq85B\nSrI/8K6qelmSx7HE2iHJHjSD6I8CrgGOpBlMXWrtsJLmjcF9wI+BPwQ2Zwm0Q5JTgAOArYExYCXw\nVeA0JnnuSd4LvImmrY6uqrOnPcZCDANJ0mAtxG4iSdKAGQaSJMNAkmQYSJIwDCRJGAaSJAwDLTJJ\n3p9k/ySHzvT250m2SfKDJBe013t0LtuvvX3yhUk2mmTbc+Zae7uf5UkunY99STNhGGixeT5wHrA/\nzcV7M/Ei4JKqenZVfW/CstcA76+qvarq3vGZ7Ve4UlX7zaHmibz4RwPnRWdaFJIcR3O/9xXA1TSX\n4F8DnF5Vfzdh3eU0t/zYhuZ+LkfSXNl5JrAxzX1c9h5/0U/yJuA44Hbg+zRXA/9PmlsK71JVT0uy\nrqo2b9d/N8395B8NfKWq/kd7zG8A59DcU+cXwKFVdW+SZwOfpQmBbwKHVNXu7X3pT6S58ngD4Per\n6up5bThpXFX548+i+AGeQ3Mbkw2B73ZZ70zgte3jI2lesKG5v8tHp9jmROAV7eP9gXXATh3L72x/\nvxj4ZPs4NPdS2o/mbpu/AXZrl30ROKJ9fDGwb/v4OJqzE4CPAoe3j5cBGw27jf1ZvD92E2kx2Qu4\nhOYbnq7sst7eNHdCBTiZ5g6pM3V+VV03yfyDgBcnuRC4ENgFeGq77OdVNT4ecAGwov2qwi3roW6p\nkzv2dS7wl0n+HFhRHd1T0nxbNuwCpLlqb+T2TzS36r0Z2LSdfyEd3T0d5qNv9O6pygH+V1V9ekKN\ny4HOOh6g6ZIa3+YRquoLSX4AvBT4epI/qqrROVUtTcEzA633quriqnoWcFVVPZ3m+2APqgmDvR2+\nT/P9uQCvBb47D2WMv6CfBRyVZDyQtk/y+AnrdNZ+B3Bbkn066qHd9klV9fOq+hjN7Yl3n4c6pUl5\nZqBFIck2PPQdsbtU1VVdVn87cGI70HszzbjBdKY7myiAqvpmkqcB57bfQb6O5gX+wS77OAo4IcmD\nQOethl+V5HU0tyG+EXhfD3VKs+KniSRJdhNJkgwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkScD/\nB9irH8HozEp6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f25a6147fd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "plt.bar(xs,ys)\n",
    "plt.axis([0,101,0,25])\n",
    "plt.title(\"Histogram of Friend Counts\")\n",
    "plt.xlabel(\"# of friends\")\n",
    "plt.ylabel(\"# of people\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is still not massively useful. So we will now generate some simple statistics: the number of data points:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "204\n"
     ]
    }
   ],
   "source": [
    "num_points  = len(num_friends)\n",
    "print(num_points)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# largest and smallest values\n",
    "largest_value = max(num_friends)\n",
    "smallest_value = min(num_friends)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "1\n",
      "49\n"
     ]
    }
   ],
   "source": [
    "sorted_values = sorted(num_friends)\n",
    "smallest_value = sorted_values[0]\n",
    "second_smallest_value = sorted_values[1]\n",
    "second_largest_value = sorted_values[-2]\n",
    "\n",
    "print(smallest_value)\n",
    "print(second_smallest_value)\n",
    "print(second_largest_value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Central Tendencies\n",
    "\n",
    "Usually we'll want some notion of where our data is centered. Most commonly we'll use the *mean*, which is just the sum of the data divided by its count. The *median* is also commonly used.\n",
    "The mean is very sensitive to outliers. \n",
    "A generalization of the median is the *quantile* which represents the value less than which a certain percentil of the data lies. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dispersion\n",
    "\n",
    "How spread out is our data? \n",
    "More complicated than just the range is *variance*:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "81.543513957307155"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def de_mean(x):\n",
    "    \"\"\"translate x by subtracting its mean (so the result has mean 0)\"\"\"\n",
    "    x_bar = np.mean(x)\n",
    "    return [x_i - x_bar for x_i in x]\n",
    "\n",
    "def variance(x):\n",
    "    \"\"\"assumes x has a least two elements\"\"\"\n",
    "    n = len(x)\n",
    "    deviations = de_mean(x) # deviation from the mean for each element in x\n",
    "    return sum(np.square(deviations)) / (n - 1)\n",
    "\n",
    "variance(num_friends)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As variance is in units that are the \"square\" of the original units, it is often better to look instead at the standard deviation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.03014473623248"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def standard_deviation(x):\n",
    "    return math.sqrt(variance(x))\n",
    "\n",
    "standard_deviation(num_friends)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Both the range and the standard deviation (square root of variance) have the same problem with outliers. \n",
    "\n",
    "A more robust alternative computes the difference between the 75th and 25th percentile value:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def quantile(x, p):\n",
    "    \"\"\"returns the pth-percentile values in x\"\"\"\n",
    "    p_index = int(p * len(x))\n",
    "    return sorted(x)[p_index]\n",
    "\n",
    "def interquartile_range(x):\n",
    "    return quantile(x,0.75) - quantile(x, 0.25)\n",
    "\n",
    "interquartile_range(num_friends)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is quite nicely not affected by a small number of outliers."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Correlation\n",
    "\n",
    "Does the amount of time people people spend on the site relate to the number of friends they have there? \n",
    "We use a list `daily_minutes` that shows how many mins per day each user spends.\n",
    "\n",
    "First we look at *covariance*, the paired analogue of variances. Where variance measures how a single variable deviates from its mean, covariance measures how two variables vary in tandem form their means:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "daily_minutes = [1,68.77,51.25,52.08,38.36,44.54,57.13,51.4,41.42,31.22,34.76,54.01,38.79,47.59,49.1,27.66,41.03,36.73,\n",
    "                 48.65,28.12,46.62,35.57,32.98,35,26.07,23.77,39.73,40.57,31.65,31.21,36.32,20.45,21.93,26.02,27.34,23.49,\n",
    "                 46.94,30.5,33.8,24.23,21.4,27.94,32.24,40.57,25.07,19.42,22.39,18.42,46.96,23.72,26.41,26.97,36.76,40.32,\n",
    "                 35.02,29.47,30.2,31,38.11,38.18,36.31,21.03,30.86,36.07,28.66,29.08,37.28,15.28,24.17,22.31,30.17,25.53,\n",
    "                 19.85,35.37,44.6,17.23,13.47,26.33,35.02,32.09,24.81,19.33,28.77,24.26,31.98,25.73,24.86,16.28,34.51,15.23\n",
    "                 ,39.72,40.8,26.06,35.76,34.76,16.13,44.04,18.03,19.65,32.62,35.59,39.43,14.18,35.24,40.13,41.82,35.45,36.07\n",
    "                 ,43.67,24.61,20.9,21.9,18.79,27.61,27.21,26.61,29.77,20.59,27.53,13.82,33.2,25,33.1,36.65,18.63,14.87,22.2,\n",
    "                 36.81,25.53,24.62,26.25,18.21,28.08,19.42,29.79,32.8,35.99,28.32,27.79,35.88,29.06,36.28,14.1,36.63,37.49,\n",
    "                 26.9,18.58,38.48,24.48,18.95,33.55,14.24,29.04,32.51,25.63,22.22,19,32.73,15.16,13.9,27.2,32.01,29.27,33,\n",
    "                 13.74,20.42,27.32,18.23,35.35,28.48,9.08,24.62,20.12,35.26,19.92,31.02,16.49,12.16,30.7,31.22,34.65,13.13,\n",
    "                 27.51,33.2,31.57,14.1,33.42,17.44,10.12,24.42,9.82,23.39,30.93,15.03,21.67,31.09,33.29,22.61,26.89,23.48,\n",
    "                 8.38,27.81,32.35,23.84]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def dot(v, w):\n",
    "    \"\"\"v_1 * w_1 + ...+ v_n * w_n\"\"\"\n",
    "    return sum(v_i * w_i for v_i, w_i in zip(v, w))\n",
    "\n",
    "def covariance(x, y):\n",
    "    \"\"\"For each corresponding variance value in x and y arrays add together\n",
    "    takes into account if variance is below for one variable and variance is above mean \n",
    "    in another\"\"\"\n",
    "    n = len(x)\n",
    "    return dot(de_mean(x), de_mean(y)) / (n-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22.425435139573057"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "covariance(num_friends, daily_minutes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A \"large\" +ve covariance means that x tends to be large when y is large and small when y is small. A \"large\" negative covariance means the opposite, when x is small, y is large and vice versa. A covariance cose to zero means that no such relationship exists."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Even then, there are a number of issues to take into account:\n",
    "- It is hard to say what a \"large\" covariance is when a variable twice as large would have twice as large covariance. If each user had twice as many friends\n",
    "- the units are the product of the inputs units.\n",
    "\n",
    "So, it is more common to look at the correlation, which divides out the standard deviations of both variables:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.24736957366478204"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def correlation(x, y):\n",
    "    stdev_x = standard_deviation(x)\n",
    "    stdev_y = standard_deviation(y)\n",
    "    if stdev_x > 0 and stdev_y > 0:\n",
    "        return covariance(x, y) / stdev_x /stdev_y\n",
    "    else:\n",
    "        return 0 # if no variance, correlation is zero\n",
    "correlation(num_friends, daily_minutes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `correlation` is unitless and always lies between -1 (perfect anti-correlation) and 1 (perfect correlation). 0.25 is a relative weak positive correlation. \n",
    "Remove outliers based on other information if you can."
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}