pandas_outliers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import scipy.stats as stats\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv(\"http://www.mosaic-web.org/go/datasets/galton.csv\")\n",
    "df = df[df['sex']=='F']\n",
    "#df = df[df['sex']=='M']\n",
    "\n",
    "print('This is the famous Galton Height Data collection set of adult heights and their parents heights from 1850\\'s')\n",
    "print(df.head(10))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df.describe(), \"\\n\")\n",
    "print(df.describe()['height']) #note df.describe() returns a data frame, can access columns individually by name\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def outliers_z_score(data):\n",
    "    threshold = 3\n",
    "    mean = np.mean(data)\n",
    "    std = np.std(data)\n",
    "    z_scores = [(y - mean) / std for y in data]\n",
    "    return np.where(np.abs(z_scores) > threshold)\n",
    "\n",
    "def outliers_iqr(data):\n",
    "    q1, q3 = np.percentile(data, [25, 75])\n",
    "    iqr = q3 - q1\n",
    "    lower_bound = q1 - (iqr * 1.5)\n",
    "    upper_bound = q3 + (iqr * 1.5)\n",
    "    return np.where((data > upper_bound) | (data < lower_bound))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lets look at the percentiles \n",
    "num_records = len(df)\n",
    "mu = df.height.mean()\n",
    "sigma = df.height.std()\n",
    "q1, q3 = np.percentile(df.height, [25, 75]) # get the values where 25% of data is below this and 75% is below this\n",
    "iqr = q3 - q1                               # inter - quartile range\n",
    "lower_bound = q1 - (iqr * 1.5)              # tukey outlier lower limit which is the mean value - 1.5 * iqr\n",
    "upper_bound = q3 + (iqr * 1.5)              # tukey outlier upper limit which is the mean value + 1.5 * iqr\n",
    "#  print(mu, sigma, q1, q3, iqr, lower_bound, upper_bound)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "green_diamond = dict(markerfacecolor='g', marker='D')\n",
    "#mybox = plt.boxplot(df.height, flierprops=green_diamond,whis=.75,vert=False)\n",
    "mybox = plt.boxplot(df.height, flierprops=green_diamond,vert=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Number of Records: %d' % num_records)\n",
    "print('mean height %.3f' % mu)\n",
    "print('std height %.3f' % (sigma))\n",
    "print('Quartile 1 %.2f' % q1)\n",
    "print('Quartile 3 %.2f' % q3)\n",
    "print('Inter Quartile Range %.2f' % iqr)\n",
    "print('Tukey Limit Boundaries Lower: %.3f Upper: %.3f' % (lower_bound, upper_bound))\n",
    "x = np.linspace(mu - 4*sigma, mu + 4*sigma, 100)    # set up a 100 linear spaced values between +/-4 std's of mean\n",
    "n, bins, patches = plt.hist(df.height, 20, density=True, facecolor='green', alpha=0.75)\n",
    "print(n, bins)\n",
    "\n",
    "plt.plot(x, stats.norm.pdf(x, mu, sigma))\n",
    "plt.plot([mu, mu], [0, stats.norm.pdf(mu, mu, sigma)], 'k')\n",
    "for i in range(0,4):\n",
    "    lower = mu - i*sigma\n",
    "    upper = mu + i*sigma\n",
    "    record_count = len(df[(df.height>=lower) & (df.height<=upper)])  # number of elements in the range\n",
    "    print('+/- %d std: %.3f-%.3f num_records %d percentage %.3f' % (i, lower, upper, record_count, record_count/num_records))\n",
    "    plt.plot([lower, upper], [stats.norm.pdf(lower, mu, sigma), stats.norm.pdf(upper, mu, sigma) ], 'k')\n",
    "    \n",
    "plt.grid(True)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Outliers using outliers_iqr()\")\n",
    "print(\"=============================\")\n",
    "for i in outliers_iqr(df.height)[0]:\n",
    "    print(df[i:i+1])\n",
    "\n",
    "\n",
    "print(\"Outliers using outliers_z_score()\")\n",
    "print(\"=================================\")\n",
    "for i in outliers_z_score(df.height)[0]:\n",
    "    print(df[i:i+1])\n",
    "print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Avg height %.3f inches: %.3f %.3f' % (mu, stats.norm.pdf(mu, mu, sigma), 1-stats.norm.pdf(mu, mu, sigma)))\n",
    "print('my height 74 inches: %.5f' % (stats.norm.pdf(74, mu, sigma)))\n",
    "print(stats.norm.cdf(74, mu, sigma))\n",
    "print('Taller then %.5f%%' % stats.percentileofscore(df.height, 74, kind='rank'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# separate the 2 modes of the data\n",
    "df_m = df[df['sex']=='M']\n",
    "#df_m.info()\n",
    "df_m.describe()['height']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_w = df[df['sex']=='F']\n",
    "df_w.describe()['height']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}