windspeed/notebooks/033-group_stations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# continuing attempts to only plot months which have > 10 obs that month. \n",
    "\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime as datetime\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "os.chdir(\"/home/sophie/projects/windspeed/bin/\") # change to directory where the module is. Must be a better way?\n",
    "from group_tseries import read_file\n",
    "\n",
    "Sud=['62600WadiHalfa','62640AbuHamed','62650Dongola','62660Karima',\n",
    "'62680Atbara'] \n",
    "\n",
    "wind = read_file(Sud[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Adding rows to the wind dataFrame, ready for grouping\n",
    "\n",
    "wind['ws_0']= wind['ws'][wind['hour'].isin([0])]\n",
    "wind['ws_06']= wind['ws'][wind['hour'].isin([6])]\n",
    "wind['ws_12']= wind['ws'][wind['hour'].isin([12])]\n",
    "wind['ws_18']= wind['ws'][wind['hour'].isin([18])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                              date_time  year month day  hour       ws  ws_0  \\\n",
      "date_time                                                                      \n",
      "1984-03-01 12:00:00 1984-03-01 12:00:00  1984     3   1    12  5.16082   NaN   \n",
      "1984-03-02 12:00:00 1984-03-02 12:00:00  1984     3   2    12  3.49453   NaN   \n",
      "1984-03-04 06:00:00 1984-03-04 06:00:00  1984     3   4     6  3.24567   NaN   \n",
      "1984-03-05 12:00:00 1984-03-05 12:00:00  1984     3   5    12  3.89541   NaN   \n",
      "1984-03-05 18:00:00 1984-03-05 18:00:00  1984     3   5    18  5.70983   NaN   \n",
      "1984-03-06 18:00:00 1984-03-06 18:00:00  1984     3   6    18  4.36439   NaN   \n",
      "1984-03-07 12:00:00 1984-03-07 12:00:00  1984     3   7    12  3.57281   NaN   \n",
      "1984-03-08 12:00:00 1984-03-08 12:00:00  1984     3   8    12  2.92309   NaN   \n",
      "1984-03-09 06:00:00 1984-03-09 06:00:00  1984     3   9     6  4.48241   NaN   \n",
      "1984-03-09 12:00:00 1984-03-09 12:00:00  1984     3   9    12  1.13027   NaN   \n",
      "\n",
      "                       ws_06    ws_12    ws_18  \n",
      "date_time                                       \n",
      "1984-03-01 12:00:00      NaN  5.16082      NaN  \n",
      "1984-03-02 12:00:00      NaN  3.49453      NaN  \n",
      "1984-03-04 06:00:00  3.24567      NaN      NaN  \n",
      "1984-03-05 12:00:00      NaN  3.89541      NaN  \n",
      "1984-03-05 18:00:00      NaN      NaN  5.70983  \n",
      "1984-03-06 18:00:00      NaN      NaN  4.36439  \n",
      "1984-03-07 12:00:00      NaN  3.57281      NaN  \n",
      "1984-03-08 12:00:00      NaN  2.92309      NaN  \n",
      "1984-03-09 06:00:00  4.48241      NaN      NaN  \n",
      "1984-03-09 12:00:00      NaN  1.13027      NaN  \n"
     ]
    }
   ],
   "source": [
    "print wind[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "group = wind.groupby(['year','month'])\n",
    "\n",
    "#print group\n",
    "#print b\n",
    "\n",
    "group_cols=[group.ws, group.ws_0, group.ws_06, group.ws_12, group.ws_18]\n",
    "\n",
    "c = map(lambda x: x > 10, group_cols) # Can only use dropna\n",
    "#print group.ws[0:10]\n",
    "#print c[0:10]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                mean       std  count\n",
      "year month                           \n",
      "1984 3      4.978945  2.058805     43\n"
     ]
    }
   ],
   "source": [
    "wind_group = group['ws','ws_0','ws_06','ws_12','ws_18'].agg(['mean','std','count'])\n",
    "\n",
    "# This gives me the count for each individual column.\n",
    "wind_group[0:3]\n",
    "\n",
    "# Now I want a function which can turn any groups with a count < 10 into Nan\n",
    "print wind_group['ws'][2:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                  ws                 ws_0              ws_06            \\\n",
      "                mean       std count mean std count     mean std count   \n",
      "year month                                                               \n",
      "1984 10     3.875280       NaN     1  NaN NaN     0  3.87528 NaN     1   \n",
      "     12     3.751893  1.428763     4  NaN NaN     0  5.05555 NaN     1   \n",
      "\n",
      "              ws_12                 ws_18            \n",
      "               mean       std count  mean std count  \n",
      "year month                                           \n",
      "1984 10         NaN       NaN     0   NaN NaN     0  \n",
      "     12     3.31734  1.388895     3   NaN NaN     0  \n"
     ]
    }
   ],
   "source": [
    "# create two seperate functions to be fed into agg\n",
    "group = wind.groupby(['year','month'])\n",
    "\n",
    "def meanf(x):\n",
    "    #print x.count()\n",
    "    #if x.count() > 10:\n",
    "    np.mean(x)\n",
    "    #else: x == np.nan\n",
    "\n",
    "def sdf(x):\n",
    "    # if x.count() > 10:\n",
    "    np.std(x)\n",
    "    # else: x == np.nan\n",
    "\n",
    "    \n",
    "wind_group = group['ws','ws_0','ws_06','ws_12','ws_18'].agg(['mean','std','count'])\n",
    "\n",
    "\n",
    "print wind_group[0:2]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ws_group = group['ws'].agg(['mean','std','count'])\n",
    "\n",
    "#print ws_group[0:5]\n",
    "\n",
    "def meanf(x):\n",
    "    #print x.count()\n",
    "    if x.count() > 10:\n",
    "        x == np.mean\n",
    "    else: x == np.nan\n",
    "\n",
    "def sdf(x):\n",
    "    # if x.count() > 10:\n",
    "    np.std(x)\n",
    "    # else: x == np.nan\n",
    "\n",
    "#group_filter = wind.groupby(['year','month']).filter(lambda x: len(x) < 10, dropna=False)    \n",
    "    \n",
    "#print group_filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                   A         B         C\n",
      "2000-01-01 -1.087578 -0.489254 -0.778082\n",
      "2000-01-02 -0.565381  1.851663 -1.231089\n",
      "2000-01-03 -0.322152 -1.162601  0.453183\n",
      "2000-01-04  1.144332 -0.092297 -1.079771\n",
      "2000-01-05 -0.236221 -1.207431  1.056300\n",
      "2000-01-06 -1.265461  1.939401  0.543690\n",
      "2000-01-07  0.642592  0.095540 -0.810490\n",
      "2000-01-08 -0.279442 -0.349765 -1.045544\n",
      "2000-01-09 -1.313917  0.909841 -0.327049\n",
      "2000-01-10  0.158442  1.127992 -1.199688\n",
      "                   A         B         C\n",
      "2000-01-01       NaN       NaN       NaN\n",
      "2000-01-02 -0.565381  1.851663 -1.231089\n",
      "2000-01-03       NaN       NaN       NaN\n",
      "2000-01-04  1.144332 -0.092297 -1.079771\n",
      "2000-01-05       NaN       NaN       NaN\n",
      "2000-01-06 -1.265461  1.939401  0.543690\n",
      "2000-01-07       NaN       NaN       NaN\n",
      "2000-01-08 -0.279442 -0.349765 -1.045544\n",
      "2000-01-09       NaN       NaN       NaN\n",
      "2000-01-10  0.158442  1.127992 -1.199688\n"
     ]
    }
   ],
   "source": [
    "tsdf = pd.DataFrame(np.random.randn(1000, 3), index=pd.date_range('1/1/2000', periods=1000), columns=['A', 'B', 'C'])\n",
    "\n",
    "print tsdf[0:10]\n",
    "\n",
    "tsdf.ix[::2] = np.nan  # every second one is a nan\n",
    "\n",
    "print tsdf[0:10]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "IndentationError",
     "evalue": "expected an indented block (<ipython-input-198-7c708c57e44c>, line 7)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-198-7c708c57e44c>\"\u001b[1;36m, line \u001b[1;32m7\u001b[0m\n\u001b[1;33m    \u001b[0m\n\u001b[1;37m    ^\u001b[0m\n\u001b[1;31mIndentationError\u001b[0m\u001b[1;31m:\u001b[0m expected an indented block\n"
     ]
    }
   ],
   "source": [
    "#print wind_group\n",
    "\n",
    "for name, group in wind.groupby(['year','month']):\n",
    "    #print(name)\n",
    "    if len(group) > 10:\n",
    "            #print name\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}