DF 3.2.1 - Summary Statistics, Sampling and Random Data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pyspark.context.SparkContext at 0x7fa7263fc790>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!rm -rf metastore_db/*.lck\n",
    "\n",
    "from pyspark.sql import SQLContext\n",
    "sqlc = SQLContext(sc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import namedtuple\n",
    "Record = namedtuple('Record',['desc','value1','value2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "recDF = sc.parallelize([Record(\"first\",1,3.7),\n",
    "                        Record(\"second\",-2,2.1),\n",
    "                        Record(\"third\",6,0.7)]).toDF()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+------------------+\n",
      "|summary|            value1|            value2|\n",
      "+-------+------------------+------------------+\n",
      "|  count|                 3|                 3|\n",
      "|   mean|1.6666666666666667| 2.166666666666667|\n",
      "| stddev| 4.041451884327381|1.5011106998930273|\n",
      "|    min|                -2|               0.7|\n",
      "|    max|                 6|               3.7|\n",
      "+-------+------------------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "recStats = recDF.describe()\n",
    "recStats.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>value1</th>\n",
       "      <th>value2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>summary</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.6666666666666667</td>\n",
       "      <td>2.166666666666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stddev</th>\n",
       "      <td>4.041451884327381</td>\n",
       "      <td>1.5011106998930273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-2</td>\n",
       "      <td>0.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>6</td>\n",
       "      <td>3.7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     value1              value2\n",
       "summary                                        \n",
       "count                     3                   3\n",
       "mean     1.6666666666666667   2.166666666666667\n",
       "stddev    4.041451884327381  1.5011106998930273\n",
       "min                      -2                 0.7\n",
       "max                       6                 3.7"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recStatsPandas = recStats.toPandas().set_index('summary')\n",
    "recStatsPandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'1.6666666666666667'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recStatsPandas.loc['mean'].value1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.5879120879120878"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recDF.stat.corr('value1','value2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-3.566666666666667"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recDF.stat.cov('value1','value2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>value1_freqItems</th>\n",
       "      <th>value2_freqItems</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[-2, 1, 6]</td>\n",
       "      <td>[0.7, 2.1, 3.7]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  value1_freqItems value2_freqItems\n",
       "0       [-2, 1, 6]  [0.7, 2.1, 3.7]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recDF.stat.freqItems(['value1','value2']).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = sqlc.createDataFrame([(1, 10), (1, 20), (2, 10),(2, 20), (2, 30), (3, 20), (3, 30)]).toDF(\"key\", \"value\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-----+\n",
      "|key|value|\n",
      "+---+-----+\n",
      "|  1|   10|\n",
      "|  1|   20|\n",
      "|  2|   10|\n",
      "|  2|   20|\n",
      "|  2|   30|\n",
      "|  3|   20|\n",
      "|  3|   30|\n",
      "+---+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-----+\n",
      "|key|value|\n",
      "+---+-----+\n",
      "|  1|   10|\n",
      "|  2|   10|\n",
      "|  2|   30|\n",
      "+---+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "dfSampled = df.sample(withReplacement=False, fraction=0.3, seed=11)\n",
    "dfSampled.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training, testing = df.randomSplit(weights=[0.3, 0.7], seed=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-----+\n",
      "|key|value|\n",
      "+---+-----+\n",
      "|  1|   10|\n",
      "|  2|   10|\n",
      "|  2|   30|\n",
      "+---+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "training.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-----+\n",
      "|key|value|\n",
      "+---+-----+\n",
      "|  1|   20|\n",
      "|  2|   20|\n",
      "|  3|   20|\n",
      "|  3|   30|\n",
      "+---+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "testing.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stratified Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-----+\n",
      "|key|value|\n",
      "+---+-----+\n",
      "|  1|   10|\n",
      "|  1|   20|\n",
      "|  2|   10|\n",
      "|  2|   30|\n",
      "|  3|   20|\n",
      "|  3|   30|\n",
      "+---+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "dfStrat = df.stat.sampleBy(col=\"key\", fractions={1: 0.7, 2: 0.7, 3: 0.7}, seed=11)\n",
    "dfStrat.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Data Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import rand, randn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = sqlc.range(0, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+\n",
      "| id|\n",
      "+---+\n",
      "|  0|\n",
      "|  1|\n",
      "|  2|\n",
      "|  3|\n",
      "|  4|\n",
      "+---+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df2 = df.select(\"id\").withColumn(\"uniform\", rand(5)).withColumn(\"normal\", randn(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-------------------+--------------------+\n",
      "| id|            uniform|              normal|\n",
      "+---+-------------------+--------------------+\n",
      "|  0|0.47611851579756026|-0.21311682946326227|\n",
      "|  1|0.06498948189958098|-0.05248092572410684|\n",
      "|  2| 0.7069655052310547|  1.3682472758997855|\n",
      "|  3| 0.1982919638208397|  -0.256535324205377|\n",
      "|  4|0.12030715258495939|  -0.506853671746243|\n",
      "+---+-------------------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df2.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}