1.2.2 - Sampling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pyspark.context.SparkContext at 0x7fbc7ddc7790>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SQLContext\n",
    "sqlc = SQLContext(sc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.mllib.linalg import Vectors, Vector\n",
    "\n",
    "elements = sc.parallelize([Vectors.dense(4.0,7.0,13.0),\n",
    "                           Vectors.dense(-2.0,8.0,4.0),\n",
    "                           Vectors.dense(3.0,-11.0,19.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[DenseVector([4.0, 7.0, 13.0]), DenseVector([-2.0, 8.0, 4.0])]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "elements.sample(withReplacement=False, fraction=0.5, seed=10L).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[DenseVector([4.0, 7.0, 13.0]),\n",
       " DenseVector([4.0, 7.0, 13.0]),\n",
       " DenseVector([-2.0, 8.0, 4.0]),\n",
       " DenseVector([-2.0, 8.0, 4.0]),\n",
       " DenseVector([-2.0, 8.0, 4.0]),\n",
       " DenseVector([3.0, -11.0, 19.0]),\n",
       " DenseVector([3.0, -11.0, 19.0])]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "elements.sample(withReplacement=True, fraction=3.0, seed=10L).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = sc.parallelize(range(1,1000000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[601691, 198898, 199410]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "splits = data.randomSplit([0.6, 0.2, 0.2], seed = 13L)\n",
    "map(lambda rdd: rdd.count(), splits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training, test, validation = splits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stratified Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pyspark.mllib.linalg.distributed import IndexedRow\n",
    "\n",
    "rows = sc.parallelize([IndexedRow(0,Vectors.dense(1.0,2.0)),\n",
    "                       IndexedRow(1,Vectors.dense(4.0,5.0)),\n",
    "                       IndexedRow(1,Vectors.dense(7.0,8.0))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fractions = {0: 4.0, 1: 0.5}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "approxSample = rows.map(lambda row: (row.index, row.vector)).sampleByKey(True, fractions, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0L, DenseVector([1.0, 2.0])),\n",
       " (0L, DenseVector([1.0, 2.0])),\n",
       " (0L, DenseVector([1.0, 2.0])),\n",
       " (0L, DenseVector([1.0, 2.0])),\n",
       " (0L, DenseVector([1.0, 2.0])),\n",
       " (1L, DenseVector([7.0, 8.0]))]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "approxSample.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}