SparkTitanic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# First model using Apache Spark\n",
    "\n",
    "### By: Favio Vázquez :)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName('Titanic Data') \\\n",
    "    .getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - in-memory</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://192.168.100.23:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.0.0</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>Titanic Data</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x7f98906c9c50>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = (spark.read\n",
    "          .format(\"csv\")\n",
    "          .option('header', 'true')\n",
    "          .load(\"train.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+--------+------+--------------------+------+---+-----+-----+----------------+-------+-----+--------+\n",
      "|PassengerId|Survived|Pclass|                Name|   Sex|Age|SibSp|Parch|          Ticket|   Fare|Cabin|Embarked|\n",
      "+-----------+--------+------+--------------------+------+---+-----+-----+----------------+-------+-----+--------+\n",
      "|          1|       0|     3|Braund, Mr. Owen ...|  male| 22|    1|    0|       A/5 21171|   7.25| null|       S|\n",
      "|          2|       1|     1|Cumings, Mrs. Joh...|female| 38|    1|    0|        PC 17599|71.2833|  C85|       C|\n",
      "|          3|       1|     3|Heikkinen, Miss. ...|female| 26|    0|    0|STON/O2. 3101282|  7.925| null|       S|\n",
      "|          4|       1|     1|Futrelle, Mrs. Ja...|female| 35|    1|    0|          113803|   53.1| C123|       S|\n",
      "|          5|       0|     3|Allen, Mr. Willia...|  male| 35|    0|    0|          373450|   8.05| null|       S|\n",
      "+-----------+--------+------+--------------------+------+---+-----+-----+----------------+-------+-----+--------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.25</td>\n",
       "      <td>None</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.925</td>\n",
       "      <td>None</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.05</td>\n",
       "      <td>None</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>886</td>\n",
       "      <td>887</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>Montvila, Rev. Juozas</td>\n",
       "      <td>male</td>\n",
       "      <td>27</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>211536</td>\n",
       "      <td>13</td>\n",
       "      <td>None</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>887</td>\n",
       "      <td>888</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Graham, Miss. Margaret Edith</td>\n",
       "      <td>female</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>112053</td>\n",
       "      <td>30</td>\n",
       "      <td>B42</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>888</td>\n",
       "      <td>889</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>\"Johnston, Miss. Catherine Helen \"\"Carrie\"\"\"</td>\n",
       "      <td>female</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>W./C. 6607</td>\n",
       "      <td>23.45</td>\n",
       "      <td>None</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>889</td>\n",
       "      <td>890</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Behr, Mr. Karl Howell</td>\n",
       "      <td>male</td>\n",
       "      <td>26</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>111369</td>\n",
       "      <td>30</td>\n",
       "      <td>C148</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>890</td>\n",
       "      <td>891</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Dooley, Mr. Patrick</td>\n",
       "      <td>male</td>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>370376</td>\n",
       "      <td>7.75</td>\n",
       "      <td>None</td>\n",
       "      <td>Q</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>891 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    PassengerId Survived Pclass  \\\n",
       "0             1        0      3   \n",
       "1             2        1      1   \n",
       "2             3        1      3   \n",
       "3             4        1      1   \n",
       "4             5        0      3   \n",
       "..          ...      ...    ...   \n",
       "886         887        0      2   \n",
       "887         888        1      1   \n",
       "888         889        0      3   \n",
       "889         890        1      1   \n",
       "890         891        0      3   \n",
       "\n",
       "                                                  Name     Sex   Age SibSp  \\\n",
       "0                              Braund, Mr. Owen Harris    male    22     1   \n",
       "1    Cumings, Mrs. John Bradley (Florence Briggs Th...  female    38     1   \n",
       "2                               Heikkinen, Miss. Laina  female    26     0   \n",
       "3         Futrelle, Mrs. Jacques Heath (Lily May Peel)  female    35     1   \n",
       "4                             Allen, Mr. William Henry    male    35     0   \n",
       "..                                                 ...     ...   ...   ...   \n",
       "886                              Montvila, Rev. Juozas    male    27     0   \n",
       "887                       Graham, Miss. Margaret Edith  female    19     0   \n",
       "888       \"Johnston, Miss. Catherine Helen \"\"Carrie\"\"\"  female  None     1   \n",
       "889                              Behr, Mr. Karl Howell    male    26     0   \n",
       "890                                Dooley, Mr. Patrick    male    32     0   \n",
       "\n",
       "    Parch            Ticket     Fare Cabin Embarked  \n",
       "0       0         A/5 21171     7.25  None        S  \n",
       "1       0          PC 17599  71.2833   C85        C  \n",
       "2       0  STON/O2. 3101282    7.925  None        S  \n",
       "3       0            113803     53.1  C123        S  \n",
       "4       0            373450     8.05  None        S  \n",
       "..    ...               ...      ...   ...      ...  \n",
       "886     0            211536       13  None        S  \n",
       "887     0            112053       30   B42        S  \n",
       "888     2        W./C. 6607    23.45  None        S  \n",
       "889     0            111369       30  C148        C  \n",
       "890     0            370376     7.75  None        Q  \n",
       "\n",
       "[891 rows x 12 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking information about your data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "891"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# How many rows we have\n",
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['PassengerId',\n",
       " 'Survived',\n",
       " 'Pclass',\n",
       " 'Name',\n",
       " 'Sex',\n",
       " 'Age',\n",
       " 'SibSp',\n",
       " 'Parch',\n",
       " 'Ticket',\n",
       " 'Fare',\n",
       " 'Cabin',\n",
       " 'Embarked']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The names of our columns\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>summary</th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>count</td>\n",
       "      <td>891</td>\n",
       "      <td>891</td>\n",
       "      <td>891</td>\n",
       "      <td>891</td>\n",
       "      <td>891</td>\n",
       "      <td>714</td>\n",
       "      <td>891</td>\n",
       "      <td>891</td>\n",
       "      <td>891</td>\n",
       "      <td>891</td>\n",
       "      <td>204</td>\n",
       "      <td>889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>mean</td>\n",
       "      <td>446.0</td>\n",
       "      <td>0.3838383838383838</td>\n",
       "      <td>2.308641975308642</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>29.69911764705882</td>\n",
       "      <td>0.5230078563411896</td>\n",
       "      <td>0.38159371492704824</td>\n",
       "      <td>260318.54916792738</td>\n",
       "      <td>32.2042079685746</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>stddev</td>\n",
       "      <td>257.3538420152301</td>\n",
       "      <td>0.48659245426485753</td>\n",
       "      <td>0.8360712409770491</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>14.526497332334035</td>\n",
       "      <td>1.1027434322934315</td>\n",
       "      <td>0.8060572211299488</td>\n",
       "      <td>471609.26868834975</td>\n",
       "      <td>49.69342859718089</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>min</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\"Andersson, Mr. August Edvard (\"\"Wennerstrom\"\")\"</td>\n",
       "      <td>female</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>110152</td>\n",
       "      <td>0</td>\n",
       "      <td>A10</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>max</td>\n",
       "      <td>99</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>van Melkebeke, Mr. Philemon</td>\n",
       "      <td>male</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>WE/P 5735</td>\n",
       "      <td>93.5</td>\n",
       "      <td>T</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  summary        PassengerId             Survived              Pclass  \\\n",
       "0   count                891                  891                 891   \n",
       "1    mean              446.0   0.3838383838383838   2.308641975308642   \n",
       "2  stddev  257.3538420152301  0.48659245426485753  0.8360712409770491   \n",
       "3     min                  1                    0                   1   \n",
       "4     max                 99                    1                   3   \n",
       "\n",
       "                                               Name     Sex  \\\n",
       "0                                               891     891   \n",
       "1                                              None    None   \n",
       "2                                              None    None   \n",
       "3  \"Andersson, Mr. August Edvard (\"\"Wennerstrom\"\")\"  female   \n",
       "4                       van Melkebeke, Mr. Philemon    male   \n",
       "\n",
       "                  Age               SibSp                Parch  \\\n",
       "0                 714                 891                  891   \n",
       "1   29.69911764705882  0.5230078563411896  0.38159371492704824   \n",
       "2  14.526497332334035  1.1027434322934315   0.8060572211299488   \n",
       "3                0.42                   0                    0   \n",
       "4                   9                   8                    6   \n",
       "\n",
       "               Ticket               Fare Cabin Embarked  \n",
       "0                 891                891   204      889  \n",
       "1  260318.54916792738   32.2042079685746  None     None  \n",
       "2  471609.26868834975  49.69342859718089  None     None  \n",
       "3              110152                  0   A10        C  \n",
       "4           WE/P 5735               93.5     T        S  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Basics stats from our columns\n",
    "df.describe().toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('PassengerId', 'string'),\n",
       " ('Survived', 'string'),\n",
       " ('Pclass', 'string'),\n",
       " ('Name', 'string'),\n",
       " ('Sex', 'string'),\n",
       " ('Age', 'string'),\n",
       " ('SibSp', 'string'),\n",
       " ('Parch', 'string'),\n",
       " ('Ticket', 'string'),\n",
       " ('Fare', 'string'),\n",
       " ('Cabin', 'string'),\n",
       " ('Embarked', 'string')]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Types of our columns\n",
    "df.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data preparation and feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+------+------+----+-------+--------+\n",
      "|Survived|Pclass|   Sex| Age|   Fare|Embarked|\n",
      "+--------+------+------+----+-------+--------+\n",
      "|     0.0|   3.0|  male|22.0|   7.25|       S|\n",
      "|     1.0|   1.0|female|38.0|71.2833|       C|\n",
      "|     1.0|   3.0|female|26.0|  7.925|       S|\n",
      "|     1.0|   1.0|female|35.0|   53.1|       S|\n",
      "|     0.0|   3.0|  male|35.0|   8.05|       S|\n",
      "|     0.0|   3.0|  male|null| 8.4583|       Q|\n",
      "|     0.0|   1.0|  male|54.0|51.8625|       S|\n",
      "|     0.0|   3.0|  male| 2.0| 21.075|       S|\n",
      "|     1.0|   3.0|female|27.0|11.1333|       S|\n",
      "|     1.0|   2.0|female|14.0|30.0708|       C|\n",
      "|     1.0|   3.0|female| 4.0|   16.7|       S|\n",
      "|     1.0|   1.0|female|58.0|  26.55|       S|\n",
      "|     0.0|   3.0|  male|20.0|   8.05|       S|\n",
      "|     0.0|   3.0|  male|39.0| 31.275|       S|\n",
      "|     0.0|   3.0|female|14.0| 7.8542|       S|\n",
      "|     1.0|   2.0|female|55.0|   16.0|       S|\n",
      "|     0.0|   3.0|  male| 2.0| 29.125|       Q|\n",
      "|     1.0|   2.0|  male|null|   13.0|       S|\n",
      "|     0.0|   3.0|female|31.0|   18.0|       S|\n",
      "|     1.0|   3.0|female|null|  7.225|       C|\n",
      "+--------+------+------+----+-------+--------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Cast numeric columns\n",
    "\n",
    "from pyspark.sql.functions import col\n",
    "\n",
    "dataset = df.select(col('Survived').cast('float'),\n",
    "                         col('Pclass').cast('float'),\n",
    "                         col('Sex'),\n",
    "                         col('Age').cast('float'),\n",
    "                         col('Fare').cast('float'),\n",
    "                         col('Embarked')\n",
    "                        )\n",
    "\n",
    "dataset.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+------+---+---+----+--------+\n",
      "|Survived|Pclass|Sex|Age|Fare|Embarked|\n",
      "+--------+------+---+---+----+--------+\n",
      "|       0|     0|  0|177|   0|       2|\n",
      "+--------+------+---+---+----+--------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## See if we have missing values\n",
    "from pyspark.sql.functions import isnull, when, count, col\n",
    "\n",
    "dataset.select([count(when(isnull(c), c)).alias(c) for c in dataset.columns]).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop missing values\n",
    "dataset = dataset.replace('null', None)\\\n",
    "        .dropna(how='any')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+------+------+----+-------+--------+------+-------+\n",
      "|Survived|Pclass|   Sex| Age|   Fare|Embarked|Gender|Boarded|\n",
      "+--------+------+------+----+-------+--------+------+-------+\n",
      "|     0.0|   3.0|  male|22.0|   7.25|       S|   0.0|    0.0|\n",
      "|     1.0|   1.0|female|38.0|71.2833|       C|   1.0|    1.0|\n",
      "|     1.0|   3.0|female|26.0|  7.925|       S|   1.0|    0.0|\n",
      "|     1.0|   1.0|female|35.0|   53.1|       S|   1.0|    0.0|\n",
      "|     0.0|   3.0|  male|35.0|   8.05|       S|   0.0|    0.0|\n",
      "|     0.0|   1.0|  male|54.0|51.8625|       S|   0.0|    0.0|\n",
      "|     0.0|   3.0|  male| 2.0| 21.075|       S|   0.0|    0.0|\n",
      "|     1.0|   3.0|female|27.0|11.1333|       S|   1.0|    0.0|\n",
      "|     1.0|   2.0|female|14.0|30.0708|       C|   1.0|    1.0|\n",
      "|     1.0|   3.0|female| 4.0|   16.7|       S|   1.0|    0.0|\n",
      "|     1.0|   1.0|female|58.0|  26.55|       S|   1.0|    0.0|\n",
      "|     0.0|   3.0|  male|20.0|   8.05|       S|   0.0|    0.0|\n",
      "|     0.0|   3.0|  male|39.0| 31.275|       S|   0.0|    0.0|\n",
      "|     0.0|   3.0|female|14.0| 7.8542|       S|   1.0|    0.0|\n",
      "|     1.0|   2.0|female|55.0|   16.0|       S|   1.0|    0.0|\n",
      "|     0.0|   3.0|  male| 2.0| 29.125|       Q|   0.0|    2.0|\n",
      "|     0.0|   3.0|female|31.0|   18.0|       S|   1.0|    0.0|\n",
      "|     0.0|   2.0|  male|35.0|   26.0|       S|   0.0|    0.0|\n",
      "|     1.0|   2.0|  male|34.0|   13.0|       S|   0.0|    0.0|\n",
      "|     1.0|   3.0|female|15.0| 8.0292|       Q|   1.0|    2.0|\n",
      "+--------+------+------+----+-------+--------+------+-------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Index categorical columns with StringIndexer\n",
    "from pyspark.ml.feature import StringIndexer\n",
    "dataset = StringIndexer(\n",
    "    inputCol='Sex', \n",
    "    outputCol='Gender', \n",
    "    handleInvalid='keep').fit(dataset).transform(dataset)\n",
    "dataset = StringIndexer(\n",
    "    inputCol='Embarked', \n",
    "    outputCol='Boarded', \n",
    "    handleInvalid='keep').fit(dataset).transform(dataset)\n",
    "dataset.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Survived', 'float'),\n",
       " ('Pclass', 'float'),\n",
       " ('Sex', 'string'),\n",
       " ('Age', 'float'),\n",
       " ('Fare', 'float'),\n",
       " ('Embarked', 'string'),\n",
       " ('Gender', 'double'),\n",
       " ('Boarded', 'double')]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check data types\n",
    "dataset.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+------+----+-------+------+-------+\n",
      "|Survived|Pclass| Age|   Fare|Gender|Boarded|\n",
      "+--------+------+----+-------+------+-------+\n",
      "|     0.0|   3.0|22.0|   7.25|   0.0|    0.0|\n",
      "|     1.0|   1.0|38.0|71.2833|   1.0|    1.0|\n",
      "|     1.0|   3.0|26.0|  7.925|   1.0|    0.0|\n",
      "|     1.0|   1.0|35.0|   53.1|   1.0|    0.0|\n",
      "|     0.0|   3.0|35.0|   8.05|   0.0|    0.0|\n",
      "|     0.0|   1.0|54.0|51.8625|   0.0|    0.0|\n",
      "|     0.0|   3.0| 2.0| 21.075|   0.0|    0.0|\n",
      "|     1.0|   3.0|27.0|11.1333|   1.0|    0.0|\n",
      "|     1.0|   2.0|14.0|30.0708|   1.0|    1.0|\n",
      "|     1.0|   3.0| 4.0|   16.7|   1.0|    0.0|\n",
      "|     1.0|   1.0|58.0|  26.55|   1.0|    0.0|\n",
      "|     0.0|   3.0|20.0|   8.05|   0.0|    0.0|\n",
      "|     0.0|   3.0|39.0| 31.275|   0.0|    0.0|\n",
      "|     0.0|   3.0|14.0| 7.8542|   1.0|    0.0|\n",
      "|     1.0|   2.0|55.0|   16.0|   1.0|    0.0|\n",
      "|     0.0|   3.0| 2.0| 29.125|   0.0|    2.0|\n",
      "|     0.0|   3.0|31.0|   18.0|   1.0|    0.0|\n",
      "|     0.0|   2.0|35.0|   26.0|   0.0|    0.0|\n",
      "|     1.0|   2.0|34.0|   13.0|   0.0|    0.0|\n",
      "|     1.0|   3.0|15.0| 8.0292|   1.0|    2.0|\n",
      "+--------+------+----+-------+------+-------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Drop unnecessary columns\n",
    "dataset = dataset.drop('Sex')\n",
    "dataset = dataset.drop('Embarked')\n",
    "\n",
    "dataset.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assemble all the features with VectorAssembler\n",
    "\n",
    "required_features = ['Pclass',\n",
    "                    'Age',\n",
    "                    'Fare',\n",
    "                    'Gender',\n",
    "                    'Boarded'\n",
    "                   ]\n",
    "\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "\n",
    "assembler = VectorAssembler(inputCols=required_features, outputCol='features')\n",
    "\n",
    "transformed_data = assembler.transform(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+------+----+-------+------+-------+--------------------+\n",
      "|Survived|Pclass| Age|   Fare|Gender|Boarded|            features|\n",
      "+--------+------+----+-------+------+-------+--------------------+\n",
      "|     0.0|   3.0|22.0|   7.25|   0.0|    0.0|[3.0,22.0,7.25,0....|\n",
      "|     1.0|   1.0|38.0|71.2833|   1.0|    1.0|[1.0,38.0,71.2833...|\n",
      "|     1.0|   3.0|26.0|  7.925|   1.0|    0.0|[3.0,26.0,7.92500...|\n",
      "|     1.0|   1.0|35.0|   53.1|   1.0|    0.0|[1.0,35.0,53.0999...|\n",
      "|     0.0|   3.0|35.0|   8.05|   0.0|    0.0|[3.0,35.0,8.05000...|\n",
      "|     0.0|   1.0|54.0|51.8625|   0.0|    0.0|[1.0,54.0,51.8624...|\n",
      "|     0.0|   3.0| 2.0| 21.075|   0.0|    0.0|[3.0,2.0,21.07500...|\n",
      "|     1.0|   3.0|27.0|11.1333|   1.0|    0.0|[3.0,27.0,11.1332...|\n",
      "|     1.0|   2.0|14.0|30.0708|   1.0|    1.0|[2.0,14.0,30.0708...|\n",
      "|     1.0|   3.0| 4.0|   16.7|   1.0|    0.0|[3.0,4.0,16.70000...|\n",
      "|     1.0|   1.0|58.0|  26.55|   1.0|    0.0|[1.0,58.0,26.5499...|\n",
      "|     0.0|   3.0|20.0|   8.05|   0.0|    0.0|[3.0,20.0,8.05000...|\n",
      "|     0.0|   3.0|39.0| 31.275|   0.0|    0.0|[3.0,39.0,31.2749...|\n",
      "|     0.0|   3.0|14.0| 7.8542|   1.0|    0.0|[3.0,14.0,7.85419...|\n",
      "|     1.0|   2.0|55.0|   16.0|   1.0|    0.0|[2.0,55.0,16.0,1....|\n",
      "|     0.0|   3.0| 2.0| 29.125|   0.0|    2.0|[3.0,2.0,29.125,0...|\n",
      "|     0.0|   3.0|31.0|   18.0|   1.0|    0.0|[3.0,31.0,18.0,1....|\n",
      "|     0.0|   2.0|35.0|   26.0|   0.0|    0.0|[2.0,35.0,26.0,0....|\n",
      "|     1.0|   2.0|34.0|   13.0|   0.0|    0.0|[2.0,34.0,13.0,0....|\n",
      "|     1.0|   3.0|15.0| 8.0292|   1.0|    2.0|[3.0,15.0,8.02919...|\n",
      "+--------+------+----+-------+------+-------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "transformed_data.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the data\n",
    "(training_data, test_data) = transformed_data.randomSplit([0.8,0.2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the model\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "\n",
    "rf = RandomForestClassifier(labelCol='Survived', \n",
    "                            featuresCol='features',\n",
    "                            maxDepth=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit the model\n",
    "model = rf.fit(training_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predict with the test dataset\n",
    "predictions = model.transform(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate our model\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "\n",
    "evaluator = MulticlassClassificationEvaluator(\n",
    "    labelCol='Survived', \n",
    "    predictionCol='prediction', \n",
    "    metricName='accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy =  0.8435374149659864\n"
     ]
    }
   ],
   "source": [
    "# Accuracy\n",
    "accuracy = evaluator.evaluate(predictions)\n",
    "print('Test Accuracy = ', accuracy)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}