Encoding/Testing Pandas and Spark Encoding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pyspark import SparkConf\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.ml.feature import StringIndexer\n",
    "from pyspark.ml.feature import OneHotEncoder\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.sql.functions import when"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sidharth/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2717: DtypeWarning: Columns (31,83,86,87,98,99,109,116,123,124,127,129,130,164) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GenderSelect</th>\n",
       "      <th>Country</th>\n",
       "      <th>Age</th>\n",
       "      <th>EmploymentStatus</th>\n",
       "      <th>StudentStatus</th>\n",
       "      <th>LearningDataScience</th>\n",
       "      <th>CodeWriter</th>\n",
       "      <th>CareerSwitcher</th>\n",
       "      <th>CurrentJobTitleSelect</th>\n",
       "      <th>TitleFit</th>\n",
       "      <th>...</th>\n",
       "      <th>JobFactorExperienceLevel</th>\n",
       "      <th>JobFactorDepartment</th>\n",
       "      <th>JobFactorTitle</th>\n",
       "      <th>JobFactorCompanyFunding</th>\n",
       "      <th>JobFactorImpact</th>\n",
       "      <th>JobFactorRemote</th>\n",
       "      <th>JobFactorIndustry</th>\n",
       "      <th>JobFactorLeaderReputation</th>\n",
       "      <th>JobFactorDiversity</th>\n",
       "      <th>JobFactorPublishingOpportunity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Non-binary, genderqueer, or gender non-conforming</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Employed full-time</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DBA/Database Engineer</td>\n",
       "      <td>Fine</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Female</td>\n",
       "      <td>United States</td>\n",
       "      <td>30.0</td>\n",
       "      <td>Not employed, but looking for work</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Somewhat important</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Male</td>\n",
       "      <td>Canada</td>\n",
       "      <td>28.0</td>\n",
       "      <td>Not employed, but looking for work</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>Very Important</td>\n",
       "      <td>Very Important</td>\n",
       "      <td>Very Important</td>\n",
       "      <td>Very Important</td>\n",
       "      <td>Very Important</td>\n",
       "      <td>Very Important</td>\n",
       "      <td>Very Important</td>\n",
       "      <td>Very Important</td>\n",
       "      <td>Very Important</td>\n",
       "      <td>Very Important</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Male</td>\n",
       "      <td>United States</td>\n",
       "      <td>56.0</td>\n",
       "      <td>Independent contractor, freelancer, or self-em...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Operations Research Practitioner</td>\n",
       "      <td>Poorly</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Male</td>\n",
       "      <td>Taiwan</td>\n",
       "      <td>38.0</td>\n",
       "      <td>Employed full-time</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Yes</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Computer Scientist</td>\n",
       "      <td>Fine</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        GenderSelect        Country   Age  \\\n",
       "0  Non-binary, genderqueer, or gender non-conforming            NaN   NaN   \n",
       "1                                             Female  United States  30.0   \n",
       "2                                               Male         Canada  28.0   \n",
       "3                                               Male  United States  56.0   \n",
       "4                                               Male         Taiwan  38.0   \n",
       "\n",
       "                                    EmploymentStatus StudentStatus  \\\n",
       "0                                 Employed full-time           NaN   \n",
       "1                 Not employed, but looking for work           NaN   \n",
       "2                 Not employed, but looking for work           NaN   \n",
       "3  Independent contractor, freelancer, or self-em...           NaN   \n",
       "4                                 Employed full-time           NaN   \n",
       "\n",
       "  LearningDataScience CodeWriter CareerSwitcher  \\\n",
       "0                 NaN        Yes            NaN   \n",
       "1                 NaN        NaN            NaN   \n",
       "2                 NaN        NaN            NaN   \n",
       "3                 NaN        Yes            NaN   \n",
       "4                 NaN        Yes            NaN   \n",
       "\n",
       "              CurrentJobTitleSelect TitleFit              ...                \\\n",
       "0             DBA/Database Engineer     Fine              ...                 \n",
       "1                               NaN      NaN              ...                 \n",
       "2                               NaN      NaN              ...                 \n",
       "3  Operations Research Practitioner   Poorly              ...                 \n",
       "4                Computer Scientist     Fine              ...                 \n",
       "\n",
       "  JobFactorExperienceLevel JobFactorDepartment  JobFactorTitle  \\\n",
       "0                      NaN                 NaN             NaN   \n",
       "1                      NaN                 NaN             NaN   \n",
       "2           Very Important      Very Important  Very Important   \n",
       "3                      NaN                 NaN             NaN   \n",
       "4                      NaN                 NaN             NaN   \n",
       "\n",
       "  JobFactorCompanyFunding JobFactorImpact JobFactorRemote JobFactorIndustry  \\\n",
       "0                     NaN             NaN             NaN               NaN   \n",
       "1                     NaN             NaN             NaN               NaN   \n",
       "2          Very Important  Very Important  Very Important    Very Important   \n",
       "3                     NaN             NaN             NaN               NaN   \n",
       "4                     NaN             NaN             NaN               NaN   \n",
       "\n",
       "  JobFactorLeaderReputation JobFactorDiversity JobFactorPublishingOpportunity  \n",
       "0                       NaN                NaN                            NaN  \n",
       "1        Somewhat important                NaN                            NaN  \n",
       "2            Very Important     Very Important                 Very Important  \n",
       "3                       NaN                NaN                            NaN  \n",
       "4                       NaN                NaN                            NaN  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputDF = pd.read_csv(\"multipleChoiceResponses.csv\")\n",
    "inputDF.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "catColumns = ['GenderSelect','Country','EmploymentStatus']\n",
    "smallDF = inputDF[catColumns]\n",
    "dummyDF = pd.get_dummies(data=smallDF, columns=catColumns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GenderSelect_A different identity</th>\n",
       "      <th>GenderSelect_Female</th>\n",
       "      <th>GenderSelect_Male</th>\n",
       "      <th>GenderSelect_Non-binary, genderqueer, or gender non-conforming</th>\n",
       "      <th>Country_Argentina</th>\n",
       "      <th>Country_Australia</th>\n",
       "      <th>Country_Belarus</th>\n",
       "      <th>Country_Belgium</th>\n",
       "      <th>Country_Brazil</th>\n",
       "      <th>Country_Canada</th>\n",
       "      <th>...</th>\n",
       "      <th>Country_United Kingdom</th>\n",
       "      <th>Country_United States</th>\n",
       "      <th>Country_Vietnam</th>\n",
       "      <th>EmploymentStatus_Employed full-time</th>\n",
       "      <th>EmploymentStatus_Employed part-time</th>\n",
       "      <th>EmploymentStatus_I prefer not to say</th>\n",
       "      <th>EmploymentStatus_Independent contractor, freelancer, or self-employed</th>\n",
       "      <th>EmploymentStatus_Not employed, and not looking for work</th>\n",
       "      <th>EmploymentStatus_Not employed, but looking for work</th>\n",
       "      <th>EmploymentStatus_Retired</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 63 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   GenderSelect_A different identity  GenderSelect_Female  GenderSelect_Male  \\\n",
       "0                                  0                    0                  0   \n",
       "1                                  0                    1                  0   \n",
       "2                                  0                    0                  1   \n",
       "3                                  0                    0                  1   \n",
       "4                                  0                    0                  1   \n",
       "\n",
       "   GenderSelect_Non-binary, genderqueer, or gender non-conforming  \\\n",
       "0                                                  1                \n",
       "1                                                  0                \n",
       "2                                                  0                \n",
       "3                                                  0                \n",
       "4                                                  0                \n",
       "\n",
       "   Country_Argentina  Country_Australia  Country_Belarus  Country_Belgium  \\\n",
       "0                  0                  0                0                0   \n",
       "1                  0                  0                0                0   \n",
       "2                  0                  0                0                0   \n",
       "3                  0                  0                0                0   \n",
       "4                  0                  0                0                0   \n",
       "\n",
       "   Country_Brazil  Country_Canada            ...             \\\n",
       "0               0               0            ...              \n",
       "1               0               0            ...              \n",
       "2               0               1            ...              \n",
       "3               0               0            ...              \n",
       "4               0               0            ...              \n",
       "\n",
       "   Country_United Kingdom  Country_United States  Country_Vietnam  \\\n",
       "0                       0                      0                0   \n",
       "1                       0                      1                0   \n",
       "2                       0                      0                0   \n",
       "3                       0                      1                0   \n",
       "4                       0                      0                0   \n",
       "\n",
       "   EmploymentStatus_Employed full-time  EmploymentStatus_Employed part-time  \\\n",
       "0                                    1                                    0   \n",
       "1                                    0                                    0   \n",
       "2                                    0                                    0   \n",
       "3                                    0                                    0   \n",
       "4                                    1                                    0   \n",
       "\n",
       "   EmploymentStatus_I prefer not to say  \\\n",
       "0                                     0   \n",
       "1                                     0   \n",
       "2                                     0   \n",
       "3                                     0   \n",
       "4                                     0   \n",
       "\n",
       "   EmploymentStatus_Independent contractor, freelancer, or self-employed  \\\n",
       "0                                                  0                       \n",
       "1                                                  0                       \n",
       "2                                                  0                       \n",
       "3                                                  1                       \n",
       "4                                                  0                       \n",
       "\n",
       "   EmploymentStatus_Not employed, and not looking for work  \\\n",
       "0                                                  0         \n",
       "1                                                  0         \n",
       "2                                                  0         \n",
       "3                                                  0         \n",
       "4                                                  0         \n",
       "\n",
       "   EmploymentStatus_Not employed, but looking for work  \\\n",
       "0                                                  0     \n",
       "1                                                  1     \n",
       "2                                                  1     \n",
       "3                                                  0     \n",
       "4                                                  0     \n",
       "\n",
       "   EmploymentStatus_Retired  \n",
       "0                         0  \n",
       "1                         0  \n",
       "2                         0  \n",
       "3                         0  \n",
       "4                         0  \n",
       "\n",
       "[5 rows x 63 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummyDF.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sparkConf = SparkConf(loadDefaults=True).setAppName(\"Encoding\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "spark = SparkSession.builder.master(\"local\").config(conf=sparkConf).getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sparkDF = spark.read.csv('multipleChoiceResponses.csv',header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sparkDF = sparkDF.select(catColumns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for col in catColumns:\n",
    "    sparkDF = sparkDF.withColumn(col, when(sparkDF[col].isNull(),\"__NA__\").otherwise(sparkDF[col]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "indexers = [StringIndexer(inputCol=col, outputCol=col+\"_encoded\").fit(sparkDF) for col in catColumns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "encoders = [OneHotEncoder(inputCol=col+\"_encoded\", outputCol=col+\"_onehot\") for col in catColumns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "p1 = Pipeline(stages=indexers)\n",
    "indexedDF = p1.fit(sparkDF).transform(sparkDF)\n",
    "p2 = Pipeline(stages=encoders)\n",
    "encodedDF = p2.fit(indexedDF).transform(indexedDF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+\n",
      "| Country_onehot|\n",
      "+---------------+\n",
      "|(84,[21],[1.0])|\n",
      "| (84,[0],[1.0])|\n",
      "| (84,[9],[1.0])|\n",
      "| (84,[0],[1.0])|\n",
      "|(84,[13],[1.0])|\n",
      "| (84,[6],[1.0])|\n",
      "| (84,[0],[1.0])|\n",
      "| (84,[1],[1.0])|\n",
      "|(84,[10],[1.0])|\n",
      "| (84,[3],[1.0])|\n",
      "| (84,[3],[1.0])|\n",
      "| (84,[1],[1.0])|\n",
      "| (84,[6],[1.0])|\n",
      "|(84,[15],[1.0])|\n",
      "|(84,[13],[1.0])|\n",
      "| (84,[0],[1.0])|\n",
      "|(84,[14],[1.0])|\n",
      "| (84,[4],[1.0])|\n",
      "| (84,[0],[1.0])|\n",
      "| (84,[6],[1.0])|\n",
      "+---------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "encodedDF.select('Country_onehot').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "oneHotPandasDF = encodedDF.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for col in catColumns:\n",
    "    \n",
    "    col_sep = '_'\n",
    "    numCat = len(oneHotPandasDF[col+\"_onehot\"][0])\n",
    "    recodedCols = []\n",
    "    \n",
    "    for i in range(0,numCat):\n",
    "        recodedCols.append(col+\"_onehot\"+col_sep+str(i))\n",
    "    \n",
    "    tempDF = pd.DataFrame(np.asarray(list(oneHotPandasDF[col+\"_onehot\"].values)), columns=recodedCols)\n",
    "    tempDF1 = oneHotPandasDF.drop(col+\"_onehot\", axis=1)\n",
    "    \n",
    "    finalDF = pd.concat([tempDF1,tempDF], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GenderSelect</th>\n",
       "      <th>Country</th>\n",
       "      <th>EmploymentStatus</th>\n",
       "      <th>GenderSelect_encoded</th>\n",
       "      <th>Country_encoded</th>\n",
       "      <th>EmploymentStatus_encoded</th>\n",
       "      <th>GenderSelect_onehot</th>\n",
       "      <th>Country_onehot</th>\n",
       "      <th>EmploymentStatus_onehot_0</th>\n",
       "      <th>EmploymentStatus_onehot_1</th>\n",
       "      <th>...</th>\n",
       "      <th>EmploymentStatus_onehot_14</th>\n",
       "      <th>EmploymentStatus_onehot_15</th>\n",
       "      <th>EmploymentStatus_onehot_16</th>\n",
       "      <th>EmploymentStatus_onehot_17</th>\n",
       "      <th>EmploymentStatus_onehot_18</th>\n",
       "      <th>EmploymentStatus_onehot_19</th>\n",
       "      <th>EmploymentStatus_onehot_20</th>\n",
       "      <th>EmploymentStatus_onehot_21</th>\n",
       "      <th>EmploymentStatus_onehot_22</th>\n",
       "      <th>EmploymentStatus_onehot_23</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Non-binary, genderqueer, or gender non-conforming</td>\n",
       "      <td>__NA__</td>\n",
       "      <td>Employed full-time</td>\n",
       "      <td>4.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Female</td>\n",
       "      <td>United States</td>\n",
       "      <td>Not employed, but looking for work</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>(0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>(1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Male</td>\n",
       "      <td>Canada</td>\n",
       "      <td>Not employed, but looking for work</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>(1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Male</td>\n",
       "      <td>United States</td>\n",
       "      <td>Independent contractor, freelancer, or self-em...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>(1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>(1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Male</td>\n",
       "      <td>Taiwan</td>\n",
       "      <td>Employed full-time</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>(1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        GenderSelect        Country  \\\n",
       "0  Non-binary, genderqueer, or gender non-conforming         __NA__   \n",
       "1                                             Female  United States   \n",
       "2                                               Male         Canada   \n",
       "3                                               Male  United States   \n",
       "4                                               Male         Taiwan   \n",
       "\n",
       "                                    EmploymentStatus  GenderSelect_encoded  \\\n",
       "0                                 Employed full-time                   4.0   \n",
       "1                 Not employed, but looking for work                   1.0   \n",
       "2                 Not employed, but looking for work                   0.0   \n",
       "3  Independent contractor, freelancer, or self-em...                   0.0   \n",
       "4                                 Employed full-time                   0.0   \n",
       "\n",
       "   Country_encoded  EmploymentStatus_encoded  \\\n",
       "0             21.0                       0.0   \n",
       "1              0.0                       1.0   \n",
       "2              9.0                       1.0   \n",
       "3              0.0                       2.0   \n",
       "4             13.0                       0.0   \n",
       "\n",
       "                                 GenderSelect_onehot  \\\n",
       "0  (0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1  (0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2  (1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3  (1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "4  (1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                      Country_onehot  \\\n",
       "0  (0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1  (1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2  (0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3  (1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "4  (0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "   EmploymentStatus_onehot_0  EmploymentStatus_onehot_1  \\\n",
       "0                        1.0                        0.0   \n",
       "1                        0.0                        1.0   \n",
       "2                        0.0                        1.0   \n",
       "3                        0.0                        0.0   \n",
       "4                        1.0                        0.0   \n",
       "\n",
       "              ...              EmploymentStatus_onehot_14  \\\n",
       "0             ...                                     0.0   \n",
       "1             ...                                     0.0   \n",
       "2             ...                                     0.0   \n",
       "3             ...                                     0.0   \n",
       "4             ...                                     0.0   \n",
       "\n",
       "   EmploymentStatus_onehot_15  EmploymentStatus_onehot_16  \\\n",
       "0                         0.0                         0.0   \n",
       "1                         0.0                         0.0   \n",
       "2                         0.0                         0.0   \n",
       "3                         0.0                         0.0   \n",
       "4                         0.0                         0.0   \n",
       "\n",
       "   EmploymentStatus_onehot_17  EmploymentStatus_onehot_18  \\\n",
       "0                         0.0                         0.0   \n",
       "1                         0.0                         0.0   \n",
       "2                         0.0                         0.0   \n",
       "3                         0.0                         0.0   \n",
       "4                         0.0                         0.0   \n",
       "\n",
       "   EmploymentStatus_onehot_19  EmploymentStatus_onehot_20  \\\n",
       "0                         0.0                         0.0   \n",
       "1                         0.0                         0.0   \n",
       "2                         0.0                         0.0   \n",
       "3                         0.0                         0.0   \n",
       "4                         0.0                         0.0   \n",
       "\n",
       "   EmploymentStatus_onehot_21  EmploymentStatus_onehot_22  \\\n",
       "0                         0.0                         0.0   \n",
       "1                         0.0                         0.0   \n",
       "2                         0.0                         0.0   \n",
       "3                         0.0                         0.0   \n",
       "4                         0.0                         0.0   \n",
       "\n",
       "   EmploymentStatus_onehot_23  \n",
       "0                         0.0  \n",
       "1                         0.0  \n",
       "2                         0.0  \n",
       "3                         0.0  \n",
       "4                         0.0  \n",
       "\n",
       "[5 rows x 32 columns]"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finalDF.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}