date_time_clean_check.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result:           date date_modify_1 date_modify_2\n",
      "0   05-07-201    wrong_date              \n",
      "1   01-10-209    wrong_date              \n",
      "2   01-08-200    wrong_date              \n",
      "3   10-1-2011    wrong_date              \n"
     ]
    }
   ],
   "source": [
    "import gls_preprocess_standard_update_v1 as pp\n",
    "import pandas as pd\n",
    "import datetime as dt\n",
    "import time\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"data/test/jobTitle_employment.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = pp.gls_preprocess(df.c_employment_job_start, None,None,None,None,None,None)\n",
    "result = x.gls_date()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>date_modify_1</th>\n",
       "      <th>date_modify_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2011</td>\n",
       "      <td>01-01-2011</td>\n",
       "      <td>2011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2008</td>\n",
       "      <td>01-01-2008</td>\n",
       "      <td>2008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2006</td>\n",
       "      <td>01-01-2006</td>\n",
       "      <td>2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2002</td>\n",
       "      <td>01-01-2002</td>\n",
       "      <td>2002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2011</td>\n",
       "      <td>01-01-2011</td>\n",
       "      <td>2011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2008</td>\n",
       "      <td>01-01-2008</td>\n",
       "      <td>2008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2006</td>\n",
       "      <td>01-01-2006</td>\n",
       "      <td>2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2002</td>\n",
       "      <td>01-01-2002</td>\n",
       "      <td>2002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>August 2012</td>\n",
       "      <td>01-08-2012</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>December 2010</td>\n",
       "      <td>01-12-2010</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>March 2004</td>\n",
       "      <td>01-03-2004</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>January 2008</td>\n",
       "      <td>01-01-2008</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>February 2004</td>\n",
       "      <td>01-02-2004</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>November 2000</td>\n",
       "      <td>01-11-2000</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2007-11</td>\n",
       "      <td>01-11-2007</td>\n",
       "      <td>11-2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2005</td>\n",
       "      <td>01-01-2005</td>\n",
       "      <td>2005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>NaN</td>\n",
       "      <td>wrong_date</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2004-07</td>\n",
       "      <td>01-07-2004</td>\n",
       "      <td>07-2004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2002-07</td>\n",
       "      <td>01-07-2002</td>\n",
       "      <td>07-2002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2005-03</td>\n",
       "      <td>01-03-2005</td>\n",
       "      <td>03-2005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>September 2011</td>\n",
       "      <td>01-09-2011</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>June 2010</td>\n",
       "      <td>01-06-2010</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>April 2007</td>\n",
       "      <td>01-04-2007</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2007</td>\n",
       "      <td>01-01-2007</td>\n",
       "      <td>2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2002-09</td>\n",
       "      <td>01-09-2002</td>\n",
       "      <td>09-2002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2007</td>\n",
       "      <td>01-01-2007</td>\n",
       "      <td>2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2002-09</td>\n",
       "      <td>01-09-2002</td>\n",
       "      <td>09-2002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>August 2011</td>\n",
       "      <td>01-08-2011</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>July 2008</td>\n",
       "      <td>01-07-2008</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>September 2011</td>\n",
       "      <td>01-09-2011</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199970</th>\n",
       "      <td>2000-03</td>\n",
       "      <td>01-03-2000</td>\n",
       "      <td>03-2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199971</th>\n",
       "      <td>2012-07</td>\n",
       "      <td>01-07-2012</td>\n",
       "      <td>07-2012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199972</th>\n",
       "      <td>2010-03</td>\n",
       "      <td>01-03-2010</td>\n",
       "      <td>03-2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199973</th>\n",
       "      <td>2009-09</td>\n",
       "      <td>01-09-2009</td>\n",
       "      <td>09-2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199974</th>\n",
       "      <td>2006-01</td>\n",
       "      <td>01-01-2006</td>\n",
       "      <td>01-2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199975</th>\n",
       "      <td>2005-02</td>\n",
       "      <td>01-02-2005</td>\n",
       "      <td>02-2005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199976</th>\n",
       "      <td>2003-09</td>\n",
       "      <td>01-09-2003</td>\n",
       "      <td>09-2003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199977</th>\n",
       "      <td>February 2002</td>\n",
       "      <td>01-02-2002</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199978</th>\n",
       "      <td>2012-07</td>\n",
       "      <td>01-07-2012</td>\n",
       "      <td>07-2012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199979</th>\n",
       "      <td>2010-03</td>\n",
       "      <td>01-03-2010</td>\n",
       "      <td>03-2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199980</th>\n",
       "      <td>2009-09</td>\n",
       "      <td>01-09-2009</td>\n",
       "      <td>09-2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199981</th>\n",
       "      <td>2006-01</td>\n",
       "      <td>01-01-2006</td>\n",
       "      <td>01-2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199982</th>\n",
       "      <td>2005-02</td>\n",
       "      <td>01-02-2005</td>\n",
       "      <td>02-2005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199983</th>\n",
       "      <td>2003-09</td>\n",
       "      <td>01-09-2003</td>\n",
       "      <td>09-2003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199984</th>\n",
       "      <td>February 2002</td>\n",
       "      <td>01-02-2002</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199985</th>\n",
       "      <td>July 2012</td>\n",
       "      <td>01-07-2012</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199986</th>\n",
       "      <td>February 2010</td>\n",
       "      <td>01-02-2010</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199987</th>\n",
       "      <td>2012-01</td>\n",
       "      <td>01-01-2012</td>\n",
       "      <td>01-2012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199988</th>\n",
       "      <td>2009-03</td>\n",
       "      <td>01-03-2009</td>\n",
       "      <td>03-2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199989</th>\n",
       "      <td>2007-04</td>\n",
       "      <td>01-04-2007</td>\n",
       "      <td>04-2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199990</th>\n",
       "      <td>2006-06</td>\n",
       "      <td>01-06-2006</td>\n",
       "      <td>06-2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199991</th>\n",
       "      <td>2007-08</td>\n",
       "      <td>01-08-2007</td>\n",
       "      <td>08-2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199992</th>\n",
       "      <td>2004-08</td>\n",
       "      <td>01-08-2004</td>\n",
       "      <td>08-2004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199993</th>\n",
       "      <td>2001-10</td>\n",
       "      <td>01-10-2001</td>\n",
       "      <td>10-2001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199994</th>\n",
       "      <td>1997-10</td>\n",
       "      <td>01-10-1997</td>\n",
       "      <td>10-1997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199995</th>\n",
       "      <td>1996-03</td>\n",
       "      <td>01-03-1996</td>\n",
       "      <td>03-1996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199996</th>\n",
       "      <td>1994-03</td>\n",
       "      <td>01-03-1994</td>\n",
       "      <td>03-1994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199997</th>\n",
       "      <td>Sep 2010</td>\n",
       "      <td>01-09-2010</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199998</th>\n",
       "      <td>Apr 2007</td>\n",
       "      <td>01-04-2007</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199999</th>\n",
       "      <td>Sep 1997</td>\n",
       "      <td>01-09-1997</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>200000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  date date_modify_1 date_modify_2\n",
       "0                 2011    01-01-2011          2011\n",
       "1                 2008    01-01-2008          2008\n",
       "2                 2006    01-01-2006          2006\n",
       "3                 2002    01-01-2002          2002\n",
       "4                 2011    01-01-2011          2011\n",
       "5                 2008    01-01-2008          2008\n",
       "6                 2006    01-01-2006          2006\n",
       "7                 2002    01-01-2002          2002\n",
       "8          August 2012    01-08-2012              \n",
       "9        December 2010    01-12-2010              \n",
       "10          March 2004    01-03-2004              \n",
       "11        January 2008    01-01-2008              \n",
       "12       February 2004    01-02-2004              \n",
       "13       November 2000    01-11-2000              \n",
       "14             2007-11    01-11-2007       11-2007\n",
       "15                2005    01-01-2005          2005\n",
       "16                 NaN    wrong_date              \n",
       "17             2004-07    01-07-2004       07-2004\n",
       "18             2002-07    01-07-2002       07-2002\n",
       "19             2005-03    01-03-2005       03-2005\n",
       "20      September 2011    01-09-2011              \n",
       "21           June 2010    01-06-2010              \n",
       "22          April 2007    01-04-2007              \n",
       "23                2007    01-01-2007          2007\n",
       "24             2002-09    01-09-2002       09-2002\n",
       "25                2007    01-01-2007          2007\n",
       "26             2002-09    01-09-2002       09-2002\n",
       "27         August 2011    01-08-2011              \n",
       "28           July 2008    01-07-2008              \n",
       "29      September 2011    01-09-2011              \n",
       "...                ...           ...           ...\n",
       "199970         2000-03    01-03-2000       03-2000\n",
       "199971         2012-07    01-07-2012       07-2012\n",
       "199972         2010-03    01-03-2010       03-2010\n",
       "199973         2009-09    01-09-2009       09-2009\n",
       "199974         2006-01    01-01-2006       01-2006\n",
       "199975         2005-02    01-02-2005       02-2005\n",
       "199976         2003-09    01-09-2003       09-2003\n",
       "199977   February 2002    01-02-2002              \n",
       "199978         2012-07    01-07-2012       07-2012\n",
       "199979         2010-03    01-03-2010       03-2010\n",
       "199980         2009-09    01-09-2009       09-2009\n",
       "199981         2006-01    01-01-2006       01-2006\n",
       "199982         2005-02    01-02-2005       02-2005\n",
       "199983         2003-09    01-09-2003       09-2003\n",
       "199984   February 2002    01-02-2002              \n",
       "199985       July 2012    01-07-2012              \n",
       "199986   February 2010    01-02-2010              \n",
       "199987         2012-01    01-01-2012       01-2012\n",
       "199988         2009-03    01-03-2009       03-2009\n",
       "199989         2007-04    01-04-2007       04-2007\n",
       "199990         2006-06    01-06-2006       06-2006\n",
       "199991         2007-08    01-08-2007       08-2007\n",
       "199992         2004-08    01-08-2004       08-2004\n",
       "199993         2001-10    01-10-2001       10-2001\n",
       "199994         1997-10    01-10-1997       10-1997\n",
       "199995         1996-03    01-03-1996       03-1996\n",
       "199996         1994-03    01-03-1994       03-1994\n",
       "199997        Sep 2010    01-09-2010              \n",
       "199998        Apr 2007    01-04-2007              \n",
       "199999        Sep 1997    01-09-1997              \n",
       "\n",
       "[200000 rows x 3 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = pp.gls_preprocess(df.c_employment_job_end, None,None,None,None,None,None)\n",
    "result_2 = x.gls_date()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "frames = [result, result_2]\n",
    "final_result = pd.concat(frames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_result.to_csv('clean_date_result.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "400000"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(final_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4998"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(final_result[final_result.date_modify_1 == 'wrong_date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_rows', len(df))\n",
    "print(final_result[final_result.date_modify_1 == 'wrong_date'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}