hw1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    },
    {
     "ename": "ImportError",
     "evalue": "No module named pyspark",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-92206d77dcf2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mfindspark\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mpyspark\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpyspark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msql\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunctions\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msqlFunctions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;31m# import matplotlib.pyplot as plt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: No module named pyspark"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import findspark\n",
    "import pyspark\n",
    "import pyspark.sql.functions as sqlFunctions\n",
    "# import matplotlib.pyplot as plt\n",
    "import config\n",
    "import summary\n",
    "import etl\n",
    "\n",
    "from operator import add\n",
    "from pyspark.sql import SQLContext\n",
    "from pyspark.ml.feature import VectorAssembler, StandardScaler\n",
    "from dataset import CriteoDataSets, CriteoData\n",
    "\n",
    "# %matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'CriteoDataSets' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-e238091520fb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCriteoDataSets\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSPARK_CONTEXT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSPARK_SQL_CONTEXT\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mtrain\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDEBUG\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtrain_5m\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_5m\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDEBUG\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'CriteoDataSets' is not defined"
     ]
    }
   ],
   "source": [
    "data = CriteoDataSets(config.SPARK_CONTEXT, config.SPARK_SQL_CONTEXT)\n",
    "train = data.train if not config.DEBUG else data.debug\n",
    "train_5m = data.train_5m if not config.DEBUG else data.debug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'config' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-c428e38fdc0d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Label data histogram\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDEBUG\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0msummary\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabel_histogram\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'config' is not defined"
     ]
    }
   ],
   "source": [
    "# Label data histogram\n",
    "if config.DEBUG:\n",
    "    summary.label_histogram(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'summary' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-822c9e0d7adb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mcol_num\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhistogram\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msummary\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint_columns_histograms_iter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m     \u001b[0mbins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcounts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhistogram\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Column %d:\\n\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mcol_num\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'summary' is not defined"
     ]
    }
   ],
   "source": [
    "def pretty_print_bin(bins, index):\n",
    "    return \"{:0,.3f} - {:0,.0f}\".format(bins[i], bins[i + 1])\n",
    "\n",
    "\n",
    "for col_num, histogram in summary.int_columns_histograms_iter(train):\n",
    "    bins, counts = histogram\n",
    "    print(\"Column %d:\\n\" % col_num)\n",
    "    for i, count in enumerate(counts):\n",
    "        print (\"%20s: %s\\n\" % (pretty_print_bin(bins, i), \"{:,}\".format(count)))\n",
    "    print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "categorical_counts = {col_name: counts for (col_name, counts) in summary.cat_column_counts_iter(train)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+-----+\n",
      "|      C1|count|\n",
      "+--------+-----+\n",
      "|05db9164|50197|\n",
      "|68fd1e64|16676|\n",
      "|5a9ed9b0| 8329|\n",
      "|8cf07265| 4887|\n",
      "|be589b51| 3336|\n",
      "|5bfa8ab5| 2364|\n",
      "|87552397| 1806|\n",
      "|f473b8dc| 1339|\n",
      "|39af2607| 1151|\n",
      "|ae82ea21|  873|\n",
      "|9a89b36c|  767|\n",
      "|17f69355|  611|\n",
      "|241546e0|  563|\n",
      "|09ca0b81|  472|\n",
      "|75ac2fe6|  417|\n",
      "|41edac3d|  357|\n",
      "|439a44a4|  342|\n",
      "|7e5c2ff4|  275|\n",
      "|1464facd|  260|\n",
      "|b455c6d7|  253|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|      C2|count|\n",
      "+--------+-----+\n",
      "|38a947a1|12895|\n",
      "|09e68b86| 6826|\n",
      "|80e26c9b| 4043|\n",
      "|38d50e09| 3724|\n",
      "|287130e0| 3269|\n",
      "|4f25e98b| 3268|\n",
      "|1cfdf714| 2903|\n",
      "|207b2d81| 2366|\n",
      "|08d6d899| 1988|\n",
      "|d833535f| 1892|\n",
      "|58e67aaf| 1757|\n",
      "|942f9a8d| 1735|\n",
      "|8947f767| 1701|\n",
      "|8084ee93| 1408|\n",
      "|0468d672| 1392|\n",
      "|0a519c5c| 1272|\n",
      "|39dfaa0d| 1142|\n",
      "|2c16a946| 1138|\n",
      "|e5fb1af3| 1126|\n",
      "|78ccd99e| 1087|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|      C3|count|\n",
      "+--------+-----+\n",
      "|    null| 3935|\n",
      "|d032c263| 3703|\n",
      "|b00d1501| 1900|\n",
      "|02cf9876| 1707|\n",
      "|aa8c1539| 1579|\n",
      "|77f2f2e5| 1571|\n",
      "|74e1a23a| 1058|\n",
      "|9143c832|  972|\n",
      "|2cbec47f|  900|\n",
      "|ad4b77ff|  855|\n",
      "|4470baf4|  725|\n",
      "|02391f51|  507|\n",
      "|ba1947d0|  506|\n",
      "|003f419b|  387|\n",
      "|b009d929|  370|\n",
      "|b1ecc6c4|  362|\n",
      "|e346a5fd|  342|\n",
      "|223b0e16|  321|\n",
      "|c86b2d8d|  310|\n",
      "|6f67f7e5|  307|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|      C4|count|\n",
      "+--------+-----+\n",
      "|c18be181| 5410|\n",
      "|d16679b9| 4380|\n",
      "|    null| 3935|\n",
      "|85dd697c| 3235|\n",
      "|13508380| 2330|\n",
      "|f922efad| 1487|\n",
      "|9a6888fb| 1058|\n",
      "|f56b7dd5|  972|\n",
      "|3e2bfbda|  908|\n",
      "|29998ed1|  895|\n",
      "|6a14f9b9|  854|\n",
      "|8c8a4c47|  730|\n",
      "|82a61820|  635|\n",
      "|b9c629a9|  507|\n",
      "|ff852091|  420|\n",
      "|657dc3b9|  399|\n",
      "|22504558|  395|\n",
      "|40ed41e5|  389|\n",
      "|c7043c4b|  371|\n",
      "|5dff9b29|  370|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|      C5|count|\n",
      "+--------+-----+\n",
      "|25c83c98|67056|\n",
      "|4cf72387|15783|\n",
      "|43b19349| 6228|\n",
      "|384874ce| 3224|\n",
      "|30903e74| 1990|\n",
      "|0942e0a7| 1313|\n",
      "|f281d2a7|  874|\n",
      "|b0530c50|  560|\n",
      "|b2241560|  483|\n",
      "|f3474129|  374|\n",
      "|307e775a|  266|\n",
      "|2c6b8ded|  201|\n",
      "|5a3e1872|  196|\n",
      "|89ff5705|  176|\n",
      "|f1d40cbe|  115|\n",
      "|b706ee81|  112|\n",
      "|a9411994|   93|\n",
      "|bf9f7f48|   93|\n",
      "|65be028e|   91|\n",
      "|db679829|   56|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|      C6|count|\n",
      "+--------+-----+\n",
      "|7e0ccccf|46438|\n",
      "|fbad5c96|19331|\n",
      "|    null|13708|\n",
      "|fe6b92e5|11495|\n",
      "|13718bbd| 3802|\n",
      "|6f6d9be8| 3460|\n",
      "|3bf701e7| 1728|\n",
      "|e3520422|   16|\n",
      "|f1f2de2d|    9|\n",
      "|c05778d5|    6|\n",
      "|c76aecf6|    5|\n",
      "|3c46edb2|    2|\n",
      "+--------+-----+\n",
      "\n",
      "+--------+-----+\n",
      "|      C7|count|\n",
      "+--------+-----+\n",
      "|3f4ec687|  957|\n",
      "|49b74ebc|  902|\n",
      "|7195046d|  776|\n",
      "|970f01b2|  772|\n",
      "|88002ee1|  639|\n",
      "|9b98e9fc|  633|\n",
      "|38eb9cf4|  585|\n",
      "|d2d741ca|  581|\n",
      "|81bb0302|  541|\n",
      "|dc7659bd|  534|\n",
      "|1c86e0eb|  531|\n",
      "|e7698644|  504|\n",
      "|468a0854|  485|\n",
      "|197b4575|  479|\n",
      "|ce4f7f55|  454|\n",
      "|afa309bd|  442|\n",
      "|d0bdaa98|  401|\n",
      "|7227c706|  393|\n",
      "|af0809a5|  386|\n",
      "|b87f4a4a|  384|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|      C8|count|\n",
      "+--------+-----+\n",
      "|0b153874|59264|\n",
      "|5b392875|16689|\n",
      "|1f89b562| 7485|\n",
      "|37e4aa92| 4234|\n",
      "|062b5529| 2638|\n",
      "|51d76abe| 1774|\n",
      "|c8ddd494| 1258|\n",
      "|64523cfa|  979|\n",
      "|6c41e35e|  709|\n",
      "|985e3fcb|  628|\n",
      "|25239412|  474|\n",
      "|a674580f|  396|\n",
      "|66f29b89|  335|\n",
      "|f504a6f4|  264|\n",
      "|a61cc0ef|  224|\n",
      "|361384ce|  201|\n",
      "|a25968f2|  187|\n",
      "|0fb392dd|  167|\n",
      "|45f7c2dd|  134|\n",
      "|233428af|  126|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|      C9|count|\n",
      "+--------+-----+\n",
      "|a73ee510|88523|\n",
      "|7cc72ec2|11445|\n",
      "|a18233ea|   32|\n",
      "+--------+-----+\n",
      "\n",
      "+--------+-----+\n",
      "|     C10|count|\n",
      "+--------+-----+\n",
      "|3b08e48b|26595|\n",
      "|fbbf2c95|  887|\n",
      "|efea433b|  824|\n",
      "|6c47047a|  764|\n",
      "|0e9ead52|  749|\n",
      "|fa7d0797|  705|\n",
      "|5ba575e7|  565|\n",
      "|5162b19c|  483|\n",
      "|451bd4e4|  446|\n",
      "|7f79890b|  426|\n",
      "|f6f942d1|  425|\n",
      "|7259dc52|  393|\n",
      "|726f00fd|  383|\n",
      "|305a0646|  361|\n",
      "|dcbc7c2b|  340|\n",
      "|49d1ad89|  311|\n",
      "|9a1250bd|  271|\n",
      "|299aecf1|  264|\n",
      "|e70742b0|  261|\n",
      "|18139a78|  248|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|     C11|count|\n",
      "+--------+-----+\n",
      "|c4adf918| 2118|\n",
      "|7f8ffe57| 1539|\n",
      "|e51ddf94| 1196|\n",
      "|4d8549da| 1157|\n",
      "|f25fe7e9|  932|\n",
      "|755e4a50|  822|\n",
      "|36bccca0|  771|\n",
      "|b7094596|  761|\n",
      "|5874c9c9|  745|\n",
      "|a7b606c4|  721|\n",
      "|c1ee56d0|  668|\n",
      "|77212bd7|  659|\n",
      "|319687c9|  638|\n",
      "|f1b78ab4|  605|\n",
      "|1054ae5c|  587|\n",
      "|ea4adb47|  581|\n",
      "|9e511730|  538|\n",
      "|a0a5e9d7|  520|\n",
      "|371dae82|  518|\n",
      "|434d6c13|  515|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|     C12|count|\n",
      "+--------+-----+\n",
      "|    null| 3935|\n",
      "|dfbb09fb| 3703|\n",
      "|e0d76380| 1900|\n",
      "|8fe001f4| 1707|\n",
      "|d8c29807| 1579|\n",
      "|9f32b866| 1571|\n",
      "|fb8fab62| 1058|\n",
      "|ae1bb660|  972|\n",
      "|21a23bfe|  900|\n",
      "|6aaba33c|  895|\n",
      "|a2f4e8b5|  855|\n",
      "|bb669e25|  725|\n",
      "|2397259a|  507|\n",
      "|34a238e0|  506|\n",
      "|b99ddbc8|  422|\n",
      "|349450bc|  387|\n",
      "|3563ab62|  370|\n",
      "|2436ff75|  362|\n",
      "|539c5644|  342|\n",
      "|156f99ef|  321|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|     C13|count|\n",
      "+--------+-----+\n",
      "|85dbe138| 2327|\n",
      "|46f42a63| 1669|\n",
      "|3516f6e6| 1327|\n",
      "|51b97b8f| 1157|\n",
      "|80467802| 1111|\n",
      "|6e5da64f| 1093|\n",
      "|ebd756bd| 1067|\n",
      "|740c210d| 1009|\n",
      "|dd183b4c|  932|\n",
      "|5978055e|  822|\n",
      "|025225f2|  813|\n",
      "|1f9d2c38|  761|\n",
      "|eae197fd|  723|\n",
      "|b55434a9|  693|\n",
      "|1aa94af3|  690|\n",
      "|7203f04e|  659|\n",
      "|62036f49|  638|\n",
      "|d7ce3abd|  587|\n",
      "|05781932|  581|\n",
      "|949ea585|  545|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|     C14|count|\n",
      "+--------+-----+\n",
      "|07d13a8f|36315|\n",
      "|b28479f6|34577|\n",
      "|1adce6ef|15964|\n",
      "|64c94865| 3553|\n",
      "|cfef1c29| 2268|\n",
      "|8ceecbc8| 1830|\n",
      "|051219e6| 1717|\n",
      "|f862f261| 1260|\n",
      "|f7c1b33f|  617|\n",
      "|d2dfe871|  538|\n",
      "|ad1cc976|  385|\n",
      "|32813e21|  288|\n",
      "|0601d3b5|  124|\n",
      "|243a4e68|  118|\n",
      "|687dfaf4|   90|\n",
      "|ec19f520|   65|\n",
      "|dcd762ee|   56|\n",
      "|26ac7cf4|   55|\n",
      "|91233270|   55|\n",
      "|ab7390e9|   33|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|     C15|count|\n",
      "+--------+-----+\n",
      "|36721ddc| 2054|\n",
      "|52baadf5| 1526|\n",
      "|42b3012c|  976|\n",
      "|10040656|  972|\n",
      "|dbc5e126|  930|\n",
      "|d345b1a0|  881|\n",
      "|0f942372|  804|\n",
      "|9efd8b77|  777|\n",
      "|f3635baf|  755|\n",
      "|d2f03b75|  733|\n",
      "|a8e962af|  683|\n",
      "|e3209fc2|  680|\n",
      "|dfab705f|  679|\n",
      "|310d155b|  675|\n",
      "|8ab5b746|  653|\n",
      "|ac182643|  649|\n",
      "|ee569ce2|  649|\n",
      "|41f10449|  646|\n",
      "|3628a186|  628|\n",
      "|10935a85|  616|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|     C16|count|\n",
      "+--------+-----+\n",
      "|    null| 3935|\n",
      "|84898b2a| 3703|\n",
      "|1203a270| 1900|\n",
      "|36103458| 1707|\n",
      "|31ca40b6| 1587|\n",
      "|c64d548f| 1579|\n",
      "|c6b1e1b2| 1058|\n",
      "|bad5ee18|  972|\n",
      "|587267a3|  900|\n",
      "|b041b04a|  895|\n",
      "|89052618|  872|\n",
      "|f8b34416|  815|\n",
      "|da441c7e|  783|\n",
      "|2b2ce127|  725|\n",
      "|87acb535|  509|\n",
      "|d37efe8c|  507|\n",
      "|2d0bbe92|  479|\n",
      "|01adbab4|  445|\n",
      "|6615ffe6|  387|\n",
      "|b688c8cc|  370|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|     C17|count|\n",
      "+--------+-----+\n",
      "|e5ba7672|42705|\n",
      "|07c540c4|13033|\n",
      "|d4bb7bd8|12639|\n",
      "|3486227d| 7548|\n",
      "|776ce399| 6418|\n",
      "|2005abd1| 4760|\n",
      "|1e88c74f| 4559|\n",
      "|27c07bd6| 4199|\n",
      "|8efede7f| 4138|\n",
      "|af5d780c|    1|\n",
      "+--------+-----+\n",
      "\n",
      "+--------+-----+\n",
      "|     C18|count|\n",
      "+--------+-----+\n",
      "|5aed7436| 5348|\n",
      "|891589e7| 2879|\n",
      "|e88ffc9d| 2590|\n",
      "|582152eb| 2156|\n",
      "|7ef5affa| 2133|\n",
      "|c21c3e4c| 1757|\n",
      "|005c6740| 1735|\n",
      "|1f868fdd| 1735|\n",
      "|f54016b9| 1731|\n",
      "|bd17c3da| 1570|\n",
      "|63cdbb21| 1252|\n",
      "|13145934| 1126|\n",
      "|e7e991cb| 1087|\n",
      "|9880032b| 1048|\n",
      "|2804effd|  986|\n",
      "|a78bd508|  966|\n",
      "|df4fffb7|  927|\n",
      "|281769c2|  781|\n",
      "|5bb2ec8e|  771|\n",
      "|ffd53157|  737|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|     C19|count|\n",
      "+--------+-----+\n",
      "|    null|41471|\n",
      "|21ddcdc9|34151|\n",
      "|55dd3565| 2385|\n",
      "|cf99e5de| 1215|\n",
      "|9437f62f|  904|\n",
      "|1d1eb838|  597|\n",
      "|5b885066|  489|\n",
      "|712d530c|  443|\n",
      "|1d04f4a4|  417|\n",
      "|9653bb65|  363|\n",
      "|a153cea2|  362|\n",
      "|2b558521|  352|\n",
      "|6f3756eb|  330|\n",
      "|efa3470f|  316|\n",
      "|4764bf77|  314|\n",
      "|3014a4b1|  312|\n",
      "|083e89d9|  286|\n",
      "|f44bef3c|  260|\n",
      "|2f4b9dd2|  234|\n",
      "|f30f7842|  223|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|     C20|count|\n",
      "+--------+-----+\n",
      "|    null|41471|\n",
      "|5840adea|21517|\n",
      "|a458ea53|18983|\n",
      "|b1252a9d|18029|\n",
      "+--------+-----+\n",
      "\n",
      "+--------+-----+\n",
      "|     C21|count|\n",
      "+--------+-----+\n",
      "|    null| 3935|\n",
      "|0014c32a| 3703|\n",
      "|73d06dde| 1900|\n",
      "|e587c466| 1707|\n",
      "|5f957280| 1579|\n",
      "|dfcfc3fa| 1571|\n",
      "|99c09e97| 1058|\n",
      "|0429f84b|  972|\n",
      "|c2a93b37|  900|\n",
      "|723b4dfd|  895|\n",
      "|d4703ebd|  855|\n",
      "|f3ddd519|  815|\n",
      "|2b796e4a|  725|\n",
      "|a4b7004c|  509|\n",
      "|b6119319|  507|\n",
      "|8717ea07|  506|\n",
      "|21c9516a|  445|\n",
      "|3dd38d65|  387|\n",
      "|2754aaf1|  370|\n",
      "|4f1aa25f|  362|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|     C22|count|\n",
      "+--------+-----+\n",
      "|    null|81566|\n",
      "|ad3062eb| 9640|\n",
      "|c9d4222a| 7023|\n",
      "|8ec974f4|  751|\n",
      "|78e2e389|  582|\n",
      "|c0061c6d|  402|\n",
      "|8651fddb|   13|\n",
      "|ccfd4002|   11|\n",
      "|49e825c5|    7|\n",
      "|28f45308|    3|\n",
      "|d9ce1838|    2|\n",
      "+--------+-----+\n",
      "\n",
      "+--------+-----+\n",
      "|     C23|count|\n",
      "+--------+-----+\n",
      "|32c7478e|43970|\n",
      "|3a171ecb|19944|\n",
      "|423fab69|11383|\n",
      "|be7c41b4| 7079|\n",
      "|bcdee96c| 6835|\n",
      "|c7dc6720| 4522|\n",
      "|55dd3565| 3029|\n",
      "|dbb486d7| 1396|\n",
      "|93bad2c0|  988|\n",
      "|c3dc6cef|  385|\n",
      "|85d5a995|  320|\n",
      "|b264a060|  122|\n",
      "|25e3c76b|   19|\n",
      "|72592995|    8|\n",
      "+--------+-----+\n",
      "\n",
      "+--------+-----+\n",
      "|     C24|count|\n",
      "+--------+-----+\n",
      "|1793a828| 6941|\n",
      "|3fdb382b| 6504|\n",
      "|3b183c5c| 5991|\n",
      "|aee52b6f| 4519|\n",
      "|    null| 3935|\n",
      "|b34f3128| 3441|\n",
      "|45ab94c8| 2851|\n",
      "|9117a34a| 1479|\n",
      "|335a6a1e| 1075|\n",
      "|c0d61a5c|  972|\n",
      "|ded4aac9|  833|\n",
      "|df487a73|  791|\n",
      "|8d365d3b|  730|\n",
      "|b258af68|  715|\n",
      "|cafb4e4d|  635|\n",
      "|6c1cdd05|  569|\n",
      "|f96a556f|  550|\n",
      "|8f079aa5|  540|\n",
      "|08b0ce98|  537|\n",
      "|c2fe6ca4|  523|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|     C25|count|\n",
      "+--------+-----+\n",
      "|    null|41471|\n",
      "|e8b83407|18487|\n",
      "|001f3601|11451|\n",
      "|ea9a246c| 5976|\n",
      "|010f6491| 3806|\n",
      "|cb079c2d| 3291|\n",
      "|9b3e8820| 2900|\n",
      "|2bf691b1| 2209|\n",
      "|445bbe3b| 1768|\n",
      "|f0f449dd| 1494|\n",
      "|9d93af03| 1239|\n",
      "|724b04da| 1225|\n",
      "|47907db5|  852|\n",
      "|b9266ff0|  754|\n",
      "|7a402766|  398|\n",
      "|c9f3bea7|  385|\n",
      "|3d2bedd7|  309|\n",
      "|c243e98b|  210|\n",
      "|3a6f6b59|  205|\n",
      "|ce62e669|  196|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------+-----+\n",
      "|     C26|count|\n",
      "+--------+-----+\n",
      "|    null|41471|\n",
      "|49d68486| 5046|\n",
      "|c84c4aec| 2330|\n",
      "|2fede552| 1574|\n",
      "|984e0db0| 1523|\n",
      "|b7d9c3bc| 1299|\n",
      "|aa5f0a15| 1271|\n",
      "|c27f155b|  944|\n",
      "|9904c656|  883|\n",
      "|b9809574|  864|\n",
      "|56be3401|  743|\n",
      "|b13f4ade|  512|\n",
      "|c986348f|  463|\n",
      "|b29c74dc|  372|\n",
      "|4e7af834|  357|\n",
      "|731c3655|  307|\n",
      "|70b6702c|  299|\n",
      "|d15c0cc8|  297|\n",
      "|6935065e|  282|\n",
      "|54ca28ff|  277|\n",
      "+--------+-----+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "if config.DEBUG:\n",
    "    for col_name in train.categorical_column_names:\n",
    "        counts = categorical_counts[col_name]\n",
    "        counts.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if config.DEBUG:\n",
    "    column_distinct_counts = {col_name: summary.column_distinct_count(train, col_name) \n",
    "                              for col_name in train.df.columns}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column C1: 541 distinct values\n",
      "Column C2: 497 distinct values\n",
      "Column C3: 43,870 distinct values\n",
      "Column C4: 25,184 distinct values\n",
      "Column C5: 145 distinct values\n",
      "Column C6: 12 distinct values\n",
      "Column C7: 7,623 distinct values\n",
      "Column C8: 257 distinct values\n",
      "Column C9: 3 distinct values\n",
      "Column C10: 10,997 distinct values\n",
      "Column C11: 3,799 distinct values\n",
      "Column C12: 41,312 distinct values\n",
      "Column C13: 2,796 distinct values\n",
      "Column C14: 26 distinct values\n",
      "Column C15: 5,238 distinct values\n",
      "Column C16: 34,617 distinct values\n",
      "Column C17: 10 distinct values\n",
      "Column C18: 2,548 distinct values\n",
      "Column C19: 1,303 distinct values\n",
      "Column C20: 4 distinct values\n",
      "Column C21: 38,618 distinct values\n",
      "Column C22: 11 distinct values\n",
      "Column C23: 14 distinct values\n",
      "Column C24: 12,335 distinct values\n",
      "Column C25: 51 distinct values\n",
      "Column C26: 9,527 distinct values\n"
     ]
    }
   ],
   "source": [
    "# Distinct value count for categorical features\n",
    "if config.DEBUG:\n",
    "    for col_name in train.categorical_column_names:\n",
    "        count = \"{:,}\".format(column_distinct_counts.get(col_name))\n",
    "        print(\"Column %s: %s distinct values\" % (col_name, count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "integer_column_stats = {name: stats for (name, stats) in summary.integer_column_stats_iter(train)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I1: \tskewness: 13.045\n",
      "\tkurtosis: 373.759\n",
      "\tstddev: 10.451\n",
      "\tmean: 3.769\n",
      "\n",
      "I2: \tskewness: 7.264\n",
      "\tkurtosis: 100.708\n",
      "\tstddev: 401.523\n",
      "\tmean: 112.864\n",
      "\n",
      "I3: \tskewness: 74.676\n",
      "\tkurtosis: 7,426.228\n",
      "\tstddev: 538.819\n",
      "\tmean: 40.745\n",
      "\n",
      "I4: \tskewness: 5.891\n",
      "\tkurtosis: 115.569\n",
      "\tstddev: 10.836\n",
      "\tmean: 8.280\n",
      "\n",
      "I5: \tskewness: 9.146\n",
      "\tkurtosis: 115.043\n",
      "\tstddev: 65,797.898\n",
      "\tmean: 17,592.599\n",
      "\n",
      "I6: \tskewness: 11.409\n",
      "\tkurtosis: 251.298\n",
      "\tstddev: 371.776\n",
      "\tmean: 139.685\n",
      "\n",
      "I7: \tskewness: 42.413\n",
      "\tkurtosis: 4,233.619\n",
      "\tstddev: 65.460\n",
      "\tmean: 15.222\n",
      "\n",
      "I8: \tskewness: 68.854\n",
      "\tkurtosis: 5,577.523\n",
      "\tstddev: 46.542\n",
      "\tmean: 13.575\n",
      "\n",
      "I9: \tskewness: 9.010\n",
      "\tkurtosis: 163.166\n",
      "\tstddev: 286.416\n",
      "\tmean: 125.295\n",
      "\n",
      "I10: \tskewness: 1.125\n",
      "\tkurtosis: 2.495\n",
      "\tstddev: 0.677\n",
      "\tmean: 0.620\n",
      "\n",
      "I11: \tskewness: 6.399\n",
      "\tkurtosis: 66.442\n",
      "\tstddev: 4.630\n",
      "\tmean: 2.400\n",
      "\n",
      "I12: \tskewness: 42.705\n",
      "\tkurtosis: 3,362.307\n",
      "\tstddev: 5.328\n",
      "\tmean: 0.938\n",
      "\n",
      "I13: \tskewness: 67.003\n",
      "\tkurtosis: 5,899.185\n",
      "\tstddev: 52.045\n",
      "\tmean: 11.608\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def pretty_print_stats(stats):\n",
    "    return \"\\n\".join([\"\\t{}: {:0,.3f}\".format(key, val) for (key, val) in stats.items()])\n",
    "\n",
    "if config.DEBUG:\n",
    "    for col_name in train.integer_column_names:\n",
    "        stats = integer_column_stats[col_name]\n",
    "        print(\"%s: %s\\n\" % (col_name, pretty_print_stats(stats)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Row count: 100,000\n"
     ]
    }
   ],
   "source": [
    "row_count = summary.row_count(train)\n",
    "print(\"Row count: %s\" % \"{:,}\".format(row_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Pull just the mean values from the integer column stats\n",
    "integer_column_means = {col_name: round(integer_column_stats[col_name][\"mean\"])\n",
    "                        for col_name in train.integer_column_names}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cat_features = [\"C5\", \"C9\", \"C22\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "scaled_df, scaler, crm = etl.transform_train(train, integer_column_means, cat_features, \n",
    "                                                  categorical_counts, row_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+--------------------+\n",
      "|label|     features_scaled|\n",
      "+-----+--------------------+\n",
      "|    0|[0.12832207225118...|\n",
      "|    0|[0.25664414450236...|\n",
      "|    0|[0.25664414450236...|\n",
      "|    0|[0.51328828900472...|\n",
      "|    0|[0.38496621675354...|\n",
      "|    0|[0.51328828900472...|\n",
      "|    0|[0.51328828900472...|\n",
      "|    1|[0.12832207225118...|\n",
      "|    0|[0.51328828900472...|\n",
      "|    0|[0.51328828900472...|\n",
      "|    0|[0.51328828900472...|\n",
      "|    0|[0.0,0.0149431176...|\n",
      "|    1|[0.0,-0.002490519...|\n",
      "|    1|[0.51328828900472...|\n",
      "|    0|[0.0,0.1270165002...|\n",
      "|    0|[0.51328828900472...|\n",
      "|    1|[0.12832207225118...|\n",
      "|    0|[0.0,0.0024905196...|\n",
      "|    0|[0.0,0.0597724707...|\n",
      "|    0|[0.89825450575827...|\n",
      "+-----+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "scaled_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}