Project_1026_002.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import findspark\n",
    "findspark.init()\n",
    "import pyspark\n",
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.appName(\"SalesPrediction1026\").getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import lit\n",
    "from pyspark.sql import functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#from pyspark.sql.functions import broadcast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "holidays_events = spark.read.csv(\"holidays_events.csv\", header=True)\n",
    "items = spark.read.csv(\"items.csv\", header=True)\n",
    "oil = spark.read.csv(\"oil.csv\", header=True)\n",
    "stores = spark.read.csv(\"stores.csv\", header=True)\n",
    "test = spark.read.csv(\"test.csv\", header=True)\n",
    "train = spark.read.csv(\"train.csv\", header=True)\n",
    "train_sample = train.sample(False, 0.05, 1)\n",
    "transactions = spark.read.csv(\"transactions.csv\", header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Check the number of partitions:\n",
    "# train_rdd = train.rdd\n",
    "# train_rdd.getNumPartitions()\n",
    "train = train_sample.repartition(800)\n",
    "# test = test.repartition(3000)\n",
    "# train_rdd = train.rdd\n",
    "# train_rdd.getNumPartitions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# stores_rdd = stores.rdd\n",
    "# stores_rdd.getNumPartitions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#train = train.coalesce(20)\n",
    "# test = test.coalesce(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#deal with oil dataframe in pandas with backfill, then transform it to spark dataframe\n",
    "oil_pandas = pd.read_csv(\"oil.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#spark.sql(\"\"\"SET spark.sql.autoBroadcastJoinThreshold = -1\"\"\")\n",
    "spark.conf.set(\"spark.sql.autoBroadcastJoinThreshold\", -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+----------+---------+--------+-----------+------+\n",
      "| id|      date|store_nbr|item_nbr|onpromotion|source|\n",
      "+---+----------+---------+--------+-----------+------+\n",
      "|  2|2013-01-01|       25|  105575|       null| train|\n",
      "| 15|2013-01-01|       25|  115850|       null| train|\n",
      "| 27|2013-01-01|       25|  153267|       null| train|\n",
      "| 34|2013-01-01|       25|  158956|       null| train|\n",
      "| 44|2013-01-01|       25|  165705|       null| train|\n",
      "| 57|2013-01-01|       25|  208498|       null| train|\n",
      "| 68|2013-01-01|       25|  219150|       null| train|\n",
      "|107|2013-01-01|       25|  305344|       null| train|\n",
      "|111|2013-01-01|       25|  310644|       null| train|\n",
      "|195|2013-01-01|       25|  457688|       null| train|\n",
      "|228|2013-01-01|       25|  518091|       null| train|\n",
      "|232|2013-01-01|       25|  522721|       null| train|\n",
      "|250|2013-01-01|       25|  567623|       null| train|\n",
      "|261|2013-01-01|       25|  582863|       null| train|\n",
      "|302|2013-01-01|       25|  660503|       null| train|\n",
      "|360|2013-01-01|       25|  781797|       null| train|\n",
      "|379|2013-01-01|       25|  819194|       null| train|\n",
      "|382|2013-01-01|       25|  819209|       null| train|\n",
      "|425|2013-01-01|       25|  864510|       null| train|\n",
      "|427|2013-01-01|       25|  870545|       null| train|\n",
      "+---+----------+---------+--------+-----------+------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#combine test set and train set together\n",
    "#add 'source' column to both train and test set so we can separate them after data cleaning\n",
    "train = train_sample.withColumn('source', lit('train'))\n",
    "test = test.withColumn('source', lit('test'))\n",
    "train_without_target = train.select(\"id\", 'date', 'store_nbr', 'item_nbr', 'onpromotion', 'source')\n",
    "#use .union() to add them together\n",
    "train_test_set = train_without_target.union(test)\n",
    "train_test_set.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_test_set.rdd.getNumPartitions()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Join train and holiday_events dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-------+---------+--------+-----------+------+----+------+-----------+-----------+-----------+\n",
      "|      date|     id|store_nbr|item_nbr|onpromotion|source|type|locale|locale_name|description|transferred|\n",
      "+----------+-------+---------+--------+-----------+------+----+------+-----------+-----------+-----------+\n",
      "|2013-03-14|2924518|        1|  168927|       null| train|null|  null|       null|       null|       null|\n",
      "|2013-03-14|2924548|        1|  213653|       null| train|null|  null|       null|       null|       null|\n",
      "+----------+-------+---------+--------+-----------+------+----+------+-----------+-----------+-----------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train_holiday = train_test_set.join(holidays_events, 'date', 'left_outer')\n",
    "train_holiday.show(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean Oil Dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>dcoilwtico</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2013-01-01</td>\n",
       "      <td>93.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2013-01-02</td>\n",
       "      <td>93.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2013-01-03</td>\n",
       "      <td>92.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2013-01-04</td>\n",
       "      <td>93.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2013-01-07</td>\n",
       "      <td>93.20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date  dcoilwtico\n",
       "0  2013-01-01       93.14\n",
       "1  2013-01-02       93.14\n",
       "2  2013-01-03       92.97\n",
       "3  2013-01-04       93.12\n",
       "4  2013-01-07       93.20"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oil_pandas = oil_pandas.fillna(method='bfill')\n",
    "oil_pandas = oil_pandas.fillna(method='ffill')\n",
    "oil_pandas.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "date          0\n",
       "dcoilwtico    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oil_pandas.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transform pandas dataframe back to spark dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----------+\n",
      "|      date|dcoilwtico|\n",
      "+----------+----------+\n",
      "|2013-01-01|     93.14|\n",
      "|2013-01-02|     93.14|\n",
      "|2013-01-03|     92.97|\n",
      "|2013-01-04|     93.12|\n",
      "|2013-01-07|      93.2|\n",
      "|2013-01-08|     93.21|\n",
      "|2013-01-09|     93.08|\n",
      "|2013-01-10|     93.81|\n",
      "|2013-01-11|      93.6|\n",
      "|2013-01-14|     94.27|\n",
      "|2013-01-15|     93.26|\n",
      "|2013-01-16|     94.28|\n",
      "|2013-01-17|     95.49|\n",
      "|2013-01-18|     95.61|\n",
      "|2013-01-21|     96.09|\n",
      "|2013-01-22|     96.09|\n",
      "|2013-01-23|     95.06|\n",
      "|2013-01-24|     95.35|\n",
      "|2013-01-25|     95.15|\n",
      "|2013-01-28|     95.95|\n",
      "+----------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "oil_spark = spark.createDataFrame(oil_pandas)\n",
    "oil_spark.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-------+---------+--------+-----------+------+----+------+-----------+-----------+-----------+----------+\n",
      "|      date|     id|store_nbr|item_nbr|onpromotion|source|type|locale|locale_name|description|transferred|dcoilwtico|\n",
      "+----------+-------+---------+--------+-----------+------+----+------+-----------+-----------+-----------+----------+\n",
      "|2013-03-14|2924518|        1|  168927|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924548|        1|  213653|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924588|        1|  260669|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924654|        1|  315178|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924662|        1|  315463|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924699|        1|  360705|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924706|        1|  364724|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924712|        1|  368140|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924723|        1|  371434|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924734|        1|  378330|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924735|        1|  378332|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924744|        1|  407608|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924749|        1|  410866|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924750|        1|  411555|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924774|        1|  426155|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924779|        1|  454426|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924787|        1|  457688|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924794|        1|  463598|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924810|        1|  469643|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "|2013-03-14|2924829|        1|  508069|       null| train|null|  null|       null|       null|       null|     93.03|\n",
      "+----------+-------+---------+--------+-----------+------+----+------+-----------+-----------+-----------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train_holiday_oil = train_holiday.join(oil_spark, 'date', 'left_outer')\n",
    "train_holiday_oil.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Join train_holiday_oil with store"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rename store.type so it would not duplicate with holiday.type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----------+-------+--------+-----------+------+----+------+-----------+-----------+-----------+----------+---------+------+----------+-------+\n",
      "|store_nbr|      date|     id|item_nbr|onpromotion|source|type|locale|locale_name|description|transferred|dcoilwtico|     city| state|store_type|cluster|\n",
      "+---------+----------+-------+--------+-----------+------+----+------+-----------+-----------+-----------+----------+---------+------+----------+-------+\n",
      "|       51|2013-03-14|2962533|  213066|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962562|  229368|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962566|  252970|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962601|  268676|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962605|  269084|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962617|  305079|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962629|  311954|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962648|  315463|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962652|  318787|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962682|  360313|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962694|  364724|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962709|  369432|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962720|  374464|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962731|  404442|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962733|  406965|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962752|  414455|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962762|  419729|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962766|  428054|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962785|  461752|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "|       51|2013-03-14|2962795|  464536|       null| train|null|  null|       null|       null|       null|     93.03|Guayaquil|Guayas|         A|     17|\n",
      "+---------+----------+-------+--------+-----------+------+----+------+-----------+-----------+-----------+----------+---------+------+----------+-------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stores = stores.withColumnRenamed(\"type\", \"store_type\")\n",
    "train_holiday_oil_store = train_holiday_oil.join(stores, 'store_nbr', 'left_outer')\n",
    "train_holiday_oil_store.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Join train_holiday_oil_store and transaction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+---------+------+--------+-----------+------+----+------+-----------+-----------+-----------+----------+---------+------+----------+-------+------------+\n",
      "|      date|store_nbr|    id|item_nbr|onpromotion|source|type|locale|locale_name|description|transferred|dcoilwtico|     city| state|store_type|cluster|transactions|\n",
      "+----------+---------+------+--------+-----------+------+----+------+-----------+-----------+-----------+----------+---------+------+----------+-------+------------+\n",
      "|2013-01-14|       51|523965|  105737|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|523979|  115693|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|523998|  123927|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524003|  153078|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524020|  159242|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524063|  213652|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524065|  213788|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524081|  223434|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524088|  252698|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524093|  255161|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524114|  265086|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524141|  302995|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524145|  305080|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524180|  318787|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524191|  329361|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524197|  355090|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524201|  358096|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524210|  360313|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524214|  361502|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "|2013-01-14|       51|524230|  368260|       null| train|null|  null|       null|       null|       null|     94.27|Guayaquil|Guayas|         A|     17|        1491|\n",
      "+----------+---------+------+--------+-----------+------+----+------+-----------+-----------+-----------+----------+---------+------+----------+-------+------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#try to avoid the duplicated column name:\n",
    "train_holiday_oil_store_transaction = train_holiday_oil_store.join(transactions, ['date', 'store_nbr'], 'left_outer')\n",
    "train_holiday_oil_store_transaction.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Join items table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_holiday_oil_store_transaction_item = train_holiday_oil_store_transaction.join(items, 'item_nbr', 'left_outer')\n",
    "#train_holiday_oil_store_transaction_item.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write to CSV file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#train_holiday_oil_store_transaction_item.write.csv('train_without_clean.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#train_holiday_oil_store_transaction_item.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Count distinct values for each column\n",
    "from pyspark.sql.functions import col, countDistinct\n",
    "#train_holiday_oil_store_transaction_item.agg(countDistinct(col(\"locale\")).alias(\"count_locale\")).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#check number of null values in columns\n",
    "from pyspark.sql.functions import isnan, when, count, col\n",
    "#train_holiday_oil_store_transaction_item.where(col('locale').isNull()).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# null_count = [[col_name, train_holiday_oil_store_transaction_item.where(col(col_name).isNull()).count()] for col_name in train_holiday_oil_store_transaction.columns]\n",
    "# print(null_count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Begin data cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Created a new dataframe called test, so I would not damage the previous one. \n",
    "train_holiday_oil_store_transaction_item_test = train_holiday_oil_store_transaction_item.fillna('False', subset=['onpromotion'])\n",
    "#train_holiday_oil_store_transaction_item_test.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#train_holiday_oil_store_transaction_item_test.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create a table for on_promotion, encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import functions as F\n",
    "onpromotions = train_holiday_oil_store_transaction_item_test.select(\"onpromotion\").distinct().rdd.flatMap(lambda x: x).collect()\n",
    "\n",
    "exprs = [F.when(F.col(\"onpromotion\") == onpromotion, 1).otherwise(0).alias('onpromotion_' + onpromotion)\n",
    "         for onpromotion in onpromotions]\n",
    "\n",
    "#train_holiday_oil_store_transaction_item_test.select(\"item_nbr\", *exprs).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#item table with onpromotion encoded to 1 and 0\n",
    "item_onpromotion = train_holiday_oil_store_transaction_item_test.select(\"item_nbr\", *exprs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encode item_family and create a new table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+-------------------------+----------------+---------------------+---------------+---------------+-------------+----------------+---------------+---------------+------------------+-----------+--------------+-----------------+----------------+-----------+---------------+-------------+----------------------+------------------------+----------+-----------------------------+-------------+------------------+--------------------------------+-------------------+--------------+--------------+------------------+------------+----------+---------------------+-----------+\n",
      "|item_nbr|familyPREPARED FOODS|familyHOME AND KITCHEN II|familyLADIESWEAR|familyLAWN AND GARDEN|familyGROCERY I|familyBABY CARE|familyPRODUCE|familyAUTOMOTIVE|familyBEVERAGES|familyHOME CARE|familyBREAD/BAKERY|familyBOOKS|familyLINGERIE|familyCELEBRATION|familyGROCERY II|familyDAIRY|familyMAGAZINES|familySEAFOOD|familyLIQUOR,WINE,BEER|familyHOME AND KITCHEN I|familyDELI|familyPLAYERS AND ELECTRONICS|familyPOULTRY|familyFROZEN FOODS|familySCHOOL AND OFFICE SUPPLIES|familyPERSONAL CARE|familyCLEANING|familyHARDWARE|familyPET SUPPLIES|familyBEAUTY|familyEGGS|familyHOME APPLIANCES|familyMEATS|\n",
      "+--------+--------------------+-------------------------+----------------+---------------------+---------------+---------------+-------------+----------------+---------------+---------------+------------------+-----------+--------------+-----------------+----------------+-----------+---------------+-------------+----------------------+------------------------+----------+-----------------------------+-------------+------------------+--------------------------------+-------------------+--------------+--------------+------------------+------------+----------+---------------------+-----------+\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "+--------+--------------------+-------------------------+----------------+---------------------+---------------+---------------+-------------+----------------+---------------+---------------+------------------+-----------+--------------+-----------------+----------------+-----------+---------------+-------------+----------------------+------------------------+----------+-----------------------------+-------------+------------------+--------------------------------+-------------------+--------------+--------------+------------------+------------+----------+---------------------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "families = train_holiday_oil_store_transaction_item_test.select(\"family\").distinct().rdd.flatMap(lambda x: x).collect()\n",
    "\n",
    "exprs = [F.when(F.col(\"family\") == family, 1).otherwise(0).alias('family' + family)\n",
    "         for family in families]\n",
    "\n",
    "train_holiday_oil_store_transaction_item_test.select(\"item_nbr\", *exprs).show()\n",
    "item_families = train_holiday_oil_store_transaction_item_test.select(\"item_nbr\", *exprs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+------------+-----------+-------------+------------------+-----------+---------+----------+----------+--------------+--------------+---------+-----------+--------------+-----------+------------+----------+------------+------------+-------------+-------------+-------------+---------------+---------+----------+----------+---------+---------+----------+---------+----------+---------+---------+---------+----------+---------+----------+----------+----------+---------+\n",
      "|store_nbr|city_Quevedo|city_Cuenca|city_Guaranda|city_Santo Domingo|city_Playas|city_Puyo|city_Quito|city_Manta|city_Latacunga|city_Guayaquil|city_Loja|city_Ibarra|city_El Carmen|city_Ambato|city_Machala|city_Daule|city_Cayambe|city_Salinas|city_Libertad|city_Babahoyo|city_Riobamba|city_Esmeraldas|cluster_7|cluster_15|cluster_11|cluster_3|cluster_8|cluster_16|cluster_5|cluster_17|cluster_6|cluster_9|cluster_1|cluster_10|cluster_4|cluster_12|cluster_13|cluster_14|cluster_2|\n",
      "+---------+------------+-----------+-------------+------------------+-----------+---------+----------+----------+--------------+--------------+---------+-----------+--------------+-----------+------------+----------+------------+------------+-------------+-------------+-------------+---------------+---------+----------+----------+---------+---------+----------+---------+----------+---------+---------+---------+----------+---------+----------+----------+----------+---------+\n",
      "|       51|           0|          0|            0|                 0|          0|        0|         0|         0|             0|             1|        0|          0|             0|          0|           0|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        0|        0|         0|        0|         1|        0|        0|        0|         0|        0|         0|         0|         0|        0|\n",
      "|       47|           0|          0|            0|                 0|          0|        0|         1|         0|             0|             0|        0|          0|             0|          0|           0|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        0|        0|         0|        0|         0|        0|        0|        0|         0|        0|         0|         0|         1|        0|\n",
      "|        1|           0|          0|            0|                 0|          0|        0|         1|         0|             0|             0|        0|          0|             0|          0|           0|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        0|        0|         0|        0|         0|        0|        0|        0|         0|        0|         0|         1|         0|        0|\n",
      "|        6|           0|          0|            0|                 0|          0|        0|         1|         0|             0|             0|        0|          0|             0|          0|           0|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        0|        0|         0|        0|         0|        0|        0|        0|         0|        0|         0|         1|         0|        0|\n",
      "|        2|           0|          0|            0|                 0|          0|        0|         1|         0|             0|             0|        0|          0|             0|          0|           0|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        0|        0|         0|        0|         0|        0|        0|        0|         0|        0|         0|         1|         0|        0|\n",
      "|        1|           0|          0|            0|                 0|          0|        0|         1|         0|             0|             0|        0|          0|             0|          0|           0|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        0|        0|         0|        0|         0|        0|        0|        0|         0|        0|         0|         1|         0|        0|\n",
      "|       28|           0|          0|            0|                 0|          0|        0|         0|         0|             0|             1|        0|          0|             0|          0|           0|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        0|        0|         0|        0|         0|        0|        0|        0|         1|        0|         0|         0|         0|        0|\n",
      "|       51|           0|          0|            0|                 0|          0|        0|         0|         0|             0|             1|        0|          0|             0|          0|           0|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        0|        0|         0|        0|         1|        0|        0|        0|         0|        0|         0|         0|         0|        0|\n",
      "|       45|           0|          0|            0|                 0|          0|        0|         1|         0|             0|             0|        0|          0|             0|          0|           0|         0|           0|           0|            0|            0|            0|              0|        0|         0|         1|        0|        0|         0|        0|         0|        0|        0|        0|         0|        0|         0|         0|         0|        0|\n",
      "|       12|           0|          0|            0|                 0|          0|        0|         0|         0|             1|             0|        0|          0|             0|          0|           0|         0|           0|           0|            0|            0|            0|              0|        0|         1|         0|        0|        0|         0|        0|         0|        0|        0|        0|         0|        0|         0|         0|         0|        0|\n",
      "|        4|           0|          0|            0|                 0|          0|        0|         1|         0|             0|             0|        0|          0|             0|          0|           0|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        0|        0|         0|        0|         0|        0|        1|        0|         0|        0|         0|         0|         0|        0|\n",
      "|       24|           0|          0|            0|                 0|          0|        0|         0|         0|             0|             1|        0|          0|             0|          0|           0|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        0|        0|         0|        0|         0|        0|        0|        1|         0|        0|         0|         0|         0|        0|\n",
      "|       50|           0|          0|            0|                 0|          0|        0|         0|         0|             0|             0|        0|          0|             0|          1|           0|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        0|        0|         0|        0|         0|        0|        0|        0|         0|        0|         0|         0|         1|        0|\n",
      "|       40|           0|          0|            0|                 0|          0|        0|         0|         0|             0|             0|        0|          0|             0|          0|           1|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        1|        0|         0|        0|         0|        0|        0|        0|         0|        0|         0|         0|         0|        0|\n",
      "|       51|           0|          0|            0|                 0|          0|        0|         0|         0|             0|             1|        0|          0|             0|          0|           0|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        0|        0|         0|        0|         1|        0|        0|        0|         0|        0|         0|         0|         0|        0|\n",
      "|       40|           0|          0|            0|                 0|          0|        0|         0|         0|             0|             0|        0|          0|             0|          0|           1|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        1|        0|         0|        0|         0|        0|        0|        0|         0|        0|         0|         0|         0|        0|\n",
      "|        9|           0|          0|            0|                 0|          0|        0|         1|         0|             0|             0|        0|          0|             0|          0|           0|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        0|        0|         0|        0|         0|        1|        0|        0|         0|        0|         0|         0|         0|        0|\n",
      "|       30|           0|          0|            0|                 0|          0|        0|         0|         0|             0|             1|        0|          0|             0|          0|           0|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        1|        0|         0|        0|         0|        0|        0|        0|         0|        0|         0|         0|         0|        0|\n",
      "|       41|           0|          0|            0|                 0|          0|        0|         0|         0|             0|             0|        0|          0|             0|          0|           1|         0|           0|           0|            0|            0|            0|              0|        0|         0|         0|        0|        0|         0|        0|         0|        0|        0|        0|         0|        1|         0|         0|         0|        0|\n",
      "|       45|           0|          0|            0|                 0|          0|        0|         1|         0|             0|             0|        0|          0|             0|          0|           0|         0|           0|           0|            0|            0|            0|              0|        0|         0|         1|        0|        0|         0|        0|         0|        0|        0|        0|         0|        0|         0|         0|         0|        0|\n",
      "+---------+------------+-----------+-------------+------------------+-----------+---------+----------+----------+--------------+--------------+---------+-----------+--------------+-----------+------------+----------+------------+------------+-------------+-------------+-------------+---------------+---------+----------+----------+---------+---------+----------+---------+----------+---------+---------+---------+----------+---------+----------+----------+----------+---------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cities = train_holiday_oil_store_transaction_item_test.select(\"city\").distinct().rdd.flatMap(lambda x: x).collect()\n",
    "store_types = train_holiday_oil_store_transaction_item_test.select(\"store_type\").distinct().rdd.flatMap(lambda x: x).collect()\n",
    "clusters = train_holiday_oil_store_transaction_item_test.select(\"cluster\").distinct().rdd.flatMap(lambda x: x).collect()\n",
    "cities_expr = [F.when(F.col(\"city\") == city, 1).otherwise(0).alias(\"city_\" + city) for city in cities]\n",
    "#store_types_expr = [F.when(F.col(\"store_type\") == tp, 1).otherwise(0).alias(\"store_type_\" + tp) for tp in store_types]\n",
    "clusters_expr = [F.when(F.col(\"cluster\") == cluster, 1).otherwise(0).alias(\"cluster_\" + cluster) for cluster in clusters]\n",
    "stores_df = train_holiday_oil_store_transaction_item_test.select(\"store_nbr\", *cities_expr+clusters_expr)\n",
    "stores_df.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deal with holiday_event dataframe:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#fill NaN value in transferred column:\n",
    "train_holiday_oil_store_transaction_item_test_002 = train_holiday_oil_store_transaction_item_test.fillna('False', subset=['transferred', 'type', 'locale', 'locale_name', 'description'])\n",
    "#train_holiday_oil_store_transaction_item_test_002.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#encode holiday_event and create a new dataframe for it\n",
    "train_holiday_oil_store_transaction_item_test_003 = train_holiday_oil_store_transaction_item_test_002.filter(train_holiday_oil_store_transaction_item_test_002['transferred'] == 'False')\n",
    "#train_holiday_oil_store_transaction_item_test_003.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#train_holiday_oil_store_transaction_item_test_003[train_holiday_oil_store_transaction_item_test_003['transferred'] == 'True'].show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----------+----------+------------+-------------+-----------+---------------+-------------+-------------------+------------------+--------------------+--------------------+-------------------------+-----------------+----------------+-----------------+-------------------+-----------------+---------------------+---------------------+----------------+-----------------------+--------------------+------------------+---------------------+------------------+-------------------+-------------------+-------------------+--------------------+------------------------------------------+--------------------+----------------------+\n",
      "|      date|type_False|type_Event|type_Holiday|type_Transfer|type_Bridge|type_Additional|type_Work Day|locale_name_Quevedo|locale_name_Cuenca|locale_name_Cotopaxi|locale_name_Guaranda|locale_name_Santo Domingo|locale_name_False|locale_name_Puyo|locale_name_Quito|locale_name_Ecuador|locale_name_Manta|locale_name_Latacunga|locale_name_Guayaquil|locale_name_Loja|locale_name_Santa Elena|locale_name_Imbabura|locale_name_Ibarra|locale_name_El Carmen|locale_name_Ambato|locale_name_Machala|locale_name_Cayambe|locale_name_Salinas|locale_name_Libertad|locale_name_Santo Domingo de los Tsachilas|locale_name_Riobamba|locale_name_Esmeraldas|\n",
      "+----------+----------+----------+------------+-------------+-----------+---------------+-------------+-------------------+------------------+--------------------+--------------------+-------------------------+-----------------+----------------+-----------------+-------------------+-----------------+---------------------+---------------------+----------------+-----------------------+--------------------+------------------+---------------------+------------------+-------------------+-------------------+-------------------+--------------------+------------------------------------------+--------------------+----------------------+\n",
      "|2013-02-10|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2013-10-11|         0|         0|           0|            1|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                0|               0|                0|                  1|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2013-12-07|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2014-02-02|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2015-02-01|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2015-05-02|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2016-02-18|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2016-06-20|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2016-08-07|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2016-09-27|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2017-02-28|         0|         0|           1|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                0|               0|                0|                  1|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2017-03-05|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2017-06-15|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2017-07-06|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2017-08-16|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2017-08-24|         0|         0|           1|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                0|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 1|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2017-08-25|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2017-08-31|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2013-07-15|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "|2015-05-19|         1|         0|           0|            0|          0|              0|            0|                  0|                 0|                   0|                   0|                        0|                1|               0|                0|                  0|                0|                    0|                    0|               0|                      0|                   0|                 0|                    0|                 0|                  0|                  0|                  0|                   0|                                         0|                   0|                     0|\n",
      "+----------+----------+----------+------------+-------------+-----------+---------------+-------------+-------------------+------------------+--------------------+--------------------+-------------------------+-----------------+----------------+-----------------+-------------------+-----------------+---------------------+---------------------+----------------+-----------------------+--------------------+------------------+---------------------+------------------+-------------------+-------------------+-------------------+--------------------+------------------------------------------+--------------------+----------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "types = train_holiday_oil_store_transaction_item_test_003.select(\"type\").distinct().rdd.flatMap(lambda x: x).collect()\n",
    "locale_names = train_holiday_oil_store_transaction_item_test_003.select(\"locale_name\").distinct().rdd.flatMap(lambda x: x).collect()\n",
    "types_expr = [F.when(F.col(\"type\") == ty, 1).otherwise(0).alias(\"type_\" + ty) for ty in types]\n",
    "locale_names_expr = [F.when(F.col(\"locale_name\") == locale_name, 1).otherwise(0).alias(\"locale_name_\" + locale_name) for locale_name in locale_names]\n",
    "holiday_events_df = train_holiday_oil_store_transaction_item_test_003.select(\"date\", *types_expr+locale_names_expr)\n",
    "holiday_events_df.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine all dataframes together, then drop original columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#combine train_holiday_oil_store_transaction_item_test_003 with item\n",
    "train_holiday_oil_store_transaction_item_test_004 = train_holiday_oil_store_transaction_item_test_003.join(item_families, 'item_nbr', 'left_outer')\n",
    "#train_holiday_oil_store_transaction_item_test_004.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#train_holiday_oil_store_transaction_item_test_004.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+----------+---------+-------+-----------+------+-----+------+-----------+-----------+-----------+----------+---------+------+----------+-------+------------+----------+--------------------+-------------------------+----------------+---------------------+---------------+---------------+-------------+----------------+---------------+---------------+------------------+-----------+--------------+-----------------+----------------+-----------+---------------+-------------+----------------------+------------------------+----------+-----------------------------+-------------+------------------+--------------------------------+-------------------+--------------+--------------+------------------+------------+----------+---------------------+-----------+\n",
      "|item_nbr|      date|store_nbr|     id|onpromotion|source| type|locale|locale_name|description|transferred|dcoilwtico|     city| state|store_type|cluster|transactions|perishable|familyPREPARED FOODS|familyHOME AND KITCHEN II|familyLADIESWEAR|familyLAWN AND GARDEN|familyGROCERY I|familyBABY CARE|familyPRODUCE|familyAUTOMOTIVE|familyBEVERAGES|familyHOME CARE|familyBREAD/BAKERY|familyBOOKS|familyLINGERIE|familyCELEBRATION|familyGROCERY II|familyDAIRY|familyMAGAZINES|familySEAFOOD|familyLIQUOR,WINE,BEER|familyHOME AND KITCHEN I|familyDELI|familyPLAYERS AND ELECTRONICS|familyPOULTRY|familyFROZEN FOODS|familySCHOOL AND OFFICE SUPPLIES|familyPERSONAL CARE|familyCLEANING|familyHARDWARE|familyPET SUPPLIES|familyBEAUTY|familyEGGS|familyHOME APPLIANCES|familyMEATS|\n",
      "+--------+----------+---------+-------+-----------+------+-----+------+-----------+-----------+-----------+----------+---------+------+----------+-------+------------+----------+--------------------+-------------------------+----------------+---------------------+---------------+---------------+-------------+----------------+---------------+---------------+------------------+-----------+--------------+-----------------+----------------+-----------+---------------+-------------+----------------------+------------------------+----------+-----------------------------+-------------+------------------+--------------------------------+-------------------+--------------+--------------+------------------+------------+----------+---------------------+-----------+\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "| 1040170|2013-02-10|       51|1631577|      False| train|False| False|      False|      False|      False|      null|Guayaquil|Guayas|         A|     17|        1297|         0|                   0|                        0|               0|                    0|              0|              0|            0|               0|              0|              0|                 0|          0|             0|                0|               0|          0|              0|            0|                     0|                       0|         0|                            0|            0|                 1|                               0|                  0|             0|             0|                 0|           0|         0|                    0|          0|\n",
      "+--------+----------+---------+-------+-----------+------+-----+------+-----------+-----------+-----------+----------+---------+------+----------+-------+------------+----------+--------------------+-------------------------+----------------+---------------------+---------------+---------------+-------------+----------------+---------------+---------------+------------------+-----------+--------------+-----------------+----------------+-----------+---------------+-------------+----------------------+------------------------+----------+-----------------------------+-------------+------------------+--------------------------------+-------------------+--------------+--------------+------------------+------------+----------+---------------------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#drop original columns\n",
    "train_holiday_oil_store_transaction_item_test_004 = train_holiday_oil_store_transaction_item_test_004.drop('family', 'class')\n",
    "train_holiday_oil_store_transaction_item_test_004.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_holiday_oil_store_transaction_item_test_004 = train_holiday_oil_store_transaction_item_test_004.join(item_onpromotion, 'item_nbr', 'left_outer')\n",
    "#train_holiday_oil_store_transaction_item_test_004.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_holiday_oil_store_transaction_item_test_004 = train_holiday_oil_store_transaction_item_test_004.drop('onpromotion')\n",
    "#train_holiday_oil_store_transaction_item_test_004.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_holiday_oil_store_transaction_item_test_004 = train_holiday_oil_store_transaction_item_test_004.join(stores_df, 'store_nbr', 'left_outer')\n",
    "#train_holiday_oil_store_transaction_item_test_004.show()\n",
    "#trying to join on columns:\n",
    "# train_holiday_oil_store_transaction_item_test_004 = train_holiday_oil_store_transaction_item_test_004.repartition(\"store_nbr\")\n",
    "# stores_df = stores_df.repartition(\"store_nbr\")\n",
    "# train_holiday_oil_store_transaction_item_test_004 = train_holiday_oil_store_transaction_item_test_004.join(stores_df, 'store_nbr', 'left_outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_holiday_oil_store_transaction_item_test_004 = train_holiday_oil_store_transaction_item_test_004.drop('city', 'state', 'store_type', 'cluster')\n",
    "#train_holiday_oil_store_transaction_item_test_004.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_holiday_oil_store_transaction_item_test_004 = train_holiday_oil_store_transaction_item_test_004.join(holiday_events_df, 'date', 'left_outer')\n",
    "#train_holiday_oil_store_transaction_item_test_004.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "ename": "Py4JJavaError",
     "evalue": "An error occurred while calling o788.showString.\n: org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 187.0 failed 1 times, most recent failure: Lost task 2.0 in stage 187.0 (TID 13757, localhost, executor driver): java.io.IOException: No space left on device\n\tat java.io.FileOutputStream.writeBytes(Native Method)\n\tat java.io.FileOutputStream.write(FileOutputStream.java:326)\n\tat org.apache.spark.storage.TimeTrackingOutputStream.write(TimeTrackingOutputStream.java:58)\n\tat java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)\n\tat java.io.BufferedOutputStream.write(BufferedOutputStream.java:126)\n\tat net.jpountz.lz4.LZ4BlockOutputStream.flushBufferedData(LZ4BlockOutputStream.java:205)\n\tat net.jpountz.lz4.LZ4BlockOutputStream.write(LZ4BlockOutputStream.java:158)\n\tat java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)\n\tat java.io.BufferedOutputStream.write(BufferedOutputStream.java:126)\n\tat java.io.DataOutputStream.write(DataOutputStream.java:107)\n\tat org.apache.spark.sql.catalyst.expressions.UnsafeRow.writeToStream(UnsafeRow.java:554)\n\tat org.apache.spark.sql.execution.UnsafeRowSerializerInstance$$anon$1.writeValue(UnsafeRowSerializer.scala:69)\n\tat org.apache.spark.storage.DiskBlockObjectWriter.write(DiskBlockObjectWriter.scala:239)\n\tat org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:151)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:108)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:335)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1499)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1487)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1486)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1486)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:814)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:814)\n\tat scala.Option.foreach(Option.scala:257)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:814)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1714)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1669)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1658)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)\n\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:630)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2022)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2043)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2062)\n\tat org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:336)\n\tat org.apache.spark.sql.execution.CollectLimitExec.executeCollect(limit.scala:38)\n\tat org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:2853)\n\tat org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2153)\n\tat org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2153)\n\tat org.apache.spark.sql.Dataset$$anonfun$55.apply(Dataset.scala:2837)\n\tat org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:65)\n\tat org.apache.spark.sql.Dataset.withAction(Dataset.scala:2836)\n\tat org.apache.spark.sql.Dataset.head(Dataset.scala:2153)\n\tat org.apache.spark.sql.Dataset.take(Dataset.scala:2366)\n\tat org.apache.spark.sql.Dataset.showString(Dataset.scala:245)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n\tat py4j.Gateway.invoke(Gateway.java:280)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:214)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: java.io.IOException: No space left on device\n\tat java.io.FileOutputStream.writeBytes(Native Method)\n\tat java.io.FileOutputStream.write(FileOutputStream.java:326)\n\tat org.apache.spark.storage.TimeTrackingOutputStream.write(TimeTrackingOutputStream.java:58)\n\tat java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)\n\tat java.io.BufferedOutputStream.write(BufferedOutputStream.java:126)\n\tat net.jpountz.lz4.LZ4BlockOutputStream.flushBufferedData(LZ4BlockOutputStream.java:205)\n\tat net.jpountz.lz4.LZ4BlockOutputStream.write(LZ4BlockOutputStream.java:158)\n\tat java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)\n\tat java.io.BufferedOutputStream.write(BufferedOutputStream.java:126)\n\tat java.io.DataOutputStream.write(DataOutputStream.java:107)\n\tat org.apache.spark.sql.catalyst.expressions.UnsafeRow.writeToStream(UnsafeRow.java:554)\n\tat org.apache.spark.sql.execution.UnsafeRowSerializerInstance$$anon$1.writeValue(UnsafeRowSerializer.scala:69)\n\tat org.apache.spark.storage.DiskBlockObjectWriter.write(DiskBlockObjectWriter.scala:239)\n\tat org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:151)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:108)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:335)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\t... 1 more\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mPy4JJavaError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-43-d940918c3fe6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mtrain_holiday_oil_store_transaction_item_test_004\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_holiday_oil_store_transaction_item_test_004\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'type'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'locale'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'locale_name'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'description'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'transferred'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtrain_holiday_oil_store_transaction_item_test_004\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/opt/apache-spark/libexec/python/pyspark/sql/dataframe.py\u001b[0m in \u001b[0;36mshow\u001b[0;34m(self, n, truncate)\u001b[0m\n\u001b[1;32m    334\u001b[0m         \"\"\"\n\u001b[1;32m    335\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtruncate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mtruncate\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 336\u001b[0;31m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshowString\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m20\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    337\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    338\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshowString\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtruncate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/opt/apache-spark/libexec/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m   1131\u001b[0m         \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1132\u001b[0m         return_value = get_return_value(\n\u001b[0;32m-> 1133\u001b[0;31m             answer, self.gateway_client, self.target_id, self.name)\n\u001b[0m\u001b[1;32m   1134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1135\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mtemp_arg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtemp_args\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/opt/apache-spark/libexec/python/pyspark/sql/utils.py\u001b[0m in \u001b[0;36mdeco\u001b[0;34m(*a, **kw)\u001b[0m\n\u001b[1;32m     61\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mdeco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     64\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mpy4j\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPy4JJavaError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m             \u001b[0ms\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjava_exception\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoString\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/opt/apache-spark/libexec/python/lib/py4j-0.10.4-src.zip/py4j/protocol.py\u001b[0m in \u001b[0;36mget_return_value\u001b[0;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[1;32m    317\u001b[0m                 raise Py4JJavaError(\n\u001b[1;32m    318\u001b[0m                     \u001b[0;34m\"An error occurred while calling {0}{1}{2}.\\n\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m                     format(target_id, \".\", name), value)\n\u001b[0m\u001b[1;32m    320\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    321\u001b[0m                 raise Py4JError(\n",
      "\u001b[0;31mPy4JJavaError\u001b[0m: An error occurred while calling o788.showString.\n: org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 187.0 failed 1 times, most recent failure: Lost task 2.0 in stage 187.0 (TID 13757, localhost, executor driver): java.io.IOException: No space left on device\n\tat java.io.FileOutputStream.writeBytes(Native Method)\n\tat java.io.FileOutputStream.write(FileOutputStream.java:326)\n\tat org.apache.spark.storage.TimeTrackingOutputStream.write(TimeTrackingOutputStream.java:58)\n\tat java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)\n\tat java.io.BufferedOutputStream.write(BufferedOutputStream.java:126)\n\tat net.jpountz.lz4.LZ4BlockOutputStream.flushBufferedData(LZ4BlockOutputStream.java:205)\n\tat net.jpountz.lz4.LZ4BlockOutputStream.write(LZ4BlockOutputStream.java:158)\n\tat java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)\n\tat java.io.BufferedOutputStream.write(BufferedOutputStream.java:126)\n\tat java.io.DataOutputStream.write(DataOutputStream.java:107)\n\tat org.apache.spark.sql.catalyst.expressions.UnsafeRow.writeToStream(UnsafeRow.java:554)\n\tat org.apache.spark.sql.execution.UnsafeRowSerializerInstance$$anon$1.writeValue(UnsafeRowSerializer.scala:69)\n\tat org.apache.spark.storage.DiskBlockObjectWriter.write(DiskBlockObjectWriter.scala:239)\n\tat org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:151)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:108)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:335)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1499)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1487)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1486)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1486)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:814)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:814)\n\tat scala.Option.foreach(Option.scala:257)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:814)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1714)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1669)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1658)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)\n\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:630)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2022)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2043)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2062)\n\tat org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:336)\n\tat org.apache.spark.sql.execution.CollectLimitExec.executeCollect(limit.scala:38)\n\tat org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:2853)\n\tat org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2153)\n\tat org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2153)\n\tat org.apache.spark.sql.Dataset$$anonfun$55.apply(Dataset.scala:2837)\n\tat org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:65)\n\tat org.apache.spark.sql.Dataset.withAction(Dataset.scala:2836)\n\tat org.apache.spark.sql.Dataset.head(Dataset.scala:2153)\n\tat org.apache.spark.sql.Dataset.take(Dataset.scala:2366)\n\tat org.apache.spark.sql.Dataset.showString(Dataset.scala:245)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n\tat py4j.Gateway.invoke(Gateway.java:280)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:214)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: java.io.IOException: No space left on device\n\tat java.io.FileOutputStream.writeBytes(Native Method)\n\tat java.io.FileOutputStream.write(FileOutputStream.java:326)\n\tat org.apache.spark.storage.TimeTrackingOutputStream.write(TimeTrackingOutputStream.java:58)\n\tat java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)\n\tat java.io.BufferedOutputStream.write(BufferedOutputStream.java:126)\n\tat net.jpountz.lz4.LZ4BlockOutputStream.flushBufferedData(LZ4BlockOutputStream.java:205)\n\tat net.jpountz.lz4.LZ4BlockOutputStream.write(LZ4BlockOutputStream.java:158)\n\tat java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)\n\tat java.io.BufferedOutputStream.write(BufferedOutputStream.java:126)\n\tat java.io.DataOutputStream.write(DataOutputStream.java:107)\n\tat org.apache.spark.sql.catalyst.expressions.UnsafeRow.writeToStream(UnsafeRow.java:554)\n\tat org.apache.spark.sql.execution.UnsafeRowSerializerInstance$$anon$1.writeValue(UnsafeRowSerializer.scala:69)\n\tat org.apache.spark.storage.DiskBlockObjectWriter.write(DiskBlockObjectWriter.scala:239)\n\tat org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:151)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:108)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:335)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\t... 1 more\n"
     ]
    }
   ],
   "source": [
    "train_holiday_oil_store_transaction_item_test_004 = train_holiday_oil_store_transaction_item_test_004.drop('type', 'locale', 'locale_name', 'description', 'transferred')\n",
    "train_holiday_oil_store_transaction_item_test_004.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#train_holiday_oil_store_transaction_item_test_004.rdd.partitions.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stores_df.rdd.getNumPartitions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}