02_employee_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8fe33bd6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "22/02/12 10:20:09 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "22/02/12 10:20:11 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n"
     ]
    }
   ],
   "source": [
    "# import classes for python application\n",
    "from pyspark.sql import SparkSession, DataFrame, functions, Row\n",
    "from pyspark import SparkContext, SparkConf\n",
    "from pyspark.sql.types import *\n",
    "\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import pyarrow as pa\n",
    "import pyarrow.parquet as pq\n",
    "os.environ['PYSPARK_PYTHON'] = sys.executable\n",
    "os.environ['PYSPARK_DRIVER_PYTHON'] = sys.executable\n",
    "\n",
    "# create spark session, configuration and context\n",
    "spark = SparkSession.builder.appName(\"DataFrame\").getOrCreate()\n",
    "conf = SparkConf().setMaster(\"local\").setAppName(\"Assignment 2\")\n",
    "sc = SparkContext.getOrCreate(conf=conf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c46bd6fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 0:>                                                          (0 + 1) / 1]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n",
      "|               value|\n",
      "+--------------------+\n",
      "|Michael, 29, 3000.30|\n",
      "|   Andy, 30, 2500.25|\n",
      "| Justin, 19, 4000.99|\n",
      "+--------------------+\n",
      "\n",
      "root\n",
      " |-- value: string (nullable = true)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "# load text data and create dataframe\n",
    "df1 = spark.read.text('emps.txt')\n",
    "#show schema\n",
    "df1.show()\n",
    "#print schema\n",
    "df1.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "595bfa6f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+---+-------+\n",
      "|   name|age| salary|\n",
      "+-------+---+-------+\n",
      "|Michael| 29| 3000.3|\n",
      "|   Andy| 30|2500.25|\n",
      "| Justin| 19|4000.99|\n",
      "+-------+---+-------+\n",
      "\n",
      "root\n",
      " |-- name: string (nullable = true)\n",
      " |-- age: integer (nullable = true)\n",
      " |-- salary: float (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#load text data \n",
    "emps_rdd = sc.textFile('emps.txt')\n",
    "#transform data to tuple for dataframe\n",
    "emps = emps_rdd.map(lambda e: tuple(e.split(\",\")))\n",
    "\n",
    "# create spark dataframe and schema during import\n",
    "df2 = spark.createDataFrame(emps, schema=['name', 'age', 'salary'])\n",
    "# update types for columns\n",
    "df2 = df2.withColumn(\"age\", df2.age.cast(IntegerType()))\n",
    "df2 = df2.withColumn(\"salary\", df2.salary.cast(FloatType()))\n",
    "#show schema\n",
    "df2.show()\n",
    "#print schema\n",
    "df2.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "79fda274",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+---+-------+\n",
      "|   name|age| salary|\n",
      "+-------+---+-------+\n",
      "|Michael| 29| 3000.3|\n",
      "|   Andy| 30|2500.25|\n",
      "| Justin| 19|4000.99|\n",
      "+-------+---+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#select content of new df\n",
    "df2.select(\"*\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d45e1857",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 4:>                                                          (0 + 1) / 1]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------+-----------------+------------------+\n",
      "|summary|   name|              age|            salary|\n",
      "+-------+-------+-----------------+------------------+\n",
      "|  count|      3|                3|                 3|\n",
      "|   mean|   null|             26.0|3167.1800130208335|\n",
      "| stddev|   null|6.082762530298219| 764.1608609506821|\n",
      "|    min|   Andy|               19|           2500.25|\n",
      "|    max|Michael|               30|           4000.99|\n",
      "+-------+-------+-----------------+------------------+\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "#display all details \n",
    "df2.select(\"*\").describe().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0511e33a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+---+-------+\n",
      "|   name|age| salary|\n",
      "+-------+---+-------+\n",
      "|Michael| 29| 3000.3|\n",
      "|   Andy| 30|2500.25|\n",
      "| Justin| 19|4000.99|\n",
      "+-------+---+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#transform data into Global temp table and show table\n",
    "table = pa.Table.from_pandas(df2.toPandas())\n",
    "pq.write_table(table, 'df.parquet')\n",
    "df_parquet = spark.read.parquet('df.parquet')\n",
    "df_parquet.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "43df9979",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+---+-------+\n",
      "|  name|age| salary|\n",
      "+------+---+-------+\n",
      "|Justin| 19|4000.99|\n",
      "+------+---+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# select names of employees with salary greater than 3500 using filter\n",
    "df_parquet.filter(df_parquet.salary > 3500).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e3a879a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+---------------+\n",
      "|   name|(salary > 3500)|\n",
      "+-------+---------------+\n",
      "|Michael|          false|\n",
      "|   Andy|          false|\n",
      "| Justin|           true|\n",
      "+-------+---------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#select names of employees with salary greater than 3500 using select\n",
    "df_parquet.select(df_parquet.name, df_parquet.salary > 3500).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "32315bb4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[age: bigint, name: string]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#parse json string and convert to pandas dataframe\n",
    "df3 = spark.read.format('json').load(\"employee.json\")\n",
    "df3 = df3.drop('id')\n",
    "df3 = df3.withColumn(\"age\", df3.age.cast('bigint'))\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6186a9b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------+--------+\n",
      "|numberOfEmployees|ageGroup|\n",
      "+-----------------+--------+\n",
      "|                1|      39|\n",
      "|                2|      33|\n",
      "|                1|      35|\n",
      "|                1|      38|\n",
      "+-----------------+--------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#group by age, select count using pyspark.sql functions, change names and show dataframe\n",
    "df3.groupBy('age')\\\n",
    "        .count()\\\n",
    "        .select(functions.col('count')\\\n",
    "        .alias('numberOfEmployees'), df3.age.alias('ageGroup')).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cb97bb54",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Michael, 29, 3000.30', 'Andy, 30, 2500.25', 'Justin, 19, 4000.99']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#load text in an RDD instance and view data\n",
    "result = emps_rdd.collect()\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "eba204b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# split lines into tuples and create rows for spark dataframe\n",
    "emps = emps_rdd.map(lambda e: tuple(e.split(\",\")))\\\n",
    "                .map(lambda e: Row(name=e[0], age=int(e[1]), salary=float(e[2])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1c18197e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+---+-------+\n",
      "|   name|age| salary|\n",
      "+-------+---+-------+\n",
      "|Michael| 29| 3000.3|\n",
      "|   Andy| 30|2500.25|\n",
      "| Justin| 19|4000.99|\n",
      "+-------+---+-------+\n",
      "\n",
      "root\n",
      " |-- name: string (nullable = true)\n",
      " |-- age: long (nullable = true)\n",
      " |-- salary: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# create and show spark dataframe and print schema\n",
    "emps_df = spark.createDataFrame(emps)\n",
    "emps_df.show()\n",
    "emps_df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "025f2d29",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+---+-------+\n",
      "|  name|age| salary|\n",
      "+------+---+-------+\n",
      "|Justin| 19|4000.99|\n",
      "+------+---+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# select names of employees with salary greater than 3500\n",
    "emps_df.filter(emps_df.salary > 3500).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "495af357",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+---+-------+\n",
      "|  name|age| salary|\n",
      "+------+---+-------+\n",
      "|Justin| 19|4000.99|\n",
      "+------+---+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#select names of employees with salary greater than 3500 with temporary table\n",
    "emps_df.createOrReplaceTempView(\"tableEmps\")\n",
    "spark.sql(\"SELECT * FROM tableEmps WHERE salary > 3500\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "013e9d49",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#full answers are in problem4_01.py and problem4_02.py script files\n",
    "\n",
    "emps_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "22ea78bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "emps_df.toPandas().to_csv('emps_update.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0e8a7740",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-------+---+-------+\n",
      "|_c0|   name|age| salary|\n",
      "+---+-------+---+-------+\n",
      "|  0|Michael| 29| 3000.3|\n",
      "|  1|   Andy| 30|2500.25|\n",
      "|  2| Justin| 19|4000.99|\n",
      "+---+-------+---+-------+\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "22/02/12 10:20:32 WARN CSVHeaderChecker: CSV header does not conform to the schema.\n",
      " Header: , name, age, salary\n",
      " Schema: _c0, name, age, salary\n",
      "Expected: _c0 but found: \n",
      "CSV file: file:///home/f_dev/big_data/emps_update.csv\n"
     ]
    }
   ],
   "source": [
    "spark.read.csv('emps_update.csv', header=True).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "53b01223",
   "metadata": {},
   "outputs": [],
   "source": [
    "#stop spark context and spark\n",
    "sc.stop()\n",
    "spark.stop()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}