PySpark Tutorial 3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ea31566c",
   "metadata": {},
   "source": [
    "# Pyspark Dataframe Handling Missing Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "d39839f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#import paskages\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.ml.feature import Imputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c2d56c5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "spark = SparkSession.builder.appName(\"practise\").getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6146d83e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#load data\n",
    "df_pyspark = spark.read.csv(\"text3.csv\" , header=True , inferSchema=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d1c8953a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----+----------+------+\n",
      "|     Name| age|Experience|Salary|\n",
      "+---------+----+----------+------+\n",
      "|    Krish|  31|        10| 30000|\n",
      "|Sudhanshu|  30|         8| 25000|\n",
      "|    Sunny|  29|         4| 20000|\n",
      "|     Paul|  24|         3| 20000|\n",
      "|   Harsha|  21|         1| 15000|\n",
      "|  Shubham|  23|         2| 18000|\n",
      "|   Mahesh|NULL|      NULL| 40000|\n",
      "|     NULL|  34|        10| 38000|\n",
      "|     NULL|  36|      NULL|  NULL|\n",
      "+---------+----+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#show data \n",
    "df_pyspark.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "99e9755d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+----------+------+\n",
      "| age|Experience|Salary|\n",
      "+----+----------+------+\n",
      "|  31|        10| 30000|\n",
      "|  30|         8| 25000|\n",
      "|  29|         4| 20000|\n",
      "|  24|         3| 20000|\n",
      "|  21|         1| 15000|\n",
      "|  23|         2| 18000|\n",
      "|NULL|      NULL| 40000|\n",
      "|  34|        10| 38000|\n",
      "|  36|      NULL|  NULL|\n",
      "+----+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#deop columns Name\n",
    "df_pyspark.drop('Name').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "8bda4afe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----+----------+------+\n",
      "|     Name| age|Experience|Salary|\n",
      "+---------+----+----------+------+\n",
      "|    Krish|  31|        10| 30000|\n",
      "|Sudhanshu|  30|         8| 25000|\n",
      "|    Sunny|  29|         4| 20000|\n",
      "|     Paul|  24|         3| 20000|\n",
      "|   Harsha|  21|         1| 15000|\n",
      "|  Shubham|  23|         2| 18000|\n",
      "|   Mahesh|NULL|      NULL| 40000|\n",
      "|     NULL|  34|        10| 38000|\n",
      "|     NULL|  36|      NULL|  NULL|\n",
      "+---------+----+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#show data \n",
    "df_pyspark.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "902b0d24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+------+\n",
      "|     Name|age|Experience|Salary|\n",
      "+---------+---+----------+------+\n",
      "|    Krish| 31|        10| 30000|\n",
      "|Sudhanshu| 30|         8| 25000|\n",
      "|    Sunny| 29|         4| 20000|\n",
      "|     Paul| 24|         3| 20000|\n",
      "|   Harsha| 21|         1| 15000|\n",
      "|  Shubham| 23|         2| 18000|\n",
      "+---------+---+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#drop rows it null or missing\n",
    "df_pyspark.na.drop().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "165e0204",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----+----------+------+\n",
      "|     Name| age|Experience|Salary|\n",
      "+---------+----+----------+------+\n",
      "|    Krish|  31|        10| 30000|\n",
      "|Sudhanshu|  30|         8| 25000|\n",
      "|    Sunny|  29|         4| 20000|\n",
      "|     Paul|  24|         3| 20000|\n",
      "|   Harsha|  21|         1| 15000|\n",
      "|  Shubham|  23|         2| 18000|\n",
      "|   Mahesh|NULL|      NULL| 40000|\n",
      "|     NULL|  34|        10| 38000|\n",
      "|     NULL|  36|      NULL|  NULL|\n",
      "+---------+----+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#drops rows where all values are null\n",
    "df_pyspark.na.drop(how='all').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "71b589d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+------+\n",
      "|     Name|age|Experience|Salary|\n",
      "+---------+---+----------+------+\n",
      "|    Krish| 31|        10| 30000|\n",
      "|Sudhanshu| 30|         8| 25000|\n",
      "|    Sunny| 29|         4| 20000|\n",
      "|     Paul| 24|         3| 20000|\n",
      "|   Harsha| 21|         1| 15000|\n",
      "|  Shubham| 23|         2| 18000|\n",
      "+---------+---+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# drops rows where any value is null\n",
    "# It removes rows if there is at least one null value in any column\n",
    "df_pyspark.na.drop(how='any').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4344b6da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+------+\n",
      "|     Name|age|Experience|Salary|\n",
      "+---------+---+----------+------+\n",
      "|    Krish| 31|        10| 30000|\n",
      "|Sudhanshu| 30|         8| 25000|\n",
      "|    Sunny| 29|         4| 20000|\n",
      "|     Paul| 24|         3| 20000|\n",
      "|   Harsha| 21|         1| 15000|\n",
      "|  Shubham| 23|         2| 18000|\n",
      "|     NULL| 34|        10| 38000|\n",
      "+---------+---+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#threshold\n",
    "#detect  minimum number of non-null values \n",
    "df_pyspark.na.drop(how='any',thresh=3).show() #3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "a703a34e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----+----------+------+\n",
      "|     Name| age|Experience|Salary|\n",
      "+---------+----+----------+------+\n",
      "|    Krish|  31|        10| 30000|\n",
      "|Sudhanshu|  30|         8| 25000|\n",
      "|    Sunny|  29|         4| 20000|\n",
      "|     Paul|  24|         3| 20000|\n",
      "|   Harsha|  21|         1| 15000|\n",
      "|  Shubham|  23|         2| 18000|\n",
      "|   Mahesh|NULL|      NULL| 40000|\n",
      "|     NULL|  34|        10| 38000|\n",
      "+---------+----+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#threshold\n",
    "#detect  minimum number of non-null values \n",
    "df_pyspark.na.drop(how='any',thresh=2).show()#4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17c3f4a4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "35389ce2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----+----------+------+\n",
      "|     Name| age|Experience|Salary|\n",
      "+---------+----+----------+------+\n",
      "|    Krish|  31|        10| 30000|\n",
      "|Sudhanshu|  30|         8| 25000|\n",
      "|    Sunny|  29|         4| 20000|\n",
      "|     Paul|  24|         3| 20000|\n",
      "|   Harsha|  21|         1| 15000|\n",
      "|  Shubham|  23|         2| 18000|\n",
      "|   Mahesh|NULL|      NULL| 40000|\n",
      "|     NULL|  34|        10| 38000|\n",
      "+---------+----+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#subset\n",
    "#specifically drops rows where the value in the 'Salary' column is null\n",
    "df_pyspark.na.drop(how='any' , subset=['Salary']).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfb0a9fb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "21f4e369",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+----+----------+------+\n",
      "|          Name| age|Experience|Salary|\n",
      "+--------------+----+----------+------+\n",
      "|         Krish|  31|        10| 30000|\n",
      "|     Sudhanshu|  30|         8| 25000|\n",
      "|         Sunny|  29|         4| 20000|\n",
      "|          Paul|  24|         3| 20000|\n",
      "|        Harsha|  21|         1| 15000|\n",
      "|       Shubham|  23|         2| 18000|\n",
      "|        Mahesh|NULL|      NULL| 40000|\n",
      "|Missing values|  34|        10| 38000|\n",
      "|Missing values|  36|      NULL|  NULL|\n",
      "+--------------+----+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#fill missing values\n",
    "df_pyspark.na.fill(\"Missing values\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "862c6317",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+----+----------+------+\n",
      "|          Name| age|Experience|Salary|\n",
      "+--------------+----+----------+------+\n",
      "|         Krish|  31|        10| 30000|\n",
      "|     Sudhanshu|  30|         8| 25000|\n",
      "|         Sunny|  29|         4| 20000|\n",
      "|          Paul|  24|         3| 20000|\n",
      "|        Harsha|  21|         1| 15000|\n",
      "|       Shubham|  23|         2| 18000|\n",
      "|        Mahesh|NULL|      NULL| 40000|\n",
      "|Missing Values|  34|        10| 38000|\n",
      "|Missing Values|  36|      NULL|  NULL|\n",
      "+--------------+----+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#fill missing values detct columns age, salary\n",
    "df_pyspark.na.fill(\"Missing Values\", ['Name','Salary']).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa86a682",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "315aa5e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----+----------+------+\n",
      "|     Name| age|Experience|Salary|\n",
      "+---------+----+----------+------+\n",
      "|    Krish|  31|        10| 30000|\n",
      "|Sudhanshu|  30|         8| 25000|\n",
      "|    Sunny|  29|         4| 20000|\n",
      "|     Paul|  24|         3| 20000|\n",
      "|   Harsha|  21|         1| 15000|\n",
      "|  Shubham|  23|         2| 18000|\n",
      "|   Mahesh|NULL|      NULL| 40000|\n",
      "|     NULL|  34|        10| 38000|\n",
      "|     NULL|  36|      NULL|  NULL|\n",
      "+---------+----+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#show data \n",
    "df_pyspark.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94d8da68",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "bd7aac25",
   "metadata": {},
   "source": [
    "###### Fill missing data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "6c55daf8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#used for handling missing values in a Spark DataFrame.\n",
    "from pyspark.ml.feature import Imputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "d9f6bce3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create an Imputer \n",
    "#add new columns it  imputes wih mean\n",
    "imputer=Imputer(\n",
    "            inputCols=['age','Experience','Salary'],\n",
    "            outputCols=[\"{}_imputes\".format(c) for c in ['age','Experience','Salary']]\n",
    "            ).setStrategy(\"mean\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "31542cbb",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----+----------+------+-----------+------------------+--------------+\n",
      "|     Name| age|Experience|Salary|age_imputes|Experience_imputes|Salary_imputes|\n",
      "+---------+----+----------+------+-----------+------------------+--------------+\n",
      "|    Krish|  31|        10| 30000|         31|                10|         30000|\n",
      "|Sudhanshu|  30|         8| 25000|         30|                 8|         25000|\n",
      "|    Sunny|  29|         4| 20000|         29|                 4|         20000|\n",
      "|     Paul|  24|         3| 20000|         24|                 3|         20000|\n",
      "|   Harsha|  21|         1| 15000|         21|                 1|         15000|\n",
      "|  Shubham|  23|         2| 18000|         23|                 2|         18000|\n",
      "|   Mahesh|NULL|      NULL| 40000|         28|                 5|         40000|\n",
      "|     NULL|  34|        10| 38000|         34|                10|         38000|\n",
      "|     NULL|  36|      NULL|  NULL|         36|                 5|         25750|\n",
      "+---------+----+----------+------+-----------+------------------+--------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# fit the Imputer and transform the DataFrame\n",
    "imputer.fit(df_pyspark).transform(df_pyspark).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18385643",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c725af5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "b1f92b98",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create an Imputer \n",
    "#add new columns it  imputes wih Median\n",
    "imputer=Imputer(\n",
    "            inputCols=['age','Experience','Salary'],\n",
    "            outputCols=[\"{}_imputes\".format(c) for c in ['age','Experience','Salary']]\n",
    "            ).setStrategy(\"mean\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "708743ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----+----------+------+-----------+------------------+--------------+\n",
      "|     Name| age|Experience|Salary|age_imputes|Experience_imputes|Salary_imputes|\n",
      "+---------+----+----------+------+-----------+------------------+--------------+\n",
      "|    Krish|  31|        10| 30000|         31|                10|         30000|\n",
      "|Sudhanshu|  30|         8| 25000|         30|                 8|         25000|\n",
      "|    Sunny|  29|         4| 20000|         29|                 4|         20000|\n",
      "|     Paul|  24|         3| 20000|         24|                 3|         20000|\n",
      "|   Harsha|  21|         1| 15000|         21|                 1|         15000|\n",
      "|  Shubham|  23|         2| 18000|         23|                 2|         18000|\n",
      "|   Mahesh|NULL|      NULL| 40000|         28|                 5|         40000|\n",
      "|     NULL|  34|        10| 38000|         34|                10|         38000|\n",
      "|     NULL|  36|      NULL|  NULL|         36|                 5|         25750|\n",
      "+---------+----+----------+------+-----------+------------------+--------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# fit the Imputer and transform the DataFrame\n",
    "imputer.fit(df_pyspark).transform(df_pyspark).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6030fc9a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eda9760c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e28d880c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "a6ad3033",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+------+\n",
      "|     Name|age|Experience|Salary|\n",
      "+---------+---+----------+------+\n",
      "|    Krish| 31|        10| 30000|\n",
      "|Sudhanshu| 30|         8| 25000|\n",
      "|    Sunny| 29|         4| 20000|\n",
      "|     Paul| 24|         3| 20000|\n",
      "|   Harsha| 21|         1| 15000|\n",
      "|  Shubham| 23|         2| 18000|\n",
      "|   Mahesh| 28|         5| 40000|\n",
      "|     NULL| 34|        10| 38000|\n",
      "|     NULL| 36|         5| 25750|\n",
      "+---------+---+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#imputes on same columns \n",
    "imputer=Imputer(\n",
    "            inputCols=['age','Experience','Salary'],\n",
    "            outputCols=['age','Experience','Salary']\n",
    "            ).setStrategy(\"mean\")\n",
    "\n",
    "# fit the Imputer and transform the DataFrame\n",
    "imputer.fit(df_pyspark).transform(df_pyspark).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfe61b19",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e035bdeb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f26ec66",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}