PySpark Tutorial 4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f961de91",
   "metadata": {},
   "source": [
    "# Pyspark Dataframes Filter Operation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a5da0d0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#import paskages \n",
    "\n",
    "from pyspark.sql import SparkSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "daf8e46a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating a SparkSession \n",
    "spark=SparkSession.builder.appName(\"dataframe\").getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "841eba4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+------+\n",
      "|     Name|age|Experience|Salary|\n",
      "+---------+---+----------+------+\n",
      "|    Krish| 31|        10| 30000|\n",
      "|Sudhanshu| 30|         8| 25000|\n",
      "|    Sunny| 29|         4| 20000|\n",
      "|     Paul| 24|         3| 20000|\n",
      "|   Harsha| 21|         1| 15000|\n",
      "|  Shubham| 23|         2| 18000|\n",
      "+---------+---+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#read data\n",
    "df_pyspark=spark.read.csv('text4.csv', header=True , inferSchema=True)\n",
    "df_pyspark.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27247022",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "2aa45bfd",
   "metadata": {},
   "source": [
    "## Filter Operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "68472721",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+---+----------+------+\n",
      "|   Name|age|Experience|Salary|\n",
      "+-------+---+----------+------+\n",
      "|  Sunny| 29|         4| 20000|\n",
      "|   Paul| 24|         3| 20000|\n",
      "| Harsha| 21|         1| 15000|\n",
      "|Shubham| 23|         2| 18000|\n",
      "+-------+---+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#salary of people less than  or equal to 20000\n",
    "df_pyspark.filter(\"Salary<=20000\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db7e410f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "33bc8999",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+---+\n",
      "|   Name|age|\n",
      "+-------+---+\n",
      "|  Sunny| 29|\n",
      "|   Paul| 24|\n",
      "| Harsha| 21|\n",
      "|Shubham| 23|\n",
      "+-------+---+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#salary of people less than  or equal to 20000 and selects only the 'Name' and 'age' columns for display.\n",
    "\n",
    "df_pyspark.filter(\"Salary<=20000\").select(['Name',\"age\"]).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "696a1335",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+----------+\n",
      "|Salary|Experience|\n",
      "+------+----------+\n",
      "| 30000|        10|\n",
      "| 25000|         8|\n",
      "| 20000|         4|\n",
      "+------+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#filter age more 25 and show just Salary and Experience columns\n",
    "df_pyspark.filter(\"age>25\").select([\"Salary\",'Experience']).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "54914d6c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+---+----------+------+\n",
      "|   Name|age|Experience|Salary|\n",
      "+-------+---+----------+------+\n",
      "|  Sunny| 29|         4| 20000|\n",
      "|   Paul| 24|         3| 20000|\n",
      "| Harsha| 21|         1| 15000|\n",
      "|Shubham| 23|         2| 18000|\n",
      "+-------+---+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# filter rows based on a condition where the 'Salary' is between 15000 and 20000\n",
    "df_pyspark.filter((df_pyspark['Salary']<=20000   ) &\n",
    "                  (df_pyspark['Salary']>=15000)  ).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "74b8a19f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+------+\n",
      "|     Name|age|Experience|Salary|\n",
      "+---------+---+----------+------+\n",
      "|    Krish| 31|        10| 30000|\n",
      "|Sudhanshu| 30|         8| 25000|\n",
      "+---------+---+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#filters  based on two conditions: age greater than or equal to 25 and salary greater than or equal to 25000. \n",
    "df_pyspark.filter((df_pyspark['age']>=25) &\n",
    "                  (df_pyspark['Salary']>=25000)  ).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "faa9ac3e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "42613be5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+------+\n",
      "|     Name|age|Experience|Salary|\n",
      "+---------+---+----------+------+\n",
      "|    Krish| 31|        10| 30000|\n",
      "|Sudhanshu| 30|         8| 25000|\n",
      "|    Sunny| 29|         4| 20000|\n",
      "+---------+---+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#checks whether  'age' is not less than 25.\n",
    "df_pyspark.filter(~(df_pyspark[\"age\"]<25) ).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbffcfcd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e3608e90",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+------+\n",
      "|     Name|age|Experience|Salary|\n",
      "+---------+---+----------+------+\n",
      "|    Krish| 31|        10| 30000|\n",
      "|Sudhanshu| 30|         8| 25000|\n",
      "+---------+---+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#checks whether  'age' is not less than 25 or  'Salary' is not less than 25000.\n",
    "df_pyspark.filter( ~((df_pyspark['age']<25) | (df_pyspark['Salary']<25000)) ).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82d59e14",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "067d332f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7349f46c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79e1a741",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}