Vine_Review_Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Vine_Review_Analysis.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMTqousv/i5tucdVXEV0Fak"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aFVNBlHTcghw"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "gtihGvDBZmtM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8462a799-557a-4845-9e9c-b7c5a06ec871"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r0% [Working]\r            \rGet:1 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "\r0% [Connecting to archive.ubuntu.com] [1 InRelease 2,589 B/88.7 kB 3%] [Connect\r                                                                               \rGet:2 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]\n",
            "Hit:3 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Get:4 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease [15.9 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "Ign:6 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Ign:7 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Get:8 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release [696 B]\n",
            "Get:9 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Hit:10 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Get:11 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release.gpg [836 B]\n",
            "Hit:12 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "Get:13 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease [15.9 kB]\n",
            "Get:14 http://security.ubuntu.com/ubuntu bionic-security/restricted amd64 Packages [783 kB]\n",
            "Get:15 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease [21.3 kB]\n",
            "Get:16 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [1,470 kB]\n",
            "Get:17 http://security.ubuntu.com/ubuntu bionic-security/multiverse amd64 Packages [21.1 kB]\n",
            "Get:18 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [2,565 kB]\n",
            "Get:19 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main Sources [1,826 kB]\n",
            "Get:20 http://archive.ubuntu.com/ubuntu bionic-updates/multiverse amd64 Packages [29.0 kB]\n",
            "Get:21 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [3,004 kB]\n",
            "Get:22 http://archive.ubuntu.com/ubuntu bionic-updates/restricted amd64 Packages [816 kB]\n",
            "Get:23 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [2,248 kB]\n",
            "Get:24 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main amd64 Packages [936 kB]\n",
            "Get:26 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Packages [917 kB]\n",
            "Get:27 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic/main amd64 Packages [45.3 kB]\n",
            "Get:28 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic/main amd64 Packages [42.8 kB]\n",
            "Fetched 15.0 MB in 7s (2,162 kB/s)\n",
            "Reading package lists... Done\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "# Find the latest version of spark 3.0  from http://www.apache.org/dist/spark/ and enter as the spark version\n",
        "# For example:\n",
        "# spark_version = 'spark-3.0.3'\n",
        "spark_version = 'spark-3.0.3'\n",
        "os.environ['SPARK_VERSION']=spark_version\n",
        "\n",
        "# Install Spark and Java\n",
        "!apt-get update\n",
        "!apt-get install openjdk-11-jdk-headless -qq > /dev/null\n",
        "!wget -q http://www.apache.org/dist/spark/spark-3.0.3/spark-3.0.3-bin-hadoop2.7.tgz  \n",
        "!tar xf spark-3.0.3-bin-hadoop2.7.tgz\n",
        "!pip install -q findspark\n",
        "\n",
        "# Set Environment Variables\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-11-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = f\"/content/spark-3.0.3-bin-hadoop2.7\"\n",
        "\n",
        "# Start a SparkSession\n",
        "import findspark\n",
        "findspark.init()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.builder.appName(\"Vine-Challenge\").config(\"spark.driver.extraClassPath\",\"/content/postgresql-42.2.16.jar\").getOrCreate()"
      ],
      "metadata": {
        "id": "vis9kiPocqpt"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark import SparkFiles\n",
        "url = \"https://amz-review.s3.us-west-1.amazonaws.com/amazon_reviews_us_Books_v1_02.tsv\"\n",
        "spark.sparkContext.addFile(url)\n",
        "df = spark.read.option(\"encoding\", \"UTF-8\").csv(SparkFiles.get(\"amazon_reviews_us_Books_v1_02.tsv\"), sep=\"\\t\", header=True, inferSchema=True)\n",
        "df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EYk5VnCRc_ZP",
        "outputId": "32919ed9-9903-4881-9898-d8a42257f013"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+\n",
            "|marketplace|customer_id|     review_id|product_id|product_parent|       product_title|product_category|star_rating|helpful_votes|total_votes|vine|verified_purchase|     review_headline|         review_body|review_date|\n",
            "+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+\n",
            "|         US|   12076615| RQ58W7SMO911M|0385730586|     122662979|Sisterhood of the...|           Books|          4|            2|          3|   N|                N|this book was a g...|this boook was a ...| 2005-10-14|\n",
            "|         US|   12703090|  RF6IUKMGL8SF|0811828964|      56191234|The Bad Girl's Gu...|           Books|          3|            5|          5|   N|                N|           Fun Fluff|If you are lookin...| 2005-10-14|\n",
            "|         US|   12257412|R1DOSHH6AI622S|1844161560|     253182049|Eisenhorn (A Warh...|           Books|          4|            1|         22|   N|                N| this isn't a review|never read it-a y...| 2005-10-14|\n",
            "|         US|   50732546| RATOTLA3OF70O|0373836635|     348672532|Colby Conspiracy ...|           Books|          5|            2|          2|   N|                N|fine author on he...|Though she is hon...| 2005-10-14|\n",
            "|         US|   51964897|R1TNWRKIVHVYOV|0262181533|     598678717|The Psychology of...|           Books|          4|            0|          2|   N|                N|Execellent cursor...|Review based on a...| 2005-10-14|\n",
            "|         US|   31048862|R2F53LI9KK9MOY|0316769487|     862964341|The Catcher in th...|           Books|          4|            2|          2|   N|                N|   Interesting, fun.|My only complaint...| 2005-10-14|\n",
            "|         US|   53000124|R1KJ6MB7MRSQFF|0805076069|     145341889|Bait and Switch: ...|           Books|          4|            9|         11|   N|                N|I viewed this the...|This book is chil...| 2005-10-14|\n",
            "|         US|   29732693|R2XIM9LT335WHE|1581603681|     640542054|Opening Combinati...|           Books|          4|            3|         10|   N|                N|No Frills - Just ...|When looking for ...| 2005-10-14|\n",
            "|         US|   48541186|R1VE0FQQ0QTQJN|0300108834|     915891133|A Little History ...|           Books|          5|           16|         20|   N|                Y|Simple, entertain...|Never been much f...| 2005-10-14|\n",
            "|         US|   15280864|R1VKEE2NWSWDRU|0446531081|     880645124|Hour Game (King &...|           Books|          4|            0|          0|   N|                N|        A good read!|If you enjoy a we...| 2005-10-14|\n",
            "|         US|   24209213|R2UP6XSVYJBJ2H|0976822105|     731931430|     Faith is a Verb|           Books|          5|            0|          0|   N|                N|Made me Question ...|I thoroughly enjo...| 2005-10-14|\n",
            "|         US|   52253037|R21SYDQ70ILUC0|1580085695|     586052746|Furry Logic: A Gu...|           Books|          5|            1|          1|   N|                Y|    The Perfect Gift|If you are stumpe...| 2005-10-14|\n",
            "|         US|   27925116| R7M06Z88PD7SX|0029148510|      72387289|Acts of War: Beha...|           Books|          4|           14|         14|   N|                N|Solid Book About ...|Whether intention...| 2005-10-14|\n",
            "|         US|   15005044| RRS38KZ4WB5O2|1592285570|     702385650|Temple to the Win...|           Books|          5|            1|          1|   N|                N|A riveting accoun...|Temple to the win...| 2005-10-14|\n",
            "|         US|   52534781|R2YDYRSLGNHPHR|0811848833|     102333549|The Star Wars Pos...|           Books|          4|           16|         19|   N|                Y|Comprehensive vis...|At last... a comp...| 2005-10-14|\n",
            "|         US|   26217071|R1BMPM18O6VZOR|0782144276|     971452783|Photoshop for Nat...|           Books|          4|           13|         14|   N|                N|          Great Book|I find \\\\\"Photosh...| 2005-10-14|\n",
            "|         US|   37684582|R1AABFZH0J0C0C|0375757996|     760991156|The Basic Works o...|           Books|          3|           17|         35|   N|                N|Problems with thi...|I love Aristotle ...| 2005-10-14|\n",
            "|         US|   45232107|R3SJYLG07EHNE9|013146308X|     332774292|The Pterosaurs: F...|           Books|          2|           16|         34|   N|                N|Warmed over lefto...|To his discredit,...| 2005-10-14|\n",
            "|         US|   15437121|R2TIO43ND87XVO|0375701907|     635714194|Straight Man: A N...|           Books|          5|            1|          1|   N|                N|       Funniest Book|Hands down, the f...| 2005-10-14|\n",
            "|         US|   52157117|R18TY4WD19OUAO|141378240X|     656010659|   Lipstick's Legacy|           Books|          5|            1|          1|   N|                N|          Great Work|I read this book ...| 2005-10-14|\n",
            "+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create the vine_table. DataFrame\n",
        "vine_df = df.select([\"review_id\", \"star_rating\", \"helpful_votes\", \"total_votes\", \"vine\", \"verified_purchase\"])\n",
        "vine_df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZXQM-CNfdFKR",
        "outputId": "e400b567-c41d-4fcc-d185-daed927d59a7"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "| RQ58W7SMO911M|          4|            2|          3|   N|                N|\n",
            "|  RF6IUKMGL8SF|          3|            5|          5|   N|                N|\n",
            "|R1DOSHH6AI622S|          4|            1|         22|   N|                N|\n",
            "| RATOTLA3OF70O|          5|            2|          2|   N|                N|\n",
            "|R1TNWRKIVHVYOV|          4|            0|          2|   N|                N|\n",
            "|R2F53LI9KK9MOY|          4|            2|          2|   N|                N|\n",
            "|R1KJ6MB7MRSQFF|          4|            9|         11|   N|                N|\n",
            "|R2XIM9LT335WHE|          4|            3|         10|   N|                N|\n",
            "|R1VE0FQQ0QTQJN|          5|           16|         20|   N|                Y|\n",
            "|R1VKEE2NWSWDRU|          4|            0|          0|   N|                N|\n",
            "|R2UP6XSVYJBJ2H|          5|            0|          0|   N|                N|\n",
            "|R21SYDQ70ILUC0|          5|            1|          1|   N|                Y|\n",
            "| R7M06Z88PD7SX|          4|           14|         14|   N|                N|\n",
            "| RRS38KZ4WB5O2|          5|            1|          1|   N|                N|\n",
            "|R2YDYRSLGNHPHR|          4|           16|         19|   N|                Y|\n",
            "|R1BMPM18O6VZOR|          4|           13|         14|   N|                N|\n",
            "|R1AABFZH0J0C0C|          3|           17|         35|   N|                N|\n",
            "|R3SJYLG07EHNE9|          2|           16|         34|   N|                N|\n",
            "|R2TIO43ND87XVO|          5|            1|          1|   N|                N|\n",
            "|R18TY4WD19OUAO|          5|            1|          1|   N|                N|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#retrieve all the rows where the total_votes count is equal to or greater than 20\n",
        "vote_df = vine_df.filter(vine_df.total_votes>=20)"
      ],
      "metadata": {
        "id": "jHPoZMDaeHIV"
      },
      "execution_count": 77,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vote_df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eU9HwSsNeuOR",
        "outputId": "279866cc-1710-4b1d-b962-32a219284b36"
      },
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|R1DOSHH6AI622S|          4|            1|         22|   N|                N|\n",
            "|R1VE0FQQ0QTQJN|          5|           16|         20|   N|                Y|\n",
            "|R1AABFZH0J0C0C|          3|           17|         35|   N|                N|\n",
            "|R3SJYLG07EHNE9|          2|           16|         34|   N|                N|\n",
            "|R3MD5TZJGIPTDU|          5|           21|         31|   N|                N|\n",
            "|R1KPOIT5WT3NEF|          1|           13|         29|   N|                N|\n",
            "| RX6OWUF8EFZ72|          5|           27|         28|   N|                N|\n",
            "| R33IFUWCNYFSO|          2|            4|         45|   N|                N|\n",
            "| RWMT2MXG3WLDP|          5|            6|         20|   N|                N|\n",
            "|R1OPUF771LL5O5|          4|           27|         27|   N|                N|\n",
            "|R3AW5JRZYH66M5|          1|          268|        307|   N|                N|\n",
            "|R2DXLNKF0EL4CI|          4|           24|         27|   N|                Y|\n",
            "|R12EHXBC0GP2C2|          5|           23|         24|   N|                N|\n",
            "|R2JSWNTAO83X3V|          5|          145|        184|   N|                N|\n",
            "|R3THZWOBS1086J|          5|           46|         47|   N|                N|\n",
            "|R160DHJPJ4B20R|          4|           28|         30|   N|                N|\n",
            "|R27UDFNE6MY1LO|          2|           15|         24|   N|                N|\n",
            "|R34X92SSRBI0N4|          5|           26|         29|   N|                N|\n",
            "| RDBVYHGTTICMV|          1|            4|         25|   N|                N|\n",
            "| RGTKXJ8P5O7LR|          4|           16|         21|   N|                N|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#create a new DataFrame or table to retrieve all the rows where the number of helpful_votes divided by total_votes is equal to or greater than 50%.\n",
        "helpful_votes_df = vote_df.filter((vote_df.helpful_votes/vote_df.total_votes)>=0.5)"
      ],
      "metadata": {
        "id": "_ty_Rl43exBs"
      },
      "execution_count": 79,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "helpful_votes_df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "swzpOkXDfYpk",
        "outputId": "3b4d3cca-14d9-44d8-f27c-e4987440316e"
      },
      "execution_count": 80,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|R1VE0FQQ0QTQJN|          5|           16|         20|   N|                Y|\n",
            "|R3MD5TZJGIPTDU|          5|           21|         31|   N|                N|\n",
            "| RX6OWUF8EFZ72|          5|           27|         28|   N|                N|\n",
            "|R1OPUF771LL5O5|          4|           27|         27|   N|                N|\n",
            "|R3AW5JRZYH66M5|          1|          268|        307|   N|                N|\n",
            "|R2DXLNKF0EL4CI|          4|           24|         27|   N|                Y|\n",
            "|R12EHXBC0GP2C2|          5|           23|         24|   N|                N|\n",
            "|R2JSWNTAO83X3V|          5|          145|        184|   N|                N|\n",
            "|R3THZWOBS1086J|          5|           46|         47|   N|                N|\n",
            "|R160DHJPJ4B20R|          4|           28|         30|   N|                N|\n",
            "|R27UDFNE6MY1LO|          2|           15|         24|   N|                N|\n",
            "|R34X92SSRBI0N4|          5|           26|         29|   N|                N|\n",
            "| RGTKXJ8P5O7LR|          4|           16|         21|   N|                N|\n",
            "|R1TK5Q90GGJSTN|          4|           59|         68|   N|                N|\n",
            "|R3E8UZOCG84HNO|          5|           18|         24|   N|                N|\n",
            "|R33O2810BC4UR0|          5|          101|        103|   N|                Y|\n",
            "|R1M60NYB4UPRJ8|          5|           19|         24|   N|                N|\n",
            "|R38RXR8USISV94|          4|         1150|       1228|   N|                N|\n",
            "|R2E2KP6K6OUBEX|          5|           59|         60|   N|                N|\n",
            "|R3F583X11I1RP6|          1|           40|         43|   N|                N|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#create a new DataFrame or table that retrieves all the rows where a review was written as part of the Vine program (paid), vine == 'Y'.\n",
        "\n",
        "paid_vine_df=helpful_votes_df.filter(helpful_votes_df.vine=='Y')\n",
        "paid_vine_df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ivej4pW0fbMb",
        "outputId": "695cec3a-dd78-4ece-e19b-84d99052504d"
      },
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+-----------+-------------+-----------+----+-----------------+\n",
            "|review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|\n",
            "+---------+-----------+-------------+-----------+----+-----------------+\n",
            "+---------+-----------+-------------+-----------+----+-----------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#create a new DataFrame or table that retrieves all the rows where a review was written as part of the Vine program (unpaid), vine == 'N'.\n",
        "\n",
        "unpaid_vine_df= helpful_votes_df.filter(helpful_votes_df.vine =='N')\n",
        "unpaid_vine_df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jsC5dTvmhmgz",
        "outputId": "49fade3f-d45c-4599-83fc-8f6bab39aa5c"
      },
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "|R1VE0FQQ0QTQJN|          5|           16|         20|   N|                Y|\n",
            "|R3MD5TZJGIPTDU|          5|           21|         31|   N|                N|\n",
            "| RX6OWUF8EFZ72|          5|           27|         28|   N|                N|\n",
            "|R1OPUF771LL5O5|          4|           27|         27|   N|                N|\n",
            "|R3AW5JRZYH66M5|          1|          268|        307|   N|                N|\n",
            "|R2DXLNKF0EL4CI|          4|           24|         27|   N|                Y|\n",
            "|R12EHXBC0GP2C2|          5|           23|         24|   N|                N|\n",
            "|R2JSWNTAO83X3V|          5|          145|        184|   N|                N|\n",
            "|R3THZWOBS1086J|          5|           46|         47|   N|                N|\n",
            "|R160DHJPJ4B20R|          4|           28|         30|   N|                N|\n",
            "|R27UDFNE6MY1LO|          2|           15|         24|   N|                N|\n",
            "|R34X92SSRBI0N4|          5|           26|         29|   N|                N|\n",
            "| RGTKXJ8P5O7LR|          4|           16|         21|   N|                N|\n",
            "|R1TK5Q90GGJSTN|          4|           59|         68|   N|                N|\n",
            "|R3E8UZOCG84HNO|          5|           18|         24|   N|                N|\n",
            "|R33O2810BC4UR0|          5|          101|        103|   N|                Y|\n",
            "|R1M60NYB4UPRJ8|          5|           19|         24|   N|                N|\n",
            "|R38RXR8USISV94|          4|         1150|       1228|   N|                N|\n",
            "|R2E2KP6K6OUBEX|          5|           59|         60|   N|                N|\n",
            "|R3F583X11I1RP6|          1|           40|         43|   N|                N|\n",
            "+--------------+-----------+-------------+-----------+----+-----------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# paid total number of reviews\n",
        "total_paid_reviews = paid_vine_df.count()\n",
        "total_paid_reviews"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YdZIjajrmHHr",
        "outputId": "e9e85b50-4184-4c49-84e6-b32774d3bfab"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {},
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#unpaid total number of reviews\n",
        "total_unpaid_reviews=unpaid_vine_df.count()\n",
        "total_unpaid_reviews"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NK_JDdJGkLS6",
        "outputId": "7ef54b32-7a8b-421b-9241-00b979b8583e"
      },
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "403807"
            ]
          },
          "metadata": {},
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# paid 5-star reviews\n",
        "paid_five_star_reviews = paid_vine_df.filter(paid_vine_df.star_rating == 5).count()\n",
        "paid_five_star_reviews"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OVVZxszxkfzF",
        "outputId": "d81d6249-93d2-445a-fd22-0393d2af4bab"
      },
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {},
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#unpaid 5-star reviews\n",
        "unpaid_five_star_reviews=unpaid_vine_df.filter(unpaid_vine_df.star_rating==5).count()\n",
        "unpaid_five_star_reviews"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "URYlj11ykzB-",
        "outputId": "5381c796-7bf2-43d9-b3b7-fe3e7fca5d12"
      },
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "242889"
            ]
          },
          "metadata": {},
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# paid 5-star reviews percentage\n",
        "#paid_five_star_percentage = (paid_five_star_reviews/total_paid_reviews)*100 is equal to 0 as there are no fields in the paid_vine_df.\n"
      ],
      "metadata": {
        "id": "-H2JrOOQox-Y"
      },
      "execution_count": 75,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#unpaid 5 star reviews\n",
        "unpaid_five_star_percentage = (unpaid_five_star_reviews/total_unpaid_reviews)*100\n",
        "unpaid_five_star_percentage"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kLAS0vBXsu2j",
        "outputId": "80baa9c1-9870-4adf-bf05-639a419904ee"
      },
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "60.14977452099642"
            ]
          },
          "metadata": {},
          "execution_count": 76
        }
      ]
    }
  ]
}