10_ALS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f7fa4290",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.ml.recommendation import ALS # Alternating least squared\n",
    "from pyspark.ml.evaluation import RegressionEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "691d7a79",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession.builder.appName(\"app\").getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8a8e2391",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = spark.read.csv(\"./Python-and-Spark-for-Big-Data-master/Python-and-Spark-for-Big-Data-master/Spark_for_Machine_Learning/Recommender_Systems/movielens_ratings.csv\",\n",
    "              inferSchema=True, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "69c261b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------+------+\n",
      "|movieId|rating|userId|\n",
      "+-------+------+------+\n",
      "|      2|   3.0|     0|\n",
      "|      3|   1.0|     0|\n",
      "|      5|   2.0|     0|\n",
      "|      9|   4.0|     0|\n",
      "|     11|   1.0|     0|\n",
      "|     12|   2.0|     0|\n",
      "|     15|   1.0|     0|\n",
      "|     17|   1.0|     0|\n",
      "|     19|   1.0|     0|\n",
      "|     21|   1.0|     0|\n",
      "|     23|   1.0|     0|\n",
      "|     26|   3.0|     0|\n",
      "|     27|   1.0|     0|\n",
      "|     28|   1.0|     0|\n",
      "|     29|   1.0|     0|\n",
      "|     30|   1.0|     0|\n",
      "|     31|   1.0|     0|\n",
      "|     34|   1.0|     0|\n",
      "|     37|   1.0|     0|\n",
      "|     41|   2.0|     0|\n",
      "+-------+------+------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7da1e92d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1501"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3aed922d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- movieId: integer (nullable = true)\n",
      " |-- rating: double (nullable = true)\n",
      " |-- userId: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0c988133",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['movieId', 'rating', 'userId']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "062aa5f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------+------+\n",
      "|movieId|rating|userId|\n",
      "+-------+------+------+\n",
      "|      0|     0|     0|\n",
      "+-------+------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.select([pyspark.sql.functions.sum(df[col].isNull().cast(\"integer\")).alias(col) for col in df.columns]).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "cd84490d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df, test_df = df.randomSplit([0.8, 0.2], seed=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f32a19d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "als = ALS(maxIter=5, regParam=0.01, userCol='userId', itemCol='movieId', ratingCol='rating')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "632740a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = als.fit(train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "387a6b0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction = model.transform(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "fb8fe711",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------+------+------------+\n",
      "|movieId|rating|userId|  prediction|\n",
      "+-------+------+------+------------+\n",
      "|      0|   3.0|    28|   1.2480105|\n",
      "|      0|   1.0|    27|  0.83925146|\n",
      "|      5|   2.0|    22|    1.818285|\n",
      "|      2|   1.0|    16|  0.68108505|\n",
      "|      2|   3.0|     6|    1.642898|\n",
      "|      2|   1.0|     3|    2.638785|\n",
      "|      2|   1.0|    17|   3.3182456|\n",
      "|      3|   1.0|     9|   2.2142448|\n",
      "|      1|   1.0|     4|   1.6648022|\n",
      "|      2|   4.0|     8|    2.037578|\n",
      "|      3|   2.0|     8|  0.07634701|\n",
      "|      7|   1.0|     8|   1.1374704|\n",
      "|      0|   3.0|    10|   1.0688026|\n",
      "|      4|   3.0|    10|   1.6321839|\n",
      "|      5|   1.0|    29|-0.096546054|\n",
      "|      3|   1.0|    21|   2.3792627|\n",
      "|      0|   1.0|    11|   1.2660939|\n",
      "|      4|   1.0|    14|   0.8775302|\n",
      "|      5|   1.0|    14|   1.7154177|\n",
      "|      4|   3.0|     2|   2.8543346|\n",
      "+-------+------+------+------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "prediction.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "f22482fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluator = RegressionEvaluator(predictionCol=\"prediction\", labelCol=\"rating\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "a25b8bf7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.7563629531848577"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.evaluate(prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1dd01d45",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}