movie_recomendation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "name": "movie_recomendation.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/stxupengyu/MF-for-Movie-Recommendation/blob/main/movie_recomendation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PvDM6Lhkj-L3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bca0127c-4bdf-4718-8adb-e89b48a6cd40"
      },
      "source": [
        "#link google drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1y171CHjj-FI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "399e6cd2-513f-486a-eb61-02a3035cc2db"
      },
      "source": [
        "#enter the folder\n",
        "!ls\n",
        "import os\n",
        "os.chdir('/content/drive/My Drive/Project/recsys')\n",
        "!ls"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "drive  sample_data\n",
            "'movie_data(大于等于20).csv'   rating.csv       微信图片_20201105195005.gdoc\n",
            " movie_recomendation.ipynb     rating_new.csv   微信图片_20201105195005.jpg\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YLPN3Avmb3nR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b255d095-09f1-4421-be82-401ad603f539"
      },
      "source": [
        "#构造数据集\n",
        "!pip install scikit-surprise\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from surprise import SVD,SVDpp,NormalPredictor,NMF,KNNBasic\n",
        "from surprise import Dataset\n",
        "from surprise.model_selection import cross_validate\n",
        "from surprise import Reader\n",
        "from surprise import accuracy\n",
        "from surprise.model_selection import KFold, split\n",
        "#读取数据\n",
        "origin_data=pd.read_csv('movie_data(大于等于20).csv',encoding=\"unicode_escape\") \n",
        "origin_data=origin_data.iloc[:,[3,1,5]]\n",
        "origin_data=origin_data.dropna()\n",
        "#store the dataset and then read it \n",
        "origin_data.to_csv('rating.csv',header=0,index=0)\n",
        "# 指定文件所在路径\n",
        "file_path = os.path.expanduser('rating.csv')\n",
        "# 告诉文本阅读器，文本的格式是怎么样的\n",
        "reader = Reader(line_format='user item rating', sep=',')\n",
        "# 加载数据\n",
        "data = Dataset.load_from_file(file_path, reader=reader)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting scikit-surprise\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/97/37/5d334adaf5ddd65da99fc65f6507e0e4599d092ba048f4302fe8775619e8/scikit-surprise-1.1.1.tar.gz (11.8MB)\n",
            "\u001b[K     |████████████████████████████████| 11.8MB 2.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise) (0.17.0)\n",
            "Requirement already satisfied: numpy>=1.11.2 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise) (1.18.5)\n",
            "Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise) (1.4.1)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from scikit-surprise) (1.15.0)\n",
            "Building wheels for collected packages: scikit-surprise\n",
            "  Building wheel for scikit-surprise (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for scikit-surprise: filename=scikit_surprise-1.1.1-cp36-cp36m-linux_x86_64.whl size=1670899 sha256=96a7005ea12c2c31a9506dbc467c52d7e2c3296f094a3b2ee11cbe1dd846fa41\n",
            "  Stored in directory: /root/.cache/pip/wheels/78/9c/3d/41b419c9d2aff5b6e2b4c0fc8d25c538202834058f9ed110d0\n",
            "Successfully built scikit-surprise\n",
            "Installing collected packages: scikit-surprise\n",
            "Successfully installed scikit-surprise-1.1.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5q0TlFMU5_Ny",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7a1bbc9e-7b66-4a44-fa38-96c0ecd8ac89"
      },
      "source": [
        "#使用基于用户相似度的协同过滤算法\n",
        "sim_options = {'name': 'pearson_baseline'}\n",
        "algo1 = KNNBasic(k=40, min_k=1,sim_options=sim_options)\n",
        "#k近邻数量 sim_options利用的相似度标准\n",
        "\n",
        "# 使用biasSVD\n",
        "algo2 = SVD(biased=True,n_factors=100,n_epochs=20,lr_all=0.005,reg_all=0.02)\n",
        "#biased是否使用biasedSVD，n_factors特征维度，n_epochs迭代次数,lr_all学习率,reg_all=0.02正则化系数\n",
        "\n",
        "# Run 5-fold cross-validation and print results\n",
        "algo1_result=cross_validate(algo1, data, measures=['RMSE', 'MAE'], cv=5, verbose=True)\n",
        "algo2_result=cross_validate(algo2, data, measures=['RMSE', 'MAE'], cv=5, verbose=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Estimating biases using als...\n",
            "Computing the pearson_baseline similarity matrix...\n",
            "Done computing similarity matrix.\n",
            "Estimating biases using als...\n",
            "Computing the pearson_baseline similarity matrix...\n",
            "Done computing similarity matrix.\n",
            "Estimating biases using als...\n",
            "Computing the pearson_baseline similarity matrix...\n",
            "Done computing similarity matrix.\n",
            "Estimating biases using als...\n",
            "Computing the pearson_baseline similarity matrix...\n",
            "Done computing similarity matrix.\n",
            "Estimating biases using als...\n",
            "Computing the pearson_baseline similarity matrix...\n",
            "Done computing similarity matrix.\n",
            "Evaluating RMSE, MAE of algorithm KNNBasic on 5 split(s).\n",
            "\n",
            "                  Fold 1  Fold 2  Fold 3  Fold 4  Fold 5  Mean    Std     \n",
            "RMSE (testset)    0.8271  0.8335  0.8277  0.8255  0.8330  0.8294  0.0033  \n",
            "MAE (testset)     0.6534  0.6574  0.6534  0.6514  0.6584  0.6548  0.0027  \n",
            "Fit time          10.00   9.86    9.78    9.71    9.92    9.85    0.10    \n",
            "Test time         13.51   13.80   13.69   13.81   13.59   13.68   0.12    \n",
            "Evaluating RMSE, MAE of algorithm SVD on 5 split(s).\n",
            "\n",
            "                  Fold 1  Fold 2  Fold 3  Fold 4  Fold 5  Mean    Std     \n",
            "RMSE (testset)    0.7903  0.7925  0.7944  0.7917  0.7906  0.7919  0.0015  \n",
            "MAE (testset)     0.6222  0.6230  0.6242  0.6215  0.6210  0.6224  0.0011  \n",
            "Fit time          11.88   11.91   11.94   11.84   11.94   11.90   0.04    \n",
            "Test time         0.56    0.50    0.61    0.50    0.50    0.53    0.05    \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xbjr0ek7XcS5"
      },
      "source": [
        "#构造数据集\n",
        "origin_data=pd.read_csv('rating.csv')\n",
        "user = origin_data.iloc[:,0]\n",
        "item = origin_data.iloc[:,1]\n",
        "rating = origin_data.iloc[:,2]\n",
        "user_num, item_num, none = origin_data.max()\n",
        "#统计每名用户的评分总和\n",
        "user2rating=np.zeros(int(user_num))\n",
        "#统计每名用户的评价次数\n",
        "user2times=np.zeros(int(user_num))\n",
        "#统计每名用户评价过的物品\n",
        "user2items = [[] for _ in range(int(user_num))]\n",
        "for i in range(len(user)):\n",
        "  user2rating[int(user[i])-1] = user2rating[int(user[i])-1] + rating[i]#这里的-1是因为用户index从1开始\n",
        "  user2times[int(user[i])-1] = user2times[int(user[i])-1] + 1\n",
        "  user2items[int(user[i])-1] = user2items[int(user[i])-1]+[(int(item[i])-1)]\n",
        "#每名用户的平均评分\n",
        "user2average=user2rating/user2times\n",
        "k=0.01#补全矩阵系数\n",
        "#生成每名用户将要评价的物品id\n",
        "user2items_new = [[] for _ in range(int(user_num))]\n",
        "#总的物品列表\n",
        "item_list = [id for id in range(int(item_num))]\n",
        "for id, inst in enumerate(user2items):\n",
        "  temp_list=list(set(item_list).difference(set(inst)))  # item_list中有而inst中没有的\n",
        "  temp_list=temp_list[:int(k*len(temp_list))]# 只选择其中的0.1份\n",
        "  user2items_new[id] = user2items_new[id] + temp_list\n",
        "#生成要补全的数组\n",
        "generate_list=[]\n",
        "for user_i in range(int(user_num)):\n",
        "  for item_i in user2items_new[user_i]:\n",
        "    generate_list.append([user_i+1, item_i+1, user2average[user_i]])#这里要+1 恢复由1开始的index\n",
        "#将补全的数组和原始数据合并，并保存为文件\n",
        "pd.DataFrame(np.vstack([origin_data.values,np.array(generate_list)])).to_csv('rating_new.csv',header=0,index=0)\n",
        "# 指定文件所在路径\n",
        "file_path = os.path.expanduser('rating_new.csv')\n",
        "# 告诉文本阅读器，文本的格式是怎么样的\n",
        "reader = Reader(line_format='user item rating', sep=',')\n",
        "# 加载数据\n",
        "data = Dataset.load_from_file(file_path, reader=reader)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bJ2KUTL9wYsf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5155fb06-c71c-4c9e-c745-94864939b454"
      },
      "source": [
        "# 使用biasSVD\n",
        "algo3 = SVD(biased=True,n_factors=100,n_epochs=20,lr_all=0.005,reg_all=0.02)\n",
        "#biased是否使用biasedSVD，n_factors特征维度，n_epochs迭代次数,lr_all学习率,reg_all=0.02正则化系数\n",
        "\n",
        "# Run 5-fold cross-validation and print results\n",
        "algo3_result=cross_validate(algo3, data, measures=['RMSE', 'MAE'], cv=5, verbose=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Evaluating RMSE, MAE of algorithm SVD on 5 split(s).\n",
            "\n",
            "                  Fold 1  Fold 2  Fold 3  Fold 4  Fold 5  Mean    Std     \n",
            "RMSE (testset)    0.7444  0.7418  0.7452  0.7442  0.7435  0.7438  0.0012  \n",
            "MAE (testset)     0.5657  0.5627  0.5662  0.5632  0.5642  0.5644  0.0014  \n",
            "Fit time          13.81   13.72   13.83   13.77   13.83   13.79   0.04    \n",
            "Test time         0.61    0.61    0.60    0.60    0.59    0.60    0.01    \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gQRd3EZu8Qj-"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}