collaborative_filter_ver2.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "id": "84de56a8",
      "metadata": {
        "id": "84de56a8"
      },
      "source": [
        "# Sample Code"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "688675f5",
      "metadata": {
        "id": "688675f5"
      },
      "source": [
        "## 基礎建設"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "id": "b32226db",
      "metadata": {
        "scrolled": true,
        "id": "b32226db"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import gzip, json\n",
        "\n",
        "def parse(path):\n",
        "    g = gzip.open(path, 'rb')\n",
        "    for l in g:\n",
        "        yield json.loads(l)\n",
        "\n",
        "def getDF(path):\n",
        "    i = 0\n",
        "    df = {}\n",
        "    for d in parse(path):\n",
        "        df[i] = d\n",
        "        i += 1\n",
        "    return pd.DataFrame.from_dict(df, orient='index')"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "84ff0f9d",
      "metadata": {
        "id": "84ff0f9d"
      },
      "source": [
        "## 載入資料"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "id": "9402e9c7",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9402e9c7",
        "outputId": "62ed4e08-938a-45b9-a931-ea8c9d42b3e4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-01-07 03:55:17--  http://deepyeti.ucsd.edu/jianmo/amazon/categoryFilesSmall/All_Beauty.csv\n",
            "Resolving deepyeti.ucsd.edu (deepyeti.ucsd.edu)... 169.228.63.50\n",
            "Connecting to deepyeti.ucsd.edu (deepyeti.ucsd.edu)|169.228.63.50|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 15499476 (15M) [application/octet-stream]\n",
            "Saving to: ‘All_Beauty.csv.5’\n",
            "\n",
            "All_Beauty.csv.5    100%[===================>]  14.78M  70.2MB/s    in 0.2s    \n",
            "\n",
            "2022-01-07 03:55:17 (70.2 MB/s) - ‘All_Beauty.csv.5’ saved [15499476/15499476]\n",
            "\n",
            "--2022-01-07 03:55:17--  http://deepyeti.ucsd.edu/jianmo/amazon/metaFiles2/meta_All_Beauty.json.gz\n",
            "Resolving deepyeti.ucsd.edu (deepyeti.ucsd.edu)... 169.228.63.50\n",
            "Connecting to deepyeti.ucsd.edu (deepyeti.ucsd.edu)|169.228.63.50|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 10329961 (9.9M) [application/octet-stream]\n",
            "Saving to: ‘meta_All_Beauty.json.gz.5’\n",
            "\n",
            "meta_All_Beauty.jso 100%[===================>]   9.85M  65.7MB/s    in 0.2s    \n",
            "\n",
            "2022-01-07 03:55:17 (65.7 MB/s) - ‘meta_All_Beauty.json.gz.5’ saved [10329961/10329961]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!wget http://deepyeti.ucsd.edu/jianmo/amazon/categoryFilesSmall/All_Beauty.csv\n",
        "!wget http://deepyeti.ucsd.edu/jianmo/amazon/metaFiles2/meta_All_Beauty.json.gz"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "id": "84b66cc7",
      "metadata": {
        "id": "84b66cc7"
      },
      "outputs": [],
      "source": [
        "#metadata = getDF('/content/meta_All_Beauty.json.gz')\n",
        "ratings = pd.read_csv('/content/All_Beauty.csv', names=['asin', 'reviewerID', 'overall', 'unixReviewTime'], header=None)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "id": "9fd77203",
      "metadata": {
        "id": "9fd77203"
      },
      "outputs": [],
      "source": [
        "#metadata.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "id": "b402792c",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "b402792c",
        "outputId": "92b2e6f0-2e49-4e19-e79b-47c6dc74a0a0"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-22292587-1517-4a56-a413-4ffc9eed8c3e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>asin</th>\n",
              "      <th>reviewerID</th>\n",
              "      <th>overall</th>\n",
              "      <th>unixReviewTime</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0143026860</td>\n",
              "      <td>A1V6B6TNIC10QE</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1424304000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0143026860</td>\n",
              "      <td>A2F5GHSXFQ0W6J</td>\n",
              "      <td>4.0</td>\n",
              "      <td>1418860800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0143026860</td>\n",
              "      <td>A1572GUYS7DGSR</td>\n",
              "      <td>4.0</td>\n",
              "      <td>1407628800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0143026860</td>\n",
              "      <td>A1PSGLFK1NSVO</td>\n",
              "      <td>5.0</td>\n",
              "      <td>1362960000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0143026860</td>\n",
              "      <td>A6IKXKZMTKGSC</td>\n",
              "      <td>5.0</td>\n",
              "      <td>1324771200</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-22292587-1517-4a56-a413-4ffc9eed8c3e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-22292587-1517-4a56-a413-4ffc9eed8c3e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-22292587-1517-4a56-a413-4ffc9eed8c3e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "         asin      reviewerID  overall  unixReviewTime\n",
              "0  0143026860  A1V6B6TNIC10QE      1.0      1424304000\n",
              "1  0143026860  A2F5GHSXFQ0W6J      4.0      1418860800\n",
              "2  0143026860  A1572GUYS7DGSR      4.0      1407628800\n",
              "3  0143026860   A1PSGLFK1NSVO      5.0      1362960000\n",
              "4  0143026860   A6IKXKZMTKGSC      5.0      1324771200"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "ratings.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "695362e3",
      "metadata": {
        "id": "695362e3"
      },
      "source": [
        "## 資料整理"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "id": "b552fb23",
      "metadata": {
        "id": "b552fb23"
      },
      "outputs": [],
      "source": [
        "ratings['DATE'] = pd.to_datetime(ratings['unixReviewTime'], unit='s')"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "054eeba4",
      "metadata": {
        "id": "054eeba4"
      },
      "source": [
        "## 資料切分"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "id": "284b14a8",
      "metadata": {
        "id": "284b14a8"
      },
      "outputs": [],
      "source": [
        "ratings_trainings = ratings[(ratings['DATE'] < '2018-09-01')]\n",
        "ratings_testings = ratings[\n",
        "    (ratings['DATE'] >= '2018-09-01') & \n",
        "    (ratings['DATE'] <= '2018-09-30')\n",
        "]\n",
        "ratings_testings_by_user = ratings_testings.groupby('reviewerID').agg(list).reset_index()[['reviewerID', 'asin']].to_dict('records')\n",
        "ratings_testings_by_user = { rating['reviewerID']: rating['asin'] for rating in ratings_testings_by_user }\n",
        "users = list(ratings_testings_by_user.keys())"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 產生推薦"
      ],
      "metadata": {
        "id": "vIVjg1LvIrMp"
      },
      "id": "vIVjg1LvIrMp"
    },
    {
      "cell_type": "code",
      "source": [
        "#! pip install surprise"
      ],
      "metadata": {
        "id": "Cp4me5IYatUM"
      },
      "id": "Cp4me5IYatUM",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from itertools import combinations\n",
        "from collections import defaultdict\n",
        "from surprise import Reader, Dataset, KNNBasic\n",
        "\n",
        "class RecSur:\n",
        "\n",
        "    def __init__(self, training_data):\n",
        "        self.training_data = training_data.sort_values(\"DATE\", ascending=False).groupby(['reviewerID', 'asin']).head(1)\n",
        "\n",
        "    def user_similarity(self, remove_obscure_user = True, user_rating_threshold = 3):\n",
        "\n",
        "        # loading data from dataframe\n",
        "        # user_to_items dict:\n",
        "        # {\n",
        "        #   'user': {\n",
        "        #       'item': ratings...\n",
        "        #   }...\n",
        "        # }  \n",
        "        user_to_items = defaultdict(dict)\n",
        "        for _, row in self.training_data.iterrows():\n",
        "            row = dict(row)\n",
        "            user = row['reviewerID']\n",
        "            item = row['asin']\n",
        "            rating = float(row['overall'])\n",
        "\n",
        "            user_to_items[user][item] = rating\n",
        "\n",
        "        #print(\"total users before filtering: \", len(user_to_items))\n",
        "\n",
        "        # remove obscure user to decrease data size\n",
        "        # filtering params\n",
        "        all_users = list(user_to_items.keys())\n",
        "        for user in all_users:\n",
        "            ratings = user_to_items[user]\n",
        "            if remove_obscure_user and len(ratings) < user_rating_threshold:\n",
        "                del user_to_items[user]\n",
        "\n",
        "        # print(\"total users  after filtering: \", len(user_to_items))\n",
        "\n",
        "        # generate item to user mapping dict\n",
        "        # {\n",
        "        #   'item': {\n",
        "        #       'user': ratings...\n",
        "        #   }...\n",
        "        # }\n",
        "        item_to_users = defaultdict(dict)\n",
        "        for user, items in user_to_items.items():\n",
        "            for item, rating in items.items():\n",
        "                item_to_users[item][user] = rating\n",
        "\n",
        "        # prepare data of computing user similarity \n",
        "        init_sim = lambda: [0 for _ in range(3)]\n",
        "        factory = lambda: defaultdict(init_sim)\n",
        "        pre_user_similarity = defaultdict(factory)\n",
        "        n = len(item_to_users)\n",
        "        index = 0\n",
        "        for item, user_ratings in item_to_users.items():\n",
        "            if len(user_ratings) > 1:\n",
        "                # print(f\"item: {item} have been rated by {len(user_ratings)} users progress: {index}/{n}\")\n",
        "                for user1, user2 in combinations(user_ratings.keys(), 2):\n",
        "                    xy = user_ratings[user1] * user_ratings[user2]\n",
        "                    xx = user_ratings[user1] ** 2\n",
        "                    yy = user_ratings[user2] ** 2\n",
        "                    pre_user_similarity[user1][user2][0] += xy\n",
        "                    pre_user_similarity[user1][user2][1] += xx\n",
        "                    pre_user_similarity[user1][user2][2] += yy\n",
        "\n",
        "                    pre_user_similarity[user2][user1][0] += xy\n",
        "                    pre_user_similarity[user2][user1][1] += xx\n",
        "                    pre_user_similarity[user2][user1][2] += yy\n",
        "            index += 1\n",
        "\n",
        "        user_similarity = {}\n",
        "        for src_user in pre_user_similarity:\n",
        "            user_similarity_order = []\n",
        "            for dst_user, val in pre_user_similarity[src_user].items():\n",
        "                xy = val[0]\n",
        "                xx = val[1]\n",
        "                yy = val[2]\n",
        "                div = ((xx*yy) ** 0.5)\n",
        "                if div == 0:\n",
        "                    continue\n",
        "                similarity = xy / div\n",
        "                if similarity < 0:\n",
        "                    continue\n",
        "                for i, s in enumerate(user_similarity_order):\n",
        "                    target_similarity = s[1]\n",
        "                    if target_similarity < similarity:\n",
        "                        user_similarity_order.insert(i, (dst_user, similarity))\n",
        "                        break\n",
        "                else:\n",
        "                    user_similarity_order.append((dst_user, similarity))\n",
        "            user_similarity[src_user] = user_similarity_order\n",
        "        \n",
        "        return user_similarity, user_to_items\n",
        "\n",
        "    def user_based_recommender(self, user_similarity, user_to_items, start_date, users=[], k=10, training_data = ''):\n",
        "        # 建立 simple rule based建立 simple rule based\n",
        "        if isinstance(training_data, pd.DataFrame):\n",
        "            hot = training_data[(training_data['DATE'] >= start_date) & (training_data['DATE'] < '2018-09-01') & (training_data['overall'] >= 4)]\n",
        "            hot = pd.DataFrame(hot.asin.value_counts())\n",
        "            hot.reset_index(inplace=True)\n",
        "            hot.columns = ['asin', 'count']\n",
        "\n",
        "        recommendation = {}\n",
        "        for user in users:\n",
        "            if user in user_similarity:\n",
        "                sim_users = user_similarity[user]\n",
        "                recommended_items = []\n",
        "                recommended_items_set = set()\n",
        "                user_have_rated = set(user_to_items[user])\n",
        "                stop_recommend = False\n",
        "                for sim_user, _ in sim_users:\n",
        "                    items_from_sim_user = sorted(list(user_to_items[sim_user].items()), key=lambda item: item[1])\n",
        "                    for item, _ in items_from_sim_user:\n",
        "                        if item not in recommended_items_set and item not in user_have_rated:\n",
        "                            recommended_items.append(item)\n",
        "                            recommended_items_set.add(item)\n",
        "                        if len(recommended_items) >= k:\n",
        "                            stop_recommend = True\n",
        "                            break\n",
        "\n",
        "                    # 推薦不滿 k 個用 rule-based 且使用者尚未評論過的補滿\n",
        "                    if isinstance(training_data, pd.DataFrame):\n",
        "                        if len(recommended_items) < k:\n",
        "                            for item in hot.asin.tolist():\n",
        "                                if item not in recommended_items and item not in user_have_rated:\n",
        "                                    recommended_items.append(item)\n",
        "                                if len(recommended_items) >= k:\n",
        "                                    stop_recommend = True\n",
        "                                    break\n",
        "\n",
        "                    if stop_recommend:\n",
        "                        break\n",
        "\n",
        "                recommendation[user] = recommended_items\n",
        "            else:\n",
        "                # rule-based\n",
        "                if isinstance(training_data, pd.DataFrame):\n",
        "                    recommendation[user] = hot.asin[:k].tolist()\n",
        "                else:\n",
        "                    recommendation[user] = []\n",
        "        return recommendation\n",
        "\n",
        "    def item_based_recommender(self, users=[], k=10, rule_base=False, start_date = ''):\n",
        "        if rule_base:\n",
        "            assert start_date != '', \"please fill start_date 'YYYY-MM-DD'\"\n",
        "            if isinstance(self.training_data, pd.DataFrame):\n",
        "                hot = self.training_data[(self.training_data['DATE'] >= start_date) & (self.training_data['DATE'] < '2018-09-01') & (self.training_data['overall'] >= 4)]\n",
        "                hot = pd.DataFrame(hot.asin.value_counts())\n",
        "                hot.reset_index(inplace=True)\n",
        "                hot.columns = ['asin', 'count']\n",
        "\n",
        "        # loading data from dataframe\n",
        "        # item_to_users dict:\n",
        "        # {\n",
        "        #   'item': {\n",
        "        #       'user': ratings...\n",
        "        #   }...\n",
        "        # }\n",
        "        item_to_users = defaultdict(dict)\n",
        "        for _, row in self.training_data.iterrows():\n",
        "            row = dict(row)\n",
        "            user = row['reviewerID']\n",
        "            item = row['asin']\n",
        "            rating = float(row['overall'])\n",
        "            item_to_users[item][user] = rating\n",
        "\n",
        "        # print(\"data converted\")\n",
        "\n",
        "        user_to_items = defaultdict(dict)\n",
        "        for item, rating_users in item_to_users.items():\n",
        "            for user, rating in rating_users.items():\n",
        "                user_to_items[user][item] = rating\n",
        "\n",
        "        # print(\"data inverted\")\n",
        "\n",
        "        init_sim = lambda: [0, 0, 0]\n",
        "        factory = lambda: defaultdict(init_sim)\n",
        "        pre_item_similarity = defaultdict(factory)\n",
        "        for user, items in user_to_items.items():\n",
        "            if len(items) > 1:\n",
        "                for i1, i2 in combinations(items.keys(), 2):\n",
        "                    xy = items[i1] * items[i2]\n",
        "                    xx = items[i1] ** 2\n",
        "                    yy = items[i2] ** 2\n",
        "                    pre_item_similarity[i1][i2][0] += xy\n",
        "                    pre_item_similarity[i1][i2][1] += xx\n",
        "                    pre_item_similarity[i1][i2][2] += yy\n",
        "\n",
        "                    pre_item_similarity[i2][i1][0] += xy\n",
        "                    pre_item_similarity[i2][i1][1] += xx\n",
        "                    pre_item_similarity[i2][i1][2] += yy\n",
        "\n",
        "        # print(\"sim data prepared\")\n",
        "\n",
        "        item_similarity = {}\n",
        "        for src_item in pre_item_similarity:\n",
        "            item_similarity_order = []\n",
        "            for dst_item, val in pre_item_similarity[src_item].items():\n",
        "                xy = val[0]\n",
        "                xx = val[1]\n",
        "                yy = val[2]\n",
        "                div = ((xx*yy) ** 0.5)\n",
        "                if div == 0:\n",
        "                    continue\n",
        "                similarity = xy / div\n",
        "                if similarity < 0:\n",
        "                    continue\n",
        "                for i, s in enumerate(item_similarity_order):\n",
        "                    target_similarity = s[1]\n",
        "                    if target_similarity < similarity:\n",
        "                        item_similarity_order.insert(i, (dst_item, similarity))\n",
        "                        break\n",
        "                else:\n",
        "                    item_similarity_order.append((dst_item, similarity))\n",
        "            item_similarity[src_item] = item_similarity_order\n",
        "\n",
        "        # print(f\"get {k} recommendation items for for user: {users}\")\n",
        "\n",
        "        recommendation = {}\n",
        "        for user in users:\n",
        "            items = []\n",
        "            items_set = set()\n",
        "            stop = False\n",
        "            user_has_rated = set(user_to_items[user])\n",
        "            for item in user_has_rated:\n",
        "                if item in item_similarity:\n",
        "                    for sim_item, _ in item_similarity[item]:\n",
        "                        # skip the item user has rated\n",
        "                        if sim_item not in user_has_rated and sim_item not in items_set:\n",
        "                            items.append(sim_item)\n",
        "                            items_set.add(sim_item)\n",
        "                        if len(items) >= k:\n",
        "                            stop = True\n",
        "                            break\n",
        "                    if stop:\n",
        "                        break\n",
        "            \n",
        "            if rule_base and len(items) < k:\n",
        "                hotlist = hot.asin.tolist()\n",
        "                for item in hotlist:\n",
        "                    if item not in items:\n",
        "                        items.append(item)\n",
        "                    if len(items) >= k:\n",
        "                        break\n",
        "\n",
        "            recommendation[user] = items\n",
        "        return recommendation    \n",
        "\n",
        "    def sur_algo(self, user_based=False, algo=KNNBasic):\n",
        "\n",
        "        reader = Reader(rating_scale=(0, 5))\n",
        "        self.data = self.training_data[self.training_data['DATE'] >= '2017-09-01'][['reviewerID', 'asin', 'overall']]\n",
        "        self.data = Dataset.load_from_df(self.data, reader=reader)\n",
        "\n",
        "        sim_options = {\n",
        "            'name': 'cosine',\n",
        "            'user_based': user_based\n",
        "        }\n",
        "        algo_impl = algo(sim_options = sim_options)\n",
        "        trainset = self.data.build_full_trainset()\n",
        "        algo_impl.fit(trainset)\n",
        "        return algo_impl\n",
        "    \n",
        "    def sur_recommender(self, algo_impl, users=[], k=10, rule_base = False, start_date = ''):\n",
        "        if rule_base:\n",
        "            assert start_date != '', \"please fill start_date 'YYYY-MM-DD'\"\n",
        "            if isinstance(self.training_data, pd.DataFrame):\n",
        "                hot = self.training_data[(self.training_data['DATE'] >= start_date) & (self.training_data['DATE'] < '2018-09-01') & (self.training_data['overall'] >= 4)]\n",
        "                hot = pd.DataFrame(hot.asin.value_counts())\n",
        "                hot.reset_index(inplace=True)\n",
        "                hot.columns = ['asin', 'count']\n",
        "\n",
        "        recommendation = {}\n",
        "        training_data = self.training_data[self.training_data['DATE'] >= '2017-09-01'][['reviewerID', 'asin', 'overall']]\n",
        "        for user in users:\n",
        "            items_user_rated = set(training_data.loc[training_data['reviewerID']==user]['asin'].tolist())\n",
        "            recommend_item_list = []\n",
        "            recommend_item_set = set()\n",
        "            for item in items_user_rated:\n",
        "                iid = algo_impl.trainset.to_inner_iid(item)\n",
        "                recommend_item_iid = algo_impl.get_neighbors(iid, k)\n",
        "                for sim_item_iid in recommend_item_iid:\n",
        "                    item_raw_id = algo_impl.trainset.to_raw_iid(sim_item_iid)\n",
        "                    if item_raw_id not in items_user_rated and item_raw_id not in recommend_item_set:\n",
        "                        recommend_item_list.append(item_raw_id)\n",
        "                        recommend_item_set.add(item_raw_id)\n",
        "\n",
        "                if len(recommend_item_list) >= k:\n",
        "                    recommend_item_list = recommend_item_list[:k]\n",
        "                    break\n",
        "                        \n",
        "            if rule_base and len(recommend_item_list) < k:\n",
        "                hotlist = hot.asin.tolist()\n",
        "                for item in hotlist:\n",
        "                    if item not in recommend_item_list:\n",
        "                        recommend_item_list.append(item)\n",
        "                    if len(recommend_item_list) >= k:\n",
        "                        break\n",
        "            \n",
        "            recommendation[user] = recommend_item_list\n",
        "        \n",
        "        return recommendation\n",
        "\n",
        "    def evaluate(self, ratings_testings_by_user={}, ratings_by_user={}, method=None):\n",
        "        '''\n",
        "        * ratings_testings_by_user: dict 真實被購買的商品資料（2018-09-01 以後資料）\n",
        "        * ratings_by_user: dict 利用訓練資料學習的推薦商品\n",
        "        * method: str\n",
        "        * score: float\n",
        "        '''\n",
        "        total = 0\n",
        "        for d in ratings_testings_by_user:\n",
        "            if d in ratings_by_user:\n",
        "                total += len(set(ratings_by_user[d]) & set(ratings_testings_by_user[d]))\n",
        "\n",
        "        score = total / len(ratings_testings)\n",
        "        return score\n"
      ],
      "metadata": {
        "id": "g9u3u7t7Iwep"
      },
      "id": "g9u3u7t7Iwep",
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 結果評估"
      ],
      "metadata": {
        "id": "C7Dt7JbmqjhP"
      },
      "id": "C7Dt7JbmqjhP"
    },
    {
      "cell_type": "code",
      "source": [
        "start_dates = ['2017-09-01','2018-03-01','2018-06-01','2018-08-01']\n",
        "ks = [10, 20, 30]\n",
        "\n",
        "# set object\n",
        "recsur = RecSur(ratings_trainings)\n",
        "\n",
        "# user-based\n",
        "user_simi, user_to_items = recsur.user_similarity()\n",
        "for k in ks:\n",
        "    ratings_by_user = recsur.user_based_recommender(user_simi, user_to_items, '', users, k=k)\n",
        "    print(f'User-based 協同過濾, {k} 個推薦, Recall： {round(recsur.evaluate(ratings_testings_by_user, ratings_by_user), 3)}')\n",
        "    for start_date in start_dates:\n",
        "        ratings_by_user = recsur.user_based_recommender(user_simi, user_to_items, start_date, users, k=k, training_data=ratings_trainings)\n",
        "        print(f'混合 Rule-based, 資料起始日：{start_date}, {k} 個推薦, Recall： {round(recsur.evaluate(ratings_testings_by_user, ratings_by_user), 3)}')\n",
        "    print(\"-\"*60)\n",
        "\n",
        "del user_simi, user_to_items"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6Q2HA-nhhDgw",
        "outputId": "38bc922a-6e43-4fb0-e65a-b1e2c1bc359e"
      },
      "id": "6Q2HA-nhhDgw",
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "User-based 協同過濾, 10 個推薦, Recall： 0.0\n",
            "混合 Rule-based, 資料起始日：2017-09-01, 10 個推薦, Recall： 0.098\n",
            "混合 Rule-based, 資料起始日：2018-03-01, 10 個推薦, Recall： 0.1\n",
            "混合 Rule-based, 資料起始日：2018-06-01, 10 個推薦, Recall： 0.112\n",
            "混合 Rule-based, 資料起始日：2018-08-01, 10 個推薦, Recall： 0.142\n",
            "------------------------------------------------------------\n",
            "User-based 協同過濾, 20 個推薦, Recall： 0.0\n",
            "混合 Rule-based, 資料起始日：2017-09-01, 20 個推薦, Recall： 0.102\n",
            "混合 Rule-based, 資料起始日：2018-03-01, 20 個推薦, Recall： 0.115\n",
            "混合 Rule-based, 資料起始日：2018-06-01, 20 個推薦, Recall： 0.185\n",
            "混合 Rule-based, 資料起始日：2018-08-01, 20 個推薦, Recall： 0.205\n",
            "------------------------------------------------------------\n",
            "User-based 協同過濾, 30 個推薦, Recall： 0.0\n",
            "混合 Rule-based, 資料起始日：2017-09-01, 30 個推薦, Recall： 0.124\n",
            "混合 Rule-based, 資料起始日：2018-03-01, 30 個推薦, Recall： 0.168\n",
            "混合 Rule-based, 資料起始日：2018-06-01, 30 個推薦, Recall： 0.231\n",
            "混合 Rule-based, 資料起始日：2018-08-01, 30 個推薦, Recall： 0.258\n",
            "------------------------------------------------------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# item-based\n",
        "for k in ks:\n",
        "    ratings_by_user = recsur.item_based_recommender(users, k=k)\n",
        "    print(f'Item-based 協同過濾, {k} 個推薦, Recall： {round(recsur.evaluate(ratings_testings_by_user, ratings_by_user), 3)}')\n",
        "    for start_date in start_dates:\n",
        "        ratings_by_user =  recsur.item_based_recommender(users, k=k, rule_base=True, start_date=start_date)\n",
        "        print(f'混合 Rule-based, 資料起始日：{start_date}, {k} 個推薦, Recall： {round(recsur.evaluate(ratings_testings_by_user, ratings_by_user), 3)}')\n",
        "    print(\"-\"*60)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_eCmQua8FAl0",
        "outputId": "805a6d78-b4cb-4d4b-f711-7d5b7bb7a641"
      },
      "id": "_eCmQua8FAl0",
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Item-based 協同過濾, 10 個推薦, Recall： 0.002\n",
            "混合 Rule-based, 資料起始日：2017-09-01, 10 個推薦, Recall： 0.098\n",
            "混合 Rule-based, 資料起始日：2018-03-01, 10 個推薦, Recall： 0.1\n",
            "混合 Rule-based, 資料起始日：2018-06-01, 10 個推薦, Recall： 0.112\n",
            "混合 Rule-based, 資料起始日：2018-08-01, 10 個推薦, Recall： 0.139\n",
            "------------------------------------------------------------\n",
            "Item-based 協同過濾, 20 個推薦, Recall： 0.002\n",
            "混合 Rule-based, 資料起始日：2017-09-01, 20 個推薦, Recall： 0.105\n",
            "混合 Rule-based, 資料起始日：2018-03-01, 20 個推薦, Recall： 0.129\n",
            "混合 Rule-based, 資料起始日：2018-06-01, 20 個推薦, Recall： 0.185\n",
            "混合 Rule-based, 資料起始日：2018-08-01, 20 個推薦, Recall： 0.205\n",
            "------------------------------------------------------------\n",
            "Item-based 協同過濾, 30 個推薦, Recall： 0.002\n",
            "混合 Rule-based, 資料起始日：2017-09-01, 30 個推薦, Recall： 0.122\n",
            "混合 Rule-based, 資料起始日：2018-03-01, 30 個推薦, Recall： 0.161\n",
            "混合 Rule-based, 資料起始日：2018-06-01, 30 個推薦, Recall： 0.239\n",
            "混合 Rule-based, 資料起始日：2018-08-01, 30 個推薦, Recall： 0.237\n",
            "------------------------------------------------------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# surprise item-based\n",
        "algo_impl = recsur.sur_algo()\n",
        "for k in ks:\n",
        "    ratings_by_user = recsur.sur_recommender(algo_impl, users, k=k)\n",
        "    print(f'Surprise Item-based 協同過濾, {k} 個推薦, Recall： {round(recsur.evaluate(ratings_testings_by_user, ratings_by_user), 3)}')\n",
        "    for start_date in start_dates:\n",
        "        ratings_by_user =  recsur.sur_recommender(algo_impl, users, k=k, rule_base=True, start_date=start_date)\n",
        "        print(f'混合 Rule-based, 資料起始日：{start_date}, {k} 個推薦, Recall： {round(recsur.evaluate(ratings_testings_by_user, ratings_by_user), 3)}')\n",
        "    print(\"-\"*60)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HSvRc0gzqD1V",
        "outputId": "8f5e84e3-f69a-44ee-82f5-44f8818b0d62"
      },
      "id": "HSvRc0gzqD1V",
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Computing the cosine similarity matrix...\n",
            "Done computing similarity matrix.\n",
            "Surprise Item-based 協同過濾, 10 個推薦, Recall： 0.002\n",
            "混合 Rule-based, 資料起始日：2017-09-01, 10 個推薦, Recall： 0.098\n",
            "混合 Rule-based, 資料起始日：2018-03-01, 10 個推薦, Recall： 0.1\n",
            "混合 Rule-based, 資料起始日：2018-06-01, 10 個推薦, Recall： 0.112\n",
            "混合 Rule-based, 資料起始日：2018-08-01, 10 個推薦, Recall： 0.139\n",
            "------------------------------------------------------------\n",
            "Surprise Item-based 協同過濾, 20 個推薦, Recall： 0.002\n",
            "混合 Rule-based, 資料起始日：2017-09-01, 20 個推薦, Recall： 0.105\n",
            "混合 Rule-based, 資料起始日：2018-03-01, 20 個推薦, Recall： 0.132\n",
            "混合 Rule-based, 資料起始日：2018-06-01, 20 個推薦, Recall： 0.186\n",
            "混合 Rule-based, 資料起始日：2018-08-01, 20 個推薦, Recall： 0.207\n",
            "------------------------------------------------------------\n",
            "Surprise Item-based 協同過濾, 30 個推薦, Recall： 0.003\n",
            "混合 Rule-based, 資料起始日：2017-09-01, 30 個推薦, Recall： 0.125\n",
            "混合 Rule-based, 資料起始日：2018-03-01, 30 個推薦, Recall： 0.168\n",
            "混合 Rule-based, 資料起始日：2018-06-01, 30 個推薦, Recall： 0.244\n",
            "混合 Rule-based, 資料起始日：2018-08-01, 30 個推薦, Recall： 0.242\n",
            "------------------------------------------------------------\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.0"
    },
    "colab": {
      "name": "sample-cf-used-based.ipynb",
      "provenance": [],
      "toc_visible": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}