_unit-test-jaccard-recommender.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"2021-08-07-unit-test-jaccard-recommender.ipynb","provenance":[],"collapsed_sections":[],"toc_visible":true,"authorship_tag":"ABX9TyM5Srtym69W8EdQ/HhFIklO"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","metadata":{"id":"5NR6O-NIyXvO"},"source":["# Unit Test - Jaccard Similarity based Recommender\n","> This test verifies the functionality of JaccardRecommender\n","\n","- toc: true\n","- badges: true\n","- comments: true\n","- categories: [JaccardModel, UnitTest, Concept]\n","- image:"]},{"cell_type":"code","metadata":{"id":"RsZQz_Usw06d","executionInfo":{"status":"ok","timestamp":1625725064242,"user_tz":-330,"elapsed":521,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}}},"source":["import sys\n","import pathlib\n","import numpy as np\n","from scipy.sparse.csc import csc_matrix"],"execution_count":3,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"LT8i_1Zxw1uF"},"source":["## Base Recommender"]},{"cell_type":"code","metadata":{"id":"Y7X4Ulcwwfzo","executionInfo":{"status":"ok","timestamp":1625725038309,"user_tz":-330,"elapsed":10,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}}},"source":["class Recommender(object):\n","    def __init__(self):\n","        raise NotImplementedError\n","\n","    def recommend(self, userid=None, username=None, N=10):\n","        raise NotImplementedError\n","\n","    def recommend_all(self, userids, num_recs, **kwargs):\n","        recs = {}\n","        with tqdm(total=len(userids), leave=True) as progress:\n","            for u in userids:\n","                recs[u] = self.recommend(userid=u, N=num_recs, **kwargs)\n","                progress.update(1)\n","\n","        return recs"],"execution_count":1,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"ksGAr1gPw3bp"},"source":["## Jaccard Recommender"]},{"cell_type":"code","metadata":{"id":"UD-h1S5ZwqIs","executionInfo":{"status":"ok","timestamp":1625725051482,"user_tz":-330,"elapsed":797,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}}},"source":["class JaccardRecommender(Recommender):\n","    def __init__(self, implicit_matrix, p2i, t2i, i2t, i2p, n2i, u2i, i2u):\n","        self.implicit_matrix = implicit_matrix\n","        self.p2i = p2i\n","        self.t2i = t2i\n","        self.i2t = i2t\n","        self.i2p = i2p\n","        self.n2i = n2i\n","        self.i2p = i2p\n","        self.u2i = u2i\n","        self.i2u = i2u\n","\n","    def jaccard_multiple(self, page_indices, exclude_index=None):\n","        X = self.implicit_matrix.astype(bool).astype(int)\n","        if exclude_index is None:\n","            intrsct = X.dot(X[page_indices, :].T)\n","            totals = X[page_indices, :].sum(axis=1).T + X.sum(axis=1)\n","        else:\n","            use_indices = np.full(X.shape[1], True)\n","            use_indices[exclude_index] = False\n","            # print(X[:, use_indices].shape)\n","            # print(X[page_indices, :][:, use_indices].T.shape)\n","\n","            intrsct = X[:, use_indices].dot(X[page_indices, :][:, use_indices].T)\n","            totals = X[page_indices, :][:, use_indices].sum(axis=1).T + X[\n","                :, use_indices\n","            ].sum(axis=1)\n","\n","        return intrsct / (totals - intrsct)\n","\n","    def recommend(\n","        self,\n","        N=10,\n","        userid=None,\n","        user=None,\n","        num_lookpage_pages=None,\n","        recent_pages_dict=None,\n","        interactions=None,\n","    ):\n","        if user is not None:\n","            user_index = self.n2i[user]\n","        elif userid is not None:\n","            user_index = self.u2i[userid]\n","        else:\n","            raise ValueError(\"Either user or userid must be non-null\")\n","\n","        recent_pages = recent_pages_dict[self.i2u[user_index]][:num_lookpage_pages]\n","\n","        user_page_indices = [self.p2i[p] for p in recent_pages]\n","        d = self.jaccard_multiple(user_page_indices, exclude_index=user_index)\n","\n","        d = np.nan_to_num(d)\n","        d[d == 1] = np.nan\n","\n","        mean_jaccard = np.nanmean(d, axis=1).A.squeeze()\n","        order = np.argsort(mean_jaccard)[::-1]\n","        return [self.i2p[o] for o in order[:N]]\n","\n","    def item_to_item(self, N=10, title=None, pageid=None):\n","        if title is not None:\n","            page_index = self.t2i.get(title, None)\n","        elif pageid is not None:\n","            page_index = self.p2i.get(pageid, None)\n","        else:\n","            raise ValueError(\"Either title or pageid must be non-null\")\n","\n","        if page_index is None:\n","            raise ValueError(\n","                \"Page {} not found\".format(pageid if title is None else title)\n","            )\n","\n","        target_page_editors = np.flatnonzero(\n","            self.implicit_matrix[page_index, :].toarray()\n","        )\n","        # print(\"target_page_editors {}\".format(target_page_editors))\n","\n","        num_target_editors = len(target_page_editors)\n","\n","        edited_indices = np.flatnonzero(\n","            np.sum(self.implicit_matrix[:, target_page_editors] > 0, axis=1)\n","        )\n","\n","        # print(\"edited_indices {}\".format(edited_indices))\n","\n","        num_shared_editors = np.asarray(\n","            np.sum(self.implicit_matrix[:, target_page_editors] > 0, axis=1)[\n","                edited_indices\n","            ]\n","        ).squeeze()\n","\n","        # print(\"num_shared_editors {}\".format(num_shared_editors))\n","\n","        num_item_editors = np.asarray(\n","            np.sum(self.implicit_matrix[edited_indices, :] > 0, axis=1)\n","        ).squeeze()\n","\n","        # print(\"num_item_editors {}\".format(num_item_editors))\n","        # print(\"Type num_item_editors {}\".format(type(num_item_editors)))\n","        # print(\"num_item_editors dims {}\".format(num_item_editors.shape))\n","\n","        jaccard_scores = (\n","            num_shared_editors.astype(float)\n","            / ((num_target_editors + num_item_editors) - num_shared_editors)\n","        ).squeeze()\n","\n","        # print(\"jaccard_scores {}\".format(jaccard_scores))\n","\n","        sorted_order = np.argsort(jaccard_scores)\n","        sorted_order = sorted_order.squeeze()\n","\n","        rec_indices = edited_indices.squeeze()[sorted_order][::-1]\n","        sorted_scores = jaccard_scores.squeeze()[sorted_order][::-1]\n","        sorted_num_shared_editors = num_shared_editors.squeeze()[sorted_order][::-1]\n","        sorted_num_item_editors = num_item_editors.squeeze()[sorted_order][::-1]\n","\n","        if title is None:\n","            return list(\n","                zip(\n","                    [self.i2p[i] for i in rec_indices[:N]],\n","                    sorted_scores[:N],\n","                    sorted_num_shared_editors[:N],\n","                    sorted_num_item_editors[:N],\n","                )\n","            )\n","        else:\n","            return list(\n","                zip(\n","                    [self.i2t[i] for i in rec_indices[:N]],\n","                    sorted_scores[:N],\n","                    sorted_num_shared_editors[:N],\n","                    sorted_num_item_editors[:N],\n","                )\n","            )"],"execution_count":2,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"jQJKv6O9yS1o"},"source":["## Test data"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"tj2e22n4xF3P","executionInfo":{"status":"ok","timestamp":1625725248341,"user_tz":-330,"elapsed":591,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"588f1118-cc30-4d47-98b8-a7831c0d089f"},"source":["implicit_matrix = np.array([[1, 1, 1, 1], [1, 1, 0, 0], [0, 0, 1, 0]])\n","assert implicit_matrix.shape == (3, 4)\n","print(implicit_matrix)\n","implicit_matrix = csc_matrix(implicit_matrix)"],"execution_count":8,"outputs":[{"output_type":"stream","text":["[[1 1 1 1]\n"," [1 1 0 0]\n"," [0 0 1 0]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"2N58ZOLXxKIa","executionInfo":{"status":"ok","timestamp":1625725119369,"user_tz":-330,"elapsed":511,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}}},"source":["n2i = {\"huey\": 0, \"dewey\": 1, \"louie\": 2, \"chewy\": 3}\n","t2i = {\"Batman\": 0, \"Mystery Men\": 1, \"Taxi Driver\": 2}\n","i2n = {v: k for k, v in n2i.items()}\n","i2t = {v: k for k, v in t2i.items()}"],"execution_count":5,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"KCiz9b0oyUtn"},"source":["## Test"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"dV28uFTRwWzp","executionInfo":{"status":"ok","timestamp":1625725182156,"user_tz":-330,"elapsed":559,"user":{"displayName":"Sparsh Agarwal","photoUrl":"","userId":"13037694610922482904"}},"outputId":"f18046bf-d3af-4d39-ec02-aac6e8bb693d"},"source":["jrec = JaccardRecommender(implicit_matrix, p2i=None, t2i=t2i, i2t=i2t, i2p=None, n2i=None, u2i=None, i2u=None)\n","print(jrec.item_to_item(N=10, title=\"Batman\"))"],"execution_count":7,"outputs":[{"output_type":"stream","text":["[('Batman', 1.0, 4, 4), ('Mystery Men', 0.5, 2, 2), ('Taxi Driver', 0.25, 1, 1)]\n"],"name":"stdout"}]}]}