scripts/PageRankRating.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 559
    },
    "colab_type": "code",
    "id": "mwDpQGn4SLL1",
    "outputId": "35e1036b-f778-497f-a872-e1b93eebb703"
   },
   "outputs": [],
   "source": [
    "!rm -f ru_tg_train.tar.gz\n",
    "!wget https://www.dropbox.com/s/1ecl9orr2tagcgi/ru_tg_train.tar.gz\n",
    "!rm -f ru_tg_train.json\n",
    "!tar -xzvf ru_tg_train.tar.gz\n",
    "!rm ru_tg_train.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 559
    },
    "colab_type": "code",
    "id": "nhBjAoQ2Gwni",
    "outputId": "6280bc40-d82d-4a33-a66b-c824fe8beb74"
   },
   "outputs": [],
   "source": [
    "!rm -f en_tg_train.tar.gz\n",
    "!wget https://www.dropbox.com/s/umd8tyx4wz1wquq/en_tg_train.tar.gz\n",
    "!rm -f en_tg_train.json\n",
    "!tar -xzvf en_tg_train.tar.gz\n",
    "!rm en_tg_train.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 127
    },
    "colab_type": "code",
    "id": "-SUTRNosH3yP",
    "outputId": "3723939c-1882-495f-faee-58e7985f5df1"
   },
   "outputs": [],
   "source": [
    "!pip install url-normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yUQPs1FNG15N"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import json\n",
    "from url_normalize import url_normalize\n",
    "from urllib.parse import urlparse   \n",
    "\n",
    "\n",
    "def normalize_url(url):\n",
    "    url_n = url_normalize(url)\n",
    "    parsed = urlparse(url_n)\n",
    "    scheme = \"%s://\" % parsed.scheme\n",
    "    res = parsed.geturl().replace(scheme, '', 1)\n",
    "    host = parsed.netloc\n",
    "    if res.startswith('www.'):\n",
    "        res = res.replace('www.', '')\n",
    "        host = host.replace('www.', '')\n",
    "    return res, host\n",
    "\n",
    "data_files = [json.load(open(\"en_tg_train.json\")), json.load(open(\"ru_tg_train.json\"))]\n",
    "\n",
    "url_set = set()\n",
    "host_set = set()\n",
    "for data in data_files:\n",
    "    for row in data:\n",
    "        url = row[\"url\"]\n",
    "        url_n, url_host = normalize_url(url)\n",
    "        url_set.add(url_n)\n",
    "        host_set.add(url_host)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cH7nJXdOHZuc"
   },
   "outputs": [],
   "source": [
    "from collections import Counter, defaultdict\n",
    "\n",
    "count = Counter()\n",
    "host_to_host = defaultdict(Counter)\n",
    "for data in data_files:\n",
    "    for row in data:\n",
    "        url = row[\"url\"]\n",
    "        url_n, url_host = normalize_url(url)\n",
    "        if \"out_links\" not in row:\n",
    "            continue\n",
    "        for link in row[\"out_links\"]:\n",
    "            link_n, link_host = normalize_url(link)\n",
    "            if url_host != link_host and link_n in url_set:\n",
    "                count[link] += 1\n",
    "            if url_host != link_host and link_host in host_set:\n",
    "                host_to_host[url_host][link_host] += 1\n",
    "\n",
    "h2i = {}\n",
    "i2h = []\n",
    "for host in host_to_host:\n",
    "    h2i[host] = len(i2h)\n",
    "    i2h.append(host)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "EpVrR8n3IVb8",
    "outputId": "2a06fd31-a4d6-4832-a5c9-6bc2abe48a52"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "size = len(i2h)\n",
    "E = np.ones((size, size))\n",
    "E = E / np.sum(E, axis=1, keepdims=True)\n",
    "print(E)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "WSuQ-mzaIZro",
    "outputId": "0701bba6-34f4-4ef1-871b-fac5b99fa33f"
   },
   "outputs": [],
   "source": [
    "matrix = np.zeros((size, size))\n",
    "\n",
    "for host in host_to_host:\n",
    "    for link_host in host_to_host[host]:\n",
    "        if link_host not in h2i:\n",
    "            continue\n",
    "        matrix[h2i[host]][h2i[link_host]] = host_to_host[host][link_host]\n",
    "print(matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "hOemXEtYIaGH",
    "outputId": "413576fd-d2e0-4bbc-bea7-601454a0d46a"
   },
   "outputs": [],
   "source": [
    "for i in range(size):\n",
    "    if np.sum(matrix[i]) == 0:\n",
    "        matrix[i] = np.ones(size)\n",
    "print(matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "BgPF2X7VIffi",
    "outputId": "21a7050e-c284-4201-d753-0e34d093937e"
   },
   "outputs": [],
   "source": [
    "matrix = matrix / np.sum(matrix, axis=1, keepdims=True)\n",
    "matrix = 0.7 * matrix + 0.3 * E\n",
    "print(matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "pagJcs2eIhxW",
    "outputId": "dc20deb4-5684-4e85-a92b-dc6c5d712454"
   },
   "outputs": [],
   "source": [
    "x = np.ones((1, size))\n",
    "x = x / np.sum(x, axis=1, keepdims=True)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "n2ou7HjRIkc2",
    "outputId": "c7de356f-f8a0-42e9-dcf0-ec5e840c4428"
   },
   "outputs": [],
   "source": [
    "y = np.dot(x, np.linalg.matrix_power(matrix, 20))\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "dDSIKRW3Il5q",
    "outputId": "fe9356b2-828f-4964-9bf3-978f6526e189"
   },
   "outputs": [],
   "source": [
    "rating = []\n",
    "for i, host in enumerate(i2h):\n",
    "    rating.append((i, y[0][i], host))\n",
    "rating = sorted(rating, key=lambda x:-x[1])\n",
    "for i, pr, host in rating:\n",
    "    print(pr, host)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "PageRankRating.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}