notebooks/data/pruning_first_pass.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pruning_first_pass.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Santosh-Gupta/NaturalLanguageRecommendations/blob/master/pruning_first_pass.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fHUtJvY4NKqS",
        "colab_type": "code",
        "outputId": "d62f46be-4842-45ae-c0cb-862a0238453c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R339MZyXNZoH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gWx5yq3LNvmT",
        "colab_type": "code",
        "outputId": "bad63dff-b9f8-44e2-c3ac-331b937fbb9b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "import os\n",
        "os.listdir('/content/gdrive/My Drive/text2doc_data')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['test_ignore',\n",
              " 'med_data_all',\n",
              " 'cs_data_english_only',\n",
              " 'med_data_english_only']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "USBA7qHE_rJ1",
        "colab_type": "code",
        "outputId": "2ed2092b-c4fe-4977-cdda-19b78f8a39bf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 581
        }
      },
      "source": [
        "fileTemp = '/content/gdrive/My Drive/text2doc_data/cs_data_english_only/' + 's2-cleaned-corpus-{:03}.json'\n",
        "df = pd.read_json(fileTemp.format(4), orient='index')\n",
        "df.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sources3</th>\n",
              "      <th>authors</th>\n",
              "      <th>doi</th>\n",
              "      <th>doiUrl</th>\n",
              "      <th>entities</th>\n",
              "      <th>id</th>\n",
              "      <th>inCitations</th>\n",
              "      <th>journalName</th>\n",
              "      <th>journalPages</th>\n",
              "      <th>journalVolume</th>\n",
              "      <th>outCitations</th>\n",
              "      <th>paperAbstract</th>\n",
              "      <th>pdfUrls</th>\n",
              "      <th>pmid</th>\n",
              "      <th>s2PdfUrl</th>\n",
              "      <th>s2Url</th>\n",
              "      <th>sources</th>\n",
              "      <th>title</th>\n",
              "      <th>totalCites2</th>\n",
              "      <th>venue</th>\n",
              "      <th>year</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>100089</th>\n",
              "      <td>['DBLP']</td>\n",
              "      <td>[{'name': 'Yi Wan', 'ids': ['144704982']}, {'n...</td>\n",
              "      <td>10.1109/LSP.2009.2032487</td>\n",
              "      <td>https://doi.org/10.1109/LSP.2009.2032487</td>\n",
              "      <td>[]</td>\n",
              "      <td>aa7f4d7176f5a09f9d82d6b7f0029da212f9e699</td>\n",
              "      <td>[13c71cdd55940b742a3f2b11755029d7bc6fb890, 1c2...</td>\n",
              "      <td>IEEE Signal Processing Letters</td>\n",
              "      <td>71-74</td>\n",
              "      <td>17</td>\n",
              "      <td>[e4482d429ddd155f4c5b1299dec71d4faa10d241, b19...</td>\n",
              "      <td>The subject of 2-D and higher dimensional obje...</td>\n",
              "      <td>[https://doi.org/10.1109/LSP.2009.2032487]</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>https://semanticscholar.org/paper/aa7f4d7176f5...</td>\n",
              "      <td>[DBLP]</td>\n",
              "      <td>A Fast Algorithm for Recognizing Translated, R...</td>\n",
              "      <td>[e4482d429ddd155f4c5b1299dec71d4faa10d241, b19...</td>\n",
              "      <td>IEEE Signal Processing Letters</td>\n",
              "      <td>2010.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100121</th>\n",
              "      <td>['DBLP']</td>\n",
              "      <td>[{'name': 'Hossein Samimi', 'ids': ['1944983']}]</td>\n",
              "      <td>10.1007/s11277-010-0034-4</td>\n",
              "      <td>https://doi.org/10.1007/s11277-010-0034-4</td>\n",
              "      <td>[]</td>\n",
              "      <td>ba7f53381ddf889a517d14dd35d6ff40e27e9b86</td>\n",
              "      <td>[dfcf48d499c51776c5e8f0c0d23a630373ce32c9]</td>\n",
              "      <td>Wireless Personal Communications</td>\n",
              "      <td>477-490</td>\n",
              "      <td>61</td>\n",
              "      <td>[]</td>\n",
              "      <td>Among the proposed models for land mobile sate...</td>\n",
              "      <td>[https://doi.org/10.1007/s11277-010-0034-4]</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>https://semanticscholar.org/paper/ba7f53381ddf...</td>\n",
              "      <td>[DBLP]</td>\n",
              "      <td>Approximate Outage Analysis of Land Mobile Sat...</td>\n",
              "      <td>[dfcf48d499c51776c5e8f0c0d23a630373ce32c9]</td>\n",
              "      <td>Wireless Personal Communications</td>\n",
              "      <td>2011.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100189</th>\n",
              "      <td>['DBLP']</td>\n",
              "      <td>[{'name': 'Stéphane Demri', 'ids': ['2049900']...</td>\n",
              "      <td>10.1023/A:1008341521750</td>\n",
              "      <td>https://doi.org/10.1023/A%3A1008341521750</td>\n",
              "      <td>[]</td>\n",
              "      <td>b419b69e250120dfe032e47926581861713f4cd1</td>\n",
              "      <td>[ca00a00c65d090bd1b2953bcb31cf1235fceb6d2, 40f...</td>\n",
              "      <td>Journal of Logic, Language and Information</td>\n",
              "      <td>213-236</td>\n",
              "      <td>9</td>\n",
              "      <td>[3090d9c11861962103f75892f1644ca777772a03, 8c5...</td>\n",
              "      <td>We define cut-free display calculi for knowled...</td>\n",
              "      <td>[ftp://arp.anu.edu.au/pub/techreports/1998/TR-...</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>https://semanticscholar.org/paper/b419b69e2501...</td>\n",
              "      <td>[DBLP]</td>\n",
              "      <td>Display Calculi for Logics with Relative Acces...</td>\n",
              "      <td>[3090d9c11861962103f75892f1644ca777772a03, 8c5...</td>\n",
              "      <td>Journal of Logic, Language and Information</td>\n",
              "      <td>2000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10023</th>\n",
              "      <td>[]</td>\n",
              "      <td>[{'name': 'Rajat K. Deb', 'ids': ['144785806']...</td>\n",
              "      <td>10.1007/s001990050348</td>\n",
              "      <td>https://doi.org/10.1007/s001990050348</td>\n",
              "      <td>[]</td>\n",
              "      <td>955007a3cee4a7f84214daa23afaf5e1753f8801</td>\n",
              "      <td>[c36a95f845a643c4b8005de89e8cfae944bc9aed, fbe...</td>\n",
              "      <td>Economic Theory</td>\n",
              "      <td>685-689</td>\n",
              "      <td>14</td>\n",
              "      <td>[]</td>\n",
              "      <td>Summary. Serizawa [3] characterized the set of...</td>\n",
              "      <td>[]</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>https://semanticscholar.org/paper/955007a3cee4...</td>\n",
              "      <td>[]</td>\n",
              "      <td>Strategy-proof and individually rational socia...</td>\n",
              "      <td>[c36a95f845a643c4b8005de89e8cfae944bc9aed, fbe...</td>\n",
              "      <td></td>\n",
              "      <td>1999.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100380</th>\n",
              "      <td>['DBLP']</td>\n",
              "      <td>[{'name': 'Paolo Dini', 'ids': ['87349437']}, ...</td>\n",
              "      <td>10.1109/WCNC.2013.6554903</td>\n",
              "      <td>https://doi.org/10.1109/WCNC.2013.6554903</td>\n",
              "      <td>[]</td>\n",
              "      <td>03dabd8fc8af013f5098d6e85a80e0aa0b43cbd2</td>\n",
              "      <td>[]</td>\n",
              "      <td>2013 IEEE Wireless Communications and Networki...</td>\n",
              "      <td>2202-2207</td>\n",
              "      <td></td>\n",
              "      <td>[3db03bfe04b8ade3aedea43d60b34b1176bfc4ec, a18...</td>\n",
              "      <td>In this paper we consider the use of inter-pac...</td>\n",
              "      <td>[https://doi.org/10.1109/WCNC.2013.6554903]</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "      <td>https://semanticscholar.org/paper/03dabd8fc8af...</td>\n",
              "      <td>[DBLP]</td>\n",
              "      <td>Inter-packet encoding to minimize data block t...</td>\n",
              "      <td>[3db03bfe04b8ade3aedea43d60b34b1176bfc4ec, a18...</td>\n",
              "      <td>2013 IEEE Wireless Communications and Networki...</td>\n",
              "      <td>2013.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        Sources3  ...    year\n",
              "100089  ['DBLP']  ...  2010.0\n",
              "100121  ['DBLP']  ...  2011.0\n",
              "100189  ['DBLP']  ...  2000.0\n",
              "10023         []  ...  1999.0\n",
              "100380  ['DBLP']  ...  2013.0\n",
              "\n",
              "[5 rows x 21 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AoOGcoPBYnNR",
        "colab_type": "text"
      },
      "source": [
        "# Create a set up paperIDs. This will be used to filter out rows  that do not have a citation in the list "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qpv9KTM5ez0X",
        "colab_type": "code",
        "outputId": "94639065-878a-48cd-f1c1-381783e6b1e3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 629
        }
      },
      "source": [
        "fileTemp = '/content/gdrive/My Drive/text2doc_data/cs_data_english_only/' + 's2-cleaned-corpus-{:03}.json'\n",
        "paperSet = set()\n",
        "\n",
        "for i in range(179):\n",
        "    if i%5==0:\n",
        "        print(i)\n",
        "    df = pd.read_json(fileTemp.format(i), orient='index')\n",
        "    b = set(df['id'])\n",
        "    paperSet.update(b)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0\n",
            "5\n",
            "10\n",
            "15\n",
            "20\n",
            "25\n",
            "30\n",
            "35\n",
            "40\n",
            "45\n",
            "50\n",
            "55\n",
            "60\n",
            "65\n",
            "70\n",
            "75\n",
            "80\n",
            "85\n",
            "90\n",
            "95\n",
            "100\n",
            "105\n",
            "110\n",
            "115\n",
            "120\n",
            "125\n",
            "130\n",
            "135\n",
            "140\n",
            "145\n",
            "150\n",
            "155\n",
            "160\n",
            "165\n",
            "170\n",
            "175\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "53d3ec29-4103-43d5-8faf-af09133afc89",
        "id": "dDDvlKEbZNrx",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(paperSet)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1795148"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ahezvP1KZT6O",
        "colab_type": "text"
      },
      "source": [
        "# Remove any rows that does Not have a citation in the list of papers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b0aiwjn9ftXW",
        "colab_type": "code",
        "outputId": "9aad6fbf-9011-4f66-d2c1-b61f912cee1a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# Prune and save\n",
        "\n",
        "# saveTemp = gdrive_path + 'textPruned-{:03}.json'\n",
        "fileTemp = '/content/gdrive/My Drive/text2doc_data/cs_data_english_only/' + 's2-cleaned-corpus-{:03}.json'\n",
        "\n",
        "intermedTemp = 'firstP-{:03}.json'\n",
        "\n",
        "for i in range(179):\n",
        "    print(i)\n",
        "    df = pd.read_json(fileTemp.format(i), orient='index')\n",
        "    print(df.shape)\n",
        "    #pruning removed rows that do not have a paper in the dictionary \n",
        "    df2 = df[df[\"totalCites2\"].transform(lambda x: bool(set(x)&paperSet))] \n",
        "    df2.to_json( intermedTemp.format(i) )\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0\n",
            "(9912, 21)\n",
            "1\n",
            "(10177, 21)\n",
            "2\n",
            "(9911, 21)\n",
            "3\n",
            "(10244, 21)\n",
            "4\n",
            "(9993, 21)\n",
            "5\n",
            "(10204, 21)\n",
            "6\n",
            "(9998, 21)\n",
            "7\n",
            "(9994, 21)\n",
            "8\n",
            "(10230, 21)\n",
            "9\n",
            "(9745, 21)\n",
            "10\n",
            "(10248, 21)\n",
            "11\n",
            "(10077, 21)\n",
            "12\n",
            "(10059, 21)\n",
            "13\n",
            "(10016, 21)\n",
            "14\n",
            "(9880, 21)\n",
            "15\n",
            "(10180, 21)\n",
            "16\n",
            "(10076, 21)\n",
            "17\n",
            "(9941, 21)\n",
            "18\n",
            "(10049, 21)\n",
            "19\n",
            "(10029, 21)\n",
            "20\n",
            "(9945, 21)\n",
            "21\n",
            "(10206, 21)\n",
            "22\n",
            "(10106, 21)\n",
            "23\n",
            "(9940, 21)\n",
            "24\n",
            "(10001, 21)\n",
            "25\n",
            "(10021, 21)\n",
            "26\n",
            "(10035, 21)\n",
            "27\n",
            "(10093, 21)\n",
            "28\n",
            "(9932, 21)\n",
            "29\n",
            "(9998, 21)\n",
            "30\n",
            "(9831, 21)\n",
            "31\n",
            "(10038, 21)\n",
            "32\n",
            "(9925, 21)\n",
            "33\n",
            "(9945, 21)\n",
            "34\n",
            "(10121, 21)\n",
            "35\n",
            "(9932, 21)\n",
            "36\n",
            "(9829, 21)\n",
            "37\n",
            "(10008, 21)\n",
            "38\n",
            "(9879, 21)\n",
            "39\n",
            "(10008, 21)\n",
            "40\n",
            "(10072, 21)\n",
            "41\n",
            "(10028, 21)\n",
            "42\n",
            "(10097, 21)\n",
            "43\n",
            "(9960, 21)\n",
            "44\n",
            "(10034, 21)\n",
            "45\n",
            "(10159, 21)\n",
            "46\n",
            "(10048, 21)\n",
            "47\n",
            "(10009, 21)\n",
            "48\n",
            "(10132, 21)\n",
            "49\n",
            "(9947, 21)\n",
            "50\n",
            "(10066, 21)\n",
            "51\n",
            "(10016, 21)\n",
            "52\n",
            "(10060, 21)\n",
            "53\n",
            "(10023, 21)\n",
            "54\n",
            "(10071, 21)\n",
            "55\n",
            "(9861, 21)\n",
            "56\n",
            "(10141, 21)\n",
            "57\n",
            "(9950, 21)\n",
            "58\n",
            "(10056, 21)\n",
            "59\n",
            "(10094, 21)\n",
            "60\n",
            "(10013, 21)\n",
            "61\n",
            "(10005, 21)\n",
            "62\n",
            "(10017, 21)\n",
            "63\n",
            "(9982, 21)\n",
            "64\n",
            "(9834, 21)\n",
            "65\n",
            "(10059, 21)\n",
            "66\n",
            "(10164, 21)\n",
            "67\n",
            "(9939, 21)\n",
            "68\n",
            "(10169, 21)\n",
            "69\n",
            "(10026, 21)\n",
            "70\n",
            "(10190, 21)\n",
            "71\n",
            "(10029, 21)\n",
            "72\n",
            "(9972, 21)\n",
            "73\n",
            "(10257, 21)\n",
            "74\n",
            "(10017, 21)\n",
            "75\n",
            "(9858, 21)\n",
            "76\n",
            "(10179, 21)\n",
            "77\n",
            "(10062, 21)\n",
            "78\n",
            "(10042, 21)\n",
            "79\n",
            "(9843, 21)\n",
            "80\n",
            "(9833, 21)\n",
            "81\n",
            "(9959, 21)\n",
            "82\n",
            "(9958, 21)\n",
            "83\n",
            "(10037, 21)\n",
            "84\n",
            "(10122, 21)\n",
            "85\n",
            "(10087, 21)\n",
            "86\n",
            "(10023, 21)\n",
            "87\n",
            "(10099, 21)\n",
            "88\n",
            "(10189, 21)\n",
            "89\n",
            "(9793, 21)\n",
            "90\n",
            "(10005, 21)\n",
            "91\n",
            "(10078, 21)\n",
            "92\n",
            "(10198, 21)\n",
            "93\n",
            "(10164, 21)\n",
            "94\n",
            "(10051, 21)\n",
            "95\n",
            "(10186, 21)\n",
            "96\n",
            "(10187, 21)\n",
            "97\n",
            "(9982, 21)\n",
            "98\n",
            "(10031, 21)\n",
            "99\n",
            "(9874, 21)\n",
            "100\n",
            "(9856, 21)\n",
            "101\n",
            "(9964, 21)\n",
            "102\n",
            "(10018, 21)\n",
            "103\n",
            "(9836, 21)\n",
            "104\n",
            "(9969, 21)\n",
            "105\n",
            "(10038, 21)\n",
            "106\n",
            "(9973, 21)\n",
            "107\n",
            "(10040, 21)\n",
            "108\n",
            "(9935, 21)\n",
            "109\n",
            "(10099, 21)\n",
            "110\n",
            "(9994, 21)\n",
            "111\n",
            "(9882, 21)\n",
            "112\n",
            "(9845, 21)\n",
            "113\n",
            "(9992, 21)\n",
            "114\n",
            "(10120, 21)\n",
            "115\n",
            "(10017, 21)\n",
            "116\n",
            "(10018, 21)\n",
            "117\n",
            "(9990, 21)\n",
            "118\n",
            "(10238, 21)\n",
            "119\n",
            "(9946, 21)\n",
            "120\n",
            "(10027, 21)\n",
            "121\n",
            "(10073, 21)\n",
            "122\n",
            "(10046, 21)\n",
            "123\n",
            "(10057, 21)\n",
            "124\n",
            "(10055, 21)\n",
            "125\n",
            "(10059, 21)\n",
            "126\n",
            "(10141, 21)\n",
            "127\n",
            "(9868, 21)\n",
            "128\n",
            "(10153, 21)\n",
            "129\n",
            "(10022, 21)\n",
            "130\n",
            "(10022, 21)\n",
            "131\n",
            "(9995, 21)\n",
            "132\n",
            "(10150, 21)\n",
            "133\n",
            "(9896, 21)\n",
            "134\n",
            "(10169, 21)\n",
            "135\n",
            "(10075, 21)\n",
            "136\n",
            "(10108, 21)\n",
            "137\n",
            "(10132, 21)\n",
            "138\n",
            "(9993, 21)\n",
            "139\n",
            "(10030, 21)\n",
            "140\n",
            "(9900, 21)\n",
            "141\n",
            "(10140, 21)\n",
            "142\n",
            "(9978, 21)\n",
            "143\n",
            "(10216, 21)\n",
            "144\n",
            "(9977, 21)\n",
            "145\n",
            "(9962, 21)\n",
            "146\n",
            "(9973, 21)\n",
            "147\n",
            "(10023, 21)\n",
            "148\n",
            "(9922, 21)\n",
            "149\n",
            "(10079, 21)\n",
            "150\n",
            "(10143, 21)\n",
            "151\n",
            "(9961, 21)\n",
            "152\n",
            "(10149, 21)\n",
            "153\n",
            "(10019, 21)\n",
            "154\n",
            "(10079, 21)\n",
            "155\n",
            "(10155, 21)\n",
            "156\n",
            "(10019, 21)\n",
            "157\n",
            "(9830, 21)\n",
            "158\n",
            "(10027, 21)\n",
            "159\n",
            "(10114, 21)\n",
            "160\n",
            "(10187, 21)\n",
            "161\n",
            "(9967, 21)\n",
            "162\n",
            "(9810, 21)\n",
            "163\n",
            "(9861, 21)\n",
            "164\n",
            "(10215, 21)\n",
            "165\n",
            "(10017, 21)\n",
            "166\n",
            "(10003, 21)\n",
            "167\n",
            "(9988, 21)\n",
            "168\n",
            "(10101, 21)\n",
            "169\n",
            "(9967, 21)\n",
            "170\n",
            "(9904, 21)\n",
            "171\n",
            "(9873, 21)\n",
            "172\n",
            "(10301, 21)\n",
            "173\n",
            "(9934, 21)\n",
            "174\n",
            "(10013, 21)\n",
            "175\n",
            "(10073, 21)\n",
            "176\n",
            "(10078, 21)\n",
            "177\n",
            "(10055, 21)\n",
            "178\n",
            "(10201, 21)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NQbgq99vZirA",
        "colab_type": "text"
      },
      "source": [
        "# Create a new set of PaperIDs "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C3v1DhVuEM0Z",
        "colab_type": "code",
        "outputId": "6feab30f-05aa-4674-b859-712417615e5a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 639
        }
      },
      "source": [
        "intermedTemp = 'firstP-{:03}.json'\n",
        "paperSet = set()\n",
        "\n",
        "for i in range(179):\n",
        "    if i%5==0:\n",
        "        print(i)\n",
        "    df = pd.read_json(intermedTemp.format(i))\n",
        "    b = set(df['id'])\n",
        "    paperSet.update(b)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0\n",
            "5\n",
            "10\n",
            "15\n",
            "20\n",
            "25\n",
            "30\n",
            "35\n",
            "40\n",
            "45\n",
            "50\n",
            "55\n",
            "60\n",
            "65\n",
            "70\n",
            "75\n",
            "80\n",
            "85\n",
            "90\n",
            "95\n",
            "100\n",
            "105\n",
            "110\n",
            "115\n",
            "120\n",
            "125\n",
            "130\n",
            "135\n",
            "140\n",
            "145\n",
            "150\n",
            "155\n",
            "160\n",
            "165\n",
            "170\n",
            "175\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "czQakVIhyCkb",
        "colab_type": "code",
        "outputId": "0caaed67-ad11-419d-b5a1-b0d3f3b5eff8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(paperSet)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1262996"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "32T7ewtzZ_w4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Map a unique EmbeddingID for each paperID"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4EgzQIjXETSh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "paperID2EmbeddingID = {ni: indi for indi, ni in enumerate(paperSet)}\n",
        "EmbeddingID2paperID = {indi: ni for indi, ni in enumerate(paperSet)}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qmZ92DjqN8DO",
        "colab_type": "code",
        "outputId": "2c232770-06fc-4992-b8fb-a9dfc8b7f32a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(paperID2EmbeddingID)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1262996"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bv4p9-PyOBgd",
        "colab_type": "code",
        "outputId": "7c64f1cf-cf02-4a12-f3ad-97688e0bd2a3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(set(paperID2EmbeddingID))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1262996"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ddJ3oS_9OEyL",
        "colab_type": "code",
        "outputId": "3908f5b1-f83b-45d6-a3e9-2e2100b94873",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(set(EmbeddingID2paperID))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1262996"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2WBFYPiON-Rq",
        "colab_type": "code",
        "outputId": "29b93018-a508-493e-e64e-98d7e3ce20f8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(EmbeddingID2paperID)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1262996"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p5PHB8x5QHq2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "paperID2EmbeddingID"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G5RyS67SaMPO",
        "colab_type": "text"
      },
      "source": [
        "# Save the paperID2EMbeddingID dictionary to your Gdrive. This will be used to retrieve papers from the EmbedID at inference"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1rm_TESLrWMZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import json\n",
        "\n",
        "# gdrive_path = ''\n",
        "gdrive_path =  '/content/gdrive/My Drive/NaturalLanguageRecommendations/CS/' \n",
        "\n",
        "# Save paperID2EmbeddingID Dictionary to json\n",
        "with open(gdrive_path+'paperID2EmbeddingID.json', 'w') as fp:\n",
        "    json.dump(paperID2EmbeddingID, fp)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UZaeCeSKQ9EE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Check if it saved correctly\n",
        "\n",
        "import json\n",
        "\n",
        "with open(gdrive_path+'paperID2EmbeddingID.json') as json_file:\n",
        "    data = json.load(json_file)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ojKEGpF6akMG",
        "colab_type": "text"
      },
      "source": [
        "# Open intermediate files and map the paperID's in the 'id' and citation columns to EmbeddingIDs. Save to 2nd intermediate file. You might want to save this to your gdrive account instead of on the colab instance, because there's a good chance the instance may crash due to running out of memory, and you may need to switch instances. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RNDDT25blfXo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# saveTemp = gdrive_path + 'textPruned-{:03}.json'\n",
        "\n",
        "intermedTemp = 'firstP-{:03}.json'\n",
        "\n",
        "# gdriveTemp = '/content/gdrive/My Drive/NaturalLanguageRecommendations/CS/firstPruneCorrected-{:03}.json'\n",
        "\n",
        "intermedTemp2 = 'firstP2-{:03}.json'\n",
        "\n",
        "# intermedTemp2 = '/content/gdrive/My Drive/NaturalLanguageRecommendations/CS/firstPruneCorrected-{:03}.json'\n",
        "\n",
        "for i in range(179):\n",
        "    dfAfterPruned = pd.read_json(intermedTemp.format(i))\n",
        "\n",
        "    df3 = dfAfterPruned.assign(citEmebeddingIds=[[paperID2EmbeddingID[k] for k in row if paperID2EmbeddingID.get(k)] for row in dfAfterPruned[\"totalCites2\"]])\n",
        "    df3['EmbeddingID'] = df3['id'].map(paperID2EmbeddingID)\n",
        "    # Save \n",
        "    df3[['EmbeddingID', 'paperAbstract', 'title', 'citEmebeddingIds']].to_json( intermedTemp2.format(i) )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0t_d9czcONzM",
        "colab_type": "code",
        "outputId": "eb70dd33-0815-4c75-a227-c7b9dceff7ce",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "EmbeddingID2paperID[33645]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'88e2bbd9b1b3113c88e8c3c60ae88b81e2da1807'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "43F-Bm4CO0oI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pd.set_option('max_colwidth', 800)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DLFXZLUzbS4G",
        "colab_type": "text"
      },
      "source": [
        "# Combine all dataframe files. You may not have enough RAM for this. If it crashes, you might get an option to restart in a higher RAM instance, which would be 25 gb in the CPU and GPU instances. If you saved the 2nd intermediate files in you Google drive, you can just start from this step. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r_2hiGnbcQFE",
        "colab_type": "text"
      },
      "source": [
        "# If this crashes again, then you can switch to the TPU instance, and crash it. When you get the option for a higher ram instance, the TPU one will have 36 GB instead of 25 GB. If it crashes again, let me know, I'll alter the code to process one file at a time"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6qSXfClVmCKH",
        "colab_type": "code",
        "outputId": "de1a2296-21b3-42c5-cf4c-e5ad543aeeb6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 639
        }
      },
      "source": [
        "afColl = []\n",
        "for idx in range(179):\n",
        "    if idx%5==0:\n",
        "        print(idx)\n",
        "    afColl.append( pd.read_json( intermedTemp2.format(idx) ) )\n",
        "\n",
        "allDf = pd.concat(afColl)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0\n",
            "5\n",
            "10\n",
            "15\n",
            "20\n",
            "25\n",
            "30\n",
            "35\n",
            "40\n",
            "45\n",
            "50\n",
            "55\n",
            "60\n",
            "65\n",
            "70\n",
            "75\n",
            "80\n",
            "85\n",
            "90\n",
            "95\n",
            "100\n",
            "105\n",
            "110\n",
            "115\n",
            "120\n",
            "125\n",
            "130\n",
            "135\n",
            "140\n",
            "145\n",
            "150\n",
            "155\n",
            "160\n",
            "165\n",
            "170\n",
            "175\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E67YW3XuI3Zu",
        "colab_type": "code",
        "outputId": "20df5632-a58e-4519-b96f-bffba6e7cc75",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "allDf.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1262996, 4)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hwrB8kyJXEZV",
        "colab_type": "code",
        "outputId": "1a422813-0839-4be2-8ca0-5b6b178c1c06",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# bb = set(allDf['EmbeddingID'])\n",
        "# len(bb)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1262996"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AlIVOxhEWHMb",
        "colab_type": "code",
        "outputId": "94b5f667-4c36-48b3-e918-07e74f3926c3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 407
        }
      },
      "source": [
        "allDf"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>EmbeddingID</th>\n",
              "      <th>paperAbstract</th>\n",
              "      <th>title</th>\n",
              "      <th>citEmebeddingIds</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>100113</th>\n",
              "      <td>235092</td>\n",
              "      <td>This paper proposes simple analytic formulas f...</td>\n",
              "      <td>Simple analytic formulas for PID tuning</td>\n",
              "      <td>[931311, 1018430, 523305, 822033, 529623, 9381...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100534</th>\n",
              "      <td>868823</td>\n",
              "      <td>Due to the advantages of high efficiency and h...</td>\n",
              "      <td>Effect of structure parameters on the losses a...</td>\n",
              "      <td>[336997]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100756</th>\n",
              "      <td>350063</td>\n",
              "      <td>The recently developed IEEE 802.11ad (WiGig) s...</td>\n",
              "      <td>Saturated Throughput Analysis of IEEE 802.11ad...</td>\n",
              "      <td>[153164, 395362, 294290, 189116, 178737, 84760...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>101147</th>\n",
              "      <td>664061</td>\n",
              "      <td>Position localization is an important problem ...</td>\n",
              "      <td>A Fuzzy Logic Algorithm for Minimizing Error (...</td>\n",
              "      <td>[313973, 591548, 571510, 479610, 760686]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>101286</th>\n",
              "      <td>884696</td>\n",
              "      <td>This paper presents a modified CMOS implementa...</td>\n",
              "      <td>FVF based CDTA and Its application</td>\n",
              "      <td>[970299, 8221, 1207310]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996022</th>\n",
              "      <td>519998</td>\n",
              "      <td>A multi-band (bands I, II, V, VIII [option]) s...</td>\n",
              "      <td>A multi-band high performance single-chip tran...</td>\n",
              "      <td>[383204, 1174990]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99698</th>\n",
              "      <td>1022390</td>\n",
              "      <td>We propose a data-driven method to realize hig...</td>\n",
              "      <td>Data-driven detailed hair animation for game c...</td>\n",
              "      <td>[1026226]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99709</th>\n",
              "      <td>946513</td>\n",
              "      <td>Fire monitoring and evacuation for building en...</td>\n",
              "      <td>A real-time and robust routing protocol for bu...</td>\n",
              "      <td>[87528, 1252523, 1155882, 204501]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9989</th>\n",
              "      <td>296967</td>\n",
              "      <td>We propose a digital optical phase locked loop...</td>\n",
              "      <td>Digital optical phase locked loop for real-tim...</td>\n",
              "      <td>[953075, 913188, 307815]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99984</th>\n",
              "      <td>35043</td>\n",
              "      <td>We are in the era of artificial intelligence w...</td>\n",
              "      <td>A study on merging mechanisms of simple hopfie...</td>\n",
              "      <td>[77923]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1262996 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        EmbeddingID  ...                                   citEmebeddingIds\n",
              "100113       235092  ...  [931311, 1018430, 523305, 822033, 529623, 9381...\n",
              "100534       868823  ...                                           [336997]\n",
              "100756       350063  ...  [153164, 395362, 294290, 189116, 178737, 84760...\n",
              "101147       664061  ...           [313973, 591548, 571510, 479610, 760686]\n",
              "101286       884696  ...                            [970299, 8221, 1207310]\n",
              "...             ...  ...                                                ...\n",
              "996022       519998  ...                                  [383204, 1174990]\n",
              "99698       1022390  ...                                          [1026226]\n",
              "99709        946513  ...                  [87528, 1252523, 1155882, 204501]\n",
              "9989         296967  ...                           [953075, 913188, 307815]\n",
              "99984         35043  ...                                            [77923]\n",
              "\n",
              "[1262996 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EIR29gjncviN",
        "colab_type": "text"
      },
      "source": [
        "# Reset the index, then delete the extra index column"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P4RqWP4GB22I",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "allDf.reset_index(inplace =True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QF5neXMmB80d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "del allDf['index']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v9A_OsmysGXh",
        "colab_type": "code",
        "outputId": "738b0870-f7cb-4452-f1f1-dc71b3f496c3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        }
      },
      "source": [
        "allDf.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>EmbeddingID</th>\n",
              "      <th>paperAbstract</th>\n",
              "      <th>title</th>\n",
              "      <th>citEmebeddingIds</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>235092</td>\n",
              "      <td>This paper proposes simple analytic formulas f...</td>\n",
              "      <td>Simple analytic formulas for PID tuning</td>\n",
              "      <td>[931311, 1018430, 523305, 822033, 529623, 9381...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>868823</td>\n",
              "      <td>Due to the advantages of high efficiency and h...</td>\n",
              "      <td>Effect of structure parameters on the losses a...</td>\n",
              "      <td>[336997]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>350063</td>\n",
              "      <td>The recently developed IEEE 802.11ad (WiGig) s...</td>\n",
              "      <td>Saturated Throughput Analysis of IEEE 802.11ad...</td>\n",
              "      <td>[153164, 395362, 294290, 189116, 178737, 84760...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>664061</td>\n",
              "      <td>Position localization is an important problem ...</td>\n",
              "      <td>A Fuzzy Logic Algorithm for Minimizing Error (...</td>\n",
              "      <td>[313973, 591548, 571510, 479610, 760686]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>884696</td>\n",
              "      <td>This paper presents a modified CMOS implementa...</td>\n",
              "      <td>FVF based CDTA and Its application</td>\n",
              "      <td>[970299, 8221, 1207310]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   EmbeddingID  ...                                   citEmebeddingIds\n",
              "0       235092  ...  [931311, 1018430, 523305, 822033, 529623, 9381...\n",
              "1       868823  ...                                           [336997]\n",
              "2       350063  ...  [153164, 395362, 294290, 189116, 178737, 84760...\n",
              "3       664061  ...           [313973, 591548, 571510, 479610, 760686]\n",
              "4       884696  ...                            [970299, 8221, 1207310]\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GJLHCfmlc-ef",
        "colab_type": "text"
      },
      "source": [
        "# Remove duplicates?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VnrvuvTYEQpN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "allDf.drop_duplicates(subset=['EmbeddingID'],  keep='first', inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sz0tbYuLcpLi",
        "colab_type": "code",
        "outputId": "1dbbe7c0-8368-4d5a-a952-3c798d6ca617",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "#check how many removed\n",
        "\n",
        "allDf.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1262996, 4)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6h1dw9ZydtR6",
        "colab_type": "text"
      },
      "source": [
        "# Create the HDF5 dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k1rpbPnwsJ5Q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "allNums = allDf[['EmbeddingID', 'citEmebeddingIds']]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zZnPY-HVswgX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BeojJCsfsxmv",
        "colab_type": "code",
        "outputId": "52da9745-1b94-4363-ae0a-96312cc61c95",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        }
      },
      "source": [
        "allNums['citEmebeddingIds'] = allNums['citEmebeddingIds'].apply(lambda x: np.array(x))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q3EhQzxXteEv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "numpyAll = allNums.values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eYbJWUkasrnp",
        "colab_type": "code",
        "outputId": "7e3ba8cf-12f9-4f24-f20d-abc7f6c10778",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 276
        }
      },
      "source": [
        "numpyAll[0:10]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[235092,\n",
              "        array([ 931311, 1018430,  523305,  822033,  529623,  938165,  927913])],\n",
              "       [868823, array([336997])],\n",
              "       [350063,\n",
              "        array([ 153164,  395362,  294290,  189116,  178737,  847601,  528571,\n",
              "        552113,  901146, 1071567,  439787])],\n",
              "       [664061, array([313973, 591548, 571510, 479610, 760686])],\n",
              "       [884696, array([ 970299,    8221, 1207310])],\n",
              "       [969635, array([353121, 263616, 243836])],\n",
              "       [418364,\n",
              "        array([ 422597,  507650, 1258624,  936568,  612901,  418330, 1222164,\n",
              "        391924, 1095247,   84533])],\n",
              "       [531667, array([ 785602,  947367, 1256856])],\n",
              "       [1827, array([ 809294, 1245639])],\n",
              "       [506588, array([951349, 208114])]], dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U7w2hq1qd1dL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BRdkb4xx--gz",
        "colab_type": "code",
        "outputId": "5be94e72-1ce0-4b2a-8fa9-b9588a882a9a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        }
      },
      "source": [
        "numpyAll[:,1]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([array([ 931311, 1018430,  523305,  822033,  529623,  938165,  927913]),\n",
              "       array([336997]),\n",
              "       array([ 153164,  395362,  294290,  189116,  178737,  847601,  528571,\n",
              "        552113,  901146, 1071567,  439787]),\n",
              "       ..., array([  87528, 1252523, 1155882,  204501]),\n",
              "       array([953075, 913188, 307815]), array([77923])], dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BPUSFLl8c5w5",
        "colab_type": "code",
        "outputId": "5f9edbee-0a69-415b-9520-f102e4350136",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "np.max(numpyAll[:,0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1262995"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "90dbf80e-8a0d-4d57-c29f-21704d6e3a39",
        "id": "pgqDi_KRd4VT",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# np.max(numpyAll[:,0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1262790"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 77
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aLbwTKhO15pU",
        "colab_type": "code",
        "outputId": "d64898b1-e401-4218-e7d2-382ac34234e6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "numpyAll[:,0].max()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1262995"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lZikHDrhsveG",
        "colab_type": "code",
        "outputId": "df803473-ca55-4b86-faa6-1951cdbb526f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(numpyAll)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1262996"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FrJUKF5Vd8lS",
        "colab_type": "text"
      },
      "source": [
        "# Save to gdrive and delete allDf to save memory "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_lK3FDvsd6k-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "allDf[['EmbeddingID', 'paperAbstract', 'title']].to_json( '/content/gdrive/My Drive/NaturalLanguageRecommendations/CS/TitlesAbstractsEmbedIds.json.gzip', compression = 'gzip' )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PV2rTUc2eBKd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "del allDf"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ViC2DbveeGZJ",
        "colab_type": "text"
      },
      "source": [
        "# Create and save HDF5 datateset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gb7376FBtnc9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import h5py\n",
        "h5f = h5py.File('/content/gdrive/My Drive/NaturalLanguageRecommendations/CS/SemanticScholarMed1p26M-CS-12-26-19.h5', 'w')\n",
        "dt = h5py.special_dtype(vlen=np.dtype('int32'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cp4UugSAtsFw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "h5f.create_dataset('cites', data=numpyAll[:,1], dtype=dt, compression='gzip', compression_opts=9, chunks=True, maxshape=(None,) )\n",
        "h5f.create_dataset('paper', data=numpyAll[:,0].astype(np.int32), compression='gzip', compression_opts=9, chunks=True, maxshape=(None,) )\n",
        "h5f.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UFk9gh3Nt3Ge",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}