notebooks/data/medical_preprocessing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "medical_preprocessing.ipynb",
      "provenance": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Santosh-Gupta/NaturalLanguageRecommendations/blob/master/medical_preprocessing.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9O1eOO0FjQYR",
        "colab_type": "code",
        "outputId": "90190b07-57cf-48af-95b8-9c00f502213f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SWD-VJLXjeKB",
        "colab_type": "code",
        "outputId": "84abc25f-1eb5-4614-8e0a-faa53cea2009",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 231
        }
      },
      "source": [
        "!pip install langid\n",
        "!pip install tqdm"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting langid\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ea/4c/0fb7d900d3b0b9c8703be316fbddffecdab23c64e1b46c7a83561d78bd43/langid-1.1.6.tar.gz (1.9MB)\n",
            "\u001b[K     |████████████████████████████████| 1.9MB 4.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from langid) (1.17.4)\n",
            "Building wheels for collected packages: langid\n",
            "  Building wheel for langid (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for langid: filename=langid-1.1.6-cp36-none-any.whl size=1941190 sha256=699bf2d6b835147f8fff54168cb6831f3786472bf5c66de0e144b1ea52b4847f\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/bc/61/50a93be85d1afe9436c3dc61f38da8ad7b637a38af4824e86e\n",
            "Successfully built langid\n",
            "Installing collected packages: langid\n",
            "Successfully installed langid-1.1.6\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (4.28.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uv-KNIA_jf_Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import zipfile\n",
        "import os\n",
        "import sys\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import gc\n",
        "from urllib import request\n",
        "import json\n",
        "import itertools\n",
        "import gzip\n",
        "import shutil\n",
        "import ast\n",
        "import pickle \n",
        "import nltk\n",
        "from langid.langid import LanguageIdentifier, model\n",
        "from tqdm import tqdm"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I6iS_D19jlGx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Function to unip a file from semantic scholar to a json file, then open that json file\n",
        "def dlandopen(url, file, json ):\n",
        "    request.urlretrieve(url, file )\n",
        "    with gzip.open(file, 'rb') as f_in:\n",
        "        with open(json, 'wb') as f_out:\n",
        "            shutil.copyfileobj(f_in, f_out)\n",
        "\n",
        "    os.remove(file)\n",
        "    f = open(json)\n",
        "    mylistH = f.readlines()\n",
        "    del f\n",
        "    return mylistH"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "35eLVFI8jpMZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# This is probably not the best way to go about it, but this is the code to convert the opened file into a pandas dataframe\n",
        "#Akul, it may be worth figuring out if you can make a dataframe directly, without going line-by-line \n",
        "def firstDataframe(mylist, start, finish):\n",
        "    data = []\n",
        "\n",
        "    for line in itertools.islice(mylist , start , finish):\n",
        "       data.append(json.loads(line))\n",
        "\n",
        "    df = pd.DataFrame(data)\n",
        "    del data\n",
        "    gc.collect()\n",
        "    return df"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V0YBFdwGjrPX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def medicalRefinement(dftbr):\n",
        "  dftbr['strsources'] = dftbr['sources'].astype('str')\n",
        "  dftbr['totalCites2'] = dftbr['outCitations'] + dftbr['inCitations'] \n",
        "  dftbr = dftbr[dftbr['totalCites2'].str.len() >= 1]  #outCitations - all papers where papers cited is greater than 1\n",
        "  dftbr = dftbr[ (dftbr.pmid != '' ) | (dftbr.strsources.str.contains(\"Medline\"))]\n",
        "  dftbr.drop(['journalVolume','journalPages','year','authors','sources','doiUrl', 'strsources'], axis = 1)\n",
        "  return dftbr"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jBlLOuCNju3q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def remove_nonenglish(df, jSet, vSet):\n",
        "  # Gets list of English Journals\n",
        "  listOfEnglishJournals = []\n",
        "  identifier = LanguageIdentifier.from_modelstring(model, norm_probs=True)\n",
        "  count = 0\n",
        "  for i in jSet:\n",
        "    lang = identifier.classify(str(i))\n",
        "    langstr = str(lang)\n",
        "    if langstr.find(\"en\") != -1 and lang[1] > 0.5:\n",
        "      listOfEnglishJournals.append(i)\n",
        "  \n",
        "  # Gets List of English Venues\n",
        "  listOfEnglishVenues = []\n",
        "  count = 0\n",
        "  for i in vSet:\n",
        "    lang = identifier.classify(str(i))\n",
        "    langstr = str(lang)\n",
        "    if langstr.find(\"en\") != -1 and lang[1] > 0.5:\n",
        "      listOfEnglishVenues.append(i)\n",
        "\n",
        "  filter1 = df_refined['journalName'].isin(listOfEnglishJournals)\n",
        "  filter2 = df_refined['venue'].isin(listOfEnglishVenues)\n",
        "\n",
        "  dfFinal = df_refined[df_refined['journalName'].isin(listOfEnglishJournals) | df_refined['venue'].isin(listOfEnglishVenues)]\n",
        "\n",
        "  return dfFinal"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r5OmTsAFj0C0",
        "colab_type": "code",
        "outputId": "7f71cdc8-fab7-483c-aa6f-64682fc43813",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 390
        }
      },
      "source": [
        "url_template = 'https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2019-11-01/s2-corpus-{:03}.gz'\n",
        "file_template = 'json{:03}.gz'\n",
        "json_template = 'json{:03}'\n",
        "\n",
        "i=55\n",
        "mylist = dlandopen(url_template.format(i), file_template.format(i), json_template.format(i) )\n",
        "df = firstDataframe(mylist, 0, len(mylist))\n",
        "df_refined = medicalRefinement(df)\n",
        "print(\"REFINED\")"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-8-9643aa3bccc5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m55\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mmylist\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdlandopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl_template\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile_template\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson_template\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfirstDataframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmylist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmylist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0mdf_refined\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmedicalRefinement\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"REFINED\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-5-375744c40b10>\u001b[0m in \u001b[0;36mfirstDataframe\u001b[0;34m(mylist, start, finish)\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mitertools\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mislice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmylist\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mstart\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mfinish\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m        \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.6/json/__init__.py\u001b[0m in \u001b[0;36mloads\u001b[0;34m(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m    352\u001b[0m             \u001b[0mparse_int\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mparse_float\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    353\u001b[0m             parse_constant is None and object_pairs_hook is None and not kw):\n\u001b[0;32m--> 354\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_default_decoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    355\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcls\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m         \u001b[0mcls\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mJSONDecoder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.6/json/decoder.py\u001b[0m in \u001b[0;36mdecode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m    337\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    338\u001b[0m         \"\"\"\n\u001b[0;32m--> 339\u001b[0;31m         \u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraw_decode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_w\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    340\u001b[0m         \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_w\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    341\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mend\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.6/json/decoder.py\u001b[0m in \u001b[0;36mraw_decode\u001b[0;34m(self, s, idx)\u001b[0m\n\u001b[1;32m    353\u001b[0m         \"\"\"\n\u001b[1;32m    354\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 355\u001b[0;31m             \u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscan_once\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    356\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    357\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mJSONDecodeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Expecting value\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v95sJN4xk0TO",
        "colab_type": "code",
        "outputId": "136528cf-15bc-4b7a-bfbb-5773e0d472f3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "df_refined.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(112419, 21)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7eivzkoikg56",
        "colab_type": "code",
        "outputId": "7d82aac8-041c-4430-f2c5-1b7641e9e834",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "journalSet = set()\n",
        "venueSet = set()\n",
        "journalSet = journalSet|set(df_refined['journalName'])\n",
        "venueSet = journalSet|set(df_refined['venue'])\n",
        "english_only_df = remove_nonenglish(df_refined, journalSet, venueSet)\n",
        "print(\"NONENGLISH REMOVED\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "NONENGLISH REMOVED\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p_XwJd4-k2M5",
        "colab_type": "code",
        "outputId": "17105f07-1959-4224-8871-8095afb5249e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "english_only_df.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(76791, 21)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nWcJto9skrZJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "english_only_df.to_json(\"/content/gdrive/My Drive/text2doc_data/med_data/med-{:03}.json\".format(i), orient='index')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DVIgq5XwzbhX",
        "colab_type": "code",
        "outputId": "e438671f-485f-408c-8faf-c681eac155c7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "url_template = 'https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2019-11-01/s2-corpus-{:03}.gz'\n",
        "file_template = 'json{:03}.gz'\n",
        "json_template = 'json{:03}'\n",
        "\n",
        "all_length = 0\n",
        "english_length = 0\n",
        "\n",
        "i = 175\n",
        "\n",
        "while i <= 178:\n",
        "  mylist = dlandopen(url_template.format(i), file_template.format(i), json_template.format(i) )\n",
        "  df = firstDataframe(mylist, 0, len(mylist))\n",
        "  df_refined = medicalRefinement(df)\n",
        "  df_refined.to_json(\"/content/gdrive/My Drive/text2doc_data/med_data_all/med-{:03}.json\".format(i), orient='index')\n",
        "  all_length += df_refined.shape[0]\n",
        "  print(\"REFINED\")\n",
        "  journalSet = set()\n",
        "  venueSet = set()\n",
        "  journalSet = journalSet|set(df_refined['journalName'])\n",
        "  venueSet = journalSet|set(df_refined['venue'])\n",
        "  english_only_df = remove_nonenglish(df_refined, journalSet, venueSet)\n",
        "  print(\"NONENGLISH REMOVED\")\n",
        "  english_length += english_only_df.shape[0]\n",
        "  english_only_df.to_json(\"/content/gdrive/My Drive/text2doc_data/med_data_english_only/med-{:03}.json\".format(i), orient='index')\n",
        "  i+=1\n",
        "\n",
        "print(\"All length: \" + all_length)\n",
        "print(\"English length: \" + english_length)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n",
            "REFINED\n",
            "NONENGLISH REMOVED\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "OSError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-8-b85b62ecab47>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m   \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfirstDataframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmylist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmylist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m   \u001b[0mdf_refined\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmedicalRefinement\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m   \u001b[0mdf_refined\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_json\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"/content/gdrive/My Drive/text2doc_data/med_data_all/med-{:03}.json\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morient\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'index'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m   \u001b[0mall_length\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mdf_refined\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m   \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"REFINED\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36mto_json\u001b[0;34m(self, path_or_buf, orient, date_format, double_precision, force_ascii, date_unit, default_handler, lines, compression, index)\u001b[0m\n\u001b[1;32m   2422\u001b[0m             \u001b[0mlines\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlines\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2423\u001b[0m             \u001b[0mcompression\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcompression\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2424\u001b[0;31m             \u001b[0mindex\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2425\u001b[0m         )\n\u001b[1;32m   2426\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/json/_json.py\u001b[0m in \u001b[0;36mto_json\u001b[0;34m(path_or_buf, obj, orient, date_format, double_precision, force_ascii, date_unit, default_handler, lines, compression, index)\u001b[0m\n\u001b[1;32m     85\u001b[0m         \u001b[0mfh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_get_handle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_or_buf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"w\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcompression\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcompression\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m             \u001b[0mfh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m             \u001b[0mfh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mOSError\u001b[0m: [Errno 28] No space left on device"
          ]
        }
      ]
    }
  ]
}