THOR_Conceptualization.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dtim-upc/THOR/blob/main/THOR_Conceptualization.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "isColab = True\n",
        "try:\n",
        "  # running on CoLAB Hosted runtime\n",
        "  print(os.environ['HOSTNAME'])\n",
        "  print('Running on CoLAB Hosted runtime.')\n",
        "except:\n",
        "  # running on Local runtime\n",
        "  print('You are running this script on your local machine!')\n",
        "  isColab = False"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ny8ua_Jehm_h",
        "outputId": "2de61ed9-5c76-464d-8fa1-8a5d78c844b4"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ed53d33efac0\n",
            "Running on CoLAB Hosted runtime.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "'''Installing/Upgrading the Libraries Not Available to CoLAB'''\n",
        "if isColab:\n",
        "  !pip install -U 'spacy'==3.6.1\n",
        "  # # Install this if you want to run on Colab GPU\n",
        "  # !pip install -U spacy[cuda-autodetect]\n",
        "  !pip install -U rdflib\n",
        "  !pip install -U spaczz\n",
        "  !pip install nervaluate\n",
        "  !pip install -U print-dict\n",
        "  !pip3 install lemminflect"
      ],
      "metadata": {
        "id": "8Ky2-1coaGjj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "380fc19e-3c3f-4fd9-f97c-0a5fdbc0cd2b"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: spacy==3.6.1 in /usr/local/lib/python3.10/dist-packages (3.6.1)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (1.0.10)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (2.0.8)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (3.0.9)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.8 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (8.1.12)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (1.1.2)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (2.4.8)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (2.0.10)\n",
            "Requirement already satisfied: typer<0.10.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (0.9.0)\n",
            "Requirement already satisfied: pathy>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (0.11.0)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (6.4.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (4.66.2)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (1.25.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (2.31.0)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (2.6.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (3.1.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (67.7.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (23.2)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (3.3.0)\n",
            "Requirement already satisfied: pathlib-abc==0.1.1 in /usr/local/lib/python3.10/dist-packages (from pathy>=0.10.0->spacy==3.6.1) (0.1.1)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy==3.6.1) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy==3.6.1) (2.16.3)\n",
            "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy==3.6.1) (4.10.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy==3.6.1) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy==3.6.1) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy==3.6.1) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy==3.6.1) (2024.2.2)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy==3.6.1) (0.7.11)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy==3.6.1) (0.1.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer<0.10.0,>=0.3.0->spacy==3.6.1) (8.1.7)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy==3.6.1) (2.1.5)\n",
            "Requirement already satisfied: rdflib in /usr/local/lib/python3.10/dist-packages (7.0.0)\n",
            "Requirement already satisfied: isodate<0.7.0,>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from rdflib) (0.6.1)\n",
            "Requirement already satisfied: pyparsing<4,>=2.1.0 in /usr/local/lib/python3.10/dist-packages (from rdflib) (3.1.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from isodate<0.7.0,>=0.6.0->rdflib) (1.16.0)\n",
            "Requirement already satisfied: spaczz in /usr/local/lib/python3.10/dist-packages (0.6.0)\n",
            "Requirement already satisfied: catalogue in /usr/local/lib/python3.10/dist-packages (from spaczz) (2.0.10)\n",
            "Requirement already satisfied: numpy>=1.21.2 in /usr/local/lib/python3.10/dist-packages (from spaczz) (1.25.2)\n",
            "Requirement already satisfied: rapidfuzz>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spaczz) (3.6.1)\n",
            "Requirement already satisfied: regex>=2020.6.8 in /usr/local/lib/python3.10/dist-packages (from spaczz) (2023.12.25)\n",
            "Requirement already satisfied: spacy<4.0,>=3.0 in /usr/local/lib/python3.10/dist-packages (from spaczz) (3.6.1)\n",
            "Requirement already satisfied: srsly in /usr/local/lib/python3.10/dist-packages (from spaczz) (2.4.8)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (1.0.10)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (2.0.8)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (3.0.9)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.8 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (8.1.12)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (1.1.2)\n",
            "Requirement already satisfied: typer<0.10.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (0.9.0)\n",
            "Requirement already satisfied: pathy>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (0.11.0)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (6.4.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (4.66.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (2.31.0)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (2.6.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (3.1.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (67.7.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (23.2)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from spacy<4.0,>=3.0->spaczz) (3.3.0)\n",
            "Requirement already satisfied: pathlib-abc==0.1.1 in /usr/local/lib/python3.10/dist-packages (from pathy>=0.10.0->spacy<4.0,>=3.0->spaczz) (0.1.1)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<4.0,>=3.0->spaczz) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<4.0,>=3.0->spaczz) (2.16.3)\n",
            "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<4.0,>=3.0->spaczz) (4.10.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<4.0,>=3.0->spaczz) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<4.0,>=3.0->spaczz) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<4.0,>=3.0->spaczz) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<4.0,>=3.0->spaczz) (2024.2.2)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy<4.0,>=3.0->spaczz) (0.7.11)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy<4.0,>=3.0->spaczz) (0.1.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer<0.10.0,>=0.3.0->spacy<4.0,>=3.0->spaczz) (8.1.7)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy<4.0,>=3.0->spaczz) (2.1.5)\n",
            "Requirement already satisfied: nervaluate in /usr/local/lib/python3.10/dist-packages (0.1.8)\n",
            "Requirement already satisfied: print-dict in /usr/local/lib/python3.10/dist-packages (0.1.19)\n",
            "Requirement already satisfied: yapf<0.31.0,>=0.30.0 in /usr/local/lib/python3.10/dist-packages (from print-dict) (0.30.0)\n",
            "Collecting lemminflect\n",
            "  Downloading lemminflect-0.2.3-py3-none-any.whl (769 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m769.7/769.7 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from lemminflect) (1.25.2)\n",
            "Installing collected packages: lemminflect\n",
            "Successfully installed lemminflect-0.2.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "'''Downloading Model lg = larger - with word vectors'''\n",
        "if isColab:\n",
        "  !python -m spacy download en_core_web_lg\n",
        "  !python -m spacy download en_core_web_trf\n",
        "\n",
        "  # Adding Word Vectors: https://spacy.io/usage/linguistic-features#adding-vectors\n",
        "  #!wget https://dl.fbaipublicfiles.com/fasttext/vectors-english/wiki-news-300d-1M-subword.vec.zip\n",
        "  #!python -m spacy init vectors en wiki-news-300d-1M-subword.vec.zip /content/wiki-news-300d-1M-subword"
      ],
      "metadata": {
        "id": "irzN7YEJwKjx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a0e03d36-0ec6-4505-a431-c3ce05514b64"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-03-03 15:25:18.675833: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-03-03 15:25:18.675931: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-03-03 15:25:18.679071: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-03-03 15:25:18.697074: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-03-03 15:25:22.093779: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Collecting en-core-web-lg==3.6.0\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.6.0/en_core_web_lg-3.6.0-py3-none-any.whl (587.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m587.7/587.7 MB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: spacy<3.7.0,>=3.6.0 in /usr/local/lib/python3.10/dist-packages (from en-core-web-lg==3.6.0) (3.6.1)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (1.0.10)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (2.0.8)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (3.0.9)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.8 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (8.1.12)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (1.1.2)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (2.4.8)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (2.0.10)\n",
            "Requirement already satisfied: typer<0.10.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (0.9.0)\n",
            "Requirement already satisfied: pathy>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (0.11.0)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (6.4.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (4.66.2)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (1.25.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (2.31.0)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (2.6.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (3.1.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (67.7.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (23.2)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (3.3.0)\n",
            "Requirement already satisfied: pathlib-abc==0.1.1 in /usr/local/lib/python3.10/dist-packages (from pathy>=0.10.0->spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (0.1.1)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (2.16.3)\n",
            "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (4.10.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (2024.2.2)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (0.7.11)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (0.1.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer<0.10.0,>=0.3.0->spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (8.1.7)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy<3.7.0,>=3.6.0->en-core-web-lg==3.6.0) (2.1.5)\n",
            "Installing collected packages: en-core-web-lg\n",
            "Successfully installed en-core-web-lg-3.6.0\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_lg')\n",
            "2024-03-03 15:26:10.589505: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-03-03 15:26:10.589589: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-03-03 15:26:10.591235: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-03-03 15:26:10.600195: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-03-03 15:26:11.993955: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Collecting en-core-web-trf==3.6.1\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.6.1/en_core_web_trf-3.6.1-py3-none-any.whl (460.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m460.3/460.3 MB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: spacy<3.7.0,>=3.6.0 in /usr/local/lib/python3.10/dist-packages (from en-core-web-trf==3.6.1) (3.6.1)\n",
            "Collecting spacy-transformers<1.3.0,>=1.2.2 (from en-core-web-trf==3.6.1)\n",
            "  Downloading spacy_transformers-1.2.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (190 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m190.8/190.8 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (1.0.10)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2.0.8)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (3.0.9)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.8 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (8.1.12)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (1.1.2)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2.4.8)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2.0.10)\n",
            "Requirement already satisfied: typer<0.10.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (0.9.0)\n",
            "Requirement already satisfied: pathy>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (0.11.0)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (6.4.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (4.66.2)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (1.25.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2.31.0)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2.6.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (3.1.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (67.7.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (23.2)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (3.3.0)\n",
            "Collecting transformers<4.31.0,>=3.4.0 (from spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1)\n",
            "  Downloading transformers-4.30.2-py3-none-any.whl (7.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.2/7.2 MB\u001b[0m \u001b[31m58.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch>=1.8.0 in /usr/local/lib/python3.10/dist-packages (from spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (2.1.0+cu121)\n",
            "Collecting spacy-alignments<1.0.0,>=0.7.2 (from spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1)\n",
            "  Downloading spacy_alignments-0.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (313 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m314.0/314.0 kB\u001b[0m \u001b[31m25.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pathlib-abc==0.1.1 in /usr/local/lib/python3.10/dist-packages (from pathy>=0.10.0->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (0.1.1)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2.16.3)\n",
            "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (4.10.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2024.2.2)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (0.7.11)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (0.1.4)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (3.13.1)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (3.2.1)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (2023.6.0)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (2.1.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /usr/local/lib/python3.10/dist-packages (from transformers<4.31.0,>=3.4.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (0.20.3)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers<4.31.0,>=3.4.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<4.31.0,>=3.4.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (2023.12.25)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers<4.31.0,>=3.4.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m72.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers<4.31.0,>=3.4.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (0.4.2)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer<0.10.0,>=0.3.0->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (8.1.7)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2.1.5)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.8.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (1.3.0)\n",
            "Installing collected packages: tokenizers, spacy-alignments, transformers, spacy-transformers, en-core-web-trf\n",
            "  Attempting uninstall: tokenizers\n",
            "    Found existing installation: tokenizers 0.15.2\n",
            "    Uninstalling tokenizers-0.15.2:\n",
            "      Successfully uninstalled tokenizers-0.15.2\n",
            "  Attempting uninstall: transformers\n",
            "    Found existing installation: transformers 4.38.1\n",
            "    Uninstalling transformers-4.38.1:\n",
            "      Successfully uninstalled transformers-4.38.1\n",
            "Successfully installed en-core-web-trf-3.6.1 spacy-alignments-0.9.1 spacy-transformers-1.2.5 tokenizers-0.13.3 transformers-4.30.2\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_trf')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "fUhgep3hK-XZ"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "import os\n",
        "import shutil\n",
        "import spacy\n",
        "from spacy.lang.en import English\n",
        "from spacy.tokenizer import Tokenizer\n",
        "from spacy.pipeline import Sentencizer\n",
        "from nervaluate import Evaluator\n",
        "import bs4\n",
        "from print_dict import pd as pdic\n",
        "import requests\n",
        "from spacy import displacy\n",
        "from spacy.matcher import Matcher\n",
        "from spacy.tokens import Span\n",
        "import matplotlib.pyplot as plt\n",
        "from tqdm import tqdm\n",
        "import csv\n",
        "from textwrap import wrap\n",
        "import os.path\n",
        "import functools\n",
        "import operator\n",
        "from collections import deque\n",
        "from operator import itemgetter\n",
        "from spacy.symbols import nsubj, VERB, AUX\n",
        "\n",
        "# spaczz lirrary for Similarity Matching\n",
        "from spaczz.matcher import SimilarityMatcher\n",
        "\n",
        "# RDFLib libraries\n",
        "from rdflib import Graph\n",
        "import pprint\n",
        "from rdflib import RDFS\n",
        "from rdflib import URIRef\n",
        "from rdflib.namespace import RDF\n",
        "\n",
        "# Colab specific libraries\n",
        "if isColab:\n",
        "  from google.colab import files\n",
        "\n",
        "# General Libraries\n",
        "import pandas as pd\n",
        "import srsly\n",
        "import json\n",
        "import os\n",
        "import re\n",
        "import time\n",
        "import warnings\n",
        "\n",
        "import subprocess\n",
        "import sys\n",
        "\n",
        "# Spacy Related Imports\n",
        "import spacy\n",
        "from spacy.util import minibatch, compounding, compile_infix_regex, get_words_and_spaces\n",
        "from spacy.tokens import Span, DocBin, Doc\n",
        "from spacy.vocab import Vocab\n",
        "from spacy.tokenizer import Tokenizer\n",
        "from spacy.lang.en import English\n",
        "from spacy.scorer import Scorer\n",
        "from spacy.training import Example\n",
        "# import spacy_sentence_bert\n",
        "\n",
        "# Ignore OOV warnings\n",
        "warnings.filterwarnings(\"ignore\", message=r\"\\[W008\\]\", category=UserWarning)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Downloading Disease A-Z Datasets"
      ],
      "metadata": {
        "id": "dNbIXSCaplIp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data_path = \"/content/data\"\n",
        "if not os.path.exists(data_path):\n",
        "  '''Downloading the Dataset into CoLab temporary directory'''\n",
        "  !mkdir /content/data\n",
        "  !wget -P /content/data/ 'https://raw.githubusercontent.com/dtim-upc/THOR/main/Dataset/Disease%20A-Z/Annotated_Text.zip'\n",
        "  !wget -P /content/data/ 'https://raw.githubusercontent.com/dtim-upc/THOR/main/Dataset/Disease%20A-Z/Structured_Data.zip'\n",
        "  !wget -P /content/data/ 'https://raw.githubusercontent.com/dtim-upc/THOR/main/Dataset/Disease%20A-Z/Schema.zip'\n",
        "\n",
        "  # unzipping the Dataset contents\n",
        "  !unzip \"/content/data/Annotated_Text.zip\" -d \"/content/data/Annotated_Text/\" && rm \"/content/data/Annotated_Text.zip\"\n",
        "  !unzip \"/content/data/Structured_Data.zip\" -d \"/content/data/Structured_Data/\" && rm \"/content/data/Structured_Data.zip\"\n",
        "  !unzip \"/content/data/Schema.zip\" -d \"/content/data/Schema\" && rm \"/content/data/Schema.zip\""
      ],
      "metadata": {
        "id": "x2Ifhur9YGdw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9ff7b01c-33a2-45b9-a40b-6bb2bc989001"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-03-03 15:33:22--  https://raw.githubusercontent.com/dtim-upc/THOR/main/Dataset/Disease%20A-Z/Annotated_Text.zip\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.111.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1078004 (1.0M) [application/zip]\n",
            "Saving to: ‘/content/data/Annotated_Text.zip’\n",
            "\n",
            "Annotated_Text.zip  100%[===================>]   1.03M  --.-KB/s    in 0.05s   \n",
            "\n",
            "2024-03-03 15:33:22 (19.6 MB/s) - ‘/content/data/Annotated_Text.zip’ saved [1078004/1078004]\n",
            "\n",
            "--2024-03-03 15:33:22--  https://raw.githubusercontent.com/dtim-upc/THOR/main/Dataset/Disease%20A-Z/Structured_Data.zip\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 22228 (22K) [application/zip]\n",
            "Saving to: ‘/content/data/Structured_Data.zip’\n",
            "\n",
            "Structured_Data.zip 100%[===================>]  21.71K  --.-KB/s    in 0.001s  \n",
            "\n",
            "2024-03-03 15:33:22 (30.2 MB/s) - ‘/content/data/Structured_Data.zip’ saved [22228/22228]\n",
            "\n",
            "--2024-03-03 15:33:22--  https://raw.githubusercontent.com/dtim-upc/THOR/main/Dataset/Disease%20A-Z/Schema.zip\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1258 (1.2K) [application/zip]\n",
            "Saving to: ‘/content/data/Schema.zip’\n",
            "\n",
            "Schema.zip          100%[===================>]   1.23K  --.-KB/s    in 0s      \n",
            "\n",
            "2024-03-03 15:33:22 (50.3 MB/s) - ‘/content/data/Schema.zip’ saved [1258/1258]\n",
            "\n",
            "Archive:  /content/data/Annotated_Text.zip\n",
            "   creating: /content/data/Annotated_Text/test/\n",
            "  inflating: /content/data/Annotated_Text/test/Abdominal_aortic_aneurysm-BD50.4Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/test/Absence_seizure-8A68.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/test/Acanthosis_nigricans-ED51.0Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/test/Achalasia-DA21.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/test/Achilles_tendinitis-FB40.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/test/Achilles_tendon_rupture-NC96.02.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/test/ACL_injury-ND56.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/test/Acne-ED80.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/test/Acoustic_neuroma-2A02.3_XA6LY7.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/test/Acromegaly-5A60.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/test/Actinic_keratosis-EK90.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/test/Acute_coronary_syndrome-BA4Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/test/Acute_flaccid_myelitis_(AFM)-1C81.jsonl  \n",
            "   creating: /content/data/Annotated_Text/train/\n",
            "  inflating: /content/data/Annotated_Text/train/Abdominal_Pain-MD81.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Abducens_nerve_Palsy-9C81.2.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Abnormal_uterine_bleeding-GA2Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Acquired_Immuno_Deficiency_Syndrome_(AIDS)-1C62.3.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Acute_encephalitis-1D00.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Adult_Inclusion_Conjunctivitis-1D84.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Alcohol_Abuse_and_Alcoholism-6C40.1Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Alopecia_(hair_loss)-ED70.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Alzheimers_Disease-8A20.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Amaurosis_Fugax-8B10.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Amblyopia-9D46.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Amoebiasis-1A36.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Anaemia-3A9Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Aniseikonia-9D00.5.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Anisometropia-9D00.4.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Ankyloblepharon-LA14.07.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Antepartum_hemorrhage-JA41.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Anthrax-1B97.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Anxiety-MB24.3.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Aphakia-9B11.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Appendicitis-DB10.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Arthritis-FA2Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Asbestos-CB20.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Asthma-CA23.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Astigmatism-9D00.2.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Atherosclerosis-BD40.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Atopic_Keratoconjunctivitis-9A60.0Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Attention_Deficit_Hyperactivity_Disorder-6A05.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Autism-6A02.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Avian_Influenza-1E31.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Back_Pain-ME84.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Bad_Breath-MD94.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Band_Shaped_Keratopathy-9A78.4.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Bedsores-EH90.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Bells_Palsy-8B88.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Benign_Essential_Blepharospasm-8A02.00.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/birth_defects-LD2Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Bleeding_Gums-MD80.Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Blepharitis-9A01.3.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Blepharochalasis-9A06.8.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Blindness-9D90.Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Brain_Tumour-2A00.5.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Breast_Cancer-2C6Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Bronchitis-CA20.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Brucellosis-1B95.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Bruxism-DA0E.7.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Burns-NE2Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Cancer-2D4Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Capillary_Haemangioma-2E81.2Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Carpal_Tunnel_Syndrome-8C10.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Cavernous_Haemangioma-LD23.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Cavities-1F25.01.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Cervical_cancer-2C77.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Chalazion-9A02.0Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Chalcosis-9A78.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Chemical_Injuries_to_the_eyes-ND99.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Chickenpox-1E90.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Chikungunya_Fever-1D40.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Child_abuse-QE82.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Cholera-1A00.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Chronic_obstructive_pulmonary_disease-CA22.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Cleft_Lip_and_Cleft_Palate-LA4Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Colorectal_Cancer-2B91.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Commotio_Retina-NA06.62.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Computer_Vision_Syndrome-9D92.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Congenital_Capillary_Haemangioma-2E81.2Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Conjunctival_Concretions-9A61.6.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Conjunctivochalasis-9A11.3.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Corneal_Abrasion-NA06.4.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Coronary_Heart_Disease-BA5Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Coronavirus_disease-RA01.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Cough-MD12.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Crimean_Congo_haemorrhagic_fever-1D49.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Dacryoadenitis-1D80.Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Dacryocystitis-9A11.2.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Deafness-AB52.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Dementia-6D8Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Dengue_Fever-1D2Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Dermatochalasis-9A03.5.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Diabetes_Mellitus-5A14.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Diabetic_Retinopathy-9B71.0Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Diarrhea-ME05.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Digit_Sucking-F98.8.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Diphtheria-1C17.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Disabilities-6A00.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Distichiasis-LA14.0Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Double_Elevator_Palsy-9C83.02.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Downs_Syndrome-LD40.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Dracunculiasis-1F64.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Duane_Retraction_Syndrome-9C82.2.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Dyslexia_and_Ocular_Features-MB4B.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Early_Pregnancy_Loss-JA40.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Ebola_Virus_Disease-1D60.01.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Eclampsia-JA25.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Ectopic_Pregnancy-JA01.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Ectropion-9A03.2Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Eczema-EA8Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Endometriosis-GA10.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Entropion-LA14.02.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Epiblepharon-LA14.0Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Epibulbar_Dermoid-LA14.06.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Epicanthus-LD21.Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Epidemic_Dropsy-MG29.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Epidemic_Keratoconjunctivitis-1D84.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Epilepsy-8A6Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Euryblepharon-LA14.0Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Exotropia-9C80.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Exposure_Keratopathy-9A78.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Eyelid_Varix-BD75.Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Factitious_Keratoconjunctivitis-1B12.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Female_Genital_Mutilation-GC51.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Fibroids-2E86.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Fibromyalgia-MG30.01.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Filamentary_Keratitis-9A78.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Filariasis-1F66.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Floppy_Eyelid_Syndrome-9A03.24.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Fluorosis-FB80.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Food_Poisoning-1A10.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Fracture-ND56.2.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Frey_s_Syndrome-8B82.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Frost_Bite-NE4Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Gaming_Disorder-6C51.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Gangrene-MC85.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Gastro_Esophageal_Reflux_Disease-DA22.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Giant_Papillary_Conjunctivitis-9A60.00.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Glaucoma-9C61.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Goitre-5A01.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Gonorrhea-1A7Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Guillain_Barre_syndrome-8C01.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Gum_Disease-DA0D.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Haemophilia-3B10.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Hand_Foot_And_Mouth_Disease-1F05.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Heat_Waves-NF01.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Helminthiasis-1F90.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Hemangioma-2E81.0Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Hepatitis-DB97.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Herpes_Simplex-1D02.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Heterophoria-9C80.4.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/High_Risk_Pregnancy-QA43.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Horner_Syndrome-8D8A.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Human_Papillomavirus_Infection-1A9Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Hyperemesis_gravidarum-JA60.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Hypermetropia-9D00.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Hypertension-BA00.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Hypertensive_Retinopathy-9B71.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Hypervitaminosis_A-5B90.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Ichthyosis-5C81.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Infertility-GA31.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Inflammatory_Bowel_Disease-DD7Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Insomnia-7A00.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Intrauterine_Growth_Restriction-KA20.1Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Iron_Deficiency_Anemia-3A00.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Japanese_Encephalitis-1C85.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Jaundice-ME10.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Kala_Azar-1F54.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Kaposis_Sarcoma-2B57.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Keratoconjunctivitis_Sicca-9A79.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Keratoconus-9A78.50.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Keratoglobus-LA11.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Keratosis_Pilaris-ED56.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Kyasanur_Forest_Disease-1D4B.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/lactose_intolerance-5C61.6Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Lagophthalmos-9A03.4Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Lead_poisoning-NE61.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Leprosy-1B20.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Leptospirosis-1B91.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Leukemia-2B33.4.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Lichen_Planus-EA91.4.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Lid_Imbrication_Syndrome-LA14.02.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Ligneous_Conjunctivitis-9A02.2.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Long_COVID-RA02.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Low_Vision-9D90.Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Lymphoedema-BD93.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Madarosis-9A04.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Malaria-1F45.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Malocclusion-DA0E.5Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Mastitis-GB21.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Maternal_anemia-KA00.B.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Measles-1F03.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Meningitis-1D01.Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Middle_East_Respiratory_Syndrome-1D64.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Migraine-8A80.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Monkeypox-1E71.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Moorens_Ulcer-9A76.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Mouth_Breathing-MD11.8Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Mouth_Ulcers-DA01.15.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Mucormycosis-1F2C.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Mumps-1D80.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Myocardial_Infarction-BA41.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Myopia-9D00.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Narcolepsy-7A20.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Nasal_Polyps-CA0J.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Neonatal_Conjunctivitis-KA65.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Neonatal_Respiratory_Failure-KB2D.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Neuralgia-MG30.5Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Neurotrophic_Keratopathy-1F00.10.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Nipah_Virus_Infection-1D63.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Nonalcoholic_fatty_liver_disease-DB92.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Obesity-5B81.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Obsessive_Compulsive_Disorder-6B20.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Ocular_Graft_Versus_Host_Disease-4B24.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Oral_Cancer-2B6E.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Orbital_Cavernous_Haemangioma-LD23.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Orbital_Dermoid-2F36.Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Orbital_Fat_Prolapse-9A03.5.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Orbital_Haemangiopericytoma-2A00.5.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Orbital_Lymphangioma-LA90.12.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Orbital_Solitary_Fibrous_Tumour-EE6Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Orbital_Varix-BD75.Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Osteoarthritis-FA0Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Osteomyelitis-FB84.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Osteoporosis-FB83.1Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Otitis_Media-AB0Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Papilloedema-9C40.A0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Parkinsons_Disease-8A00.0Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Pellucid_Marginal_Degeneration-9A78.4.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Perennial_Allergic_Conjunctivitis-9A60.02.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Pericarditis-BB2Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Pharyngoconjunctival_Fever-1D84.Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Phlyctenular_Keratoconjunctivitis-1B12.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Photophthalmia-9A72.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Pinguecula-9A61.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Plague-1B93.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Pneumonia-CA40.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Poliomyelitis-1C81.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Polycystic_Ovary_Syndrome-5A80.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Postherpetic_Neuralgia-1E91.5.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Postpartum_Depression-6E20.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Postpartum_Haemorrhage-JA43.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Post_Menopausal_Bleeding-GA30.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Post_Polio_Syndrome-8B62.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Preeclampsia-JA24.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Premenstrual_syndrome-GA34.40.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Presbyopia-9D00.3.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Preterm_Birth-KA21.4Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Pseudophakia-QB51.2.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Pseudostrabismus-Q10.3.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Psoriasis-EA90.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Pterygium-9A61.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/train/Puerperal_Sepsis-JB40.0.jsonl  \n",
            "   creating: /content/data/Annotated_Text/valid/\n",
            "  inflating: /content/data/Annotated_Text/valid/Quinsy-CA0K.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Q_Fever-1C33.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Rabies-1C82.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Ramsay_Hunt_Syndrome-8B88.Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Rare_Diseases-8A44.3.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Raynauds_Phenomenon-BD42.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Recurrent_Corneal_Erosion_Syndrome-9A78.8.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Reiters_Syndrome-FA11.2.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Retinopathy_of_Prematurity-9B71.3.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Rheumatic_Heart_Disease-BC20.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Rubella-1F02.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Salzmanns_Nodular_Degeneration-9A78.4.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Sarcoidosis-4B20.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Sarcoma-2A60.39.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Scabies-1G04.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Schizophrenia-6A20.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Scrub_Typhus-1C30.3.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Scurvy-5B56.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Seasonal_Allergic_Conjunctivitis-9A60.02.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Severe_Acute_Respiratory_Syndrome-1D65.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Sexually_Transmitted_Infections-1A9Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Shaken_Baby_Syndrome-PJ20.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Siderosis_Bulbi-CA60.8.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Silicosis-CA60.0Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Solar_Retinopathy-9B62.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Spheroidal_Degeneration-8C72.0Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Spinal_cord_Injury-ND51.2.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Steatoablepharon-H02.36.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Stroke-8B20.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Substance_abuse-6C4Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Sub_Conjunctival_Haemorrhage-9A61.5.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Superior_Limbic_Keratoconjunctivitis-9A60.Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Swine_Flu-1E31.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Syphilis-1A6Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Taeniasis-1F76.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Terriens_Marginal_Degeneration-H18.469.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Tetanus-1C13.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Thalassaemia-3A50.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Thermal_Injuries_to_the_Eye-NA06.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Tics_and_Eye_Manifestations-8A05.0Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Tomato_Fever-MG26.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Tooth_Sensitivity-DA08.Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Toxic_conjunctivitis-9A60.2.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Trachoma-1C23.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Traumatic_Hyphaema-NA06.9.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Trichiasis-9A04.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Trochlear_Nerve_Palsy-9C81.1.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Tuberculosis-1B1Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Turners_Syndrome-LD50.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Typhoid-1A07.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Ulcerative_Colitis-DD71.Z.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Urticaria-EB05.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Varicose_Veins-JA61.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Vernal_Keratoconjunctivitis-9A60.5.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Vitamin_B12_Deficiency-5B5F.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Vitiligo-ED63.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Welding_Arc_Maculopathy-9B78.3Y.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Whooping_Cough-1C12.0.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Xanthelasma-9A06.4.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Yaws-1C1D.jsonl  \n",
            "  inflating: /content/data/Annotated_Text/valid/Yellow_Fever-1D47.jsonl  \n",
            "Archive:  /content/data/Structured_Data.zip\n",
            "  inflating: /content/data/Structured_Data/anatomy_surgery.csv  \n",
            "  inflating: /content/data/Structured_Data/disease_anatomy.csv  \n",
            "  inflating: /content/data/Structured_Data/disease_causes.csv  \n",
            "  inflating: /content/data/Structured_Data/disease_complications.csv  \n",
            "  inflating: /content/data/Structured_Data/disease_diagnosis.csv  \n",
            "  inflating: /content/data/Structured_Data/disease_medicine.csv  \n",
            "  inflating: /content/data/Structured_Data/disease_precuation.csv  \n",
            "  inflating: /content/data/Structured_Data/disease_risk_factors.csv  \n",
            "  inflating: /content/data/Structured_Data/disease_symptoms.csv  \n",
            "  inflating: /content/data/Structured_Data/medicine_composition.csv  \n",
            "Archive:  /content/data/Schema.zip\n",
            "  inflating: /content/data/Schema/Disease_KG_Extended.ttl  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def create_out_dir(OUTPUT_DIR):\n",
        "  '''This function will be used to remove the outputs in different run'''\n",
        "  if os.path.exists(OUTPUT_DIR):\n",
        "    shutil.rmtree(OUTPUT_DIR)\n",
        "\n",
        "  os.makedirs(OUTPUT_DIR)"
      ],
      "metadata": {
        "id": "RhHDedZfceMV"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Defining some of the Directories... Please also check the Main function at the bottom\n",
        "SCHEMA_FILE = \"data/Schema/Disease_KG_Extended.ttl\"\n",
        "STRUCTURED_DATA_DIR = \"data/Structured_Data\""
      ],
      "metadata": {
        "id": "dP5QwTYkH7J2"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Data Conversion Part"
      ],
      "metadata": {
        "id": "InQ4__x1UkH1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def trim_entity_spans(text, spans):\n",
        "  '''Data Cleaning: Removes leading and trailing white spaces from entity spans.'''\n",
        "  invalid_span_tokens = re.compile(r'\\s')\n",
        "\n",
        "  valid_spans = []\n",
        "  for start, end, label in spans:\n",
        "    valid_start = start\n",
        "    valid_end = end\n",
        "    while valid_start < len(text) and invalid_span_tokens.match(text[valid_start]):\n",
        "      valid_start += 1\n",
        "    while valid_end > 1 and invalid_span_tokens.match(text[valid_end - 1]):\n",
        "      valid_end -= 1\n",
        "\n",
        "    if valid_start < valid_end:\n",
        "      valid_spans.append((valid_start, valid_end, label))\n",
        "\n",
        "  return valid_spans"
      ],
      "metadata": {
        "id": "BCAkc7AYUrnM"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def docanno_to_spacy_ner_db(DATA_DIR):\n",
        "  '''\n",
        "  This function takes a directory of docanno annotated datasets for NER/RE\n",
        "  and converts them into SpaCy DocBin Object which is Trainable via commandline\n",
        "  '''\n",
        "  # Creates a blank Tokenizer with just the English vocab\n",
        "  nlp = spacy.blank(\"en\")\n",
        "\n",
        "  Doc.set_extension(\"rel\", default={},force=True)\n",
        "  vocab = Vocab()\n",
        "\n",
        "  word_count = 0\n",
        "  no_files = 0\n",
        "  no_doc = 0\n",
        "  no_entities = 0\n",
        "  error_cnt = 0\n",
        "\n",
        "  # the DocBin will store the example documents\n",
        "  db = DocBin()\n",
        "\n",
        "  for dirname, _, filenames in os.walk(DATA_DIR):\n",
        "    for filename in filenames:\n",
        "      file_path = os.path.join(dirname, filename)\n",
        "\n",
        "      try:\n",
        "        \"\"\" Iterate through the Jsonl file to create serialize Docbin object / .spacy IOB File \"\"\"\n",
        "        for json_line in srsly.read_jsonl(file_path):\n",
        "\n",
        "          # parsing the docanno JSON data (per-line)\n",
        "          text = json_line[\"text\"]\n",
        "          spans = json_line[\"entities\"]\n",
        "\n",
        "          new_spans = []\n",
        "          for span in spans:\n",
        "            new_spans.append((span[\"start_offset\"], span[\"end_offset\"], span[\"label\"]))\n",
        "\n",
        "          # cleaning and validating the leading and trailing spaces from the annotated entities\n",
        "          valid_spans = trim_entity_spans(text, new_spans)\n",
        "\n",
        "          \"\"\" Parsing tokens from Text \"\"\"\n",
        "          tokens = nlp(text)\n",
        "\n",
        "          entities = []\n",
        "\n",
        "          spaces = [True if tok.whitespace_ else False for tok in tokens]\n",
        "          words = [t.text for t in tokens]\n",
        "          doc = Doc(nlp.vocab, words=words, spaces=spaces)\n",
        "\n",
        "          for start, end, label in valid_spans:\n",
        "            \"\"\" The modes should be: strict, contract, and expand \"\"\"\n",
        "            # print(eg['text'][int(span[\"start_offset\"]):int(span[\"end_offset\"])])\n",
        "            entity = doc.char_span(start, end, label=label, alignment_mode='contract')\n",
        "\n",
        "            # Not considering the spans which are Erroneous\n",
        "            if entity is None:\n",
        "              error_cnt += 1\n",
        "              # file_name = ttext.split('\\n')[0]\n",
        "              # print(f'Error Found in File: {file_name};\\n Span = {start}-{end}; Phrase = {doc.text[start:end]}; Label = {label}\\n')\n",
        "\n",
        "            else:\n",
        "              no_entities += 1\n",
        "              entities.append(entity)\n",
        "\n",
        "          # print(entities)\n",
        "          try:\n",
        "            doc.ents = entities\n",
        "            word_count += len(words)-int(7)\n",
        "          except:\n",
        "            # print(\"=>> Error\")\n",
        "            continue\n",
        "\n",
        "          db.add(doc)\n",
        "          no_doc += 1\n",
        "\n",
        "      except:\n",
        "        print('Error While Loading JSON Data From Input Directory. Please check if you have other file type...')\n",
        "\n",
        "      no_files +=1\n",
        "  print(f\"- Files: {no_files} \\n- Processed Documents: {no_doc} \\n- Total Entities: {no_entities} \\n- Erroneous Entities (Ignored): {error_cnt} \\n- Total Words: {word_count}\")\n",
        "\n",
        "  return doc, db"
      ],
      "metadata": {
        "id": "s18Rxcy3UuSR"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def load_json_from_docbin(file_path: str, nlp):\n",
        "  '''This function loads data from SpaCy docbin formatted files into spacy compitable JSON format'''\n",
        "  doc_bin = DocBin().from_disk(file_path)\n",
        "  samples, entities_count = [], 0\n",
        "  for doc in doc_bin.get_docs(nlp.vocab):\n",
        "    sample = {\n",
        "      \"text\": doc.text,\n",
        "      \"entities\": []\n",
        "    }\n",
        "    if len(doc.ents) > 0:\n",
        "      # print(doc.ents)\n",
        "      entities = [(e.start_char, e.end_char, e.label_) for e in doc.ents]\n",
        "      sample[\"entities\"] = entities\n",
        "      entities_count += len(entities)\n",
        "    else:\n",
        "      warnings.warn(\"Sample without entities!\")\n",
        "    samples.append(sample)\n",
        "  return samples, entities_count"
      ],
      "metadata": {
        "id": "5ygazl30VHTT"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Reading Data"
      ],
      "metadata": {
        "id": "EpRbrkfFjU8i"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import en_core_web_trf\n",
        "# TODO - we can aditionally remove the unnecessary pipeline from it -- **speed improvement**\n",
        "nlp_trf = en_core_web_trf.load()"
      ],
      "metadata": {
        "id": "SHey33Q1Nnna"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# #Using custom Word Vector Models\n",
        "# if isColab:\n",
        "#   nlp = spacy.load(\"/content/wiki-news-300d-1M-subword\")\n",
        "# else:\n",
        "#   nlp = spacy.load(\"Embeddings/glove-840B-300d\")"
      ],
      "metadata": {
        "id": "k_ZkNY2vNjur"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# '''Import the appropriate model and adding the extra pipeline needed'''\n",
        "# # import en_core_web_md\n",
        "import en_core_web_lg\n",
        "\n",
        "nlp = en_core_web_lg.load()\n",
        "\n",
        "# # sbd = nlp.create_pipe('sentencizer')\n",
        "nlp.add_pipe('sentencizer')"
      ],
      "metadata": {
        "id": "qjgX9rbuBqQy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f97298b6-1d6c-41d1-d64c-dbc3dace7733"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<spacy.pipeline.sentencizer.Sentencizer at 0x7f0a0e5dee40>"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_rdf_graph(file_name=\"\"):\n",
        "  '''Getting the RDF file from the user (.nt/.ttl/.xml etc)'''\n",
        "\n",
        "  # parsing the graph\n",
        "  g = Graph()\n",
        "  g.parse(file_name)\n",
        "\n",
        "  # # Loop through some of the triples in the graph (subj, pred, obj)\n",
        "  # print('\\nTotal Triples Found = {}\\n'.format(len(g)))\n",
        "  # print('First 10 Triples:')\n",
        "  # for triple in list(g)[:10]:\n",
        "  #     # Check if there is at least one triple in the Graph\n",
        "  #     # if (subj, pred, obj) not in g:\n",
        "  #     #    raise Exception(\"It better be!\")\n",
        "  #     print(triple)\n",
        "\n",
        "  return g"
      ],
      "metadata": {
        "id": "IBlyu7fRPr6S"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_templates(g):\n",
        "  '''Given a RDF graph 'g' this function will return a list of (S, P, O) triples having only names'''\n",
        "\n",
        "  # Getting all the unique S, P, O from the graph using the RDFS.domain and RDFS.range\n",
        "  preds_subs = list(g.subject_objects(predicate=RDFS.domain))\n",
        "  preds_objs = list(g.subject_objects(predicate=RDFS.range))\n",
        "\n",
        "  # Getting all the subclasses with corresponding superclasses\n",
        "  sup_sub = g.subject_objects(predicate=RDFS.subClassOf)\n",
        "\n",
        "  # dictionary having a superclasses and it's subclasses {'Treatment':['Medicine', 'Precaution', 'Surgery']}\n",
        "  sup_sub_dic = {}\n",
        "\n",
        "  # populating the dictionary from the graph\n",
        "  for sc in sup_sub:\n",
        "    subj = sc[0].split('#')[1]\n",
        "    obj = sc[1].split('#')[1]\n",
        "\n",
        "    if obj in sup_sub_dic:\n",
        "      sup_sub_dic[obj].append(subj)\n",
        "    else:\n",
        "      sup_sub_dic[obj] = [subj]\n",
        "\n",
        "  # dictionary having a structure {'P':['S', 'O']}\n",
        "  dic_triples = {}\n",
        "\n",
        "  # gets only the name of Predicates and Subjects splitting from the URI's\n",
        "  for ps in preds_subs:\n",
        "    pred = ps[0].split('#')[1]\n",
        "    subj = ps[1].split('#')[1]\n",
        "    dic_triples[pred] = [subj]\n",
        "    # # print(subj, pred)\n",
        "\n",
        "  # matches the Subjects having specific Predicates with Objects\n",
        "  for po in preds_objs:\n",
        "    pred = po[0].split('#')[1]\n",
        "    obj = po[1].split('#')[1]\n",
        "    dic_triples[pred].append(obj)\n",
        "    # # print(pred, obj)\n",
        "\n",
        "  # saves the triples from the dictionary into a list of tuple -> [(S, P, O)]\n",
        "  triples_name = []\n",
        "  for pred in dic_triples:\n",
        "    subj = dic_triples[pred][0]\n",
        "    obj = dic_triples[pred][1]\n",
        "\n",
        "    # checking if the subject is a superclass - we ignore the superclass and only consider it's subclasses\n",
        "    if subj in sup_sub_dic:\n",
        "      # copy it's predicate to all it's subclasses (sub_cls) along with the range\n",
        "      for sub_cls in sup_sub_dic[subj]:\n",
        "        triples_name.append((sub_cls, pred, obj))\n",
        "    else:\n",
        "      triples_name.append((subj, pred, obj))\n",
        "      # print('({}, {}, {})'.format(subj, pred, obj))\n",
        "\n",
        "  # print(\"Total Templates = {}\\n\".format(len(triples_name)))\n",
        "  # for triple in triples_name:\n",
        "  #   print(triple)\n",
        "\n",
        "  return triples_name"
      ],
      "metadata": {
        "id": "9iqjhWhuPthP"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# for local runtime - upload the file (first time) in the file upload option (left)\n",
        "graph = get_rdf_graph(file_name = SCHEMA_FILE)\n",
        "\n",
        "templates = get_templates(graph)\n",
        "# Keeping only those templates/triples that has Object Type Property as it's Object\n",
        "print(f'Extracted a Tolta of {len(templates)} Relationship Templates Having Only Object Properties from the Knowledge Graph:\\n')\n",
        "templates = [template for template in templates if template[2] != 'string']\n",
        "pdic(templates)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YRl81M6fQSWy",
        "outputId": "010ad88a-d51f-4bd9-ea5c-f7ad72d05a38"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracted a Tolta of 17 Relationship Templates Having Only Object Properties from the Knowledge Graph:\n",
            "\n",
            "[('Disease', 'affects', 'Anatomy'), ('Disease', 'caused_by', 'Cause'),\n",
            " ('Disease', 'has_complication', 'Complication'), ('Disease', 'has_diagnosis', 'Diagnosis'),\n",
            " ('Disease', 'has_precaution', 'Precaution'), ('Disease', 'has_risk_factor', 'Riskfactor'),\n",
            " ('Disease', 'has_symptom', 'Symptom'), ('Diagnosis', 'diagnosis_on', 'Anatomy'),\n",
            " ('Diagnosis', 'needs', 'Surgery'), ('Medicine', 'has_side_effect', 'Complication'),\n",
            " ('Precaution', 'has_side_effect', 'Complication'),\n",
            " ('Surgery', 'has_side_effect', 'Complication'), ('Complication', 'influence', 'Anatomy'),\n",
            " ('Medicine', 'made_with', 'Composition'), ('Medicine', 'prescribed_for', 'Disease'),\n",
            " ('Surgery', 'surgery_for', 'Disease'), ('Surgery', 'surgery_on', 'Anatomy')]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\"\"\" This is just to SHOW the Concepts\"\"\"\n",
        "unique_entities = set()\n",
        "\n",
        "# Extracting unique subjects and objects from the list\n",
        "for sub, pred, obj in templates:\n",
        "  # Adding the '_E' at the end of the Entity Labels since our Ground Truth annotation followed this in order to separate Entities from Relation names\n",
        "  unique_entities.add(sub + '_E')\n",
        "  unique_entities.add(obj + '_E')\n",
        "\n",
        "ENTITY_LABELS = list(unique_entities)\n",
        "print(ENTITY_LABELS)"
      ],
      "metadata": {
        "id": "sIZD1ccBUnLJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1fed49e5-ff60-4f16-98ec-cb117fda86bf"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['Symptom_E', 'Medicine_E', 'Complication_E', 'Precaution_E', 'Anatomy_E', 'Cause_E', 'Surgery_E', 'Disease_E', 'Riskfactor_E', 'Diagnosis_E', 'Composition_E']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def input_file(file_name=\"\"):\n",
        "  '''This function uploads a file and retunrs it's filename'''\n",
        "\n",
        "  # if file_name is empty, that means we are using CoLAB Google Hosted Runtime\n",
        "  if not file_name:\n",
        "    input_file = files.upload()\n",
        "    file_name = list(input_file.keys())[0]\n",
        "\n",
        "  # removing the new file if already exits\n",
        "  path = os.path.abspath(file_name.split('.')[0]+' (1).'+file_name.split('.')[1])\n",
        "  if os.path.exists(path) == True:\n",
        "    os.remove(path)\n",
        "\n",
        "  return file_name"
      ],
      "metadata": {
        "id": "0AP7eo4e1JiQ"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_data_from_csv(file_name=\"\"):\n",
        "  '''\n",
        "  This function will get a CSV from the user having TWO COL i.e., Disease_Name, Affected_Anatomy and\n",
        "  returns a Dictionary having a structure: {\"Tuberculosis\": ['lungs', 'brain', 'kidneys', 'spine'], ...}\n",
        "  '''\n",
        "  # if file_name is empty, that means we are using CoLAB Google Hosted Runtime\n",
        "  if not file_name:\n",
        "    file_name = input_file()\n",
        "\n",
        "  structured_data = {}\n",
        "\n",
        "  with open(file_name, 'r', encoding='utf-8') as file:\n",
        "      csvreader = csv.reader(file)\n",
        "      # we can ignore the header\n",
        "      header = next(csvreader)\n",
        "      # print('Reading New Structured Data Source: {}'.format(file_name.split('\\\\')[1]))\n",
        "      # print('Data Headers: ', header)\n",
        "\n",
        "      for row in csvreader:\n",
        "        # # print(row)\n",
        "        # splits the comma separated values from the 1st column\n",
        "        first_cols = row[0].split(',')\n",
        "\n",
        "        # each of the instances of the first col (subject) will have the same value domain\n",
        "        for instance in first_cols:\n",
        "          instance = instance.strip()\n",
        "\n",
        "          # splits the comma separated values from the 2nd column\n",
        "          value_domain = row[1].split(',')\n",
        "          # removes the leading and trailing spaces\n",
        "          value_domain = [x.strip() for x in value_domain]\n",
        "          # making instance/value-domain dictionary\n",
        "          structured_data[instance] = value_domain\n",
        "\n",
        "  # print('Instances/Values:')\n",
        "  # print(structured_data)\n",
        "  # print()\n",
        "\n",
        "  return header, structured_data"
      ],
      "metadata": {
        "id": "SQeMzNp80ssQ"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def accumulate_structured_data(STRUCTURED_DATA_DIR=STRUCTURED_DATA_DIR):\n",
        "  '''This will read the Structured Data From the CSV Files of a Directory and accumulate them'''\n",
        "  # DATA: gets the structured data from CSV files\n",
        "  accu_data = {}\n",
        "\n",
        "  for dirname, _, filenames in os.walk(STRUCTURED_DATA_DIR):\n",
        "    for filename in filenames:\n",
        "      file_path = os.path.join(dirname, filename)\n",
        "      # # print(file_path)\n",
        "      sd_header, sd_data = get_data_from_csv(file_path)\n",
        "      accu_data = accumulate_unique_data(prev_data=accu_data, keys_list=sd_header, data_dic=sd_data)\n",
        "\n",
        "  # print('\\nAfter Structured Data Accumulation:')\n",
        "  # print(accu_data)\n",
        "\n",
        "  return accu_data"
      ],
      "metadata": {
        "id": "zLTZuSuozoHw"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_text(file_name=\"\"):\n",
        "  '''Getting the TEXT file from the user - and renaming it'''\n",
        "\n",
        "  # if file_name is empty, that means we are using CoLAB Google Hosted Runtime\n",
        "  input_txt = \"\"\n",
        "  if not file_name:\n",
        "    text_file = files.upload()\n",
        "    file_name = list(text_file.keys())[0]\n",
        "    input_txt = text_file[file_name].decode(\"utf-8\").strip()\n",
        "\n",
        "    given_text_file_name = str(file_name)\n",
        "\n",
        "    old_name = '/content/'+ given_text_file_name\n",
        "    new_name = \"/content/\" + 'inference_text.txt'\n",
        "\n",
        "    # removing the new file if already exits\n",
        "    if os.path.exists(new_name) == True:\n",
        "      os.remove(new_name)\n",
        "\n",
        "    os.rename(old_name, new_name)\n",
        "    file_name = new_name\n",
        "\n",
        "  # if the file is called from local runtime with filename specified\n",
        "\n",
        "  with open(file_name, 'r', encoding='utf-8') as text_file:\n",
        "    input_txt = text_file.read().strip()\n",
        "\n",
        "  # print(\"Contents of the Input Text File:\")\n",
        "  # print(input_txt)\n",
        "\n",
        "  return input_txt"
      ],
      "metadata": {
        "id": "jNhRCQEkMFl3"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def sentence_segmentation(input_txt, print_indx = False):\n",
        "    '''This Function Segments the Text into Sentences and Tokens'''\n",
        "\n",
        "    doc = nlp(input_txt)\n",
        "\n",
        "    sentences = []\n",
        "    # sentences_tokens = []\n",
        "    for sent in doc.sents:\n",
        "        # print(sent)\n",
        "        # here sentences are strings\n",
        "        sentences.append(sent.text.strip())\n",
        "        # here a  particular sentence is a list of strings containing the words/tokens\n",
        "        # sentences_tokens.append([token.text for token in sent])\n",
        "\n",
        "    # print('\\nTotal Sentence Found = ', len(sentences))\n",
        "\n",
        "    if print_indx is not False:\n",
        "        print('\\nExample Sentence and Tokens:')\n",
        "        print(sentences[print_indx])\n",
        "        # print(sentences_tokens[print_indx])\n",
        "\n",
        "    return doc, sentences"
      ],
      "metadata": {
        "id": "i-zCn0yMRKqe"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Conceptualization"
      ],
      "metadata": {
        "id": "ktC0mpAf6rw_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def accumulate_unique_data(prev_data, keys_list, data_dic):\n",
        "  ''' This function creates a set (unique lists) of data for each of the Concepts (e.g., Disease, Sysmtoms, Anatomy etc.)\n",
        "      Parameters:\n",
        "        pre_data  -> already created dictionary with other data sources previously\n",
        "        keys_list -> list of keys (column names) for the new data source\n",
        "        data_dic  -> a dictionary holding the source\n",
        "\n",
        "      Returns:  a dictionary having the keys_list as key and a list of unique (set) values from the data source\n",
        "  '''\n",
        "  new_data = {}\n",
        "  # if we have data already residing in the prev_dic\n",
        "  if prev_data:\n",
        "    new_data = prev_data\n",
        "\n",
        "  # get a list from the keys of the data_dic - for example list of disease names\n",
        "  keys_data_list = list(data_dic.keys())\n",
        "\n",
        "  values_data_list = list(data_dic.values())\n",
        "  # flattening the list of lists and taking only a the unique values\n",
        "  values_data_list = list(set(functools.reduce(operator.iconcat, values_data_list)))\n",
        "\n",
        "  # merging the above tow in a single list of lists\n",
        "  data_lists = [keys_data_list, values_data_list]\n",
        "\n",
        "  # Accumulates the new data into Key and Value while merging prev values if the key is already present\n",
        "  for key, val in zip(keys_list, data_lists):\n",
        "    if key in new_data:\n",
        "      pre_val = new_data[key]\n",
        "      # # print(pre_val)\n",
        "      val = list(set(functools.reduce(operator.iconcat, [val + pre_val])))\n",
        "      # # print(val)\n",
        "\n",
        "    # removing empty string from the instances\n",
        "    if '' in val:\n",
        "      val.remove('')\n",
        "\n",
        "    new_data[key] = val\n",
        "\n",
        "  # # print(new_data,'\\n')\n",
        "  return new_data"
      ],
      "metadata": {
        "id": "QcfJcDbZhNoN"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def initiate_matcher(patterns, threshold=80):\n",
        "  '''Initializing and Fine-tuning the matcher'''\n",
        "  # For more information of how it works please see: https://github.com/gandersen101/spaczz\n",
        "  # changing min_r2 from default of 75 to produce matches in this example -- using custom vocab doesn't work for now\n",
        "  matcher = SimilarityMatcher(vocab=nlp.vocab, min_r2=threshold)\n",
        "\n",
        "  # UNCOMMENT - this if you set the threshold level to below 50\n",
        "  # matcher = SimilarityMatcher(vocab=nlp.vocab, min_r2=threshold, min_r1=1)\n",
        "\n",
        "  for key_concept in patterns:\n",
        "    # print(key_concept)\n",
        "\n",
        "    pattern = [nlp(instance.lower()) for instance in patterns[key_concept]]\n",
        "    # pattern = [nlp(patterns[key_concept])]\n",
        "\n",
        "    # we can add as many patterns as we want with specified names of Keys/Concepts\n",
        "    matcher.add(key_concept, pattern)\n",
        "    # Ex: Dimitris adding the concept as the embedding\n",
        "    # matcher.add(key_concept, [nlp(key_concept.lower())])\n",
        "\n",
        "  return matcher"
      ],
      "metadata": {
        "id": "QsGH_roxCIwF"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def strip_noun_chunks(np_chunks, sent):\n",
        "  ''' Removes leading and trailing stop words from noun chunks '''\n",
        "  cleaned_chunks = []\n",
        "  for chunk in np_chunks:\n",
        "    start = chunk.start\n",
        "    end = chunk.end\n",
        "\n",
        "    # Move start and end while they point to a stop word\n",
        "    while start < end and sent[start].is_stop:\n",
        "      start += 1\n",
        "    while start < end and sent[end - 1].is_stop:\n",
        "      end -= 1\n",
        "\n",
        "    # If there's any non-stop word left, create a new span with the new start and end\n",
        "    if start < end:\n",
        "      cleaned_chunks.append(Span(sent, start, end, label=chunk.label))\n",
        "\n",
        "  return cleaned_chunks"
      ],
      "metadata": {
        "id": "khrIU0xS7oky"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def identify_noun_chunks(SENTENCES):\n",
        "  ''' This function iterates over the sentences and splits them into NP (Noun Chunks)\n",
        "  '''\n",
        "  # doc = nlp(\"Typical symptoms of active Tuberculosis are chronic cough with blood-containing mucus, fever, night sweats, and weight loss.\")\n",
        "  # NP per sentence {indx:(sen_indx, [chunk1, chunk1,...])}\n",
        "  np_sentences = {}\n",
        "\n",
        "  for indx, sentence in enumerate(SENTENCES):\n",
        "    # print(\"\\nSENTENCE-{}: {}\\n\".format(indx, sentence))\n",
        "    # This is using Transformer Based Model to generate better Dependency Parsing.\n",
        "    sent = nlp_trf(sentence)\n",
        "\n",
        "    # Extracts noun chunks possibly with leading and trailing stop words from the sentence\n",
        "    np_chunks = [chunk for chunk in sent.noun_chunks]\n",
        "\n",
        "    # We are cleaning the chunks from leading/trailing/fully-containing stop words\n",
        "    clean_np_chunks = strip_noun_chunks(np_chunks, sent)\n",
        "\n",
        "    np_sentences[indx] = (sent, clean_np_chunks)\n",
        "\n",
        "  return np_sentences"
      ],
      "metadata": {
        "id": "I3-e6OBFvPah"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from difflib import SequenceMatcher\n",
        "import lemminflect\n",
        "from nltk.stem import PorterStemmer\n",
        "\n",
        "def sequence_similarity(phrase, pattern):\n",
        "  return SequenceMatcher(None, phrase, pattern).ratio()\n",
        "\n",
        "def word_overlap_lemma(phrase, pattern):\n",
        "  '''This function computes the no. of overlaps between the lemmatized tokens of two nlp objects '''\n",
        "  phrase_words_set = {token._.lemma().lower() for token in phrase if not token.is_punct}\n",
        "  pattern_words_set = {token._.lemma().lower() for token in pattern if not token.is_punct}\n",
        "\n",
        "  # score = Intersection / Union (similar to IoU score)\n",
        "  # The & operator gives the intersection of two sets, while | gives set union\n",
        "  word_overlap_score = (len(phrase_words_set & pattern_words_set)/len(phrase_words_set | pattern_words_set))\n",
        "\n",
        "  return word_overlap_score\n",
        "\n",
        "def word_overlap_stem(phrase, pattern):\n",
        "  '''This function computes the no. of overlaps between the lemmatized tokens of two nlp objects '''\n",
        "  ps = PorterStemmer()\n",
        "\n",
        "  phrase_words_set = {ps.stem(token.text.lower()) for token in phrase if not token.is_punct}\n",
        "  pattern_words_set = {ps.stem(token.text.lower()) for token in pattern if not token.is_punct}\n",
        "\n",
        "  # score = Intersection / Union (similar to IoU score)\n",
        "  # The & operator gives the intersection of two sets, while | gives set union\n",
        "  word_overlap_score = (len(phrase_words_set & pattern_words_set)/len(phrase_words_set | pattern_words_set))\n",
        "\n",
        "  return word_overlap_score"
      ],
      "metadata": {
        "id": "h9IjtPnMmhaU"
      },
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_dup_match(matches):\n",
        "  '''This function will remove the matched instances where they have same span and entity type but different confidence score\n",
        "    Example:\n",
        "      (abdomen, 35, 36, Anatomy, 1.0)\n",
        "      (abdomen, 35, 36, Anatomy, 0.5)\n",
        "      (abdomen, 35, 36, Anatomy, 0.3)\n",
        "  '''\n",
        "  filtered_dict = {}\n",
        "\n",
        "  for match_tup in matches:\n",
        "    # We'll create a dictionary with keys as a combination of (match_id, start, end) and values as the original tuple.\n",
        "    match_id, start, end, confidence_score, pattern = match_tup\n",
        "    key = (match_id, start, end)\n",
        "\n",
        "    # If we encounter a tuple with the same (match_id, start, end) but a higher confidence_score, we'll update the dictionary value.\n",
        "    if key not in filtered_dict or confidence_score > filtered_dict[key][3]:\n",
        "      filtered_dict[key] = match_tup\n",
        "\n",
        "  return list(filtered_dict.values())"
      ],
      "metadata": {
        "id": "dL5VR9reILbu"
      },
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def matching_and_ranking(np_sentences, matcher, top_k=1):\n",
        "  ''' Need More Details: This is a Core function that uses Word Vectors internally to match similar concepts...!! '''\n",
        "\n",
        "  # matches per sentence {i:[(np_chunk, matched_sub_chunk, pattern_concept, confidence), ...]}\n",
        "  matched_sentences = {}\n",
        "\n",
        "  for i in np_sentences:\n",
        "\n",
        "    sent, clean_np_chunks = np_sentences[i]\n",
        "    matches_sent = []\n",
        "    # print(f\"\\nSENTENCE-{i+1}: {sent}\")\n",
        "\n",
        "    for chunk in clean_np_chunks:\n",
        "      phrase = nlp(chunk.text.lower())\n",
        "\n",
        "      # TODO: we could apply the Matcher for the whole DOCUMENT - to check **speed improvement**\n",
        "      matches_orig = matcher(phrase)\n",
        "\n",
        "      matches = []\n",
        "\n",
        "      # We go for further ranking (+sorting) using two more matching techniques\n",
        "      for match_id, start, end, ratio, pattern in matches_orig:\n",
        "\n",
        "        # Exception: This is a special case of possessive noun which we need to ignore from the matching results\n",
        "        if(phrase.text ==\"'s\"):\n",
        "          continue\n",
        "\n",
        "        # Word overlaps - providing nlp objects for better tokenization\n",
        "        # word_matched = word_overlap_lemma(phrase[start:end], nlp(pattern))\n",
        "        word_matched = word_overlap_stem(phrase[start:end], nlp(pattern))\n",
        "\n",
        "        # Gestalt pattern matching - difflib\n",
        "        seq_sim = sequence_similarity(phrase[start:end].text, pattern)\n",
        "\n",
        "        # Mean Similarity Score from the 2 matchers - 'ratio' is out of 100 so we normalized it\n",
        "        # confidence_score = (ratio/100 + word_matched)/2\n",
        "        confidence_score = (ratio/100 + word_matched + seq_sim)/3\n",
        "        confidence_score = round(confidence_score, 2)\n",
        "\n",
        "        matches.append((match_id, start, end, confidence_score, pattern))\n",
        "\n",
        "      # Removing duplicate matches having same span as well as same entity type (matching id)\n",
        "      # matches = remove_dup_match(matches)\n",
        "\n",
        "      # Sorting the list of tuples by 'confidence score'\n",
        "      matches.sort(key=itemgetter(3), reverse=True)\n",
        "\n",
        "      # TODO: For now we are only taking the TOP-3 matching results.\n",
        "      for match_id, match_start, match_end, conf, pattern in matches[0:top_k]:\n",
        "        # This is to track down the matches within the whole Sentence\n",
        "        sent_start = chunk.start + match_start\n",
        "        sent_end = chunk.start + match_end\n",
        "        matched_sub_chunk = Span(sent, sent_start, sent_end, label=chunk.label)\n",
        "\n",
        "        # TODO: If we need we can also include the NP (chunk) here for future use\n",
        "        matches_sent.append((chunk, matched_sub_chunk, match_id, conf))\n",
        "\n",
        "        # print(f'Chunk = {chunk}, Match Start = {match_start}, Match End = {match_end}, Matched Sub-Chunk = {matched_sub_chunk}, Concept = {match_id}, Confidence = {conf}, Pattern = {pattern}')\n",
        "\n",
        "    # Adding the (sent, matches) to the dictionary with Sentence Index as Key for return purpose\n",
        "    matched_sentences[i] = (sent, matches_sent)\n",
        "\n",
        "  return matched_sentences"
      ],
      "metadata": {
        "id": "eu2ENipU-OVy"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def print_match(matched_sentences):\n",
        "  # print('Final List of Concepts in this Sentence:\\n')\n",
        "\n",
        "  count = 0\n",
        "  for S_i in matched_sentences:\n",
        "    sent, matches = matched_sentences[S_i]\n",
        "\n",
        "    print(\"Sentence-{}: '{}'\".format(S_i, sent))\n",
        "    # print(matches)\n",
        "\n",
        "    for np_chunk, matched_sub_chunk, match_id, conf in matches:\n",
        "      print(f'\\t\\tNP Chunk = {np_chunk}, Matched Sub-Chunk = {matched_sub_chunk}, Match Start = {matched_sub_chunk.start}, Match End = {matched_sub_chunk.end}, Concept = {match_id}, Confidence = {conf}')\n",
        "      count += 1\n",
        "    print()\n",
        "\n",
        "  print(f'\\nTotal Entities Recognized in Sentences = {count}')"
      ],
      "metadata": {
        "id": "pxWD_Ejyux0Q"
      },
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def conceptualized_preprocessed_doc(prep_doc, matched_sentences):\n",
        "  ''' This function will map the sentence based matching results into the coreferred documents.\n",
        "      The chunks produces here uses token based span according to the pre-processed document as a whole.\n",
        "\n",
        "      Returns a list of 3 tuples (np_chunk_doc, matched_sub_chunk_doc, match_id, conf)\n",
        "      --> [(Abdominal aortic aneurysm, Abdominal aortic aneurysm, 'Disease', 1.0), (Abdominal aortic aneurysm, Abdominal, 'Anatomy', 0.62), ...]\n",
        "  '''\n",
        "\n",
        "  entities_pre_doc = []\n",
        "  for i, sent in enumerate(prep_doc.sents):\n",
        "    sent_from_matcher, matched_results = matched_sentences[i]\n",
        "\n",
        "    for np_chunk, matched_sub_chunk, match_id, conf in matched_results:\n",
        "      # Adjust the start and end indices of NP Chunk according to the doc\n",
        "      np_chunk_doc_start = sent.start + np_chunk.start\n",
        "      np_chunk_doc_end = sent.start + np_chunk.end\n",
        "\n",
        "      # Adjust the start and end indices of matched sub-chunk according to the doc\n",
        "      sub_chunk_doc_start = sent.start + matched_sub_chunk.start\n",
        "      sub_chunk_doc_end = sent.start + matched_sub_chunk.end\n",
        "\n",
        "      # Making token based span according to the doc\n",
        "      np_chunk_doc = Span(prep_doc, np_chunk_doc_start, np_chunk_doc_end, label=np_chunk.label)\n",
        "      matched_sub_chunk_doc = Span(prep_doc, sub_chunk_doc_start, sub_chunk_doc_end, label=matched_sub_chunk.label)\n",
        "      entities_pre_doc.append((np_chunk_doc, matched_sub_chunk_doc, match_id, conf))\n",
        "\n",
        "      # print(matched_sub_chunk, matched_sub_chunk_doc)\n",
        "\n",
        "  # print(f'\\nTotal Entities Mapped in Document = {len(entities_pre_doc)}')\n",
        "  return entities_pre_doc"
      ],
      "metadata": {
        "id": "3HNW3GBqJZjk"
      },
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_char_span(orig_doc, input_txt, entities_pre_doc, tag_np_chunk_only = False):\n",
        "  '''This function conversts token span into character span for the evaluation script to work.\n",
        "     You can choose either to tag the NP Chunk or to be more granular by tagging with Sub-Chunk level matching (default).\n",
        "     The original document object is also populated with the token span so that we can visualize the results of our prediction\n",
        "  '''\n",
        "\n",
        "  entities_span = []\n",
        "  doc_span = []\n",
        "\n",
        "  if tag_np_chunk_only:\n",
        "    for np_chunk_doc, matched_sub_chunk_doc, match_id, conf in entities_pre_doc:\n",
        "      # This will be used for SemEval Evaluation\n",
        "      entities_span.append((np_chunk_doc.start_char, np_chunk_doc.end_char, match_id+'_E'))\n",
        "      # This will be used to visualize the Span\n",
        "      doc_span.append(Span(orig_doc, np_chunk_doc.start, np_chunk_doc.end, match_id+'_E'))\n",
        "  else:\n",
        "    for np_chunk_doc, matched_sub_chunk_doc, match_id, conf in entities_pre_doc:\n",
        "      # This will be used for SemEval Evaluation\n",
        "      entities_span.append((matched_sub_chunk_doc.start_char, matched_sub_chunk_doc.end_char, match_id+'_E'))\n",
        "      # This will be used to visualize the Span\n",
        "      doc_span.append(Span(orig_doc, matched_sub_chunk_doc.start, matched_sub_chunk_doc.end, match_id+'_E'))\n",
        "\n",
        "  # A dictionary in the format {'text': 'Tuberculosis generally damages the lungs', 'entities': [(0, 12, 'Disease_E'), (35, 40, 'Anatomy_E')]}\n",
        "  ner_prediction = {'text': input_txt, 'entities': entities_span}\n",
        "  orig_doc.spans[\"sc\"] = doc_span\n",
        "\n",
        "  return orig_doc, ner_prediction"
      ],
      "metadata": {
        "id": "otK3hsiA74t8"
      },
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def list_to_spacy_ner_doc(ner_pred):\n",
        "  '''\n",
        "  This function takes a list of directory of NER predictions of the form\n",
        "  {'text': '...', 'entities':[(start, end, tag)]} and converts them into SpaCy Doc Object\n",
        "  '''\n",
        "  # Creates a blank Tokenizer with just the English vocab\n",
        "  nlp = spacy.blank(\"en\")\n",
        "\n",
        "  Doc.set_extension(\"rel\", default={},force=True)\n",
        "  vocab = Vocab()\n",
        "\n",
        "  # try:\n",
        "  # parsing the docanno JSON data (per-line)\n",
        "  text = ner_pred[\"text\"]\n",
        "  spans = ner_pred[\"entities\"]\n",
        "\n",
        "  \"\"\" Parsing tokens from Text \"\"\"\n",
        "  tokens = nlp(text)\n",
        "\n",
        "  entities = []\n",
        "\n",
        "  spaces = [True if tok.whitespace_ else False for tok in tokens]\n",
        "  words = [t.text for t in tokens]\n",
        "  doc = Doc(nlp.vocab, words=words, spaces=spaces)\n",
        "\n",
        "  for start, end, label in spans:\n",
        "    \"\"\" The modes should be: strict, contract, and expand \"\"\"\n",
        "      # print(eg['text'][int(span[\"start_offset\"]):int(span[\"end_offset\"])])\n",
        "    entity = doc.char_span(start, end, label=label, alignment_mode='contract')\n",
        "\n",
        "    # Not considering the spans which are Erroneous\n",
        "    if entity is None:\n",
        "      # file_name = text.split('\\n')[0]\n",
        "      # print(f'No Entity Found in File: {file_name};\\n Span = {start}-{end}; Phrase = {doc.text[start:end]}; Label = {label}\\n')\n",
        "      continue\n",
        "    else:\n",
        "      entities.append(entity)\n",
        "\n",
        "  # print(entities[0].label_)\n",
        "  try:\n",
        "    doc.ents = entities\n",
        "  except:\n",
        "    print(\"=>> Error\")\n",
        "    print(text)\n",
        "\n",
        "  # except:\n",
        "  #   print('Error While Loading Predicted List...')\n",
        "\n",
        "  return doc"
      ],
      "metadata": {
        "id": "1rL4VUMArPxu"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''Assigns different colors to the Entities during visualization.'''\n",
        "\n",
        "color_list = ['yellow', 'white', 'orange', '#0D9CB4', '#5813C7', '#0D350E', '#1AA436',\n",
        "          '#1AE0F9', '#BADCA1', '#78A2E5', '#D845FB', '#54B69E']\n",
        "\n",
        "colors = dict(zip(ENTITY_LABELS, color_list))\n",
        "options = {\"colors\": colors}"
      ],
      "metadata": {
        "id": "uVqNZk4vJRmg"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def render_pred_span_cat(doc):\n",
        "  '''Desired Format: doc.spans[\"sc\"] = [Span(doc, 3, 6, \"ORG\"), Span(doc, 5, 6, \"GPE\")]\n",
        "  '''\n",
        "  spacy.displacy.render(doc, style=\"span\", options=options, jupyter=True)"
      ],
      "metadata": {
        "id": "5X5IdFY4A6Pi"
      },
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def render_groud_truth_entities(doc):\n",
        "  spacy.displacy.render(doc, style=\"ent\", options=options, jupyter=True)"
      ],
      "metadata": {
        "id": "0X3Zmm_SJWKO"
      },
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def save_predictions(ner_predictions, filename, semeval_format=True):\n",
        "  # Saving the predictions as JSON - each dictionary on a line\n",
        "  semeval_ent = []\n",
        "  with open(OUTPUT_DIR+'/'+filename, 'w', encoding='UTF-8') as json_file:\n",
        "    for pred in ner_predictions:\n",
        "      if semeval_format:\n",
        "        # prodigy format to work with nereval library - for SemEval 2013 - 9.1 task.\n",
        "        tmp_ent = []\n",
        "        for ent in pred['entities']:\n",
        "          # saved in this format: [{\"label\": \"PER\", \"start\": 2, \"end\": 4}, ... ]\n",
        "          tmp_ent.append({\"label\": ent[2], \"start\": ent[0], \"end\": ent[1]})\n",
        "\n",
        "        semeval_ent.append(tmp_ent)\n",
        "\n",
        "      else:\n",
        "        # this is regullar spacy format, can be used for spacy's default evaluation later\n",
        "        json_file.write(json.dumps(pred, ensure_ascii=False))\n",
        "        json_file.write('\\n')\n",
        "\n",
        "    if semeval_format:\n",
        "      # dumping it into a JSON file\n",
        "      json_file.write(json.dumps(semeval_ent, ensure_ascii=False))\n",
        "\n",
        "  return semeval_ent\n",
        "  # # This is single line JSON Dump of the entile list of dictionary - parser cannot parse it directly\n",
        "  # with open(OUTPUT_DIR+'/predition.jsonl', 'w') as fout:\n",
        "  #     json.dump(ner_predictions, fout)"
      ],
      "metadata": {
        "id": "TKvCpbC5ov7I"
      },
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def process_results(results_by_tag):\n",
        "  '''This fucntion is used to process the individual entity based scores together'''\n",
        "  results_by_entity = []\n",
        "  for entity in ENTITY_LABELS:\n",
        "    df = pd.DataFrame(results_by_tag[entity])\n",
        "    df = df.round(decimals = 2)\n",
        "    df.insert(0,'Entity','')\n",
        "    df['Entity'] = entity\n",
        "    results_by_entity.append(df)\n",
        "\n",
        "  return results_by_entity"
      ],
      "metadata": {
        "id": "QPUwmNTfu0Tw"
      },
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def semeval_evaluation(true, pred):\n",
        "    evaluator = Evaluator(true, pred, tags=ENTITY_LABELS)\n",
        "    results, results_by_tag = evaluator.evaluate()\n",
        "\n",
        "    results = pd.DataFrame(results)\n",
        "    results.to_excel(OUTPUT_DIR+'/'+'overall_benchmark.xlsx')\n",
        "\n",
        "    results_by_entity = pd.concat(process_results(results_by_tag))\n",
        "    results_by_entity.to_excel(OUTPUT_DIR+'/'+'entity_benchmark.xlsx')\n",
        "\n",
        "    return results, results_by_entity"
      ],
      "metadata": {
        "id": "6FlqR5Dqt_nP"
      },
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Main Function"
      ],
      "metadata": {
        "id": "_x4dRIoBxInd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "if __name__=='__main__':\n",
        "\n",
        "  # Prompting for evaluation set - validation/test\n",
        "  EVAL_SET = ['valid', 'test', 'train', 'inf']\n",
        "  EVAL_SET = EVAL_SET[int(input('Chose Dataset for Evaluation:\\n[0] Validation\\n[1] Test\\n[2] Training\\n[3] Inferencing\\nEnter your choice:'))]\n",
        "\n",
        "  # Input/Output Directories\n",
        "  INPUT_DIR = \"\"\n",
        "  OUTPUT_DIR = \"\"\n",
        "\n",
        "  if EVAL_SET == 'inf':\n",
        "    # uploaded_text = files.upload()\n",
        "    # text_filename = inference_file(uploaded_text)\n",
        "    input_text = get_text()\n",
        "    samples = [{'text':input_text}]\n",
        "    # print(input_text)\n",
        "  else:\n",
        "    # Input/Output Directories\n",
        "    INPUT_DIR = f'/content/data/Annotated_Text/{EVAL_SET}'\n",
        "    OUTPUT_DIR = f'/content/output/{EVAL_SET}'\n",
        "    create_out_dir(OUTPUT_DIR)\n",
        "\n",
        "    print('\\nPreparing Evaluation Dataset:')\n",
        "    doc_valid, db_valid = docanno_to_spacy_ner_db(INPUT_DIR)\n",
        "    db_valid.to_disk(OUTPUT_DIR + f\"/disease_A-Z_{EVAL_SET}.spacy\")\n",
        "\n",
        "    # reading the ground truth entities with text file from spacy docbin\n",
        "    samples, ground_entities_count = load_json_from_docbin(OUTPUT_DIR + f\"/disease_A-Z_{EVAL_SET}.spacy\", nlp)\n",
        "\n",
        "  # reading and accumulating (integrated) structured data\n",
        "  accu_data = accumulate_structured_data()\n",
        "  # pdic(accu_data)\n",
        "\n",
        "  # initializing and fine-tuning the matcher\n",
        "  matcher = initiate_matcher(patterns=accu_data, threshold=80)\n",
        "\n",
        "  ner_predictions = []\n",
        "  all_docs = []\n",
        "  count = 0\n",
        "  total_doc = len(samples)\n",
        "\n",
        "  print('\\nConceptualization Process Started...')\n",
        "  # each sample represents one document..\n",
        "  for ground in samples:\n",
        "    input_txt = ground['text']\n",
        "    # print(f'\\n{input_txt}')\n",
        "\n",
        "    # Segmenting the document into sentences - for Conceptualization we can also process the whole document\n",
        "    doc, sentences = sentence_segmentation(input_txt, print_indx=False)\n",
        "\n",
        "    # Detecting the NP Chunks - also removes leading and trailing stop words in the NP chunks\n",
        "    np_sentences = identify_noun_chunks(sentences)\n",
        "    # print(np_sentences)\n",
        "\n",
        "    # Getting the initial Top-k matrching results\n",
        "    matched_sentences = matching_and_ranking(np_sentences, matcher=matcher, top_k=1)\n",
        "    # print_match(matched_sentences)\n",
        "\n",
        "    # Conceptualizing the Text Document - token based indices\n",
        "    entities_pre_doc = conceptualized_preprocessed_doc(doc, matched_sentences)\n",
        "    # print(f'\\nEntities in the Coreference Resolved Document:\\n{entities_pre_doc}')\n",
        "\n",
        "    # Organizing indexes of those entitties according to the original text, also creating span categories for visualization\n",
        "    doc, ner_pred_tmp = get_char_span(doc, input_txt, entities_pre_doc, tag_np_chunk_only = True)\n",
        "    # print(f'\\nNER Prediction for current document:\\n{ner_pred_tmp}')\n",
        "    ner_predictions.append(ner_pred_tmp)\n",
        "    all_docs.append(doc)\n",
        "\n",
        "    count +=1\n",
        "    # print(f'Document-{count}')\n",
        "    if count % 10 == 0:\n",
        "      print(f'{count}/{total_doc} Document Processed...')\n",
        "\n",
        "  if EVAL_SET == 'inf':\n",
        "    # Visualizing the Predictions for the Given Input Text\n",
        "    print('\\n########### Prediction ###########\\n')\n",
        "    render_pred_span_cat(all_docs[0])\n",
        "\n",
        "  else:\n",
        "    # saving the grond and predictions into a JSONL file for later evaluation.\n",
        "    semeval_ground = save_predictions(samples, filename='ground.jsonl')\n",
        "    semeval_pred = save_predictions(ner_predictions, filename='predition.jsonl')\n",
        "\n",
        "    # doing the evaluation following SemEval 2013 metrics\n",
        "    results, results_by_entity = semeval_evaluation(true=semeval_ground, pred=semeval_pred)\n",
        "\n",
        "    # Saving this for Future Experiments... Spacy Format\n",
        "    _ = save_predictions(ner_predictions, filename='predition_THOR_spacy.jsonl', semeval_format=False)\n",
        "\n",
        "    print('\\n########### Overall Results ###########\\n')\n",
        "    print(f\"Precision: {results['partial']['precision']}\\nRecall: {results['partial']['recall']}\\nF1: {results['partial']['f1']}\\n\")"
      ],
      "metadata": {
        "id": "1OzJAh7XDTVA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a5eb61ce-f7c1-4b16-a7cf-123df1d9298a"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Chose Dataset for Evaluation:\n",
            "[0] Validation\n",
            "[1] Test\n",
            "[2] Training\n",
            "[3] Inferencing\n",
            "Enter your choice:1\n",
            "\n",
            "Preparing Evaluation Dataset:\n",
            "- Files: 13 \n",
            "- Processed Documents: 90 \n",
            "- Total Entities: 2222 \n",
            "- Erroneous Entities (Ignored): 6 \n",
            "- Total Words: 19237\n",
            "\n",
            "Conceptualization Process Started...\n",
            "10/90 Document Processed...\n",
            "20/90 Document Processed...\n",
            "30/90 Document Processed...\n",
            "40/90 Document Processed...\n",
            "50/90 Document Processed...\n",
            "60/90 Document Processed...\n",
            "70/90 Document Processed...\n",
            "80/90 Document Processed...\n",
            "90/90 Document Processed...\n",
            "\n",
            "########### Overall Results ###########\n",
            "\n",
            "Precision: 0.5635613779718583\n",
            "Recall: 0.5227272727272727\n",
            "F1: 0.5423768386644875\n",
            "\n"
          ]
        }
      ]
    }
  ]
}