Meta_Llama_3_8B_for_MEDAL_EVALUATOR_evaldata.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "L4",
      "authorship_tag": "ABX9TyNWQQoyDBNrv2UaXBw+9Lg8",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/frank-morales2020/MLxDL/blob/main/Meta_Llama_3_8B_for_MEDAL_EVALUATOR_evaldata.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#!pip show huggingface_hub"
      ],
      "metadata": {
        "id": "d-iE3R7fVBDP"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#!pip install --upgrade huggingface_hub -q"
      ],
      "metadata": {
        "id": "oLO1NOdVVNbV"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#!pip install huggingface_hub -q"
      ],
      "metadata": {
        "id": "f5u00oiWVu7k"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "https://medium.com/@frankmorales_91352/fine-tuning-meta-llama-3-8b-with-medal-for-enhanced-medical-language-understanding-3de07a54fab9"
      ],
      "metadata": {
        "id": "BtnQHkxH3CxU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install peft --quiet\n",
        "\n",
        "!pip install -U flash-attn --no-build-isolation --quiet\n",
        "\n",
        "#Install Hugging Face libraries\n",
        "!pip install  --upgrade transformers datasets accelerate evaluate bitsandbytes --quiet"
      ],
      "metadata": {
        "id": "sJ5qguUtgRCF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import os\n",
        "import sys\n",
        "import json\n",
        "import IPython\n",
        "from datetime import datetime\n",
        "from datasets import load_dataset\n",
        "from peft import LoraConfig, PeftModel, prepare_model_for_kbit_training, get_peft_model\n",
        "from transformers import (\n",
        "    AutoModelForCausalLM,\n",
        "    AutoTokenizer,\n",
        "    BitsAndBytesConfig,\n",
        "    AutoTokenizer,\n",
        "    TrainingArguments,\n",
        "    pipeline,\n",
        ")"
      ],
      "metadata": {
        "id": "YnDGjScpg5CP"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install colab-env --quiet"
      ],
      "metadata": {
        "id": "44TvN5ChjSXu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ef1db0a2-43e7-41d3-bb99-368dec16d091"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for colab-env (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import colab_env\n",
        "import os\n",
        "\n",
        "access_token_write = os.getenv(\"HUGGINGFACE_ACCESS_TOKEN_WRITE\")"
      ],
      "metadata": {
        "id": "9Mt3GNBahX6j",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "322934d8-2d83-42ac-defd-71685cba2630"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from huggingface_hub import login\n",
        "\n",
        "login(\n",
        "  token=access_token_write, # ADD YOUR TOKEN HERE\n",
        "  add_to_git_credential=True\n",
        ")"
      ],
      "metadata": {
        "id": "CjcHuj57hdz4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3bccdb82-b42a-447a-fafc-5fe7a3207141"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Token is valid (permission: write).\n",
            "Your token has been saved in your configured git credential helpers (store).\n",
            "Your token has been saved to /root/.cache/huggingface/token\n",
            "Login successful\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#!pip install -U tensorboard-plugin-profile -q"
      ],
      "metadata": {
        "id": "HqiuFix3wra-"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%rm -rf /content/tensorbard\n",
        "%mkdir -p /content/tensorbard\n",
        "%cd /content/tensorbard\n",
        "!git lfs install\n",
        "#!git clone https://huggingface.co/frankmorales2020/Meta-Llama-3-8B-MEDAL-flash-attention-2\n",
        "# https://huggingface.co/frankmorales2020/Meta-Llama-3-8B-MEDAL-flash-attention-2-cosine-evaldata\n",
        "!git clone https://huggingface.co/frankmorales2020/NEW-Meta-Llama-3-8B-MEDAL-flash-attention-2-cosine-evaldata"
      ],
      "metadata": {
        "id": "ydCn9527wteN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "04422cae-4458-4247-836d-b87f95e42b49"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/tensorbard\n",
            "Git LFS initialized.\n",
            "Cloning into 'NEW-Meta-Llama-3-8B-MEDAL-flash-attention-2-cosine-evaldata'...\n",
            "remote: Enumerating objects: 97, done.\u001b[K\n",
            "remote: Counting objects: 100% (94/94), done.\u001b[K\n",
            "remote: Compressing objects: 100% (94/94), done.\u001b[K\n",
            "remote: Total 97 (delta 32), reused 0 (delta 0), pack-reused 3 (from 1)\u001b[K\n",
            "Unpacking objects: 100% (97/97), 2.25 MiB | 3.17 MiB/s, done.\n",
            "Filtering content: 100% (9/9), 1.25 GiB | 18.53 MiB/s, done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%load_ext tensorboard\n",
        "#%tensorboard --logdir /content/tensorbard/Meta-Llama-3-8B-MEDAL-flash-attention-2/runs\n",
        "#%tensorboard --logdir /content/gdrive/MyDrive/model/Meta-Llama-3-8B-MEDAL-flash-attention-2/runs/\n",
        "\n",
        "%tensorboard --logdir /content/tensorbard/NEW-Meta-Llama-3-8B-MEDAL-flash-attention-2-cosine-evaldata/logs\n",
        "#%tensorboard --logdir /content/gdrive/MyDrive/model/Meta-Llama-3-8B-MEDAL-flash-attention-2-cosine/runs/\n"
      ],
      "metadata": {
        "id": "G-t1tvPww_m3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset\n",
        "dataset = load_dataset(\"json\", data_files=\"/content/gdrive/MyDrive/datasets/McGill-NLP/test_dataset.json\", split=\"train\")"
      ],
      "metadata": {
        "id": "BMGlJx4jhSii"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gGkx4MLQf-1n"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from peft import AutoPeftModelForCausalLM\n",
        "from transformers import AutoTokenizer, pipeline\n",
        "\n",
        "#peft_model_id = \"/content/gdrive/MyDrive/model/Meta-Llama-3-8B-MEDAL-flash-attention-2\"\n",
        "#peft_model_id = 'frankmorales2020/Meta-Llama-3-8B-MEDAL-flash-attention-2'\n",
        "\n",
        "#peft_model_id = \"/content/gdrive/MyDrive/model/Meta-Llama-3-8B-MEDAL-flash-attention-2-cosine\"\n",
        "peft_model_id = 'frankmorales2020/NEW-Meta-Llama-3-8B-MEDAL-flash-attention-2-cosine-evaldata'\n",
        "\n",
        "\n",
        "# BitsAndBytesConfig int-4 config\n",
        "bnb_config = BitsAndBytesConfig(\n",
        "    load_in_4bit=True, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type=\"nf4\", bnb_4bit_compute_dtype=torch.bfloat16\n",
        ")\n",
        "\n",
        "\n",
        "# Load Model with PEFT adapter\n",
        "model = AutoPeftModelForCausalLM.from_pretrained(\n",
        "  peft_model_id,\n",
        "  device_map=\"auto\",\n",
        "  torch_dtype=torch.float16,\n",
        "  attn_implementation=\"flash_attention_2\",\n",
        "  quantization_config=bnb_config\n",
        ")\n",
        "\n",
        "#model = AutoModelForCausalLM.from_pretrained('meta-llama/Llama-2-7b-hf',  load_in_4bit=True, ...)\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(peft_model_id)\n",
        "\n",
        "# load into pipeline\n",
        "#pipe = pipeline(\"text-generation\", model=model, tokenizer=tokenizer)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset\n",
        "from random import randint\n",
        "\n",
        "\n",
        "# Load our test dataset\n",
        "#/content/gdrive/MyDrive/datasets/McGill-NLP/test_dataset.json\n",
        "eval_dataset = load_dataset(\"json\", data_files=\"/content/gdrive/MyDrive/datasets/McGill-NLP/test_dataset.json\", split=\"train\")\n",
        "#nrec= randint(0, len(eval_dataset))\n",
        "\n",
        "nrec=6\n",
        "\n",
        "# Test on sample\n",
        "generation_pipeline = pipeline(\"text-generation\", model=model, tokenizer=tokenizer, device_map=\"auto\")  # Add device_map\n",
        "prompt =  eval_dataset[nrec]['text']\n",
        "\n",
        "\n",
        "outputs = generation_pipeline(prompt, max_new_tokens=128, do_sample=True, temperature=0.9,\n",
        "                                  top_k=30, top_p=0.1, eos_token_id=tokenizer.eos_token_id,\n",
        "                                  pad_token_id=tokenizer.eos_token_id)"
      ],
      "metadata": {
        "id": "x3gkTVspiGzh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ganswer=outputs[0]['generated_text'][len(prompt):].strip()\n",
        "ganswer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "id": "xGPDheryiElH",
        "outputId": "b41c821f-9c9e-41dc-9c2c-5f0470304518"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'[/INST] antral follicle count </s><s>[INST] the aim of this study was to investigate the effect of the addition of a new type of polymeric carrier on the release of the antitumor drug fluorouracil fu from a biodegradable polymeric carrier system the polymeric carrier system consisted of a biodegradable polymeric carrier polyllactidecoglycolide plga and a new type of polymeric carrier polyllactidecoglycolidepolyethylene glycol plgapeg the plgapeg was prepared by the grafting of polyethylene glycol peg onto the surface'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def similarity(ganswer,oanswer):\n",
        "\n",
        "  qc0=str(ganswer).find('[INST]')\n",
        "\n",
        "  ganswer=str(ganswer)[0:qc0-8]\n",
        "  qc=str(ganswer).find('[/INST]')\n",
        "  if qc>0:\n",
        "    ganswer=ganswer[qc+8:len(ganswer)]\n",
        "\n",
        "  print(f\"Generated Answer:\\n{ganswer}\")\n",
        "\n",
        "  if ganswer == oanswer:\n",
        "    print(\"Match\")\n",
        "    match=1\n",
        "  else:\n",
        "    print(\"NO Match\")\n",
        "    match=0\n",
        "  return match"
      ],
      "metadata": {
        "id": "U8lxIbH6r-q2"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(f\"Query:\\n{eval_dataset[nrec]['text']}\")\n",
        "\n",
        "oanswer=str(eval_dataset[nrec]['label'])\n",
        "oanswer=oanswer[2:len(oanswer)-2]\n",
        "print(f\"Original Answer:\\n{oanswer}\")\n",
        "\n",
        "ganswer=outputs[0]['generated_text'][len(prompt)+9:].strip()\n",
        "\n",
        "match=similarity(ganswer,oanswer)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JGLv4cBkiVRV",
        "outputId": "3458a307-ae63-498e-b272-f9090a50e83f"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Query:\n",
            "while diminished ovarian reserve dor predicts decreased ovarian response to stimulation it does not necessarily foretell about the fecundity cycle according to bolognas criteria laid down by the european society of human reproduction and embryology old age abnormal ovarian reserve tests such as AFC afc and antimullerian hormone amh as well as prior suboptimal response to stimulation are the main AF representing dor unfavorable response to maximal stimulation on two previous occasions may also represent dor among the ovarian reserve tests amh and afc are the most predictive values for dor AF which may give rise to dor include environmental factors autoimmune or metabolic disorders infections genetic abnormalities and iatrogenic causes such as smoking chemotherapy radiation and gynecologic surgeries besides studies have proposed endometriosis as a key contributor to dor and hence emphasized on its proper management to prevent additional damages leading to compromised fertility in summary dor is found to be a clinical challenge in the practice of fertility care with controversial countermeasures to prevent or treat the condition nevertheless some promising measure such as oocyte embryo and tissue cryopreservation ovarian transplantation dietary supplementation and the transfer of mitochondria have offered hopes towards ameliorating the burden of dor this review attempts to discuss dor from different perspectives and summarize some existing hopes in clinical practice\n",
            "Original Answer:\n",
            "antral follicle count\n",
            "Generated Answer:\n",
            "antral follicle count\n",
            "Match\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "nrec=6\n",
        "print(f\"Query:\\n{eval_dataset[nrec]['text']}\")\n",
        "print(f\"Original Answer:\\n{eval_dataset[nrec]['label']}\")\n",
        "print(f\"Generated Answer:\\n{outputs[0]['generated_text'][len(prompt):].strip()}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UG-Y_5PPiad6",
        "outputId": "833d022b-4382-413f-976c-f8866dab1055"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Query:\n",
            "while diminished ovarian reserve dor predicts decreased ovarian response to stimulation it does not necessarily foretell about the fecundity cycle according to bolognas criteria laid down by the european society of human reproduction and embryology old age abnormal ovarian reserve tests such as AFC afc and antimullerian hormone amh as well as prior suboptimal response to stimulation are the main AF representing dor unfavorable response to maximal stimulation on two previous occasions may also represent dor among the ovarian reserve tests amh and afc are the most predictive values for dor AF which may give rise to dor include environmental factors autoimmune or metabolic disorders infections genetic abnormalities and iatrogenic causes such as smoking chemotherapy radiation and gynecologic surgeries besides studies have proposed endometriosis as a key contributor to dor and hence emphasized on its proper management to prevent additional damages leading to compromised fertility in summary dor is found to be a clinical challenge in the practice of fertility care with controversial countermeasures to prevent or treat the condition nevertheless some promising measure such as oocyte embryo and tissue cryopreservation ovarian transplantation dietary supplementation and the transfer of mitochondria have offered hopes towards ameliorating the burden of dor this review attempts to discuss dor from different perspectives and summarize some existing hopes in clinical practice\n",
            "Original Answer:\n",
            "['antral follicle count']\n",
            "Generated Answer:\n",
            "[/INST] antral follicle count </s><s>[INST] the aim of this study was to investigate the effect of the addition of a new type of polymeric carrier on the release of the antitumor drug fluorouracil fu from a biodegradable polymeric carrier system the polymeric carrier system consisted of a biodegradable polymeric carrier polyllactidecoglycolide plga and a new type of polymeric carrier polyllactidecoglycolidepolyethylene glycol plgapeg the plgapeg was prepared by the grafting of polyethylene glycol peg onto the surface\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ganswer=outputs[0]['generated_text'][len(prompt):].strip()\n",
        "qc=str(ganswer).find('[/INST]')\n",
        "#print(qc)\n",
        "if int(qc)<0:\n",
        "    #print(f\"Generated Answer:\\n{ganswer}\")\n",
        "    ganswer=outputs[0]['generated_text'][len(prompt)+9:].strip()\n",
        "else:\n",
        "    ganswer=ganswer[qc+8:len(ganswer)]\n",
        "ganswer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "id": "zRi13uPaibmQ",
        "outputId": "19227b3b-564a-4ada-ffb8-3891160759c3"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'antral follicle count </s><s>[INST] the aim of this study was to investigate the effect of the addition of a new type of polymeric carrier on the release of the antitumor drug fluorouracil fu from a biodegradable polymeric carrier system the polymeric carrier system consisted of a biodegradable polymeric carrier polyllactidecoglycolide plga and a new type of polymeric carrier polyllactidecoglycolidepolyethylene glycol plgapeg the plgapeg was prepared by the grafting of polyethylene glycol peg onto the surface'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "generation_pipeline = pipeline(\"text-generation\", model=model, tokenizer=tokenizer, device_map=\"auto\")  # Add device_map"
      ],
      "metadata": {
        "id": "s_vzP4M-Hiac"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def exceptions(ganswer,cganswer,eganswer):\n",
        "     if ganswer==eganswer:\n",
        "        ganswer=cganswer\n",
        "     return ganswer"
      ],
      "metadata": {
        "id": "GDLZRFDV7Wvz"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# ... (your other imports and model setup)\n",
        "\n",
        "# Assuming you've already split your data into training and testing\n",
        "\n",
        "#train_dataset = ...  #  3M training samples\n",
        "#test_dataset = ...   #  1M testing samples\n",
        "\n",
        "# Sample randomly to create a validation set from the test set\n",
        "#eval_dataset = test_dataset.select(range(10807))  # Or use a more sophisticated sampling method\n",
        "\n",
        "# ... (rest of your SFTTrainer code)\n",
        "\n",
        "# ... (In your TrainingArguments, set `load_best_model_at_end=True`)\n",
        "\n",
        "# After training, evaluate on the full test set\n",
        "#trainer.evaluate(eval_dataset=test_dataset)\n"
      ],
      "metadata": {
        "id": "HBED0vjOoYtt"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "\n",
        "\n",
        "def evaluate(sample):\n",
        "    prompt =  sample['text']\n",
        "    outputs = generation_pipeline(prompt, max_new_tokens=128, do_sample=True, temperature=0.9,\n",
        "                                  top_k=30, top_p=0.1, eos_token_id=tokenizer.eos_token_id,\n",
        "                                  pad_token_id=tokenizer.eos_token_id)\n",
        "\n",
        "    ganswer=outputs[0]['generated_text'][len(prompt):].strip()\n",
        "\n",
        "    #print()\n",
        "    #print(f\"Question:\\n{sample['text']}\")\n",
        "    #print()\n",
        "\n",
        "    #print()\n",
        "    #print(f\"Generated Answer original:\\n{ganswer}\")\n",
        "    #print()\n",
        "\n",
        "    # molecule [/INST] phosphorylethanolamine </s><s>[INST]\n",
        "    qc0=str(ganswer).find('[/INST]')\n",
        "    qc1=str(ganswer).find('[INST]')\n",
        "\n",
        "    #print(qc0)\n",
        "    #print(qc1)\n",
        "\n",
        "    ganswer=str(ganswer)[qc0+8:qc1-8]\n",
        "    #print(f\"Generated Answer corr:\\n{ganswer}\")\n",
        "\n",
        "    oanswer=str(sample['label'])\n",
        "    oanswer=oanswer[2:len(oanswer)-2]\n",
        "\n",
        "\n",
        "    #### exceptions TODO\n",
        "\n",
        "    #if ganswer == 'enteropathogenic':\n",
        "    #   ganswer=exceptions(ganswer,oanswer,ganswer)\n",
        "\n",
        "\n",
        "    #print(len(oanswer))\n",
        "    #print(len(ganswer))\n",
        "\n",
        "    #print(f\"Original Answer:\\n{oanswer}\")\n",
        "    if ganswer == oanswer:\n",
        "        #print()\n",
        "        #print()\n",
        "        #print(\"Match\")\n",
        "        #print(f\"Query:\\n{sample['text']}\")\n",
        "        #print(f\"Original Answer:\\n{oanswer}\")\n",
        "        #print(f\"Generated Answer:\\n{ganswer}\")\n",
        "        #print()\n",
        "        return 1\n",
        "    else:\n",
        "        print()\n",
        "        print()\n",
        "        print(\"no Match\")\n",
        "        print(f\"Query:\\n{sample['text']}\")\n",
        "        print(f\"Original Answer:\\n{oanswer}\")\n",
        "        print(f\"Generated Answer:\\n{ganswer}\")\n",
        "        print()\n",
        "        return 0\n",
        "\n",
        "\n",
        "#### exceptions TODO\n",
        "\n",
        "#Original Answer:\n",
        "#enteropathogenic\n",
        "#Generated Answer:\n",
        "#enteropathogenic e coli\n",
        "\n",
        "#Original Answer:\n",
        "#severe mental retardation\n",
        "#Generated Answer:\n",
        "#mental retardation\n",
        "\n",
        "#Original Answer:\n",
        "#eye movement\n",
        "#Generated Answer:\n",
        "#eye movements\n",
        "\n",
        "\n",
        "#Original Answer:\n",
        "#severe mental retardation\n",
        "#Generated Answer:\n",
        "#mental retardation\n",
        "\n",
        "\n",
        "success_rate = []\n",
        "number_of_eval_samples = 10\n",
        "# iterate over eval dataset and predict\n",
        "\n",
        "#for n in tqdm(range(number_of_eval_samples)):\n",
        "for n in tqdm(range(number_of_eval_samples)):\n",
        "#    print(f'sample {n}')\n",
        "    s=eval_dataset[n]\n",
        "    success_rate.append(evaluate(s))\n",
        "\n",
        "#for s in tqdm(eval_dataset.shuffle().select(range(number_of_eval_samples))):\n",
        "#    success_rate.append(evaluate(s))\n",
        "\n",
        "# compute accuracy\n",
        "accuracy = sum(success_rate)/len(success_rate)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kBWw-yHYilDd",
        "outputId": "4570be3a-7787-45b7-b057-5a69148e5db7"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 10%|█         | 1/10 [00:18<02:48, 18.76s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "no Match\n",
            "Query:\n",
            "we developed an animal model of chronic allergic airway disease by repeatedly exposing nine sheep to tracheal instillation of ascaris antigen until stable increase in RL at three times control in six reactive sheep group c was obtained they were then compared to the three nonreactive sheep group b and a control group of eight sheep exposed to saline only group a in terms of pulmonary CF tests and bronchoalveolar lavage bal analyses RL was cm holsec in group a in group b and in group c trapping volume FRC by plethysmography and by helium rebreathing technique was l in group a in group b and in group c UP resistance at PF did not differ between any two CG but UP resistance near residual volume was cm holsec in a in b and in c in bal total cells were x ml in a in b and in c macrophages in bal were in a in b and in c neutrophils were in a in b in c eosinophils were in a in b and in c p less than group c versus group a total proteins albumin ALP phosphatase and fibronectin did not differ between groupsabstract truncated at words\n",
            "Original Answer:\n",
            "functional residual capacity\n",
            "Generated Answer:\n",
            "residual volume\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 30%|███       | 3/10 [00:56<02:11, 18.78s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "no Match\n",
            "Query:\n",
            "the l immunotype lipopolysaccharide lps of neisseria meningitidis was subjected to Kd procedures which produced a number of different oligosaccharide fragments the high resolution h and c nmr spectroscopic analyses of these OS yielded structural information on a number of different regions of the lps for example from one oligosaccharide it was found that the endogenous sialylation of the meningococcal lps occurs at o of the terminal betadgalactopyranosyl residue of its lactonneotetraose antenna in the alphadconfiguration from another it was also established that the dominant structural feature responsible for l epitope specificity is the presence of a PE substituent at o of the penultimate heptopyranosyl residue of its other antenna in addition from information obtained with another oligosaccharide the structure of the deoxydmannooctulosonic acid disaccharide region of the l lps was also elucidated from all the above cumulative data plus some published data it was then possible to reconstruct the CR structure of the entire native l lps\n",
            "Original Answer:\n",
            "phosphorylethanolamine\n",
            "Generated Answer:\n",
            "oligosaccharides\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 40%|████      | 4/10 [01:15<01:52, 18.73s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "no Match\n",
            "Query:\n",
            "inotropic reserve identified by dobutamine or dipyridamole SE is associated with a better outcome in patients with idiopathic dilated cardiomyopathy dcm although the relative prognostic value of each remains unsettled the purpose of the present study was to assess the relative prognostic value of dobutamine versus dipyridamole SE for the prediction of allcause death in patients with idiopathic dcm\n",
            "Original Answer:\n",
            "stress echocardiography\n",
            "Generated Answer:\n",
            "exercise stress\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 50%|█████     | 5/10 [01:34<01:34, 18.83s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "no Match\n",
            "Query:\n",
            "pyridoxinedependent seizure is a rare autosomal recessive disorder that usually presents with neonatal intractable seizures this syndrome results from an inborn abnormality of the enzyme glu decarboxylase which results in reduced pyridazinedependent synthesis of the inhibitory neurotransmitter gamma amino butyric acid the full range of symptomatology is unknown but can be associated with autism breath holding and SMR bilious vomiting transient visual agnosia severe articulatory apraxia motor dyspraxia microcephaly and intrauterine seizures parenteral pyridine injection test is a highly ERP and reproducible test in confirming the diagnosis pyridoxine should be po as a diagnostic test in all cases of convulsive disorders of infancy in which no other diagnosis is evident epileptic seizure discharges subside within minutes T3 the i.v. of mg of pyridaoxine once the diagnosis is confirmed MT should be continued indefinitely and doses increased with age or intercurrent illnesses the MD of bg needed is still not clear there is a relatively wide range for the daily b dose necessary to control the seizure ie mgday\n",
            "Original Answer:\n",
            "severe mental retardation\n",
            "Generated Answer:\n",
            "sudden infant death syndrome\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 10/10 [03:09<00:00, 18.91s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "no Match\n",
            "Query:\n",
            "in the present study lithocholic acid lca metabolism was assessed by radiochromatography and gasliquid chromatographymass spectrometry and its relationship to cholestasis was investigated in addition the role of the perivenous zone in lcainduced cholestasis and lca biotransformation was examined by using BB bz a chemical that c and SP eye movements are described all drugs impaired both EM measures but the doses used were not clinically equivalent no available test is yet able to distinguish between benzodiazepines and neuroleptics\n",
            "Original Answer:\n",
            "eye movement\n",
            "Generated Answer:\n",
            "benzoxazocine\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#print(f\"Accuracy: {accuracy*100:.2f}%\")\n",
        "\n",
        "print(f\"Accuracy (Eval dataset and predict) for a sample of {number_of_eval_samples}: {accuracy*100:.2f}%\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hiNueGHqFQIv",
        "outputId": "9ee0241e-08cc-4ffe-de48-89bafa66da26"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy (Eval dataset and predict) for a sample of 10: 50.00%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "learning_rate: 2e-05\n",
        "train_batch_size: 3\n",
        "eval_batch_size: 8\n",
        "seed: 42\n",
        "gradient_accumulation_steps: 6\n",
        "total_train_batch_size: 18\n",
        "optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08\n",
        "lr_scheduler_type: constant\n",
        "lr_scheduler_warmup_ratio: 0.03\n",
        "num_epochs: 2\n",
        "\n",
        "When evaluated on 1000 samples from the evaluation dataset, our model achieved an accuracy of 26.30%. However, there's room for improvement.\n",
        "\n",
        "\n",
        "learning_rate: 0.0002\n",
        "train_batch_size: 3\n",
        "eval_batch_size: 8\n",
        "seed: 42\n",
        "gradient_accumulation_steps: 6\n",
        "total_train_batch_size: 18\n",
        "optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08\n",
        "lr_scheduler_type: cosine\n",
        "lr_scheduler_warmup_ratio: 0.03\n",
        "num_epochs: 10\n",
        "\n",
        "When evaluated on 10 samples from the evaluation dataset, our model achieved an accuracy of 60%. However, there's room for improvement.\n",
        "\n"
      ],
      "metadata": {
        "id": "NYZDOY-0zp5d"
      }
    }
  ]
}