evaluations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ea9401b-2165-4324-a628-31e8f993de74",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install --upgrade pip\n",
    "# !pip install evaluate\n",
    "# !pip install sentence_transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "243489e3-6ef1-4324-8207-46988c5f8896",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import sys\n",
    "import numpy as np\n",
    " \n",
    "from transformers import GPT2Tokenizer, GPT2LMHeadModel\n",
    "# Load pre-trained model (weights)\n",
    "with torch.no_grad():\n",
    "        GPT2_model = GPT2LMHeadModel.from_pretrained('gpt2').cuda()\n",
    "        GPT2_model.eval()\n",
    "# Load pre-trained model tokenizer (vocabulary)\n",
    "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
    " \n",
    "def GPT2_score(sentence):\n",
    "    tokenize_input = tokenizer.encode(sentence)\n",
    "    tensor_input = torch.tensor([tokenize_input]).cuda()\n",
    "    loss=GPT2_model(tensor_input, labels=tensor_input)[0]\n",
    "    return np.exp(loss.cpu().detach().numpy())\n",
    "#print(GPT2_score(sentence='Humans have many basic needs  and one of them is to have an environment that can sustain their lives.'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17e0e874-8d8a-41b6-9caa-427e51b87f61",
   "metadata": {},
   "outputs": [],
   "source": [
    "import evaluate\n",
    "bleu = evaluate.load(\"bleu\")\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "sim_model = SentenceTransformer('bert-base-nli-mean-tokens')\n",
    "from sklearn.metrics.pairwise import cosine_similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8da952df-5ba0-4b52-979c-9e167ccd7716",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d400986a-e563-4cc5-bf89-d63c89d876fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def similarity(sentence1, sentence2):\n",
    "    sentence_embedding1 = sim_model.encode(sentence1)\n",
    "    sentence_embedding2 = sim_model.encode(sentence2)\n",
    "    sim_score = cosine_similarity([sentence_embedding1], [sentence_embedding2])\n",
    "    return sim_score[0][0]\n",
    "\n",
    "def evaluate(preds, refs):\n",
    "    with open(preds, 'rb') as f:\n",
    "        predictions = pickle.load(f)\n",
    "        predictions = predictions[:1000]\n",
    "    with open(refs, 'rb') as f:\n",
    "        references = pickle.load(f)\n",
    "        references = references[:1000]\n",
    "        \n",
    "    references1 = list()\n",
    "    sim_scores = list()\n",
    "    GPT2_scores_preds = list()\n",
    "    for idx, reference in enumerate(references):\n",
    "        references1.append([reference])\n",
    "        sim_scores.append(similarity(predictions[idx], reference))\n",
    "        if(len(predictions[idx])!=0):\n",
    "            GPT2_scores_preds.append(GPT2_score(sentence = predictions[idx]))\n",
    "    \n",
    "    res_bleu_1 = bleu.compute(predictions=predictions, references=references1, max_order=1)\n",
    "    res_bleu_2 = bleu.compute(predictions=predictions, references=references1, max_order=2)\n",
    "    res_bleu_3 = bleu.compute(predictions=predictions, references=references1, max_order=3)\n",
    "    res_bleu_4 = bleu.compute(predictions=predictions, references=references1, max_order=4)\n",
    "    # sum(GPT2_scores_preds)/len(GPT2_scores_preds)\n",
    "    return sum(sim_scores)/len(sim_scores), [res_bleu_1['bleu'],res_bleu_2['bleu'],res_bleu_3['bleu'],res_bleu_4['bleu']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd00c4e5-943b-43e6-81d6-9ebb624513f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "res_bleu, sim = evaluate('politeness_transfer/direct/trg_direct_lists.pkl', 'politeness_transfer/direct/src_direct_lists.pkl')\n",
    "print(res_bleu)\n",
    "print(sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83787512-4936-44c1-9d0e-de38eb0df8ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "res_bleu, sim = evaluate('politeness_transfer/direct/pred_direct_lists.pkl', 'politeness_transfer/direct/src_direct_lists.pkl')\n",
    "print(res_bleu)\n",
    "print(sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d89a5b04-59df-4f2b-a167-1910cf642dc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "res_bleu, sim = evaluate('politeness_transfer/direct/pred_direct_lists.pkl', 'politeness_transfer/direct/trg_direct_lists.pkl')\n",
    "print(res_bleu)\n",
    "print(sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08993e78-12c9-4b02-84a6-30a6241cf9b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, bleus = evaluate('politeness_transfer/synthetic_test_polite_tagandgen.pkl', 'politeness_transfer/direct/src_direct_lists.pkl')\n",
    "print(sim)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "348026de-ae1e-412b-a738-e2ea5e9bee86",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, bleus = evaluate('politeness_transfer/synthetic_test_polite_tagandgen.pkl', 'politeness_transfer/direct/trg_direct_lists.pkl')\n",
    "print(sim)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "607e005e-d465-4151-9316-3a3e6ae64d55",
   "metadata": {},
   "outputs": [],
   "source": [
    "res_bleu, sim = evaluate('politeness_transfer/gen/pred_gen_lists.pkl', 'politeness_transfer/direct/src_direct_lists.pkl')\n",
    "print(res_bleu)\n",
    "print(sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e4d3e71-a1cf-48b5-bd9b-da7e41e1487d",
   "metadata": {},
   "outputs": [],
   "source": [
    "res_bleu, sim = evaluate('politeness_transfer/gen/pred_gen_lists.pkl', 'politeness_transfer/direct/trg_direct_lists.pkl')\n",
    "print(res_bleu)\n",
    "print(sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9bea3e3f-cc11-44c4-92e2-638aa684d8e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "res_bleu, sim = evaluate('politeness_transfer/direct/P_0_test_polite_direct.pkl', 'polite_multicls_classifier/P_0_test.pkl')\n",
    "print(res_bleu)\n",
    "print(sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c30c45bb-9437-4f84-b85c-73de7ce26516",
   "metadata": {},
   "outputs": [],
   "source": [
    "res_bleu, sim = evaluate('politeness_transfer/gen/P_0_test_polite_gen.pkl', 'polite_multicls_classifier/P_0_test.pkl')\n",
    "print(res_bleu)\n",
    "print(sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfa15ca9-1e81-4f7b-997c-069eb18c725a",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, bleus = evaluate('dialog/daily_dialog/finetune-polite/polite_dlg_responses_direct_pred_lists.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2b2d958-f472-4efa-aff1-8967142a68d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, res_bleu = evaluate('dialog/daily_dialog/finetune/dlg_responses_direct_pred_lists.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75ae107f-a594-4e41-8ef8-5cf21f64594a",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, res_bleu = evaluate('dialog/daily_dialog/without-finetune/dlg_responses_wo_pred_lists.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32184a52-7596-4ee7-8efe-8b8fe3563d16",
   "metadata": {},
   "outputs": [],
   "source": [
    "res_bleu, sim = evaluate('dialog/daily_dialog/finetune-polite/polite_dlg_responses_direct_pred_lists.pkl', 'dialog/daily_dialog/responses/test_polite_res_direct.pkl')\n",
    "print(res_bleu)\n",
    "print(sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "805327ae-864b-48ff-b04a-70ed483f445b",
   "metadata": {},
   "outputs": [],
   "source": [
    "res_bleu, sim = evaluate('dialog/daily_dialog/finetune/dlg_responses_direct_pred_lists.pkl', 'dialog/daily_dialog/responses/test_polite_res_direct.pkl')\n",
    "print(res_bleu)\n",
    "print(sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38e528bb-dc0e-49c8-9d44-7742951bdbc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, bleus = evaluate('dialog/daily_dialog/responses/test_polite_res_direct.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb98b557-c8b5-4e40-8a87-a077c13fffff",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, bleus = evaluate('dialog/daily_dialog/responses/tag-gen/test_polite_res.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3355295-81cd-4bae-9494-23c34e351c6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, bleus = evaluate('dialog/daily_dialog/finetune/DialoGPT_finetune_lastmodel.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af77eea8-28ac-40fe-9a0e-71f5fb035f03",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, bleus = evaluate('dialog/daily_dialog/finetune/DialoGPT_finetune_lastmodel_polite.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e4430f4-5b82-4b99-8f84-76c66c9aa678",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, bleus = evaluate('dialog/daily_dialog/finetune/GPT2_finetune_last.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa4fb399-abf4-4653-9969-89ba80fb3338",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, PPL_Score, bleus = evaluate('dialog/daily_dialog/finetune/GPT2_finetune_last_polite.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "print(PPL_Score)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "569c524d-3250-4180-b6a9-3f6cd3b566ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, PPL_Score, bleus = evaluate('dialog/daily_dialog/finetune/GPT2_finetune_last_polite.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "#print(PPL_Score)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "128dae29-61fa-4fd9-ae62-975478301f15",
   "metadata": {},
   "outputs": [],
   "source": [
    "# resps = list()\n",
    "# with open('dialog/daily_dialog/responses/test_res.pkl', 'rb') as f:\n",
    "#         resps = pickle.load(f)\n",
    "# print(len(resps))\n",
    "\n",
    "# with open('dialog/daily_dialog/responses/test_res.txt', 'w') as f:\n",
    "#     for line in resps:\n",
    "#         f.write(f\"{line}\\n\")\n",
    "\n",
    "# with open(r\"dialog/daily_dialog/responses/test_res.txt\", 'r') as fp:\n",
    "#     lines = len(fp.readlines())\n",
    "#     print(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70befb46-d3a3-4cf6-ba83-53921f00bd45",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, bleus = evaluate('dialog/daily_dialog/without-finetune/dlg_responses_wo_pred_lists_GPT2.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "#print(PPL_Score)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff71eb1b-ec5f-4068-8b6d-4a0e17039025",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, bleus = evaluate('dialog/daily_dialog/without-finetune/dlg_responses_wo_pred_lists_dialoGPT.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "#print(PPL_Score)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "245a12ef-de13-474b-8dc6-defc34cd6837",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, bleus = evaluate('dialog/daily_dialog/finetune/GPT2_finetune_best.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "#print(PPL_Score)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06ebc933-2db1-4019-b74c-6b28ae9d4daf",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, bleus = evaluate('dialog/daily_dialog/finetune/GPT2_finetune_polite_best.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "#print(PPL_Score)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b268dce-49a2-4896-95b0-16daeab9eb3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, bleus = evaluate('dialog/daily_dialog/finetune/DialoGPT_finetune_best.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "#print(PPL_Score)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cc8e226-e1c1-4eb4-a9a6-82a01646c315",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, bleus = evaluate('dialog/daily_dialog/finetune/DialoGPT_finetune_polite_best.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "#print(PPL_Score)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53da37d9-39f3-4276-9c90-bed5aa3a4e40",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, bleus = evaluate('dialog/daily_dialog/finetune/GPT2_finetune_tag-gen_polite.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "#print(PPL_Score)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11fba764-a588-413f-b354-e65cd5c1e717",
   "metadata": {},
   "outputs": [],
   "source": [
    "sim, bleus = evaluate('dialog/daily_dialog/finetune-polite/Blenderbot_polite_pred_tag-gen.pkl', 'dialog/daily_dialog/responses/test_res.pkl')\n",
    "print(sim)\n",
    "#print(PPL_Score)\n",
    "print(bleus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9c132d3-13e6-4abc-8a12-f4d3d47b177b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}