ACL2022.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XAT4dX4nJkHw"
   },
   "outputs": [],
   "source": [
    "# Pipeline: Pun -> 2 sense def -> Reverse dictionary -> all keywords -> generation\n",
    "\n",
    "## set path here\n",
    "path=\"path/to/folder\"\n",
    "openaikey=\"xxxxxx\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 973
    },
    "id": "8EVZ5MTNf2YF",
    "outputId": "3e99c918-1406-43e2-9dd5-5b781cab6f31"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fatal: destination path 'WantWords' already exists and is not an empty directory.\n",
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: torch in /home/development/nihars/.local/lib/python3.7/site-packages (1.9.0)\n",
      "Requirement already satisfied: typing-extensions in /home/development/nihars/.local/lib/python3.7/site-packages (from torch) (3.10.0.0)\n",
      "\u001b[33mWARNING: You are using pip version 21.2.3; however, version 21.3.1 is available.\n",
      "You should consider upgrading via the '/opt/anaconda/envs/env_pytorch/bin/python -m pip install --upgrade pip' command.\u001b[0m\n",
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: pytorch_transformers in /home/development/nihars/.local/lib/python3.7/site-packages (1.2.0)\n",
      "Requirement already satisfied: tqdm in /home/development/nihars/.local/lib/python3.7/site-packages (from pytorch_transformers) (4.62.3)\n",
      "Requirement already satisfied: regex in /home/development/nihars/.local/lib/python3.7/site-packages (from pytorch_transformers) (2021.10.21)\n",
      "Requirement already satisfied: torch>=1.0.0 in /home/development/nihars/.local/lib/python3.7/site-packages (from pytorch_transformers) (1.9.0)\n",
      "Requirement already satisfied: sentencepiece in /home/development/nihars/.local/lib/python3.7/site-packages (from pytorch_transformers) (0.1.96)\n",
      "Requirement already satisfied: boto3 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from pytorch_transformers) (1.9.233)\n",
      "Requirement already satisfied: sacremoses in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from pytorch_transformers) (0.0.35)\n",
      "Requirement already satisfied: requests in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from pytorch_transformers) (2.22.0)\n",
      "Requirement already satisfied: numpy in /home/development/nihars/.local/lib/python3.7/site-packages (from pytorch_transformers) (1.21.1)\n",
      "Requirement already satisfied: typing-extensions in /home/development/nihars/.local/lib/python3.7/site-packages (from torch>=1.0.0->pytorch_transformers) (3.10.0.0)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from boto3->pytorch_transformers) (0.9.4)\n",
      "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from boto3->pytorch_transformers) (0.2.1)\n",
      "Requirement already satisfied: botocore<1.13.0,>=1.12.233 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from boto3->pytorch_transformers) (1.12.233)\n",
      "Requirement already satisfied: urllib3<1.26,>=1.20 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from botocore<1.13.0,>=1.12.233->boto3->pytorch_transformers) (1.25.5)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from botocore<1.13.0,>=1.12.233->boto3->pytorch_transformers) (2.8.0)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from botocore<1.13.0,>=1.12.233->boto3->pytorch_transformers) (0.15.2)\n",
      "Requirement already satisfied: six>=1.5 in /home/development/nihars/.local/lib/python3.7/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.13.0,>=1.12.233->boto3->pytorch_transformers) (1.16.0)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from requests->pytorch_transformers) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from requests->pytorch_transformers) (2019.9.11)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from requests->pytorch_transformers) (2.8)\n",
      "Requirement already satisfied: click in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from sacremoses->pytorch_transformers) (7.0)\n",
      "Requirement already satisfied: joblib in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from sacremoses->pytorch_transformers) (0.13.2)\n",
      "\u001b[33mWARNING: You are using pip version 21.2.3; however, version 21.3.1 is available.\n",
      "You should consider upgrading via the '/opt/anaconda/envs/env_pytorch/bin/python -m pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/thunlp/WantWords.git\n",
    "!pip install torch\n",
    "!pip install pytorch_transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "pLCV46jna9vC"
   },
   "outputs": [],
   "source": [
    "## Sense keys for a word from the dataset\n",
    "import pandas as pd \n",
    "def get_senses(pun):\n",
    "  df = pd.read_csv(path+\"Dataset1.11.csv\")\n",
    "  for index, row in df.iterrows():\n",
    "    if row['Pun']==pun:\n",
    "      senseKeys=[row['Sense 1'],row['sense 2']]\n",
    "  # senseKeys=[row['Sense 1'],row['sense 2']]\n",
    "  return senseKeys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Z2j1aONIJwfh",
    "outputId": "ebccda5b-8de3-4671-95f8-37d1119f8a08"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     /home/development/nihars/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /home/development/nihars/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /home/development/nihars/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "### Sense definitions from the pun word\n",
    "import nltk\n",
    "nltk.download('wordnet')\n",
    "from nltk.corpus import wordnet as wn\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords\n",
    "stop_words = set(stopwords.words('english'))\n",
    "from nltk.tokenize import word_tokenize\n",
    "nltk.download('punkt')\n",
    "\n",
    "def sense_def(pun):\n",
    "  senseKeys= get_senses(pun)\n",
    "  sense_definitions=[]\n",
    "  for senseKey in senseKeys:\n",
    "    m=wn.lemma_from_key(senseKey).synset().definition()\n",
    "    sense_definitions.append(m)\n",
    "  return sense_definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "v-PK5fkDfpMt",
    "outputId": "7d63f40b-7160-4a3c-dec5-aa50cded1737"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/development/nihars/anirudh/Anirudh-UCLA/Code/WantWords\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/development/nihars/.local/lib/python3.7/site-packages/torch/serialization.py:671: SourceChangeWarning: source code of class 'model_en.Encoder' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n",
      "  warnings.warn(msg, SourceChangeWarning)\n",
      "/home/development/nihars/.local/lib/python3.7/site-packages/torch/serialization.py:671: SourceChangeWarning: source code of class 'torch.nn.modules.sparse.Embedding' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n",
      "  warnings.warn(msg, SourceChangeWarning)\n",
      "/home/development/nihars/.local/lib/python3.7/site-packages/torch/serialization.py:671: SourceChangeWarning: source code of class 'torch.nn.modules.dropout.Dropout' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n",
      "  warnings.warn(msg, SourceChangeWarning)\n",
      "/home/development/nihars/.local/lib/python3.7/site-packages/torch/serialization.py:671: SourceChangeWarning: source code of class 'torch.nn.modules.normalization.LayerNorm' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n",
      "  warnings.warn(msg, SourceChangeWarning)\n",
      "/home/development/nihars/.local/lib/python3.7/site-packages/torch/serialization.py:671: SourceChangeWarning: source code of class 'torch.nn.modules.container.ModuleList' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n",
      "  warnings.warn(msg, SourceChangeWarning)\n",
      "/home/development/nihars/.local/lib/python3.7/site-packages/torch/serialization.py:671: SourceChangeWarning: source code of class 'torch.nn.modules.linear.Linear' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n",
      "  warnings.warn(msg, SourceChangeWarning)\n",
      "/home/development/nihars/.local/lib/python3.7/site-packages/torch/serialization.py:671: SourceChangeWarning: source code of class 'torch.nn.modules.activation.Tanh' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n",
      "  warnings.warn(msg, SourceChangeWarning)\n",
      "/home/development/nihars/.local/lib/python3.7/site-packages/torch/serialization.py:671: SourceChangeWarning: source code of class 'torch.nn.modules.loss.CrossEntropyLoss' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n",
      "  warnings.warn(msg, SourceChangeWarning)\n",
      "/home/development/nihars/.local/lib/python3.7/site-packages/torch/serialization.py:671: SourceChangeWarning: source code of class 'torch.nn.modules.activation.ReLU' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n",
      "  warnings.warn(msg, SourceChangeWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------2\n"
     ]
    }
   ],
   "source": [
    "#### Reverse Dicitonary\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch, gc, json, os, string, re, requests, hashlib, urllib.parse \n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "from pytorch_transformers import *\n",
    "\n",
    "path_loc = path + \"Code/WantWords/\"\n",
    "%cd $path_loc\n",
    "ani= torch.load(\"../website_RD_data/1rsl_saved.model\", map_location=lambda storage, loc: storage)\n",
    "tokenizer_class = BertTokenizer\n",
    "tokenizer_Ch = tokenizer_class.from_pretrained('bert-base-chinese')\n",
    "tokenizer_En = tokenizer_class.from_pretrained('bert-base-uncased')\n",
    "(_, (_, label_size, _, _), (word2index_en, index2word_en, index2sememe, index2lexname, index2rootaffix)) = np.load(\"../website_RD_data/\"+ 'data_inUse1_en.npy', allow_pickle=True)\n",
    "device = torch.device('cpu')\n",
    "words_t = torch.tensor(np.array([0]))\n",
    "def label_multihot(labels, num):\n",
    "    sm = np.zeros((len(labels), num), dtype=np.float32)\n",
    "    for i in range(len(labels)):\n",
    "        for s in labels[i]:\n",
    "            if s >= num:\n",
    "                break\n",
    "            sm[i, s] = 1\n",
    "    return sm\n",
    "\n",
    "\n",
    "\n",
    "def word2feature(dataset, word_num, feature_num, feature_name):\n",
    "    max_feature_num = max([len(instance[feature_name]) for instance in dataset])\n",
    "    ret = np.zeros((word_num, max_feature_num), dtype=np.int64)\n",
    "    ret.fill(feature_num)\n",
    "    for instance in dataset:\n",
    "        if ret[instance['word'], 0] != feature_num: \n",
    "            continue # this target_words has been given a feature mapping, because same word with different definition in dataset\n",
    "        feature = instance[feature_name]\n",
    "        ret[instance['word'], :len(feature)] = np.array(feature)\n",
    "    return torch.tensor(ret, dtype=torch.int64, device=device)\n",
    "    \n",
    "def mask_noFeature(label_size, wd2fea, feature_num):\n",
    "    mask_nofea = torch.zeros(label_size, dtype=torch.float32, device=device)\n",
    "    for i in range(label_size):\n",
    "        feas = set(wd2fea[i].detach().cpu().numpy().tolist())-set([feature_num])\n",
    "        if len(feas)==0:\n",
    "            mask_nofea[i] = 1\n",
    "    return mask_nofea\n",
    "    \n",
    "(data_train_idx, data_dev_idx, data_test_500_seen_idx, data_test_500_unseen_idx, data_defi_c_idx, data_desc_c_idx) = np.load(\"../website_RD_data/\" + 'data_inUse2_en.npy', allow_pickle=True)\n",
    "data_all_idx = data_train_idx + data_dev_idx + data_test_500_seen_idx + data_test_500_unseen_idx + data_defi_c_idx\n",
    "index2word_en = np.array(index2word_en)\n",
    "print('-------------------------2')\n",
    "sememe_num = len(index2sememe)\n",
    "wd2sem = word2feature(data_all_idx, label_size, sememe_num, 'sememes')\n",
    "wd_sems_ = label_multihot(wd2sem, sememe_num)\n",
    "wd_sems_ = torch.from_numpy(np.array(wd_sems_)).to(device) \n",
    "lexname_num = len(index2lexname)\n",
    "wd2lex = word2feature(data_all_idx, label_size, lexname_num, 'lexnames') \n",
    "wd_lex = label_multihot(wd2lex, lexname_num)\n",
    "wd_lex = torch.from_numpy(np.array(wd_lex)).to(device)\n",
    "rootaffix_num = len(index2rootaffix)\n",
    "wd2ra = word2feature(data_all_idx, label_size, rootaffix_num, 'root_affix') \n",
    "wd_ra = label_multihot(wd2ra, rootaffix_num)\n",
    "wd_ra = torch.from_numpy(np.array(wd_ra)).to(device)\n",
    "mask_s_ = mask_noFeature(label_size, wd2sem, sememe_num)\n",
    "mask_l = mask_noFeature(label_size, wd2lex, lexname_num)\n",
    "mask_r = mask_noFeature(label_size, wd2ra, rootaffix_num)\n",
    "\n",
    "\n",
    "MODE_en = 'rsl'\n",
    "itemsPerCol = 20\n",
    "GET_NUM = 100\n",
    "NUM_RESPONSE = 500\n",
    "def Score2Hexstr(score, maxsc):\n",
    "    thr = maxsc/1.5\n",
    "    l = len(score)\n",
    "    ret = ['00']*l\n",
    "    for i in range(l):\n",
    "        res = int(200*(score[i] - thr)/thr)\n",
    "        if res>15:\n",
    "            ret[i] = hex(res)[2:]\n",
    "        else:\n",
    "            break\n",
    "    return ret\n",
    "\n",
    "def keywords(sentences):\n",
    "  output=[]\n",
    "  for description in sentences:\n",
    "  \n",
    "    RD_mode='EE'\n",
    "    with torch.no_grad():\n",
    "      def_words = re.sub('[%s]' % re.escape(string.punctuation), ' ', description)\n",
    "      def_words =def_words.lower()\n",
    "      def_words = def_words.strip().split()\n",
    "      def_word_idx = []\n",
    "      # print(\"def words:\", def_words)\n",
    "      if len(def_words) > 0:\n",
    "        for def_word in def_words:\n",
    "            if def_word in word2index_en:\n",
    "              def_word_idx.append(word2index_en[def_word])\n",
    "            else:\n",
    "              # print(\"hi\")\n",
    "              def_word_idx.append(word2index_en['<OOV>'])\n",
    "        # print(\"def_word_idx: \", def_word_idx, print(word2index_en['<OOV>']),set(def_word_idx),{word2index_en['<OOV>']})\n",
    "        x_len = len(def_word_idx)\n",
    "        if set(def_word_idx)=={word2index_en['<OOV>']}:\n",
    "            x_len = 1\n",
    "        if x_len==1:\n",
    "            if def_word_idx[0]>1:\n",
    "                #词向量找相关词，排序后，如果在WordNet的synset里，则对应的同义词的分数乘以2？\n",
    "                score = ((model_en.embedding.weight.data).mm((model_en.embedding.weight.data[def_word_idx[0]]).unsqueeze(1))).squeeze(1)\n",
    "                if RD_mode=='EE': #当EE的时候，排除自身，CE的时候自身是最准确的，不排除。\n",
    "                    score[def_word_idx[0]] = -10.\n",
    "                score[np.array(index2synset_en[def_word_idx[0]])] *= 2\n",
    "                sc, indices = torch.sort(score, descending=True)\n",
    "                predicted = indices[:NUM_RESPONSE].detach().cpu().numpy()\n",
    "                \n",
    "                score = sc[:NUM_RESPONSE].detach().numpy()\n",
    "                maxsc = sc[0].detach().item()\n",
    "                s2h = Score2Hexstr(score, maxsc)\n",
    "            else:\n",
    "                predicted= []\n",
    "                ret = {'error': 1} # 字符无法识别\n",
    "        else:\n",
    "            defi = '[CLS] ' + description\n",
    "            def_word_idx = tokenizer_En.encode(defi)[:60]\n",
    "            def_word_idx.extend(tokenizer_En.encode('[SEP]'))\n",
    "            definition_words_t = torch.tensor(np.array(def_word_idx), dtype=torch.int64, device=device)\n",
    "            definition_words_t = definition_words_t.unsqueeze(0) # batch_size = 1\n",
    "            score = ani('test', x=definition_words_t, w=words_t, ws=wd_sems_, wl=wd_lex, wr=wd_ra, msk_s=mask_s_, msk_l=mask_l, msk_r=mask_r, mode=MODE_en)\n",
    "            sc, indices = torch.sort(score, descending=True)\n",
    "            predicted = indices[0, :NUM_RESPONSE].detach().cpu().numpy()\n",
    "            score = sc[0, :NUM_RESPONSE].detach().numpy()\n",
    "            maxsc = sc[0, 0].detach().item()\n",
    "            s2h = Score2Hexstr(score, maxsc)\n",
    "              \n",
    "      else:\n",
    "        print(\"hiiii\")\n",
    "        predicted= []\n",
    "        ret = {'error': 0} # 输入为空\n",
    "    # print(\"predicted: \", predicted)\n",
    "    if len(predicted)>0:\n",
    "        res = index2word_en[predicted]\n",
    "        # print(\"res: \",res)\n",
    "        ret = [] # 不能以字典形式返回，因为字典是无序的，这里需要用列表来保持顺序。\n",
    "        cn = -1\n",
    "        if RD_mode == \"EE\":\n",
    "            def_words = set(def_words)\n",
    "            for wd in res:\n",
    "                cn += 1\n",
    "                if len(wd)>1 and (wd not in def_words):\n",
    "                    # ret.append(wd_data_en[wd]) # wd_data_en[wd] = {'word': word, 'definition':defis, 'POS':['n']}]\n",
    "                    # ret[len(ret)-1]['c'] = s2h[cn]\n",
    "                    try:\n",
    "                        ret.append(wd_data_en[wd]) # wd_data_en[wd] = {'word': word, 'definition':defis, 'POS':['n']}]\n",
    "                        ret[len(ret)-1]['c'] = s2h[cn]\n",
    "                    except:\n",
    "                        continue\n",
    "        else:\n",
    "            for wd in res:\n",
    "                cn += 1\n",
    "                if len(wd)>1:\n",
    "                    try:\n",
    "                        ret.append(wd_data_en[wd]) # wd_data_en[wd] = {'word': word, 'definition':defis, 'POS':['n']}]\n",
    "                        ret[len(ret)-1]['c'] = s2h[cn]\n",
    "                    except:\n",
    "                        continue\n",
    "    output.append(res[0:8])\n",
    "  return output\n",
    "\n",
    "    # json.dumps(ret,ensure_ascii=False),content_type=\"application/json,charset=utf-8\")\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "H38ZLAMhJwlJ",
    "outputId": "cbed0d6a-873a-49fa-ba89-599e9bb35d4d"
   },
   "outputs": [],
   "source": [
    "#### Getting related keywords for TFIDF\n",
    "\n",
    "!pip install rake-nltk\n",
    "from rake_nltk import Rake\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "import os\n",
    "oneb=[]\n",
    "# paths=[path+\"Data/1-billion-word-language-modeling-benchmark-r13output/training-monolingual.tokenized.shuffled/\",path+\"Data/1-billion-word-language-modeling-benchmark-r13output/heldout-monolingual.tokenized.shuffled/\"]\n",
    "# for path in paths:\n",
    "#   print(len(os.listdir(path)))\n",
    "#   for filename in os.listdir(path):\n",
    "#     file=open(path+filename,\"r\")\n",
    "#     try:\n",
    "#       oneb.append(file.read())\n",
    "#     except:\n",
    "#       print(filename)\n",
    "import pickle\n",
    "with open(path+'oneb.p', 'rb') as fp:\n",
    "    data = pickle.load(fp)\n",
    "oneb=data\n",
    "\n",
    "\n",
    "\n",
    "def generate_sentences(word):\n",
    "  sentences=[]\n",
    "  # word='contact'\n",
    "  for article in oneb:\n",
    "    for sentence in article.split('.'):\n",
    "      if word in sentence:\n",
    "        sentences.append(sentence)\n",
    "\n",
    "  # print(word, len(sentences))\n",
    "  # print(sentences)\n",
    "  return sentences\n",
    "\n",
    "\n",
    "# master={}\n",
    "\n",
    "# for article in oneb:\n",
    "#   for sentence in article.split('.'):\n",
    "#     for word in sentence:\n",
    "#       if word in master:\n",
    "#         master[word]+=1\n",
    "#       else:\n",
    "#         master[word]=1\n",
    "\n",
    "# try:\n",
    "#     import cPickle as pickle\n",
    "# except ImportError:  # Python 3.x\n",
    "#     import pickle\n",
    "# # print(path,len(oneb))\n",
    "# with open(path+'master.p', 'wb') as fp:\n",
    "#     pickle.dump(master, fp, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "def find_number(wordx):\n",
    "  if wordx in master:\n",
    "    return master[wordx]\n",
    "  else:\n",
    "    return 0\n",
    "with open(path+'master.p', 'rb') as fp:\n",
    "    data1 = pickle.load(fp)\n",
    "master=data1\n",
    "\n",
    "from scipy import spatial\n",
    "!pip install -U sentence-transformers\n",
    "from sentence_transformers import SentenceTransformer\n",
    "model_sentence = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')\n",
    "\n",
    "def refine_sentences(sentences, senseDef):\n",
    "  sentences_refined=[]\n",
    "  for sentence in sentences:\n",
    "    sentence=sentence.strip()\n",
    "    embeddings = model_sentence.encode([sentence, senseDef])\n",
    "    result = 1 - spatial.distance.cosine(embeddings[0], embeddings[1])\n",
    "    # print(\"result: \", result)\n",
    "    if result> 0.10:\n",
    "      sentences_refined.append(sentence)\n",
    "  # print(\"sentence_refined: \", sentences_refined[0:5])\n",
    "  return sentences_refined\n",
    "\n",
    "\n",
    "\n",
    "def final_keywords(interim_keywords, senseDef):\n",
    "  final_keywords=[]\n",
    "  for words in interim_keywords:\n",
    "    final_keyword=[]\n",
    "    # print(\"words:\", words)\n",
    "    for word in words:\n",
    "      sentences = generate_sentences(word)\n",
    "      # print(\"index: \",interim_keywords.index(words) )\n",
    "      # sentences_refined= refine_sentences(sentences, senseDef[interim_keywords.index(words)])\n",
    "      r = Rake(max_length=1)\n",
    "      r.extract_keywords_from_sentences(sentences)\n",
    "      # print(sentences[0:5])\n",
    "      keywordsa= r.get_ranked_phrases()\n",
    "      freq = {}\n",
    "      for item in keywordsa:\n",
    "        if (item in freq):\n",
    "            freq[item] += 1\n",
    "        else:\n",
    "            freq[item] = 1\n",
    "      freq= sorted(freq.items(), key=lambda x: x[1], reverse=True)\n",
    "      freq1={}\n",
    "      for freqz in freq[:100]:\n",
    "        num = find_number(freqz[0]) # freqz[0] -> word , freqz[1] -> number\n",
    "        freq1[freqz[0]]=freqz[1]/(num+1)\n",
    "      freq1= sorted(freq1.items(), key=lambda x: x[1], reverse=True)\n",
    "      checker=0\n",
    "      for freqz in freq1[:10]:\n",
    "        if freqz[0] != word and freqz[0] not in stop_words:\n",
    "          final_keyword.append(freqz[0])\n",
    "          checker=checker+1\n",
    "        if checker>=5:\n",
    "          break\n",
    "\n",
    "    final_keywords.append(final_keyword)\n",
    "  return final_keywords\n",
    "\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### Getting related keywords for Word2Vec\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "stop_words = set(stopwords.words('english'))\n",
    "\n",
    "with open(path+\"LM_model_distance.txt\") as file:\n",
    "  lines= file.readlines()\n",
    "\n",
    "def final_keywords(interim_keywords):\n",
    "  final_keywords=[]\n",
    "  for words in interim_keywords:\n",
    "    final_keyword=[]\n",
    "    for word in words:\n",
    "      for linex in lines:\n",
    "        line = linex.split(\" \")\n",
    "        if line[0]==word:\n",
    "          \n",
    "          for i in range(2,120,2):\n",
    "            if i not in stop_words and len(final_keyword)<=30 and i != word:\n",
    "              final_keyword.append(line[i])\n",
    "    final_keywords.append(final_keyword)\n",
    "  return final_keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python3 -m pip install openai\n",
    "import os\n",
    "import openai\n",
    "openai.api_key = os.getenv(openaikey)\n",
    "!export OPENAI_API_KEY= openaikey\n",
    "openai.api_key = openaikey\n",
    "engine_name = \"davinci-instruct-beta\"\n",
    "def final_keywords(keywords, pun):\n",
    "    flist=[]\n",
    "\n",
    "    for klist in keywords:\n",
    "        fflist=[]\n",
    "        for word in klist:\n",
    "            prompt = '''Generate seven keywords for computer and internet: mouse, hardware, fast, smart, email, technology, software.\n",
    "            Generate seven keywords for '''+ word+\" and \"+pun +':'\n",
    "            response = openai.Completion.create(\n",
    "            engine= engine_name,\n",
    "            prompt=prompt,\n",
    "            temperature=0.2,\n",
    "            max_tokens=15,\n",
    "            top_p=0.95,\n",
    "            frequency_penalty=0.5,\n",
    "            presence_penalty=0.5\n",
    "            )\n",
    "            # print(\"-----\"+keyword)\n",
    "            # print(response['choices'][0]['text'])\n",
    "            nlp = response['choices'][0]['text']\n",
    "            nlp=nlp.split(\".\")[0]\n",
    "            nlp=nlp.split(\",\")\n",
    "            # cfflist.append(nlp)\n",
    "            nlp = [a.strip(\" \") for a in nlp]\n",
    "            if word in nlp:\n",
    "                nlp.remove(word)\n",
    "            # print(nlp)\n",
    "            fflist=fflist+nlp\n",
    "        # print(\"fflist: \", fflist)\n",
    "        flist.append(fflist)\n",
    "    # print(flist)\n",
    "    return flist\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "mBrNatn4Jwnl"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: simplet5 in /home/development/nihars/.local/lib/python3.7/site-packages (0.1.3)\n",
      "Requirement already satisfied: tqdm in /home/development/nihars/.local/lib/python3.7/site-packages (from simplet5) (4.62.3)\n",
      "Requirement already satisfied: transformers==4.10.0 in /home/development/nihars/.local/lib/python3.7/site-packages (from simplet5) (4.10.0)\n",
      "Requirement already satisfied: pytorch-lightning==1.4.5 in /home/development/nihars/.local/lib/python3.7/site-packages (from simplet5) (1.4.5)\n",
      "Requirement already satisfied: sentencepiece in /home/development/nihars/.local/lib/python3.7/site-packages (from simplet5) (0.1.96)\n",
      "Requirement already satisfied: torch!=1.8.0,>=1.7.0 in /home/development/nihars/.local/lib/python3.7/site-packages (from simplet5) (1.9.0)\n",
      "Requirement already satisfied: numpy>=1.17.2 in /home/development/nihars/.local/lib/python3.7/site-packages (from pytorch-lightning==1.4.5->simplet5) (1.21.1)\n",
      "Requirement already satisfied: fsspec[http]!=2021.06.0,>=2021.05.0 in /home/development/nihars/.local/lib/python3.7/site-packages (from pytorch-lightning==1.4.5->simplet5) (2021.10.1)\n",
      "Requirement already satisfied: PyYAML>=5.1 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from pytorch-lightning==1.4.5->simplet5) (5.1.2)\n",
      "Requirement already satisfied: tensorboard>=2.2.0 in /home/development/nihars/.local/lib/python3.7/site-packages (from pytorch-lightning==1.4.5->simplet5) (2.6.0)\n",
      "Requirement already satisfied: torchmetrics>=0.4.0 in /home/development/nihars/.local/lib/python3.7/site-packages (from pytorch-lightning==1.4.5->simplet5) (0.5.1)\n",
      "Requirement already satisfied: typing-extensions in /home/development/nihars/.local/lib/python3.7/site-packages (from pytorch-lightning==1.4.5->simplet5) (3.10.0.0)\n",
      "Requirement already satisfied: packaging>=17.0 in /home/development/nihars/.local/lib/python3.7/site-packages (from pytorch-lightning==1.4.5->simplet5) (21.0)\n",
      "Requirement already satisfied: future>=0.17.1 in /home/development/nihars/.local/lib/python3.7/site-packages (from pytorch-lightning==1.4.5->simplet5) (0.18.2)\n",
      "Requirement already satisfied: pyDeprecate==0.3.1 in /home/development/nihars/.local/lib/python3.7/site-packages (from pytorch-lightning==1.4.5->simplet5) (0.3.1)\n",
      "Requirement already satisfied: importlib-metadata in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from transformers==4.10.0->simplet5) (0.23)\n",
      "Requirement already satisfied: filelock in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from transformers==4.10.0->simplet5) (3.0.12)\n",
      "Requirement already satisfied: requests in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from transformers==4.10.0->simplet5) (2.22.0)\n",
      "Requirement already satisfied: sacremoses in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from transformers==4.10.0->simplet5) (0.0.35)\n",
      "Requirement already satisfied: huggingface-hub>=0.0.12 in /home/development/nihars/.local/lib/python3.7/site-packages (from transformers==4.10.0->simplet5) (0.0.17)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/development/nihars/.local/lib/python3.7/site-packages (from transformers==4.10.0->simplet5) (2021.10.21)\n",
      "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /home/development/nihars/.local/lib/python3.7/site-packages (from transformers==4.10.0->simplet5) (0.10.3)\n",
      "Requirement already satisfied: aiohttp in /home/development/nihars/.local/lib/python3.7/site-packages (from fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning==1.4.5->simplet5) (3.7.4.post0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from packaging>=17.0->pytorch-lightning==1.4.5->simplet5) (2.4.2)\n",
      "Requirement already satisfied: wheel>=0.26 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (0.33.4)\n",
      "Requirement already satisfied: google-auth<2,>=1.6.3 in /home/development/nihars/.local/lib/python3.7/site-packages (from tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (1.34.0)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /home/development/nihars/.local/lib/python3.7/site-packages (from tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (0.4.5)\n",
      "Requirement already satisfied: grpcio>=1.24.3 in /home/development/nihars/.local/lib/python3.7/site-packages (from tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (1.39.0)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /home/development/nihars/.local/lib/python3.7/site-packages (from tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (1.8.0)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (0.16.0)\n",
      "Requirement already satisfied: absl-py>=0.4 in /home/development/nihars/.local/lib/python3.7/site-packages (from tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (0.13.0)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (41.0.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /home/development/nihars/.local/lib/python3.7/site-packages (from tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (3.3.4)\n",
      "Requirement already satisfied: protobuf>=3.6.0 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (3.9.2)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /home/development/nihars/.local/lib/python3.7/site-packages (from tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (0.6.1)\n",
      "Requirement already satisfied: six in /home/development/nihars/.local/lib/python3.7/site-packages (from absl-py>=0.4->tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (1.16.0)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /home/development/nihars/.local/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (4.2.2)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/development/nihars/.local/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /home/development/nihars/.local/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (4.7.2)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /home/development/nihars/.local/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (1.3.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /home/development/nihars/.local/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (0.4.8)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from requests->transformers==4.10.0->simplet5) (1.25.5)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from requests->transformers==4.10.0->simplet5) (2019.9.11)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from requests->transformers==4.10.0->simplet5) (3.0.4)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from requests->transformers==4.10.0->simplet5) (2.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /home/development/nihars/.local/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->pytorch-lightning==1.4.5->simplet5) (3.1.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from aiohttp->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning==1.4.5->simplet5) (19.1.0)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/development/nihars/.local/lib/python3.7/site-packages (from aiohttp->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning==1.4.5->simplet5) (1.7.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/development/nihars/.local/lib/python3.7/site-packages (from aiohttp->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning==1.4.5->simplet5) (5.2.0)\n",
      "Requirement already satisfied: async-timeout<4.0,>=3.0 in /home/development/nihars/.local/lib/python3.7/site-packages (from aiohttp->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning==1.4.5->simplet5) (3.0.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from importlib-metadata->transformers==4.10.0->simplet5) (0.6.0)\n",
      "Requirement already satisfied: more-itertools in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from zipp>=0.5->importlib-metadata->transformers==4.10.0->simplet5) (7.2.0)\n",
      "Requirement already satisfied: click in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from sacremoses->transformers==4.10.0->simplet5) (7.0)\n",
      "Requirement already satisfied: joblib in /opt/anaconda/envs/env_pytorch/lib/python3.7/site-packages (from sacremoses->transformers==4.10.0->simplet5) (0.13.2)\n",
      "\u001b[33mWARNING: You are using pip version 21.2.3; however, version 21.3.1 is available.\n",
      "You should consider upgrading via the '/opt/anaconda/envs/env_pytorch/bin/python -m pip install --upgrade pip' command.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 42\n"
     ]
    }
   ],
   "source": [
    "###Generating candidate sentences\n",
    "\n",
    "!pip install simplet5\n",
    "from simplet5 import SimpleT5\n",
    "import random\n",
    "# from transformers import GPT2LMHeadModel,  GPT2Tokenizer, GPT2Config, GPT2LMHeadModel\n",
    "\n",
    "# device = torch.device('cuda')\n",
    "\n",
    "model_t5 = SimpleT5()\n",
    "# output_dir=path+\"CandidateGeneration/GPT2/wo_sampling/epoch3\"\n",
    "# model_gpt2 = GPT2LMHeadModel.from_pretrained(output_dir)\n",
    "# tokenizer_gpt2 = GPT2Tokenizer.from_pretrained(output_dir)\n",
    "# # def load_models():\n",
    "# device = torch.device(\"cuda:1\")\n",
    "# model_gpt2.to(device)\n",
    "model_t5.from_pretrained(model_type=\"t5\", model_name=\"t5-base\")\n",
    "model_t5.load_model(\"t5\",path+\"simplet5-epoch-5-train-loss-1.4723\", use_gpu=True)\n",
    "  \n",
    "\n",
    "def generate_candidates1(x, pun, flag=\"mix\"):\n",
    "  sense1=x[0]\n",
    "  sense2=x[1]\n",
    "  candidates=[]\n",
    "\n",
    "  if flag==\"one\" or flag==\"mix\":\n",
    "    for i in range(50):\n",
    "      a = random.randint(0,len(sense1)-1)\n",
    "      b = random.randint(0,len(sense2)-1)\n",
    "      if i<16:\n",
    "        text=\"generate sentence: \"+str(pun)+\", \"+str(sense1[a])+\", \"+str(sense2[b])\n",
    "      elif i<37:\n",
    "        text=\"generate sentence: \"+str(sense1[a])+\", \"+str(pun)+\", \"+str(sense2[b])\n",
    "      else:\n",
    "        text=\"generate sentence: \"+str(sense1[a])+\", \"+str(sense2[b])+\", \"+str(pun)\n",
    "\n",
    "      candidates.append([model_t5.predict(text)[0],text])\n",
    "  if flag==\"two\" or flag==\"mix\":\n",
    "    for i in range(50):\n",
    "      a = random.randint(0,len(sense1)-1)\n",
    "      b = random.randint(0,len(sense1)-1)\n",
    "      c = random.randint(0,len(sense2)-1)\n",
    "      d = random.randint(0,len(sense2)-1)\n",
    "      if i<0:\n",
    "        text=\"generate sentence: \"+str(pun)+\", \"+str(sense1[a])+\", \"+str(sense1[b])+\", \"+str(sense2[c])+\", \"+str(sense2[d])\n",
    "      elif i<8:\n",
    "        text=\"generate sentence: \"+str(sense1[a])+\", \"+str(sense1[b])+\", \"+str(pun)+\", \"+str(sense2[c])+\", \"+str(sense2[d])\n",
    "      else:\n",
    "        text=\"generate sentence: \"+str(sense1[a])+\", \"+str(sense1[b])+\", \"+str(sense2[c])+\", \"+str(sense2[d])+\", \"+str(pun)\n",
    "      candidates.append([model_t5.predict(text)[0],text])\n",
    "  if flag==\"two\" or flag==\"mix\":\n",
    "    for i in range(75):\n",
    "      a = random.randint(0,len(sense1)-1)\n",
    "      b = random.randint(0,len(sense1)-1)\n",
    "      c = random.randint(0,len(sense2)-1)\n",
    "      d = random.randint(0,len(sense2)-1)\n",
    "      if i<0:\n",
    "        text=\"generate sentence: \"+str(pun)+\", \"+str(sense1[a])+\", \"+str(sense1[b])+\", \"+str(sense2[c])\n",
    "      elif i<15:\n",
    "        text=\"generate sentence: \"+str(sense1[a])+\", \"+str(sense1[b])+\", \"+str(pun)+\", \"+str(sense2[c])\n",
    "      elif i<40:\n",
    "        text=\"generate sentence: \"+str(sense1[a])+\", \"+str(sense2[d])+\", \"+str(sense2[c])+\", \"+str(pun)\n",
    "      else:\n",
    "        text=\"generate sentence: \"+str(sense1[a])+\", \"+str(sense1[b])+\", \"+str(sense2[c])+\", \"+str(pun)\n",
    "      candidates.append([model_t5.predict(text)[0],text])\n",
    "\n",
    "  return candidates\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "22bTjN7qvk7g"
   },
   "outputs": [],
   "source": [
    "flag = \"two\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Data refinement process\n",
    "\n",
    "import pickle\n",
    "import re\n",
    "df_aloha = pd.read_csv(path+\"Dataset1.11.csv\")\n",
    "aloha=df_aloha['Pun'].to_list()\n",
    "def Refine(keywords_lists, word , num=100):\n",
    "    aa=[]\n",
    "    # print(\"num:\",num)\n",
    "    for keyword_list in keywords_lists:\n",
    "        a=[]\n",
    "        for item in keyword_list:\n",
    "            item=item.strip()\n",
    "            item=item.strip(\" \")\n",
    "            item= re.sub(r'\\W+', '', item)\n",
    "            item = ''.join([i for i in item if not i.isdigit()])\n",
    "            if item != word and item not in stop_words and item not in a and item not in aloha and len(item)>0:\n",
    "                a.append(item)\n",
    "        if len(a)>num:\n",
    "            aa.append(a[:num])\n",
    "        else:\n",
    "            aa.append(a)\n",
    "    return aa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "words=[\"addresses\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pun word:  adresses\n",
      "Error:  adresses local variable 'senseKeys' referenced before assignment\n",
      "Pun word:  bugs\n",
      "sense definitions:  ['general term for any insect or similar creeping or crawling invertebrate', 'a fault or defect in a computer program, system, or machine']\n",
      "ReverseDicKeywords_refine:  [['spider', 'cockroach', 'termite', 'snail', 'earthworm'], ['flaw', 'checksum', 'error', 'misstep', 'problem']]\n",
      "finalKeywords_refine [['spiders', 'said', 'one', 'cockroaches', 'rats', 'mice', 'termites', 'insects', 'company', 'found', 'ants', 'snails', 'pace', 'slugs', 'shell', 'earthworms', 'soil', 'species'], ['flawed', 'flaws', 'said', 'one', 'flawless', 'method', 'woke', 'verify', 'user', 'tool', 'terrorism', 'terrorists', 'terror', 'war', 'missteps', 'series', 'people', 'solve']]\n",
      "Candidate sentences 1:  [[\"A user can verify if he has spiders cockroaches bugs or a person's facebook status.\", 'generate sentence: cockroaches, spiders, bugs, user, verify'], ['What do insects and rats have in common? one of them can solve a problem.', 'generate sentence: insects, rats, bugs, one, solve'], ['One slugs bugs. the other flaws are flawed.', 'generate sentence: one, slugs, bugs, flaws, flawed'], [\"I can verify that termites are bugs......but most people don't like them.\", 'generate sentence: one, termites, bugs, verify, people'], ['I woke up with spiders as a species of bugs to solve my problem.', 'generate sentence: spiders, species, bugs, woke, solve']]\n"
     ]
    }
   ],
   "source": [
    "for word in words:\n",
    "    try:\n",
    "        print(\"Pun word: \", word)\n",
    "        senseDefinitions = sense_def(word)\n",
    "        print(\"sense definitions: \",senseDefinitions)\n",
    "        ReverseDicKeywords = keywords(senseDefinitions)\n",
    "        print(\"ReverseDicKeywords: \",ReverseDicKeywords)\n",
    "        ReverseDicKeywords_refine = Refine(ReverseDicKeywords, word,5 )\n",
    "        print(\"ReverseDicKeywords_refine: \",ReverseDicKeywords_refine)\n",
    "        finalKeywords= final_keywords(ReverseDicKeywords_refine, word)\n",
    "        print(\"finalKeywords: \",finalKeywords)\n",
    "        finalKeywords_refine= Refine(finalKeywords, word)\n",
    "        print(\"finalKeywords_refine\",finalKeywords_refine)\n",
    "        candidateSentences1 = generate_candidates1(finalKeywords_refine, word, flag)\n",
    "        print(\"Candidate sentences 1: \", candidateSentences1[0:5])\n",
    "        with open(path+str(word_temp)+'_.p', 'wb') as fx:\n",
    "            pickle.dump(candidateSentences1, fx, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "    except Exception as e:\n",
    "        print(\"Error: \", word, e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "UGkKgHMWyJP6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pickle\n",
    "with open(path+'hit.p', 'rb') as fp:\n",
    "    data1 = pickle.load(fp)\n",
    "data1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "import transformers\n",
    "from transformers import AutoModel, BertTokenizerFast\n",
    "import matplotlib\n",
    "from scipy.special import softmax\n",
    "device = torch.device(\"cuda:1\")\n",
    "\n",
    "import pickle\n",
    "# import BERT-base pretrained model\n",
    "bert = AutoModel.from_pretrained('bert-large-uncased',return_dict=False)\n",
    "\n",
    "# Load the BERT tokenizer\n",
    "tokenizer = BertTokenizerFast.from_pretrained('bert-large-uncased')\n",
    "\n",
    "max_seq_len = 20\n",
    "class BERT_Arch(nn.Module):\n",
    "\n",
    "    def __init__(self, bert):\n",
    "      \n",
    "      super(BERT_Arch, self).__init__()\n",
    "\n",
    "      self.bert = bert \n",
    "      \n",
    "      # dropout layer\n",
    "      self.dropout = nn.Dropout(0.1)\n",
    "      \n",
    "      # relu activation function\n",
    "      self.relu =  nn.ReLU()\n",
    "\n",
    "      # dense layer 1\n",
    "      self.fc1 = nn.Linear(1024,512)\n",
    "      \n",
    "      # dense layer 2 (Output layer)\n",
    "      self.fc2 = nn.Linear(512,2)\n",
    "\n",
    "      #softmax activation function\n",
    "      self.softmax = nn.LogSoftmax(dim=1)\n",
    "\n",
    "    #define the forward pass\n",
    "    def forward(self, sent_id, mask):\n",
    "\n",
    "      #pass the inputs to the model  \n",
    "      _, cls_hs = self.bert(sent_id, attention_mask=mask)\n",
    "      \n",
    "      x = self.fc1(cls_hs)\n",
    "\n",
    "      x = self.relu(x)\n",
    "\n",
    "      x = self.dropout(x)\n",
    "\n",
    "      # output layer\n",
    "      x = self.fc2(x)\n",
    "      \n",
    "      # apply softmax activation\n",
    "      x = self.softmax(x)\n",
    "\n",
    "      return x\n",
    "\n",
    "# pass the pre-trained BERT to our define architecture\n",
    "model = BERT_Arch(bert)\n",
    "\n",
    "# push the model to GPU\n",
    "model = model.to(device)\n",
    "\n",
    "#load weights of best model\n",
    "path1 = path+'saved_weights_colbert.pt'\n",
    "model.load_state_dict(torch.load(path1))\n",
    "\n",
    "with torch.no_grad():\n",
    "  preds = model(infer_seq.to(device), infer_mask.to(device))\n",
    "  preds = preds.detach().cpu().numpy()\n",
    "logits = softmax(preds, axis = 1)\n",
    "\n",
    "score_dict={}\n",
    "for text, score in zip(infer_text,logits[:,1]):\n",
    "    score_dict[text] = score\n",
    "\n",
    "ranked_dict = {k: v for k, v in sorted(score_dict.items(), key=lambda item: item[1], reverse = True)}\n",
    "for item in ranked_dict:\n",
    "    if word.strip('_new') in item:\n",
    "        print(item, ranked_dict[item], sep = '\\t')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Pipeline.ipynb",
   "provenance": []
  },
  "interpreter": {
   "hash": "98564face510875289c8489aecbca6068466fb105cf6ae67d89e53a37d233d89"
  },
  "kernelspec": {
   "display_name": "Python 3.7.4 64-bit ('env_pytorch': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}