Toxic_Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of training and testing is: 159571 153164\n"
     ]
    }
   ],
   "source": [
    "train_df=pd.read_csv('/home/ruicao/NLP/datasets/hate-speech/toxic-comment-detection/train.csv')\n",
    "test_df=pd.read_csv('/home/ruicao/NLP/datasets/hate-speech/toxic-comment-detection/test.csv')\n",
    "print('Length of training and testing is:',len(train_df),len(test_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle as pkl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_names=['toxic', 'severe_toxic', 'obscene', 'threat', 'insult','identity_hate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Explanation\n",
      "Why the edits made under my username Hardcore Metallica Fan were reverted? They weren't vandalisms, just closure on some GAs after I voted at New York Dolls FAC. And please don't remove the template from the talk page since I'm retired now.89.205.38.27\n",
      "[0 0 0 0 0 0]\n",
      "1 D'aww! He matches this background colour I'm seemingly stuck with. Thanks.  (talk) 21:51, January 11, 2016 (UTC)\n",
      "[0 0 0 0 0 0]\n",
      "2 Hey man, I'm really not trying to edit war. It's just that this guy is constantly removing relevant information and talking to me through edits instead of my talk page. He seems to care more about the formatting than the actual info.\n",
      "[0 0 0 0 0 0]\n",
      "3 \"\n",
      "More\n",
      "I can't make any real suggestions on improvement - I wondered if the section statistics should be later on, or a subsection of \"\"types of accidents\"\"  -I think the references may need tidying so that they are all in the exact same format ie date format etc. I can do that later on, if no-one else does first - if you have any preferences for formatting style on references or want to do it yourself please let me know.\n",
      "\n",
      "There appears to be a backlog on articles for review so I guess there may be a delay until a reviewer turns up. It's listed in the relevant form eg Wikipedia:Good_article_nominations#Transport  \"\n",
      "[0 0 0 0 0 0]\n",
      "4 You, sir, are my hero. Any chance you remember what page that's on?\n",
      "[0 0 0 0 0 0]\n",
      "5 \"\n",
      "\n",
      "Congratulations from me as well, use the tools well.  · talk \"\n",
      "[0 0 0 0 0 0]\n",
      "6 COCKSUCKER BEFORE YOU PISS AROUND ON MY WORK\n",
      "[1 1 1 0 1 0]\n",
      "7 Your vandalism to the Matt Shirvington article has been reverted.  Please don't do it again, or you will be banned.\n",
      "[0 0 0 0 0 0]\n",
      "8 Sorry if the word 'nonsense' was offensive to you. Anyway, I'm not intending to write anything in the article(wow they would jump on me for vandalism), I'm merely requesting that it be more encyclopedic so one can use it for school as a reference. I have been to the selective breeding page but it's almost a stub. It points to 'animal breeding' which is a short messy article that gives you no info. There must be someone around with expertise in eugenics? 93.161.107.169\n",
      "[0 0 0 0 0 0]\n",
      "9 alignment on this subject and which are contrary to those of DuLithgow\n",
      "[0 0 0 0 0 0]\n"
     ]
    }
   ],
   "source": [
    "for i in range (10):\n",
    "    print (i,train_df['comment_text'][i])\n",
    "    print (train_df[label_names].values[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['id', 'toxic', 'severe_toxic', 'obscene', 'threat', 'insult',\n",
      "       'identity_hate'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "labels=pd.read_csv('/home/ruicao/NLP/datasets/hate-speech/toxic-comment-detection/test_labels.csv')\n",
    "print (labels.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0]\n",
      " [1 1 1 0 1 0]\n",
      " [0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0]]\n",
      "<class 'numpy.ndarray'>\n"
     ]
    }
   ],
   "source": [
    "print(train_df[label_names].values[:10])\n",
    "print (type(train_df[label_names].values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_test=pd.concat([test_df['comment_text'],labels],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['id', 'comment_text'], dtype='object') Index(['comment_text', 'id', 'toxic', 'severe_toxic', 'obscene', 'threat',\n",
      "       'insult', 'identity_hate'],\n",
      "      dtype='object')\n",
      "Index(['id', 'comment_text', 'toxic', 'severe_toxic', 'obscene', 'threat',\n",
      "       'insult', 'identity_hate'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print (test_df.columns,total_test.columns)\n",
    "print (train_df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "153164\n",
      "63978\n",
      "{'0': 6090, '1': 367, '2': 3691, '3': 211, '4': 3427, '5': 712}\n"
     ]
    }
   ],
   "source": [
    "label=total_test[label_names].values\n",
    "print (len(label))\n",
    "count={str(i):0 for i in range(6)}\n",
    "counting=0\n",
    "for row in label:\n",
    "    if row[0]==-1:\n",
    "        #print (row)\n",
    "        continue\n",
    "    counting+=1\n",
    "    for j in range(6):\n",
    "        if row[j]>0.5:\n",
    "            count[str(j)]+=1\n",
    "print (counting)\n",
    "print (count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize.treebank import TreebankWordTokenizer\n",
    "from preprocessing import clean_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "symbols_to_delete  = '@＼・ω+=”“[]^–>\\\\°<~•≠™ˈʊɒ∞§{}·τα❤☺ɡ|¢→̶`❥━┣┫┗Ｏ►★©―ɪ✔®\\x96\\x92●£♥➤´¹☕≈÷♡◐║▬′ɔː€۩۞†μ✒➥═☆ˌ◄½ʻπδηλσερνʃ✬ＳＵＰＥＲＩＴ☻±♍µº¾✓◾؟．⬅℅»Вав❣⋅¿¬♫ＣＭβ█▓▒░⇒⭐›¡₂₃❧▰▔◞▀▂▃▄▅▆▇↙γ̄″☹➡«φ⅓„✋：¥̲̅́∙‛◇✏▷❓❗¶˚˙）сиʿ✨。ɑ\\x80◕！％¯−ﬂﬁ₁²ʌ¼⁴⁄₄⌠♭✘╪▶☭✭♪☔☠♂☃☎✈✌✰❆☙○‣⚓年∎ℒ▪▙☏⅛ｃａｓǀ℮¸ｗ‚∼‖ℳ❄←☼⋆ʒ⊂、⅔¨͡๏⚾⚽Φ×θ￦？（℃⏩☮⚠月✊❌⭕▸■⇌☐☑⚡☄ǫ╭∩╮，例＞ʕɐ̣Δ₀✞┈╱╲▏▕┃╰▊▋╯┳┊≥☒↑☝ɹ✅☛♩☞ＡＪＢ◔◡↓♀⬆̱ℏ\\x91⠀ˤ╚↺⇤∏✾◦♬³の｜／∵∴√Ω¤☜▲↳▫‿⬇✧ｏｖｍ－２０８＇‰≤∕ˆ⚜☁.,?!-;*\"…:—()%#$&_/\\n🍕\\r🐵😑\\xa0\\ue014\\t\\uf818\\uf04a\\xad😢🐶️\\uf0e0😜😎👊\\u200b\\u200e😁عدويهصقأناخلىبمغر😍💖💵Е👎😀😂\\u202a\\u202c🔥😄🏻💥ᴍʏʀᴇɴᴅᴏᴀᴋʜᴜʟᴛᴄᴘʙғᴊᴡɢ😋👏שלוםבי😱‼\\x81エンジ故障\\u2009🚌ᴵ͞🌟😊😳😧🙀😐😕\\u200f👍😮😃😘אעכח💩💯⛽🚄🏼ஜ😖ᴠ🚲‐😟😈💪🙏🎯🌹😇💔😡\\x7f👌ἐὶήιὲκἀίῃἴξ🙄Ｈ😠\\ufeff\\u2028😉😤⛺🙂\\u3000تحكسة👮💙فزط😏🍾🎉😞\\u2008🏾😅😭👻😥😔😓🏽🎆🍻🍽🎶🌺🤔😪\\x08‑🐰🐇🐱🙆😨🙃💕𝘊𝘦𝘳𝘢𝘵𝘰𝘤𝘺𝘴𝘪𝘧𝘮𝘣💗💚地獄谷улкнПоАН🐾🐕😆ה🔗🚽歌舞伎🙈😴🏿🤗🇺🇸мυтѕ⤵🏆🎃😩\\u200a🌠🐟💫💰💎эпрд\\x95🖐🙅⛲🍰🤐👆🙌\\u2002💛🙁👀🙊🙉\\u2004ˢᵒʳʸᴼᴷᴺʷᵗʰᵉᵘ\\x13🚬🤓\\ue602😵άοόςέὸתמדףנרךצט😒͝🆕👅👥👄🔄🔤👉👤👶👲🔛🎓\\uf0b7\\uf04c\\x9f\\x10成都😣⏺😌🤑🌏😯ех😲Ἰᾶὁ💞🚓🔔📚🏀👐\\u202d💤🍇\\ue613小土豆🏡❔⁉\\u202f👠》कर्मा🇹🇼🌸蔡英文🌞🎲レクサス😛外国人关系Сб💋💀🎄💜🤢َِьыгя不是\\x9c\\x9d🗑\\u2005💃📣👿༼つ༽😰ḷЗз▱ц￼🤣卖温哥华议会下降你失去所有的钱加拿大坏税骗子🐝ツ🎅\\x85🍺آإشء🎵🌎͟ἔ油别克🤡🤥😬🤧й\\u2003🚀🤴ʲшчИОРФДЯМюж😝🖑ὐύύ特殊作戦群щ💨圆明园קℐ🏈😺🌍⏏ệ🍔🐮🍁🍆🍑🌮🌯🤦\\u200d𝓒𝓲𝓿𝓵안영하세요ЖљКћ🍀😫🤤ῦ我出生在了可以说普通话汉语好极🎼🕺🍸🥂🗽🎇🎊🆘🤠👩🖒🚪天一家⚲\\u2006⚭⚆⬭⬯⏖新✀╌🇫🇷🇩🇪🇮🇬🇧😷🇨🇦ХШ🌐\\x1f杀鸡给猴看ʁ𝗪𝗵𝗲𝗻𝘆𝗼𝘂𝗿𝗮𝗹𝗶𝘇𝗯𝘁𝗰𝘀𝘅𝗽𝘄𝗱📺ϖ\\u2000үսᴦᎥһͺ\\u2007հ\\u2001ɩｙｅ൦ｌƽｈ𝐓𝐡𝐞𝐫𝐮𝐝𝐚𝐃𝐜𝐩𝐭𝐢𝐨𝐧Ƅᴨןᑯ໐ΤᏧ௦Іᴑ܁𝐬𝐰𝐲𝐛𝐦𝐯𝐑𝐙𝐣𝐇𝐂𝐘𝟎ԜТᗞ౦〔Ꭻ𝐳𝐔𝐱𝟔𝟓𝐅🐋ﬃ💘💓ё𝘥𝘯𝘶💐🌋🌄🌅𝙬𝙖𝙨𝙤𝙣𝙡𝙮𝙘𝙠𝙚𝙙𝙜𝙧𝙥𝙩𝙪𝙗𝙞𝙝𝙛👺🐷ℋ𝐀𝐥𝐪🚶𝙢Ἱ🤘ͦ💸ج패티Ｗ𝙇ᵻ👂👃ɜ🎫\\uf0a7БУі🚢🚂ગુજરાતીῆ🏃𝓬𝓻𝓴𝓮𝓽𝓼☘﴾̯﴿₽\\ue807𝑻𝒆𝒍𝒕𝒉𝒓𝒖𝒂𝒏𝒅𝒔𝒎𝒗𝒊👽😙\\u200cЛ‒🎾👹⎌🏒⛸公寓养宠物吗🏄🐀🚑🤷操美𝒑𝒚𝒐𝑴🤙🐒欢迎来到阿拉斯ספ𝙫🐈𝒌𝙊𝙭𝙆𝙋𝙍𝘼𝙅ﷻ🦄巨收赢得白鬼愤怒要买额ẽ🚗🐳𝟏𝐟𝟖𝟑𝟕𝒄𝟗𝐠𝙄𝙃👇锟斤拷𝗢𝟳𝟱𝟬⦁マルハニチロ株式社⛷한국어ㄸㅓ니͜ʖ𝘿𝙔₵𝒩ℯ𝒾𝓁𝒶𝓉𝓇𝓊𝓃𝓈𝓅ℴ𝒻𝒽𝓀𝓌𝒸𝓎𝙏ζ𝙟𝘃𝗺𝟮𝟭𝟯𝟲👋🦊多伦🐽🎻🎹⛓🏹🍷🦆为和中友谊祝贺与其想象对法如直接问用自己猜本传教士没积唯认识基督徒曾经让相信耶稣复活死怪他但当们聊些政治题时候战胜因圣把全堂结婚孩恐惧且栗谓这样还♾🎸🤕🤒⛑🎁批判检讨🏝🦁🙋😶쥐스탱트뤼도석유가격인상이경제황을렵게만들지않록잘관리해야합다캐나에서대마초와화약금의품런성분갈때는반드시허된사용🔫👁凸ὰ💲🗯𝙈Ἄ𝒇𝒈𝒘𝒃𝑬𝑶𝕾𝖙𝖗𝖆𝖎𝖌𝖍𝖕𝖊𝖔𝖑𝖉𝖓𝖐𝖜𝖞𝖚𝖇𝕿𝖘𝖄𝖛𝖒𝖋𝖂𝕴𝖟𝖈𝕸👑🚿💡知彼百\\uf005𝙀𝒛𝑲𝑳𝑾𝒋𝟒😦𝙒𝘾𝘽🏐𝘩𝘨ὼṑ𝑱𝑹𝑫𝑵𝑪🇰🇵👾ᓇᒧᔭᐃᐧᐦᑳᐨᓃᓂᑲᐸᑭᑎᓀᐣ🐄🎈🔨🐎🤞🐸💟🎰🌝🛳点击查版🍭𝑥𝑦𝑧ＮＧ👣\\uf020っ🏉ф💭🎥Ξ🐴👨🤳🦍\\x0b🍩𝑯𝒒😗𝟐🏂👳🍗🕉🐲چی𝑮𝗕𝗴🍒ꜥⲣⲏ🐑⏰鉄リ事件ї💊「」\\uf203\\uf09a\\uf222\\ue608\\uf202\\uf099\\uf469\\ue607\\uf410\\ue600燻製シ虚偽屁理屈Г𝑩𝑰𝒀𝑺🌤𝗳𝗜𝗙𝗦𝗧🍊ὺἈἡχῖΛ⤏🇳𝒙ψՁմեռայինրւդձ冬至ὀ𝒁🔹🤚🍎𝑷🐂💅𝘬𝘱𝘸𝘷𝘐𝘭𝘓𝘖𝘹𝘲𝘫کΒώ💢ΜΟΝΑΕ🇱♲𝝈↴💒⊘Ȼ🚴🖕🖤🥘📍👈➕🚫🎨🌑🐻𝐎𝐍𝐊𝑭🤖🎎😼🕷ｇｒｎｔｉｄｕｆｂｋ𝟰🇴🇭🇻🇲𝗞𝗭𝗘𝗤👼📉🍟🍦🌈🔭《🐊🐍\\uf10aლڡ🐦\\U0001f92f\\U0001f92a🐡💳ἱ🙇𝗸𝗟𝗠𝗷🥜さようなら🔼'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = TreebankWordTokenizer()\n",
    "#isolate_dict = {ord(c): ' {} '.format(c) for c in symbols_to_isolate}\n",
    "remove_dict = {ord(c): '' for c in symbols_to_delete}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def handle_contractions(x):\n",
    "    x = tokenizer.tokenize(x)\n",
    "    return x\n",
    "\n",
    "def handle_punctuation(x):\n",
    "    x = x.translate(remove_dict)\n",
    "    #x = x.translate(isolate_dict)\n",
    "    return x\n",
    "\n",
    "def fix_quote(x):\n",
    "    x = [x_[1:] if x_.startswith(\"'\") else x_ for x_ in x]\n",
    "    x = ' '.join(x)\n",
    "    return x\n",
    "\n",
    "def preprocess(x):\n",
    "    x=clean_text(x)\n",
    "    x = handle_punctuation(x)\n",
    "    x = handle_contractions(x)\n",
    "    x = fix_quote(x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of sentences: 159571\n",
      "52034\n",
      "The lenght of the dictionary is: 52035\n"
     ]
    }
   ],
   "source": [
    "#create dictionary and glove embeddings\n",
    "MIN_OCC=5\n",
    "DICT_PATH='/home/ruicao/NLP/textual/hate-speech-detection/toxic/dictionary/dictionary.pkl'\n",
    "from collections import defaultdict\n",
    "sents=[]\n",
    "for i in range(len(train_df)):\n",
    "    sent=train_df['comment_text'][i]\n",
    "    sents.append(sent)\n",
    "print ('Total number of sentences:',len(sents))\n",
    "count_word=defaultdict(int)\n",
    "for s in sents:\n",
    "    words=preprocess(s).split()\n",
    "    for w in words:\n",
    "        count_word[w]+=1\n",
    "        \n",
    "count=0\n",
    "idx2word=[]\n",
    "word2idx={}\n",
    "for word in count_word.keys():\n",
    "    if count_word[word]>=MIN_OCC:\n",
    "        word2idx[word]=count\n",
    "        count+=1\n",
    "        idx2word.append(word)\n",
    "word2idx['UNK']=count\n",
    "idx2word.append('UNK')\n",
    "print (word2idx['UNK'])\n",
    "print ('The lenght of the dictionary is:',len(idx2word))        \n",
    "pkl.dump([word2idx,idx2word],open(DICT_PATH,'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(52035, 300)\n"
     ]
    }
   ],
   "source": [
    "GLOVE_PATH='/home/ruicao/trained/embeddings/glove.6B.300d.txt'\n",
    "word2emb={}\n",
    "emb_path='/home/ruicao/NLP/textual/hate-speech-detection/toxic/dictionary/glove_embedding.npy'\n",
    "with open(GLOVE_PATH,'r') as f:\n",
    "    entries=f.readlines()\n",
    "    emb_dim=len(entries[0].split(' '))-1\n",
    "    weights=np.zeros((len(idx2word),emb_dim),dtype=np.float32)\n",
    "    for entry in entries:\n",
    "        word=entry.split(' ')[0]\n",
    "        word2emb[word]=np.array(list(map(float,entry.split(' ')[1:])))\n",
    "    for idx,word in enumerate(idx2word):\n",
    "        if word not in word2emb:\n",
    "            continue\n",
    "        weights[idx]=word2emb[word]\n",
    "    print (weights.shape)\n",
    "    np.save(emb_path,weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Data_Set(object):\n",
    "    def __init__(self,pd,mode='train'):\n",
    "        self.length=220\n",
    "        \n",
    "        self.dict_path='/home/ruicao/NLP/textual/hate-speech-detection/toxic/dictionary/dictionary.pkl'\n",
    "        \n",
    "        self.pd=pd\n",
    "        self.mode=mode\n",
    "        \n",
    "        self.init_dict_info()\n",
    "        self.entries=self.load_entries()\n",
    "        print ('Number of entries is:',len(self.entries))\n",
    "        self.tokenize()\n",
    "        self.tensorize()\n",
    "    \n",
    "    def init_dict_info(self):\n",
    "        self.word2idx=pkl.load(open(self.dict_path,'rb'))[0]\n",
    "        self.ntokens=len(self.word2idx)\n",
    "        print ('The length of the dictionary is:',self.ntokens)\n",
    "    \n",
    "    def get_tokens(self,sent):\n",
    "        words=preprocess(sent).split()\n",
    "        tokens=[]\n",
    "        for w in words:\n",
    "            if w in self.word2idx:\n",
    "                tokens.append(self.word2idx[w])\n",
    "            else:\n",
    "                tokens.append(self.word2idx['UNK'])\n",
    "        return tokens\n",
    "    \n",
    "    def pad_sent(self,tokens):\n",
    "        if len(tokens)<self.length:\n",
    "            padding=[self.ntokens]*(self.length-len(tokens))\n",
    "            tokens=padding+tokens\n",
    "        else:\n",
    "            tokens=tokens[:self.length]\n",
    "        return tokens\n",
    "    \n",
    "    def tokenize(self):\n",
    "        print ('Tokenizing tweets...')\n",
    "        for entry in self.entries:\n",
    "            tokens=self.get_tokens(entry['sent'])\n",
    "            #print(tokens)\n",
    "            pad_tokens=self.pad_sent(tokens)\n",
    "            entry['tokens']=np.array((pad_tokens),dtype=np.int64)\n",
    "    \n",
    "    def tensorize(self):\n",
    "        print ('Tensorizing all information...')\n",
    "        for entry in self.entries:\n",
    "            entry['tokens']=torch.from_numpy(entry['tokens'])\n",
    "            entry['label']=torch.from_numpy(entry['label'])\n",
    "    \n",
    "    def load_entries(self):\n",
    "        entries=[]\n",
    "        texts=self.pd['comment_text']\n",
    "        labels=self.pd[label_names].values\n",
    "        for i,sent in enumerate(texts):\n",
    "            if self.mode=='test' and labels[i][0]==-1:\n",
    "                continue\n",
    "            entry={\n",
    "                'sent':sent,\n",
    "                'label':labels[i]\n",
    "            }\n",
    "            entries.append(entry)\n",
    "        return entries    \n",
    "        \n",
    "    def __getitem__(self,index):\n",
    "        entry=self.entries[index]\n",
    "        tokens=entry['tokens']\n",
    "        label=entry['label']\n",
    "        return tokens,label\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.entries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The length of the dictionary is: 52035\n",
      "Number of entries is: 159571\n",
      "Tokenizing tweets...\n",
      "Tensorizing all information...\n"
     ]
    }
   ],
   "source": [
    "train_set=Data_Set(train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The length of the dictionary is: 52035\n",
      "Number of entries is: 63978\n",
      "Tokenizing tweets...\n",
      "Tensorizing all information...\n"
     ]
    }
   ],
   "source": [
    "test_set=Data_Set(total_test,mode='test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "class Base_Model(nn.Module):\n",
    "    #vocab_size is ntokens + 1\n",
    "    def __init__(self,emb_dim,vocab_size,num_hidden,final_dim,emb_dropout,fc_dropout,emb_path):\n",
    "        super(Base_Model,self).__init__()\n",
    "        self.emb=nn.Embedding(vocab_size+1,emb_dim,padding_idx=vocab_size)\n",
    "        self.lstm1=nn.LSTM(emb_dim,num_hidden,bidirectional=True,batch_first=True)\n",
    "        self.lstm2=nn.LSTM(num_hidden*2,num_hidden,bidirectional=True,batch_first=True)\n",
    "        self.out=nn.Linear(num_hidden*4,6)\n",
    "        \n",
    "        self.emb_drop=nn.Dropout(emb_dropout)\n",
    "        self.init_word_emb(emb_path)\n",
    "        \n",
    "    def init_word_emb(self,path):\n",
    "        print ('Initializing the pre-trained embeddings')\n",
    "        emb_weights=torch.from_numpy(np.load(path))\n",
    "        self.emb.weight.data[:-1]=emb_weights\n",
    "        \n",
    "    def forward(self,tokens):\n",
    "        t_emb=self.emb(tokens)\n",
    "        t_emb=self.emb_drop(t_emb)\n",
    "        hidden1,_=self.lstm1(t_emb)\n",
    "        hidden2,_=self.lstm2(hidden1)\n",
    "        avg_pool=torch.mean(hidden2,1)\n",
    "        max_pool,_=torch.max(hidden2,1)\n",
    "        concat_hidden=torch.cat((max_pool,avg_pool),1)\n",
    "        result=torch.sigmoid(self.out(concat_hidden)) \n",
    "        return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMB_DIM=300\n",
    "VOCAB_SIZE=train_set.ntokens\n",
    "NUM_HIDDEN=128\n",
    "FINAL_DIM=6\n",
    "EMB_DROPOUT=0.3\n",
    "FC_DROPOUT=0.5\n",
    "EMB_PATH='/home/ruicao/NLP/textual/hate-speech-detection/toxic/dictionary/glove_embedding.npy'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "CUDA_DEVICE=1\n",
    "SEED=1111\n",
    "torch.cuda.set_device(CUDA_DEVICE)\n",
    "torch.manual_seed(SEED)\n",
    "device = torch.device(\"cuda:\"+str(CUDA_DEVICE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing the pre-trained embeddings\n"
     ]
    }
   ],
   "source": [
    "model=Base_Model(EMB_DIM,VOCAB_SIZE,NUM_HIDDEN,FINAL_DIM,EMB_DROPOUT,FC_DROPOUT,EMB_PATH).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE=256\n",
    "EPOCHS=8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "test_loader=DataLoader(test_set,BATCH_SIZE,shuffle=True,num_workers=1)\n",
    "train_loader=DataLoader(train_set,BATCH_SIZE,shuffle=True,num_workers=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion=nn.BCELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_multi_loss(r_pred,r_labels):\n",
    "    loss=criterion(r_pred,r_labels)\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_score(r_pred,r_labels):\n",
    "    r_pred=r_pred.detach().cpu().numpy()\n",
    "    r_labels=r_labels.detach().cpu().numpy()\n",
    "    result=roc_auc_score(r_labels,r_pred,average='weighted')\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_model(baseline,test_info):\n",
    "    t_loss=0.0\n",
    "    print ('Length of iterations for evaluation is:',len(test_info))\n",
    "    for i,(tokens,labels) in enumerate(test_info):\n",
    "        with torch.no_grad():\n",
    "            tokens=tokens.to(device)\n",
    "            labels=labels.float().to(device)\n",
    "            #print (labels)\n",
    "            pred=baseline(tokens)\n",
    "            b_loss=compute_multi_loss(pred,labels)\n",
    "            t_loss+=b_loss\n",
    "        if i==0:\n",
    "            t_pred=pred\n",
    "            t_labels=labels\n",
    "        else:\n",
    "            t_pred=torch.cat((t_pred,pred),dim=0)\n",
    "            t_labels=torch.cat((t_labels,labels),dim=0)\n",
    "    avg_loss=t_loss\n",
    "    avg_score=compute_score(t_pred,t_labels)\n",
    "    return avg_loss,avg_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "optim=torch.optim.Adamax(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training loss and score is tensor(53.2302, device='cuda:1', grad_fn=<AddBackward0>) 0.9210070221431018 in Epoch 0\n",
      "Length of iterations for evaluation is: 250\n",
      "Evaluation loss and score is tensor(20.8438, device='cuda:1') 0.9475581242127151 in Epoch 0\n",
      "Training loss and score is tensor(34.8857, device='cuda:1', grad_fn=<AddBackward0>) 0.971088738889286 in Epoch 1\n",
      "Length of iterations for evaluation is: 250\n",
      "Evaluation loss and score is tensor(20.1914, device='cuda:1') 0.9571523526913058 in Epoch 1\n",
      "Training loss and score is tensor(31.8086, device='cuda:1', grad_fn=<AddBackward0>) 0.9772264089984338 in Epoch 2\n",
      "Length of iterations for evaluation is: 250\n",
      "Evaluation loss and score is tensor(19.4898, device='cuda:1') 0.9607476777147577 in Epoch 2\n",
      "Training loss and score is tensor(29.6564, device='cuda:1', grad_fn=<AddBackward0>) 0.9813206903534551 in Epoch 3\n",
      "Length of iterations for evaluation is: 250\n",
      "Evaluation loss and score is tensor(18.3606, device='cuda:1') 0.9657028118996226 in Epoch 3\n",
      "Training loss and score is tensor(27.7792, device='cuda:1', grad_fn=<AddBackward0>) 0.9844246964405461 in Epoch 4\n",
      "Length of iterations for evaluation is: 250\n",
      "Evaluation loss and score is tensor(17.3896, device='cuda:1') 0.9675574221133868 in Epoch 4\n",
      "Training loss and score is tensor(26.2073, device='cuda:1', grad_fn=<AddBackward0>) 0.9864626271627298 in Epoch 5\n",
      "Length of iterations for evaluation is: 250\n",
      "Evaluation loss and score is tensor(16.6415, device='cuda:1') 0.9693697361587172 in Epoch 5\n",
      "Training loss and score is tensor(24.7913, device='cuda:1', grad_fn=<AddBackward0>) 0.9882986947355293 in Epoch 6\n",
      "Length of iterations for evaluation is: 250\n",
      "Evaluation loss and score is tensor(18.3227, device='cuda:1') 0.9700237364954787 in Epoch 6\n",
      "Training loss and score is tensor(23.3442, device='cuda:1', grad_fn=<AddBackward0>) 0.989951874216018 in Epoch 7\n",
      "Length of iterations for evaluation is: 250\n",
      "Evaluation loss and score is tensor(18.8487, device='cuda:1') 0.9685633197725426 in Epoch 7\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(EPOCHS):\n",
    "    total_loss=0.0\n",
    "    model.train(True)\n",
    "    for i,(tokens,labels) in enumerate(train_loader):\n",
    "        tokens=tokens.to(device)\n",
    "        labels=labels.float().to(device)\n",
    "        pred=model(tokens)\n",
    "        #print (type(labels))\n",
    "        loss=compute_multi_loss(pred,labels)\n",
    "        total_loss+=loss\n",
    "        loss.backward()\n",
    "        optim.step()\n",
    "        optim.zero_grad()\n",
    "        if i==0:\n",
    "            t_pred=pred\n",
    "            t_labels=labels\n",
    "        else:\n",
    "            t_pred=torch.cat((t_pred,pred),dim=0)\n",
    "            t_labels=torch.cat((t_labels,labels),dim=0)\n",
    "    train_score=compute_score(t_pred,t_labels)\n",
    "    print ('Training loss and score is',total_loss,train_score,'in Epoch',epoch)\n",
    "    model.train(False)\n",
    "    eval_loss,eval_score=evaluate_model(model,test_loader)\n",
    "    print ('Evaluation loss and score is',eval_loss,eval_score,'in Epoch',epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(),'/home/ruicao/NLP/textual/hate-speech-detection/toxic/model.pth')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3.6",
   "language": "python",
   "name": "py36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}