end_to_end_Corpus_Classifier_Test_FPC.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "! if you have prepared data - you can load these in the section marked 'pre-loaded data - start here!'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:47:12.648554Z",
     "start_time": "2020-03-19T09:47:09.735374Z"
    }
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os, random, glob, json, nltk, re\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.neural_network import MLPClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Filter Gigaword"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Read in the full corpus \n",
    "this is the full corpus with only information about date_location and concatenated head and sentences - requires a lot of RAM but it resolves the overfitting problem and is quick to choose a balanced corpus. \n",
    "\n",
    "When I ran this, I added the words poverty, slave ' aid '(with spaces) to Jan's list.\n",
    "\n",
    "I have turned the entire corpus into a single json file ~ 7GB. It has 2 columns - name/date and text - text includes headline + all 5 sentences in 1 column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T17:14:09.985379Z",
     "start_time": "2020-03-17T17:12:05.069733Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "big_df = pd.read_json('./pickles/frame_to_rule2.json', encoding = 'utf-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Dataframe Construction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Keyword Lookup\n",
    "\n",
    "### The threshold set to 4 produces 3000+ articles / set to 5 produces 880\n",
    "experimented with threshold 3 - produces 21000 articles but scores were worse on both baseline and bert - seems that it's not a high-quality search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T17:19:00.823401Z",
     "start_time": "2020-03-17T17:19:00.792147Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def keyword_search(keywords, big_df, threshold = 0):\n",
    "    articles = []\n",
    "    article_keywords = []\n",
    "    for text in big_df['text']:\n",
    "        keywords_found = set()\n",
    "        for keyword in keywords:\n",
    "            if keyword in text:\n",
    "                keywords_found.add(keyword)\n",
    "        if len(keywords_found) >= threshold:\n",
    "            articles.append(text)\n",
    "            article_keywords.append(keywords_found)\n",
    "    return articles, article_keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T17:19:01.151536Z",
     "start_time": "2020-03-17T17:19:01.135910Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "topic = 'poverty'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T17:19:02.651649Z",
     "start_time": "2020-03-17T17:19:02.245382Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "with open(f'keywords/{topic}.txt', 'r', encoding = 'utf-8') as infile:\n",
    "    keywords = infile.read().splitlines()\n",
    "\n",
    "#fine-grained search, kwarg 'threshold' indicates the number of keywords that should be present in the text\n",
    "articles, keywords_per_article = keyword_search(keywords, big_df, threshold = 4)\n",
    "\n",
    "print('Number of articles found:', len(articles))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### add 'text', 'labels' column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T17:19:02.886042Z",
     "start_time": "2020-03-17T17:19:02.854790Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "columns = ['text']\n",
    "filtered_df = pd.DataFrame(articles, columns = columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T17:17:16.969224Z",
     "start_time": "2020-03-17T17:17:16.953597Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "filtered_df['label'] = 'related'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T17:17:17.000474Z",
     "start_time": "2020-03-17T17:17:16.969224Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# there are duplicate texts in the dataset - this should drop them. \n",
    "filtered_df.drop_duplicates(subset = 'text', inplace = True, keep = 'first')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T17:17:17.500512Z",
     "start_time": "2020-03-17T17:17:17.000474Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# in order to clear the memory to construct the balancing - unrelated - datset, the file is dumped to a pickle file\n",
    "# and the kernel was restarted. \n",
    "\n",
    "pickle.dump(filtered_df, open('./saved_models/interim_threshold4_related_only_delete_immediately.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "#### potential memory wipe point"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## random selection of non-related articles to balance the training set \n",
    "\n",
    "remember to clear the kernel before continuing:\n",
    "this method takes a long time - watch a vid; play a tune. \n",
    "\n",
    "For some reason if I just randomly sample the full dataframe (very fast) the stats consistently drop 10+ points!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T17:19:18.246603Z",
     "start_time": "2020-03-17T17:19:18.215350Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "filtered_df = pickle.load(open('./saved_models/interim_threshold4_related_only_delete_immediately.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T17:19:20.918678Z",
     "start_time": "2020-03-17T17:19:20.903053Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#Tip: to save memory write the line texts to a csv file instead of yield (I think it should work)\n",
    "\n",
    "def random_list (filename):\n",
    "    with open (filename, 'r') as infile:\n",
    "#         print(filename)\n",
    "        file = pd.read_json(infile, orient = 'index')\n",
    "    random_index = random.randint(0,file.shape[0])\n",
    "    if keyword not in file.iloc[random_index-1].text:\n",
    "        #if memory is an issue, uncomment these 3 line:\n",
    "#         text_only = file.iloc[random_index-1]['text'].rstrip('\\n')+'\\n'\n",
    "#         with open ('balancing_data.csv', 'a') as outfile:\n",
    "#             outfile.write(text_only)\n",
    "        # and comment out this yield statement\n",
    "        yield file.iloc[random_index-1]\n",
    "        \n",
    "    else:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T18:06:02.710476Z",
     "start_time": "2020-03-17T17:19:21.168697Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "num = (round (filtered_df.shape[0]*1.1))\n",
    "keyword = 'poverty'\n",
    "syphon = []\n",
    "\n",
    "while len(syphon) < num:\n",
    "    filename = f'pickles/jsons2/{random.choice(os.listdir(\"pickles/jsons2\"))}'\n",
    "    syphon.extend(random_list(filename))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T18:06:02.741728Z",
     "start_time": "2020-03-17T18:06:02.710476Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "list_for_balancing = []\n",
    "\n",
    "for gen_list in syphon:\n",
    "    list_for_balancing.append(gen_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T18:06:06.554514Z",
     "start_time": "2020-03-17T18:06:02.741728Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "balance_df = pd.DataFrame(list_for_balancing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T18:06:06.570141Z",
     "start_time": "2020-03-17T18:06:06.554514Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "balance_df.drop('date', inplace=True, axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### add label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T18:06:06.601395Z",
     "start_time": "2020-03-17T18:06:06.570141Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "balance_df['label']= 'unrelated'\n",
    "balance_df = balance_df[['text', 'label']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T18:06:06.788907Z",
     "start_time": "2020-03-17T18:06:06.601395Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# check to see if there are any duplicates in this section - make sure len (balance_df) is longer than len(filtered_df)\n",
    "balance_df.drop_duplicates(subset = 'text', inplace = True, keep = 'first')\n",
    "len(balance_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Concat related and unrelated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T18:06:06.851411Z",
     "start_time": "2020-03-17T18:06:06.788907Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "balanced_labeled_df = pd.concat([filtered_df, balance_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T18:06:06.929544Z",
     "start_time": "2020-03-17T18:06:06.851411Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# the missing id column is not an issue since they are not used for representation\n",
    "balanced_labeled_df = balanced_labeled_df.reset_index(drop = True)\n",
    "#check for duplicates again - keeping all 'first' (related) texts\n",
    "balanced_labeled_df.drop_duplicates(subset = 'text', inplace = True, keep = 'first')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T18:06:06.945170Z",
     "start_time": "2020-03-17T18:06:06.929544Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#prune the excess unrelated texts to make a 50:50 balanced dataset.\n",
    "balanced_labeled_df = balanced_labeled_df.iloc[:filtered_df.shape[0]*2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Save for portable version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T18:08:22.700502Z",
     "start_time": "2020-03-17T18:08:21.966082Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# change threshold value in name depending on threshold set (example here is threshhold 4)\n",
    "balanced_labeled_df.to_json('./saved_models/balanced_df_WITH_aid_with_slave_threshold_4_keep.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "#### potential memory wipe point"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Load portable version - of df to save time intensive problems above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T18:09:39.836515Z",
     "start_time": "2020-03-17T18:09:39.758394Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "balanced_labeled_df = pd.read_json('./saved_models/balanced_df_WITH_aid_with_slave_threshold_4_keep.json', encoding = 'utf-8')\n",
    "train_y = list(balanced_labeled_df.label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# BERT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Train - BERT \n",
    "\n",
    "### not necessary if you have pickled embeddings already - this step is performed below \n",
    "\n",
    "if using previously saved embeddings, do make sure the embeddings you have match the dataframe loaded above. If you re-populate the dataframe, you have to re-encode the embeddings. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### load model and encode sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T18:09:56.415885Z",
     "start_time": "2020-03-17T18:09:46.086985Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "model = SentenceTransformer ('roberta-large-nli-mean-tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T18:09:56.431512Z",
     "start_time": "2020-03-17T18:09:56.415885Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "texts = balanced_labeled_df.text.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T18:19:19.619441Z",
     "start_time": "2020-03-17T18:09:56.431512Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "embedded_texts = []\n",
    "for text in texts:\n",
    "    sentences = nltk.sent_tokenize(text)\n",
    "    if len(sentences) >=6:\n",
    "        sentence_embeddings = model.encode(sentences[:6])\n",
    "    else:\n",
    "        for n in range (6-len(sentences)):\n",
    "            sentences.append(sentences[0])\n",
    "            sentence_embeddings = model.encode(sentences[:6])\n",
    "    embedded_texts.append(sentence_embeddings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Concatenate embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T18:20:01.248480Z",
     "start_time": "2020-03-17T18:20:01.139108Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# concatenate the embeddings to get 1 vector per document\n",
    "train_X = []\n",
    "\n",
    "for six_embeds in embedded_texts:\n",
    "    new = np.concatenate(six_embeds)\n",
    "    train_X.append(new)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### save for portable version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-17T18:20:19.546766Z",
     "start_time": "2020-03-17T18:20:14.202615Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# once the above is done, dumping the embeds to memory here means you can skip the above. \n",
    "pickle.dump(concatenated_X, open('./saved_models/balanced_df_WITH_aid_with_slave_threshold_4_keep_RoBERTa.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# pre-loaded data - start here!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### potential memory wipe point\n",
    "feel free to purge your overburdened memory here. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:47:12.923797Z",
     "start_time": "2020-03-19T09:47:12.648554Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "train_X = pickle.load(open('./saved_models/balanced_df_WITH_aid_with_slave_threshold_4_keep_RoBERTa.pkl', 'rb'))\n",
    "\n",
    "balanced_labeled_df = pd.read_json('./saved_models/balanced_df_WITH_aid_with_slave_threshold_4_keep.json', encoding = 'utf-8')\n",
    "train_y = list(balanced_labeled_df.label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load test data and encode sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:47:13.455322Z",
     "start_time": "2020-03-19T09:47:13.124332Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# sheet name 'Related_SDG1' is both costa and starbucks aggregated data\n",
    "dfs = pd.read_excel('./classify_only/combined_testset.xlsx', sheet_name='Related_SDG1')\n",
    "dfs['text']=dfs['Headline'].astype(str)+' '+dfs['First 5 sentences']\n",
    "dfs = dfs[['text', 'label']]\n",
    "test_text = dfs['text'].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:47:23.368099Z",
     "start_time": "2020-03-19T09:47:13.465355Z"
    }
   },
   "outputs": [],
   "source": [
    "model = SentenceTransformer ('roberta-large-nli-mean-tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:47:30.695600Z",
     "start_time": "2020-03-19T09:47:23.370118Z"
    }
   },
   "outputs": [],
   "source": [
    "# # use same model as previously - RoBERTa\n",
    "embedded_test = []\n",
    "for text in test_text:\n",
    "    sentences = nltk.sent_tokenize(text)\n",
    "    if len(sentences) >=6:\n",
    "        sentence_embeddings = model.encode(sentences[:6])\n",
    "    else:\n",
    "        for n in range (6-len(sentences)):\n",
    "            sentences.append(sentences[0])\n",
    "            sentence_embeddings = model.encode(sentences[:6])\n",
    "    embedded_test.append(sentence_embeddings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### concatenate test embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:47:30.715744Z",
     "start_time": "2020-03-19T09:47:30.695699Z"
    }
   },
   "outputs": [],
   "source": [
    "BERT_test_X = []\n",
    "\n",
    "for six_embeds in embedded_test:\n",
    "    new = np.concatenate(six_embeds)\n",
    "    BERT_test_X.append(new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:47:30.731374Z",
     "start_time": "2020-03-19T09:47:30.715744Z"
    }
   },
   "outputs": [],
   "source": [
    "BERT_test_y = dfs['label'].to_list()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVM Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:47:49.621693Z",
     "start_time": "2020-03-19T09:47:30.731374Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\python\\lib\\site-packages\\sklearn\\svm\\base.py:929: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     related       0.88      0.90      0.89        41\n",
      "   unrelated       0.82      0.78      0.80        23\n",
      "\n",
      "    accuracy                           0.86        64\n",
      "   macro avg       0.85      0.84      0.85        64\n",
      "weighted avg       0.86      0.86      0.86        64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "BERT_classifier = LinearSVC(random_state=0, tol=1e-5)\n",
    "BERT_classifier.fit(train_X, train_y)\n",
    "SVM_predictions = list(BERT_classifier.predict(BERT_test_X))\n",
    "predicted_test_scores= BERT_classifier.decision_function(BERT_test_X) \n",
    "print(classification_report(BERT_test_y, SVM_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:47:49.641795Z",
     "start_time": "2020-03-19T09:47:49.623711Z"
    }
   },
   "outputs": [],
   "source": [
    "predicted_test_scores = np.round(abs(predicted_test_scores),3).tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Multi-level Perceptron Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:48:57.587745Z",
     "start_time": "2020-03-19T09:47:49.643812Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     related       0.90      0.93      0.92        41\n",
      "   unrelated       0.86      0.83      0.84        23\n",
      "\n",
      "    accuracy                           0.89        64\n",
      "   macro avg       0.88      0.88      0.88        64\n",
      "weighted avg       0.89      0.89      0.89        64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "MLPclf = MLPClassifier(solver='sgd', alpha=1e-5,hidden_layer_sizes=(15,), random_state=1, max_iter=500)\n",
    "MLPclf.fit(train_X, train_y)\n",
    "predictionsMLP=MLPclf.predict(BERT_test_X)\n",
    "print(classification_report(BERT_test_y, predictionsMLP))\n",
    "predicted_NN_scores= MLPclf.predict_proba(BERT_test_X) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:48:57.603371Z",
     "start_time": "2020-03-19T09:48:57.587745Z"
    }
   },
   "outputs": [],
   "source": [
    "predicted_NN_scores_percent = [round(max(score),3) for score in predicted_NN_scores]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Baseline System"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## prepare BOW model from training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:48:57.618997Z",
     "start_time": "2020-03-19T09:48:57.603371Z"
    }
   },
   "outputs": [],
   "source": [
    "texts = balanced_labeled_df.text.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:49:00.234802Z",
     "start_time": "2020-03-19T09:48:57.618997Z"
    }
   },
   "outputs": [],
   "source": [
    "# cleaning the texts for lower case words only\n",
    "BOW_texts_list = []\n",
    "\n",
    "for text in texts:\n",
    "    sentences = nltk.sent_tokenize(text)\n",
    "    \n",
    "    new_text =[]\n",
    "    if len(sentences) >=6:\n",
    "        for sent in sentences [:6]:\n",
    "            sent = sent.lower()\n",
    "            sent = re.sub(r'\\W',' ',sent)\n",
    "            sent = re.sub(r'\\s+',' ',sent)\n",
    "            new_text.append(sent)\n",
    "            \n",
    "    else:\n",
    "        for sent in sentences:\n",
    "            sent = sent.lower()\n",
    "            sent = re.sub(r'\\W',' ',sent)\n",
    "            sent = re.sub(r'\\s+',' ',sent)\n",
    "            new_text.append(sent)\n",
    "    BOW_texts_list.append(new_text)\n",
    "\n",
    "# each text is a single string after this ( i feel this is an inefficient way to do this, but it works)\n",
    "concatenated_sents = [''.join(item)for item in BOW_texts_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:49:00.840658Z",
     "start_time": "2020-03-19T09:49:00.234802Z"
    }
   },
   "outputs": [],
   "source": [
    "BOW_train_y = list(balanced_labeled_df.label)\n",
    "count_vec = CountVectorizer(stop_words=stopwords.words('english')) \n",
    "# this is the model\n",
    "BOW_model = count_vec.fit_transform(concatenated_sents)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classify - BoW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:49:01.356310Z",
     "start_time": "2020-03-19T09:49:00.840658Z"
    }
   },
   "outputs": [],
   "source": [
    "dfs = pd.read_excel('./classify_only/combined_testset.xlsx', sheet_name='Related_SDG1')\n",
    "dfs['text']=dfs['Headline'].astype(str)+' '+dfs['First 5 sentences']\n",
    "dfs = dfs[['text', 'label']]\n",
    "test_text = dfs['text'].to_list()\n",
    "test_y = dfs['label'].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:49:01.376767Z",
     "start_time": "2020-03-19T09:49:01.356310Z"
    }
   },
   "outputs": [],
   "source": [
    "#this vector transforms the texts to fit the BOW model (put test texts in to the function)\n",
    "test_X = count_vec.transform(test_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:49:01.891092Z",
     "start_time": "2020-03-19T09:49:01.376767Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\python\\lib\\site-packages\\sklearn\\svm\\base.py:929: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     related       1.00      0.61      0.76        41\n",
      "   unrelated       0.59      1.00      0.74        23\n",
      "\n",
      "    accuracy                           0.75        64\n",
      "   macro avg       0.79      0.80      0.75        64\n",
      "weighted avg       0.85      0.75      0.75        64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "BOW_classifier = LinearSVC(random_state=0, tol=1e-5)\n",
    "BOW_classifier.fit(BOW_model,BOW_train_y)\n",
    "predicted_label = BOW_classifier.predict(test_X)\n",
    "print(classification_report(test_y, predicted_label))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# output results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-19T09:52:05.710957Z",
     "start_time": "2020-03-19T09:52:05.695317Z"
    }
   },
   "outputs": [],
   "source": [
    "dfs['SVM_predictions']=SVM_predictions\n",
    "dfs['SVM_confidence'] = predicted_test_scores\n",
    "dfs['NN_predictions'] = predictionsMLP\n",
    "dfs['NN_confidence'] = predicted_NN_scores_percent\n",
    "dfs['Baseline_predictions']=predicted_label\n",
    "dfs.to_csv('results.tsv', sep='\\t')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}