Classifier_Builder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn import model_selection, preprocessing, linear_model, naive_bayes, metrics, svm\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn import decomposition, ensemble\n",
    "import pickle\n",
    "import pandas, numpy, string\n",
    "from keras.preprocessing import text, sequence\n",
    "from keras import layers, models, optimizers\n",
    "import re\n",
    "import nltk\n",
    "import csv\n",
    "from nltk.corpus import stopwords\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stopword_remover(text):\n",
    "    leng = len(texts)\n",
    "    \n",
    "    final_string = \"\"\n",
    "    all_sentences = nltk.sent_tokenize(processed_article)\n",
    "\n",
    "    all_words = [nltk.word_tokenize(sent) for sent in all_sentences]\n",
    "\n",
    "    # Removing Stop Words\n",
    "    from nltk.corpus import stopwords  \n",
    "    for i in range(len(all_words)):  \n",
    "        #print(all_words[i])\n",
    "        all_words[i] = [w for w in all_words[i] if w not in stopwords.words('english')]\n",
    "        final_string = final_string +\" \"+str(all_words[i])\n",
    "    print(all_words[i])    \n",
    "#     print(type(final_string))\n",
    "#     print(final_string)\n",
    "#     return final_string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import string\n",
    "\n",
    "labels, texts = [], []\n",
    "\n",
    "\n",
    "path = '/home/raj/UB_Stuff/CSE_635/Phase2/Invalid_Data/Windows/Windows_Invalid_json'\n",
    "for file in os.listdir(path):\n",
    "     xpath = os.path.join(path,file)\n",
    "     if xpath.endswith('.json'):\n",
    "         with open(xpath, \"r+\") as f:\n",
    "            data = json.load(f)\n",
    "\n",
    "         temp = data[\"text\"] \n",
    "         \n",
    "         \n",
    "         if temp is not None: \n",
    "             processed_article = temp.lower() \n",
    "             processed_article = re.sub('[^a-zA-Z]', ' ', processed_article )\n",
    "             processed_article = re.sub(r'\\s+', ' ', processed_article)\n",
    "             processed_article = ' '.join([word for word in processed_article.split() if word not in (stopwords.words('english'))])\n",
    "             texts.append(processed_article)\n",
    "             labels.append(data[\"label\"])\n",
    "                \n",
    "counter = 1\n",
    "with open('/home/raj/UB_Stuff/CSE_635/Final_Data/Thailand/Protest.csv','r') as csvinput:\n",
    "    reader = csv.reader(csvinput)\n",
    "    for row in reader:\n",
    "        try:\n",
    "            if(counter == 40) :\n",
    "                break\n",
    "            processed_article = row[27]\n",
    "            processed_article = re.sub(r'\\([^)]*\\)', '', processed_article)\n",
    "            processed_article = \" \".join(re.findall(r\"[.',a-zA-Z0-9]+\", processed_article))\n",
    "            processed_article = ' '.join([word for word in processed_article.split() if word not in (stopwords.words('english'))])\n",
    "            ##print(processed_article)\n",
    "            texts.append(processed_article)\n",
    "            counter+=1\n",
    "            labels.append(1)\n",
    "\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            pass\n",
    "\n",
    "counter = 1\n",
    "with open('/home/raj/UB_Stuff/CSE_635/Final_Data/Thailand/Violence.csv','r') as csvinput:\n",
    "    reader = csv.reader(csvinput)\n",
    "    for row in reader:\n",
    "        try:\n",
    "            if(counter == 40) :\n",
    "                break\n",
    "            processed_article = row[27]\n",
    "            processed_article = re.sub(r'\\([^)]*\\)', '', processed_article)\n",
    "            processed_article = \" \".join(re.findall(r\"[.',a-zA-Z0-9]+\", processed_article))\n",
    "            processed_article = ' '.join([word for word in processed_article.split() if word not in (stopwords.words('english'))])\n",
    "            #print(processed_article)\n",
    "            texts.append(processed_article)\n",
    "            counter+=1\n",
    "            labels.append(2)\n",
    "\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            pass\n",
    "        \n",
    "\n",
    "        \n",
    "# create a dataframe using texts and lables\n",
    "trainDF = pandas.DataFrame()\n",
    "trainDF['text'] = texts\n",
    "trainDF['label'] = labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split the dataset into training and validation datasets \n",
    "sentences_train, sentences_test, y_train, y_test = model_selection.train_test_split(trainDF['text'], trainDF['label'], test_size=0.33, random_state=42)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# ngram level tf-idf \n",
    "#tfidf_vect_ngram = TfidfVectorizer(analyzer='word', token_pattern=r'\\w{1,}', ngram_range=(2,3))\n",
    "filename = '/home/raj/UB_Stuff/CSE_635/Phase4/TFIDF_Vector.pickle'\n",
    "tfidf_vect_ngram = TfidfVectorizer(sublinear_tf=True, min_df=5, norm='l2', ngram_range=(1, 2), stop_words='english')\n",
    "tfidf_vect_ngram.fit(trainDF['text'])\n",
    "pickle.dump(tfidf_vect_ngram, open(filename, \"wb\"))\n",
    "\n",
    "X_train =  tfidf_vect_ngram.transform(sentences_train)\n",
    "X_test =  tfidf_vect_ngram.transform(sentences_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.941834451901566\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "classifier = LogisticRegression()\n",
    "classifier.fit(X_train, y_train)\n",
    "score = classifier.score(X_test, y_test)\n",
    "print(\"Accuracy:\", score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9440715883668904\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "clf = MultinomialNB().fit(X_train, y_train)\n",
    "score = clf.score(X_test, y_test)\n",
    "print(\"Accuracy:\", score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9955257270693513"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Import the Classifier.\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "## Instantiate the model with 5 neighbors. \n",
    "knn = KNeighborsClassifier(n_neighbors=3)\n",
    "## Fit the model on the training data.\n",
    "knn.fit(X_train, y_train)\n",
    "## See how the model performs on the test data.\n",
    "knn.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9955257270693513\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "# save the model to disk\n",
    "filename = '/home/raj/UB_Stuff/CSE_635/Phase4/phasr_4_model.sav'\n",
    "pickle.dump(knn, open(filename, 'wb'))\n",
    "\n",
    "# load the model from disk\n",
    "loaded_model = pickle.load(open(filename, 'rb'))\n",
    "result = loaded_model.score(X_test, y_test)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Arnold Schwarzenegger was part of a terrorist atttack on unarmed civilians that killed 10']\n",
      "[2]\n"
     ]
    }
   ],
   "source": [
    "input= \"Arnold Schwarzenegger was part of a terrorist atttack on unarmed civilians that killed 10\"\n",
    "input=[input]\n",
    "# create a dataframe using texts and lables\n",
    "#predictDF = pandas.DataFrame()\n",
    "something = tfidf_vect_ngram.transform(input)\n",
    "output = knn.predict(something)\n",
    "print(input)\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}