pos_neg_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Lexicon - Dictionary of words\n",
    "- Bag of words - Every word has an ID (Index) (One-Hot array)\n",
    "- Here input vectors have to be of same length\n",
    "- Stemming - (Not necessarily a real Word) - Stemming handles matching “car” to “cars” .\n",
    "- Lematizing - (Is a Real Word) - Lemmatization handles matching “car” to “cars” along with matching “car” to “automobile”. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### NLP Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "import numpy as np\n",
    "import random\n",
    "import pickle\n",
    "from collections import Counter\n",
    "from nltk.stem import WordNetLemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmatizer = WordNetLemmatizer()\n",
    "hm_lines = 100000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Pre-Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_lexicon(pos,neg):\n",
    "\n",
    "    lexicon = []\n",
    "    with open(pos,'r') as f:\n",
    "        contents = f.readlines()\n",
    "        for l in contents[:hm_lines]:\n",
    "            all_words = word_tokenize(l)\n",
    "            lexicon += list(all_words)\n",
    "\n",
    "    with open(neg,'r') as f:\n",
    "        contents = f.readlines()\n",
    "        for l in contents[:hm_lines]:\n",
    "            all_words = word_tokenize(l)\n",
    "            lexicon += list(all_words)\n",
    "\n",
    "    lexicon = [lemmatizer.lemmatize(i) for i in lexicon]\n",
    "    w_counts = Counter(lexicon)\n",
    "    l2 = []\n",
    "    for w in w_counts:\n",
    "        #print(w_counts[w])\n",
    "        if 1000 > w_counts[w] > 50:\n",
    "            l2.append(w)\n",
    "    print(len(l2))\n",
    "    return l2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_handling(sample,lexicon,classification):\n",
    "\n",
    "    featureset = []\n",
    "\n",
    "    with open(sample,'r') as f:\n",
    "        contents = f.readlines()\n",
    "        for l in contents[:hm_lines]:\n",
    "            current_words = word_tokenize(l.lower())\n",
    "            current_words = [lemmatizer.lemmatize(i) for i in current_words]\n",
    "            features = np.zeros(len(lexicon))\n",
    "            for word in current_words:\n",
    "                if word.lower() in lexicon:\n",
    "                    index_value = lexicon.index(word.lower())\n",
    "                    features[index_value] += 1\n",
    "\n",
    "            features = list(features)\n",
    "            featureset.append([features,classification])\n",
    "\n",
    "    return featureset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train-Test-Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_feature_sets_and_labels(pos,neg,test_size = 0.1):\n",
    "    lexicon = create_lexicon(pos,neg)\n",
    "    features = []\n",
    "    features += sample_handling('pos.txt',lexicon,[1,0])\n",
    "    features += sample_handling('neg.txt',lexicon,[0,1])\n",
    "    random.shuffle(features)\n",
    "    features = np.array(features)\n",
    "\n",
    "    testing_size = int(test_size*len(features))\n",
    "\n",
    "    train_x = list(features[:,0][:-testing_size])\n",
    "    train_y = list(features[:,1][:-testing_size])\n",
    "    test_x = list(features[:,0][-testing_size:])\n",
    "    test_y = list(features[:,1][-testing_size:])\n",
    "\n",
    "    return train_x,train_y,test_x,test_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Return length of Lexicon"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If the Dataset is too large you can pickle it and store it so that you don't have to do it again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "423\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    train_x,train_y,test_x,test_y = create_feature_sets_and_labels('/Users/gaddamnitish/Desktop/TensorFlow/SentDex/pos.txt','/Users/gaddamnitish/Desktop/TensorFlow/SentDex/neg.txt')\n",
    "    # if you want to pickle this data:\n",
    "    with open('/Users/gaddamnitish/Desktop/TensorFlow/SentDex/sentiment_set.pickle','wb') as f:\n",
    "        pickle.dump([train_x,train_y,test_x,test_y],f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Basically means when we are sending Data to the Neural Network, the length of every string of ours is 423 characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}