SentimentAnalysis/jupyternotebook/preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "from nltk.tokenize import WordPunctTokenizer\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "from replacers import RegexReplacer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# init function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove HTML tag pattern\n",
    "rc = re.compile(r\"\\<.*?\\>\")  \n",
    "# Replacer class\n",
    "replacer = RegexReplacer()\n",
    "# split sentence into word\n",
    "pattern = r'[\\d.,]+|[A-Z][.A-Z]+\\b\\.*|\\w+|\\S'\n",
    "tokenizer = RegexpTokenizer(pattern)\n",
    "# Lemmatizer\n",
    "wordnet_lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Row data path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [],
   "source": [
    "LabelTrainDataPath = \"RowData\\\\labeledTrainData.tsv\"\n",
    "unLabelTrainDataPath = \"RowData\\\\unlabeledTrainData.tsv\"\n",
    "testDataPath = \"RowData\\\\testData.tsv\"\n",
    "LabelTrainDataFrame = pd.read_csv(LabelTrainDataPath, sep='\\t',quoting=3).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>\"5814_8\"</td>\n",
       "      <td>1</td>\n",
       "      <td>\"With all this stuff going down at the moment ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"2381_9\"</td>\n",
       "      <td>1</td>\n",
       "      <td>\"\\\"The Classic War of the Worlds\\\" by Timothy ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\"7759_3\"</td>\n",
       "      <td>0</td>\n",
       "      <td>\"The film starts with a manager (Nicholas Bell...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\"3630_4\"</td>\n",
       "      <td>0</td>\n",
       "      <td>\"It must be assumed that those who praised thi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\"9495_8\"</td>\n",
       "      <td>1</td>\n",
       "      <td>\"Superbly trashy and wondrously unpretentious ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>\"8196_8\"</td>\n",
       "      <td>1</td>\n",
       "      <td>\"I dont know why people think this is such a b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>\"7166_2\"</td>\n",
       "      <td>0</td>\n",
       "      <td>\"This movie could have been very good, but com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>\"10633_1\"</td>\n",
       "      <td>0</td>\n",
       "      <td>\"I watched this video at a friend's house. I'm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>\"319_1\"</td>\n",
       "      <td>0</td>\n",
       "      <td>\"A friend of mine bought this film for £1, and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>\"8713_10\"</td>\n",
       "      <td>1</td>\n",
       "      <td>\"&lt;br /&gt;&lt;br /&gt;This movie is full of references....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  sentiment                                             review\n",
       "0   \"5814_8\"          1  \"With all this stuff going down at the moment ...\n",
       "1   \"2381_9\"          1  \"\\\"The Classic War of the Worlds\\\" by Timothy ...\n",
       "2   \"7759_3\"          0  \"The film starts with a manager (Nicholas Bell...\n",
       "3   \"3630_4\"          0  \"It must be assumed that those who praised thi...\n",
       "4   \"9495_8\"          1  \"Superbly trashy and wondrously unpretentious ...\n",
       "5   \"8196_8\"          1  \"I dont know why people think this is such a b...\n",
       "6   \"7166_2\"          0  \"This movie could have been very good, but com...\n",
       "7  \"10633_1\"          0  \"I watched this video at a friend's house. I'm...\n",
       "8    \"319_1\"          0  \"A friend of mine bought this film for £1, and...\n",
       "9  \"8713_10\"          1  \"<br /><br />This movie is full of references...."
      ]
     },
     "execution_count": 305,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LabelTrainDataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# define apply function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SplitPhase(row):\n",
    "    \"\"\" split paragraph to sentence \"\"\"\n",
    "    return PunktTokenizer.tokenize(row['review'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [],
   "source": [
    "def RemoveHTML(row):\n",
    "    \"\"\" remove HTML tags \"\"\"\n",
    "    return [rc.sub('',sentence) for sentence in row['review']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ReplaceAbbre(row):\n",
    "    \"\"\" Replace abbreviation \"\"\"\n",
    "    return [replacer.replace(sentence) for sentence in row['review']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SplitSent(row):\n",
    "    \"\"\" split sentence to words \"\"\"\n",
    "    return [tokenizer.tokenize(sentence) for sentence in row['review']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## using WordNetLemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemma(tags):\n",
    "    WORD = []\n",
    "    for word, tag in tags:\n",
    "         wntag = tag[0].lower()\n",
    "         wntag = wntag if wntag in ['a', 'r', 'n', 'v','n','s'] else None\n",
    "         if not wntag:\n",
    "            lemma = word\n",
    "         else:\n",
    "            lemma = wordnet_lemmatizer.lemmatize(word, wntag)\n",
    "         WORD.append(lemma)\n",
    "    return WORD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Lemmatizer(row):\n",
    "    \"\"\" Lemmatizer words use WordNet \"\"\"\n",
    "    return [ lemma(nltk.pos_tag(sentence)) for sentence in row['review']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {},
   "outputs": [],
   "source": [
    "def CleanWords(sentence):\n",
    "    return [word.lower() for word in sentence if len(word)>=2 and (word.isalpha() or word.isdigit())]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## if using doc2vec model, does not remove stops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {},
   "outputs": [],
   "source": [
    "def CleanWords(self, sentence):\n",
    "        \"\"\" remove len < 3 and non alpha and lowercase \"\"\"\n",
    "        if self.word2vector:\n",
    "            return [word.lower() for word in sentence if (word.isalpha() and len(word) >= 2) or word.isdigit()]\n",
    "        else:\n",
    "            stops = set(stopwords.words(\"english\"))\n",
    "            return [word.lower() for word in sentence if len(word) >= 3 and word.isalpha() and not word in stops]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ToStr(row):\n",
    "    str=\"\"\n",
    "    for sentence in row['review']:\n",
    "        for word in sentence:\n",
    "            str += (word + \" \")\n",
    "    return str[:-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# apply to dataframe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## split into sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {},
   "outputs": [],
   "source": [
    "PunktTokenizer = nltk.tokenize.punkt.PunktSentenceTokenizer()\n",
    "LabelTrainDataFrame['review'] = LabelTrainDataFrame.apply(SplitPhase, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\"With all this stuff going down at the moment with MJ i\\'ve started listening to his music, watching the odd documentary here and there, watched The Wiz and watched Moonwalker again.', 'Maybe i just want to get a certain insight into this guy who i thought was really cool in the eighties just to maybe make up my mind whether he is guilty or innocent.', 'Moonwalker is part biography, part feature film which i remember going to see at the cinema when it was originally released.', \"Some of it has subtle messages about MJ's feeling towards the press and also the obvious message of drugs are bad m'kay.<br /><br />Visually impressive but of course this is all about Michael Jackson so unless you remotely like MJ in anyway then you are going to hate this and find it boring.\", 'Some may call MJ an egotist for consenting to the making of this movie BUT MJ and most of his fans would say that he made it for the fans which if true is really nice of him.<br /><br />The actual feature film bit when it finally starts is only on for 20 minutes or so excluding the Smooth Criminal sequence and Joe Pesci is convincing as a psychopathic all powerful drug lord.', 'Why he wants MJ dead so bad is beyond me.', 'Because MJ overheard his plans?', \"Nah, Joe Pesci's character ranted that he wanted people to know it is he who is supplying drugs etc so i dunno, maybe he just hates MJ's music.<br /><br />Lots of cool things in this like MJ turning into a car and a robot and the whole Speed Demon sequence.\", 'Also, the director must have had the patience of a saint when it came to filming the kiddy Bad sequence as usually directors hate working with one kid let alone a whole bunch of them performing a complex dance scene.<br /><br />Bottom line, this movie is for people who like MJ on one level or another (which i think is most people).', 'If not, then stay away.', \"It does try and give off a wholesome message and ironically MJ's bestest buddy in this movie is a girl!\", 'Michael Jackson is truly one of the most talented people ever to grace this planet but is he guilty?', \"Well, with all the attention i've gave this subject....hmmm well i don't know because people can be different behind closed doors, i know this for a fact.\", 'He is either an extremely nice but stupid guy or one of the most sickest liars.', 'I hope he is not the latter.\"']\n"
     ]
    }
   ],
   "source": [
    "for i in LabelTrainDataFrame['review']:\n",
    "    print(i)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove HTMLtags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {},
   "outputs": [],
   "source": [
    "LabelTrainDataFrame['review'] = LabelTrainDataFrame.apply(RemoveHTML, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\"With all this stuff going down at the moment with MJ i\\'ve started listening to his music, watching the odd documentary here and there, watched The Wiz and watched Moonwalker again.', 'Maybe i just want to get a certain insight into this guy who i thought was really cool in the eighties just to maybe make up my mind whether he is guilty or innocent.', 'Moonwalker is part biography, part feature film which i remember going to see at the cinema when it was originally released.', \"Some of it has subtle messages about MJ's feeling towards the press and also the obvious message of drugs are bad m'kay.Visually impressive but of course this is all about Michael Jackson so unless you remotely like MJ in anyway then you are going to hate this and find it boring.\", 'Some may call MJ an egotist for consenting to the making of this movie BUT MJ and most of his fans would say that he made it for the fans which if true is really nice of him.The actual feature film bit when it finally starts is only on for 20 minutes or so excluding the Smooth Criminal sequence and Joe Pesci is convincing as a psychopathic all powerful drug lord.', 'Why he wants MJ dead so bad is beyond me.', 'Because MJ overheard his plans?', \"Nah, Joe Pesci's character ranted that he wanted people to know it is he who is supplying drugs etc so i dunno, maybe he just hates MJ's music.Lots of cool things in this like MJ turning into a car and a robot and the whole Speed Demon sequence.\", 'Also, the director must have had the patience of a saint when it came to filming the kiddy Bad sequence as usually directors hate working with one kid let alone a whole bunch of them performing a complex dance scene.Bottom line, this movie is for people who like MJ on one level or another (which i think is most people).', 'If not, then stay away.', \"It does try and give off a wholesome message and ironically MJ's bestest buddy in this movie is a girl!\", 'Michael Jackson is truly one of the most talented people ever to grace this planet but is he guilty?', \"Well, with all the attention i've gave this subject....hmmm well i don't know because people can be different behind closed doors, i know this for a fact.\", 'He is either an extremely nice but stupid guy or one of the most sickest liars.', 'I hope he is not the latter.\"']\n"
     ]
    }
   ],
   "source": [
    "for i in LabelTrainDataFrame['review']:\n",
    "    print(i)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## replace abbreviation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [],
   "source": [
    "LabelTrainDataFrame['review'] = LabelTrainDataFrame.apply(ReplaceAbbre, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\"With all this stuff going down at the moment with MJ i have started listening to his music, watching the odd documentary here and there, watched The Wiz and watched Moonwalker again.', 'Maybe i just want to get a certain insight into this guy who i thought was really cool in the eighties just to maybe make up my mind whether he is guilty or innocent.', 'Moonwalker is part biography, part feature film which i remember going to see at the cinema when it was originally released.', \"Some of it has subtle messages about MJ is feeling towards the press and also the obvious message of drugs are bad m'kay.Visually impressive but of course this is all about Michael Jackson so unless you remotely like MJ in anyway then you are going to hate this and find it boring.\", 'Some may call MJ an egotist for consenting to the making of this movie BUT MJ and most of his fans would say that he made it for the fans which if true is really nice of him.The actual feature film bit when it finally starts is only on for 20 minutes or so excluding the Smooth Criminal sequence and Joe Pesci is convincing as a psychopathic all powerful drug lord.', 'Why he wants MJ dead so bad is beyond me.', 'Because MJ overheard his plans?', 'Nah, Joe Pesci is character ranted that he wanted people to know it is he who is supplying drugs etc so i dunno, maybe he just hates MJ is music.Lots of cool things in this like MJ turning into a car and a robot and the whole Speed Demon sequence.', 'Also, the director must have had the patience of a saint when it came to filming the kiddy Bad sequence as usually directors hate working with one kid let alone a whole bunch of them performing a complex dance scene.Bottom line, this movie is for people who like MJ on one level or another (which i think is most people).', 'If not, then stay away.', 'It does try and give off a wholesome message and ironically MJ is bestest buddy in this movie is a girl!', 'Michael Jackson is truly one of the most talented people ever to grace this planet but is he guilty?', 'Well, with all the attention i have gave this subject....hmmm well i do not know because people can be different behind closed doors, i know this for a fact.', 'He is either an extremely nice but stupid guy or one of the most sickest liars.', 'I hope he is not the latter.\"']\n"
     ]
    }
   ],
   "source": [
    "for i in LabelTrainDataFrame['review']:\n",
    "    print(i)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## split into words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {},
   "outputs": [],
   "source": [
    "LabelTrainDataFrame['review'] = LabelTrainDataFrame.apply(SplitSent, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['\"', 'With', 'all', 'this', 'stuff', 'going', 'down', 'at', 'the', 'moment', 'with', 'MJ', 'i', 'have', 'started', 'listening', 'to', 'his', 'music', ',', 'watching', 'the', 'odd', 'documentary', 'here', 'and', 'there', ',', 'watched', 'The', 'Wiz', 'and', 'watched', 'Moonwalker', 'again', '.'], ['Maybe', 'i', 'just', 'want', 'to', 'get', 'a', 'certain', 'insight', 'into', 'this', 'guy', 'who', 'i', 'thought', 'was', 'really', 'cool', 'in', 'the', 'eighties', 'just', 'to', 'maybe', 'make', 'up', 'my', 'mind', 'whether', 'he', 'is', 'guilty', 'or', 'innocent', '.'], ['Moonwalker', 'is', 'part', 'biography', ',', 'part', 'feature', 'film', 'which', 'i', 'remember', 'going', 'to', 'see', 'at', 'the', 'cinema', 'when', 'it', 'was', 'originally', 'released', '.'], ['Some', 'of', 'it', 'has', 'subtle', 'messages', 'about', 'MJ', 'is', 'feeling', 'towards', 'the', 'press', 'and', 'also', 'the', 'obvious', 'message', 'of', 'drugs', 'are', 'bad', 'm', \"'\", 'kay', '.', 'Visually', 'impressive', 'but', 'of', 'course', 'this', 'is', 'all', 'about', 'Michael', 'Jackson', 'so', 'unless', 'you', 'remotely', 'like', 'MJ', 'in', 'anyway', 'then', 'you', 'are', 'going', 'to', 'hate', 'this', 'and', 'find', 'it', 'boring', '.'], ['Some', 'may', 'call', 'MJ', 'an', 'egotist', 'for', 'consenting', 'to', 'the', 'making', 'of', 'this', 'movie', 'BUT', 'MJ', 'and', 'most', 'of', 'his', 'fans', 'would', 'say', 'that', 'he', 'made', 'it', 'for', 'the', 'fans', 'which', 'if', 'true', 'is', 'really', 'nice', 'of', 'him', '.', 'The', 'actual', 'feature', 'film', 'bit', 'when', 'it', 'finally', 'starts', 'is', 'only', 'on', 'for', '20', 'minutes', 'or', 'so', 'excluding', 'the', 'Smooth', 'Criminal', 'sequence', 'and', 'Joe', 'Pesci', 'is', 'convincing', 'as', 'a', 'psychopathic', 'all', 'powerful', 'drug', 'lord', '.'], ['Why', 'he', 'wants', 'MJ', 'dead', 'so', 'bad', 'is', 'beyond', 'me', '.'], ['Because', 'MJ', 'overheard', 'his', 'plans', '?'], ['Nah', ',', 'Joe', 'Pesci', 'is', 'character', 'ranted', 'that', 'he', 'wanted', 'people', 'to', 'know', 'it', 'is', 'he', 'who', 'is', 'supplying', 'drugs', 'etc', 'so', 'i', 'dunno', ',', 'maybe', 'he', 'just', 'hates', 'MJ', 'is', 'music', '.', 'Lots', 'of', 'cool', 'things', 'in', 'this', 'like', 'MJ', 'turning', 'into', 'a', 'car', 'and', 'a', 'robot', 'and', 'the', 'whole', 'Speed', 'Demon', 'sequence', '.'], ['Also', ',', 'the', 'director', 'must', 'have', 'had', 'the', 'patience', 'of', 'a', 'saint', 'when', 'it', 'came', 'to', 'filming', 'the', 'kiddy', 'Bad', 'sequence', 'as', 'usually', 'directors', 'hate', 'working', 'with', 'one', 'kid', 'let', 'alone', 'a', 'whole', 'bunch', 'of', 'them', 'performing', 'a', 'complex', 'dance', 'scene', '.', 'Bottom', 'line', ',', 'this', 'movie', 'is', 'for', 'people', 'who', 'like', 'MJ', 'on', 'one', 'level', 'or', 'another', '(', 'which', 'i', 'think', 'is', 'most', 'people', ')', '.'], ['If', 'not', ',', 'then', 'stay', 'away', '.'], ['It', 'does', 'try', 'and', 'give', 'off', 'a', 'wholesome', 'message', 'and', 'ironically', 'MJ', 'is', 'bestest', 'buddy', 'in', 'this', 'movie', 'is', 'a', 'girl', '!'], ['Michael', 'Jackson', 'is', 'truly', 'one', 'of', 'the', 'most', 'talented', 'people', 'ever', 'to', 'grace', 'this', 'planet', 'but', 'is', 'he', 'guilty', '?'], ['Well', ',', 'with', 'all', 'the', 'attention', 'i', 'have', 'gave', 'this', 'subject', '....', 'hmmm', 'well', 'i', 'do', 'not', 'know', 'because', 'people', 'can', 'be', 'different', 'behind', 'closed', 'doors', ',', 'i', 'know', 'this', 'for', 'a', 'fact', '.'], ['He', 'is', 'either', 'an', 'extremely', 'nice', 'but', 'stupid', 'guy', 'or', 'one', 'of', 'the', 'most', 'sickest', 'liars', '.'], ['I', 'hope', 'he', 'is', 'not', 'the', 'latter', '.', '\"']]\n"
     ]
    }
   ],
   "source": [
    "for i in LabelTrainDataFrame['review']:\n",
    "    print(i)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['\"', 'With', 'all', 'this', 'stuff', 'go', 'down', 'at', 'the', 'moment', 'with', 'MJ', 'i', 'have', 'start', 'listen', 'to', 'his', 'music', ',', 'watch', 'the', 'odd', 'documentary', 'here', 'and', 'there', ',', 'watch', 'The', 'Wiz', 'and', 'watch', 'Moonwalker', 'again', '.'], ['Maybe', 'i', 'just', 'want', 'to', 'get', 'a', 'certain', 'insight', 'into', 'this', 'guy', 'who', 'i', 'think', 'be', 'really', 'cool', 'in', 'the', 'eighty', 'just', 'to', 'maybe', 'make', 'up', 'my', 'mind', 'whether', 'he', 'be', 'guilty', 'or', 'innocent', '.'], ['Moonwalker', 'be', 'part', 'biography', ',', 'part', 'feature', 'film', 'which', 'i', 'remember', 'go', 'to', 'see', 'at', 'the', 'cinema', 'when', 'it', 'be', 'originally', 'release', '.'], ['Some', 'of', 'it', 'have', 'subtle', 'message', 'about', 'MJ', 'be', 'feel', 'towards', 'the', 'press', 'and', 'also', 'the', 'obvious', 'message', 'of', 'drug', 'be', 'bad', 'm', \"'\", 'kay', '.', 'Visually', 'impressive', 'but', 'of', 'course', 'this', 'be', 'all', 'about', 'Michael', 'Jackson', 'so', 'unless', 'you', 'remotely', 'like', 'MJ', 'in', 'anyway', 'then', 'you', 'be', 'go', 'to', 'hate', 'this', 'and', 'find', 'it', 'bore', '.'], ['Some', 'may', 'call', 'MJ', 'an', 'egotist', 'for', 'consent', 'to', 'the', 'making', 'of', 'this', 'movie', 'BUT', 'MJ', 'and', 'most', 'of', 'his', 'fan', 'would', 'say', 'that', 'he', 'make', 'it', 'for', 'the', 'fan', 'which', 'if', 'true', 'be', 'really', 'nice', 'of', 'him', '.', 'The', 'actual', 'feature', 'film', 'bit', 'when', 'it', 'finally', 'start', 'be', 'only', 'on', 'for', '20', 'minute', 'or', 'so', 'exclude', 'the', 'Smooth', 'Criminal', 'sequence', 'and', 'Joe', 'Pesci', 'be', 'convince', 'as', 'a', 'psychopathic', 'all', 'powerful', 'drug', 'lord', '.'], ['Why', 'he', 'want', 'MJ', 'dead', 'so', 'bad', 'be', 'beyond', 'me', '.'], ['Because', 'MJ', 'overhear', 'his', 'plan', '?'], ['Nah', ',', 'Joe', 'Pesci', 'be', 'character', 'rant', 'that', 'he', 'want', 'people', 'to', 'know', 'it', 'be', 'he', 'who', 'be', 'supply', 'drug', 'etc', 'so', 'i', 'dunno', ',', 'maybe', 'he', 'just', 'hat', 'MJ', 'be', 'music', '.', 'Lots', 'of', 'cool', 'thing', 'in', 'this', 'like', 'MJ', 'turn', 'into', 'a', 'car', 'and', 'a', 'robot', 'and', 'the', 'whole', 'Speed', 'Demon', 'sequence', '.'], ['Also', ',', 'the', 'director', 'must', 'have', 'have', 'the', 'patience', 'of', 'a', 'saint', 'when', 'it', 'come', 'to', 'film', 'the', 'kiddy', 'Bad', 'sequence', 'as', 'usually', 'director', 'hate', 'work', 'with', 'one', 'kid', 'let', 'alone', 'a', 'whole', 'bunch', 'of', 'them', 'perform', 'a', 'complex', 'dance', 'scene', '.', 'Bottom', 'line', ',', 'this', 'movie', 'be', 'for', 'people', 'who', 'like', 'MJ', 'on', 'one', 'level', 'or', 'another', '(', 'which', 'i', 'think', 'be', 'most', 'people', ')', '.'], ['If', 'not', ',', 'then', 'stay', 'away', '.'], ['It', 'do', 'try', 'and', 'give', 'off', 'a', 'wholesome', 'message', 'and', 'ironically', 'MJ', 'be', 'bestest', 'buddy', 'in', 'this', 'movie', 'be', 'a', 'girl', '!'], ['Michael', 'Jackson', 'be', 'truly', 'one', 'of', 'the', 'most', 'talented', 'people', 'ever', 'to', 'grace', 'this', 'planet', 'but', 'be', 'he', 'guilty', '?'], ['Well', ',', 'with', 'all', 'the', 'attention', 'i', 'have', 'give', 'this', 'subject', '....', 'hmmm', 'well', 'i', 'do', 'not', 'know', 'because', 'people', 'can', 'be', 'different', 'behind', 'closed', 'door', ',', 'i', 'know', 'this', 'for', 'a', 'fact', '.'], ['He', 'be', 'either', 'an', 'extremely', 'nice', 'but', 'stupid', 'guy', 'or', 'one', 'of', 'the', 'most', 'sickest', 'liar', '.'], ['I', 'hope', 'he', 'be', 'not', 'the', 'latter', '.', '\"']]\n"
     ]
    }
   ],
   "source": [
    "LabelTrainDataFrame['review'] = LabelTrainDataFrame.apply(Lemmatizer, axis=1)\n",
    "for i in LabelTrainDataFrame['review']:\n",
    "    print(i)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean words, if word2vector, not  remove stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['with', 'all', 'this', 'stuff', 'go', 'down', 'at', 'the', 'moment', 'with', 'mj', 'have', 'start', 'listen', 'to', 'his', 'music', 'watch', 'the', 'odd', 'documentary', 'here', 'and', 'there', 'watch', 'the', 'wiz', 'and', 'watch', 'moonwalker', 'again'], ['maybe', 'just', 'want', 'to', 'get', 'certain', 'insight', 'into', 'this', 'guy', 'who', 'think', 'be', 'really', 'cool', 'in', 'the', 'eighty', 'just', 'to', 'maybe', 'make', 'up', 'my', 'mind', 'whether', 'he', 'be', 'guilty', 'or', 'innocent'], ['moonwalker', 'be', 'part', 'biography', 'part', 'feature', 'film', 'which', 'remember', 'go', 'to', 'see', 'at', 'the', 'cinema', 'when', 'it', 'be', 'originally', 'release'], ['some', 'of', 'it', 'have', 'subtle', 'message', 'about', 'mj', 'be', 'feel', 'towards', 'the', 'press', 'and', 'also', 'the', 'obvious', 'message', 'of', 'drug', 'be', 'bad', 'kay', 'visually', 'impressive', 'but', 'of', 'course', 'this', 'be', 'all', 'about', 'michael', 'jackson', 'so', 'unless', 'you', 'remotely', 'like', 'mj', 'in', 'anyway', 'then', 'you', 'be', 'go', 'to', 'hate', 'this', 'and', 'find', 'it', 'bore'], ['some', 'may', 'call', 'mj', 'an', 'egotist', 'for', 'consent', 'to', 'the', 'making', 'of', 'this', 'movie', 'but', 'mj', 'and', 'most', 'of', 'his', 'fan', 'would', 'say', 'that', 'he', 'make', 'it', 'for', 'the', 'fan', 'which', 'if', 'true', 'be', 'really', 'nice', 'of', 'him', 'the', 'actual', 'feature', 'film', 'bit', 'when', 'it', 'finally', 'start', 'be', 'only', 'on', 'for', '20', 'minute', 'or', 'so', 'exclude', 'the', 'smooth', 'criminal', 'sequence', 'and', 'joe', 'pesci', 'be', 'convince', 'as', 'psychopathic', 'all', 'powerful', 'drug', 'lord'], ['why', 'he', 'want', 'mj', 'dead', 'so', 'bad', 'be', 'beyond', 'me'], ['because', 'mj', 'overhear', 'his', 'plan'], ['nah', 'joe', 'pesci', 'be', 'character', 'rant', 'that', 'he', 'want', 'people', 'to', 'know', 'it', 'be', 'he', 'who', 'be', 'supply', 'drug', 'etc', 'so', 'dunno', 'maybe', 'he', 'just', 'hat', 'mj', 'be', 'music', 'lots', 'of', 'cool', 'thing', 'in', 'this', 'like', 'mj', 'turn', 'into', 'car', 'and', 'robot', 'and', 'the', 'whole', 'speed', 'demon', 'sequence'], ['also', 'the', 'director', 'must', 'have', 'have', 'the', 'patience', 'of', 'saint', 'when', 'it', 'come', 'to', 'film', 'the', 'kiddy', 'bad', 'sequence', 'as', 'usually', 'director', 'hate', 'work', 'with', 'one', 'kid', 'let', 'alone', 'whole', 'bunch', 'of', 'them', 'perform', 'complex', 'dance', 'scene', 'bottom', 'line', 'this', 'movie', 'be', 'for', 'people', 'who', 'like', 'mj', 'on', 'one', 'level', 'or', 'another', 'which', 'think', 'be', 'most', 'people'], ['if', 'not', 'then', 'stay', 'away'], ['it', 'do', 'try', 'and', 'give', 'off', 'wholesome', 'message', 'and', 'ironically', 'mj', 'be', 'bestest', 'buddy', 'in', 'this', 'movie', 'be', 'girl'], ['michael', 'jackson', 'be', 'truly', 'one', 'of', 'the', 'most', 'talented', 'people', 'ever', 'to', 'grace', 'this', 'planet', 'but', 'be', 'he', 'guilty'], ['well', 'with', 'all', 'the', 'attention', 'have', 'give', 'this', 'subject', 'hmmm', 'well', 'do', 'not', 'know', 'because', 'people', 'can', 'be', 'different', 'behind', 'closed', 'door', 'know', 'this', 'for', 'fact'], ['he', 'be', 'either', 'an', 'extremely', 'nice', 'but', 'stupid', 'guy', 'or', 'one', 'of', 'the', 'most', 'sickest', 'liar'], ['hope', 'he', 'be', 'not', 'the', 'latter']]\n"
     ]
    }
   ],
   "source": [
    "LabelTrainDataFrame['review'] = LabelTrainDataFrame.apply(CleanSentences, axis=1)\n",
    "for i in LabelTrainDataFrame['review']:\n",
    "    print(i)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# convert to str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "with all this stuff go down at the moment with mj have start listen to his music watch the odd documentary here and there watch the wiz and watch moonwalker again maybe just want to get certain insight into this guy who think be really cool in the eighty just to maybe make up my mind whether he be guilty or innocent moonwalker be part biography part feature film which remember go to see at the cinema when it be originally release some of it have subtle message about mj be feel towards the press and also the obvious message of drug be bad kay visually impressive but of course this be all about michael jackson so unless you remotely like mj in anyway then you be go to hate this and find it bore some may call mj an egotist for consent to the making of this movie but mj and most of his fan would say that he make it for the fan which if true be really nice of him the actual feature film bit when it finally start be only on for 20 minute or so exclude the smooth criminal sequence and joe pesci be convince as psychopathic all powerful drug lord why he want mj dead so bad be beyond me because mj overhear his plan nah joe pesci be character rant that he want people to know it be he who be supply drug etc so dunno maybe he just hat mj be music lots of cool thing in this like mj turn into car and robot and the whole speed demon sequence also the director must have have the patience of saint when it come to film the kiddy bad sequence as usually director hate work with one kid let alone whole bunch of them perform complex dance scene bottom line this movie be for people who like mj on one level or another which think be most people if not then stay away it do try and give off wholesome message and ironically mj be bestest buddy in this movie be girl michael jackson be truly one of the most talented people ever to grace this planet but be he guilty well with all the attention have give this subject hmmm well do not know because people can be different behind closed door know this for fact he be either an extremely nice but stupid guy or one of the most sickest liar hope he be not the latter\n"
     ]
    }
   ],
   "source": [
    "LabelTrainDataFrame['review']=LabelTrainDataFrame.apply(ToStr, axis=1)\n",
    "for i in LabelTrainDataFrame['review']:\n",
    "    print(i)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## save to txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {},
   "outputs": [],
   "source": [
    "a.to_csv(\"test.txt\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [],
   "source": [
    "DataPath = \"test.txt\"\n",
    "DataFrame = pd.read_csv(DataPath,header=None)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}