preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.26.7) or chardet (3.0.4) doesn't match a supported version!\n",
      "  RequestsDependencyWarning)\n"
     ]
    }
   ],
   "source": [
    "## NLP library\n",
    "import re\n",
    "import string\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "## ML Library\n",
    "from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer, TfidfTransformer\n",
    "from sklearn.model_selection import RepeatedStratifiedKFold,cross_val_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pickle\n",
    "\n",
    "import orchest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = orchest.get_inputs()\n",
    "train, test = data[\"data\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    Our Deeds are the Reason of this #earthquake M...\n",
       "1               Forest fire near La Ronge Sask. Canada\n",
       "2    All residents asked to 'shelter in place' are ...\n",
       "3    13,000 people receive #wildfires evacuation or...\n",
       "4    Just got sent this photo from Ruby #Alaska as ...\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.text.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_processing(data):\n",
    "    # lowering the text\n",
    "    data.text=data.text.apply(lambda x:x.lower() )\n",
    "    #removing square brackets\n",
    "    data.text=data.text.apply(lambda x:re.sub('\\[.*?\\]', '', x) )\n",
    "    data.text=data.text.apply(lambda x:re.sub('<.*?>+', '', x) )\n",
    "    #removing hyperlink\n",
    "    data.text=data.text.apply(lambda x:re.sub('https?://\\S+|www\\.\\S+', '', x) )\n",
    "    #removing puncuation\n",
    "    data.text=data.text.apply(lambda x:re.sub(\n",
    "               '[%s]' % re.escape(string.punctuation), '', x\n",
    "                                                ))\n",
    "    data.text=data.text.apply(lambda x:re.sub('\\n' , '', x) )\n",
    "    #remove words containing numbers\n",
    "    data.text=data.text.apply(lambda x:re.sub('\\w*\\d\\w*' , '', x) )\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    our deeds are the reason of this earthquake ma...\n",
       "1                forest fire near la ronge sask canada\n",
       "2    all residents asked to shelter in place are be...\n",
       "3     people receive wildfires evacuation orders in...\n",
       "4    just got sent this photo from ruby alaska as s...\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = text_processing(train)\n",
    "test = text_processing(test)\n",
    "train.text.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenization\n",
    "Tokenization is the process of tokenizing or splitting a string, text into a list of tokens. One can think of token as parts like a word is a token in a sentence, and a sentence is a token in a paragraph."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    [our, deeds, are, the, reason, of, this, earth...\n",
       "1        [forest, fire, near, la, ronge, sask, canada]\n",
       "2    [all, residents, asked, to, shelter, in, place...\n",
       "3    [people, receive, wildfires, evacuation, order...\n",
       "4    [just, got, sent, this, photo, from, ruby, ala...\n",
       "Name: text, dtype: object"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Tokenizer\n",
    "token=nltk.tokenize.RegexpTokenizer(r'\\w+')\n",
    "#applying token\n",
    "train.text=train.text.apply(lambda x:token.tokenize(x))\n",
    "test.text=test.text.apply(lambda x:token.tokenize(x))\n",
    "#view\n",
    "display(train.text.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## removing stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/jovyan/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    [deeds, reason, earthquake, may, allah, forgiv...\n",
       "1        [forest, fire, near, la, ronge, sask, canada]\n",
       "2    [residents, asked, shelter, place, notified, o...\n",
       "3    [people, receive, wildfires, evacuation, order...\n",
       "4    [got, sent, photo, ruby, alaska, smoke, wildfi...\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('stopwords')\n",
    "#removing stop words\n",
    "train.text=train.text.apply(lambda x:[w for w in x if w not in stopwords.words('english')])\n",
    "test.text=test.text.apply(lambda x:[w for w in x if w not in stopwords.words('english')])\n",
    "#view\n",
    "train.text.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stemming\n",
    "Stemming and Lemmatization in Python NLTK are text normalization techniques for Natural Language Processing. These techniques are widely used for text preprocessing. The difference between stemming and lemmatization is that stemming is faster as it cuts words without knowing the context, while lemmatization is slower as it knows the context of words before processing.\n",
    "\n",
    "**In this case PoerterStemmer performed well then lemmatization**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0            deed reason earthquak may allah forgiv us\n",
       "1                 forest fire near la rong sask canada\n",
       "2    resid ask shelter place notifi offic evacu she...\n",
       "3          peopl receiv wildfir evacu order california\n",
       "4    got sent photo rubi alaska smoke wildfir pour ...\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#stemmering the text and joining\n",
    "stemmer = nltk.stem.PorterStemmer()\n",
    "train.text=train.text.apply(lambda x:\" \".join(stemmer.stem(token) for token in x))\n",
    "test.text=test.text.apply(lambda x:\" \".join(stemmer.stem(token) for token in x))\n",
    "#View\n",
    "train.text.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text Vectorization\n",
    "Machine learning algorithms most often take numeric feature vectors as input. Thus, when working with text documents, we need a way to convert each document into a numeric vector.\n",
    "\n",
    "**In this case Countvectorizer is best performing.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer()\n",
    "train_vectors_count = vectorizer.fit_transform(train['text'])\n",
    "test_vectors_count = vectorizer.transform(test[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(train_vectors_count, train[\"target\"], test_size=0.2, random_state=40,stratify=train[\"target\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "orchest.output((X_train, y_train,X_val, y_val,test_vectors_count), name='train_val_test')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "orchest-kernel-29cddbd9-ee64-4453-9b86-1fbfff20b541"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}