notebooks/galina/count_words_and_chars_dirty.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import math\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "from sklearn import metrics\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer, WordNetLemmatizer\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.tokenize import word_tokenize\n",
    "import spacy\n",
    "from spacy.lang.en import English\n",
    "\n",
    "import nlp\n",
    "import string\n",
    "import re\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from string import punctuation as PUNCTS\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')\n",
    "nltk.download('wordnet')\n",
    "spacy.load('en_core_web_sm')\n",
    "nlp = English()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Guys, is there any way to shut up nltk lib?"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Before lematization"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "df = pd.read_csv('train_dup.csv').drop(columns='id', axis=1)\n",
    "df = df.dropna()\n",
    "df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "**Unique sentences df**"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "unique_set = pd.DataFrame(set(df.question1.append(df.question2).values),columns=['unique_sentence'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "unique_set"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def word_count_sentence(sentence):\n",
    "    \"\"\" The function takes a string, leave only words and ' ', split by ' ' and\n",
    "   return len of word_array\"\"\"\n",
    "\n",
    "    sentence = re.sub(\"[^A-Za-z]\", \" \", sentence)\n",
    "    sentence = sentence.split()\n",
    "    return len(sentence)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "**Adding column with len of sentence in words**"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "unique_set['sentence_len_word'] = unique_set.stack(). \\\n",
    "apply(lambda x: word_count_sentence(x)).reset_index(drop=True) "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "**Adding column with len of sentence in chars**"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "unique_set['sentence_len_char'] = unique_set.unique_sentence. \\\n",
    "apply(lambda x: len(x)).reset_index(drop=True) "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "**Here we can see the top20 longest sentences**"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "df_u_s = unique_set.sort_values('sentence_len_word', ascending=False).head(20)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "df_u_s"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "**Print top 20 longest sentences**"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for num,indx in enumerate(df_u_s.unique_sentence.index):\n",
    "    print(str(num)+'\\n==================================\\n'+df_u_s.unique_sentence[indx]+'\\n\\n')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "**Plot of Top 20 Longest Sentences By Words(with stop-words)**"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "x = np.linspace(df_u_s.sentence_len_word.mean(),df_u_s.sentence_len_word.mean(), 20)\n",
    "y = np.linspace(0, 20, 20)\n",
    "x_m = np.linspace(df_u_s.sentence_len_word.median(),df_u_s.sentence_len_word.median(), 20)\n",
    "\n",
    "fig, (ax1) = plt.subplots( figsize = (15,8))\n",
    "g=sns.barplot(x=df_u_s.sentence_len_word,y=df_u_s.unique_sentence, ax=ax1)\n",
    "for p in g.patches:\n",
    "    width = p.get_width()\n",
    "    g.text(width +5  ,\n",
    "            p.get_y()+ p.get_height()/2+0.2 ,\n",
    "            f'{int(width)}',\n",
    "            ha=\"center\", size=13)\n",
    "    \n",
    "plt.plot(x_m,y, color='g',linestyle='--',label='the median length of all sentences in words')\n",
    "plt.plot(x,y, color='r',linestyle='--',label='the mean length of all sentences in words')\n",
    "plt.title('Top 20 Longest Sentences By Words(with stop-words)', fontsize = 20)\n",
    "plt.yticks([])\n",
    "plt.xticks(np.arange(0,251,10), fontsize=12)\n",
    "plt.grid(axis='x',which='major',color='grey', alpha=0.2)\n",
    "plt.legend();"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "**Sort df by chars**"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "df_u_ch = unique_set.sort_values('sentence_len_char', ascending=False).head(20)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "**Plot of Top 20 Longest Sentences By Characters(with stop-words)**"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "Sort df by chars\n",
    "df_u_ch = unique_set.sort_values('sentence_len_char', ascending=False).head(20)\n",
    "Plot of Top 20 Longest Sentences By Characters(with stop-words)\n",
    "x = np.linspace(df_u_ch.sentence_len_char.mean(),df_u_ch.sentence_len_char.mean(), 20)\n",
    "y = np.linspace(0, 20, 20)\n",
    "x_m = np.linspace(df_u_ch.sentence_len_char.median(),df_u_ch.sentence_len_char.median(), 20)\n",
    "\n",
    "\n",
    "fig, (ax1) = plt.subplots( figsize = (15,8))\n",
    "g=sns.barplot(x=df_u_ch.sentence_len_char,y=df_u_ch.unique_sentence, ax=ax1)\n",
    "for p in g.patches:\n",
    "    width = p.get_width()\n",
    "    g.text(width +23  ,\n",
    "            p.get_y()+ p.get_height()/2+0.2 ,\n",
    "            f'{int(width)}',\n",
    "            ha=\"center\", size=13)\n",
    "plt.plot(x,y, color='r',linestyle='--',label='the mean length of all sentences in chars')\n",
    "plt.plot(x_m,y, color='g',linestyle='--',label='the median length of all sentences in chars')\n",
    "\n",
    "plt.title('Top 20 Longest Sentences By Chars (with stop-words)', fontsize = 20)\n",
    "plt.yticks([])\n",
    "plt.xticks(np.arange(0,1201,30), fontsize=8)\n",
    "plt.grid(axis='x',which='major',color='grey', alpha=0.2)\n",
    "plt.legend(loc=(0.6,0.01),fontsize=14);\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# After lematization"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "**Download the set (after lematization)**"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "df = pd.read_csv('preprocess_all_good_lm.csv').drop(columns=['id','Unnamed: 0'], axis=1)\n",
    "df = df.dropna()\n",
    "df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "**Unique sentences df (after lematization)**"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "unique_set_after = pd.DataFrame(set(df.preprocessed_q1.append(df.preprocessed_q2).values),\n",
    "                          columns=['unique_sentence'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "unique_set_after"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "**Adding column with len of sentence in words (after lematization)**\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "unique_set_after['sentence_len_word'] = unique_set_after.stack(). \\\n",
    "apply(lambda x: len(x.split())).reset_index(drop=True) "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "\n",
    "**Adding column with len of sentence in chars (after lematization)**\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "unique_set_after['sentence_len_char'] = unique_set_after.unique_sentence. \\\n",
    "apply(lambda x: len(x)).reset_index(drop=True) "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "**Here we can see the top20 longest sentences (after lematization)**\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "df_u_after = unique_set_after.sort_values('sentence_len_word', ascending=False).head(20)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "df_u_after"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "**Print top 20 longest sentences (after lematization)**\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "for num,indx in enumerate(df_u_after.unique_sentence.index):\n",
    "    print(str(num)+'\\n==================================\\n'+df_u_after.unique_sentence[indx]+'\\n\\n')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "**Plot of Top 20 Longest Sentences By Words(after lematization)**\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "x = np.linspace(df_u_after.sentence_len_word.mean(),df_u_after.sentence_len_word.mean(), 20)\n",
    "y = np.linspace(0, 20, 20)\n",
    "x_m = np.linspace(df_u_after.sentence_len_word.median(),df_u_after.sentence_len_word.median(), 20)\n",
    "\n",
    "fig, (ax1) = plt.subplots( figsize = (15,8))\n",
    "g=sns.barplot(x=df_u_after.sentence_len_word,y=df_u_after.unique_sentence, ax=ax1)\n",
    "for p in g.patches:\n",
    "    width = p.get_width()\n",
    "    g.text(width +2  ,\n",
    "            p.get_y()+ p.get_height()/2+0.2 ,\n",
    "            f'{int(width)}',\n",
    "            ha=\"center\", size=13)\n",
    "    \n",
    "plt.plot(x_m,y, color='g',linestyle='--',label='the median length of all sentences in words')\n",
    "plt.plot(x,y, color='r',linestyle='--',label='the mean length of all sentences in words')\n",
    "plt.title('Top 20 Longest Sentences By Words (after lematization)', fontsize = 20)\n",
    "plt.yticks([])\n",
    "plt.xticks(np.arange(0,110,5), fontsize=12)\n",
    "plt.grid(axis='x',which='major',color='grey', alpha=0.2)\n",
    "plt.legend(loc=(0.6,0.01),fontsize=14);"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "**Sort df by chars (after lematization)**\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "df_u_ch_after = unique_set_after.sort_values('sentence_len_char', ascending=False).head(20)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "df_u_ch_after"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "**Plot of Top 20 Longest Sentences By Characters (after lematization)**\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "x = np.linspace(df_u_ch_after.sentence_len_char.mean(),df_u_ch_after.sentence_len_char.mean(), 20)\n",
    "y = np.linspace(0, 20, 20)\n",
    "x_m = np.linspace(df_u_ch_after.sentence_len_char.median(),df_u_ch_after.sentence_len_char.median(), 20)\n",
    "\n",
    "\n",
    "fig, (ax1) = plt.subplots( figsize = (15,8))\n",
    "g=sns.barplot(x=df_u_ch_after.sentence_len_char,y=df_u_ch_after.unique_sentence, ax=ax1)\n",
    "for p in g.patches:\n",
    "    width = p.get_width()\n",
    "    g.text(width +12  ,\n",
    "            p.get_y()+ p.get_height()/2+0.2 ,\n",
    "            f'{int(width)}',\n",
    "            ha=\"center\", size=13)\n",
    "plt.plot(x,y, color='r',linestyle='--',label='the mean length of all sentences in chars')\n",
    "plt.plot(x_m,y, color='g',linestyle='--',label='the median length of all sentences in chars')\n",
    "\n",
    "plt.title('Top 20 Longest Sentences By Chars (after lematization)', fontsize = 20)\n",
    "plt.yticks([])\n",
    "plt.xticks(np.arange(0,641,20), fontsize=8)\n",
    "plt.grid(axis='x',which='major',color='grey', alpha=0.2)\n",
    "plt.legend(loc=(0.6,0.01),fontsize=14);"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}