tweets - collect - EDA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import LatentDirichletAllocation, NMF\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stops =  set(stopwords.words('english')+['com'])\n",
    "co = CountVectorizer(stop_words=stops)\n",
    "counts = co.fit_transform(data.Tweet_Text)\n",
    "pd.DataFrame(counts.sum(axis=0),columns=co.get_feature_names()).T.sort_values(0,ascending=False).head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "co = CountVectorizer(ngram_range=(2,2),stop_words=stops)\n",
    "counts = co.fit_transform(data.Tweet_Text)\n",
    "pd.DataFrame(counts.sum(axis=0),columns=co.get_feature_names()).T.sort_values(0,ascending=False).head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = pd.read_table('https://raw.githubusercontent.com/dwyl/english-words/master/words.txt')\n",
    "\n",
    "words.columns=['word']\n",
    "words = words['word'].str.lower().values.tolist()\n",
    "\n",
    "data['clean_text'] = data.Tweet_Text.apply(lambda x: ' '.join([i for i in x.split(' ') if not (i.startswith('@') or i.startswith('#'))]))\n",
    "data['clean_text'] = data.clean_text.str.lower().str.replace('[^a-zA]', ' ')\n",
    "\n",
    "non_list = {}\n",
    "\n",
    "for sent in tqdm(data.clean_text.str.split().values):\n",
    "    for token in sent:\n",
    "        if token not in words:\n",
    "            non_list[token] = 1 if token not in non_list else non_list[token]+1\n",
    "            \n",
    "pd.Series(non_list).sort_values(ascending=False).head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.Tweet_Text.str.extractall(r'(\\#\\w+)')[0].value_counts().head(20).plot.bar()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "vectorizer = CountVectorizer(stop_words=stops)\n",
    "model = vectorizer.fit(data.clean_text)\n",
    "docs = vectorizer.transform(data.clean_text)\n",
    "lda = LatentDirichletAllocation(20)\n",
    "lda.fit(docs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_top_words(model, feature_names, n_top_words):\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "        message = \"Topic #%d: \" % topic_idx\n",
    "        message += \" \".join([(feature_names[i])\n",
    "                             for i in topic.argsort()[:-n_top_words - 1:-1]])\n",
    "        print(message)\n",
    "    print()\n",
    "    \n",
    "print_top_words(lda,vectorizer.get_feature_names(),10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['topic']=lda.transform(docs).argmax(axis=1)\n",
    "data.topic.value_counts(normalize=True).plot.bar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[data.topic==0].Tweet_Text.sample(5).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['Date']=pd.to_datetime(data.Date,yearfirst=True)\n",
    "data.index = data.Date\n",
    "for i in range(10):\n",
    "    temp = data[data.topic==i]\n",
    "    temp.resample('7D').size().plot()\n",
    "    plt.title('topic %s' %i)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf-gpu-2.0",
   "language": "python",
   "name": "tf_gpu2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}