07.3 Tweet Embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "import twitter\n",
    "import emoji\n",
    "import gensim\n",
    "import unicodedata\n",
    "import html\n",
    "from keras.preprocessing.text import text_to_word_sequence\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Fill these in!\n",
    "\n",
    "CONSUMER_KEY = 'xbMuxcJpRTiVGt2C2EYnA'\n",
    "CONSUMER_SECRET = '2DbQTsvIptkPTdaUcos8DDvQH9fzO0hNjJpUT2uVzQ'\n",
    "ACCESS_TOKEN = '7319442-EDm4CPxL7W4KkZcGWRMJNVHp88W5OH9vgblu898fg'\n",
    "ACCESS_SECRET = '5ZxJSbqXhG7uhgXzTFWf9XhkfsxxinlPRXyDTzbA9w'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['filter_level', 'in_reply_to_screen_name', 'user', 'coordinates', 'id_str', 'favorited', 'truncated', 'contributors', 'in_reply_to_status_id_str', 'source', 'retweet_count', 'in_reply_to_status_id', 'entities', 'extended_entities', 'lang', 'favorite_count', 'retweeted_status', 'reply_count', 'id', 'place', 'text', 'is_quote_status', 'quote_count', 'timestamp_ms', 'in_reply_to_user_id_str', 'geo', 'possibly_sensitive', 'created_at', 'retweeted', 'in_reply_to_user_id'])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auth=twitter.OAuth(\n",
    "    consumer_key=CONSUMER_KEY,\n",
    "    consumer_secret=CONSUMER_SECRET,\n",
    "    token=ACCESS_TOKEN,\n",
    "    token_secret=ACCESS_SECRET,\n",
    ")\n",
    "\n",
    "status_stream = twitter.TwitterStream(auth=auth).statuses\n",
    "next(status_stream.sample()).keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "!\n",
      "['narendramodi', 'best', 'wishes', 'to', 'our', 'former', 'prime', 'minister', 'shri', 'hd', 'deve', 'gowda', 'ji', 'on', 'his', 'birthday', 'i', 'pray', 'for', 'his', 'long', 'and', 'healthy', 'life']\n",
      "['ty', 'lightwood', 'aristotle', 'and', 'dante', 'discover', 'the', 'secrets', 'of', 'the', 'universe', '2019']\n",
      "['goaaaaaalllll', 'gabrieeeell', 'jesuss']\n"
     ]
    }
   ],
   "source": [
    "RE_URL = re.compile(r'(http|ftp|https)://([\\w_-]+(?:(?:\\.[\\w_-]+)+))([\\w.,@?^=%&:/~+#-]*[\\w@?^=%&/~+#-])?')\n",
    "RE_WHITESPACE = re.compile(r'\\s+')\n",
    "\n",
    "def strip_accents(s):\n",
    "     return ''.join(c for c in unicodedata.normalize('NFD', s)\n",
    "                    if unicodedata.category(c) != 'Mn')\n",
    "\n",
    "class TokensYielder(object):\n",
    "    def __init__(self, tweet_count, stream):\n",
    "        self.tweet_count = tweet_count\n",
    "        self.stream = stream\n",
    "\n",
    "    def __iter__(self):\n",
    "        count = self.tweet_count\n",
    "        for tweet in self.stream:\n",
    "            if tweet.get('lang') != 'en':\n",
    "                continue\n",
    "            text = tweet['text']\n",
    "            text = html.unescape(text)\n",
    "            text = RE_WHITESPACE.sub(' ', text)\n",
    "            text = RE_URL.sub(' ', text)\n",
    "            text = strip_accents(text)\n",
    "            text = ''.join(ch for ch in text if ord(ch) < 128)\n",
    "            if text.startswith('RT '):\n",
    "                text = text[3:]\n",
    "            text = text.strip()\n",
    "            if text:\n",
    "                yield text_to_word_sequence(text)\n",
    "                count -= 1\n",
    "                if count <= 0:\n",
    "                    break\n",
    "\n",
    "for t in TokensYielder(3, twitter.TwitterStream(auth=auth).statuses.sample()):\n",
    "    print(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "!\n"
     ]
    }
   ],
   "source": [
    "tweets = list(TokensYielder(70000, twitter.TwitterStream(auth=auth).statuses.sample()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = gensim.models.Word2Vec(tweets, \n",
    "                               workers=5,\n",
    "                               min_count=2,\n",
    "                              )\n",
    "model.save('zoo/07/twitter_stream_w2v.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/douwe/checkout/deep_learning_cookbook/venv3/lib/python3.5/site-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
      "  if np.issubdtype(vec.dtype, np.int):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('miss', 0.822679877281189),\n",
       " ('hope', 0.8068050146102905),\n",
       " ('loved', 0.8038904666900635),\n",
       " ('appreciate', 0.8034697771072388),\n",
       " ('ramblingsloa', 0.8009338974952698)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.most_similar(positive=['love'], topn=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}