flock.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from random import random\n",
    "\n",
    "from gensim.models.doc2vec import LabeledSentence, Doc2Vec\n",
    "\n",
    "from sqlalchemy import create_engine, Column, Integer, String\n",
    "from sqlalchemy.ext.declarative import declarative_base\n",
    "from sqlalchemy.orm import sessionmaker\n",
    "from sqlalchemy.dialects import postgresql\n",
    "from  sqlalchemy.sql.expression import func\n",
    "\n",
    "from poultry import readline_dir\n",
    "from poultry import tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "logging.basicConfig(filename='flock.log', mode='w', level=logging.DEBUG)\n",
    "logger = logging.getLogger('flock')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "engine = create_engine('postgresql://dm303@localhost/sample')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Session = sessionmaker(bind=engine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "session = Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Base = declarative_base()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class Tweet(Base):\n",
    "    __tablename__ = 'tweets'\n",
    "    \n",
    "    id = Column(Integer, primary_key=True)\n",
    "    \n",
    "    raw_data = Column(postgresql.JSONB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Base.metadata.drop_all(engine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Base.metadata.create_all(engine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "for i, tweet in enumerate(readline_dir('/import/dima-scratch/flock/data/brexit.jul23-24')):\n",
    "\n",
    "    if (i  % 100000) == 1:\n",
    "        logger.debug('Processed %s tweets, it\\'s time to commit %s items.', i, len(session.new))\n",
    "        session.commit()\n",
    "\n",
    "    session.add(Tweet(raw_data=tweet.parsed))\n",
    "\n",
    "session.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3703196L"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "session.query(Tweet).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2459735L"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "session.query(Tweet).filter(Tweet.raw_data.contains({'lang': 'en'})).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "116503L"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "session.query(Tweet).filter(Tweet.raw_data.contains({'lang': 'it'})).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "session.query(Tweet).filter(Tweet.raw_data.contains({'lang': 'de'})).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "# Random selection of tweets.\n",
    "[t.raw_data['id'] for t in session.query(Tweet).order_by(func.random()).limit(10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "t.raw_data['id_str']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def labeled_sentences(tweets):\n",
    "    for tweet in tweets:\n",
    "        yield LabeledSentence(\n",
    "            words=tweet.raw_data['text'].split(),  # TODO: tokenise.\n",
    "            tags=[\n",
    "                u'id:{}'.format(tweet.raw_data['id_str']),\n",
    "                u'@{}'.format(tweet.raw_data['user']['screen_name']),\n",
    "            ]\n",
    "            + [u'#{}'.format(ht['text']) for ht in tweet.raw_data['entities']['hashtags']]\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# list(\n",
    "#     labeled_sentences(\n",
    "#         session.query(Tweet).order_by(func.random()).limit(10)\n",
    "#     )\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sentences():\n",
    "    return labeled_sentences(session.query(Tweet).order_by(func.random()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Doc2Vec(alpha=0.025, min_alpha=0.025, workers=64)  # use fixed learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "model.build_vocab(sentences())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "for epoch in range(10):\n",
    "    print('Epoch: {}'.format(epoch))\n",
    "    model.train(sentences())\n",
    "    model.alpha -= 0.002  # decrease the learning rate\n",
    "    model.min_alpha = model.alpha  # fix the learning rate, no decay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'Leave', 0.3594135344028473),\n",
       " (u'#leave?', 0.3591740131378174),\n",
       " (u'#Rexit', 0.34052973985671997),\n",
       " (u'https://t.co/Bdbs3ZfY9D', 0.3333190083503723),\n",
       " (u'#Remain', 0.33204835653305054),\n",
       " (u'#leave.', 0.3283088207244873),\n",
       " (u'#leave!', 0.3253001570701599),\n",
       " (u'#EuropeanUnion', 0.3220774233341217),\n",
       " (u'counter', 0.3179582953453064),\n",
       " (u'#Bremain', 0.30828213691711426)]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.most_similar('#leave')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'#UK', 0.4775325059890747),\n",
       " (u'#remain,', 0.4151751399040222),\n",
       " (u'Leave', 0.38699883222579956),\n",
       " (u'#remain.', 0.38196349143981934),\n",
       " (u'#EURef', 0.3802741765975952),\n",
       " (u'Remain', 0.355394184589386),\n",
       " (u'https://t.co/IQGSj1V3pU', 0.35470372438430786),\n",
       " (u'#Remain', 0.34487384557724),\n",
       " (u'#ShoutNews', 0.3439306616783142),\n",
       " (u'#canneslions', 0.3398371636867523)]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.most_similar('#remain')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'Boris', 0.3845505118370056),\n",
       " (u'https://t.co/IFdi5TuKqE', 0.3593224883079529),\n",
       " (u'POTUS', 0.3321494162082672),\n",
       " (u'he', 0.3153676688671112),\n",
       " (u'EWING', 0.306194543838501),\n",
       " (u'trump', 0.30460125207901),\n",
       " (u'@Nivo0o0:', 0.294739305973053),\n",
       " (u'naus\\xe9abond', 0.2929772138595581),\n",
       " (u'https://t.co/Z8hp1DQO2j', 0.2872544527053833),\n",
       " (u'she', 0.28533172607421875)]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.most_similar('boris')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'fire', 0.5007041692733765),\n",
       " (u'table', 0.39386045932769775),\n",
       " (u'sun', 0.3847666382789612),\n",
       " (u'air', 0.38156476616859436),\n",
       " (u'weekend', 0.37769317626953125),\n",
       " (u'kettle', 0.37703073024749756),\n",
       " (u'band', 0.3667536973953247),\n",
       " (u'island', 0.35691404342651367),\n",
       " (u'water', 0.35450342297554016),\n",
       " (u'bag', 0.35022270679473877)]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.most_similar('bus')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}