wrangle_act.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import requests as re\n",
    "import tweepy\n",
    "import json\n",
    "import io"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gathering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter_archive_df = pd.read_csv('twitter-archive-enhanced.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['tweet_id', 'in_reply_to_status_id', 'in_reply_to_user_id', 'timestamp',\n",
      "       'source', 'text', 'retweeted_status_id', 'retweeted_status_user_id',\n",
      "       'retweeted_status_timestamp', 'expanded_urls', 'rating_numerator',\n",
      "       'rating_denominator', 'name', 'doggo', 'floofer', 'pupper', 'puppo'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>retweeted_status_id</th>\n",
       "      <th>retweeted_status_user_id</th>\n",
       "      <th>retweeted_status_timestamp</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>name</th>\n",
       "      <th>doggo</th>\n",
       "      <th>floofer</th>\n",
       "      <th>pupper</th>\n",
       "      <th>puppo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892420643555336193</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-08-01 16:23:56 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Phineas. He's a mystical boy. Only eve...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/892420643...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>Phineas</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>892177421306343426</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-08-01 00:17:27 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Tilly. She's just checking pup on you....</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/892177421...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>Tilly</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>891815181378084864</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-31 00:18:03 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Archie. He is a rare Norwegian Pouncin...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891815181...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>Archie</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>891689557279858688</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-30 15:58:51 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Darla. She commenced a snooze mid meal...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891689557...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>Darla</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>891327558926688256</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-29 16:00:24 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Franklin. He would like you to stop ca...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891327558...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>Franklin</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id  in_reply_to_status_id  in_reply_to_user_id  \\\n",
       "0  892420643555336193                    NaN                  NaN   \n",
       "1  892177421306343426                    NaN                  NaN   \n",
       "2  891815181378084864                    NaN                  NaN   \n",
       "3  891689557279858688                    NaN                  NaN   \n",
       "4  891327558926688256                    NaN                  NaN   \n",
       "\n",
       "                   timestamp  \\\n",
       "0  2017-08-01 16:23:56 +0000   \n",
       "1  2017-08-01 00:17:27 +0000   \n",
       "2  2017-07-31 00:18:03 +0000   \n",
       "3  2017-07-30 15:58:51 +0000   \n",
       "4  2017-07-29 16:00:24 +0000   \n",
       "\n",
       "                                              source  \\\n",
       "0  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "2  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "3  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "4  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "\n",
       "                                                text  retweeted_status_id  \\\n",
       "0  This is Phineas. He's a mystical boy. Only eve...                  NaN   \n",
       "1  This is Tilly. She's just checking pup on you....                  NaN   \n",
       "2  This is Archie. He is a rare Norwegian Pouncin...                  NaN   \n",
       "3  This is Darla. She commenced a snooze mid meal...                  NaN   \n",
       "4  This is Franklin. He would like you to stop ca...                  NaN   \n",
       "\n",
       "   retweeted_status_user_id retweeted_status_timestamp  \\\n",
       "0                       NaN                        NaN   \n",
       "1                       NaN                        NaN   \n",
       "2                       NaN                        NaN   \n",
       "3                       NaN                        NaN   \n",
       "4                       NaN                        NaN   \n",
       "\n",
       "                                       expanded_urls  rating_numerator  \\\n",
       "0  https://twitter.com/dog_rates/status/892420643...                13   \n",
       "1  https://twitter.com/dog_rates/status/892177421...                13   \n",
       "2  https://twitter.com/dog_rates/status/891815181...                12   \n",
       "3  https://twitter.com/dog_rates/status/891689557...                13   \n",
       "4  https://twitter.com/dog_rates/status/891327558...                12   \n",
       "\n",
       "   rating_denominator      name doggo floofer pupper puppo  \n",
       "0                  10   Phineas  None    None   None  None  \n",
       "1                  10     Tilly  None    None   None  None  \n",
       "2                  10    Archie  None    None   None  None  \n",
       "3                  10     Darla  None    None   None  None  \n",
       "4                  10  Franklin  None    None   None  None  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(twitter_archive_df.columns)\n",
    "twitter_archive_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# connecting to image predictions tsv file via requests library\n",
    "image_predictions = re.get('https://d17h27t6h515a5.cloudfront.net/topher/2017/August/599fd2ad_image-predictions/image-predictions.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_predictions.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "rawData = image_predictions.content\n",
    "image_predictions_df = pd.read_csv(io.StringIO(rawData.decode('utf-8')), sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bytes"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(rawData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>jpg_url</th>\n",
       "      <th>img_num</th>\n",
       "      <th>p1</th>\n",
       "      <th>p1_conf</th>\n",
       "      <th>p1_dog</th>\n",
       "      <th>p2</th>\n",
       "      <th>p2_conf</th>\n",
       "      <th>p2_dog</th>\n",
       "      <th>p3</th>\n",
       "      <th>p3_conf</th>\n",
       "      <th>p3_dog</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>666020888022790149</td>\n",
       "      <td>https://pbs.twimg.com/media/CT4udn0WwAA0aMy.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>Welsh_springer_spaniel</td>\n",
       "      <td>0.465074</td>\n",
       "      <td>True</td>\n",
       "      <td>collie</td>\n",
       "      <td>0.156665</td>\n",
       "      <td>True</td>\n",
       "      <td>Shetland_sheepdog</td>\n",
       "      <td>0.061428</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>666029285002620928</td>\n",
       "      <td>https://pbs.twimg.com/media/CT42GRgUYAA5iDo.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>redbone</td>\n",
       "      <td>0.506826</td>\n",
       "      <td>True</td>\n",
       "      <td>miniature_pinscher</td>\n",
       "      <td>0.074192</td>\n",
       "      <td>True</td>\n",
       "      <td>Rhodesian_ridgeback</td>\n",
       "      <td>0.072010</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>666033412701032449</td>\n",
       "      <td>https://pbs.twimg.com/media/CT4521TWwAEvMyu.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>German_shepherd</td>\n",
       "      <td>0.596461</td>\n",
       "      <td>True</td>\n",
       "      <td>malinois</td>\n",
       "      <td>0.138584</td>\n",
       "      <td>True</td>\n",
       "      <td>bloodhound</td>\n",
       "      <td>0.116197</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>666044226329800704</td>\n",
       "      <td>https://pbs.twimg.com/media/CT5Dr8HUEAA-lEu.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>Rhodesian_ridgeback</td>\n",
       "      <td>0.408143</td>\n",
       "      <td>True</td>\n",
       "      <td>redbone</td>\n",
       "      <td>0.360687</td>\n",
       "      <td>True</td>\n",
       "      <td>miniature_pinscher</td>\n",
       "      <td>0.222752</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>666049248165822465</td>\n",
       "      <td>https://pbs.twimg.com/media/CT5IQmsXIAAKY4A.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>miniature_pinscher</td>\n",
       "      <td>0.560311</td>\n",
       "      <td>True</td>\n",
       "      <td>Rottweiler</td>\n",
       "      <td>0.243682</td>\n",
       "      <td>True</td>\n",
       "      <td>Doberman</td>\n",
       "      <td>0.154629</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id                                          jpg_url  \\\n",
       "0  666020888022790149  https://pbs.twimg.com/media/CT4udn0WwAA0aMy.jpg   \n",
       "1  666029285002620928  https://pbs.twimg.com/media/CT42GRgUYAA5iDo.jpg   \n",
       "2  666033412701032449  https://pbs.twimg.com/media/CT4521TWwAEvMyu.jpg   \n",
       "3  666044226329800704  https://pbs.twimg.com/media/CT5Dr8HUEAA-lEu.jpg   \n",
       "4  666049248165822465  https://pbs.twimg.com/media/CT5IQmsXIAAKY4A.jpg   \n",
       "\n",
       "   img_num                      p1   p1_conf  p1_dog                  p2  \\\n",
       "0        1  Welsh_springer_spaniel  0.465074    True              collie   \n",
       "1        1                 redbone  0.506826    True  miniature_pinscher   \n",
       "2        1         German_shepherd  0.596461    True            malinois   \n",
       "3        1     Rhodesian_ridgeback  0.408143    True             redbone   \n",
       "4        1      miniature_pinscher  0.560311    True          Rottweiler   \n",
       "\n",
       "    p2_conf  p2_dog                   p3   p3_conf  p3_dog  \n",
       "0  0.156665    True    Shetland_sheepdog  0.061428    True  \n",
       "1  0.074192    True  Rhodesian_ridgeback  0.072010    True  \n",
       "2  0.138584    True           bloodhound  0.116197    True  \n",
       "3  0.360687    True   miniature_pinscher  0.222752    True  \n",
       "4  0.243682    True             Doberman  0.154629    True  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_predictions_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#connecting to twitter API (consumer_key, consumer_secret, access_token, access_secret removed for privacy)\n",
    "# consumer_key = \n",
    "# consumer_secret =\n",
    "# access_token = \n",
    "# access_secret = \n",
    "\n",
    "auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
    "auth.set_access_token(access_token, access_secret)\n",
    "#json parser will output tweet info in dictionary format\n",
    "api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True, parser=tweepy.parsers.JSONParser())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 483,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tweepy.auth.OAuthHandler"
      ]
     },
     "execution_count": 483,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(auth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tweepy.api.API"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(api)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dict'>\n",
      "dict_keys(['id', 'geo', 'place', 'source', 'in_reply_to_screen_name', 'in_reply_to_status_id_str', 'text', 'retweeted', 'contributors', 'retweet_count', 'in_reply_to_user_id_str', 'truncated', 'coordinates', 'possibly_sensitive_appealable', 'id_str', 'created_at', 'extended_entities', 'in_reply_to_status_id', 'lang', 'possibly_sensitive', 'is_quote_status', 'favorite_count', 'user', 'entities', 'favorited', 'in_reply_to_user_id'])\n"
     ]
    }
   ],
   "source": [
    "tweet = api.get_status(id = '892420643555336193')\n",
    "# print(tweet)\n",
    "print(type(tweet))\n",
    "print(tweet.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet = api.get_status(id = '892177421306343426', tweet_mode='extended')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'contributors': None,\n",
       " 'coordinates': None,\n",
       " 'created_at': 'Tue Aug 01 00:17:27 +0000 2017',\n",
       " 'display_text_range': [0, 138],\n",
       " 'entities': {'hashtags': [],\n",
       "  'media': [{'display_url': 'pic.twitter.com/0Xxu71qeIV',\n",
       "    'expanded_url': 'https://twitter.com/dog_rates/status/892177421306343426/photo/1',\n",
       "    'id': 892177413194625024,\n",
       "    'id_str': '892177413194625024',\n",
       "    'indices': [139, 162],\n",
       "    'media_url': 'http://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg',\n",
       "    'media_url_https': 'https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg',\n",
       "    'sizes': {'large': {'h': 1600, 'resize': 'fit', 'w': 1407},\n",
       "     'medium': {'h': 1200, 'resize': 'fit', 'w': 1055},\n",
       "     'small': {'h': 680, 'resize': 'fit', 'w': 598},\n",
       "     'thumb': {'h': 150, 'resize': 'crop', 'w': 150}},\n",
       "    'type': 'photo',\n",
       "    'url': 'https://t.co/0Xxu71qeIV'}],\n",
       "  'symbols': [],\n",
       "  'urls': [],\n",
       "  'user_mentions': []},\n",
       " 'extended_entities': {'media': [{'display_url': 'pic.twitter.com/0Xxu71qeIV',\n",
       "    'expanded_url': 'https://twitter.com/dog_rates/status/892177421306343426/photo/1',\n",
       "    'id': 892177413194625024,\n",
       "    'id_str': '892177413194625024',\n",
       "    'indices': [139, 162],\n",
       "    'media_url': 'http://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg',\n",
       "    'media_url_https': 'https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg',\n",
       "    'sizes': {'large': {'h': 1600, 'resize': 'fit', 'w': 1407},\n",
       "     'medium': {'h': 1200, 'resize': 'fit', 'w': 1055},\n",
       "     'small': {'h': 680, 'resize': 'fit', 'w': 598},\n",
       "     'thumb': {'h': 150, 'resize': 'crop', 'w': 150}},\n",
       "    'type': 'photo',\n",
       "    'url': 'https://t.co/0Xxu71qeIV'}]},\n",
       " 'favorite_count': 33657,\n",
       " 'favorited': False,\n",
       " 'full_text': \"This is Tilly. She's just checking pup on you. Hopes you're doing ok. If not, she's available for pats, snugs, boops, the whole bit. 13/10 https://t.co/0Xxu71qeIV\",\n",
       " 'geo': None,\n",
       " 'id': 892177421306343426,\n",
       " 'id_str': '892177421306343426',\n",
       " 'in_reply_to_screen_name': None,\n",
       " 'in_reply_to_status_id': None,\n",
       " 'in_reply_to_status_id_str': None,\n",
       " 'in_reply_to_user_id': None,\n",
       " 'in_reply_to_user_id_str': None,\n",
       " 'is_quote_status': False,\n",
       " 'lang': 'en',\n",
       " 'place': None,\n",
       " 'possibly_sensitive': False,\n",
       " 'possibly_sensitive_appealable': False,\n",
       " 'retweet_count': 6431,\n",
       " 'retweeted': False,\n",
       " 'source': '<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>',\n",
       " 'truncated': False,\n",
       " 'user': {'contributors_enabled': False,\n",
       "  'created_at': 'Sun Nov 15 21:41:29 +0000 2015',\n",
       "  'default_profile': False,\n",
       "  'default_profile_image': False,\n",
       "  'description': 'Only Legit Source for Professional Dog Ratings STORE: @ShopWeRateDogs | IG, FB & SC: WeRateDogs | MOBILE APP: @GoodDogsGame Business: dogratingtwitter@gmail.com',\n",
       "  'entities': {'description': {'urls': []},\n",
       "   'url': {'urls': [{'display_url': 'weratedogs.com',\n",
       "      'expanded_url': 'http://weratedogs.com',\n",
       "      'indices': [0, 23],\n",
       "      'url': 'https://t.co/N7sNNHSfPq'}]}},\n",
       "  'favourites_count': 126213,\n",
       "  'follow_request_sent': False,\n",
       "  'followers_count': 4735289,\n",
       "  'following': False,\n",
       "  'friends_count': 109,\n",
       "  'geo_enabled': True,\n",
       "  'has_extended_profile': True,\n",
       "  'id': 4196983835,\n",
       "  'id_str': '4196983835',\n",
       "  'is_translation_enabled': False,\n",
       "  'is_translator': False,\n",
       "  'lang': 'en',\n",
       "  'listed_count': 3573,\n",
       "  'location': 'DM YOUR DOGS. WE WILL RATE',\n",
       "  'name': 'WeRateDogs™',\n",
       "  'notifications': False,\n",
       "  'profile_background_color': '000000',\n",
       "  'profile_background_image_url': 'http://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "  'profile_background_image_url_https': 'https://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "  'profile_background_tile': False,\n",
       "  'profile_banner_url': 'https://pbs.twimg.com/profile_banners/4196983835/1510812288',\n",
       "  'profile_image_url': 'http://pbs.twimg.com/profile_images/936608706107772929/GwbLQRxf_normal.jpg',\n",
       "  'profile_image_url_https': 'https://pbs.twimg.com/profile_images/936608706107772929/GwbLQRxf_normal.jpg',\n",
       "  'profile_link_color': 'F5ABB5',\n",
       "  'profile_sidebar_border_color': '000000',\n",
       "  'profile_sidebar_fill_color': '000000',\n",
       "  'profile_text_color': '000000',\n",
       "  'profile_use_background_image': False,\n",
       "  'protected': False,\n",
       "  'screen_name': 'dog_rates',\n",
       "  'statuses_count': 6308,\n",
       "  'time_zone': None,\n",
       "  'translator_type': 'none',\n",
       "  'url': 'https://t.co/N7sNNHSfPq',\n",
       "  'utc_offset': None,\n",
       "  'verified': True}}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'contributors': None,\n",
       " 'coordinates': None,\n",
       " 'created_at': 'Tue Aug 01 00:17:27 +0000 2017',\n",
       " 'display_text_range': [0, 138],\n",
       " 'entities': {'hashtags': [],\n",
       "  'media': [{'display_url': 'pic.twitter.com/0Xxu71qeIV',\n",
       "    'expanded_url': 'https://twitter.com/dog_rates/status/892177421306343426/photo/1',\n",
       "    'id': 892177413194625024,\n",
       "    'id_str': '892177413194625024',\n",
       "    'indices': [139, 162],\n",
       "    'media_url': 'http://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg',\n",
       "    'media_url_https': 'https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg',\n",
       "    'sizes': {'large': {'h': 1600, 'resize': 'fit', 'w': 1407},\n",
       "     'medium': {'h': 1200, 'resize': 'fit', 'w': 1055},\n",
       "     'small': {'h': 680, 'resize': 'fit', 'w': 598},\n",
       "     'thumb': {'h': 150, 'resize': 'crop', 'w': 150}},\n",
       "    'type': 'photo',\n",
       "    'url': 'https://t.co/0Xxu71qeIV'}],\n",
       "  'symbols': [],\n",
       "  'urls': [],\n",
       "  'user_mentions': []},\n",
       " 'extended_entities': {'media': [{'display_url': 'pic.twitter.com/0Xxu71qeIV',\n",
       "    'expanded_url': 'https://twitter.com/dog_rates/status/892177421306343426/photo/1',\n",
       "    'id': 892177413194625024,\n",
       "    'id_str': '892177413194625024',\n",
       "    'indices': [139, 162],\n",
       "    'media_url': 'http://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg',\n",
       "    'media_url_https': 'https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg',\n",
       "    'sizes': {'large': {'h': 1600, 'resize': 'fit', 'w': 1407},\n",
       "     'medium': {'h': 1200, 'resize': 'fit', 'w': 1055},\n",
       "     'small': {'h': 680, 'resize': 'fit', 'w': 598},\n",
       "     'thumb': {'h': 150, 'resize': 'crop', 'w': 150}},\n",
       "    'type': 'photo',\n",
       "    'url': 'https://t.co/0Xxu71qeIV'}]},\n",
       " 'favorite_count': 33657,\n",
       " 'favorited': False,\n",
       " 'full_text': \"This is Tilly. She's just checking pup on you. Hopes you're doing ok. If not, she's available for pats, snugs, boops, the whole bit. 13/10 https://t.co/0Xxu71qeIV\",\n",
       " 'geo': None,\n",
       " 'id': 892177421306343426,\n",
       " 'id_str': '892177421306343426',\n",
       " 'in_reply_to_screen_name': None,\n",
       " 'in_reply_to_status_id': None,\n",
       " 'in_reply_to_status_id_str': None,\n",
       " 'in_reply_to_user_id': None,\n",
       " 'in_reply_to_user_id_str': None,\n",
       " 'is_quote_status': False,\n",
       " 'lang': 'en',\n",
       " 'place': None,\n",
       " 'possibly_sensitive': False,\n",
       " 'possibly_sensitive_appealable': False,\n",
       " 'retweet_count': 6431,\n",
       " 'retweeted': False,\n",
       " 'source': '<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>',\n",
       " 'truncated': False,\n",
       " 'user': {'contributors_enabled': False,\n",
       "  'created_at': 'Sun Nov 15 21:41:29 +0000 2015',\n",
       "  'default_profile': False,\n",
       "  'default_profile_image': False,\n",
       "  'description': 'Only Legit Source for Professional Dog Ratings STORE: @ShopWeRateDogs | IG, FB & SC: WeRateDogs | MOBILE APP: @GoodDogsGame Business: dogratingtwitter@gmail.com',\n",
       "  'entities': {'description': {'urls': []},\n",
       "   'url': {'urls': [{'display_url': 'weratedogs.com',\n",
       "      'expanded_url': 'http://weratedogs.com',\n",
       "      'indices': [0, 23],\n",
       "      'url': 'https://t.co/N7sNNHSfPq'}]}},\n",
       "  'favourites_count': 126213,\n",
       "  'follow_request_sent': False,\n",
       "  'followers_count': 4735289,\n",
       "  'following': False,\n",
       "  'friends_count': 109,\n",
       "  'geo_enabled': True,\n",
       "  'has_extended_profile': True,\n",
       "  'id': 4196983835,\n",
       "  'id_str': '4196983835',\n",
       "  'is_translation_enabled': False,\n",
       "  'is_translator': False,\n",
       "  'lang': 'en',\n",
       "  'listed_count': 3573,\n",
       "  'location': 'DM YOUR DOGS. WE WILL RATE',\n",
       "  'name': 'WeRateDogs™',\n",
       "  'notifications': False,\n",
       "  'profile_background_color': '000000',\n",
       "  'profile_background_image_url': 'http://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "  'profile_background_image_url_https': 'https://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "  'profile_background_tile': False,\n",
       "  'profile_banner_url': 'https://pbs.twimg.com/profile_banners/4196983835/1510812288',\n",
       "  'profile_image_url': 'http://pbs.twimg.com/profile_images/936608706107772929/GwbLQRxf_normal.jpg',\n",
       "  'profile_image_url_https': 'https://pbs.twimg.com/profile_images/936608706107772929/GwbLQRxf_normal.jpg',\n",
       "  'profile_link_color': 'F5ABB5',\n",
       "  'profile_sidebar_border_color': '000000',\n",
       "  'profile_sidebar_fill_color': '000000',\n",
       "  'profile_text_color': '000000',\n",
       "  'profile_use_background_image': False,\n",
       "  'protected': False,\n",
       "  'screen_name': 'dog_rates',\n",
       "  'statuses_count': 6308,\n",
       "  'time_zone': None,\n",
       "  'translator_type': 'none',\n",
       "  'url': 'https://t.co/N7sNNHSfPq',\n",
       "  'utc_offset': None,\n",
       "  'verified': True}}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "892177421306343426"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet['id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33657"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet['favorite_count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_list = [892420643555336193, 892177421306343426, 891815181378084864, 891689557279858688, 891327558926688256]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[892420643555336193,\n",
       " 892177421306343426,\n",
       " 891815181378084864,\n",
       " 891689557279858688,\n",
       " 891327558926688256]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate limit reached. Sleeping for: 675\n"
     ]
    }
   ],
   "source": [
    "# takes a while to run. uncomment lines below if necessary\n",
    "\n",
    "# #for testing Create a list with some ids 4-5 are ok and loop in this, so you don't have to wait half an hour each time.\n",
    "# for tweet_id in test_list:\n",
    "\n",
    "# with open('tweet_json.txt', 'a+', encoding='utf-8') as file:\n",
    "#     for tweet_id in twitter_archive_df['tweet_id']:\n",
    "#         try:\n",
    "#             tweet = api.get_status(id = tweet_id, tweet_mode='extended')\n",
    "#             file.write(json.dumps(tweet))\n",
    "#             file.write('\\n')\n",
    "#         except:\n",
    "#             pass\n",
    "    \n",
    "# file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import json objects from .txt file into a list\n",
    "\n",
    "with open('tweet_json.txt') as file:\n",
    "    status = []\n",
    "    for line in file:\n",
    "        status.append(json.loads(line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_df = pd.DataFrame(status)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['contributors', 'coordinates', 'created_at', 'display_text_range',\n",
       "       'entities', 'extended_entities', 'favorite_count', 'favorited',\n",
       "       'full_text', 'geo', 'id', 'id_str', 'in_reply_to_screen_name',\n",
       "       'in_reply_to_status_id', 'in_reply_to_status_id_str',\n",
       "       'in_reply_to_user_id', 'in_reply_to_user_id_str', 'is_quote_status',\n",
       "       'lang', 'place', 'possibly_sensitive', 'possibly_sensitive_appealable',\n",
       "       'quoted_status', 'quoted_status_id', 'quoted_status_id_str',\n",
       "       'retweet_count', 'retweeted', 'retweeted_status', 'source', 'truncated',\n",
       "       'user'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2352 entries, 0 to 2351\n",
      "Data columns (total 31 columns):\n",
      "contributors                     0 non-null object\n",
      "coordinates                      0 non-null object\n",
      "created_at                       2352 non-null object\n",
      "display_text_range               2352 non-null object\n",
      "entities                         2352 non-null object\n",
      "extended_entities                2074 non-null object\n",
      "favorite_count                   2352 non-null int64\n",
      "favorited                        2352 non-null bool\n",
      "full_text                        2352 non-null object\n",
      "geo                              0 non-null object\n",
      "id                               2352 non-null int64\n",
      "id_str                           2352 non-null object\n",
      "in_reply_to_screen_name          77 non-null object\n",
      "in_reply_to_status_id            77 non-null float64\n",
      "in_reply_to_status_id_str        77 non-null object\n",
      "in_reply_to_user_id              77 non-null float64\n",
      "in_reply_to_user_id_str          77 non-null object\n",
      "is_quote_status                  2352 non-null bool\n",
      "lang                             2352 non-null object\n",
      "place                            1 non-null object\n",
      "possibly_sensitive               2212 non-null object\n",
      "possibly_sensitive_appealable    2212 non-null object\n",
      "quoted_status                    28 non-null object\n",
      "quoted_status_id                 29 non-null float64\n",
      "quoted_status_id_str             29 non-null object\n",
      "retweet_count                    2352 non-null int64\n",
      "retweeted                        2352 non-null bool\n",
      "retweeted_status                 174 non-null object\n",
      "source                           2352 non-null object\n",
      "truncated                        2352 non-null bool\n",
      "user                             2352 non-null object\n",
      "dtypes: bool(4), float64(3), int64(3), object(21)\n",
      "memory usage: 505.4+ KB\n"
     ]
    }
   ],
   "source": [
    "tweets_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_df = pd.DataFrame(tweets_df, columns = ['id', 'retweeted','retweet_count','favorite_count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>retweeted</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892420643555336193</td>\n",
       "      <td>False</td>\n",
       "      <td>8778</td>\n",
       "      <td>39308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>892177421306343426</td>\n",
       "      <td>False</td>\n",
       "      <td>6431</td>\n",
       "      <td>33657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>891815181378084864</td>\n",
       "      <td>False</td>\n",
       "      <td>4269</td>\n",
       "      <td>25363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>891689557279858688</td>\n",
       "      <td>False</td>\n",
       "      <td>8867</td>\n",
       "      <td>42692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>891327558926688256</td>\n",
       "      <td>False</td>\n",
       "      <td>9646</td>\n",
       "      <td>40841</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id  retweeted  retweet_count  favorite_count\n",
       "0  892420643555336193      False           8778           39308\n",
       "1  892177421306343426      False           6431           33657\n",
       "2  891815181378084864      False           4269           25363\n",
       "3  891689557279858688      False           8867           42692\n",
       "4  891327558926688256      False           9646           40841"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Assessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['tweet_id', 'in_reply_to_status_id', 'in_reply_to_user_id', 'timestamp',\n",
       "       'source', 'text', 'retweeted_status_id', 'retweeted_status_user_id',\n",
       "       'retweeted_status_timestamp', 'expanded_urls', 'rating_numerator',\n",
       "       'rating_denominator', 'name', 'doggo', 'floofer', 'pupper', 'puppo'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>retweeted_status_id</th>\n",
       "      <th>retweeted_status_user_id</th>\n",
       "      <th>retweeted_status_timestamp</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>name</th>\n",
       "      <th>doggo</th>\n",
       "      <th>floofer</th>\n",
       "      <th>pupper</th>\n",
       "      <th>puppo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2353</th>\n",
       "      <td>666033412701032449</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-11-15 23:21:54 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Here is a very happy pup. Big fan of well-main...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666033412...</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>a</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2354</th>\n",
       "      <td>666029285002620928</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-11-15 23:05:30 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is a western brown Mitsubishi terrier. Up...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666029285...</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>a</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2355</th>\n",
       "      <td>666020888022790149</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-11-15 22:32:08 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Here we have a Japanese Irish Setter. Lost eye...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/666020888...</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                tweet_id  in_reply_to_status_id  in_reply_to_user_id  \\\n",
       "2353  666033412701032449                    NaN                  NaN   \n",
       "2354  666029285002620928                    NaN                  NaN   \n",
       "2355  666020888022790149                    NaN                  NaN   \n",
       "\n",
       "                      timestamp  \\\n",
       "2353  2015-11-15 23:21:54 +0000   \n",
       "2354  2015-11-15 23:05:30 +0000   \n",
       "2355  2015-11-15 22:32:08 +0000   \n",
       "\n",
       "                                                 source  \\\n",
       "2353  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "2354  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "2355  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "\n",
       "                                                   text  retweeted_status_id  \\\n",
       "2353  Here is a very happy pup. Big fan of well-main...                  NaN   \n",
       "2354  This is a western brown Mitsubishi terrier. Up...                  NaN   \n",
       "2355  Here we have a Japanese Irish Setter. Lost eye...                  NaN   \n",
       "\n",
       "      retweeted_status_user_id retweeted_status_timestamp  \\\n",
       "2353                       NaN                        NaN   \n",
       "2354                       NaN                        NaN   \n",
       "2355                       NaN                        NaN   \n",
       "\n",
       "                                          expanded_urls  rating_numerator  \\\n",
       "2353  https://twitter.com/dog_rates/status/666033412...                 9   \n",
       "2354  https://twitter.com/dog_rates/status/666029285...                 7   \n",
       "2355  https://twitter.com/dog_rates/status/666020888...                 8   \n",
       "\n",
       "      rating_denominator  name doggo floofer pupper puppo  \n",
       "2353                  10     a  None    None   None  None  \n",
       "2354                  10     a  None    None   None  None  \n",
       "2355                  10  None  None    None   None  None  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# archived tweets imported from .csv file\n",
    "twitter_archive_df.tail(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>retweeted_status_id</th>\n",
       "      <th>retweeted_status_user_id</th>\n",
       "      <th>retweeted_status_timestamp</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>name</th>\n",
       "      <th>doggo</th>\n",
       "      <th>floofer</th>\n",
       "      <th>pupper</th>\n",
       "      <th>puppo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>880221127280381952</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-06-29 00:27:25 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Meet Jesse. He's a Fetty Woof. His tongue ejec...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/880221127...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>Jesse</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>863471782782697472</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-05-13 19:11:30 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>RT @dog_rates: Say hello to Quinn. She's quite...</td>\n",
       "      <td>8.630625e+17</td>\n",
       "      <td>4.196984e+09</td>\n",
       "      <td>2017-05-12 16:05:02 +0000</td>\n",
       "      <td>https://www.gofundme.com/helpquinny,https://tw...</td>\n",
       "      <td>13</td>\n",
       "      <td>10</td>\n",
       "      <td>Quinn</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>965</th>\n",
       "      <td>750429297815552001</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-07-05 20:41:01 +0000</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Arnie. He's a Nova Scotian Fridge Floo...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://twitter.com/dog_rates/status/750429297...</td>\n",
       "      <td>12</td>\n",
       "      <td>10</td>\n",
       "      <td>Arnie</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               tweet_id  in_reply_to_status_id  in_reply_to_user_id  \\\n",
       "61   880221127280381952                    NaN                  NaN   \n",
       "146  863471782782697472                    NaN                  NaN   \n",
       "965  750429297815552001                    NaN                  NaN   \n",
       "\n",
       "                     timestamp  \\\n",
       "61   2017-06-29 00:27:25 +0000   \n",
       "146  2017-05-13 19:11:30 +0000   \n",
       "965  2016-07-05 20:41:01 +0000   \n",
       "\n",
       "                                                source  \\\n",
       "61   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "146  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "965  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "\n",
       "                                                  text  retweeted_status_id  \\\n",
       "61   Meet Jesse. He's a Fetty Woof. His tongue ejec...                  NaN   \n",
       "146  RT @dog_rates: Say hello to Quinn. She's quite...         8.630625e+17   \n",
       "965  This is Arnie. He's a Nova Scotian Fridge Floo...                  NaN   \n",
       "\n",
       "     retweeted_status_user_id retweeted_status_timestamp  \\\n",
       "61                        NaN                        NaN   \n",
       "146              4.196984e+09  2017-05-12 16:05:02 +0000   \n",
       "965                       NaN                        NaN   \n",
       "\n",
       "                                         expanded_urls  rating_numerator  \\\n",
       "61   https://twitter.com/dog_rates/status/880221127...                12   \n",
       "146  https://www.gofundme.com/helpquinny,https://tw...                13   \n",
       "965  https://twitter.com/dog_rates/status/750429297...                12   \n",
       "\n",
       "     rating_denominator   name doggo floofer pupper puppo  \n",
       "61                   10  Jesse  None    None   None  None  \n",
       "146                  10  Quinn  None    None   None  None  \n",
       "965                  10  Arnie  None    None   None  None  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df.sample(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2356 entries, 0 to 2355\n",
      "Data columns (total 17 columns):\n",
      "tweet_id                      2356 non-null int64\n",
      "in_reply_to_status_id         78 non-null float64\n",
      "in_reply_to_user_id           78 non-null float64\n",
      "timestamp                     2356 non-null object\n",
      "source                        2356 non-null object\n",
      "text                          2356 non-null object\n",
      "retweeted_status_id           181 non-null float64\n",
      "retweeted_status_user_id      181 non-null float64\n",
      "retweeted_status_timestamp    181 non-null object\n",
      "expanded_urls                 2297 non-null object\n",
      "rating_numerator              2356 non-null int64\n",
      "rating_denominator            2356 non-null int64\n",
      "name                          2356 non-null object\n",
      "doggo                         2356 non-null object\n",
      "floofer                       2356 non-null object\n",
      "pupper                        2356 non-null object\n",
      "puppo                         2356 non-null object\n",
      "dtypes: float64(4), int64(3), object(10)\n",
      "memory usage: 313.0+ KB\n"
     ]
    }
   ],
   "source": [
    "twitter_archive_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>retweeted_status_id</th>\n",
       "      <th>retweeted_status_user_id</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>2.356000e+03</td>\n",
       "      <td>7.800000e+01</td>\n",
       "      <td>7.800000e+01</td>\n",
       "      <td>1.810000e+02</td>\n",
       "      <td>1.810000e+02</td>\n",
       "      <td>2356.000000</td>\n",
       "      <td>2356.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>7.427716e+17</td>\n",
       "      <td>7.455079e+17</td>\n",
       "      <td>2.014171e+16</td>\n",
       "      <td>7.720400e+17</td>\n",
       "      <td>1.241698e+16</td>\n",
       "      <td>13.126486</td>\n",
       "      <td>10.455433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>6.856705e+16</td>\n",
       "      <td>7.582492e+16</td>\n",
       "      <td>1.252797e+17</td>\n",
       "      <td>6.236928e+16</td>\n",
       "      <td>9.599254e+16</td>\n",
       "      <td>45.876648</td>\n",
       "      <td>6.745237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>6.660209e+17</td>\n",
       "      <td>6.658147e+17</td>\n",
       "      <td>1.185634e+07</td>\n",
       "      <td>6.661041e+17</td>\n",
       "      <td>7.832140e+05</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>6.783989e+17</td>\n",
       "      <td>6.757419e+17</td>\n",
       "      <td>3.086374e+08</td>\n",
       "      <td>7.186315e+17</td>\n",
       "      <td>4.196984e+09</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>7.196279e+17</td>\n",
       "      <td>7.038708e+17</td>\n",
       "      <td>4.196984e+09</td>\n",
       "      <td>7.804657e+17</td>\n",
       "      <td>4.196984e+09</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.993373e+17</td>\n",
       "      <td>8.257804e+17</td>\n",
       "      <td>4.196984e+09</td>\n",
       "      <td>8.203146e+17</td>\n",
       "      <td>4.196984e+09</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>8.924206e+17</td>\n",
       "      <td>8.862664e+17</td>\n",
       "      <td>8.405479e+17</td>\n",
       "      <td>8.874740e+17</td>\n",
       "      <td>7.874618e+17</td>\n",
       "      <td>1776.000000</td>\n",
       "      <td>170.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           tweet_id  in_reply_to_status_id  in_reply_to_user_id  \\\n",
       "count  2.356000e+03           7.800000e+01         7.800000e+01   \n",
       "mean   7.427716e+17           7.455079e+17         2.014171e+16   \n",
       "std    6.856705e+16           7.582492e+16         1.252797e+17   \n",
       "min    6.660209e+17           6.658147e+17         1.185634e+07   \n",
       "25%    6.783989e+17           6.757419e+17         3.086374e+08   \n",
       "50%    7.196279e+17           7.038708e+17         4.196984e+09   \n",
       "75%    7.993373e+17           8.257804e+17         4.196984e+09   \n",
       "max    8.924206e+17           8.862664e+17         8.405479e+17   \n",
       "\n",
       "       retweeted_status_id  retweeted_status_user_id  rating_numerator  \\\n",
       "count         1.810000e+02              1.810000e+02       2356.000000   \n",
       "mean          7.720400e+17              1.241698e+16         13.126486   \n",
       "std           6.236928e+16              9.599254e+16         45.876648   \n",
       "min           6.661041e+17              7.832140e+05          0.000000   \n",
       "25%           7.186315e+17              4.196984e+09         10.000000   \n",
       "50%           7.804657e+17              4.196984e+09         11.000000   \n",
       "75%           8.203146e+17              4.196984e+09         12.000000   \n",
       "max           8.874740e+17              7.874618e+17       1776.000000   \n",
       "\n",
       "       rating_denominator  \n",
       "count         2356.000000  \n",
       "mean            10.455433  \n",
       "std              6.745237  \n",
       "min              0.000000  \n",
       "25%             10.000000  \n",
       "50%             10.000000  \n",
       "75%             10.000000  \n",
       "max            170.000000  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2015-11-23 00:30:28 +0000    1\n",
       "2016-07-15 17:56:40 +0000    1\n",
       "2016-08-31 15:58:28 +0000    1\n",
       "2015-11-21 02:07:05 +0000    1\n",
       "2015-12-28 05:02:37 +0000    1\n",
       "2016-08-08 02:13:34 +0000    1\n",
       "2016-02-25 01:47:04 +0000    1\n",
       "2016-02-12 04:07:53 +0000    1\n",
       "2016-12-27 18:24:12 +0000    1\n",
       "2017-02-08 01:35:19 +0000    1\n",
       "Name: timestamp, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df['timestamp'].value_counts(ascending=True)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "182"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(twitter_archive_df['retweeted_status_id'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "None       745\n",
       "a           55\n",
       "Charlie     12\n",
       "Cooper      11\n",
       "Lucy        11\n",
       "Oliver      11\n",
       "Lola        10\n",
       "Tucker      10\n",
       "Penny       10\n",
       "Bo           9\n",
       "Winston      9\n",
       "Sadie        8\n",
       "the          8\n",
       "Daisy        7\n",
       "Buddy        7\n",
       "Bailey       7\n",
       "Toby         7\n",
       "an           7\n",
       "Jax          6\n",
       "Rusty        6\n",
       "Name: name, dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df['name'].value_counts(ascending = False)[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>retweeted_status_id</th>\n",
       "      <th>retweeted_status_user_id</th>\n",
       "      <th>retweeted_status_timestamp</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>name</th>\n",
       "      <th>doggo</th>\n",
       "      <th>floofer</th>\n",
       "      <th>pupper</th>\n",
       "      <th>puppo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [tweet_id, in_reply_to_status_id, in_reply_to_user_id, timestamp, source, text, retweeted_status_id, retweeted_status_user_id, retweeted_status_timestamp, expanded_urls, rating_numerator, rating_denominator, name, doggo, floofer, pupper, puppo]\n",
       "Index: []"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df[twitter_archive_df['tweet_id'].duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(twitter_archive_df[pd.isnull(twitter_archive_df['expanded_urls'])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>retweeted</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2349</th>\n",
       "      <td>666033412701032449</td>\n",
       "      <td>False</td>\n",
       "      <td>46</td>\n",
       "      <td>127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2350</th>\n",
       "      <td>666029285002620928</td>\n",
       "      <td>False</td>\n",
       "      <td>47</td>\n",
       "      <td>131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2351</th>\n",
       "      <td>666020888022790149</td>\n",
       "      <td>False</td>\n",
       "      <td>527</td>\n",
       "      <td>2526</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      id  retweeted  retweet_count  favorite_count\n",
       "2349  666033412701032449      False             46             127\n",
       "2350  666029285002620928      False             47             131\n",
       "2351  666020888022790149      False            527            2526"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#additional data on retweet count and favorite count extracted from tweepy API\n",
    "\n",
    "tweets_df.tail(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>retweeted</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1118</th>\n",
       "      <td>730573383004487680</td>\n",
       "      <td>False</td>\n",
       "      <td>2411</td>\n",
       "      <td>5351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>314</th>\n",
       "      <td>835172783151792128</td>\n",
       "      <td>False</td>\n",
       "      <td>6454</td>\n",
       "      <td>28338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1795</th>\n",
       "      <td>677187300187611136</td>\n",
       "      <td>False</td>\n",
       "      <td>1023</td>\n",
       "      <td>2948</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      id  retweeted  retweet_count  favorite_count\n",
       "1118  730573383004487680      False           2411            5351\n",
       "314   835172783151792128      False           6454           28338\n",
       "1795  677187300187611136      False           1023            2948"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df.sample(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    2352\n",
       "Name: retweeted, dtype: int64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df.retweeted.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2352 entries, 0 to 2351\n",
      "Data columns (total 4 columns):\n",
      "id                2352 non-null int64\n",
      "retweeted         2352 non-null bool\n",
      "retweet_count     2352 non-null int64\n",
      "favorite_count    2352 non-null int64\n",
      "dtypes: bool(1), int64(3)\n",
      "memory usage: 57.5 KB\n"
     ]
    }
   ],
   "source": [
    "tweets_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>2.352000e+03</td>\n",
       "      <td>2352.000000</td>\n",
       "      <td>2352.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>7.427301e+17</td>\n",
       "      <td>3105.955357</td>\n",
       "      <td>8167.393707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>6.868610e+16</td>\n",
       "      <td>5132.152436</td>\n",
       "      <td>12115.422425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>6.660209e+17</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>6.783949e+17</td>\n",
       "      <td>615.750000</td>\n",
       "      <td>1415.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>7.193536e+17</td>\n",
       "      <td>1452.500000</td>\n",
       "      <td>3586.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.991219e+17</td>\n",
       "      <td>3608.250000</td>\n",
       "      <td>10116.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>8.924206e+17</td>\n",
       "      <td>78712.000000</td>\n",
       "      <td>131758.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id  retweet_count  favorite_count\n",
       "count  2.352000e+03    2352.000000     2352.000000\n",
       "mean   7.427301e+17    3105.955357     8167.393707\n",
       "std    6.868610e+16    5132.152436    12115.422425\n",
       "min    6.660209e+17       0.000000        0.000000\n",
       "25%    6.783949e+17     615.750000     1415.000000\n",
       "50%    7.193536e+17    1452.500000     3586.500000\n",
       "75%    7.991219e+17    3608.250000    10116.000000\n",
       "max    8.924206e+17   78712.000000   131758.000000"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>retweeted</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>892420643555336193</td>\n",
       "      <td>False</td>\n",
       "      <td>8778</td>\n",
       "      <td>39308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>892177421306343426</td>\n",
       "      <td>False</td>\n",
       "      <td>6431</td>\n",
       "      <td>33657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>891815181378084864</td>\n",
       "      <td>False</td>\n",
       "      <td>4269</td>\n",
       "      <td>25363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>891689557279858688</td>\n",
       "      <td>False</td>\n",
       "      <td>8867</td>\n",
       "      <td>42692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>891327558926688256</td>\n",
       "      <td>False</td>\n",
       "      <td>9646</td>\n",
       "      <td>40841</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id  retweeted  retweet_count  favorite_count\n",
       "5  892420643555336193      False           8778           39308\n",
       "6  892177421306343426      False           6431           33657\n",
       "7  891815181378084864      False           4269           25363\n",
       "8  891689557279858688      False           8867           42692\n",
       "9  891327558926688256      False           9646           40841"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df[tweets_df['id'].duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>jpg_url</th>\n",
       "      <th>img_num</th>\n",
       "      <th>p1</th>\n",
       "      <th>p1_conf</th>\n",
       "      <th>p1_dog</th>\n",
       "      <th>p2</th>\n",
       "      <th>p2_conf</th>\n",
       "      <th>p2_dog</th>\n",
       "      <th>p3</th>\n",
       "      <th>p3_conf</th>\n",
       "      <th>p3_dog</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2072</th>\n",
       "      <td>891815181378084864</td>\n",
       "      <td>https://pbs.twimg.com/media/DGBdLU1WsAANxJ9.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>Chihuahua</td>\n",
       "      <td>0.716012</td>\n",
       "      <td>True</td>\n",
       "      <td>malamute</td>\n",
       "      <td>0.078253</td>\n",
       "      <td>True</td>\n",
       "      <td>kelpie</td>\n",
       "      <td>0.031379</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2073</th>\n",
       "      <td>892177421306343426</td>\n",
       "      <td>https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>Chihuahua</td>\n",
       "      <td>0.323581</td>\n",
       "      <td>True</td>\n",
       "      <td>Pekinese</td>\n",
       "      <td>0.090647</td>\n",
       "      <td>True</td>\n",
       "      <td>papillon</td>\n",
       "      <td>0.068957</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2074</th>\n",
       "      <td>892420643555336193</td>\n",
       "      <td>https://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>orange</td>\n",
       "      <td>0.097049</td>\n",
       "      <td>False</td>\n",
       "      <td>bagel</td>\n",
       "      <td>0.085851</td>\n",
       "      <td>False</td>\n",
       "      <td>banana</td>\n",
       "      <td>0.076110</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                tweet_id                                          jpg_url  \\\n",
       "2072  891815181378084864  https://pbs.twimg.com/media/DGBdLU1WsAANxJ9.jpg   \n",
       "2073  892177421306343426  https://pbs.twimg.com/media/DGGmoV4XsAAUL6n.jpg   \n",
       "2074  892420643555336193  https://pbs.twimg.com/media/DGKD1-bXoAAIAUK.jpg   \n",
       "\n",
       "      img_num         p1   p1_conf  p1_dog        p2   p2_conf  p2_dog  \\\n",
       "2072        1  Chihuahua  0.716012    True  malamute  0.078253    True   \n",
       "2073        1  Chihuahua  0.323581    True  Pekinese  0.090647    True   \n",
       "2074        1     orange  0.097049   False     bagel  0.085851   False   \n",
       "\n",
       "            p3   p3_conf  p3_dog  \n",
       "2072    kelpie  0.031379    True  \n",
       "2073  papillon  0.068957    True  \n",
       "2074    banana  0.076110   False  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# image predictions imported from website address in .tsv format using requests library\n",
    "image_predictions_df.tail(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>jpg_url</th>\n",
       "      <th>img_num</th>\n",
       "      <th>p1</th>\n",
       "      <th>p1_conf</th>\n",
       "      <th>p1_dog</th>\n",
       "      <th>p2</th>\n",
       "      <th>p2_conf</th>\n",
       "      <th>p2_dog</th>\n",
       "      <th>p3</th>\n",
       "      <th>p3_conf</th>\n",
       "      <th>p3_dog</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1723</th>\n",
       "      <td>819924195358416896</td>\n",
       "      <td>https://pbs.twimg.com/ext_tw_video_thumb/81992...</td>\n",
       "      <td>1</td>\n",
       "      <td>bathtub</td>\n",
       "      <td>0.100896</td>\n",
       "      <td>False</td>\n",
       "      <td>shower_curtain</td>\n",
       "      <td>0.091866</td>\n",
       "      <td>False</td>\n",
       "      <td>tub</td>\n",
       "      <td>0.049176</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1379</th>\n",
       "      <td>764857477905154048</td>\n",
       "      <td>https://pbs.twimg.com/media/Cp1R0ZTWcAAaPO4.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>Bernese_mountain_dog</td>\n",
       "      <td>0.792059</td>\n",
       "      <td>True</td>\n",
       "      <td>Appenzeller</td>\n",
       "      <td>0.155034</td>\n",
       "      <td>True</td>\n",
       "      <td>EntleBucher</td>\n",
       "      <td>0.038374</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>657</th>\n",
       "      <td>682303737705140231</td>\n",
       "      <td>https://pbs.twimg.com/media/CXgHoLnWAAA8i52.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>seat_belt</td>\n",
       "      <td>0.997659</td>\n",
       "      <td>False</td>\n",
       "      <td>Lakeland_terrier</td>\n",
       "      <td>0.001731</td>\n",
       "      <td>True</td>\n",
       "      <td>Airedale</td>\n",
       "      <td>0.000204</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                tweet_id                                            jpg_url  \\\n",
       "1723  819924195358416896  https://pbs.twimg.com/ext_tw_video_thumb/81992...   \n",
       "1379  764857477905154048    https://pbs.twimg.com/media/Cp1R0ZTWcAAaPO4.jpg   \n",
       "657   682303737705140231    https://pbs.twimg.com/media/CXgHoLnWAAA8i52.jpg   \n",
       "\n",
       "      img_num                    p1   p1_conf  p1_dog                p2  \\\n",
       "1723        1               bathtub  0.100896   False    shower_curtain   \n",
       "1379        1  Bernese_mountain_dog  0.792059    True       Appenzeller   \n",
       "657         1             seat_belt  0.997659   False  Lakeland_terrier   \n",
       "\n",
       "       p2_conf  p2_dog           p3   p3_conf  p3_dog  \n",
       "1723  0.091866   False          tub  0.049176   False  \n",
       "1379  0.155034    True  EntleBucher  0.038374    True  \n",
       "657   0.001731    True     Airedale  0.000204    True  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_predictions_df.sample(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2075 entries, 0 to 2074\n",
      "Data columns (total 12 columns):\n",
      "tweet_id    2075 non-null int64\n",
      "jpg_url     2075 non-null object\n",
      "img_num     2075 non-null int64\n",
      "p1          2075 non-null object\n",
      "p1_conf     2075 non-null float64\n",
      "p1_dog      2075 non-null bool\n",
      "p2          2075 non-null object\n",
      "p2_conf     2075 non-null float64\n",
      "p2_dog      2075 non-null bool\n",
      "p3          2075 non-null object\n",
      "p3_conf     2075 non-null float64\n",
      "p3_dog      2075 non-null bool\n",
      "dtypes: bool(3), float64(3), int64(2), object(4)\n",
      "memory usage: 152.1+ KB\n"
     ]
    }
   ],
   "source": [
    "image_predictions_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>img_num</th>\n",
       "      <th>p1_conf</th>\n",
       "      <th>p2_conf</th>\n",
       "      <th>p3_conf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>2.075000e+03</td>\n",
       "      <td>2075.000000</td>\n",
       "      <td>2075.000000</td>\n",
       "      <td>2.075000e+03</td>\n",
       "      <td>2.075000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>7.384514e+17</td>\n",
       "      <td>1.203855</td>\n",
       "      <td>0.594548</td>\n",
       "      <td>1.345886e-01</td>\n",
       "      <td>6.032417e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>6.785203e+16</td>\n",
       "      <td>0.561875</td>\n",
       "      <td>0.271174</td>\n",
       "      <td>1.006657e-01</td>\n",
       "      <td>5.090593e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>6.660209e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.044333</td>\n",
       "      <td>1.011300e-08</td>\n",
       "      <td>1.740170e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>6.764835e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.364412</td>\n",
       "      <td>5.388625e-02</td>\n",
       "      <td>1.622240e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>7.119988e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.588230</td>\n",
       "      <td>1.181810e-01</td>\n",
       "      <td>4.944380e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.932034e+17</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.843855</td>\n",
       "      <td>1.955655e-01</td>\n",
       "      <td>9.180755e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>8.924206e+17</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.880140e-01</td>\n",
       "      <td>2.734190e-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           tweet_id      img_num      p1_conf       p2_conf       p3_conf\n",
       "count  2.075000e+03  2075.000000  2075.000000  2.075000e+03  2.075000e+03\n",
       "mean   7.384514e+17     1.203855     0.594548  1.345886e-01  6.032417e-02\n",
       "std    6.785203e+16     0.561875     0.271174  1.006657e-01  5.090593e-02\n",
       "min    6.660209e+17     1.000000     0.044333  1.011300e-08  1.740170e-10\n",
       "25%    6.764835e+17     1.000000     0.364412  5.388625e-02  1.622240e-02\n",
       "50%    7.119988e+17     1.000000     0.588230  1.181810e-01  4.944380e-02\n",
       "75%    7.932034e+17     1.000000     0.843855  1.955655e-01  9.180755e-02\n",
       "max    8.924206e+17     4.000000     1.000000  4.880140e-01  2.734190e-01"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_predictions_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>jpg_url</th>\n",
       "      <th>img_num</th>\n",
       "      <th>p1</th>\n",
       "      <th>p1_conf</th>\n",
       "      <th>p1_dog</th>\n",
       "      <th>p2</th>\n",
       "      <th>p2_conf</th>\n",
       "      <th>p2_dog</th>\n",
       "      <th>p3</th>\n",
       "      <th>p3_conf</th>\n",
       "      <th>p3_dog</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [tweet_id, jpg_url, img_num, p1, p1_conf, p1_dog, p2, p2_conf, p2_dog, p3, p3_conf, p3_dog]\n",
       "Index: []"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_predictions_df[image_predictions_df['tweet_id'].duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    1780\n",
       "2     198\n",
       "3      66\n",
       "4      31\n",
       "Name: img_num, dtype: int64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_predictions_df.img_num.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quality Issues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## need to adjust: \n",
    "# remove retweets from twitter archive\n",
    "# do I then need to remove duplicates from tweets df? or will this be handled when I merge\n",
    "# need to update quality issues"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Timestamp values in twitter_archive dataframe are string type.\n",
    "2. Timestamp values include more than just date.\n",
    "3. Duplicates & retweets need to be in twitter_archive and tweets dataframe.\n",
    "4. There are many inaccurate names under 'names' column in archived tweets dataframe. \n",
    "5. Rating numerator and rating denominator are separated in archived tweets dataframe.\n",
    "6. The following columns in twitter archive dataframe are primarily missing values: ['in_reply_to_status_id', 'in_reply_to_user_id', 'retweeted_status_id', 'retweeted_status_user_id', 'retweeted_status_timestamp']\n",
    "7. 'expanded_urls' column has 59 null values in archived tweets dataframe.\n",
    "8. Entire 'retweeted' column in tweets dataframe all have the same value of 'false'."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tidiness Issues"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. 'id' column in tweets dataframe and 'tweet_id' in other two dataframes.\n",
    "2. Dog stages are in seperated columns\n",
    "3. Datasets are seperated"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quality Issues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# creating copies of df's to ensure I wont lose original material from any mistakes cleaning\n",
    "twitter_archive_df_clean = twitter_archive_df.copy()\n",
    "tweets_df_clean = tweets_df.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define\n",
    "Convert timestamp column in twitter archive dataframe to timestamp and keep just the date."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# checking the type first element in series\n",
    "type(twitter_archive_df_clean.timestamp[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# converting the timestamp column to dateime object and then extracting out the date\n",
    "twitter_archive_df_clean['timestamp'] = twitter_archive_df_clean['timestamp'].map(lambda x: pd.to_datetime(x).date())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2017-08-01\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "datetime.date"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(twitter_archive_df_clean['timestamp'][0])\n",
    "type(twitter_archive_df_clean['timestamp'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2017-08-01\n",
       "1    2017-08-01\n",
       "2    2017-07-31\n",
       "3    2017-07-30\n",
       "4    2017-07-29\n",
       "Name: timestamp, dtype: object"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df_clean['timestamp'].sort_values(ascending=False).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are many inaccurate names under 'names' column in archived tweets dataframe. ['None', 'a', 'the', 'an']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "None       745\n",
       "a           55\n",
       "Charlie     12\n",
       "Cooper      11\n",
       "Lucy        11\n",
       "Oliver      11\n",
       "Lola        10\n",
       "Tucker      10\n",
       "Penny       10\n",
       "Bo           9\n",
       "Winston      9\n",
       "Sadie        8\n",
       "the          8\n",
       "Daisy        7\n",
       "Buddy        7\n",
       "Bailey       7\n",
       "Toby         7\n",
       "an           7\n",
       "Jax          6\n",
       "Rusty        6\n",
       "Name: name, dtype: int64"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df_clean.name.value_counts()[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>retweeted_status_id</th>\n",
       "      <th>retweeted_status_user_id</th>\n",
       "      <th>retweeted_status_timestamp</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>rating_numerator</th>\n",
       "      <th>rating_denominator</th>\n",
       "      <th>name</th>\n",
       "      <th>doggo</th>\n",
       "      <th>floofer</th>\n",
       "      <th>pupper</th>\n",
       "      <th>puppo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [tweet_id, in_reply_to_status_id, in_reply_to_user_id, timestamp, source, text, retweeted_status_id, retweeted_status_user_id, retweeted_status_timestamp, expanded_urls, rating_numerator, rating_denominator, name, doggo, floofer, pupper, puppo]\n",
       "Index: []"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df_clean[twitter_archive_df_clean['name'] == None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "twitter_archive_df_clean['name'].replace(to_replace=['None', 'a', 'the', 'an'], value=\"Unsure\", inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unsure     815\n",
       "Charlie     12\n",
       "Oliver      11\n",
       "Cooper      11\n",
       "Lucy        11\n",
       "Penny       10\n",
       "Lola        10\n",
       "Tucker      10\n",
       "Winston      9\n",
       "Bo           9\n",
       "Sadie        8\n",
       "Bailey       7\n",
       "Daisy        7\n",
       "Buddy        7\n",
       "Toby         7\n",
       "Oscar        6\n",
       "Jack         6\n",
       "Leo          6\n",
       "Koda         6\n",
       "Jax          6\n",
       "Name: name, dtype: int64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df_clean.name.value_counts()[:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define\n",
    "\n",
    "Rating numerator and rating denominator are separated in archived tweets dataframe."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter_archive_df_clean['rating'] = twitter_archive_df_clean['rating_numerator'] / twitter_archive_df_clean['rating_denominator'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter_archive_df_clean.drop(['rating_numerator', 'rating_denominator'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.3\n",
       "1    1.3\n",
       "2    1.2\n",
       "3    1.3\n",
       "4    1.2\n",
       "Name: rating, dtype: float64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df_clean.rating.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['tweet_id', 'in_reply_to_status_id', 'in_reply_to_user_id', 'timestamp',\n",
       "       'source', 'text', 'retweeted_status_id', 'retweeted_status_user_id',\n",
       "       'retweeted_status_timestamp', 'expanded_urls', 'name', 'doggo',\n",
       "       'floofer', 'pupper', 'puppo', 'rating'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df_clean.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Replace NaN values in 'expanded_urls' column in archived tweets dataframe."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df_clean['expanded_urls'].isnull().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(twitter_archive_df_clean[pd.isnull(twitter_archive_df_clean['expanded_urls'])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter_archive_df_clean['expanded_urls'].fillna(\"Unsure\", inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df_clean['expanded_urls'].isnull().any()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define\n",
    "\n",
    "Remove duplicates & retweets in twitter_archive and tweets dataframe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>retweeted_status_id</th>\n",
       "      <th>retweeted_status_user_id</th>\n",
       "      <th>retweeted_status_timestamp</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>name</th>\n",
       "      <th>doggo</th>\n",
       "      <th>floofer</th>\n",
       "      <th>pupper</th>\n",
       "      <th>puppo</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>888202515573088257</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-21</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>RT @dog_rates: This is Canela. She attempted s...</td>\n",
       "      <td>8.874740e+17</td>\n",
       "      <td>4.196984e+09</td>\n",
       "      <td>2017-07-19 00:47:34 +0000</td>\n",
       "      <td>https://twitter.com/dog_rates/status/887473957...</td>\n",
       "      <td>Canela</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>886054160059072513</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-15</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>RT @Athletics: 12/10 #BATP https://t.co/WxwJmv...</td>\n",
       "      <td>8.860537e+17</td>\n",
       "      <td>1.960740e+07</td>\n",
       "      <td>2017-07-15 02:44:07 +0000</td>\n",
       "      <td>https://twitter.com/dog_rates/status/886053434...</td>\n",
       "      <td>Unsure</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>885311592912609280</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-07-13</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>RT @dog_rates: This is Lilly. She just paralle...</td>\n",
       "      <td>8.305833e+17</td>\n",
       "      <td>4.196984e+09</td>\n",
       "      <td>2017-02-12 01:04:29 +0000</td>\n",
       "      <td>https://twitter.com/dog_rates/status/830583320...</td>\n",
       "      <td>Lilly</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>879130579576475649</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-06-26</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>RT @dog_rates: This is Emmy. She was adopted t...</td>\n",
       "      <td>8.780576e+17</td>\n",
       "      <td>4.196984e+09</td>\n",
       "      <td>2017-06-23 01:10:23 +0000</td>\n",
       "      <td>https://twitter.com/dog_rates/status/878057613...</td>\n",
       "      <td>Emmy</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>878404777348136964</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2017-06-24</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>RT @dog_rates: Meet Shadow. In an attempt to r...</td>\n",
       "      <td>8.782815e+17</td>\n",
       "      <td>4.196984e+09</td>\n",
       "      <td>2017-06-23 16:00:04 +0000</td>\n",
       "      <td>https://www.gofundme.com/3yd6y1c,https://twitt...</td>\n",
       "      <td>Shadow</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              tweet_id  in_reply_to_status_id  in_reply_to_user_id  \\\n",
       "19  888202515573088257                    NaN                  NaN   \n",
       "32  886054160059072513                    NaN                  NaN   \n",
       "36  885311592912609280                    NaN                  NaN   \n",
       "68  879130579576475649                    NaN                  NaN   \n",
       "73  878404777348136964                    NaN                  NaN   \n",
       "\n",
       "     timestamp                                             source  \\\n",
       "19  2017-07-21  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "32  2017-07-15  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "36  2017-07-13  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "68  2017-06-26  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "73  2017-06-24  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "\n",
       "                                                 text  retweeted_status_id  \\\n",
       "19  RT @dog_rates: This is Canela. She attempted s...         8.874740e+17   \n",
       "32  RT @Athletics: 12/10 #BATP https://t.co/WxwJmv...         8.860537e+17   \n",
       "36  RT @dog_rates: This is Lilly. She just paralle...         8.305833e+17   \n",
       "68  RT @dog_rates: This is Emmy. She was adopted t...         8.780576e+17   \n",
       "73  RT @dog_rates: Meet Shadow. In an attempt to r...         8.782815e+17   \n",
       "\n",
       "    retweeted_status_user_id retweeted_status_timestamp  \\\n",
       "19              4.196984e+09  2017-07-19 00:47:34 +0000   \n",
       "32              1.960740e+07  2017-07-15 02:44:07 +0000   \n",
       "36              4.196984e+09  2017-02-12 01:04:29 +0000   \n",
       "68              4.196984e+09  2017-06-23 01:10:23 +0000   \n",
       "73              4.196984e+09  2017-06-23 16:00:04 +0000   \n",
       "\n",
       "                                        expanded_urls    name doggo floofer  \\\n",
       "19  https://twitter.com/dog_rates/status/887473957...  Canela  None    None   \n",
       "32  https://twitter.com/dog_rates/status/886053434...  Unsure  None    None   \n",
       "36  https://twitter.com/dog_rates/status/830583320...   Lilly  None    None   \n",
       "68  https://twitter.com/dog_rates/status/878057613...    Emmy  None    None   \n",
       "73  https://www.gofundme.com/3yd6y1c,https://twitt...  Shadow  None    None   \n",
       "\n",
       "   pupper puppo  rating  \n",
       "19   None  None     1.3  \n",
       "32   None  None     1.2  \n",
       "36   None  None     1.3  \n",
       "68   None  None     1.4  \n",
       "73   None  None     1.3  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df_clean[~pd.isnull(twitter_archive_df_clean['retweeted_status_id'])].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "twitter_archive_df_clean = twitter_archive_df_clean[pd.isnull(twitter_archive_df_clean['retweeted_status_id'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>retweeted</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>892420643555336193</td>\n",
       "      <td>False</td>\n",
       "      <td>8778</td>\n",
       "      <td>39308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>892177421306343426</td>\n",
       "      <td>False</td>\n",
       "      <td>6431</td>\n",
       "      <td>33657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>891815181378084864</td>\n",
       "      <td>False</td>\n",
       "      <td>4269</td>\n",
       "      <td>25363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>891689557279858688</td>\n",
       "      <td>False</td>\n",
       "      <td>8867</td>\n",
       "      <td>42692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>891327558926688256</td>\n",
       "      <td>False</td>\n",
       "      <td>9646</td>\n",
       "      <td>40841</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id  retweeted  retweet_count  favorite_count\n",
       "5  892420643555336193      False           8778           39308\n",
       "6  892177421306343426      False           6431           33657\n",
       "7  891815181378084864      False           4269           25363\n",
       "8  891689557279858688      False           8867           42692\n",
       "9  891327558926688256      False           9646           40841"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df_clean[tweets_df_clean['id'].duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tweets_df_clean.drop_duplicates('id', inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>retweeted</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [id, retweeted, retweet_count, favorite_count]\n",
       "Index: []"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df_clean[tweets_df_clean['id'].duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>retweeted_status_id</th>\n",
       "      <th>retweeted_status_user_id</th>\n",
       "      <th>retweeted_status_timestamp</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>name</th>\n",
       "      <th>doggo</th>\n",
       "      <th>floofer</th>\n",
       "      <th>pupper</th>\n",
       "      <th>puppo</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [tweet_id, in_reply_to_status_id, in_reply_to_user_id, timestamp, source, text, retweeted_status_id, retweeted_status_user_id, retweeted_status_timestamp, expanded_urls, name, doggo, floofer, pupper, puppo, rating]\n",
       "Index: []"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df_clean[~pd.isnull(twitter_archive_df_clean['retweeted_status_id'])]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define\n",
    "\n",
    "Remove the following columns in twitter archive dataframe that are primarily missing values: ['in_reply_to_status_id', 'in_reply_to_user_id', 'retweeted_status_id', 'retweeted_status_user_id', 'retweeted_status_timestamp']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['tweet_id', 'in_reply_to_status_id', 'in_reply_to_user_id', 'timestamp',\n",
       "       'source', 'text', 'retweeted_status_id', 'retweeted_status_user_id',\n",
       "       'retweeted_status_timestamp', 'expanded_urls', 'name', 'doggo',\n",
       "       'floofer', 'pupper', 'puppo', 'rating'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df_clean.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "twitter_archive_df_clean.drop(['in_reply_to_status_id', 'in_reply_to_user_id', 'retweeted_status_id', 'retweeted_status_user_id', 'retweeted_status_timestamp'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['tweet_id', 'timestamp', 'source', 'text', 'expanded_urls', 'name',\n",
       "       'doggo', 'floofer', 'pupper', 'puppo', 'rating'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df_clean.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define\n",
    "\n",
    "Remove 'retweeted' column in tweets dataframe since they all have the same value of 'false'."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_df_clean.drop(['retweeted'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892420643555336193</td>\n",
       "      <td>8778</td>\n",
       "      <td>39308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>892177421306343426</td>\n",
       "      <td>6431</td>\n",
       "      <td>33657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>891815181378084864</td>\n",
       "      <td>4269</td>\n",
       "      <td>25363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>891689557279858688</td>\n",
       "      <td>8867</td>\n",
       "      <td>42692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>891327558926688256</td>\n",
       "      <td>9646</td>\n",
       "      <td>40841</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id  retweet_count  favorite_count\n",
       "0  892420643555336193           8778           39308\n",
       "1  892177421306343426           6431           33657\n",
       "2  891815181378084864           4269           25363\n",
       "3  891689557279858688           8867           42692\n",
       "4  891327558926688256           9646           40841"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df_clean.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tidiness Issues"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define\n",
    "\n",
    "'id' column in tweets dataframe should be \"tweet_id\" to match the other two dataframes column name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id', 'retweet_count', 'favorite_count'], dtype='object')"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df_clean.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tweets_df_clean.columns = ['tweet_id', 'retweet_count', 'favorite_count']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['tweet_id', 'retweet_count', 'favorite_count'], dtype='object')"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df_clean.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define\n",
    "\n",
    "The dog stages should be combined into a single column e.g. \"stage\" column where the value could be either one of the four dog stages."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 2175 entries, 0 to 2355\n",
      "Data columns (total 11 columns):\n",
      "tweet_id         2175 non-null int64\n",
      "timestamp        2175 non-null object\n",
      "source           2175 non-null object\n",
      "text             2175 non-null object\n",
      "expanded_urls    2175 non-null object\n",
      "name             2175 non-null object\n",
      "doggo            2175 non-null object\n",
      "floofer          2175 non-null object\n",
      "pupper           2175 non-null object\n",
      "puppo            2175 non-null object\n",
      "rating           2175 non-null float64\n",
      "dtypes: float64(1), int64(1), object(9)\n",
      "memory usage: 203.9+ KB\n"
     ]
    }
   ],
   "source": [
    "twitter_archive_df_clean.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>name</th>\n",
       "      <th>doggo</th>\n",
       "      <th>floofer</th>\n",
       "      <th>pupper</th>\n",
       "      <th>puppo</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892420643555336193</td>\n",
       "      <td>2017-08-01</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Phineas. He's a mystical boy. Only eve...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/892420643...</td>\n",
       "      <td>Phineas</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>892177421306343426</td>\n",
       "      <td>2017-08-01</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Tilly. She's just checking pup on you....</td>\n",
       "      <td>https://twitter.com/dog_rates/status/892177421...</td>\n",
       "      <td>Tilly</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>891815181378084864</td>\n",
       "      <td>2017-07-31</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Archie. He is a rare Norwegian Pouncin...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/891815181...</td>\n",
       "      <td>Archie</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id   timestamp  \\\n",
       "0  892420643555336193  2017-08-01   \n",
       "1  892177421306343426  2017-08-01   \n",
       "2  891815181378084864  2017-07-31   \n",
       "\n",
       "                                              source  \\\n",
       "0  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "2  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "\n",
       "                                                text  \\\n",
       "0  This is Phineas. He's a mystical boy. Only eve...   \n",
       "1  This is Tilly. She's just checking pup on you....   \n",
       "2  This is Archie. He is a rare Norwegian Pouncin...   \n",
       "\n",
       "                                       expanded_urls     name doggo floofer  \\\n",
       "0  https://twitter.com/dog_rates/status/892420643...  Phineas  None    None   \n",
       "1  https://twitter.com/dog_rates/status/892177421...    Tilly  None    None   \n",
       "2  https://twitter.com/dog_rates/status/891815181...   Archie  None    None   \n",
       "\n",
       "  pupper puppo  rating  \n",
       "0   None  None     1.3  \n",
       "1   None  None     1.3  \n",
       "2   None  None     1.2  "
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df_clean.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "twitter_archive_df_clean['stage'] = twitter_archive_df_clean[['doggo','floofer','pupper', 'puppo']].replace('None','').sum(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "twitter_archive_df_clean.drop(['doggo', 'puppo', 'pupper', 'floofer'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['', 'doggo', 'puppo', 'pupper', 'floofer', 'doggopuppo',\n",
       "       'doggofloofer', 'doggopupper'], dtype=object)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df_clean['stage'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['tweet_id', 'timestamp', 'source', 'text', 'expanded_urls', 'name',\n",
       "       'rating', 'stage'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_archive_df_clean.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define\n",
    "\n",
    "Dataframes need to be merged into one tidy masterdataframe by 'tweet_id' column and remove any rows with null values."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tweets_master_df = pd.merge(twitter_archive_df_clean, tweets_df_clean, \n",
    "                            on='tweet_id', how='inner')\n",
    "\n",
    "tweets_master_df = pd.merge(tweets_master_df, image_predictions_df, \n",
    "                            on='tweet_id', how='inner')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['tweet_id', 'timestamp', 'source', 'text', 'expanded_urls', 'name',\n",
       "       'rating', 'stage', 'retweet_count', 'favorite_count', 'jpg_url',\n",
       "       'img_num', 'p1', 'p1_conf', 'p1_dog', 'p2', 'p2_conf', 'p2_dog', 'p3',\n",
       "       'p3_conf', 'p3_dog'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_master_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1993"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tweets_master_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>name</th>\n",
       "      <th>rating</th>\n",
       "      <th>stage</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>...</th>\n",
       "      <th>img_num</th>\n",
       "      <th>p1</th>\n",
       "      <th>p1_conf</th>\n",
       "      <th>p1_dog</th>\n",
       "      <th>p2</th>\n",
       "      <th>p2_conf</th>\n",
       "      <th>p2_dog</th>\n",
       "      <th>p3</th>\n",
       "      <th>p3_conf</th>\n",
       "      <th>p3_dog</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [tweet_id, timestamp, source, text, expanded_urls, name, rating, stage, retweet_count, favorite_count, jpg_url, img_num, p1, p1_conf, p1_dog, p2, p2_conf, p2_dog, p3, p3_conf, p3_dog]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 21 columns]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remaining rows with null values\n",
    "tweets_master_df[tweets_master_df.isnull().any(axis=1)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Analyis/Visualization & Storage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Insights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### #1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "timestamp\n",
       "2015    663\n",
       "2016    973\n",
       "2017    357\n",
       "Name: timestamp, dtype: int64"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_master_df.groupby(tweets_master_df[\"timestamp\"].apply(lambda x: x.year))['timestamp'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The majority of tweets were published in 2016."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### #2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>name</th>\n",
       "      <th>rating</th>\n",
       "      <th>stage</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>...</th>\n",
       "      <th>img_num</th>\n",
       "      <th>p1</th>\n",
       "      <th>p1_conf</th>\n",
       "      <th>p1_dog</th>\n",
       "      <th>p2</th>\n",
       "      <th>p2_conf</th>\n",
       "      <th>p2_dog</th>\n",
       "      <th>p3</th>\n",
       "      <th>p3_conf</th>\n",
       "      <th>p3_dog</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>774</th>\n",
       "      <td>744234799360020481</td>\n",
       "      <td>2016-06-18</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Here's a doggo realizing you can stand in a po...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/744234799...</td>\n",
       "      <td>Unsure</td>\n",
       "      <td>1.3</td>\n",
       "      <td>doggo</td>\n",
       "      <td>78712</td>\n",
       "      <td>130050</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>Labrador_retriever</td>\n",
       "      <td>0.825333</td>\n",
       "      <td>True</td>\n",
       "      <td>ice_bear</td>\n",
       "      <td>0.044681</td>\n",
       "      <td>False</td>\n",
       "      <td>whippet</td>\n",
       "      <td>0.018442</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400</th>\n",
       "      <td>807106840509214720</td>\n",
       "      <td>2016-12-09</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Stephan. He just wants to help. 13/10 ...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/807106840...</td>\n",
       "      <td>Stephan</td>\n",
       "      <td>1.3</td>\n",
       "      <td></td>\n",
       "      <td>62145</td>\n",
       "      <td>123953</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>Chihuahua</td>\n",
       "      <td>0.505370</td>\n",
       "      <td>True</td>\n",
       "      <td>Pomeranian</td>\n",
       "      <td>0.120358</td>\n",
       "      <td>True</td>\n",
       "      <td>toy_terrier</td>\n",
       "      <td>0.077008</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>809</th>\n",
       "      <td>739238157791694849</td>\n",
       "      <td>2016-06-04</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Here's a doggo blowing bubbles. It's downright...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/739238157...</td>\n",
       "      <td>Unsure</td>\n",
       "      <td>1.3</td>\n",
       "      <td>doggo</td>\n",
       "      <td>51810</td>\n",
       "      <td>74480</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>Eskimo_dog</td>\n",
       "      <td>0.503372</td>\n",
       "      <td>True</td>\n",
       "      <td>Siberian_husky</td>\n",
       "      <td>0.390413</td>\n",
       "      <td>True</td>\n",
       "      <td>malamute</td>\n",
       "      <td>0.080901</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309</th>\n",
       "      <td>822872901745569793</td>\n",
       "      <td>2017-01-21</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Here's a super supportive puppo participating ...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/822872901...</td>\n",
       "      <td>Unsure</td>\n",
       "      <td>1.3</td>\n",
       "      <td>puppo</td>\n",
       "      <td>47668</td>\n",
       "      <td>131758</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>Lakeland_terrier</td>\n",
       "      <td>0.196015</td>\n",
       "      <td>True</td>\n",
       "      <td>Labrador_retriever</td>\n",
       "      <td>0.160329</td>\n",
       "      <td>True</td>\n",
       "      <td>Irish_terrier</td>\n",
       "      <td>0.069126</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>879415818425184262</td>\n",
       "      <td>2017-06-26</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Duddles. He did an attempt. 13/10 some...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/879415818...</td>\n",
       "      <td>Duddles</td>\n",
       "      <td>1.3</td>\n",
       "      <td></td>\n",
       "      <td>45443</td>\n",
       "      <td>107357</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>English_springer</td>\n",
       "      <td>0.383404</td>\n",
       "      <td>True</td>\n",
       "      <td>Boston_bull</td>\n",
       "      <td>0.134967</td>\n",
       "      <td>True</td>\n",
       "      <td>Cardigan</td>\n",
       "      <td>0.110481</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333</th>\n",
       "      <td>819004803107983360</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Bo. He was a very good First Doggo. 14...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/819004803...</td>\n",
       "      <td>Bo</td>\n",
       "      <td>1.4</td>\n",
       "      <td>doggo</td>\n",
       "      <td>41836</td>\n",
       "      <td>94805</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>standard_poodle</td>\n",
       "      <td>0.351308</td>\n",
       "      <td>True</td>\n",
       "      <td>toy_poodle</td>\n",
       "      <td>0.271929</td>\n",
       "      <td>True</td>\n",
       "      <td>Tibetan_terrier</td>\n",
       "      <td>0.094759</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>402</th>\n",
       "      <td>806629075125202948</td>\n",
       "      <td>2016-12-07</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>\"Good afternoon class today we're going to lea...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/806629075...</td>\n",
       "      <td>Unsure</td>\n",
       "      <td>1.3</td>\n",
       "      <td></td>\n",
       "      <td>40273</td>\n",
       "      <td>83347</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>Arabian_camel</td>\n",
       "      <td>0.366248</td>\n",
       "      <td>False</td>\n",
       "      <td>house_finch</td>\n",
       "      <td>0.209852</td>\n",
       "      <td>False</td>\n",
       "      <td>cocker_spaniel</td>\n",
       "      <td>0.046403</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>636</th>\n",
       "      <td>761672994376806400</td>\n",
       "      <td>2016-08-05</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Ohboyohboyohboyohboyohboyohboyohboyohboyohboyo...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/761672994...</td>\n",
       "      <td>Unsure</td>\n",
       "      <td>1.0</td>\n",
       "      <td></td>\n",
       "      <td>32997</td>\n",
       "      <td>54474</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>gondola</td>\n",
       "      <td>0.318851</td>\n",
       "      <td>False</td>\n",
       "      <td>sea_lion</td>\n",
       "      <td>0.306525</td>\n",
       "      <td>False</td>\n",
       "      <td>pool_table</td>\n",
       "      <td>0.111565</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>866450705531457537</td>\n",
       "      <td>2017-05-22</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Jamesy. He gives a kiss to every other...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/866450705...</td>\n",
       "      <td>Jamesy</td>\n",
       "      <td>1.3</td>\n",
       "      <td>pupper</td>\n",
       "      <td>32514</td>\n",
       "      <td>106071</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>French_bulldog</td>\n",
       "      <td>0.905334</td>\n",
       "      <td>True</td>\n",
       "      <td>Boston_bull</td>\n",
       "      <td>0.078060</td>\n",
       "      <td>True</td>\n",
       "      <td>pug</td>\n",
       "      <td>0.001771</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>859196978902773760</td>\n",
       "      <td>2017-05-02</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>We only rate dogs. This is quite clearly a smo...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/859196978...</td>\n",
       "      <td>quite</td>\n",
       "      <td>1.2</td>\n",
       "      <td></td>\n",
       "      <td>32388</td>\n",
       "      <td>93826</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>Angora</td>\n",
       "      <td>0.224218</td>\n",
       "      <td>False</td>\n",
       "      <td>malamute</td>\n",
       "      <td>0.216163</td>\n",
       "      <td>True</td>\n",
       "      <td>Persian_cat</td>\n",
       "      <td>0.128383</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               tweet_id   timestamp  \\\n",
       "774  744234799360020481  2016-06-18   \n",
       "400  807106840509214720  2016-12-09   \n",
       "809  739238157791694849  2016-06-04   \n",
       "309  822872901745569793  2017-01-21   \n",
       "58   879415818425184262  2017-06-26   \n",
       "333  819004803107983360  2017-01-11   \n",
       "402  806629075125202948  2016-12-07   \n",
       "636  761672994376806400  2016-08-05   \n",
       "108  866450705531457537  2017-05-22   \n",
       "135  859196978902773760  2017-05-02   \n",
       "\n",
       "                                                source  \\\n",
       "774  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "400  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "809  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "309  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "58   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "333  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "402  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "636  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "108  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "135  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "\n",
       "                                                  text  \\\n",
       "774  Here's a doggo realizing you can stand in a po...   \n",
       "400  This is Stephan. He just wants to help. 13/10 ...   \n",
       "809  Here's a doggo blowing bubbles. It's downright...   \n",
       "309  Here's a super supportive puppo participating ...   \n",
       "58   This is Duddles. He did an attempt. 13/10 some...   \n",
       "333  This is Bo. He was a very good First Doggo. 14...   \n",
       "402  \"Good afternoon class today we're going to lea...   \n",
       "636  Ohboyohboyohboyohboyohboyohboyohboyohboyohboyo...   \n",
       "108  This is Jamesy. He gives a kiss to every other...   \n",
       "135  We only rate dogs. This is quite clearly a smo...   \n",
       "\n",
       "                                         expanded_urls     name  rating  \\\n",
       "774  https://twitter.com/dog_rates/status/744234799...   Unsure     1.3   \n",
       "400  https://twitter.com/dog_rates/status/807106840...  Stephan     1.3   \n",
       "809  https://twitter.com/dog_rates/status/739238157...   Unsure     1.3   \n",
       "309  https://twitter.com/dog_rates/status/822872901...   Unsure     1.3   \n",
       "58   https://twitter.com/dog_rates/status/879415818...  Duddles     1.3   \n",
       "333  https://twitter.com/dog_rates/status/819004803...       Bo     1.4   \n",
       "402  https://twitter.com/dog_rates/status/806629075...   Unsure     1.3   \n",
       "636  https://twitter.com/dog_rates/status/761672994...   Unsure     1.0   \n",
       "108  https://twitter.com/dog_rates/status/866450705...   Jamesy     1.3   \n",
       "135  https://twitter.com/dog_rates/status/859196978...    quite     1.2   \n",
       "\n",
       "      stage  retweet_count  favorite_count   ...   img_num  \\\n",
       "774   doggo          78712          130050   ...         1   \n",
       "400                  62145          123953   ...         1   \n",
       "809   doggo          51810           74480   ...         1   \n",
       "309   puppo          47668          131758   ...         1   \n",
       "58                   45443          107357   ...         1   \n",
       "333   doggo          41836           94805   ...         1   \n",
       "402                  40273           83347   ...         2   \n",
       "636                  32997           54474   ...         1   \n",
       "108  pupper          32514          106071   ...         2   \n",
       "135                  32388           93826   ...         1   \n",
       "\n",
       "                     p1   p1_conf  p1_dog                  p2   p2_conf  \\\n",
       "774  Labrador_retriever  0.825333    True            ice_bear  0.044681   \n",
       "400           Chihuahua  0.505370    True          Pomeranian  0.120358   \n",
       "809          Eskimo_dog  0.503372    True      Siberian_husky  0.390413   \n",
       "309    Lakeland_terrier  0.196015    True  Labrador_retriever  0.160329   \n",
       "58     English_springer  0.383404    True         Boston_bull  0.134967   \n",
       "333     standard_poodle  0.351308    True          toy_poodle  0.271929   \n",
       "402       Arabian_camel  0.366248   False         house_finch  0.209852   \n",
       "636             gondola  0.318851   False            sea_lion  0.306525   \n",
       "108      French_bulldog  0.905334    True         Boston_bull  0.078060   \n",
       "135              Angora  0.224218   False            malamute  0.216163   \n",
       "\n",
       "     p2_dog               p3   p3_conf  p3_dog  \n",
       "774   False          whippet  0.018442    True  \n",
       "400    True      toy_terrier  0.077008    True  \n",
       "809    True         malamute  0.080901    True  \n",
       "309    True    Irish_terrier  0.069126    True  \n",
       "58     True         Cardigan  0.110481    True  \n",
       "333    True  Tibetan_terrier  0.094759    True  \n",
       "402   False   cocker_spaniel  0.046403    True  \n",
       "636   False       pool_table  0.111565   False  \n",
       "108    True              pug  0.001771    True  \n",
       "135    True      Persian_cat  0.128383   False  \n",
       "\n",
       "[10 rows x 21 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#filtering dataframe by indexes that are in the top 10 most retweets\n",
    "tweets_master_df.iloc[tweets_master_df['retweet_count'].nlargest(10).index, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Keenan\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:1: DeprecationWarning: \n",
      ".ix is deprecated. Please use\n",
      ".loc for label based indexing or\n",
      ".iloc for positional indexing\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated\n",
      "  if __name__ == '__main__':\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"Here's a doggo realizing you can stand in a pool. 13/10 enlightened af (vid by Tina Conrad) https://t.co/7wE9LTEXC4\""
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_master_df.ix[774, 'text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://twitter.com/dog_rates/status/744234799360020481/video/1'"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_master_df.ix[774, 'expanded_urls']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<blockquote class=\"twitter-video\" data-lang=\"en\"><p lang=\"en\" dir=\"ltr\">Here&#39;s a doggo realizing you can stand in a pool. 13/10 enlightened af (vid by Tina Conrad) <a href=\"https://t.co/7wE9LTEXC4\">pic.twitter.com/7wE9LTEXC4</a></p>&mdash; WeRateDogs™ (@dog_rates) <a href=\"https://twitter.com/dog_rates/status/744234799360020481?ref_src=twsrc%5Etfw\">June 18, 2016</a></blockquote>\n",
    "<script async src=\"https://platform.twitter.com/widgets.js\" charset=\"utf-8\"></script>\n",
    "\n",
    " [![Enlightened AF](https://pbs.twimg.com/ext_tw_video_thumb/744234667679821824/pu/img/1GaWmtJtdqzZV7jy.jpg)](https://twitter.com/dog_rates/status/744234799360020481/video/1 \"Enlightened AF\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The tweet with the most retweets was a video of a dog in a pool walking on his hindlegs.  Pretty darn cute!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### #3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>expanded_urls</th>\n",
       "      <th>name</th>\n",
       "      <th>rating</th>\n",
       "      <th>stage</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>...</th>\n",
       "      <th>img_num</th>\n",
       "      <th>p1</th>\n",
       "      <th>p1_conf</th>\n",
       "      <th>p1_dog</th>\n",
       "      <th>p2</th>\n",
       "      <th>p2_conf</th>\n",
       "      <th>p2_dog</th>\n",
       "      <th>p3</th>\n",
       "      <th>p3_conf</th>\n",
       "      <th>p3_dog</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>309</th>\n",
       "      <td>822872901745569793</td>\n",
       "      <td>2017-01-21</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Here's a super supportive puppo participating ...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/822872901...</td>\n",
       "      <td>Unsure</td>\n",
       "      <td>1.3</td>\n",
       "      <td>puppo</td>\n",
       "      <td>47668</td>\n",
       "      <td>131758</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>Lakeland_terrier</td>\n",
       "      <td>0.196015</td>\n",
       "      <td>True</td>\n",
       "      <td>Labrador_retriever</td>\n",
       "      <td>0.160329</td>\n",
       "      <td>True</td>\n",
       "      <td>Irish_terrier</td>\n",
       "      <td>0.069126</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>774</th>\n",
       "      <td>744234799360020481</td>\n",
       "      <td>2016-06-18</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>Here's a doggo realizing you can stand in a po...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/744234799...</td>\n",
       "      <td>Unsure</td>\n",
       "      <td>1.3</td>\n",
       "      <td>doggo</td>\n",
       "      <td>78712</td>\n",
       "      <td>130050</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>Labrador_retriever</td>\n",
       "      <td>0.825333</td>\n",
       "      <td>True</td>\n",
       "      <td>ice_bear</td>\n",
       "      <td>0.044681</td>\n",
       "      <td>False</td>\n",
       "      <td>whippet</td>\n",
       "      <td>0.018442</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400</th>\n",
       "      <td>807106840509214720</td>\n",
       "      <td>2016-12-09</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Stephan. He just wants to help. 13/10 ...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/807106840...</td>\n",
       "      <td>Stephan</td>\n",
       "      <td>1.3</td>\n",
       "      <td></td>\n",
       "      <td>62145</td>\n",
       "      <td>123953</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>Chihuahua</td>\n",
       "      <td>0.505370</td>\n",
       "      <td>True</td>\n",
       "      <td>Pomeranian</td>\n",
       "      <td>0.120358</td>\n",
       "      <td>True</td>\n",
       "      <td>toy_terrier</td>\n",
       "      <td>0.077008</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>879415818425184262</td>\n",
       "      <td>2017-06-26</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Duddles. He did an attempt. 13/10 some...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/879415818...</td>\n",
       "      <td>Duddles</td>\n",
       "      <td>1.3</td>\n",
       "      <td></td>\n",
       "      <td>45443</td>\n",
       "      <td>107357</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>English_springer</td>\n",
       "      <td>0.383404</td>\n",
       "      <td>True</td>\n",
       "      <td>Boston_bull</td>\n",
       "      <td>0.134967</td>\n",
       "      <td>True</td>\n",
       "      <td>Cardigan</td>\n",
       "      <td>0.110481</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>866450705531457537</td>\n",
       "      <td>2017-05-22</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Jamesy. He gives a kiss to every other...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/866450705...</td>\n",
       "      <td>Jamesy</td>\n",
       "      <td>1.3</td>\n",
       "      <td>pupper</td>\n",
       "      <td>32514</td>\n",
       "      <td>106071</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>French_bulldog</td>\n",
       "      <td>0.905334</td>\n",
       "      <td>True</td>\n",
       "      <td>Boston_bull</td>\n",
       "      <td>0.078060</td>\n",
       "      <td>True</td>\n",
       "      <td>pug</td>\n",
       "      <td>0.001771</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333</th>\n",
       "      <td>819004803107983360</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Bo. He was a very good First Doggo. 14...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/819004803...</td>\n",
       "      <td>Bo</td>\n",
       "      <td>1.4</td>\n",
       "      <td>doggo</td>\n",
       "      <td>41836</td>\n",
       "      <td>94805</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>standard_poodle</td>\n",
       "      <td>0.351308</td>\n",
       "      <td>True</td>\n",
       "      <td>toy_poodle</td>\n",
       "      <td>0.271929</td>\n",
       "      <td>True</td>\n",
       "      <td>Tibetan_terrier</td>\n",
       "      <td>0.094759</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>859196978902773760</td>\n",
       "      <td>2017-05-02</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>We only rate dogs. This is quite clearly a smo...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/859196978...</td>\n",
       "      <td>quite</td>\n",
       "      <td>1.2</td>\n",
       "      <td></td>\n",
       "      <td>32388</td>\n",
       "      <td>93826</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>Angora</td>\n",
       "      <td>0.224218</td>\n",
       "      <td>False</td>\n",
       "      <td>malamute</td>\n",
       "      <td>0.216163</td>\n",
       "      <td>True</td>\n",
       "      <td>Persian_cat</td>\n",
       "      <td>0.128383</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>870374049280663552</td>\n",
       "      <td>2017-06-01</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Zoey. She really likes the planet. Wou...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/870374049...</td>\n",
       "      <td>Zoey</td>\n",
       "      <td>1.3</td>\n",
       "      <td></td>\n",
       "      <td>27343</td>\n",
       "      <td>84412</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>golden_retriever</td>\n",
       "      <td>0.841001</td>\n",
       "      <td>True</td>\n",
       "      <td>Great_Pyrenees</td>\n",
       "      <td>0.099278</td>\n",
       "      <td>True</td>\n",
       "      <td>Labrador_retriever</td>\n",
       "      <td>0.032621</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>402</th>\n",
       "      <td>806629075125202948</td>\n",
       "      <td>2016-12-07</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>\"Good afternoon class today we're going to lea...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/806629075...</td>\n",
       "      <td>Unsure</td>\n",
       "      <td>1.3</td>\n",
       "      <td></td>\n",
       "      <td>40273</td>\n",
       "      <td>83347</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>Arabian_camel</td>\n",
       "      <td>0.366248</td>\n",
       "      <td>False</td>\n",
       "      <td>house_finch</td>\n",
       "      <td>0.209852</td>\n",
       "      <td>False</td>\n",
       "      <td>cocker_spaniel</td>\n",
       "      <td>0.046403</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>877736472329191424</td>\n",
       "      <td>2017-06-22</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>This is Aja. She was just told she's a good do...</td>\n",
       "      <td>https://twitter.com/dog_rates/status/877736472...</td>\n",
       "      <td>Aja</td>\n",
       "      <td>1.3</td>\n",
       "      <td></td>\n",
       "      <td>19804</td>\n",
       "      <td>81132</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>Chesapeake_Bay_retriever</td>\n",
       "      <td>0.837956</td>\n",
       "      <td>True</td>\n",
       "      <td>Labrador_retriever</td>\n",
       "      <td>0.062034</td>\n",
       "      <td>True</td>\n",
       "      <td>Weimaraner</td>\n",
       "      <td>0.040599</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               tweet_id   timestamp  \\\n",
       "309  822872901745569793  2017-01-21   \n",
       "774  744234799360020481  2016-06-18   \n",
       "400  807106840509214720  2016-12-09   \n",
       "58   879415818425184262  2017-06-26   \n",
       "108  866450705531457537  2017-05-22   \n",
       "333  819004803107983360  2017-01-11   \n",
       "135  859196978902773760  2017-05-02   \n",
       "93   870374049280663552  2017-06-01   \n",
       "402  806629075125202948  2016-12-07   \n",
       "65   877736472329191424  2017-06-22   \n",
       "\n",
       "                                                source  \\\n",
       "309  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "774  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "400  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "58   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "108  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "333  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "135  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "93   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "402  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "65   <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "\n",
       "                                                  text  \\\n",
       "309  Here's a super supportive puppo participating ...   \n",
       "774  Here's a doggo realizing you can stand in a po...   \n",
       "400  This is Stephan. He just wants to help. 13/10 ...   \n",
       "58   This is Duddles. He did an attempt. 13/10 some...   \n",
       "108  This is Jamesy. He gives a kiss to every other...   \n",
       "333  This is Bo. He was a very good First Doggo. 14...   \n",
       "135  We only rate dogs. This is quite clearly a smo...   \n",
       "93   This is Zoey. She really likes the planet. Wou...   \n",
       "402  \"Good afternoon class today we're going to lea...   \n",
       "65   This is Aja. She was just told she's a good do...   \n",
       "\n",
       "                                         expanded_urls     name  rating  \\\n",
       "309  https://twitter.com/dog_rates/status/822872901...   Unsure     1.3   \n",
       "774  https://twitter.com/dog_rates/status/744234799...   Unsure     1.3   \n",
       "400  https://twitter.com/dog_rates/status/807106840...  Stephan     1.3   \n",
       "58   https://twitter.com/dog_rates/status/879415818...  Duddles     1.3   \n",
       "108  https://twitter.com/dog_rates/status/866450705...   Jamesy     1.3   \n",
       "333  https://twitter.com/dog_rates/status/819004803...       Bo     1.4   \n",
       "135  https://twitter.com/dog_rates/status/859196978...    quite     1.2   \n",
       "93   https://twitter.com/dog_rates/status/870374049...     Zoey     1.3   \n",
       "402  https://twitter.com/dog_rates/status/806629075...   Unsure     1.3   \n",
       "65   https://twitter.com/dog_rates/status/877736472...      Aja     1.3   \n",
       "\n",
       "      stage  retweet_count  favorite_count   ...   img_num  \\\n",
       "309   puppo          47668          131758   ...         1   \n",
       "774   doggo          78712          130050   ...         1   \n",
       "400                  62145          123953   ...         1   \n",
       "58                   45443          107357   ...         1   \n",
       "108  pupper          32514          106071   ...         2   \n",
       "333   doggo          41836           94805   ...         1   \n",
       "135                  32388           93826   ...         1   \n",
       "93                   27343           84412   ...         1   \n",
       "402                  40273           83347   ...         2   \n",
       "65                   19804           81132   ...         2   \n",
       "\n",
       "                           p1   p1_conf  p1_dog                  p2   p2_conf  \\\n",
       "309          Lakeland_terrier  0.196015    True  Labrador_retriever  0.160329   \n",
       "774        Labrador_retriever  0.825333    True            ice_bear  0.044681   \n",
       "400                 Chihuahua  0.505370    True          Pomeranian  0.120358   \n",
       "58           English_springer  0.383404    True         Boston_bull  0.134967   \n",
       "108            French_bulldog  0.905334    True         Boston_bull  0.078060   \n",
       "333           standard_poodle  0.351308    True          toy_poodle  0.271929   \n",
       "135                    Angora  0.224218   False            malamute  0.216163   \n",
       "93           golden_retriever  0.841001    True      Great_Pyrenees  0.099278   \n",
       "402             Arabian_camel  0.366248   False         house_finch  0.209852   \n",
       "65   Chesapeake_Bay_retriever  0.837956    True  Labrador_retriever  0.062034   \n",
       "\n",
       "     p2_dog                  p3   p3_conf  p3_dog  \n",
       "309    True       Irish_terrier  0.069126    True  \n",
       "774   False             whippet  0.018442    True  \n",
       "400    True         toy_terrier  0.077008    True  \n",
       "58     True            Cardigan  0.110481    True  \n",
       "108    True                 pug  0.001771    True  \n",
       "333    True     Tibetan_terrier  0.094759    True  \n",
       "135    True         Persian_cat  0.128383   False  \n",
       "93     True  Labrador_retriever  0.032621    True  \n",
       "402   False      cocker_spaniel  0.046403    True  \n",
       "65     True          Weimaraner  0.040599    True  \n",
       "\n",
       "[10 rows x 21 columns]"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#filtering dataframe by indexes that are in the top 10 most favorite\n",
    "tweets_master_df.iloc[tweets_master_df['favorite_count'].nlargest(10).index, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Here's a super supportive puppo participating in the Toronto  #WomensMarch today. 13/10 https://t.co/nTz3FtorBc\""
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_master_df.ix[309, 'text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://twitter.com/dog_rates/status/822872901745569793/photo/1'"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_master_df.ix[309, 'expanded_urls']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<blockquote class=\"twitter-tweet\" data-lang=\"en\"><p lang=\"en\" dir=\"ltr\">Here&#39;s a super supportive puppo participating in the Toronto  <a href=\"https://twitter.com/hashtag/WomensMarch?src=hash&amp;ref_src=twsrc%5Etfw\">#WomensMarch</a> today. 13/10 <a href=\"https://t.co/nTz3FtorBc\">pic.twitter.com/nTz3FtorBc</a></p>&mdash; WeRateDogs™ (@dog_rates) <a href=\"https://twitter.com/dog_rates/status/822872901745569793?ref_src=twsrc%5Etfw\">January 21, 2017</a></blockquote>\n",
    "<script async src=\"https://platform.twitter.com/widgets.js\" charset=\"utf-8\"></script>\n",
    "\n",
    "![Women's March](https://pbs.twimg.com/media/C2tugXLXgAArJO4.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The tweet with the most favorites was a photo of a dog wearing a sign at the womens march last year. I'm not suprised that gained a lot of traction because that movement and hashtag went viral."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1993 entries, 0 to 1992\n",
      "Data columns (total 21 columns):\n",
      "tweet_id          1993 non-null int64\n",
      "timestamp         1993 non-null object\n",
      "source            1993 non-null object\n",
      "text              1993 non-null object\n",
      "expanded_urls     1993 non-null object\n",
      "name              1993 non-null object\n",
      "rating            1993 non-null float64\n",
      "stage             1993 non-null object\n",
      "retweet_count     1993 non-null int64\n",
      "favorite_count    1993 non-null int64\n",
      "jpg_url           1993 non-null object\n",
      "img_num           1993 non-null int64\n",
      "p1                1993 non-null object\n",
      "p1_conf           1993 non-null float64\n",
      "p1_dog            1993 non-null bool\n",
      "p2                1993 non-null object\n",
      "p2_conf           1993 non-null float64\n",
      "p2_dog            1993 non-null bool\n",
      "p3                1993 non-null object\n",
      "p3_conf           1993 non-null float64\n",
      "p3_dog            1993 non-null bool\n",
      "dtypes: bool(3), float64(4), int64(4), object(10)\n",
      "memory usage: 381.7+ KB\n"
     ]
    }
   ],
   "source": [
    "tweets_master_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_master_df['tweet_id'] = tweets_master_df['tweet_id'].astype(object, copy=False)\n",
    "tweets_master_df['timestamp'] = pd.to_datetime(tweets_master_df['timestamp'])\n",
    "tweets_master_df['retweet_count'] = tweets_master_df['retweet_count'].astype(int, copy=False)\n",
    "tweets_master_df['favorite_count'] = tweets_master_df['favorite_count'].astype(int, copy=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1993 entries, 0 to 1992\n",
      "Data columns (total 21 columns):\n",
      "tweet_id          1993 non-null object\n",
      "timestamp         1993 non-null datetime64[ns]\n",
      "source            1993 non-null object\n",
      "text              1993 non-null object\n",
      "expanded_urls     1993 non-null object\n",
      "name              1993 non-null object\n",
      "rating            1993 non-null float64\n",
      "stage             1993 non-null object\n",
      "retweet_count     1993 non-null int32\n",
      "favorite_count    1993 non-null int32\n",
      "jpg_url           1993 non-null object\n",
      "img_num           1993 non-null int64\n",
      "p1                1993 non-null object\n",
      "p1_conf           1993 non-null float64\n",
      "p1_dog            1993 non-null bool\n",
      "p2                1993 non-null object\n",
      "p2_conf           1993 non-null float64\n",
      "p2_dog            1993 non-null bool\n",
      "p3                1993 non-null object\n",
      "p3_conf           1993 non-null float64\n",
      "p3_dog            1993 non-null bool\n",
      "dtypes: bool(3), datetime64[ns](1), float64(4), int32(2), int64(1), object(10)\n",
      "memory usage: 366.1+ KB\n"
     ]
    }
   ],
   "source": [
    "tweets_master_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas._libs.tslib.Timestamp"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(tweets_master_df['timestamp'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAENCAYAAAABh67pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFfRJREFUeJzt3X2wXHd93/H3x7IxD8YPjG9UIQlkGNFGpkU0t4IptHEw\nYMVmIruhjqClIpgIBoeHaZKODJ0C09FEJDyEaWI6IhjUBFDFUywekmCEHQaSsZBB2JZt1QqWsVRZ\nuoEAJlAlkr/9Y4/JIqR7997dtayf36+Znf2d33n4/ta6/uzZs2fPSVUhSWrXaSd7AJKk8TLoJalx\nBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcaef7AEAnH/++bVkyZKTPQxJOqXc\ncsstf1NVEzMt94gI+iVLlrBjx46TPQxJOqUkuXeQ5Tx0I0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINe\nkhpn0EtS4wx6SWrcI+IHU5LUuiXrPjun9fZuuGzo2u7RS1LjDHpJapxBL0mNM+glqXEGvSQ1buCg\nTzIvydeTfKabflKSG5Lc3T2f17fsNUn2JNmd5JJxDFySNJjZ7NG/Ebizb3odsK2qlgLbummSLANW\nAxcCK4Frk8wbzXAlSbM1UNAnWQRcBvxhX/cqYFPX3gRc3te/uaoOV9U9wB5gxWiGK0marUH36H8P\n+C/Ag31986vqQNe+H5jftRcC9/Utt6/rkySdBDMGfZKXAIeq6pYTLVNVBdRsCidZm2RHkh1TU1Oz\nWVWSNAuD7NE/D/ilJHuBzcALkvwxcDDJAoDu+VC3/H5gcd/6i7q+n1BVG6tqsqomJyZmvLetJGmO\nZgz6qrqmqhZV1RJ6X7J+sar+I7AVWNMttga4vmtvBVYnOTPJBcBSYPvIRy5JGsgwFzXbAGxJchVw\nL3AlQFXtSrIFuAM4AlxdVUeHHqkkaU5mFfRVdRNwU9f+NnDxCZZbD6wfcmySpBHwl7GS1DiDXpIa\nZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEG\nvSQ1zqCXpMYNcnPwxybZnuQbSXYleXvX/7Yk+5Ps7B6X9q1zTZI9SXYnuWScL0CSNL1B7jB1GHhB\nVf0gyRnAl5P8aTfvPVX1zv6Fkyyjd2/ZC4EnA19I8gxvJyhJJ8cgNwevqvpBN3lG96hpVlkFbK6q\nw1V1D7AHWDH0SCVJczLQMfok85LsBA4BN1TVzd2s1ye5Ncl1Sc7r+hYC9/Wtvq/rkySdBAMFfVUd\nrarlwCJgRZJnAu8DngYsBw4A75pN4SRrk+xIsmNqamqWw5YkDWqQY/Q/VlXfTXIjsLL/2HyS9wOf\n6Sb3A4v7VlvU9R27rY3ARoDJycnpDgVJ0sgtWffZOa23d8NlIx7J+A1y1s1EknO79uOAFwF3JVnQ\nt9gVwO1deyuwOsmZSS4AlgLbRztsSdKgBtmjXwBsSjKP3hvDlqr6TJI/SrKc3heze4HXAFTVriRb\ngDuAI8DVnnEjSSfPjEFfVbcCzz5O/yumWWc9sH64oUmSRsFfxkpS4wx6SWqcQS9JjTPoJalxBr0k\nNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGjfIrQQf\nm2R7km8k2ZXk7V3/k5LckOTu7vm8vnWuSbInye4kl4zzBUiSpjfIHv1h4AVV9SxgObAyyXOBdcC2\nqloKbOumSbIMWA1cCKwEru1uQyhJOglmDPrq+UE3eUb3KGAVsKnr3wRc3rVXAZur6nBV3QPsAVaM\ndNSSpIENdIw+ybwkO4FDwA1VdTMwv6oOdIvcD8zv2guB+/pW39f1HbvNtUl2JNkxNTU15xcgSZre\nQEFfVUerajmwCFiR5JnHzC96e/kDq6qNVTVZVZMTExOzWVWSNAuzOuumqr4L3Ejv2PvBJAsAuudD\n3WL7gcV9qy3q+iRJJ8EgZ91MJDm3az8OeBFwF7AVWNMttga4vmtvBVYnOTPJBcBSYPuoBy5JGszp\nAyyzANjUnTlzGrClqj6T5K+ALUmuAu4FrgSoql1JtgB3AEeAq6vq6HiGL0mayYxBX1W3As8+Tv+3\ngYtPsM56YP3Qo5MkDc1fxkpS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMM\neklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktS4Qe4wtTjJjUnuSLIryRu7/rcl2Z9kZ/e4tG+da5Ls\nSbI7ySXjfAGSpOkNcoepI8BvVNXXkjwRuCXJDd2891TVO/sXTrIMWA1cCDwZ+EKSZ3iXKUk6OWbc\no6+qA1X1ta79AHAnsHCaVVYBm6vqcFXdA+wBVoxisJKk2ZvVMfokS+jdVvDmruv1SW5Ncl2S87q+\nhcB9favtY/o3BknSGA0c9EnOAj4BvKmqvg+8D3gasBw4ALxrNoWTrE2yI8mOqamp2awqSZqFgYI+\nyRn0Qv7DVfVJgKo6WFVHq+pB4P384+GZ/cDivtUXdX0/oao2VtVkVU1OTEwM8xokSdMY5KybAB8A\n7qyqd/f1L+hb7Arg9q69FVid5MwkFwBLge2jG7IkaTYGOevmecArgNuS7Oz63gy8LMlyoIC9wGsA\nqmpXki3AHfTO2LnaM24k6eSZMeir6stAjjPrc9Ossx5YP8S4JEkj4i9jJalxBr0kNW6QY/Qn1ZJ1\nn53Tens3XDbikUjSqck9eklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxB\nL0mNM+glqXEGvSQ1zqCXpMYNcivBxUluTHJHkl1J3tj1PynJDUnu7p7P61vnmiR7kuxOcsk4X4Ak\naXqD7NEfAX6jqpYBzwWuTrIMWAdsq6qlwLZumm7eauBCYCVwbZJ54xi8JGlmMwZ9VR2oqq917QeA\nO4GFwCpgU7fYJuDyrr0K2FxVh6vqHmAPsGLUA5ckDWZWx+iTLAGeDdwMzK+qA92s+4H5XXshcF/f\navu6PknSSTBw0Cc5C/gE8Kaq+n7/vKoqoGZTOMnaJDuS7JiamprNqpKkWRgo6JOcQS/kP1xVn+y6\nDyZZ0M1fABzq+vcDi/tWX9T1/YSq2lhVk1U1OTExMdfxS5JmMMhZNwE+ANxZVe/um7UVWNO11wDX\n9/WvTnJmkguApcD20Q1ZkjQbg9wc/HnAK4Dbkuzs+t4MbAC2JLkKuBe4EqCqdiXZAtxB74ydq6vq\n6MhHLkkayIxBX1VfBnKC2RefYJ31wPohxiVJGhF/GStJjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIa\nZ9BLUuMG+cHUo8qSdZ+d03p7N1w24pFI0mi4Ry9JjTPoJalxHro5yTxUJGnc3KOXpMYZ9JLUOINe\nkhpn0EtS4wa5w9R1SQ4lub2v721J9ifZ2T0u7Zt3TZI9SXYnuWRcA5ckDWaQPfoPASuP0/+eqlre\nPT4HkGQZsBq4sFvn2iTzRjVYSdLszRj0VfUl4DsDbm8VsLmqDlfVPcAeYMUQ45MkDWmYY/SvT3Jr\nd2jnvK5vIXBf3zL7uj5J0kky16B/H/A0YDlwAHjXbDeQZG2SHUl2TE1NzXEYkqSZzCnoq+pgVR2t\nqgeB9/OPh2f2A4v7Fl3U9R1vGxurarKqJicmJuYyDEnSAOYU9EkW9E1eATx0Rs5WYHWSM5NcACwF\ntg83REnSMGa81k2SjwIXAecn2Qe8FbgoyXKggL3AawCqaleSLcAdwBHg6qo6Op6hS5IGMWPQV9XL\njtP9gWmWXw+sH2ZQkqTR8ZexktQ4g16SGuf16KUheD8BnQrco5ekxhn0ktQ4g16SGmfQS1LjDHpJ\napxn3agpngUj/TT36CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJatyMQZ/kuiSHktze1/ek\nJDckubt7Pq9v3jVJ9iTZneSScQ1ckjSYQfboPwSsPKZvHbCtqpYC27ppkiwDVgMXdutcm2TeyEYr\nSZq1GYO+qr4EfOeY7lXApq69Cbi8r39zVR2uqnuAPcCKEY1VkjQHcz1GP7+qDnTt+4H5XXshcF/f\ncvu6PknSSTL0tW6qqpLUbNdLshZYC/CUpzxl2GHoEcprz0gn31z36A8mWQDQPR/q+vcDi/uWW9T1\n/ZSq2lhVk1U1OTExMcdhSJJmMtc9+q3AGmBD93x9X/9HkrwbeDKwFNg+7CA1Ou5hS48+MwZ9ko8C\nFwHnJ9kHvJVewG9JchVwL3AlQFXtSrIFuAM4AlxdVUfHNHZJ0gBmDPqqetkJZl18guXXA+uHGZQk\naXT8ZawkNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16S\nGjf0jUckaRS8hPb4GPSSTmgu4WvwPvIY9NIpxL1ezYXH6CWpcUPt0SfZCzwAHAWOVNVkkicB/xtY\nAuwFrqyqvx1umJKkuRrFHv0vVNXyqprsptcB26pqKbCtm5YknSTjOHSzCtjUtTcBl4+hhiRpQMMG\nfQFfSHJLkrVd3/yqOtC17wfmH2/FJGuT7EiyY2pqashhSJJOZNizbp5fVfuT/AxwQ5K7+mdWVSWp\n461YVRuBjQCTk5PHXUaSNLyh9uiran/3fAj4FLACOJhkAUD3fGjYQUqS5m7OQZ/kCUme+FAbeDFw\nO7AVWNMttga4fthBSpLmbphDN/OBTyV5aDsfqao/S/JVYEuSq4B7gSuHH6Ykaa7mHPRV9U3gWcfp\n/zZw8TCDkiSNjr+MlaTGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4\ng16SGmfQS1LjDHpJapxBL0mNM+glqXFjC/okK5PsTrInybpx1ZEkTW8sQZ9kHvAHwC8Cy4CXJVk2\njlqSpOmNa49+BbCnqr5ZVX8PbAZWjamWJGka4wr6hcB9fdP7uj5J0sMsVTX6jSYvBVZW1au76VcA\nz6mqX+9bZi2wtpv8p8DuOZQ6H/ibIYdrPetZ75Fdy3on9tSqmphpoTnfHHwG+4HFfdOLur4fq6qN\nwMZhiiTZUVWTw2zDetaz3iO7lvWGN65DN18Flia5IMljgNXA1jHVkiRNYyx79FV1JMmvA38OzAOu\nq6pd46glSZreuA7dUFWfAz43ru13hjr0Yz3rWe+UqGW9IY3ly1hJ0iOHl0CQpMYZ9JLUOINekhpn\n0E8jyT9LcnGSs47pXzmmeiuS/KuuvSzJf05y6ThqHaf2/3o46vTVe373+l48pu0/J8nZXftxSd6e\n5NNJ3pHknBHXekOSxTMvObJ6j0nyn5K8sJt+eZLfT3J1kjPGVPNpSX4zyXuTvDvJax/676tHvia+\njE3yq1X1wRFv8w3A1cCdwHLgjVV1fTfva1X1L0dc7630LgJ3OnAD8BzgRuBFwJ9X1foR1jr2Nw0B\nfgH4IkBV/dKoavXV3F5VK7r2r9H7b/sp4MXAp6tqw4jr7QKe1Z3quxH4IfBx4OKu/9+NsNb3gL8D\n/hr4KPCxqpoa1faPU+/D9P5OHg98FzgL+CS915aqWjPiem8AXgJ8CbgU+HpX9wrgdVV10yjraQyq\n6pR/AN8awzZvA87q2kuAHfTCHuDrY6o3j97/vN8Hzu76HwfcOuJaXwP+GLgI+Pnu+UDX/vkx/Rt9\nva/9VWCiaz8BuG0M9e7sf73HzNs56tdG79Pxi4EPAFPAnwFrgCeO4bXd2j2fDhwE5nXTGfXfSrfd\n2/pqPB64qWs/ZUz/L5wDbADuAr4DfJveDtcG4NxR15thLH86hm2eDfw28EfAy4+Zd+04XsfYzqMf\ntSS3nmgWMH8MJU+rqh8AVNXeJBcBH0/y1K7mqB2pqqPAD5P8dVV9v6v9oyQPjrjWJPBG4C3Ab1XV\nziQ/qqq/GHGdfqclOY9eIM6rbo+3qv4uyZEx1Lu975PeN5JMVtWOJM8A/mHEtaqqHgQ+D3y+O3zy\ni8DLgHcCM16LZJZO635x/gR6wXsOvUA8ExjLoRt6bypHuxpnAVTVt8Z0qGgLvU+XF1XV/QBJ/gm9\nN84t9N5QRybJiT6dh96n+VH7IHA38AngVUl+mV7gHwaeO4Z6p07Q0wvzS4C/PaY/wF+Ood7BJMur\naidAVf0gyUuA64B/PoZ6f5/k8VX1Q+DnHursjiePNOi7UHpPko91zwcZ/9/COcAt9P69KsmCqjrQ\nff8xjjfOVwPvTfJf6V0s6q+S3EfvqqqvHnGtnxh/Vf0DvUt+bE3y+BHXgt6nhrvofQJ8C/CxJN+k\nFxKbx1DvD4GvJrkZ+DfAOwCSTNB7gxm1JVX1jv6OLvDfkeRVY6j3VeAvOP7f4bljqPf0qvrlrv0n\nSd4CfDHJyA+ZPuSUOUaf5APAB6vqy8eZ95GqevmI6y2it5d9/3HmPa+qvjLiemd27+jH9p8PLKiq\n20ZZ75galwHPq6o3j6vGNLUfD8yvqnvGtP2zgQvovZHtq6qDY6jxjKr6P6Pe7gw1nwxQVf83ybnA\nC+kdwtw+pnoXAj8L3F5Vd42jRl+tzwNfADY99O+VZD7wSuBFVfXCEde7Hbiiqu4+zrz7qmqkX7Qn\nuRO4sNvheqjvlcBv0Ttc/NRR1oNTKOglPTp0h/jW0btZ0c903QfpfUraUFXHfqoftt5L6X1P9FOX\nSk9yeVX9yYjr/Q7w+ar6wjH9K4H/UVVLR1kPDHpJp5BxnGH3aKhn0Es6ZST5VlU9xXqzcyp9GSvp\nUeDhPsOu9Xpg0Et65Hm4z7BrvZ5BL+kR5zP0zj7ZeeyMJDdZb/Y8Ri9JjfOiZpLUOINekhpn0OuU\nlOTcJK/r2k9O8vEx1lr+cF0uWhoHg16nqnOB10HvMgBV9dIx1lpO7/K80inJL2N1Skqymd5P5HfT\nuxLgz1bVM7trhlxO78qOS+ldPfIxwCuAw8ClVfWdJE8H/oDelSV/CPxaVd2V5N8Db6V3pcbv0buG\nzB56l4veT+/ysvcA7wUeC/wI+NWq2j2L2jcB36B3WejTgVeN6xo1EtDG9eh9PPoe9O4RcPtx2q+k\nF8xPpBfi3wNe2817D/Cmrr0NWNq1nwN8sWvfBizs2uf2bfP3+2qfDZzetV8IfGKWtW8C3t+1/+1D\nY/fhY1wPz6NXi26sqgeAB7q7P326678N+BfdpZH/Nb3L+z60zpnd81eADyXZQu+uTcdzDrApyVKg\n+MlrwE9bu2+5jwJU1ZeSnJ3k3Kr67hxfrzQtg14t6r/c84N90w/S+5s/DfhuVf3UTSWq6rVJngNc\nBtyS5OeOXQb47/QC/YokS+jtoQ9a+8elji09zeuRhuKXsTpVPUDvEMmsVe/uXfd0x+NJz7O69tOr\n6uaq+m/0bgm4+Di1zqF3vB56h2vm4le6es8HvldV35vjdqQZGfQ6JVXVt4GvdDeN+N05bOI/AFcl\n+Qawi94XuwC/m+S2brt/Se9L0xuBZUl2JvkV4HeA307ydeb+qfj/dev/T+CqOW5DGohn3UgPs+6s\nm9+sqh0neyx6dHCPXpIa5x69JDXOPXpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUuP8P4Qg5dMki\nZJ0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1cd013db400>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "# creating a bar chart grouped by months. \";\" at the end prevents additional irrelevant text output with plot.\n",
    "tweets_master_df.groupby(tweets_master_df[\"timestamp\"].apply(lambda x: x.month))['timestamp'].count().plot(kind=\"bar\");"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\Keenan\\\\datascience\\\\Udacity\\\\DA Nanodegree\\\\Data Wrangling\\\\data wrangling project'"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tweets_master_df.to_csv('twitter_archive_master.csv', header=True)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}