Scrapping Images - Twitter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Modules\n",
    "import tweepy, json\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import requests\n",
    "from PIL import Image\n",
    "from io import BytesIO\n",
    "\n",
    "# Athentication\n",
    "consumer_key = '#'\n",
    "consumer_secret = '#'\n",
    "\n",
    "access_token = '#'\n",
    "access_token_secret = '#'\n",
    "\n",
    "print('Connecting to Twitter API....')\n",
    "auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
    "auth.set_access_token(access_token, access_token_secret)\n",
    "\n",
    "api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving images in the retrieved tweets\n",
    "def download(search_results, location='data'):\n",
    "    for i, tweet in enumerate(search_results):\n",
    "        for j,media in enumerate(tweet.entities.get(\"media\",[{}])):\n",
    "            #checks if there is any media-entity\n",
    "            if media.get(\"type\",None) == \"photo\":\n",
    "                # checks if the entity is of the type \"photo\"\n",
    "                image_content=requests.get(media[\"media_url\"])\n",
    "                # save images\n",
    "                img = Image.open(BytesIO(image_content.content))\n",
    "                img.save(f\"{location}/{str(tweet.id)}_{str(j)}.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scrapping images at `#faceapp`\n",
    "original code was provided [here](https://bhaskarvk.github.io/2015/01/how-to-use-twitters-search-rest-api-most-effectively./)\n",
    "- Added the ability to download images for each tweet\n",
    "- Specified the `lang` to be English\n",
    "\n",
    "### Scraps all images from the hashtag\n",
    "- Real images\n",
    "- Fake generated images\n",
    "- A collage of Real and Fake images\n",
    "- Non-relevant images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "searchQuery = '#faceapp'  # this is what we're searching for\n",
    "maxTweets = 10000000 # Some arbitrary large number\n",
    "tweetsPerQry = 100  # this is the max the API permits\n",
    "tweetCount = 0\n",
    "max_id = -1\n",
    "sinceId = None\n",
    "\n",
    "while tweetCount < maxTweets:\n",
    "    try:\n",
    "        if (max_id <= 0):\n",
    "            if (not sinceId):\n",
    "                new_tweets = api.search(q=searchQuery, lang='en', count=tweetsPerQry)\n",
    "            else:\n",
    "                new_tweets = api.search(q=searchQuery, lang='en', count=tweetsPerQry,\n",
    "                                        since_id=sinceId)\n",
    "        else:\n",
    "            if (not sinceId):\n",
    "                new_tweets = api.search(q=searchQuery, lang='en', count=tweetsPerQry,\n",
    "                                        max_id=str(max_id - 1))\n",
    "            else:\n",
    "                new_tweets = api.search(q=searchQuery, lang='en', count=tweetsPerQry,\n",
    "                                        max_id=str(max_id - 1),\n",
    "                                        since_id=sinceId)\n",
    "        if not new_tweets:\n",
    "            print(\"No more tweets found\")\n",
    "        #for tweet in new_tweets:\n",
    "        download(new_tweets)\n",
    "            \n",
    "        tweetCount += len(new_tweets)\n",
    "        print(\"Downloaded {0} tweets\".format(tweetCount))\n",
    "        max_id = new_tweets[-1].id\n",
    "    except tweepy.TweepError as e:\n",
    "        # Just exit if any error\n",
    "        print(\"some error : \" + str(e))\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Downloading more real images to balance the data \n",
    "- This time we are using `#new_avatar` as people will tend to put their real images\n",
    "- All the images are downloaded to the `more_original` folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "searchQuery = '#new_avatar'  # this is what we're searching for\n",
    "maxTweets = 10000 # Some arbitrary large number\n",
    "tweetsPerQry = 100  # this is the max the API permits\n",
    "tweetCount = 0\n",
    "max_id = -1\n",
    "sinceId = None\n",
    "\n",
    "\n",
    "while tweetCount < maxTweets:\n",
    "    try:\n",
    "        if (max_id <= 0):\n",
    "            if (not sinceId):\n",
    "                new_tweets = api.search(q=searchQuery, count=tweetsPerQry)\n",
    "            else:\n",
    "                new_tweets = api.search(q=searchQuery, count=tweetsPerQry,\n",
    "                                        since_id=sinceId)\n",
    "        else:\n",
    "            if (not sinceId):\n",
    "                new_tweets = api.search(q=searchQuery, count=tweetsPerQry,\n",
    "                                        max_id=str(max_id - 1))\n",
    "            else:\n",
    "                new_tweets = api.search(q=searchQuery, count=tweetsPerQry,\n",
    "                                        max_id=str(max_id - 1),\n",
    "                                        since_id=sinceId)\n",
    "        if not new_tweets:\n",
    "            print(\"No more tweets found\")\n",
    "            break\n",
    "        #for tweet in new_tweets:\n",
    "        download(new_tweets, location='more_original')\n",
    "            \n",
    "        tweetCount += len(new_tweets)\n",
    "        print(\"Downloaded {0} tweets\".format(tweetCount))\n",
    "        max_id = new_tweets[-1].id\n",
    "    except tweepy.TweepError as e:\n",
    "        # Just exit if any error\n",
    "        print(\"some error : \" + str(e))\n",
    "        break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}