WebscrapingInstagram_completeNotebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Web Scraping Instagram with Selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "#imports here\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.wait import WebDriverWait"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download ChromeDriver\n",
    "Now we need to download latest stable release of ChromeDriver from:\n",
    "<br>\n",
    "https://chromedriver.chromium.org/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "#specify the path to chromedriver.exe (download and save on your computer)\n",
    "driver = webdriver.Chrome('C:/Users/goaim/chromedriver.exe')\n",
    "\n",
    "#open the webpage\n",
    "driver.get(\"http://www.instagram.com\")\n",
    "\n",
    "#target username\n",
    "username = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, \"input[name='username']\")))\n",
    "password = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, \"input[name='password']\")))\n",
    "\n",
    "#enter username and password\n",
    "username.clear()\n",
    "username.send_keys(\"my_username\")\n",
    "password.clear()\n",
    "password.send_keys(\"my_password\")\n",
    "\n",
    "#target the login button and click it\n",
    "button = WebDriverWait(driver, 2).until(EC.element_to_be_clickable((By.CSS_SELECTOR, \"button[type='submit']\"))).click()\n",
    "\n",
    "#We are logged in!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "#nadle NOT NOW\n",
    "not_now = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//button[contains(text(), \"Not Now\")]'))).click()\n",
    "not_now2 = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//button[contains(text(), \"Not Now\")]'))).click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Search keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "#target the search input field\n",
    "searchbox = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, \"//input[@placeholder='Search']\")))\n",
    "searchbox.clear()\n",
    "\n",
    "#search for the hashtag cat\n",
    "keyword = \"#cat\"\n",
    "searchbox.send_keys(keyword)\n",
    " \n",
    "# Wait for 5 seconds\n",
    "time.sleep(5)\n",
    "searchbox.send_keys(Keys.ENTER)\n",
    "time.sleep(5)\n",
    "searchbox.send_keys(Keys.ENTER)\n",
    "time.sleep(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of scraped images:  46\n"
     ]
    }
   ],
   "source": [
    "#scroll down to scrape more images\n",
    "driver.execute_script(\"window.scrollTo(0, 4000);\")\n",
    "\n",
    "#target all images on the page\n",
    "images = driver.find_elements_by_tag_name('img')\n",
    "images = [image.get_attribute('src') for image in images]\n",
    "images = images[:-2]\n",
    "\n",
    "print('Number of scraped images: ', len(images))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save images to computer\n",
    "\n",
    "First we'll create a new folder for our images somewhere on our computer.\n",
    "<br>\n",
    "Then, we'll save all the images there."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\goaim\\\\cats'"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import wget\n",
    "\n",
    "path = os.getcwd()\n",
    "path = os.path.join(path, keyword[1:] + \"s\")\n",
    "\n",
    "#create the directory\n",
    "os.mkdir(path)\n",
    "\n",
    "path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100% [..............................................................................] 53506 / 53506"
     ]
    }
   ],
   "source": [
    "#download images\n",
    "counter = 0\n",
    "for image in images:\n",
    "    save_as = os.path.join(path, keyword[1:] + str(counter) + '.jpg')\n",
    "    wget.download(image, save_as)\n",
    "    counter += 1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}